scope/search/web/search.go

package web

import (
	"net/http"
	"sort"
	"sync"
	"time"

	"golang.org/x/sync/errgroup"
)

func init() {
	http.DefaultClient.Timeout = 5 * time.Second
}

type Result struct {
	Title   string
	Link    string
	Desc    string
	Engines []string
	Rank    int
}

type Engine interface {
	// Set search keyword for engine
	SetKeyword(string)

	// Set User Agent. If string is empty,
	// an acceptable will should be used.
	SetUserAgent(string)

	// Set page number to search
	SetPage(int)

	// Initialize engine (make requests, set variables, etc.)
	Init() error

	// Run function for each search result,
	// inputting index
	Each(func(int) error) error

	// Get title from index given by Each()
	Title(int) (string, error)
	// Get link from index given by Each()
	Link(int) (string, error)
	// Get description from index given by Each()
	Desc(int) (string, error)

	// Return shortened name of search engine.
	// Should be lowercase (e.g. google, ddg, bing)
	Name() string
}

type Options struct {
	Keyword   string
	UserAgent string
	Page      int
}

func Search(opts Options, engines ...Engine) ([]*Result, error) {
	var outMtx sync.Mutex
	var out []*Result

	wg := errgroup.Group{}
	for index, engine := range engines {
		curIndex, curEngine := index, engine
		wg.Go(func() error {

			curEngine.SetKeyword(opts.Keyword)
			curEngine.SetUserAgent(opts.UserAgent)
			curEngine.SetPage(opts.Page)

			if err := curEngine.Init(); err != nil {
				return err
			}

			err := curEngine.Each(func(i int) error {
				link, err := curEngine.Link(i)
				if err != nil {
					return err
				}

				rank := (curIndex * 100) + i

				index, exists := linkExists(out, link)
				if exists {
					out[index].Engines = append(out[index].Engines, curEngine.Name())
					if rank < out[index].Rank {
						out[index].Rank = rank
					}
					return nil
				}

				title, err := curEngine.Title(i)
				if err != nil {
					return err
				}

				desc, err := curEngine.Desc(i)
				if err != nil {
					return err
				}

				if title == "" || link == "" || desc == "" {
					return nil
				}

				if len(desc) > 500 {
					desc = desc[:500] + "..."
				}

				result := &Result{
					Title: title,
					Link:  link,
					Desc:  desc,
					Rank:  rank,
				}
				result.Engines = append(result.Engines, curEngine.Name())

				outMtx.Lock()
				out = append(out, result)
				outMtx.Unlock()

				return nil
			})
			if err != nil {
				return err
			}

			sort.Slice(out, func(i, j int) bool {
				return out[i].Rank < out[j].Rank
			})
			return nil
		})
	}

	if err := wg.Wait(); err != nil {
		return out, err
	}

	return out, nil
}

func linkExists(results []*Result, link string) (int, bool) {
	for index, result := range results {
		if result.Link == link {
			return index, true
		}
	}
	return -1, false
}