Elara6331/scope
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix golint errors and gofmt

Browse Source
This commit is contained in:
Elara
2021-12-08 13:18:14 -08:00
parent c8bec472be
commit 254155d442
10 changed files with 144 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
search/web/bing.go
View File

@@ -9,6 +9,7 @@ import (
var bingURL = urlMustParse("https://www.bing.com/search?count=10")
// Bing represents the Bing search engine
type Bing struct {
keyword string
userAgent string
@@ -18,29 +19,37 @@ type Bing struct {
baseSel *goquery.Selection
}
// SetKeyword sets the keyword for searching
func (b *Bing) SetKeyword(keyword string) {
b.keyword = keyword
}
// SetPage sets the page number for searching
func (b *Bing) SetPage(page int) {
b.first = page * 10
}
// SetUserAgent sets the user agent to use for the request
func (b *Bing) SetUserAgent(ua string) {
b.userAgent = ua
}
// Init runs requests for Bing search engine
func (b *Bing) Init() error {
// Copy URL so it can be changed
initURL := copyURL(bingURL)
query := initURL.Query()
// Set query
query.Set("q", b.keyword)
if b.first > 0 {
query.Set("first", strconv.Itoa(b.first))
} else {
query.Set("first", "1")
}
// Update URL query parameters
initURL.RawQuery = query.Encode()
// Create new request for modified URL
req, err := http.NewRequest(
http.MethodGet,
initURL.String(),
@@ -49,17 +58,21 @@ func (b *Bing) Init() error {
if err != nil {
return err
}
// If no user agent, use default
if b.userAgent == "" {
b.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
}
// Set request user agent
req.Header.Set("User-Agent", b.userAgent)
// Perform request
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
// Create new goquery document
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
return err
@@ -70,6 +83,7 @@ func (b *Bing) Init() error {
return nil
}
// Each runs eachCb with the index of each search result
func (b *Bing) Each(eachCb func(int) error) error {
for i := 0; i < b.baseSel.Length(); i++ {
err := eachCb(i)
@@ -80,18 +94,22 @@ func (b *Bing) Each(eachCb func(int) error) error {
return nil
}
// Title returns the title of the search result corresponding to i
func (b *Bing) Title(i int) (string, error) {
return get(b.baseSel, i).ChildrenFiltered("h2").Children().First().Text(), nil
}
// Link returns the link to the search result corresponding to i
func (b *Bing) Link(i int) (string, error) {
return get(b.baseSel, i).ChildrenFiltered("h2").Children().First().AttrOr("href", ""), nil
}
// Desc returns the description of the search result corresponding to i
func (b *Bing) Desc(i int) (string, error) {
return get(b.baseSel, i).ChildrenFiltered(".b_caption").Children().Last().Text(), nil
}
// Name returns "bing"
func (b *Bing) Name() string {
return "bing"
}

21
search/web/ddg.go
View File

@@ -12,6 +12,7 @@ var ddgURL = urlMustParse("https://html.duckduckgo.com/html")
const uddgPrefix = "//duckduckgo.com/l/?uddg="
// DDG represents the DuckDuckGo search engine
type DDG struct {
keyword string
userAgent string
@@ -21,28 +22,37 @@ type DDG struct {
baseSel *goquery.Selection
}
// SetKeyword sets the keyword for searching
func (d *DDG) SetKeyword(keyword string) {
d.keyword = keyword
}
// SetPage sets the page number for searching
func (d *DDG) SetPage(page int) {
d.page = page * 30
}
// SetUserAgent sets the user agent for the request
func (d *DDG) SetUserAgent(ua string) {
d.userAgent = "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10" //ua
d.userAgent = ua
}
// Init runs requests for the DuckDuckGo search engine
func (d *DDG) Init() error {
// Copy URL so that it can be changed
initURL := copyURL(ddgURL)
// Get query parameters
query := initURL.Query()
// Set query
query.Set("q", d.keyword)
if d.page > 0 {
query.Set("s", strconv.Itoa(d.page))
query.Set("dc", strconv.Itoa(d.page+1))
}
// Update URL query
initURL.RawQuery = query.Encode()
// Create new request for modified URL
req, err := http.NewRequest(
http.MethodGet,
initURL.String(),
@@ -51,17 +61,21 @@ func (d *DDG) Init() error {
if err != nil {
return err
}
// If user agent empty, use default
if d.userAgent == "" {
d.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
}
// Set user agent of request
req.Header.Set("User-Agent", d.userAgent)
// Perform request
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
// Create goquery document from reader
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
return err
@@ -72,6 +86,7 @@ func (d *DDG) Init() error {
return nil
}
// Each runs eachCb with the index of each search result
func (d *DDG) Each(eachCb func(int) error) error {
for i := 0; i < d.baseSel.Length(); i++ {
err := eachCb(i)
@@ -82,10 +97,12 @@ func (d *DDG) Each(eachCb func(int) error) error {
return nil
}
// Title returns the title of the search result corresponding to i
func (d *DDG) Title(i int) (string, error) {
return strings.TrimSpace(get(d.baseSel, i).Children().First().ChildrenFiltered("h2").Text()), nil
}
// Link returns the link to the search result corresponding to i
func (d *DDG) Link(i int) (string, error) {
link := get(d.baseSel, i).Children().First().ChildrenFiltered("a").AttrOr("href", "")
if strings.HasPrefix(link, uddgPrefix) {
@@ -94,10 +111,12 @@ func (d *DDG) Link(i int) (string, error) {
return link, nil
}
// Desc returns the description of the search result corresponding to i
func (d *DDG) Desc(i int) (string, error) {
return get(d.baseSel, i).Children().First().ChildrenFiltered("a").Text(), nil
}
// Name returns "ddg"
func (d *DDG) Name() string {
return "ddg"
}

24
search/web/google.go
View File

@@ -10,6 +10,7 @@ import (
var googleURL = urlMustParse("https://www.google.com/search")
// Google represents the Google search engine
type Google struct {
keyword string
userAgent string
@@ -19,24 +20,35 @@ type Google struct {
baseSel *goquery.Selection
}
// SetKeyword sets the keyword for searching
func (g *Google) SetKeyword(keyword string) {
g.keyword = keyword
}
// SetPage sets the page number for searching
func (g *Google) SetPage(page int) {
g.page = page * 10
}
// SetUserAgent sets the user agent for the request
func (g *Google) SetUserAgent(ua string) {
g.userAgent = ua
}
// Init runs requests for the Google search engine
func (g *Google) Init() error {
// Copy URL so that it can be changed
initURL := copyURL(googleURL)
// Get query parameters
query := initURL.Query()
// Set query
query.Set("q", g.keyword)
// Set starting result (page number)
query.Set("start", strconv.Itoa(g.page))
// Update URL query
initURL.RawQuery = query.Encode()
// Create new request for modified URL
req, err := http.NewRequest(
http.MethodGet,
initURL.String(),
@@ -45,17 +57,21 @@ func (g *Google) Init() error {
if err != nil {
return err
}
// If user agent empty, use default
if g.userAgent == "" {
g.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
}
// Set user agent of request
req.Header.Set("User-Agent", g.userAgent)
// Perform request
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
// Create goquery document from reader
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
return err
@@ -66,6 +82,7 @@ func (g *Google) Init() error {
return nil
}
// Each runs eachCb with the index of each search result
func (g *Google) Each(eachCb func(int) error) error {
for i := 0; i < g.baseSel.Length(); i++ {
err := eachCb(i)
@@ -76,31 +93,38 @@ func (g *Google) Each(eachCb func(int) error) error {
return nil
}
// Title returns the title of the search result corresponding to i
func (g *Google) Title(i int) (string, error) {
return get(g.baseSel, i).Text(), nil
}
// Link returns the link to the search result corresponding to i
func (g *Google) Link(i int) (string, error) {
return get(g.baseSel, i).Parent().AttrOr("href", ""), nil
}
// Desc returns the description of the search result corresponding to i
func (g *Google) Desc(i int) (string, error) {
return get(g.baseSel, i).Parent().Parent().Next().Text(), nil
}
// Name returns "google"
func (g *Google) Name() string {
return "google"
}
// get gets an element and given index from given selection
func get(sel *goquery.Selection, i int) *goquery.Selection {
return sel.Slice(i, i+1)
}
// Parse url ignoring error
func urlMustParse(urlStr string) *url.URL {
out, _ := url.Parse(urlStr)
return out
}
// copyURL makes a copy of the url and returns it
func copyURL(orig *url.URL) *url.URL {
newURL := new(url.URL)
*newURL = *orig

41
search/web/search.go
View File

@@ -13,6 +13,7 @@ func init() {
http.DefaultClient.Timeout = 5 * time.Second
}
// Result represents a search result
type Result struct {
Title string
Link string
@@ -21,6 +22,7 @@ type Result struct {
Rank int
}
// Engine represents a search engine for web results (not images, shopping, erc.)
type Engine interface {
// Set search keyword for engine
SetKeyword(string)
@@ -51,74 +53,95 @@ type Engine interface {
Name() string
}
// Options represents search options
type Options struct {
Keyword string
UserAgent string
Page int
}
// Search searches the given engines concurrently and returns the results
func Search(opts Options, engines ...Engine) ([]*Result, error) {
var outMtx sync.Mutex
var out []*Result
// Create new error group
wg := errgroup.Group{}
// For every engine
for index, engine := range engines {
// Copy index and engine (for goroutine)
curIndex, curEngine := index, engine
wg.Go(func() error {
// Set options
curEngine.SetKeyword(opts.Keyword)
curEngine.SetUserAgent(opts.UserAgent)
curEngine.SetPage(opts.Page)
// Attempt to init engine
if err := curEngine.Init(); err != nil {
return err
}
// For each result
err := curEngine.Each(func(i int) error {
// Get result link
link, err := curEngine.Link(i)
if err != nil {
return err
}
// Calculate result rank
rank := (curIndex * 100) + i
// Check if result exists
index, exists := linkExists(out, link)
// If result already exists
if exists {
// Add engine to the existing result
out[index].Engines = append(out[index].Engines, curEngine.Name())
// If the rank is higher than the old one, update it
if rank < out[index].Rank {
out[index].Rank = rank
}
return nil
}
// Get result title
title, err := curEngine.Title(i)
if err != nil {
return err
}
// Get result description
desc, err := curEngine.Desc(i)
if err != nil {
return err
}
// If title, link, or description empty, ignore
if title == "" || link == "" || desc == "" {
return nil
}
// If length of description, truncate
if len(desc) > 500 {
desc = desc[:500] + "..."
}
// Create result struct
result := &Result{
Title: title,
Link: link,
Desc: desc,
Rank: rank,
Title: title,
Link: link,
Desc: desc,
Rank: rank,
Engines: []string{curEngine.Name()},
}
result.Engines = append(result.Engines, curEngine.Name())
// Lock out mutex
outMtx.Lock()
// Add result to slice
out = append(out, result)
// Unlock out mutex
outMtx.Unlock()
return nil
@@ -127,6 +150,7 @@ func Search(opts Options, engines ...Engine) ([]*Result, error) {
return err
}
// Sort slice by rank
sort.Slice(out, func(i, j int) bool {
return out[i].Rank < out[j].Rank
})
@@ -134,6 +158,7 @@ func Search(opts Options, engines ...Engine) ([]*Result, error) {
})
}
// Wait for error group
if err := wg.Wait(); err != nil {
return out, err
}
@@ -141,9 +166,13 @@ func Search(opts Options, engines ...Engine) ([]*Result, error) {
return out, nil
}
// linkExists checks if a link exists in the results
func linkExists(results []*Result, link string) (int, bool) {
// For every result
for index, result := range results {
// If link is the same as provided
if result.Link == link {
// Return index with true
return index, true
}
}