scope/search/web/ddg.go

/*
 *	Scope - A simple and minimal metasearch engine
 *	Copyright (C) 2021 Arsen Musayelyan
 *
 *	This program is free software: you can redistribute it and/or modify
 *	it under the terms of the GNU Affero General Public License as published by
 *	the Free Software Foundation, either version 3 of the License, or
 *	(at your option) any later version.
 *
 *	This program is distributed in the hope that it will be useful,
 *	but WITHOUT ANY WARRANTY; without even the implied warranty of
 *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *	GNU Affero General Public License for more details.
 *
 *	You should have received a copy of the GNU Affero General Public License
 *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package web

import (
	"net/http"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

var ddgURL = urlMustParse("https://html.duckduckgo.com/html")

const uddgPrefix = "//duckduckgo.com/l/?uddg="

// DDG represents the DuckDuckGo search engine
type DDG struct {
	keyword   string
	userAgent string
	page      int
	doc       *goquery.Document
	initDone  bool
	baseSel   *goquery.Selection
}

// SetKeyword sets the keyword for searching
func (d *DDG) SetKeyword(keyword string) {
	d.keyword = keyword
}

// SetPage sets the page number for searching
func (d *DDG) SetPage(page int) {
	d.page = page * 30
}

// SetUserAgent sets the user agent for the request
func (d *DDG) SetUserAgent(ua string) {
	d.userAgent = ua
}

// Init runs requests for the DuckDuckGo search engine
func (d *DDG) Init() error {
	// Copy URL so that it can be changed
	initURL := copyURL(ddgURL)
	// Get query parameters
	query := initURL.Query()
	// Set query
	query.Set("q", d.keyword)
	if d.page > 0 {
		query.Set("s", strconv.Itoa(d.page))
		query.Set("dc", strconv.Itoa(d.page+1))
	}
	// Update URL query
	initURL.RawQuery = query.Encode()

	// Create new request for modified URL
	req, err := http.NewRequest(
		http.MethodGet,
		initURL.String(),
		nil,
	)
	if err != nil {
		return err
	}
	// If user agent empty, use default
	if d.userAgent == "" {
		d.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
	}
	// Set user agent of request
	req.Header.Set("User-Agent", d.userAgent)

	// Perform request
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	// Create goquery document from reader
	doc, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		return err
	}
	d.doc = doc
	d.baseSel = doc.Find(`#links > .result`)
	d.initDone = true
	return nil
}

// Each runs eachCb with the index of each search result
func (d *DDG) Each(eachCb func(int) error) error {
	for i := 0; i < d.baseSel.Length(); i++ {
		err := eachCb(i)
		if err != nil {
			return err
		}
	}
	return nil
}

// Title returns the title of the search result corresponding to i
func (d *DDG) Title(i int) (string, error) {
	return strings.TrimSpace(get(d.baseSel, i).Children().First().ChildrenFiltered("h2").Text()), nil
}

// Link returns the link to the search result corresponding to i
func (d *DDG) Link(i int) (string, error) {
	link := get(d.baseSel, i).Children().First().ChildrenFiltered("a").AttrOr("href", "")
	if strings.HasPrefix(link, uddgPrefix) {
		link = urlMustParse(link).Query().Get("uddg")
	}
	return link, nil
}

// Desc returns the description of the search result corresponding to i
func (d *DDG) Desc(i int) (string, error) {
	return get(d.baseSel, i).Children().First().ChildrenFiltered("a").Text(), nil
}

// Name returns "ddg"
func (d *DDG) Name() string {
	return "ddg"
}