Add AOL engine

.gitignore

@@ -1,2 +1,4 @@
-/static/
+/static/ext/
 /scope
+/cmd/test/
+/test

Makefile

@@ -6,18 +6,18 @@ build:
 .PHONY: build
 
 static:
-	mkdir -p static
+	mkdir -p static/ext
 
-	wget https://cdn.jsdelivr.net/npm/bulma@0.9.3/css/bulma.min.css -O static/bulma.min.css
+	wget https://cdn.jsdelivr.net/npm/bulma@0.9.3/css/bulma.min.css -O static/ext/bulma.min.css
 
-	wget https://code.iconify.design/2/2.1.0/iconify.min.js -O static/iconify.min.js
+	wget https://code.iconify.design/2/2.1.0/iconify.min.js -O static/ext/iconify.min.js
 	
-	wget https://cdn.jsdelivr.net/npm/katex@0.15.1/dist/katex.min.css -O static/katex.min.css
-	wget https://cdn.jsdelivr.net/npm/katex@0.15.1/dist/katex.min.js -O static/katex.min.js
+	wget https://cdn.jsdelivr.net/npm/katex@0.15.1/dist/katex.min.css -O static/ext/katex.min.css
+	wget https://cdn.jsdelivr.net/npm/katex@0.15.1/dist/katex.min.js -O static/ext/katex.min.js
 
-	wget https://cdn.jsdelivr.net/npm/nerdamer@latest/nerdamer.core.js -O static/nerdamer.core.js
-	wget https://cdn.jsdelivr.net/npm/nerdamer@latest/Algebra.js -O static/Algebra.js
-	wget https://cdn.jsdelivr.net/npm/nerdamer@latest/Calculus.js -O static/Calculus.js
-	wget https://cdn.jsdelivr.net/npm/nerdamer@latest/Solve.js -O static/Solve.js
+	wget https://cdn.jsdelivr.net/npm/nerdamer@latest/nerdamer.core.js -O static/ext/nerdamer.core.js
+	wget https://cdn.jsdelivr.net/npm/nerdamer@latest/Algebra.js -O static/ext/Algebra.js
+	wget https://cdn.jsdelivr.net/npm/nerdamer@latest/Calculus.js -O static/ext/Calculus.js
+	wget https://cdn.jsdelivr.net/npm/nerdamer@latest/Solve.js -O static/ext/Solve.js
 
-	wget https://unpkg.com/function-plot/dist/function-plot.js -O static/function-plot.js
+	wget https://unpkg.com/function-plot/dist/function-plot.js -O static/ext/function-plot.js

README.md

@@ -94,3 +94,4 @@ This project uses many other projects. Those projects and the reasons for using
 - [Function Plot](https://mauriciopoppe.github.io/function-plot/): Equation grapher
 - [Metaweather](https://www.metaweather.com/): Weather API
 - [DuckDuckGo](https://duckduckgo.com/): Provides instant answers
+- [Dark Reader](https://darkreader.org/): Dark mode CSS

internal/cards/calc.go

@@ -1,3 +1,21 @@
+/*
+ *	Scope - A simple and minimal metasearch engine
+ *	Copyright (C) 2021 Arsen Musayelyan
+ *
+ *	This program is free software: you can redistribute it and/or modify
+ *	it under the terms of the GNU Affero General Public License as published by
+ *	the Free Software Foundation, either version 3 of the License, or
+ *	(at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU Affero General Public License for more details.
+ *
+ *	You should have received a copy of the GNU Affero General Public License
+ *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package cards
 
 import (
@@ -8,19 +26,23 @@ import (
 )
 
 const calcExtraHead = `
-<link rel="stylesheet" href="/static/katex.min.css">
-<script defer src="/static/katex.min.js"></script>
+<!-- Import KaTeX for math rendering -->
+<link rel="stylesheet" href="/static/ext/katex.min.css">
+<script defer src="/static/ext/katex.min.js"></script>
 
-<script src="/static/nerdamer.core.js"></script>
-<script src="/static/Algebra.js"></script>
-<script src="/static/Calculus.js"></script>
-<script src="/static/Solve.js"></script>`
+<!-- Import Nerdamer for equation evaluator -->
+<script src="/static/ext/nerdamer.core.js"></script>
+<script src="/static/ext/Algebra.js"></script>
+<script src="/static/ext/Calculus.js"></script>
+<script src="/static/ext/Solve.js"></script>`
 
 const solveRenderScript = `
 <div id="calc-content" class="subtitle mx-2 my-0"></div>
 <script>
 window.onload = () => {
-	latex = nerdamer.convertToLaTeX(nerdamer('%s').toString())
+	// Execute input and get output as LaTeX
+	latex = nerdamer('%s').latex()
+	// Use KaTeX to render output to #calc-content div
 	katex.render(latex, document.getElementById('calc-content'))
 }
 </script>`

internal/cards/cards.go

@@ -1,3 +1,21 @@
+/*
+ *	Scope - A simple and minimal metasearch engine
+ *	Copyright (C) 2021 Arsen Musayelyan
+ *
+ *	This program is free software: you can redistribute it and/or modify
+ *	it under the terms of the GNU Affero General Public License as published by
+ *	the Free Software Foundation, either version 3 of the License, or
+ *	(at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU Affero General Public License for more details.
+ *
+ *	You should have received a copy of the GNU Affero General Public License
+ *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package cards
 
 import (

internal/cards/ddg.go

@@ -1,3 +1,21 @@
+/*
+ *	Scope - A simple and minimal metasearch engine
+ *	Copyright (C) 2021 Arsen Musayelyan
+ *
+ *	This program is free software: you can redistribute it and/or modify
+ *	it under the terms of the GNU Affero General Public License as published by
+ *	the Free Software Foundation, either version 3 of the License, or
+ *	(at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU Affero General Public License for more details.
+ *
+ *	You should have received a copy of the GNU Affero General Public License
+ *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package cards
 
 import (

internal/cards/metaweather.go

@@ -1,3 +1,21 @@
+/*
+ *	Scope - A simple and minimal metasearch engine
+ *	Copyright (C) 2021 Arsen Musayelyan
+ *
+ *	This program is free software: you can redistribute it and/or modify
+ *	it under the terms of the GNU Affero General Public License as published by
+ *	the Free Software Foundation, either version 3 of the License, or
+ *	(at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU Affero General Public License for more details.
+ *
+ *	You should have received a copy of the GNU Affero General Public License
+ *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package cards
 
 import (
@@ -39,9 +57,9 @@ const metaweatherContent = `
 		<p><b>Predictability:</b> %d%%</p>
 	</div>
 	<div class="column has-text-centered">
+		<p><b>Air Pressure:</b> %s</p>
 		<p><b>Wind Direction:</b> %s</p>
 		<p><b>Location:</b> %s</p>
-		<p><b>Timezone:</b> %s</p>
 	</div>
 </div>`
 
@@ -210,12 +228,12 @@ func (mc *MetaweatherCard) Content() template.HTML {
 		convert(mc.resp.Consolidated[0].Visibility, "visibility", units.Mile),
 		// Write predictability percentage
 		mc.resp.Consolidated[0].Predictability,
+		// Write air pressure
+		convert(mc.resp.Consolidated[0].AirPressure, "pressure", units.HectoPascal),
 		// Write compass wind direction
 		mc.resp.Consolidated[0].WindCompass,
 		// Write title
 		mc.resp.Title,
-		// Write timezone
-		mc.resp.Timezone,
 	))
 }
 

internal/cards/plot.go

@@ -1,3 +1,21 @@
+/*
+ *	Scope - A simple and minimal metasearch engine
+ *	Copyright (C) 2021 Arsen Musayelyan
+ *
+ *	This program is free software: you can redistribute it and/or modify
+ *	it under the terms of the GNU Affero General Public License as published by
+ *	the Free Software Foundation, either version 3 of the License, or
+ *	(at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU Affero General Public License for more details.
+ *
+ *	You should have received a copy of the GNU Affero General Public License
+ *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package cards
 
 import (
@@ -7,7 +25,7 @@ import (
 )
 
 const plotExtraHead = `
-<script src="/static/function-plot.js"></script>
+<script src="/static/ext/function-plot.js"></script>
 <style>
 .top-right-legend {
 	display: none;
@@ -17,6 +35,7 @@ const plotExtraHead = `
 const plotScript = `
 <div id="plot-content" class="container"></div>
 <script>
+// Create function to draw plot in #plot-content
 plotFn = () => functionPlot({
 	target: '#plot-content',
 	grid: true,
@@ -25,6 +44,7 @@ plotFn = () => functionPlot({
 		fn: '%s'
 	}]
 })
+// Create resize observer that runs plot function
 new ResizeObserver(plotFn).observe(document.getElementById('plot-content'))
 </script>`
 

scope.toml

@@ -5,7 +5,8 @@
 [site]
     name = "Scope"
     baseURL = "http://localhost:8080"
-    sourceURL = "https://example.com"
+    sourceURL = "https://gitea.arsenm.dev/Arsen6331/scope"
+    theme = "dark"
 
 [search]
     engines = ["google", "ddg", "bing"]

search/web/aol.go

@@ -0,0 +1,144 @@
+package web
+
+import (
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+var aolURL = urlMustParse("https://search.aol.com/aol/search?rp=&s_chn=prt_bon&s_it=comsearch")
+
+type AOL struct {
+	keyword   string
+	userAgent string
+	page      int
+	doc       *goquery.Document
+	initDone  bool
+	baseSel   *goquery.Selection
+}
+
+// SetKeyword sets the keyword for searching
+func (a *AOL) SetKeyword(keyword string) {
+	a.keyword = keyword
+}
+
+// SetPage sets the page number for searching
+func (a *AOL) SetPage(page int) {
+	a.page = page * 10
+	if a.page > 0 {
+		a.page++
+	}
+}
+
+// SetUserAgent sets the user agent to use for the request
+func (a *AOL) SetUserAgent(ua string) {
+	a.userAgent = ua
+}
+
+// Init runs requests for Bing search engine
+func (a *AOL) Init() error {
+	// Copy URL so it can be changed
+	initURL := copyURL(aolURL)
+	query := initURL.Query()
+	// Set query
+	query.Set("q", a.keyword)
+	if a.page > 0 {
+		query.Set("b", strconv.Itoa(a.page))
+	}
+	// Update URL query parameters
+	initURL.RawQuery = query.Encode()
+
+	// Create new request for modified URL
+	req, err := http.NewRequest(
+		http.MethodGet,
+		initURL.String(),
+		nil,
+	)
+	if err != nil {
+		return err
+	}
+	// If no user agent, use default
+	if a.userAgent == "" {
+		a.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
+	}
+	// Set request user agent
+	req.Header.Set("User-Agent", a.userAgent)
+
+	// Perform request
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	// Create new goquery document
+	doc, err := goquery.NewDocumentFromReader(res.Body)
+	if err != nil {
+		return err
+	}
+	a.doc = doc
+	a.baseSel = doc.Find(`h3.title > a[href]`)
+	a.initDone = true
+	return nil
+}
+
+// Each runs eachCb with the index of each search result
+func (a *AOL) Each(eachCb func(int) error) error {
+	for i := 0; i < a.baseSel.Length(); i++ {
+		err := eachCb(i)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Title returns the title of the search result corresponding to i
+func (a *AOL) Title(i int) (string, error) {
+	return get(a.baseSel, i).Text(), nil
+}
+
+// Link returns the link to the search result corresponding to i
+func (a *AOL) Link(i int) (string, error) {
+	href := get(a.baseSel, i).AttrOr("href", "")
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return "", err
+	}
+
+	var ru string
+	splitPath := strings.Split(hrefURL.RawPath, "/")
+	for _, item := range splitPath {
+		if strings.HasPrefix(item, "RU=") {
+			ru = strings.TrimPrefix(item, "RU=")
+			break
+		}
+	}
+	if ru == "" {
+		return href, nil
+	}
+
+	return url.PathUnescape(ru)
+}
+
+// Desc returns the description of the search result corresponding to i
+func (a *AOL) Desc(i int) (string, error) {
+	return a.baseSel.
+		First().
+		Parent().
+		Parent().
+		Next().
+		Children().
+		First().
+		Text(), nil
+}
+
+// Name returns "aol"
+func (*AOL) Name() string {
+	return "aol"
+}
+
+// https://search.aol.com/aol/search?q=site%3Alinkedin.com%2Fin%2F+%22Senior+Developer%22+%22Nvidia%22&rp=&s_chn=prt_bon&s_it=comsearch

search/web/bing.go

@@ -1,3 +1,21 @@
+/*
+ *	Scope - A simple and minimal metasearch engine
+ *	Copyright (C) 2021 Arsen Musayelyan
+ *
+ *	This program is free software: you can redistribute it and/or modify
+ *	it under the terms of the GNU Affero General Public License as published by
+ *	the Free Software Foundation, either version 3 of the License, or
+ *	(at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU Affero General Public License for more details.
+ *
+ *	You should have received a copy of the GNU Affero General Public License
+ *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package web
 
 import (

search/web/ddg.go

@@ -1,3 +1,21 @@
+/*
+ *	Scope - A simple and minimal metasearch engine
+ *	Copyright (C) 2021 Arsen Musayelyan
+ *
+ *	This program is free software: you can redistribute it and/or modify
+ *	it under the terms of the GNU Affero General Public License as published by
+ *	the Free Software Foundation, either version 3 of the License, or
+ *	(at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU Affero General Public License for more details.
+ *
+ *	You should have received a copy of the GNU Affero General Public License
+ *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package web
 
 import (

search/web/google.go

@@ -1,3 +1,21 @@
+/*
+ *	Scope - A simple and minimal metasearch engine
+ *	Copyright (C) 2021 Arsen Musayelyan
+ *
+ *	This program is free software: you can redistribute it and/or modify
+ *	it under the terms of the GNU Affero General Public License as published by
+ *	the Free Software Foundation, either version 3 of the License, or
+ *	(at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU Affero General Public License for more details.
+ *
+ *	You should have received a copy of the GNU Affero General Public License
+ *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package web
 
 import (

search/web/web.go

@@ -1,3 +1,21 @@
+/*
+ *	Scope - A simple and minimal metasearch engine
+ *	Copyright (C) 2021 Arsen Musayelyan
+ *
+ *	This program is free software: you can redistribute it and/or modify
+ *	it under the terms of the GNU Affero General Public License as published by
+ *	the Free Software Foundation, either version 3 of the License, or
+ *	(at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU Affero General Public License for more details.
+ *
+ *	You should have received a copy of the GNU Affero General Public License
+ *	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package web
 
 import (

search/web/web_test.go

@@ -0,0 +1,198 @@
+package web_test
+
+import (
+	"errors"
+	"testing"
+
+	"go.arsenm.dev/scope/search/web"
+)
+
+const (
+	ErrInNone = iota
+	ErrInInit
+	ErrInLink
+	ErrInTitle
+	ErrInDesc
+)
+
+var (
+	ErrInit  = errors.New("error in init")
+	ErrLink  = errors.New("error in link")
+	ErrTitle = errors.New("error in title")
+	ErrDesc  = errors.New("error in description")
+)
+
+type TestEngine struct {
+	errIn     int
+	name      string
+	query     string
+	userAgent string
+	page      int
+	results   []TestResult
+}
+
+type TestResult struct {
+	title string
+	link  string
+	desc  string
+}
+
+func (te *TestEngine) SetKeyword(query string) {
+	te.query = query
+}
+
+func (te *TestEngine) SetUserAgent(ua string) {
+	te.userAgent = ua
+}
+
+func (te *TestEngine) SetPage(page int) {
+	te.page = page
+}
+
+func (te *TestEngine) Init() error {
+	if te.errIn == ErrInInit {
+		return ErrInit
+	}
+	te.results = append(te.results,
+		TestResult{
+			"Google",
+			"https://www.google.com",
+			"Google search engine",
+		},
+		TestResult{
+			"Wikipedia",
+			"https://wikipedia.org",
+			"Online wiki encyclopedia",
+		},
+		TestResult{
+			"Reddit",
+			"https://reddit.com",
+			"600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars 600 chars ",
+		},
+		TestResult{
+			"Example",
+			"https://example.com",
+			"",
+		},
+	)
+	return nil
+}
+
+func (te *TestEngine) Each(eachCb func(int) error) error {
+	for index := range te.results {
+		if err := eachCb(index); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (te *TestEngine) Title(i int) (string, error) {
+	if te.errIn == ErrInTitle {
+		return "", ErrTitle
+	}
+	return te.results[i].title, nil
+}
+
+func (te *TestEngine) Link(i int) (string, error) {
+	if te.errIn == ErrInLink {
+		return "", ErrLink
+	}
+	return te.results[i].link, nil
+}
+
+func (te *TestEngine) Desc(i int) (string, error) {
+	if te.errIn == ErrInDesc {
+		return "", ErrDesc
+	}
+	return te.results[i].desc, nil
+}
+
+func (te *TestEngine) Name() string {
+	return te.name
+}
+
+func TestSearch(t *testing.T) {
+	engine := &TestEngine{name: "one"}
+
+	results, err := web.Search(
+		web.Options{
+			Keyword:   "test keyword",
+			UserAgent: "TestEngine/0.0.0",
+			Page:      0,
+		},
+		engine,
+		&TestEngine{name: "two"},
+	)
+	if err != nil {
+		t.Fatalf("Error in Search(): %s", err)
+	}
+
+	if len(results) < len(engine.results)-1 {
+		t.Fatalf(
+			"Expected %d results, got %d",
+			len(engine.results),
+			len(results),
+		)
+	}
+
+	for index, result := range results {
+		if engine.results[index].desc == "" {
+			continue
+		}
+		if result.Title != engine.results[index].title {
+			t.Fatalf(
+				"Result %d: expected title %s, got %s",
+				index,
+				engine.results[index].title,
+				result.Title,
+			)
+		} else if result.Link != engine.results[index].link {
+			t.Fatalf(
+				"Result %d: expected link %s, got %s",
+				index,
+				engine.results[index].link,
+				result.Link,
+			)
+		} else if result.Desc != engine.results[index].desc &&
+			len(engine.results[index].desc) <= 500 {
+			t.Fatalf(
+				"Result %d: expected description %s, got %s",
+				index,
+				engine.results[index].desc,
+				result.Desc,
+			)
+		}
+	}
+}
+
+func TestSearchError(t *testing.T) {
+	engines := []*TestEngine{
+		{errIn: ErrInInit},
+		{errIn: ErrInTitle},
+		{errIn: ErrInLink},
+		{errIn: ErrInDesc},
+	}
+
+	for index, engine := range engines {
+		_, err := web.Search(
+			web.Options{
+				Keyword:   "test keyword",
+				UserAgent: "TestEngine/0.0.0",
+				Page:      0,
+			},
+			engine,
+		)
+		if err == nil {
+			t.Fatalf("Expected error in engine %d, received nil", index)
+		} else if err != ErrTitle && engines[index].errIn == ErrInTitle {
+			t.Fatalf("Expected error in title (index %d), received %s", index, err)
+		} else if err != ErrLink && engines[index].errIn == ErrInLink {
+			t.Fatalf("Expected error in link (index %d), received %s", index, err)
+		} else if err != ErrDesc && engines[index].errIn == ErrInDesc {
+			t.Fatalf("Expected error in description (index %d), received %s", index, err)
+		} else if err != ErrInit && engines[index].errIn == ErrInInit {
+			t.Fatalf("Expected error in init (index %d), received %s", index, err)
+		}
+	}
+}

templates/base.html

@@ -3,8 +3,11 @@
     <head>
         <title>{{template "page" .}} - {{.Config "site.name"}}</title>
         <link rel="search" type="application/opensearchdescription+xml" title='{{.Config "site.name"}}' href="/opensearch">
-        <link rel="stylesheet" href="/static/bulma.min.css">
-        <script async src="/static/iconify.min.js"></script>
+        <link rel="stylesheet" href="/static/ext/bulma.min.css">
+        <script async src="/static/ext/iconify.min.js"></script>
+        {{if .Config "site.theme" | ne "light"}}
+        <link rel="stylesheet" href="/static/DarkReader-scope.css">
+        {{end}}
         {{if .Card}}
         {{.Card.Head}}
         {{end}}