Compare commits

...

11 Commits

Author SHA1 Message Date
d6698b3a03 Update import path
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-10-07 17:35:22 -07:00
8b97df59e1 Set platform to amd64
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-10-07 12:02:43 -07:00
e429c904d4 Move lure-updater to updater.lure.sh
All checks were successful
ci/woodpecker/manual/woodpecker Pipeline was successful
2023-10-07 11:50:57 -07:00
d5775b9fa2 Update lure-repo URL 2023-10-06 19:17:08 -07:00
029dfab35f Create a readerValue for values that can implement the io.Reader interface 2023-09-30 16:59:10 -07:00
6272e5e044 Move all HTTP error handling into error handling middleware
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-06-26 19:43:27 -07:00
d90da5dcf9 Accept reader as a valid type for bodyReader
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-06-26 19:28:00 -07:00
f5c8ba2282 Add html builtin module
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-06-25 13:07:38 -07:00
a30d805f9b Add utils.ver_cmp() builtin
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-06-22 14:53:42 -07:00
228bf59c85 Add starlark reader implementation
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-06-22 14:47:22 -07:00
2cdea934eb Change the nomad service name to fix a collision
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-06-21 21:24:32 +00:00
16 changed files with 661 additions and 72 deletions

3
.gitignore vendored
View File

@ -1,3 +1,4 @@
/lure-updater.toml
/lure-updater
/dist/
/dist/
*.star

View File

@ -29,7 +29,7 @@ blobs:
folder: "/"
release:
gitea:
owner: Elara6331
owner: lure
name: lure-updater
gitea_urls:
api: 'https://gitea.elara.ws/api/v1/'

View File

@ -1,3 +1,4 @@
platform: linux/amd64
pipeline:
release:
image: goreleaser/goreleaser

View File

@ -1,12 +1,12 @@
# LURE Updater
Modular bot that automatically checks for upstream updates and pushes new packages to [lure-repo](https://github.com/Elara6331/lure-repo).
Modular bot that automatically checks for upstream updates and pushes new packages to [lure-repo](https://github.com/lure-sh/lure-repo).
---
### How it works
Since LURE is meant to be able to install many different types of packages, this bot accepts [plugins](https://gitea.elara.ws/Elara6331/updater-plugins) in the form of [Starlark](https://github.com/bazelbuild/starlark) files rather than hardcoding each package. These plugins can schedule functions to be run at certain intervals, or when a webhook is received, and they have access to persistent key/value storage to keep track of information. This allows plugins to use many different ways to detect upstream updates.
Since LURE is meant to be able to install many different types of packages, this bot accepts [plugins](https://gitea.elara.ws/lure/updater-plugins) in the form of [Starlark](https://github.com/bazelbuild/starlark) files rather than hardcoding each package. These plugins can schedule functions to be run at certain intervals, or when a webhook is received, and they have access to persistent key/value storage to keep track of information. This allows plugins to use many different ways to detect upstream updates.
For example, the plugin for `discord-bin` repeatedly polls discord's API every hour for the current latest download link. It puts the link in persistent storage, and if it has changed since last time, it parses the URL to extract the version number, and uses that to update the build script for `discord-bin`.

8
go.mod
View File

@ -1,14 +1,17 @@
module go.elara.ws/lure-updater
module lure.sh/lure-updater
go 1.20
require (
github.com/PuerkitoBio/goquery v1.8.1
github.com/caarlos0/env/v8 v8.0.0
github.com/go-git/go-git/v5 v5.7.0
github.com/pelletier/go-toml/v2 v2.0.8
github.com/spf13/pflag v1.0.5
github.com/vmihailenco/msgpack/v5 v5.3.5
go.elara.ws/logger v0.0.0-20230421022458-e80700db2090
go.elara.ws/pcre v0.0.0-20230421030233-daf2d2e6973f
go.elara.ws/vercmp v0.0.0-20230622214216-0b2b067575c4
go.etcd.io/bbolt v1.3.7
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca
golang.org/x/crypto v0.9.0
@ -19,6 +22,7 @@ require (
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
@ -34,8 +38,10 @@ require (
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/skeema/knownhosts v1.1.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sys v0.8.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect

15
go.sum
View File

@ -4,8 +4,12 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VM
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek=
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
@ -104,11 +108,16 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
@ -119,6 +128,8 @@ go.elara.ws/logger v0.0.0-20230421022458-e80700db2090 h1:RVC8XvWo6Yw4HUshqx4TSzu
go.elara.ws/logger v0.0.0-20230421022458-e80700db2090/go.mod h1:qng49owViqsW5Aey93lwBXONw20oGbJIoLVscB16mPM=
go.elara.ws/pcre v0.0.0-20230421030233-daf2d2e6973f h1:ZwR0xvBeP5BHHv63fgfuwhZIj+Si5rp79WSDUE73ZVA=
go.elara.ws/pcre v0.0.0-20230421030233-daf2d2e6973f/go.mod h1:EF48C6VnP4wBayzFGk6lXqbiLucH7EfiaYOgiiCe5k4=
go.elara.ws/vercmp v0.0.0-20230622214216-0b2b067575c4 h1:Ep54XceQlKhcCHl9awG+wWP4kz4kIP3c3Lzw/Gc/zwY=
go.elara.ws/vercmp v0.0.0-20230622214216-0b2b067575c4/go.mod h1:/7PNW7nFnDR5W7UXZVc04gdVLR/wBNgkm33KgIz0OBk=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
@ -132,6 +143,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU
golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@ -146,9 +159,11 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=

216
internal/builtins/html.go Normal file
View File

@ -0,0 +1,216 @@
package builtins
import (
"github.com/PuerkitoBio/goquery"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
)
var (
_ starlark.Iterable = (*starlarkSelection)(nil)
_ starlark.Sliceable = (*starlarkSelection)(nil)
_ starlark.Sequence = (*starlarkSelection)(nil)
_ starlark.Value = (*starlarkSelection)(nil)
)
var htmlModule = &starlarkstruct.Module{
Name: "html",
Members: starlark.StringDict{
"parse": starlark.NewBuiltin("html.parse", htmlParse),
},
}
func htmlParse(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var r readerValue
err := starlark.UnpackArgs("html.selection.find", args, kwargs, "from", &r)
if err != nil {
return nil, err
}
defer r.Close()
doc, err := goquery.NewDocumentFromReader(r)
if err != nil {
return nil, err
}
return newStarlarkSelection(doc.Selection), nil
}
type starlarkSelection struct {
sel *goquery.Selection
*starlarkstruct.Struct
}
func newStarlarkSelection(sel *goquery.Selection) starlarkSelection {
ss := starlarkSelection{sel: sel}
ss.Struct = starlarkstruct.FromStringDict(starlark.String("html.selection"), starlark.StringDict{
"text": starlark.NewBuiltin("html.selection.text", ss.text),
"html": starlark.NewBuiltin("html.selection.html", ss.html),
"children": starlark.NewBuiltin("html.selection.children", ss.children),
"parent": starlark.NewBuiltin("html.selection.parent", ss.parent),
"find": starlark.NewBuiltin("html.selection.find", ss.find),
"add": starlark.NewBuiltin("html.selection.add", ss.add),
"attr": starlark.NewBuiltin("html.selection.attr", ss.attr),
"has_class": starlark.NewBuiltin("html.selection.has_class", ss.hasClass),
"index_selector": starlark.NewBuiltin("html.selection.index_selector", ss.indexSelector),
"and_self": starlark.NewBuiltin("html.selection.and_self", ss.andSelf),
"first": starlark.NewBuiltin("html.selection.first", ss.first),
"last": starlark.NewBuiltin("html.selection.last", ss.last),
"next": starlark.NewBuiltin("html.selection.last", ss.next),
"next_all": starlark.NewBuiltin("html.selection.next_all", ss.nextAll),
"next_until": starlark.NewBuiltin("html.selection.next_until", ss.nextUntil),
"prev": starlark.NewBuiltin("html.selection.prev", ss.prev),
"prev_all": starlark.NewBuiltin("html.selection.prev_all", ss.prevAll),
"prev_until": starlark.NewBuiltin("html.selection.prev_until", ss.prevUntil),
})
return ss
}
func (ss starlarkSelection) Truth() starlark.Bool {
return len(ss.sel.Nodes) > 0
}
func (ss starlarkSelection) Len() int {
return ss.sel.Length()
}
func (ss starlarkSelection) Index(i int) starlark.Value {
return newStarlarkSelection(ss.sel.Slice(i, i+1))
}
func (ss starlarkSelection) Slice(start, end, _ int) starlark.Value {
return newStarlarkSelection(ss.sel.Slice(start, end))
}
func (ss starlarkSelection) Iterate() starlark.Iterator {
return newSelectionIterator(ss.sel)
}
func (ss starlarkSelection) text(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return starlark.String(ss.sel.Text()), nil
}
func (ss starlarkSelection) html(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
s, err := ss.sel.Html()
return starlark.String(s), err
}
func (ss starlarkSelection) children(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return newStarlarkSelection(ss.sel.Children()), nil
}
func (ss starlarkSelection) parent(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return newStarlarkSelection(ss.sel.Parent()), nil
}
func (ss starlarkSelection) find(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var selector string
err := starlark.UnpackArgs("html.selection.find", args, kwargs, "selector", &selector)
if err != nil {
return nil, err
}
return newStarlarkSelection(ss.sel.Find(selector)), nil
}
func (ss starlarkSelection) add(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var selector string
err := starlark.UnpackArgs("html.selection.add", args, kwargs, "selector", &selector)
if err != nil {
return nil, err
}
return newStarlarkSelection(ss.sel.Add(selector)), nil
}
func (ss starlarkSelection) indexSelector(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var selector string
err := starlark.UnpackArgs("html.selection.index_selector", args, kwargs, "selector", &selector)
if err != nil {
return nil, err
}
return starlark.MakeInt(ss.sel.IndexSelector(selector)), nil
}
func (ss starlarkSelection) attr(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name, def string
err := starlark.UnpackArgs("html.selection.find", args, kwargs, "name", &name, "default??", &def)
if err != nil {
return nil, err
}
return starlark.String(ss.sel.AttrOr(name, def)), nil
}
func (ss starlarkSelection) hasClass(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
err := starlark.UnpackArgs("html.selection.has_class", args, kwargs, "name", &name)
if err != nil {
return nil, err
}
return starlark.Bool(ss.sel.HasClass(name)), nil
}
func (ss starlarkSelection) andSelf(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return newStarlarkSelection(ss.sel.AndSelf()), nil
}
func (ss starlarkSelection) first(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return newStarlarkSelection(ss.sel.First()), nil
}
func (ss starlarkSelection) last(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return newStarlarkSelection(ss.sel.Last()), nil
}
func (ss starlarkSelection) next(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return newStarlarkSelection(ss.sel.Next()), nil
}
func (ss starlarkSelection) nextAll(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return newStarlarkSelection(ss.sel.NextAll()), nil
}
func (ss starlarkSelection) nextUntil(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var selector string
err := starlark.UnpackArgs("html.selection.next_until", args, kwargs, "selector", &selector)
if err != nil {
return nil, err
}
return newStarlarkSelection(ss.sel.NextUntil(selector)), nil
}
func (ss starlarkSelection) prev(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return newStarlarkSelection(ss.sel.Prev()), nil
}
func (ss starlarkSelection) prevAll(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
return newStarlarkSelection(ss.sel.PrevAll()), nil
}
func (ss starlarkSelection) prevUntil(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var selector string
err := starlark.UnpackArgs("html.selection.prev_until", args, kwargs, "selector", &selector)
if err != nil {
return nil, err
}
return newStarlarkSelection(ss.sel.PrevUntil(selector)), nil
}
type starlarkSelectionIterator struct {
sel *goquery.Selection
index int
}
func newSelectionIterator(sel *goquery.Selection) *starlarkSelectionIterator {
return &starlarkSelectionIterator{sel: sel}
}
func (ssi *starlarkSelectionIterator) Next(v *starlark.Value) bool {
if ssi.index == ssi.sel.Length() {
return false
}
*v = newStarlarkSelection(ssi.sel.Slice(ssi.index, ssi.index+1))
ssi.index++
return true
}
func (ssi *starlarkSelectionIterator) Done() {}

View File

@ -27,7 +27,7 @@ import (
"strings"
"go.elara.ws/logger/log"
"go.elara.ws/lure-updater/internal/config"
"lure.sh/lure-updater/internal/config"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
"golang.org/x/crypto/bcrypt"
@ -80,6 +80,8 @@ func (sbr *starlarkBodyReader) Unpack(v starlark.Value) error {
sbr.Reader = strings.NewReader(string(v))
case starlark.Bytes:
sbr.Reader = strings.NewReader(string(v))
case starlarkReader:
sbr.Reader = v
default:
return fmt.Errorf("%w: %s", ErrInvalidBodyType, v.Type())
}
@ -174,7 +176,7 @@ func starlarkResponse(res *http.Response) *starlarkstruct.Struct {
return starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{
"code": starlark.MakeInt(res.StatusCode),
"headers": starlarkStringSliceMap(res.Header),
"body": starlarkBody(res.Body),
"body": newStarlarkReader(res.Body),
})
}
@ -184,7 +186,7 @@ func starlarkRequest(req *http.Request) *starlarkstruct.Struct {
"remote_addr": starlark.String(req.RemoteAddr),
"headers": starlarkStringSliceMap(req.Header),
"query": starlarkStringSliceMap(req.URL.Query()),
"body": starlarkBody(req.Body),
"body": newStarlarkReader(req.Body),
})
}
@ -200,44 +202,6 @@ func starlarkStringSliceMap(ssm map[string][]string) *starlark.Dict {
return dict
}
func starlarkBody(body io.ReadCloser) *starlarkstruct.Struct {
return starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{
"string": bodyAsString(body),
"bytes": bodyAsBytes(body),
"close": bodyClose(body),
})
}
func bodyAsBytes(body io.ReadCloser) *starlark.Builtin {
return starlark.NewBuiltin("http.response.body.bytes", func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
data, err := io.ReadAll(io.LimitReader(body, maxBodySize))
if err != nil {
return nil, err
}
return starlark.Bytes(data), nil
})
}
func bodyAsString(body io.ReadCloser) *starlark.Builtin {
return starlark.NewBuiltin("http.response.body.string", func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
data, err := io.ReadAll(io.LimitReader(body, maxBodySize))
if err != nil {
return nil, err
}
return starlark.String(data), nil
})
}
func bodyClose(body io.ReadCloser) *starlark.Builtin {
return starlark.NewBuiltin("http.response.body.close", func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
err := body.Close()
if err != nil {
return nil, err
}
return starlark.None, nil
})
}
func registerWebhook(mux *http.ServeMux, cfg *config.Config, pluginName string) *starlark.Builtin {
return starlark.NewBuiltin("register_webhook", func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var fn *starlark.Function
@ -259,7 +223,7 @@ func registerWebhook(mux *http.ServeMux, cfg *config.Config, pluginName string)
}
func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *starlark.Thread, fn *starlark.Function) http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
return handleError(func(res http.ResponseWriter, req *http.Request) *HTTPError {
defer req.Body.Close()
res.Header().Add("X-Updater-Plugin", pluginName)
@ -267,20 +231,22 @@ func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *
if secure {
err := verifySecure(cfg.Webhook.PasswordHash, pluginName, req)
if err != nil {
log.Error("Error verifying webhook").Err(err).Send()
res.WriteHeader(http.StatusForbidden)
_, _ = io.WriteString(res, err.Error())
return
return &HTTPError{
Message: "Error verifying webhook",
Code: http.StatusForbidden,
Err: err,
}
}
}
log.Debug("Calling webhook function").Str("name", fn.Name()).Stringer("pos", fn.Position()).Send()
val, err := starlark.Call(thread, fn, starlark.Tuple{starlarkRequest(req)}, nil)
if err != nil {
log.Error("Error while executing webhook").Err(err).Stringer("pos", fn.Position()).Send()
res.WriteHeader(http.StatusInternalServerError)
_, _ = io.WriteString(res, err.Error())
return
return &HTTPError{
Message: "Error while executing webhook",
Code: http.StatusInternalServerError,
Err: err,
}
}
switch val := val.(type) {
@ -298,13 +264,20 @@ func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *
body := newBodyReader()
err = body.Unpack(val)
if err != nil {
log.Error("Error unpacking returned body").Err(err).Send()
return
return &HTTPError{
Message: "Error unpacking returned body",
Code: http.StatusInternalServerError,
Err: err,
}
}
_, err = io.Copy(res, body)
if err != nil {
log.Error("Error writing body").Err(err).Send()
return
return &HTTPError{
Message: "Error writing body",
Code: http.StatusInternalServerError,
Err: err,
}
}
case *starlark.Dict:
code := http.StatusOK
@ -312,8 +285,11 @@ func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *
if ok {
err = starlark.AsInt(codeVal, &code)
if err != nil {
log.Error("Error decoding returned status code").Err(err).Send()
return
return &HTTPError{
Message: "Error decoding returned status code",
Code: http.StatusInternalServerError,
Err: err,
}
}
res.WriteHeader(code)
}
@ -323,16 +299,40 @@ func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *
if ok {
err = body.Unpack(bodyVal)
if err != nil {
log.Error("Error unpacking returned body").Err(err).Send()
return
return &HTTPError{
Message: "Error unpacking returned body",
Code: http.StatusInternalServerError,
Err: err,
}
}
_, err = io.Copy(res, body)
if err != nil {
log.Error("Error writing body").Err(err).Send()
return
return &HTTPError{
Message: "Error writing body",
Code: http.StatusInternalServerError,
Err: err,
}
}
}
}
return nil
})
}
type HTTPError struct {
Code int
Message string
Err error
}
func handleError(h func(res http.ResponseWriter, req *http.Request) *HTTPError) http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
httpErr := h(res, req)
if httpErr != nil {
log.Error(httpErr.Message).Err(httpErr.Err).Send()
res.WriteHeader(httpErr.Code)
fmt.Sprintf("%s: %s", httpErr.Message, httpErr.Err)
}
}
}

238
internal/builtins/reader.go Normal file
View File

@ -0,0 +1,238 @@
package builtins
import (
"bufio"
"encoding/json"
"errors"
"io"
"strings"
"github.com/vmihailenco/msgpack/v5"
"lure.sh/lure-updater/internal/convert"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
)
type starlarkReader struct {
closeFunc func() error
br *bufio.Reader
*starlarkstruct.Struct
}
func newStarlarkReader(r io.Reader) starlarkReader {
sr := starlarkReader{br: bufio.NewReader(r)}
if rc, ok := r.(io.ReadCloser); ok {
sr.closeFunc = rc.Close
}
sr.Struct = starlarkstruct.FromStringDict(starlark.String("regex"), starlark.StringDict{
"read": starlark.NewBuiltin("reader.read", sr.read),
"peek": starlark.NewBuiltin("reader.peek", sr.peek),
"discard": starlark.NewBuiltin("reader.discard", sr.discard),
"read_string": starlark.NewBuiltin("reader.read_string", sr.readString),
"read_until": starlark.NewBuiltin("reader.read_until", sr.readUntil),
"read_string_until": starlark.NewBuiltin("reader.read_string_until", sr.readStringUntil),
"read_all": starlark.NewBuiltin("reader.read_all", sr.readAll),
"read_all_string": starlark.NewBuiltin("reader.read_all_string", sr.readAllString),
"read_json": starlark.NewBuiltin("reader.read_json", sr.readJSON),
"read_msgpack": starlark.NewBuiltin("reader.read_msgpack", sr.readMsgpack),
"close": starlark.NewBuiltin("reader.close", sr.closeReader),
})
return sr
}
func (sr starlarkReader) read(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var n int
err := starlark.UnpackArgs("reader.read", args, kwargs, "n", &n)
if err != nil {
return nil, err
}
buf := make([]byte, n)
_, err = io.ReadFull(sr.br, buf)
if err != nil {
return nil, err
}
return starlark.Bytes(buf), nil
}
func (sr starlarkReader) readString(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var n int
err := starlark.UnpackArgs("reader.read_string", args, kwargs, "n", &n)
if err != nil {
return nil, err
}
buf := make([]byte, n)
_, err = io.ReadFull(sr.br, buf)
if err != nil {
return nil, err
}
return starlark.String(buf), nil
}
func (sr starlarkReader) readUntil(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var delimiter string
err := starlark.UnpackArgs("reader.read_until", args, kwargs, "delimiter", &delimiter)
if err != nil {
return nil, err
}
buf, err := sr.br.ReadBytes(delimiter[0])
if err != nil {
return nil, err
}
return starlark.Bytes(buf), nil
}
func (sr starlarkReader) readStringUntil(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var delimiter string
err := starlark.UnpackArgs("reader.read_string_until", args, kwargs, "delimiter", &delimiter)
if err != nil {
return nil, err
}
buf, err := sr.br.ReadString(delimiter[0])
if err != nil {
return nil, err
}
return starlark.String(buf), nil
}
func (sr starlarkReader) peek(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var n int
err := starlark.UnpackArgs("reader.peek", args, kwargs, "n", &n)
if err != nil {
return nil, err
}
buf, err := sr.br.Peek(n)
if err != nil {
return nil, err
}
return starlark.Bytes(buf), nil
}
func (sr starlarkReader) discard(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var n int
err := starlark.UnpackArgs("reader.discard", args, kwargs, "n", &n)
if err != nil {
return nil, err
}
dn, err := sr.br.Discard(n)
if err != nil {
return nil, err
}
return starlark.MakeInt(dn), nil
}
func (sr starlarkReader) readAll(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var limit int64 = 102400
err := starlark.UnpackArgs("reader.read_all", args, kwargs, "limit??", &limit)
if err != nil {
return nil, err
}
var r io.Reader = sr.br
if limit > 0 {
r = io.LimitReader(sr.br, limit)
}
buf, err := io.ReadAll(r)
if err != nil {
return nil, err
}
return starlark.Bytes(buf), nil
}
func (sr starlarkReader) readAllString(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var limit int64 = 102400
err := starlark.UnpackArgs("reader.read_all_string", args, kwargs, "limit??", &limit)
if err != nil {
return nil, err
}
var r io.Reader = sr.br
if limit > 0 {
r = io.LimitReader(sr.br, limit)
}
buf, err := io.ReadAll(r)
if err != nil {
return nil, err
}
return starlark.String(buf), nil
}
func (sr starlarkReader) readJSON(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var v any
err := json.NewDecoder(sr.br).Decode(&v)
if err != nil {
return nil, err
}
return convert.Convert(v)
}
func (sr starlarkReader) readMsgpack(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var v any
err := msgpack.NewDecoder(sr.br).Decode(&v)
if err != nil {
return nil, err
}
return convert.Convert(v)
}
func (sr starlarkReader) closeReader(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
if sr.closeFunc != nil {
err := sr.closeFunc()
if err != nil {
return nil, err
}
}
return starlark.None, nil
}
// Read implements the io.ReadCloser interface
func (sr starlarkReader) Read(b []byte) (int, error) {
return sr.br.Read(b)
}
// Close implements the io.ReadCloser interface
func (sr starlarkReader) Close() error {
if sr.closeFunc != nil {
return sr.closeFunc()
}
return nil
}
type readerValue struct {
io.ReadCloser
}
func (rv *readerValue) Unpack(v starlark.Value) error {
switch val := v.(type) {
case starlark.String:
rv.ReadCloser = io.NopCloser(strings.NewReader(string(val)))
case starlark.Bytes:
rv.ReadCloser = io.NopCloser(strings.NewReader(string(val)))
case starlarkReader:
rv.ReadCloser = val
}
if rv.ReadCloser == nil {
return errors.New("invalid type for reader")
}
return nil
}

View File

@ -21,7 +21,7 @@ package builtins
import (
"net/http"
"go.elara.ws/lure-updater/internal/config"
"lure.sh/lure-updater/internal/config"
"go.etcd.io/bbolt"
"go.starlark.net/starlark"
"go.starlark.net/starlarkjson"
@ -43,5 +43,7 @@ func Register(sd starlark.StringDict, opts *Options) {
sd["updater"] = updaterModule(opts.Config)
sd["log"] = logModule(opts.Name)
sd["json"] = starlarkjson.Module
sd["utils"] = utilsModule
sd["html"] = htmlModule
sd["register_webhook"] = registerWebhook(opts.Mux, opts.Config, opts.Name)
}

View File

@ -30,7 +30,7 @@ import (
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"go.elara.ws/logger/log"
"go.elara.ws/lure-updater/internal/config"
"lure.sh/lure-updater/internal/config"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
)

View File

@ -0,0 +1,23 @@
package builtins
import (
"go.elara.ws/vercmp"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
)
var utilsModule = &starlarkstruct.Module{
Name: "utils",
Members: starlark.StringDict{
"ver_cmp": starlark.NewBuiltin("utils.ver_cmp", utilsVerCmp),
},
}
func utilsVerCmp(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var v1, v2 string
err := starlark.UnpackArgs("utils.ver_cmp", args, kwargs, "v1", &v1, "v2", &v2)
if err != nil {
return nil, err
}
return starlark.MakeInt(vercmp.Compare(v1, v2)), nil
}

View File

@ -0,0 +1,87 @@
package convert
import (
"errors"
"fmt"
"reflect"
"go.starlark.net/starlark"
)
var ErrInvalidType = errors.New("unknown type")
func Convert(v any) (starlark.Value, error) {
if v == nil {
return starlark.None, nil
}
val := reflect.ValueOf(v)
kind := val.Kind()
for kind == reflect.Pointer || kind == reflect.Interface {
val = val.Elem()
}
return convert(val)
}
func convert(val reflect.Value) (starlark.Value, error) {
switch val.Kind() {
case reflect.Interface:
return convert(val.Elem())
case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8:
return starlark.MakeInt64(val.Int()), nil
case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8:
return starlark.MakeUint64(val.Uint()), nil
case reflect.Float64, reflect.Float32:
return starlark.Float(val.Float()), nil
case reflect.Bool:
return starlark.Bool(val.Bool()), nil
case reflect.String:
return starlark.String(val.String()), nil
case reflect.Slice, reflect.Array:
return convertSlice(val)
case reflect.Map:
return convertMap(val)
default:
return nil, fmt.Errorf("%w: %s", ErrInvalidType, val.Type())
}
}
func convertSlice(val reflect.Value) (starlark.Value, error) {
// Detect byte slice
if val.Type().Elem().Kind() == reflect.Uint8 {
return starlark.Bytes(val.Bytes()), nil
}
elems := make([]starlark.Value, val.Len())
for i := 0; i < val.Len(); i++ {
elem, err := convert(val.Index(i))
if err != nil {
return nil, err
}
elems[i] = elem
}
return starlark.NewList(elems), nil
}
func convertMap(val reflect.Value) (starlark.Value, error) {
dict := starlark.NewDict(val.Len())
iter := val.MapRange()
for iter.Next() {
k, err := convert(iter.Key())
if err != nil {
return nil, err
}
v, err := convert(iter.Value())
if err != nil {
return nil, err
}
err = dict.SetKey(k, v)
if err != nil {
return nil, err
}
}
return dict, nil
}

View File

@ -1,5 +1,5 @@
[git]
repoURL = "https://github.com/Elara6331/lure-repo.git"
repoURL = "https://github.com/lure-sh/lure-repo.git"
repoDir = "/etc/lure-updater/repo"
[git.commit]
# The name and email to use in the git commit

View File

@ -31,8 +31,8 @@ import (
"github.com/spf13/pflag"
"go.elara.ws/logger"
"go.elara.ws/logger/log"
"go.elara.ws/lure-updater/internal/builtins"
"go.elara.ws/lure-updater/internal/config"
"lure.sh/lure-updater/internal/builtins"
"lure.sh/lure-updater/internal/config"
"go.etcd.io/bbolt"
"go.starlark.net/starlark"
"golang.org/x/crypto/bcrypt"

View File

@ -27,7 +27,7 @@ job "lure-updater" {
env {
GIT_REPO_DIR = "/etc/lure-updater/repo"
GIT_REPO_URL = "https://github.com/Elara6331/lure-repo.git"
GIT_REPO_URL = "https://github.com/lure-sh/lure-repo.git"
GIT_CREDENTIALS_USERNAME = "lure-repo-bot"
GIT_CREDENTIALS_PASSWORD = "${GITHUB_PASSWORD}"
GIT_COMMIT_NAME = "lure-repo-bot"
@ -53,12 +53,12 @@ job "lure-updater" {
}
service {
name = "site"
name = "lure-updater"
port = "webhook"
tags = [
"traefik.enable=true",
"traefik.http.routers.lure-updater.rule=Host(`updater.lure.elara.ws`)",
"traefik.http.routers.lure-updater.rule=Host(`updater.lure.sh`)",
"traefik.http.routers.lure-updater.tls.certResolver=letsencrypt",
]
}