Add html builtin module

Add utils.ver_cmp() builtin
Add starlark reader implementation
2023-06-25 13:07:38 -07:00 · 2023-06-22 14:53:42 -07:00 · 2023-06-22 14:47:22 -07:00 · 2023-06-21 21:24:32 +00:00
10 changed files with 582 additions and 42 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /lure-updater.toml
 /lure-updater
-/dist/
+/dist/
+*.star
--- a/go.mod
+++ b/go.mod
@@ -3,12 +3,15 @@ module go.elara.ws/lure-updater
 go 1.20

 require (
+	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/caarlos0/env/v8 v8.0.0
 	github.com/go-git/go-git/v5 v5.7.0
 	github.com/pelletier/go-toml/v2 v2.0.8
 	github.com/spf13/pflag v1.0.5
+	github.com/vmihailenco/msgpack/v5 v5.3.5
 	go.elara.ws/logger v0.0.0-20230421022458-e80700db2090
 	go.elara.ws/pcre v0.0.0-20230421030233-daf2d2e6973f
+	go.elara.ws/vercmp v0.0.0-20230622214216-0b2b067575c4
 	go.etcd.io/bbolt v1.3.7
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca
 	golang.org/x/crypto v0.9.0
@@ -19,6 +22,7 @@ require (
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
@@ -34,8 +38,10 @@ require (
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/skeema/knownhosts v1.1.1 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
+	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
--- a/go.sum
+++ b/go.sum
@@ -4,8 +4,12 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VM
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek=
 github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
+github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
+github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
 github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
 github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
@@ -104,11 +108,16 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
+github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
@@ -119,6 +128,8 @@ go.elara.ws/logger v0.0.0-20230421022458-e80700db2090 h1:RVC8XvWo6Yw4HUshqx4TSzu
 go.elara.ws/logger v0.0.0-20230421022458-e80700db2090/go.mod h1:qng49owViqsW5Aey93lwBXONw20oGbJIoLVscB16mPM=
 go.elara.ws/pcre v0.0.0-20230421030233-daf2d2e6973f h1:ZwR0xvBeP5BHHv63fgfuwhZIj+Si5rp79WSDUE73ZVA=
 go.elara.ws/pcre v0.0.0-20230421030233-daf2d2e6973f/go.mod h1:EF48C6VnP4wBayzFGk6lXqbiLucH7EfiaYOgiiCe5k4=
+go.elara.ws/vercmp v0.0.0-20230622214216-0b2b067575c4 h1:Ep54XceQlKhcCHl9awG+wWP4kz4kIP3c3Lzw/Gc/zwY=
+go.elara.ws/vercmp v0.0.0-20230622214216-0b2b067575c4/go.mod h1:/7PNW7nFnDR5W7UXZVc04gdVLR/wBNgkm33KgIz0OBk=
 go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
 go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
@@ -132,6 +143,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU
 golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
 golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -146,9 +159,11 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
--- a/internal/builtins/html.go
+++ b/internal/builtins/html.go
@@ -0,0 +1,229 @@
+package builtins
+
+import (
+	"io"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"go.starlark.net/starlark"
+	"go.starlark.net/starlarkstruct"
+)
+
+var (
+	_ starlark.Iterable  = (*starlarkSelection)(nil)
+	_ starlark.Sliceable = (*starlarkSelection)(nil)
+	_ starlark.Sequence  = (*starlarkSelection)(nil)
+	_ starlark.Value     = (*starlarkSelection)(nil)
+)
+
+var htmlModule = &starlarkstruct.Module{
+	Name: "html",
+	Members: starlark.StringDict{
+		"parse": starlark.NewBuiltin("html.parse", htmlParse),
+	},
+}
+
+func htmlParse(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var val starlark.Value
+	err := starlark.UnpackArgs("html.selection.find", args, kwargs, "from", &val)
+	if err != nil {
+		return nil, err
+	}
+
+	var r io.ReadCloser
+	switch val := val.(type) {
+	case starlark.String:
+		r = io.NopCloser(strings.NewReader(string(val)))
+	case starlark.Bytes:
+		r = io.NopCloser(strings.NewReader(string(val)))
+	case starlarkReader:
+		r = val
+	}
+	defer r.Close()
+
+	doc, err := goquery.NewDocumentFromReader(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return newStarlarkSelection(doc.Selection), nil
+}
+
+type starlarkSelection struct {
+	sel *goquery.Selection
+	*starlarkstruct.Struct
+}
+
+func newStarlarkSelection(sel *goquery.Selection) starlarkSelection {
+	ss := starlarkSelection{sel: sel}
+	ss.Struct = starlarkstruct.FromStringDict(starlark.String("html.selection"), starlark.StringDict{
+		"text":           starlark.NewBuiltin("html.selection.text", ss.text),
+		"html":           starlark.NewBuiltin("html.selection.html", ss.html),
+		"children":       starlark.NewBuiltin("html.selection.children", ss.children),
+		"parent":         starlark.NewBuiltin("html.selection.parent", ss.parent),
+		"find":           starlark.NewBuiltin("html.selection.find", ss.find),
+		"add":            starlark.NewBuiltin("html.selection.add", ss.add),
+		"attr":           starlark.NewBuiltin("html.selection.attr", ss.attr),
+		"has_class":      starlark.NewBuiltin("html.selection.has_class", ss.hasClass),
+		"index_selector": starlark.NewBuiltin("html.selection.index_selector", ss.indexSelector),
+		"and_self":       starlark.NewBuiltin("html.selection.and_self", ss.andSelf),
+		"first":          starlark.NewBuiltin("html.selection.first", ss.first),
+		"last":           starlark.NewBuiltin("html.selection.last", ss.last),
+		"next":           starlark.NewBuiltin("html.selection.last", ss.next),
+		"next_all":       starlark.NewBuiltin("html.selection.next_all", ss.nextAll),
+		"next_until":     starlark.NewBuiltin("html.selection.next_until", ss.nextUntil),
+		"prev":           starlark.NewBuiltin("html.selection.prev", ss.prev),
+		"prev_all":       starlark.NewBuiltin("html.selection.prev_all", ss.prevAll),
+		"prev_until":     starlark.NewBuiltin("html.selection.prev_until", ss.prevUntil),
+	})
+	return ss
+}
+
+func (ss starlarkSelection) Truth() starlark.Bool {
+	return len(ss.sel.Nodes) > 0
+}
+
+func (ss starlarkSelection) Len() int {
+	return ss.sel.Length()
+}
+
+func (ss starlarkSelection) Index(i int) starlark.Value {
+	return newStarlarkSelection(ss.sel.Slice(i, i+1))
+}
+
+func (ss starlarkSelection) Slice(start, end, _ int) starlark.Value {
+	return newStarlarkSelection(ss.sel.Slice(start, end))
+}
+
+func (ss starlarkSelection) Iterate() starlark.Iterator {
+	return newSelectionIterator(ss.sel)
+}
+
+func (ss starlarkSelection) text(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return starlark.String(ss.sel.Text()), nil
+}
+
+func (ss starlarkSelection) html(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	s, err := ss.sel.Html()
+	return starlark.String(s), err
+}
+
+func (ss starlarkSelection) children(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return newStarlarkSelection(ss.sel.Children()), nil
+}
+
+func (ss starlarkSelection) parent(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return newStarlarkSelection(ss.sel.Parent()), nil
+}
+
+func (ss starlarkSelection) find(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var selector string
+	err := starlark.UnpackArgs("html.selection.find", args, kwargs, "selector", &selector)
+	if err != nil {
+		return nil, err
+	}
+
+	return newStarlarkSelection(ss.sel.Find(selector)), nil
+}
+
+func (ss starlarkSelection) add(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var selector string
+	err := starlark.UnpackArgs("html.selection.add", args, kwargs, "selector", &selector)
+	if err != nil {
+		return nil, err
+	}
+	return newStarlarkSelection(ss.sel.Add(selector)), nil
+}
+
+func (ss starlarkSelection) indexSelector(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var selector string
+	err := starlark.UnpackArgs("html.selection.index_selector", args, kwargs, "selector", &selector)
+	if err != nil {
+		return nil, err
+	}
+	return starlark.MakeInt(ss.sel.IndexSelector(selector)), nil
+}
+
+func (ss starlarkSelection) attr(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var name, def string
+	err := starlark.UnpackArgs("html.selection.find", args, kwargs, "name", &name, "default??", &def)
+	if err != nil {
+		return nil, err
+	}
+	return starlark.String(ss.sel.AttrOr(name, def)), nil
+}
+
+func (ss starlarkSelection) hasClass(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var name string
+	err := starlark.UnpackArgs("html.selection.has_class", args, kwargs, "name", &name)
+	if err != nil {
+		return nil, err
+	}
+	return starlark.Bool(ss.sel.HasClass(name)), nil
+}
+
+func (ss starlarkSelection) andSelf(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return newStarlarkSelection(ss.sel.AndSelf()), nil
+}
+
+func (ss starlarkSelection) first(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return newStarlarkSelection(ss.sel.First()), nil
+}
+
+func (ss starlarkSelection) last(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return newStarlarkSelection(ss.sel.Last()), nil
+}
+
+func (ss starlarkSelection) next(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return newStarlarkSelection(ss.sel.Next()), nil
+}
+
+func (ss starlarkSelection) nextAll(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return newStarlarkSelection(ss.sel.NextAll()), nil
+}
+
+func (ss starlarkSelection) nextUntil(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var selector string
+	err := starlark.UnpackArgs("html.selection.next_until", args, kwargs, "selector", &selector)
+	if err != nil {
+		return nil, err
+	}
+	return newStarlarkSelection(ss.sel.NextUntil(selector)), nil
+}
+
+func (ss starlarkSelection) prev(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return newStarlarkSelection(ss.sel.Prev()), nil
+}
+
+func (ss starlarkSelection) prevAll(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	return newStarlarkSelection(ss.sel.PrevAll()), nil
+}
+
+func (ss starlarkSelection) prevUntil(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var selector string
+	err := starlark.UnpackArgs("html.selection.prev_until", args, kwargs, "selector", &selector)
+	if err != nil {
+		return nil, err
+	}
+	return newStarlarkSelection(ss.sel.PrevUntil(selector)), nil
+}
+
+type starlarkSelectionIterator struct {
+	sel   *goquery.Selection
+	index int
+}
+
+func newSelectionIterator(sel *goquery.Selection) *starlarkSelectionIterator {
+	return &starlarkSelectionIterator{sel: sel}
+}
+
+func (ssi *starlarkSelectionIterator) Next(v *starlark.Value) bool {
+	if ssi.index == ssi.sel.Length() {
+		return false
+	}
+	*v = newStarlarkSelection(ssi.sel.Slice(ssi.index, ssi.index+1))
+	ssi.index++
+	return true
+}
+
+func (ssi *starlarkSelectionIterator) Done() {}
--- a/internal/builtins/http.go
+++ b/internal/builtins/http.go
@@ -174,7 +174,7 @@ func starlarkResponse(res *http.Response) *starlarkstruct.Struct {
 	return starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{
 		"code":    starlark.MakeInt(res.StatusCode),
 		"headers": starlarkStringSliceMap(res.Header),
-		"body":    starlarkBody(res.Body),
+		"body":    newStarlarkReader(res.Body),
 	})
 }

@@ -184,7 +184,7 @@ func starlarkRequest(req *http.Request) *starlarkstruct.Struct {
 		"remote_addr": starlark.String(req.RemoteAddr),
 		"headers":     starlarkStringSliceMap(req.Header),
 		"query":       starlarkStringSliceMap(req.URL.Query()),
-		"body":        starlarkBody(req.Body),
+		"body":        newStarlarkReader(req.Body),
 	})
 }

@@ -200,44 +200,6 @@ func starlarkStringSliceMap(ssm map[string][]string) *starlark.Dict {
 	return dict
 }

-func starlarkBody(body io.ReadCloser) *starlarkstruct.Struct {
-	return starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{
-		"string": bodyAsString(body),
-		"bytes":  bodyAsBytes(body),
-		"close":  bodyClose(body),
-	})
-}
-
-func bodyAsBytes(body io.ReadCloser) *starlark.Builtin {
-	return starlark.NewBuiltin("http.response.body.bytes", func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
-		data, err := io.ReadAll(io.LimitReader(body, maxBodySize))
-		if err != nil {
-			return nil, err
-		}
-		return starlark.Bytes(data), nil
-	})
-}
-
-func bodyAsString(body io.ReadCloser) *starlark.Builtin {
-	return starlark.NewBuiltin("http.response.body.string", func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
-		data, err := io.ReadAll(io.LimitReader(body, maxBodySize))
-		if err != nil {
-			return nil, err
-		}
-		return starlark.String(data), nil
-	})
-}
-
-func bodyClose(body io.ReadCloser) *starlark.Builtin {
-	return starlark.NewBuiltin("http.response.body.close", func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
-		err := body.Close()
-		if err != nil {
-			return nil, err
-		}
-		return starlark.None, nil
-	})
-}
-
 func registerWebhook(mux *http.ServeMux, cfg *config.Config, pluginName string) *starlark.Builtin {
 	return starlark.NewBuiltin("register_webhook", func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
 		var fn *starlark.Function
--- a/internal/builtins/reader.go
+++ b/internal/builtins/reader.go
@@ -0,0 +1,215 @@
+package builtins
+
+import (
+	"bufio"
+	"encoding/json"
+	"io"
+
+	"github.com/vmihailenco/msgpack/v5"
+	"go.elara.ws/lure-updater/internal/convert"
+	"go.starlark.net/starlark"
+	"go.starlark.net/starlarkstruct"
+)
+
+type starlarkReader struct {
+	closeFunc func() error
+	br        *bufio.Reader
+	*starlarkstruct.Struct
+}
+
+func newStarlarkReader(r io.Reader) starlarkReader {
+	sr := starlarkReader{br: bufio.NewReader(r)}
+
+	if rc, ok := r.(io.ReadCloser); ok {
+		sr.closeFunc = rc.Close
+	}
+
+	sr.Struct = starlarkstruct.FromStringDict(starlark.String("regex"), starlark.StringDict{
+		"read":              starlark.NewBuiltin("reader.read", sr.read),
+		"peek":              starlark.NewBuiltin("reader.peek", sr.peek),
+		"discard":           starlark.NewBuiltin("reader.discard", sr.discard),
+		"read_string":       starlark.NewBuiltin("reader.read_string", sr.readString),
+		"read_until":        starlark.NewBuiltin("reader.read_until", sr.readUntil),
+		"read_string_until": starlark.NewBuiltin("reader.read_string_until", sr.readStringUntil),
+		"read_all":          starlark.NewBuiltin("reader.read_all", sr.readAll),
+		"read_all_string":   starlark.NewBuiltin("reader.read_all_string", sr.readAllString),
+		"read_json":         starlark.NewBuiltin("reader.read_json", sr.readJSON),
+		"read_msgpack":      starlark.NewBuiltin("reader.read_msgpack", sr.readMsgpack),
+		"close":             starlark.NewBuiltin("reader.close", sr.closeReader),
+	})
+
+	return sr
+}
+
+func (sr starlarkReader) read(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var n int
+	err := starlark.UnpackArgs("reader.read", args, kwargs, "n", &n)
+	if err != nil {
+		return nil, err
+	}
+
+	buf := make([]byte, n)
+	_, err = io.ReadFull(sr.br, buf)
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.Bytes(buf), nil
+}
+
+func (sr starlarkReader) readString(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var n int
+	err := starlark.UnpackArgs("reader.read_string", args, kwargs, "n", &n)
+	if err != nil {
+		return nil, err
+	}
+
+	buf := make([]byte, n)
+	_, err = io.ReadFull(sr.br, buf)
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.String(buf), nil
+}
+
+func (sr starlarkReader) readUntil(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var delimiter string
+	err := starlark.UnpackArgs("reader.read_until", args, kwargs, "delimiter", &delimiter)
+	if err != nil {
+		return nil, err
+	}
+
+	buf, err := sr.br.ReadBytes(delimiter[0])
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.Bytes(buf), nil
+}
+
+func (sr starlarkReader) readStringUntil(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var delimiter string
+	err := starlark.UnpackArgs("reader.read_string_until", args, kwargs, "delimiter", &delimiter)
+	if err != nil {
+		return nil, err
+	}
+
+	buf, err := sr.br.ReadString(delimiter[0])
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.String(buf), nil
+}
+
+func (sr starlarkReader) peek(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var n int
+	err := starlark.UnpackArgs("reader.peek", args, kwargs, "n", &n)
+	if err != nil {
+		return nil, err
+	}
+
+	buf, err := sr.br.Peek(n)
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.Bytes(buf), nil
+}
+
+func (sr starlarkReader) discard(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var n int
+	err := starlark.UnpackArgs("reader.discard", args, kwargs, "n", &n)
+	if err != nil {
+		return nil, err
+	}
+
+	dn, err := sr.br.Discard(n)
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.MakeInt(dn), nil
+}
+
+func (sr starlarkReader) readAll(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var limit int64 = 102400
+	err := starlark.UnpackArgs("reader.read_all", args, kwargs, "limit??", &limit)
+	if err != nil {
+		return nil, err
+	}
+
+	var r io.Reader = sr.br
+	if limit > 0 {
+		r = io.LimitReader(sr.br, limit)
+	}
+
+	buf, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.Bytes(buf), nil
+}
+
+func (sr starlarkReader) readAllString(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var limit int64 = 102400
+	err := starlark.UnpackArgs("reader.read_all_string", args, kwargs, "limit??", &limit)
+	if err != nil {
+		return nil, err
+	}
+
+	var r io.Reader = sr.br
+	if limit > 0 {
+		r = io.LimitReader(sr.br, limit)
+	}
+
+	buf, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.String(buf), nil
+}
+
+func (sr starlarkReader) readJSON(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var v any
+	err := json.NewDecoder(sr.br).Decode(&v)
+	if err != nil {
+		return nil, err
+	}
+	return convert.Convert(v)
+}
+
+func (sr starlarkReader) readMsgpack(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var v any
+	err := msgpack.NewDecoder(sr.br).Decode(&v)
+	if err != nil {
+		return nil, err
+	}
+	return convert.Convert(v)
+}
+
+func (sr starlarkReader) closeReader(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	if sr.closeFunc != nil {
+		err := sr.closeFunc()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return starlark.None, nil
+}
+
+// Read implements the io.ReadCloser interface
+func (sr starlarkReader) Read(b []byte) (int, error) {
+	return sr.br.Read(b)
+}
+
+// Close implements the io.ReadCloser interface
+func (sr starlarkReader) Close() error {
+	if sr.closeFunc != nil {
+		return sr.closeFunc()
+	}
+	return nil
+}
--- a/internal/builtins/register.go
+++ b/internal/builtins/register.go
@@ -43,5 +43,7 @@ func Register(sd starlark.StringDict, opts *Options) {
 	sd["updater"] = updaterModule(opts.Config)
 	sd["log"] = logModule(opts.Name)
 	sd["json"] = starlarkjson.Module
+	sd["utils"] = utilsModule
+	sd["html"] = htmlModule
 	sd["register_webhook"] = registerWebhook(opts.Mux, opts.Config, opts.Name)
 }
--- a/internal/builtins/utils.go
+++ b/internal/builtins/utils.go
@@ -0,0 +1,23 @@
+package builtins
+
+import (
+	"go.elara.ws/vercmp"
+	"go.starlark.net/starlark"
+	"go.starlark.net/starlarkstruct"
+)
+
+var utilsModule = &starlarkstruct.Module{
+	Name: "utils",
+	Members: starlark.StringDict{
+		"ver_cmp": starlark.NewBuiltin("utils.ver_cmp", utilsVerCmp),
+	},
+}
+
+func utilsVerCmp(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var v1, v2 string
+	err := starlark.UnpackArgs("utils.ver_cmp", args, kwargs, "v1", &v1, "v2", &v2)
+	if err != nil {
+		return nil, err
+	}
+	return starlark.MakeInt(vercmp.Compare(v1, v2)), nil
+}
--- a/internal/convert/convert.go
+++ b/internal/convert/convert.go
@@ -0,0 +1,87 @@
+package convert
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+
+	"go.starlark.net/starlark"
+)
+
+var ErrInvalidType = errors.New("unknown type")
+
+func Convert(v any) (starlark.Value, error) {
+	if v == nil {
+		return starlark.None, nil
+	}
+	val := reflect.ValueOf(v)
+	kind := val.Kind()
+	for kind == reflect.Pointer || kind == reflect.Interface {
+		val = val.Elem()
+	}
+	return convert(val)
+}
+
+func convert(val reflect.Value) (starlark.Value, error) {
+	switch val.Kind() {
+	case reflect.Interface:
+		return convert(val.Elem())
+	case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8:
+		return starlark.MakeInt64(val.Int()), nil
+	case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8:
+		return starlark.MakeUint64(val.Uint()), nil
+	case reflect.Float64, reflect.Float32:
+		return starlark.Float(val.Float()), nil
+	case reflect.Bool:
+		return starlark.Bool(val.Bool()), nil
+	case reflect.String:
+		return starlark.String(val.String()), nil
+	case reflect.Slice, reflect.Array:
+		return convertSlice(val)
+	case reflect.Map:
+		return convertMap(val)
+	default:
+		return nil, fmt.Errorf("%w: %s", ErrInvalidType, val.Type())
+	}
+}
+
+func convertSlice(val reflect.Value) (starlark.Value, error) {
+	// Detect byte slice
+	if val.Type().Elem().Kind() == reflect.Uint8 {
+		return starlark.Bytes(val.Bytes()), nil
+	}
+
+	elems := make([]starlark.Value, val.Len())
+
+	for i := 0; i < val.Len(); i++ {
+		elem, err := convert(val.Index(i))
+		if err != nil {
+			return nil, err
+		}
+		elems[i] = elem
+	}
+
+	return starlark.NewList(elems), nil
+}
+
+func convertMap(val reflect.Value) (starlark.Value, error) {
+	dict := starlark.NewDict(val.Len())
+	iter := val.MapRange()
+	for iter.Next() {
+		k, err := convert(iter.Key())
+		if err != nil {
+			return nil, err
+		}
+
+		v, err := convert(iter.Value())
+		if err != nil {
+			return nil, err
+		}
+
+		err = dict.SetKey(k, v)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return dict, nil
+}
--- a/template.nomad
+++ b/template.nomad
@@ -53,7 +53,7 @@ job "lure-updater" {
      }

      service {
-        name = "site"
+        name = "lure-updater"
        port = "webhook"

        tags = [
Author	SHA1	Message	Date
Elara Musayelyan	f5c8ba2282	Add html builtin module All checks were successful ci/woodpecker/push/woodpecker Pipeline was successful Details	2023-06-25 13:07:38 -07:00
Elara Musayelyan	a30d805f9b	Add utils.ver_cmp() builtin All checks were successful ci/woodpecker/push/woodpecker Pipeline was successful Details	2023-06-22 14:53:42 -07:00
Elara Musayelyan	228bf59c85	Add starlark reader implementation All checks were successful ci/woodpecker/push/woodpecker Pipeline was successful Details	2023-06-22 14:47:22 -07:00
Elara Musayelyan	2cdea934eb	Change the nomad service name to fix a collision All checks were successful ci/woodpecker/push/woodpecker Pipeline was successful Details	2023-06-21 21:24:32 +00:00