Compare commits

...

7 Commits

Author SHA1 Message Date
d6698b3a03 Update import path
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-10-07 17:35:22 -07:00
8b97df59e1 Set platform to amd64
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-10-07 12:02:43 -07:00
e429c904d4 Move lure-updater to updater.lure.sh
All checks were successful
ci/woodpecker/manual/woodpecker Pipeline was successful
2023-10-07 11:50:57 -07:00
d5775b9fa2 Update lure-repo URL 2023-10-06 19:17:08 -07:00
029dfab35f Create a readerValue for values that can implement the io.Reader interface 2023-09-30 16:59:10 -07:00
6272e5e044 Move all HTTP error handling into error handling middleware
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-06-26 19:43:27 -07:00
d90da5dcf9 Accept reader as a valid type for bodyReader
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-06-26 19:28:00 -07:00
12 changed files with 95 additions and 46 deletions

View File

@ -29,7 +29,7 @@ blobs:
folder: "/" folder: "/"
release: release:
gitea: gitea:
owner: Elara6331 owner: lure
name: lure-updater name: lure-updater
gitea_urls: gitea_urls:
api: 'https://gitea.elara.ws/api/v1/' api: 'https://gitea.elara.ws/api/v1/'

View File

@ -1,3 +1,4 @@
platform: linux/amd64
pipeline: pipeline:
release: release:
image: goreleaser/goreleaser image: goreleaser/goreleaser

View File

@ -1,12 +1,12 @@
# LURE Updater # LURE Updater
Modular bot that automatically checks for upstream updates and pushes new packages to [lure-repo](https://github.com/Elara6331/lure-repo). Modular bot that automatically checks for upstream updates and pushes new packages to [lure-repo](https://github.com/lure-sh/lure-repo).
--- ---
### How it works ### How it works
Since LURE is meant to be able to install many different types of packages, this bot accepts [plugins](https://gitea.elara.ws/Elara6331/updater-plugins) in the form of [Starlark](https://github.com/bazelbuild/starlark) files rather than hardcoding each package. These plugins can schedule functions to be run at certain intervals, or when a webhook is received, and they have access to persistent key/value storage to keep track of information. This allows plugins to use many different ways to detect upstream updates. Since LURE is meant to be able to install many different types of packages, this bot accepts [plugins](https://gitea.elara.ws/lure/updater-plugins) in the form of [Starlark](https://github.com/bazelbuild/starlark) files rather than hardcoding each package. These plugins can schedule functions to be run at certain intervals, or when a webhook is received, and they have access to persistent key/value storage to keep track of information. This allows plugins to use many different ways to detect upstream updates.
For example, the plugin for `discord-bin` repeatedly polls discord's API every hour for the current latest download link. It puts the link in persistent storage, and if it has changed since last time, it parses the URL to extract the version number, and uses that to update the build script for `discord-bin`. For example, the plugin for `discord-bin` repeatedly polls discord's API every hour for the current latest download link. It puts the link in persistent storage, and if it has changed since last time, it parses the URL to extract the version number, and uses that to update the build script for `discord-bin`.

2
go.mod
View File

@ -1,4 +1,4 @@
module go.elara.ws/lure-updater module lure.sh/lure-updater
go 1.20 go 1.20

View File

@ -1,9 +1,6 @@
package builtins package builtins
import ( import (
"io"
"strings"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"go.starlark.net/starlark" "go.starlark.net/starlark"
"go.starlark.net/starlarkstruct" "go.starlark.net/starlarkstruct"
@ -24,21 +21,11 @@ var htmlModule = &starlarkstruct.Module{
} }
func htmlParse(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { func htmlParse(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var val starlark.Value var r readerValue
err := starlark.UnpackArgs("html.selection.find", args, kwargs, "from", &val) err := starlark.UnpackArgs("html.selection.find", args, kwargs, "from", &r)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var r io.ReadCloser
switch val := val.(type) {
case starlark.String:
r = io.NopCloser(strings.NewReader(string(val)))
case starlark.Bytes:
r = io.NopCloser(strings.NewReader(string(val)))
case starlarkReader:
r = val
}
defer r.Close() defer r.Close()
doc, err := goquery.NewDocumentFromReader(r) doc, err := goquery.NewDocumentFromReader(r)

View File

@ -27,7 +27,7 @@ import (
"strings" "strings"
"go.elara.ws/logger/log" "go.elara.ws/logger/log"
"go.elara.ws/lure-updater/internal/config" "lure.sh/lure-updater/internal/config"
"go.starlark.net/starlark" "go.starlark.net/starlark"
"go.starlark.net/starlarkstruct" "go.starlark.net/starlarkstruct"
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"
@ -80,6 +80,8 @@ func (sbr *starlarkBodyReader) Unpack(v starlark.Value) error {
sbr.Reader = strings.NewReader(string(v)) sbr.Reader = strings.NewReader(string(v))
case starlark.Bytes: case starlark.Bytes:
sbr.Reader = strings.NewReader(string(v)) sbr.Reader = strings.NewReader(string(v))
case starlarkReader:
sbr.Reader = v
default: default:
return fmt.Errorf("%w: %s", ErrInvalidBodyType, v.Type()) return fmt.Errorf("%w: %s", ErrInvalidBodyType, v.Type())
} }
@ -221,7 +223,7 @@ func registerWebhook(mux *http.ServeMux, cfg *config.Config, pluginName string)
} }
func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *starlark.Thread, fn *starlark.Function) http.HandlerFunc { func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *starlark.Thread, fn *starlark.Function) http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) { return handleError(func(res http.ResponseWriter, req *http.Request) *HTTPError {
defer req.Body.Close() defer req.Body.Close()
res.Header().Add("X-Updater-Plugin", pluginName) res.Header().Add("X-Updater-Plugin", pluginName)
@ -229,20 +231,22 @@ func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *
if secure { if secure {
err := verifySecure(cfg.Webhook.PasswordHash, pluginName, req) err := verifySecure(cfg.Webhook.PasswordHash, pluginName, req)
if err != nil { if err != nil {
log.Error("Error verifying webhook").Err(err).Send() return &HTTPError{
res.WriteHeader(http.StatusForbidden) Message: "Error verifying webhook",
_, _ = io.WriteString(res, err.Error()) Code: http.StatusForbidden,
return Err: err,
}
} }
} }
log.Debug("Calling webhook function").Str("name", fn.Name()).Stringer("pos", fn.Position()).Send() log.Debug("Calling webhook function").Str("name", fn.Name()).Stringer("pos", fn.Position()).Send()
val, err := starlark.Call(thread, fn, starlark.Tuple{starlarkRequest(req)}, nil) val, err := starlark.Call(thread, fn, starlark.Tuple{starlarkRequest(req)}, nil)
if err != nil { if err != nil {
log.Error("Error while executing webhook").Err(err).Stringer("pos", fn.Position()).Send() return &HTTPError{
res.WriteHeader(http.StatusInternalServerError) Message: "Error while executing webhook",
_, _ = io.WriteString(res, err.Error()) Code: http.StatusInternalServerError,
return Err: err,
}
} }
switch val := val.(type) { switch val := val.(type) {
@ -260,13 +264,20 @@ func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *
body := newBodyReader() body := newBodyReader()
err = body.Unpack(val) err = body.Unpack(val)
if err != nil { if err != nil {
log.Error("Error unpacking returned body").Err(err).Send() return &HTTPError{
return Message: "Error unpacking returned body",
Code: http.StatusInternalServerError,
Err: err,
}
} }
_, err = io.Copy(res, body) _, err = io.Copy(res, body)
if err != nil { if err != nil {
log.Error("Error writing body").Err(err).Send() log.Error("Error writing body").Err(err).Send()
return return &HTTPError{
Message: "Error writing body",
Code: http.StatusInternalServerError,
Err: err,
}
} }
case *starlark.Dict: case *starlark.Dict:
code := http.StatusOK code := http.StatusOK
@ -274,8 +285,11 @@ func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *
if ok { if ok {
err = starlark.AsInt(codeVal, &code) err = starlark.AsInt(codeVal, &code)
if err != nil { if err != nil {
log.Error("Error decoding returned status code").Err(err).Send() return &HTTPError{
return Message: "Error decoding returned status code",
Code: http.StatusInternalServerError,
Err: err,
}
} }
res.WriteHeader(code) res.WriteHeader(code)
} }
@ -285,16 +299,40 @@ func webhookHandler(pluginName string, secure bool, cfg *config.Config, thread *
if ok { if ok {
err = body.Unpack(bodyVal) err = body.Unpack(bodyVal)
if err != nil { if err != nil {
log.Error("Error unpacking returned body").Err(err).Send() return &HTTPError{
return Message: "Error unpacking returned body",
Code: http.StatusInternalServerError,
Err: err,
}
} }
_, err = io.Copy(res, body) _, err = io.Copy(res, body)
if err != nil { if err != nil {
log.Error("Error writing body").Err(err).Send() return &HTTPError{
return Message: "Error writing body",
Code: http.StatusInternalServerError,
Err: err,
}
} }
} }
} }
return nil
})
}
type HTTPError struct {
Code int
Message string
Err error
}
func handleError(h func(res http.ResponseWriter, req *http.Request) *HTTPError) http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
httpErr := h(res, req)
if httpErr != nil {
log.Error(httpErr.Message).Err(httpErr.Err).Send()
res.WriteHeader(httpErr.Code)
fmt.Sprintf("%s: %s", httpErr.Message, httpErr.Err)
}
} }
} }

View File

@ -3,10 +3,12 @@ package builtins
import ( import (
"bufio" "bufio"
"encoding/json" "encoding/json"
"errors"
"io" "io"
"strings"
"github.com/vmihailenco/msgpack/v5" "github.com/vmihailenco/msgpack/v5"
"go.elara.ws/lure-updater/internal/convert" "lure.sh/lure-updater/internal/convert"
"go.starlark.net/starlark" "go.starlark.net/starlark"
"go.starlark.net/starlarkstruct" "go.starlark.net/starlarkstruct"
) )
@ -213,3 +215,24 @@ func (sr starlarkReader) Close() error {
} }
return nil return nil
} }
type readerValue struct {
io.ReadCloser
}
func (rv *readerValue) Unpack(v starlark.Value) error {
switch val := v.(type) {
case starlark.String:
rv.ReadCloser = io.NopCloser(strings.NewReader(string(val)))
case starlark.Bytes:
rv.ReadCloser = io.NopCloser(strings.NewReader(string(val)))
case starlarkReader:
rv.ReadCloser = val
}
if rv.ReadCloser == nil {
return errors.New("invalid type for reader")
}
return nil
}

View File

@ -21,7 +21,7 @@ package builtins
import ( import (
"net/http" "net/http"
"go.elara.ws/lure-updater/internal/config" "lure.sh/lure-updater/internal/config"
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
"go.starlark.net/starlark" "go.starlark.net/starlark"
"go.starlark.net/starlarkjson" "go.starlark.net/starlarkjson"

View File

@ -30,7 +30,7 @@ import (
"github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/plumbing/transport/http"
"go.elara.ws/logger/log" "go.elara.ws/logger/log"
"go.elara.ws/lure-updater/internal/config" "lure.sh/lure-updater/internal/config"
"go.starlark.net/starlark" "go.starlark.net/starlark"
"go.starlark.net/starlarkstruct" "go.starlark.net/starlarkstruct"
) )

View File

@ -1,5 +1,5 @@
[git] [git]
repoURL = "https://github.com/Elara6331/lure-repo.git" repoURL = "https://github.com/lure-sh/lure-repo.git"
repoDir = "/etc/lure-updater/repo" repoDir = "/etc/lure-updater/repo"
[git.commit] [git.commit]
# The name and email to use in the git commit # The name and email to use in the git commit

View File

@ -31,8 +31,8 @@ import (
"github.com/spf13/pflag" "github.com/spf13/pflag"
"go.elara.ws/logger" "go.elara.ws/logger"
"go.elara.ws/logger/log" "go.elara.ws/logger/log"
"go.elara.ws/lure-updater/internal/builtins" "lure.sh/lure-updater/internal/builtins"
"go.elara.ws/lure-updater/internal/config" "lure.sh/lure-updater/internal/config"
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
"go.starlark.net/starlark" "go.starlark.net/starlark"
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"

View File

@ -27,7 +27,7 @@ job "lure-updater" {
env { env {
GIT_REPO_DIR = "/etc/lure-updater/repo" GIT_REPO_DIR = "/etc/lure-updater/repo"
GIT_REPO_URL = "https://github.com/Elara6331/lure-repo.git" GIT_REPO_URL = "https://github.com/lure-sh/lure-repo.git"
GIT_CREDENTIALS_USERNAME = "lure-repo-bot" GIT_CREDENTIALS_USERNAME = "lure-repo-bot"
GIT_CREDENTIALS_PASSWORD = "${GITHUB_PASSWORD}" GIT_CREDENTIALS_PASSWORD = "${GITHUB_PASSWORD}"
GIT_COMMIT_NAME = "lure-repo-bot" GIT_COMMIT_NAME = "lure-repo-bot"
@ -58,7 +58,7 @@ job "lure-updater" {
tags = [ tags = [
"traefik.enable=true", "traefik.enable=true",
"traefik.http.routers.lure-updater.rule=Host(`updater.lure.elara.ws`)", "traefik.http.routers.lure-updater.rule=Host(`updater.lure.sh`)",
"traefik.http.routers.lure-updater.tls.certResolver=letsencrypt", "traefik.http.routers.lure-updater.tls.certResolver=letsencrypt",
] ]
} }