From bc91986e6c37f8081e560b7182fe73e1245165f0 Mon Sep 17 00:00:00 2001 From: Elara Musayelyan Date: Fri, 18 Aug 2023 19:10:43 -0700 Subject: [PATCH] Remove deprecated WebSocket API, switch to SQLite for reply store --- .gitignore | 3 +- README.md | 3 +- go.mod | 32 +++-- go.sum | 90 +++++++------- internal/store/db.go | 31 +++++ internal/store/models.go | 13 +++ internal/store/queries.sql.go | 42 +++++++ internal/store/types.go | 6 + main.go | 214 +++++++++++++++++----------------- sql/queries.sql | 5 + sql/schema.sql | 6 + sqlc.yaml | 9 ++ 12 files changed, 289 insertions(+), 165 deletions(-) create mode 100644 internal/store/db.go create mode 100644 internal/store/models.go create mode 100644 internal/store/queries.sql.go create mode 100644 internal/store/types.go create mode 100644 sql/queries.sql create mode 100644 sql/schema.sql create mode 100644 sqlc.yaml diff --git a/.gitignore b/.gitignore index 240f63e..0b2b714 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /lemmy-reply-bot.toml /lemmy-reply-bot -/replied.bin \ No newline at end of file +/replied.bin +/replied.db \ No newline at end of file diff --git a/README.md b/README.md index 49c3bc9..5217449 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Lemmy Reply Bot -This project is a simple bot that replies to comments and posts on Lemmy. It uses Lemmy's WebSocket API to get notified of any new comments or posts, and sees if they match any regex configured in the config file. If it finds one that does, it replies with the message corresponding to that regex. +This project is a simple bot that replies to comments and posts on Lemmy. It gets the newest posts and comments every 15 seconds, and sees if they match any regex configured in the config file. If it finds one that does, it replies with the message corresponding to that regex. ### Features @@ -8,7 +8,6 @@ This project is a simple bot that replies to comments and posts on Lemmy. It use - Powerful PCRE2 regular expressions for detecting triggers - Ability to use regex capture groups in replies - Persistent duplicate reply prevention via a filesystem store -- Uses event-based WebSocket API, which means near-instant replies and no rate limiting ### Configuration diff --git a/go.mod b/go.mod index 18df31d..69cba5d 100644 --- a/go.mod +++ b/go.mod @@ -8,31 +8,39 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible github.com/pelletier/go-toml/v2 v2.0.6 github.com/spf13/pflag v1.0.5 - github.com/vmihailenco/msgpack/v5 v5.3.5 - go.elara.ws/go-lemmy v0.17.2 + go.elara.ws/go-lemmy v0.18.0 go.elara.ws/logger v0.0.0-20230421022458-e80700db2090 - go.elara.ws/pcre v0.0.0-20230421030233-daf2d2e6973f + go.elara.ws/pcre v0.0.0-20230805032557-4ce849193f64 + modernc.org/sqlite v1.25.0 ) require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gookit/color v1.5.1 // indirect - github.com/gorilla/websocket v1.5.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.13 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect - github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect - golang.org/x/sys v0.1.0 // indirect - modernc.org/libc v1.16.8 // indirect - modernc.org/mathutil v1.4.1 // indirect - modernc.org/memory v1.1.1 // indirect + golang.org/x/mod v0.3.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + lukechampine.com/uint128 v1.2.0 // indirect + modernc.org/cc/v3 v3.40.0 // indirect + modernc.org/ccgo/v3 v3.16.13 // indirect + modernc.org/libc v1.24.1 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.0 // indirect + modernc.org/opt v0.1.3 // indirect + modernc.org/strutil v1.1.3 // indirect + modernc.org/token v1.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1945f55..ec0dd73 100644 --- a/go.sum +++ b/go.sum @@ -4,31 +4,29 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gookit/color v1.5.1 h1:Vjg2VEcdHpwq+oY63s/ksHrgJYCTo0bwWvmmYWdE9fQ= github.com/gookit/color v1.5.1/go.mod h1:wZFzea4X8qN6vHOSP2apMb4/+w/orMznEzYsIHPaqKM= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -37,36 +35,32 @@ github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvI github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= -github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= -github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= -github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.elara.ws/go-lemmy v0.17.2 h1:EGj/dzowxuU6KYAUN5s1eClNVXifpcLQLQDCN0BltUw= -go.elara.ws/go-lemmy v0.17.2/go.mod h1:rurQND/HT3yWfX/T4w+hb6vEwRAeAlV+9bSGFkkx5rA= +go.elara.ws/go-lemmy v0.18.0 h1:c83VhfGtePegDZRV8X4PWfkl4cqBqug4li20VChN6sE= +go.elara.ws/go-lemmy v0.18.0/go.mod h1:rurQND/HT3yWfX/T4w+hb6vEwRAeAlV+9bSGFkkx5rA= go.elara.ws/logger v0.0.0-20230421022458-e80700db2090 h1:RVC8XvWo6Yw4HUshqx4TSzuBDScDghafU6QFRJ4xPZg= go.elara.ws/logger v0.0.0-20230421022458-e80700db2090/go.mod h1:qng49owViqsW5Aey93lwBXONw20oGbJIoLVscB16mPM= -go.elara.ws/pcre v0.0.0-20230421030233-daf2d2e6973f h1:ZwR0xvBeP5BHHv63fgfuwhZIj+Si5rp79WSDUE73ZVA= -go.elara.ws/pcre v0.0.0-20230421030233-daf2d2e6973f/go.mod h1:EF48C6VnP4wBayzFGk6lXqbiLucH7EfiaYOgiiCe5k4= +go.elara.ws/pcre v0.0.0-20230805032557-4ce849193f64 h1:QixGnJE1jP08Hs1G3rS7tZGd8DeBRtz9RBpk08WlGh4= +go.elara.ws/pcre v0.0.0-20230805032557-4ce849193f64/go.mod h1:EF48C6VnP4wBayzFGk6lXqbiLucH7EfiaYOgiiCe5k4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -75,44 +69,48 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 h1:M8tBwCtWD/cZV9DZpFYRUgaymAYAr+aIUTWzDaM3uPs= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= -modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= -modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= -modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= -modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= -modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= -modernc.org/libc v1.16.8 h1:Ux98PaOMvolgoFX/YwusFOHBnanXdGRmWgI8ciI2z4o= -modernc.org/libc v1.16.8/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= -modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8= -modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU= -modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= -modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= -modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= +modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.0 h1:2pXdbgdP5hIyDp2JqIwkHNZ1sAjEbh8GnRpcqFWBf7E= +modernc.org/memory v1.7.0/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA= +modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= +modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= +modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= diff --git a/internal/store/db.go b/internal/store/db.go new file mode 100644 index 0000000..f28762c --- /dev/null +++ b/internal/store/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.19.1 + +package store + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/store/models.go b/internal/store/models.go new file mode 100644 index 0000000..1e20037 --- /dev/null +++ b/internal/store/models.go @@ -0,0 +1,13 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.19.1 + +package store + +import () + +type RepliedItem struct { + ID int64 + ItemType string + UpdatedTime int64 +} diff --git a/internal/store/queries.sql.go b/internal/store/queries.sql.go new file mode 100644 index 0000000..9603254 --- /dev/null +++ b/internal/store/queries.sql.go @@ -0,0 +1,42 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.19.1 +// source: queries.sql + +package store + +import ( + "context" +) + +const addItem = `-- name: AddItem :exec +INSERT OR REPLACE INTO replied_items (id, item_type, updated_time) VALUES (?, ?, ?) +` + +type AddItemParams struct { + ID int64 + ItemType string + UpdatedTime int64 +} + +func (q *Queries) AddItem(ctx context.Context, arg AddItemParams) error { + _, err := q.db.ExecContext(ctx, addItem, arg.ID, arg.ItemType, arg.UpdatedTime) + return err +} + +const itemExists = `-- name: ItemExists :one +SELECT COUNT(1) FROM replied_items WHERE item_type = ? AND id = ? AND updated_time = ? +` + +type ItemExistsParams struct { + ItemType string + ID int64 + UpdatedTime int64 +} + +func (q *Queries) ItemExists(ctx context.Context, arg ItemExistsParams) (int64, error) { + row := q.db.QueryRowContext(ctx, itemExists, arg.ItemType, arg.ID, arg.UpdatedTime) + var count int64 + err := row.Scan(&count) + return count, err +} diff --git a/internal/store/types.go b/internal/store/types.go new file mode 100644 index 0000000..f4e0f00 --- /dev/null +++ b/internal/store/types.go @@ -0,0 +1,6 @@ +package store + +const ( + Comment = "c" + Post = "p" +) diff --git a/main.go b/main.go index 5821030..6b0f37b 100644 --- a/main.go +++ b/main.go @@ -2,43 +2,23 @@ package main import ( "context" - "os" + "database/sql" + _ "embed" "os/signal" "strings" "syscall" "text/template" + "time" "unsafe" "github.com/spf13/pflag" - "github.com/vmihailenco/msgpack/v5" "go.elara.ws/go-lemmy" "go.elara.ws/go-lemmy/types" + "go.elara.ws/lemmy-reply-bot/internal/store" "go.elara.ws/logger/log" + _ "modernc.org/sqlite" ) -type itemType uint8 - -const ( - comment itemType = iota - post -) - -type item struct { - Type itemType - ID int -} - -func (it itemType) String() string { - switch it { - case comment: - return "comment" - case post: - return "post" - default: - return "" - } -} - type Submatches []string func (sm Submatches) Item(i int) string { @@ -47,13 +27,31 @@ func (sm Submatches) Item(i int) string { type TmplContext struct { Matches []Submatches - Type itemType + Type string } func (tc TmplContext) Match(i, j int) string { return tc.Matches[i][j] } +//go:embed sql/schema.sql +var schema string + +func init() { + db, err := sql.Open("sqlite", "replied.db") + if err != nil { + log.Fatal("Error opening database during init").Err(err).Send() + } + _, err = db.Exec(schema) + if err != nil { + log.Fatal("Error initializing database").Err(err).Send() + } + err = db.Close() + if err != nil { + log.Fatal("Error closing database after init").Err(err).Send() + } +} + func main() { configPath := pflag.StringP("config", "c", "./lemmy-reply-bot.toml", "Path to the config file") dryRun := pflag.BoolP("dry-run", "D", false, "Don't actually send comments, just check for matches") @@ -68,7 +66,7 @@ func main() { log.Fatal("Error loading config file").Err(err).Send() } - c, err := lemmy.NewWebSocket(cfg.Lemmy.InstanceURL) + c, err := lemmy.New(cfg.Lemmy.InstanceURL) if err != nil { log.Fatal("Error creating new Lemmy API client").Err(err).Send() } @@ -83,13 +81,6 @@ func main() { log.Info("Successfully logged in to Lemmy instance").Send() - joinAll(c) - - c.OnReconnect(func(c *lemmy.WSClient) { - joinAll(c) - log.Info("Successfully reconnected to WebSocket").Send() - }) - replyCh := make(chan replyJob, 200) if !*dryRun { @@ -99,57 +90,64 @@ func main() { commentWorker(ctx, c, replyCh) } -func commentWorker(ctx context.Context, c *lemmy.WSClient, replyCh chan<- replyJob) { - repliedIDs := map[item]struct{}{} - - repliedStore, err := os.Open("replied.bin") - if err == nil { - err = msgpack.NewDecoder(repliedStore).Decode(&repliedIDs) - if err != nil { - log.Warn("Error decoding reply store").Err(err).Send() - } - repliedStore.Close() +func commentWorker(ctx context.Context, c *lemmy.Client, replyCh chan<- replyJob) { + db, err := sql.Open("sqlite", "replied.db") + if err != nil { + log.Fatal("Error opening reply database").Err(err).Send() } + rs := store.New(db) for { select { - case res := <-c.Responses(): - if res.IsOneOf(types.UserOperationCRUDCreateComment, types.UserOperationCRUDEditComment) { - var cr types.CommentResponse - err = lemmy.DecodeResponse(res.Data, &cr) - if err != nil { - log.Warn("Error while trying to decode comment").Err(err).Send() + case <-time.After(15 * time.Second): + comments, err := c.Comments(ctx, types.GetComments{ + Type: types.NewOptional(types.ListingTypeLocal), + Sort: types.NewOptional(types.CommentSortTypeNew), + Limit: types.NewOptional[int64](50), + }) + if err != nil { + log.Warn("Error getting comments").Err(err).Send() + continue + } + + for _, c := range comments.Comments { + // Skip all non-local comments + if !c.Community.Local { continue } - if !cr.CommentView.Community.Local { + // If the item we're checking for already exists, we've already replied, so skip it + if c, err := rs.ItemExists(ctx, store.ItemExistsParams{ + ID: int64(c.Comment.ID), + ItemType: store.Comment, + UpdatedTime: c.Comment.Updated.Unix(), + }); c > 0 && err == nil { continue - } - - if _, ok := repliedIDs[item{comment, cr.CommentView.Comment.ID}]; ok { + } else if err != nil { + log.Warn("Error checking if item exists").Err(err).Send() continue } for i, reply := range cfg.Replies { re := compiledRegexes[reply.Regex] - if !re.MatchString(cr.CommentView.Comment.Content) { + if !re.MatchString(c.Comment.Content) { continue } log.Info("Matched comment body"). Int("reply-index", i). - Int("comment-id", cr.CommentView.Comment.ID). + Int("comment-id", c.Comment.ID). Send() job := replyJob{ - CommentID: types.NewOptional(cr.CommentView.Comment.ID), - PostID: cr.CommentView.Comment.PostID, + CommentID: types.NewOptional(c.Comment.ID), + PostID: c.Comment.PostID, } - matches := re.FindAllStringSubmatch(cr.CommentView.Comment.Content, -1) + matches := re.FindAllStringSubmatch(c.Comment.Content, -1) job.Content, err = executeTmpl(compiledTmpls[reply.Regex], TmplContext{ Matches: toSubmatches(matches), - Type: comment, + Type: "comment", }) if err != nil { log.Warn("Error while executing template").Err(err).Send() @@ -158,25 +156,47 @@ func commentWorker(ctx context.Context, c *lemmy.WSClient, replyCh chan<- replyJ replyCh <- job - repliedIDs[item{comment, cr.CommentView.Comment.ID}] = struct{}{} + err = rs.AddItem(ctx, store.AddItemParams{ + ID: int64(c.Comment.ID), + ItemType: store.Comment, + UpdatedTime: c.Comment.Updated.Unix(), + }) + if err != nil { + log.Warn("Error adding comment to the reply store").Err(err).Send() + continue + } } - } else if res.IsOneOf(types.UserOperationCRUDCreatePost, types.UserOperationCRUDEditPost) { - var pr types.PostResponse - err = lemmy.DecodeResponse(res.Data, &pr) - if err != nil { - log.Warn("Error while trying to decode comment").Err(err).Send() + } + + posts, err := c.Posts(ctx, types.GetPosts{ + Type: types.NewOptional(types.ListingTypeLocal), + Sort: types.NewOptional(types.SortTypeNew), + Limit: types.NewOptional[int64](20), + }) + if err != nil { + log.Warn("Error getting comments").Err(err).Send() + continue + } + + for _, p := range posts.Posts { + // Skip all non-local posts + if !p.Community.Local { continue } - if !pr.PostView.Community.Local { + // If the item we're checking for already exists, we've already replied, so skip it + if c, err := rs.ItemExists(ctx, store.ItemExistsParams{ + ID: int64(p.Post.ID), + ItemType: store.Post, + UpdatedTime: p.Post.Updated.Unix(), + }); c > 0 && err == nil { + continue + } else if err != nil { + log.Warn("Error checking if item exists").Err(err).Send() continue } - if _, ok := repliedIDs[item{post, pr.PostView.Post.ID}]; ok { - continue - } - - body := pr.PostView.Post.URL.ValueOr("") + "\n\n" + pr.PostView.Post.Body.ValueOr("") + body := p.Post.URL.ValueOr("") + "\n\n" + p.Post.Body.ValueOr("") for i, reply := range cfg.Replies { re := compiledRegexes[reply.Regex] if !re.MatchString(body) { @@ -185,15 +205,15 @@ func commentWorker(ctx context.Context, c *lemmy.WSClient, replyCh chan<- replyJ log.Info("Matched post body"). Int("reply-index", i). - Int("post-id", pr.PostView.Post.ID). + Int("post-id", p.Post.ID). Send() - job := replyJob{PostID: pr.PostView.Post.ID} + job := replyJob{PostID: p.Post.ID} matches := re.FindAllStringSubmatch(body, -1) job.Content, err = executeTmpl(compiledTmpls[reply.Regex], TmplContext{ Matches: toSubmatches(matches), - Type: post, + Type: "post", }) if err != nil { log.Warn("Error while executing template").Err(err).Send() @@ -202,24 +222,23 @@ func commentWorker(ctx context.Context, c *lemmy.WSClient, replyCh chan<- replyJ replyCh <- job - repliedIDs[item{post, pr.PostView.Post.ID}] = struct{}{} + err = rs.AddItem(ctx, store.AddItemParams{ + ID: int64(p.Post.ID), + ItemType: store.Post, + UpdatedTime: p.Post.Updated.Unix(), + }) + if err != nil { + log.Warn("Error adding post to the reply store").Err(err).Send() + continue + } } } - case err := <-c.Errors(): - log.Warn("Lemmy client error").Err(err).Send() case <-ctx.Done(): - repliedStore, err := os.Create("replied.bin") + err = db.Close() if err != nil { - log.Warn("Error creating reply store file").Err(err).Send() - return + log.Warn("Error closing database").Err(err).Send() + continue } - - err = msgpack.NewEncoder(repliedStore).Encode(repliedIDs) - if err != nil { - log.Warn("Error encoding replies to reply store").Err(err).Send() - } - - repliedStore.Close() return } } @@ -231,11 +250,11 @@ type replyJob struct { PostID int } -func commentReplyWorker(ctx context.Context, c *lemmy.WSClient, ch <-chan replyJob) { +func commentReplyWorker(ctx context.Context, c *lemmy.Client, ch <-chan replyJob) { for { select { case reply := <-ch: - err := c.Request(types.UserOperationCRUDCreateComment, types.CreateComment{ + cr, err := c.CreateComment(ctx, types.CreateComment{ PostID: reply.PostID, ParentID: reply.CommentID, Content: reply.Content, @@ -247,6 +266,7 @@ func commentReplyWorker(ctx context.Context, c *lemmy.WSClient, ch <-chan replyJ log.Info("Created new comment"). Int("post-id", reply.PostID). Int("parent-id", reply.CommentID.ValueOr(-1)). + Int("comment-id", cr.CommentView.Comment.ID). Send() case <-ctx.Done(): return @@ -260,20 +280,6 @@ func executeTmpl(tmpl *template.Template, tc TmplContext) (string, error) { return sb.String(), err } -func joinAll(c *lemmy.WSClient) { - err := c.Request(types.UserOperationUserJoin, nil) - if err != nil { - log.Fatal("Error joining WebSocket user context").Err(err).Send() - } - - err = c.Request(types.UserOperationCommunityJoin, types.CommunityJoin{ - CommunityID: 0, - }) - if err != nil { - log.Fatal("Error joining WebSocket community context").Err(err).Send() - } -} - // toSubmatches converts matches coming from PCRE2 to a // submatch array used for the template func toSubmatches(s [][]string) []Submatches { diff --git a/sql/queries.sql b/sql/queries.sql new file mode 100644 index 0000000..8ee34af --- /dev/null +++ b/sql/queries.sql @@ -0,0 +1,5 @@ +/* name: ItemExists :one */ +SELECT COUNT(1) FROM replied_items WHERE item_type = ? AND id = ? AND updated_time = ?; + +/* name: AddItem :exec */ +INSERT OR REPLACE INTO replied_items (id, item_type, updated_time) VALUES (?, ?, ?); \ No newline at end of file diff --git a/sql/schema.sql b/sql/schema.sql new file mode 100644 index 0000000..6549e97 --- /dev/null +++ b/sql/schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS replied_items ( + id INT NOT NULL PRIMARY KEY, + item_type TEXT NOT NULL CHECK( item_type IN ('p', 'c') ), + updated_time INT NOT NULL, + UNIQUE(id, item_type) +); \ No newline at end of file diff --git a/sqlc.yaml b/sqlc.yaml new file mode 100644 index 0000000..5deedc9 --- /dev/null +++ b/sqlc.yaml @@ -0,0 +1,9 @@ +version: '2' +sql: + - schema: sql/schema.sql + queries: sql/queries.sql + engine: sqlite + gen: + go: + package: store + out: internal/store \ No newline at end of file