Use aria2 for torrent downloads
ci/woodpecker/push/woodpecker Pipeline was successful Details

This commit is contained in:
Elara 2023-07-11 13:14:54 -07:00
parent 36d3be72a7
commit 7a65835816
6 changed files with 442 additions and 711 deletions

View File

@ -43,6 +43,8 @@ nfpms:
- linux-user-repository
conflicts:
- linux-user-repository
recommends:
- aria2
contents:
- src: scripts/completion/bash
dst: /usr/share/bash-completion/completions/lure
@ -64,6 +66,8 @@ aurs:
depends:
- sudo
- pacman
optdepends:
- 'aria2: for downloading torrent sources'
package: |-
# binaries
install -Dm755 ./lure "${pkgdir}/usr/bin/lure"

View File

@ -380,7 +380,7 @@ func buildPackage(ctx context.Context, script string, mgr manager.Manager, clean
} else {
log.Fatal("The package() function is required").Send()
}
log.Info("Building package metadata").Str("name", vars.Name).Send()
uniq(
@ -393,6 +393,7 @@ func buildPackage(ctx context.Context, script string, mgr manager.Manager, clean
Name: vars.Name,
Description: vars.Description,
Arch: cpu.Arch(),
Platform: "linux",
Version: vars.Version,
Release: strconv.Itoa(vars.Release),
Homepage: vars.Homepage,
@ -500,7 +501,7 @@ func buildPackage(ctx context.Context, script string, mgr manager.Manager, clean
if err != nil {
return nil, nil, err
}
log.Info("Compressing package").Str("name", pkgName).Send()
err = packager.Package(pkgInfo, pkgFile)

172
go.mod
View File

@ -3,168 +3,120 @@ module go.elara.ws/lure
go 1.18
require (
github.com/AlecAivazis/survey/v2 v2.3.6
github.com/AlecAivazis/survey/v2 v2.3.7
github.com/PuerkitoBio/purell v1.2.0
github.com/alecthomas/chroma/v2 v2.4.0
github.com/anacrolix/log v0.14.0
github.com/anacrolix/torrent v1.51.3
github.com/charmbracelet/bubbles v0.14.0
github.com/charmbracelet/bubbletea v0.23.1
github.com/charmbracelet/lipgloss v0.6.0
github.com/alecthomas/chroma/v2 v2.8.0
github.com/charmbracelet/bubbles v0.16.1
github.com/charmbracelet/bubbletea v0.24.2
github.com/charmbracelet/lipgloss v0.7.1
github.com/go-chi/chi/v5 v5.0.8
github.com/go-git/go-billy/v5 v5.3.1
github.com/go-git/go-git/v5 v5.4.2
github.com/goreleaser/nfpm/v2 v2.20.0
github.com/go-git/go-billy/v5 v5.4.1
github.com/go-git/go-git/v5 v5.7.0
github.com/goreleaser/nfpm/v2 v2.31.0
github.com/jmoiron/sqlx v1.3.5
github.com/mattn/go-isatty v0.0.18
github.com/mholt/archiver/v4 v4.0.0-alpha.7
github.com/mattn/go-isatty v0.0.19
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/mitchellh/mapstructure v1.5.0
github.com/muesli/reflow v0.3.0
github.com/pelletier/go-toml/v2 v2.0.7
github.com/schollz/progressbar/v3 v3.13.0
github.com/pelletier/go-toml/v2 v2.0.8
github.com/schollz/progressbar/v3 v3.13.1
github.com/twitchtv/twirp v8.1.3+incompatible
github.com/urfave/cli/v2 v2.23.7
github.com/urfave/cli/v2 v2.25.7
github.com/vmihailenco/msgpack/v5 v5.3.5
go.elara.ws/logger v0.0.0-20230421022458-e80700db2090
go.elara.ws/translate v0.0.0-20230421025926-32ccfcd110e6
go.elara.ws/vercmp v0.0.0-20230622214216-0b2b067575c4
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
golang.org/x/sys v0.7.0
golang.org/x/text v0.9.0
google.golang.org/protobuf v1.28.1
golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb
golang.org/x/sys v0.10.0
golang.org/x/text v0.11.0
google.golang.org/protobuf v1.31.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.20.0
mvdan.cc/sh/v3 v3.5.1
modernc.org/sqlite v1.24.0
mvdan.cc/sh/v3 v3.7.0
)
require (
crawshaw.io/sqlite v0.3.3-0.20220618202545-d1964889ea3c // indirect
github.com/AlekSi/pointer v1.2.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/semver/v3 v3.1.1 // indirect
github.com/Masterminds/sprig v2.22.0+incompatible // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/Masterminds/sprig/v3 v3.2.3 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20210512092938-c05353c2d58c // indirect
github.com/RoaringBitmap/roaring v1.2.3 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/anacrolix/chansync v0.3.0 // indirect
github.com/anacrolix/dht/v2 v2.19.2-0.20221121215055-066ad8494444 // indirect
github.com/anacrolix/envpprof v1.2.1 // indirect
github.com/anacrolix/generics v0.0.0-20230428105757-683593396d68 // indirect
github.com/anacrolix/go-libutp v1.2.0 // indirect
github.com/anacrolix/missinggo v1.3.0 // indirect
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
github.com/anacrolix/missinggo/v2 v2.7.0 // indirect
github.com/anacrolix/mmsg v1.0.0 // indirect
github.com/anacrolix/multiless v0.3.0 // indirect
github.com/anacrolix/stm v0.4.0 // indirect
github.com/anacrolix/sync v0.4.0 // indirect
github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 // indirect
github.com/anacrolix/utp v0.1.0 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230626094100-7e9e0395ebec // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/aymanbagabas/go-osc52 v1.0.3 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/benbjohnson/immutable v0.3.0 // indirect
github.com/bits-and-blooms/bitset v1.2.2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
github.com/bodgit/plumbing v1.2.0 // indirect
github.com/bodgit/sevenzip v1.3.0 // indirect
github.com/bodgit/windows v1.0.0 // indirect
github.com/cavaliergopher/cpio v1.0.1 // indirect
github.com/containerd/console v1.0.3 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/connesc/cipherio v0.2.1 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.4.0 // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gookit/color v1.5.3 // indirect
github.com/goreleaser/chglog v0.2.2 // indirect
github.com/gookit/color v1.5.1 // indirect
github.com/goreleaser/chglog v0.5.0 // indirect
github.com/goreleaser/fileglob v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/huandu/xstrings v1.3.3 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kevinburke/ssh_config v1.1.0 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/lispad/go-generics-tools v1.1.0 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.16.6 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-colorable v0.1.11 // indirect
github.com/mattn/go-colorable v0.1.2 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.13.0 // indirect
github.com/muesli/termenv v0.15.1 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pion/datachannel v1.5.2 // indirect
github.com/pion/dtls/v2 v2.2.4 // indirect
github.com/pion/ice/v2 v2.2.6 // indirect
github.com/pion/interceptor v0.1.11 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns v0.0.5 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.9 // indirect
github.com/pion/rtp v1.7.13 // indirect
github.com/pion/sctp v1.8.2 // indirect
github.com/pion/sdp/v3 v3.0.5 // indirect
github.com/pion/srtp/v2 v2.0.9 // indirect
github.com/pion/stun v0.3.5 // indirect
github.com/pion/transport v0.13.1 // indirect
github.com/pion/transport/v2 v2.0.0 // indirect
github.com/pion/turn/v2 v2.0.8 // indirect
github.com/pion/udp v0.1.4 // indirect
github.com/pion/webrtc/v3 v3.1.42 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/stretchr/testify v1.8.2 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/skeema/knownhosts v1.1.1 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/tidwall/btree v1.6.0 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.1 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
go.opentelemetry.io/otel v1.8.0 // indirect
go.opentelemetry.io/otel/trace v1.8.0 // indirect
golang.org/x/crypto v0.5.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/crypto v0.9.0 // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/term v0.8.0 // indirect
golang.org/x/tools v0.6.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13 // indirect
modernc.org/libc v1.21.5 // indirect
modernc.org/libc v1.22.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.4.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/strutil v1.1.3 // indirect
modernc.org/token v1.0.1 // indirect

816
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -26,6 +26,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
"github.com/PuerkitoBio/purell"
"github.com/vmihailenco/msgpack/v5"
@ -319,5 +320,12 @@ func normalizeURL(u string) (string, error) {
purell.FlagDecodeUnnecessaryEscapes |
purell.FlagRemoveEmptyPortSeparator
return purell.NormalizeURLString(u, normalizationFlags)
u, err := purell.NormalizeURLString(u, normalizationFlags)
if err != nil {
return "", err
}
// Fix magnet URLs after normalization
u = strings.Replace(u, "magnet://", "magnet:", 1)
return u, nil
}

View File

@ -2,20 +2,18 @@ package dl
import (
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
alog "github.com/anacrolix/log"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
"github.com/schollz/progressbar/v3"
"go.elara.ws/logger/log"
"errors"
)
var urlMatchRegex = regexp.MustCompile(`(magnet|torrent\+https?):.*`)
var ErrAria2NotFound = errors.New("aria2 must be installed for torrent functionality")
var ErrDestinationEmpty = errors.New("the destination directory is empty")
type TorrentDownloader struct{}
@ -32,90 +30,60 @@ func (TorrentDownloader) MatchURL(u string) bool {
// Download downloads a file over the BitTorrent protocol.
func (TorrentDownloader) Download(opts Options) (Type, string, error) {
cfg := torrent.NewDefaultClientConfig()
cfg.DataDir = opts.Destination
cfg.DisableWebseeds = true
cfg.Logger.SetHandlers(alog.DiscardHandler)
aria2Path, err := exec.LookPath("aria2c")
if err != nil {
return 0, "", ErrAria2NotFound
}
c, err := torrent.NewClient(cfg)
opts.URL = strings.TrimPrefix(opts.URL, "torrent+")
cmd := exec.Command(aria2Path, "--summary-interval=0", "--log-level=warn", "--seed-time=0", "--dir="+opts.Destination, opts.URL)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
if err != nil {
return 0, "", fmt.Errorf("aria2c returned an error: %w", err)
}
err = removeTorrentFiles(opts.Destination)
if err != nil {
return 0, "", err
}
defer c.Close()
var t *torrent.Torrent
if strings.HasPrefix(opts.URL, "magnet:") {
t, err = c.AddMagnet(opts.URL)
if err != nil {
return 0, "", err
}
log.Info("Waiting for torrent metadata").Str("source", opts.Name).Send()
<-t.GotInfo()
} else if strings.HasPrefix(opts.URL, "torrent+") {
log.Info("Downloading torrent file").Str("source", opts.Name).Send()
res, err := http.Get(strings.TrimPrefix(opts.URL, "torrent+"))
if err != nil {
return 0, "", err
}
meta, err := metainfo.Load(res.Body)
if err != nil {
return 0, "", err
}
t, err = c.AddTorrent(meta)
if err != nil {
return 0, "", err
}
}
t.DownloadAll()
info := t.Info()
info.BestName()
var bar *progressbar.ProgressBar
if opts.Progress != nil {
bar = progressbar.NewOptions64(
info.TotalLength(),
progressbar.OptionSetDescription(info.Name),
progressbar.OptionSetWriter(opts.Progress),
progressbar.OptionShowBytes(true),
progressbar.OptionSetWidth(10),
progressbar.OptionThrottle(65*time.Millisecond),
progressbar.OptionShowCount(),
progressbar.OptionOnCompletion(func() {
_, _ = io.WriteString(opts.Progress, "\n")
}),
progressbar.OptionSpinnerType(14),
progressbar.OptionFullWidth(),
progressbar.OptionSetRenderBlankState(true),
)
defer bar.Close()
}
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for range ticker.C {
info = t.Info()
if t.Complete.Bool() {
if info.IsDir() {
return TypeDir, info.Name, nil
} else {
return TypeFile, info.Name, nil
}
}
if bar != nil {
bar.ChangeMax64(info.TotalLength())
bar.Set64(t.BytesCompleted())
stats := t.Stats()
bar.Describe(fmt.Sprintf("%s [%d/%d]", info.Name, stats.ActivePeers, stats.TotalPeers))
}
}
// This code should never execute because the loop will return from the function
// once the torrent has finished downloading.
panic("unreachable")
return determineType(opts.Destination)
}
func removeTorrentFiles(path string) error {
filePaths, err := filepath.Glob(filepath.Join(path, "*.torrent"))
if err != nil {
return err
}
for _, filePath := range filePaths {
err = os.Remove(filePath)
if err != nil {
return err
}
}
return nil
}
func determineType(path string) (Type, string, error) {
files, err := os.ReadDir(path)
if err != nil {
return 0, "", err
}
if len(files) > 1 {
return TypeDir, "", nil
} else if len(files) == 1 {
if files[0].IsDir() {
return TypeDir, files[0].Name(), nil
} else {
return TypeFile, files[0].Name(), nil
}
}
return 0, "", ErrDestinationEmpty
}