Add comments for internal/{dl,dlcache}
ci/woodpecker/push/woodpecker Pipeline was successful
Details
ci/woodpecker/push/woodpecker Pipeline was successful
Details
This commit is contained in:
parent
163ad12575
commit
fa4604c26c
|
@ -1,3 +1,5 @@
|
||||||
|
// Package dl contains abstractions for downloadingfiles and directories
|
||||||
|
// from various sources.
|
||||||
package dl
|
package dl
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
@ -14,13 +16,18 @@ import (
|
||||||
|
|
||||||
const manifestFileName = ".lure_cache_manifest"
|
const manifestFileName = ".lure_cache_manifest"
|
||||||
|
|
||||||
|
// ErrChecksumMismatch occurs when the checksum of a downloaded file
|
||||||
|
// does not match the expected checksum provided in the Options struct.
|
||||||
var ErrChecksumMismatch = errors.New("dl: checksums did not match")
|
var ErrChecksumMismatch = errors.New("dl: checksums did not match")
|
||||||
|
|
||||||
|
// Downloaders contains all the downloaders in the order in which
|
||||||
|
// they should be checked
|
||||||
var Downloaders = []Downloader{
|
var Downloaders = []Downloader{
|
||||||
GitDownloader{},
|
GitDownloader{},
|
||||||
FileDownloader{},
|
FileDownloader{},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Type represents the type of download (file or directory)
|
||||||
type Type uint8
|
type Type uint8
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -38,6 +45,8 @@ func (t Type) String() string {
|
||||||
return "<unknown>"
|
return "<unknown>"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Options contains the options for downloading
|
||||||
|
// files and directories
|
||||||
type Options struct {
|
type Options struct {
|
||||||
SHA256 []byte
|
SHA256 []byte
|
||||||
Name string
|
Name string
|
||||||
|
@ -48,6 +57,9 @@ type Options struct {
|
||||||
Progress io.Writer
|
Progress io.Writer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Manifest holds information about the type and name
|
||||||
|
// of a downloaded file or directory. It is stored inside
|
||||||
|
// each cache directory for later use.
|
||||||
type Manifest struct {
|
type Manifest struct {
|
||||||
Type Type
|
Type Type
|
||||||
Name string
|
Name string
|
||||||
|
@ -59,11 +71,22 @@ type Downloader interface {
|
||||||
Download(Options) (Type, string, error)
|
Download(Options) (Type, string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UpdatingDownloader extends the Downloader interface
|
||||||
|
// with an Update method for protocols such as git, which
|
||||||
|
// allow for incremental updates without changing the URL.
|
||||||
type UpdatingDownloader interface {
|
type UpdatingDownloader interface {
|
||||||
Downloader
|
Downloader
|
||||||
Update(Options) (bool, error)
|
Update(Options) (bool, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Download downloads a file or directory using the specified options.
|
||||||
|
// It first gets the appropriate downloader for the URL, then checks
|
||||||
|
// if caching is enabled. If caching is enabled, it attempts to get
|
||||||
|
// the cache directory for the URL and update it if necessary.
|
||||||
|
// If the source is found in the cache, it links it to the destination
|
||||||
|
// using hard links. If the source is not found in the cache,
|
||||||
|
// it downloads the source to a new cache directory and links it
|
||||||
|
// to the destination.
|
||||||
func Download(ctx context.Context, opts Options) (err error) {
|
func Download(ctx context.Context, opts Options) (err error) {
|
||||||
d := getDownloader(opts.URL)
|
d := getDownloader(opts.URL)
|
||||||
|
|
||||||
|
@ -144,6 +167,7 @@ func Download(ctx context.Context, opts Options) (err error) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeManifest writes the manifest to the specified cache directory.
|
||||||
func writeManifest(cacheDir string, m Manifest) error {
|
func writeManifest(cacheDir string, m Manifest) error {
|
||||||
fl, err := os.Create(filepath.Join(cacheDir, manifestFileName))
|
fl, err := os.Create(filepath.Join(cacheDir, manifestFileName))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -153,6 +177,7 @@ func writeManifest(cacheDir string, m Manifest) error {
|
||||||
return msgpack.NewEncoder(fl).Encode(m)
|
return msgpack.NewEncoder(fl).Encode(m)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getManifest reads the manifest from the specified cache directory.
|
||||||
func getManifest(cacheDir string) (m Manifest, err error) {
|
func getManifest(cacheDir string) (m Manifest, err error) {
|
||||||
fl, err := os.Open(filepath.Join(cacheDir, manifestFileName))
|
fl, err := os.Open(filepath.Join(cacheDir, manifestFileName))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -164,6 +189,7 @@ func getManifest(cacheDir string) (m Manifest, err error) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handleCache links the cache directory or a file within it to the destination
|
||||||
func handleCache(cacheDir, dest string, t Type) (bool, error) {
|
func handleCache(cacheDir, dest string, t Type) (bool, error) {
|
||||||
switch t {
|
switch t {
|
||||||
case TypeFile:
|
case TypeFile:
|
||||||
|
@ -202,6 +228,12 @@ func handleCache(cacheDir, dest string, t Type) (bool, error) {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// linkDir recursively walks through a directory, creating
|
||||||
|
// hard links for each file from the src directory to the
|
||||||
|
// dest directory. If it encounters a directory, it will
|
||||||
|
// create a directory with the same name and permissions
|
||||||
|
// in the dest directory, because hard links cannot be
|
||||||
|
// created for directories.
|
||||||
func linkDir(src, dest string) error {
|
func linkDir(src, dest string) error {
|
||||||
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
|
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -18,20 +18,22 @@ import (
|
||||||
"go.arsenm.dev/lure/internal/shutils"
|
"go.arsenm.dev/lure/internal/shutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// FileDownloader downloads files using HTTP
|
||||||
type FileDownloader struct{}
|
type FileDownloader struct{}
|
||||||
|
|
||||||
|
// Name always returns "file"
|
||||||
func (FileDownloader) Name() string {
|
func (FileDownloader) Name() string {
|
||||||
return "file"
|
return "file"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (FileDownloader) Type() Type {
|
// MatchURL always returns true, as FileDownloader
|
||||||
return TypeFile
|
// is used as a fallback if nothing else matches
|
||||||
}
|
|
||||||
|
|
||||||
func (FileDownloader) MatchURL(string) bool {
|
func (FileDownloader) MatchURL(string) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Download downloads a file using HTTP. If the file is
|
||||||
|
// compressed using a supported format, it will be extracted
|
||||||
func (FileDownloader) Download(opts Options) (Type, string, error) {
|
func (FileDownloader) Download(opts Options) (Type, string, error) {
|
||||||
res, err := http.Get(opts.URL)
|
res, err := http.Get(opts.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -115,6 +117,7 @@ func (FileDownloader) Download(opts Options) (Type, string, error) {
|
||||||
return TypeDir, "", err
|
return TypeDir, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// extractFile extracts an archive or decompresses a file
|
||||||
func extractFile(r io.Reader, format archiver.Format, name string, opts Options) (err error) {
|
func extractFile(r io.Reader, format archiver.Format, name string, opts Options) (err error) {
|
||||||
fname := format.Name()
|
fname := format.Name()
|
||||||
|
|
||||||
|
@ -185,6 +188,10 @@ func extractFile(r io.Reader, format archiver.Format, name string, opts Options)
|
||||||
|
|
||||||
var cdHeaderRgx = regexp.MustCompile(`filename="(.+)"`)
|
var cdHeaderRgx = regexp.MustCompile(`filename="(.+)"`)
|
||||||
|
|
||||||
|
// getFilename attempts to parse the Content-Disposition
|
||||||
|
// HTTP response header and extract a filename. If the
|
||||||
|
// header does not exist, it will use the last element
|
||||||
|
// of the path.
|
||||||
func getFilename(res *http.Response) (name string) {
|
func getFilename(res *http.Response) (name string) {
|
||||||
cd := res.Header.Get("Content-Disposition")
|
cd := res.Header.Get("Content-Disposition")
|
||||||
matches := cdHeaderRgx.FindStringSubmatch(cd)
|
matches := cdHeaderRgx.FindStringSubmatch(cd)
|
||||||
|
|
|
@ -11,20 +11,22 @@ import (
|
||||||
"github.com/go-git/go-git/v5/plumbing"
|
"github.com/go-git/go-git/v5/plumbing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// GitDownloader downloads Git repositories
|
||||||
type GitDownloader struct{}
|
type GitDownloader struct{}
|
||||||
|
|
||||||
|
// Name always returns "git"
|
||||||
func (GitDownloader) Name() string {
|
func (GitDownloader) Name() string {
|
||||||
return "git"
|
return "git"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (GitDownloader) Type() Type {
|
// MatchURL matches any URLs that start with "git+"
|
||||||
return TypeDir
|
|
||||||
}
|
|
||||||
|
|
||||||
func (GitDownloader) MatchURL(u string) bool {
|
func (GitDownloader) MatchURL(u string) bool {
|
||||||
return strings.HasPrefix(u, "git+")
|
return strings.HasPrefix(u, "git+")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Download uses git to clone the repository from the specified URL.
|
||||||
|
// It allows specifying the revision, depth and recursion options
|
||||||
|
// via query string
|
||||||
func (GitDownloader) Download(opts Options) (Type, string, error) {
|
func (GitDownloader) Download(opts Options) (Type, string, error) {
|
||||||
u, err := url.Parse(opts.URL)
|
u, err := url.Parse(opts.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -92,6 +94,11 @@ func (GitDownloader) Download(opts Options) (Type, string, error) {
|
||||||
return TypeDir, name, nil
|
return TypeDir, name, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update uses git to pull the repository and update it
|
||||||
|
// to the latest revision. It allows specifying the depth
|
||||||
|
// and recursion options via query string. It returns
|
||||||
|
// true if update was successful and false if the
|
||||||
|
// repository is already up-to-date
|
||||||
func (GitDownloader) Update(opts Options) (bool, error) {
|
func (GitDownloader) Update(opts Options) (bool, error) {
|
||||||
u, err := url.Parse(opts.URL)
|
u, err := url.Parse(opts.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -10,8 +10,12 @@ import (
|
||||||
"go.arsenm.dev/lure/internal/config"
|
"go.arsenm.dev/lure/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// BasePath stores the base path to the download cache
|
||||||
var BasePath = filepath.Join(config.CacheDir, "dl")
|
var BasePath = filepath.Join(config.CacheDir, "dl")
|
||||||
|
|
||||||
|
// New creates a new directory with the given ID in the cache.
|
||||||
|
// If a directory with the same ID already exists,
|
||||||
|
// it will be deleted before creating a new one.
|
||||||
func New(id string) (string, error) {
|
func New(id string) (string, error) {
|
||||||
h, err := hashID(id)
|
h, err := hashID(id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -35,6 +39,11 @@ func New(id string) (string, error) {
|
||||||
return itemPath, nil
|
return itemPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get checks if an entry with the given ID
|
||||||
|
// already exists in the cache, and if so,
|
||||||
|
// returns the directory and true. If it
|
||||||
|
// does not exist, it returns an empty string
|
||||||
|
// and false.
|
||||||
func Get(id string) (string, bool) {
|
func Get(id string) (string, bool) {
|
||||||
h, err := hashID(id)
|
h, err := hashID(id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -50,6 +59,9 @@ func Get(id string) (string, bool) {
|
||||||
return itemPath, true
|
return itemPath, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// hashID hashes the input ID with SHA1
|
||||||
|
// and returns the hex string of the hashed
|
||||||
|
// ID.
|
||||||
func hashID(id string) (string, error) {
|
func hashID(id string) (string, error) {
|
||||||
h := sha1.New()
|
h := sha1.New()
|
||||||
_, err := io.WriteString(h, id)
|
_, err := io.WriteString(h, id)
|
||||||
|
|
Loading…
Reference in New Issue