Add the ability to change the hashing algorithm
ci/woodpecker/push/woodpecker Pipeline was successful Details

This commit is contained in:
Elara 2023-07-11 17:00:30 -07:00
parent e8cb614b2e
commit 365634f4d9
4 changed files with 69 additions and 20 deletions

View File

@ -609,11 +609,21 @@ func getSources(ctx context.Context, srcdir string, bv *BuildVars) error {
}
if !strings.EqualFold(bv.Checksums[i], "SKIP") {
checksum, err := hex.DecodeString(bv.Checksums[i])
if err != nil {
return err
algo, hashData, ok := strings.Cut(bv.Checksums[i], ":")
if ok {
checksum, err := hex.DecodeString(hashData)
if err != nil {
return err
}
opts.Hash = checksum
opts.HashAlgorithm = algo
} else {
checksum, err := hex.DecodeString(bv.Checksums[i])
if err != nil {
return err
}
opts.Hash = checksum
}
opts.SHA256 = checksum
}
err := dl.Download(ctx, opts)

View File

@ -199,7 +199,9 @@ git+https://gitea.arsenm.dev/Arsen6331/lure?~rev=v0.0.1&~recursive=true
### checksums
The `checksums` array must be the same length as the `sources` array. It contains sha256 checksums for the source files. The files are checked against the checksums and the build fails if they don't match.
The `checksums` array must be the same length as the `sources` array. It contains checksums for the source files. The files are checked against the checksums and the build fails if they don't match.
By default, checksums are expected to be sha256. To change the algorithm, add it before the hash with a colon in between. For example, `md5:bc0c6f5dcd06bddbca9a0163e4c9f2e1`. The following algorithms are currently supported: `sha256`, `sha224`, `sha512`, `sha384`, `sha1`, and `md5`.
To skip the check for a particular source, set the corresponding checksum to `SKIP`.

View File

@ -22,7 +22,13 @@ package dl
import (
"context"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"errors"
"fmt"
"hash"
"io"
"os"
"path/filepath"
@ -39,7 +45,10 @@ const manifestFileName = ".lure_cache_manifest"
// ErrChecksumMismatch occurs when the checksum of a downloaded file
// does not match the expected checksum provided in the Options struct.
var ErrChecksumMismatch = errors.New("dl: checksums did not match")
var (
ErrChecksumMismatch = errors.New("dl: checksums did not match")
ErrNoSuchHashAlgo = errors.New("dl: invalid hashing algorithm")
)
// Downloaders contains all the downloaders in the order in which
// they should be checked
@ -70,7 +79,8 @@ func (t Type) String() string {
// Options contains the options for downloading
// files and directories
type Options struct {
SHA256 []byte
Hash []byte
HashAlgorithm string
Name string
URL string
Destination string
@ -79,6 +89,27 @@ type Options struct {
Progress io.Writer
}
func (opts Options) NewHash() (hash.Hash, error) {
if opts.HashAlgorithm == "" {
opts.HashAlgorithm = "sha256"
}
switch opts.HashAlgorithm {
case "sha256":
return sha256.New(), nil
case "sha224":
return sha256.New224(), nil
case "sha512":
return sha512.New(), nil
case "sha384":
return sha512.New384(), nil
case "sha1":
return sha1.New(), nil
case "md5":
return md5.New(), nil
}
return nil, fmt.Errorf("%w: %s", ErrNoSuchHashAlgo, opts.HashAlgorithm)
}
// Manifest holds information about the type and name
// of a downloaded file or directory. It is stored inside
// each cache directory for later use.
@ -144,10 +175,12 @@ func Download(ctx context.Context, opts Options) (err error) {
log.Info("Source can be updated, updating if required").Str("source", opts.Name).Str("downloader", d.Name()).Send()
updated, err = d.Update(Options{
Name: opts.Name,
URL: opts.URL,
Destination: cacheDir,
Progress: opts.Progress,
Hash: opts.Hash,
HashAlgorithm: opts.HashAlgorithm,
Name: opts.Name,
URL: opts.URL,
Destination: cacheDir,
Progress: opts.Progress,
})
if err != nil {
return err
@ -189,10 +222,12 @@ func Download(ctx context.Context, opts Options) (err error) {
}
t, name, err := d.Download(Options{
Name: opts.Name,
URL: opts.URL,
Destination: cacheDir,
Progress: opts.Progress,
Hash: opts.Hash,
HashAlgorithm: opts.HashAlgorithm,
Name: opts.Name,
URL: opts.URL,
Destination: cacheDir,
Progress: opts.Progress,
})
if err != nil {
return err

View File

@ -21,7 +21,6 @@ package dl
import (
"bytes"
"context"
"crypto/sha256"
"io"
"net/http"
"net/url"
@ -109,10 +108,13 @@ func (FileDownloader) Download(opts Options) (Type, string, error) {
bar = shutils.NopRWC{}
}
h := sha256.New()
h, err := opts.NewHash()
if err != nil {
return 0, "", err
}
var w io.Writer
if opts.SHA256 != nil {
if opts.Hash != nil {
w = io.MultiWriter(fl, h, bar)
} else {
w = io.MultiWriter(fl, bar)
@ -124,9 +126,9 @@ func (FileDownloader) Download(opts Options) (Type, string, error) {
}
res.Body.Close()
if opts.SHA256 != nil {
if opts.Hash != nil {
sum := h.Sum(nil)
if !bytes.Equal(sum, opts.SHA256) {
if !bytes.Equal(sum, opts.Hash) {
return 0, "", ErrChecksumMismatch
}
}