Normalize URL for caching
ci/woodpecker/push/woodpecker Pipeline was successful
Details
ci/woodpecker/push/woodpecker Pipeline was successful
Details
This commit is contained in:
parent
192e3e8a4f
commit
0ece536810
1
go.mod
1
go.mod
|
@ -4,6 +4,7 @@ go 1.18
|
|||
|
||||
require (
|
||||
github.com/AlecAivazis/survey/v2 v2.3.6
|
||||
github.com/PuerkitoBio/purell v1.2.0
|
||||
github.com/alecthomas/chroma/v2 v2.4.0
|
||||
github.com/charmbracelet/bubbles v0.14.0
|
||||
github.com/charmbracelet/bubbletea v0.23.1
|
||||
|
|
2
go.sum
2
go.sum
|
@ -23,6 +23,8 @@ github.com/ProtonMail/go-crypto v0.0.0-20210512092938-c05353c2d58c h1:bNpaLLv2Y4
|
|||
github.com/ProtonMail/go-crypto v0.0.0-20210512092938-c05353c2d58c/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f h1:CGq7OieOz3wyQJ1fO8S0eO9TCW1JyvLrf8fhzz1i8ko=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.2.2 h1:u2m7xt+CZWj88qK1UUNBoXeJCFJwJCZ/Ff4ymGoxEXs=
|
||||
github.com/PuerkitoBio/purell v1.2.0 h1:/Jdm5QfyM8zdlqT6WVZU4cfP23sot6CEHA4CS49Ezig=
|
||||
github.com/PuerkitoBio/purell v1.2.0/go.mod h1:OhLRTaaIzhvIyofkJfB24gokC7tM42Px5UhoT32THBk=
|
||||
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
|
||||
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
||||
github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw=
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/PuerkitoBio/purell"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go.arsenm.dev/logger/log"
|
||||
"go.arsenm.dev/lure/internal/dlcache"
|
||||
|
@ -88,6 +89,12 @@ type UpdatingDownloader interface {
|
|||
// it downloads the source to a new cache directory and links it
|
||||
// to the destination.
|
||||
func Download(ctx context.Context, opts Options) (err error) {
|
||||
normalized, err := normalizeURL(opts.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts.URL = normalized
|
||||
|
||||
d := getDownloader(opts.URL)
|
||||
|
||||
if opts.CacheDisabled {
|
||||
|
@ -266,3 +273,22 @@ func getDownloader(u string) Downloader {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeURL normalizes a URL string, so that insignificant
|
||||
// don't change the hash.
|
||||
func normalizeURL(u string) (string, error) {
|
||||
const normalizationFlags = purell.FlagRemoveTrailingSlash |
|
||||
purell.FlagRemoveDefaultPort |
|
||||
purell.FlagLowercaseHost |
|
||||
purell.FlagLowercaseScheme |
|
||||
purell.FlagRemoveDuplicateSlashes |
|
||||
purell.FlagRemoveFragment |
|
||||
purell.FlagRemoveUnnecessaryHostDots |
|
||||
purell.FlagSortQuery |
|
||||
purell.FlagDecodeHexHost |
|
||||
purell.FlagDecodeOctalHost |
|
||||
purell.FlagDecodeUnnecessaryEscapes |
|
||||
purell.FlagRemoveEmptyPortSeparator
|
||||
|
||||
return purell.NormalizeURLString(u, normalizationFlags)
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue