From 6ed31f252cf110a9d0a91807f73434398c0b83e9 Mon Sep 17 00:00:00 2001 From: Elara Musayelyan Date: Thu, 21 Sep 2023 13:27:23 -0700 Subject: [PATCH] Add some comments to internal/build --- internal/build/build.go | 113 +++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 37 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 364f02d..f8b4345 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -69,6 +69,9 @@ func BuildPackage(ctx context.Context, opts types.BuildOpts) ([]string, []string return nil, nil, err } + // The first pass is just used to get variable values and runs before + // the script is displayed, so it's restricted so as to prevent malicious + // code from executing. vars, err := executeFirstPass(ctx, info, fl, opts.Script) if err != nil { return nil, nil, err @@ -76,6 +79,8 @@ func BuildPackage(ctx context.Context, opts types.BuildOpts) ([]string, []string dirs := getDirs(vars, opts.Script) + // If opts.Clean isn't set and we find the package already built, + // just return it rather than rebuilding if !opts.Clean { builtPkgPath, ok, err := checkForBuiltPackage(opts.Manager, vars, getPkgFormat(opts.Manager), dirs.BaseDir) if err != nil { @@ -87,6 +92,7 @@ func BuildPackage(ctx context.Context, opts types.BuildOpts) ([]string, []string } } + // Ask the user if they'd like to see the build script err = cliutils.PromptViewScript(opts.Script, vars.Name, config.Config().PagerStyle, opts.Interactive) if err != nil { log.Fatal("Failed to prompt user to view build script").Err(err).Send() @@ -94,11 +100,15 @@ func BuildPackage(ctx context.Context, opts types.BuildOpts) ([]string, []string log.Info("Building package").Str("name", vars.Name).Str("version", vars.Version).Send() + // The second pass will be used to execute the actual code, + // so it's unrestricted. The script has already been displayed + // to the user by this point, so it should be safe dec, err := executeSecondPass(ctx, info, fl, dirs) if err != nil { return nil, nil, err } + // Get the installed packages on the system installed, err := opts.Manager.ListInstalled(nil) if err != nil { return nil, nil, err @@ -111,6 +121,7 @@ func BuildPackage(ctx context.Context, opts types.BuildOpts) ([]string, []string os.Exit(1) } + // Prepare the directories for building err = prepareDirs(dirs) if err != nil { return nil, nil, err @@ -126,7 +137,7 @@ func BuildPackage(ctx context.Context, opts types.BuildOpts) ([]string, []string return nil, nil, err } - builtPaths, builtNames, repoDeps, err := installDeps(ctx, opts, vars) + builtPaths, builtNames, repoDeps, err := buildLUREDeps(ctx, opts, vars) if err != nil { return nil, nil, err } @@ -180,12 +191,16 @@ func BuildPackage(ctx context.Context, opts types.BuildOpts) ([]string, []string pkgPaths := append(builtPaths, pkgPath) pkgNames := append(builtNames, vars.Name) + // Remove any duplicates from the pkgPaths and pkgNames. + // Duplicates can be introduced if several of the dependencies + // depend on the same packages. pkgPaths = removeDuplicates(pkgPaths) pkgNames = removeDuplicates(pkgNames) return pkgPaths, pkgNames, nil } +// parseScript parses the build script using the built-in bash implementation func parseScript(info *distro.OSRelease, script string) (*syntax.File, error) { fl, err := os.Open(script) if err != nil { @@ -201,13 +216,12 @@ func parseScript(info *distro.OSRelease, script string) (*syntax.File, error) { return file, nil } +// executeFirstPass executes the parsed script in a restricted environment +// to extract the build variables without executing any actual code. func executeFirstPass(ctx context.Context, info *distro.OSRelease, fl *syntax.File, script string) (*types.BuildVars, error) { scriptDir := filepath.Dir(script) env := createBuildEnvVars(info, types.Directories{ScriptDir: scriptDir}) - // The first pass is just used to get variable values and runs before - // the script is displayed, so it is restricted so as to prevent malicious - // code from executing. runner, err := interp.New( interp.Env(expand.ListEnviron(env...)), interp.StdIO(os.Stdin, os.Stdout, os.Stderr), @@ -236,6 +250,7 @@ func executeFirstPass(ctx context.Context, info *distro.OSRelease, fl *syntax.Fi return &vars, nil } +// getDirs returns the appropriate directories for the script func getDirs(vars *types.BuildVars, script string) types.Directories { baseDir := filepath.Join(config.GetPaths().PkgsDir, vars.Name) return types.Directories{ @@ -246,11 +261,11 @@ func getDirs(vars *types.BuildVars, script string) types.Directories { } } +// executeSecondPass executes the build script for the second time, this time without any restrictions. +// It returns a decoder that can be used to retrieve functions and variables from the script. func executeSecondPass(ctx context.Context, info *distro.OSRelease, fl *syntax.File, dirs types.Directories) (*decoder.Decoder, error) { env := createBuildEnvVars(info, dirs) - // The second pass will be used to execute the actual functions, - // so it cannot be restricted. The script has already been displayed - // to the user by this point, so it should be safe + runner, err := interp.New( interp.Env(expand.ListEnviron(env...)), interp.StdIO(os.Stdin, os.Stdout, os.Stderr), @@ -268,6 +283,7 @@ func executeSecondPass(ctx context.Context, info *distro.OSRelease, fl *syntax.F return decoder.New(info, runner), nil } +// prepareDirs prepares the directories for building. func prepareDirs(dirs types.Directories) error { err := os.RemoveAll(dirs.BaseDir) if err != nil { @@ -280,6 +296,7 @@ func prepareDirs(dirs types.Directories) error { return os.MkdirAll(dirs.PkgDir, 0o755) } +// performChecks checks various things on the system to ensure that the package can be installed. func performChecks(vars *types.BuildVars, interactive bool, installed map[string]string) (bool, error) { if !cpu.IsCompatibleWith(cpu.Arch(), vars.Architectures) { cont, err := cliutils.YesNoPrompt("Your system's CPU architecture doesn't match this package. Do you want to build anyway?", interactive, true) @@ -302,6 +319,8 @@ func performChecks(vars *types.BuildVars, interactive bool, installed map[string return true, nil } +// installBuildDeps installs any build dependencies that aren't already installed and returns +// a slice containing the names of all the packages it installed. func installBuildDeps(ctx context.Context, vars *types.BuildVars, opts types.BuildOpts, installed map[string]string) ([]string, error) { var buildDeps []string if len(vars.BuildDepends) > 0 { @@ -310,7 +329,7 @@ func installBuildDeps(ctx context.Context, vars *types.BuildVars, opts types.Bui return nil, err } - found = filterBuildDeps(found, installed) + found = removeAlreadyInstalled(found, installed) log.Info("Installing build dependencies").Send() @@ -321,6 +340,8 @@ func installBuildDeps(ctx context.Context, vars *types.BuildVars, opts types.Bui return buildDeps, nil } +// installOptDeps asks the user which, if any, optional dependencies they want to install. +// If the user chooses to install any optional dependencies, it performs the installation. func installOptDeps(ctx context.Context, vars *types.BuildVars, opts types.BuildOpts, installed map[string]string) error { if len(vars.OptDepends) > 0 { optDeps, err := cliutils.ChooseOptDepends(vars.OptDepends, "install", opts.Interactive) @@ -337,15 +358,17 @@ func installOptDeps(ctx context.Context, vars *types.BuildVars, opts types.Build return err } - found = filterBuildDeps(found, installed) + found = removeAlreadyInstalled(found, installed) flattened := cliutils.FlattenPkgs(found, "install", opts.Interactive) - optDeps = packageNames(flattened) InstallPkgs(ctx, flattened, notFound, opts) } return nil } -func installDeps(ctx context.Context, opts types.BuildOpts, vars *types.BuildVars) (builtPaths, builtNames, repoDeps []string, err error) { +// buildLUREDeps builds all the LURE dependencies of the package. It returns the paths and names +// of the packages it built, as well as all the dependencies it didn't find in the LURE repo so +// they can be installed from the system repos. +func buildLUREDeps(ctx context.Context, opts types.BuildOpts, vars *types.BuildVars) (builtPaths, builtNames, repoDeps []string, err error) { if len(vars.Depends) > 0 { log.Info("Installing dependencies").Send() @@ -377,12 +400,15 @@ func installDeps(ctx context.Context, opts types.BuildOpts, vars *types.BuildVar } } + // Remove any potential duplicates, which can be introduced if + // several of the dependencies depend on the same packages. repoDeps = removeDuplicates(repoDeps) builtPaths = removeDuplicates(builtPaths) builtNames = removeDuplicates(builtNames) return builtPaths, builtNames, repoDeps, nil } +// executeFunctions executes the special LURE functions, such as version(), prepare(), etc. func executeFunctions(ctx context.Context, dec *decoder.Decoder, dirs types.Directories, vars *types.BuildVars) (err error) { version, ok := dec.GetFunc("version") if ok { @@ -444,6 +470,7 @@ func executeFunctions(ctx context.Context, dec *decoder.Decoder, dirs types.Dire return nil } +// buildPkgMetadata builds the metadata for the package that's going to be built. func buildPkgMetadata(vars *types.BuildVars, dirs types.Directories, deps []string) (*nfpm.Info, error) { pkgInfo := &nfpm.Info{ Name: vars.Name, @@ -482,6 +509,8 @@ func buildPkgMetadata(vars *types.BuildVars, dirs types.Directories, deps []stri return pkgInfo, nil } +// buildContents builds the contents section of the package, which contains the files +// that will be placed into the final package. func buildContents(vars *types.BuildVars, dirs types.Directories) ([]*files.Content, error) { contents := []*files.Content{} err := filepath.Walk(dirs.PkgDir, func(path string, fi os.FileInfo, err error) error { @@ -493,6 +522,7 @@ func buildContents(vars *types.BuildVars, dirs types.Directories) ([]*files.Cont return err } + // If the directory is empty, skip it _, err = f.Readdirnames(1) if err != io.EOF { return nil @@ -507,8 +537,7 @@ func buildContents(vars *types.BuildVars, dirs types.Directories) ([]*files.Cont }, }) - f.Close() - return nil + return f.Close() } if fi.Mode()&os.ModeSymlink != 0 { @@ -516,6 +545,7 @@ func buildContents(vars *types.BuildVars, dirs types.Directories) ([]*files.Cont if err != nil { return err } + // Remove pkgdir from the symlink's path link = strings.TrimPrefix(link, dirs.PkgDir) contents = append(contents, &files.Content{ @@ -541,6 +571,7 @@ func buildContents(vars *types.BuildVars, dirs types.Directories) ([]*files.Cont }, } + // If the file is supposed to be backed up, set its type to config|noreplace if slices.Contains(vars.Backup, trimmed) { fileContent.Type = "config|noreplace" } @@ -552,14 +583,16 @@ func buildContents(vars *types.BuildVars, dirs types.Directories) ([]*files.Cont return contents, err } +// removeBuildDeps asks the user if they'd like to remove the build dependencies that were +// installed by installBuildDeps. If so, it uses the package manager to do that. func removeBuildDeps(buildDeps []string, opts types.BuildOpts) error { if len(buildDeps) > 0 { - removeBuildDeps, err := cliutils.YesNoPrompt("Would you like to remove the build dependencies?", opts.Interactive, false) + remove, err := cliutils.YesNoPrompt("Would you like to remove the build dependencies?", opts.Interactive, false) if err != nil { return err } - if removeBuildDeps { + if remove { err = opts.Manager.Remove( &manager.Opts{ AsRoot: true, @@ -575,6 +608,8 @@ func removeBuildDeps(buildDeps []string, opts types.BuildOpts) error { return nil } +// checkForBuiltPackage tries to detect a previously-built package and returns its path +// and true if it finds one. If it doesn't find it, it returns "", false, nil. func checkForBuiltPackage(mgr manager.Manager, vars *types.BuildVars, pkgFormat, baseDir string) (string, bool, error) { filename, err := pkgFileName(vars, pkgFormat) if err != nil { @@ -591,6 +626,8 @@ func checkForBuiltPackage(mgr manager.Manager, vars *types.BuildVars, pkgFormat, return pkgPath, true, nil } +// pkgFileName returns the filename of the package if it were to be built. +// This is used to check if the package has already been built. func pkgFileName(vars *types.BuildVars, pkgFormat string) (string, error) { pkgInfo := &nfpm.Info{ Name: vars.Name, @@ -608,6 +645,8 @@ func pkgFileName(vars *types.BuildVars, pkgFormat string) (string, error) { return packager.ConventionalFileName(pkgInfo), nil } +// getPkgFormat returns the package format of the package manager, +// or LURE_PKG_FORMAT if that's set. func getPkgFormat(mgr manager.Manager) string { pkgFormat := mgr.Format() if format, ok := os.LookupEnv("LURE_PKG_FORMAT"); ok { @@ -616,6 +655,8 @@ func getPkgFormat(mgr manager.Manager) string { return pkgFormat } +// createBuildEnvVars creates the environment variables that will be set in the +// build script when it's executed. func createBuildEnvVars(info *distro.OSRelease, dirs types.Directories) []string { env := os.Environ() @@ -645,6 +686,7 @@ func createBuildEnvVars(info *distro.OSRelease, dirs types.Directories) []string return env } +// getSources downloads the sources from the script. func getSources(ctx context.Context, srcdir string, bv *types.BuildVars) error { if len(bv.Sources) != len(bv.Checksums) { log.Fatal("The checksums array must be the same length as sources").Send() @@ -659,6 +701,9 @@ func getSources(ctx context.Context, srcdir string, bv *types.BuildVars) error { } if !strings.EqualFold(bv.Checksums[i], "SKIP") { + // If the checksum contains a colon, use the part before the colon + // as the algorithm and the part after as the actual checksum. + // Otherwise, use the default sha256 with the whole string as the checksum. algo, hashData, ok := strings.Cut(bv.Checksums[i], ":") if ok { checksum, err := hex.DecodeString(hashData) @@ -685,6 +730,7 @@ func getSources(ctx context.Context, srcdir string, bv *types.BuildVars) error { return nil } +// setScripts adds any hook scripts to the package metadata. func setScripts(vars *types.BuildVars, info *nfpm.Info, scriptDir string) { if vars.Scripts.PreInstall != "" { info.Scripts.PreInstall = filepath.Join(scriptDir, vars.Scripts.PreInstall) @@ -721,6 +767,8 @@ func setScripts(vars *types.BuildVars, info *nfpm.Info, scriptDir string) { } } +// setVersion changes the version variable in the script runner. +// It's used to set the version to the output of the version() function. func setVersion(ctx context.Context, r *interp.Runner, to string) error { fl, err := syntax.NewParser().Parse(strings.NewReader("version='"+to+"'"), "") if err != nil { @@ -729,34 +777,24 @@ func setVersion(ctx context.Context, r *interp.Runner, to string) error { return r.Run(ctx, fl) } -// filterBuildDeps returns a map without any dependencies that are already installed -func filterBuildDeps(found map[string][]db.Package, installed map[string]string) map[string][]db.Package { - out := map[string][]db.Package{} - for name, pkgs := range found { - var inner []db.Package - for _, pkg := range pkgs { - if _, ok := installed[pkg.Name]; !ok { - addToFiltered := true - for _, provides := range pkg.Provides.Val { - if _, ok := installed[provides]; ok { - addToFiltered = false - break - } - } +// removeAlreadyInstalled returns a map without any dependencies that are already installed +func removeAlreadyInstalled(found map[string][]db.Package, installed map[string]string) map[string][]db.Package { + filteredPackages := make(map[string][]db.Package) - if addToFiltered { - inner = append(inner, pkg) - } + for name, pkgList := range found { + filteredPkgList := []db.Package{} + for _, pkg := range pkgList { + if _, isInstalled := installed[pkg.Name]; !isInstalled { + filteredPkgList = append(filteredPkgList, pkg) } } - - if len(inner) > 0 { - out[name] = inner - } + filteredPackages[name] = filteredPkgList } - return out + + return filteredPackages } +// packageNames returns the names of all the given packages func packageNames(pkgs []db.Package) []string { names := make([]string, len(pkgs)) for i, p := range pkgs { @@ -765,6 +803,7 @@ func packageNames(pkgs []db.Package) []string { return names } +// removeDuplicates removes any duplicates from the given slice func removeDuplicates(slice []string) []string { seen := map[string]struct{}{} result := []string{}