Move database open code into internal/db

cmd/lure-api-server/db.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"os"
-
 	"github.com/jmoiron/sqlx"
 	"go.arsenm.dev/logger/log"
 	"go.arsenm.dev/lure/internal/config"
@@ -12,21 +10,9 @@ import (
 var gdb *sqlx.DB
 
 func init() {
-	fi, err := os.Stat(config.DBPath)
-	if err == nil {
-		// TODO: This should be removed by the first stable release.
-		if fi.IsDir() {
-			log.Fatal("Your package cache database is using the old database engine. Please remove ~/.cache/lure and then run `lure ref`.").Send()
-		}
-	}
-
-	gdb, err = sqlx.Open("sqlite", config.DBPath)
+	var err error
+	gdb, err = db.Open(config.DBPath)
 	if err != nil {
 		log.Fatal("Error opening database").Err(err).Send()
 	}
-
-	err = db.Init(gdb)
-	if err != nil {
-		log.Fatal("Error initializing database").Err(err).Send()
-	}
 }

db.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"os"
-
 	"github.com/jmoiron/sqlx"
 	"go.arsenm.dev/logger/log"
 	"go.arsenm.dev/lure/internal/config"
@@ -12,21 +10,9 @@ import (
 var gdb *sqlx.DB
 
 func init() {
-	fi, err := os.Stat(config.DBPath)
-	if err == nil {
-		// TODO: This should be removed by the first stable release.
-		if fi.IsDir() {
-			log.Fatal("Your package cache database is using the old database engine. Please remove ~/.cache/lure and then run `lure ref`.").Send()
-		}
-	}
-
-	gdb, err = sqlx.Open("sqlite", config.DBPath)
+	var err error
+	gdb, err = db.Open(config.DBPath)
 	if err != nil {
 		log.Fatal("Error opening database").Err(err).Send()
 	}
-
-	err = db.Init(gdb)
-	if err != nil {
-		log.Fatal("Error initializing database").Err(err).Send()
-	}
 }

fix.go

@@ -3,7 +3,6 @@ package main
 import (
 	"os"
 
-	"github.com/jmoiron/sqlx"
 	"github.com/urfave/cli/v2"
 	"go.arsenm.dev/logger/log"
 	"go.arsenm.dev/lure/internal/config"
@@ -28,18 +27,12 @@ func fixCmd(c *cli.Context) error {
 		log.Fatal("Unable to create new cache directory").Err(err).Send()
 	}
 
-	gdb, err = sqlx.Open("sqlite", config.DBPath)
-	if err != nil {
-		log.Fatal("Unable to create new database").Err(err).Send()
-	}
-
 	// Make sure the DB is rebuilt when repos are pulled
-	config.DBPresent = false
-
-	err = db.Init(gdb)
+	gdb, err = db.Open(config.DBPath)
 	if err != nil {
 		log.Fatal("Error initializing database").Err(err).Send()
 	}
+	config.DBPresent = false
 
 	err = repos.Pull(c.Context, gdb, cfg.Repos)
 	if err != nil {

info.go

@@ -62,7 +62,10 @@ func infoCmd(c *cli.Context) error {
 		if err != nil {
 			log.Fatal("Error parsing os-release file").Err(err).Send()
 		}
-		names = overrides.Resolve(info, overrides.DefaultOpts)
+		names, err = overrides.Resolve(info, overrides.DefaultOpts)
+		if err != nil {
+			log.Fatal("Error resolving overrides").Err(err).Send()
+		}
 	}
 
 	for _, pkg := range pkgs {

internal/db/db.go

@@ -6,12 +6,17 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 
 	"github.com/jmoiron/sqlx"
+	"go.arsenm.dev/logger/log"
+	"go.arsenm.dev/lure/internal/config"
 	"golang.org/x/exp/slices"
 	"modernc.org/sqlite"
 )
 
+const CurrentVersion = 1
+
 func init() {
 	sqlite.MustRegisterScalarFunction("json_array_contains", 2, JsonArrayContains)
 }
@@ -35,8 +40,42 @@ type Package struct {
 	Repository    string                    `db:"repository"`
 }
 
+type version struct {
+	Version int `db:"version"`
+}
+
+func Open(dsn string) (*sqlx.DB, error) {
+	if dsn != ":memory:" {
+		fi, err := os.Stat(config.DBPath)
+		if err == nil {
+			// TODO: This should be removed by the first stable release.
+			if fi.IsDir() {
+				log.Warn("Your database is using the old database engine; rebuilding").Send()
+
+				err = os.RemoveAll(config.DBPath)
+				if err != nil {
+					log.Fatal("Error removing old database").Err(err).Send()
+				}
+				config.DBPresent = false
+			}
+		}
+	}
+
+	db, err := sqlx.Open("sqlite", dsn)
+	if err != nil {
+		log.Fatal("Error opening database").Err(err).Send()
+	}
+
+	err = Init(db, dsn)
+	if err != nil {
+		log.Fatal("Error initializing database").Err(err).Send()
+	}
+
+	return db, nil
+}
+
 // Init initializes the database
-func Init(db *sqlx.DB) error {
+func Init(db *sqlx.DB, dsn string) error {
 	*db = *db.Unsafe()
 	_, err := db.Exec(`
 		CREATE TABLE IF NOT EXISTS pkgs (
@@ -57,7 +96,51 @@ func Init(db *sqlx.DB) error {
 			builddepends  TEXT CHECK(builddepends = 'null' OR (JSON_VALID(builddepends) AND JSON_TYPE(builddepends) = 'object')),
 			UNIQUE(name, repository)
 		);
+
+		CREATE TABLE IF NOT EXISTS lure_db_version (
+			version INT NOT NULL
+		);
 	`)
+	if err != nil {
+		return err
+	}
+
+	ver, ok := GetVersion(db)
+	if !ok {
+		return addVersion(db, CurrentVersion)
+	}
+
+	if ver != CurrentVersion {
+		log.Warn("Database version mismatch; rebuilding").Int("version", ver).Int("expected", CurrentVersion).Send()
+
+		db.Close()
+		err = os.Remove(config.DBPath)
+		if err != nil {
+			return err
+		}
+		config.DBPresent = false
+
+		tdb, err := Open(dsn)
+		if err != nil {
+			return err
+		}
+		*db = *tdb
+	}
+
+	return nil
+}
+
+func GetVersion(db *sqlx.DB) (int, bool) {
+	var ver version
+	err := db.Get(&ver, "SELECT * FROM lure_db_version LIMIT 1;")
+	if err != nil {
+		return 0, false
+	}
+	return ver.Version, true
+}
+
+func addVersion(db *sqlx.DB, ver int) error {
+	_, err := db.Exec(`INSERT INTO lure_db_version(version) VALUES (?);`, ver)
 	return err
 }
 

internal/db/db_test.go

@@ -32,18 +32,6 @@ var testPkg = db.Package{
 	Repository: "default",
 }
 
-func getDB(t *testing.T) (*sqlx.DB, error) {
-	t.Helper()
-
-	gdb, err := sqlx.Open("sqlite", ":memory:")
-	if err != nil {
-		return nil, err
-	}
-
-	err = db.Init(gdb)
-	return gdb, err
-}
-
 func TestInit(t *testing.T) {
 	gdb, err := sqlx.Open("sqlite", ":memory:")
 	if err != nil {
@@ -51,7 +39,7 @@ func TestInit(t *testing.T) {
 	}
 	defer gdb.Close()
 
-	err = db.Init(gdb)
+	err = db.Init(gdb, ":memory:")
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}
@@ -60,10 +48,17 @@ func TestInit(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}
+
+	ver, ok := db.GetVersion(gdb)
+	if !ok {
+		t.Errorf("Expected version to be present")
+	} else if ver != db.CurrentVersion {
+		t.Errorf("Expected version %d, got %d", db.CurrentVersion, ver)
+	}
 }
 
 func TestInsertPackage(t *testing.T) {
-	gdb, err := getDB(t)
+	gdb, err := db.Open(":memory:")
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}
@@ -86,7 +81,7 @@ func TestInsertPackage(t *testing.T) {
 }
 
 func TestGetPkgs(t *testing.T) {
-	gdb, err := getDB(t)
+	gdb, err := db.Open(":memory:")
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}
@@ -126,7 +121,7 @@ func TestGetPkgs(t *testing.T) {
 }
 
 func TestGetPkg(t *testing.T) {
-	gdb, err := getDB(t)
+	gdb, err := db.Open(":memory:")
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}
@@ -162,7 +157,7 @@ func TestGetPkg(t *testing.T) {
 }
 
 func TestDeletePkgs(t *testing.T) {
-	gdb, err := getDB(t)
+	gdb, err := db.Open(":memory:")
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}
@@ -200,7 +195,7 @@ func TestDeletePkgs(t *testing.T) {
 }
 
 func TestJsonArrayContains(t *testing.T) {
-	gdb, err := getDB(t)
+	gdb, err := db.Open(":memory:")
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}

internal/overrides/overrides.go

@@ -1,32 +1,42 @@
 package overrides
 
 import (
+	"os"
 	"runtime"
 	"strings"
 
 	"go.arsenm.dev/lure/distro"
 	"go.arsenm.dev/lure/internal/cpu"
+	"golang.org/x/exp/slices"
+	"golang.org/x/text/language"
 )
 
 type Opts struct {
 	Name        string
 	Overrides   bool
 	LikeDistros bool
+	Languages   []string
 }
 
 var DefaultOpts = &Opts{
 	Overrides:   true,
 	LikeDistros: true,
+	Languages:   []string{"en"},
 }
 
 // Resolve generates a slice of possible override names in the order that they should be checked
-func Resolve(info *distro.OSRelease, opts *Opts) []string {
+func Resolve(info *distro.OSRelease, opts *Opts) ([]string, error) {
 	if opts == nil {
 		opts = DefaultOpts
 	}
 
 	if !opts.Overrides {
-		return []string{opts.Name}
+		return []string{opts.Name}, nil
+	}
+
+	langs, err := parseLangs(opts.Languages)
+	if err != nil {
+		return nil, err
 	}
 
 	architectures := []string{runtime.GOARCH}
@@ -71,7 +81,22 @@ func Resolve(info *distro.OSRelease, opts *Opts) []string {
 		out[index] = strings.ReplaceAll(item, "-", "_")
 	}
 
-	return out
+	if len(langs) > 0 {
+		tmp := out
+		out = make([]string, 0, len(tmp)+(len(tmp)*len(langs)))
+		for _, lang := range langs {
+			for _, val := range tmp {
+				if val == "" {
+					continue
+				}
+
+				out = append(out, val+"_"+lang)
+			}
+		}
+		out = append(out, tmp...)
+	}
+
+	return out, nil
 }
 
 func (o *Opts) WithName(name string) *Opts {
@@ -97,3 +122,26 @@ func (o *Opts) WithLikeDistros(v bool) *Opts {
 	out.LikeDistros = v
 	return out
 }
+
+func parseLangs(langs []string) ([]string, error) {
+	out := make([]string, len(langs))
+	for i, lang := range langs {
+		tag, err := language.Parse(lang)
+		if err != nil {
+			return nil, err
+		}
+		base, _ := tag.Base()
+		out[i] = base.String()
+	}
+	slices.Sort(out)
+	out = slices.Compact(out)
+	return out, nil
+}
+
+func SystemLang() string {
+	lang := os.Getenv("LANG")
+	if lang == "" {
+		lang = "en"
+	}
+	return lang
+}

internal/overrides/overrides_test.go

@@ -14,9 +14,19 @@ var info = &distro.OSRelease{
 }
 
 func TestResolve(t *testing.T) {
-	names := overrides.Resolve(info, nil)
+	names, err := overrides.Resolve(info, nil)
+	if err != nil {
+		t.Fatalf("Expected no error, got %s", err)
+	}
 
 	expected := []string{
+		"amd64_centos_en",
+		"centos_en",
+		"amd64_rhel_en",
+		"rhel_en",
+		"amd64_fedora_en",
+		"fedora_en",
+		"amd64_en",
 		"amd64_centos",
 		"centos",
 		"amd64_rhel",
@@ -33,11 +43,14 @@ func TestResolve(t *testing.T) {
 }
 
 func TestResolveName(t *testing.T) {
-	names := overrides.Resolve(info, &overrides.Opts{
+	names, err := overrides.Resolve(info, &overrides.Opts{
 		Name:        "deps",
 		Overrides:   true,
 		LikeDistros: true,
 	})
+	if err != nil {
+		t.Fatalf("Expected no error, got %s", err)
+	}
 
 	expected := []string{
 		"deps_amd64_centos",
@@ -56,10 +69,13 @@ func TestResolveName(t *testing.T) {
 }
 
 func TestResolveNoLikeDistros(t *testing.T) {
-	names := overrides.Resolve(info, &overrides.Opts{
+	names, err := overrides.Resolve(info, &overrides.Opts{
 		Overrides:   true,
 		LikeDistros: false,
 	})
+	if err != nil {
+		t.Fatalf("Expected no error, got %s", err)
+	}
 
 	expected := []string{
 		"amd64_centos",
@@ -74,11 +90,14 @@ func TestResolveNoLikeDistros(t *testing.T) {
 }
 
 func TestResolveNoOverrides(t *testing.T) {
-	names := overrides.Resolve(info, &overrides.Opts{
+	names, err := overrides.Resolve(info, &overrides.Opts{
 		Name:        "deps",
 		Overrides:   false,
 		LikeDistros: false,
 	})
+	if err != nil {
+		t.Fatalf("Expected no error, got %s", err)
+	}
 
 	expected := []string{"deps"}
 
@@ -86,3 +105,30 @@ func TestResolveNoOverrides(t *testing.T) {
 		t.Errorf("expected %v, got %v", expected, names)
 	}
 }
+
+func TestResolveLangs(t *testing.T) {
+	names, err := overrides.Resolve(info, &overrides.Opts{
+		Overrides: true,
+		Languages: []string{"ru_RU", "en", "en_US"},
+	})
+	if err != nil {
+		t.Fatalf("Expected no error, got %s", err)
+	}
+
+	expected := []string{
+		"amd64_centos_en",
+		"centos_en",
+		"amd64_en",
+		"amd64_centos_ru",
+		"centos_ru",
+		"amd64_ru",
+		"amd64_centos",
+		"centos",
+		"amd64",
+		"",
+	}
+
+	if !reflect.DeepEqual(names, expected) {
+		t.Errorf("expected %v, got %v", expected, names)
+	}
+}

internal/repos/find_test.go

@@ -6,24 +6,18 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/jmoiron/sqlx"
 	"go.arsenm.dev/lure/internal/db"
 	"go.arsenm.dev/lure/internal/repos"
 	"go.arsenm.dev/lure/internal/types"
 )
 
 func TestFindPkgs(t *testing.T) {
-	gdb, err := sqlx.Open("sqlite", ":memory:")
+	gdb, err := db.Open(":memory:")
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}
 	defer gdb.Close()
 
-	err = db.Init(gdb)
-	if err != nil {
-		t.Fatalf("Expected no error, got %s", err)
-	}
-
 	setCfgDirs(t)
 	defer removeCacheDir(t)
 
@@ -69,17 +63,12 @@ func TestFindPkgs(t *testing.T) {
 }
 
 func TestFindPkgsEmpty(t *testing.T) {
-	gdb, err := sqlx.Open("sqlite", ":memory:")
+	gdb, err := db.Open(":memory:")
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}
 	defer gdb.Close()
 
-	err = db.Init(gdb)
-	if err != nil {
-		t.Fatalf("Expected no error, got %s", err)
-	}
-
 	setCfgDirs(t)
 	defer removeCacheDir(t)
 

internal/repos/pull_test.go

@@ -6,7 +6,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/jmoiron/sqlx"
 	"go.arsenm.dev/lure/internal/config"
 	"go.arsenm.dev/lure/internal/db"
 	"go.arsenm.dev/lure/internal/repos"
@@ -48,17 +47,12 @@ func removeCacheDir(t *testing.T) {
 }
 
 func TestPull(t *testing.T) {
-	gdb, err := sqlx.Open("sqlite", ":memory:")
+	gdb, err := db.Open(":memory:")
 	if err != nil {
 		t.Fatalf("Expected no error, got %s", err)
 	}
 	defer gdb.Close()
 
-	err = db.Init(gdb)
-	if err != nil {
-		t.Fatalf("Expected no error, got %s", err)
-	}
-
 	setCfgDirs(t)
 	defer removeCacheDir(t)
 

internal/shutils/decoder/decoder.go

@@ -182,7 +182,11 @@ func (d *Decoder) GetFunc(name string) (ScriptFunc, bool) {
 }
 
 func (d *Decoder) getFunc(name string) *syntax.Stmt {
-	names := overrides.Resolve(d.info, overrides.DefaultOpts.WithName(name))
+	names, err := overrides.Resolve(d.info, overrides.DefaultOpts.WithName(name))
+	if err != nil {
+		return nil
+	}
+
 	for _, fnName := range names {
 		fn, ok := d.runner.Funcs[fnName]
 		if ok {
@@ -195,7 +199,11 @@ func (d *Decoder) getFunc(name string) *syntax.Stmt {
 // getVar gets a variable based on its name, taking into account
 // override variables and nameref variables.
 func (d *Decoder) getVar(name string) *expand.Variable {
-	names := overrides.Resolve(d.info, overrides.DefaultOpts.WithName(name))
+	names, err := overrides.Resolve(d.info, overrides.DefaultOpts.WithName(name))
+	if err != nil {
+		return nil
+	}
+
 	for _, varName := range names {
 		val, ok := d.runner.Vars[varName]
 		if ok {