fix(offsite): fetch-db latest 跳过新装机器的空库包，取包后显示包内账号和服务器数 (fa467338) · Commits · FelisMC / Felis · GitLab

cmd/felis/db.go

+5 −4

Changes for cmd/felis/db.go: 5 added lines, 4 removed lines.
Original line number	Diff line number	Diff line
		@@ -211,8 +211,8 @@ func dbRestore(fs flag.FlagSet, dir string, args []string, stdout, stderr io.W
		return 1
		}
		if !*yes {
		fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d).\n",
		filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion)
		fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d, holding %s).\n",
		filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion, m.Counts.String())
		fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.")
		return 2
		}
		@@ -268,8 +268,9 @@ func dbVerify(fs flag.FlagSet, dir string, args []string, stdout, stderr io.Wr
		fmt.Fprintf(stderr, "felis db verify: %v\n", err)
		return 1
		}
		fmt.Fprintf(stdout, "%s: ok\n taken %s (%s)\n felis %s\n schema %d\n %s\n",
		filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion, orUnknown(m.PGDumpVersion))
		fmt.Fprintf(stdout, "%s: ok\n taken %s (%s)\n felis %s\n schema %d\n holds %s\n %s\n",
		filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion,
		m.Counts.String(), orUnknown(m.PGDumpVersion))
		for _, f := range m.Files {
		if f.Link != "" {
		fmt.Fprintf(stdout, " %-40s -> %s\n", f.Name, f.Link)

cmd/felis/db_test.go

+22 −1

Changes for cmd/felis/db_test.go: 22 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -29,6 +29,26 @@ func TestDBUsage(t *testing.T) {
		}
		}

		func TestDBVerifySaysWhatTheBundleHolds(t *testing.T) {
		dir := newPodRig(t)
		cfg := podConfig(t, dir)
		bundles := filepath.Join(dir, "bundles")
		var out, errBuf bytes.Buffer
		if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
		}
		bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
		out.Reset()
		if code := run([]string{"db", "verify", "-dir", bundles, filepath.Base(bundle)}, &out, &errBuf); code != 0 {
		t.Fatalf("verify: exit %d: %s", code, errBuf.String())
		}
		for _, want := range []string{filepath.Base(bundle) + ": ok", "schema 3", "holds 4 accounts, 2 servers", "pg_dump (PostgreSQL) 18.6"} {
		if !strings.Contains(out.String(), want) {
		t.Errorf("verify output lacks %q:\n%s", want, out.String())
		}
		}
		}

		// TestDBRestoreNeedsYes: without -yes a restore describes the bundle and stops
		// before anything reaches the database, even with -force and
		// -no-safety-backup, which would otherwise let the replay run at once.
		@@ -49,7 +69,7 @@ func TestDBRestoreNeedsYes(t *testing.T) {
		if code != 2 {
		t.Fatalf("exit %d, want 2; stderr %q", code, errBuf.String())
		}
		if want := filepath.Base(bundle) + " (manual, taken "; !strings.Contains(errBuf.String(), want) \|\| !strings.Contains(errBuf.String(), "schema 3).") {
		if want := filepath.Base(bundle) + " (manual, taken "; !strings.Contains(errBuf.String(), want) \|\| !strings.Contains(errBuf.String(), "schema 3, holding 4 accounts, 2 servers).") {
		t.Errorf("stderr %q does not describe the bundle", errBuf.String())
		}
		if !strings.Contains(errBuf.String(), "re-run with -yes") {
		@@ -238,6 +258,7 @@ printf 'PGDMP-fake-archive'
		cat > /dev/null
		`,
		"psql": `#!/bin/sh
		for a in "$@"; do case "$a" in "FROM users") echo "4\|2"; exit 0 ;; esac; done
		for a in "$@"; do [ "$a" = "-c" ] && { echo 3; exit 0; }; done
		cat > /dev/null
		`,

cmd/felis/offsite.go

+33 −15

Changes for cmd/felis/offsite.go: 33 added lines, 15 removed lines.
Original line number	Diff line number	Diff line
		@@ -501,10 +501,7 @@ func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
		return 1
		}
		fmt.Fprintf(stdout, "database bundles (%d, newest first):\n", len(bundles))
		for _, b := range bundles {
		fmt.Fprintf(stdout, " %s %s\n", b.Key, offsite.HumanBytes(b.Size))
		}
		printDBBundles(ctx, env.bucket, env.key, bundles, stdout)
		var total int64
		for _, w := range worlds {
		total += w.Size
		@@ -541,6 +538,20 @@ func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
		return 0
		}

		// printDBBundles lists the database bundles with what each one's database
		// held, read off the front of each, so a restore can pick one by its contents.
		func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles []offsite.Object, stdout io.Writer) {
		fmt.Fprintf(stdout, "database bundles (%d, newest first; restore one with fetch-db):\n", len(bundles))
		for _, o := range bundles {
		m, err := offsite.PeekDB(ctx, b, key, o.Key)
		if err != nil {
		fmt.Fprintf(stdout, " %s %s unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err)
		continue
		}
		fmt.Fprintf(stdout, " %s %s %s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String())
		}
		}

		func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
		cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
		envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
		@@ -583,37 +594,44 @@ func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) i
		}
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
		defer cancel()
		return fetchDB(ctx, env.bucket, env.key, arg, *dir, time.Now(), stdout, stderr)
		}

		// fetchDB is fetch-db once the bucket is open: arg is a bundle name or latest.
		func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string, now time.Time, stdout, stderr io.Writer) int {
		name := arg
		if name == "latest" {
		bundles, err := offsite.ListDB(ctx, env.bucket)
		if err != nil {
		var err error
		if name, _, err = offsite.ChooseDB(ctx, b, key); err != nil {
		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
		return 1
		}
		if len(bundles) == 0 {
		fmt.Fprintln(stderr, "felis offsite fetch-db: the bucket holds no database bundle")
		return 1
		}
		name = bundles[0].Key
		}
		if _, _, ok := dbbackup.ParseBundleName(name); !ok {
		fmt.Fprintf(stderr, "felis offsite fetch-db: %q is not a bundle name (felis-db-<stamp>-<label>.tar); see `felis offsite list`\n", name)
		return 2
		}
		if err := os.MkdirAll(*dir, 0o700); err != nil {
		if err := os.MkdirAll(dir, 0o700); err != nil {
		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
		return 1
		}
		dst := filepath.Join(*dir, name)
		if err := offsite.FetchObject(ctx, env.bucket, env.key, offsite.DBKey(name), dst, 0o600); err != nil {
		dst := filepath.Join(dir, name)
		if err := offsite.FetchObject(ctx, b, key, offsite.DBKey(name), dst, 0o600); err != nil {
		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
		return 1
		}
		if _, err := dbbackup.Verify(dst); err != nil {
		m, err := dbbackup.Verify(dst)
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite fetch-db: fetched %s but it does not verify: %v\n", dst, err)
		return 1
		}
		fmt.Fprintf(stdout, "felis offsite fetch-db: wrote %s (verified)\n", dst)
		fmt.Fprintf(stdout, " taken %s (%s, %s ago)\n felis %s, schema %d\n holds %s\n",
		m.CreatedAt.Format(time.RFC3339), m.Label, dbbackup.Age(now.Sub(m.CreatedAt)),
		orUnknown(m.FelisVersion), m.SchemaVersion, m.Counts.String())
		if m.Counts.Fresh() {
		fmt.Fprintln(stdout, " This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.")
		}
		return 0
		}

cmd/felis/offsite_test.go

+197 −0

Changes for cmd/felis/offsite_test.go: 197 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"archive/tar"
		"bytes"
		"context"
		"crypto/sha256"
		"encoding/hex"
		"encoding/json"
		"errors"
		"io"
		"os"
		"path/filepath"
		"slices"
		"strings"
		"testing"
		"time"

		"felis.lolicon.best/internal/config"
		"felis.lolicon.best/internal/dbbackup"
		"felis.lolicon.best/internal/imagepush"
		"felis.lolicon.best/internal/offsite"
		)
		@@ -120,3 +129,191 @@ func TestRegistryGoneMarksNotFound(t *testing.T) {
		t.Fatalf("503 = %v, want it kept an ordinary failure", err)
		}
		}

		// mapBucket is an in-memory offsite.Bucket.
		type mapBucket map[string][]byte

		func (b mapBucket) Put(_ context.Context, key string, r io.Reader, _ int64) error {
		data, err := io.ReadAll(r)
		b[key] = data
		return err
		}

		func (b mapBucket) Get(_ context.Context, key string) (io.ReadCloser, error) {
		data, ok := b[key]
		if !ok {
		return nil, offsite.ErrNotFound
		}
		return io.NopCloser(bytes.NewReader(data)), nil
		}

		func (b mapBucket) List(_ context.Context, prefix string) ([]offsite.Object, error) {
		var out []offsite.Object
		for k, v := range b {
		if strings.HasPrefix(k, prefix) {
		out = append(out, offsite.Object{Key: k, Size: int64(len(v))})
		}
		}
		return out, nil
		}

		func (b mapBucket) Remove(_ context.Context, key string) error {
		delete(b, key)
		return nil
		}

		var fetchT0 = time.Date(2026, 9, 20, 3, 30, 0, 0, time.UTC)

		// putBundle seals a bundle that verifies, taken daysAgo days before fetchT0,
		// into b and returns its name.
		func putBundle(t testing.T, b mapBucket, key []byte, daysAgo int, counts dbbackup.Counts) string {
		t.Helper()
		created := fetchT0.AddDate(0, 0, -daysAgo)
		dump := []byte("PGDMP " + created.String())
		sum := sha256.Sum256(dump)
		manifest, err := json.Marshal(dbbackup.Manifest{
		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts,
		Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}},
		})
		if err != nil {
		t.Fatal(err)
		}
		var plain bytes.Buffer
		tw := tar.NewWriter(&plain)
		for _, f := range []struct {
		name string
		data []byte
		}{{"MANIFEST.json", manifest}, {"db.dump", dump}} {
		if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o600, Size: int64(len(f.data)), Typeflag: tar.TypeReg}); err != nil {
		t.Fatal(err)
		}
		if _, err := tw.Write(f.data); err != nil {
		t.Fatal(err)
		}
		}
		if err := tw.Close(); err != nil {
		t.Fatal(err)
		}
		var sealed bytes.Buffer
		if err := offsite.Encrypt(&sealed, &plain, key); err != nil {
		t.Fatal(err)
		}
		name := dbbackup.BundleName(created, dbbackup.LabelDaily)
		b[offsite.DBKey(name)] = sealed.Bytes()
		return name
		}

		func TestOffsiteFetchDB(t *testing.T) {
		rawKey, _ := offsite.NewKey()
		key, _ := offsite.ParseKey(rawKey)
		now := fetchT0.Add(2 * time.Hour)
		fetch := func(b mapBucket, arg string) (dir string, code int, stdout, stderr string) {
		dir = t.TempDir()
		var out, errb bytes.Buffer
		code = fetchDB(context.Background(), b, key, arg, dir, now, &out, &errb)
		return dir, code, out.String(), errb.String()
		}
		fetched := func(t *testing.T, dir string) []string {
		t.Helper()
		var names []string
		entries, _ := os.ReadDir(dir)
		for _, e := range entries {
		names = append(names, e.Name())
		}
		return names
		}

		t.Run("latest skips a rebuilt host's empty bundle", func(t *testing.T) {
		b := mapBucket{}
		full := putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
		empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
		dir, code, out, errb := fetch(b, "latest")
		if code != 1 \|\| !strings.Contains(errb, empty) \|\| !strings.Contains(errb, full+" (5 accounts, 3 servers)") {
		t.Fatalf("exit %d, stdout %q, stderr %q; want a refusal naming %s", code, out, errb, full)
		}
		if got := fetched(t, dir); len(got) != 0 {
		t.Errorf("a refused fetch wrote %v", got)
		}
		})

		t.Run("latest takes the newest bundle and says what it holds", func(t *testing.T) {
		b := mapBucket{}
		putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 2})
		newest := putBundle(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3})
		dir, code, out, errb := fetch(b, "latest")
		if code != 0 {
		t.Fatalf("exit %d: %s", code, errb)
		}
		if got := fetched(t, dir); !slices.Equal(got, []string{newest}) {
		t.Errorf("wrote %v, want %s", got, newest)
		}
		for _, want := range []string{
		"wrote " + filepath.Join(dir, newest) + " (verified)",
		"taken 2026-09-19T03:30:00Z (daily, 26h0m ago)",
		"felis v1.2.3, schema 21",
		"holds 5 accounts, 3 servers",
		} {
		if !strings.Contains(out, want) {
		t.Errorf("stdout lacks %q:\n%s", want, out)
		}
		}
		if strings.Contains(out, "new install") {
		t.Errorf("a bundle with servers flagged as a new install's:\n%s", out)
		}
		})

		t.Run("an empty bundle named outright is fetched with a warning", func(t *testing.T) {
		b := mapBucket{}
		putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
		empty := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 1})
		dir, code, out, errb := fetch(b, empty)
		if code != 0 \|\| !slices.Equal(fetched(t, dir), []string{empty}) {
		t.Fatalf("exit %d, wrote %v: %s", code, fetched(t, dir), errb)
		}
		if !strings.Contains(out, "holds 1 account, 0 servers") \|\| !strings.Contains(out, "like a new install's") {
		t.Errorf("stdout = %s", out)
		}
		})

		t.Run("a bundle from before counts says so", func(t *testing.T) {
		b := mapBucket{}
		old := putBundle(t, b, key, 0, nil)
		_, code, out, errb := fetch(b, "latest")
		if code != 0 \|\| !strings.Contains(out, old) \|\| !strings.Contains(out, "holds not recorded") \|\| strings.Contains(out, "new install") {
		t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
		}
		})
		}

		func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
		rawKey, _ := offsite.NewKey()
		key, _ := offsite.ParseKey(rawKey)
		otherRaw, _ := offsite.NewKey()
		other, _ := offsite.ParseKey(otherRaw)
		b := mapBucket{}
		old := putBundle(t, b, key, 3, nil)
		full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3})
		sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3})
		empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
		bundles, err := offsite.ListDB(context.Background(), b)
		if err != nil {
		t.Fatal(err)
		}
		var out bytes.Buffer
		printDBBundles(context.Background(), b, key, bundles, &out)
		lines := strings.Split(strings.TrimSpace(out.String()), "\n")
		want := []struct{ name, holds string }{
		{empty, "0 accounts, 0 servers"},
		{sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"},
		{full, "5 accounts, 3 servers"},
		{old, "not recorded"},
		}
		if len(lines) != len(want)+1 \|\| !strings.HasPrefix(lines[0], "database bundles (4, newest first") {
		t.Fatalf("output:\n%s", out.String())
		}
		for i, w := range want {
		if l := lines[i+1]; !strings.HasPrefix(l, " "+w.name+" ") \|\| !strings.Contains(l, w.holds) {
		t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds)
		}
		}
		}

docs/troubleshooting.md

+13 −1

Changes for docs/troubleshooting.md: 13 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -2145,9 +2145,21 @@ host yourself, plus the off-site encryption key if the copy is in the bucket.
		```

		It writes the bundle into `/var/lib/felis/db-backups` (`-dir` to change),
		checks it (`felis db verify`) and names it. A wrong key fails with
		checks it (`felis db verify`) and names it, with when it was taken, the
		release and schema that took it, and how many accounts and servers its
		database holds. Read those before going on. A wrong key fails with
		`object does not decrypt with this key` and writes nothing. For a copy you
		made yourself, check it with `sha256sum -c felis-db-....tar.sha256`.

		`latest` is the newest bundle, unless that one holds no servers and at
		most one account while an older one holds more. That is the database a
		rebuilt host backs up and copies off-site within its first hour, before
		anyone restores onto it, so `fetch-db latest` refuses it and names up to
		three older bundles with their counts; fetch the one you want by name in
		place of `latest` (`felis offsite list` shows them all, each with its
		counts). A bundle fetched by name that looks like a new install's is
		written with a warning. Bundles from releases before the counts were
		recorded show `not recorded`.
		2. Put the old host's state in place before installing, so the installer
		reuses the same DB password, session secret, forwarding secret, the mail
		relay password and uploads bucket keys `felis setup` took, and the