fix(dbbackup): 缺 MinecraftServer 导出时重试、CLI 失败退出、面板与 watchdog 告警，异地快照要求完整

cmd/felis/db.go

@@ -172,6 +172,14 @@ func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
 	path, err := dbbackup.Backup(ctx, o)
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// The bundle is on disk and holds the database; the exit status fails
+		// the timer's run so the gap shows in systemctl and the journal, and
+		// the panel and the watchdog read it from the record and the manifest.
+		fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+		fmt.Fprintf(stderr, "felis db backup: %v\n  a restore from %s brings back the database but no servers; check `k3s kubectl get minecraftservers -A`, then run `felis db backup` again\n", err, filepath.Base(path))
+		return 1
+	}
 	if err != nil {
 		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
 		return 1
@@ -423,10 +431,36 @@ func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wri
 	return 0
 }
 
-// exportMinecraftServers reads every MinecraftServer through the host's k3s
+// serverExportTries and serverExportRetry are how long a backup waits out a
+// cluster that is briefly away (an apiserver restart) before its bundle goes
+// without the MinecraftServer objects.
+const serverExportTries = 3
+
+var serverExportRetry = 10 * time.Second
+
+// exportMinecraftServers is dbbackup's ExportServers on the host: the
+// MinecraftServer objects through k3s kubectl, tried serverExportTries times.
+func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+	for try := 1; ; try++ {
+		out, err := getMinecraftServers(ctx)
+		if err == nil {
+			return out, nil
+		}
+		if try == serverExportTries {
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		}
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		case <-time.After(serverExportRetry):
+		}
+	}
+}
+
+// getMinecraftServers reads every MinecraftServer through the host's k3s
 // kubectl and strips what the API server owns, so the result can be fed back
 // with `kubectl apply -f` on a rebuilt cluster.
-func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+func getMinecraftServers(ctx context.Context) ([]byte, error) {
 	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
 	// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning

cmd/felis/db_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"archive/tar"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -240,8 +241,20 @@ func TestAuditExportBounds(t *testing.T) {
 
 // podK3s stands in for `k3s kubectl exec ... --`: it logs its argv and runs the
 // command after -- from the "container" directory, which is the only place the
-// PostgreSQL tools exist, as on an installed host.
+// PostgreSQL tools exist, as on an installed host. `kubectl get` lists one
+// MinecraftServer, logged to k3s.get, and refuses its first N calls while
+// servers_fail holds N.
 const podK3s = `#!/bin/sh
+if [ "$1" = kubectl ] && [ "$2" = get ]; then
+  printf '%s\n' "$*" >> "$FAKE_DIR/k3s.get"
+  n=$(/usr/bin/wc -l < "$FAKE_DIR/k3s.get")
+  if [ -f "$FAKE_DIR/servers_fail" ] && [ "$n" -le "$(/bin/cat "$FAKE_DIR/servers_fail")" ]; then
+    echo "The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?" >&2
+    exit 1
+  fi
+  echo '{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer","metadata":{"name":"lobby","namespace":"felis-servers","uid":"u-1"},"spec":{"type":"PAPER"},"status":{"phase":"Running"}}]}'
+  exit 0
+fi
 printf '%s\n' "$*" >> "$FAKE_DIR/k3s.args"
 while [ $# -gt 0 ] && [ "$1" != "--" ]; do shift; done
 shift
@@ -410,3 +423,142 @@ func TestDBToolsNeedTheRoleAndDatabase(t *testing.T) {
 }
 
 var dbbackupVerify = dbbackup.Verify
+
+func noServerExportWait(t *testing.T) {
+	t.Helper()
+	old := serverExportRetry
+	serverExportRetry = 0
+	t.Cleanup(func() { serverExportRetry = old })
+}
+
+// serverGets counts the `kubectl get` calls the fake k3s answered or refused.
+func serverGets(dir string) int {
+	b, _ := os.ReadFile(filepath.Join(dir, "k3s.get"))
+	return strings.Count(string(b), "\n")
+}
+
+// bundleServers returns the bundle's k8s/minecraftservers.json, or nil.
+func bundleServers(t *testing.T, bundle string) []byte {
+	t.Helper()
+	f, err := os.Open(bundle)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	tr := tar.NewReader(f)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		if h.Name == "k8s/minecraftservers.json" {
+			data, err := io.ReadAll(tr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return data
+		}
+	}
+}
+
+// TestDBBackupWithoutServersFails: when the cluster stays away the daily
+// bundle is still written, and `felis db backup` exits 1, so the timer's run
+// shows failed, saying a restore from the bundle brings back no servers.
+func TestDBBackupWithoutServersFails(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	var out, errBuf bytes.Buffer
+	code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", "", "-label", "daily"}, &out, &errBuf)
+	if code != 1 {
+		t.Fatalf("exit %d, want 1: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	m, err := dbbackupVerify(bundle)
+	if err != nil {
+		t.Fatalf("the database must still be bundled: %v", err)
+	}
+	if !strings.Contains(m.ServersError, "6443 was refused") || !strings.Contains(m.ServersError, "(tried 3 times)") || bundleServers(t, bundle) != nil {
+		t.Errorf("manifest servers error = %q", m.ServersError)
+	}
+	if n := serverGets(dir); n != serverExportTries {
+		t.Errorf("export tried %d times, want %d", n, serverExportTries)
+	}
+	if msg := errBuf.String(); !strings.Contains(msg, "a restore from "+filepath.Base(bundle)+" brings back the database but no servers") {
+		t.Errorf("stderr = %q", msg)
+	}
+}
+
+// TestDBBackupRetriesTheServerExport: a cluster back on the last try costs the
+// bundle nothing, and what it holds is ready for kubectl apply.
+func TestDBBackupRetriesTheServerExport(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "2", 0o600)
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", ""}, &out, &errBuf); code != 0 {
+		t.Fatalf("exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	servers := string(bundleServers(t, bundle))
+	if !strings.Contains(servers, `"name": "lobby"`) || strings.Contains(servers, "status") || strings.Contains(servers, "u-1") {
+		t.Errorf("k8s/minecraftservers.json = %s", servers)
+	}
+	if n := serverGets(dir); n != 3 {
+		t.Errorf("export tried %d times, want 3", n)
+	}
+}
+
+// TestPreMigrateBackupExportsServers: the snapshot every upgrade takes, often
+// the newest bundle, carries the MinecraftServer objects too; a cluster that
+// is away does not hold back the migration, whose rollback needs the database
+// alone.
+func TestPreMigrateBackupExportsServers(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	old := preMigrateStateDir
+	preMigrateStateDir = ""
+	t.Cleanup(func() { preMigrateStateDir = old })
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	bundles := filepath.Join(dir, "bundles")
+	pending := appliedDriver{done: map[int]struct{}{1: {}}}
+
+	path, err := preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil {
+		t.Fatalf("snapshot: %v", err)
+	}
+	if !strings.Contains(string(bundleServers(t, path)), `"name": "lobby"`) {
+		t.Errorf("%s holds no MinecraftServer objects", path)
+	}
+
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	path, err = preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil || path == "" {
+		t.Fatalf("snapshot with the cluster away = %q, %v; want the bundle and no error", path, err)
+	}
+	if m, err := dbbackupVerify(path); err != nil || m.ServersError == "" || bundleServers(t, path) != nil {
+		t.Errorf("snapshot with the cluster away: %+v, %v", m, err)
+	}
+}
+
+// TestServerExportStopsWaitingWithTheContext: a backup whose time is up stops
+// waiting for the cluster between tries.
+func TestServerExportStopsWaitingWithTheContext(t *testing.T) {
+	dir := newPodRig(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	_, err := exportMinecraftServers(ctx)
+	if err == nil || !strings.Contains(err.Error(), "(tried 1 times)") || serverGets(dir) != 1 {
+		t.Fatalf("err = %v after %d tries, want the first failure alone", err, serverGets(dir))
+	}
+	if took := time.Since(start); took > 5*time.Second {
+		t.Errorf("took %s, want the context's deadline, not the %s retry wait", took, serverExportRetry)
+	}
+}

cmd/felis/migrate.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -83,6 +84,10 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	return 0
 }
 
+// preMigrateStateDir is the host state a pre-migrate bundle carries; tests
+// point it at a directory of their own.
+var preMigrateStateDir = dbbackup.DefaultStateDir
+
 // preMigrateBackup bundles the database when it already carries a schema and
 // some of migrations are not applied yet, and returns the bundle's path ("" when
 // there was nothing to protect: a fresh database, or nothing pending).
@@ -101,11 +106,18 @@ func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.
 	if err != nil {
 		return "", err
 	}
-	return dbbackup.Backup(ctx, dbbackup.BackupOptions{
+	path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
 		DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelPreMigrate,
-		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir,
-		Version: resolvedVersion(), Log: log, Record: true,
+		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: preMigrateStateDir,
+		Version: resolvedVersion(), ExportServers: exportMinecraftServers, Log: log, Record: true,
 	})
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// Rolling the migration back needs the database alone. Backup logged
+		// the gap, and the panel and the watchdog show it while this is the
+		// newest bundle.
+		return path, nil
+	}
+	return path, err
 }
 
 func hasPending(done map[int]struct{}, migrations []store.Migration) bool {

cmd/felis/offsite.go

@@ -382,7 +382,10 @@ func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, arch
 // (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite,
 // with the newest one kept in dir. It is not recorded for the panel, whose
 // backup card watches felis-db-backup.timer: snapshots come only when archives
-// are copied, and would hide a daily timer that stopped.
+// are copied, and would hide a daily timer that stopped. It requires the
+// MinecraftServer objects: it becomes the newest bundle in the bucket, which a
+// lost host restores from, and a pass that cannot take a whole one fails and
+// tries again next hour.
 func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error {
 	return func(ctx context.Context) error {
 		tools, err := dbTools(db)
@@ -394,7 +397,7 @@ func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writ
 		path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
 			DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite,
 			Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(),
-			ExportServers: exportMinecraftServers, Log: log,
+			ExportServers: exportMinecraftServers, RequireServers: true, Log: log,
 		})
 		if err == nil {
 			fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path))
@@ -667,7 +670,11 @@ func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles [
 			fmt.Fprintf(stdout, "  %s  %s  unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err)
 			continue
 		}
-		fmt.Fprintf(stdout, "  %s  %s  %s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String())
+		gap := ""
+		if m.ServersError != "" {
+			gap = ", no MinecraftServer objects"
+		}
+		fmt.Fprintf(stdout, "  %s  %s  %s%s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String(), gap)
 	}
 }
 
@@ -891,6 +898,9 @@ func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string,
 	if m.Counts.Fresh() {
 		fmt.Fprintln(stdout, "  This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.")
 	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  This bundle lacks the MinecraftServer objects (%s): `felis db restore` brings back the database, and the servers come from k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap.\n", m.ServersError)
+	}
 	return 0
 }
 

cmd/felis/offsite_test.go

@@ -168,12 +168,19 @@ var fetchT0 = time.Date(2026, 9, 20, 3, 30, 0, 0, time.UTC)
 // putBundle seals a bundle that verifies, taken daysAgo days before fetchT0,
 // into b and returns its name.
 func putBundle(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts) string {
+	t.Helper()
+	return putBundleWith(t, b, key, daysAgo, counts, "")
+}
+
+// putBundleWith is putBundle for a bundle whose server export failed with
+// serversError, when that is not empty.
+func putBundleWith(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts, serversError string) string {
 	t.Helper()
 	created := fetchT0.AddDate(0, 0, -daysAgo)
 	dump := []byte("PGDMP " + created.String())
 	sum := sha256.Sum256(dump)
 	manifest, err := json.Marshal(dbbackup.Manifest{
-		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts,
+		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, ServersError: serversError,
 		Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}},
 	})
 	if err != nil {
@@ -276,6 +283,20 @@ func TestOffsiteFetchDB(t *testing.T) {
 		}
 	})
 
+	t.Run("a bundle without the servers says where they come from", func(t *testing.T) {
+		b := mapBucket{}
+		gapped := putBundleWith(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused (tried 3 times)")
+		_, code, out, errb := fetch(b, gapped)
+		if code != 0 || !strings.Contains(out, "This bundle lacks the MinecraftServer objects (connection refused (tried 3 times))") ||
+			!strings.Contains(out, "k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap") {
+			t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
+		}
+		whole := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+		if _, _, out, _ := fetch(b, whole); strings.Contains(out, "lacks the MinecraftServer objects") {
+			t.Errorf("a whole bundle flagged:\n%s", out)
+		}
+	})
+
 	t.Run("a bundle from before counts says so", func(t *testing.T) {
 		b := mapBucket{}
 		old := putBundle(t, b, key, 0, nil)
@@ -601,6 +622,7 @@ func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
 	otherRaw, _ := offsite.NewKey()
 	other, _ := offsite.ParseKey(otherRaw)
 	b := mapBucket{}
+	gapped := putBundleWith(t, b, key, 4, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused")
 	old := putBundle(t, b, key, 3, nil)
 	full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3})
 	sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3})
@@ -617,12 +639,13 @@ func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
 		{sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"},
 		{full, "5 accounts, 3 servers"},
 		{old, "not recorded"},
+		{gapped, "5 accounts, 3 servers, no MinecraftServer objects"},
 	}
-	if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (4, newest first") {
+	if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (5, newest first") {
 		t.Fatalf("output:\n%s", out.String())
 	}
 	for i, w := range want {
-		if l := lines[i+1]; !strings.HasPrefix(l, "  "+w.name+"  ") || !strings.Contains(l, w.holds) {
+		if l := lines[i+1]; !strings.HasPrefix(l, "  "+w.name+"  ") || !strings.Contains(l, w.holds) || strings.Contains(l, "no MinecraftServer") != (w.name == gapped) {
 			t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds)
 		}
 	}
@@ -708,12 +731,25 @@ func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) {
 	}
 	// The MinecraftServer objects are exported alongside, as in the daily bundle.
 	argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args"))
-	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") || !strings.Contains(ran, "kubectl get minecraftservers") {
-		t.Errorf("k3s ran %q, want pg_dump in the pod and the server export", ran)
+	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") {
+		t.Errorf("k3s ran %q, want pg_dump in the pod", ran)
+	}
+	if !strings.Contains(string(bundleServers(t, got[0].Path)), `"name": "lobby"`) {
+		t.Errorf("the snapshot holds no MinecraftServer objects")
 	}
 	if !strings.Contains(log.String(), "took database bundle "+got[0].Name) {
 		t.Errorf("the snapshot is not logged:\n%s", log.String())
 	}
+	// It becomes the newest bundle in the bucket, so with the cluster away it
+	// fails, leaves no bundle, and the next pass tries again.
+	noServerExportWait(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	if err := s.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "export the MinecraftServer objects") {
+		t.Errorf("snapshot with the cluster away: %v, want a failure", err)
+	}
+	if again, _ := dbbackup.List(bundles); len(again) != 1 || again[0].Name != got[0].Name {
+		t.Errorf("bundle directory after the failed snapshot = %+v, want %s alone", again, got[0].Name)
+	}
 
 	for _, c := range []struct {
 		archive config.ArchiveConfig