fix(dbbackup): 缺 MinecraftServer 导出时重试、CLI 失败退出、面板与 watchdog 告警，异地快照要求完整

cmd/felis/db.go

@@ -172,6 +172,14 @@ func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
 	path, err := dbbackup.Backup(ctx, o)
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// The bundle is on disk and holds the database; the exit status fails
+		// the timer's run so the gap shows in systemctl and the journal, and
+		// the panel and the watchdog read it from the record and the manifest.
+		fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+		fmt.Fprintf(stderr, "felis db backup: %v\n  a restore from %s brings back the database but no servers; check `k3s kubectl get minecraftservers -A`, then run `felis db backup` again\n", err, filepath.Base(path))
+		return 1
+	}
 	if err != nil {
 		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
 		return 1
@@ -423,10 +431,36 @@ func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wri
 	return 0
 }
 
-// exportMinecraftServers reads every MinecraftServer through the host's k3s
+// serverExportTries and serverExportRetry are how long a backup waits out a
+// cluster that is briefly away (an apiserver restart) before its bundle goes
+// without the MinecraftServer objects.
+const serverExportTries = 3
+
+var serverExportRetry = 10 * time.Second
+
+// exportMinecraftServers is dbbackup's ExportServers on the host: the
+// MinecraftServer objects through k3s kubectl, tried serverExportTries times.
+func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+	for try := 1; ; try++ {
+		out, err := getMinecraftServers(ctx)
+		if err == nil {
+			return out, nil
+		}
+		if try == serverExportTries {
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		}
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		case <-time.After(serverExportRetry):
+		}
+	}
+}
+
+// getMinecraftServers reads every MinecraftServer through the host's k3s
 // kubectl and strips what the API server owns, so the result can be fed back
 // with `kubectl apply -f` on a rebuilt cluster.
-func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+func getMinecraftServers(ctx context.Context) ([]byte, error) {
 	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
 	// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning

cmd/felis/db_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"archive/tar"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -240,8 +241,20 @@ func TestAuditExportBounds(t *testing.T) {
 
 // podK3s stands in for `k3s kubectl exec ... --`: it logs its argv and runs the
 // command after -- from the "container" directory, which is the only place the
-// PostgreSQL tools exist, as on an installed host.
+// PostgreSQL tools exist, as on an installed host. `kubectl get` lists one
+// MinecraftServer, logged to k3s.get, and refuses its first N calls while
+// servers_fail holds N.
 const podK3s = `#!/bin/sh
+if [ "$1" = kubectl ] && [ "$2" = get ]; then
+  printf '%s\n' "$*" >> "$FAKE_DIR/k3s.get"
+  n=$(/usr/bin/wc -l < "$FAKE_DIR/k3s.get")
+  if [ -f "$FAKE_DIR/servers_fail" ] && [ "$n" -le "$(/bin/cat "$FAKE_DIR/servers_fail")" ]; then
+    echo "The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?" >&2
+    exit 1
+  fi
+  echo '{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer","metadata":{"name":"lobby","namespace":"felis-servers","uid":"u-1"},"spec":{"type":"PAPER"},"status":{"phase":"Running"}}]}'
+  exit 0
+fi
 printf '%s\n' "$*" >> "$FAKE_DIR/k3s.args"
 while [ $# -gt 0 ] && [ "$1" != "--" ]; do shift; done
 shift
@@ -410,3 +423,142 @@ func TestDBToolsNeedTheRoleAndDatabase(t *testing.T) {
 }
 
 var dbbackupVerify = dbbackup.Verify
+
+func noServerExportWait(t *testing.T) {
+	t.Helper()
+	old := serverExportRetry
+	serverExportRetry = 0
+	t.Cleanup(func() { serverExportRetry = old })
+}
+
+// serverGets counts the `kubectl get` calls the fake k3s answered or refused.
+func serverGets(dir string) int {
+	b, _ := os.ReadFile(filepath.Join(dir, "k3s.get"))
+	return strings.Count(string(b), "\n")
+}
+
+// bundleServers returns the bundle's k8s/minecraftservers.json, or nil.
+func bundleServers(t *testing.T, bundle string) []byte {
+	t.Helper()
+	f, err := os.Open(bundle)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	tr := tar.NewReader(f)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		if h.Name == "k8s/minecraftservers.json" {
+			data, err := io.ReadAll(tr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return data
+		}
+	}
+}
+
+// TestDBBackupWithoutServersFails: when the cluster stays away the daily
+// bundle is still written, and `felis db backup` exits 1, so the timer's run
+// shows failed, saying a restore from the bundle brings back no servers.
+func TestDBBackupWithoutServersFails(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	var out, errBuf bytes.Buffer
+	code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", "", "-label", "daily"}, &out, &errBuf)
+	if code != 1 {
+		t.Fatalf("exit %d, want 1: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	m, err := dbbackupVerify(bundle)
+	if err != nil {
+		t.Fatalf("the database must still be bundled: %v", err)
+	}
+	if !strings.Contains(m.ServersError, "6443 was refused") || !strings.Contains(m.ServersError, "(tried 3 times)") || bundleServers(t, bundle) != nil {
+		t.Errorf("manifest servers error = %q", m.ServersError)
+	}
+	if n := serverGets(dir); n != serverExportTries {
+		t.Errorf("export tried %d times, want %d", n, serverExportTries)
+	}
+	if msg := errBuf.String(); !strings.Contains(msg, "a restore from "+filepath.Base(bundle)+" brings back the database but no servers") {
+		t.Errorf("stderr = %q", msg)
+	}
+}
+
+// TestDBBackupRetriesTheServerExport: a cluster back on the last try costs the
+// bundle nothing, and what it holds is ready for kubectl apply.
+func TestDBBackupRetriesTheServerExport(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "2", 0o600)
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", ""}, &out, &errBuf); code != 0 {
+		t.Fatalf("exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	servers := string(bundleServers(t, bundle))
+	if !strings.Contains(servers, `"name": "lobby"`) || strings.Contains(servers, "status") || strings.Contains(servers, "u-1") {
+		t.Errorf("k8s/minecraftservers.json = %s", servers)
+	}
+	if n := serverGets(dir); n != 3 {
+		t.Errorf("export tried %d times, want 3", n)
+	}
+}
+
+// TestPreMigrateBackupExportsServers: the snapshot every upgrade takes, often
+// the newest bundle, carries the MinecraftServer objects too; a cluster that
+// is away does not hold back the migration, whose rollback needs the database
+// alone.
+func TestPreMigrateBackupExportsServers(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	old := preMigrateStateDir
+	preMigrateStateDir = ""
+	t.Cleanup(func() { preMigrateStateDir = old })
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	bundles := filepath.Join(dir, "bundles")
+	pending := appliedDriver{done: map[int]struct{}{1: {}}}
+
+	path, err := preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil {
+		t.Fatalf("snapshot: %v", err)
+	}
+	if !strings.Contains(string(bundleServers(t, path)), `"name": "lobby"`) {
+		t.Errorf("%s holds no MinecraftServer objects", path)
+	}
+
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	path, err = preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil || path == "" {
+		t.Fatalf("snapshot with the cluster away = %q, %v; want the bundle and no error", path, err)
+	}
+	if m, err := dbbackupVerify(path); err != nil || m.ServersError == "" || bundleServers(t, path) != nil {
+		t.Errorf("snapshot with the cluster away: %+v, %v", m, err)
+	}
+}
+
+// TestServerExportStopsWaitingWithTheContext: a backup whose time is up stops
+// waiting for the cluster between tries.
+func TestServerExportStopsWaitingWithTheContext(t *testing.T) {
+	dir := newPodRig(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	_, err := exportMinecraftServers(ctx)
+	if err == nil || !strings.Contains(err.Error(), "(tried 1 times)") || serverGets(dir) != 1 {
+		t.Fatalf("err = %v after %d tries, want the first failure alone", err, serverGets(dir))
+	}
+	if took := time.Since(start); took > 5*time.Second {
+		t.Errorf("took %s, want the context's deadline, not the %s retry wait", took, serverExportRetry)
+	}
+}

cmd/felis/migrate.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -83,6 +84,10 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	return 0
 }
 
+// preMigrateStateDir is the host state a pre-migrate bundle carries; tests
+// point it at a directory of their own.
+var preMigrateStateDir = dbbackup.DefaultStateDir
+
 // preMigrateBackup bundles the database when it already carries a schema and
 // some of migrations are not applied yet, and returns the bundle's path ("" when
 // there was nothing to protect: a fresh database, or nothing pending).
@@ -101,11 +106,18 @@ func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.
 	if err != nil {
 		return "", err
 	}
-	return dbbackup.Backup(ctx, dbbackup.BackupOptions{
+	path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
 		DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelPreMigrate,
-		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir,
-		Version: resolvedVersion(), Log: log, Record: true,
+		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: preMigrateStateDir,
+		Version: resolvedVersion(), ExportServers: exportMinecraftServers, Log: log, Record: true,
 	})
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// Rolling the migration back needs the database alone. Backup logged
+		// the gap, and the panel and the watchdog show it while this is the
+		// newest bundle.
+		return path, nil
+	}
+	return path, err
 }
 
 func hasPending(done map[int]struct{}, migrations []store.Migration) bool {

cmd/felis/offsite.go

@@ -382,7 +382,10 @@ func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, arch
 // (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite,
 // with the newest one kept in dir. It is not recorded for the panel, whose
 // backup card watches felis-db-backup.timer: snapshots come only when archives
-// are copied, and would hide a daily timer that stopped.
+// are copied, and would hide a daily timer that stopped. It requires the
+// MinecraftServer objects: it becomes the newest bundle in the bucket, which a
+// lost host restores from, and a pass that cannot take a whole one fails and
+// tries again next hour.
 func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error {
 	return func(ctx context.Context) error {
 		tools, err := dbTools(db)
@@ -394,7 +397,7 @@ func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writ
 		path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
 			DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite,
 			Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(),
-			ExportServers: exportMinecraftServers, Log: log,
+			ExportServers: exportMinecraftServers, RequireServers: true, Log: log,
 		})
 		if err == nil {
 			fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path))
@@ -667,7 +670,11 @@ func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles [
 			fmt.Fprintf(stdout, "  %s  %s  unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err)
 			continue
 		}
-		fmt.Fprintf(stdout, "  %s  %s  %s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String())
+		gap := ""
+		if m.ServersError != "" {
+			gap = ", no MinecraftServer objects"
+		}
+		fmt.Fprintf(stdout, "  %s  %s  %s%s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String(), gap)
 	}
 }
 
@@ -891,6 +898,9 @@ func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string,
 	if m.Counts.Fresh() {
 		fmt.Fprintln(stdout, "  This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.")
 	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  This bundle lacks the MinecraftServer objects (%s): `felis db restore` brings back the database, and the servers come from k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap.\n", m.ServersError)
+	}
 	return 0
 }
 

cmd/felis/offsite_test.go

@@ -168,12 +168,19 @@ var fetchT0 = time.Date(2026, 9, 20, 3, 30, 0, 0, time.UTC)
 // putBundle seals a bundle that verifies, taken daysAgo days before fetchT0,
 // into b and returns its name.
 func putBundle(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts) string {
+	t.Helper()
+	return putBundleWith(t, b, key, daysAgo, counts, "")
+}
+
+// putBundleWith is putBundle for a bundle whose server export failed with
+// serversError, when that is not empty.
+func putBundleWith(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts, serversError string) string {
 	t.Helper()
 	created := fetchT0.AddDate(0, 0, -daysAgo)
 	dump := []byte("PGDMP " + created.String())
 	sum := sha256.Sum256(dump)
 	manifest, err := json.Marshal(dbbackup.Manifest{
-		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts,
+		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, ServersError: serversError,
 		Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}},
 	})
 	if err != nil {
@@ -276,6 +283,20 @@ func TestOffsiteFetchDB(t *testing.T) {
 		}
 	})
 
+	t.Run("a bundle without the servers says where they come from", func(t *testing.T) {
+		b := mapBucket{}
+		gapped := putBundleWith(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused (tried 3 times)")
+		_, code, out, errb := fetch(b, gapped)
+		if code != 0 || !strings.Contains(out, "This bundle lacks the MinecraftServer objects (connection refused (tried 3 times))") ||
+			!strings.Contains(out, "k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap") {
+			t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
+		}
+		whole := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+		if _, _, out, _ := fetch(b, whole); strings.Contains(out, "lacks the MinecraftServer objects") {
+			t.Errorf("a whole bundle flagged:\n%s", out)
+		}
+	})
+
 	t.Run("a bundle from before counts says so", func(t *testing.T) {
 		b := mapBucket{}
 		old := putBundle(t, b, key, 0, nil)
@@ -601,6 +622,7 @@ func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
 	otherRaw, _ := offsite.NewKey()
 	other, _ := offsite.ParseKey(otherRaw)
 	b := mapBucket{}
+	gapped := putBundleWith(t, b, key, 4, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused")
 	old := putBundle(t, b, key, 3, nil)
 	full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3})
 	sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3})
@@ -617,12 +639,13 @@ func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
 		{sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"},
 		{full, "5 accounts, 3 servers"},
 		{old, "not recorded"},
+		{gapped, "5 accounts, 3 servers, no MinecraftServer objects"},
 	}
-	if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (4, newest first") {
+	if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (5, newest first") {
 		t.Fatalf("output:\n%s", out.String())
 	}
 	for i, w := range want {
-		if l := lines[i+1]; !strings.HasPrefix(l, "  "+w.name+"  ") || !strings.Contains(l, w.holds) {
+		if l := lines[i+1]; !strings.HasPrefix(l, "  "+w.name+"  ") || !strings.Contains(l, w.holds) || strings.Contains(l, "no MinecraftServer") != (w.name == gapped) {
 			t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds)
 		}
 	}
@@ -708,12 +731,25 @@ func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) {
 	}
 	// The MinecraftServer objects are exported alongside, as in the daily bundle.
 	argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args"))
-	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") || !strings.Contains(ran, "kubectl get minecraftservers") {
-		t.Errorf("k3s ran %q, want pg_dump in the pod and the server export", ran)
+	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") {
+		t.Errorf("k3s ran %q, want pg_dump in the pod", ran)
+	}
+	if !strings.Contains(string(bundleServers(t, got[0].Path)), `"name": "lobby"`) {
+		t.Errorf("the snapshot holds no MinecraftServer objects")
 	}
 	if !strings.Contains(log.String(), "took database bundle "+got[0].Name) {
 		t.Errorf("the snapshot is not logged:\n%s", log.String())
 	}
+	// It becomes the newest bundle in the bucket, so with the cluster away it
+	// fails, leaves no bundle, and the next pass tries again.
+	noServerExportWait(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	if err := s.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "export the MinecraftServer objects") {
+		t.Errorf("snapshot with the cluster away: %v, want a failure", err)
+	}
+	if again, _ := dbbackup.List(bundles); len(again) != 1 || again[0].Name != got[0].Name {
+		t.Errorf("bundle directory after the failed snapshot = %+v, want %s alone", again, got[0].Name)
+	}
 
 	for _, c := range []struct {
 		archive config.ArchiveConfig

deploy/bootstrap.sh

@@ -4074,7 +4074,9 @@ migrate_host_postgres() {
   remember_temp "$fresh"
   write_felis_toml "$legacy" "127.0.0.1:5432"
   write_felis_toml "$fresh" "127.0.0.1:${PG_HOST_PORT}" "${CONTROL_NS}/${PG_DEPLOYMENT}"
-  out="$("$HOST_BIN" db backup -config "$legacy" -dir "$FELIS_DB_BACKUP_DIR" -label pre-pg-move -keep 0)" \
+  # The move carries the database alone; the MinecraftServer objects stay in the
+  # cluster, so a cluster slow to answer their export must not stop it.
+  out="$("$HOST_BIN" db backup -config "$legacy" -dir "$FELIS_DB_BACKUP_DIR" -label pre-pg-move -keep 0 -no-servers)" \
     || die "could not take a bundle of the host database; nothing was moved"
   bundle="$(printf '%s\n' "$out" | sed -n 's/^felis db backup: wrote //p' | tail -n 1)"
   [ -n "$bundle" ] && [ -f "$bundle" ] || die "felis db backup reported no bundle; nothing was moved"

deploy/bootstrap_test.sh

@@ -3534,7 +3534,7 @@ run_move() { # holds(0/1) backup(ok|silent|fail) restore-exit compare-exit
 out="$(run_move 1 ok)"
 calls="$(cat "$mgdir/calls")"
 expect "the bundle is taken from the host server, never pruned" \
-  "FELIS db backup [127.0.0.1:5432 no-deployment] -dir $mgdir/db -label pre-pg-move -keep 0" "$calls"
+  "FELIS db backup [127.0.0.1:5432 no-deployment] -dir $mgdir/db -label pre-pg-move -keep 0 -no-servers" "$calls"
 expect "the bundle is restored into the pod, which served nobody yet" \
   "FELIS db restore [127.0.0.1:15432 felis/felis-postgres] -dir $mgdir/db -yes -no-safety-backup $mgdir/db/b.tar" "$calls"
 before "writers are stopped before the bundle" "QUIESCE" "FELIS db backup" "$calls"

docs/openapi.yaml

@@ -363,7 +363,8 @@ components:
               description: Bundle file name, felis-db-<UTC stamp>-<label>.tar.
             label:
               type: string
-              enum: [daily, pre-migrate, pre-restore, manual]
+              enum: [daily, pre-migrate, pre-restore, offsite, manual]
+              description: offsite is the bundle `felis offsite sync` takes after copying world archives; a restore records the newest bundle on disk, which may be one.
             size_bytes:
               type: integer
               format: int64
@@ -375,6 +376,12 @@ components:
             dir:
               type: string
               description: Backup directory on the host.
+            servers_error:
+              type: string
+              description: >
+                Why the bundle lacks the MinecraftServer objects (the cluster did not
+                answer the export), when it does. A restore from it brings back the
+                database but no servers.
         stale:
           type: boolean
           description: True when there is no record or it is older than max_age_seconds.

docs/troubleshooting.md

@@ -2020,7 +2020,7 @@ along). One bundle is `felis-db-<UTC stamp>-<label>.tar`:
 | `MANIFEST.json` | version, schema version, `pg_dump --version`, sha256 of every member |
 | `db.dump` | `pg_dump --format=custom` of the `felis` database |
 | `state/etc/felis/...` | every file in `/etc/felis`: `secrets.env` (DB password, session/forwarding secrets, registry tokens), `felis.host.toml`, `felis.pod.toml`, the `felis.toml` symlink, `offsite.env` (bucket credentials and encryption key), `smtp-password`, `uploads-s3-access-key` and `uploads-s3-secret-key` (the mail relay password and the uploads bucket keys `felis setup` took), the panel TLS pair, and the installer's own markers (`system-server-images`, `velocity.fingerprint`). `bootstrap.done` is left out on purpose |
-| `k8s/minecraftservers.json` | every MinecraftServer, status and server-side metadata stripped, ready for `kubectl apply` (best effort: when the cluster did not answer, the manifest records why) |
+| `k8s/minecraftservers.json` | every MinecraftServer, status and server-side metadata stripped, ready for `kubectl apply`. The export is tried 3 times, 10 s apart; when the cluster still does not answer, the bundle is written without it and the manifest records why (next section) |
 
 next to a `.sha256` sidecar in `sha256sum` format. **A bundle contains the
 secrets; treat it like `/etc/felis` itself.** Retention per label: `daily` 14
@@ -2056,6 +2056,22 @@ sudo journalctl -u felis-db-backup -n 50 --no-pager   # why the last run failed
 sudo felis db backup                                  # take one now (label manual)
 ```
 
+**A bundle without the MinecraftServer objects.** When the cluster does not
+answer the export (`k3s kubectl get minecraftservers`) three times running,
+the bundle is still written, since it holds the database, but a restore from it
+brings back no servers. `felis db backup` then exits 1 (the timer's run shows
+failed) after `wrote ...` and the reason; the panel card turns amber
+(**不完整**) with the reason and the commands; the watchdog mails the owners
+(`the newest control-plane database backup ... lacks the MinecraftServer
+objects`) while that bundle is the newest; `felis db verify` and
+`felis offsite list`/`fetch-db` name the gap. A pre-migrate or pre-restore
+bundle goes the same way without stopping the upgrade or the restore, whose
+rollback needs the database alone. The bundle the off-site copy takes after
+copying archives refuses to go without them and is tried again next pass. Once
+`sudo k3s kubectl get minecraftservers -A` answers, run `sudo felis db backup`.
+[GO-TESTED: `internal/dbbackup`, `cmd/felis`, `internal/watchdog`; the panel card
+in `DBBackupCard.test.tsx`]
+
 Common failures: felis-postgres not running (`kubectl exec` reports no running
 pod, or `pg_dump: ... connection refused`; next section); `k3s: executable file not
 found` from a `felis` that runs with neither `/usr/local/bin` on PATH nor k3s
@@ -2303,6 +2319,12 @@ host yourself, plus the off-site encryption key if the copy is in the bucket.
    tar -xOf felis-db-....tar k8s/minecraftservers.json | kubectl apply -f -
    ```
 
+   When `tar` answers `Not found in archive`, the bundle lacks the
+   MinecraftServer objects (`fetch-db` said so when it fetched it). Fetch the
+   newest bundle `felis offsite list` shows without `no MinecraftServer
+   objects` and apply its `k8s/minecraftservers.json` instead; servers that
+   bundle does not list do not come back from it.
+
 7. Bring the world archives back into the archive volume:
 
    ```

internal/dbbackup/dbbackup.go

@@ -126,11 +126,17 @@ type BackupOptions struct {
 	StateDir string // host state to bundle; "" bundles none
 	Version  string // felis build stamp, recorded in the manifest
 	Tools    Tools
-	// ExportServers returns the cluster's MinecraftServer objects as JSON. A
-	// failure is recorded in the manifest and does not fail the backup: the
-	// database is what must not be lost, and a nightly run cannot hang on a
-	// cluster that happens to be down.
+	// ExportServers returns the cluster's MinecraftServer objects as JSON.
+	// When it fails the bundle is still written, with the reason in its
+	// manifest and in the Record, and Backup returns its path with
+	// ErrServersMissing: the database is what must not be lost, and a nightly
+	// run cannot hang on a cluster that happens to be down. A restore from
+	// such a bundle brings back no servers, so the caller has to make that
+	// heard.
 	ExportServers func(ctx context.Context) ([]byte, error)
+	// RequireServers makes an ExportServers failure fail the backup before a
+	// bundle is written, for a caller that can simply try again later.
+	RequireServers bool
 	// MetricsFile, when set, is rewritten after a successful backup with
 	// node-exporter textfile metrics (felis_db_backup_last_success_timestamp_seconds
 	// and felis_db_backup_last_size_bytes), which FelisDBBackupStale alerts on.
@@ -146,6 +152,11 @@ type BackupOptions struct {
 // StatusKey is the platform_settings key Record writes; internal/api reads it.
 const StatusKey = "db_backup_last"
 
+// ErrServersMissing comes back from Backup, together with the path of the
+// bundle it wrote, when the bundle holds the database but ExportServers
+// failed every try: a restore from it brings back no servers.
+var ErrServersMissing = errors.New("the bundle holds the database but not the MinecraftServer objects")
+
 // StaleAfter is how old the newest backup may get before it counts as missed:
 // a day plus the timer's randomized delay and a slow dump. `felis db check`,
 // the admin panel and the FelisDBBackupStale alert (deploy/alerts) share it.
@@ -160,6 +171,9 @@ type Status struct {
 	FelisVersion  string    `json:"felis_version,omitempty"`
 	SchemaVersion int       `json:"schema_version,omitempty"`
 	Dir           string    `json:"dir"`
+	// ServersError is why the bundle lacks the MinecraftServer objects, when
+	// it does.
+	ServersError string `json:"servers_error,omitempty"`
 }
 
 // Manifest describes a bundle.
@@ -444,7 +458,9 @@ func removeStalePartials(dir string) {
 	}
 }
 
-// Backup writes one bundle and returns its path.
+// Backup writes one bundle and returns its path. With ErrServersMissing the
+// bundle is written and holds the database, but not the MinecraftServer
+// objects.
 func Backup(ctx context.Context, o BackupOptions) (string, error) {
 	if !labelRe.MatchString(o.Label) {
 		return "", fmt.Errorf("invalid label %q (want [a-z0-9-], e.g. daily or manual)", o.Label)
@@ -527,6 +543,9 @@ func Backup(ctx context.Context, o BackupOptions) (string, error) {
 	}
 	if o.ExportServers != nil {
 		if data, err := o.ExportServers(ctx); err != nil {
+			if o.RequireServers {
+				return "", fmt.Errorf("export the MinecraftServer objects: %w", err)
+			}
 			m.ServersError = err.Error()
 			fmt.Fprintf(logw, "felis db backup: MinecraftServer objects not included: %v\n", err)
 		} else {
@@ -546,7 +565,7 @@ func Backup(ctx context.Context, o BackupOptions) (string, error) {
 	}
 	if info, err := os.Stat(final); err == nil {
 		st := Status{At: created, Name: name, Label: o.Label, SizeBytes: info.Size(),
-			FelisVersion: o.Version, SchemaVersion: m.SchemaVersion, Dir: o.Dir}
+			FelisVersion: o.Version, SchemaVersion: m.SchemaVersion, Dir: o.Dir, ServersError: m.ServersError}
 		if o.Record {
 			if err := record(ctx, c, o.Tools, st); err != nil {
 				fmt.Fprintf(logw, "felis db backup: record the backup for the panel: %v\n", err)
@@ -563,6 +582,9 @@ func Backup(ctx context.Context, o BackupOptions) (string, error) {
 	} else if len(removed) > 0 {
 		fmt.Fprintf(logw, "felis db backup: pruned %d old %s bundle(s)\n", len(removed), o.Label)
 	}
+	if m.ServersError != "" {
+		return final, fmt.Errorf("%w: %s", ErrServersMissing, m.ServersError)
+	}
 	return final, nil
 }
 

internal/dbbackup/dbbackup_test.go

@@ -307,22 +307,73 @@ func TestBackupRecordsFreshness(t *testing.T) {
 	}
 }
 
+// failingExport stands in for a cluster that does not answer.
+func failingExport(context.Context) ([]byte, error) {
+	return nil, errors.New("connection refused")
+}
+
+func hasServers(m Manifest) bool {
+	return slices.ContainsFunc(m.Files, func(f ManifestEntry) bool { return f.Name == serversEntry })
+}
+
+// TestBackupRecordsAClusterThatDidNotAnswer: the database still gets its
+// bundle when the cluster is away, and the caller learns the bundle restores
+// no servers, as do the manifest and the panel's record.
 func TestBackupRecordsAClusterThatDidNotAnswer(t *testing.T) {
 	pg := newFakePG(t, "x\n")
 	dir := t.TempDir()
+	var log bytes.Buffer
 	path, err := Backup(context.Background(), BackupOptions{
 		DatabaseURL: testURL, Dir: dir, Label: LabelDaily, Tools: pg.tools, Now: at(t0),
-		ExportServers: func(context.Context) ([]byte, error) { return nil, errors.New("connection refused") },
+		ExportServers: failingExport, Record: true, Log: &log,
 	})
-	if err != nil {
-		t.Fatalf("a cluster outage must not fail the database backup: %v", err)
+	if !errors.Is(err, ErrServersMissing) || !strings.HasSuffix(err.Error(), ": connection refused") {
+		t.Fatalf("err = %v, want ErrServersMissing with the export's reason", err)
 	}
-	m, err := Verify(path)
+	if !strings.Contains(log.String(), "MinecraftServer objects not included: connection refused") {
+		t.Errorf("log = %q", log.String())
+	}
+	m, verr := Verify(path)
+	if verr != nil {
+		t.Fatalf("the database must still be bundled: %v", verr)
+	}
+	if m.ServersError != "connection refused" || hasServers(m) || len(m.Files) != 1 {
+		t.Errorf("manifest = %+v", m)
+	}
+	if st := pg.recorded(t); st.Name != filepath.Base(path) || st.ServersError != m.ServersError {
+		t.Errorf("recorded %+v, want the bundle with its servers error", st)
+	}
+}
+
+// TestBackupRequiringServersWritesNothing: a caller that can try again later
+// gets no bundle rather than one that restores no servers.
+func TestBackupRequiringServersWritesNothing(t *testing.T) {
+	pg := newFakePG(t, "x\n")
+	dir := t.TempDir()
+	path, err := Backup(context.Background(), BackupOptions{
+		DatabaseURL: testURL, Dir: dir, Label: LabelOffsite, Tools: pg.tools, Now: at(t0),
+		ExportServers: failingExport, RequireServers: true, Record: true,
+	})
+	if err == nil || errors.Is(err, ErrServersMissing) || path != "" {
+		t.Fatalf("Backup = %q, %v; want a plain failure and no path", path, err)
+	}
+	if all, _ := List(dir); len(all) != 0 {
+		t.Errorf("bundles left behind: %v", all)
+	}
+	if st := pg.recorded(t); st.Name != "" {
+		t.Errorf("recorded %+v for a backup that failed", st)
+	}
+	// With the cluster answering, the same backup is whole.
+	path, err = Backup(context.Background(), BackupOptions{
+		DatabaseURL: testURL, Dir: dir, Label: LabelOffsite, Tools: pg.tools, Now: at(t0),
+		ExportServers:  func(context.Context) ([]byte, error) { return []byte(`{"kind":"List","items":[]}`), nil },
+		RequireServers: true,
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if m.ServersError != "connection refused" || len(m.Files) != 1 {
-		t.Errorf("manifest = %+v", m)
+	if m, err := Verify(path); err != nil || !hasServers(m) || m.ServersError != "" {
+		t.Errorf("manifest = %+v, %v", m, err)
 	}
 }
 

internal/dbbackup/restore.go

@@ -247,7 +247,9 @@ func Restore(ctx context.Context, o RestoreOptions) (Manifest, string, error) {
 		if so.Log == nil {
 			so.Log = logw
 		}
-		if safety, err = Backup(ctx, so); err != nil {
+		// The restore replaces the database alone, so a safety bundle the cluster
+		// did not add its objects to still holds everything it replaces.
+		if safety, err = Backup(ctx, so); err != nil && !errors.Is(err, ErrServersMissing) {
 			return m, "", fmt.Errorf("safety backup of the current database: %w (pass -no-safety-backup to restore without one)", err)
 		}
 		fmt.Fprintf(logw, "felis db restore: current database saved to %s\n", safety)
@@ -274,7 +276,7 @@ func recordNewest(ctx context.Context, c conn, t Tools, dir string) error {
 	b := all[0]
 	st := Status{At: b.Created, Name: b.Name, Label: b.Label, SizeBytes: b.Size, Dir: dir}
 	if m, err := Verify(b.Path); err == nil {
-		st.FelisVersion, st.SchemaVersion = m.FelisVersion, m.SchemaVersion
+		st.FelisVersion, st.SchemaVersion, st.ServersError = m.FelisVersion, m.SchemaVersion, m.ServersError
 	}
 	return record(ctx, c, t, st)
 }

internal/dbbackup/restore_test.go

@@ -137,3 +137,30 @@ func TestRestoreRefusesACorruptBundle(t *testing.T) {
 		}
 	}
 }
+
+// TestRestoreKeepsASafetyBundleWithoutServers: the restore replaces only the
+// database, so a cluster that is away does not stop it, and the record the
+// restore leaves says the newest bundle restores no servers.
+func TestRestoreKeepsASafetyBundleWithoutServers(t *testing.T) {
+	pg := newFakePG(t, "alice\n")
+	dir := t.TempDir()
+	bundle := takeBackup(t, pg, dir, t0)
+	pg.setDB(t, "alice\nbob\n")
+
+	safety, err := restore(pg, dir, bundle, func(o *RestoreOptions) {
+		o.Safety.ExportServers = failingExport
+	})
+	if err != nil {
+		t.Fatalf("Restore: %v", err)
+	}
+	if got := pg.db(t); got != "alice\n" {
+		t.Fatalf("db after restore = %q", got)
+	}
+	m, err := Verify(safety)
+	if err != nil || m.ServersError == "" || m.Label != LabelPreRestore {
+		t.Fatalf("safety bundle %s: %+v, %v", safety, m, err)
+	}
+	if st := pg.recorded(t); st.Name != filepath.Base(safety) || st.ServersError != m.ServersError {
+		t.Fatalf("recorded after restore = %+v", st)
+	}
+}

internal/watchdog/probes.go

@@ -342,7 +342,7 @@ func PostgresDown(err error) Finding {
 }
 
 // BackupFinding reports a control-plane database backup older than a day, or
-// none at all, in dir.
+// none at all, in dir, and a fresh one that would restore no servers.
 func BackupFinding(dir string, now time.Time) *Finding {
 	bundles, err := dbbackup.List(dir)
 	if err != nil {
@@ -354,7 +354,7 @@ func BackupFinding(dir string, now time.Time) *Finding {
 		}
 	}
 	if len(bundles) > 0 && now.Sub(bundles[0].Created) <= maxBackupAge {
-		return nil
+		return serversFinding(bundles[0])
 	}
 	f := &Finding{
 		Key: "db-backup", Severity: Critical, For: backupFor,
@@ -370,6 +370,29 @@ func BackupFinding(dir string, now time.Time) *Finding {
 	return f
 }
 
+// serversFinding reports a bundle whose MinecraftServer export failed: a
+// restore from it, which a lost host would take from the newest bundle, brings
+// back the database and no servers. A bundle it cannot read is left to the
+// restore that verifies it; a bundle taken without the export on purpose
+// (`felis db backup -no-servers`) records no failure.
+func serversFinding(b dbbackup.Bundle) *Finding {
+	f, err := os.Open(b.Path)
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+	m, err := dbbackup.ReadManifest(f)
+	if err != nil || m.ServersError == "" {
+		return nil
+	}
+	return &Finding{
+		Key: "db-backup-servers", Severity: Warning, For: backupFor,
+		Summary:   fmt.Sprintf("最新的控制面数据库备份 %s 缺少 MinecraftServer 对象（%s）：用它恢复能找回数据库，但集群里不会有任何服务器", b.Name, m.ServersError),
+		SummaryEN: fmt.Sprintf("the newest control-plane database backup %s lacks the MinecraftServer objects (%s): a restore from it brings back the database but no servers", b.Name, m.ServersError),
+		Hint:      "k3s kubectl get minecraftservers -A; once the cluster answers, take one now with `sudo felis db backup` (docs/troubleshooting.md §16)",
+	}
+}
+
 // OffsiteFinding reports an off-site copy that has not completed a clean run
 // within offsite.StaleAfter, going by the record `felis offsite sync` leaves
 // in statusFile. It is a warning: the local copies are intact, but a lost

internal/watchdog/probes_test.go

@@ -1,7 +1,10 @@
 package watchdog
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"sort"
@@ -161,6 +164,46 @@ func TestBackupFinding(t *testing.T) {
 	}
 }
 
+// TestBackupFindingServers: a fresh newest bundle whose MinecraftServer export
+// failed is reported, since a lost host restores from it; an older bundle
+// with the same gap, a bundle taken without the export and a stale newest
+// bundle are left to the checks that own them.
+func TestBackupFindingServers(t *testing.T) {
+	dir := t.TempDir()
+	bundle := func(at time.Time, label, serversError string) {
+		t.Helper()
+		m, err := json.Marshal(map[string]any{"format": 1, "label": label, "servers_error": serversError})
+		if err != nil {
+			t.Fatal(err)
+		}
+		var buf bytes.Buffer
+		tw := tar.NewWriter(&buf)
+		if err := tw.WriteHeader(&tar.Header{Name: "MANIFEST.json", Mode: 0o600, Size: int64(len(m))}); err != nil {
+			t.Fatal(err)
+		}
+		tw.Write(m)
+		tw.Close()
+		if err := os.WriteFile(filepath.Join(dir, dbbackup.BundleName(at, label)), buf.Bytes(), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	bundle(t0.Add(-5*time.Hour), "daily", "k3s kubectl get minecraftservers: connection refused")
+	bundle(t0.Add(-3*time.Hour), "manual", "")
+	if f := BackupFinding(dir, t0); f != nil {
+		t.Fatalf("an older bundle's gap reported under a whole newer one: %+v", f)
+	}
+	bundle(t0.Add(-2*time.Hour), "pre-migrate", "k3s kubectl get minecraftservers: connection refused")
+	f := BackupFinding(dir, t0)
+	if f == nil || f.Key != "db-backup-servers" || f.Severity != Warning ||
+		!strings.Contains(f.SummaryEN, "felis-db-20260924T100000Z-pre-migrate.tar lacks the MinecraftServer objects (k3s kubectl get minecraftservers: connection refused)") ||
+		!strings.Contains(f.Summary, "缺少 MinecraftServer 对象") || !strings.Contains(f.Hint, "felis db backup") {
+		t.Fatalf("newest bundle without servers: %+v", f)
+	}
+	if f := BackupFinding(dir, t0.Add(30*time.Hour)); f == nil || f.Key != "db-backup" {
+		t.Fatalf("stale: %+v, want the staleness finding", f)
+	}
+}
+
 func TestOffsiteFinding(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "offsite", "status.json")
 	if f := OffsiteFinding(path, t0); f == nil || !strings.Contains(f.SummaryEN, "never completed") {

panel/src/i18n/resources/en-US/admin.json

@@ -116,6 +116,7 @@
   "dbbackup_status_ok": "Healthy",
   "dbbackup_status_stale": "Overdue",
   "dbbackup_status_never": "Never backed up",
+  "dbbackup_status_incomplete": "Incomplete",
   "dbbackup_refresh": "Refresh",
   "dbbackup_loading": "Reading backup status…",
   "dbbackup_field_when": "Last backup",
@@ -126,9 +127,12 @@
   "dbbackup_label_daily": "Daily",
   "dbbackup_label_pre_migrate": "Pre-upgrade snapshot",
   "dbbackup_label_pre_restore": "Pre-restore snapshot",
+  "dbbackup_label_offsite": "Off-site copy snapshot",
   "dbbackup_label_manual": "Manual",
   "dbbackup_never_title": "No database backup has been recorded yet",
   "dbbackup_stale_title": "The newest backup is more than {{hours}} hours old",
+  "dbbackup_servers_title": "The newest backup lacks the server definitions",
+  "dbbackup_servers_hint": "The cluster did not answer when the backup was taken. Restoring from it brings back accounts and the database, but no servers. Once the cluster answers, take a backup on the host again:",
   "dbbackup_fix_hint": "If the host failed now, accounts, server ownership and the archive index could not be recovered. Take a backup on the host now, then read the timer's log to find out why it did not run:",
   "dbbackup_copy": "Copy command",
   "dbbackup_offsite_note": "Backups are kept on this host only and are lost with its disk. Copy the backup directory to another machine regularly; restore and disaster-recovery steps are in the troubleshooting guide, §16.",

panel/src/i18n/resources/zh-CN/admin.json

@@ -116,6 +116,7 @@
   "dbbackup_status_ok": "正常",
   "dbbackup_status_stale": "已过期",
   "dbbackup_status_never": "从未备份",
+  "dbbackup_status_incomplete": "不完整",
   "dbbackup_refresh": "刷新",
   "dbbackup_loading": "正在读取备份状态…",
   "dbbackup_field_when": "最近一次备份",
@@ -126,9 +127,12 @@
   "dbbackup_label_daily": "每日定时",
   "dbbackup_label_pre_migrate": "升级前快照",
   "dbbackup_label_pre_restore": "恢复前快照",
+  "dbbackup_label_offsite": "异地复制快照",
   "dbbackup_label_manual": "手动",
   "dbbackup_never_title": "还没有记录到任何数据库备份",
   "dbbackup_stale_title": "最近一次备份已超过 {{hours}} 小时",
+  "dbbackup_servers_title": "最近一次备份缺少服务器定义",
+  "dbbackup_servers_hint": "备份时集群没有响应。用它恢复能找回账号和数据库，但一台服务器都不会有。等集群恢复响应后，在主机上重新备份一次：",
   "dbbackup_fix_hint": "此时主机出故障，账号、服务器归属和存档索引都无法恢复。在主机上立即备份一次，再查看定时任务日志找出它没有运行的原因：",
   "dbbackup_copy": "复制命令",
   "dbbackup_offsite_note": "备份只保存在这台主机上，硬盘损坏或主机丢失时会一起丢失。请定期把备份目录复制到另一台机器；恢复与灾备步骤见故障排查文档 §16。",

panel/src/lib/openapi.gen.ts

@@ -2329,8 +2329,11 @@ export interface components {
                 at: string;
                 /** @description Bundle file name, felis-db-<UTC stamp>-<label>.tar. */
                 name: string;
-                /** @enum {string} */
-                label: "daily" | "pre-migrate" | "pre-restore" | "manual";
+                /**
+                 * @description offsite is the bundle `felis offsite sync` takes after copying world archives; a restore records the newest bundle on disk, which may be one.
+                 * @enum {string}
+                 */
+                label: "daily" | "pre-migrate" | "pre-restore" | "offsite" | "manual";
                 /** Format: int64 */
                 size_bytes: number;
                 felis_version?: string;
@@ -2338,6 +2341,8 @@ export interface components {
                 schema_version?: number;
                 /** @description Backup directory on the host. */
                 dir: string;
+                /** @description Why the bundle lacks the MinecraftServer objects (the cluster did not answer the export), when it does. A restore from it brings back the database but no servers. */
+                servers_error?: string;
             } | null;
             /** @description True when there is no record or it is older than max_age_seconds. */
             stale: boolean;

panel/src/lib/types.ts

@@ -457,7 +457,7 @@ export interface UpdateWindow {
 
 // ---- Control-plane database backup (internal/api/handlers_dbbackup.go dbBackupView) ----
 
-export type DBBackupLabel = "daily" | "pre-migrate" | "pre-restore" | "manual";
+export type DBBackupLabel = "daily" | "pre-migrate" | "pre-restore" | "offsite" | "manual";
 
 export interface DBBackupRecord {
   at: string;
@@ -467,6 +467,8 @@ export interface DBBackupRecord {
   felis_version?: string;
   schema_version?: number;
   dir: string;
+  /** Why the bundle lacks the MinecraftServer objects, when it does: a restore from it brings back no servers. */
+  servers_error?: string;
 }
 
 export interface DBBackupStatus {

panel/src/pages/admin/DBBackupCard.test.tsx

@@ -0,0 +1,76 @@
+// @vitest-environment jsdom
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { render, screen } from "@testing-library/react";
+import i18next from "i18next";
+import { DBBackupCard } from "./DBBackupCard";
+import type { DBBackupStatus } from "@/lib/types";
+
+const calls = vi.hoisted(() => ({ getDBBackup: vi.fn() }));
+vi.mock("@/lib/config", () => ({ loadConfig: () => Promise.resolve({}) }));
+vi.mock("@/lib/api", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("@/lib/api")>();
+  return { ...actual, api: { ...actual.api, ...calls } };
+});
+
+const AT = new Date(Date.now() - 3 * 3600 * 1000).toISOString();
+const REASON = "k3s kubectl get minecraftservers: exit status 1: connection refused (tried 3 times)";
+
+function status(last: Partial<NonNullable<DBBackupStatus["last"]>>, stale = false): DBBackupStatus {
+  return {
+    last: {
+      at: AT,
+      name: "felis-db-20260926T033000Z-daily.tar",
+      label: "daily",
+      size_bytes: 4 << 20,
+      schema_version: 42,
+      dir: "/var/lib/felis/db-backups",
+      ...last,
+    },
+    stale,
+    max_age_seconds: 93600,
+  };
+}
+
+beforeEach(() => {
+  calls.getDBBackup.mockReset();
+});
+afterEach(() => {
+  vi.restoreAllMocks();
+  return i18next.changeLanguage("en-US");
+});
+
+describe("DBBackupCard", () => {
+  it("shows a whole, fresh backup as healthy", async () => {
+    calls.getDBBackup.mockResolvedValue(status({}));
+    render(<DBBackupCard />);
+    expect(await screen.findByText("Healthy")).toBeTruthy();
+    expect(screen.queryByText("Incomplete")).toBeNull();
+    expect(screen.queryByText("The newest backup lacks the server definitions")).toBeNull();
+  });
+
+  it("warns that a fresh backup without the MinecraftServer objects restores no servers", async () => {
+    calls.getDBBackup.mockResolvedValue(status({ servers_error: REASON }));
+    render(<DBBackupCard />);
+    expect(await screen.findByText("Incomplete")).toBeTruthy();
+    expect(screen.queryByText("Healthy")).toBeNull();
+    expect(screen.getByText("The newest backup lacks the server definitions")).toBeTruthy();
+    expect(screen.getByText(/Restoring from it brings back accounts and the database, but no servers/)).toBeTruthy();
+    expect(screen.getByText(REASON)).toBeTruthy();
+    expect(screen.getByTitle("sudo k3s kubectl get minecraftservers -A")).toBeTruthy();
+    expect(screen.getByTitle("sudo felis db backup")).toBeTruthy();
+  });
+
+  it("keeps the overdue alarm first and still names the missing servers", async () => {
+    calls.getDBBackup.mockResolvedValue(status({ servers_error: REASON }, true));
+    render(<DBBackupCard />);
+    expect(await screen.findByText("Overdue")).toBeTruthy();
+    expect(screen.queryByText("Incomplete")).toBeNull();
+    expect(screen.getByText("The newest backup lacks the server definitions")).toBeTruthy();
+  });
+
+  it("names an off-site copy snapshot by its kind", async () => {
+    calls.getDBBackup.mockResolvedValue(status({ label: "offsite", name: "felis-db-20260926T101500Z-offsite.tar" }));
+    render(<DBBackupCard />);
+    expect(await screen.findByText("Off-site copy snapshot")).toBeTruthy();
+  });
+});

panel/src/pages/admin/DBBackupCard.tsx

@@ -19,10 +19,14 @@ const LABEL_KEY: Record<DBBackupLabel, string> = {
   daily: "dbbackup_label_daily",
   "pre-migrate": "dbbackup_label_pre_migrate",
   "pre-restore": "dbbackup_label_pre_restore",
+  offsite: "dbbackup_label_offsite",
   manual: "dbbackup_label_manual",
 };
 
 const FIX_COMMANDS = ["sudo felis db backup", "journalctl -u felis-db-backup -n 50 --no-pager"];
+// A bundle the cluster did not add its MinecraftServer objects to: see why the
+// cluster did not answer, then take a whole one.
+const SERVERS_COMMANDS = ["sudo k3s kubectl get minecraftservers -A", "sudo felis db backup"];
 
 export function CopyCommand({ command }: { command: string }) {
   const { t } = useTranslation("admin");
@@ -73,6 +77,7 @@ export function DBBackupCard() {
   const { data, error, loading, reload } = useAsync(() => api.getDBBackup(), []);
   const last = data?.last ?? null;
   const maxAgeHours = data ? Math.round(data.max_age_seconds / 3600) : 26;
+  const serversError = last?.servers_error ?? "";
 
   const state: "loading" | "error" | "never" | "stale" | "ok" = !data
     ? error
@@ -87,6 +92,14 @@ export function DBBackupCard() {
   const badge = (() => {
     switch (state) {
       case "ok":
+        if (serversError) {
+          return (
+            <Badge className="gap-1 border-transparent bg-amber-500/15 text-amber-600 dark:text-amber-400">
+              <AlertTriangle className="h-3 w-3" />
+              {t("dbbackup_status_incomplete")}
+            </Badge>
+          );
+        }
         return (
           <Badge className="gap-1 border-transparent bg-emerald-500/15 text-emerald-500">
             <CheckCircle2 className="h-3 w-3" />
@@ -121,6 +134,8 @@ export function DBBackupCard() {
               "hidden rounded-md p-2 sm:block",
               state === "stale" || state === "never"
                 ? "bg-destructive/10 text-destructive"
+                : serversError
+                  ? "bg-amber-500/10 text-amber-600 dark:text-amber-400"
                   : "bg-primary/10 text-primary",
             )}
           >
@@ -205,6 +220,24 @@ export function DBBackupCard() {
           </div>
         )}
 
+        {serversError && (
+          <div className="space-y-3 rounded-lg border border-amber-500/30 bg-amber-500/10 p-4">
+            <div className="flex items-start gap-2 text-amber-800 dark:text-amber-200">
+              <AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" />
+              <div className="min-w-0 space-y-1">
+                <p className="font-semibold">{t("dbbackup_servers_title")}</p>
+                <p className="text-xs leading-relaxed">{t("dbbackup_servers_hint")}</p>
+                <p className="break-all font-mono text-[11px] opacity-80">{serversError}</p>
+              </div>
+            </div>
+            <div className="space-y-2">
+              {SERVERS_COMMANDS.map((c) => (
+                <CopyCommand key={c} command={c} />
+              ))}
+            </div>
+          </div>
+        )}
+
         {data && (
           <p className="rounded-md border border-border/40 bg-muted/15 p-3 text-[11px] leading-relaxed text-muted-foreground">
             {t("dbbackup_offsite_note")}