fix(dbbackup): 缺 MinecraftServer 导出时重试、CLI 失败退出、面板与 watchdog 告警，异地快照要求完整

internal/dbbackup/dbbackup.go

@@ -126,11 +126,17 @@ type BackupOptions struct {
 	StateDir string // host state to bundle; "" bundles none
 	Version  string // felis build stamp, recorded in the manifest
 	Tools    Tools
-	// ExportServers returns the cluster's MinecraftServer objects as JSON. A
-	// failure is recorded in the manifest and does not fail the backup: the
-	// database is what must not be lost, and a nightly run cannot hang on a
-	// cluster that happens to be down.
+	// ExportServers returns the cluster's MinecraftServer objects as JSON.
+	// When it fails the bundle is still written, with the reason in its
+	// manifest and in the Record, and Backup returns its path with
+	// ErrServersMissing: the database is what must not be lost, and a nightly
+	// run cannot hang on a cluster that happens to be down. A restore from
+	// such a bundle brings back no servers, so the caller has to make that
+	// heard.
 	ExportServers func(ctx context.Context) ([]byte, error)
+	// RequireServers makes an ExportServers failure fail the backup before a
+	// bundle is written, for a caller that can simply try again later.
+	RequireServers bool
 	// MetricsFile, when set, is rewritten after a successful backup with
 	// node-exporter textfile metrics (felis_db_backup_last_success_timestamp_seconds
 	// and felis_db_backup_last_size_bytes), which FelisDBBackupStale alerts on.
@@ -146,6 +152,11 @@ type BackupOptions struct {
 // StatusKey is the platform_settings key Record writes; internal/api reads it.
 const StatusKey = "db_backup_last"
 
+// ErrServersMissing comes back from Backup, together with the path of the
+// bundle it wrote, when the bundle holds the database but ExportServers
+// failed every try: a restore from it brings back no servers.
+var ErrServersMissing = errors.New("the bundle holds the database but not the MinecraftServer objects")
+
 // StaleAfter is how old the newest backup may get before it counts as missed:
 // a day plus the timer's randomized delay and a slow dump. `felis db check`,
 // the admin panel and the FelisDBBackupStale alert (deploy/alerts) share it.
@@ -160,6 +171,9 @@ type Status struct {
 	FelisVersion  string    `json:"felis_version,omitempty"`
 	SchemaVersion int       `json:"schema_version,omitempty"`
 	Dir           string    `json:"dir"`
+	// ServersError is why the bundle lacks the MinecraftServer objects, when
+	// it does.
+	ServersError string `json:"servers_error,omitempty"`
 }
 
 // Manifest describes a bundle.
@@ -444,7 +458,9 @@ func removeStalePartials(dir string) {
 	}
 }
 
-// Backup writes one bundle and returns its path.
+// Backup writes one bundle and returns its path. With ErrServersMissing the
+// bundle is written and holds the database, but not the MinecraftServer
+// objects.
 func Backup(ctx context.Context, o BackupOptions) (string, error) {
 	if !labelRe.MatchString(o.Label) {
 		return "", fmt.Errorf("invalid label %q (want [a-z0-9-], e.g. daily or manual)", o.Label)
@@ -527,6 +543,9 @@ func Backup(ctx context.Context, o BackupOptions) (string, error) {
 	}
 	if o.ExportServers != nil {
 		if data, err := o.ExportServers(ctx); err != nil {
+			if o.RequireServers {
+				return "", fmt.Errorf("export the MinecraftServer objects: %w", err)
+			}
 			m.ServersError = err.Error()
 			fmt.Fprintf(logw, "felis db backup: MinecraftServer objects not included: %v\n", err)
 		} else {
@@ -546,7 +565,7 @@ func Backup(ctx context.Context, o BackupOptions) (string, error) {
 	}
 	if info, err := os.Stat(final); err == nil {
 		st := Status{At: created, Name: name, Label: o.Label, SizeBytes: info.Size(),
-			FelisVersion: o.Version, SchemaVersion: m.SchemaVersion, Dir: o.Dir}
+			FelisVersion: o.Version, SchemaVersion: m.SchemaVersion, Dir: o.Dir, ServersError: m.ServersError}
 		if o.Record {
 			if err := record(ctx, c, o.Tools, st); err != nil {
 				fmt.Fprintf(logw, "felis db backup: record the backup for the panel: %v\n", err)
@@ -563,6 +582,9 @@ func Backup(ctx context.Context, o BackupOptions) (string, error) {
 	} else if len(removed) > 0 {
 		fmt.Fprintf(logw, "felis db backup: pruned %d old %s bundle(s)\n", len(removed), o.Label)
 	}
+	if m.ServersError != "" {
+		return final, fmt.Errorf("%w: %s", ErrServersMissing, m.ServersError)
+	}
 	return final, nil
 }
 

internal/dbbackup/dbbackup_test.go

@@ -307,22 +307,73 @@ func TestBackupRecordsFreshness(t *testing.T) {
 	}
 }
 
+// failingExport stands in for a cluster that does not answer.
+func failingExport(context.Context) ([]byte, error) {
+	return nil, errors.New("connection refused")
+}
+
+func hasServers(m Manifest) bool {
+	return slices.ContainsFunc(m.Files, func(f ManifestEntry) bool { return f.Name == serversEntry })
+}
+
+// TestBackupRecordsAClusterThatDidNotAnswer: the database still gets its
+// bundle when the cluster is away, and the caller learns the bundle restores
+// no servers, as do the manifest and the panel's record.
 func TestBackupRecordsAClusterThatDidNotAnswer(t *testing.T) {
 	pg := newFakePG(t, "x\n")
 	dir := t.TempDir()
+	var log bytes.Buffer
 	path, err := Backup(context.Background(), BackupOptions{
 		DatabaseURL: testURL, Dir: dir, Label: LabelDaily, Tools: pg.tools, Now: at(t0),
-		ExportServers: func(context.Context) ([]byte, error) { return nil, errors.New("connection refused") },
+		ExportServers: failingExport, Record: true, Log: &log,
 	})
-	if err != nil {
-		t.Fatalf("a cluster outage must not fail the database backup: %v", err)
+	if !errors.Is(err, ErrServersMissing) || !strings.HasSuffix(err.Error(), ": connection refused") {
+		t.Fatalf("err = %v, want ErrServersMissing with the export's reason", err)
 	}
-	m, err := Verify(path)
+	if !strings.Contains(log.String(), "MinecraftServer objects not included: connection refused") {
+		t.Errorf("log = %q", log.String())
+	}
+	m, verr := Verify(path)
+	if verr != nil {
+		t.Fatalf("the database must still be bundled: %v", verr)
+	}
+	if m.ServersError != "connection refused" || hasServers(m) || len(m.Files) != 1 {
+		t.Errorf("manifest = %+v", m)
+	}
+	if st := pg.recorded(t); st.Name != filepath.Base(path) || st.ServersError != m.ServersError {
+		t.Errorf("recorded %+v, want the bundle with its servers error", st)
+	}
+}
+
+// TestBackupRequiringServersWritesNothing: a caller that can try again later
+// gets no bundle rather than one that restores no servers.
+func TestBackupRequiringServersWritesNothing(t *testing.T) {
+	pg := newFakePG(t, "x\n")
+	dir := t.TempDir()
+	path, err := Backup(context.Background(), BackupOptions{
+		DatabaseURL: testURL, Dir: dir, Label: LabelOffsite, Tools: pg.tools, Now: at(t0),
+		ExportServers: failingExport, RequireServers: true, Record: true,
+	})
+	if err == nil || errors.Is(err, ErrServersMissing) || path != "" {
+		t.Fatalf("Backup = %q, %v; want a plain failure and no path", path, err)
+	}
+	if all, _ := List(dir); len(all) != 0 {
+		t.Errorf("bundles left behind: %v", all)
+	}
+	if st := pg.recorded(t); st.Name != "" {
+		t.Errorf("recorded %+v for a backup that failed", st)
+	}
+	// With the cluster answering, the same backup is whole.
+	path, err = Backup(context.Background(), BackupOptions{
+		DatabaseURL: testURL, Dir: dir, Label: LabelOffsite, Tools: pg.tools, Now: at(t0),
+		ExportServers:  func(context.Context) ([]byte, error) { return []byte(`{"kind":"List","items":[]}`), nil },
+		RequireServers: true,
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if m.ServersError != "connection refused" || len(m.Files) != 1 {
-		t.Errorf("manifest = %+v", m)
+	if m, err := Verify(path); err != nil || !hasServers(m) || m.ServersError != "" {
+		t.Errorf("manifest = %+v, %v", m, err)
 	}
 }
 

internal/dbbackup/restore.go

@@ -247,7 +247,9 @@ func Restore(ctx context.Context, o RestoreOptions) (Manifest, string, error) {
 		if so.Log == nil {
 			so.Log = logw
 		}
-		if safety, err = Backup(ctx, so); err != nil {
+		// The restore replaces the database alone, so a safety bundle the cluster
+		// did not add its objects to still holds everything it replaces.
+		if safety, err = Backup(ctx, so); err != nil && !errors.Is(err, ErrServersMissing) {
 			return m, "", fmt.Errorf("safety backup of the current database: %w (pass -no-safety-backup to restore without one)", err)
 		}
 		fmt.Fprintf(logw, "felis db restore: current database saved to %s\n", safety)
@@ -274,7 +276,7 @@ func recordNewest(ctx context.Context, c conn, t Tools, dir string) error {
 	b := all[0]
 	st := Status{At: b.Created, Name: b.Name, Label: b.Label, SizeBytes: b.Size, Dir: dir}
 	if m, err := Verify(b.Path); err == nil {
-		st.FelisVersion, st.SchemaVersion = m.FelisVersion, m.SchemaVersion
+		st.FelisVersion, st.SchemaVersion, st.ServersError = m.FelisVersion, m.SchemaVersion, m.ServersError
 	}
 	return record(ctx, c, t, st)
 }

internal/dbbackup/restore_test.go

@@ -137,3 +137,30 @@ func TestRestoreRefusesACorruptBundle(t *testing.T) {
 		}
 	}
 }
+
+// TestRestoreKeepsASafetyBundleWithoutServers: the restore replaces only the
+// database, so a cluster that is away does not stop it, and the record the
+// restore leaves says the newest bundle restores no servers.
+func TestRestoreKeepsASafetyBundleWithoutServers(t *testing.T) {
+	pg := newFakePG(t, "alice\n")
+	dir := t.TempDir()
+	bundle := takeBackup(t, pg, dir, t0)
+	pg.setDB(t, "alice\nbob\n")
+
+	safety, err := restore(pg, dir, bundle, func(o *RestoreOptions) {
+		o.Safety.ExportServers = failingExport
+	})
+	if err != nil {
+		t.Fatalf("Restore: %v", err)
+	}
+	if got := pg.db(t); got != "alice\n" {
+		t.Fatalf("db after restore = %q", got)
+	}
+	m, err := Verify(safety)
+	if err != nil || m.ServersError == "" || m.Label != LabelPreRestore {
+		t.Fatalf("safety bundle %s: %+v, %v", safety, m, err)
+	}
+	if st := pg.recorded(t); st.Name != filepath.Base(safety) || st.ServersError != m.ServersError {
+		t.Fatalf("recorded after restore = %+v", st)
+	}
+}

internal/watchdog/probes.go

@@ -342,7 +342,7 @@ func PostgresDown(err error) Finding {
 }
 
 // BackupFinding reports a control-plane database backup older than a day, or
-// none at all, in dir.
+// none at all, in dir, and a fresh one that would restore no servers.
 func BackupFinding(dir string, now time.Time) *Finding {
 	bundles, err := dbbackup.List(dir)
 	if err != nil {
@@ -354,7 +354,7 @@ func BackupFinding(dir string, now time.Time) *Finding {
 		}
 	}
 	if len(bundles) > 0 && now.Sub(bundles[0].Created) <= maxBackupAge {
-		return nil
+		return serversFinding(bundles[0])
 	}
 	f := &Finding{
 		Key: "db-backup", Severity: Critical, For: backupFor,
@@ -370,6 +370,29 @@ func BackupFinding(dir string, now time.Time) *Finding {
 	return f
 }
 
+// serversFinding reports a bundle whose MinecraftServer export failed: a
+// restore from it, which a lost host would take from the newest bundle, brings
+// back the database and no servers. A bundle it cannot read is left to the
+// restore that verifies it; a bundle taken without the export on purpose
+// (`felis db backup -no-servers`) records no failure.
+func serversFinding(b dbbackup.Bundle) *Finding {
+	f, err := os.Open(b.Path)
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+	m, err := dbbackup.ReadManifest(f)
+	if err != nil || m.ServersError == "" {
+		return nil
+	}
+	return &Finding{
+		Key: "db-backup-servers", Severity: Warning, For: backupFor,
+		Summary:   fmt.Sprintf("最新的控制面数据库备份 %s 缺少 MinecraftServer 对象（%s）：用它恢复能找回数据库，但集群里不会有任何服务器", b.Name, m.ServersError),
+		SummaryEN: fmt.Sprintf("the newest control-plane database backup %s lacks the MinecraftServer objects (%s): a restore from it brings back the database but no servers", b.Name, m.ServersError),
+		Hint:      "k3s kubectl get minecraftservers -A; once the cluster answers, take one now with `sudo felis db backup` (docs/troubleshooting.md §16)",
+	}
+}
+
 // OffsiteFinding reports an off-site copy that has not completed a clean run
 // within offsite.StaleAfter, going by the record `felis offsite sync` leaves
 // in statusFile. It is a warning: the local copies are intact, but a lost

internal/watchdog/probes_test.go

@@ -1,7 +1,10 @@
 package watchdog
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"sort"
@@ -161,6 +164,46 @@ func TestBackupFinding(t *testing.T) {
 	}
 }
 
+// TestBackupFindingServers: a fresh newest bundle whose MinecraftServer export
+// failed is reported, since a lost host restores from it; an older bundle
+// with the same gap, a bundle taken without the export and a stale newest
+// bundle are left to the checks that own them.
+func TestBackupFindingServers(t *testing.T) {
+	dir := t.TempDir()
+	bundle := func(at time.Time, label, serversError string) {
+		t.Helper()
+		m, err := json.Marshal(map[string]any{"format": 1, "label": label, "servers_error": serversError})
+		if err != nil {
+			t.Fatal(err)
+		}
+		var buf bytes.Buffer
+		tw := tar.NewWriter(&buf)
+		if err := tw.WriteHeader(&tar.Header{Name: "MANIFEST.json", Mode: 0o600, Size: int64(len(m))}); err != nil {
+			t.Fatal(err)
+		}
+		tw.Write(m)
+		tw.Close()
+		if err := os.WriteFile(filepath.Join(dir, dbbackup.BundleName(at, label)), buf.Bytes(), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	bundle(t0.Add(-5*time.Hour), "daily", "k3s kubectl get minecraftservers: connection refused")
+	bundle(t0.Add(-3*time.Hour), "manual", "")
+	if f := BackupFinding(dir, t0); f != nil {
+		t.Fatalf("an older bundle's gap reported under a whole newer one: %+v", f)
+	}
+	bundle(t0.Add(-2*time.Hour), "pre-migrate", "k3s kubectl get minecraftservers: connection refused")
+	f := BackupFinding(dir, t0)
+	if f == nil || f.Key != "db-backup-servers" || f.Severity != Warning ||
+		!strings.Contains(f.SummaryEN, "felis-db-20260924T100000Z-pre-migrate.tar lacks the MinecraftServer objects (k3s kubectl get minecraftservers: connection refused)") ||
+		!strings.Contains(f.Summary, "缺少 MinecraftServer 对象") || !strings.Contains(f.Hint, "felis db backup") {
+		t.Fatalf("newest bundle without servers: %+v", f)
+	}
+	if f := BackupFinding(dir, t0.Add(30*time.Hour)); f == nil || f.Key != "db-backup" {
+		t.Fatalf("stale: %+v, want the staleness finding", f)
+	}
+}
+
 func TestOffsiteFinding(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "offsite", "status.json")
 	if f := OffsiteFinding(path, t0); f == nil || !strings.Contains(f.SummaryEN, "never completed") {