fix(dbbackup): 缺 MinecraftServer 导出时重试、CLI 失败退出、面板与 watchdog 告警，异地快照要求完整

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)

	defer cancel()

	path, err := dbbackup.Backup(ctx, o)

	if errors.Is(err, dbbackup.ErrServersMissing) {

		// The bundle is on disk and holds the database; the exit status fails

		// the timer's run so the gap shows in systemctl and the journal, and

		// the panel and the watchdog read it from the record and the manifest.

		fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)

		fmt.Fprintf(stderr, "felis db backup: %v\n  a restore from %s brings back the database but no servers; check `k3s kubectl get minecraftservers -A`, then run `felis db backup` again\n", err, filepath.Base(path))

		return 1

	}

	if err != nil {

		fmt.Fprintf(stderr, "felis db backup: %v\n", err)

		return 1

	return 0

}

// exportMinecraftServers reads every MinecraftServer through the host's k3s

// serverExportTries and serverExportRetry are how long a backup waits out a

// cluster that is briefly away (an apiserver restart) before its bundle goes

// without the MinecraftServer objects.

const serverExportTries = 3

var serverExportRetry = 10 * time.Second

// exportMinecraftServers is dbbackup's ExportServers on the host: the

// MinecraftServer objects through k3s kubectl, tried serverExportTries times.

func exportMinecraftServers(ctx context.Context) ([]byte, error) {

	for try := 1; ; try++ {

		out, err := getMinecraftServers(ctx)

		if err == nil {

			return out, nil

		}

		if try == serverExportTries {

			return nil, fmt.Errorf("%w (tried %d times)", err, try)

		}

		select {

		case <-ctx.Done():

			return nil, fmt.Errorf("%w (tried %d times)", err, try)

		case <-time.After(serverExportRetry):

		}

	}

}

// getMinecraftServers reads every MinecraftServer through the host's k3s

// kubectl and strips what the API server owns, so the result can be fed back

// with `kubectl apply -f` on a rebuilt cluster.

func exportMinecraftServers(ctx context.Context) ([]byte, error) {

func getMinecraftServers(ctx context.Context) ([]byte, error) {

	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)

	defer cancel()

	// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning

package main

import (

	"archive/tar"

	"bytes"

	"context"

	"encoding/json"

// podK3s stands in for `k3s kubectl exec ... --`: it logs its argv and runs the

// command after -- from the "container" directory, which is the only place the

// PostgreSQL tools exist, as on an installed host.

// PostgreSQL tools exist, as on an installed host. `kubectl get` lists one

// MinecraftServer, logged to k3s.get, and refuses its first N calls while

// servers_fail holds N.

const podK3s = `#!/bin/sh

if [ "$1" = kubectl ] && [ "$2" = get ]; then

  printf '%s\n' "$*" >> "$FAKE_DIR/k3s.get"

  n=$(/usr/bin/wc -l < "$FAKE_DIR/k3s.get")

  if [ -f "$FAKE_DIR/servers_fail" ] && [ "$n" -le "$(/bin/cat "$FAKE_DIR/servers_fail")" ]; then

    echo "The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?" >&2

    exit 1

  fi

  echo '{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer","metadata":{"name":"lobby","namespace":"felis-servers","uid":"u-1"},"spec":{"type":"PAPER"},"status":{"phase":"Running"}}]}'

  exit 0

fi

printf '%s\n' "$*" >> "$FAKE_DIR/k3s.args"

while [ $# -gt 0 ] && [ "$1" != "--" ]; do shift; done

shift

}

var dbbackupVerify = dbbackup.Verify

func noServerExportWait(t *testing.T) {

	t.Helper()

	old := serverExportRetry

	serverExportRetry = 0

	t.Cleanup(func() { serverExportRetry = old })

}

// serverGets counts the `kubectl get` calls the fake k3s answered or refused.

func serverGets(dir string) int {

	b, _ := os.ReadFile(filepath.Join(dir, "k3s.get"))

	return strings.Count(string(b), "\n")

}

// bundleServers returns the bundle's k8s/minecraftservers.json, or nil.

func bundleServers(t *testing.T, bundle string) []byte {

	t.Helper()

	f, err := os.Open(bundle)

	if err != nil {

		t.Fatal(err)

	}

	defer f.Close()

	tr := tar.NewReader(f)

	for {

		h, err := tr.Next()

		if err == io.EOF {

			return nil

		}

		if err != nil {

			t.Fatal(err)

		}

		if h.Name == "k8s/minecraftservers.json" {

			data, err := io.ReadAll(tr)

			if err != nil {

				t.Fatal(err)

			}

			return data

		}

	}

}

// TestDBBackupWithoutServersFails: when the cluster stays away the daily

// bundle is still written, and `felis db backup` exits 1, so the timer's run

// shows failed, saying a restore from the bundle brings back no servers.

func TestDBBackupWithoutServersFails(t *testing.T) {

	noServerExportWait(t)

	dir := newPodRig(t)

	cfg := podConfig(t, dir)

	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)

	var out, errBuf bytes.Buffer

	code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", "", "-label", "daily"}, &out, &errBuf)

	if code != 1 {

		t.Fatalf("exit %d, want 1: %s", code, errBuf.String())

	}

	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))

	m, err := dbbackupVerify(bundle)

	if err != nil {

		t.Fatalf("the database must still be bundled: %v", err)

	}

	if !strings.Contains(m.ServersError, "6443 was refused") || !strings.Contains(m.ServersError, "(tried 3 times)") || bundleServers(t, bundle) != nil {

		t.Errorf("manifest servers error = %q", m.ServersError)

	}

	if n := serverGets(dir); n != serverExportTries {

		t.Errorf("export tried %d times, want %d", n, serverExportTries)

	}

	if msg := errBuf.String(); !strings.Contains(msg, "a restore from "+filepath.Base(bundle)+" brings back the database but no servers") {

		t.Errorf("stderr = %q", msg)

	}

}

// TestDBBackupRetriesTheServerExport: a cluster back on the last try costs the

// bundle nothing, and what it holds is ready for kubectl apply.

func TestDBBackupRetriesTheServerExport(t *testing.T) {

	noServerExportWait(t)

	dir := newPodRig(t)

	cfg := podConfig(t, dir)

	writeTestFile(t, filepath.Join(dir, "servers_fail"), "2", 0o600)

	var out, errBuf bytes.Buffer

	if code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", ""}, &out, &errBuf); code != 0 {

		t.Fatalf("exit %d: %s", code, errBuf.String())

	}

	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))

	servers := string(bundleServers(t, bundle))

	if !strings.Contains(servers, `"name": "lobby"`) || strings.Contains(servers, "status") || strings.Contains(servers, "u-1") {

		t.Errorf("k8s/minecraftservers.json = %s", servers)

	}

	if n := serverGets(dir); n != 3 {

		t.Errorf("export tried %d times, want 3", n)

	}

}

// TestPreMigrateBackupExportsServers: the snapshot every upgrade takes, often

// the newest bundle, carries the MinecraftServer objects too; a cluster that

// is away does not hold back the migration, whose rollback needs the database

// alone.

func TestPreMigrateBackupExportsServers(t *testing.T) {

	noServerExportWait(t)

	dir := newPodRig(t)

	old := preMigrateStateDir

	preMigrateStateDir = ""

	t.Cleanup(func() { preMigrateStateDir = old })

	ms := []store.Migration{{Version: 1}, {Version: 2}}

	bundles := filepath.Join(dir, "bundles")

	pending := appliedDriver{done: map[int]struct{}{1: {}}}

	path, err := preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)

	if err != nil {

		t.Fatalf("snapshot: %v", err)

	}

	if !strings.Contains(string(bundleServers(t, path)), `"name": "lobby"`) {

		t.Errorf("%s holds no MinecraftServer objects", path)

	}

	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)

	path, err = preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)

	if err != nil || path == "" {

		t.Fatalf("snapshot with the cluster away = %q, %v; want the bundle and no error", path, err)

	}

	if m, err := dbbackupVerify(path); err != nil || m.ServersError == "" || bundleServers(t, path) != nil {

		t.Errorf("snapshot with the cluster away: %+v, %v", m, err)

	}

}

// TestServerExportStopsWaitingWithTheContext: a backup whose time is up stops

// waiting for the cluster between tries.

func TestServerExportStopsWaitingWithTheContext(t *testing.T) {

	dir := newPodRig(t)

	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)

	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)

	defer cancel()

	start := time.Now()

	_, err := exportMinecraftServers(ctx)

	if err == nil || !strings.Contains(err.Error(), "(tried 1 times)") || serverGets(dir) != 1 {

		t.Fatalf("err = %v after %d tries, want the first failure alone", err, serverGets(dir))

	}

	if took := time.Since(start); took > 5*time.Second {

		t.Errorf("took %s, want the context's deadline, not the %s retry wait", took, serverExportRetry)

	}

}

import (

	"context"

	"errors"

	"flag"

	"fmt"

	"io"

	return 0

}

// preMigrateStateDir is the host state a pre-migrate bundle carries; tests

// point it at a directory of their own.

var preMigrateStateDir = dbbackup.DefaultStateDir

// preMigrateBackup bundles the database when it already carries a schema and

// some of migrations are not applied yet, and returns the bundle's path ("" when

// there was nothing to protect: a fresh database, or nothing pending).

	if err != nil {

		return "", err

	}

	return dbbackup.Backup(ctx, dbbackup.BackupOptions{

	path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{

		DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelPreMigrate,

		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir,

		Version: resolvedVersion(), Log: log, Record: true,

		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: preMigrateStateDir,

		Version: resolvedVersion(), ExportServers: exportMinecraftServers, Log: log, Record: true,

	})

	if errors.Is(err, dbbackup.ErrServersMissing) {

		// Rolling the migration back needs the database alone. Backup logged

		// the gap, and the panel and the watchdog show it while this is the

		// newest bundle.

		return path, nil

	}

	return path, err

}

func hasPending(done map[int]struct{}, migrations []store.Migration) bool {

// (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite,

// with the newest one kept in dir. It is not recorded for the panel, whose

// backup card watches felis-db-backup.timer: snapshots come only when archives

// are copied, and would hide a daily timer that stopped.

// are copied, and would hide a daily timer that stopped. It requires the

// MinecraftServer objects: it becomes the newest bundle in the bucket, which a

// lost host restores from, and a pass that cannot take a whole one fails and

// tries again next hour.

func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error {

	return func(ctx context.Context) error {

		tools, err := dbTools(db)

		path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{

			DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite,

			Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(),

			ExportServers: exportMinecraftServers, Log: log,

			ExportServers: exportMinecraftServers, RequireServers: true, Log: log,

		})

		if err == nil {

			fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path))

			fmt.Fprintf(stdout, "  %s  %s  unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err)

			continue

		}

		fmt.Fprintf(stdout, "  %s  %s  %s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String())

		gap := ""

		if m.ServersError != "" {

			gap = ", no MinecraftServer objects"

		}

		fmt.Fprintf(stdout, "  %s  %s  %s%s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String(), gap)

	}

}

	if m.Counts.Fresh() {

		fmt.Fprintln(stdout, "  This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.")

	}

	if m.ServersError != "" {

		fmt.Fprintf(stdout, "  This bundle lacks the MinecraftServer objects (%s): `felis db restore` brings back the database, and the servers come from k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap.\n", m.ServersError)

	}

	return 0

}

// putBundle seals a bundle that verifies, taken daysAgo days before fetchT0,

// into b and returns its name.

func putBundle(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts) string {

	t.Helper()

	return putBundleWith(t, b, key, daysAgo, counts, "")

}

// putBundleWith is putBundle for a bundle whose server export failed with

// serversError, when that is not empty.

func putBundleWith(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts, serversError string) string {

	t.Helper()

	created := fetchT0.AddDate(0, 0, -daysAgo)

	dump := []byte("PGDMP " + created.String())

	sum := sha256.Sum256(dump)

	manifest, err := json.Marshal(dbbackup.Manifest{

		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts,

		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, ServersError: serversError,

		Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}},

	})

	if err != nil {

		}

	})

	t.Run("a bundle without the servers says where they come from", func(t *testing.T) {

		b := mapBucket{}

		gapped := putBundleWith(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused (tried 3 times)")

		_, code, out, errb := fetch(b, gapped)

		if code != 0 || !strings.Contains(out, "This bundle lacks the MinecraftServer objects (connection refused (tried 3 times))") ||

			!strings.Contains(out, "k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap") {

			t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)

		}

		whole := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})

		if _, _, out, _ := fetch(b, whole); strings.Contains(out, "lacks the MinecraftServer objects") {

			t.Errorf("a whole bundle flagged:\n%s", out)

		}

	})

	t.Run("a bundle from before counts says so", func(t *testing.T) {

		b := mapBucket{}

		old := putBundle(t, b, key, 0, nil)

	otherRaw, _ := offsite.NewKey()

	other, _ := offsite.ParseKey(otherRaw)

	b := mapBucket{}

	gapped := putBundleWith(t, b, key, 4, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused")

	old := putBundle(t, b, key, 3, nil)

	full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3})

	sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3})

		{sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"},

		{full, "5 accounts, 3 servers"},

		{old, "not recorded"},

		{gapped, "5 accounts, 3 servers, no MinecraftServer objects"},

	}

	if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (4, newest first") {

	if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (5, newest first") {

		t.Fatalf("output:\n%s", out.String())

	}

	for i, w := range want {

		if l := lines[i+1]; !strings.HasPrefix(l, "  "+w.name+"  ") || !strings.Contains(l, w.holds) {

		if l := lines[i+1]; !strings.HasPrefix(l, "  "+w.name+"  ") || !strings.Contains(l, w.holds) || strings.Contains(l, "no MinecraftServer") != (w.name == gapped) {

			t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds)

		}

	}

	}

	// The MinecraftServer objects are exported alongside, as in the daily bundle.

	argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args"))

	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") || !strings.Contains(ran, "kubectl get minecraftservers") {

		t.Errorf("k3s ran %q, want pg_dump in the pod and the server export", ran)

	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") {

		t.Errorf("k3s ran %q, want pg_dump in the pod", ran)

	}

	if !strings.Contains(string(bundleServers(t, got[0].Path)), `"name": "lobby"`) {

		t.Errorf("the snapshot holds no MinecraftServer objects")

	}

	if !strings.Contains(log.String(), "took database bundle "+got[0].Name) {

		t.Errorf("the snapshot is not logged:\n%s", log.String())

	}

	// It becomes the newest bundle in the bucket, so with the cluster away it

	// fails, leaves no bundle, and the next pass tries again.

	noServerExportWait(t)

	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)

	if err := s.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "export the MinecraftServer objects") {

		t.Errorf("snapshot with the cluster away: %v, want a failure", err)

	}

	if again, _ := dbbackup.List(bundles); len(again) != 1 || again[0].Name != got[0].Name {

		t.Errorf("bundle directory after the failed snapshot = %+v, want %s alone", again, got[0].Name)

	}

	for _, c := range []struct {

		archive config.ArchiveConfig