diff --git a/cmd/felis/db.go b/cmd/felis/db.go index b7fc609..ab5ae17 100644 --- a/cmd/felis/db.go +++ b/cmd/felis/db.go @@ -172,6 +172,14 @@ func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() path, err := dbbackup.Backup(ctx, o) + if errors.Is(err, dbbackup.ErrServersMissing) { + // The bundle is on disk and holds the database; the exit status fails + // the timer's run so the gap shows in systemctl and the journal, and + // the panel and the watchdog read it from the record and the manifest. + fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path) + fmt.Fprintf(stderr, "felis db backup: %v\n a restore from %s brings back the database but no servers; check `k3s kubectl get minecraftservers -A`, then run `felis db backup` again\n", err, filepath.Base(path)) + return 1 + } if err != nil { fmt.Fprintf(stderr, "felis db backup: %v\n", err) return 1 @@ -423,10 +431,36 @@ func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wri return 0 } -// exportMinecraftServers reads every MinecraftServer through the host's k3s +// serverExportTries and serverExportRetry are how long a backup waits out a +// cluster that is briefly away (an apiserver restart) before its bundle goes +// without the MinecraftServer objects. +const serverExportTries = 3 + +var serverExportRetry = 10 * time.Second + +// exportMinecraftServers is dbbackup's ExportServers on the host: the +// MinecraftServer objects through k3s kubectl, tried serverExportTries times. +func exportMinecraftServers(ctx context.Context) ([]byte, error) { + for try := 1; ; try++ { + out, err := getMinecraftServers(ctx) + if err == nil { + return out, nil + } + if try == serverExportTries { + return nil, fmt.Errorf("%w (tried %d times)", err, try) + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("%w (tried %d times)", err, try) + case <-time.After(serverExportRetry): + } + } +} + +// getMinecraftServers reads every MinecraftServer through the host's k3s // kubectl and strips what the API server owns, so the result can be fed back // with `kubectl apply -f` on a rebuilt cluster. -func exportMinecraftServers(ctx context.Context) ([]byte, error) { +func getMinecraftServers(ctx context.Context) ([]byte, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() // Output, not the CombinedOutput kubectlOutput uses: a deprecation warning diff --git a/cmd/felis/db_test.go b/cmd/felis/db_test.go index 39de253..ddc8812 100644 --- a/cmd/felis/db_test.go +++ b/cmd/felis/db_test.go @@ -1,6 +1,7 @@ package main import ( + "archive/tar" "bytes" "context" "encoding/json" @@ -240,8 +241,20 @@ func TestAuditExportBounds(t *testing.T) { // podK3s stands in for `k3s kubectl exec ... --`: it logs its argv and runs the // command after -- from the "container" directory, which is the only place the -// PostgreSQL tools exist, as on an installed host. +// PostgreSQL tools exist, as on an installed host. `kubectl get` lists one +// MinecraftServer, logged to k3s.get, and refuses its first N calls while +// servers_fail holds N. const podK3s = `#!/bin/sh +if [ "$1" = kubectl ] && [ "$2" = get ]; then + printf '%s\n' "$*" >> "$FAKE_DIR/k3s.get" + n=$(/usr/bin/wc -l < "$FAKE_DIR/k3s.get") + if [ -f "$FAKE_DIR/servers_fail" ] && [ "$n" -le "$(/bin/cat "$FAKE_DIR/servers_fail")" ]; then + echo "The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?" >&2 + exit 1 + fi + echo '{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer","metadata":{"name":"lobby","namespace":"felis-servers","uid":"u-1"},"spec":{"type":"PAPER"},"status":{"phase":"Running"}}]}' + exit 0 +fi printf '%s\n' "$*" >> "$FAKE_DIR/k3s.args" while [ $# -gt 0 ] && [ "$1" != "--" ]; do shift; done shift @@ -410,3 +423,142 @@ func TestDBToolsNeedTheRoleAndDatabase(t *testing.T) { } var dbbackupVerify = dbbackup.Verify + +func noServerExportWait(t *testing.T) { + t.Helper() + old := serverExportRetry + serverExportRetry = 0 + t.Cleanup(func() { serverExportRetry = old }) +} + +// serverGets counts the `kubectl get` calls the fake k3s answered or refused. +func serverGets(dir string) int { + b, _ := os.ReadFile(filepath.Join(dir, "k3s.get")) + return strings.Count(string(b), "\n") +} + +// bundleServers returns the bundle's k8s/minecraftservers.json, or nil. +func bundleServers(t *testing.T, bundle string) []byte { + t.Helper() + f, err := os.Open(bundle) + if err != nil { + t.Fatal(err) + } + defer f.Close() + tr := tar.NewReader(f) + for { + h, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + t.Fatal(err) + } + if h.Name == "k8s/minecraftservers.json" { + data, err := io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + return data + } + } +} + +// TestDBBackupWithoutServersFails: when the cluster stays away the daily +// bundle is still written, and `felis db backup` exits 1, so the timer's run +// shows failed, saying a restore from the bundle brings back no servers. +func TestDBBackupWithoutServersFails(t *testing.T) { + noServerExportWait(t) + dir := newPodRig(t) + cfg := podConfig(t, dir) + writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600) + var out, errBuf bytes.Buffer + code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", "", "-label", "daily"}, &out, &errBuf) + if code != 1 { + t.Fatalf("exit %d, want 1: %s", code, errBuf.String()) + } + bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote ")) + m, err := dbbackupVerify(bundle) + if err != nil { + t.Fatalf("the database must still be bundled: %v", err) + } + if !strings.Contains(m.ServersError, "6443 was refused") || !strings.Contains(m.ServersError, "(tried 3 times)") || bundleServers(t, bundle) != nil { + t.Errorf("manifest servers error = %q", m.ServersError) + } + if n := serverGets(dir); n != serverExportTries { + t.Errorf("export tried %d times, want %d", n, serverExportTries) + } + if msg := errBuf.String(); !strings.Contains(msg, "a restore from "+filepath.Base(bundle)+" brings back the database but no servers") { + t.Errorf("stderr = %q", msg) + } +} + +// TestDBBackupRetriesTheServerExport: a cluster back on the last try costs the +// bundle nothing, and what it holds is ready for kubectl apply. +func TestDBBackupRetriesTheServerExport(t *testing.T) { + noServerExportWait(t) + dir := newPodRig(t) + cfg := podConfig(t, dir) + writeTestFile(t, filepath.Join(dir, "servers_fail"), "2", 0o600) + var out, errBuf bytes.Buffer + if code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", ""}, &out, &errBuf); code != 0 { + t.Fatalf("exit %d: %s", code, errBuf.String()) + } + bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote ")) + servers := string(bundleServers(t, bundle)) + if !strings.Contains(servers, `"name": "lobby"`) || strings.Contains(servers, "status") || strings.Contains(servers, "u-1") { + t.Errorf("k8s/minecraftservers.json = %s", servers) + } + if n := serverGets(dir); n != 3 { + t.Errorf("export tried %d times, want 3", n) + } +} + +// TestPreMigrateBackupExportsServers: the snapshot every upgrade takes, often +// the newest bundle, carries the MinecraftServer objects too; a cluster that +// is away does not hold back the migration, whose rollback needs the database +// alone. +func TestPreMigrateBackupExportsServers(t *testing.T) { + noServerExportWait(t) + dir := newPodRig(t) + old := preMigrateStateDir + preMigrateStateDir = "" + t.Cleanup(func() { preMigrateStateDir = old }) + ms := []store.Migration{{Version: 1}, {Version: 2}} + bundles := filepath.Join(dir, "bundles") + pending := appliedDriver{done: map[int]struct{}{1: {}}} + + path, err := preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + if !strings.Contains(string(bundleServers(t, path)), `"name": "lobby"`) { + t.Errorf("%s holds no MinecraftServer objects", path) + } + + writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600) + path, err = preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard) + if err != nil || path == "" { + t.Fatalf("snapshot with the cluster away = %q, %v; want the bundle and no error", path, err) + } + if m, err := dbbackupVerify(path); err != nil || m.ServersError == "" || bundleServers(t, path) != nil { + t.Errorf("snapshot with the cluster away: %+v, %v", m, err) + } +} + +// TestServerExportStopsWaitingWithTheContext: a backup whose time is up stops +// waiting for the cluster between tries. +func TestServerExportStopsWaitingWithTheContext(t *testing.T) { + dir := newPodRig(t) + writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600) + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + start := time.Now() + _, err := exportMinecraftServers(ctx) + if err == nil || !strings.Contains(err.Error(), "(tried 1 times)") || serverGets(dir) != 1 { + t.Fatalf("err = %v after %d tries, want the first failure alone", err, serverGets(dir)) + } + if took := time.Since(start); took > 5*time.Second { + t.Errorf("took %s, want the context's deadline, not the %s retry wait", took, serverExportRetry) + } +} diff --git a/cmd/felis/migrate.go b/cmd/felis/migrate.go index 9d33180..00fe82c 100644 --- a/cmd/felis/migrate.go +++ b/cmd/felis/migrate.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "flag" "fmt" "io" @@ -83,6 +84,10 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int { return 0 } +// preMigrateStateDir is the host state a pre-migrate bundle carries; tests +// point it at a directory of their own. +var preMigrateStateDir = dbbackup.DefaultStateDir + // preMigrateBackup bundles the database when it already carries a schema and // some of migrations are not applied yet, and returns the bundle's path ("" when // there was nothing to protect: a fresh database, or nothing pending). @@ -101,11 +106,18 @@ func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store. if err != nil { return "", err } - return dbbackup.Backup(ctx, dbbackup.BackupOptions{ + path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{ DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelPreMigrate, - Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir, - Version: resolvedVersion(), Log: log, Record: true, + Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: preMigrateStateDir, + Version: resolvedVersion(), ExportServers: exportMinecraftServers, Log: log, Record: true, }) + if errors.Is(err, dbbackup.ErrServersMissing) { + // Rolling the migration back needs the database alone. Backup logged + // the gap, and the panel and the watchdog show it while this is the + // newest bundle. + return path, nil + } + return path, err } func hasPending(done map[int]struct{}, migrations []store.Migration) bool { diff --git a/cmd/felis/offsite.go b/cmd/felis/offsite.go index 0dae59d..6b10a54 100644 --- a/cmd/felis/offsite.go +++ b/cmd/felis/offsite.go @@ -382,7 +382,10 @@ func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, arch // (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite, // with the newest one kept in dir. It is not recorded for the panel, whose // backup card watches felis-db-backup.timer: snapshots come only when archives -// are copied, and would hide a daily timer that stopped. +// are copied, and would hide a daily timer that stopped. It requires the +// MinecraftServer objects: it becomes the newest bundle in the bucket, which a +// lost host restores from, and a pass that cannot take a whole one fails and +// tries again next hour. func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error { return func(ctx context.Context) error { tools, err := dbTools(db) @@ -394,7 +397,7 @@ func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writ path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{ DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite, Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(), - ExportServers: exportMinecraftServers, Log: log, + ExportServers: exportMinecraftServers, RequireServers: true, Log: log, }) if err == nil { fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path)) @@ -667,7 +670,11 @@ func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles [ fmt.Fprintf(stdout, " %s %s unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err) continue } - fmt.Fprintf(stdout, " %s %s %s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String()) + gap := "" + if m.ServersError != "" { + gap = ", no MinecraftServer objects" + } + fmt.Fprintf(stdout, " %s %s %s%s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String(), gap) } } @@ -891,6 +898,9 @@ func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string, if m.Counts.Fresh() { fmt.Fprintln(stdout, " This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.") } + if m.ServersError != "" { + fmt.Fprintf(stdout, " This bundle lacks the MinecraftServer objects (%s): `felis db restore` brings back the database, and the servers come from k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap.\n", m.ServersError) + } return 0 } diff --git a/cmd/felis/offsite_test.go b/cmd/felis/offsite_test.go index d8390a3..6353b61 100644 --- a/cmd/felis/offsite_test.go +++ b/cmd/felis/offsite_test.go @@ -168,12 +168,19 @@ var fetchT0 = time.Date(2026, 9, 20, 3, 30, 0, 0, time.UTC) // putBundle seals a bundle that verifies, taken daysAgo days before fetchT0, // into b and returns its name. func putBundle(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts) string { + t.Helper() + return putBundleWith(t, b, key, daysAgo, counts, "") +} + +// putBundleWith is putBundle for a bundle whose server export failed with +// serversError, when that is not empty. +func putBundleWith(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts, serversError string) string { t.Helper() created := fetchT0.AddDate(0, 0, -daysAgo) dump := []byte("PGDMP " + created.String()) sum := sha256.Sum256(dump) manifest, err := json.Marshal(dbbackup.Manifest{ - Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, + Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, ServersError: serversError, Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}}, }) if err != nil { @@ -276,6 +283,20 @@ func TestOffsiteFetchDB(t *testing.T) { } }) + t.Run("a bundle without the servers says where they come from", func(t *testing.T) { + b := mapBucket{} + gapped := putBundleWith(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused (tried 3 times)") + _, code, out, errb := fetch(b, gapped) + if code != 0 || !strings.Contains(out, "This bundle lacks the MinecraftServer objects (connection refused (tried 3 times))") || + !strings.Contains(out, "k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap") { + t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb) + } + whole := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3}) + if _, _, out, _ := fetch(b, whole); strings.Contains(out, "lacks the MinecraftServer objects") { + t.Errorf("a whole bundle flagged:\n%s", out) + } + }) + t.Run("a bundle from before counts says so", func(t *testing.T) { b := mapBucket{} old := putBundle(t, b, key, 0, nil) @@ -601,6 +622,7 @@ func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) { otherRaw, _ := offsite.NewKey() other, _ := offsite.ParseKey(otherRaw) b := mapBucket{} + gapped := putBundleWith(t, b, key, 4, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused") old := putBundle(t, b, key, 3, nil) full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3}) sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3}) @@ -617,12 +639,13 @@ func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) { {sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"}, {full, "5 accounts, 3 servers"}, {old, "not recorded"}, + {gapped, "5 accounts, 3 servers, no MinecraftServer objects"}, } - if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (4, newest first") { + if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (5, newest first") { t.Fatalf("output:\n%s", out.String()) } for i, w := range want { - if l := lines[i+1]; !strings.HasPrefix(l, " "+w.name+" ") || !strings.Contains(l, w.holds) { + if l := lines[i+1]; !strings.HasPrefix(l, " "+w.name+" ") || !strings.Contains(l, w.holds) || strings.Contains(l, "no MinecraftServer") != (w.name == gapped) { t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds) } } @@ -708,12 +731,25 @@ func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) { } // The MinecraftServer objects are exported alongside, as in the daily bundle. argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args")) - if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") || !strings.Contains(ran, "kubectl get minecraftservers") { - t.Errorf("k3s ran %q, want pg_dump in the pod and the server export", ran) + if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") { + t.Errorf("k3s ran %q, want pg_dump in the pod", ran) + } + if !strings.Contains(string(bundleServers(t, got[0].Path)), `"name": "lobby"`) { + t.Errorf("the snapshot holds no MinecraftServer objects") } if !strings.Contains(log.String(), "took database bundle "+got[0].Name) { t.Errorf("the snapshot is not logged:\n%s", log.String()) } + // It becomes the newest bundle in the bucket, so with the cluster away it + // fails, leaves no bundle, and the next pass tries again. + noServerExportWait(t) + writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600) + if err := s.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "export the MinecraftServer objects") { + t.Errorf("snapshot with the cluster away: %v, want a failure", err) + } + if again, _ := dbbackup.List(bundles); len(again) != 1 || again[0].Name != got[0].Name { + t.Errorf("bundle directory after the failed snapshot = %+v, want %s alone", again, got[0].Name) + } for _, c := range []struct { archive config.ArchiveConfig diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index c457ec3..b3b1944 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -4074,7 +4074,9 @@ migrate_host_postgres() { remember_temp "$fresh" write_felis_toml "$legacy" "127.0.0.1:5432" write_felis_toml "$fresh" "127.0.0.1:${PG_HOST_PORT}" "${CONTROL_NS}/${PG_DEPLOYMENT}" - out="$("$HOST_BIN" db backup -config "$legacy" -dir "$FELIS_DB_BACKUP_DIR" -label pre-pg-move -keep 0)" \ + # The move carries the database alone; the MinecraftServer objects stay in the + # cluster, so a cluster slow to answer their export must not stop it. + out="$("$HOST_BIN" db backup -config "$legacy" -dir "$FELIS_DB_BACKUP_DIR" -label pre-pg-move -keep 0 -no-servers)" \ || die "could not take a bundle of the host database; nothing was moved" bundle="$(printf '%s\n' "$out" | sed -n 's/^felis db backup: wrote //p' | tail -n 1)" [ -n "$bundle" ] && [ -f "$bundle" ] || die "felis db backup reported no bundle; nothing was moved" diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index de4efb3..d83d5f8 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -3534,7 +3534,7 @@ run_move() { # holds(0/1) backup(ok|silent|fail) restore-exit compare-exit out="$(run_move 1 ok)" calls="$(cat "$mgdir/calls")" expect "the bundle is taken from the host server, never pruned" \ - "FELIS db backup [127.0.0.1:5432 no-deployment] -dir $mgdir/db -label pre-pg-move -keep 0" "$calls" + "FELIS db backup [127.0.0.1:5432 no-deployment] -dir $mgdir/db -label pre-pg-move -keep 0 -no-servers" "$calls" expect "the bundle is restored into the pod, which served nobody yet" \ "FELIS db restore [127.0.0.1:15432 felis/felis-postgres] -dir $mgdir/db -yes -no-safety-backup $mgdir/db/b.tar" "$calls" before "writers are stopped before the bundle" "QUIESCE" "FELIS db backup" "$calls" diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e1ed1d9..e489fcd 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -363,7 +363,8 @@ components: description: Bundle file name, felis-db--