feat(db): 控制面 PG 定时备份、迁移前快照与原子恢复

README.md

@@ -18,6 +18,7 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy,
 - **即开即玩**：玩家尝试连接时自动唤醒服务器，空闲后自动休眠，像游戏主机一样省资源。
 - **Web 控制面板**：浏览器中查看服务器状态、在线玩家与资源用量，管理备份与恢复。
 - **备份与恢复**：一键把整服数据（世界、配置、插件/模组，即整个 /data 卷）打包进集群内的归档库，支持从任意备份点回滚；默认安装就已启用（归档 PVC 与路径由安装器一并生成）。
+- **控制面数据库备份**：账号、服务器归属、配额与存档索引所在的数据库每天自动备份，每次升级迁移前先快照，出错可用 `felis db restore` 整库原子回滚；面板「维护与备份」页显示备份是否新鲜（见 [故障排查 §16](docs/troubleshooting.md)）。
 - **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。
 - **多核心支持**：兼容 Paper、Fabric、Forge、NeoForge，经由 Velocity 代理统一入口。
 - **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建；构建产物进入镜像白名单，可直接选用为服务器镜像完成部署。

README_EN.md

@@ -18,6 +18,7 @@ Table of Contents
 - **Wake on Join**: Servers start automatically when a player connects, and stop when idle — like hibernate for your server.
 - **Web Dashboard**: Monitor server status, online players, and resource usage from your browser, with backup and restore management.
 - **Backup & Restore**: One-click snapshots of a server's whole data volume (worlds, config, plugins/mods — the entire /data volume) into the cluster's archive store, with rollback from any backup point — enabled by default (the installer renders the archive PVC and its path).
+- **Control-plane database backups**: The database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migrates it; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows whether the newest backup is fresh (see [troubleshooting §16](docs/troubleshooting.md)).
 - **World Reaper** (opt in): Worlds idle for more than 15 days are automatically backed up and removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is ever deleted.
 - **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, federated behind a Velocity proxy.
 - **Modpack Submission**: Players submit custom modpacks; admin approval triggers an automatic build, and the result is whitelisted as a server image you can select to deploy.

cmd/felis/db.go

@@ -0,0 +1,334 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+)
+
+const dbUsage = `usage:
+  felis db backup  [-config path] [-dir dir] [-label daily|manual|...] [-keep n] [-state-dir dir]
+                   [-no-servers] [-metrics-file path]
+  felis db restore [-config path] [-dir dir] [-yes] [-force] [-no-safety-backup] <bundle>
+  felis db verify  [-dir dir] <bundle>
+  felis db list    [-dir dir]
+  felis db check   [-dir dir] [-max-age 26h]
+`
+
+// defaultKeep is how many bundles of a label a backup leaves behind. Manual
+// bundles are the operator's own and are never pruned.
+var defaultKeep = map[string]int{
+	dbbackup.LabelDaily:      14,
+	dbbackup.LabelPreMigrate: 10,
+	dbbackup.LabelPreRestore: 5,
+}
+
+// cmdDB implements `felis db`: logical backups of the control-plane database
+// together with the host state a rebuild needs (internal/dbbackup). The verb
+// comes first for the same reason as `felis migrate up`.
+func cmdDB(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("db "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, dbUsage) }
+	dir := fs.String("dir", dbbackup.DefaultDir, "bundle directory")
+	switch verb {
+	case "backup":
+		return dbBackup(fs, dir, rest, stdout, stderr)
+	case "restore":
+		return dbRestore(fs, dir, rest, stdout, stderr)
+	case "verify":
+		return dbVerify(fs, dir, rest, stdout, stderr)
+	case "list":
+		return dbList(fs, dir, rest, stdout, stderr)
+	case "check":
+		return dbCheck(fs, dir, rest, stdout, stderr)
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, dbUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis db: unknown verb %q\n%s", verb, dbUsage)
+	return 2
+}
+
+// parseWithArg parses flags that may sit on either side of one positional
+// argument (`restore -yes x.tar` and `restore x.tar -yes` both work) and
+// returns that argument.
+func parseWithArg(fs *flag.FlagSet, args []string) (string, bool) {
+	if err := fs.Parse(args); err != nil {
+		return "", false
+	}
+	if fs.NArg() == 0 {
+		return "", true
+	}
+	arg := fs.Arg(0)
+	if err := fs.Parse(fs.Args()[1:]); err != nil {
+		return "", false
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprintf(fs.Output(), "felis db: unexpected argument %q\n", fs.Arg(0))
+		return "", false
+	}
+	return arg, true
+}
+
+func dbDatabaseURL(path string) (string, error) {
+	cfg, err := config.Load(path)
+	if err != nil {
+		return "", err
+	}
+	return cfg.Database.URL, nil
+}
+
+func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never")
+	keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, manual all)")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`)
+	noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle")
+	metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	if *keep < 0 {
+		*keep = defaultKeep[*label]
+	}
+	o := dbbackup.BackupOptions{
+		DatabaseURL: url, Dir: *dir, Label: *label, Keep: *keep,
+		StateDir: *stateDir, Version: resolvedVersion(), Log: stderr,
+		MetricsFile: *metrics, Record: true,
+	}
+	if !*noServers {
+		o.ExportServers = exportMinecraftServers
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	path, err := dbbackup.Backup(ctx, o)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+	return 0
+}
+
+// resolveBundle accepts a path, or a bare bundle name looked up in dir.
+func resolveBundle(dir, arg string) string {
+	if strings.ContainsRune(arg, os.PathSeparator) {
+		return arg
+	}
+	if _, err := os.Stat(arg); err == nil {
+		return arg
+	}
+	return filepath.Join(dir, arg)
+}
+
+func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	yes := fs.Bool("yes", false, "replace the database's contents (required)")
+	force := fs.Bool("force", false, "restore even while other clients are connected")
+	noSafety := fs.Bool("no-safety-backup", false, "skip the bundle of the current database taken first")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, "host state directory for the safety bundle")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	if !*yes {
+		fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d).\n",
+			filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion)
+		fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.")
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
+	defer cancel()
+	_, safety, err := dbbackup.Restore(ctx, dbbackup.RestoreOptions{
+		DatabaseURL: url, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
+		Safety: dbbackup.BackupOptions{Keep: defaultKeep[dbbackup.LabelPreRestore], StateDir: *stateDir,
+			Version: resolvedVersion(), ExportServers: exportMinecraftServers},
+		Log: stderr,
+	})
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		if errors.Is(err, dbbackup.ErrClientsConnected) {
+			fmt.Fprintln(stderr, "  kubectl -n felis scale deployment felis-api felis-operator --replicas=0")
+		}
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db restore: restored %s (schema %d)\n", filepath.Base(bundle), m.SchemaVersion)
+	if safety != "" {
+		fmt.Fprintf(stdout, "  the database as it was before is in %s\n", safety)
+	}
+	// Nothing migrates at startup, so a control plane newer than the bundle needs
+	// its migrations re-applied; rolling back to the release that wrote the bundle
+	// must skip that, or the rollback is undone.
+	fmt.Fprintf(stdout, "  next: felis migrate up -config %s (skip it when rolling back to felis %s, which wrote this bundle)\n", *cfgPath, orUnknown(m.FelisVersion))
+	fmt.Fprintln(stdout, "        kubectl -n felis scale deployment felis-api felis-operator --replicas=1")
+	return 0
+}
+
+func dbVerify(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db verify: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "%s: ok\n  taken   %s (%s)\n  felis   %s\n  schema  %d\n  %s\n",
+		filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion, orUnknown(m.PGDumpVersion))
+	for _, f := range m.Files {
+		if f.Link != "" {
+			fmt.Fprintf(stdout, "  %-40s -> %s\n", f.Name, f.Link)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %-40s %d bytes\n", f.Name, f.Size)
+	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  (no MinecraftServer objects: %s)\n", m.ServersError)
+	}
+	return 0
+}
+
+func orUnknown(s string) string {
+	if s == "" {
+		return "unknown"
+	}
+	return s
+}
+
+func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	all, err := dbbackup.List(*dir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db list: %v\n", err)
+		return 1
+	}
+	if len(all) == 0 {
+		fmt.Fprintf(stdout, "no database backups in %s\n", *dir)
+		return 0
+	}
+	now := time.Now()
+	for _, b := range all {
+		fmt.Fprintf(stdout, "%-50s %-12s %10s  %s ago\n", b.Name, b.Label, humanBytes(b.Size), dbbackup.Age(now.Sub(b.Created)))
+	}
+	return 0
+}
+
+func humanBytes(n int64) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%d B", n)
+	}
+	div, exp := int64(unit), 0
+	for m := n / unit; m >= unit; m /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
+}
+
+// dbCheck is the freshness probe: exit 1 when the newest bundle is missing or
+// older than -max-age, for a monitor or the break-glass console to act on.
+func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest bundle")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	b, err := dbbackup.Check(*dir, *maxAge, time.Now())
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db check: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db check: ok, newest backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
+	return 0
+}
+
+// exportMinecraftServers reads every MinecraftServer through the host's k3s
+// kubectl and strips what the API server owns, so the result can be fed back
+// with `kubectl apply -f` on a rebuilt cluster.
+func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning
+	// on stderr must not end up inside the JSON.
+	cmd := exec.CommandContext(ctx, "k3s", "kubectl", "get", "minecraftservers.felis.lolicon.best", "-A", "-o", "json")
+	cmd.Env = append(os.Environ(), "KUBECONFIG="+hostBootstrapKubeconfigPath)
+	var errBuf strings.Builder
+	cmd.Stderr = &errBuf
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("k3s kubectl get minecraftservers: %w: %s", err, strings.TrimSpace(errBuf.String()))
+	}
+	return cleanServerList(out)
+}
+
+// cleanServerList drops status and the server-assigned metadata from a
+// `kubectl get -o json` List.
+func cleanServerList(raw []byte) ([]byte, error) {
+	var list struct {
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(raw, &list); err != nil {
+		return nil, fmt.Errorf("parse MinecraftServer list: %w", err)
+	}
+	for _, it := range list.Items {
+		delete(it, "status")
+		if md, ok := it["metadata"].(map[string]any); ok {
+			for _, k := range []string{"resourceVersion", "uid", "creationTimestamp", "generation", "managedFields", "selfLink"} {
+				delete(md, k)
+			}
+		}
+	}
+	if list.Items == nil {
+		list.Items = []map[string]any{}
+	}
+	return json.MarshalIndent(map[string]any{"apiVersion": "v1", "kind": "List", "items": list.Items}, "", "  ")
+}

cmd/felis/db_test.go

@@ -0,0 +1,151 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"io"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/store"
+)
+
+func TestDBUsage(t *testing.T) {
+	for _, args := range [][]string{{"db"}, {"db", "frobnicate"}, {"db", "restore"}, {"db", "verify"}, {"db", "backup", "extra"}} {
+		var out, errBuf bytes.Buffer
+		if code := run(args, &out, &errBuf); code != 2 {
+			t.Errorf("%v: exit %d, want 2", args, code)
+		}
+		if !strings.Contains(errBuf.String(), "felis db restore") {
+			t.Errorf("%v: no usage on stderr: %q", args, errBuf.String())
+		}
+	}
+}
+
+func TestDBRestoreNeedsYes(t *testing.T) {
+	// A bundle that does not exist fails verification (1) before -yes matters;
+	// the -yes gate itself is exercised against a real bundle in internal/dbbackup
+	// and on the VM. Here: the refusal path never reaches the config or database.
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "restore", "-dir", t.TempDir(), "missing.tar"}, &out, &errBuf); code != 1 {
+		t.Fatalf("exit %d, stderr %q", code, errBuf.String())
+	}
+}
+
+func TestParseWithArg(t *testing.T) {
+	for _, args := range [][]string{{"-yes", "b.tar"}, {"b.tar", "-yes"}} {
+		fs := flag.NewFlagSet("t", flag.ContinueOnError)
+		fs.SetOutput(io.Discard)
+		yes := fs.Bool("yes", false, "")
+		arg, ok := parseWithArg(fs, args)
+		if !ok || arg != "b.tar" || !*yes {
+			t.Errorf("%v -> %q ok=%v yes=%v", args, arg, ok, *yes)
+		}
+	}
+	fs := flag.NewFlagSet("t", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	if _, ok := parseWithArg(fs, []string{"a.tar", "b.tar"}); ok {
+		t.Error("two positional arguments accepted")
+	}
+}
+
+func TestResolveBundle(t *testing.T) {
+	if got := resolveBundle("/var/lib/felis/db-backups", "felis-db-x.tar"); got != "/var/lib/felis/db-backups/felis-db-x.tar" {
+		t.Errorf("bare name -> %s", got)
+	}
+	if got := resolveBundle("/var/lib/felis/db-backups", "/root/copy.tar"); got != "/root/copy.tar" {
+		t.Errorf("path -> %s", got)
+	}
+}
+
+func TestCleanServerList(t *testing.T) {
+	raw := `{"apiVersion":"v1","kind":"List","metadata":{"resourceVersion":""},"items":[{
+		"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer",
+		"metadata":{"name":"survival","namespace":"minecraft","uid":"u","resourceVersion":"42","generation":3,
+			"creationTimestamp":"2026-09-01T00:00:00Z","managedFields":[{}],"labels":{"a":"b"}},
+		"spec":{"desiredState":"Running"},"status":{"phase":"Running"}}]}`
+	out, err := cleanServerList([]byte(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Kind  string           `json:"kind"`
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatal(err)
+	}
+	if got.Kind != "List" || len(got.Items) != 1 {
+		t.Fatalf("got %s", out)
+	}
+	it := got.Items[0]
+	if _, ok := it["status"]; ok {
+		t.Error("status kept")
+	}
+	md := it["metadata"].(map[string]any)
+	for _, k := range []string{"uid", "resourceVersion", "generation", "creationTimestamp", "managedFields"} {
+		if _, ok := md[k]; ok {
+			t.Errorf("metadata.%s kept", k)
+		}
+	}
+	if md["name"] != "survival" || md["namespace"] != "minecraft" || md["labels"] == nil {
+		t.Errorf("identity lost: %v", md)
+	}
+	if it["spec"].(map[string]any)["desiredState"] != "Running" {
+		t.Error("spec lost")
+	}
+
+	empty, err := cleanServerList([]byte(`{"items":null}`))
+	if err != nil || !strings.Contains(string(empty), `"items": []`) {
+		t.Errorf("empty list -> %s, %v", empty, err)
+	}
+	if _, err := cleanServerList([]byte("Warning: x\n{")); err == nil {
+		t.Error("garbage parsed")
+	}
+}
+
+func TestHasPending(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}, {Version: 3}}
+	if hasPending(map[int]struct{}{1: {}, 2: {}, 3: {}}, ms) {
+		t.Error("fully applied reported pending")
+	}
+	if !hasPending(map[int]struct{}{1: {}, 2: {}}, ms) {
+		t.Error("missing 3 not reported")
+	}
+}
+
+type appliedDriver struct {
+	store.Driver
+	done map[int]struct{}
+}
+
+func (d appliedDriver) EnsureVersionTable(context.Context) error { return nil }
+func (d appliedDriver) AppliedVersions(context.Context) (map[int]struct{}, error) {
+	return d.done, nil
+}
+
+func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	// An unusable URL makes an attempted backup observable as an error without
+	// any PostgreSQL tooling.
+	const badURL = "not-a-url"
+	for _, tc := range []struct {
+		name    string
+		done    map[int]struct{}
+		attempt bool
+	}{
+		{"fresh database", map[int]struct{}{}, false},
+		{"up to date", map[int]struct{}{1: {}, 2: {}}, false},
+		{"pending on a populated database", map[int]struct{}{1: {}}, true},
+	} {
+		path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, badURL, t.TempDir(), io.Discard)
+		if attempted := err != nil; attempted != tc.attempt {
+			t.Errorf("%s: attempted = %v (err %v), want %v", tc.name, attempted, err, tc.attempt)
+		}
+		if path != "" {
+			t.Errorf("%s: path = %q", tc.name, path)
+		}
+	}
+}

cmd/felis/migrate.go

@@ -7,15 +7,24 @@ import (
 	"io"
 
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
 // cmdMigrate implements `felis migrate up`: load config, open the database, and
 // apply every pending embedded migration under the advisory lock (spec §6).
+//
+// Migrations only roll forward, and some drop data (0017_drop_password), so a
+// database that already holds a schema and has migrations pending is bundled
+// first (internal/dbbackup, label pre-migrate). A failed snapshot stops the
+// upgrade; -no-backup is the explicit way past it, e.g. for an external
+// database whose server is newer than the host's pg_dump.
 func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("migrate", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	backupDir := fs.String("backup-dir", dbbackup.DefaultDir, "where the pre-migration snapshot goes")
+	noBackup := fs.Bool("no-backup", false, "apply pending migrations without snapshotting the database first")
 	// The "up" verb precedes any flags (felis migrate up -config path). Go's
 	// flag.Parse stops at the first non-flag token and would never see a flag
 	// placed after "up", silently falling back to the default -config. Pull the
@@ -48,6 +57,18 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	if !*noBackup {
+		path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database.URL, *backupDir, stderr)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis migrate: pre-migration backup failed, nothing applied: %v\n", err)
+			fmt.Fprintln(stderr, "  fix the backup, or re-run with -no-backup to migrate without one")
+			return 1
+		}
+		if path != "" {
+			fmt.Fprintf(stdout, "felis migrate: database snapshot %s\n", path)
+		}
+	}
+
 	applied, err := store.Up(ctx, drv, migrations)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis migrate: %v\n", err)
@@ -60,3 +81,33 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// preMigrateBackup bundles the database when it already carries a schema and
+// some of migrations are not applied yet, and returns the bundle's path ("" when
+// there was nothing to protect: a fresh database, or nothing pending).
+func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, dbURL, dir string, log io.Writer) (string, error) {
+	if err := drv.EnsureVersionTable(ctx); err != nil {
+		return "", fmt.Errorf("ensure version table: %w", err)
+	}
+	done, err := drv.AppliedVersions(ctx)
+	if err != nil {
+		return "", fmt.Errorf("read applied versions: %w", err)
+	}
+	if len(done) == 0 || !hasPending(done, migrations) {
+		return "", nil
+	}
+	return dbbackup.Backup(ctx, dbbackup.BackupOptions{
+		DatabaseURL: dbURL, Dir: dir, Label: dbbackup.LabelPreMigrate,
+		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir,
+		Version: resolvedVersion(), Log: log, Record: true,
+	})
+}
+
+func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
+	for _, m := range migrations {
+		if _, ok := done[m.Version]; !ok {
+			return true
+		}
+	}
+	return false
+}

cmd/felis/run.go

@@ -11,7 +11,8 @@ Usage:
   felis <command> [flags]
 
 Commands:
-  migrate up        Apply embedded database migrations under an advisory lock
+  migrate up        Apply embedded database migrations under an advisory lock (snapshots the database first)
+  db                Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check)
   operator          Run the MinecraftServer controller-manager
   api               Run the felis-api HTTP server
   nano              Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)
@@ -44,6 +45,7 @@ Run "felis <command> -h" for command-specific flags.
 // subcommand, and listing them would make the table disagree with the command list.
 var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
 	"migrate":          cmdMigrate,
+	"db":               cmdDB,
 	"operator":         cmdOperator,
 	"api":              cmdAPI,
 	"nano":             cmdNano,

cmd/felis/tui_migration.go

@@ -3,8 +3,10 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"time"
 
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
@@ -12,7 +14,7 @@ import (
 // applied count. Used by the preflight stage to self-heal a freshly bootstrapped
 // (or upgraded) database.
 func applyMigrations(dbURL string) (int, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
 	drv, err := store.Open(ctx, dbURL)
 	if err != nil {
@@ -23,6 +25,11 @@ func applyMigrations(dbURL string) (int, error) {
 	if err != nil {
 		return 0, err
 	}
+	// Same guard as `felis migrate up`: never roll a populated database forward
+	// without a snapshot to roll back to.
+	if _, err := preMigrateBackup(ctx, drv, migrations, dbURL, dbbackup.DefaultDir, io.Discard); err != nil {
+		return 0, fmt.Errorf("pre-migration backup: %w", err)
+	}
 	if _, err := store.Up(ctx, drv, migrations); err != nil {
 		return 0, err
 	}

deploy/alerts/felis-alerts.yaml

@@ -5,6 +5,7 @@
 # felis_* series come from two processes:
 #   - felis-operator pod :8080/metrics      → felis_servers_total, felis_start_duration_seconds
 #   - felis-api internal :8081/metrics      → felis_image_build_failures_total
+#   - node-exporter textfile collector      → felis_db_backup_* (felis-db-backup.timer)
 # node_* / kube_* series come from node-exporter / kube-state-metrics.
 groups:
   - name: felis.rules
@@ -67,3 +68,29 @@ groups:
           description: >-
             PostgreSQL, the control plane, the registry and game servers share one
             node; sustained memory pressure risks OOM kills.
+  - name: felis.backup.rules
+    rules:
+      - alert: FelisDBBackupStale
+        expr: time() - max(felis_db_backup_last_success_timestamp_seconds) > 26 * 3600
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "no control-plane database backup in over 26h"
+          description: >-
+            felis-db-backup.timer runs daily; the newest bundle is more than a day
+            old. Read `journalctl -u felis-db-backup` on the host, then take one now
+            with `sudo felis db backup` (troubleshooting §16).
+      - alert: FelisDBBackupMetricMissing
+        expr: absent(felis_db_backup_last_success_timestamp_seconds)
+        for: 2h
+        labels:
+          severity: warning
+        annotations:
+          summary: "database backup freshness is not being scraped"
+          description: >-
+            No felis_db_backup_last_success_timestamp_seconds series, so
+            FelisDBBackupStale cannot fire. Point node-exporter's
+            --collector.textfile.directory at the directory of
+            FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
+            (troubleshooting §16).

deploy/alerts/felis-alerts_test.yml

@@ -105,3 +105,49 @@ tests:
               description: >-
                 PostgreSQL, the control plane, the registry and game servers share one
                 node; sustained memory pressure risks OOM kills.
+  - name: database backup freshness
+    interval: 1m
+    input_series:
+      # The newest bundle was taken at t=0 and none since.
+      - series: 'felis_db_backup_last_success_timestamp_seconds{instance="node1",job="node-exporter",label="daily"}'
+        values: '0x1630'
+    alert_rule_test:
+      - eval_time: 25h
+        alertname: FelisDBBackupStale
+        exp_alerts: []
+      - eval_time: 27h
+        alertname: FelisDBBackupStale
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+            exp_annotations:
+              summary: "no control-plane database backup in over 26h"
+              description: >-
+                felis-db-backup.timer runs daily; the newest bundle is more than a day
+                old. Read `journalctl -u felis-db-backup` on the host, then take one now
+                with `sudo felis db backup` (troubleshooting §16).
+      - eval_time: 27h
+        alertname: FelisDBBackupMetricMissing
+        exp_alerts: []
+  - name: database backup freshness not scraped
+    interval: 1m
+    input_series:
+      - series: 'up{job="node-exporter"}'
+        values: '1x200'
+    alert_rule_test:
+      - eval_time: 1h
+        alertname: FelisDBBackupMetricMissing
+        exp_alerts: []
+      - eval_time: 3h
+        alertname: FelisDBBackupMetricMissing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "database backup freshness is not being scraped"
+              description: >-
+                No felis_db_backup_last_success_timestamp_seconds series, so
+                FelisDBBackupStale cannot fire. Point node-exporter's
+                --collector.textfile.directory at the directory of
+                FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
+                (troubleshooting §16).

deploy/alerts/felis-prometheusrule.yaml

@@ -72,3 +72,29 @@ spec:
             description: >-
               PostgreSQL, the control plane, the registry and game servers share one
               node; sustained memory pressure risks OOM kills.
+    - name: felis.backup.rules
+      rules:
+        - alert: FelisDBBackupStale
+          expr: time() - max(felis_db_backup_last_success_timestamp_seconds) > 26 * 3600
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "no control-plane database backup in over 26h"
+            description: >-
+              felis-db-backup.timer runs daily; the newest bundle is more than a day
+              old. Read `journalctl -u felis-db-backup` on the host, then take one now
+              with `sudo felis db backup` (troubleshooting §16).
+        - alert: FelisDBBackupMetricMissing
+          expr: absent(felis_db_backup_last_success_timestamp_seconds)
+          for: 2h
+          labels:
+            severity: warning
+          annotations:
+            summary: "database backup freshness is not being scraped"
+            description: >-
+              No felis_db_backup_last_success_timestamp_seconds series, so
+              FelisDBBackupStale cannot fire. Point node-exporter's
+              --collector.textfile.directory at the directory of
+              FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
+              (troubleshooting §16).

deploy/bootstrap.sh

@@ -140,6 +140,19 @@ FELIS_ARCHIVE_LOCAL_PATH="${FELIS_ARCHIVE_LOCAL_PATH:-/var/lib/felis/archives}"
 # from its volumeName. Left unset, no reaper CronJob renders and archives accumulate until
 # the backup PVC fills (then backups fail loudly; nothing is deleted).
 FELIS_WORLDS_HOST_PATH="${FELIS_WORLDS_HOST_PATH:-}"
+# Control-plane database backups (felis db backup): a daily timer bundles pg_dump with the
+# /etc/felis state a rebuild needs, and every upgrade that has migrations to apply snapshots
+# the database first (felis migrate up). The directory sits outside /var/lib/rancher on
+# purpose: reinstalling k3s must not take the database backups with it. Copy it off the
+# host for anything beyond "undo a bad upgrade or a mistaken delete" (troubleshooting §16).
+FELIS_DB_BACKUP_DIR="${FELIS_DB_BACKUP_DIR:-/var/lib/felis/db-backups}"
+FELIS_DB_BACKUP_KEEP="${FELIS_DB_BACKUP_KEEP:-14}"
+FELIS_DB_BACKUP_TIME="${FELIS_DB_BACKUP_TIME:-*-*-* 03:30:00}"
+# node-exporter textfile collector target; FelisDBBackupStale (deploy/alerts) reads it.
+FELIS_DB_BACKUP_METRICS="${FELIS_DB_BACKUP_METRICS:-/var/lib/node_exporter/textfile_collector/felis_db_backup.prom}"
+# 0 migrates without the pre-migration snapshot, e.g. against an external database newer
+# than this host's pg_dump. The upgrade stops if the snapshot fails and this is not set.
+FELIS_PRE_MIGRATE_BACKUP="${FELIS_PRE_MIGRATE_BACKUP:-1}"
 INSTALL_MODE="${FELIS_INSTALL_MODE:-}"
 # Loopback by default: hasJoined is an unauthenticated endpoint by protocol (Velocity
 # sends no token), so a public bind is a free auth relay — anyone can point their own
@@ -237,6 +250,8 @@ HOST_BIN="/usr/local/bin/felis"
 # version sits here, and an operator's Go at the conventional path is not ours to swap.
 GOROOT_DIR="/opt/felis/go"
 NANO_SERVICE="/etc/systemd/system/felis-nano.service"
+DB_BACKUP_SERVICE="/etc/systemd/system/felis-db-backup.service"
+DB_BACKUP_TIMER="/etc/systemd/system/felis-db-backup.timer"
 VELOCITY_DIR="/opt/felis/velocity"
 VELOCITY_USER="felis-velocity"
 VELOCITY_SERVICE="/etc/systemd/system/felis-velocity.service"
@@ -2364,13 +2379,62 @@ ensure_default_config() {
 # 8. Migrate + deploy bundle
 # ---------------------------------------------------------------------------
 run_migrations() {
+  local backup_flags=(-backup-dir "$FELIS_DB_BACKUP_DIR")
   write_felis_toml "${STATE_DIR}/felis.host.toml" "127.0.0.1"
   ensure_default_config
+  if [ "$FELIS_PRE_MIGRATE_BACKUP" = 0 ]; then
+    warn "FELIS_PRE_MIGRATE_BACKUP=0: pending migrations run without a database snapshot"
+    backup_flags=(-no-backup)
+  fi
+  # Migrations only roll forward. On an existing database with migrations pending, the
+  # binary bundles the database into FELIS_DB_BACKUP_DIR first and refuses to migrate
+  # if that fails; a fresh database has nothing to protect and is migrated directly.
   log "running database migrations (host binary -> 127.0.0.1)"
-  "$HOST_BIN" migrate up -config "${STATE_DIR}/felis.host.toml"
+  "$HOST_BIN" migrate up -config "${STATE_DIR}/felis.host.toml" "${backup_flags[@]}"
   ok "migrations applied"
 }
 
+# The daily database backup. The first run happens now, so a broken pipeline (pg_dump
+# missing, directory unwritable) shows up in this install rather than in the first
+# restore someone needs.
+install_db_backup_timer() {
+  install -d -m 0700 "$FELIS_DB_BACKUP_DIR"
+  cat > "$DB_BACKUP_SERVICE" <<EOF
+[Unit]
+Description=Felis control-plane database backup (pg_dump + /etc/felis state)
+After=postgresql.service k3s.service
+Wants=postgresql.service
+
+[Service]
+Type=oneshot
+ExecStart=${HOST_BIN} db backup -config ${STATE_DIR}/felis.host.toml -dir ${FELIS_DB_BACKUP_DIR} -label daily -keep ${FELIS_DB_BACKUP_KEEP} -metrics-file ${FELIS_DB_BACKUP_METRICS}
+Nice=10
+IOSchedulingClass=idle
+PrivateTmp=yes
+NoNewPrivileges=yes
+EOF
+  cat > "$DB_BACKUP_TIMER" <<EOF
+[Unit]
+Description=Daily Felis control-plane database backup
+
+[Timer]
+OnCalendar=${FELIS_DB_BACKUP_TIME}
+RandomizedDelaySec=15min
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+  systemctl daemon-reload
+  systemctl enable --now felis-db-backup.timer
+  if systemctl start felis-db-backup.service; then
+    ok "database backups: daily at ${FELIS_DB_BACKUP_TIME}, newest ${FELIS_DB_BACKUP_KEEP} kept in ${FELIS_DB_BACKUP_DIR} (first one taken now)"
+  else
+    journalctl -u felis-db-backup.service -n 20 --no-pager >&2 || true
+    warn "the first database backup failed (log above); fix it before relying on the daily timer: sudo systemctl start felis-db-backup.service"
+  fi
+}
+
 deploy_bundle() {
   local had_api=0 had_operator=0
   export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
@@ -2970,6 +3034,8 @@ main() {
   # After deploy_bundle: the proxy dials felis-api's internal ClusterIP, which does not
   # exist until the bundle is applied.
   install_velocity
+  # After deploy_bundle: the bundle's MinecraftServer export reads the cluster.
+  install_db_backup_timer
   mark_bootstrap_done
   summary
 }

deploy/bootstrap_test.sh

@@ -1017,6 +1017,70 @@ fi
 
 rm -f "$fnfile"
 
+# --- database backups: the pre-migration snapshot and the daily timer --------------------
+# Migrations only roll forward, so an upgrade must hand `migrate up` the snapshot directory,
+# and only an explicit FELIS_PRE_MIGRATE_BACKUP=0 may take that away.
+
+mblock="$(awk '/^run_migrations\(\) \{/,/^}/' "$BS")"
+[ -n "$mblock" ] || { echo "FAIL: no run_migrations found in $BS"; exit 1; }
+[ "$(printf '%s\n' "$mblock" | wc -l)" -lt 30 ] \
+  || { echo "FAIL: the extracted block is not run_migrations -- did its closing brace move?"; exit 1; }
+
+run_migrate() { # FELIS_PRE_MIGRATE_BACKUP
+  FELIS_PRE_MIGRATE_BACKUP="$1" FELIS_DB_BACKUP_DIR=/var/lib/felis/db-backups STATE_DIR=/etc/felis \
+    HOST_BIN=fakefelis bash -c '
+    log() { :; }; ok() { :; }; warn() { printf "WARN: %s\n" "$*"; }
+    write_felis_toml() { :; }; ensure_default_config() { :; }
+    fakefelis() { printf "RUN: %s\n" "$*"; }
+    '"$mblock"'
+    run_migrations' 2>&1
+}
+
+out="$(run_migrate 1)"
+expect "an upgrade snapshots into the backup dir" "RUN: migrate up -config /etc/felis/felis.host.toml -backup-dir /var/lib/felis/db-backups" "$out"
+out="$(run_migrate 0)"
+expect "FELIS_PRE_MIGRATE_BACKUP=0 opts out explicitly" "RUN: migrate up -config /etc/felis/felis.host.toml -no-backup" "$out"
+expect "the opt-out is loud" "WARN: FELIS_PRE_MIGRATE_BACKUP=0" "$out"
+
+tblock="$(awk '/^install_db_backup_timer\(\) \{/,/^}/' "$BS")"
+[ -n "$tblock" ] || { echo "FAIL: no install_db_backup_timer found in $BS"; exit 1; }
+[ "$(printf '%s\n' "$tblock" | wc -l)" -lt 60 ] \
+  || { echo "FAIL: the extracted block is not install_db_backup_timer -- did its closing brace move?"; exit 1; }
+
+tdir="$(mktemp -d)"
+run_timer() { # exit status of the first backup
+  FIRST="$1" DB_BACKUP_SERVICE="$tdir/felis-db-backup.service" DB_BACKUP_TIMER="$tdir/felis-db-backup.timer" \
+    FELIS_DB_BACKUP_DIR="$tdir/db-backups" FELIS_DB_BACKUP_KEEP=7 FELIS_DB_BACKUP_TIME='*-*-* 04:00:00' \
+    FELIS_DB_BACKUP_METRICS=/var/lib/node_exporter/textfile_collector/felis_db_backup.prom \
+    HOST_BIN=/usr/local/bin/felis STATE_DIR=/etc/felis bash -c '
+    ok() { printf "OK: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; }
+    systemctl() { printf "SYSTEMCTL: %s\n" "$*"; [ "$1" != start ] || return "$FIRST"; }
+    journalctl() { printf "JOURNAL: pg_dump: connection refused\n"; }
+    '"$tblock"'
+    install_db_backup_timer' 2>&1
+}
+
+out="$(run_timer 0)"
+unit="$(cat "$tdir/felis-db-backup.service")"
+timer="$(cat "$tdir/felis-db-backup.timer")"
+expect "the unit runs a daily-labelled backup with the configured retention" \
+  "ExecStart=/usr/local/bin/felis db backup -config /etc/felis/felis.host.toml -dir $tdir/db-backups -label daily -keep 7 -metrics-file /var/lib/node_exporter/textfile_collector/felis_db_backup.prom" "$unit"
+expect "the timer fires at the configured time" "OnCalendar=*-*-* 04:00:00" "$timer"
+expect "a missed run (host off at 03:30) catches up at boot" "Persistent=true" "$timer"
+expect "the timer is enabled" "SYSTEMCTL: enable --now felis-db-backup.timer" "$out"
+expect "the first backup runs during the install" "SYSTEMCTL: start felis-db-backup.service" "$out"
+expect "a working first backup is reported" "OK: database backups: daily" "$out"
+if [ "$(stat -c %a "$tdir/db-backups" 2>/dev/null || stat -f %Lp "$tdir/db-backups")" = 700 ]; then
+  echo "PASS the backup directory is private"
+else
+  echo "FAIL the backup directory must be 0700"; fails=$((fails + 1))
+fi
+
+out="$(run_timer 1)"
+expect "a failed first backup shows its log" "JOURNAL: pg_dump: connection refused" "$out"
+expect "a failed first backup is a loud warning" "WARN: the first database backup failed" "$out"
+rm -rf "$tdir"
+
 # ---------------------------------------------------------------------------------------
 if [ "$fails" -eq 0 ]; then
   echo "ALL PASS"

docs/openapi.yaml

@@ -201,6 +201,49 @@ components:
           nullable: true
           description: Window end (RFC3339, exclusive), or null when unset.
 
+    DBBackupStatus:
+      type: object
+      description: >
+        The newest control-plane database backup the host recorded
+        (internal/api/handlers_dbbackup.go dbBackupView; the record itself is
+        internal/dbbackup Status, written by `felis db backup`).
+      required: [last, stale, max_age_seconds]
+      properties:
+        last:
+          type: object
+          nullable: true
+          description: Null until the first backup has been recorded.
+          required: [at, name, label, size_bytes, dir]
+          properties:
+            at:
+              type: string
+              format: date-time
+              description: When the bundle was written.
+            name:
+              type: string
+              description: Bundle file name, felis-db-<UTC stamp>-<label>.tar.
+            label:
+              type: string
+              enum: [daily, pre-migrate, pre-restore, manual]
+            size_bytes:
+              type: integer
+              format: int64
+            felis_version:
+              type: string
+            schema_version:
+              type: integer
+              description: Newest applied migration at backup time.
+            dir:
+              type: string
+              description: Backup directory on the host.
+        stale:
+          type: boolean
+          description: True when there is no record or it is older than max_age_seconds.
+        max_age_seconds:
+          type: integer
+          format: int64
+          description: The freshness limit (26h), shared with `felis db check` and FelisDBBackupStale.
+
     PasskeyCredential:
       type: object
       description: >
@@ -2724,6 +2767,31 @@ paths:
         '403':
           $ref: '#/components/responses/Forbidden'
 
+  /api/v1/platform/db-backup:
+    get:
+      tags: [admin-updates]
+      operationId: getDBBackup
+      summary: Freshness of the newest control-plane database backup (admin).
+      description: >-
+        What the host's felis-db-backup.timer (or a manual `felis db backup`)
+        last recorded in platform_settings. last is null before the first
+        backup; stale is true then, and whenever the newest backup is older than
+        max_age_seconds. Read-only: backups run on the host, never through the API.
+      x-felis-face: [external]
+      x-felis-tier: admin
+      security: [{ accessJWT: [] }]
+      responses:
+        '200':
+          description: The newest recorded backup and whether it is stale.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DBBackupStatus'
+        '401':
+          $ref: '#/components/responses/Unauthorized'
+        '403':
+          $ref: '#/components/responses/Forbidden'
+
   /api/v1/fleet:
     get:
       tags: [admin-servers]

docs/troubleshooting.md

@@ -941,7 +941,8 @@ The series come from two processes:
 ### Alert rules
 
 `deploy/alerts/` ships ready-made rules: build failures, slow starts, node
-disk/memory thresholds, and the kubelet `DiskPressure` condition.
+disk/memory thresholds, the kubelet `DiskPressure` condition, and control-plane
+database backup freshness (§16; needs node-exporter's textfile collector).
 
 - Plain Prometheus: add `felis-alerts.yaml` to `rule_files`. Check and unit-test
   it standalone with `promtool check rules felis-alerts.yaml` and
@@ -982,6 +983,192 @@ previous ReplicaSet, whose image is normally still on the node; if the image GC
 collected it, the registry re-serves it automatically (§13b) for every tag the
 installer built — only hand-built tags need a manual re-mirror.
 
+`rollout undo` reverts the image only. The upgrade's database migrations stay
+applied; when they are the problem, restore the `pre-migrate` bundle the upgrade
+took (§16, "Roll back an upgrade that broke the database").
+
+## 16. Control-plane database backups and disaster recovery
+
+The PostgreSQL database behind felis-api holds everything that is not a world:
+accounts, passkeys, Minecraft account links, server ownership, quotas, audit
+logs, and the `world_backups` index that maps an archive (§10) back to its
+owner. Losing it orphans every world archive. It lives on the host (not in
+k3s), so it is backed up on the host too.
+
+### What runs, and where the bundles go
+
+- **`felis-db-backup.timer`** runs `felis db backup` daily at
+  `FELIS_DB_BACKUP_TIME` (default `*-*-* 03:30:00`, plus up to 15 min random
+  delay). `Persistent=true` catches up at boot after the host was off at that
+  time. The installer takes the first backup during the install, so a broken
+  pipeline shows up there. [CODE-ONLY; unit and timer content GO-TESTED in
+  `deploy/bootstrap_test.sh`]
+- **Every upgrade** (`felis migrate up`, which the installer runs) takes a
+  `pre-migrate` bundle first when the database already has data and a
+  migration is pending, and **applies nothing** if that backup fails
+  (`pre-migration backup failed, nothing applied`). [GO-TESTED]
+- **Every restore** takes a `pre-restore` bundle of the database it is about to
+  replace (skip with `-no-safety-backup`). [GO-TESTED]
+
+Bundles land in `FELIS_DB_BACKUP_DIR` (default `/var/lib/felis/db-backups`,
+mode 0700; outside `/var/lib/rancher` so a k3s reinstall cannot take them
+along). One bundle is `felis-db-<UTC stamp>-<label>.tar`:
+
+| Member | Content |
+|---|---|
+| `MANIFEST.json` | version, schema version, `pg_dump --version`, sha256 of every member |
+| `db.dump` | `pg_dump --format=custom` of the `felis` database |
+| `state/etc/felis/...` | `secrets.env` (DB password, session/forwarding secrets, registry tokens), `felis.host.toml`, `felis.pod.toml`, the `felis.toml` symlink, the panel TLS pair. `bootstrap.done` is left out on purpose |
+| `k8s/minecraftservers.json` | every MinecraftServer, status and server-side metadata stripped, ready for `kubectl apply` (best effort: when the cluster did not answer, the manifest records why) |
+
+next to a `.sha256` sidecar in `sha256sum` format. **A bundle contains the
+secrets; treat it like `/etc/felis` itself.** Retention per label: `daily` 14
+(`FELIS_DB_BACKUP_KEEP`), `pre-migrate` 10, `pre-restore` 5, `manual` never
+pruned.
+
+Installer knobs: `FELIS_DB_BACKUP_DIR`, `FELIS_DB_BACKUP_KEEP`,
+`FELIS_DB_BACKUP_TIME`, `FELIS_DB_BACKUP_METRICS` and
+`FELIS_PRE_MIGRATE_BACKUP` (below).
+
+### Is the newest backup fresh?
+
+Three places answer, all with the same 26 h limit:
+
+- The panel: **管理 → 维护与备份** shows the newest backup, its kind and size,
+  and turns red with the fix commands when it is missing or overdue (read from
+  the `db_backup_last` platform setting each backup writes).
+- `sudo felis db check` exits 1 with the reason; `sudo felis db list` shows every
+  bundle with its age.
+- Prometheus: `FelisDBBackupStale` (critical) and `FelisDBBackupMetricMissing`
+  (warning) in `deploy/alerts/`. They read
+  `felis_db_backup_last_success_timestamp_seconds`, which each daily run writes
+  to `FELIS_DB_BACKUP_METRICS` (default
+  `/var/lib/node_exporter/textfile_collector/felis_db_backup.prom`). Point
+  node-exporter's `--collector.textfile.directory` at that directory, or
+  `FelisDBBackupMetricMissing` fires after 2 h.
+
+When a backup is overdue:
+
+```
+sudo systemctl status felis-db-backup.timer          # enabled? next run?
+sudo journalctl -u felis-db-backup -n 50 --no-pager   # why the last run failed
+sudo felis db backup                                  # take one now (label manual)
+```
+
+Common failures: PostgreSQL down (`pg_dump: ... connection refused`); the
+backup directory's disk full (the half-written `.partial` is removed and the
+previous bundles stay intact); `pg_dump: server version mismatch` when an
+external database is newer than the host's client tools (install the matching
+`postgresql` client package).
+
+### Check a bundle
+
+```
+sudo felis db verify felis-db-20260924T033012Z-daily.tar   # bare names resolve in the backup dir
+sha256sum -c felis-db-20260924T033012Z-daily.tar.sha256    # on a copy, without felis
+```
+
+`verify` checks the sidecar, every member against the manifest, that nothing
+is missing or unlisted, and that the manifest comes first. It does not touch
+the database.
+
+### Restore on the same host (undo a mistake)
+
+```
+kubectl -n felis scale deployment felis-api felis-operator --replicas=0
+sudo felis db restore -yes felis-db-20260924T033012Z-daily.tar
+sudo felis migrate up -config /etc/felis/felis.host.toml
+kubectl -n felis scale deployment felis-api felis-operator --replicas=1
+```
+
+- Without `-yes`, restore prints what the bundle holds and exits 2.
+- It refuses while other clients are connected (`other clients are connected to the database (N)`)
+  and prints the scale command; `-force` overrides, for a client you know is
+  idle.
+- The replay is one transaction: it drops everything the `felis` role owns and
+  loads the dump. **Any failure rolls back and leaves the database exactly as it
+  was** (`rolled back, the database is unchanged`, with the psql and pg_restore
+  errors). [GO-TESTED]
+- The database before the restore is in the `pre-restore` bundle it names;
+  restoring that one undoes the restore.
+- `migrate up` brings an older bundle's schema up to the running release.
+  Nothing migrates at startup, so skip it only when you are rolling back to the
+  release that wrote the bundle (next section).
+- Restore replaces the database only. World data (PVCs and archives, §10, §13)
+  is not in the bundle and is not touched; a server created after the bundle
+  keeps its PVC but loses its owner row.
+
+### Roll back an upgrade that broke the database
+
+The installer's migrations only roll forward. The `pre-migrate` bundle taken by
+the upgrade is the way back:
+
+```
+kubectl -n felis scale deployment felis-api felis-operator --replicas=0
+sudo felis db list | grep pre-migrate                  # newest one is the upgrade's
+sudo felis db restore -yes <that bundle>
+kubectl -n felis rollout undo deploy/felis-api
+kubectl -n felis rollout undo deploy/felis-operator
+kubectl -n felis scale deployment felis-api felis-operator --replicas=1
+```
+
+Do **not** run `felis migrate up` here: the host binary is already the new
+release and would re-apply the migrations you are rolling back. Re-run the
+older installer version to bring the host binary back in line.
+
+### Rebuild on a new host (the old one is gone)
+
+This needs a bundle that was copied off the old host (next section).
+
+1. Check the copy: `sha256sum -c felis-db-....tar.sha256`.
+2. Put the old host's state in place **before** installing, so the installer
+   reuses the same DB password, session secret and forwarding secret (the
+   Velocity proxy and existing sessions keep working):
+
+   ```
+   sudo install -d -m 0700 /etc/felis
+   sudo tar -xpf felis-db-....tar -C / --strip-components=1 state/etc/felis
+   ```
+
+3. Run the installer as for a first install. `bootstrap.done` is not in the
+   bundle, so it takes the fresh-install path, creates the empty database with
+   the restored password and migrates it.
+4. Restore the database and bring the servers back:
+
+   ```
+   kubectl -n felis scale deployment felis-api felis-operator --replicas=0
+   sudo felis db restore -yes -no-safety-backup /path/to/felis-db-....tar
+   sudo felis migrate up -config /etc/felis/felis.host.toml
+   kubectl -n felis scale deployment felis-api felis-operator --replicas=1
+   tar -xOf felis-db-....tar k8s/minecraftservers.json | kubectl apply -f -
+   ```
+
+5. Worlds come back from their own archives (§10), which are a separate volume
+   and need their own off-host copy. Custom images built on the old host are
+   rebuilt from their submissions (§8), or re-pushed.
+
+### Keep a copy somewhere else
+
+A bundle on the same disk as the database protects against mistakes and bad
+upgrades, not against losing the disk. Copy the directory off the host on a
+schedule of your own, for example from another machine:
+
+```
+rsync -a --delete root@felis-host:/var/lib/felis/db-backups/ /backups/felis-db/
+```
+
+or with `rclone copy /var/lib/felis/db-backups remote:felis-db` from a systemd
+timer on the host. Copy the `.sha256` sidecars too; `sha256sum -c` on the far
+side proves the copy.
+
+### `FELIS_PRE_MIGRATE_BACKUP=0`
+
+Skips the pre-migration snapshot (`migrate up -no-backup`). The installer warns
+loudly when it is set. Use it only when the snapshot cannot work and you have
+another backup, e.g. an external database newer than the host's `pg_dump`.
+
+---
+
 ## Quick reference: symptom → section
 
 | Symptom | Section |
@@ -1007,3 +1194,7 @@ installer built — only hand-built tags need a manual re-mirror.
 | Node out of disk; pods evicted / ImagePullBackOff | §13b |
 | Which metric to scrape | §14 |
 | Upgrade / roll back a bad control-plane image | §15 |
+| Database backup overdue / `FelisDBBackupStale` / panel shows 从未备份 | §16 |
+| `pre-migration backup failed, nothing applied` during an upgrade | §16 |
+| Undo a mistaken change / restore the control-plane database | §16 |
+| Host lost: rebuild from a database bundle | §16 |

internal/api/api.go

@@ -565,6 +565,9 @@ func (a *API) externalAPIRoutes() []apiRoute {
 		// only — the runner/executors that consume the window are still INTEGRATION-ONLY.
 		{Method: "GET", Pattern: "/api/v1/updates/window", Admin: true, h: a.handleGetUpdateWindow},
 		{Method: "PUT", Pattern: "/api/v1/updates/window", Admin: true, h: a.handleSetUpdateWindow},
+		// Control-plane database backup freshness, as the host's felis-db-backup.timer
+		// last recorded it. Admin-tier: it names the host backup directory.
+		{Method: "GET", Pattern: "/api/v1/platform/db-backup", Admin: true, h: a.handleGetDBBackup},
 
 		// User admin (spec §7, owner-only). Every route gates on the admin Zero-Trust
 		// path AND the owner role: listing, mutating, disabling, or deleting users is

internal/api/handlers_dbbackup.go

@@ -0,0 +1,49 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+
+	"felis.lolicon.best/internal/dbbackup"
+)
+
+// Control-plane database backup freshness (admin-tier, read-only). The backups
+// themselves run on the host — felis-db-backup.timer calls `felis db backup`,
+// which records its newest success in platform_settings[dbbackup.StatusKey] — so
+// the API can report them without reaching the host's backup directory. The
+// panel shows this next to the update window: both answer "is it safe to change
+// something on this install right now".
+
+// dbBackupView is the wire shape. Last is null until the first backup has been
+// recorded; Stale is true for a missing record too, so the panel has a single
+// flag for "nobody could restore today's state".
+type dbBackupView struct {
+	Last          *dbbackup.Status `json:"last"`
+	Stale         bool             `json:"stale"`
+	MaxAgeSeconds int64            `json:"max_age_seconds"`
+}
+
+// handleGetDBBackup reports the newest recorded control-plane database backup.
+// Only a missing key reads as "never"; any other store error is a 500 so a DB
+// blip is never shown as a healthy or an absent backup.
+func (a *API) handleGetDBBackup(w http.ResponseWriter, r *http.Request) {
+	view := dbBackupView{Stale: true, MaxAgeSeconds: int64(dbbackup.StaleAfter.Seconds())}
+	raw, err := a.Repo.GetSetting(r.Context(), dbbackup.StatusKey)
+	switch {
+	case errors.Is(err, ErrNotFound):
+		writeJSON(w, http.StatusOK, view)
+		return
+	case err != nil:
+		writeError(w, r, err)
+		return
+	}
+	var st dbbackup.Status
+	if err := json.Unmarshal(raw, &st); err != nil {
+		writeError(w, r, err)
+		return
+	}
+	view.Last = &st
+	view.Stale = st.At.IsZero() || a.now().Sub(st.At) > dbbackup.StaleAfter
+	writeJSON(w, http.StatusOK, view)
+}

internal/api/handlers_dbbackup_test.go

@@ -0,0 +1,87 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/dbbackup"
+)
+
+// The panel's backup card reads one flag, stale, so these pin when it is set:
+// never backed up, too old, and not for a fresh backup. A store outage must
+// not read as either answer.
+
+func getDBBackup(t *testing.T, api *API) (int, dbBackupView) {
+	t.Helper()
+	w := do(api.ExternalHandler(), "GET", "/api/v1/platform/db-backup", "", nil)
+	var v dbBackupView
+	if w.Code == http.StatusOK {
+		if err := json.Unmarshal(w.Body.Bytes(), &v); err != nil {
+			t.Fatalf("body not JSON: %v (%s)", err, w.Body.String())
+		}
+	}
+	return w.Code, v
+}
+
+func TestDBBackupNeverRecordedIsStale(t *testing.T) {
+	api, _ := seedUpdatesAPI(t)
+	code, v := getDBBackup(t, api)
+	if code != http.StatusOK || v.Last != nil || !v.Stale {
+		t.Fatalf("never backed up = %d %+v, want 200 last=null stale", code, v)
+	}
+	if v.MaxAgeSeconds != int64(dbbackup.StaleAfter/time.Second) {
+		t.Fatalf("max_age_seconds = %d", v.MaxAgeSeconds)
+	}
+}
+
+func TestDBBackupFreshness(t *testing.T) {
+	now := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
+	for _, tc := range []struct {
+		name  string
+		age   time.Duration
+		stale bool
+	}{
+		{"taken this morning", 8 * time.Hour, false},
+		{"yesterday's, timer slightly late", 25 * time.Hour, false},
+		{"missed a day", 27 * time.Hour, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			api, repo := seedUpdatesAPI(t)
+			api.Now = func() time.Time { return now }
+			st := dbbackup.Status{At: now.Add(-tc.age), Name: "felis-db-x-daily.tar", Label: "daily", SizeBytes: 4096, SchemaVersion: 31, Dir: "/var/lib/felis/db-backups"}
+			raw, _ := json.Marshal(st)
+			repo.settings[dbbackup.StatusKey] = raw
+
+			code, v := getDBBackup(t, api)
+			if code != http.StatusOK || v.Last == nil {
+				t.Fatalf("code %d, view %+v", code, v)
+			}
+			if v.Stale != tc.stale {
+				t.Fatalf("stale = %v, want %v", v.Stale, tc.stale)
+			}
+			if v.Last.Name != st.Name || v.Last.SizeBytes != 4096 || v.Last.SchemaVersion != 31 || !v.Last.At.Equal(st.At) {
+				t.Fatalf("record not passed through: %+v", v.Last)
+			}
+		})
+	}
+}
+
+func TestDBBackupStoreOutageIsAnError(t *testing.T) {
+	api, repo := seedUpdatesAPI(t)
+	repo.failGetSetting = errors.New("connection reset")
+	if code, _ := getDBBackup(t, api); code == http.StatusOK {
+		t.Fatal("a failed settings read answered 200")
+	}
+}
+
+func TestDBBackupAdminOnly(t *testing.T) {
+	repo := newFakeRepo()
+	api := newTestAPI(repo, newFakeCluster())
+	api.External = staticExternal{p: &Principal{UserID: "u1", Role: "user"}}
+	if code, _ := getDBBackup(t, api); code != http.StatusForbidden {
+		t.Fatalf("player read = %d, want 403", code)
+	}
+}

internal/dbbackup/dbbackup.go

@@ -0,0 +1,707 @@
+// Package dbbackup takes and restores logical backups of the control-plane
+// PostgreSQL: users, passkeys, account links, server ownership, quotas, audit
+// logs and the world_backups index that maps a world archive back to its owner.
+// World archives live on their own volume (internal/archive); without this
+// database they are files nobody can be matched to.
+//
+// A backup is one bundle, felis-db-<UTC stamp>-<label>.tar, holding
+//
+//	MANIFEST.json                   what is in the bundle and each member's sha256
+//	db.dump                         pg_dump --format=custom of the felis database
+//	state/etc/felis/...             the host state a rebuild needs: secrets.env
+//	                                (the DB password, session and forwarding
+//	                                secrets, registry tokens), felis.{host,pod}.toml,
+//	                                the panel TLS pair
+//	k8s/minecraftservers.json       the MinecraftServer objects, when the cluster
+//	                                answered (best effort)
+//
+// plus a felis-db-....tar.sha256 sidecar in sha256sum format, so a copy shipped
+// off the host can be checked with `sha256sum -c` before anyone relies on it.
+//
+// Bundles are written as a hidden .partial and renamed into place after an
+// fsync, so a crash or a full disk leaves either a complete bundle or none.
+// The dump is proven readable (pg_restore --list) before the bundle counts.
+//
+// Restore is all-or-nothing: the dump is replayed through psql in a single
+// transaction that first drops everything the felis role owns, so a failure
+// anywhere leaves the database exactly as it was, and objects a newer schema
+// added do not survive to collide with the next `felis migrate up`.
+package dbbackup
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+)
+
+const (
+	// DefaultDir is where the host keeps its bundles. It is deliberately off the
+	// k3s storage tree: `rm -rf /var/lib/rancher` (a k3s reinstall) must not take
+	// the database backups with it.
+	DefaultDir = "/var/lib/felis/db-backups"
+	// DefaultStateDir is the host state directory bootstrap writes.
+	DefaultStateDir = "/etc/felis"
+
+	LabelDaily      = "daily"
+	LabelPreMigrate = "pre-migrate"
+	LabelPreRestore = "pre-restore"
+	LabelManual     = "manual"
+
+	manifestEntry = "MANIFEST.json"
+	dumpEntry     = "db.dump"
+	stateEntry    = "state"
+	serversEntry  = "k8s/minecraftservers.json"
+
+	bundlePrefix = "felis-db-"
+	bundleExt    = ".tar"
+	sumExt       = ".sha256"
+	stampLayout  = "20060102T150405Z"
+	formatV1     = 1
+)
+
+// stateSkip are files in the state directory a bundle leaves out:
+// bootstrap.done marks THIS host as installed, and carrying it to a fresh host
+// would make the installer treat a first install as an upgrade.
+var stateSkip = map[string]bool{"bootstrap.done": true}
+
+var labelRe = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`)
+
+// Tools names the PostgreSQL client binaries. Empty fields take the names on
+// PATH; tests point them at fakes.
+type Tools struct {
+	PGDump, PGRestore, PSQL string
+}
+
+func (t Tools) pgDump() string    { return orDefault(t.PGDump, "pg_dump") }
+func (t Tools) pgRestore() string { return orDefault(t.PGRestore, "pg_restore") }
+func (t Tools) psql() string      { return orDefault(t.PSQL, "psql") }
+
+func orDefault(s, d string) string {
+	if s == "" {
+		return d
+	}
+	return s
+}
+
+// BackupOptions configures one backup.
+type BackupOptions struct {
+	DatabaseURL string
+	Dir         string // bundle directory; created 0700
+	Label       string // daily | pre-migrate | pre-restore | manual | any [a-z0-9-]
+	// Keep is how many bundles of this label survive the post-backup prune;
+	// zero or less prunes nothing.
+	Keep     int
+	StateDir string // host state to bundle; "" bundles none
+	Version  string // felis build stamp, recorded in the manifest
+	Tools    Tools
+	// ExportServers returns the cluster's MinecraftServer objects as JSON. A
+	// failure is recorded in the manifest and does not fail the backup: the
+	// database is what must not be lost, and a nightly run cannot hang on a
+	// cluster that happens to be down.
+	ExportServers func(ctx context.Context) ([]byte, error)
+	// MetricsFile, when set, is rewritten after a successful backup with
+	// node-exporter textfile metrics (felis_db_backup_last_success_timestamp_seconds
+	// and felis_db_backup_last_size_bytes), which FelisDBBackupStale alerts on.
+	MetricsFile string
+	// Record stores a summary of the backup in platform_settings under
+	// StatusKey, which the admin panel reads to show how fresh the newest
+	// backup is. A failure to record is logged, not fatal.
+	Record bool
+	Now    func() time.Time
+	Log    io.Writer
+}
+
+// StatusKey is the platform_settings key Record writes; internal/api reads it.
+const StatusKey = "db_backup_last"
+
+// StaleAfter is how old the newest backup may get before it counts as missed:
+// a day plus the timer's randomized delay and a slow dump. `felis db check`,
+// the admin panel and the FelisDBBackupStale alert (deploy/alerts) share it.
+const StaleAfter = 26 * time.Hour
+
+// Status is the value stored under StatusKey.
+type Status struct {
+	At            time.Time `json:"at"`
+	Name          string    `json:"name"`
+	Label         string    `json:"label"`
+	SizeBytes     int64     `json:"size_bytes"`
+	FelisVersion  string    `json:"felis_version,omitempty"`
+	SchemaVersion int       `json:"schema_version,omitempty"`
+	Dir           string    `json:"dir"`
+}
+
+// Manifest describes a bundle.
+type Manifest struct {
+	Format        int             `json:"format"`
+	CreatedAt     time.Time       `json:"created_at"`
+	Label         string          `json:"label"`
+	FelisVersion  string          `json:"felis_version,omitempty"`
+	Database      DatabaseInfo    `json:"database"`
+	SchemaVersion int             `json:"schema_version,omitempty"`
+	PGDumpVersion string          `json:"pg_dump_version,omitempty"`
+	Files         []ManifestEntry `json:"files"`
+	// ServersError is why k8s/minecraftservers.json is absent, when it is.
+	ServersError string `json:"servers_error,omitempty"`
+}
+
+// DatabaseInfo is the connection a bundle was taken from, password excluded.
+type DatabaseInfo struct {
+	Host string `json:"host"`
+	Port string `json:"port,omitempty"`
+	Name string `json:"name"`
+	User string `json:"user"`
+}
+
+// ManifestEntry is one bundle member.
+type ManifestEntry struct {
+	Name   string `json:"name"`
+	Size   int64  `json:"size"`
+	SHA256 string `json:"sha256,omitempty"` // absent for symlinks
+	Mode   uint32 `json:"mode"`
+	Link   string `json:"link,omitempty"`
+}
+
+// Bundle is one bundle on disk.
+type Bundle struct {
+	Name    string
+	Path    string
+	Label   string
+	Created time.Time
+	Size    int64
+}
+
+// conn splits a postgres:// URL into the URL libpq should see (password
+// removed) and the password, which goes to the child through PGPASSWORD so it
+// never shows up in ps.
+type conn struct {
+	uri      string
+	password string
+	info     DatabaseInfo
+}
+
+func parseConn(raw string) (conn, error) {
+	u, err := url.Parse(raw)
+	if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") {
+		return conn{}, errors.New("database url must be a postgres:// URL")
+	}
+	c := conn{info: DatabaseInfo{Host: u.Hostname(), Port: u.Port(), Name: strings.TrimPrefix(u.Path, "/")}}
+	if u.User != nil {
+		c.info.User = u.User.Username()
+		c.password, _ = u.User.Password()
+		u.User = url.User(c.info.User)
+	}
+	if c.info.Name == "" {
+		return conn{}, errors.New("database url names no database")
+	}
+	c.uri = u.String()
+	return c, nil
+}
+
+func (c conn) env() []string {
+	env := os.Environ()
+	if c.password != "" {
+		env = append(env, "PGPASSWORD="+c.password)
+	}
+	// Never prompt: a timer-driven run with a wrong password must fail, not hang.
+	return append(env, "PGCONNECT_TIMEOUT=15")
+}
+
+func (c conn) command(ctx context.Context, bin string, args ...string) *exec.Cmd {
+	cmd := exec.CommandContext(ctx, bin, args...)
+	cmd.Env = c.env()
+	return cmd
+}
+
+// run executes cmd and folds its stderr into the error.
+func run(cmd *exec.Cmd) ([]byte, error) {
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	out, err := cmd.Output()
+	if err != nil {
+		msg := strings.TrimSpace(stderr.String())
+		if msg == "" {
+			return out, fmt.Errorf("%s: %w", filepath.Base(cmd.Path), err)
+		}
+		return out, fmt.Errorf("%s: %w: %s", filepath.Base(cmd.Path), err, msg)
+	}
+	return out, nil
+}
+
+// foreignObjectRe finds the object pg_dump could not read, and its kind.
+var foreignObjectRe = regexp.MustCompile(`permission denied for (table|sequence|schema|view|materialized view) ([^\s]+)`)
+
+// dumpHint explains the one pg_dump failure an operator causes without noticing:
+// an object created in the felis database by another role (typically postgres,
+// from a manual psql session). The dump runs as the felis role and must read
+// everything; leaving the object out would make the bundle an incomplete restore.
+func dumpHint(err error, db DatabaseInfo) string {
+	m := foreignObjectRe.FindStringSubmatch(err.Error())
+	if m == nil {
+		return ""
+	}
+	kind, name := strings.ToUpper(m[1]), m[2]
+	return fmt.Sprintf("\n  %s %s belongs to another role, so %s cannot dump it. Hand it over with\n"+
+		"    sudo -u postgres psql -d %s -c 'ALTER %s %s OWNER TO %s'\n"+
+		"  or drop it if it is a leftover.", strings.ToLower(kind), name, db.User, db.Name, kind, name, db.User)
+}
+
+// BundleName is the file name of a bundle taken at t with label.
+func BundleName(t time.Time, label string) string {
+	return bundlePrefix + t.UTC().Format(stampLayout) + "-" + label + bundleExt
+}
+
+// parseBundleName reverses BundleName.
+func parseBundleName(name string) (time.Time, string, bool) {
+	if !strings.HasPrefix(name, bundlePrefix) || !strings.HasSuffix(name, bundleExt) {
+		return time.Time{}, "", false
+	}
+	rest := strings.TrimSuffix(strings.TrimPrefix(name, bundlePrefix), bundleExt)
+	if len(rest) < len(stampLayout)+2 || rest[len(stampLayout)] != '-' {
+		return time.Time{}, "", false
+	}
+	t, err := time.Parse(stampLayout, rest[:len(stampLayout)])
+	label := rest[len(stampLayout)+1:]
+	if err != nil || !labelRe.MatchString(label) {
+		return time.Time{}, "", false
+	}
+	return t, label, true
+}
+
+// List returns the bundles in dir, newest first. A missing dir is no bundles.
+func List(dir string) ([]Bundle, error) {
+	entries, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var out []Bundle
+	for _, e := range entries {
+		if !e.Type().IsRegular() {
+			continue
+		}
+		t, label, ok := parseBundleName(e.Name())
+		if !ok {
+			continue
+		}
+		info, err := e.Info()
+		if err != nil {
+			continue
+		}
+		out = append(out, Bundle{Name: e.Name(), Path: filepath.Join(dir, e.Name()), Label: label, Created: t, Size: info.Size()})
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if !out[i].Created.Equal(out[j].Created) {
+			return out[i].Created.After(out[j].Created)
+		}
+		return out[i].Name > out[j].Name
+	})
+	return out, nil
+}
+
+// Prune deletes all but the newest keep bundles of label (and their sidecars)
+// and returns what it removed. keep <= 0 removes nothing.
+func Prune(dir, label string, keep int) ([]string, error) {
+	if keep <= 0 {
+		return nil, nil
+	}
+	all, err := List(dir)
+	if err != nil {
+		return nil, err
+	}
+	var removed []string
+	n := 0
+	for _, b := range all {
+		if b.Label != label {
+			continue
+		}
+		if n++; n <= keep {
+			continue
+		}
+		if err := os.Remove(b.Path); err != nil && !errors.Is(err, os.ErrNotExist) {
+			return removed, err
+		}
+		_ = os.Remove(b.Path + sumExt)
+		removed = append(removed, b.Name)
+	}
+	return removed, nil
+}
+
+// lockDir serializes bundle writers (the nightly timer, a pre-migrate snapshot
+// and an operator's manual run) on dir/.lock.
+func lockDir(dir string) (func(), error) {
+	f, err := os.OpenFile(filepath.Join(dir, ".lock"), os.O_CREATE|os.O_RDWR, 0o600)
+	if err != nil {
+		return nil, err
+	}
+	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
+		f.Close()
+		return nil, fmt.Errorf("lock %s: %w", dir, err)
+	}
+	return func() {
+		_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+		f.Close()
+	}, nil
+}
+
+// removeStalePartials drops leftovers of a writer that died mid-bundle. Only
+// called under the directory lock, so no live writer owns them.
+func removeStalePartials(dir string) {
+	entries, _ := os.ReadDir(dir)
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), "."+bundlePrefix) && strings.HasSuffix(e.Name(), ".partial") {
+			_ = os.Remove(filepath.Join(dir, e.Name()))
+		}
+	}
+}
+
+// Backup writes one bundle and returns its path.
+func Backup(ctx context.Context, o BackupOptions) (string, error) {
+	if !labelRe.MatchString(o.Label) {
+		return "", fmt.Errorf("invalid label %q (want [a-z0-9-], e.g. daily or manual)", o.Label)
+	}
+	c, err := parseConn(o.DatabaseURL)
+	if err != nil {
+		return "", err
+	}
+	now := time.Now
+	if o.Now != nil {
+		now = o.Now
+	}
+	logw := o.Log
+	if logw == nil {
+		logw = io.Discard
+	}
+	if err := os.MkdirAll(o.Dir, 0o700); err != nil {
+		return "", err
+	}
+	if err := os.Chmod(o.Dir, 0o700); err != nil {
+		return "", err
+	}
+	unlock, err := lockDir(o.Dir)
+	if err != nil {
+		return "", err
+	}
+	defer unlock()
+	removeStalePartials(o.Dir)
+
+	// Names have one-second resolution. A second bundle of the same label within
+	// that second (a restore retried right after a failed one takes two
+	// pre-restore snapshots) moves to the next free second; the lock makes the
+	// probe race-free.
+	created := now().UTC().Truncate(time.Second)
+	name := BundleName(created, o.Label)
+	for i := 0; ; i++ {
+		if _, err := os.Lstat(filepath.Join(o.Dir, name)); errors.Is(err, fs.ErrNotExist) {
+			break
+		} else if err != nil {
+			return "", err
+		}
+		if i == 60 {
+			return "", fmt.Errorf("no free bundle name after %s", name)
+		}
+		created = created.Add(time.Second)
+		name = BundleName(created, o.Label)
+	}
+	final := filepath.Join(o.Dir, name)
+
+	dump := filepath.Join(o.Dir, "."+name+".dump.partial")
+	defer os.Remove(dump)
+	if _, err := run(c.command(ctx, o.Tools.pgDump(), "--format=custom", "--no-password", "--file="+dump, "--dbname="+c.uri)); err != nil {
+		return "", fmt.Errorf("dump the database: %w%s", err, dumpHint(err, c.info))
+	}
+	if err := os.Chmod(dump, 0o600); err != nil {
+		return "", err
+	}
+	// A dump pg_restore cannot read is not a backup; find out now, not on the
+	// day it is needed.
+	if _, err := run(exec.CommandContext(ctx, o.Tools.pgRestore(), "--list", dump)); err != nil {
+		return "", fmt.Errorf("the dump does not read back: %w", err)
+	}
+
+	m := Manifest{Format: formatV1, CreatedAt: created, Label: o.Label, FelisVersion: o.Version, Database: c.info}
+	if out, err := run(exec.CommandContext(ctx, o.Tools.pgDump(), "--version")); err == nil {
+		m.PGDumpVersion = strings.TrimSpace(string(out))
+	}
+	m.SchemaVersion = schemaVersion(ctx, c, o.Tools)
+
+	var members []member
+	dm, err := fileMember(dumpEntry, dump)
+	if err != nil {
+		return "", err
+	}
+	members = append(members, dm)
+	if o.StateDir != "" {
+		sm, err := stateMembers(o.StateDir)
+		if err != nil {
+			return "", fmt.Errorf("read host state: %w", err)
+		}
+		members = append(members, sm...)
+	}
+	if o.ExportServers != nil {
+		if data, err := o.ExportServers(ctx); err != nil {
+			m.ServersError = err.Error()
+			fmt.Fprintf(logw, "felis db backup: MinecraftServer objects not included: %v\n", err)
+		} else {
+			members = append(members, bytesMember(serversEntry, data))
+		}
+	}
+	for _, mb := range members {
+		m.Files = append(m.Files, mb.entry)
+	}
+
+	sum, err := writeBundle(o.Dir, final, m, members)
+	if err != nil {
+		return "", err
+	}
+	if err := writeFileAtomic(final+sumExt, []byte(sum+"  "+name+"\n")); err != nil {
+		return "", fmt.Errorf("write checksum: %w", err)
+	}
+	if info, err := os.Stat(final); err == nil {
+		st := Status{At: created, Name: name, Label: o.Label, SizeBytes: info.Size(),
+			FelisVersion: o.Version, SchemaVersion: m.SchemaVersion, Dir: o.Dir}
+		if o.Record {
+			if err := record(ctx, c, o.Tools, st); err != nil {
+				fmt.Fprintf(logw, "felis db backup: record the backup for the panel: %v\n", err)
+			}
+		}
+		if o.MetricsFile != "" {
+			if err := writeMetrics(o.MetricsFile, st); err != nil {
+				fmt.Fprintf(logw, "felis db backup: write %s: %v\n", o.MetricsFile, err)
+			}
+		}
+	}
+	if removed, err := Prune(o.Dir, o.Label, o.Keep); err != nil {
+		fmt.Fprintf(logw, "felis db backup: prune old %s bundles: %v\n", o.Label, err)
+	} else if len(removed) > 0 {
+		fmt.Fprintf(logw, "felis db backup: pruned %d old %s bundle(s)\n", len(removed), o.Label)
+	}
+	return final, nil
+}
+
+// record upserts st into platform_settings. The JSON travels as a psql
+// variable, quoted by psql itself, over stdin (-c does not interpolate).
+func record(ctx context.Context, c conn, t Tools, st Status) error {
+	v, err := json.Marshal(st)
+	if err != nil {
+		return err
+	}
+	cmd := c.command(ctx, t.psql(), "-X", "-q", "-w", "-v", "ON_ERROR_STOP=1", "-v", "v="+string(v), "-d", c.uri)
+	cmd.Stdin = strings.NewReader("INSERT INTO platform_settings (key, value) VALUES ('" + StatusKey + "', :'v'::jsonb)\n" +
+		"ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now();\n")
+	_, err = run(cmd)
+	return err
+}
+
+// writeMetrics rewrites a node-exporter textfile-collector file for st.
+func writeMetrics(path string, st Status) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return err
+	}
+	body := fmt.Sprintf(`# HELP felis_db_backup_last_success_timestamp_seconds Unix time of the newest successful control-plane database backup.
+# TYPE felis_db_backup_last_success_timestamp_seconds gauge
+felis_db_backup_last_success_timestamp_seconds{label=%q} %d
+# HELP felis_db_backup_last_size_bytes Size of the newest control-plane database backup bundle.
+# TYPE felis_db_backup_last_size_bytes gauge
+felis_db_backup_last_size_bytes{label=%q} %d
+`, st.Label, st.At.Unix(), st.Label, st.SizeBytes)
+	if err := writeFileAtomic(path, []byte(body)); err != nil {
+		return err
+	}
+	// Read by node-exporter, which usually runs unprivileged.
+	return os.Chmod(path, 0o644)
+}
+
+// schemaVersion reads the newest applied migration, or 0 when it cannot.
+func schemaVersion(ctx context.Context, c conn, t Tools) int {
+	out, err := run(c.command(ctx, t.psql(), "-X", "-q", "-t", "-A", "-w", "-d", c.uri,
+		"-c", "SELECT coalesce(max(version), 0) FROM schema_migrations"))
+	if err != nil {
+		return 0
+	}
+	v, _ := strconv.Atoi(strings.TrimSpace(string(out)))
+	return v
+}
+
+// member is one bundle entry: a file on disk, bytes, or a symlink.
+type member struct {
+	entry ManifestEntry
+	path  string
+	data  []byte
+}
+
+func fileMember(name, path string) (member, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return member{}, err
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return member{}, err
+	}
+	h := sha256.New()
+	n, err := io.Copy(h, f)
+	if err != nil {
+		return member{}, err
+	}
+	return member{entry: ManifestEntry{Name: name, Size: n, SHA256: hex.EncodeToString(h.Sum(nil)), Mode: uint32(info.Mode().Perm())}, path: path}, nil
+}
+
+func bytesMember(name string, data []byte) member {
+	s := sha256.Sum256(data)
+	return member{entry: ManifestEntry{Name: name, Size: int64(len(data)), SHA256: hex.EncodeToString(s[:]), Mode: 0o600}, data: data}
+}
+
+// stateMembers bundles the regular files and symlinks directly in dir, under
+// state/<absolute dir>/. Subdirectories are not state bootstrap writes.
+func stateMembers(dir string) ([]member, error) {
+	abs, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, err
+	}
+	entries, err := os.ReadDir(abs)
+	if err != nil {
+		return nil, err
+	}
+	prefix := stateEntry + filepath.ToSlash(abs) + "/"
+	var out []member
+	for _, e := range entries {
+		if stateSkip[e.Name()] {
+			continue
+		}
+		p := filepath.Join(abs, e.Name())
+		switch {
+		case e.Type()&os.ModeSymlink != 0:
+			target, err := os.Readlink(p)
+			if err != nil {
+				return nil, err
+			}
+			out = append(out, member{entry: ManifestEntry{Name: prefix + e.Name(), Mode: 0o777, Link: target}})
+		case e.Type().IsRegular():
+			m, err := fileMember(prefix+e.Name(), p)
+			if err != nil {
+				return nil, err
+			}
+			out = append(out, m)
+		}
+	}
+	return out, nil
+}
+
+// writeBundle writes MANIFEST.json and the members to final via a .partial and
+// returns the bundle's sha256.
+func writeBundle(dir, final string, m Manifest, members []member) (string, error) {
+	manifest, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return "", err
+	}
+	partial := filepath.Join(dir, "."+filepath.Base(final)+".partial")
+	f, err := os.OpenFile(partial, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
+	if err != nil {
+		return "", err
+	}
+	defer os.Remove(partial)
+	h := sha256.New()
+	if err := writeTar(io.MultiWriter(f, h), m.CreatedAt, manifest, members); err != nil {
+		f.Close()
+		return "", fmt.Errorf("write bundle: %w", err)
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		return "", err
+	}
+	if err := f.Close(); err != nil {
+		return "", err
+	}
+	if err := os.Rename(partial, final); err != nil {
+		return "", err
+	}
+	if err := syncDir(dir); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+func writeFileAtomic(path string, data []byte) error {
+	dir := filepath.Dir(path)
+	partial := filepath.Join(dir, "."+filepath.Base(path)+".partial")
+	defer os.Remove(partial)
+	f, err := os.OpenFile(partial, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
+	if err != nil {
+		return err
+	}
+	if _, err := f.Write(data); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	if err := os.Rename(partial, path); err != nil {
+		return err
+	}
+	return syncDir(dir)
+}
+
+func syncDir(dir string) error {
+	d, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	return d.Sync()
+}
+
+// Age renders a bundle's age for a human: seconds under a minute, then
+// minutes, then days and hours past two days.
+func Age(d time.Duration) string {
+	switch {
+	case d < 0:
+		return "0s"
+	case d < time.Minute:
+		return d.Truncate(time.Second).String()
+	case d < 48*time.Hour:
+		return strings.TrimSuffix(d.Truncate(time.Minute).String(), "0s")
+	default:
+		return fmt.Sprintf("%dd%dh", d/(24*time.Hour), d%(24*time.Hour)/time.Hour)
+	}
+}
+
+// Check reports the newest bundle in dir and an error when there is none or it
+// is older than maxAge.
+func Check(dir string, maxAge time.Duration, now time.Time) (*Bundle, error) {
+	all, err := List(dir)
+	if err != nil {
+		return nil, err
+	}
+	if len(all) == 0 {
+		return nil, fmt.Errorf("no database backup in %s", dir)
+	}
+	newest := all[0]
+	if age := now.Sub(newest.Created); age > maxAge {
+		return &newest, fmt.Errorf("newest database backup %s is %s old (limit %s)", newest.Name, Age(age), maxAge)
+	}
+	return &newest, nil
+}

internal/dbbackup/dbbackup_test.go

@@ -0,0 +1,512 @@
+package dbbackup
+
+import (
+	"archive/tar"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+)
+
+// The PostgreSQL client tools are faked with shell scripts over a one-file
+// "database" ($FAKE_DIR/db). The fake psql only writes the replayed rows back
+// when its input ends in COMMIT, which is how a real server treats an open
+// transaction at disconnect, so rollback-on-failure is observable.
+
+const fakePGDump = `#!/bin/sh
+D="$FAKE_DIR"
+if [ "$1" = "--version" ]; then echo "pg_dump (PostgreSQL) 13.23"; exit 0; fi
+printf '%s\n' "$*" > "$D/pg_dump.args"
+printf '%s' "$PGPASSWORD" > "$D/pg_dump.password"
+[ -f "$D/dump_fail" ] && { echo "pg_dump: error: connection refused" >&2; exit 1; }
+[ -f "$D/dump_denied" ] && { printf 'pg_dump: error: query failed: ERROR:  permission denied for table servers_preserve\npg_dump: error: query was: LOCK TABLE public.servers_preserve IN ACCESS SHARE MODE\n' >&2; exit 1; }
+for a in "$@"; do case "$a" in --file=*) out="${a#--file=}";; esac; done
+if [ -f "$D/dump_garbage" ]; then echo garbage > "$out"; exit 0; fi
+{ printf 'PGDMP\n'; cat "$D/db"; } > "$out"
+`
+
+const fakePGRestore = `#!/bin/sh
+D="$FAKE_DIR"
+list=0
+for a in "$@"; do case "$a" in --list) list=1;; esac; last="$a"; done
+head -n 1 "$last" | grep -q '^PGDMP$' || { echo "pg_restore: error: input file does not appear to be a valid archive" >&2; exit 1; }
+[ $list = 1 ] && { echo "; Archive created"; exit 0; }
+echo "-- restore script"
+tail -n +2 "$last" | sed 's/^/DATA /'
+[ -f "$D/restore_fail" ] && { echo "pg_restore: error: could not read input" >&2; exit 1; }
+exit 0
+`
+
+const fakePSQL = `#!/bin/sh
+D="$FAKE_DIR"
+printf '%s\n' "$@" > "$D/psql.args"
+q=""
+while [ $# -gt 0 ]; do case "$1" in -c) q="$2"; shift;; esac; shift; done
+if [ -n "$q" ]; then
+  case "$q" in
+    *schema_migrations*) echo 21;;
+    *pg_stat_activity*) cat "$D/clients" 2>/dev/null || echo 0;;
+  esac
+  exit 0
+fi
+cat > "$D/psql.in"
+# The freshness record is its own psql call; keep it apart from the replay.
+if grep -q platform_settings "$D/psql.in"; then
+  mv "$D/psql.in" "$D/record.stdin"; cp "$D/psql.args" "$D/record.args"; exit 0
+fi
+mv "$D/psql.in" "$D/psql.stdin"
+[ -f "$D/psql_fail" ] && { echo 'ERROR:  relation "x" already exists' >&2; exit 3; }
+tail -n 1 "$D/psql.stdin" | grep -q '^COMMIT;$' || exit 0
+grep '^DATA ' "$D/psql.stdin" | sed 's/^DATA //' > "$D/db"
+`
+
+const testURL = "postgres://felis:[email protected]:5432/felis?sslmode=disable"
+
+type fakePG struct {
+	dir   string
+	tools Tools
+}
+
+func newFakePG(t *testing.T, db string) *fakePG {
+	t.Helper()
+	dir := t.TempDir()
+	write := func(name, body string) string {
+		p := filepath.Join(dir, name)
+		if err := os.WriteFile(p, []byte(body), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		return p
+	}
+	f := &fakePG{dir: dir, tools: Tools{
+		PGDump: write("pg_dump", fakePGDump), PGRestore: write("pg_restore", fakePGRestore), PSQL: write("psql", fakePSQL),
+	}}
+	f.setDB(t, db)
+	t.Setenv("FAKE_DIR", dir)
+	return f
+}
+
+func (f *fakePG) setDB(t *testing.T, s string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(f.dir, "db"), []byte(s), 0o600); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (f *fakePG) db(t *testing.T) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(f.dir, "db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
+func (f *fakePG) flag(t *testing.T, name, content string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(f.dir, name), []byte(content), 0o600); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func stateDir(t *testing.T) string {
+	t.Helper()
+	d := t.TempDir()
+	for name, body := range map[string]string{
+		"secrets.env":     "DB_PASSWORD=s3cret-pw\n",
+		"felis.host.toml": "[database]\n",
+		"bootstrap.done":  "2026-09-24T00:00:00Z\n",
+	} {
+		if err := os.WriteFile(filepath.Join(d, name), []byte(body), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.Symlink(filepath.Join(d, "felis.host.toml"), filepath.Join(d, "felis.toml")); err != nil {
+		t.Fatal(err)
+	}
+	return d
+}
+
+var t0 = time.Date(2026, 9, 24, 3, 30, 0, 0, time.UTC)
+
+// recorded is the Status of the last freshness record, or the zero Status.
+func (pg *fakePG) recorded(t *testing.T) Status {
+	t.Helper()
+	args, _ := os.ReadFile(filepath.Join(pg.dir, "record.args"))
+	var st Status
+	for _, a := range strings.Split(string(args), "\n") {
+		if v, ok := strings.CutPrefix(a, "v="); ok {
+			if err := json.Unmarshal([]byte(v), &st); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	return st
+}
+
+func at(t time.Time) func() time.Time { return func() time.Time { return t } }
+
+func TestBackupWritesAVerifiableBundle(t *testing.T) {
+	pg := newFakePG(t, "users: alice\n")
+	dir := filepath.Join(t.TempDir(), "db-backups")
+	state := stateDir(t)
+	path, err := Backup(context.Background(), BackupOptions{
+		DatabaseURL: testURL, Dir: dir, Label: LabelDaily, StateDir: state, Version: "v1.2.3",
+		Tools: pg.tools, Now: at(t0),
+		ExportServers: func(context.Context) ([]byte, error) { return []byte(`{"kind":"List","items":[]}`), nil },
+	})
+	if err != nil {
+		t.Fatalf("Backup: %v", err)
+	}
+	if want := filepath.Join(dir, "felis-db-20260924T033000Z-daily.tar"); path != want {
+		t.Fatalf("path = %s, want %s", path, want)
+	}
+	for p, mode := range map[string]os.FileMode{dir: 0o700, path: 0o600, path + ".sha256": 0o600} {
+		info, err := os.Stat(p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if info.Mode().Perm() != mode {
+			t.Errorf("%s mode = %v, want %v", p, info.Mode().Perm(), mode)
+		}
+	}
+
+	// The password reaches pg_dump through the environment, never argv.
+	args, _ := os.ReadFile(filepath.Join(pg.dir, "pg_dump.args"))
+	if strings.Contains(string(args), "s3cret-pw") {
+		t.Errorf("password on the pg_dump command line: %s", args)
+	}
+	if pw, _ := os.ReadFile(filepath.Join(pg.dir, "pg_dump.password")); string(pw) != "s3cret-pw" {
+		t.Errorf("PGPASSWORD = %q", pw)
+	}
+
+	m, err := Verify(path)
+	if err != nil {
+		t.Fatalf("Verify: %v", err)
+	}
+	if m.Label != LabelDaily || m.FelisVersion != "v1.2.3" || m.SchemaVersion != 21 || !m.CreatedAt.Equal(t0) {
+		t.Errorf("manifest = %+v", m)
+	}
+	if m.Database != (DatabaseInfo{Host: "127.0.0.1", Port: "5432", Name: "felis", User: "felis"}) {
+		t.Errorf("database = %+v", m.Database)
+	}
+	if !strings.Contains(m.PGDumpVersion, "13.23") {
+		t.Errorf("pg_dump version = %q", m.PGDumpVersion)
+	}
+	var names []string
+	for _, f := range m.Files {
+		names = append(names, f.Name)
+	}
+	abs, _ := filepath.Abs(state)
+	prefix := "state" + filepath.ToSlash(abs) + "/"
+	want := []string{"db.dump", prefix + "felis.host.toml", prefix + "felis.toml", prefix + "secrets.env", "k8s/minecraftservers.json"}
+	if !slices.Equal(names, want) {
+		t.Errorf("members = %v, want %v (bootstrap.done left out)", names, want)
+	}
+	for _, f := range m.Files {
+		if f.Name == prefix+"felis.toml" && f.Link != filepath.Join(state, "felis.host.toml") {
+			t.Errorf("symlink recorded as %+v", f)
+		}
+	}
+
+	// No scratch or partial files survive a successful run.
+	entries, _ := os.ReadDir(dir)
+	for _, e := range entries {
+		if strings.HasSuffix(e.Name(), ".partial") || strings.HasSuffix(e.Name(), ".dump") {
+			t.Errorf("leftover %s", e.Name())
+		}
+	}
+}
+
+func TestBackupRecordsFreshness(t *testing.T) {
+	pg := newFakePG(t, "x\n")
+	dir := t.TempDir()
+	metrics := filepath.Join(t.TempDir(), "textfile", "felis_db_backup.prom")
+	path, err := Backup(context.Background(), BackupOptions{
+		DatabaseURL: testURL, Dir: dir, Label: LabelDaily, Version: "v9", Tools: pg.tools, Now: at(t0),
+		Record: true, MetricsFile: metrics,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	stdin, _ := os.ReadFile(filepath.Join(pg.dir, "record.stdin"))
+	if !strings.Contains(string(stdin), "INSERT INTO platform_settings") || !strings.Contains(string(stdin), ":'v'::jsonb") {
+		t.Fatalf("record SQL = %q", stdin)
+	}
+	st := pg.recorded(t)
+	info, _ := os.Stat(path)
+	if st.Name != filepath.Base(path) || st.Label != LabelDaily || !st.At.Equal(t0) || st.SizeBytes != info.Size() || st.SchemaVersion != 21 {
+		t.Fatalf("recorded %+v", st)
+	}
+
+	prom, err := os.ReadFile(metrics)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := fmt.Sprintf("felis_db_backup_last_success_timestamp_seconds{label=\"daily\"} %d\n", t0.Unix())
+	if !strings.Contains(string(prom), want) {
+		t.Fatalf("metrics = %s, want %s", prom, want)
+	}
+	if fi, _ := os.Stat(metrics); fi.Mode().Perm() != 0o644 {
+		t.Errorf("metrics mode %v", fi.Mode().Perm())
+	}
+}
+
+func TestBackupRecordsAClusterThatDidNotAnswer(t *testing.T) {
+	pg := newFakePG(t, "x\n")
+	dir := t.TempDir()
+	path, err := Backup(context.Background(), BackupOptions{
+		DatabaseURL: testURL, Dir: dir, Label: LabelDaily, Tools: pg.tools, Now: at(t0),
+		ExportServers: func(context.Context) ([]byte, error) { return nil, errors.New("connection refused") },
+	})
+	if err != nil {
+		t.Fatalf("a cluster outage must not fail the database backup: %v", err)
+	}
+	m, err := Verify(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if m.ServersError != "connection refused" || len(m.Files) != 1 {
+		t.Errorf("manifest = %+v", m)
+	}
+}
+
+func TestBackupsWithinOneSecondGetDistinctNames(t *testing.T) {
+	pg := newFakePG(t, "x\n")
+	dir := t.TempDir()
+	o := BackupOptions{DatabaseURL: testURL, Dir: dir, Label: LabelPreRestore, Tools: pg.tools, Now: at(t0)}
+	first, err := Backup(context.Background(), o)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := Backup(context.Background(), o)
+	if err != nil {
+		t.Fatalf("second backup in the same second: %v", err)
+	}
+	if first == second {
+		t.Fatalf("both backups wrote %s", first)
+	}
+	all, err := List(dir)
+	if err != nil || len(all) != 2 || !all[0].Created.After(all[1].Created) {
+		t.Fatalf("list = %+v, %v", all, err)
+	}
+	if _, err := Verify(second); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestBackupFailures(t *testing.T) {
+	for _, tc := range []struct {
+		flag, want string
+	}{
+		{"dump_fail", "connection refused"},
+		{"dump_garbage", "does not read back"},
+		// An object another role created in the database: the error names the fix.
+		{"dump_denied", "sudo -u postgres psql -d felis -c 'ALTER TABLE servers_preserve OWNER TO felis'"},
+	} {
+		t.Run(tc.flag, func(t *testing.T) {
+			pg := newFakePG(t, "x\n")
+			pg.flag(t, tc.flag, "")
+			dir := t.TempDir()
+			_, err := Backup(context.Background(), BackupOptions{DatabaseURL: testURL, Dir: dir, Label: LabelDaily, Tools: pg.tools, Now: at(t0)})
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err = %v, want %q", err, tc.want)
+			}
+			entries, _ := os.ReadDir(dir)
+			for _, e := range entries {
+				if e.Name() != ".lock" {
+					t.Errorf("a failed backup left %s behind", e.Name())
+				}
+			}
+		})
+	}
+
+	t.Run("bad label and url", func(t *testing.T) {
+		pg := newFakePG(t, "x\n")
+		if _, err := Backup(context.Background(), BackupOptions{DatabaseURL: testURL, Dir: t.TempDir(), Label: "../x", Tools: pg.tools}); err == nil {
+			t.Error("label with a path separator accepted")
+		}
+		if _, err := Backup(context.Background(), BackupOptions{DatabaseURL: "host=x dbname=y", Dir: t.TempDir(), Label: "daily", Tools: pg.tools}); err == nil {
+			t.Error("non-URL database accepted")
+		}
+	})
+
+	t.Run("stale partials from a crashed run are cleared", func(t *testing.T) {
+		pg := newFakePG(t, "x\n")
+		dir := t.TempDir()
+		stale := filepath.Join(dir, ".felis-db-20260101T000000Z-daily.tar.partial")
+		if err := os.WriteFile(stale, []byte("half"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := Backup(context.Background(), BackupOptions{DatabaseURL: testURL, Dir: dir, Label: LabelDaily, Tools: pg.tools, Now: at(t0)}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := os.Stat(stale); !errors.Is(err, os.ErrNotExist) {
+			t.Error("stale partial survived")
+		}
+	})
+}
+
+func TestVerifyCatchesCorruption(t *testing.T) {
+	pg := newFakePG(t, strings.Repeat("row\n", 64))
+	dir := t.TempDir()
+	path, err := Backup(context.Background(), BackupOptions{DatabaseURL: testURL, Dir: dir, Label: LabelManual, Tools: pg.tools, Now: at(t0)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, _ := os.ReadFile(path)
+	i := strings.Index(string(raw), "PGDMP")
+	raw[i+10] ^= 0x20
+	if err := os.WriteFile(path, raw, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Verify(path); err == nil || !strings.Contains(err.Error(), "corrupt") {
+		t.Fatalf("flipped dump byte: err = %v, want member corrupt", err)
+	}
+
+	// A bundle intact inside but not the one the sidecar vouches for.
+	raw[i+10] ^= 0x20
+	raw = append(raw, make([]byte, 512)...)
+	if err := os.WriteFile(path, raw, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Verify(path); err == nil || !strings.Contains(err.Error(), ".sha256") {
+		t.Fatalf("sidecar mismatch: err = %v", err)
+	}
+
+	junk := filepath.Join(dir, "junk.tar")
+	if err := os.WriteFile(junk, []byte("not a tar"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Verify(junk); err == nil {
+		t.Fatal("junk verified")
+	}
+}
+
+func TestVerifyRejectsUnlistedMember(t *testing.T) {
+	pg := newFakePG(t, "x\n")
+	dir := t.TempDir()
+	path, err := Backup(context.Background(), BackupOptions{DatabaseURL: testURL, Dir: dir, Label: LabelManual, Tools: pg.tools, Now: at(t0)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Re-pack with an extra member the manifest does not list.
+	src, _ := os.Open(path)
+	defer src.Close()
+	out := filepath.Join(dir, "felis-db-20260924T040000Z-manual.tar")
+	dst, _ := os.Create(out)
+	tr, tw := tar.NewReader(src), tar.NewWriter(dst)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = tw.WriteHeader(h)
+		_, _ = io.Copy(tw, tr)
+	}
+	_ = tw.WriteHeader(&tar.Header{Name: "state/etc/cron.d/evil", Mode: 0o644, Size: 1, Typeflag: tar.TypeReg})
+	_, _ = tw.Write([]byte("x"))
+	_ = tw.Close()
+	_ = dst.Close()
+	if _, err := Verify(out); err == nil || !strings.Contains(err.Error(), "not in the manifest") {
+		t.Fatalf("err = %v", err)
+	}
+}
+
+func TestListPruneCheck(t *testing.T) {
+	pg := newFakePG(t, "x\n")
+	dir := t.TempDir()
+	backup := func(label string, when time.Time, keep int) {
+		t.Helper()
+		if _, err := Backup(context.Background(), BackupOptions{DatabaseURL: testURL, Dir: dir, Label: label, Keep: keep, Tools: pg.tools, Now: at(when)}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	backup(LabelManual, t0.Add(-100*time.Hour), 0)
+	for i := range 5 {
+		backup(LabelDaily, t0.Add(time.Duration(i-5)*24*time.Hour), 3)
+	}
+	backup(LabelPreMigrate, t0.Add(-time.Hour), 3)
+
+	all, err := List(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for _, b := range all {
+		got = append(got, b.Label+"@"+b.Created.Format("0102T15"))
+	}
+	want := []string{"pre-migrate@0924T02", "daily@0923T03", "daily@0922T03", "daily@0921T03", "manual@0919T23"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("List = %v, want %v (dailies pruned to 3, others untouched)", got, want)
+	}
+	if _, err := os.Stat(filepath.Join(dir, BundleName(t0.Add(-5*24*time.Hour), LabelDaily)+".sha256")); !errors.Is(err, os.ErrNotExist) {
+		t.Error("a pruned bundle's sidecar survived")
+	}
+
+	if b, err := Check(dir, 26*time.Hour, t0); err != nil || b.Label != LabelPreMigrate {
+		t.Errorf("Check fresh = %v, %v", b, err)
+	}
+	if _, err := Check(dir, 26*time.Hour, t0.Add(30*time.Hour)); err == nil {
+		t.Error("Check accepted a 31h-old newest bundle")
+	}
+	if _, err := Check(filepath.Join(dir, "none"), time.Hour, t0); err == nil {
+		t.Error("Check accepted an empty directory")
+	}
+}
+
+func TestParseBundleName(t *testing.T) {
+	for name, ok := range map[string]bool{
+		"felis-db-20260924T033000Z-daily.tar":        true,
+		"felis-db-20260924T033000Z-pre-migrate.tar":  true,
+		"felis-db-20260924T033000Z-.tar":             false,
+		"felis-db-20260924T033000Z-Daily.tar":        false,
+		"felis-db-2026-daily.tar":                    false,
+		"felis-db-20260924T033000Z-daily.tar.sha256": false,
+		"other.tar": false,
+	} {
+		if _, _, got := parseBundleName(name); got != ok {
+			t.Errorf("parseBundleName(%q) ok = %v, want %v", name, got, ok)
+		}
+	}
+}
+
+func TestParseConnStripsPassword(t *testing.T) {
+	c, err := parseConn(testURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(c.uri, "s3cret") || c.password != "s3cret-pw" {
+		t.Fatalf("conn = %+v", c)
+	}
+	if !strings.Contains(c.uri, "sslmode=disable") || !strings.HasPrefix(c.uri, "postgres://[email protected]:5432/felis") {
+		t.Fatalf("uri = %s", c.uri)
+	}
+	if _, err := parseConn("postgres://127.0.0.1/"); err == nil {
+		t.Fatal("URL without a database accepted")
+	}
+}
+
+func TestAge(t *testing.T) {
+	for d, want := range map[time.Duration]string{
+		-time.Second:                 "0s",
+		44 * time.Second:             "44s",
+		90 * time.Second:             "1m",
+		26*time.Hour + 5*time.Minute: "26h5m",
+		3*24*time.Hour + 4*time.Hour: "3d4h",
+		2*time.Hour + 30*time.Second: "2h0m",
+	} {
+		if got := Age(d); got != want {
+			t.Errorf("Age(%s) = %q, want %q", d, got, want)
+		}
+	}
+}

internal/dbbackup/restore.go

@@ -0,0 +1,346 @@
+package dbbackup
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// writeTar streams MANIFEST.json and then every member.
+func writeTar(w io.Writer, mtime time.Time, manifest []byte, members []member) error {
+	tw := tar.NewWriter(w)
+	hdr := func(name string, mode uint32, size int64) *tar.Header {
+		return &tar.Header{Name: name, Mode: int64(mode), Size: size, ModTime: mtime, Typeflag: tar.TypeReg, Format: tar.FormatPAX}
+	}
+	if err := tw.WriteHeader(hdr(manifestEntry, 0o600, int64(len(manifest)))); err != nil {
+		return err
+	}
+	if _, err := tw.Write(manifest); err != nil {
+		return err
+	}
+	for _, m := range members {
+		if m.entry.Link != "" {
+			if err := tw.WriteHeader(&tar.Header{Name: m.entry.Name, Linkname: m.entry.Link, Mode: 0o777,
+				ModTime: mtime, Typeflag: tar.TypeSymlink, Format: tar.FormatPAX}); err != nil {
+				return err
+			}
+			continue
+		}
+		if err := tw.WriteHeader(hdr(m.entry.Name, m.entry.Mode, m.entry.Size)); err != nil {
+			return err
+		}
+		if m.data != nil {
+			if _, err := tw.Write(m.data); err != nil {
+				return err
+			}
+			continue
+		}
+		f, err := os.Open(m.path)
+		if err != nil {
+			return err
+		}
+		// CopyN: the size went into the header from the hash pass; a file that
+		// changed since is an error here, not a silently short member.
+		_, err = io.CopyN(tw, f, m.entry.Size)
+		f.Close()
+		if err != nil {
+			return fmt.Errorf("%s: %w", m.entry.Name, err)
+		}
+	}
+	return tw.Close()
+}
+
+// Verify reads a whole bundle, checks it against its sidecar (when present)
+// and every member against the manifest, and returns the manifest.
+func Verify(path string) (Manifest, error) {
+	return readBundle(path, nil)
+}
+
+// readBundle is Verify that also copies db.dump to dumpTo when non-nil.
+func readBundle(path string, dumpTo io.Writer) (Manifest, error) {
+	var m Manifest
+	f, err := os.Open(path)
+	if err != nil {
+		return m, err
+	}
+	defer f.Close()
+	whole := sha256.New()
+	tr := tar.NewReader(io.TeeReader(f, whole))
+
+	first, err := tr.Next()
+	if err != nil || first.Name != manifestEntry {
+		return m, fmt.Errorf("%s is not a felis database bundle (no %s)", filepath.Base(path), manifestEntry)
+	}
+	raw, err := io.ReadAll(io.LimitReader(tr, 1<<20))
+	if err != nil {
+		return m, err
+	}
+	if err := json.Unmarshal(raw, &m); err != nil {
+		return m, fmt.Errorf("read %s: %w", manifestEntry, err)
+	}
+	if m.Format != formatV1 {
+		return m, fmt.Errorf("bundle format %d is not one this felis reads (want %d)", m.Format, formatV1)
+	}
+	want := map[string]ManifestEntry{}
+	for _, e := range m.Files {
+		want[e.Name] = e
+	}
+	seen := map[string]bool{}
+	for {
+		h, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return m, fmt.Errorf("read bundle: %w", err)
+		}
+		e, ok := want[h.Name]
+		if !ok {
+			return m, fmt.Errorf("bundle member %s is not in the manifest", h.Name)
+		}
+		seen[h.Name] = true
+		if h.Typeflag == tar.TypeSymlink {
+			if h.Linkname != e.Link {
+				return m, fmt.Errorf("bundle member %s links to %q, manifest says %q", h.Name, h.Linkname, e.Link)
+			}
+			continue
+		}
+		dst := io.Discard
+		if h.Name == dumpEntry && dumpTo != nil {
+			dst = dumpTo
+		}
+		sum := sha256.New()
+		n, err := io.Copy(io.MultiWriter(dst, sum), tr)
+		if err != nil {
+			return m, fmt.Errorf("read %s: %w", h.Name, err)
+		}
+		if n != e.Size || hex.EncodeToString(sum.Sum(nil)) != e.SHA256 {
+			return m, fmt.Errorf("bundle member %s is corrupt (size or sha256 differs from the manifest)", h.Name)
+		}
+	}
+	for name := range want {
+		if !seen[name] {
+			return m, fmt.Errorf("bundle is missing %s", name)
+		}
+	}
+	if _, ok := want[dumpEntry]; !ok {
+		return m, fmt.Errorf("bundle holds no %s", dumpEntry)
+	}
+	// The tar end marker is not the end of the file; the sidecar covers every byte.
+	if _, err := io.Copy(io.Discard, io.TeeReader(f, whole)); err != nil {
+		return m, err
+	}
+	if sidecar, err := os.ReadFile(path + sumExt); err == nil {
+		fields := strings.Fields(string(sidecar))
+		if len(fields) == 0 || fields[0] != hex.EncodeToString(whole.Sum(nil)) {
+			return m, fmt.Errorf("%s does not match %s%s", filepath.Base(path), filepath.Base(path), sumExt)
+		}
+	}
+	return m, nil
+}
+
+// RestoreOptions configures a restore.
+type RestoreOptions struct {
+	DatabaseURL string
+	Bundle      string
+	// Dir holds the scratch copy of the dump and the pre-restore safety bundle.
+	Dir string
+	// Force restores even while other clients are connected to the database.
+	Force bool
+	// SkipSafetyBackup skips the bundle of the current database taken before it
+	// is replaced.
+	SkipSafetyBackup bool
+	// Safety configures that bundle; its DatabaseURL, Dir and Label are set here.
+	Safety BackupOptions
+	Tools  Tools
+	Log    io.Writer
+}
+
+// ErrClientsConnected refuses a restore under live clients: the replay needs
+// exclusive locks on every table, and felis-api would be serving from a
+// database that is about to change under it.
+var ErrClientsConnected = errors.New("other clients are connected to the database")
+
+// Restore replaces the database's contents with the bundle's dump, atomically.
+// It returns the bundle's manifest and, unless skipped, the path of the safety
+// bundle of what was there before.
+func Restore(ctx context.Context, o RestoreOptions) (Manifest, string, error) {
+	c, err := parseConn(o.DatabaseURL)
+	if err != nil {
+		return Manifest{}, "", err
+	}
+	logw := o.Log
+	if logw == nil {
+		logw = io.Discard
+	}
+	if err := os.MkdirAll(o.Dir, 0o700); err != nil {
+		return Manifest{}, "", err
+	}
+	scratch, err := os.CreateTemp(o.Dir, ".restore-*.dump")
+	if err != nil {
+		return Manifest{}, "", err
+	}
+	defer os.Remove(scratch.Name())
+	m, err := readBundle(o.Bundle, scratch)
+	if cerr := scratch.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return m, "", err
+	}
+	if _, err := run(exec.CommandContext(ctx, o.Tools.pgRestore(), "--list", scratch.Name())); err != nil {
+		return m, "", fmt.Errorf("the bundle's dump does not read: %w", err)
+	}
+
+	if !o.Force {
+		n, err := otherClients(ctx, c, o.Tools)
+		if err != nil {
+			return m, "", fmt.Errorf("count connected clients: %w", err)
+		}
+		if n > 0 {
+			return m, "", fmt.Errorf("%w (%d); scale felis-api and felis-operator to 0 first, or pass -force", ErrClientsConnected, n)
+		}
+	}
+
+	var safety string
+	if !o.SkipSafetyBackup {
+		so := o.Safety
+		so.DatabaseURL, so.Dir, so.Label, so.Tools = o.DatabaseURL, o.Dir, LabelPreRestore, o.Tools
+		if so.Log == nil {
+			so.Log = logw
+		}
+		if safety, err = Backup(ctx, so); err != nil {
+			return m, "", fmt.Errorf("safety backup of the current database: %w (pass -no-safety-backup to restore without one)", err)
+		}
+		fmt.Fprintf(logw, "felis db restore: current database saved to %s\n", safety)
+	}
+
+	if err := replay(ctx, c, o.Tools, scratch.Name()); err != nil {
+		return m, safety, err
+	}
+	// The dump carried its own freshness record, older than the bundle it is in
+	// (the record is written after the bundle). Point it at the newest bundle on
+	// disk, or the panel reports a missing backup right after a restore.
+	if err := recordNewest(ctx, c, o.Tools, o.Dir); err != nil {
+		fmt.Fprintf(logw, "felis db restore: record the newest backup for the panel: %v\n", err)
+	}
+	return m, safety, nil
+}
+
+// recordNewest records the newest bundle in dir as the latest backup.
+func recordNewest(ctx context.Context, c conn, t Tools, dir string) error {
+	all, err := List(dir)
+	if err != nil || len(all) == 0 {
+		return err
+	}
+	b := all[0]
+	st := Status{At: b.Created, Name: b.Name, Label: b.Label, SizeBytes: b.Size, Dir: dir}
+	if m, err := Verify(b.Path); err == nil {
+		st.FelisVersion, st.SchemaVersion = m.FelisVersion, m.SchemaVersion
+	}
+	return record(ctx, c, t, st)
+}
+
+// otherClients counts client sessions on the database other than this one.
+func otherClients(ctx context.Context, c conn, t Tools) (int, error) {
+	out, err := run(c.command(ctx, t.psql(), "-X", "-q", "-t", "-A", "-w", "-d", c.uri, "-c",
+		"SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid() AND backend_type = 'client backend'"))
+	if err != nil {
+		return 0, err
+	}
+	return strconv.Atoi(strings.TrimSpace(string(out)))
+}
+
+// dropOwned clears everything the connecting role owns in the database, the
+// first statement of the restore transaction.
+const dropOwned = "DROP OWNED BY CURRENT_USER;\n"
+
+// replay pipes `pg_restore --file=-` into one psql transaction that starts by
+// dropping what the role owns. The COMMIT is only written once pg_restore has
+// exited cleanly: a generator that dies mid-stream leaves psql at EOF inside an
+// open transaction, which the server rolls back when psql disconnects. psql's
+// own --single-transaction would commit whatever arrived before that EOF.
+func replay(ctx context.Context, c conn, t Tools, dump string) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	r, w, err := os.Pipe()
+	if err != nil {
+		return err
+	}
+	var restoreErr, psqlErr bytes.Buffer
+	gen := exec.CommandContext(ctx, t.pgRestore(), "--no-owner", "--no-privileges", "--file=-", dump)
+	gen.Stdout, gen.Stderr = w, &restoreErr
+	if err := gen.Start(); err != nil {
+		r.Close()
+		w.Close()
+		return fmt.Errorf("pg_restore: %w", err)
+	}
+	w.Close()
+
+	tail := &commitAfter{gen: gen}
+	apply := c.command(ctx, t.psql(), "-X", "-q", "-w", "-v", "ON_ERROR_STOP=1", "-d", c.uri)
+	apply.Stdin = io.MultiReader(strings.NewReader("BEGIN;\n"+dropOwned), r, tail)
+	apply.Stdout, apply.Stderr = io.Discard, &psqlErr
+	aerr := apply.Run()
+	// Closing the read end makes a pg_restore still writing (psql stopped early)
+	// die on the broken pipe instead of blocking, and it is reaped exactly once.
+	r.Close()
+	gerr := tail.wait()
+	if aerr == nil {
+		// psql read to the end, and the end is a COMMIT only pg_restore's clean
+		// exit releases.
+		return nil
+	}
+	msg := "replay the dump (rolled back, the database is unchanged)"
+	if s := strings.TrimSpace(psqlErr.String()); s != "" {
+		msg += ": psql: " + s
+	}
+	if s := strings.TrimSpace(restoreErr.String()); gerr != nil && s != "" {
+		msg += ": pg_restore: " + s
+	}
+	return fmt.Errorf("%s: %w", msg, aerr)
+}
+
+// commitAfter yields "COMMIT;" once, and only if the pg_restore feeding the
+// pipe exited cleanly; otherwise it fails the stream so psql never sees one.
+type commitAfter struct {
+	gen       *exec.Cmd
+	waited    bool
+	err       error
+	committed bool
+	rest      []byte
+}
+
+func (c *commitAfter) wait() error {
+	if !c.waited {
+		c.waited, c.err = true, c.gen.Wait()
+	}
+	return c.err
+}
+
+func (c *commitAfter) Read(p []byte) (int, error) {
+	if !c.committed {
+		if err := c.wait(); err != nil {
+			return 0, err
+		}
+		c.committed, c.rest = true, []byte("COMMIT;\n")
+	}
+	if len(c.rest) == 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, c.rest)
+	c.rest = c.rest[n:]
+	return n, nil
+}

internal/dbbackup/restore_test.go

@@ -0,0 +1,139 @@
+package dbbackup
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func takeBackup(t *testing.T, pg *fakePG, dir string, when time.Time) string {
+	t.Helper()
+	path, err := Backup(context.Background(), BackupOptions{DatabaseURL: testURL, Dir: dir, Label: LabelManual, Tools: pg.tools, Now: at(when)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+func restore(pg *fakePG, dir, bundle string, mut func(*RestoreOptions)) (string, error) {
+	o := RestoreOptions{DatabaseURL: testURL, Bundle: bundle, Dir: dir, Tools: pg.tools,
+		Safety: BackupOptions{Now: at(t0.Add(time.Hour))}}
+	if mut != nil {
+		mut(&o)
+	}
+	_, safety, err := Restore(context.Background(), o)
+	return safety, err
+}
+
+func TestRestoreReplacesTheDatabase(t *testing.T) {
+	pg := newFakePG(t, "alice\n")
+	dir := t.TempDir()
+	bundle := takeBackup(t, pg, dir, t0)
+	pg.setDB(t, "alice\nbob\n")
+
+	safety, err := restore(pg, dir, bundle, nil)
+	if err != nil {
+		t.Fatalf("Restore: %v", err)
+	}
+	if got := pg.db(t); got != "alice\n" {
+		t.Fatalf("db after restore = %q", got)
+	}
+	// The replay dropped what the role owns inside the same transaction.
+	stdin, _ := os.ReadFile(filepath.Join(pg.dir, "psql.stdin"))
+	if !strings.HasPrefix(string(stdin), "BEGIN;\n"+dropOwned) || !strings.HasSuffix(string(stdin), "COMMIT;\n") {
+		t.Fatalf("psql input = %q", stdin)
+	}
+
+	// The safety bundle holds what was replaced, and restores it.
+	if filepath.Base(safety) != "felis-db-20260924T043000Z-pre-restore.tar" {
+		t.Fatalf("safety = %s", safety)
+	}
+	// The dump brought back its own, older freshness record; the restore
+	// points it at the newest bundle on disk again.
+	if st := pg.recorded(t); st.Name != filepath.Base(safety) || st.Label != LabelPreRestore || st.SchemaVersion != 21 || st.Dir != dir {
+		t.Fatalf("recorded after restore = %+v", st)
+	}
+	if _, err := restore(pg, dir, safety, func(o *RestoreOptions) { o.SkipSafetyBackup = true }); err != nil {
+		t.Fatal(err)
+	}
+	if got := pg.db(t); got != "alice\nbob\n" {
+		t.Fatalf("db after undo = %q", got)
+	}
+}
+
+func TestRestoreRefusesLiveClients(t *testing.T) {
+	pg := newFakePG(t, "alice\n")
+	dir := t.TempDir()
+	bundle := takeBackup(t, pg, dir, t0)
+	pg.setDB(t, "changed\n")
+	pg.flag(t, "clients", "2\n")
+
+	if _, err := restore(pg, dir, bundle, nil); !errors.Is(err, ErrClientsConnected) {
+		t.Fatalf("err = %v, want ErrClientsConnected", err)
+	}
+	if pg.db(t) != "changed\n" {
+		t.Fatal("database touched despite the refusal")
+	}
+	if _, err := restore(pg, dir, bundle, func(o *RestoreOptions) { o.Force = true }); err != nil {
+		t.Fatalf("forced: %v", err)
+	}
+	if pg.db(t) != "alice\n" {
+		t.Fatal("forced restore did not apply")
+	}
+}
+
+func TestRestoreFailureLeavesTheDatabaseAlone(t *testing.T) {
+	for _, tc := range []struct {
+		flag, want string
+	}{
+		// The replay fails in the server: psql stops, the transaction dies with it.
+		{"psql_fail", "already exists"},
+		// The dump reader dies after streaming everything: psql never gets the
+		// COMMIT that only a clean pg_restore exit releases.
+		{"restore_fail", "could not read input"},
+	} {
+		t.Run(tc.flag, func(t *testing.T) {
+			pg := newFakePG(t, "alice\n")
+			dir := t.TempDir()
+			bundle := takeBackup(t, pg, dir, t0)
+			pg.setDB(t, "current\n")
+			pg.flag(t, tc.flag, "")
+			_, err := restore(pg, dir, bundle, func(o *RestoreOptions) { o.SkipSafetyBackup = true })
+			if err == nil || !strings.Contains(err.Error(), "rolled back") || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err = %v, want rolled back + %q", err, tc.want)
+			}
+			if got := pg.db(t); got != "current\n" {
+				t.Fatalf("db = %q, want it untouched", got)
+			}
+		})
+	}
+}
+
+func TestRestoreRefusesACorruptBundle(t *testing.T) {
+	pg := newFakePG(t, "alice\n")
+	dir := t.TempDir()
+	bundle := takeBackup(t, pg, dir, t0)
+	if err := os.WriteFile(bundle+".sha256", []byte("0000  x\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	pg.setDB(t, "current\n")
+	if _, err := restore(pg, dir, bundle, nil); err == nil {
+		t.Fatal("restored a bundle its checksum disowns")
+	}
+	if pg.db(t) != "current\n" {
+		t.Fatal("database touched")
+	}
+	entries, _ := os.ReadDir(dir)
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), ".restore-") {
+			t.Errorf("scratch dump %s left behind", e.Name())
+		}
+		if strings.Contains(e.Name(), LabelPreRestore) {
+			t.Errorf("safety bundle %s taken before the bundle was verified", e.Name())
+		}
+	}
+}

panel/dev/mockApi.ts

@@ -721,6 +721,29 @@ async function handlePublic(ctx: RequestContext): Promise<boolean> {
 
 async function handleSession(ctx: SessionContext): Promise<boolean> {
   switch (route(ctx)) {
+    case "GET platform/db-backup": {
+      if (!isAdmin(ctx.account.role)) {
+        sendError(ctx.res, 403, "forbidden", "admin account required");
+        return true;
+      }
+      // Yesterday's daily run: fresh, so the card shows its healthy state.
+      const at = new Date(Date.now() - 9 * 3600 * 1000);
+      const stamp = at.toISOString().replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z");
+      sendJSON(ctx.res, 200, {
+        last: {
+          at: at.toISOString(),
+          name: `felis-db-${stamp}-daily.tar`,
+          label: "daily",
+          size_bytes: 3_482_112,
+          felis_version: "dev",
+          schema_version: 31,
+          dir: "/var/lib/felis/db-backups",
+        },
+        stale: false,
+        max_age_seconds: 26 * 3600,
+      });
+      return true;
+    }
     case "GET updates/window":
       if (!isAdmin(ctx.account.role)) {
         sendError(ctx.res, 403, "forbidden", "admin account required");

panel/src/i18n/resources/en-US/admin.json

@@ -69,8 +69,8 @@
   "reviewer": "Reviewer",
   "reviewed_at": "Reviewed At",
   "reject_reason": "Rejection Reason",
-  "updates_title": "Maintenance Window",
-  "updates_subtitle": "Configure the platform-wide maintenance window. A Scheduled auto-update component may only be applied by Felis within this window; outside it, updates are notify-only.",
+  "updates_title": "Maintenance & Backups",
+  "updates_subtitle": "Check that the control-plane database backup is fresh, and configure the platform-wide maintenance window. Felis may apply a Scheduled update only inside the window; outside it, updates are notify-only.",
   "updates_current_unset": "No maintenance window set. Scheduled updates will degrade to notify-only and will not be applied automatically.",
   "updates_start_label": "Start Time",
   "updates_end_label": "End Time",
@@ -90,13 +90,33 @@
   "updates_stat_start": "Start Time",
   "updates_stat_end": "End Time",
   "updates_stat_timezone": "Local Timezone",
+  "dbbackup_title": "Control-plane database backup",
+  "dbbackup_subtitle": "Users, passkeys, server ownership, quotas, audit logs and the world-archive index live in this database. felis-db-backup.timer on the host backs it up daily, and every upgrade snapshots it before migrating.",
+  "dbbackup_status_ok": "Healthy",
+  "dbbackup_status_stale": "Overdue",
+  "dbbackup_status_never": "Never backed up",
+  "dbbackup_refresh": "Refresh",
+  "dbbackup_loading": "Reading backup status…",
+  "dbbackup_field_when": "Last backup",
+  "dbbackup_field_label": "Kind",
+  "dbbackup_field_size": "Size",
+  "dbbackup_field_schema": "Schema version",
+  "dbbackup_field_file": "Bundle (host path)",
+  "dbbackup_label_daily": "Daily",
+  "dbbackup_label_pre_migrate": "Pre-upgrade snapshot",
+  "dbbackup_label_pre_restore": "Pre-restore snapshot",
+  "dbbackup_label_manual": "Manual",
+  "dbbackup_never_title": "No database backup has been recorded yet",
+  "dbbackup_stale_title": "The newest backup is more than {{hours}} hours old",
+  "dbbackup_fix_hint": "If the host failed now, accounts, server ownership and the archive index could not be recovered. Take a backup on the host now, then read the timer's log to find out why it did not run:",
+  "dbbackup_copy": "Copy command",
+  "dbbackup_offsite_note": "Backups are kept on this host only and are lost with its disk. Copy the backup directory to another machine regularly; restore and disaster-recovery steps are in the troubleshooting guide, §16.",
   "build_import_submission_label": "Import parameters from submission",
   "build_import_submission_placeholder": "Select a user submission...",
   "build_import_submission_none": "No matching submissions found or not loaded",
   "build_import_submission_hint": "Selecting a submission automatically populates the Image Reference, Context Reference, and the corresponding Dockerfile audit header.",
   "build_import_submission_warning_title": "Warning: This submission is currently \"{{status}}\"",
   "build_import_submission_warning_desc": "Manually triggering a build will not automatically mark this submission as approved, nor will it update its associated build status in the database. Use for emergency debugging or testing only.",
-
   "_users_comment": "User administration (admin-tier only).",
   "users_title": "Users",
   "users_subtitle": "Manage platform user accounts, quotas, and sessions.",
@@ -112,9 +132,9 @@
   "users_filter_status_all": "All Status",
   "users_status_active": "Active",
   "users_status_disabled": "Disabled",
-   "users_role_admin": "Admin",
-     "users_role_owner": "Owner",
-     "users_role_user": "User",
+  "users_role_admin": "Admin",
+  "users_role_owner": "Owner",
+  "users_role_user": "User",
   "users_col_user": "User",
   "users_col_role": "Role",
   "users_col_servers": "Servers",

panel/src/i18n/resources/en-US/navigation.json

@@ -9,5 +9,5 @@
   "admin_images": "Images",
   "admin_builds": "Build Pipeline",
   "admin_submissions": "Submissions",
-  "admin_updates": "Maintenance Window"
+  "admin_updates": "Maintenance & Backups"
 }

panel/src/i18n/resources/zh-CN/admin.json

@@ -69,8 +69,8 @@
   "reviewer": "审核人",
   "reviewed_at": "审核时间",
   "reject_reason": "驳回理由",
-  "updates_title": "维护窗口",
-  "updates_subtitle": "配置全局系统维护窗口。在此窗口内，Felis 可以自动应用系统更新；在窗口外，更新将降级为仅通知，不会自动执行。",
+  "updates_title": "维护与备份",
+  "updates_subtitle": "查看控制面数据库备份是否新鲜，并配置全局维护窗口。在窗口内 Felis 可以自动应用系统更新；在窗口外，更新降级为仅通知。",
   "updates_current_unset": "当前未设置维护窗口。自动更新将降级为仅通知，不会自动执行。",
   "updates_start_label": "开始时间",
   "updates_end_label": "结束时间",
@@ -90,13 +90,33 @@
   "updates_stat_start": "维护开始时间",
   "updates_stat_end": "维护结束时间",
   "updates_stat_timezone": "本地时区",
+  "dbbackup_title": "控制面数据库备份",
+  "dbbackup_subtitle": "用户、Passkey、服务器归属、配额、审计日志和世界存档索引都在这个数据库里。主机上的 felis-db-backup.timer 每天备份一次，每次升级迁移前也会先快照。",
+  "dbbackup_status_ok": "正常",
+  "dbbackup_status_stale": "已过期",
+  "dbbackup_status_never": "从未备份",
+  "dbbackup_refresh": "刷新",
+  "dbbackup_loading": "正在读取备份状态…",
+  "dbbackup_field_when": "最近一次备份",
+  "dbbackup_field_label": "类型",
+  "dbbackup_field_size": "大小",
+  "dbbackup_field_schema": "数据库版本",
+  "dbbackup_field_file": "备份文件（主机路径）",
+  "dbbackup_label_daily": "每日定时",
+  "dbbackup_label_pre_migrate": "升级前快照",
+  "dbbackup_label_pre_restore": "恢复前快照",
+  "dbbackup_label_manual": "手动",
+  "dbbackup_never_title": "还没有记录到任何数据库备份",
+  "dbbackup_stale_title": "最近一次备份已超过 {{hours}} 小时",
+  "dbbackup_fix_hint": "此时主机出故障，账号、服务器归属和存档索引都无法恢复。在主机上立即备份一次，再查看定时任务日志找出它没有运行的原因：",
+  "dbbackup_copy": "复制命令",
+  "dbbackup_offsite_note": "备份只保存在这台主机上，硬盘损坏或主机丢失时会一起丢失。请定期把备份目录复制到另一台机器；恢复与灾备步骤见故障排查文档 §16。",
   "build_import_submission_label": "从已有的审核提交导入参数",
   "build_import_submission_placeholder": "选择一个用户提交...",
   "build_import_submission_none": "无匹配的提交或暂未加载",
   "build_import_submission_hint": "选择提交将自动填充镜像引用、构建上下文引用和对应的 Dockerfile 审计头。",
   "build_import_submission_warning_title": "警告：该提交状态为「{{status}}」",
   "build_import_submission_warning_desc": "手动触发构建不会自动将该提交标记为已同意，也不会更新其关联的构建状态。仅适用于紧急调试或测试。",
-
   "_users_comment": "用户管理（仅管理员可见）。",
   "users_title": "用户管理",
   "users_subtitle": "管理平台用户账号、配额和会话。",
@@ -113,8 +133,8 @@
   "users_status_active": "正常",
   "users_status_disabled": "已禁用",
   "users_role_admin": "管理员",
-    "users_role_owner": "所有者",
-    "users_role_user": "普通用户",
+  "users_role_owner": "所有者",
+  "users_role_user": "普通用户",
   "users_col_user": "用户",
   "users_col_role": "角色",
   "users_col_servers": "服务器数",

panel/src/i18n/resources/zh-CN/navigation.json

@@ -9,5 +9,5 @@
   "admin_images": "镜像",
   "admin_builds": "构建流水线",
   "admin_submissions": "审核提交",
-  "admin_updates": "维护窗口"
+  "admin_updates": "维护与备份"
 }

panel/src/lib/api.test.ts

@@ -580,6 +580,19 @@ describe("image whitelist and builds wire shapes", () => {
     });
   });
 
+  describe("control-plane database backup", () => {
+    it("getDBBackup GETs /platform/db-backup and keeps a null last", async () => {
+      const status = { last: null, stale: true, max_age_seconds: 93600 };
+      const fetchSpy = fakeFetch(status);
+      vi.stubGlobal("fetch", fetchSpy);
+      const res = await api.getDBBackup();
+      expect(res).toEqual(status);
+      const [url, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
+      expect(String(url)).toBe("/platform/db-backup");
+      expect((opts as RequestInit).method).toBe("GET");
+    });
+  });
+
   describe("backup now and server jobs wire shapes", () => {
     it("backupNow POSTs to /servers/{name}/backup with no body and parses the 202", async () => {
       const fetchSpy = fakeFetch({ name: "survival", status: "backing_up" }, { status: 202 });

panel/src/lib/api.ts

@@ -27,6 +27,7 @@ import type {
   WhitelistResult,
   Submission,
   UpdateWindow,
+  DBBackupStatus,
 } from "./types";
 import { loadConfig } from "./config";
 import i18next from "i18next";
@@ -549,6 +550,9 @@ export const api = {
 
   setUpdateWindow: (window: UpdateWindow) => request<UpdateWindow>("PUT", "/updates/window", window),
 
+  // Freshness of the host's control-plane database backup (felis-db-backup.timer).
+  getDBBackup: () => request<DBBackupStatus>("GET", "/platform/db-backup"),
+
   // ---- User admin (admin-tier, spec §7 user admin) ----
 
   listUsers: (params?: {

panel/src/lib/types.ts

@@ -289,6 +289,28 @@ export interface UpdateWindow {
   end: string | null;
 }
 
+// ---- Control-plane database backup (internal/api/handlers_dbbackup.go dbBackupView) ----
+
+export type DBBackupLabel = "daily" | "pre-migrate" | "pre-restore" | "manual";
+
+export interface DBBackupRecord {
+  at: string;
+  name: string;
+  label: DBBackupLabel;
+  size_bytes: number;
+  felis_version?: string;
+  schema_version?: number;
+  dir: string;
+}
+
+export interface DBBackupStatus {
+  /** Null until the host has recorded its first backup. */
+  last: DBBackupRecord | null;
+  /** True when there is no record or it is older than max_age_seconds. */
+  stale: boolean;
+  max_age_seconds: number;
+}
+
 // ---- User admin types (internal/api/repo.go UserView, UserDetail, QuotaView, SessionView) ----
 
 export interface UserView {

panel/src/pages/admin/DBBackupCard.tsx

@@ -0,0 +1,216 @@
+import { useState } from "react";
+import { AlertTriangle, Check, CheckCircle2, Copy, Database, Loader2, RefreshCw } from "lucide-react";
+import { useTranslation } from "react-i18next";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { MessageLine } from "@/components/MessageLine";
+import { api, humanizeError } from "@/lib/api";
+import { useAsync } from "@/lib/hooks";
+import { formatAbsolute, formatBytes, formatRelative } from "@/lib/format";
+import { cn } from "@/lib/utils";
+import type { DBBackupLabel } from "@/lib/types";
+
+// The control-plane database backup is taken on the host (felis-db-backup.timer),
+// never through the API, so the card is read-only: it says how fresh the newest
+// backup is and, when it is not, hands the admin the exact host commands.
+
+const LABEL_KEY: Record<DBBackupLabel, string> = {
+  daily: "dbbackup_label_daily",
+  "pre-migrate": "dbbackup_label_pre_migrate",
+  "pre-restore": "dbbackup_label_pre_restore",
+  manual: "dbbackup_label_manual",
+};
+
+const FIX_COMMANDS = ["sudo felis db backup", "journalctl -u felis-db-backup -n 50 --no-pager"];
+
+function CopyCommand({ command }: { command: string }) {
+  const { t } = useTranslation("admin");
+  const [copied, setCopied] = useState(false);
+  async function copy() {
+    try {
+      await navigator.clipboard.writeText(command);
+      setCopied(true);
+      window.setTimeout(() => setCopied(false), 1500);
+    } catch {
+      // Clipboard denied (non-secure context): the command stays selectable.
+    }
+  }
+  return (
+    <div className="flex items-center gap-2 rounded-md border border-border/60 bg-muted/40 pl-3 pr-1 py-1">
+      <code className="min-w-0 flex-1 overflow-x-auto whitespace-nowrap py-1 font-mono text-xs text-foreground" title={command}>
+        {command}
+      </code>
+      <Button
+        type="button"
+        variant="ghost"
+        size="icon"
+        className="h-7 w-7 shrink-0"
+        onClick={copy}
+        aria-label={t("dbbackup_copy")}
+        title={t("dbbackup_copy")}
+      >
+        {copied ? <Check className="h-3.5 w-3.5 text-emerald-500" /> : <Copy className="h-3.5 w-3.5" />}
+      </Button>
+    </div>
+  );
+}
+
+function Field({ label, children, title }: { label: string; children: React.ReactNode; title?: string }) {
+  return (
+    <div className="min-w-0 space-y-1">
+      <dt className="text-[11px] font-medium text-muted-foreground">{label}</dt>
+      <dd className="truncate text-sm font-semibold text-foreground" title={title}>
+        {children}
+      </dd>
+    </div>
+  );
+}
+
+export function DBBackupCard() {
+  const { t, i18n } = useTranslation("admin");
+  const locale = i18n.language;
+  const { data, error, loading, reload } = useAsync(() => api.getDBBackup(), []);
+  const last = data?.last ?? null;
+  const maxAgeHours = data ? Math.round(data.max_age_seconds / 3600) : 26;
+
+  const state: "loading" | "error" | "never" | "stale" | "ok" = !data
+    ? error
+      ? "error"
+      : "loading"
+    : !last
+      ? "never"
+      : data.stale
+        ? "stale"
+        : "ok";
+
+  const badge = (() => {
+    switch (state) {
+      case "ok":
+        return (
+          <Badge className="gap-1 border-transparent bg-emerald-500/15 text-emerald-500">
+            <CheckCircle2 className="h-3 w-3" />
+            {t("dbbackup_status_ok")}
+          </Badge>
+        );
+      case "stale":
+        return (
+          <Badge variant="destructive" className="gap-1">
+            <AlertTriangle className="h-3 w-3" />
+            {t("dbbackup_status_stale")}
+          </Badge>
+        );
+      case "never":
+        return (
+          <Badge variant="destructive" className="gap-1">
+            <AlertTriangle className="h-3 w-3" />
+            {t("dbbackup_status_never")}
+          </Badge>
+        );
+      default:
+        return null;
+    }
+  })();
+
+  return (
+    <Card className="w-full">
+      <CardHeader className="flex flex-row items-center justify-between gap-3 space-y-0">
+        <div className="flex min-w-0 items-center gap-3">
+          <div
+            className={cn(
+              "hidden rounded-md p-2 sm:block",
+              state === "stale" || state === "never"
+                ? "bg-destructive/10 text-destructive"
+                : "bg-primary/10 text-primary",
+            )}
+          >
+            <Database className="h-5 w-5" />
+          </div>
+          <div className="min-w-0">
+            <CardTitle className="flex flex-wrap items-center gap-2 text-base font-semibold">
+              {t("dbbackup_title")}
+              {badge}
+            </CardTitle>
+            <p className="mt-0.5 text-xs text-muted-foreground">{t("dbbackup_subtitle")}</p>
+          </div>
+        </div>
+        <Button
+          type="button"
+          variant="ghost"
+          size="icon"
+          className="h-8 w-8 shrink-0"
+          onClick={reload}
+          disabled={loading}
+          aria-label={t("dbbackup_refresh")}
+          title={t("dbbackup_refresh")}
+        >
+          <RefreshCw className={cn("h-4 w-4", loading && "animate-spin")} />
+        </Button>
+      </CardHeader>
+
+      <CardContent className="space-y-4 text-sm">
+        {state === "loading" && (
+          <div className="flex items-center gap-2 text-xs text-muted-foreground">
+            <Loader2 className="h-4 w-4 animate-spin" />
+            {t("dbbackup_loading")}
+          </div>
+        )}
+
+        {state === "error" && <MessageLine kind="error" message={humanizeError(error)} />}
+
+        {last && (
+          <dl className="grid grid-cols-2 gap-x-6 gap-y-4 lg:grid-cols-4">
+            <Field label={t("dbbackup_field_when")} title={formatAbsolute(last.at, locale)}>
+              <span className={cn(state === "stale" && "text-destructive")}>
+                {formatRelative(last.at, Date.now(), locale) || "—"}
+              </span>
+              <span className="block truncate text-[11px] font-normal text-muted-foreground">
+                {formatAbsolute(last.at, locale)}
+              </span>
+            </Field>
+            <Field label={t("dbbackup_field_label")}>
+              {LABEL_KEY[last.label] ? t(LABEL_KEY[last.label]) : last.label}
+            </Field>
+            <Field label={t("dbbackup_field_size")}>
+              <span className="font-mono">{formatBytes(last.size_bytes)}</span>
+            </Field>
+            <Field label={t("dbbackup_field_schema")}>
+              <span className="font-mono">{last.schema_version ? `#${last.schema_version}` : "—"}</span>
+            </Field>
+            <div className="col-span-2 min-w-0 space-y-1 lg:col-span-4">
+              <dt className="text-[11px] font-medium text-muted-foreground">{t("dbbackup_field_file")}</dt>
+              <dd className="break-all font-mono text-xs text-foreground">
+                {last.dir.replace(/\/+$/, "")}/{last.name}
+              </dd>
+            </div>
+          </dl>
+        )}
+
+        {(state === "stale" || state === "never") && (
+          <div className="space-y-3 rounded-lg border border-destructive/25 bg-destructive/5 p-4">
+            <div className="flex items-start gap-2 text-destructive">
+              <AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" />
+              <div className="space-y-1">
+                <p className="font-semibold">
+                  {state === "never" ? t("dbbackup_never_title") : t("dbbackup_stale_title", { hours: maxAgeHours })}
+                </p>
+                <p className="text-xs leading-relaxed text-destructive/90">{t("dbbackup_fix_hint")}</p>
+              </div>
+            </div>
+            <div className="space-y-2">
+              {FIX_COMMANDS.map((c) => (
+                <CopyCommand key={c} command={c} />
+              ))}
+            </div>
+          </div>
+        )}
+
+        {data && (
+          <p className="rounded-md border border-border/40 bg-muted/15 p-3 text-[11px] leading-relaxed text-muted-foreground">
+            {t("dbbackup_offsite_note")}
+          </p>
+        )}
+      </CardContent>
+    </Card>
+  );
+}

panel/src/pages/admin/UpdatesPage.tsx

@@ -13,6 +13,7 @@ import { Loading, ErrorState } from "@/components/States";
 import { api, humanizeError } from "@/lib/api";
 import { useAsync } from "@/lib/hooks";
 import { formatAbsolute } from "@/lib/format";
+import { DBBackupCard } from "./DBBackupCard";
 
 function toLocalDatetimeString(dateOrStr: Date | string | null | undefined): string {
   if (!dateOrStr) return "";
@@ -166,6 +167,9 @@ export function UpdatesPage() {
     <div className="space-y-6">
       <PageHeader icon={Clock} title={t("updates_title")} subtitle={t("updates_subtitle")} />
 
+      {/* Control-plane database backup freshness (read-only, host timer) */}
+      <DBBackupCard />
+
       {/* Stats Cards Row */}
       <div className="grid grid-cols-1 gap-4 sm:grid-cols-4">
         <StatCard