From c7db7d41260790a19d08890c32c680c903541673 Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Thu, 24 Sep 2026 15:19:42 +0800 Subject: [PATCH] =?UTF-8?q?feat(db):=20=E6=8E=A7=E5=88=B6=E9=9D=A2=20PG=20?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E5=A4=87=E4=BB=BD=E3=80=81=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E5=89=8D=E5=BF=AB=E7=85=A7=E4=B8=8E=E5=8E=9F=E5=AD=90=E6=81=A2?= =?UTF-8?q?=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + README_EN.md | 1 + cmd/felis/db.go | 334 +++++++++ cmd/felis/db_test.go | 151 ++++ cmd/felis/migrate.go | 51 ++ cmd/felis/run.go | 4 +- cmd/felis/tui_migration.go | 9 +- deploy/alerts/felis-alerts.yaml | 27 + deploy/alerts/felis-alerts_test.yml | 46 ++ deploy/alerts/felis-prometheusrule.yaml | 26 + deploy/bootstrap.sh | 68 +- deploy/bootstrap_test.sh | 64 ++ docs/openapi.yaml | 68 ++ docs/troubleshooting.md | 193 ++++- internal/api/api.go | 3 + internal/api/handlers_dbbackup.go | 49 ++ internal/api/handlers_dbbackup_test.go | 87 +++ internal/dbbackup/dbbackup.go | 707 ++++++++++++++++++ internal/dbbackup/dbbackup_test.go | 512 +++++++++++++ internal/dbbackup/restore.go | 346 +++++++++ internal/dbbackup/restore_test.go | 139 ++++ panel/dev/mockApi.ts | 23 + panel/src/i18n/resources/en-US/admin.json | 32 +- .../src/i18n/resources/en-US/navigation.json | 2 +- panel/src/i18n/resources/zh-CN/admin.json | 30 +- .../src/i18n/resources/zh-CN/navigation.json | 2 +- panel/src/lib/api.test.ts | 13 + panel/src/lib/api.ts | 4 + panel/src/lib/types.ts | 22 + panel/src/pages/admin/DBBackupCard.tsx | 216 ++++++ panel/src/pages/admin/UpdatesPage.tsx | 4 + 31 files changed, 3217 insertions(+), 17 deletions(-) create mode 100644 cmd/felis/db.go create mode 100644 cmd/felis/db_test.go create mode 100644 internal/api/handlers_dbbackup.go create mode 100644 internal/api/handlers_dbbackup_test.go create mode 100644 internal/dbbackup/dbbackup.go create mode 100644 internal/dbbackup/dbbackup_test.go create mode 100644 internal/dbbackup/restore.go create mode 100644 internal/dbbackup/restore_test.go create mode 100644 panel/src/pages/admin/DBBackupCard.tsx diff --git a/README.md b/README.md index 075dfac..bc85702 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy, - **即开即玩**：玩家尝试连接时自动唤醒服务器，空闲后自动休眠，像游戏主机一样省资源。 - **Web 控制面板**：浏览器中查看服务器状态、在线玩家与资源用量，管理备份与恢复。 - **备份与恢复**：一键把整服数据（世界、配置、插件/模组，即整个 /data 卷）打包进集群内的归档库，支持从任意备份点回滚；默认安装就已启用（归档 PVC 与路径由安装器一并生成）。 +- **控制面数据库备份**：账号、服务器归属、配额与存档索引所在的数据库每天自动备份，每次升级迁移前先快照，出错可用 `felis db restore` 整库原子回滚；面板「维护与备份」页显示备份是否新鲜（见 [故障排查 §16](docs/troubleshooting.md)）。 - **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。 - **多核心支持**：兼容 Paper、Fabric、Forge、NeoForge，经由 Velocity 代理统一入口。 - **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建；构建产物进入镜像白名单，可直接选用为服务器镜像完成部署。 diff --git a/README_EN.md b/README_EN.md index 774e6f8..50f5c10 100644 --- a/README_EN.md +++ b/README_EN.md @@ -18,6 +18,7 @@ Table of Contents - **Wake on Join**: Servers start automatically when a player connects, and stop when idle — like hibernate for your server. - **Web Dashboard**: Monitor server status, online players, and resource usage from your browser, with backup and restore management. - **Backup & Restore**: One-click snapshots of a server's whole data volume (worlds, config, plugins/mods — the entire /data volume) into the cluster's archive store, with rollback from any backup point — enabled by default (the installer renders the archive PVC and its path). +- **Control-plane database backups**: The database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migrates it; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows whether the newest backup is fresh (see [troubleshooting §16](docs/troubleshooting.md)). - **World Reaper** (opt in): Worlds idle for more than 15 days are automatically backed up and removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is ever deleted. - **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, federated behind a Velocity proxy. - **Modpack Submission**: Players submit custom modpacks; admin approval triggers an automatic build, and the result is whitelisted as a server image you can select to deploy. diff --git a/cmd/felis/db.go b/cmd/felis/db.go new file mode 100644 index 0000000..8050221 --- /dev/null +++ b/cmd/felis/db.go @@ -0,0 +1,334 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/dbbackup" +) + +const dbUsage = `usage: + felis db backup [-config path] [-dir dir] [-label daily|manual|...] [-keep n] [-state-dir dir] + [-no-servers] [-metrics-file path] + felis db restore [-config path] [-dir dir] [-yes] [-force] [-no-safety-backup] + felis db verify [-dir dir] + felis db list [-dir dir] + felis db check [-dir dir] [-max-age 26h] +` + +// defaultKeep is how many bundles of a label a backup leaves behind. Manual +// bundles are the operator's own and are never pruned. +var defaultKeep = map[string]int{ + dbbackup.LabelDaily: 14, + dbbackup.LabelPreMigrate: 10, + dbbackup.LabelPreRestore: 5, +} + +// cmdDB implements `felis db`: logical backups of the control-plane database +// together with the host state a rebuild needs (internal/dbbackup). The verb +// comes first for the same reason as `felis migrate up`. +func cmdDB(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprint(stderr, dbUsage) + return 2 + } + verb, rest := args[0], args[1:] + fs := flag.NewFlagSet("db "+verb, flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprint(stderr, dbUsage) } + dir := fs.String("dir", dbbackup.DefaultDir, "bundle directory") + switch verb { + case "backup": + return dbBackup(fs, dir, rest, stdout, stderr) + case "restore": + return dbRestore(fs, dir, rest, stdout, stderr) + case "verify": + return dbVerify(fs, dir, rest, stdout, stderr) + case "list": + return dbList(fs, dir, rest, stdout, stderr) + case "check": + return dbCheck(fs, dir, rest, stdout, stderr) + case "-h", "--help", "help": + fmt.Fprint(stdout, dbUsage) + return 0 + } + fmt.Fprintf(stderr, "felis db: unknown verb %q\n%s", verb, dbUsage) + return 2 +} + +// parseWithArg parses flags that may sit on either side of one positional +// argument (`restore -yes x.tar` and `restore x.tar -yes` both work) and +// returns that argument. +func parseWithArg(fs *flag.FlagSet, args []string) (string, bool) { + if err := fs.Parse(args); err != nil { + return "", false + } + if fs.NArg() == 0 { + return "", true + } + arg := fs.Arg(0) + if err := fs.Parse(fs.Args()[1:]); err != nil { + return "", false + } + if fs.NArg() > 0 { + fmt.Fprintf(fs.Output(), "felis db: unexpected argument %q\n", fs.Arg(0)) + return "", false + } + return arg, true +} + +func dbDatabaseURL(path string) (string, error) { + cfg, err := config.Load(path) + if err != nil { + return "", err + } + return cfg.Database.URL, nil +} + +func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int { + cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml") + label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never") + keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, manual all)") + stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`) + noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle") + metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)") + if err := fs.Parse(args); err != nil { + return 2 + } + if fs.NArg() > 0 { + fmt.Fprint(stderr, dbUsage) + return 2 + } + url, err := dbDatabaseURL(*cfgPath) + if err != nil { + fmt.Fprintf(stderr, "felis db backup: %v\n", err) + return 1 + } + if *keep < 0 { + *keep = defaultKeep[*label] + } + o := dbbackup.BackupOptions{ + DatabaseURL: url, Dir: *dir, Label: *label, Keep: *keep, + StateDir: *stateDir, Version: resolvedVersion(), Log: stderr, + MetricsFile: *metrics, Record: true, + } + if !*noServers { + o.ExportServers = exportMinecraftServers + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + path, err := dbbackup.Backup(ctx, o) + if err != nil { + fmt.Fprintf(stderr, "felis db backup: %v\n", err) + return 1 + } + fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path) + return 0 +} + +// resolveBundle accepts a path, or a bare bundle name looked up in dir. +func resolveBundle(dir, arg string) string { + if strings.ContainsRune(arg, os.PathSeparator) { + return arg + } + if _, err := os.Stat(arg); err == nil { + return arg + } + return filepath.Join(dir, arg) +} + +func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int { + cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml") + yes := fs.Bool("yes", false, "replace the database's contents (required)") + force := fs.Bool("force", false, "restore even while other clients are connected") + noSafety := fs.Bool("no-safety-backup", false, "skip the bundle of the current database taken first") + stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, "host state directory for the safety bundle") + arg, ok := parseWithArg(fs, args) + if !ok { + return 2 + } + if arg == "" { + fmt.Fprint(stderr, dbUsage) + return 2 + } + bundle := resolveBundle(*dir, arg) + m, err := dbbackup.Verify(bundle) + if err != nil { + fmt.Fprintf(stderr, "felis db restore: %v\n", err) + return 1 + } + if !*yes { + fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d).\n", + filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion) + fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.") + return 2 + } + url, err := dbDatabaseURL(*cfgPath) + if err != nil { + fmt.Fprintf(stderr, "felis db restore: %v\n", err) + return 1 + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + _, safety, err := dbbackup.Restore(ctx, dbbackup.RestoreOptions{ + DatabaseURL: url, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety, + Safety: dbbackup.BackupOptions{Keep: defaultKeep[dbbackup.LabelPreRestore], StateDir: *stateDir, + Version: resolvedVersion(), ExportServers: exportMinecraftServers}, + Log: stderr, + }) + if err != nil { + fmt.Fprintf(stderr, "felis db restore: %v\n", err) + if errors.Is(err, dbbackup.ErrClientsConnected) { + fmt.Fprintln(stderr, " kubectl -n felis scale deployment felis-api felis-operator --replicas=0") + } + return 1 + } + fmt.Fprintf(stdout, "felis db restore: restored %s (schema %d)\n", filepath.Base(bundle), m.SchemaVersion) + if safety != "" { + fmt.Fprintf(stdout, " the database as it was before is in %s\n", safety) + } + // Nothing migrates at startup, so a control plane newer than the bundle needs + // its migrations re-applied; rolling back to the release that wrote the bundle + // must skip that, or the rollback is undone. + fmt.Fprintf(stdout, " next: felis migrate up -config %s (skip it when rolling back to felis %s, which wrote this bundle)\n", *cfgPath, orUnknown(m.FelisVersion)) + fmt.Fprintln(stdout, " kubectl -n felis scale deployment felis-api felis-operator --replicas=1") + return 0 +} + +func dbVerify(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int { + arg, ok := parseWithArg(fs, args) + if !ok { + return 2 + } + if arg == "" { + fmt.Fprint(stderr, dbUsage) + return 2 + } + bundle := resolveBundle(*dir, arg) + m, err := dbbackup.Verify(bundle) + if err != nil { + fmt.Fprintf(stderr, "felis db verify: %v\n", err) + return 1 + } + fmt.Fprintf(stdout, "%s: ok\n taken %s (%s)\n felis %s\n schema %d\n %s\n", + filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion, orUnknown(m.PGDumpVersion)) + for _, f := range m.Files { + if f.Link != "" { + fmt.Fprintf(stdout, " %-40s -> %s\n", f.Name, f.Link) + continue + } + fmt.Fprintf(stdout, " %-40s %d bytes\n", f.Name, f.Size) + } + if m.ServersError != "" { + fmt.Fprintf(stdout, " (no MinecraftServer objects: %s)\n", m.ServersError) + } + return 0 +} + +func orUnknown(s string) string { + if s == "" { + return "unknown" + } + return s +} + +func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int { + if err := fs.Parse(args); err != nil { + return 2 + } + all, err := dbbackup.List(*dir) + if err != nil { + fmt.Fprintf(stderr, "felis db list: %v\n", err) + return 1 + } + if len(all) == 0 { + fmt.Fprintf(stdout, "no database backups in %s\n", *dir) + return 0 + } + now := time.Now() + for _, b := range all { + fmt.Fprintf(stdout, "%-50s %-12s %10s %s ago\n", b.Name, b.Label, humanBytes(b.Size), dbbackup.Age(now.Sub(b.Created))) + } + return 0 +} + +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for m := n / unit; m >= unit; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} + +// dbCheck is the freshness probe: exit 1 when the newest bundle is missing or +// older than -max-age, for a monitor or the break-glass console to act on. +func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int { + maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest bundle") + if err := fs.Parse(args); err != nil { + return 2 + } + b, err := dbbackup.Check(*dir, *maxAge, time.Now()) + if err != nil { + fmt.Fprintf(stderr, "felis db check: %v\n", err) + return 1 + } + fmt.Fprintf(stdout, "felis db check: ok, newest backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created))) + return 0 +} + +// exportMinecraftServers reads every MinecraftServer through the host's k3s +// kubectl and strips what the API server owns, so the result can be fed back +// with `kubectl apply -f` on a rebuilt cluster. +func exportMinecraftServers(ctx context.Context) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + // Output, not the CombinedOutput kubectlOutput uses: a deprecation warning + // on stderr must not end up inside the JSON. + cmd := exec.CommandContext(ctx, "k3s", "kubectl", "get", "minecraftservers.felis.lolicon.best", "-A", "-o", "json") + cmd.Env = append(os.Environ(), "KUBECONFIG="+hostBootstrapKubeconfigPath) + var errBuf strings.Builder + cmd.Stderr = &errBuf + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("k3s kubectl get minecraftservers: %w: %s", err, strings.TrimSpace(errBuf.String())) + } + return cleanServerList(out) +} + +// cleanServerList drops status and the server-assigned metadata from a +// `kubectl get -o json` List. +func cleanServerList(raw []byte) ([]byte, error) { + var list struct { + Items []map[string]any `json:"items"` + } + if err := json.Unmarshal(raw, &list); err != nil { + return nil, fmt.Errorf("parse MinecraftServer list: %w", err) + } + for _, it := range list.Items { + delete(it, "status") + if md, ok := it["metadata"].(map[string]any); ok { + for _, k := range []string{"resourceVersion", "uid", "creationTimestamp", "generation", "managedFields", "selfLink"} { + delete(md, k) + } + } + } + if list.Items == nil { + list.Items = []map[string]any{} + } + return json.MarshalIndent(map[string]any{"apiVersion": "v1", "kind": "List", "items": list.Items}, "", " ") +} diff --git a/cmd/felis/db_test.go b/cmd/felis/db_test.go new file mode 100644 index 0000000..7831143 --- /dev/null +++ b/cmd/felis/db_test.go @@ -0,0 +1,151 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "io" + "strings" + "testing" + + "felis.lolicon.best/internal/store" +) + +func TestDBUsage(t *testing.T) { + for _, args := range [][]string{{"db"}, {"db", "frobnicate"}, {"db", "restore"}, {"db", "verify"}, {"db", "backup", "extra"}} { + var out, errBuf bytes.Buffer + if code := run(args, &out, &errBuf); code != 2 { + t.Errorf("%v: exit %d, want 2", args, code) + } + if !strings.Contains(errBuf.String(), "felis db restore") { + t.Errorf("%v: no usage on stderr: %q", args, errBuf.String()) + } + } +} + +func TestDBRestoreNeedsYes(t *testing.T) { + // A bundle that does not exist fails verification (1) before -yes matters; + // the -yes gate itself is exercised against a real bundle in internal/dbbackup + // and on the VM. Here: the refusal path never reaches the config or database. + var out, errBuf bytes.Buffer + if code := run([]string{"db", "restore", "-dir", t.TempDir(), "missing.tar"}, &out, &errBuf); code != 1 { + t.Fatalf("exit %d, stderr %q", code, errBuf.String()) + } +} + +func TestParseWithArg(t *testing.T) { + for _, args := range [][]string{{"-yes", "b.tar"}, {"b.tar", "-yes"}} { + fs := flag.NewFlagSet("t", flag.ContinueOnError) + fs.SetOutput(io.Discard) + yes := fs.Bool("yes", false, "") + arg, ok := parseWithArg(fs, args) + if !ok || arg != "b.tar" || !*yes { + t.Errorf("%v -> %q ok=%v yes=%v", args, arg, ok, *yes) + } + } + fs := flag.NewFlagSet("t", flag.ContinueOnError) + fs.SetOutput(io.Discard) + if _, ok := parseWithArg(fs, []string{"a.tar", "b.tar"}); ok { + t.Error("two positional arguments accepted") + } +} + +func TestResolveBundle(t *testing.T) { + if got := resolveBundle("/var/lib/felis/db-backups", "felis-db-x.tar"); got != "/var/lib/felis/db-backups/felis-db-x.tar" { + t.Errorf("bare name -> %s", got) + } + if got := resolveBundle("/var/lib/felis/db-backups", "/root/copy.tar"); got != "/root/copy.tar" { + t.Errorf("path -> %s", got) + } +} + +func TestCleanServerList(t *testing.T) { + raw := `{"apiVersion":"v1","kind":"List","metadata":{"resourceVersion":""},"items":[{ + "apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer", + "metadata":{"name":"survival","namespace":"minecraft","uid":"u","resourceVersion":"42","generation":3, + "creationTimestamp":"2026-09-01T00:00:00Z","managedFields":[{}],"labels":{"a":"b"}}, + "spec":{"desiredState":"Running"},"status":{"phase":"Running"}}]}` + out, err := cleanServerList([]byte(raw)) + if err != nil { + t.Fatal(err) + } + var got struct { + Kind string `json:"kind"` + Items []map[string]any `json:"items"` + } + if err := json.Unmarshal(out, &got); err != nil { + t.Fatal(err) + } + if got.Kind != "List" || len(got.Items) != 1 { + t.Fatalf("got %s", out) + } + it := got.Items[0] + if _, ok := it["status"]; ok { + t.Error("status kept") + } + md := it["metadata"].(map[string]any) + for _, k := range []string{"uid", "resourceVersion", "generation", "creationTimestamp", "managedFields"} { + if _, ok := md[k]; ok { + t.Errorf("metadata.%s kept", k) + } + } + if md["name"] != "survival" || md["namespace"] != "minecraft" || md["labels"] == nil { + t.Errorf("identity lost: %v", md) + } + if it["spec"].(map[string]any)["desiredState"] != "Running" { + t.Error("spec lost") + } + + empty, err := cleanServerList([]byte(`{"items":null}`)) + if err != nil || !strings.Contains(string(empty), `"items": []`) { + t.Errorf("empty list -> %s, %v", empty, err) + } + if _, err := cleanServerList([]byte("Warning: x\n{")); err == nil { + t.Error("garbage parsed") + } +} + +func TestHasPending(t *testing.T) { + ms := []store.Migration{{Version: 1}, {Version: 2}, {Version: 3}} + if hasPending(map[int]struct{}{1: {}, 2: {}, 3: {}}, ms) { + t.Error("fully applied reported pending") + } + if !hasPending(map[int]struct{}{1: {}, 2: {}}, ms) { + t.Error("missing 3 not reported") + } +} + +type appliedDriver struct { + store.Driver + done map[int]struct{} +} + +func (d appliedDriver) EnsureVersionTable(context.Context) error { return nil } +func (d appliedDriver) AppliedVersions(context.Context) (map[int]struct{}, error) { + return d.done, nil +} + +func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) { + ms := []store.Migration{{Version: 1}, {Version: 2}} + // An unusable URL makes an attempted backup observable as an error without + // any PostgreSQL tooling. + const badURL = "not-a-url" + for _, tc := range []struct { + name string + done map[int]struct{} + attempt bool + }{ + {"fresh database", map[int]struct{}{}, false}, + {"up to date", map[int]struct{}{1: {}, 2: {}}, false}, + {"pending on a populated database", map[int]struct{}{1: {}}, true}, + } { + path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, badURL, t.TempDir(), io.Discard) + if attempted := err != nil; attempted != tc.attempt { + t.Errorf("%s: attempted = %v (err %v), want %v", tc.name, attempted, err, tc.attempt) + } + if path != "" { + t.Errorf("%s: path = %q", tc.name, path) + } + } +} diff --git a/cmd/felis/migrate.go b/cmd/felis/migrate.go index 17b21a8..26b23b6 100644 --- a/cmd/felis/migrate.go +++ b/cmd/felis/migrate.go @@ -7,15 +7,24 @@ import ( "io" "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/dbbackup" "felis.lolicon.best/internal/store" ) // cmdMigrate implements `felis migrate up`: load config, open the database, and // apply every pending embedded migration under the advisory lock (spec §6). +// +// Migrations only roll forward, and some drop data (0017_drop_password), so a +// database that already holds a schema and has migrations pending is bundled +// first (internal/dbbackup, label pre-migrate). A failed snapshot stops the +// upgrade; -no-backup is the explicit way past it, e.g. for an external +// database whose server is newer than the host's pg_dump. func cmdMigrate(args []string, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("migrate", flag.ContinueOnError) fs.SetOutput(stderr) cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml") + backupDir := fs.String("backup-dir", dbbackup.DefaultDir, "where the pre-migration snapshot goes") + noBackup := fs.Bool("no-backup", false, "apply pending migrations without snapshotting the database first") // The "up" verb precedes any flags (felis migrate up -config path). Go's // flag.Parse stops at the first non-flag token and would never see a flag // placed after "up", silently falling back to the default -config. Pull the @@ -48,6 +57,18 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int { return 1 } + if !*noBackup { + path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database.URL, *backupDir, stderr) + if err != nil { + fmt.Fprintf(stderr, "felis migrate: pre-migration backup failed, nothing applied: %v\n", err) + fmt.Fprintln(stderr, " fix the backup, or re-run with -no-backup to migrate without one") + return 1 + } + if path != "" { + fmt.Fprintf(stdout, "felis migrate: database snapshot %s\n", path) + } + } + applied, err := store.Up(ctx, drv, migrations) if err != nil { fmt.Fprintf(stderr, "felis migrate: %v\n", err) @@ -60,3 +81,33 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int { } return 0 } + +// preMigrateBackup bundles the database when it already carries a schema and +// some of migrations are not applied yet, and returns the bundle's path ("" when +// there was nothing to protect: a fresh database, or nothing pending). +func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, dbURL, dir string, log io.Writer) (string, error) { + if err := drv.EnsureVersionTable(ctx); err != nil { + return "", fmt.Errorf("ensure version table: %w", err) + } + done, err := drv.AppliedVersions(ctx) + if err != nil { + return "", fmt.Errorf("read applied versions: %w", err) + } + if len(done) == 0 || !hasPending(done, migrations) { + return "", nil + } + return dbbackup.Backup(ctx, dbbackup.BackupOptions{ + DatabaseURL: dbURL, Dir: dir, Label: dbbackup.LabelPreMigrate, + Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir, + Version: resolvedVersion(), Log: log, Record: true, + }) +} + +func hasPending(done map[int]struct{}, migrations []store.Migration) bool { + for _, m := range migrations { + if _, ok := done[m.Version]; !ok { + return true + } + } + return false +} diff --git a/cmd/felis/run.go b/cmd/felis/run.go index ed42a52..62be2ab 100644 --- a/cmd/felis/run.go +++ b/cmd/felis/run.go @@ -11,7 +11,8 @@ Usage: felis [flags] Commands: - migrate up Apply embedded database migrations under an advisory lock + migrate up Apply embedded database migrations under an advisory lock (snapshots the database first) + db Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check) operator Run the MinecraftServer controller-manager api Run the felis-api HTTP server nano Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane) @@ -44,6 +45,7 @@ Run "felis -h" for command-specific flags. // subcommand, and listing them would make the table disagree with the command list. var commands = map[string]func(args []string, stdout, stderr io.Writer) int{ "migrate": cmdMigrate, + "db": cmdDB, "operator": cmdOperator, "api": cmdAPI, "nano": cmdNano, diff --git a/cmd/felis/tui_migration.go b/cmd/felis/tui_migration.go index ebe1ae9..df7480e 100644 --- a/cmd/felis/tui_migration.go +++ b/cmd/felis/tui_migration.go @@ -3,8 +3,10 @@ package main import ( "context" "fmt" + "io" "time" + "felis.lolicon.best/internal/dbbackup" "felis.lolicon.best/internal/store" ) @@ -12,7 +14,7 @@ import ( // applied count. Used by the preflight stage to self-heal a freshly bootstrapped // (or upgraded) database. func applyMigrations(dbURL string) (int, error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() drv, err := store.Open(ctx, dbURL) if err != nil { @@ -23,6 +25,11 @@ func applyMigrations(dbURL string) (int, error) { if err != nil { return 0, err } + // Same guard as `felis migrate up`: never roll a populated database forward + // without a snapshot to roll back to. + if _, err := preMigrateBackup(ctx, drv, migrations, dbURL, dbbackup.DefaultDir, io.Discard); err != nil { + return 0, fmt.Errorf("pre-migration backup: %w", err) + } if _, err := store.Up(ctx, drv, migrations); err != nil { return 0, err } diff --git a/deploy/alerts/felis-alerts.yaml b/deploy/alerts/felis-alerts.yaml index c83d8dc..f4fc9c0 100644 --- a/deploy/alerts/felis-alerts.yaml +++ b/deploy/alerts/felis-alerts.yaml @@ -5,6 +5,7 @@ # felis_* series come from two processes: # - felis-operator pod :8080/metrics → felis_servers_total, felis_start_duration_seconds # - felis-api internal :8081/metrics → felis_image_build_failures_total +# - node-exporter textfile collector → felis_db_backup_* (felis-db-backup.timer) # node_* / kube_* series come from node-exporter / kube-state-metrics. groups: - name: felis.rules @@ -67,3 +68,29 @@ groups: description: >- PostgreSQL, the control plane, the registry and game servers share one node; sustained memory pressure risks OOM kills. + - name: felis.backup.rules + rules: + - alert: FelisDBBackupStale + expr: time() - max(felis_db_backup_last_success_timestamp_seconds) > 26 * 3600 + for: 10m + labels: + severity: critical + annotations: + summary: "no control-plane database backup in over 26h" + description: >- + felis-db-backup.timer runs daily; the newest bundle is more than a day + old. Read `journalctl -u felis-db-backup` on the host, then take one now + with `sudo felis db backup` (troubleshooting §16). + - alert: FelisDBBackupMetricMissing + expr: absent(felis_db_backup_last_success_timestamp_seconds) + for: 2h + labels: + severity: warning + annotations: + summary: "database backup freshness is not being scraped" + description: >- + No felis_db_backup_last_success_timestamp_seconds series, so + FelisDBBackupStale cannot fire. Point node-exporter's + --collector.textfile.directory at the directory of + FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) + (troubleshooting §16). diff --git a/deploy/alerts/felis-alerts_test.yml b/deploy/alerts/felis-alerts_test.yml index d15e00e..24071ef 100644 --- a/deploy/alerts/felis-alerts_test.yml +++ b/deploy/alerts/felis-alerts_test.yml @@ -105,3 +105,49 @@ tests: description: >- PostgreSQL, the control plane, the registry and game servers share one node; sustained memory pressure risks OOM kills. + - name: database backup freshness + interval: 1m + input_series: + # The newest bundle was taken at t=0 and none since. + - series: 'felis_db_backup_last_success_timestamp_seconds{instance="node1",job="node-exporter",label="daily"}' + values: '0x1630' + alert_rule_test: + - eval_time: 25h + alertname: FelisDBBackupStale + exp_alerts: [] + - eval_time: 27h + alertname: FelisDBBackupStale + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + summary: "no control-plane database backup in over 26h" + description: >- + felis-db-backup.timer runs daily; the newest bundle is more than a day + old. Read `journalctl -u felis-db-backup` on the host, then take one now + with `sudo felis db backup` (troubleshooting §16). + - eval_time: 27h + alertname: FelisDBBackupMetricMissing + exp_alerts: [] + - name: database backup freshness not scraped + interval: 1m + input_series: + - series: 'up{job="node-exporter"}' + values: '1x200' + alert_rule_test: + - eval_time: 1h + alertname: FelisDBBackupMetricMissing + exp_alerts: [] + - eval_time: 3h + alertname: FelisDBBackupMetricMissing + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + summary: "database backup freshness is not being scraped" + description: >- + No felis_db_backup_last_success_timestamp_seconds series, so + FelisDBBackupStale cannot fire. Point node-exporter's + --collector.textfile.directory at the directory of + FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) + (troubleshooting §16). diff --git a/deploy/alerts/felis-prometheusrule.yaml b/deploy/alerts/felis-prometheusrule.yaml index 6967395..01cb6bb 100644 --- a/deploy/alerts/felis-prometheusrule.yaml +++ b/deploy/alerts/felis-prometheusrule.yaml @@ -72,3 +72,29 @@ spec: description: >- PostgreSQL, the control plane, the registry and game servers share one node; sustained memory pressure risks OOM kills. + - name: felis.backup.rules + rules: + - alert: FelisDBBackupStale + expr: time() - max(felis_db_backup_last_success_timestamp_seconds) > 26 * 3600 + for: 10m + labels: + severity: critical + annotations: + summary: "no control-plane database backup in over 26h" + description: >- + felis-db-backup.timer runs daily; the newest bundle is more than a day + old. Read `journalctl -u felis-db-backup` on the host, then take one now + with `sudo felis db backup` (troubleshooting §16). + - alert: FelisDBBackupMetricMissing + expr: absent(felis_db_backup_last_success_timestamp_seconds) + for: 2h + labels: + severity: warning + annotations: + summary: "database backup freshness is not being scraped" + description: >- + No felis_db_backup_last_success_timestamp_seconds series, so + FelisDBBackupStale cannot fire. Point node-exporter's + --collector.textfile.directory at the directory of + FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) + (troubleshooting §16). diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 6ed6f16..7daebc7 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -140,6 +140,19 @@ FELIS_ARCHIVE_LOCAL_PATH="${FELIS_ARCHIVE_LOCAL_PATH:-/var/lib/felis/archives}" # from its volumeName. Left unset, no reaper CronJob renders and archives accumulate until # the backup PVC fills (then backups fail loudly; nothing is deleted). FELIS_WORLDS_HOST_PATH="${FELIS_WORLDS_HOST_PATH:-}" +# Control-plane database backups (felis db backup): a daily timer bundles pg_dump with the +# /etc/felis state a rebuild needs, and every upgrade that has migrations to apply snapshots +# the database first (felis migrate up). The directory sits outside /var/lib/rancher on +# purpose: reinstalling k3s must not take the database backups with it. Copy it off the +# host for anything beyond "undo a bad upgrade or a mistaken delete" (troubleshooting §16). +FELIS_DB_BACKUP_DIR="${FELIS_DB_BACKUP_DIR:-/var/lib/felis/db-backups}" +FELIS_DB_BACKUP_KEEP="${FELIS_DB_BACKUP_KEEP:-14}" +FELIS_DB_BACKUP_TIME="${FELIS_DB_BACKUP_TIME:-*-*-* 03:30:00}" +# node-exporter textfile collector target; FelisDBBackupStale (deploy/alerts) reads it. +FELIS_DB_BACKUP_METRICS="${FELIS_DB_BACKUP_METRICS:-/var/lib/node_exporter/textfile_collector/felis_db_backup.prom}" +# 0 migrates without the pre-migration snapshot, e.g. against an external database newer +# than this host's pg_dump. The upgrade stops if the snapshot fails and this is not set. +FELIS_PRE_MIGRATE_BACKUP="${FELIS_PRE_MIGRATE_BACKUP:-1}" INSTALL_MODE="${FELIS_INSTALL_MODE:-}" # Loopback by default: hasJoined is an unauthenticated endpoint by protocol (Velocity # sends no token), so a public bind is a free auth relay — anyone can point their own @@ -237,6 +250,8 @@ HOST_BIN="/usr/local/bin/felis" # version sits here, and an operator's Go at the conventional path is not ours to swap. GOROOT_DIR="/opt/felis/go" NANO_SERVICE="/etc/systemd/system/felis-nano.service" +DB_BACKUP_SERVICE="/etc/systemd/system/felis-db-backup.service" +DB_BACKUP_TIMER="/etc/systemd/system/felis-db-backup.timer" VELOCITY_DIR="/opt/felis/velocity" VELOCITY_USER="felis-velocity" VELOCITY_SERVICE="/etc/systemd/system/felis-velocity.service" @@ -2364,13 +2379,62 @@ ensure_default_config() { # 8. Migrate + deploy bundle # --------------------------------------------------------------------------- run_migrations() { + local backup_flags=(-backup-dir "$FELIS_DB_BACKUP_DIR") write_felis_toml "${STATE_DIR}/felis.host.toml" "127.0.0.1" ensure_default_config + if [ "$FELIS_PRE_MIGRATE_BACKUP" = 0 ]; then + warn "FELIS_PRE_MIGRATE_BACKUP=0: pending migrations run without a database snapshot" + backup_flags=(-no-backup) + fi + # Migrations only roll forward. On an existing database with migrations pending, the + # binary bundles the database into FELIS_DB_BACKUP_DIR first and refuses to migrate + # if that fails; a fresh database has nothing to protect and is migrated directly. log "running database migrations (host binary -> 127.0.0.1)" - "$HOST_BIN" migrate up -config "${STATE_DIR}/felis.host.toml" + "$HOST_BIN" migrate up -config "${STATE_DIR}/felis.host.toml" "${backup_flags[@]}" ok "migrations applied" } +# The daily database backup. The first run happens now, so a broken pipeline (pg_dump +# missing, directory unwritable) shows up in this install rather than in the first +# restore someone needs. +install_db_backup_timer() { + install -d -m 0700 "$FELIS_DB_BACKUP_DIR" + cat > "$DB_BACKUP_SERVICE" < "$DB_BACKUP_TIMER" <&2 || true + warn "the first database backup failed (log above); fix it before relying on the daily timer: sudo systemctl start felis-db-backup.service" + fi +} + deploy_bundle() { local had_api=0 had_operator=0 export KUBECONFIG=/etc/rancher/k3s/k3s.yaml @@ -2970,6 +3034,8 @@ main() { # After deploy_bundle: the proxy dials felis-api's internal ClusterIP, which does not # exist until the bundle is applied. install_velocity + # After deploy_bundle: the bundle's MinecraftServer export reads the cluster. + install_db_backup_timer mark_bootstrap_done summary } diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index 2beb2fd..c69a2a4 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -1017,6 +1017,70 @@ fi rm -f "$fnfile" +# --- database backups: the pre-migration snapshot and the daily timer -------------------- +# Migrations only roll forward, so an upgrade must hand `migrate up` the snapshot directory, +# and only an explicit FELIS_PRE_MIGRATE_BACKUP=0 may take that away. + +mblock="$(awk '/^run_migrations\(\) \{/,/^}/' "$BS")" +[ -n "$mblock" ] || { echo "FAIL: no run_migrations found in $BS"; exit 1; } +[ "$(printf '%s\n' "$mblock" | wc -l)" -lt 30 ] \ + || { echo "FAIL: the extracted block is not run_migrations -- did its closing brace move?"; exit 1; } + +run_migrate() { # FELIS_PRE_MIGRATE_BACKUP + FELIS_PRE_MIGRATE_BACKUP="$1" FELIS_DB_BACKUP_DIR=/var/lib/felis/db-backups STATE_DIR=/etc/felis \ + HOST_BIN=fakefelis bash -c ' + log() { :; }; ok() { :; }; warn() { printf "WARN: %s\n" "$*"; } + write_felis_toml() { :; }; ensure_default_config() { :; } + fakefelis() { printf "RUN: %s\n" "$*"; } + '"$mblock"' + run_migrations' 2>&1 +} + +out="$(run_migrate 1)" +expect "an upgrade snapshots into the backup dir" "RUN: migrate up -config /etc/felis/felis.host.toml -backup-dir /var/lib/felis/db-backups" "$out" +out="$(run_migrate 0)" +expect "FELIS_PRE_MIGRATE_BACKUP=0 opts out explicitly" "RUN: migrate up -config /etc/felis/felis.host.toml -no-backup" "$out" +expect "the opt-out is loud" "WARN: FELIS_PRE_MIGRATE_BACKUP=0" "$out" + +tblock="$(awk '/^install_db_backup_timer\(\) \{/,/^}/' "$BS")" +[ -n "$tblock" ] || { echo "FAIL: no install_db_backup_timer found in $BS"; exit 1; } +[ "$(printf '%s\n' "$tblock" | wc -l)" -lt 60 ] \ + || { echo "FAIL: the extracted block is not install_db_backup_timer -- did its closing brace move?"; exit 1; } + +tdir="$(mktemp -d)" +run_timer() { # exit status of the first backup + FIRST="$1" DB_BACKUP_SERVICE="$tdir/felis-db-backup.service" DB_BACKUP_TIMER="$tdir/felis-db-backup.timer" \ + FELIS_DB_BACKUP_DIR="$tdir/db-backups" FELIS_DB_BACKUP_KEEP=7 FELIS_DB_BACKUP_TIME='*-*-* 04:00:00' \ + FELIS_DB_BACKUP_METRICS=/var/lib/node_exporter/textfile_collector/felis_db_backup.prom \ + HOST_BIN=/usr/local/bin/felis STATE_DIR=/etc/felis bash -c ' + ok() { printf "OK: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; } + systemctl() { printf "SYSTEMCTL: %s\n" "$*"; [ "$1" != start ] || return "$FIRST"; } + journalctl() { printf "JOURNAL: pg_dump: connection refused\n"; } + '"$tblock"' + install_db_backup_timer' 2>&1 +} + +out="$(run_timer 0)" +unit="$(cat "$tdir/felis-db-backup.service")" +timer="$(cat "$tdir/felis-db-backup.timer")" +expect "the unit runs a daily-labelled backup with the configured retention" \ + "ExecStart=/usr/local/bin/felis db backup -config /etc/felis/felis.host.toml -dir $tdir/db-backups -label daily -keep 7 -metrics-file /var/lib/node_exporter/textfile_collector/felis_db_backup.prom" "$unit" +expect "the timer fires at the configured time" "OnCalendar=*-*-* 04:00:00" "$timer" +expect "a missed run (host off at 03:30) catches up at boot" "Persistent=true" "$timer" +expect "the timer is enabled" "SYSTEMCTL: enable --now felis-db-backup.timer" "$out" +expect "the first backup runs during the install" "SYSTEMCTL: start felis-db-backup.service" "$out" +expect "a working first backup is reported" "OK: database backups: daily" "$out" +if [ "$(stat -c %a "$tdir/db-backups" 2>/dev/null || stat -f %Lp "$tdir/db-backups")" = 700 ]; then + echo "PASS the backup directory is private" +else + echo "FAIL the backup directory must be 0700"; fails=$((fails + 1)) +fi + +out="$(run_timer 1)" +expect "a failed first backup shows its log" "JOURNAL: pg_dump: connection refused" "$out" +expect "a failed first backup is a loud warning" "WARN: the first database backup failed" "$out" +rm -rf "$tdir" + # --------------------------------------------------------------------------------------- if [ "$fails" -eq 0 ]; then echo "ALL PASS" diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 30f6f42..371b856 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -201,6 +201,49 @@ components: nullable: true description: Window end (RFC3339, exclusive), or null when unset. + DBBackupStatus: + type: object + description: > + The newest control-plane database backup the host recorded + (internal/api/handlers_dbbackup.go dbBackupView; the record itself is + internal/dbbackup Status, written by `felis db backup`). + required: [last, stale, max_age_seconds] + properties: + last: + type: object + nullable: true + description: Null until the first backup has been recorded. + required: [at, name, label, size_bytes, dir] + properties: + at: + type: string + format: date-time + description: When the bundle was written. + name: + type: string + description: Bundle file name, felis-db--