From 983727d9c04532e843a235a4773824e5667518d5 Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Sun, 27 Sep 2026 09:16:00 +0800 Subject: [PATCH] =?UTF-8?q?fix(offsite):=20=E5=A4=8D=E5=88=B6=E6=96=B0?= =?UTF-8?q?=E5=BD=92=E6=A1=A3=E5=90=8E=E8=A1=A5=E6=89=93=E5=B9=B6=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=20DB=20=E5=8C=85=EF=BC=8C=E6=A1=B6=E5=86=85=E6=97=A0?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E7=9A=84=E5=BD=92=E6=A1=A3=E8=BF=87=E6=9C=80?= =?UTF-8?q?=E9=95=BF=E4=BF=9D=E7=95=99=E6=9C=9F=E5=90=8E=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/felis/db.go | 3 +- cmd/felis/offsite.go | 64 ++++++- cmd/felis/offsite_test.go | 64 +++++++ docs/troubleshooting.md | 34 +++- internal/dbbackup/dbbackup.go | 4 + internal/offsite/offsite_test.go | 20 +++ internal/offsite/pgcatalog.go | 16 +- internal/offsite/snapshot_test.go | 286 ++++++++++++++++++++++++++++++ internal/offsite/sync.go | 146 +++++++++++++-- internal/pgint/offsite_test.go | 60 +++++++ 10 files changed, 669 insertions(+), 28 deletions(-) create mode 100644 internal/offsite/snapshot_test.go create mode 100644 internal/pgint/offsite_test.go diff --git a/cmd/felis/db.go b/cmd/felis/db.go index e6cbab7..b7fc609 100644 --- a/cmd/felis/db.go +++ b/cmd/felis/db.go @@ -36,6 +36,7 @@ var defaultKeep = map[string]int{ dbbackup.LabelDaily: 14, dbbackup.LabelPreMigrate: 10, dbbackup.LabelPreRestore: 5, + dbbackup.LabelOffsite: 1, } // cmdDB implements `felis db`: logical backups of the control-plane database @@ -136,7 +137,7 @@ func libpqQuote(v string) string { func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int { cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml") label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never") - keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, manual all)") + keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, offsite 1, manual all)") stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`) noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle") metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)") diff --git a/cmd/felis/offsite.go b/cmd/felis/offsite.go index 362b023..0dae59d 100644 --- a/cmd/felis/offsite.go +++ b/cmd/felis/offsite.go @@ -211,6 +211,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)") backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`) dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`) + stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory bundled into the database bundle taken after archives are copied ("" for none)`) registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`) uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC through the cluster)") uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, `the submission uploads PVC, in the control-plane namespace ("" copies no uploads)`) @@ -225,7 +226,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int } st, lease := startRun(env.cfg, env.key, *statusFile, time.Now()) res, err := runOffsiteSync(cfg, env, offsiteSources{ - archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir, + archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir, stateDir: *stateDir, registry: offsiteRegistryEndpoint(*registry, cfg.Registry), uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC, }, &lease, stderr) @@ -299,12 +300,13 @@ func recordRun(st *offsite.Status, res offsite.Result, err error, lease offsite. } // offsiteSources is where one sync pass reads from: the world archive volume -// (archiveDir, or the backupPVC's directory), the bundle directory, the -// registry's loopback endpoint and the uploads volume (uploadsDir, or the -// uploadsPVC's directory). An empty source is skipped. +// (archiveDir, or the backupPVC's directory), the bundle directory (with the +// host state the pass bundles, stateDir), the registry's loopback endpoint and +// the uploads volume (uploadsDir, or the uploadsPVC's directory). An empty +// source is skipped. type offsiteSources struct { archiveDir, backupPVC string - dbDir string + dbDir, stateDir string registry string uploadsDir, uploadsPVC string } @@ -346,10 +348,8 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lea return offsite.Result{}, fmt.Errorf("open database: %w", err) } defer drv.Close() - s := &offsite.Syncer{ - Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key, - ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log, - } + s := offsiteSyncer(cfg, env, src, archiveDir, uploadsDir, lease, log) + s.Catalog = offsite.PGCatalog{DB: drv.DB()} if src.registry != "" { s.Images = newRegistryImages(src.registry) s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL) @@ -357,6 +357,52 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lea return s.Run(ctx) } +// offsiteSyncer is the pass runOffsiteSync runs over the resolved archive and +// uploads directories, before its catalog and registry are attached. It +// snapshots the database into the bundle directory after copying archives, and +// sweeps world objects no backup records once they outlive every retention in +// [archive]; a retention that does not parse sweeps none. +func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, archiveDir, uploadsDir string, lease *offsite.Lease, log io.Writer) *offsite.Syncer { + s := &offsite.Syncer{ + Bucket: env.bucket, Key: env.key, + ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log, + } + if src.dbDir != "" { + s.Snapshot = offsiteSnapshot(cfg.Database, src.dbDir, src.stateDir, log) + } + if rc, err := reaperConfig(cfg); err != nil { + fmt.Fprintf(log, "felis offsite: world objects no backup records are kept: %v\n", err) + } else { + s.OrphanAfter = max(rc.Retention, rc.ManualRetention, rc.ScheduledRetention) + } + return s +} + +// offsiteSnapshot takes the bundle a pass sends after copying world archives +// (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite, +// with the newest one kept in dir. It is not recorded for the panel, whose +// backup card watches felis-db-backup.timer: snapshots come only when archives +// are copied, and would hide a daily timer that stopped. +func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error { + return func(ctx context.Context) error { + tools, err := dbTools(db) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + defer cancel() + path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{ + DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite, + Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(), + ExportServers: exportMinecraftServers, Log: log, + }) + if err == nil { + fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path)) + } + return err + } +} + // volumeKind names a PVC the off-site copy reads or restores, for messages, // with the flag that bypasses finding it through the cluster. type volumeKind struct{ what, dirFlag, empty string } diff --git a/cmd/felis/offsite_test.go b/cmd/felis/offsite_test.go index 02c634c..d8390a3 100644 --- a/cmd/felis/offsite_test.go +++ b/cmd/felis/offsite_test.go @@ -673,3 +673,67 @@ func TestRestoredHostKeepsStandingBy(t *testing.T) { t.Errorf("upgraded host's status = %+v", st) } } + +// TestOffsiteSyncerSnapshotsAndSweeps: the pass `offsite sync` runs takes its +// snapshot the way `felis db backup` does, in the database pod, into the +// bundle directory, keeping the newest one there; and it sweeps unrecorded +// world objects only past the longest retention [archive] gives any backup. +func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) { + dir := newPodRig(t) + bundles := filepath.Join(dir, "bundles") + env := &offsiteEnv{cfg: config.OffsiteConfig{DBKeep: 5}} + var log bytes.Buffer + cfg := &config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "120d"}} + s := offsiteSyncer(cfg, env, offsiteSources{dbDir: bundles}, "/archives", "/uploads", nil, &log) + if s.DBDir != bundles || s.DBKeep != 5 || s.ArchiveDir != "/archives" || s.UploadsDir != "/uploads" { + t.Fatalf("syncer = %+v", s) + } + if s.OrphanAfter != 120*24*time.Hour { + t.Errorf("OrphanAfter = %s, want the 120d retention", s.OrphanAfter) + } + if s.Snapshot == nil { + t.Fatal("the pass takes no snapshot after copying archives") + } + for i := 0; i < 2; i++ { + if err := s.Snapshot(context.Background()); err != nil { + t.Fatalf("snapshot %d: %v", i, err) + } + } + got, err := dbbackup.List(bundles) + if err != nil || len(got) != 1 || got[0].Label != dbbackup.LabelOffsite { + t.Fatalf("bundle directory = %+v, %v; want the newest offsite bundle alone", got, err) + } + if _, err := dbbackupVerify(got[0].Path); err != nil { + t.Fatalf("the snapshot does not verify: %v", err) + } + // The MinecraftServer objects are exported alongside, as in the daily bundle. + argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args")) + if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") || !strings.Contains(ran, "kubectl get minecraftservers") { + t.Errorf("k3s ran %q, want pg_dump in the pod and the server export", ran) + } + if !strings.Contains(log.String(), "took database bundle "+got[0].Name) { + t.Errorf("the snapshot is not logged:\n%s", log.String()) + } + + for _, c := range []struct { + archive config.ArchiveConfig + want time.Duration + }{ + {config.ArchiveConfig{}, 90 * 24 * time.Hour}, + {config.ArchiveConfig{ScheduledRetention: "200d"}, 200 * 24 * time.Hour}, + {config.ArchiveConfig{ManualRetention: "150d", Retention: "30d", ScheduledRetention: "60d"}, 150 * 24 * time.Hour}, + } { + s := offsiteSyncer(&config.Config{Database: podDB, Archive: c.archive}, env, offsiteSources{dbDir: bundles}, "", "", nil, io.Discard) + if s.OrphanAfter != c.want { + t.Errorf("%+v: OrphanAfter = %s, want %s", c.archive, s.OrphanAfter, c.want) + } + } + log.Reset() + s = offsiteSyncer(&config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "soon"}}, env, offsiteSources{}, "", "", nil, &log) + if s.OrphanAfter != 0 || !strings.Contains(log.String(), "world objects no backup records are kept") { + t.Errorf("a retention that does not parse: OrphanAfter %s, log %q; want no sweep, said", s.OrphanAfter, log.String()) + } + if s.Snapshot != nil { + t.Error("a pass that copies no bundles takes a snapshot") + } +} diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d196833..0a4b347 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2024,8 +2024,8 @@ along). One bundle is `felis-db--