fix(offsite): 复制新归档后补打并上传 DB 包，桶内无记录的归档过最长保留期后清理

cmd/felis/db.go

@@ -36,6 +36,7 @@ var defaultKeep = map[string]int{
 	dbbackup.LabelDaily:      14,
 	dbbackup.LabelPreMigrate: 10,
 	dbbackup.LabelPreRestore: 5,
+	dbbackup.LabelOffsite:    1,
 }
 
 // cmdDB implements `felis db`: logical backups of the control-plane database
@@ -136,7 +137,7 @@ func libpqQuote(v string) string {
 func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
 	label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never")
-	keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, manual all)")
+	keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, offsite 1, manual all)")
 	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`)
 	noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle")
 	metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)")

cmd/felis/offsite.go

@@ -211,6 +211,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
 	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
 	backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
 	dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory bundled into the database bundle taken after archives are copied ("" for none)`)
 	registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
 	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC through the cluster)")
 	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, `the submission uploads PVC, in the control-plane namespace ("" copies no uploads)`)
@@ -225,7 +226,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
 	}
 	st, lease := startRun(env.cfg, env.key, *statusFile, time.Now())
 	res, err := runOffsiteSync(cfg, env, offsiteSources{
-		archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir,
+		archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir, stateDir: *stateDir,
 		registry:   offsiteRegistryEndpoint(*registry, cfg.Registry),
 		uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
 	}, &lease, stderr)
@@ -299,12 +300,13 @@ func recordRun(st *offsite.Status, res offsite.Result, err error, lease offsite.
 }
 
 // offsiteSources is where one sync pass reads from: the world archive volume
-// (archiveDir, or the backupPVC's directory), the bundle directory, the
-// registry's loopback endpoint and the uploads volume (uploadsDir, or the
-// uploadsPVC's directory). An empty source is skipped.
+// (archiveDir, or the backupPVC's directory), the bundle directory (with the
+// host state the pass bundles, stateDir), the registry's loopback endpoint and
+// the uploads volume (uploadsDir, or the uploadsPVC's directory). An empty
+// source is skipped.
 type offsiteSources struct {
 	archiveDir, backupPVC  string
-	dbDir                  string
+	dbDir, stateDir        string
 	registry               string
 	uploadsDir, uploadsPVC string
 }
@@ -346,10 +348,8 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lea
 		return offsite.Result{}, fmt.Errorf("open database: %w", err)
 	}
 	defer drv.Close()
-	s := &offsite.Syncer{
-		Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key,
-		ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log,
-	}
+	s := offsiteSyncer(cfg, env, src, archiveDir, uploadsDir, lease, log)
+	s.Catalog = offsite.PGCatalog{DB: drv.DB()}
 	if src.registry != "" {
 		s.Images = newRegistryImages(src.registry)
 		s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL)
@@ -357,6 +357,52 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lea
 	return s.Run(ctx)
 }
 
+// offsiteSyncer is the pass runOffsiteSync runs over the resolved archive and
+// uploads directories, before its catalog and registry are attached. It
+// snapshots the database into the bundle directory after copying archives, and
+// sweeps world objects no backup records once they outlive every retention in
+// [archive]; a retention that does not parse sweeps none.
+func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, archiveDir, uploadsDir string, lease *offsite.Lease, log io.Writer) *offsite.Syncer {
+	s := &offsite.Syncer{
+		Bucket: env.bucket, Key: env.key,
+		ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log,
+	}
+	if src.dbDir != "" {
+		s.Snapshot = offsiteSnapshot(cfg.Database, src.dbDir, src.stateDir, log)
+	}
+	if rc, err := reaperConfig(cfg); err != nil {
+		fmt.Fprintf(log, "felis offsite: world objects no backup records are kept: %v\n", err)
+	} else {
+		s.OrphanAfter = max(rc.Retention, rc.ManualRetention, rc.ScheduledRetention)
+	}
+	return s
+}
+
+// offsiteSnapshot takes the bundle a pass sends after copying world archives
+// (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite,
+// with the newest one kept in dir. It is not recorded for the panel, whose
+// backup card watches felis-db-backup.timer: snapshots come only when archives
+// are copied, and would hide a daily timer that stopped.
+func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error {
+	return func(ctx context.Context) error {
+		tools, err := dbTools(db)
+		if err != nil {
+			return err
+		}
+		ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+		defer cancel()
+		path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
+			DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite,
+			Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(),
+			ExportServers: exportMinecraftServers, Log: log,
+		})
+		if err == nil {
+			fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path))
+		}
+		return err
+	}
+}
+
 // volumeKind names a PVC the off-site copy reads or restores, for messages,
 // with the flag that bypasses finding it through the cluster.
 type volumeKind struct{ what, dirFlag, empty string }

cmd/felis/offsite_test.go

@@ -673,3 +673,67 @@ func TestRestoredHostKeepsStandingBy(t *testing.T) {
 		t.Errorf("upgraded host's status = %+v", st)
 	}
 }
+
+// TestOffsiteSyncerSnapshotsAndSweeps: the pass `offsite sync` runs takes its
+// snapshot the way `felis db backup` does, in the database pod, into the
+// bundle directory, keeping the newest one there; and it sweeps unrecorded
+// world objects only past the longest retention [archive] gives any backup.
+func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) {
+	dir := newPodRig(t)
+	bundles := filepath.Join(dir, "bundles")
+	env := &offsiteEnv{cfg: config.OffsiteConfig{DBKeep: 5}}
+	var log bytes.Buffer
+	cfg := &config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "120d"}}
+	s := offsiteSyncer(cfg, env, offsiteSources{dbDir: bundles}, "/archives", "/uploads", nil, &log)
+	if s.DBDir != bundles || s.DBKeep != 5 || s.ArchiveDir != "/archives" || s.UploadsDir != "/uploads" {
+		t.Fatalf("syncer = %+v", s)
+	}
+	if s.OrphanAfter != 120*24*time.Hour {
+		t.Errorf("OrphanAfter = %s, want the 120d retention", s.OrphanAfter)
+	}
+	if s.Snapshot == nil {
+		t.Fatal("the pass takes no snapshot after copying archives")
+	}
+	for i := 0; i < 2; i++ {
+		if err := s.Snapshot(context.Background()); err != nil {
+			t.Fatalf("snapshot %d: %v", i, err)
+		}
+	}
+	got, err := dbbackup.List(bundles)
+	if err != nil || len(got) != 1 || got[0].Label != dbbackup.LabelOffsite {
+		t.Fatalf("bundle directory = %+v, %v; want the newest offsite bundle alone", got, err)
+	}
+	if _, err := dbbackupVerify(got[0].Path); err != nil {
+		t.Fatalf("the snapshot does not verify: %v", err)
+	}
+	// The MinecraftServer objects are exported alongside, as in the daily bundle.
+	argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args"))
+	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") || !strings.Contains(ran, "kubectl get minecraftservers") {
+		t.Errorf("k3s ran %q, want pg_dump in the pod and the server export", ran)
+	}
+	if !strings.Contains(log.String(), "took database bundle "+got[0].Name) {
+		t.Errorf("the snapshot is not logged:\n%s", log.String())
+	}
+
+	for _, c := range []struct {
+		archive config.ArchiveConfig
+		want    time.Duration
+	}{
+		{config.ArchiveConfig{}, 90 * 24 * time.Hour},
+		{config.ArchiveConfig{ScheduledRetention: "200d"}, 200 * 24 * time.Hour},
+		{config.ArchiveConfig{ManualRetention: "150d", Retention: "30d", ScheduledRetention: "60d"}, 150 * 24 * time.Hour},
+	} {
+		s := offsiteSyncer(&config.Config{Database: podDB, Archive: c.archive}, env, offsiteSources{dbDir: bundles}, "", "", nil, io.Discard)
+		if s.OrphanAfter != c.want {
+			t.Errorf("%+v: OrphanAfter = %s, want %s", c.archive, s.OrphanAfter, c.want)
+		}
+	}
+	log.Reset()
+	s = offsiteSyncer(&config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "soon"}}, env, offsiteSources{}, "", "", nil, &log)
+	if s.OrphanAfter != 0 || !strings.Contains(log.String(), "world objects no backup records are kept") {
+		t.Errorf("a retention that does not parse: OrphanAfter %s, log %q; want no sweep, said", s.OrphanAfter, log.String())
+	}
+	if s.Snapshot != nil {
+		t.Error("a pass that copies no bundles takes a snapshot")
+	}
+}