fix(backup): 手动备份按服冷却、每服保留上限与独立保留期,容量驱逐不再删除回收世界的唯一副本
This commit is contained in:
30 files changed
+801
-42
No files matched your search
@@ -24,6 +24,7 @@ import (
|
||||
"felis.lolicon.best/internal/panel"
|
||||
"felis.lolicon.best/internal/passkey"
|
||||
"felis.lolicon.best/internal/platform"
|
||||
"felis.lolicon.best/internal/reaper"
|
||||
"felis.lolicon.best/internal/restore"
|
||||
"felis.lolicon.best/internal/store"
|
||||
"felis.lolicon.best/internal/submit"
|
||||
@@ -262,6 +263,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
|
||||
// agree on what local auth knows.
|
||||
repo := api.NewPGRepo(drv.DB())
|
||||
|
||||
// The owner's on-demand backup levers come from [archive], the same keys the
|
||||
// backup Job and the reaper read. A malformed key leaves the defaults in
|
||||
// place here; the reaper Job fails on it and names it.
|
||||
rcfg, err := reaperConfig(cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis api: %v; using the default backup limits\n", err)
|
||||
rcfg = reaper.DefaultConfig()
|
||||
}
|
||||
|
||||
a := &api.API{
|
||||
Repo: repo,
|
||||
Cluster: api.NewK8sCluster(cl, cfg.K8s.Namespace),
|
||||
@@ -296,6 +306,10 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
|
||||
AdminHostname: cfg.Auth.AdminHostname,
|
||||
PanelHostname: cfg.Auth.PanelHostname,
|
||||
WakeCooldown: 30 * time.Second,
|
||||
// An owner may start one backup per server per manual_cooldown, and none
|
||||
// while the store is at max_local_bytes (data-durability-9).
|
||||
BackupCooldown: rcfg.ManualCooldown,
|
||||
BackupStoreCap: rcfg.MaxLocalBytes,
|
||||
// The user-modpack lane's per-user throttles: a create spaces out
|
||||
// review-queue rows, an upload spaces out (up to 1 GiB) context streams.
|
||||
// Separate keys, so the normal create→upload sequence stays immediate.
|
||||
|
||||
+37
-4
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"flag"
|
||||
@@ -52,8 +53,8 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintf(stderr, "felis backup: archive store %q is not implemented in this build (only tarLocal)\n", cfg.Archive.Store)
|
||||
return 1
|
||||
}
|
||||
// Reuse the reaper's retention derivation so an on-demand backup expires on the
|
||||
// same clock as an inactivity backup — one retention policy, not two.
|
||||
// The [archive] parse the reaper uses; an on-demand backup takes its
|
||||
// manual_retention and manual_keep.
|
||||
rcfg, err := reaperConfig(cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis backup: %v\n", err)
|
||||
@@ -72,6 +73,13 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
|
||||
|
||||
ctx := ctrl.SetupSignalHandler()
|
||||
|
||||
// The archive store shares the node's disk with every world and the
|
||||
// database: an owner's backup must not be what tips it into eviction.
|
||||
if err := backup.CheckRoom(cfg.Archive.LocalPath, *worldsRoot, backup.MinFreeAfter); err != nil {
|
||||
fmt.Fprintf(stderr, "felis backup: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
|
||||
@@ -92,9 +100,10 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
|
||||
BackupRef: string(ref),
|
||||
SizeBytes: size,
|
||||
Reason: "manual",
|
||||
ExpiresAt: time.Now().Add(rcfg.Retention),
|
||||
ExpiresAt: time.Now().Add(rcfg.ManualRetention),
|
||||
}
|
||||
if err := reaper.NewPGStore(drv.DB()).InsertBackup(ctx, rec); err != nil {
|
||||
st := reaper.NewPGStore(drv.DB())
|
||||
if err := st.InsertBackup(ctx, rec); err != nil {
|
||||
// The archive is written but unrecorded — an orphan the retention pass would
|
||||
// never expire. Delete it so a failed backup leaves no leaked bytes, mirroring
|
||||
// the reaper's archive-then-record atomicity.
|
||||
@@ -107,9 +116,33 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
|
||||
}
|
||||
|
||||
fmt.Fprintf(stdout, "felis backup: server=%s archived %d bytes to %s (backup %s)\n", *server, size, ref, rec.ID)
|
||||
pruneManualBackups(ctx, st, archiver, *server, rcfg.ManualKeep, stdout, stderr)
|
||||
return 0
|
||||
}
|
||||
|
||||
// pruneManualBackups keeps server's newest keep on-demand backups and removes
|
||||
// the rest, oldest first, so repeated backups of one world cannot fill the
|
||||
// shared archive store. The new backup is already recorded; a removal that
|
||||
// fails is reported and retried after the next backup.
|
||||
func pruneManualBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server string, keep int, stdout, stderr io.Writer) {
|
||||
excess, err := st.ExcessManualBackups(ctx, server, keep)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)
|
||||
return
|
||||
}
|
||||
for _, b := range excess {
|
||||
if err := archiver.Delete(ctx, backup.ArchiveRef(b.BackupRef)); err != nil {
|
||||
fmt.Fprintf(stderr, "felis backup: remove older backup %s: %v\n", b.ID, err)
|
||||
continue
|
||||
}
|
||||
if err := st.MarkBackupDeleted(ctx, b.ID, time.Now()); err != nil {
|
||||
fmt.Fprintf(stderr, "felis backup: record the removal of %s: %v\n", b.ID, err)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(stdout, "felis backup: removed older backup %s of %s (keeping the newest %d)\n", b.ID, server, keep)
|
||||
}
|
||||
}
|
||||
|
||||
// newBackupID mints a world_backups primary key, matching the reaper's "bk-"+hex
|
||||
// scheme so a manual and an inactivity backup are indistinguishable downstream.
|
||||
func newBackupID() string {
|
||||
|
||||
+23
-2
@@ -183,8 +183,9 @@ func (w *mailWarner) Warn(ctx context.Context, ownerID, server, remaining string
|
||||
}
|
||||
|
||||
// reaperConfig derives the reaper's retention windows from felis.toml. The 15d
|
||||
// idle deadline is fixed by §18; only the warning offsets, retention, and the
|
||||
// store soft-cap are configurable (§24).
|
||||
// idle deadline is fixed by §18; only the warning offsets, retention, the
|
||||
// store soft-cap and the on-demand backup bounds are configurable (§24). The
|
||||
// backup Job and felis-api read the manual_* bounds through it too.
|
||||
func reaperConfig(cfg *config.Config) (reaper.Config, error) {
|
||||
rc := reaper.DefaultConfig()
|
||||
if v := cfg.Archive.Retention; v != "" {
|
||||
@@ -212,6 +213,26 @@ func reaperConfig(cfg *config.Config) (reaper.Config, error) {
|
||||
}
|
||||
rc.MaxLocalBytes = b
|
||||
}
|
||||
if v := cfg.Archive.ManualRetention; v != "" {
|
||||
d, err := parseSpanDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
return rc, fmt.Errorf("[archive] manual_retention %q: want a positive span such as 30d", v)
|
||||
}
|
||||
rc.ManualRetention = d
|
||||
}
|
||||
switch n := cfg.Archive.ManualKeep; {
|
||||
case n < 0:
|
||||
return rc, fmt.Errorf("[archive] manual_keep %d: want 1 or more", n)
|
||||
case n > 0:
|
||||
rc.ManualKeep = n
|
||||
}
|
||||
if v := cfg.Archive.ManualCooldown; v != "" {
|
||||
d, err := parseSpanDuration(v)
|
||||
if err != nil || d < 0 {
|
||||
return rc, fmt.Errorf("[archive] manual_cooldown %q: want a span such as 10m (0s for none)", v)
|
||||
}
|
||||
rc.ManualCooldown = d
|
||||
}
|
||||
rc.RequireOffsite = cfg.Offsite.Enabled()
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
|
||||
"felis.lolicon.best/internal/config"
|
||||
"felis.lolicon.best/internal/reaper"
|
||||
)
|
||||
|
||||
@@ -46,6 +48,38 @@ func TestReportReaperRunFailsTheJob(t *testing.T) {
|
||||
// fail-closed miss. The stock local-path arm is derived from the live PVC's
|
||||
// volumeName — a name-based guess (glob) could tar a stale deleted PV's bytes and
|
||||
// then delete the current world, which is why it is read from the API instead.
|
||||
// TestReaperConfigManualKeys: the on-demand backup keys default to 30 days,
|
||||
// five per server and a ten-minute cooldown, accept overrides, and refuse
|
||||
// values that would keep nothing or throttle backwards.
|
||||
func TestReaperConfigManualKeys(t *testing.T) {
|
||||
rc, err := reaperConfig(&config.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rc.ManualRetention != 30*reaper.Day || rc.ManualKeep != 5 || rc.ManualCooldown != 10*time.Minute {
|
||||
t.Fatalf("defaults = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
|
||||
}
|
||||
rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
|
||||
ManualRetention: "7d", ManualKeep: 2, ManualCooldown: "0s"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rc.ManualRetention != 7*reaper.Day || rc.ManualKeep != 2 || rc.ManualCooldown != 0 {
|
||||
t.Fatalf("overrides = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
|
||||
}
|
||||
for _, bad := range []config.ArchiveConfig{
|
||||
{ManualRetention: "0d"},
|
||||
{ManualRetention: "soon"},
|
||||
{ManualKeep: -1},
|
||||
{ManualCooldown: "-5m"},
|
||||
{ManualCooldown: "often"},
|
||||
} {
|
||||
if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
|
||||
t.Errorf("%+v was accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWorldDir(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
|
||||
Reference in new issue
Block a user