package reaper import ( "context" "errors" "fmt" "io" "log/slog" "reflect" "sort" "testing" "time" "felis.lolicon.best/internal/backup" "felis.lolicon.best/internal/metrics" "github.com/prometheus/client_golang/prometheus/testutil" ) // testNow is the frozen clock for every hermetic case. Idle is expressed as an // offset back from here. var testNow = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) func idleBy(d time.Duration) time.Time { return testNow.Add(-d) } // recorder captures the cross-fake call order so a test can assert the strict // archive→insert→deletePVC→release→stop→audit sequence of red line ④. type recorder struct{ events []string } func (r *recorder) add(e string) { r.events = append(r.events, e) } // ---- fake backup.WorldArchiver ------------------------------------------- type fakeArchiver struct { rec *recorder archiveErr error deleteErr error archives int deletes []backup.ArchiveRef seq int } func (f *fakeArchiver) Archive(_ context.Context, server, _ string) (backup.ArchiveRef, int64, error) { if f.archiveErr != nil { return "", 0, f.archiveErr } f.archives++ f.seq++ f.rec.add("archive") return backup.ArchiveRef(fmt.Sprintf("ref-%s-%d", server, f.seq)), 10, nil } func (f *fakeArchiver) Restore(context.Context, backup.ArchiveRef, string) error { return nil } func (f *fakeArchiver) Delete(_ context.Context, ref backup.ArchiveRef) error { if f.deleteErr != nil { return f.deleteErr } f.deletes = append(f.deletes, ref) f.rec.add("delete") return nil } // ---- fake Cluster --------------------------------------------------------- type fakeCluster struct { rec *recorder crds map[string]ServerCRD inspectErr map[string]error deletePVCErr error deletePVCCalls int deletedPVCs []string stopped []string } func (c *fakeCluster) Inspect(_ context.Context, name string) (ServerCRD, error) { if e := c.inspectErr[name]; e != nil { return ServerCRD{}, e } crd, ok := c.crds[name] if !ok { return ServerCRD{}, ErrNotFound } return crd, nil } func (c *fakeCluster) DeletePVC(_ context.Context, pvc string) error { c.deletePVCCalls++ if c.deletePVCErr != nil { return c.deletePVCErr } c.deletedPVCs = append(c.deletedPVCs, pvc) c.rec.add("deletePVC") return nil } func (c *fakeCluster) Stop(_ context.Context, name string) error { c.stopped = append(c.stopped, name) c.rec.add("stop") return nil } // ---- fake Store ----------------------------------------------------------- type fakeBackup struct { id, server, ref string size int64 status string // present | deleted createdAt, expires time.Time } type fakeStore struct { rec *recorder clock time.Time order []string byName map[string]*Candidate backups []*fakeBackup audits []AuditRecord released []string listErr error insertErr error idn int } func (s *fakeStore) ListActiveServers(context.Context) ([]Candidate, error) { if s.listErr != nil { return nil, s.listErr } out := make([]Candidate, 0, len(s.order)) for _, n := range s.order { out = append(out, *s.byName[n]) } return out, nil } func (s *fakeStore) FreshBackup(_ context.Context, server string, since time.Time) (string, bool, error) { for _, b := range s.backups { if b.server == server && b.status == "present" && !b.createdAt.Before(since) { return b.ref, true, nil } } return "", false, nil } func (s *fakeStore) InsertBackup(_ context.Context, rec BackupRecord) error { if s.insertErr != nil { return s.insertErr } s.backups = append(s.backups, &fakeBackup{ id: rec.ID, server: rec.ServerName, ref: rec.BackupRef, size: rec.SizeBytes, status: "present", createdAt: s.clock, expires: rec.ExpiresAt, }) s.rec.add("insert") return nil } func (s *fakeStore) ReleaseWorld(_ context.Context, name string, at time.Time) error { c := s.byName[name] c.OwnerID = "" c.LastActiveAt = at c.Warned3dAt = time.Time{} c.Warned1dAt = time.Time{} s.released = append(s.released, name) s.rec.add("release") return nil } func (s *fakeStore) MarkWarned(_ context.Context, name string, tier Tier, at time.Time) error { c := s.byName[name] if tier == Tier1d { c.Warned1dAt = at } else { c.Warned3dAt = at } return nil } func (s *fakeStore) PresentBackupBytes(context.Context) (int64, error) { var total int64 for _, b := range s.backups { if b.status == "present" { total += b.size } } return total, nil } func (s *fakeStore) OldestPresentBackups(context.Context) ([]StoredBackup, error) { var ps []*fakeBackup for _, b := range s.backups { if b.status == "present" { ps = append(ps, b) } } sort.Slice(ps, func(i, j int) bool { return ps[i].createdAt.Before(ps[j].createdAt) }) out := make([]StoredBackup, 0, len(ps)) for _, b := range ps { out = append(out, StoredBackup{ID: b.id, ServerName: b.server, BackupRef: b.ref, SizeBytes: b.size}) } return out, nil } func (s *fakeStore) ListExpiredBackups(_ context.Context, now time.Time) ([]StoredBackup, error) { var out []StoredBackup for _, b := range s.backups { if b.status == "present" && b.expires.Before(now) { out = append(out, StoredBackup{ID: b.id, ServerName: b.server, BackupRef: b.ref, SizeBytes: b.size}) } } return out, nil } func (s *fakeStore) MarkBackupDeleted(_ context.Context, id string, at time.Time) error { for _, b := range s.backups { if b.id == id { b.status = "deleted" return nil } } return fmt.Errorf("no backup %s", id) } func (s *fakeStore) Audit(_ context.Context, rec AuditRecord) error { s.audits = append(s.audits, rec) s.rec.add("audit:" + rec.Action) return nil } // ---- fixture -------------------------------------------------------------- func newReaper(cfg Config, cands ...Candidate) (*Reaper, *fakeStore, *fakeCluster, *fakeArchiver) { rec := &recorder{} st := &fakeStore{rec: rec, clock: testNow, byName: map[string]*Candidate{}} cl := &fakeCluster{rec: rec, crds: map[string]ServerCRD{}, inspectErr: map[string]error{}} for i := range cands { cc := cands[i] st.byName[cc.Name] = &cc st.order = append(st.order, cc.Name) cl.crds[cc.Name] = ServerCRD{PVC: "world-" + cc.Name + "-0"} } ar := &fakeArchiver{rec: rec} r := &Reaper{ Cfg: cfg, Store: st, Cluster: cl, Archiver: ar, Log: slog.New(slog.NewTextHandler(io.Discard, nil)), Now: func() time.Time { return testNow }, IDGen: func() string { st.idn++; return fmt.Sprintf("bk-%d", st.idn) }, } return r, st, cl, ar } func mustRun(t *testing.T, r *Reaper) Summary { t.Helper() sum, err := r.RunOnce(context.Background()) if err != nil { t.Fatalf("RunOnce: %v", err) } return sum } // ---- tests ---------------------------------------------------------------- // Red line ①: a reaperExempt server is never archived, deleted, or warned no // matter how idle it is — losing the lobby would be total ingress loss. func TestReapExemptServerNeverTouched(t *testing.T) { r, st, cl, ar := newReaper(DefaultConfig(), Candidate{Name: "lobby", OwnerID: "", LastActiveAt: idleBy(100 * Day)}) cl.crds["lobby"] = ServerCRD{Exempt: true, PVC: "world-lobby-0"} sum := mustRun(t, r) if sum.WorldsReaped != 0 || sum.Skipped != 0 || ar.archives != 0 || cl.deletePVCCalls != 0 { t.Fatalf("exempt server was touched: %+v archives=%d deletePVC=%d", sum, ar.archives, cl.deletePVCCalls) } if st.byName["lobby"].OwnerID != "" || len(st.released) != 0 { t.Fatalf("exempt server ownership mutated") } } // The happy path, asserting the strict ordering of red line ④ and that the // reap audit carries former_owner (spec §18 audit(reap_world, s, former_owner)). func TestReapIdleWorldFullSequence(t *testing.T) { r, st, cl, _ := newReaper(DefaultConfig(), Candidate{Name: "alpha", OwnerID: "user-7", LastActiveAt: idleBy(20 * Day)}) sum := mustRun(t, r) if sum.WorldsReaped != 1 { t.Fatalf("WorldsReaped = %d, want 1", sum.WorldsReaped) } want := []string{"archive", "insert", "deletePVC", "release", "stop", "audit:" + ActionReapWorld} if !reflect.DeepEqual(st.rec.events, want) { t.Fatalf("call order = %v, want %v", st.rec.events, want) } if got := cl.deletedPVCs; len(got) != 1 || got[0] != "world-alpha-0" { t.Fatalf("deleted PVCs = %v, want [world-alpha-0]", got) } // Backup carries the former owner and a retention deadline 3mo out. if len(st.backups) != 1 || st.backups[0].server != "alpha" { t.Fatalf("backup not recorded: %+v", st.backups) } if want := testNow.Add(DefaultConfig().Retention); !st.backups[0].expires.Equal(want) { t.Fatalf("backup expires = %v, want %v", st.backups[0].expires, want) } if len(st.audits) != 1 || st.audits[0].FormerOwner != "user-7" || st.audits[0].Action != ActionReapWorld { t.Fatalf("audit = %+v, want reap_world former_owner=user-7", st.audits) } // Red line ②: the row survives (ownership released, not deleted). c := st.byName["alpha"] if c.OwnerID != "" || !c.LastActiveAt.Equal(testNow) || !c.Warned3dAt.IsZero() { t.Fatalf("post-reap state wrong: %+v", c) } } // §23 instrumentation: felis_reaper_worlds_deleted_total advances by exactly one // per world whose PVC is actually deleted — in lockstep with Summary.WorldsReaped, // and never for a skipped/preserved world. Asserted as a delta because the counter // is a process-global singleton other tests in this package also advance. func TestReapIncrementsDeletedWorldsMetric(t *testing.T) { before := testutil.ToFloat64(metrics.ReaperWorldsDeletedTotal) r, _, cl, _ := newReaper(DefaultConfig(), Candidate{Name: "metric-a", OwnerID: "user-9", LastActiveAt: idleBy(20 * Day)}, Candidate{Name: "metric-b", OwnerID: "user-9", LastActiveAt: idleBy(20 * Day)}, // Fresh server: under the deadline, must NOT be reaped or counted. Candidate{Name: "metric-fresh", OwnerID: "user-9", LastActiveAt: idleBy(2 * Day)}, ) sum := mustRun(t, r) if sum.WorldsReaped != 2 { t.Fatalf("WorldsReaped = %d, want 2", sum.WorldsReaped) } if cl.deletePVCCalls != 2 { t.Fatalf("deletePVC calls = %d, want 2", cl.deletePVCCalls) } delta := testutil.ToFloat64(metrics.ReaperWorldsDeletedTotal) - before if delta != float64(sum.WorldsReaped) { t.Fatalf("felis_reaper_worlds_deleted_total advanced by %v, want %d (one per reaped world)", delta, sum.WorldsReaped) } } // CENTERPIECE — red line ④: when the archive fails, the PVC is never deleted, // ownership is untouched, no backup row is written, and the same server is // retried (state unchanged) on the next run. func TestReapArchiveFailurePreservesWorld(t *testing.T) { r, st, cl, ar := newReaper(DefaultConfig(), Candidate{Name: "beta", OwnerID: "user-1", LastActiveAt: idleBy(20 * Day)}) ar.archiveErr = errors.New("backend offline") sum := mustRun(t, r) if sum.WorldsReaped != 0 || sum.Skipped != 1 { t.Fatalf("summary = %+v, want 0 reaped / 1 skipped", sum) } if cl.deletePVCCalls != 0 { t.Fatalf("DeletePVC was called %d times despite archive failure", cl.deletePVCCalls) } if len(st.backups) != 0 { t.Fatalf("a backup row was written despite archive failure: %+v", st.backups) } if len(st.released) != 0 { t.Fatalf("ReleaseWorld ran despite archive failure") } c := st.byName["beta"] if c.OwnerID != "user-1" || !c.LastActiveAt.Equal(idleBy(20*Day)) { t.Fatalf("server state changed despite archive failure: %+v", c) } // Recovery: backend returns, next run reaps cleanly. ar.archiveErr = nil sum2 := mustRun(t, r) if sum2.WorldsReaped != 1 || cl.deletePVCCalls != 1 { t.Fatalf("recovery run: summary=%+v deletePVC=%d, want 1 reaped / 1 delete", sum2, cl.deletePVCCalls) } } // Red line ④ (recording arm): if the archive succeeds but recording it fails, // the orphan archive is cleaned up and the PVC is still never deleted. func TestReapInsertBackupFailurePreservesWorld(t *testing.T) { r, st, cl, ar := newReaper(DefaultConfig(), Candidate{Name: "gamma", OwnerID: "user-2", LastActiveAt: idleBy(20 * Day)}) st.insertErr = errors.New("db down") sum := mustRun(t, r) if sum.WorldsReaped != 0 || sum.Skipped != 1 { t.Fatalf("summary = %+v, want 0 reaped / 1 skipped", sum) } if cl.deletePVCCalls != 0 { t.Fatalf("DeletePVC called despite insert failure") } if ar.archives != 1 || len(ar.deletes) != 1 { t.Fatalf("orphan archive not cleaned up: archives=%d deletes=%d", ar.archives, len(ar.deletes)) } if len(st.backups) != 0 { t.Fatalf("backup row present despite insert failure") } } // Idempotency (the deliberate disk-growth choice): a DeletePVC failure leaves a // recorded backup; the retry reuses it via FreshBackup instead of writing a // duplicate archive. func TestReapDeletePVCFailureIsIdempotent(t *testing.T) { r, st, cl, ar := newReaper(DefaultConfig(), Candidate{Name: "delta", OwnerID: "user-3", LastActiveAt: idleBy(20 * Day)}) cl.deletePVCErr = errors.New("apiserver timeout") sum := mustRun(t, r) if sum.WorldsReaped != 0 || sum.Skipped != 1 { t.Fatalf("run1 summary = %+v, want 0 reaped / 1 skipped", sum) } if ar.archives != 1 || len(st.backups) != 1 { t.Fatalf("run1: archives=%d backups=%d, want 1/1", ar.archives, len(st.backups)) } // apiserver recovers; the retry must NOT re-archive. cl.deletePVCErr = nil sum2 := mustRun(t, r) if sum2.WorldsReaped != 1 { t.Fatalf("run2 WorldsReaped = %d, want 1", sum2.WorldsReaped) } if ar.archives != 1 { t.Fatalf("retry re-archived: archives=%d, want 1 (reuse via FreshBackup)", ar.archives) } if len(st.backups) != 1 { t.Fatalf("retry duplicated the backup row: %d rows, want 1", len(st.backups)) } } // Red line ⑤ (reap arm): an unowned server is still reaped on time; the audit // records an empty former_owner. func TestReapUnownedServerStillReaped(t *testing.T) { r, st, cl, _ := newReaper(DefaultConfig(), Candidate{Name: "orphan", OwnerID: "", LastActiveAt: idleBy(20 * Day)}) sum := mustRun(t, r) if sum.WorldsReaped != 1 || cl.deletePVCCalls != 1 { t.Fatalf("unowned server not reaped: %+v deletePVC=%d", sum, cl.deletePVCCalls) } if len(st.audits) != 1 || st.audits[0].FormerOwner != "" { t.Fatalf("audit former_owner = %q, want empty", st.audits[0].FormerOwner) } } func TestNoReapBeforeDeadline(t *testing.T) { r, _, cl, ar := newReaper(DefaultConfig(), Candidate{Name: "fresh", OwnerID: "user-4", LastActiveAt: idleBy(10 * Day)}) sum := mustRun(t, r) if sum.WorldsReaped != 0 || sum.Warned != 0 || ar.archives != 0 || cl.deletePVCCalls != 0 { t.Fatalf("acted on a 10d-idle server with a 15d deadline: %+v", sum) } } // Warning thresholds are DERIVED from the deadline, not hardcoded. Using a // non-default 10d deadline, warnings must fire at 10d-3d=7d and 10d-1d=9d, in // elif precedence, deduped per tier, and never for an unowned server. func TestWarningsDerivedFromNonDefaultDeadline(t *testing.T) { cfg := Config{ IdleBeforeReap: 10 * Day, WarnBefore: []time.Duration{1 * Day, 3 * Day}, // intentionally unsorted Retention: 90 * Day, } r, st, cl, _ := newReaper(cfg, // past 7d, under 9d, nothing sent -> 3d (Tier3d) warning Candidate{Name: "a", OwnerID: "u-a", LastActiveAt: idleBy(8 * Day)}, // past 9d, 3d already sent -> 1d (Tier1d) warning Candidate{Name: "b", OwnerID: "u-b", LastActiveAt: idleBy(95 * Day / 10), Warned3dAt: idleBy(2 * Day)}, // past 7d, 3d already sent, under 9d -> no new warning (dedup) Candidate{Name: "c", OwnerID: "u-c", LastActiveAt: idleBy(8 * Day), Warned3dAt: idleBy(2 * Day)}, // under 7d -> no warning yet Candidate{Name: "d", OwnerID: "u-d", LastActiveAt: idleBy(6 * Day)}, // past 7d but unowned -> never warned (red line ⑤) Candidate{Name: "e", OwnerID: "", LastActiveAt: idleBy(8 * Day)}, ) sum := mustRun(t, r) if sum.WorldsReaped != 0 { t.Fatalf("nothing should be reaped under a 10d deadline at <=9.5d idle: %+v", sum) } if sum.Warned != 2 { t.Fatalf("Warned = %d, want 2 (a:3d, b:1d)", sum.Warned) } if cl.deletePVCCalls != 0 { t.Fatalf("a warning path deleted a PVC") } if st.byName["a"].Warned3dAt.IsZero() || !st.byName["a"].Warned1dAt.IsZero() { t.Fatalf("server a: expected only a 3d warning, got %+v", st.byName["a"]) } if st.byName["b"].Warned1dAt.IsZero() { t.Fatalf("server b: expected a 1d warning, got %+v", st.byName["b"]) } if !st.byName["e"].Warned3dAt.IsZero() { t.Fatalf("unowned server e was warned") } } // Red line ⑤ (best-effort): a Warner delivery error does not abort the run, and // the warned_* stamp still advances (a real join, not a failed warn, is what // resets the clock). func TestWarningBestEffortOnDeliveryFailure(t *testing.T) { r, st, _, _ := newReaper(DefaultConfig(), Candidate{Name: "h", OwnerID: "u-h", LastActiveAt: idleBy(13 * Day)}) r.Warner = failWarner{} sum := mustRun(t, r) if sum.Warned != 1 { t.Fatalf("Warned = %d, want 1 despite delivery failure", sum.Warned) } if st.byName["h"].Warned3dAt.IsZero() { t.Fatalf("warned_3d_at not stamped after best-effort warn") } } type failWarner struct{} func (failWarner) Warn(context.Context, string, string, string) error { return errors.New("smtp unavailable") } // §26 capacity: when the store is over its cap, the oldest backup is evicted // early (destructive — audited) to make room, then the reap proceeds. func TestCapacityEvictsOldestThenReaps(t *testing.T) { cfg := DefaultConfig() cfg.MaxLocalBytes = 100 r, st, _, ar := newReaper(cfg, Candidate{Name: "epsilon", OwnerID: "user-5", LastActiveAt: idleBy(20 * Day)}) // Two present backups of 75 each = 150 > 100. Oldest must be evicted first. st.backups = []*fakeBackup{ {id: "old", server: "zzz", ref: "ref-old", size: 75, status: "present", createdAt: idleBy(40 * Day), expires: testNow.Add(30 * Day)}, {id: "new", server: "yyy", ref: "ref-new", size: 75, status: "present", createdAt: idleBy(5 * Day), expires: testNow.Add(60 * Day)}, } sum := mustRun(t, r) if sum.EvictedEarly != 1 { t.Fatalf("EvictedEarly = %d, want 1", sum.EvictedEarly) } if sum.WorldsReaped != 1 { t.Fatalf("WorldsReaped = %d, want 1 after eviction freed room", sum.WorldsReaped) } var oldStatus, newStatus string for _, b := range st.backups { switch b.id { case "old": oldStatus = b.status case "new": newStatus = b.status } } if oldStatus != "deleted" || newStatus != "present" { t.Fatalf("eviction hit wrong backup: old=%s new=%s, want deleted/present", oldStatus, newStatus) } // The destructive eviction must be audited. var evicted bool for _, a := range st.audits { if a.Action == ActionEvictBackup { evicted = true } } if !evicted { t.Fatalf("early eviction was not audited") } if ar.archives != 1 { t.Fatalf("reap did not archive after eviction: archives=%d", ar.archives) } } // §26 capacity + red line ④: if eviction cannot free enough space, the reap is // skipped and the world is preserved rather than deleted unbacked. func TestCapacityStillFullSkipsReap(t *testing.T) { cfg := DefaultConfig() cfg.MaxLocalBytes = 100 r, st, cl, ar := newReaper(cfg, Candidate{Name: "zeta", OwnerID: "user-6", LastActiveAt: idleBy(20 * Day)}) st.backups = []*fakeBackup{ {id: "stuck", server: "zzz", ref: "ref-stuck", size: 150, status: "present", createdAt: idleBy(40 * Day), expires: testNow.Add(30 * Day)}, } // The archive backend can't delete, so eviction cannot free space. r.Archiver.(*fakeArchiver).deleteErr = errors.New("evict unavailable") sum := mustRun(t, r) if sum.WorldsReaped != 0 || sum.Skipped != 1 { t.Fatalf("summary = %+v, want 0 reaped / 1 skipped (store full)", sum) } if cl.deletePVCCalls != 0 { t.Fatalf("world was deleted while the store was full") } if ar.archives != 0 { t.Fatalf("archived into a full store") } if st.byName["zeta"].OwnerID != "user-6" { t.Fatalf("ownership changed while store full") } } // Retention pass: backups past expires_at are deleted from the backend and // marked deleted; unexpired backups are untouched. func TestExpiredBackupsDeleted(t *testing.T) { r, st, _, ar := newReaper(DefaultConfig()) st.backups = []*fakeBackup{ {id: "gone", server: "s1", ref: "ref-gone", size: 5, status: "present", createdAt: idleBy(120 * Day), expires: idleBy(1 * Day)}, {id: "keep", server: "s2", ref: "ref-keep", size: 5, status: "present", createdAt: idleBy(10 * Day), expires: testNow.Add(80 * Day)}, } sum := mustRun(t, r) if sum.BackupsExpired != 1 { t.Fatalf("BackupsExpired = %d, want 1", sum.BackupsExpired) } if len(ar.deletes) != 1 || ar.deletes[0] != "ref-gone" { t.Fatalf("deleted archives = %v, want [ref-gone]", ar.deletes) } byID := map[string]string{} for _, b := range st.backups { byID[b.id] = b.status } if byID["gone"] != "deleted" || byID["keep"] != "present" { t.Fatalf("expiry hit wrong rows: %v", byID) } } // A servers row whose CRD has been deleted is skipped (not a failure) — the // reaper never deletes world data it cannot first inspect for the exemption. func TestMissingCRDSkipped(t *testing.T) { r, st, cl, ar := newReaper(DefaultConfig(), Candidate{Name: "ghost", OwnerID: "u", LastActiveAt: idleBy(20 * Day)}) delete(cl.crds, "ghost") // CRD gone, servers row lingers sum := mustRun(t, r) if sum.WorldsReaped != 0 || sum.Skipped != 0 { t.Fatalf("summary = %+v, want 0/0 (skipped without error)", sum) } if ar.archives != 0 || cl.deletePVCCalls != 0 { t.Fatalf("acted on a server with no CRD") } if st.byName["ghost"].OwnerID != "u" { t.Fatalf("mutated a server with no CRD") } } // A failure to list servers is the one hard error that aborts the batch. func TestListErrorAbortsBatch(t *testing.T) { r, st, _, _ := newReaper(DefaultConfig()) st.listErr = errors.New("db unreachable") if _, err := r.RunOnce(context.Background()); err == nil { t.Fatal("expected a hard error when listing servers fails") } }