fix(reaper): 有服务器处理失败或过期备份删不掉时以非零码退出，让 Job 失败告警触达运维

internal/pgint/reaper_test.go

@@ -42,7 +42,7 @@ func (a *reclaimArchiver) Archive(_ context.Context, server, _ string) (backup.A
 }
 
 func (a *reclaimArchiver) Restore(context.Context, backup.ArchiveRef, string) error { return nil }
-func (a *reclaimArchiver) Delete(context.Context, backup.ArchiveRef) error        { return nil }
+func (a *reclaimArchiver) Delete(context.Context, backup.ArchiveRef) error          { return nil }
 
 // TestReclaimRestartsReaperClock: a world reaped weeks ago and claimed by a new
 // owner is not reaped again on the next run, and when it does go idle the reap

internal/reaper/reaper.go

@@ -241,17 +241,30 @@ type Reaper struct {
 
 // Summary is the per-run tally (feeds §23 metrics).
 type Summary struct {
-	Evaluated      int
-	WorldsReaped   int
-	Warned         int
-	Skipped        int // exempt, CRD gone, or could not back up
+	Evaluated    int
+	WorldsReaped int
+	Warned       int
+	// Skipped are servers the run failed on (archive, store, cluster or
+	// capacity errors); their worlds are kept and retried next run. Exempt
+	// servers and rows whose CRD is gone are not counted.
+	Skipped int
+	// StoreFull are the Skipped servers kept because the backup store was at
+	// capacity and eviction could not make room.
+	StoreFull      int
 	EvictedEarly   int
 	BackupsExpired int
+	// ExpireFailed are expired backups the retention pass could not remove.
+	ExpireFailed int
 	// AwaitingOffsite are idle worlds that are archived and kept until the
 	// archive's off-site copy lands.
 	AwaitingOffsite int
 }
 
+// Failed reports whether the run left work undone: a server it could not
+// process, or an expired backup it could not remove. The world is safe either
+// way, but the run did not do its job and whoever operates it must hear.
+func (s Summary) Failed() bool { return s.Skipped > 0 || s.ExpireFailed > 0 }
+
 func (r *Reaper) now() time.Time {
 	if r.Now != nil {
 		return r.Now()
@@ -281,7 +294,8 @@ func (r *Reaper) id() string {
 // retention pass over expired backups. It is idempotent and restart-safe, so a
 // Kubernetes CronJob can drive the daily cadence (spec §18). Per-server
 // failures are logged and counted as Skipped without aborting the batch; only
-// an inability to list servers is a hard error.
+// an inability to list servers is a hard error. Summary.Failed tells the
+// caller whether the run as a whole should report failure.
 func (r *Reaper) RunOnce(ctx context.Context) (Summary, error) {
 	var sum Summary
 
@@ -301,6 +315,9 @@ func (r *Reaper) RunOnce(ctx context.Context) (Summary, error) {
 		if err := r.evaluate(ctx, now, offs, c, &sum); err != nil {
 			r.log().Error("reaper: skipping server", "server", c.Name, "err", err)
 			sum.Skipped++
+			if errors.Is(err, errStoreFull) {
+				sum.StoreFull++
+			}
 		}
 	}
 
@@ -512,15 +529,18 @@ func (r *Reaper) expireBackups(ctx context.Context, now time.Time, sum *Summary)
 	exp, err := r.Store.ListExpiredBackups(ctx, now)
 	if err != nil {
 		r.log().Error("reaper: list expired backups", "err", err)
+		sum.ExpireFailed++
 		return
 	}
 	for _, b := range exp {
 		if err := r.Archiver.Delete(ctx, backup.ArchiveRef(b.BackupRef)); err != nil {
 			r.log().Error("reaper: delete expired archive", "id", b.ID, "err", err)
+			sum.ExpireFailed++
 			continue
 		}
 		if err := r.Store.MarkBackupDeleted(ctx, b.ID, now); err != nil {
 			r.log().Error("reaper: mark expired deleted", "id", b.ID, "err", err)
+			sum.ExpireFailed++
 			continue
 		}
 		sum.BackupsExpired++

internal/reaper/reaper_test.go

@@ -675,8 +675,8 @@ func TestCapacityStillFullSkipsReap(t *testing.T) {
 	r.Archiver.(*fakeArchiver).deleteErr = errors.New("evict unavailable")
 
 	sum := mustRun(t, r)
-	if sum.WorldsReaped != 0 || sum.Skipped != 1 {
-		t.Fatalf("summary = %+v, want 0 reaped / 1 skipped (store full)", sum)
+	if sum.WorldsReaped != 0 || sum.Skipped != 1 || sum.StoreFull != 1 || !sum.Failed() {
+		t.Fatalf("summary = %+v, want 0 reaped / 1 skipped, store full, failed", sum)
 	}
 	if cl.deletePVCCalls != 0 {
 		t.Fatalf("world was deleted while the store was full")
@@ -714,6 +714,24 @@ func TestExpiredBackupsDeleted(t *testing.T) {
 	}
 }
 
+// An expired backup the backend cannot delete stays present and marks the run
+// failed, so the Job reports it instead of succeeding every day.
+func TestExpiryFailureFailsTheRun(t *testing.T) {
+	r, st, _, _ := newReaper(DefaultConfig())
+	st.backups = []*fakeBackup{
+		{id: "gone", server: "s1", ref: "ref-gone", size: 5, status: "present", createdAt: idleBy(120 * Day), expires: idleBy(1 * Day)},
+	}
+	r.Archiver.(*fakeArchiver).deleteErr = errors.New("permission denied")
+
+	sum := mustRun(t, r)
+	if sum.BackupsExpired != 0 || sum.ExpireFailed != 1 || !sum.Failed() {
+		t.Fatalf("summary = %+v, want 0 expired / 1 expire_failed, failed", sum)
+	}
+	if st.backups[0].status != "present" {
+		t.Fatalf("the undeleted archive's row was marked %s", st.backups[0].status)
+	}
+}
+
 // A servers row whose CRD has been deleted is skipped (not a failure) — the
 // reaper never deletes world data it cannot first inspect for the exemption.
 func TestMissingCRDSkipped(t *testing.T) {