fix(reaper): 有服务器处理失败或过期备份删不掉时以非零码退出，让 Job 失败告警触达运维

		fmt.Fprintf(stderr, "felis reaper: %v\n", err)

		return 1

	}

	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d evicted=%d expired=%d\n",

		sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.Warned, sum.Skipped, sum.EvictedEarly, sum.BackupsExpired)

	return reportReaperRun(sum, stdout, stderr)

}

// reportReaperRun prints the run's tally and turns a run that left work undone

// into exit 1, so the Job fails and the watchdog's job-failed check (and the

// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived

// is kept, and without this nobody would learn that it is never reaped.

func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {

	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d\n",

		sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.Warned, sum.Skipped, sum.StoreFull,

		sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed)

	if !sum.Failed() {

		return 0

	}

	fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",

		sum.Skipped, sum.StoreFull, sum.ExpireFailed)

	return 1

}

// mailWarner delivers a pre-reap notice to the owner's verified email — the

// only channel this build can reach. Unowned owners and owners who never proved

package main

import (

	"bytes"

	"context"

	"errors"

	"os"

	corev1 "k8s.io/api/core/v1"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"sigs.k8s.io/controller-runtime/pkg/client/fake"

	"felis.lolicon.best/internal/reaper"

)

// TestReportReaperRunFailsTheJob: a run that could not process a server, or

// could not remove an expired backup, exits 1 so the Job shows as failed.

func TestReportReaperRunFailsTheJob(t *testing.T) {

	for _, tc := range []struct {

		name string

		sum  reaper.Summary

		want int

	}{

		{"clean", reaper.Summary{Evaluated: 3, WorldsReaped: 1, AwaitingOffsite: 1}, 0},

		{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},

		{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},

		{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},

	} {

		var out, errb bytes.Buffer

		if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {

			t.Errorf("%s: exit %d, want %d", tc.name, got, tc.want)

		}

		if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") {

			t.Errorf("%s: summary line = %q", tc.name, out.String())

		}

		if (tc.want == 1) != (errb.Len() > 0) {

			t.Errorf("%s: stderr = %q", tc.name, errb.String())

		}

	}

}

// TestResolveWorldDir pins the two world layouts the reaper must find, and the

// fail-closed miss. The stock local-path arm is derived from the live PVC's

// volumeName — a name-based guess (glob) could tar a stale deleted PV's bytes and

`awaiting_offsite` that stays above zero for more than a day means the copy is

failing: `sudo felis offsite status` (§16).

### A failed reaper Job

Each run ends with one line:

```

felis reaper: evaluated=12 reaped=1 awaiting_offsite=0 warned=2 skipped=0 store_full=0 evicted=0 expired=3 expire_failed=0

```

`skipped` counts servers the run failed on (steps 1–3 above, or the cluster or

the database answering with an error; exempt servers and rows whose CRD is gone

are not counted), `store_full` the subset kept because the backup store is full,

and `expire_failed` expired backups it could not remove. Any of them above zero

makes the process exit 1: the worlds are safe, but the Job fails so the watchdog

mails `world reaper Job … failed` and `FelisWorldJobFailed` fires. The Job retries

twice (`backoffLimit`), each retry re-running the whole batch, which is safe

because every step is idempotent. Read the error above the summary:

```sh

kubectl -n minecraft logs job/<the failed felis-reaper-… Job>

```

A server that fails every day keeps its world and is retried every day, so the

Job fails every day until the cause is fixed; after 26 hours the watchdog also

reports `the world reaper has not succeeded for …`. [GO-TESTED:

`TestReportReaperRunFailsTheJob`, `TestExpiryFailureFailsTheRun`,

`TestCapacityStillFullSkipsReap`.]

### Exemptions (world never reaped)

- `spec.reaperExempt=true` → skipped entirely (system servers). [GO-TESTED

	Evaluated    int

	WorldsReaped int

	Warned       int

	Skipped        int // exempt, CRD gone, or could not back up

	// Skipped are servers the run failed on (archive, store, cluster or

	// capacity errors); their worlds are kept and retried next run. Exempt

	// servers and rows whose CRD is gone are not counted.

	Skipped int

	// StoreFull are the Skipped servers kept because the backup store was at

	// capacity and eviction could not make room.

	StoreFull      int

	EvictedEarly   int

	BackupsExpired int

	// ExpireFailed are expired backups the retention pass could not remove.

	ExpireFailed int

	// AwaitingOffsite are idle worlds that are archived and kept until the

	// archive's off-site copy lands.

	AwaitingOffsite int

}

// Failed reports whether the run left work undone: a server it could not

// process, or an expired backup it could not remove. The world is safe either

// way, but the run did not do its job and whoever operates it must hear.

func (s Summary) Failed() bool { return s.Skipped > 0 || s.ExpireFailed > 0 }

func (r *Reaper) now() time.Time {

	if r.Now != nil {

		return r.Now()

// retention pass over expired backups. It is idempotent and restart-safe, so a

// Kubernetes CronJob can drive the daily cadence (spec §18). Per-server

// failures are logged and counted as Skipped without aborting the batch; only

// an inability to list servers is a hard error.

// an inability to list servers is a hard error. Summary.Failed tells the

// caller whether the run as a whole should report failure.

func (r *Reaper) RunOnce(ctx context.Context) (Summary, error) {

	var sum Summary

		if err := r.evaluate(ctx, now, offs, c, &sum); err != nil {

			r.log().Error("reaper: skipping server", "server", c.Name, "err", err)

			sum.Skipped++

			if errors.Is(err, errStoreFull) {

				sum.StoreFull++

			}

		}

	}

	exp, err := r.Store.ListExpiredBackups(ctx, now)

	if err != nil {

		r.log().Error("reaper: list expired backups", "err", err)

		sum.ExpireFailed++

		return

	}

	for _, b := range exp {

		if err := r.Archiver.Delete(ctx, backup.ArchiveRef(b.BackupRef)); err != nil {

			r.log().Error("reaper: delete expired archive", "id", b.ID, "err", err)

			sum.ExpireFailed++

			continue

		}

		if err := r.Store.MarkBackupDeleted(ctx, b.ID, now); err != nil {

			r.log().Error("reaper: mark expired deleted", "id", b.ID, "err", err)

			sum.ExpireFailed++

			continue

		}

		sum.BackupsExpired++

	r.Archiver.(*fakeArchiver).deleteErr = errors.New("evict unavailable")

	sum := mustRun(t, r)

	if sum.WorldsReaped != 0 || sum.Skipped != 1 {

		t.Fatalf("summary = %+v, want 0 reaped / 1 skipped (store full)", sum)

	if sum.WorldsReaped != 0 || sum.Skipped != 1 || sum.StoreFull != 1 || !sum.Failed() {

		t.Fatalf("summary = %+v, want 0 reaped / 1 skipped, store full, failed", sum)

	}

	if cl.deletePVCCalls != 0 {

		t.Fatalf("world was deleted while the store was full")

	}

}

// An expired backup the backend cannot delete stays present and marks the run

// failed, so the Job reports it instead of succeeding every day.

func TestExpiryFailureFailsTheRun(t *testing.T) {

	r, st, _, _ := newReaper(DefaultConfig())

	st.backups = []*fakeBackup{

		{id: "gone", server: "s1", ref: "ref-gone", size: 5, status: "present", createdAt: idleBy(120 * Day), expires: idleBy(1 * Day)},

	}

	r.Archiver.(*fakeArchiver).deleteErr = errors.New("permission denied")

	sum := mustRun(t, r)

	if sum.BackupsExpired != 0 || sum.ExpireFailed != 1 || !sum.Failed() {

		t.Fatalf("summary = %+v, want 0 expired / 1 expire_failed, failed", sum)

	}

	if st.backups[0].status != "present" {

		t.Fatalf("the undeleted archive's row was marked %s", st.backups[0].status)

	}

}

// A servers row whose CRD has been deleted is skipped (not a failure) — the

// reaper never deletes world data it cannot first inspect for the exemption.

func TestMissingCRDSkipped(t *testing.T) {