fix(reaper): 回收前先停服并持有世界维护锁再归档，锁丢失不删卷；无世界卷的无主服务器只重置时钟不再重复回收 (7766e8ef) · Commits · FelisMC / Felis · GitLab

cmd/felis/reaper.go

+2 −2

Changes for cmd/felis/reaper.go: 2 added lines, 2 removed lines.
Original line number	Diff line number	Diff line
		@@ -138,8 +138,8 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
		// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
		// is kept, and without this nobody would learn that it is never reaped.
		func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
		fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
		sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.Warned, sum.Skipped, sum.StoreFull,
		fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d awaiting_stop=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
		sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.AwaitingStop, sum.Warned, sum.Skipped, sum.StoreFull,
		sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
		sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
		if !sum.Failed() {

cmd/felis/reaper_test.go

+1 −0

Changes for cmd/felis/reaper_test.go: 1 added line, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -27,6 +27,7 @@ func TestReportReaperRunFailsTheJob(t *testing.T) {
		want int
		}{
		{"clean", reaper.Summary{Evaluated: 3, WorldsReaped: 1, AwaitingOffsite: 1}, 0},
		{"waiting for a stop", reaper.Summary{Evaluated: 3, AwaitingStop: 1}, 0},
		{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
		{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
		{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},

internal/api/maintenance.go

+2 −0

Changes for internal/api/maintenance.go: 2 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -37,6 +37,8 @@ func maintenanceLabel(kind string) string {
		return "a backup"
		case maintenance.KindFileWrite:
		return "a file write"
		case maintenance.KindReap:
		return "the idle-world reaper"
		}
		return "another operation"
		}

internal/maintenance/maintenance.go

+6 −1

Changes for internal/maintenance/maintenance.go: 6 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -18,7 +18,8 @@
		// conflict and re-checks.
		//
		// A lock older than Grace with no Job behind it is stale (felis-api died between
		// the two writes) and holds nothing.
		// the two writes) and holds nothing. The reaper is the one holder without a Job:
		// it keeps its lock fresh by rewriting it while it archives and reclaims a world.
		//
		// A restore that starts with a safety snapshot is two Jobs in a row: the backup
		// Job carries the restore to run after it (LabelThenRestore), and felis-api
		@@ -83,6 +84,10 @@ const (
		KindRestore = "restore"
		KindBackup = "backup"
		KindFileWrite = "file-write"
		// KindReap is the reaper archiving an idle world and reclaiming its volume.
		// It runs no Job: the reaper holds the Annotation itself and rewrites it
		// well inside Grace for as long as it works on the world.
		KindReap = "reap"
		)

		// FilesModeWrite is the LabelFilesMode value of a file write.

internal/pgint/reaper_test.go

+38 −1

Changes for internal/pgint/reaper_test.go: 38 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -33,7 +33,11 @@ func (c *reclaimCluster) DeletePVC(_ context.Context, pvc string) error {
		return nil
		}

		func (c *reclaimCluster) Stop(context.Context, string) error { return nil }
		func (c *reclaimCluster) HoldWorld(ctx context.Context, _ string) (context.Context, func(), error) {
		return ctx, func() {}, nil
		}

		func (c *reclaimCluster) WorldExists(context.Context, string) (bool, error) { return true, nil }

		type reclaimArchiver struct{ archived []string }

		@@ -387,3 +391,36 @@ func TestBackupReadBack(t *testing.T) {
		t.Fatalf("live refs after deleting %s still claim its archive", newer)
		}
		}

		// TestRestartClock: an idle server with no world to reclaim gets its clock and
		// warnings reset, and keeps its owner and resource cache (data-durability-19).
		func TestRestartClock(t *testing.T) {
		ctx := context.Background()
		st := reaper.NewPGStore(db)
		name := "clock-" + suffix(t)
		old := time.Now().Add(-20 * reaper.Day).UTC().Truncate(time.Microsecond)
		if _, err := db.ExecContext(ctx,
		`INSERT INTO servers (name, cached_cpu_milli, cached_memory_mb, cached_storage_mb) VALUES ($1, 100, 128, 1)`,
		name); err != nil {
		t.Fatalf("seed server: %v", err)
		}
		if _, err := db.ExecContext(ctx,
		`UPDATE servers SET last_active_at = $2, warned_3d_at = $2, warned_1d_at = $2 WHERE name = $1`, name, old); err != nil {
		t.Fatalf("age server: %v", err)
		}
		at := time.Now().UTC().Truncate(time.Microsecond)
		if err := st.RestartClock(ctx, name, at); err != nil {
		t.Fatalf("RestartClock: %v", err)
		}
		var last time.Time
		var w3, w1 sql.NullTime
		var cpu int
		if err := db.QueryRowContext(ctx,
		`SELECT last_active_at, warned_3d_at, warned_1d_at, cached_cpu_milli FROM servers WHERE name = $1`, name).
		Scan(&last, &w3, &w1, &cpu); err != nil {
		t.Fatalf("read back: %v", err)
		}
		if !last.Equal(at) \|\| w3.Valid \|\| w1.Valid \|\| cpu != 100 {
		t.Fatalf("after RestartClock: last_active_at=%v warned=%v/%v cpu=%d; want %v, cleared, cache kept", last, w3, w1, cpu, at)
		}
		}