fix(servers): 主人可放弃服务器、管理员可删除服务器，由 reaper 归档世界后释放或移除

internal/reaper/k8scluster.go

@@ -33,7 +33,8 @@ const gamePodComponent = "server"
 
 // K8sCluster is the production Cluster backed by a controller-runtime client
 // (spec §4, §18). It reads spec.reaperExempt, stops a server and holds its world
-// volume through the maintenance lock, and deletes the world PVC — nothing else.
+// volume through the maintenance lock, deletes the world PVC, and removes the
+// MinecraftServer of a server an admin deleted — nothing else.
 type K8sCluster struct {
 	c         client.Client
 	namespace string
@@ -60,7 +61,7 @@ func (k *K8sCluster) Inspect(ctx context.Context, name string) (ServerCRD, error
 	if err := k.get(ctx, name, &ms); err != nil {
 		return ServerCRD{}, err
 	}
-	return ServerCRD{Exempt: ms.Spec.ReaperExempt, PVC: WorldPVCName(name)}, nil
+	return ServerCRD{Exempt: ms.Spec.ReaperExempt, PVC: WorldPVCName(name), UID: string(ms.UID)}, nil
 }
 
 // HoldWorld implements Cluster. The lock is the same Annotation felis-api
@@ -232,6 +233,30 @@ func (k *K8sCluster) DeletePVC(ctx context.Context, pvc string) error {
 	return nil
 }
 
+// DeleteServer implements Cluster. The read and the delete are one step (the
+// delete carries the resourceVersion read), so the object removed is the one
+// whose uid was checked: a server made again under the name is refused.
+func (k *K8sCluster) DeleteServer(ctx context.Context, name, uid string) error {
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		var ms v1alpha1.MinecraftServer
+		switch err := k.get(ctx, name, &ms); {
+		case errors.Is(err, ErrNotFound):
+			return nil
+		case err != nil:
+			return err
+		}
+		if string(ms.UID) != uid {
+			return fmt.Errorf("MinecraftServer %s is another server now (uid %s, inspected %s)", name, ms.UID, uid)
+		}
+		rv, u := ms.ResourceVersion, ms.UID
+		err := k.c.Delete(ctx, &ms, client.Preconditions{UID: &u, ResourceVersion: &rv})
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	})
+}
+
 func (k *K8sCluster) get(ctx context.Context, name string, ms *v1alpha1.MinecraftServer) error {
 	if err := k.c.Get(ctx, types.NamespacedName{Namespace: k.namespace, Name: name}, ms); err != nil {
 		if apierrors.IsNotFound(err) {

internal/reaper/k8scluster_test.go

@@ -216,3 +216,36 @@ func TestGamePodComponentMatchesOperator(t *testing.T) {
 		t.Fatalf("gamePodComponent = %q, operator labels its pods %q", gamePodComponent, operator.ComponentValue)
 	}
 }
+
+// DeleteServer removes the MinecraftServer Inspect returned, and nothing that
+// merely carries its name: a server made again under the name (another uid)
+// stays, and one already gone is not an error.
+func TestDeleteServerRemovesOnlyTheInspectedServer(t *testing.T) {
+	ms := holdServer(v1alpha1.DesiredStopped, v1alpha1.PhaseStopped)
+	ms.UID = "uid-1"
+	k, c, _ := holdCluster(t, ms)
+	ctx := context.Background()
+
+	crd, err := k.Inspect(ctx, "survival")
+	if err != nil || crd.UID != "uid-1" {
+		t.Fatalf("Inspect = %+v, %v; want uid-1", crd, err)
+	}
+
+	if err := k.DeleteServer(ctx, "survival", "uid-0"); err == nil {
+		t.Fatal("DeleteServer with another uid succeeded")
+	}
+	var got v1alpha1.MinecraftServer
+	if err := c.Get(ctx, types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &got); err != nil {
+		t.Fatalf("the server of another uid was deleted: %v", err)
+	}
+
+	if err := k.DeleteServer(ctx, "survival", crd.UID); err != nil {
+		t.Fatalf("DeleteServer: %v", err)
+	}
+	if err := c.Get(ctx, types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &got); err == nil {
+		t.Fatal("the server is still there")
+	}
+	if err := k.DeleteServer(ctx, "survival", crd.UID); err != nil {
+		t.Fatalf("DeleteServer of a server already gone = %v, want nil", err)
+	}
+}

internal/reaper/pgstore.go

@@ -20,7 +20,7 @@ type PGStore struct {
 func NewPGStore(db *sql.DB) *PGStore { return &PGStore{db: db} }
 
 func (s *PGStore) ListActiveServers(ctx context.Context) ([]Candidate, error) {
-	const q = `SELECT name, owner_id, last_active_at, warned_3d_at, warned_1d_at
+	const q = `SELECT name, owner_id, last_active_at, warned_3d_at, warned_1d_at, retire_requested_at, retire_delete
 		FROM servers WHERE deleted_at IS NULL ORDER BY name`
 	rows, err := s.db.QueryContext(ctx, q)
 	if err != nil {
@@ -33,8 +33,9 @@ func (s *PGStore) ListActiveServers(ctx context.Context) ([]Candidate, error) {
 			c      Candidate
 			owner  sql.NullString
 			w3, w1 sql.NullTime
+			retire sql.NullTime
 		)
-		if err := rows.Scan(&c.Name, &owner, &c.LastActiveAt, &w3, &w1); err != nil {
+		if err := rows.Scan(&c.Name, &owner, &c.LastActiveAt, &w3, &w1, &retire, &c.RetireDelete); err != nil {
 			return nil, err
 		}
 		c.OwnerID = owner.String
@@ -44,18 +45,21 @@ func (s *PGStore) ListActiveServers(ctx context.Context) ([]Candidate, error) {
 		if w1.Valid {
 			c.Warned1dAt = w1.Time
 		}
+		if retire.Valid {
+			c.RetireRequestedAt = retire.Time
+		}
 		out = append(out, c)
 	}
 	return out, rows.Err()
 }
 
 func (s *PGStore) FreshBackup(ctx context.Context, server string, since time.Time) (Fresh, bool, error) {
-	// Only the reaper's own archives count, and only those taken since the
-	// current owner claimed the server: a manual backup may predate a panel edit
-	// that did not move last_active_at, and an archive from before the claim is
-	// the previous owner's world. One found corrupt is never reused.
+	// Only the reaper's own archives count (an idle reap's or a retirement's),
+	// and only those taken since the current owner claimed the server: a manual
+	// backup may predate a panel edit that did not move last_active_at, and an
+	// archive from before the claim is the previous owner's world. One found corrupt is never reused.
 	const q = `SELECT b.id, b.backup_ref, COALESCE(b.sha256, ''), b.offsite_at IS NOT NULL FROM world_backups b
-		WHERE b.server_name = $1 AND b.status = 'present' AND b.reason = 'inactive_15d' AND b.created_at >= $2
+		WHERE b.server_name = $1 AND b.status = 'present' AND b.reason IN ('inactive_15d', 'released') AND b.created_at >= $2
 		  AND b.corrupt_at IS NULL
 		  AND b.created_at >= COALESCE((SELECT s.claimed_at FROM servers s WHERE s.name = $1 AND s.deleted_at IS NULL), '-infinity')
 		ORDER BY b.offsite_at IS NOT NULL DESC, b.created_at DESC LIMIT 1`
@@ -85,17 +89,36 @@ func (s *PGStore) InsertBackup(ctx context.Context, rec BackupRecord) error {
 // gated on that size and counts it. The wake allowlist is emptied in the same
 // statement: its players were vouched for by the owner being released, and an
 // ownerless server set to autostartPolicy=allowlist would otherwise stay
-// wakeable by them.
+// wakeable by them. A pending release is done with once the world is gone;
+// a pending deletion stays, so an idle reap that lands on a server an admin is
+// deleting leaves the deletion for the next run.
 func (s *PGStore) ReleaseWorld(ctx context.Context, name string, at time.Time) error {
 	const q = `WITH released AS (
 		UPDATE servers
-		SET owner_id = NULL, last_active_at = $2, warned_3d_at = NULL, warned_1d_at = NULL
+		SET owner_id = NULL, last_active_at = $2, warned_3d_at = NULL, warned_1d_at = NULL,
+		    retire_requested_at = CASE WHEN retire_delete THEN retire_requested_at END
 		WHERE name = $1 AND deleted_at IS NULL RETURNING name)
 		DELETE FROM server_allowlist WHERE server_name IN (SELECT name FROM released)`
 	_, err := s.db.ExecContext(ctx, q, name, at)
 	return err
 }
 
+// DeleteServerRow marks a server deleted once its MinecraftServer is gone. The
+// row stays (red line ②) and so do its backups (red line ③), recorded against
+// the name; the aliases go, which frees the subdomain, and the allowlist with
+// them. A create under the name later starts the row over (api.SeedServer).
+func (s *PGStore) DeleteServerRow(ctx context.Context, name string, at time.Time) error {
+	const q = `WITH gone AS (
+		UPDATE servers
+		SET deleted_at = $2, owner_id = NULL, retire_requested_at = NULL, retire_delete = false,
+		    warned_3d_at = NULL, warned_1d_at = NULL
+		WHERE name = $1 AND deleted_at IS NULL AND retire_delete RETURNING name),
+	aliases AS (DELETE FROM server_aliases WHERE server_name IN (SELECT name FROM gone))
+	DELETE FROM server_allowlist WHERE server_name IN (SELECT name FROM gone)`
+	_, err := s.db.ExecContext(ctx, q, name, at)
+	return err
+}
+
 func (s *PGStore) RestartClock(ctx context.Context, name string, at time.Time) error {
 	const q = `UPDATE servers SET last_active_at = $2, warned_3d_at = NULL, warned_1d_at = NULL
 		WHERE name = $1 AND deleted_at IS NULL`
@@ -123,8 +146,8 @@ func (s *PGStore) PresentBackupBytes(ctx context.Context) (int64, error) {
 
 func (s *PGStore) EvictableBackups(ctx context.Context) ([]StoredBackup, error) {
 	const q = `SELECT id, server_name, backup_ref, size_bytes, reason, COALESCE(sha256, '') FROM world_backups
-		WHERE status = 'present' AND (reason <> 'inactive_15d' OR offsite_at IS NOT NULL)
-		ORDER BY reason = 'inactive_15d', created_at ASC`
+		WHERE status = 'present' AND (reason NOT IN ('inactive_15d', 'released') OR offsite_at IS NOT NULL)
+		ORDER BY reason IN ('inactive_15d', 'released'), created_at ASC`
 	return s.queryBackups(ctx, q)
 }
 

internal/reaper/reaper.go

@@ -43,11 +43,18 @@ const Day = 24 * time.Hour
 // (spec §18). It is a stable label, not a literal restatement of the deadline.
 const ReasonInactive = "inactive_15d"
 
+// ReasonReleased is the world_backups.reason for the archive of a world whose
+// server its owner gave up or an admin deleted (PUT /servers/{name}/retirement).
+// Like an idle reap's, it is the world's copy after the volume is gone.
+const ReasonReleased = "released"
+
 // Audit actions emitted by the reaper. The actor/source are a system identity
 // ("reaper") because no human Access email is in play here (spec §14).
 const (
-	ActionReapWorld   = "reap_world"
-	ActionEvictBackup = "evict_backup_early"
+	ActionReapWorld    = "reap_world"
+	ActionEvictBackup  = "evict_backup_early"
+	ActionReleaseWorld = "release_world"
+	ActionDeleteServer = "delete_server"
 )
 
 // ErrNotFound is returned by Cluster.Inspect when the MinecraftServer CRD for a
@@ -151,6 +158,12 @@ type Candidate struct {
 	LastActiveAt time.Time
 	Warned3dAt   time.Time // zero = not yet sent
 	Warned1dAt   time.Time // zero = not yet sent
+	// RetireRequestedAt is when the owner gave the server up or an admin asked
+	// for it to be deleted (zero = no request); RetireDelete marks a deletion.
+	// Either is carried out on the next run, however recently the world was
+	// played.
+	RetireRequestedAt time.Time
+	RetireDelete      bool
 }
 
 func (c Candidate) warnedAt(t Tier) time.Time {
@@ -160,11 +173,25 @@ func (c Candidate) warnedAt(t Tier) time.Time {
 	return c.Warned3dAt
 }
 
+func (c Candidate) retiring() bool { return !c.RetireRequestedAt.IsZero() }
+
+// worldSince is how recent an archive must be to hold the current world: taken
+// after the last join, and for a retirement after the request, so the world
+// archived is the one its owner left.
+func (c Candidate) worldSince() time.Time {
+	if c.RetireRequestedAt.After(c.LastActiveAt) {
+		return c.RetireRequestedAt
+	}
+	return c.LastActiveAt
+}
+
 // ServerCRD is the slice of the MinecraftServer CRD the reaper needs: the
-// exemption flag (red line ①) and the world PVC to archive then delete.
+// exemption flag (red line ①), the world PVC to archive then delete, and the
+// object's uid, so a deletion removes the server that was inspected.
 type ServerCRD struct {
 	Exempt bool
 	PVC    string
+	UID    string
 }
 
 // BackupRecord is a world_backups insert. FormerOwner is captured so the
@@ -233,9 +260,17 @@ type Store interface {
 
 	// ReleaseWorld is the post-delete business mutation: owner_id→NULL,
 	// last_active_at→at (clock reset), warned_*→NULL. It does NOT delete the
-	// row (red line ②).
+	// row (red line ②). A pending release is done with; a pending deletion
+	// stays for the next run to finish.
 	ReleaseWorld(ctx context.Context, name string, at time.Time) error
 
+	// DeleteServerRow finishes an admin's deletion once the MinecraftServer is
+	// gone: the row is marked deleted, which frees its name and subdomain, and
+	// loses its owner, aliases and allowlist. Only a row with a pending deletion
+	// is touched, so a server created again under the name is left alone. The
+	// row itself stays, like every reaped server's (red line ②).
+	DeleteServerRow(ctx context.Context, name string, at time.Time) error
+
 	// RestartClock sets last_active_at→at and clears warned_* on a server with
 	// no world to reclaim, so it is not found idle again every run.
 	RestartClock(ctx context.Context, name string, at time.Time) error
@@ -295,6 +330,11 @@ type Cluster interface {
 	WorldExists(ctx context.Context, pvc string) (bool, error)
 	// DeletePVC deletes the world PersistentVolumeClaim.
 	DeletePVC(ctx context.Context, pvc string) error
+	// DeleteServer removes the MinecraftServer whose uid Inspect returned, and
+	// with it what the operator made for it (StatefulSet, Service, Secret). The
+	// world volume is not among them: it is deleted first. A server already gone
+	// is not an error; one of the same name with another uid is left alone.
+	DeleteServer(ctx context.Context, name, uid string) error
 }
 
 // Warner delivers an impending-reap notice. It is optional and best-effort: a
@@ -323,10 +363,15 @@ type Reaper struct {
 type Summary struct {
 	Evaluated    int
 	WorldsReaped int
-	Warned       int
+	// Released are servers given up by their owner (or released by an admin)
+	// and ServersDeleted the ones an admin deleted, carried out this run.
+	Released       int
+	ServersDeleted int
+	Warned         int
 	// Skipped are servers the run failed on (archive, store, cluster or
 	// capacity errors); their worlds are kept and retried next run. Exempt
-	// servers and rows whose CRD is gone are not counted.
+	// servers and rows whose CRD is gone are not counted, except a deletion
+	// left with a world volume and no MinecraftServer to hold it by.
 	Skipped int
 	// StoreFull are the Skipped servers kept because the backup store was at
 	// capacity and eviction could not make room.
@@ -453,6 +498,11 @@ func (r *Reaper) evaluate(ctx context.Context, now time.Time, offs []time.Durati
 	crd, err := r.Cluster.Inspect(ctx, c.Name)
 	if err != nil {
 		if errors.Is(err, ErrNotFound) {
+			if c.RetireDelete {
+				// An earlier run removed the MinecraftServer and stopped before
+				// marking the row, or it was removed by hand.
+				return r.forgetServer(ctx, now, c, sum)
+			}
 			// CRD gone but the row lingers — nothing safe to do; not a failure.
 			r.log().Warn("reaper: CRD missing, skipping", "server", c.Name)
 			return nil
@@ -460,9 +510,18 @@ func (r *Reaper) evaluate(ctx context.Context, now time.Time, offs []time.Durati
 		return fmt.Errorf("inspect: %w", err)
 	}
 	if crd.Exempt {
-		// Red line ①: system servers (lobby/proxy) are never reaped.
+		// Red line ①: system servers (lobby/proxy) are never reaped. felis-api
+		// refuses to retire one, so a request here predates the flag.
+		if c.retiring() {
+			r.log().Warn("reaper: a system server is never given up or deleted; request ignored", "server", c.Name)
+		}
 		return nil
 	}
+	if c.retiring() {
+		// The owner gave the server up, or an admin is deleting it: the world
+		// goes now, archived like an idle one, and no warning is owed.
+		return r.reap(ctx, now, c, crd, sum)
+	}
 
 	idle := now.Sub(c.LastActiveAt)
 	if idle > r.Cfg.IdleBeforeReap {
@@ -490,7 +549,7 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
 		return fmt.Errorf("look up world volume: %w", err)
 	}
 	if !exists {
-		return r.reapNoWorld(ctx, now, c, sum)
+		return r.reapNoWorld(ctx, now, c, crd, sum)
 	}
 
 	// §26 soft cap: free space before adding a backup. If the store cannot be
@@ -510,7 +569,7 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
 	// world but failed before deleting the PVC, reuse that backup rather than
 	// writing a duplicate. The world has not changed since last_active_at, so
 	// any present backup created after it still describes the current world.
-	fresh, ok, err := r.Store.FreshBackup(ctx, c.Name, c.LastActiveAt)
+	fresh, ok, err := r.Store.FreshBackup(ctx, c.Name, c.worldSince())
 	if err != nil {
 		return fmt.Errorf("lookup fresh backup: %w", err)
 	}
@@ -537,13 +596,17 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
 			r.log().Warn("reaper: archive leaves out entries that are not plain files or directories",
 				"server", c.Name, "count", len(a.Skipped), "first", a.Skipped[:min(len(a.Skipped), 5)])
 		}
+		reason := ReasonInactive
+		if c.retiring() {
+			reason = ReasonReleased
+		}
 		rec := BackupRecord{
 			ID:             r.id(),
 			ServerName:     c.Name,
 			FormerOwner:    c.OwnerID,
 			BackupRef:      string(a.Ref),
 			SizeBytes:      a.Size,
-			Reason:         ReasonInactive,
+			Reason:         reason,
 			ExpiresAt:      now.Add(r.Cfg.Retention),
 			SHA256:         a.SHA256,
 			SkippedEntries: len(a.Skipped),
@@ -577,16 +640,24 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
 		// the delete, so no duplicate archive is created.
 		return fmt.Errorf("delete pvc: %w", err)
 	}
-	return r.finishReap(ctx, now, c, ref, sum)
+	return r.finishReap(ctx, now, c, crd, ref, sum)
 }
 
-// finishReap releases a world whose PVC is gone and records the reap.
-func (r *Reaper) finishReap(ctx context.Context, now time.Time, c Candidate, ref string, sum *Summary) error {
-	if err := r.Store.ReleaseWorld(ctx, c.Name, now); err != nil {
-		return fmt.Errorf("release world: %w", err)
-	}
-	if err := r.Store.Audit(ctx, AuditRecord{Action: ActionReapWorld, ServerName: c.Name, FormerOwner: c.OwnerID}); err != nil {
-		r.log().Error("reaper: audit reap_world failed", "server", c.Name, "err", err)
+// finishReap releases a world whose PVC is gone and records the reap. A
+// retirement is finished with it: the server is released, or for a deletion
+// removed.
+func (r *Reaper) finishReap(ctx context.Context, now time.Time, c Candidate, crd ServerCRD, ref string, sum *Summary) error {
+	if c.retiring() {
+		if err := r.retire(ctx, now, c, crd, sum); err != nil {
+			return err
+		}
+	} else {
+		if err := r.Store.ReleaseWorld(ctx, c.Name, now); err != nil {
+			return fmt.Errorf("release world: %w", err)
+		}
+		if err := r.Store.Audit(ctx, AuditRecord{Action: ActionReapWorld, ServerName: c.Name, FormerOwner: c.OwnerID}); err != nil {
+			r.log().Error("reaper: audit reap_world failed", "server", c.Name, "err", err)
+		}
 	}
 
 	sum.WorldsReaped++
@@ -604,13 +675,17 @@ func (r *Reaper) finishReap(ctx context.Context, now time.Time, c Candidate, ref
 // no world to reclaim: an owner who never started the server gives it up
 // (nothing to back up), and an unowned one — typically a world reaped earlier —
 // only has its clock restarted, so it is not reaped over and over.
-func (r *Reaper) reapNoWorld(ctx context.Context, now time.Time, c Candidate, sum *Summary) error {
-	fresh, ok, err := r.Store.FreshBackup(ctx, c.Name, c.LastActiveAt)
+func (r *Reaper) reapNoWorld(ctx context.Context, now time.Time, c Candidate, crd ServerCRD, sum *Summary) error {
+	fresh, ok, err := r.Store.FreshBackup(ctx, c.Name, c.worldSince())
 	if err != nil {
 		return fmt.Errorf("lookup fresh backup: %w", err)
 	}
 	if ok {
-		return r.finishReap(ctx, now, c, fresh.Ref, sum)
+		return r.finishReap(ctx, now, c, crd, fresh.Ref, sum)
+	}
+	if c.retiring() {
+		// A retired server that never had a world has nothing to archive.
+		return r.retire(ctx, now, c, crd, sum)
 	}
 	if c.OwnerID == "" {
 		if err := r.Store.RestartClock(ctx, c.Name, now); err != nil {
@@ -628,6 +703,58 @@ func (r *Reaper) reapNoWorld(ctx context.Context, now time.Time, c Candidate, su
 	return nil
 }
 
+// retire carries out a retirement once the world is archived and its volume
+// deleted, or there was none: a given-up server is released for someone else to
+// claim, and a deleted one loses its MinecraftServer and then its row.
+func (r *Reaper) retire(ctx context.Context, now time.Time, c Candidate, crd ServerCRD, sum *Summary) error {
+	if !c.RetireDelete {
+		if err := r.Store.ReleaseWorld(ctx, c.Name, now); err != nil {
+			return fmt.Errorf("release world: %w", err)
+		}
+		if err := r.Store.Audit(ctx, AuditRecord{Action: ActionReleaseWorld, ServerName: c.Name, FormerOwner: c.OwnerID}); err != nil {
+			r.log().Error("reaper: audit release_world failed", "server", c.Name, "err", err)
+		}
+		sum.Released++
+		r.log().Info("reaper: server given up and released", "server", c.Name, "former_owner", c.OwnerID)
+		return nil
+	}
+	// The row goes last: a failure in between leaves a row that still asks for
+	// its deletion, and the next run finishes it (forgetServer).
+	if err := r.Cluster.DeleteServer(ctx, c.Name, crd.UID); err != nil {
+		return fmt.Errorf("delete server: %w", err)
+	}
+	return r.deleteRow(ctx, now, c, sum)
+}
+
+// forgetServer finishes the deletion of a server whose MinecraftServer is gone.
+// A world volume left behind (the StatefulSet retains claims, so removing the
+// MinecraftServer by hand leaves it) is never deleted unarchived, and without the
+// MinecraftServer the reaper cannot hold it still to archive it: an operator
+// takes it from there, and the run reports the server until then.
+func (r *Reaper) forgetServer(ctx context.Context, now time.Time, c Candidate, sum *Summary) error {
+	pvc := WorldPVCName(c.Name)
+	exists, err := r.Cluster.WorldExists(ctx, pvc)
+	if err != nil {
+		return fmt.Errorf("look up world volume: %w", err)
+	}
+	if exists {
+		return fmt.Errorf("its MinecraftServer is gone but world volume %s is still there; archive and remove it by hand to finish the deletion", pvc)
+	}
+	return r.deleteRow(ctx, now, c, sum)
+}
+
+func (r *Reaper) deleteRow(ctx context.Context, now time.Time, c Candidate, sum *Summary) error {
+	if err := r.Store.DeleteServerRow(ctx, c.Name, now); err != nil {
+		return fmt.Errorf("mark server deleted: %w", err)
+	}
+	if err := r.Store.Audit(ctx, AuditRecord{Action: ActionDeleteServer, ServerName: c.Name, FormerOwner: c.OwnerID}); err != nil {
+		r.log().Error("reaper: audit delete_server failed", "server", c.Name, "err", err)
+	}
+	sum.ServersDeleted++
+	r.log().Info("reaper: server deleted", "server", c.Name, "former_owner", c.OwnerID)
+	return nil
+}
+
 // ensureCapacity frees the backup store down under MaxLocalBytes by evicting the
 // oldest present backups early. Early eviction is destructive (it removes
 // not-yet-expired backups), so each eviction is alerted and audited. It returns

internal/reaper/reaper_test.go

@@ -115,6 +115,9 @@ type fakeCluster struct {
 	held     map[string]bool // servers held right now
 	holds    []string
 	lost     context.CancelCauseFunc
+
+	deletedServers  []string // name/uid of each DeleteServer
+	deleteServerErr error
 }
 
 func (c *fakeCluster) HoldWorld(ctx context.Context, name string) (context.Context, func(), error) {
@@ -167,10 +170,24 @@ func (c *fakeCluster) DeletePVC(ctx context.Context, pvc string) error {
 	return nil
 }
 
+func (c *fakeCluster) DeleteServer(ctx context.Context, name, uid string) error {
+	if c.deleteServerErr != nil {
+		return c.deleteServerErr
+	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+	c.deletedServers = append(c.deletedServers, name+"/"+uid)
+	delete(c.crds, name)
+	c.rec.add("deleteServer")
+	return nil
+}
+
 // ---- fake Store -----------------------------------------------------------
 
 type fakeBackup struct {
 	id, server, ref    string
+	owner              string
 	reason             string
 	size               int64
 	status             string // present | deleted
@@ -190,6 +207,8 @@ type fakeStore struct {
 	backups   []*fakeBackup
 	audits    []AuditRecord
 	released  []string
+	deleted   []string
+	deleteErr error
 	listErr   error
 	insertErr error
 	liveErr   error
@@ -210,7 +229,7 @@ func (s *fakeStore) ListActiveServers(context.Context) ([]Candidate, error) {
 func (s *fakeStore) FreshBackup(_ context.Context, server string, since time.Time) (Fresh, bool, error) {
 	var found *fakeBackup
 	for _, b := range s.backups {
-		if b.server == server && b.status == "present" && b.reason == ReasonInactive && !b.createdAt.Before(since) && b.corruptAt.IsZero() {
+		if b.server == server && b.status == "present" && (b.reason == ReasonInactive || b.reason == ReasonReleased) && !b.createdAt.Before(since) && b.corruptAt.IsZero() {
 			if found == nil || (b.offsite && !found.offsite) {
 				found = b
 			}
@@ -227,7 +246,7 @@ func (s *fakeStore) InsertBackup(_ context.Context, rec BackupRecord) error {
 		return s.insertErr
 	}
 	s.backups = append(s.backups, &fakeBackup{
-		id: rec.ID, server: rec.ServerName, ref: rec.BackupRef, reason: rec.Reason, size: rec.SizeBytes,
+		id: rec.ID, server: rec.ServerName, owner: rec.FormerOwner, ref: rec.BackupRef, reason: rec.Reason, size: rec.SizeBytes,
 		status: "present", createdAt: s.clock, expires: rec.ExpiresAt, sha: rec.SHA256, skipped: rec.SkippedEntries,
 	})
 	s.rec.add("insert")
@@ -240,11 +259,36 @@ func (s *fakeStore) ReleaseWorld(_ context.Context, name string, at time.Time) e
 	c.LastActiveAt = at
 	c.Warned3dAt = time.Time{}
 	c.Warned1dAt = time.Time{}
+	if !c.RetireDelete {
+		c.RetireRequestedAt = time.Time{}
+	}
 	s.released = append(s.released, name)
 	s.rec.add("release")
 	return nil
 }
 
+// DeleteServerRow drops the row from the listing, as deleted_at does, and only
+// when it asks for its deletion (PGStore's condition).
+func (s *fakeStore) DeleteServerRow(_ context.Context, name string, _ time.Time) error {
+	if s.deleteErr != nil {
+		return s.deleteErr
+	}
+	c := s.byName[name]
+	if c == nil || !c.RetireDelete {
+		return nil
+	}
+	delete(s.byName, name)
+	for i, n := range s.order {
+		if n == name {
+			s.order = append(s.order[:i], s.order[i+1:]...)
+			break
+		}
+	}
+	s.deleted = append(s.deleted, name)
+	s.rec.add("deleteRow")
+	return nil
+}
+
 func (s *fakeStore) RestartClock(_ context.Context, name string, at time.Time) error {
 	c := s.byName[name]
 	c.LastActiveAt = at
@@ -277,12 +321,12 @@ func (s *fakeStore) PresentBackupBytes(context.Context) (int64, error) {
 func (s *fakeStore) EvictableBackups(context.Context) ([]StoredBackup, error) {
 	var ps []*fakeBackup
 	for _, b := range s.backups {
-		if b.status == "present" && (b.reason != ReasonInactive || b.offsite) {
+		if b.status == "present" && ((b.reason != ReasonInactive && b.reason != ReasonReleased) || b.offsite) {
 			ps = append(ps, b)
 		}
 	}
 	sort.SliceStable(ps, func(i, j int) bool {
-		if ri, rj := ps[i].reason == ReasonInactive, ps[j].reason == ReasonInactive; ri != rj {
+		if ri, rj := isArchive(ps[i].reason), isArchive(ps[j].reason); ri != rj {
 			return rj
 		}
 		return ps[i].createdAt.Before(ps[j].createdAt)
@@ -294,6 +338,8 @@ func (s *fakeStore) EvictableBackups(context.Context) ([]StoredBackup, error) {
 	return out, nil
 }
 
+func isArchive(reason string) bool { return reason == ReasonInactive || reason == ReasonReleased }
+
 func (s *fakeStore) ListExpiredBackups(_ context.Context, now time.Time) ([]StoredBackup, error) {
 	var out []StoredBackup
 	for _, b := range s.backups {
@@ -397,7 +443,7 @@ func newReaper(cfg Config, cands ...Candidate) (*Reaper, *fakeStore, *fakeCluste
 		cc := cands[i]
 		st.byName[cc.Name] = &cc
 		st.order = append(st.order, cc.Name)
-		cl.crds[cc.Name] = ServerCRD{PVC: "world-" + cc.Name + "-0"}
+		cl.crds[cc.Name] = ServerCRD{PVC: "world-" + cc.Name + "-0", UID: "uid-" + cc.Name}
 	}
 	ar := &fakeArchiver{rec: rec}
 	r := &Reaper{

internal/reaper/retire_test.go

@@ -0,0 +1,211 @@
+package reaper
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+	"time"
+)
+
+// retiring is a candidate whose owner gave it up an hour ago, a day after they
+// last played: nowhere near idle, so only the request makes the reaper act.
+func retiring(name, owner string, del bool) Candidate {
+	return Candidate{Name: name, OwnerID: owner, LastActiveAt: idleBy(Day),
+		RetireRequestedAt: idleBy(time.Hour), RetireDelete: del}
+}
+
+// A server its owner gave up goes on the next run however recently it was
+// played: archived as a "released" backup under the owner, volume deleted, and
+// released for someone else to claim, with the request done with.
+func TestRetireReleaseArchivesThenReleases(t *testing.T) {
+	r, st, cl, _ := newReaper(DefaultConfig(), retiring("alpha", "user-7", false))
+
+	sum := mustRun(t, r)
+
+	want := []string{"hold", "archive", "insert", "deletePVC", "release", "audit:" + ActionReleaseWorld, "unhold"}
+	if !reflect.DeepEqual(st.rec.events, want) {
+		t.Fatalf("call order = %v, want %v", st.rec.events, want)
+	}
+	if len(st.backups) != 1 || st.backups[0].reason != ReasonReleased || st.backups[0].owner != "user-7" {
+		t.Fatalf("backups = %+v, want one released archive recorded against user-7", st.backups)
+	}
+	if want := testNow.Add(DefaultConfig().Retention); !st.backups[0].expires.Equal(want) {
+		t.Fatalf("archive expires %v, want %v", st.backups[0].expires, want)
+	}
+	if len(st.audits) != 1 || st.audits[0].FormerOwner != "user-7" {
+		t.Fatalf("audits = %+v", st.audits)
+	}
+	c := st.byName["alpha"]
+	if c.OwnerID != "" || !c.RetireRequestedAt.IsZero() {
+		t.Fatalf("after release: %+v, want no owner and no pending request", c)
+	}
+	if len(cl.deletedServers) != 0 || len(st.deleted) != 0 {
+		t.Fatalf("a release deleted the server: %v %v", cl.deletedServers, st.deleted)
+	}
+	if sum.WorldsReaped != 1 || sum.Released != 1 || sum.ServersDeleted != 0 || sum.Skipped != 0 {
+		t.Fatalf("summary = %+v", sum)
+	}
+}
+
+// An admin's deletion archives the world the same way, then removes the
+// MinecraftServer that was inspected (by uid) and only after it the row.
+func TestRetireDeleteRemovesServerThenRow(t *testing.T) {
+	r, st, cl, _ := newReaper(DefaultConfig(), retiring("beta", "user-2", true))
+
+	sum := mustRun(t, r)
+
+	want := []string{"hold", "archive", "insert", "deletePVC", "deleteServer", "deleteRow", "audit:" + ActionDeleteServer, "unhold"}
+	if !reflect.DeepEqual(st.rec.events, want) {
+		t.Fatalf("call order = %v, want %v", st.rec.events, want)
+	}
+	if !reflect.DeepEqual(cl.deletedServers, []string{"beta/uid-beta"}) {
+		t.Fatalf("deleted servers = %v, want [beta/uid-beta]", cl.deletedServers)
+	}
+	if !reflect.DeepEqual(st.deleted, []string{"beta"}) || len(st.released) != 0 {
+		t.Fatalf("rows deleted %v released %v", st.deleted, st.released)
+	}
+	if len(st.backups) != 1 || st.backups[0].reason != ReasonReleased || st.backups[0].owner != "user-2" {
+		t.Fatalf("backups = %+v", st.backups)
+	}
+	if sum.WorldsReaped != 1 || sum.ServersDeleted != 1 || sum.Released != 0 {
+		t.Fatalf("summary = %+v", sum)
+	}
+}
+
+// The archive a retirement leaves must be of the world as its owner left it: one
+// taken before the request (a reap waiting for its off-site copy, say) is not
+// reused, one taken after it is.
+func TestRetireReusesOnlyAnArchiveTakenAfterTheRequest(t *testing.T) {
+	c := retiring("gamma", "user-3", false)
+
+	r, st, _, ar := newReaper(DefaultConfig(), c)
+	st.backups = append(st.backups, &fakeBackup{id: "old", server: "gamma", ref: "ref-old", reason: ReasonInactive,
+		status: "present", createdAt: c.RetireRequestedAt.Add(-1), sha: "sha-ref-old"})
+	mustRun(t, r)
+	if ar.archives != 1 {
+		t.Fatalf("archives = %d: an archive older than the request was reused", ar.archives)
+	}
+
+	r, st, _, ar = newReaper(DefaultConfig(), c)
+	st.backups = append(st.backups, &fakeBackup{id: "new", server: "gamma", ref: "ref-new", reason: ReasonReleased,
+		status: "present", createdAt: c.RetireRequestedAt.Add(1), sha: "sha-ref-new"})
+	mustRun(t, r)
+	if ar.archives != 0 {
+		t.Fatalf("archives = %d: the archive taken after the request was not reused", ar.archives)
+	}
+}
+
+// A retired server that never had a world has nothing to archive: it is
+// released (an unowned one too, where an idle one only restarts its clock), or
+// deleted.
+func TestRetireWithNoWorld(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		c     Candidate
+		want  []string
+		check func(*testing.T, *fakeStore, *fakeCluster, Summary)
+	}{
+		{"owned release", retiring("a", "user-1", false),
+			[]string{"hold", "release", "audit:" + ActionReleaseWorld, "unhold"},
+			func(t *testing.T, st *fakeStore, _ *fakeCluster, sum Summary) {
+				if sum.Released != 1 || sum.WorldsReaped != 0 {
+					t.Errorf("summary = %+v", sum)
+				}
+			}},
+		{"unowned release", retiring("a", "", false),
+			[]string{"hold", "release", "audit:" + ActionReleaseWorld, "unhold"},
+			func(t *testing.T, st *fakeStore, _ *fakeCluster, _ Summary) {
+				if !st.byName["a"].RetireRequestedAt.IsZero() {
+					t.Errorf("request still pending: %+v", st.byName["a"])
+				}
+			}},
+		{"delete", retiring("a", "user-1", true),
+			[]string{"hold", "deleteServer", "deleteRow", "audit:" + ActionDeleteServer, "unhold"},
+			func(t *testing.T, _ *fakeStore, cl *fakeCluster, sum Summary) {
+				if sum.ServersDeleted != 1 || sum.WorldsReaped != 0 || len(cl.deletedServers) != 1 {
+					t.Errorf("summary = %+v, deleted %v", sum, cl.deletedServers)
+				}
+			}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			r, st, cl, ar := newReaper(DefaultConfig(), tc.c)
+			cl.noWorld = map[string]bool{"world-a-0": true}
+			sum := mustRun(t, r)
+			if !reflect.DeepEqual(st.rec.events, tc.want) {
+				t.Fatalf("call order = %v, want %v", st.rec.events, tc.want)
+			}
+			if ar.archives != 0 || cl.deletePVCCalls != 0 {
+				t.Fatalf("archived %d, deleted %d PVCs of a server with no world", ar.archives, cl.deletePVCCalls)
+			}
+			tc.check(t, st, cl, sum)
+		})
+	}
+}
+
+// A deletion interrupted after the MinecraftServer went (or one removed by hand)
+// is finished from the row alone, unless a world volume is still there: that is
+// never deleted unarchived, and the run reports it. A release request with no
+// MinecraftServer is left alone as before.
+func TestRetireDeleteWithTheServerGone(t *testing.T) {
+	r, st, cl, _ := newReaper(DefaultConfig(), retiring("gone", "user-4", true))
+	delete(cl.crds, "gone")
+	cl.noWorld = map[string]bool{"world-gone-0": true}
+	sum := mustRun(t, r)
+	if !reflect.DeepEqual(st.rec.events, []string{"deleteRow", "audit:" + ActionDeleteServer}) || sum.ServersDeleted != 1 {
+		t.Fatalf("events %v, summary %+v", st.rec.events, sum)
+	}
+
+	r, st, cl, _ = newReaper(DefaultConfig(), retiring("gone", "user-4", true))
+	delete(cl.crds, "gone")
+	sum = mustRun(t, r)
+	if len(st.deleted) != 0 || sum.Skipped != 1 || sum.ServersDeleted != 0 {
+		t.Fatalf("with its world volume left: deleted %v, summary %+v", st.deleted, sum)
+	}
+
+	r, st, cl, _ = newReaper(DefaultConfig(), retiring("gone", "user-4", false))
+	delete(cl.crds, "gone")
+	sum = mustRun(t, r)
+	if len(st.rec.events) != 0 || sum.Skipped != 0 {
+		t.Fatalf("release with no server: events %v, summary %+v", st.rec.events, sum)
+	}
+}
+
+// A failed MinecraftServer delete keeps the row asking for its deletion, and the
+// next run finishes it from the archive the first one left.
+func TestRetireDeleteServerFailureIsRetried(t *testing.T) {
+	r, st, cl, ar := newReaper(DefaultConfig(), retiring("delta", "user-5", true))
+	cl.deleteServerErr = errors.New("apiserver down")
+
+	sum := mustRun(t, r)
+	if sum.Skipped != 1 || len(st.deleted) != 0 || st.byName["delta"] == nil || !st.byName["delta"].RetireDelete {
+		t.Fatalf("after a failed delete: summary %+v, deleted %v, row %+v", sum, st.deleted, st.byName["delta"])
+	}
+
+	cl.deleteServerErr = nil
+	cl.noWorld = map[string]bool{"world-delta-0": true} // the first run deleted it
+	sum = mustRun(t, r)
+	if ar.archives != 1 || !reflect.DeepEqual(st.deleted, []string{"delta"}) || sum.ServersDeleted != 1 {
+		t.Fatalf("retry: archives %d, deleted %v, summary %+v", ar.archives, st.deleted, sum)
+	}
+}
+
+// A system server is never given up, whatever its row says.
+func TestRetireExemptServerIgnored(t *testing.T) {
+	r, st, cl, ar := newReaper(DefaultConfig(), retiring("lobby", "", true))
+	cl.crds["lobby"] = ServerCRD{Exempt: true, PVC: "world-lobby-0", UID: "uid-lobby"}
+	sum := mustRun(t, r)
+	if len(st.rec.events) != 0 || ar.archives != 0 || sum.Skipped != 0 {
+		t.Fatalf("exempt server touched: events %v, summary %+v", st.rec.events, sum)
+	}
+}
+
+// With the off-site copy required, a retirement waits for it like an idle reap.
+func TestRetireWaitsForTheOffsiteCopy(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.RequireOffsite = true
+	r, st, cl, _ := newReaper(cfg, retiring("eps", "user-6", true))
+	sum := mustRun(t, r)
+	if sum.AwaitingOffsite != 1 || cl.deletePVCCalls != 0 || len(cl.deletedServers) != 0 || len(st.deleted) != 0 {
+		t.Fatalf("summary %+v, deletePVC %d, servers %v", sum, cl.deletePVCCalls, cl.deletedServers)
+	}
+}