feat(backups): 主人和管理员可删除单个备份，归档由 reaper 下一轮删除、异地副本随下次同步删除，恢复可能在读时拒绝

internal/api/api.go

@@ -533,11 +533,12 @@ func (a *API) externalAPIRoutes() []apiRoute {
 		// identity (including email_verified) to drive the setup flow.
 		{Method: "GET", Pattern: "/api/v1/me", SetupAllowed: true, h: a.handleMe},
 		{Method: "GET", Pattern: "/api/v1/me/servers", SetupAllowed: true, h: a.handleMyServers},
-		// World backups (spec §7, §466). Both are app-tier: GET /backups is scoped
+		// World backups (spec §7, §466). All are app-tier: GET /backups is scoped
 		// inside the handler (admin sees all; a user sees only worlds they formerly
-		// owned), and restore is gated by owner-or-admin PLUS a former-owner match, so
-		// neither sits behind adminOnly.
+		// owned), DELETE takes the same scope, and restore is gated by owner-or-admin
+		// PLUS a former-owner match, so none sits behind adminOnly.
 		{Method: "GET", Pattern: "/api/v1/backups", h: a.handleListBackups},
+		{Method: "DELETE", Pattern: "/api/v1/backups/{id}", h: a.handleDeleteBackup},
 		{Method: "GET", Pattern: "/api/v1/servers/{name}/jobs", h: a.handleServerJobs},
 		{Method: "POST", Pattern: "/api/v1/servers/{name}/restore-backup", h: a.handleRestoreBackup},
 		{Method: "POST", Pattern: "/api/v1/servers/{name}/backup", h: a.handleBackupNow},

internal/api/api_test.go

@@ -1058,6 +1058,20 @@ func (f *fakeRepo) BackupByID(_ context.Context, id string) (*BackupRecord, erro
 	return nil, ErrNotFound
 }
 
+func (f *fakeRepo) ExpireBackup(_ context.Context, id string, at time.Time) error {
+	for i := range f.backups {
+		b := &f.backups[i]
+		if b.view.Status == "present" && b.view.ID == id {
+			b.view.Status = "expired"
+			if at.Before(b.view.ExpiresAt) {
+				b.view.ExpiresAt = at
+			}
+			return nil
+		}
+	}
+	return ErrNotFound
+}
+
 // ---- staff / session auth fakes (spec §B, passwordless) ----
 // Each method mirrors the PGRepo contract: a returned StaffUser is copied so a
 // test cannot mutate the stored row by reference, SessionUser re-reads the

internal/api/handlers_backups.go

@@ -74,6 +74,75 @@ func (a *API) handleListBackups(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]any{"backups": backups, "total": total})
 }
 
+// handleDeleteBackup deletes one world backup (DELETE /api/v1/backups/{id}). An
+// admin may delete any; a user only one of a world they owned, the scope that
+// lists and restores it, and any other id reads as unknown (404), as it does in
+// their list. The row turns expired at once, so no list, restore or backup
+// budget counts it again; the reaper's next retention pass deletes the archive
+// and the off-site copy's next sync the bucket's copy. A reaped world's backup
+// is that world's only copy, which the panel says before it asks.
+//
+// A restore running on the backup's server may be reading the archive, so the
+// delete waits for it: a restore Job still running, or a safety snapshot whose
+// restore of this backup has yet to start, is 409 restore_in_progress. Without
+// a JobStatus reader there is nothing to ask and the delete goes ahead: the
+// reaper removes the archive at its next daily run, long after any restore
+// admitted before the delete has read it.
+func (a *API) handleDeleteBackup(w http.ResponseWriter, r *http.Request) {
+	p := principalFromContext(r.Context())
+	backup, err := a.Repo.BackupByID(r.Context(), r.PathValue("id"))
+	if err == nil && !p.IsAdmin() && (p.UserID == "" || backup.FormerOwner != p.UserID) {
+		err = ErrNotFound
+	}
+	if errors.Is(err, ErrNotFound) {
+		writeError(w, r, newError(http.StatusNotFound, "no_backup", "no matching backup exists"))
+		return
+	}
+	if err != nil {
+		writeError(w, r, err)
+		return
+	}
+	if a.JobStatus != nil {
+		jobs, err := a.JobStatus.LatestJobs(r.Context(), backup.ServerName)
+		if err != nil {
+			writeError(w, r, err)
+			return
+		}
+		if restoreMayRead(jobs, backup.ID) {
+			writeError(w, r, newError(http.StatusConflict, "restore_in_progress",
+				"a restore is running on this backup's server and may be reading it; delete it once the restore finishes"))
+			return
+		}
+	}
+	if err := a.Repo.ExpireBackup(r.Context(), backup.ID, a.now()); err != nil {
+		if errors.Is(err, ErrNotFound) {
+			writeError(w, r, newError(http.StatusNotFound, "no_backup", "no matching backup exists"))
+			return
+		}
+		writeError(w, r, err)
+		return
+	}
+	e := AuditEntry{Actor: auditActor(p), ActorUserID: p.UserID, Action: "backup.delete", ServerName: backup.ServerName}
+	e.Payload = auditPayload(map[string]any{"backup_id": backup.ID, "former_owner": backup.FormerOwner, "size_bytes": backup.SizeBytes})
+	a.auditEntry(r, e)
+	writeJSON(w, http.StatusOK, map[string]any{"id": backup.ID, "status": "expired"})
+}
+
+// restoreMayRead reports whether one of a server's Jobs may still read backup
+// id's archive: any restore Job not yet finished (which archive it extracts is
+// not on the Job), or a safety snapshot whose restore of id has not started.
+func restoreMayRead(jobs []AsyncJob, id string) bool {
+	for _, j := range jobs {
+		if j.Kind == "restore" && j.State == "running" {
+			return true
+		}
+		if j.ThenRestore == maintenance.ThenRestorePending && j.RestoreBackupID == id {
+			return true
+		}
+	}
+	return false
+}
+
 // handleRestoreBackup starts restoring a server's world from a backup (spec §7
 // POST /servers/{name}/restore-backup; spec §466). It accepts an optional JSON
 // body with a backup_id; when absent it restores the latest backup for the server

internal/api/handlers_backups_test.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"net/http"
@@ -542,3 +543,226 @@ func TestRestoreBackup(t *testing.T) {
 		}
 	})
 }
+
+// TestDeleteBackup covers DELETE /api/v1/backups/{id}: the scope that lists a
+// backup deletes it, the row turns expired (out of lists, restores and the
+// budget, expires_at pulled to now so the reaper and the off-site sync take it
+// next), an out-of-scope id reads as unknown, and a restore that may be reading
+// the archive holds the delete off.
+func TestDeleteBackup(t *testing.T) {
+	owner := &Principal{UserID: "owner1", Email: "[email protected]", Role: "user"}
+	admin := &Principal{UserID: "admin1", Email: "[email protected]", Role: "admin", ViaAdminAccess: true}
+	expires := time.Unix(1_706_000_000, 0)
+
+	mk := func(p *Principal) (*API, *fakeRepo) {
+		repo := newFakeRepo()
+		repo.backups = []fakeBackup{
+			{view: BackupView{ID: "bk1", ServerName: "survival", FormerOwner: "owner1",
+				Status: "present", Reason: "manual", SizeBytes: 1024,
+				CreatedAt: time.Unix(1_699_000_000, 0), ExpiresAt: expires}, ref: "ref-bk1"},
+			{view: BackupView{ID: "bk2", ServerName: "creative", FormerOwner: "owner2",
+				Status: "present", Reason: "inactive_15d", SizeBytes: 2048,
+				CreatedAt: time.Unix(1_699_500_000, 0), ExpiresAt: expires}, ref: "ref-bk2"},
+		}
+		api := newTestAPI(repo, newFakeCluster())
+		api.External = staticExternal{p: p}
+		return api, repo
+	}
+	del := func(api *API, id string) *httptest.ResponseRecorder {
+		return do(api.ExternalHandler(), "DELETE", "/api/v1/backups/"+id, "", nil)
+	}
+	status := func(repo *fakeRepo, id string) string {
+		for _, b := range repo.backups {
+			if b.view.ID == id {
+				return b.view.Status
+			}
+		}
+		return ""
+	}
+
+	t.Run("former owner deletes -> 200, expired, audited", func(t *testing.T) {
+		api, repo := mk(owner)
+		w := del(api, "bk1")
+		if w.Code != http.StatusOK {
+			t.Fatalf("code = %d, want 200 (%s)", w.Code, w.Body.String())
+		}
+		var resp map[string]any
+		if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+			t.Fatalf("body not JSON: %v (%s)", err, w.Body.String())
+		}
+		if len(resp) != 2 || resp["id"] != "bk1" || resp["status"] != "expired" {
+			t.Fatalf("body = %v, want {id: bk1, status: expired}", resp)
+		}
+		b := repo.backups[0].view
+		if b.Status != "expired" || !b.ExpiresAt.Equal(api.now()) {
+			t.Fatalf("row = %s expiring %v, want expired expiring %v", b.Status, b.ExpiresAt, api.now())
+		}
+		if status(repo, "bk2") != "present" {
+			t.Fatal("deleting bk1 touched bk2")
+		}
+		if n, _ := repo.BackupStoreBytes(t.Context()); n != 2048 {
+			t.Fatalf("backup budget counts %d bytes, want 2048", n)
+		}
+		lw := do(api.ExternalHandler(), "GET", "/api/v1/backups", "", nil)
+		if strings.Contains(lw.Body.String(), `"bk1"`) {
+			t.Fatalf("deleted backup still listed: %s", lw.Body.String())
+		}
+		if len(repo.audits) != 1 {
+			t.Fatalf("audits = %+v, want one", repo.audits)
+		}
+		a := repo.audits[0]
+		if a.Action != "backup.delete" || a.Actor != "[email protected]" || a.ActorUserID != "owner1" || a.ServerName != "survival" {
+			t.Fatalf("audit = %+v", a)
+		}
+		var payload map[string]any
+		if err := json.Unmarshal(a.Payload, &payload); err != nil {
+			t.Fatalf("payload not JSON: %v (%s)", err, a.Payload)
+		}
+		if payload["backup_id"] != "bk1" || payload["former_owner"] != "owner1" || payload["size_bytes"] != float64(1024) {
+			t.Fatalf("payload = %v", payload)
+		}
+		if w := del(api, "bk1"); w.Code != http.StatusNotFound || decodeErr(t, w) != "no_backup" {
+			t.Fatalf("second delete: code = %d body %s, want 404 no_backup", w.Code, w.Body.String())
+		}
+	})
+
+	t.Run("expires_at already past stays put", func(t *testing.T) {
+		api, repo := mk(owner)
+		past := api.now().Add(-time.Hour)
+		repo.backups[0].view.ExpiresAt = past
+		if w := del(api, "bk1"); w.Code != http.StatusOK {
+			t.Fatalf("code = %d (%s)", w.Code, w.Body.String())
+		}
+		if got := repo.backups[0].view.ExpiresAt; !got.Equal(past) {
+			t.Fatalf("expires_at = %v, want %v", got, past)
+		}
+	})
+
+	t.Run("admin deletes another's backup -> 200", func(t *testing.T) {
+		api, repo := mk(admin)
+		if w := del(api, "bk2"); w.Code != http.StatusOK {
+			t.Fatalf("code = %d, want 200 (%s)", w.Code, w.Body.String())
+		}
+		if status(repo, "bk2") != "expired" {
+			t.Fatalf("bk2 = %s, want expired", status(repo, "bk2"))
+		}
+	})
+
+	for _, tc := range []struct {
+		name string
+		p    *Principal
+		id   string
+		prep func(*fakeRepo)
+	}{
+		{"another user's backup", owner, "bk2", nil},
+		{"unknown id", admin, "nope", nil},
+		{"already deleted", admin, "bk1", func(r *fakeRepo) { r.backups[0].view.Status = "deleted" }},
+		{"already expired", owner, "bk1", func(r *fakeRepo) { r.backups[0].view.Status = "expired" }},
+		{"no user id against no former owner", &Principal{Role: "user"}, "bk1",
+			func(r *fakeRepo) { r.backups[0].view.FormerOwner = "" }},
+	} {
+		t.Run(tc.name+" -> 404 no_backup", func(t *testing.T) {
+			api, repo := mk(tc.p)
+			if tc.prep != nil {
+				tc.prep(repo)
+			}
+			before := []string{status(repo, "bk1"), status(repo, "bk2")}
+			w := del(api, tc.id)
+			if w.Code != http.StatusNotFound || decodeErr(t, w) != "no_backup" {
+				t.Fatalf("code = %d body %s, want 404 no_backup", w.Code, w.Body.String())
+			}
+			if after := []string{status(repo, "bk1"), status(repo, "bk2")}; after[0] != before[0] || after[1] != before[1] {
+				t.Fatalf("statuses %v -> %v, want unchanged", before, after)
+			}
+			if len(repo.audits) != 0 {
+				t.Fatalf("refused delete audited: %+v", repo.audits)
+			}
+		})
+	}
+
+	for _, tc := range []struct {
+		name string
+		jobs []AsyncJob
+		want int
+	}{
+		{"restore running", []AsyncJob{{Kind: "restore", State: "running"}}, http.StatusConflict},
+		{"snapshot before restoring this backup", []AsyncJob{{Kind: "backup", State: "running",
+			ThenRestore: "pending", RestoreBackupID: "bk1"}}, http.StatusConflict},
+		{"snapshot done, restore of this backup still to start", []AsyncJob{{Kind: "backup", State: "succeeded",
+			ThenRestore: "pending", RestoreBackupID: "bk1"}}, http.StatusConflict},
+		{"snapshot before restoring another backup", []AsyncJob{{Kind: "backup", State: "running",
+			ThenRestore: "pending", RestoreBackupID: "bk9"}}, http.StatusOK},
+		{"restore of this backup started and finished", []AsyncJob{
+			{Kind: "restore", State: "succeeded"},
+			{Kind: "backup", State: "succeeded", ThenRestore: "started", RestoreBackupID: "bk1"}}, http.StatusOK},
+		{"chain abandoned", []AsyncJob{{Kind: "backup", State: "failed",
+			ThenRestore: "abandoned", RestoreBackupID: "bk1"}}, http.StatusOK},
+		{"restore failed", []AsyncJob{{Kind: "restore", State: "failed"}}, http.StatusOK},
+		{"plain backup running", []AsyncJob{{Kind: "backup", State: "running"}}, http.StatusOK},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			api, repo := mk(owner)
+			js := &fakeJobStatus{jobs: tc.jobs}
+			api.JobStatus = js
+			w := del(api, "bk1")
+			if w.Code != tc.want {
+				t.Fatalf("code = %d, want %d (%s)", w.Code, tc.want, w.Body.String())
+			}
+			if js.got != "survival" {
+				t.Fatalf("jobs read for %q, want survival", js.got)
+			}
+			want := "expired"
+			if tc.want == http.StatusConflict {
+				want = "present"
+				if decodeErr(t, w) != "restore_in_progress" {
+					t.Fatalf("error code %q, want restore_in_progress", decodeErr(t, w))
+				}
+			}
+			if status(repo, "bk1") != want {
+				t.Fatalf("bk1 = %s, want %s", status(repo, "bk1"), want)
+			}
+		})
+	}
+
+	t.Run("deleted by someone else after the lookup -> 404, not audited", func(t *testing.T) {
+		api, repo := mk(owner)
+		api.Repo = expireFails{repo, ErrNotFound}
+		if w := del(api, "bk1"); w.Code != http.StatusNotFound || decodeErr(t, w) != "no_backup" {
+			t.Fatalf("code = %d body %s, want 404 no_backup", w.Code, w.Body.String())
+		}
+		if len(repo.audits) != 0 {
+			t.Fatalf("lost race audited: %+v", repo.audits)
+		}
+	})
+
+	t.Run("expire fails -> 500, not audited", func(t *testing.T) {
+		api, repo := mk(owner)
+		api.Repo = expireFails{repo, errors.New("connection reset")}
+		if w := del(api, "bk1"); w.Code != http.StatusInternalServerError {
+			t.Fatalf("code = %d, want 500 (%s)", w.Code, w.Body.String())
+		}
+		if len(repo.audits) != 0 {
+			t.Fatalf("failed delete audited: %+v", repo.audits)
+		}
+	})
+
+	t.Run("job status error -> 500, kept", func(t *testing.T) {
+		api, repo := mk(owner)
+		api.JobStatus = &fakeJobStatus{err: errors.New("apiserver down")}
+		if w := del(api, "bk1"); w.Code != http.StatusInternalServerError {
+			t.Fatalf("code = %d, want 500 (%s)", w.Code, w.Body.String())
+		}
+		if status(repo, "bk1") != "present" || len(repo.audits) != 0 {
+			t.Fatalf("bk1 = %s audits %d, want present and none", status(repo, "bk1"), len(repo.audits))
+		}
+	})
+}
+
+// expireFails is a repo whose ExpireBackup answers err after the handler's
+// lookup found the backup: ErrNotFound is a concurrent delete winning.
+type expireFails struct {
+	*fakeRepo
+	err error
+}
+
+func (f expireFails) ExpireBackup(context.Context, string, time.Time) error { return f.err }

internal/api/pgrepo.go

@@ -968,6 +968,25 @@ func (p *PGRepo) BackupByID(ctx context.Context, id string) (*BackupRecord, erro
 	return &b, nil
 }
 
+// ExpireBackup marks one present backup expired. The reaper's ListExpiredBackups
+// picks expired rows up whatever their expires_at; pulling expires_at in to at is
+// what lets the off-site copy drop the bucket's copy (KeptRefs, ExpiredRefs)
+// instead of keeping it to the original date.
+func (p *PGRepo) ExpireBackup(ctx context.Context, id string, at time.Time) error {
+	res, err := p.db.ExecContext(ctx,
+		`UPDATE world_backups SET status = 'expired', expires_at = LEAST(expires_at, $2)
+		 WHERE id = $1 AND status = 'present'`, id, at)
+	if err != nil {
+		return err
+	}
+	if n, err := res.RowsAffected(); err != nil {
+		return err
+	} else if n == 0 {
+		return ErrNotFound
+	}
+	return nil
+}
+
 // LastBackupRequest reads the newest backup.create audit row for the server
 // since the given time; the created_at index bounds the scan to that window.
 func (p *PGRepo) LastBackupRequest(ctx context.Context, serverName string, since time.Time) (time.Time, error) {

internal/api/repo.go

@@ -413,6 +413,11 @@ type Repo interface {
 	// none matches. Like LatestBackup the returned BackupRecord carries the
 	// server-side backup_ref the restore path needs; the client never sees it.
 	BackupByID(ctx context.Context, id string) (*BackupRecord, error)
+	// ExpireBackup takes one present backup out of every list, restore and budget
+	// at once (status expired, expires_at no later than at), leaving its archive
+	// for the reaper's next retention pass and its off-site copy for the next
+	// sync. ErrNotFound when no present backup has that id.
+	ExpireBackup(ctx context.Context, id string, at time.Time) error
 	// LastBackupRequest returns when an on-demand backup of the server was last
 	// accepted (its newest backup.create audit row) at or after since, or the zero
 	// time when there was none. The since bound keeps the lookup inside the

internal/offsite/offsite_test.go

@@ -233,7 +233,7 @@ func (c *fakeCatalog) MarkOffsite(_ context.Context, id string, at time.Time) er
 func (c *fakeCatalog) ExpiredRefs(_ context.Context, now time.Time) ([]string, error) {
 	var out []string
 	for _, r := range c.rows {
-		if r.status == "deleted" && r.expires.Before(now) && !r.offsite.IsZero() {
+		if (r.status == "deleted" || r.status == "expired") && r.expires.Before(now) && !r.offsite.IsZero() {
 			out = append(out, r.Ref)
 		}
 	}
@@ -374,23 +374,26 @@ func TestSyncFailureLeavesRowPending(t *testing.T) {
 }
 
 // TestSyncExpiresOnlyPastRetention: a remote archive goes once its row has
-// expired; one evicted early from the local disk stays until then, and an
+// expired, or once its owner deleted it (status expired, expires_at pulled to
+// the delete); one evicted early from the local disk stays until then, and an
 // object with no row at all is left alone.
 func TestSyncExpiresOnlyPastRetention(t *testing.T) {
 	cat := &fakeCatalog{rows: []*row{
 		{WorldBackup: WorldBackup{ID: "old", Ref: "/a/old.tar.gz"}, status: "deleted", expires: now.Add(-time.Hour), offsite: now.Add(-100 * 24 * time.Hour)},
 		{WorldBackup: WorldBackup{ID: "evicted", Ref: "/a/evicted.tar.gz"}, status: "deleted", expires: now.Add(30 * 24 * time.Hour), offsite: now.Add(-24 * time.Hour)},
+		{WorldBackup: WorldBackup{ID: "dropped", Ref: "/a/dropped.tar.gz"}, status: "expired", expires: now.Add(-time.Minute), offsite: now.Add(-24 * time.Hour)},
 	}}
 	s, b := newSyncer(t, cat)
-	for _, k := range []string{"worlds/old.tar.gz.fenc", "worlds/evicted.tar.gz.fenc", "worlds/unknown.tar.gz.fenc"} {
+	for _, k := range []string{"worlds/old.tar.gz.fenc", "worlds/evicted.tar.gz.fenc", "worlds/dropped.tar.gz.fenc", "worlds/unknown.tar.gz.fenc"} {
 		b.objs[k] = []byte("x")
 	}
 	res, err := s.Run(context.Background())
 	if err != nil {
 		t.Fatal(err)
 	}
-	if res.WorldsExpired != 1 || len(b.removed) != 1 || b.removed[0] != "worlds/old.tar.gz.fenc" {
-		t.Fatalf("removed %v (expired %d), want only the expired archive", b.removed, res.WorldsExpired)
+	sort.Strings(b.removed)
+	if res.WorldsExpired != 2 || len(b.removed) != 2 || b.removed[0] != "worlds/dropped.tar.gz.fenc" || b.removed[1] != "worlds/old.tar.gz.fenc" {
+		t.Fatalf("removed %v (expired %d), want the expired and the owner-deleted archive", b.removed, res.WorldsExpired)
 	}
 	if res.RemoteWorlds != 2 {
 		t.Fatalf("remote worlds = %d, want 2 left", res.RemoteWorlds)

internal/offsite/pgcatalog.go

@@ -44,7 +44,7 @@ func (c PGCatalog) MarkOffsite(ctx context.Context, id string, at time.Time) err
 
 func (c PGCatalog) ExpiredRefs(ctx context.Context, now time.Time) ([]string, error) {
 	return c.refs(ctx, `SELECT backup_ref FROM world_backups
-		WHERE status = 'deleted' AND expires_at < $1 AND offsite_at IS NOT NULL`, now)
+		WHERE status IN ('deleted', 'expired') AND expires_at < $1 AND offsite_at IS NOT NULL`, now)
 }
 
 func (c PGCatalog) refs(ctx context.Context, q string, args ...any) ([]string, error) {

internal/offsite/sync.go

@@ -78,7 +78,8 @@ type Catalog interface {
 	// MarkOffsite records that the archive of row id is in the bucket.
 	MarkOffsite(ctx context.Context, id string, at time.Time) error
 	// ExpiredRefs lists the backup_ref of every row past its retention
-	// (deleted and expires_at < now) whose archive was copied off-site.
+	// (deleted, or expired: deleted through the API; and expires_at < now)
+	// whose archive was copied off-site.
 	ExpiredRefs(ctx context.Context, now time.Time) ([]string, error)
 	// PresentWorlds lists every present archive, for a restore of the volume.
 	PresentWorlds(ctx context.Context) ([]WorldBackup, error)

internal/pgint/offsite_test.go

@@ -4,10 +4,12 @@ package pgint
 
 import (
 	"context"
+	"errors"
 	"slices"
 	"testing"
 	"time"
 
+	"felis.lolicon.best/internal/api"
 	"felis.lolicon.best/internal/offsite"
 	"felis.lolicon.best/internal/reaper"
 )
@@ -58,3 +60,112 @@ func TestOffsiteCatalogNewestCopyAndKeptRefs(t *testing.T) {
 		}
 	}
 }
+
+// TestOwnerDeletedBackup: ExpireBackup turns one present backup expired with
+// expires_at pulled to the delete (never pushed later), after which no read
+// finds it, the backup budget drops its bytes, the reaper's retention pass
+// lists it whatever its expires_at, and the off-site sweep drops its copy.
+func TestOwnerDeletedBackup(t *testing.T) {
+	ctx := context.Background()
+	sfx := suffix(t)
+	server := "owner-del-" + sfx
+	t.Cleanup(func() { db.ExecContext(ctx, `DELETE FROM world_backups WHERE server_name = $1`, server) })
+	now := time.Now().UTC().Truncate(time.Microsecond)
+	insert := func(id string, size int64, expires time.Time) (string, string) {
+		t.Helper()
+		id += "-" + sfx
+		ref := "/archives/" + id + ".tar.gz"
+		mustExec(t, `INSERT INTO world_backups (id, server_name, backup_ref, size_bytes, reason, status, created_at, expires_at, offsite_at)
+			VALUES ($1, $2, $3, $4, 'manual', 'present', $5, $6, $7)`,
+			id, server, ref, size, now.Add(-reaper.Day), expires, now.Add(-time.Hour))
+		return id, ref
+	}
+	later := now.Add(90 * reaper.Day)
+	del, delRef := insert("del", 700, later)
+	keep, keepRef := insert("keep", 11, later)
+	past, _ := insert("past", 13, now.Add(-time.Minute))
+	row := func(id string) (string, time.Time) {
+		t.Helper()
+		var status string
+		var expires time.Time
+		if err := db.QueryRowContext(ctx, `SELECT status, expires_at FROM world_backups WHERE id = $1`, id).Scan(&status, &expires); err != nil {
+			t.Fatalf("read %s: %v", id, err)
+		}
+		return status, expires
+	}
+
+	before, err := repo.BackupStoreBytes(ctx)
+	if err != nil {
+		t.Fatalf("BackupStoreBytes: %v", err)
+	}
+	if err := repo.ExpireBackup(ctx, del, now); err != nil {
+		t.Fatalf("ExpireBackup: %v", err)
+	}
+	if s, e := row(del); s != "expired" || !e.Equal(now) {
+		t.Fatalf("deleted row = %s expiring %v, want expired expiring %v", s, e, now)
+	}
+	if s, e := row(keep); s != "present" || !e.Equal(later) {
+		t.Fatalf("other row = %s expiring %v, want untouched", s, e)
+	}
+	if after, err := repo.BackupStoreBytes(ctx); err != nil || after != before-700 {
+		t.Fatalf("BackupStoreBytes = %d, %v; want %d", after, err, before-700)
+	}
+	if _, err := repo.BackupByID(ctx, del); !errors.Is(err, api.ErrNotFound) {
+		t.Fatalf("BackupByID(deleted) = %v, want ErrNotFound", err)
+	}
+	for _, id := range []string{del, "missing-" + sfx} {
+		if err := repo.ExpireBackup(ctx, id, now); !errors.Is(err, api.ErrNotFound) {
+			t.Fatalf("ExpireBackup(%s) = %v, want ErrNotFound", id, err)
+		}
+	}
+	if err := repo.ExpireBackup(ctx, past, now); err != nil {
+		t.Fatalf("ExpireBackup(past): %v", err)
+	}
+	if s, e := row(past); s != "expired" || !e.Equal(now.Add(-time.Minute)) {
+		t.Fatalf("past row = %s expiring %v, want expired keeping %v", s, e, now.Add(-time.Minute))
+	}
+
+	// The reaper takes it even at a time before its expires_at.
+	exp, err := reaper.NewPGStore(db).ListExpiredBackups(ctx, now.Add(-reaper.Day))
+	if err != nil {
+		t.Fatalf("ListExpiredBackups: %v", err)
+	}
+	var ids []string
+	for _, b := range exp {
+		ids = append(ids, b.ID)
+	}
+	if !slices.Contains(ids, del) || slices.Contains(ids, keep) {
+		t.Fatalf("ListExpiredBackups = %v, want %s and not %s", ids, del, keep)
+	}
+
+	cat := offsite.PGCatalog{DB: db}
+	gone, err := cat.ExpiredRefs(ctx, now.Add(time.Second))
+	if err != nil {
+		t.Fatalf("ExpiredRefs: %v", err)
+	}
+	if !slices.Contains(gone, delRef) || slices.Contains(gone, keepRef) {
+		t.Fatalf("ExpiredRefs = %v, want %s and not %s", gone, delRef, keepRef)
+	}
+	kept, err := cat.KeptRefs(ctx, now.Add(time.Second))
+	if err != nil {
+		t.Fatalf("KeptRefs: %v", err)
+	}
+	if slices.Contains(kept, delRef) || !slices.Contains(kept, keepRef) {
+		t.Fatalf("KeptRefs = %v, want %s and not %s", kept, keepRef, delRef)
+	}
+
+	// The undo troubleshooting.md §10 gives, before the reaper has run: the backup
+	// is restorable again and the next sync copies it off site anew.
+	mustExec(t, `UPDATE world_backups SET status = 'present', expires_at = now() + interval '30 days', offsite_at = NULL
+   WHERE id = '`+del+`' AND status = 'expired'`)
+	if b, err := repo.BackupByID(ctx, del); err != nil || b.BackupRef != delRef {
+		t.Fatalf("BackupByID after the undo = (%+v, %v), want %s", b, err, delRef)
+	}
+	pending, err := cat.PendingWorlds(ctx)
+	if err != nil {
+		t.Fatalf("PendingWorlds: %v", err)
+	}
+	if !slices.ContainsFunc(pending, func(w offsite.WorldBackup) bool { return w.ID == del }) {
+		t.Fatalf("PendingWorlds after the undo lacks %s", del)
+	}
+}

internal/reaper/pgstore.go

@@ -170,7 +170,7 @@ func (s *PGStore) ExcessBackups(ctx context.Context, server, owner, reason strin
 
 func (s *PGStore) ListExpiredBackups(ctx context.Context, now time.Time) ([]StoredBackup, error) {
 	const q = `SELECT id, server_name, backup_ref, size_bytes, reason, COALESCE(sha256, '') FROM world_backups
-		WHERE status = 'present' AND expires_at < $1 ORDER BY expires_at ASC`
+		WHERE (status = 'present' AND expires_at < $1) OR status = 'expired' ORDER BY expires_at ASC`
 	return s.queryBackups(ctx, q, now)
 }
 

internal/reaper/reaper.go

@@ -288,7 +288,9 @@ type Store interface {
 	// copy of a deleted world and is never listed.
 	EvictableBackups(ctx context.Context) ([]StoredBackup, error)
 
-	// ListExpiredBackups lists status=present backups whose expires_at < now.
+	// ListExpiredBackups lists status=present backups whose expires_at < now,
+	// and every status=expired one: a backup its owner deleted (the API marks it
+	// so and leaves the archive to this pass).
 	ListExpiredBackups(ctx context.Context, now time.Time) ([]StoredBackup, error)
 
 	// MarkBackupDeleted flips a backup to status=deleted, deleted_at=at.

internal/reaper/reaper_test.go

@@ -343,7 +343,7 @@ func isArchive(reason string) bool { return reason == ReasonInactive || reason =
 func (s *fakeStore) ListExpiredBackups(_ context.Context, now time.Time) ([]StoredBackup, error) {
 	var out []StoredBackup
 	for _, b := range s.backups {
-		if b.status == "present" && b.expires.Before(now) {
+		if b.status == "present" && b.expires.Before(now) || b.status == "expired" {
 			out = append(out, StoredBackup{ID: b.id, ServerName: b.server, BackupRef: b.ref, SizeBytes: b.size})
 		}
 	}
@@ -932,27 +932,29 @@ func TestCapacityEvictionOrderSparesSoleCopies(t *testing.T) {
 	}
 }
 
-// Retention pass: backups past expires_at are deleted from the backend and
-// marked deleted; unexpired backups are untouched.
+// Retention pass: backups past expires_at, and any its owner deleted (status
+// expired) whatever its expires_at, are deleted from the backend and marked
+// deleted; unexpired backups are untouched.
 func TestExpiredBackupsDeleted(t *testing.T) {
 	r, st, _, ar := newReaper(DefaultConfig())
 	st.backups = []*fakeBackup{
 		{id: "gone", server: "s1", ref: "ref-gone", size: 5, status: "present", createdAt: idleBy(120 * Day), expires: idleBy(1 * Day)},
 		{id: "keep", server: "s2", ref: "ref-keep", size: 5, status: "present", createdAt: idleBy(10 * Day), expires: testNow.Add(80 * Day)},
+		{id: "dropped", server: "s3", ref: "ref-dropped", size: 5, status: "expired", createdAt: idleBy(10 * Day), expires: testNow.Add(80 * Day)},
 	}
 
 	sum := mustRun(t, r)
-	if sum.BackupsExpired != 1 {
-		t.Fatalf("BackupsExpired = %d, want 1", sum.BackupsExpired)
+	if sum.BackupsExpired != 2 {
+		t.Fatalf("BackupsExpired = %d, want 2", sum.BackupsExpired)
 	}
-	if len(ar.deletes) != 1 || ar.deletes[0] != "ref-gone" {
-		t.Fatalf("deleted archives = %v, want [ref-gone]", ar.deletes)
+	if len(ar.deletes) != 2 || ar.deletes[0] != "ref-gone" || ar.deletes[1] != "ref-dropped" {
+		t.Fatalf("deleted archives = %v, want [ref-gone ref-dropped]", ar.deletes)
 	}
 	byID := map[string]string{}
 	for _, b := range st.backups {
 		byID[b.id] = b.status
 	}
-	if byID["gone"] != "deleted" || byID["keep"] != "present" {
+	if byID["gone"] != "deleted" || byID["keep"] != "present" || byID["dropped"] != "deleted" {
 		t.Fatalf("expiry hit wrong rows: %v", byID)
 	}
 }