ci(release): write the sboms straight into dist

Bumps [react-router](https://github.com/remix-run/react-router/tree/HEAD/packages/react-router) to 7.18.4 and updates ancestor dependency [react-router-dom](https://github.com/remix-run/react-router/tree/HEAD/packages/react-router-dom). These dependencies need to be updated together.


Updates `react-router` from 6.30.4 to 7.18.4
- [Release notes](https://github.com/remix-run/react-router/releases)
- [Changelog](https://github.com/remix-run/react-router/blob/[email protected]/packages/react-router/CHANGELOG.md)
- [Commits](https://github.com/remix-run/react-router/commits/[email protected]/packages/react-router)

Updates `react-router-dom` from 6.30.4 to 7.18.4
- [Release notes](https://github.com/remix-run/react-router/releases)
- [Changelog](https://github.com/remix-run/react-router/blob/[email protected]/packages/react-router-dom/CHANGELOG.md)
- [Commits](https://github.com/remix-run/react-router/commits/[email protected]/packages/react-router-dom)

---
updated-dependencies:
- dependency-name: react-router
  dependency-version: 7.18.4
  dependency-type: indirect
- dependency-name: react-router-dom
  dependency-version: 7.18.4
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.dockerignore

@@ -31,3 +31,14 @@ Dockerfile
 *.key
 felis
 felis.exe
+
+# macOS materializes extended attributes as ._<name> sidecars (BSD tar uploads,
+# Finder copies, network volumes) and leaves .DS_Store behind. Neither is
+# source, and one is actively harmful: a ._*.sql beside the migrations is
+# //go:embed-ed into the binary and makes every `felis migrate` fail
+# ("non-numeric version") — observed live on a Mac-staged tree. Same exposure
+# for any tree the other //go:embed patterns walk (deploy/, plugins/).
+._*
+**/._*
+.DS_Store
+**/.DS_Store

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# The workflows pin every action to a commit SHA, and every Dockerfile pins its base images
+# by digest. This keeps those pins moving: Dependabot reads the "# vX.Y.Z" comment next to
+# each action SHA, and the tag in front of each image digest, and opens a PR that bumps both
+# together.
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directories:
+      - /
+      - /deploy/limbo
+      - /deploy/lobby
+      - /deploy/paper
+    schedule:
+      interval: weekly
+    # A new major is a runtime change (Paper 26.x needs Java 25, Limbo's jar is Java 21
+    # bytecode), so only digests and minors are proposed; majors move by hand.
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/workflows/ci.yml

@@ -14,10 +14,18 @@
 # PR is what asks for the answer.
 name: ci
 
+#
+# release.yml calls this workflow (workflow_call) before it builds anything, so a tag passes
+# exactly these gates and there is one list of them.
+#
+# workflow_dispatch reruns the suite on a commit whose push produced no run, such as one
+# pushed while Actions was unavailable.
 on:
   push:
     branches: [main]
   pull_request:
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -30,9 +38,9 @@ jobs:
   go:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
           go-version-file: go.mod
 
@@ -43,12 +51,70 @@ jobs:
             echo "gofmt needed on:"; echo "$unformatted"; exit 1
           fi
       - run: go vet ./...
-      - run: go test ./...
+      # -race: felis-api and the operator are mostly goroutines (watchers, the
+      # registry pruner, the backup scheduler, the rate limiters).
+      - run: go test -race ./...
+      # The version is pinned here and bumped by hand; Dependabot does not read `go run`.
+      - name: staticcheck
+        run: go run honnef.co/go/tools/cmd/[email protected] ./...
+
+  # Separate from the go job so a newly published advisory reads as what it is. govulncheck
+  # exits non-zero only for vulnerable code this module can actually reach, standard
+  # library included: setup-go installs the newest patch of go.mod's Go line, so a finding
+  # there means the Dockerfile's golang digest (which ships the release) needs a bump too.
+  vuln:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go run golang.org/x/vuln/cmd/[email protected] ./...
+
+  # The business stores' SQL against a real PostgreSQL (internal/pgint): the unit suites run
+  # on fakes, and PGRepo drifted from them three times while those stayed green. 13 is the
+  # oldest server a supported distribution installs (EL9), 18 the newest (Arch).
+  # `felis db backup` and `restore` run there too, with the tools inside the service
+  # container, as production runs them inside felis-postgres: the runner's own client is one
+  # major version, and pg_dump refuses a newer server.
+  pgint:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        postgres: ['13', '18']
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres }}
+        env:
+          POSTGRES_USER: felis
+          POSTGRES_PASSWORD: pgint
+          POSTGRES_DB: felis_pgint
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U felis -d felis_pgint"
+          --health-interval 2s
+          --health-timeout 5s
+          --health-retries 30
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go test -race -tags pgint -count=1 ./internal/pgint/
+        env:
+          FELIS_TEST_PG_URL: postgres://felis:pgint@localhost:5432/felis_pgint?sslmode=disable
+          FELIS_TEST_PG_EXEC: docker exec -i ${{ job.services.postgres.id }}
 
   shell:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # bootstrap.sh is the only thing that ever runs on a fresh host, and nothing here can
       # run it — it wants root, a package manager and k3s. Syntax plus the extracted-block
@@ -66,12 +132,49 @@ jobs:
             esac
           done
 
+      # A pinned release rather than the runner image's copy, so a runner update cannot
+      # change what fails. Warnings and errors fail the job; style notes (info) do not.
+      - name: shellcheck
+        run: |
+          curl -fsSL -o shellcheck.tar.xz \
+            https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz
+          echo "8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198  shellcheck.tar.xz" | sha256sum -c
+          tar -xJf shellcheck.tar.xz
+          ./shellcheck-v0.11.0/shellcheck -S warning $(git ls-files '*.sh')
+
+      # An exit status is 8 bits, so `exit "$fails"` reads 256 failures as a pass. The suites
+      # exit 1 on any failure; this keeps the next one from carrying its count out.
+      - name: No script exits with its failure count
+        run: |
+          if git grep -nE 'exit +"?\$\{?[a-z_]*fail[a-z_]*\}?"?[[:space:]]*$' -- '*.sh'; then exit 1; fi
+
       - run: sh deploy/bootstrap_test.sh
+      - run: sh deploy/uninstall_test.sh
+      - run: bash deploy/e2e_release_test.sh
+
+  # The shipped alert rules (deploy/alerts): promtool parses them and runs their unit tests,
+  # which pin when each alert fires and that it stays quiet before. internal/metrics'
+  # alerts_test.go pins what promtool cannot see from there: the PrometheusRule twin and the
+  # metric names the rules read.
+  alerts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # A pinned release, like shellcheck's, so a new Prometheus cannot change what fails.
+      - name: promtool
+        run: |
+          curl -fsSL -o prometheus.tar.gz \
+            https://github.com/prometheus/prometheus/releases/download/v3.15.0/prometheus-3.15.0.linux-amd64.tar.gz
+          echo "2a542df32eac02ee17b9d844fb2aa1de00dafa5476579ba8a3ba862e9d572ea0  prometheus.tar.gz" | sha256sum -c
+          tar -xzf prometheus.tar.gz --strip-components=1 prometheus-3.15.0.linux-amd64/promtool
+          ./promtool check rules deploy/alerts/felis-alerts.yaml
+          ./promtool test rules deploy/alerts/felis-alerts_test.yml
 
   panel:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # The Dockerfile's `FROM node:<major>` is the only place the panel's Node version is
       # declared — there is no .nvmrc and no engines field. Reading it here rather than
@@ -84,7 +187,7 @@ jobs:
           [ -n "$version" ] || { echo "Dockerfile has no 'FROM ... node:<major>' line"; exit 1; }
           echo "version=${version}" >> "$GITHUB_OUTPUT"
 
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: ${{ steps.node.outputs.version }}
           cache: npm
@@ -96,5 +199,71 @@ jobs:
       - run: npm test
         working-directory: panel
 
+      # `tsc -b`: tsconfig.json is a solution file (files: [] plus references), so a plain
+      # `tsc --noEmit` checked nothing and passed with type errors in the tree.
       - run: npm run typecheck
         working-directory: panel
+
+      # Rules of hooks and effect dependency lists, with --max-warnings 0.
+      - run: npm run lint
+        working-directory: panel
+
+      # openapi.gen.ts is generated from docs/openapi.yaml and checked in, so the
+      # compile-time parity in src/lib/types.parity.ts needs no generator in the build;
+      # a schema edit that was not regenerated fails here.
+      - name: openapi.gen.ts matches docs/openapi.yaml
+        working-directory: panel
+        run: |
+          npm run gen:api
+          git diff --exit-code -- src/lib/openapi.gen.ts
+
+      # Browser smoke over the mock-mode dev server, in the runner's installed Chrome
+      # (playwright.config.ts sets channel: chrome, so nothing is downloaded).
+      - run: npm run test:e2e
+        working-directory: panel
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: panel-playwright-report
+          path: |
+            panel/playwright-report
+            panel/test-results
+          retention-days: 7
+
+  plugins:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The other jobs never touch the Java layer: the plugin jars were only ever
+      # compiled by bootstrap on a live host, and the test mains under plugins/*/test
+      # were run by hand. JDK 25 is what the plugin build image runs (paper-api 26.x
+      # needs it); each module's wrapper brings the Gradle the image pins.
+      - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
+        with:
+          distribution: temurin
+          java-version: '25'
+
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
+
+      - run: bash plugins/test.sh
+
+  mods:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The three loader mods (Minecraft 1.20.1 / 1.20.4, Java-17 lines) compile
+      # through their vendored Gradle wrappers, which fetch their own Gradle. Until
+      # this job nothing ever built them: no install path touches them, and their
+      # gradlew scripts were committed without the exec bit, so the README's
+      # one-liners failed on a fresh clone.
+      - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
+
+      - run: bash plugins/test-mods.sh

.github/workflows/e2e.yml

@@ -0,0 +1,333 @@
+# deploy/bootstrap.sh end to end on a fresh Ubuntu 24.04 x86_64 runner: the paths every
+# host goes through, run for real instead of by hand on a VM.
+#
+#   artifacts  this commit's release assets, built by deploy/build-release-artifacts.sh the
+#              way release.yml builds a tag's (amd64 only, the runners' architecture)
+#   install    a full install from those assets, the way a release installs, on a host with
+#              ufw enabled, then the same assets again (a rerun must converge without
+#              restarting what did not change, importing or uploading an image again, or
+#              touching Docker)
+#   readme     the README's one-line install as a new host runs it today: this commit's
+#              installer on its default channel, which installs the newest published
+#              release's binary, images and plugin from that release's assets; skipped until
+#              a release exists
+#   upgrade    the newest published release through its own installer and its own assets,
+#              rows of every kind seeded into its database, then this commit's assets on
+#              top of it; skipped until a release exists
+#   source     a full install built on the host from this checkout (FELIS_SKIP_FETCH):
+#              the fallback a release without assets, or an unsupported one, takes. It builds
+#              everything with Docker, so it runs by hand and weekly only
+#
+# This runs on pushes that touch what gets installed, by hand, and weekly (a moving
+# upstream: apt mirrors, k3s's install script and release assets, Adoptium).
+# deploy/e2e_check.sh holds the assertions, deploy/e2e_seed.sh the upgrade's seed and its check.
+name: e2e
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'deploy/**'
+      - 'cmd/**'
+      - 'internal/**'
+      - 'panel/**'
+      - 'plugins/**'
+      - 'Dockerfile'
+      - 'go.mod'
+      - 'go.sum'
+      - '.github/workflows/e2e.yml'
+  workflow_dispatch:
+  schedule:
+    - cron: '23 4 * * 1'
+
+permissions:
+  contents: read
+
+# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` fails the step
+# when the installer fails; the default shell reports tee's status.
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  artifacts:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0 # the newest tag is the version's base, as on the dev channel
+
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      # Stamped the way bootstrap stamps a build of main: "<newest tag>+g<sha>".
+      - name: Build the release assets
+        run: |
+          base="$(git describe --tags --abbrev=0 --match 'v*' 2>/dev/null || echo v0.0.0)"
+          FELIS_RELEASE_ARCHES=amd64 deploy/build-release-artifacts.sh "${base}+g$(git rev-parse --short HEAD)" dist
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: e2e-assets
+          path: dist/
+          if-no-files-found: error
+          retention-days: 1
+
+  install:
+    needs: artifacts
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The images, k3s and the JRE need more room than a stock runner leaves free.
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: e2e-assets
+          path: dist
+
+      # The runner has Docker preinstalled, so its absence proves nothing: the log shows
+      # whether bootstrap reached for it. From FELIS_ARTIFACT_DIR every image and the plugin
+      # come out of the assets, so it must not have. (`! grep` is spelled `if grep ... exit 1`
+      # below because bash -e ignores a failing `!` command.)
+      # ufw, enabled on many Ubuntu hosts, drops every inbound packet no rule admits, the
+      # pods' traffic to the API server among them; the runner ships it disabled. The
+      # runner's own traffic is outbound, which ufw lets through.
+      - name: Enable ufw
+        run: sudo ufw --force enable
+
+      - name: Install
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log
+
+      - name: Check the install
+        run: |
+          sudo bash deploy/e2e_check.sh install
+          for tag in felis-k3s-pods felis-k3s-services felis-panel felis-proxy; do
+            sudo ufw status | grep -qE "# ${tag} *$"
+          done
+          if grep -E 'docker already installed|installing docker|docker running' install.log; then exit 1; fi
+          for role in felis limbo lobby paper; do
+            grep -q "felis/${role}:[^ ]* is the release's" install.log
+          done
+          grep -q "docker.io/library/registry@sha256:[0-9a-f]* is the release's" install.log
+          grep -q "docker.io/library/postgres@sha256:[0-9a-f]* is the release's" install.log
+          grep -q "felis-velocity.jar is the release's" install.log
+
+      - name: Rerun the same assets
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log
+
+      # containerd and the registry already hold every image under the digest the listing
+      # names, so nothing is imported or uploaded twice.
+      - name: Check the rerun
+        run: |
+          sudo bash deploy/e2e_check.sh rerun
+          grep -q 'felis-velocity unchanged; left running' rerun.log
+          if grep -E 'docker already installed|installing docker|docker running' rerun.log; then exit 1; fi
+          if grep -E 'importing felis-image-|mirroring .* into the internal registry' rerun.log; then exit 1; fi
+          grep -q 'is already in the internal registry' rerun.log
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          k() { sudo -E /usr/local/bin/k3s kubectl "$@"; }
+          k get pods -A -o wide || true
+          k get events -A --sort-by=.lastTimestamp | tail -n 60 || true
+          for p in $(k get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" {print $1 "/" $2}'); do
+            k -n "${p%%/*}" describe pod "${p#*/}" | tail -n 40 || true
+            k -n "${p%%/*}" logs "${p#*/}" --all-containers --tail=60 || true
+          done
+          k -n felis logs deploy/felis-postgres --tail=60 || true
+          sudo journalctl -u k3s -u felis-velocity --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-install-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  # The README's command on a fresh host: this commit's installer, as main serves it, on its
+  # default channel with nothing pinned. It resolves the newest release and installs that
+  # release's binary, images and plugin from its assets, each checked against its
+  # SHA256SUMS; the private repo's token is the only thing added. FELIS_INSTALL_MODE picks the
+  # mode the setup console would ask for.
+  readme:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - name: Find the newest release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: bash deploy/e2e_release.sh find
+
+      - name: Install as the README does
+        if: steps.release.outputs.tag != ''
+        env:
+          TOKEN: ${{ github.token }}
+        run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log
+
+      # The binary is the release's, so the phase is `release`: its database backup and
+      # timers are the release's to have or lack.
+      - name: Check the install
+        if: steps.release.outputs.tag != ''
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          BINARY: ${{ steps.release.outputs.binary }}
+          SUMS: ${{ steps.release.outputs.sums }}
+        run: |
+          sudo bash deploy/e2e_check.sh release
+          sudo /usr/local/bin/felis version | grep -qx "felis ${TAG}"
+          bash deploy/e2e_release.sh check-readme readme.log
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
+          sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-readme-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  upgrade:
+    needs: artifacts
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: e2e-assets
+          path: dist
+
+      - name: Find the newest release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: bash deploy/e2e_release.sh find
+
+      # The release's own installer on its default channel, fetching the release's own
+      # binary: what a host that installed that release is running today. FELIS_REF would
+      # build the tag from source instead, a path no host takes by default. FELIS_RELEASE pins
+      # the tag found above; an installer older than FELIS_RELEASE ignores it and resolves
+      # the newest release, the same tag.
+      - name: Install the newest release
+        if: steps.release.outputs.tag != ''
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          TOKEN: ${{ github.token }}
+        run: |
+          git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh
+          sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log
+
+      - name: Check the release install
+        if: steps.release.outputs.tag != ''
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          BINARY: ${{ steps.release.outputs.binary }}
+        run: |
+          sudo bash deploy/e2e_check.sh release
+          bash deploy/e2e_release.sh check-own release.log
+
+      # A fresh install's database holds only what its migrations wrote: the seed gives the
+      # upgrade's move and its pending migrations existing rows to carry.
+      - name: Seed the release's database
+        if: steps.release.outputs.tag != ''
+        run: sudo bash deploy/e2e_seed.sh seed
+
+      - name: Upgrade to this commit
+        if: steps.release.outputs.tag != ''
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log
+
+      # Both checks run and report: the seeded rows go last, after the restore drill has
+      # also put them back from a bundle.
+      - name: Check the upgrade
+        if: steps.release.outputs.tag != ''
+        run: |
+          rc=0
+          sudo bash deploy/e2e_check.sh upgrade || rc=1
+          sudo bash deploy/e2e_seed.sh check || rc=1
+          exit "$rc"
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
+          # The release ran the database on the host; the upgrade moves it into k3s.
+          sudo journalctl -u k3s -u felis-velocity -u postgresql --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-upgrade-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  source:
+    if: github.event_name != 'push'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      # FELIS_SKIP_FETCH builds whatever sits in /opt/felis/src; the .git directory is what
+      # stamps the build. Root owns it so git, run as root by the installer, does not refuse
+      # it as dubious.
+      - name: Stage this commit as the installer's source
+        run: |
+          sudo mkdir -p /opt/felis
+          sudo cp -a "$GITHUB_WORKSPACE" /opt/felis/src
+          sudo chown -R root:root /opt/felis/src
+
+      - name: Install
+        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log
+
+      - name: Check the install
+        run: sudo bash deploy/e2e_check.sh install
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-source-logs
+          path: '*.log'
+          if-no-files-found: ignore

.github/workflows/release.yml

@@ -2,14 +2,15 @@
 #
 # Two things depend on this job. /repos/{repo}/releases/latest must ANSWER — that endpoint is
 # what `felis update` polls (internal/updater/github.go) and what deploy/bootstrap.sh's default
-# "release" channel resolves its ref from. And the binaries below are what that channel then
-# INSTALLS: bootstrap downloads felis-linux-<arch> instead of compiling on the target host, so
-# these are the shipped artifact, not a convenience.
+# "release" channel resolves its ref from. And the assets below are what that channel then
+# INSTALLS: bootstrap downloads the felis binary, the image tars, their listing and the Velocity
+# plugin instead of building anything on the target host, so these are the shipped artifact,
+# not a convenience. A host installing a release needs neither Docker, Gradle, Go nor Docker Hub.
 #
-# The asset NAME is a contract with deploy/bootstrap.sh (download_release_binary builds
-# "felis-linux-${arch}"). It is deliberately a plain literal on both sides: a GitHub Actions
-# YAML and a go:embed'ed shell script have no honest way to share a constant, and the failure
-# mode is benign — bootstrap warns and falls back to a source build of the same tag.
+# deploy/build-release-artifacts.sh builds every asset in one run and documents each name; the
+# names are a contract with deploy/bootstrap.sh. They are plain literals on both sides: a YAML
+# workflow and a go:embed'ed shell script have no honest way to share a constant, and the
+# failure mode is benign — bootstrap warns and falls back to building on the host.
 #
 # The binary is built through the repo Dockerfile rather than a plain `go build`.
 # internal/panel/static holds a tracked PLACEHOLDER index.html so the //go:embed
@@ -17,6 +18,15 @@
 # quietly ships a release whose panel is that placeholder. The Dockerfile runs the npm
 # build first, and is the same recipe bootstrap uses, so there is one way to build felis
 # rather than two that can drift.
+#
+# SHA256SUMS is a contract with bootstrap too: it refuses any asset whose hash is not listed
+# there, BEFORE it runs or imports it.
+#
+# The write token never meets the test suite: `gates` (ci.yml) and `build` run the tests,
+# Gradle and the Docker builds (each of which executes third-party code) with a read-only
+# token, and `build` hands the assets over as a workflow artifact; `publish` holds contents:write and runs only
+# pinned actions and gh. Every action is pinned to a commit SHA (the tag in the trailing
+# comment is for humans); .github/dependabot.yml proposes the bumps.
 name: release
 
 on:
@@ -24,59 +34,99 @@ on:
     tags: ['v*']
 
 permissions:
-  contents: write # gh release create/upload
+  contents: read
 
 jobs:
-  release:
+  # A tag that ships red is worse than a tag that fails to ship. These are ci.yml's gates,
+  # called rather than copied: Go (race, vet, staticcheck), govulncheck, the PostgreSQL
+  # contract suite, shellcheck and the bootstrap tests, promtool on the alert rules, the
+  # panel, and the Java layer the binary EMBEDS (bootstrap_asset.go ships the plugin
+  # sources, so a tag whose plugins do not compile turns every install of that release into
+  # a failed bootstrap).
+  gates:
+    uses: ./.github/workflows/ci.yml
+
+  build:
+    needs: gates
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-
-      # A tag that ships red is worse than a tag that fails to ship.
-      - run: go vet ./...
-      - run: go test ./...
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # Both architectures, because bootstrap's default release channel DOWNLOADS these
-      # rather than compiling on the target host — an arm64 host with no asset silently
-      # falls back to a slow source build. Neither stage is emulated: the Dockerfile pins
-      # both build stages to $BUILDPLATFORM and the Go stage cross-compiles via TARGETARCH,
-      # so the second architecture costs about a minute.
-      - uses: docker/setup-buildx-action@v3
+      # rather than building on the target host — an arm64 host with no asset falls back to
+      # a slow on-host build. The felis binary and the plugin jars cross-compile on the
+      # runner (their build stages are pinned to $BUILDPLATFORM); the game images' runtime
+      # stages run apt-get for the target platform, which for arm64 takes QEMU.
+      - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
 
-      - name: Build the stamped binaries
-        run: |
-          docker buildx build --platform linux/amd64,linux/arm64 \
-            --build-arg FELIS_VERSION="${GITHUB_REF_NAME}" \
-            --output type=local,dest=out .
-          mv out/linux_amd64/usr/local/bin/felis ./felis-linux-amd64
-          mv out/linux_arm64/usr/local/bin/felis ./felis-linux-arm64
-          chmod +x ./felis-linux-amd64 ./felis-linux-arm64
+      # Two architectures of five images plus BuildKit's cache outgrow the runner's free disk
+      # with its preinstalled SDKs in place; none of them is used here.
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
 
-      # The stamp is the whole point and it fails silently: an unstamped binary reports
-      # "dev", which `felis update` refuses to compare, disabling update reporting for
-      # every install built from it. Assert it end to end instead of trusting the ARG
-      # reached the linker.
-      - name: Verify the version stamp
-        run: |
-          got="$(./felis-linux-amd64 version | head -1)"
-          echo "reported: ${got}"
-          [ "$got" = "felis ${GITHUB_REF_NAME}" ] \
-            || { echo "expected 'felis ${GITHUB_REF_NAME}' — the -X main.version stamp did not reach the binary"; exit 1; }
-          # The arm64 binary is checked by ELF machine type, NOT by running it. Runners have
-          # binfmt/QEMU registered, so `./felis-linux-arm64 version` would happily succeed on
-          # an amd64 binary misnamed arm64 — which is exactly the failure the Dockerfile's
-          # ${TARGETARCH:-$(go env GOARCH)} fallback produces if buildx did not take. Both
-          # binaries come out of one RUN with one -ldflags string, so the stamp is checked once.
-          file ./felis-linux-arm64
-          file ./felis-linux-arm64 | grep -q 'ARM aarch64' \
-            || { echo "felis-linux-arm64 is not an arm64 ELF — TARGETARCH did not reach the go build"; exit 1; }
+      # The script checks what the old inline steps did: the host binary reports exactly
+      # "felis <tag>" (unstamped, `felis update` refuses to compare), and the other one is an
+      # ELF of its architecture, read with file(1) — binfmt would run a misnamed binary happily.
+      - name: Build the release assets
+        run: deploy/build-release-artifacts.sh "${GITHUB_REF_NAME}" dist
 
-      # --verify-tag refuses to invent a release for a tag that is not pushed. The upload
-      # fallback makes a re-run converge rather than failing on an existing release.
+      # A CycloneDX SBOM per binary: the Go modules (and versions) linked into it, read
+      # from the build info the linker embeds. Written after SHA256SUMS, so beside it in the
+      # release but outside it: that lists what bootstrap installs. Straight into dist/,
+      # because the action does not create a missing output directory.
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: dist/felis-linux-amd64
+          format: cyclonedx-json
+          output-file: dist/felis-linux-amd64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: dist/felis-linux-arm64
+          format: cyclonedx-json
+          output-file: dist/felis-linux-arm64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: release-assets
+          path: dist/
+          if-no-files-found: error
+          retention-days: 7
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # gh release create/upload
+      id-token: write # the Sigstore certificate behind the provenance attestation
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: release-assets
+
+      # The artifact store sits between the two jobs, so check the handover too.
+      - run: sha256sum -c SHA256SUMS
+
+      # Signed SLSA provenance: which workflow run, commit and repository produced each
+      # asset. Check one with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`.
+      # GitHub only stores attestations for private repositories on Enterprise Cloud, and a
+      # failure here would block the release, so a private repository skips the step and
+      # relies on SHA256SUMS alone.
+      - name: Attest build provenance
+        if: ${{ !github.event.repository.private }}
+        uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0
+        with:
+          subject-path: |
+            felis-linux-*
+            felis-image-*.tar
+            felis-velocity.jar
+
+      # --verify-tag refuses to invent a release for a tag that is not pushed.
       #
       # The prerelease flag has to be passed explicitly: the trigger glob is v*, so v1.2.3-rc1
       # lands here too, and gh does not read semver out of the tag name. Published as a full
@@ -84,13 +134,35 @@ jobs:
       # channel installs from and `felis update` polls — so every fresh install would get the
       # RC binary and every deployed felis-api would error on the felis component until a
       # stable tag was cut. Flagged, GitHub keeps latest pointing at the last stable release.
+      #
+      # A re-run (the release already exists) uploads only what is missing and never
+      # replaces a published asset: hosts may already have installed it, and their
+      # SHA256SUMS check would start failing against a swapped file. An asset that is
+      # there with different bytes stops the job; cut a new tag instead.
       - name: Publish the release
         env:
           GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
         run: |
+          # Every file the build handed over: SHA256SUMS names the installable ones, and the
+          # SBOMs ride along.
+          assets="$(ls)"
           flags=""
           case "$GITHUB_REF_NAME" in *-*) flags="--prerelease" ;; esac
-          gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags \
-              ./felis-linux-amd64 ./felis-linux-arm64 \
-            || gh release upload "$GITHUB_REF_NAME" \
-                 ./felis-linux-amd64 ./felis-linux-arm64 --clobber
+          if ! gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
+            # shellcheck disable=SC2086 # word-splitting the list is the point
+            gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags $assets
+            exit 0
+          fi
+          # The REST payload's per-asset "digest" is GitHub's own sha256 of the stored file.
+          published="$(gh api "repos/${GH_REPO}/releases/tags/${GITHUB_REF_NAME}" --jq '.assets[] | "\(.name) \(.digest)"')"
+          for a in $assets; do
+            have="$(printf '%s\n' "$published" | awk -v n="$a" '$1 == n { print $2 }')"
+            want="sha256:$(sha256sum < "$a" | cut -d' ' -f1)"
+            if [ -z "$have" ]; then
+              gh release upload "$GITHUB_REF_NAME" "$a"
+            elif [ "$have" != "$want" ]; then
+              echo "::error::$a is already published with $have; this run built $want. Published assets are never replaced."
+              exit 1
+            fi
+          done

CONTRIBUTING.md

@@ -67,6 +67,27 @@ go test ./internal/api
 go test ./cmd/felis
 ```
 
+The hermetic suites run against in-memory fakes; the business stores' SQL is
+verified separately against a real Postgres, on a throwaway database whose name
+must contain `pgint` (the harness drops and recreates its schema and replays the
+embedded migrations):
+
+```bash
+FELIS_TEST_PG_URL='postgres://felis:***@127.0.0.1:5432/felis_pgint?sslmode=disable' \
+  go test -tags pgint ./internal/pgint/ -v
+```
+
+Run it after touching anything under `internal/api/pgrepo.go`, `internal/submit`,
+`internal/build` or `internal/dbbackup` that speaks SQL: the fakes encode the
+contract, and this suite exists to catch the drift between the fakes and the real
+queries. The `felis db backup` and `restore` tests also run `pg_dump`, `pg_restore`
+and `psql`, which must be the server's major version. For a server in a container,
+run them in it, as production does in felis-postgres:
+
+```bash
+FELIS_TEST_PG_EXEC='docker exec -i <container>' FELIS_TEST_PG_URL=... go test -tags pgint ./internal/pgint/
+```
+
 Build the CLI:
 
 ```bash

Dockerfile

@@ -21,14 +21,18 @@
 # minutes. The FINAL stage is deliberately NOT pinned — it must stay on the target platform
 # or the published arm64 image would carry amd64 layers. It contains only COPY, which
 # BuildKit performs itself, so it needs no QEMU either; adding a RUN there would.
-FROM --platform=$BUILDPLATFORM node:22-bookworm AS panel
+#
+# Every base image here and in deploy/{limbo,lobby,paper} is pinned by digest, so a rebuild
+# of one release uses the same bytes; .github/dependabot.yml proposes the bumps (tag and
+# digest together).
+FROM --platform=$BUILDPLATFORM node:22-bookworm@sha256:363e1587494626837fa7f9a23bdb453d13b0ff3c67c705c2805cfc69c2d2fad7 AS panel
 WORKDIR /panel
 COPY panel/package*.json ./
 RUN npm ci
 COPY panel/ ./
 RUN npm run build
 
-FROM --platform=$BUILDPLATFORM golang:1.26 AS build
+FROM --platform=$BUILDPLATFORM golang:1.26@sha256:6c2a5538f964f1c82f97ad14988bf05de100d922d159d0e398b54c7b0ca0c6c9 AS build
 WORKDIR /src
 ARG TARGETOS=linux
 ARG TARGETARCH
@@ -55,7 +59,7 @@ ARG FELIS_VERSION=dev
 RUN CGO_ENABLED=0 GOOS="$TARGETOS" GOARCH="${TARGETARCH:-$(go env GOARCH)}" \
     go build -trimpath -ldflags="-s -w -X main.version=${FELIS_VERSION}" -o /out/felis ./cmd/felis
 
-FROM gcr.io/distroless/static-debian12:nonroot
+FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
 ENV PATH=/usr/local/bin:/usr/bin:/bin
 COPY --chmod=0755 --from=build /out/felis /usr/local/bin/felis
 # distroless "nonroot" is uid 65532; the rendered PodSecurityContext pins

README.md

@@ -1,5 +1,8 @@
 # Felis
 
+**此项目仍处于早期开发阶段，您不该在任何生产环境使用该项目。若产生任何问题，贵用户的使用行为与 FelisMC 团队无任何民事刑事法律关系。**
+**THIS PROJECT IS STILL WIP, YOU SHOULD DO NOT USE THIS PROJECT IN ANY PRODUCTION USAGE. WE ARE NOT RESPOND FOR ANY LEGAL OR HUMANLY PROBLEM.**
+
 一款 Kubernetes 驱动的 Minecraft 服务器托管平台，一行命令部署，自动管理生命周期与安全。  
 A Kubernetes-driven Minecraft server hosting platform — one command to deploy, automatic lifecycle, backup, and security.
 
@@ -17,22 +20,28 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy,
 
 - **即开即玩**：玩家尝试连接时自动唤醒服务器，空闲后自动休眠，像游戏主机一样省资源。
 - **Web 控制面板**：浏览器中查看服务器状态、在线玩家与资源用量，管理备份与恢复。
-- **自动备份与恢复**：定时将世界打包存档，支持从任意备份点一键回滚。
-- **智慧回收**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间。
+- **备份与恢复**：一键把整服数据（世界、配置、插件/模组，即整个 /data 卷）打包进集群内的归档库，支持从任意备份点回滚；默认安装就已启用（归档 PVC 与路径由安装器一并生成）。
+- **控制面数据库备份**：账号、服务器归属、配额与存档索引所在的数据库每天自动备份，每次升级迁移前先快照，出错可用 `felis db restore` 整库原子回滚；面板「维护与备份」页显示备份是否新鲜（见 [故障排查 §16](docs/troubleshooting.md)）。
+- **运维自检**：`sudo felis status` 一屏列出节点、控制面、游戏代理、每台服务器、备份与未解决的告警；`sudo felis doctor` 把健康检查全跑一遍，按区域给出问题和下一步去哪看，不发邮件；`sudo felis support-bundle` 打出一个脱敏的诊断包，求助时直接附上（见 [故障排查 §0](docs/troubleshooting.md)）。
+- **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。过期备份无论是否开启都会每天清理。
 - **多核心支持**：兼容 Paper、Fabric、Forge、NeoForge，经由 Velocity 代理统一入口。
-- **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建并部署。
+- **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建；构建产物进入镜像白名单，可直接选用为服务器镜像完成部署。
 - **Passkey 登录**：支持指纹、面容、硬件密钥等无密码认证方式。
 - **零信任安全**：面板流量由 Cloudflare Access 保护，集群内 API 不暴露到公网。
 
 ## 使用方式
 
-在准备好的 Linux 主机上执行：
+在准备好的 Linux 主机上执行（已验证的发行版与架构见 [运维手册 §1](docs/operations.md#1-supported-hosts)：CentOS Stream 9 aarch64 实机验证，Ubuntu 24.04 x86_64 每次推送由 CI 跑全新安装、重跑、升级和下面这条命令本身）：
 
 ```bash
-curl -fsSL https://raw.githubusercontent.com/MliroLirrorsIngenuity/Felis/main/deploy/bootstrap.sh | sudo bash
+curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
 ```
 
-脚本将自动安装 K3s、部署控制平面并启动设置向导。完成后浏览器访问已配置的域名进入控制面板即可使用。
+脚本将自动安装 K3s，在 K3s 内部署 PostgreSQL 与控制平面，并启动设置向导。完成后浏览器访问已配置的域名进入控制面板即可使用。
+
+安装发布版时，二进制、全部镜像与 Velocity 插件都取自该版本在 CI 里预构建好的 release 附件，逐个核对 `SHA256SUMS` 后导入，主机上无需 Docker、Gradle 或 Go，也不从 Docker Hub 拉取；某个附件缺失或校验不符时，只有那一个镜像退回到本机构建，并给出提示（见 [故障排查 §15c](docs/troubleshooting.md)）。附件也可以先拷到本机，再用 `FELIS_ARTIFACT_DIR=<绝对路径>` 安装，Felis 自己的二进制、镜像和插件就都取自这个目录；k3s 及其镜像、JRE、cloudflared、Velocity 和 Via 插件照旧从 GitHub 与 PaperMC 下载，RHEL、Fedora、openSUSE Leap 这类开着 SELinux 的主机还要从 rpm.rancher.io 装 k3s-selinux，系统软件包来自发行版的源。所以出网受限的主机要放行这几处的 HTTPS（或设 `https_proxy`），preflight 会在改动主机之前逐个探测，完全断网的主机目前装不了（地址清单见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。旧版本装在宿主上的 PostgreSQL 会在重跑时整库迁进 K3s，宿主上的那份停用保留，供回退（见 [运维手册 §4](docs/operations.md#4-upgrading-the-pieces-around-felis)）。
+
+动手之前，脚本先检查内存、磁盘、端口、网段冲突、已有的 Kubernetes 和外网连通，把所有问题一次列出并停下，主机上什么都没改（检查项见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
 
 > **本仓库当前为私有**，上面这条会返回 404。请改用带凭据的形式；安装器自身也需要同一个 token
 > 去解析并下载 release，所以用 `sudo -E` 把它带进去：
@@ -41,7 +50,7 @@ curl -fsSL https://raw.githubusercontent.com/MliroLirrorsIngenuity/Felis/main/de
 > export FELIS_GITHUB_TOKEN=<对本仓库有读权限的 token>
 > printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \
 >   | curl -fsSL --config - -H "Accept: application/vnd.github.raw" \
->       https://api.github.com/repos/MliroLirrorsIngenuity/Felis/contents/deploy/bootstrap.sh \
+>       https://api.github.com/repos/FelisMC/Felis/contents/deploy/bootstrap.sh \
 >   | sudo -E bash
 > ```
 >

README_EN.md

@@ -17,22 +17,50 @@ Table of Contents
 
 - **Wake on Join**: Servers start automatically when a player connects, and stop when idle — like hibernate for your server.
 - **Web Dashboard**: Monitor server status, online players, and resource usage from your browser, with backup and restore management.
-- **Auto Backup & Restore**: Scheduled world backups with one-click rollback from any backup point.
-- **World Reaper**: Worlds idle for more than 15 days are automatically backed up and removed to free disk space.
+- **Backup & Restore**: One-click snapshots of a server's whole data volume (worlds, config, plugins/mods — the entire /data volume) into the cluster's archive store, with rollback from any backup point — enabled by default (the installer renders the archive PVC and its path).
+- **Control-plane database backups**: The database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migrates it; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows whether the newest backup is fresh (see [troubleshooting §16](docs/troubleshooting.md)).
+- **Self-check for operators**: `sudo felis status` shows the node, the control plane, the game proxy, every server, the backups and the open alerts on one screen; `sudo felis doctor` runs every health check once and lists each problem by area with where to look next, mailing nothing; `sudo felis support-bundle` writes one redacted diagnostics archive to attach when asking for help (see [troubleshooting §0](docs/troubleshooting.md)).
+- **World Reaper** (opt in): Worlds idle for more than 15 days are automatically backed up and removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is ever deleted.
 - **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, federated behind a Velocity proxy.
-- **Modpack Submission**: Players submit custom modpacks; admin approval triggers automatic build and deployment.
+- **Modpack Submission**: Players submit custom modpacks; admin approval triggers an automatic build, and the result is whitelisted as a server image you can select to deploy.
 - **Passkey Login**: Passwordless authentication via fingerprint, face recognition, or hardware security keys.
 - **Zero Trust Security**: Panel traffic protected by Cloudflare Access; the internal API is never exposed to the internet.
 
 ## Getting Started
 
-On a prepared Linux host, run:
+On a prepared Linux host, run (the verified distributions and architectures are listed in
+[operations §1](docs/operations.md#1-supported-hosts): CentOS Stream 9 on aarch64 is verified
+on a real host, and Ubuntu 24.04 on x86_64 gets a fresh install, rerun and upgrade in CI on
+every push):
 
 ```bash
-curl -fsSL https://raw.githubusercontent.com/MliroLirrorsIngenuity/Felis/main/deploy/bootstrap.sh | sudo bash
+curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
 ```
 
-The script installs K3s, deploys the control plane, and launches a setup wizard. Once done, open your browser at the configured domain.
+The script installs K3s, deploys PostgreSQL and the control plane inside it, and launches a setup wizard. Once done, open your browser at the configured domain. A PostgreSQL an earlier release installed on the host is moved into K3s on the next rerun, and the host copy is stopped and kept for a rollback (see [Operations §4](docs/operations.md#4-upgrading-the-pieces-around-felis)).
+
+Before it changes anything, the script checks RAM, disk, ports, network-range clashes, any Kubernetes already there and outbound access, lists every problem at once and stops with the host untouched (the checks are in [operations §1](docs/operations.md#1-supported-hosts)).
+
+> **This repository is currently private**, so the command above returns 404. Use the
+> credentialed form instead; the installer itself needs the same token to resolve and
+> download the release, so pass it through with `sudo -E`:
+>
+> ```bash
+> export FELIS_GITHUB_TOKEN=<a token with read access to this repository>
+> printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \
+>   | curl -fsSL --config - -H "Accept: application/vnd.github.raw" \
+>       https://api.github.com/repos/FelisMC/Felis/contents/deploy/bootstrap.sh \
+>   | sudo -E bash
+> ```
+>
+> The token reaches `curl --config -` over stdin instead of the command line: argv is
+> readable by any local user via `/proc`, and that is exactly why the installer's
+> internal `github_api` uses the same form.
+
+Rerunning this command is also how you upgrade felis-api to a newer version (`felis setup`
+cannot — it uses the binary already installed on the host). The rerun keeps the installed
+root domain but **not** the channel: if this host follows main, also
+`export FELIS_VERSION_BOOTSTRAP=dev`.
 
 ## Build from Source
 

bootstrap_asset.go

@@ -22,14 +22,17 @@ var bootstrapAssets embed.FS
 // developer's working tree carries gradle output (plugins/*/build, plugins/*/bin,
 // and for the modded loaders a decompiled Minecraft under build/) which would
 // otherwise be baked into every felis binary. Keep them explicit — add a source
-// directory here, never a parent.
+// directory here, never a parent. Each module's gradle/verification-metadata.xml rides
+// along, since Gradle builds unverified without it; the wrapper stays out, because the
+// image builds run the pinned build image's own gradle.
 //
+//go:embed deploy/game-stack.lock
 //go:embed deploy/limbo/Dockerfile deploy/limbo/entrypoint.sh
 //go:embed deploy/lobby/Dockerfile deploy/lobby/entrypoint.sh
 //go:embed deploy/paper/Dockerfile deploy/paper/entrypoint.sh
-//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src
-//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src
-//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src
+//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src plugins/limbo/gradle/verification-metadata.xml
+//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src plugins/paper/gradle/verification-metadata.xml
+//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src plugins/velocity/gradle/verification-metadata.xml
 //go:embed plugins/shared/src
 var gameStackAssets embed.FS
 

bootstrap_asset_test.go

@@ -1,7 +1,9 @@
 package felis
 
 import (
+	"encoding/xml"
 	"io/fs"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -205,6 +207,299 @@ func requireEmbedded(t *testing.T, path string) {
 	}
 }
 
+// The lock file is the install's only source of upstream builds, and bootstrap.sh reads it
+// with a strict KEY=value parser that dies on anything unexpected, so a malformed lock is a
+// failed install on every host. Check the shipped copy the same way here.
+func TestGameStackLockIsComplete(t *testing.T) {
+	lock := gameStackLock(t)
+	m := regexp.MustCompile(`GAME_STACK_LOCK_KEYS="([^"]*)"`).FindStringSubmatch(BootstrapScript())
+	if m == nil {
+		t.Fatal("bootstrap.sh no longer declares GAME_STACK_LOCK_KEYS")
+	}
+	keys := strings.Fields(m[1])
+	sha := regexp.MustCompile(`^[0-9a-f]{64}$`)
+	for _, k := range keys {
+		v, ok := lock[k]
+		if !ok || v == "" {
+			t.Errorf("game-stack.lock does not set %s", k)
+			continue
+		}
+		if strings.HasSuffix(k, "_SHA256") && !sha.MatchString(v) {
+			t.Errorf("%s=%q is not a lowercase sha256", k, v)
+		}
+		// A moving URL pins nothing: the digest check would start failing the day
+		// upstream publishes the next build.
+		if strings.HasSuffix(k, "_URL") && strings.Contains(v, "lastSuccessfulBuild") {
+			t.Errorf("%s names a moving build: %s", k, v)
+		}
+	}
+	for k := range lock {
+		if !strings.Contains(" "+m[1]+" ", " "+k+" ") {
+			t.Errorf("game-stack.lock sets %s, which bootstrap.sh refuses as an unknown key", k)
+		}
+	}
+	// Fill's URLs are content-addressed; a lock whose digest disagrees with its own URL
+	// was edited by hand and half-way.
+	for _, name := range []string{"PAPER", "VELOCITY"} {
+		if !strings.Contains(lock[name+"_JAR_URL"], "/objects/"+lock[name+"_JAR_SHA256"]+"/") {
+			t.Errorf("%s_JAR_SHA256 is not the digest in %s_JAR_URL", name, name)
+		}
+	}
+	if !strings.Contains(lock["LIMBO_JAR_URL"], "-"+lock["MC_VERSION"]+".jar") {
+		t.Errorf("LIMBO_JAR_URL %s is not a Minecraft %s build", lock["LIMBO_JAR_URL"], lock["MC_VERSION"])
+	}
+	if !strings.Contains(lock["PAPER_JAR_URL"], "/paper-"+lock["MC_VERSION"]+"-") {
+		t.Errorf("PAPER_JAR_URL %s is not a Minecraft %s build; the lobby would not speak the login gate's protocol", lock["PAPER_JAR_URL"], lock["MC_VERSION"])
+	}
+}
+
+// Each downloaded jar's digest is a build-arg bootstrap.sh passes and the Dockerfile must
+// both require and spend on the file it downloaded; docker only warns about an unknown
+// --build-arg, so a renamed arg would ship an unchecked jar.
+func TestGameStackDigestsReachTheImageBuilds(t *testing.T) {
+	script := BootstrapScript()
+	for _, c := range []struct{ dockerfile, arg, path string }{
+		{"deploy/limbo/Dockerfile", "LIMBO_JAR_SHA256", "/limbo/Limbo.jar"},
+		{"deploy/limbo/Dockerfile", "LIMBO_SCHEM_SHA256", "/limbo/spawn.schem"},
+		{"deploy/lobby/Dockerfile", "LUCKPERMS_JAR_SHA256", "/paper/plugins/LuckPerms.jar"},
+	} {
+		if !strings.Contains(script, "--build-arg "+c.arg+"=\"$"+c.arg+"\"") {
+			t.Errorf("bootstrap.sh never passes --build-arg %s", c.arg)
+		}
+		dockerfile := readGameStackFile(t, c.dockerfile)
+		if !strings.Contains(dockerfile, "ARG "+c.arg) {
+			t.Errorf("%s declares no ARG %s", c.dockerfile, c.arg)
+		}
+		if !strings.Contains(dockerfile, `echo "$`+c.arg+`  `+c.path+`" | sha256sum -c`) {
+			t.Errorf("%s never verifies %s against %s", c.dockerfile, c.path, c.arg)
+		}
+	}
+}
+
+// A base image named by tag alone is whatever the tag points at on build day.
+func TestDockerfileBaseImagesArePinnedByDigest(t *testing.T) {
+	root, err := os.ReadFile("Dockerfile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	files := map[string]string{"Dockerfile": string(root)}
+	for _, name := range []string{"deploy/limbo/Dockerfile", "deploy/lobby/Dockerfile", "deploy/paper/Dockerfile"} {
+		files[name] = readGameStackFile(t, name)
+	}
+	pinned := regexp.MustCompile(`^FROM (--platform=\S+ )?\S+:\S+@sha256:[0-9a-f]{64}( AS \S+)?$`)
+	for name, body := range files {
+		n := 0
+		for line := range strings.SplitSeq(body, "\n") {
+			if !strings.HasPrefix(line, "FROM ") {
+				continue
+			}
+			n++
+			if !pinned.MatchString(line) {
+				t.Errorf("%s: %q is not pinned by digest", name, line)
+			}
+		}
+		if n == 0 {
+			t.Errorf("%s has no FROM line", name)
+		}
+	}
+}
+
+// The plugin jars are built in four places the installer controls — the lobby and limbo
+// image builds, bootstrap's Velocity build and the release build of the same jar — and
+// through each module's wrapper by a developer or CI. A tag alone is whatever it points
+// at on build day, and two Gradle versions are two chances for a build to pass in one
+// place and break in the other, so all of them run one image, pinned by digest, whose
+// Gradle is the wrappers' Gradle.
+func TestPluginBuildsRunOnePinnedGradle(t *testing.T) {
+	sources := map[string]string{
+		"deploy/lobby/Dockerfile": readGameStackFile(t, "deploy/lobby/Dockerfile"),
+		"deploy/limbo/Dockerfile": readGameStackFile(t, "deploy/limbo/Dockerfile"),
+		"deploy/bootstrap.sh":     BootstrapScript(),
+		// The release build compiles felis-velocity.jar for hosts that install prebuilt.
+		"deploy/build-release-artifacts.sh": readRepoFile(t, "deploy/build-release-artifacts.sh"),
+	}
+	anyRef := regexp.MustCompile(`gradle:[\w.-]+(@sha256:\w+)?`)
+	pinned := regexp.MustCompile(`^gradle:(\d+\.\d+(?:\.\d+)?)-jdk\d+@sha256:[0-9a-f]{64}$`)
+	images := map[string]bool{}
+	gradle := ""
+	for name, body := range sources {
+		refs := anyRef.FindAllString(body, -1)
+		if len(refs) == 0 {
+			t.Errorf("%s names no gradle image", name)
+		}
+		for _, ref := range refs {
+			m := pinned.FindStringSubmatch(ref)
+			if m == nil {
+				t.Errorf("%s: %s is not a gradle image pinned by digest", name, ref)
+				continue
+			}
+			images[ref] = true
+			gradle = m[1]
+		}
+	}
+	if len(images) != 1 {
+		t.Fatalf("the plugin builds use %d different gradle images, want one: %v", len(images), images)
+	}
+	// bootstrap names the image once and has to spend it where it builds the jar.
+	if !strings.Contains(BootstrapScript(), `"$PLUGIN_BUILD_IMAGE" gradle --no-daemon clean build`) {
+		t.Error("build_velocity_plugin does not build in $PLUGIN_BUILD_IMAGE")
+	}
+
+	for _, module := range []string{"velocity", "paper", "limbo"} {
+		props := wrapperProperties(t, module)
+		if want := "gradle-" + gradle + "-bin.zip"; !strings.HasSuffix(props["distributionUrl"], "/"+want) {
+			t.Errorf("plugins/%s wrapper runs %s; the image builds run Gradle %s", module, props["distributionUrl"], gradle)
+		}
+	}
+	// The mods are no part of the install, but a wrapper without a checksum runs whatever
+	// the download handed it.
+	for _, module := range []string{"velocity", "paper", "limbo", "fabric", "forge", "neoforge"} {
+		if sum := wrapperProperties(t, module)["distributionSha256Sum"]; !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(sum) {
+			t.Errorf("plugins/%s wrapper pins no distribution sha256 (got %q)", module, sum)
+		}
+	}
+}
+
+// Each plugin compiles against the API of the exact build the install runs, and Gradle
+// checks those bytes against the module's verification file. Nothing but this test ties
+// the three to deploy/game-stack.lock: a lock refresh that leaves them behind builds the
+// lobby against yesterday's API, or fails every image build on a checksum the file does
+// not have.
+func TestPluginApisAreTheLockedBuilds(t *testing.T) {
+	lock := gameStackLock(t)
+
+	// Paper: paper-<mc>-<build>.jar runs; paper-api <mc>.build.<build>-<channel> compiles.
+	jar := regexp.MustCompile(`/paper-([^/]+)-(\d+)\.jar$`).FindStringSubmatch(lock["PAPER_JAR_URL"])
+	if jar == nil {
+		t.Fatalf("PAPER_JAR_URL %s does not name paper-<mc>-<build>.jar", lock["PAPER_JAR_URL"])
+	}
+	dep := regexp.MustCompile(`compileOnly 'io\.papermc\.paper:paper-api:([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/paper/build.gradle"))
+	if dep == nil {
+		t.Fatal("plugins/paper/build.gradle declares no paper-api dependency")
+	}
+	if !regexp.MustCompile(`^` + regexp.QuoteMeta(jar[1]+".build."+jar[2]) + `(-[a-z]+)?$`).MatchString(dep[1]) {
+		t.Errorf("paper-api %s is not the API of the locked server paper-%s-%s.jar", dep[1], jar[1], jar[2])
+	}
+	requireVerified(t, "paper", "io.papermc.paper", "paper-api", dep[1])
+
+	// Limbo: the lock's release, passed to the image build, which refuses to guess one.
+	limbo := lock["LIMBO_VERSION"]
+	if !strings.Contains(BootstrapScript(), `--build-arg LIMBO_VERSION="$LIMBO_VERSION"`) {
+		t.Error("bootstrap.sh does not pass the locked LIMBO_VERSION to the limbo image build")
+	}
+	dockerfile := readGameStackFile(t, "deploy/limbo/Dockerfile")
+	if !regexp.MustCompile(`(?m)^ARG LIMBO_VERSION$`).MatchString(dockerfile) ||
+		!strings.Contains(dockerfile, `if [ -z "${LIMBO_VERSION:-}" ]`) {
+		t.Error("deploy/limbo/Dockerfile does not require LIMBO_VERSION; a build without it would " +
+			"compile against a version nobody chose")
+	}
+	// LOOHP publishes the jar the login gate runs as the Limbo API artifact itself, so the
+	// checksum Gradle holds for it is the lock's: compiled-against and running are one file.
+	if got := requireVerified(t, "limbo", "com.loohp", "Limbo", limbo)["Limbo-"+limbo+".jar"]; got != lock["LIMBO_JAR_SHA256"] {
+		t.Errorf("verification-metadata.xml holds %q for Limbo-%s.jar; the login gate runs %s", got, limbo, lock["LIMBO_JAR_SHA256"])
+	}
+
+	// Velocity: the API default is the proxy the install runs.
+	api := regexp.MustCompile(`findProperty\('velocityApi'\) \?: '([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/velocity/build.gradle"))
+	if api == nil {
+		t.Fatal("plugins/velocity/build.gradle has no velocityApi default")
+	}
+	if api[1] != lock["VELOCITY_VERSION"] {
+		t.Errorf("velocity-api defaults to %s; the install runs Velocity %s", api[1], lock["VELOCITY_VERSION"])
+	}
+	requireVerified(t, "velocity", "com.velocitypowered", "velocity-api", api[1])
+}
+
+// requireVerified asserts the module's shipped verification file checks metadata and
+// pins group:name:version, and returns that component's artifact sha256s by file name.
+func requireVerified(t *testing.T, module, group, name, version string) map[string]string {
+	t.Helper()
+	path := "plugins/" + module + "/gradle/verification-metadata.xml"
+	var doc struct {
+		VerifyMetadata bool `xml:"configuration>verify-metadata"`
+		Components     []struct {
+			Group     string `xml:"group,attr"`
+			Name      string `xml:"name,attr"`
+			Version   string `xml:"version,attr"`
+			Artifacts []struct {
+				Name   string `xml:"name,attr"`
+				SHA256 []struct {
+					Value string `xml:"value,attr"`
+				} `xml:"sha256"`
+			} `xml:"artifact"`
+		} `xml:"components>component"`
+	}
+	if err := xml.Unmarshal([]byte(readGameStackFile(t, path)), &doc); err != nil {
+		t.Fatalf("%s: %v", path, err)
+	}
+	if !doc.VerifyMetadata {
+		t.Errorf("%s does not verify metadata; a swapped pom could redirect the graph", path)
+	}
+	for _, c := range doc.Components {
+		if c.Group != group || c.Name != name || c.Version != version {
+			continue
+		}
+		sums := map[string]string{}
+		for _, a := range c.Artifacts {
+			if len(a.SHA256) > 0 {
+				sums[a.Name] = a.SHA256[0].Value
+			}
+		}
+		if sums[name+"-"+version+".jar"] == "" {
+			t.Errorf("%s pins %s:%s:%s but no sha256 for its jar", path, group, name, version)
+		}
+		return sums
+	}
+	t.Errorf("%s has no checksum for %s:%s:%s; the build would refuse it", path, group, name, version)
+	return nil
+}
+
+// wrapperProperties reads a module's gradle-wrapper.properties off disk: the wrappers are
+// for developers and CI, and nothing embeds them.
+func wrapperProperties(t *testing.T, module string) map[string]string {
+	t.Helper()
+	b, err := os.ReadFile("plugins/" + module + "/gradle/wrapper/gradle-wrapper.properties")
+	if err != nil {
+		t.Fatal(err)
+	}
+	props := map[string]string{}
+	for line := range strings.SplitSeq(string(b), "\n") {
+		if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok && !strings.HasPrefix(k, "#") {
+			props[k] = strings.ReplaceAll(v, `\:`, ":")
+		}
+	}
+	return props
+}
+
+// gameStackLock parses the shipped deploy/game-stack.lock the way bootstrap.sh does.
+func gameStackLock(t *testing.T) map[string]string {
+	t.Helper()
+	lock := map[string]string{}
+	for line := range strings.SplitSeq(readGameStackFile(t, "deploy/game-stack.lock"), "\n") {
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			t.Fatalf("not a KEY=value line: %q", line)
+		}
+		lock[k] = v
+	}
+	return lock
+}
+
+// readRepoFile reads a file of the checkout that the binary does not embed.
+func readRepoFile(t *testing.T, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
 func readGameStackFile(t *testing.T, name string) string {
 	t.Helper()
 	b, err := gameStackAssets.ReadFile(name)

cmd/felis/api.go

@@ -5,10 +5,14 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"felis.lolicon.best/internal/api"
@@ -17,18 +21,26 @@ import (
 	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/fileedit"
+	"felis.lolicon.best/internal/imagepin"
 	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/metrics"
+	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/panel"
 	"felis.lolicon.best/internal/passkey"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/reaper"
+	"felis.lolicon.best/internal/registryprune"
 	"felis.lolicon.best/internal/restore"
-	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/retention"
 	"felis.lolicon.best/internal/submit"
+	"felis.lolicon.best/internal/worldexport"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -38,6 +50,22 @@ import (
 // the code marks Identity (UUIDs trusted verbatim); config can never add another.
 const mojangSessionServer = "https://sessionserver.mojang.com/session/minecraft/hasJoined"
 
+// passkeyRelyingParty is the one WebAuthn relying party both web faces share: its id
+// is the player console host, derived from server.root_domain the way the panel
+// handler derives it when auth.panel_hostname is unset, and its origins are that host
+// plus the operator host. An empty id means the install names no panel host at all.
+func passkeyRelyingParty(cfg *config.Config) (string, []string) {
+	rpID := defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname)
+	if rpID == "" {
+		return "", nil
+	}
+	origins := []string{"https://" + rpID}
+	if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != rpID {
+		origins = append(origins, "https://"+admin)
+	}
+	return rpID, origins
+}
+
 // authSourcesFromConfig builds the multiplexer's priority list from the configured
 // [[auth_source]] entries: Mojang leads as the code-owned identity anchor (正版优先, the ONLY
 // Identity source — config can only append namespace-rewritten third-party sources, never a
@@ -53,10 +81,8 @@ func authSourcesFromConfig(configured []config.AuthSourceConfig) []api.AuthSourc
 }
 
 // cmdAPI runs felis-api: two listeners, two middleware chains (spec §7). The
-// internal face (service token) is fully wired. The external face is wired but
-// fails closed until an Access JWKS key function is configured — the verifier's
-// audience logic is unit-tested (internal/api), the JWKS source is a deployment
-// integration point.
+// internal face authenticates per-caller service tokens; the external face
+// authenticates the local session cookie.
 func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("api", flag.ContinueOnError)
 	fs.SetOutput(stderr)
@@ -79,9 +105,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	// Load already refused a malformed [audit] retention.
+	auditRetention, _ := cfg.Audit.RetentionPeriod()
+	if auditRetention == 0 {
+		fmt.Fprintln(stdout, "felis api: audit rows are kept forever ([audit] retention = \"forever\")")
+	} else {
+		fmt.Fprintf(stdout, "felis api: audit rows older than %d days are deleted ([audit] retention; export them first with felis db audit-export)\n", int(auditRetention/(24*time.Hour)))
+	}
+
 	ctx := ctrl.SetupSignalHandler()
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Before anything serves: an api on a schema it was not built for answers with
+	// errors, or writes rows the other version cannot read.
+	drv, err := openPodStore(ctx, cfg.Database.URL, "api", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis api: open database: %v\n", err)
 		return 1
@@ -108,15 +144,24 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	token := os.Getenv("FELIS_SERVICE_TOKEN")
-	if token == "" {
-		fmt.Fprintln(stderr, "felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers")
+	metrics.SetBuildInfo("api", resolvedVersion())
+
+	internalAuth, err := internalCallerTokens(os.Getenv)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: internal face tokens: %v\n", err)
+		return 1
+	}
+	for _, ct := range naming.CallerTokens {
+		if internalAuth[api.Caller(ct.Caller)] == "" {
+			fmt.Fprintf(stderr, "felis api: warning: %s unset — the internal face turns the %s caller away\n", ct.APIEnv, ct.Caller)
+		}
 	}
 
 	// Email one-time codes go through the [smtp] relay when one is configured; the
 	// password is read from the env var password_ref names (default SMTPPasswordEnv,
 	// injected from the felis-smtp Secret). No [smtp] host ⇒ mailer stays nil and
-	// deliverOTP logs each code server-side (the pre-SMTP bootstrap posture).
+	// every door that mails a code answers 503 mail_unavailable: a code that is
+	// not mailed is never written anywhere else either.
 	var mailer api.OTPMailer
 	if cfg.SMTP.Host != "" {
 		passRef := cfg.SMTP.PasswordRef
@@ -127,26 +172,30 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		if cfg.SMTP.Username != "" && password == "" {
 			fmt.Fprintf(stderr, "felis api: warning: [smtp] username is set but credentials env %s is empty — OTP sends will fail AUTH\n", passRef)
 		}
-		mailer = &mail.SMTP{
-			Host:     cfg.SMTP.Host,
-			Port:     cfg.SMTP.Port,
-			From:     cfg.SMTP.From,
-			Username: cfg.SMTP.Username,
-			Password: password,
+		mailer = smtpRelay(cfg.SMTP, password)
+		if !cfg.SMTP.TLSRequired() {
+			fmt.Fprintf(stderr, "felis api: warning: [smtp] %s may be sent codes without TLS (require_tls off or a relay on this host)\n", cfg.SMTP.Host)
 		}
 	} else {
-		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email one-time codes are logged, not mailed")
+		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email sign-in and verification are off (503 mail_unavailable); sign in with a passkey, or run felis setup to add a relay")
 	}
 
 	// Build subsystem (spec §16): the weak-SA build Job runs in the configured
 	// build namespace and pushes to the internal registry. The build Pod never
 	// holds DB credentials — felis-api owns the PG store and admits scanned
 	// images, so the Builder is constructed here with both bindings.
+	buildCfg := buildConfig(cfg)
+	// The fetch initContainer runs THIS image's fetch-context entrypoint, so the
+	// build config carries the api's own image (the platform sets FELIS_IMAGE).
+	buildCfg.FelisImage = os.Getenv("FELIS_IMAGE")
+	buildJobs := build.NewK8sJobs(cl, buildCfg)
 	builder := &build.Builder{
-		Store:  build.NewPGStore(drv.DB()),
-		Jobs:   build.NewK8sJobs(cl, buildConfig(cfg)),
-		Config: buildConfig(cfg),
+		Store:    build.NewPGStore(drv.DB()),
+		Jobs:     buildJobs,
+		Config:   buildCfg,
+		Outcomes: build.NewK8sOutcomes(clientset, buildCfg),
 	}
+	go probeBuildUserNamespaces(ctx, buildJobs, buildCfg, stderr)
 
 	// User-modpack approval lane (user-directed extension over §16; see
 	// internal/submit). An ordinary user may only SUBMIT a
@@ -159,14 +208,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// The blob upload transport is selected by the shape of user_uploads_context —
 	// the two backends the setup wizard chooses between. A local path wires
 	// LocalContextStore (the mounted uploads PVC); an s3:// base wires
-	// S3ContextStore when its credentials resolve. Either way the store's target is
-	// derived from the SAME config field the context ref uses, so the blob lands
-	// exactly where Kaniko's --context points. Anything else — or an s3:// base with
-	// no credentials configured — leaves Blobs nil so POST
+	// S3ContextStore when its credentials resolve. Anything else — or an s3:// base
+	// with no credentials configured — leaves Blobs nil so POST
 	// /me/submissions/{id}/context returns 503, honest like the restore executor
-	// when its PVC is not supplied. (Letting the sandboxed Kaniko build Pod READ the
-	// context — PVC mount for local, creds+egress for S3 — is a separate deployment
-	// integration.)
+	// when its PVC is not supplied.
+	//
+	// Reading the blob back is the API's job, not Kaniko's: the build Pod runs in
+	// another namespace and can neither mount the uploads PVC (a PVC does not cross
+	// namespaces) nor hold object-store credentials, so ContextBaseURL makes the
+	// derived context ref an internal-face URL that the build Job's fetch
+	// initContainer streams (cmd/felis fetch-context). The platform renders this
+	// address into the api Deployment (felis API base URL env); the fallback keeps
+	// a hand-rolled deployment working under the platform's default control
+	// namespace.
 	contextBase := cfg.Registry.UserUploadsContext
 	var blobs submit.Blobs
 	switch {
@@ -186,11 +240,27 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis api: user-uploads context %q is neither a local path nor an s3:// base — modpack upload transport disabled (POST /api/v1/me/submissions/{id}/context returns 503)\n", contextBase)
 	}
 	submissions := &submit.Manager{
-		Store:        submit.NewPGStore(drv.DB()),
-		Builds:       builder,
-		Registry:     cfg.Registry.URL,
-		ContextStore: contextBase,
-		Blobs:        blobs,
+		Store:          submit.NewPGStore(drv.DB()),
+		Builds:         builder,
+		Registry:       cfg.Registry.URL,
+		ContextStore:   contextBase,
+		ContextBaseURL: internalAPIBaseURL(),
+		Blobs:          blobs,
+	}
+	if blobs != nil {
+		submissions.Parts = &submit.PartStore{Dir: uploadPartsDir(contextBase)}
+	}
+	if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
+		if n, err := parseByteSize(v); err != nil || n <= 0 {
+			fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
+		} else {
+			submissions.MaxStoredBytesTotal = n
+		}
+	}
+	if n, err := contextMaxBytes(cfg); err != nil {
+		fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 512Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
+	} else {
+		submissions.MaxContextBytes = n
 	}
 
 	// Restore subsystem (spec §7): the weak-SA restore Job mounts the target
@@ -223,6 +293,17 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503")
 	}
 
+	// World export: a weak-SA Job mounts the world PVC, or the backup PVC, read-only
+	// and PUTs the archive to the internal face, which streams it on to the owner's
+	// browser (internal/worldexport). A backup export mounts the backup PVC, so it
+	// is wired under the restore gate; otherwise the export routes return 503.
+	var exporter api.Exporter
+	if felisImage != "" && backupPVC != "" {
+		exporter = worldexport.New(clientset, exportConfig(cfg, felisImage, backupPVC))
+	} else {
+		fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503")
+	}
+
 	// Server file editor: a weak-SA Job mounts ONLY the target world PVC and runs
 	// `felis files`, printing its result for felis-api to read back through
 	// pods/log (see internal/fileedit). It needs FELIS_IMAGE but — unlike restore
@@ -231,12 +312,22 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// endpoints honestly return 503. It takes the typed clientset rather than the
 	// controller-runtime client because the log subresource lives only on the typed
 	// CoreV1 client, and one client covers its Job create, Pod list, and log read.
+	//
+	// Uploads additionally stage their bytes on this pod's disk until the Job
+	// fetches them from the internal face; whatever a previous process staged is
+	// orphaned (the index is in memory), so the stage starts empty.
 	var files api.FileEditor
+	var fileStage *fileedit.Stage
 	if felisImage != "" {
 		files = &fileedit.Editor{
 			Runner: fileedit.NewK8sRunner(clientset),
 			Config: fileEditConfig(cfg, felisImage),
 		}
+		fileStage = &fileedit.Stage{Dir: fileStagingDir()}
+		if err := fileStage.Sweep(); err != nil {
+			fmt.Fprintf(stderr, "felis api: %v — file uploads return 503\n", err)
+			fileStage = nil
+		}
 	} else {
 		fmt.Fprintln(stderr, "felis api: file editor disabled (needs FELIS_IMAGE) — file endpoints return 503")
 	}
@@ -246,32 +337,58 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// the same store the auth handlers write to, so a login and the next request
 	// agree on what local auth knows.
 	repo := api.NewPGRepo(drv.DB())
+	if err := api.RegisterStorePool(drv.DB()); err != nil {
+		fmt.Fprintf(stderr, "felis api: store pool metrics unavailable: %v\n", err)
+	}
 
+	// The owner's on-demand backup levers come from [archive], the same keys the
+	// backup Job and the reaper read. A malformed key leaves the defaults in
+	// place here; the reaper Job fails on it and names it.
+	rcfg, err := reaperConfig(cfg)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: %v; using the default backup limits\n", err)
+		rcfg = reaper.DefaultConfig()
+	}
+
+	serverCache, serversSynced, err := startServerCache(ctx, restCfg, scheme, cfg.K8s.Namespace, stderr)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err)
+		return 1
+	}
+	cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced)
+	jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace)
 	a := &api.API{
 		Repo:    repo,
-		Cluster: api.NewK8sCluster(cl, cfg.K8s.Namespace),
+		Cluster: cluster,
 		Console: api.NewK8sConsole(cl, cfg.K8s.Namespace),
 		Logs:    api.NewK8sLogStreamer(clientset, cfg.K8s.Namespace),
 		// Build-log stream (spec §16) is scoped to the BUILD namespace — the same
 		// value the Builder renders Jobs into — so it follows where build Pods run.
-		BuildLogs:   api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
-		Internal:    api.BearerTokenAuth{Token: token},
-		Builder:     builder,
-		Restorer:    restorer,
-		Backuper:    backuper,
-		Files:       files,
-		Submissions: submissions,
-		Mailer:      mailer,
-		// The external face is fronted by SessionAuth: it prefers a local session
-		// cookie (minted by the passwordless doors) and otherwise delegates to the
-		// Cloudflare-Access JWT verifier, so both auth models coexist on one face. The
-		// delegate's Keyfunc is intentionally nil — the JWT path fails closed until a
-		// JWKS-backed key function is wired (deployment integration point) — while the
-		// local session path is live the moment `felis breakGlass` flips
-		// local_auth_enabled on.
+		BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
+		Internal:  internalAuth,
+		Builder:   builder,
+		Images:    imagePinner(cfg.Registry.URL),
+		Restorer:  restorer,
+		Backuper:  backuper,
+		JobStatus: jobStatus,
+		// A restore starts with a safety snapshot; settleRestoreChains starts the
+		// restore behind each one.
+		RestoreChains: jobStatus,
+		Files:         files,
+		FileStage:     fileStage,
+		// The file Job fetches an upload from here; it runs in the minecraft
+		// namespace, where the internal face is reachable like it is for the login
+		// gate.
+		InternalBaseURL: internalAPIBaseURL(),
+		Exporter:        exporter,
+		Submissions:     submissions,
+		Mailer:          mailer,
+		Schedules:       repo,
+		// The external face authenticates the local session cookie the sign-in doors
+		// mint, live once `felis breakGlass` flips local_auth_enabled on. Cloudflare
+		// Access, when the install sits behind it, is enforced at the edge only.
 		External: api.SessionAuth{
 			Repo:          repo,
-			Delegate:      api.AccessVerifier{Audience: cfg.Auth.AccessJWTAud},
 			RootDomain:    cfg.Server.RootDomain,
 			AdminHostname: cfg.Auth.AdminHostname,
 		},
@@ -279,12 +396,32 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		AdminHostname: cfg.Auth.AdminHostname,
 		PanelHostname: cfg.Auth.PanelHostname,
 		WakeCooldown:  30 * time.Second,
+		// An owner may start one backup per server per manual_cooldown, and none
+		// while the store is at max_local_bytes (data-durability-9).
+		BackupCooldown: rcfg.ManualCooldown,
+		BackupStoreCap: rcfg.MaxLocalBytes,
+		// The user-modpack lane's per-user throttles: a create spaces out
+		// review-queue rows, an upload spaces out (up to 1 GiB) context streams.
+		// Separate keys, so the normal create→upload sequence stays immediate.
+		SubmitCreateCooldown: 30 * time.Second,
+		SubmitUploadCooldown: 15 * time.Second,
 		// Bound concurrent console/build-log SSE streams per principal. Generous enough
 		// for legitimate multi-tab / multi-server watching, while capping how many
 		// upstream follow connections a single caller can tie up if their streams stall.
 		MaxStreamsPerPrincipal: 16,
+		// Public sign-in doors, per client address: a person signing in makes a
+		// handful of calls, so 20 at once refilled at 20 a minute never bites a
+		// real user and still turns a spray into a trickle. The client address
+		// is the edge's header when the install names one (config.AuthConfig).
+		AuthDoorLimit:  api.RateLimit{Burst: 20, PerMinute: 20},
+		ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(),
+		MailLimit:      mailLimit(cfg.SMTP.MaxPerHour),
+	}
+	if a.ClientIPHeader != "" {
+		fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader)
+	} else {
+		fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)")
 	}
-	fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)")
 
 	// Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned
 	// identity anchor (正版优先); config can only append namespace-rewritten third-party
@@ -301,22 +438,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// enrolled once asserts on either face — one binding, usable on the player console
 	// AND the operator console. Both hosts are therefore listed as permitted origins,
 	// while the RP id stays the panel host so the credential's scope is ONE relying
-	// party, not two. Wired only when auth.panel_hostname is configured; otherwise
-	// a.Passkey stays nil and the passkey routes honestly return 503 (the authenticated
-	// enrollment boundary is still enforced by the handlers).
-	if cfg.Auth.PanelHostname != "" {
-		origins := []string{"https://" + cfg.Auth.PanelHostname}
-		if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != cfg.Auth.PanelHostname {
-			origins = append(origins, "https://"+admin)
-		}
-		pv, err := passkey.New(cfg.Auth.PanelHostname, "Felis", origins)
-		if err != nil {
-			fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
-		} else {
-			a.Passkey = pv
-		}
+	// party, not two. Without a panel host (neither auth.panel_hostname nor
+	// server.root_domain) a.Passkey stays nil and the passkey routes honestly return
+	// 503 (the authenticated enrollment boundary is still enforced by the handlers).
+	if rpID, origins := passkeyRelyingParty(cfg); rpID == "" {
+		fmt.Fprintln(stderr, "felis api: passkey verifier disabled (no panel host: set server.root_domain or auth.panel_hostname) — passkey endpoints return 503")
+	} else if pv, err := passkey.New(rpID, "Felis", origins); err != nil {
+		fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
 	} else {
-		fmt.Fprintln(stderr, "felis api: passkey verifier disabled (auth.panel_hostname unset) — passkey endpoints return 503")
+		a.Passkey = pv
 	}
 
 	// Derive the console hostnames when felis.toml leaves them unset, exactly as the
@@ -325,7 +455,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain,
 		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname),
 		defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname),
-		resolvedVersion())
+		cfg.Velocity.GamePort, resolvedVersion())
 	internalSrv := newAPIServer(*internalAddr, a.InternalHandler())
 	externalSrv := newAPIServer(cfg.Server.Listen, externalHandler)
 
@@ -347,16 +477,39 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// and advance any whose Job has reached a terminal phase. GET on a build also
 	// reconciles it, but this loop converges builds nobody is polling.
 	go reconcileBuilds(ctx, builder, stderr)
+	go settleRestoreChains(ctx, a, stderr)
+	go runSchedules(ctx, a, stderr)
+	// A daily restore point of every world played since its last one, taken
+	// once the server stops ([archive] scheduled_every; 0s turns it off).
+	if backuper != nil && rcfg.ScheduledEvery > 0 {
+		go scheduleBackups(ctx, &api.BackupScheduler{API: a, Store: repo, Jobs: jobStatus, Every: rcfg.ScheduledEvery}, stderr)
+	} else {
+		fmt.Fprintln(stderr, "felis api: scheduled backups off (needs the backup executor and [archive] scheduled_every above 0s)")
+	}
+
+	if pruner := registryPruner(cfg, builder.Store, cluster, stderr); pruner != nil {
+		go pruner.Loop(ctx, registryPruneInterval)
+	}
+	go reapRejectedContexts(ctx, submissions, stderr)
+	if fileStage != nil {
+		go expireFileSessions(ctx, fileStage, fileSessionSweep, stderr)
+	}
+	if exporter != nil {
+		go expireExports(ctx, a, exportSweep)
+	}
+	go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
+
+	servers := []*http.Server{internalSrv, externalSrv}
+	if httpsSrv != nil {
+		servers = append(servers, httpsSrv)
+	}
+	for _, srv := range servers {
+		srv.RegisterOnShutdown(a.CloseStreams)
+	}
 
 	select {
 	case <-ctx.Done():
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		_ = internalSrv.Shutdown(shutdownCtx)
-		_ = externalSrv.Shutdown(shutdownCtx)
-		if httpsSrv != nil {
-			_ = httpsSrv.Shutdown(shutdownCtx)
-		}
+		shutdownServers(servers, apiShutdownGrace, stderr)
 		return 0
 	case err := <-errc:
 		if err != nil && err != http.ErrServerClosed {
@@ -376,8 +529,30 @@ const (
 	// apiIdleTimeout caps how long a kept-alive connection may sit idle between
 	// requests before the server closes it, bounding idle-connection exhaustion.
 	apiIdleTimeout = 120 * time.Second
+	// apiShutdownGrace is how long the listeners drain after SIGTERM. The pod gets
+	// the Kubernetes default of 30s before SIGKILL; this leaves the rest for the
+	// process to exit.
+	apiShutdownGrace = 20 * time.Second
 )
 
+// shutdownServers drains every listener at once under one deadline: in turn, a
+// slow first listener would spend the time the others needed. Log streams end
+// through RegisterOnShutdown (API.CloseStreams); what is still running when the
+// deadline passes is cut off with the process.
+func shutdownServers(servers []*http.Server, grace time.Duration, stderr io.Writer) {
+	ctx, cancel := context.WithTimeout(context.Background(), grace)
+	defer cancel()
+	var wg sync.WaitGroup
+	for _, srv := range servers {
+		wg.Go(func() {
+			if err := srv.Shutdown(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: shutdown %s: %v\n", srv.Addr, err)
+			}
+		})
+	}
+	wg.Wait()
+}
+
 // newAPIServer builds an http.Server with hardened header/idle timeouts (gosec
 // G112) shared by all three felis-api listeners (internal, external, https).
 // WriteTimeout and ReadTimeout are deliberately LEFT UNSET: the external and https
@@ -401,9 +576,63 @@ func buildConfig(cfg *config.Config) build.Config {
 	return build.Config{
 		Namespace:   cfg.Registry.BuildNamespace,
 		RegistryURL: cfg.Registry.URL,
+		// Empty overrides fall back to the registry's copies of the tools
+		// (build.Tools), which felis mirror-build-tools keeps current.
+		KanikoImage: cfg.Registry.KanikoImage,
+		TrivyImage:  cfg.Registry.TrivyImage,
+		CPULimit:    cfg.Registry.BuildCPULimit,
+		MemLimit:    cfg.Registry.BuildMemLimit,
+		DiskLimit:   cfg.Registry.BuildDiskLimit,
+		// "auto" follows the startup probe (see probeBuildUserNamespaces).
+		UserNamespaces:        cfg.Registry.BuildUserNamespaces,
+		UserNamespacesProbe:   new(atomic.Bool),
+		RuntimeClass:          cfg.Registry.BuildRuntimeClass,
+		MaxConcurrent:         cfg.Registry.MaxConcurrentBuilds,
+		TrivyDBRepository:     cfg.Registry.TrivyDBRepository,
+		TrivyJavaDBRepository: cfg.Registry.TrivyJavaDBRepository,
+		ScanFailOn:            cfg.Registry.ScanFailOn,
+		ScanFailUnfixed:       cfg.Registry.ScanFailUnfixed,
+		ScanAccept:            cfg.Registry.ScanAccept,
+		// The submit lane's derived context URLs live here; the fetch step's
+		// service token goes nowhere else.
+		ContextOrigin: internalAPIBaseURL(),
 	}
 }
 
+// probeBuildUserNamespaces settles build_user_namespaces = "auto": one probe
+// pod with hostUsers: false tells whether this node's kernel and runtime can run
+// build pods in a user namespace. Builds submitted before it answers run without.
+func probeBuildUserNamespaces(ctx context.Context, jobs *build.K8sJobs, cfg build.Config, stderr io.Writer) {
+	if mode := cfg.UserNamespaces; mode != "" && mode != build.UserNamespacesAuto {
+		return
+	}
+	if cfg.FelisImage == "" {
+		fmt.Fprintln(stderr, "felis api: FELIS_IMAGE unset — build pods run without a user namespace")
+		return
+	}
+	ok, err := jobs.ProbeUserNamespaces(ctx, cfg.FelisImage)
+	cfg.UserNamespacesProbe.Store(ok)
+	switch {
+	case ok:
+		fmt.Fprintln(stderr, "felis api: build pods run in a user namespace (hostUsers: false)")
+	case err != nil:
+		fmt.Fprintf(stderr, "felis api: build pods run without a user namespace: the probe failed: %v\n", err)
+	default:
+		fmt.Fprintln(stderr, "felis api: build pods run without a user namespace: this node cannot start a pod with hostUsers: false")
+	}
+}
+
+// internalAPIBaseURL resolves the platform's internal-face base URL: the address
+// the platform rendered into this pod (felis API base URL env), or — for a
+// hand-rolled deployment that set none — the platform default control namespace,
+// the same fallback setup.go uses to hand the login gate its address.
+func internalAPIBaseURL() string {
+	if base := os.Getenv(naming.EnvAPIBaseURL); base != "" {
+		return base
+	}
+	return platform.InternalAPIBaseURL(platform.DefaultControlNamespace)
+}
+
 // uploadsSchemeRE matches a leading URL scheme like "s3://" or "gs://".
 var uploadsSchemeRE = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://`)
 
@@ -477,6 +706,17 @@ func backupConfig(cfg *config.Config, image, backupPVC string) backupjob.Config
 	}
 }
 
+// exportConfig builds the world export executor's config. BackupRoot mirrors
+// restoreConfig: the stored refs are absolute paths under [archive] local_path.
+func exportConfig(cfg *config.Config, image, backupPVC string) worldexport.Config {
+	return worldexport.Config{
+		Namespace:  cfg.K8s.Namespace,
+		Image:      image,
+		BackupPVC:  backupPVC,
+		BackupRoot: cfg.Archive.LocalPath,
+	}
+}
+
 // fileEditConfig builds the file editor's config from felis.toml plus the
 // deployment-supplied image. It is the shortest of the three: the editor mounts
 // only the world PVC, so it needs no archive coordinates at all, and everything
@@ -506,3 +746,339 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
 		}
 	}
 }
+
+// settleRestoreChains starts the restore behind each safety snapshot that has
+// finished (and gives up the one behind a snapshot that failed). The world stays
+// locked in between, so the interval is how long a finished snapshot keeps the
+// server down before its restore begins.
+func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
+	t := time.NewTicker(5 * time.Second)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := a.SettleRestoreChains(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: restore chains: %v\n", err)
+			}
+		}
+	}
+}
+
+// runSchedules runs the servers' scheduled tasks (api.API.RunSchedules). The
+// interval is how late a task may start, and how often a restart or backup in
+// progress checks whether it can take its next step.
+func runSchedules(ctx context.Context, a *api.API, stderr io.Writer) {
+	t := time.NewTicker(15 * time.Second)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := a.RunSchedules(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: scheduled tasks: %v\n", err)
+			}
+		}
+	}
+}
+
+// scheduleBackups starts the scheduled backups (api.BackupScheduler). Each tick
+// starts at most one, so the interval also spaces the worlds that stopped at
+// the same time: a world that stops waits at most this long for its point to
+// start once the Jobs ahead of it are done.
+func scheduleBackups(ctx context.Context, s *api.BackupScheduler, stderr io.Writer) {
+	t := time.NewTicker(2 * time.Minute)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := s.Tick(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: scheduled backups: %v\n", err)
+			}
+		}
+	}
+}
+
+// fileSessionSweep is how often expireFileSessions looks for idle upload
+// sessions: small beside fileedit.SessionIdle, so an abandoned one gives its
+// room back within minutes of going stale.
+const fileSessionSweep = 10 * time.Minute
+
+// expireFileSessions drops the file manager's upload sessions left untouched
+// for fileedit.SessionIdle. Each reserved room on the staging disk for its whole
+// file when it began, so one abandoned would otherwise hold that room until
+// felis-api restarts.
+func expireFileSessions(ctx context.Context, s *fileedit.Stage, every time.Duration, stderr io.Writer) {
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if n := s.Expire(); n > 0 {
+				fmt.Fprintf(stderr, "felis api: dropped %d upload session(s) left idle for %s\n", n, fileedit.SessionIdle)
+			}
+		}
+	}
+}
+
+// exportSweep is how often expireExports runs: an export whose Job never
+// connected is stopped within a minute of going stale.
+const exportSweep = time.Minute
+
+// expireExports runs the export sweep (api.API.ExpireExports) on a ticker. The
+// export routes sweep as they are called, and an owner who closed the tab calls
+// none; a Job whose Pod never got going would then keep the server from
+// starting until the Job's deadline.
+func expireExports(ctx context.Context, a interface{ ExpireExports() }, every time.Duration) {
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			a.ExpireExports()
+		}
+	}
+}
+
+// reapRejectedContexts deletes, once an hour, the uploaded contexts of
+// submissions rejected more than submit.RejectedContextRetention ago, and the
+// chunked uploads left untouched for submit.StalePartRetention. Without it a
+// rejected modpack or an abandoned upload keeps its bytes on the uploads store
+// (and against its submitter's budget) until an admin deletes the row.
+func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
+	t := time.NewTicker(time.Hour)
+	defer t.Stop()
+	for {
+		n, err := m.ReapRejected(ctx, submit.RejectedContextRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap rejected uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
+		}
+		n, err = m.ReapStaleParts(submit.StalePartRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap abandoned uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted %d abandoned chunked upload(s)\n", n)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// retentionInterval spaces the runs that delete spent sign-in rows and audit rows
+// past [audit] retention. The rows are spent for weeks before they go, so a few
+// runs a day keep the tables flat.
+const retentionInterval = 6 * time.Hour
+
+// registryPruneInterval spaces the registry pruner's runs. The registry-gc
+// sidecar sweeps once a day, so pruning more often only changes which sweep frees
+// a layer.
+const registryPruneInterval = 6 * time.Hour
+
+// registryPruner deletes the registry manifests nothing references
+// (internal/registryprune); the registry-gc sidecar frees their layers on its next
+// sweep. It acts as the gate's prune principal, whose token the api Deployment
+// injects from felis-registry-auth. Without the token the registry only grows,
+// which is said once here.
+func registryPruner(cfg *config.Config, store imageRefStore, servers serverLister, stderr io.Writer) *registryprune.Pruner {
+	if cfg.Registry.URL == "" {
+		return nil
+	}
+	token := os.Getenv(platform.RegistryPruneTokenEnv)
+	if token == "" {
+		fmt.Fprintf(stderr, "felis api: registry pruner disabled (%s unset) — images nothing uses are never deleted from the registry\n", platform.RegistryPruneTokenEnv)
+		return nil
+	}
+	static := append([]string{os.Getenv("FELIS_IMAGE")}, buildConfig(cfg).ToolRefs()...)
+	return &registryprune.Pruner{
+		Registry: &registryprune.Client{Endpoint: "http://" + cfg.Registry.URL, Token: token},
+		Host:     cfg.Registry.URL,
+		Refs: func(ctx context.Context) ([]string, error) {
+			return inUseImageRefs(ctx, store, servers, static)
+		},
+		Log: slog.New(slog.NewTextHandler(stderr, nil)),
+	}
+}
+
+type imageRefStore interface {
+	ListImages(ctx context.Context) ([]build.Image, error)
+	ListUnfinishedBuilds(ctx context.Context) ([]build.Build, error)
+}
+
+type serverLister interface {
+	ListServers(ctx context.Context) ([]api.ServerInfo, error)
+	PodImages(ctx context.Context) ([]string, error)
+}
+
+// inUseImageRefs lists every image reference the platform still depends on: the
+// whitelist (disabled rows too, an admin may enable them again), every server's
+// spec, the images the game pods run, builds still running, and the images the
+// control plane and the build Jobs run. Any source failing fails the whole list,
+// so the pruner never decides on a partial view.
+//
+// The pods matter for the felis image: a running server keeps the one it started
+// with across platform upgrades (operator.PodTemplateAnnotation), which after a
+// few releases is no longer among the newest tags the pruner keeps anyway, and
+// the pod needs it again whenever it is recreated.
+func inUseImageRefs(ctx context.Context, store imageRefStore, servers serverLister, static []string) ([]string, error) {
+	refs := append([]string(nil), static...)
+	images, err := store.ListImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("image whitelist: %w", err)
+	}
+	for _, img := range images {
+		refs = append(refs, img.ImageRef)
+	}
+	srvs, err := servers.ListServers(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("servers: %w", err)
+	}
+	for _, s := range srvs {
+		refs = append(refs, s.Image)
+	}
+	podImages, err := servers.PodImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("game pods: %w", err)
+	}
+	refs = append(refs, podImages...)
+	builds, err := store.ListUnfinishedBuilds(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("running builds: %w", err)
+	}
+	for _, b := range builds {
+		refs = append(refs, b.ImageRef)
+	}
+	return refs, nil
+}
+
+// mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket:
+// the hourly cap as the refill rate, with a quarter of it (at least 5) allowed
+// at once so a burst of real sign-ins is not queued behind the average.
+func mailLimit(perHour int) api.RateLimit {
+	if perHour <= 0 {
+		perHour = config.DefaultMailPerHour
+	}
+	return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60}
+}
+
+// imagePinner resolves a new server's image against the platform registry
+// through its in-cluster Service, the address its refs already spell. An install
+// without a registry has no platform-built images to pin.
+func imagePinner(registry string) api.ImagePinner {
+	if registry == "" {
+		return nil
+	}
+	return imagepin.Resolver{Registry: registry}
+}
+
+// internalCallerTokens reads each internal caller's token from the env var the
+// Deployment feeds it from (naming.CallerTokens). Two callers sharing a value
+// would make the caller ambiguous, so that refuses to start.
+func internalCallerTokens(getenv func(string) string) (api.CallerTokens, error) {
+	tokens := map[api.Caller]string{}
+	for _, ct := range naming.CallerTokens {
+		tokens[api.Caller(ct.Caller)] = strings.TrimSpace(getenv(ct.APIEnv))
+	}
+	return api.NewCallerTokens(tokens)
+}
+
+// smtpRelay is the relay [smtp] names, with the resolved password and the TLS
+// posture config.SMTPConfig.TLSRequired picks. felis api, the reaper and the
+// watchdog all send through it, so none can drift to a weaker posture.
+func smtpRelay(c config.SMTPConfig, password string) *mail.SMTP {
+	return &mail.SMTP{
+		Host:       c.Host,
+		Port:       c.Port,
+		From:       c.From,
+		Username:   c.Username,
+		Password:   password,
+		RequireTLS: c.TLSRequired(),
+	}
+}
+
+// startServerCache starts the informer that serves the api's fleet-wide
+// MinecraftServer reads (api.K8sCluster.WithServerCache): one watch on the
+// namespace instead of a full List per velocity pull, fleet page and wake. It
+// caches MinecraftServers only — ReaderFailOnMissingInformer turns any other read
+// through it into an error rather than a new informer the api's Role cannot back —
+// indexes spec.subdomain for GetBySubdomain, and drops managedFields to keep the
+// copy small. It returns without waiting: the reads block until the first list
+// lands and /readyz reports not-ready until then.
+func startServerCache(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, stderr io.Writer) (cache.Cache, func() bool, error) {
+	c, err := cache.New(cfg, cache.Options{
+		Scheme:                      scheme,
+		DefaultNamespaces:           map[string]cache.Config{namespace: {}},
+		DefaultTransform:            cache.TransformStripManagedFields(),
+		ReaderFailOnMissingInformer: true,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := c.IndexField(ctx, &v1alpha1.MinecraftServer{}, api.SubdomainIndex, api.SubdomainOf); err != nil {
+		return nil, nil, fmt.Errorf("index %s: %w", api.SubdomainIndex, err)
+	}
+	inf, err := c.GetInformer(ctx, &v1alpha1.MinecraftServer{})
+	if err != nil {
+		return nil, nil, err
+	}
+	go func() {
+		if err := c.Start(ctx); err != nil {
+			fmt.Fprintf(stderr, "felis api: MinecraftServer cache stopped: %v\n", err)
+		}
+	}()
+	return c, inf.HasSynced, nil
+}
+
+// uploadPartsDir is where chunked uploads are staged: beside a local store's
+// contexts, so the room check and the budget see one disk and a staged upload
+// survives an API restart; for an s3:// store, on the uploads volume the
+// platform mounts either way, or the pod's /tmp when run by hand without it.
+func uploadPartsDir(contextBase string) string {
+	if isLocalUploadsPath(contextBase) {
+		return filepath.Join(strings.TrimPrefix(contextBase, "file://"), ".parts")
+	}
+	if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
+		return filepath.Join(platform.UploadsLocalPath, ".parts")
+	}
+	return filepath.Join(os.TempDir(), "felis-upload-parts")
+}
+
+// fileStagingDir is where file uploads wait for their Job: on the uploads
+// volume, whose capacity is its own, or the pod's /tmp when run by hand without
+// it — /tmp is the node's disk, which a burst of uploads should not fill.
+func fileStagingDir() string {
+	if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
+		return filepath.Join(platform.UploadsLocalPath, ".file-staging")
+	}
+	return filepath.Join(os.TempDir(), "felis-file-staging")
+}
+
+// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
+// package's own default (1 GiB). The Cloudflare edge refuses a single request
+// body over 100 MB, which the panel's chunked upload stays under, so the edge
+// does not lower the cap.
+func contextMaxBytes(cfg *config.Config) (int64, error) {
+	v := cfg.Registry.ContextMaxBytes
+	if v == "" {
+		return 0, nil
+	}
+	n, err := parseByteSize(v)
+	if err != nil || n <= 0 {
+		return 0, fmt.Errorf("not a positive size: %q", v)
+	}
+	return n, nil
+}

cmd/felis/api_test.go

@@ -1,12 +1,56 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
 	"net/http"
+	"slices"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/fileedit"
 )
 
+// The passkey relying party follows the panel host the SPA is served on: an install
+// that names only its root domain still gets passkeys, on console.<root>, with the
+// operator host as the second origin; only an install with no panel host goes without.
+func TestPasskeyRelyingParty(t *testing.T) {
+	for _, tc := range []struct {
+		name               string
+		root, panel, admin string
+		wantRP             string
+		wantOrigins        []string
+	}{
+		{"root domain only", "example.net", "", "", "console.example.net",
+			[]string{"https://console.example.net", "https://op.console.example.net"}},
+		{"configured hosts", "example.net", " play.example.net ", "ops.example.net", "play.example.net",
+			[]string{"https://play.example.net", "https://ops.example.net"}},
+		{"operator host equal to the panel host", "example.net", "console.example.net", "console.example.net",
+			"console.example.net", []string{"https://console.example.net"}},
+		{"panel host without a root domain", "", "console.example.org", "", "console.example.org",
+			[]string{"https://console.example.org"}},
+		{"no host at all", "", "", "", "", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := &config.Config{}
+			cfg.Server.RootDomain, cfg.Auth.PanelHostname, cfg.Auth.AdminHostname = tc.root, tc.panel, tc.admin
+			rp, origins := passkeyRelyingParty(cfg)
+			if rp != tc.wantRP || !slices.Equal(origins, tc.wantOrigins) {
+				t.Fatalf("relying party = %q %q, want %q %q", rp, origins, tc.wantRP, tc.wantOrigins)
+			}
+		})
+	}
+}
+
 // TestAuthSourcesFromConfig pins the one place the hasJoined identity anchor is decided:
 // Mojang is prepended in code, first, and is the only source whose UUIDs are trusted as-is.
 // The empty case matters on its own — both `felis api` and `felis nano` call this with a
@@ -44,6 +88,32 @@ func TestAuthSourcesFromConfig(t *testing.T) {
 	}
 }
 
+// TestBuildConfig_ProjectsOverrides pins the [registry] overrides reaching the
+// build subsystem: unset fields must stay EMPTY (the build package's compiled-in
+// defaults apply there, not here), and set fields must pass through verbatim —
+// an air-gapped install points these at its imported mirrors.
+func TestBuildConfig_ProjectsOverrides(t *testing.T) {
+	empty := buildConfig(&config.Config{})
+	if empty.KanikoImage != "" || empty.TrivyImage != "" || empty.CPULimit != "" || empty.MemLimit != "" {
+		t.Errorf("empty registry config must project empty overrides (defaults live in internal/build), got %+v", empty)
+	}
+	full := buildConfig(&config.Config{Registry: config.RegistryConfig{
+		URL:            "registry.felis.svc:5000",
+		BuildNamespace: "felis-build",
+		KanikoImage:    "reg/kaniko:v1",
+		TrivyImage:     "reg/trivy:v1",
+		BuildCPULimit:  "1",
+		BuildMemLimit:  "2Gi",
+	}})
+	if full.KanikoImage != "reg/kaniko:v1" || full.TrivyImage != "reg/trivy:v1" ||
+		full.CPULimit != "1" || full.MemLimit != "2Gi" {
+		t.Errorf("registry overrides did not reach build.Config: %+v", full)
+	}
+	if full.Namespace != "felis-build" || full.RegistryURL != "registry.felis.svc:5000" {
+		t.Errorf("namespace/registry url must keep projecting: %+v", full)
+	}
+}
+
 // TestNewAPIServerSetsHardenedTimeouts pins the gosec-G112 hardening on every
 // felis-api listener: the shared factory must bound the header and idle phases
 // (Slowloris + idle-connection exhaustion) while leaving WriteTimeout UNSET, because
@@ -65,3 +135,179 @@ func TestNewAPIServerSetsHardenedTimeouts(t *testing.T) {
 		t.Errorf("ReadTimeout = %v, want 0 (unset) so a slow SSE attach is not capped", srv.ReadTimeout)
 	}
 }
+
+// Every listener drains at once, and the shutdown hook (API.CloseStreams in
+// cmdAPI) runs on each: two listeners each holding a request that ends when
+// the hook fires take one hook's worth of time, well inside the deadline.
+func TestShutdownServersDrainsListenersTogether(t *testing.T) {
+	release := make(chan struct{})
+	var hooks int32
+	started := make(chan struct{}, 2)
+	var servers []*http.Server
+	for range 2 {
+		srv := newAPIServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			started <- struct{}{}
+			<-release
+		}))
+		srv.RegisterOnShutdown(func() {
+			if atomic.AddInt32(&hooks, 1) == 1 {
+				time.AfterFunc(100*time.Millisecond, func() { close(release) })
+			}
+		})
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			t.Fatal(err)
+		}
+		srv.Addr = l.Addr().String()
+		go func() { _ = srv.Serve(l) }()
+		go func() {
+			if resp, err := http.Get("http://" + srv.Addr); err == nil {
+				resp.Body.Close()
+			}
+		}()
+		servers = append(servers, srv)
+	}
+	<-started
+	<-started
+
+	begun := time.Now()
+	shutdownServers(servers, 5*time.Second, io.Discard)
+	if took := time.Since(begun); took > 2*time.Second {
+		t.Fatalf("shutdown took %v", took)
+	}
+	if got := atomic.LoadInt32(&hooks); got != 2 {
+		t.Fatalf("shutdown hook ran %d times, want once per listener", got)
+	}
+}
+
+type fakeRefStore struct {
+	images []build.Image
+	builds []build.Build
+	err    error
+}
+
+func (f fakeRefStore) ListImages(context.Context) ([]build.Image, error) { return f.images, f.err }
+func (f fakeRefStore) ListUnfinishedBuilds(context.Context) ([]build.Build, error) {
+	return f.builds, nil
+}
+
+type fakeServers struct {
+	list    []api.ServerInfo
+	pods    []string
+	podsErr error
+}
+
+func (f fakeServers) ListServers(context.Context) ([]api.ServerInfo, error) { return f.list, nil }
+func (f fakeServers) PodImages(context.Context) ([]string, error)           { return f.pods, f.podsErr }
+
+// The registry pruner deletes whatever this list does not name, so every source of
+// a reference has to be in it, and a failing source must fail the list.
+func TestInUseImageRefsCoversEverySource(t *testing.T) {
+	const reg = "registry.felis.svc:5000/"
+	store := fakeRefStore{
+		images: []build.Image{{ImageRef: reg + "modpacks/pack:*"}, {ImageRef: reg + "felis/paper:demo"}},
+		builds: []build.Build{{ImageRef: reg + "user-uploads/sub-9:latest"}},
+	}
+	servers := fakeServers{
+		list: []api.ServerInfo{{Name: "s1", Image: reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1)}},
+		pods: []string{reg + "felis/felis:v1.0.0"},
+	}
+	got, err := inUseImageRefs(context.Background(), store, servers, []string{reg + "felis/felis:b60"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		reg + "felis/felis:b60",
+		reg + "modpacks/pack:*", reg + "felis/paper:demo",
+		reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1),
+		reg + "felis/felis:v1.0.0",
+		reg + "user-uploads/sub-9:latest",
+	}
+	if fmt.Sprint(got) != fmt.Sprint(want) {
+		t.Fatalf("refs = %v\nwant %v", got, want)
+	}
+
+	servers.podsErr = errors.New("apiserver down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing pod list produced a reference list")
+	}
+	servers.podsErr = nil
+
+	store.err = errors.New("db down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing whitelist read produced a reference list")
+	}
+}
+
+// TestExpireFileSessions runs the loop against a stage whose clock the test
+// holds: the session idle past fileedit.SessionIdle goes, the one touched since
+// stays, and the drop is said once.
+func TestExpireFileSessions(t *testing.T) {
+	var mu sync.Mutex
+	now := time.Date(2026, 9, 28, 10, 0, 0, 0, time.UTC)
+	advance := func(d time.Duration) { mu.Lock(); now = now.Add(d); mu.Unlock() }
+	st := &fileedit.Stage{Dir: t.TempDir(), MinFree: 1e-9, Now: func() time.Time {
+		mu.Lock()
+		defer mu.Unlock()
+		return now
+	}}
+	idle, err := st.Begin("u1", "survival", "a.jar", 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	advance(fileedit.SessionIdle - time.Minute)
+	fresh, err := st.Begin("u1", "survival", "b.jar", 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	advance(2 * time.Minute)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var out bytes.Buffer
+	done := make(chan struct{})
+	go func() { expireFileSessions(ctx, st, time.Millisecond, &out); close(done) }()
+	for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(time.Millisecond) {
+		if _, err := st.Status("u1", "survival", idle.ID); errors.Is(err, fileedit.ErrNotStaged) {
+			break
+		}
+		if time.Now().After(deadline) {
+			cancel()
+			t.Fatal("the idle session was never dropped")
+		}
+	}
+	// A few more ticks with nothing idle, which must stay quiet.
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+	<-done
+	if _, err := st.Status("u1", "survival", fresh.ID); err != nil {
+		t.Fatalf("the session touched since went too: %v", err)
+	}
+	if got := out.String(); got != "felis api: dropped 1 upload session(s) left idle for 6h0m0s\n" {
+		t.Fatalf("said %q", got)
+	}
+}
+
+type sweepCount struct{ n atomic.Int32 }
+
+func (s *sweepCount) ExpireExports() { s.n.Add(1) }
+
+// TestExpireExports: the loop sweeps on each tick, and returns once felis-api
+// shuts down.
+func TestExpireExports(t *testing.T) {
+	var s sweepCount
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { expireExports(ctx, &s, time.Millisecond); close(done) }()
+	for deadline := time.Now().Add(5 * time.Second); s.n.Load() < 3; time.Sleep(time.Millisecond) {
+		if time.Now().After(deadline) {
+			cancel()
+			t.Fatalf("swept %d times in 5s at a 1ms tick", s.n.Load())
+		}
+	}
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the loop outlived its context")
+	}
+}

cmd/felis/api_tokens_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/api"
+)
+
+// felis-api maps each env var onto the caller the Deployment feeds it for, so the
+// build Job's token (FELIS_BUILD_TOKEN) authenticates as build and only as build.
+func TestInternalCallerTokensReadEachCallersEnv(t *testing.T) {
+	env := map[string]string{
+		"FELIS_SERVICE_TOKEN": "v-tok",
+		"FELIS_LIMBO_TOKEN":   "l-tok",
+		"FELIS_BUILD_TOKEN":   " b-tok\n",
+		"FELIS_OPS_TOKEN":     "o-tok",
+	}
+	auth, err := internalCallerTokens(func(k string) string { return env[k] })
+	if err != nil {
+		t.Fatal(err)
+	}
+	for tok, want := range map[string]api.Caller{"v-tok": api.CallerVelocity, "l-tok": api.CallerLimbo, "b-tok": api.CallerBuild, "o-tok": api.CallerOps} {
+		r := httptest.NewRequest("GET", "/", nil)
+		r.Header.Set("Authorization", "Bearer "+tok)
+		if got, err := auth.Authenticate(r); err != nil || got != want {
+			t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want)
+		}
+	}
+
+	// An install where the build namespace still holds a copy of the proxy's token
+	// would let that copy act as the proxy; the api refuses to start on it.
+	env["FELIS_BUILD_TOKEN"] = "v-tok"
+	if _, err := internalCallerTokens(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "same value") {
+		t.Fatalf("shared token: err = %v, want a same-value refusal", err)
+	}
+}

cmd/felis/apply.go

@@ -8,7 +8,9 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/naming"
@@ -16,10 +18,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -111,20 +109,15 @@ func cmdApply(args []string, stdout, stderr io.Writer) int {
 	}
 
 	// ------- K8s client (one context, one client) -------
-	// SetupSignalHandler must be called exactly once per process —
-	// controller-runtime panics on a second call. We create ctx and the
-	// K8s client here and thread both through every downstream call so no
-	// callee ever needs to call SetupSignalHandler again.
-	ctx := ctrl.SetupSignalHandler()
-
-	scheme := runtime.NewScheme()
-	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-	utilruntime.Must(v1alpha1.AddToScheme(scheme))
-
-	cfg := ctrl.GetConfigOrDie()
-	cl, err := client.New(cfg, client.Options{Scheme: scheme})
+	// The operator runs this on the node, where the kubeconfig is k3s's own file and
+	// neither $KUBECONFIG nor ~/.kube is set. buildSystemServerClient falls back to that
+	// file and names what it tried; ctrl.GetConfigOrDie exited 1 there without a word,
+	// because controller-runtime's logger is never set up in a CLI command.
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	cl, err := buildSystemServerClient()
 	if err != nil {
-		fmt.Fprintf(stderr, "felis apply: build client: %v\n", err)
+		fmt.Fprintf(stderr, "felis apply: %v\n", err)
 		return 1
 	}
 
@@ -167,6 +160,10 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 	if err := naming.ValidateServerName(req.Subdomain); err != nil {
 		return nil, fmt.Errorf("invalid subdomain: %w", err)
 	}
+	displayName, err := naming.CleanDisplayName(req.DisplayName)
+	if err != nil {
+		return nil, fmt.Errorf("invalid displayName: %w", err)
+	}
 	if strings.TrimSpace(req.Image) == "" {
 		return nil, fmt.Errorf("image is required")
 	}
@@ -236,13 +233,26 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 		},
 		Spec: v1alpha1.MinecraftServerSpec{
 			Subdomain:       req.Subdomain,
-			DisplayName:     req.DisplayName,
+			DisplayName:     displayName,
 			Image:           req.Image,
 			JavaMemory:      deriveApplyJavaHeap(memLim),
 			DesiredState:    v1alpha1.DesiredStopped,
 			AutostartPolicy: policy,
 			Storage:         v1alpha1.StorageSpec{Size: storageQ.String()},
 			Resources:       corev1.ResourceRequirements{Limits: limits, Requests: requests},
+			// The rest matches what felis-api's create writes (K8sCluster.CreateServer):
+			// fall back to the login gate while stopped, RCON on (readiness, the
+			// player count and the console all ride it; the operator mints the
+			// password), and the default idle stop.
+			FallbackServer: naming.SystemLoginServer,
+			Rcon: v1alpha1.RconSpec{
+				Enabled: true,
+				SecretRef: v1alpha1.SecretKeyRef{
+					Name: naming.RconSecretName(req.Name),
+					Key:  naming.RconSecretKey,
+				},
+			},
+			Idle: v1alpha1.DefaultIdle(),
 		},
 	}, nil
 }
@@ -251,8 +261,7 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 // request if any CRD already carries the given spec.subdomain. metadata.name
 // uniqueness is enforced by K8s on Create, but spec.subdomain must be checked
 // here because two CRDs with different names could otherwise share a subdomain.
-// It reuses the caller's context and K8s client — it never calls
-// SetupSignalHandler or builds its own client.
+// It reuses the caller's context and K8s client.
 func checkSubdomainUnique(ctx context.Context, cl client.Client, namespace, subdomain string) error {
 	var list v1alpha1.MinecraftServerList
 	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {

cmd/felis/apply_test.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
@@ -136,11 +140,12 @@ func TestDeriveApplyJavaHeap(t *testing.T) {
 
 func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	req := applyRequest{
-		Name:      "test-server",
-		Subdomain: "test-server",
-		Image:     "registry.felis.svc/paper:1.21",
-		Memory:    "4Gi",
-		Storage:   "20Gi",
+		Name:        "test-server",
+		Subdomain:   "test-server",
+		DisplayName: "  Test Server  ",
+		Image:       "registry.felis.svc/paper:1.21",
+		Memory:      "4Gi",
+		Storage:     "20Gi",
 	}
 	ms, err := buildMinecraftServerFromApplyRequest(req, "minecraft")
 	if err != nil {
@@ -155,6 +160,9 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	if ms.Spec.Subdomain != "test-server" {
 		t.Errorf("Subdomain = %q", ms.Spec.Subdomain)
 	}
+	if ms.Spec.DisplayName != "Test Server" {
+		t.Errorf("DisplayName = %q, want it trimmed to Test Server", ms.Spec.DisplayName)
+	}
 	if ms.Spec.Image != "registry.felis.svc/paper:1.21" {
 		t.Errorf("Image = %q", ms.Spec.Image)
 	}
@@ -167,6 +175,18 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	if ms.Spec.Storage.Size != "20Gi" {
 		t.Errorf("Storage.Size = %q, want 20Gi", ms.Spec.Storage.Size)
 	}
+	// Same operational defaults as the API create path: without RCON the server
+	// never reports players and the console answers 503; without spec.idle it
+	// never stops on its own.
+	if !ms.Spec.Rcon.Enabled || ms.Spec.Rcon.SecretRef.Name != naming.RconSecretName("test-server") {
+		t.Errorf("Rcon = %+v, want enabled with the operator-minted secret", ms.Spec.Rcon)
+	}
+	if ms.Spec.Idle != v1alpha1.DefaultIdle() {
+		t.Errorf("Idle = %+v, want the default %+v", ms.Spec.Idle, v1alpha1.DefaultIdle())
+	}
+	if ms.Spec.FallbackServer != naming.SystemLoginServer {
+		t.Errorf("FallbackServer = %q, want the login gate", ms.Spec.FallbackServer)
+	}
 	mem, ok := ms.Spec.Resources.Limits[corev1.ResourceMemory]
 	if !ok {
 		t.Fatal("memory limit missing")
@@ -267,6 +287,11 @@ func TestBuildMinecraftServerFromApplyRequest_Errors(t *testing.T) {
 			applyRequest{Name: ok, Subdomain: "", Image: "x", Memory: "1Gi", Storage: "1Gi"},
 			"invalid subdomain",
 		},
+		{
+			"display name with a tab",
+			applyRequest{Name: ok, Subdomain: ok, DisplayName: "a" + string(rune(0x09)) + "b", Image: "x", Memory: "1Gi", Storage: "1Gi"},
+			"invalid displayName",
+		},
 		{
 			"empty image",
 			applyRequest{Name: ok, Subdomain: ok, Image: "", Memory: "1Gi", Storage: "1Gi"},
@@ -357,3 +382,27 @@ func resList(specs ...string) corev1.ResourceList {
 	}
 	return rl
 }
+
+// TestApplyReportsAMissingKubeconfig pins the node-side failure: with no kubeconfig to
+// find, apply says which ones it tried and exits 1. It used to call
+// ctrl.GetConfigOrDie, which ended the process with exit 1 and nothing printed.
+func TestApplyReportsAMissingKubeconfig(t *testing.T) {
+	if _, err := os.Stat(hostBootstrapKubeconfigPath); err == nil {
+		t.Skipf("%s exists on this machine", hostBootstrapKubeconfigPath)
+	}
+	dir := t.TempDir()
+	t.Setenv("KUBECONFIG", filepath.Join(dir, "missing"))
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	form := filepath.Join(dir, "server.json")
+	if err := os.WriteFile(form, []byte(`{"name":"alpha","subdomain":"alpha","image":"registry.felis.svc:5000/felis/paper:demo","memory":"1Gi","storage":"1Gi"}`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var out, errw bytes.Buffer
+	if code := cmdApply([]string{"-f", form}, &out, &errw); code != 1 {
+		t.Fatalf("exit = %d, want 1; stderr %q", code, errw.String())
+	}
+	want := "felis apply: no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and " + hostBootstrapKubeconfigPath + "): stat " + hostBootstrapKubeconfigPath + ": no such file or directory\n"
+	if errw.String() != want || out.Len() != 0 {
+		t.Fatalf("stdout %q, stderr %q, want stderr %q", out.String(), errw.String(), want)
+	}
+}

cmd/felis/backup.go

@@ -1,18 +1,20 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	"felis.lolicon.best/internal/backup"
+	"felis.lolicon.best/internal/backupjob"
 	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/reaper"
-	"felis.lolicon.best/internal/store"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
@@ -35,6 +37,8 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	server := fs.String("server", "", "server name whose world is being backed up")
 	formerOwner := fs.String("former-owner", "", "owner recorded on the backup row (empty for an unowned server)")
 	worldsRoot := fs.String("worlds-root", "/world", "mount path of the world PVC being archived")
+	reason := fs.String("reason", reasonManual, "world_backups reason: manual, pre_restore for the safety snapshot in front of a restore, or scheduled for felis-api's daily restore point")
+	protect := fs.String("protect", "", "backup id the prune must keep (the one a chained restore extracts)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -42,6 +46,10 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintln(stderr, "felis backup: --server is required")
 		return 2
 	}
+	if _, _, ok := backupPolicy(*reason, reaper.DefaultConfig()); !ok {
+		fmt.Fprintf(stderr, "felis backup: unknown --reason %q (manual, %s or %s)\n", *reason, backupjob.ReasonPreRestore, backupjob.ReasonScheduled)
+		return 2
+	}
 
 	cfg, err := config.Load(*cfgPath)
 	if err != nil {
@@ -52,13 +60,14 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis backup: archive store %q is not implemented in this build (only tarLocal)\n", cfg.Archive.Store)
 		return 1
 	}
-	// Reuse the reaper's retention derivation so an on-demand backup expires on the
-	// same clock as an inactivity backup — one retention policy, not two.
+	// The [archive] parse the reaper uses: it holds each reason's keep and
+	// retention.
 	rcfg, err := reaperConfig(cfg)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: %v\n", err)
 		return 1
 	}
+	keep, retention, _ := backupPolicy(*reason, rcfg)
 
 	// The world PVC is mounted directly at worldsRoot; the resolver returns it for
 	// any target, exactly as in cmdRestore. This is the same TarLocal the reaper
@@ -72,13 +81,25 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 
 	ctx := ctrl.SetupSignalHandler()
 
-	ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
+	// The archive store shares the node's disk with every world and the
+	// database: an owner's backup must not be what tips it into eviction.
+	if err := backup.CheckRoom(cfg.Archive.LocalPath, *worldsRoot, backup.MinFreeAfter); err != nil {
+		fmt.Fprintf(stderr, "felis backup: %v\n", err)
+		return 1
+	}
+
+	a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
 		return 1
 	}
+	ref, size := a.Ref, a.Size
+	if len(a.Skipped) > 0 {
+		fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",
+			len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))
+	}
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	drv, err := openPodStore(ctx, cfg.Database.URL, "backup", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: open database: %v\n", err)
 		return 1
@@ -91,10 +112,14 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		FormerOwner: *formerOwner,
 		BackupRef:   string(ref),
 		SizeBytes:   size,
-		Reason:      "manual",
-		ExpiresAt:   time.Now().Add(rcfg.Retention),
+		Reason:      *reason,
+		ExpiresAt:   time.Now().Add(retention),
+
+		SHA256:         a.SHA256,
+		SkippedEntries: len(a.Skipped),
 	}
-	if err := reaper.NewPGStore(drv.DB()).InsertBackup(ctx, rec); err != nil {
+	st := reaper.NewPGStore(drv.DB())
+	if err := st.InsertBackup(ctx, rec); err != nil {
 		// The archive is written but unrecorded — an orphan the retention pass would
 		// never expire. Delete it so a failed backup leaves no leaked bytes, mirroring
 		// the reaper's archive-then-record atomicity.
@@ -107,9 +132,61 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	}
 
 	fmt.Fprintf(stdout, "felis backup: server=%s archived %d bytes to %s (backup %s)\n", *server, size, ref, rec.ID)
+	pruneBackups(ctx, st, archiver, *server, *formerOwner, *reason, keep, *protect, stdout, stderr)
 	return 0
 }
 
+const (
+	reasonManual = "manual"
+	// preRestoreKeep is how many safety snapshots a server keeps: enough to walk
+	// back a couple of restores in a row, without every restore adding a world's
+	// worth of bytes for the full manual retention.
+	preRestoreKeep = 3
+)
+
+// backupPolicy is how many backups of one reason a server keeps and how long
+// each lives: an owner's own backups and the safety snapshots in front of a
+// restore by [archive] manual_keep / manual_retention (the snapshots capped at
+// preRestoreKeep), felis-api's daily restore points by scheduled_keep /
+// scheduled_retention, so neither kind crowds out the other. ok is false for a
+// reason this command does not record.
+func backupPolicy(reason string, rcfg reaper.Config) (keep int, retention time.Duration, ok bool) {
+	switch reason {
+	case reasonManual:
+		return rcfg.ManualKeep, rcfg.ManualRetention, true
+	case backupjob.ReasonPreRestore:
+		return preRestoreKeep, rcfg.ManualRetention, true
+	case backupjob.ReasonScheduled:
+		return rcfg.ScheduledKeep, rcfg.ScheduledRetention, true
+	}
+	return 0, 0, false
+}
+
+// pruneBackups keeps the newest keep backups of this reason that owner holds of
+// server and removes the rest, oldest first, so repeated backups of one world
+// cannot fill the shared archive store and a new owner's backups never remove a
+// previous owner's. protect is never removed: it is the backup a chained restore
+// is about to extract. The new backup is already recorded; a removal that fails
+// is reported and retried after the next backup.
+func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) {
+	excess, err := st.ExcessBackups(ctx, server, owner, reason, keep, protect)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)
+		return
+	}
+	for _, b := range excess {
+		if err := archiver.Delete(ctx, backup.ArchiveRef(b.BackupRef)); err != nil {
+			fmt.Fprintf(stderr, "felis backup: remove older backup %s: %v\n", b.ID, err)
+			continue
+		}
+		if err := st.MarkBackupDeleted(ctx, b.ID, time.Now()); err != nil {
+			fmt.Fprintf(stderr, "felis backup: record the removal of %s: %v\n", b.ID, err)
+			continue
+		}
+		fmt.Fprintf(stdout, "felis backup: removed older %s backup %s of %s (keeping the newest %d)\n", reason, b.ID, server, keep)
+	}
+}
+
 // newBackupID mints a world_backups primary key, matching the reaper's "bk-"+hex
 // scheme so a manual and an inactivity backup are indistinguishable downstream.
 func newBackupID() string {

cmd/felis/backup_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/reaper"
+)
+
+// The reason decides which backups the new one's prune may remove, so an
+// unknown one is refused before anything is archived.
+func TestBackupSubcommandRejectsUnknownReason(t *testing.T) {
+	var stderr bytes.Buffer
+	if code := cmdBackup([]string{"--server", "survival", "--reason", "inactive_15d"}, &bytes.Buffer{}, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2 (%s)", code, stderr.String())
+	}
+	if !strings.Contains(stderr.String(), "unknown --reason") {
+		t.Fatalf("stderr = %q", stderr.String())
+	}
+}
+
+// Each reason is pruned and expired by its own [archive] keys: a daily
+// restore point must never count against, or take the lifetime of, the
+// backups an owner asked for.
+func TestBackupPolicyPerReason(t *testing.T) {
+	rcfg := reaper.DefaultConfig()
+	rcfg.ManualKeep, rcfg.ManualRetention = 5, 30*reaper.Day
+	rcfg.ScheduledKeep, rcfg.ScheduledRetention = 7, 90*reaper.Day
+	for _, tc := range []struct {
+		reason    string
+		keep      int
+		retention time.Duration
+	}{
+		{"manual", 5, 30 * reaper.Day},
+		{"pre_restore", preRestoreKeep, 30 * reaper.Day},
+		{"scheduled", 7, 90 * reaper.Day},
+	} {
+		keep, retention, ok := backupPolicy(tc.reason, rcfg)
+		if !ok || keep != tc.keep || retention != tc.retention {
+			t.Errorf("backupPolicy(%q) = (%d, %v, %v); want (%d, %v, true)", tc.reason, keep, retention, ok, tc.keep, tc.retention)
+		}
+	}
+	if _, _, ok := backupPolicy("inactive_15d", rcfg); ok {
+		t.Error("backupPolicy accepted inactive_15d; the reaper records those itself")
+	}
+}

cmd/felis/backupnow.go

@@ -4,15 +4,28 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"os/signal"
+	"sort"
+	"strings"
+	"syscall"
 	"time"
 
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/operator"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/store"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -20,9 +33,11 @@ import (
 // backupnow is the break-glass "back up a world now" op (§B4 "Sync"). Unlike halt —
 // which writes the CRD directly — a backup needs felis-api's deployment coordinates
 // (FELIS_IMAGE / FELIS_BACKUP_PVC) to render the one-shot backup Job, so the console
-// cannot do it in-process. It POSTs the felis-api INTERNAL face (service-token auth)
+// cannot do it in-process. It POSTs the felis-api INTERNAL face (ops-token auth)
 // while the API is alive, and the API renders the Job and audits the action. This file
-// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue.
+// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue. The
+// `felis backup-now` command (cmdBackupNow, below) takes the same route for every
+// user server in turn.
 
 // backupNowOutcome is the durable result of a backup request, re-printed after the TUI
 // alt-screen tears down.
@@ -33,7 +48,7 @@ type backupNowOutcome struct {
 
 // resolveInternalAPI reads the two things the on-node console needs to reach the
 // felis-api internal face: the felis-api-internal Service ClusterIP (the host's
-// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the service
+// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the ops
 // token. Both live in the control namespace.
 func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace string) (baseURL, token string, err error) {
 	var svc corev1.Service
@@ -45,13 +60,14 @@ func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace
 		return "", "", fmt.Errorf("%s Service has no ClusterIP yet", platform.APIInternalServiceName)
 	}
 
+	// The console's own token, which the api serves on the backup route alone.
 	var sec corev1.Secret
-	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.ServiceTokenSecretName}, &sec); err != nil {
-		return "", "", fmt.Errorf("get %s Secret: %w", naming.ServiceTokenSecretName, err)
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.OpsTokenSecretName}, &sec); err != nil {
+		return "", "", fmt.Errorf("get %s Secret (re-run the installer to create it): %w", naming.OpsTokenSecretName, err)
 	}
 	token = string(sec.Data[naming.ServiceTokenSecretKey])
 	if token == "" {
-		return "", "", fmt.Errorf("secret %s has no %s key", naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey)
+		return "", "", fmt.Errorf("secret %s has no %s key", naming.OpsTokenSecretName, naming.ServiceTokenSecretKey)
 	}
 
 	return fmt.Sprintf("http://%s:%d", ip, platform.APIInternalPort), token, nil
@@ -74,7 +90,7 @@ func requestBackup(ctx context.Context, hc *http.Client, baseURL, token, name, o
 
 	resp, err := hc.Do(req)
 	if err != nil {
-		return backupNowOutcome{}, fmt.Errorf("felis-api unreachable (a backup needs it alive): %w", err)
+		return backupNowOutcome{}, fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
 	}
 	defer resp.Body.Close()
 
@@ -86,23 +102,31 @@ func requestBackup(ctx context.Context, hc *http.Client, baseURL, token, name, o
 
 // backupErrorFromResponse turns a non-202 into a human message. The well-known codes get
 // an operator-facing explanation; anything else falls back to the API's
-// {"error":{message}} body, then the bare status code.
+// {"error":{code,message}} body, then the bare status code.
 func backupErrorFromResponse(resp *http.Response) error {
-	switch resp.StatusCode {
-	case http.StatusConflict: // not_stopped
-		return fmt.Errorf("the server must be stopped before its world can be backed up — halt it first")
-	case http.StatusServiceUnavailable: // backup_unavailable
-		return fmt.Errorf("the backup subsystem is not configured on felis-api (FELIS_IMAGE / FELIS_BACKUP_PVC unset)")
-	case http.StatusNotFound:
-		return fmt.Errorf("no such server")
-	}
 	var e struct {
 		Error struct {
+			Code    string `json:"code"`
 			Message string `json:"message"`
 		} `json:"error"`
 	}
 	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
 	_ = json.Unmarshal(raw, &e)
+
+	switch resp.StatusCode {
+	case http.StatusConflict:
+		// Two refusals share 409: the stopped gate and the missing-world-volume
+		// gate. The body's code distinguishes them; a code-less body reads as the
+		// stopped gate (the only 409 before the volume gate existed), and any other
+		// coded 409 falls through to the API's own operator text.
+		if e.Error.Code == "" || e.Error.Code == "not_stopped" {
+			return fmt.Errorf("the server must be stopped before its world can be backed up — halt it first")
+		}
+	case http.StatusServiceUnavailable: // backup_unavailable
+		return fmt.Errorf("the backup subsystem is not configured on felis-api (FELIS_IMAGE / FELIS_BACKUP_PVC unset)")
+	case http.StatusNotFound:
+		return fmt.Errorf("no such server")
+	}
 	if e.Error.Message != "" {
 		return fmt.Errorf("felis-api: %s", e.Error.Message)
 	}
@@ -120,3 +144,455 @@ func performBackupNow(ctx context.Context, cl client.Client, controlNamespace, n
 	hc := &http.Client{Timeout: 10 * time.Second}
 	return requestBackup(ctx, hc, baseURL, token, name, osUser)
 }
+
+// backupPickable narrows the backup picker to servers the backup API can accept.
+// System servers (login/lobby) are excluded: they have no row in the servers
+// table and carry reserved names, so every attempt dies in name validation —
+// offering them would be a dead pick. The halt picker keeps them on purpose
+// (break-glass retains full power over system servers); only the API-backed
+// backup op cannot reach them.
+func backupPickable(servers []haltableServer) []haltableServer {
+	out := make([]haltableServer, 0, len(servers))
+	for _, s := range servers {
+		if !s.system {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+// cmdBackupNow is `felis backup-now`: the world of every user server (or of the
+// named ones) archived now, one at a time, through the internal backup route the
+// console's Sync uses. A world lives only in its volume and the off-site copy holds
+// only its archives, so this is the lever in front of a planned move to another
+// host, a disk swap or anything else that could lose a volume.
+func cmdBackupNow(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("backup-now", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	yes := fs.Bool("yes", false, "back up; without it the plan is printed and nothing changes")
+	stop := fs.Bool("stop", false, "stop the running servers first: their players are disconnected and the servers stay stopped")
+	fs.Usage = func() {
+		fmt.Fprintln(stderr, "Usage: felis backup-now [-yes] [-stop] [server ...]")
+		fmt.Fprintln(stderr)
+		fmt.Fprintln(stderr, "Archives the world of every user server, or of the named ones, one at a time, and waits for each archive.")
+		fmt.Fprintln(stderr, "A running server is skipped unless -stop is given. Without -yes it prints what it would do.")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis backup-now: refused — it reads the cluster's ops token, so it must run as root (try: sudo felis backup-now)")
+		return 1
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
+		return 1
+	}
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer cancel()
+
+	ns := cfg.K8s.Namespace
+	osUser := accountableOSUser()
+	jobs := api.NewK8sJobStatus(cl, ns)
+	var baseURL, token string
+	var repo ownerStore
+	var drv *store.PostgresDriver
+	defer func() {
+		if drv != nil {
+			_ = drv.Close()
+		}
+	}()
+	hc := &http.Client{Timeout: 10 * time.Second}
+	r := backupNowRun{
+		out:  stdout,
+		errw: stderr,
+		ns:   ns,
+		list: func(ctx context.Context) ([]backupNowWorld, error) { return listBackupNowWorlds(ctx, cl, ns) },
+		stopped: func(ctx context.Context, name string) (bool, error) {
+			var ms v1alpha1.MinecraftServer
+			if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &ms); err != nil {
+				return false, err
+			}
+			return backupNowStopped(ctx, cl, &ms)
+		},
+		halt: func(ctx context.Context, name string) error {
+			// The database is opened only once a server is to be stopped: the halt
+			// is audited like the console's, and a run with nothing running needs
+			// no more than the API.
+			if repo == nil {
+				d, err := store.Open(ctx, cfg.Database.URL)
+				if err != nil {
+					return fmt.Errorf("open the database for the audit log: %w", err)
+				}
+				drv, repo = d, api.NewPGRepo(d.DB())
+			}
+			out, err := performHalt(ctx, cl, repo, ns, name, osUser)
+			if err != nil {
+				return err
+			}
+			if out.auditErr != nil {
+				fmt.Fprintf(stderr, "felis backup-now: the audit row for stopping %s was not written: %v\n", name, out.auditErr)
+			}
+			return nil
+		},
+		request: func(ctx context.Context, name string) error {
+			if baseURL == "" {
+				var err error
+				if baseURL, token, err = resolveInternalAPI(ctx, cl, platform.DefaultControlNamespace); err != nil {
+					return fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
+				}
+			}
+			_, err := requestBackup(ctx, hc, baseURL, token, name, osUser)
+			return err
+		},
+		jobs:  jobs.LatestJobs,
+		now:   time.Now,
+		sleep: func(ctx context.Context, d time.Duration) { sleepCtx(ctx, d) },
+	}
+	return r.run(ctx, fs.Args(), *yes, *stop)
+}
+
+// sleepCtx waits d or until ctx ends.
+func sleepCtx(ctx context.Context, d time.Duration) {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+	case <-t.C:
+	}
+}
+
+// backupNowWorld is one user server as backup-now sees it.
+type backupNowWorld struct {
+	name     string
+	phase    string // the observed phase, or the desired state before the operator reconciled it
+	stopped  bool   // the backup route's stopped gate admits it
+	hasWorld bool   // its world volume exists
+}
+
+// listBackupNowWorlds lists the user servers of namespace in the API server's
+// order (by name), each with what the backup route checks. System servers are
+// left out: they have no row in the servers table, so the route refuses them
+// (backupPickable).
+func listBackupNowWorlds(ctx context.Context, cl client.Client, namespace string) ([]backupNowWorld, error) {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return nil, err
+	}
+	var out []backupNowWorld
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if isSystemServer(ms.Name) {
+			continue
+		}
+		stopped, err := backupNowStopped(ctx, cl, ms)
+		if err != nil {
+			return nil, err
+		}
+		var pvc corev1.PersistentVolumeClaim
+		err = cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: naming.WorldPVCName(ms.Name)}, &pvc)
+		if err != nil && !apierrors.IsNotFound(err) {
+			return nil, fmt.Errorf("look up the world volume of %s: %w", ms.Name, err)
+		}
+		phase := string(ms.Status.Phase)
+		if phase == "" {
+			phase = string(ms.Spec.DesiredState)
+		}
+		out = append(out, backupNowWorld{name: ms.Name, phase: phase, stopped: stopped, hasWorld: err == nil})
+	}
+	return out, nil
+}
+
+// backupNowStopped is the backup route's stopped gate (api.enqueueBackup and
+// K8sCluster.AcquireMaintenance together): desired Stopped, not ready, phase
+// Stopped and no game pod left.
+func backupNowStopped(ctx context.Context, cl client.Client, ms *v1alpha1.MinecraftServer) (bool, error) {
+	if ms.Spec.DesiredState != v1alpha1.DesiredStopped || ms.Status.Ready || ms.Status.Phase != v1alpha1.PhaseStopped {
+		return false, nil
+	}
+	var pods corev1.PodList
+	if err := cl.List(ctx, &pods, client.InNamespace(ms.Namespace), client.MatchingLabels{
+		v1alpha1.LabelServer: ms.Name, v1alpha1.LabelComponent: operator.ComponentValue,
+	}); err != nil {
+		return false, fmt.Errorf("look up the pod of %s: %w", ms.Name, err)
+	}
+	return len(pods.Items) == 0, nil
+}
+
+// errBackupAPIUnreachable is a backup request that never reached felis-api. It
+// ends a backup-now run: every later world would fail the same way, and stopping
+// servers for backups that cannot be taken only takes them away from players.
+var errBackupAPIUnreachable = errors.New("felis-api unreachable (a backup needs it alive)")
+
+// Polling of backup-now. A graceful stop saves the world first; the backup Job's
+// own deadline (backupjob, 30 minutes) ends a Job that hangs, so its wait needs no
+// cap of its own.
+const (
+	backupNowPoll      = 2 * time.Second
+	backupNowStopWait  = 10 * time.Minute
+	backupNowJobAppear = time.Minute
+)
+
+// backupNowRun is backup-now over seams, so the plan and the run are tested
+// without a cluster or felis-api.
+type backupNowRun struct {
+	out, errw io.Writer
+	ns        string // where the servers and their Jobs live, for the kubectl hints
+	list      func(ctx context.Context) ([]backupNowWorld, error)
+	stopped   func(ctx context.Context, name string) (bool, error)
+	halt      func(ctx context.Context, name string) error
+	request   func(ctx context.Context, name string) error
+	jobs      func(ctx context.Context, name string) ([]api.AsyncJob, error)
+	now       func() time.Time
+	sleep     func(ctx context.Context, d time.Duration)
+}
+
+// pickBackupNowWorlds narrows worlds to names, in the order given, or keeps them
+// all when names is empty.
+func pickBackupNowWorlds(worlds []backupNowWorld, names []string) ([]backupNowWorld, error) {
+	if len(names) == 0 {
+		return worlds, nil
+	}
+	byName := make(map[string]backupNowWorld, len(worlds))
+	for _, w := range worlds {
+		byName[w.name] = w
+	}
+	var out []backupNowWorld
+	seen := map[string]bool{}
+	for _, n := range names {
+		if seen[n] {
+			continue
+		}
+		seen[n] = true
+		w, ok := byName[n]
+		switch {
+		case ok:
+			out = append(out, w)
+		case isSystemServer(n):
+			return nil, fmt.Errorf("%s is a system server: its world is rebuilt by felis setup and has no backups", n)
+		default:
+			return nil, fmt.Errorf("no server named %q", n)
+		}
+	}
+	return out, nil
+}
+
+// backupNowAction is what the plan does with a world.
+func backupNowAction(w backupNowWorld, stop bool) string {
+	switch {
+	case w.stopped && !w.hasWorld:
+		return "skip: no world volume (never started, nothing to save)"
+	case w.stopped:
+		return "back up"
+	case stop:
+		return "stop, then back up"
+	default:
+		return "skip: running (stop it first, or pass -stop)"
+	}
+}
+
+func (r *backupNowRun) run(ctx context.Context, names []string, yes, stop bool) int {
+	all, err := r.list(ctx)
+	if err != nil {
+		fmt.Fprintf(r.errw, "felis backup-now: list the servers: %v\n", err)
+		return 1
+	}
+	worlds, err := pickBackupNowWorlds(all, names)
+	if err != nil {
+		fmt.Fprintf(r.errw, "felis backup-now: %v\n", err)
+		return 2
+	}
+	if len(worlds) == 0 {
+		fmt.Fprintln(r.out, "felis backup-now: there are no user servers")
+		return 0
+	}
+
+	// The stopped worlds go first, so a felis-api that cannot take a backup is
+	// found before any server is stopped for one.
+	sort.SliceStable(worlds, func(i, j int) bool { return worlds[i].stopped && !worlds[j].stopped })
+	width := 0
+	for _, w := range worlds {
+		width = max(width, len(w.name))
+	}
+	fmt.Fprintf(r.out, "felis backup-now: %d server(s), backed up one at a time:\n", len(worlds))
+	stopping, work := false, 0
+	for _, w := range worlds {
+		fmt.Fprintf(r.out, "  %-*s  %-8s  %s\n", width, w.name, w.phase, backupNowAction(w, stop))
+		stopping = stopping || (!w.stopped && stop)
+		if (w.stopped && w.hasWorld) || (!w.stopped && stop) {
+			work++
+		}
+	}
+	if work > 0 {
+		fmt.Fprintln(r.out, "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.")
+	}
+	if stopping {
+		fmt.Fprintln(r.out, "Stopping disconnects the players on those servers, and they stay stopped afterwards.")
+	}
+	if !yes {
+		if work == 0 {
+			fmt.Fprintln(r.out, "Nothing to back up.")
+		} else {
+			fmt.Fprintln(r.out, "Nothing changed. Run again with -yes to back them up.")
+		}
+		return 0
+	}
+
+	var done, failed, running, empty int
+	var leftStopped []string
+	for _, w := range worlds {
+		if err := ctx.Err(); err != nil {
+			break
+		}
+		switch {
+		case w.stopped && !w.hasWorld:
+			empty++
+			continue
+		case !w.stopped && !stop:
+			running++
+			continue
+		}
+		if !w.stopped {
+			halted, err := r.stopWorld(ctx, w.name)
+			if halted {
+				leftStopped = append(leftStopped, w.name)
+			}
+			if err != nil {
+				fmt.Fprintf(r.out, "  %s: failed: %v\n", w.name, err)
+				failed++
+				continue
+			}
+		}
+		err := r.backUp(ctx, w.name)
+		if errors.Is(err, errBackupAPIUnreachable) {
+			fmt.Fprintf(r.out, "  %s: failed: %v\n", w.name, err)
+			fmt.Fprintln(r.out, "Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).")
+			failed++
+			break
+		}
+		if err != nil {
+			fmt.Fprintf(r.out, "  %s: failed: %v\n", w.name, err)
+			failed++
+			continue
+		}
+		done++
+	}
+
+	interrupted := ctx.Err() != nil
+	if interrupted {
+		fmt.Fprintln(r.out, "Interrupted: a backup Job already started runs to its end.")
+	}
+	parts := []string{fmt.Sprintf("%d backed up", done)}
+	if failed > 0 {
+		parts = append(parts, fmt.Sprintf("%d failed", failed))
+	}
+	if running > 0 {
+		parts = append(parts, fmt.Sprintf("%d skipped (running)", running))
+	}
+	if empty > 0 {
+		parts = append(parts, fmt.Sprintf("%d without a world", empty))
+	}
+	fmt.Fprintf(r.out, "%s.\n", strings.Join(parts, ", "))
+	if len(leftStopped) > 0 {
+		fmt.Fprintf(r.out, "Left stopped: %s. Start them from the panel when you are done.\n", strings.Join(leftStopped, ", "))
+	}
+	if done > 0 {
+		fmt.Fprintln(r.out, "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.")
+	}
+	if failed > 0 || running > 0 || interrupted {
+		return 1
+	}
+	return 0
+}
+
+// stopWorld stops one server and waits until the backup route would admit it.
+// halted is whether the stop was asked for: the server then stays stopped, even
+// when it takes longer than the wait.
+func (r *backupNowRun) stopWorld(ctx context.Context, name string) (halted bool, err error) {
+	fmt.Fprintf(r.out, "  %s: stopping\n", name)
+	if err := r.halt(ctx, name); err != nil {
+		return false, err
+	}
+	start := r.now()
+	for {
+		ok, err := r.stopped(ctx, name)
+		if err != nil {
+			return true, err
+		}
+		if ok {
+			fmt.Fprintf(r.out, "  %s: stopped after %s\n", name, r.now().Sub(start).Round(time.Second))
+			return true, nil
+		}
+		if r.now().Sub(start) >= backupNowStopWait {
+			return true, fmt.Errorf("did not stop within %s (kubectl -n %s describe minecraftserver %s)", backupNowStopWait, r.ns, name)
+		}
+		if err := ctx.Err(); err != nil {
+			return true, err
+		}
+		r.sleep(ctx, backupNowPoll)
+	}
+}
+
+// backUp requests one world's backup and waits for its Job to finish. The Job is
+// the one of this server that was not there before the request.
+func (r *backupNowRun) backUp(ctx context.Context, name string) error {
+	before, err := r.jobs(ctx, name)
+	if err != nil {
+		return fmt.Errorf("list its backup Jobs: %w", err)
+	}
+	known := make(map[string]bool, len(before))
+	for _, j := range before {
+		known[j.Name] = true
+	}
+	if err := r.request(ctx, name); err != nil {
+		return err
+	}
+	fmt.Fprintf(r.out, "  %s: backing up\n", name)
+	start := r.now()
+	seen := ""
+	for {
+		jobs, err := r.jobs(ctx, name)
+		if err != nil {
+			return fmt.Errorf("list its backup Jobs: %w", err)
+		}
+		var job *api.AsyncJob
+		for i := range jobs {
+			if jobs[i].Kind == "backup" && (jobs[i].Name == seen || seen == "" && !known[jobs[i].Name]) {
+				job = &jobs[i]
+				break
+			}
+		}
+		switch {
+		case job == nil && seen != "":
+			return fmt.Errorf("its backup Job %s was deleted before it finished", seen)
+		case job == nil && r.now().Sub(start) >= backupNowJobAppear:
+			return fmt.Errorf("felis-api accepted the backup, but no backup Job appeared within %s (kubectl -n %s get jobs)", backupNowJobAppear, r.ns)
+		case job != nil && job.State == "succeeded":
+			fmt.Fprintf(r.out, "  %s: archived in %s\n", name, r.now().Sub(start).Round(time.Second))
+			return nil
+		case job != nil && job.State == "failed":
+			msg := job.Message
+			if msg == "" {
+				msg = "the backup Job failed"
+			}
+			return fmt.Errorf("%s (kubectl -n %s logs job/%s)", msg, r.ns, job.Name)
+		case job != nil:
+			seen = job.Name
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		r.sleep(ctx, backupNowPoll)
+	}
+}

cmd/felis/backupnow_cmd_test.go

@@ -0,0 +1,563 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// bnRig drives backupNowRun against scripted servers and Jobs on a fake clock that
+// moves only when the run sleeps.
+type bnRig struct {
+	out, errw bytes.Buffer
+	clock     time.Time
+	events    []string
+	worlds    []backupNowWorld
+	// stopAfter is how many polls a halted server takes to stop; -1 never does.
+	stopAfter map[string]int
+	polls     map[string]int
+	reqErr    map[string]error
+	// states is what the server's new backup Job reports on each poll after the
+	// request, the last one repeating; "" is no Job.
+	states   map[string][]string
+	messages map[string]string
+	jobPolls map[string]int
+	// stopErr fails a server's stop polls; jobsFail fails the nth (1-based) Job
+	// list of a server.
+	stopErr  map[string]error
+	jobsFail map[string]int
+	jobCalls map[string]int
+	// ctx is the run's context, and onAct runs after each halt and request.
+	ctx   context.Context
+	onAct func(event string)
+}
+
+func newBNRig(worlds ...backupNowWorld) *bnRig {
+	return &bnRig{
+		clock:     time.Unix(1_800_000_000, 0),
+		worlds:    worlds,
+		stopAfter: map[string]int{},
+		polls:     map[string]int{},
+		reqErr:    map[string]error{},
+		states:    map[string][]string{},
+		messages:  map[string]string{},
+		jobPolls:  map[string]int{},
+		stopErr:   map[string]error{},
+		jobsFail:  map[string]int{},
+		jobCalls:  map[string]int{},
+		ctx:       context.Background(),
+		onAct:     func(string) {},
+	}
+}
+
+func (g *bnRig) run(names []string, yes, stop bool) int {
+	r := backupNowRun{
+		out: &g.out, errw: &g.errw, ns: "minecraft",
+		list: func(context.Context) ([]backupNowWorld, error) {
+			return append([]backupNowWorld(nil), g.worlds...), nil
+		},
+		stopped: func(_ context.Context, name string) (bool, error) {
+			g.polls[name]++
+			if err := g.stopErr[name]; err != nil {
+				return false, err
+			}
+			n := g.stopAfter[name]
+			return n >= 0 && g.polls[name] > n, nil
+		},
+		halt: func(_ context.Context, name string) error {
+			g.events = append(g.events, "halt "+name)
+			g.onAct("halt " + name)
+			return nil
+		},
+		request: func(_ context.Context, name string) error {
+			g.events = append(g.events, "request "+name)
+			g.onAct("request " + name)
+			if err := g.reqErr[name]; err != nil {
+				return err
+			}
+			g.jobPolls[name] = 0
+			return nil
+		},
+		jobs: func(_ context.Context, name string) ([]api.AsyncJob, error) {
+			// Every server has an older finished backup, and a restore Job that
+			// shows up with the new backup: neither is the Job to wait for.
+			g.jobCalls[name]++
+			if g.jobCalls[name] == g.jobsFail[name] {
+				return nil, errors.New("the apiserver is gone")
+			}
+			out := []api.AsyncJob{{Name: "backup-" + name + "-old", Kind: "backup", State: "succeeded"}}
+			n, requested := g.jobPolls[name]
+			if !requested {
+				return out, nil
+			}
+			g.jobPolls[name] = n + 1
+			states := g.states[name]
+			if len(states) == 0 {
+				states = []string{"succeeded"}
+			}
+			state := states[min(n, len(states)-1)]
+			if state == "" {
+				return out, nil
+			}
+			return append([]api.AsyncJob{
+				{Name: "restore-" + name + "-x", Kind: "restore", State: "succeeded"},
+				{Name: "backup-" + name + "-new", Kind: "backup", State: state, Message: g.messages[name]},
+			}, out...), nil
+		},
+		now:   func() time.Time { return g.clock },
+		sleep: func(_ context.Context, d time.Duration) { g.clock = g.clock.Add(d) },
+	}
+	return r.run(g.ctx, names, yes, stop)
+}
+
+func stoppedWorld(name string) backupNowWorld {
+	return backupNowWorld{name: name, phase: "Stopped", stopped: true, hasWorld: true}
+}
+
+func runningWorld(name string) backupNowWorld {
+	return backupNowWorld{name: name, phase: "Running", hasWorld: true}
+}
+
+func emptyWorld(name string) backupNowWorld {
+	return backupNowWorld{name: name, phase: "Stopped", stopped: true}
+}
+
+const (
+	bnManualKeep = "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.\n"
+	bnStopping   = "Stopping disconnects the players on those servers, and they stay stopped afterwards.\n"
+	bnOffsite    = "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.\n"
+)
+
+func TestBackupNowPlanChangesNothing(t *testing.T) {
+	for _, tc := range []struct {
+		stop bool
+		want string
+	}{
+		{false, "felis backup-now: 4 server(s), backed up one at a time:\n" +
+			"  alpha    Stopped   back up\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   skip: running (stop it first, or pass -stop)\n" +
+			"  delta    Starting  skip: running (stop it first, or pass -stop)\n" +
+			bnManualKeep +
+			"Nothing changed. Run again with -yes to back them up.\n"},
+		{true, "felis backup-now: 4 server(s), backed up one at a time:\n" +
+			"  alpha    Stopped   back up\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   stop, then back up\n" +
+			"  delta    Starting  stop, then back up\n" +
+			bnManualKeep + bnStopping +
+			"Nothing changed. Run again with -yes to back them up.\n"},
+	} {
+		t.Run(fmt.Sprintf("stop=%v", tc.stop), func(t *testing.T) {
+			delta := runningWorld("delta")
+			delta.phase = "Starting"
+			g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), delta)
+			if code := g.run(nil, false, tc.stop); code != 0 {
+				t.Fatalf("exit = %d, want 0; stderr %q", code, g.errw.String())
+			}
+			if g.out.String() != tc.want {
+				t.Fatalf("plan =\n%s\nwant\n%s", g.out.String(), tc.want)
+			}
+			if len(g.events) != 0 || len(g.polls) != 0 {
+				t.Fatalf("the plan acted: events %v, polls %v", g.events, g.polls)
+			}
+		})
+	}
+}
+
+// A plan that saves nothing says so, without the manual_keep warning; a running
+// server counts as something to save once -stop is given.
+func TestBackupNowPlanWithNothingToSave(t *testing.T) {
+	for _, tc := range []struct {
+		stop bool
+		want string
+	}{
+		{false, "felis backup-now: 2 server(s), backed up one at a time:\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   skip: running (stop it first, or pass -stop)\n" +
+			"Nothing to back up.\n"},
+		{true, "felis backup-now: 2 server(s), backed up one at a time:\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   stop, then back up\n" +
+			bnManualKeep + bnStopping +
+			"Nothing changed. Run again with -yes to back them up.\n"},
+	} {
+		g := newBNRig(runningWorld("bravo"), emptyWorld("charlie"))
+		if code := g.run(nil, false, tc.stop); code != 0 {
+			t.Fatalf("stop=%v: exit = %d, want 0", tc.stop, code)
+		}
+		if g.out.String() != tc.want {
+			t.Fatalf("stop=%v: plan =\n%s\nwant\n%s", tc.stop, g.out.String(), tc.want)
+		}
+	}
+}
+
+func TestBackupNowBacksUpEachWorldInTurn(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), stoppedWorld("delta"))
+	g.states["alpha"] = []string{"", "running", "succeeded"}
+	g.states["delta"] = []string{"running", "failed"}
+	g.messages["delta"] = "felis backup: not enough free disk for the archive"
+	g.stopAfter["bravo"] = 2
+	g.states["bravo"] = []string{"running", "running", "running", "succeeded"}
+
+	code := g.run(nil, true, true)
+	if code != 1 {
+		t.Fatalf("exit = %d, want 1 (delta failed)", code)
+	}
+	if want := []string{"request alpha", "request delta", "halt bravo", "request bravo"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping+"")
+	want := "  alpha: backing up\n" +
+		"  alpha: archived in 4s\n" +
+		"  delta: backing up\n" +
+		"  delta: failed: felis backup: not enough free disk for the archive (kubectl -n minecraft logs job/backup-delta-new)\n" +
+		"  bravo: stopping\n" +
+		"  bravo: stopped after 4s\n" +
+		"  bravo: backing up\n" +
+		"  bravo: archived in 6s\n" +
+		"2 backed up, 1 failed, 1 without a world.\n" +
+		"Left stopped: bravo. Start them from the panel when you are done.\n" +
+		bnOffsite
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+}
+
+func TestBackupNowSkipsRunningServersWithoutStop(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"))
+	if code := g.run(nil, true, false); code != 1 {
+		t.Fatalf("exit = %d, want 1 (bravo was not backed up)", code)
+	}
+	if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	if len(g.polls) != 0 {
+		t.Fatalf("polled a server it did not stop: %v", g.polls)
+	}
+	_, run, _ := strings.Cut(g.out.String(), "Nothing changed")
+	if run != "" {
+		t.Fatalf("-yes printed the plan's closing line")
+	}
+	_, run, _ = strings.Cut(g.out.String(), bnManualKeep)
+	want := "  alpha: backing up\n  alpha: archived in 0s\n1 backed up, 1 skipped (running).\n" + bnOffsite
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+}
+
+func TestBackupNowExitsCleanWhenEverythingIsSaved(t *testing.T) {
+	// -stop with nothing running stops nothing and says nothing about stopping.
+	g := newBNRig(stoppedWorld("alpha"), emptyWorld("charlie"))
+	if code := g.run(nil, true, true); code != 0 {
+		t.Fatalf("exit = %d, want 0; output\n%s", code, g.out.String())
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
+	if want := "  alpha: backing up\n  alpha: archived in 0s\n1 backed up, 1 without a world.\n" + bnOffsite; run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+
+	g = newBNRig(emptyWorld("charlie"))
+	if code := g.run(nil, true, false); code != 0 {
+		t.Fatalf("exit = %d, want 0 for a server with nothing to save", code)
+	}
+	// Nothing to archive: no manual_keep warning, and no off-site hint.
+	if want := "felis backup-now: 1 server(s), backed up one at a time:\n" +
+		"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+		"0 backed up, 1 without a world.\n"; g.out.String() != want {
+		t.Fatalf("output =\n%s\nwant\n%s", g.out.String(), want)
+	}
+	if len(g.events) != 0 {
+		t.Fatalf("events = %v, want none", g.events)
+	}
+}
+
+func TestBackupNowStopsAtAnUnreachableAPI(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), runningWorld("charlie"))
+	g.reqErr["alpha"] = fmt.Errorf("%w: dial tcp 10.43.0.9:8081: connect: connection refused", errBackupAPIUnreachable)
+	if code := g.run(nil, true, true); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v: nothing after the API proved unreachable, and no server stopped", g.events, want)
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping)
+	want := "  alpha: failed: felis-api unreachable (a backup needs it alive): dial tcp 10.43.0.9:8081: connect: connection refused\n" +
+		"Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).\n" +
+		"0 backed up, 1 failed.\n"
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+
+	// Any other refusal is that world's alone: the run goes on.
+	g = newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
+	g.reqErr["alpha"] = errors.New("felis-api: the world is being restored")
+	if code := g.run(nil, true, false); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	if want := []string{"request alpha", "request bravo"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+}
+
+func TestBackupNowGivesUpOnAServerThatDoesNotStop(t *testing.T) {
+	g := newBNRig(runningWorld("bravo"), runningWorld("echo"))
+	g.stopAfter["bravo"] = -1
+	if code := g.run(nil, true, true); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	if want := []string{"halt bravo", "halt echo", "request echo"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	// One poll at the start and one per 2s sleep up to the 10-minute mark.
+	if g.polls["bravo"] != 301 {
+		t.Fatalf("bravo polled %d times, want 301", g.polls["bravo"])
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping)
+	want := "  bravo: stopping\n" +
+		"  bravo: failed: did not stop within 10m0s (kubectl -n minecraft describe minecraftserver bravo)\n" +
+		"  echo: stopping\n" +
+		"  echo: stopped after 0s\n" +
+		"  echo: backing up\n" +
+		"  echo: archived in 0s\n" +
+		"1 backed up, 1 failed.\n" +
+		"Left stopped: bravo, echo. Start them from the panel when you are done.\n" +
+		bnOffsite
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+}
+
+func TestBackupNowReportsAJobThatNeverRuns(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		states []string
+		polls  int
+		want   string
+	}{
+		{"never appears", []string{""}, 31,
+			"felis-api accepted the backup, but no backup Job appeared within 1m0s (kubectl -n minecraft get jobs)"},
+		{"deleted while running", []string{"running", "running", ""}, 3,
+			"its backup Job backup-alpha-new was deleted before it finished"},
+		{"fails without a message", []string{"failed"}, 1,
+			"the backup Job failed (kubectl -n minecraft logs job/backup-alpha-new)"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			g := newBNRig(stoppedWorld("alpha"))
+			g.states["alpha"] = tc.states
+			if code := g.run(nil, true, false); code != 1 {
+				t.Fatalf("exit = %d, want 1", code)
+			}
+			if !strings.Contains(g.out.String(), "  alpha: failed: "+tc.want+"\n") {
+				t.Fatalf("output =\n%s\nwant the line %q", g.out.String(), tc.want)
+			}
+			if g.jobPolls["alpha"] != tc.polls {
+				t.Fatalf("polled the Jobs %d times after the request, want %d", g.jobPolls["alpha"], tc.polls)
+			}
+		})
+	}
+}
+
+func TestBackupNowNamedServers(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), stoppedWorld("delta"))
+	if code := g.run([]string{"delta", "alpha", "delta"}, true, false); code != 0 {
+		t.Fatalf("exit = %d, want 0", code)
+	}
+	if want := []string{"request delta", "request alpha"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	if !strings.HasPrefix(g.out.String(), "felis backup-now: 2 server(s), backed up one at a time:\n  delta  Stopped   back up\n  alpha  Stopped   back up\n") {
+		t.Fatalf("plan =\n%s", g.out.String())
+	}
+
+	for _, tc := range []struct{ name, want string }{
+		{"login", "felis backup-now: login is a system server: its world is rebuilt by felis setup and has no backups\n"},
+		{"lobby", "felis backup-now: lobby is a system server: its world is rebuilt by felis setup and has no backups\n"},
+		{"nope", "felis backup-now: no server named \"nope\"\n"},
+	} {
+		g := newBNRig(stoppedWorld("alpha"))
+		if code := g.run([]string{"alpha", tc.name}, true, false); code != 2 {
+			t.Fatalf("%s: exit = %d, want 2", tc.name, code)
+		}
+		if g.errw.String() != tc.want {
+			t.Fatalf("%s: stderr = %q, want %q", tc.name, g.errw.String(), tc.want)
+		}
+		if len(g.events) != 0 || g.out.Len() != 0 {
+			t.Fatalf("%s: acted on a bad name: events %v, output %q", tc.name, g.events, g.out.String())
+		}
+	}
+
+	g = newBNRig()
+	if code := g.run(nil, true, false); code != 0 || g.out.String() != "felis backup-now: there are no user servers\n" {
+		t.Fatalf("empty fleet: exit %d, output %q", code, g.out.String())
+	}
+}
+
+// The world list mirrors the backup route's own gate, so the plan says exactly what
+// the route would refuse.
+func TestListBackupNowWorlds(t *testing.T) {
+	withStatus := func(ms *v1alpha1.MinecraftServer, ready bool) *v1alpha1.MinecraftServer {
+		ms.Status.Ready = ready
+		return ms
+	}
+	pvc := func(server, ns string) *corev1.PersistentVolumeClaim {
+		return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: naming.WorldPVCName(server), Namespace: ns}}
+	}
+	pod := func(name, ns string, labels map[string]string) *corev1.Pod {
+		return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}}
+	}
+	gameLabels := func(server string) map[string]string {
+		return map[string]string{v1alpha1.LabelServer: server, v1alpha1.LabelComponent: "server"}
+	}
+	other := mcServer("zulu", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped)
+	other.Namespace = "elsewhere"
+	hotel := mcServer("hotel", v1alpha1.DesiredStopped, "")
+	objs := []client.Object{
+		mcServer("alpha", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("alpha", haltNS),
+		// A backup Job's pod carries the server label with its own component, and
+		// a game pod of the same name in another namespace is somebody else's.
+		pod("backup-alpha-1-x", haltNS, map[string]string{v1alpha1.LabelServer: "alpha", "app.kubernetes.io/component": "world-backup"}),
+		pod("alpha-0", "elsewhere", gameLabels("alpha")),
+		withStatus(mcServer("bravo", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning), true), pvc("bravo", haltNS), pod("bravo-0", haltNS, gameLabels("bravo")),
+		mcServer("charlie", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped),
+		mcServer("delta", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("delta", haltNS), pod("delta-0", haltNS, gameLabels("delta")),
+		mcServer("echo", v1alpha1.DesiredStopped, v1alpha1.PhaseStopping), pvc("echo", haltNS),
+		mcServer("foxtrot", v1alpha1.DesiredRunning, v1alpha1.PhaseStopped), pvc("foxtrot", haltNS),
+		withStatus(mcServer("golf", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), true), pvc("golf", haltNS),
+		hotel, pvc("hotel", haltNS),
+		mcServer("login", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("login", haltNS),
+		mcServer("lobby", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("lobby", haltNS),
+		other, pvc("zulu", "elsewhere"),
+	}
+	got, err := listBackupNowWorlds(context.Background(), haltClient(t, objs...), haltNS)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []backupNowWorld{
+		{name: "alpha", phase: "Stopped", stopped: true, hasWorld: true},
+		{name: "bravo", phase: "Running", hasWorld: true},
+		{name: "charlie", phase: "Stopped", stopped: true},
+		{name: "delta", phase: "Stopped", hasWorld: true},   // its pod is still going
+		{name: "echo", phase: "Stopping", hasWorld: true},   // still stopping
+		{name: "foxtrot", phase: "Stopped", hasWorld: true}, // asked to start
+		{name: "golf", phase: "Stopped", hasWorld: true},    // still reports ready
+		{name: "hotel", phase: "Stopped", hasWorld: true},   // not reconciled yet: the desired state shows
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("worlds =\n%+v\nwant\n%+v", got, want)
+	}
+}
+
+func TestBackupNowStopsWhenInterrupted(t *testing.T) {
+	t.Run("between worlds", func(t *testing.T) {
+		g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		g.ctx = ctx
+		g.onAct = func(e string) {
+			if e == "request alpha" {
+				cancel()
+			}
+		}
+		if code := g.run(nil, true, false); code != 1 {
+			t.Fatalf("exit = %d, want 1", code)
+		}
+		if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
+			t.Fatalf("events = %v, want %v", g.events, want)
+		}
+		_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
+		want := "  alpha: backing up\n  alpha: archived in 0s\n" +
+			"Interrupted: a backup Job already started runs to its end.\n" +
+			"1 backed up.\n" + bnOffsite
+		if run != want {
+			t.Fatalf("run =\n%s\nwant\n%s", run, want)
+		}
+	})
+	t.Run("while a Job runs", func(t *testing.T) {
+		g := newBNRig(stoppedWorld("alpha"))
+		g.states["alpha"] = []string{"running"}
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		g.ctx = ctx
+		g.onAct = func(string) { cancel() }
+		if code := g.run(nil, true, false); code != 1 {
+			t.Fatalf("exit = %d, want 1", code)
+		}
+		if g.jobPolls["alpha"] != 1 {
+			t.Fatalf("polled the Job %d times after the interrupt, want 1", g.jobPolls["alpha"])
+		}
+		if !strings.Contains(g.out.String(), "  alpha: failed: context canceled\nInterrupted: a backup Job already started runs to its end.\n0 backed up, 1 failed.\n") {
+			t.Fatalf("output =\n%s", g.out.String())
+		}
+	})
+	t.Run("while a server stops", func(t *testing.T) {
+		g := newBNRig(runningWorld("bravo"))
+		g.stopAfter["bravo"] = -1
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		g.ctx = ctx
+		g.onAct = func(string) { cancel() }
+		if code := g.run(nil, true, true); code != 1 {
+			t.Fatalf("exit = %d, want 1", code)
+		}
+		if g.polls["bravo"] != 1 {
+			t.Fatalf("polled bravo %d times after the interrupt, want 1", g.polls["bravo"])
+		}
+		_, run, _ := strings.Cut(g.out.String(), bnStopping)
+		want := "  bravo: stopping\n  bravo: failed: context canceled\n" +
+			"Interrupted: a backup Job already started runs to its end.\n" +
+			"0 backed up, 1 failed.\n" +
+			"Left stopped: bravo. Start them from the panel when you are done.\n"
+		if run != want {
+			t.Fatalf("run =\n%s\nwant\n%s", run, want)
+		}
+	})
+}
+
+func TestBackupNowReportsClusterErrors(t *testing.T) {
+	g := newBNRig(runningWorld("bravo"))
+	g.stopErr["bravo"] = errors.New("the apiserver is gone")
+	if code := g.run(nil, true, true); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping)
+	// The stop was asked for, so the server stays stopped whatever the poll said.
+	want := "  bravo: stopping\n  bravo: failed: the apiserver is gone\n0 backed up, 1 failed.\n" +
+		"Left stopped: bravo. Start them from the panel when you are done.\n"
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+
+	for _, tc := range []struct {
+		call   int
+		events []string
+	}{
+		{1, nil},                       // before the request: nothing is asked for
+		{2, []string{"request alpha"}}, // the first poll after it
+	} {
+		g := newBNRig(stoppedWorld("alpha"))
+		g.jobsFail["alpha"] = tc.call
+		if code := g.run(nil, true, false); code != 1 {
+			t.Fatalf("call %d: exit = %d, want 1", tc.call, code)
+		}
+		if !reflect.DeepEqual(g.events, tc.events) {
+			t.Fatalf("call %d: events = %v, want %v", tc.call, g.events, tc.events)
+		}
+		if !strings.Contains(g.out.String(), "  alpha: failed: list its backup Jobs: the apiserver is gone\n") {
+			t.Fatalf("call %d: output =\n%s", tc.call, g.out.String())
+		}
+	}
+}

cmd/felis/backupnow_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,7 +12,6 @@ import (
 	"testing"
 	"time"
 
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 
 	corev1 "k8s.io/api/core/v1"
@@ -28,9 +28,15 @@ func internalAPIObjs(clusterIP, token string) []client.Object {
 			ObjectMeta: metav1.ObjectMeta{Name: platform.APIInternalServiceName, Namespace: bgControlNS},
 			Spec:       corev1.ServiceSpec{ClusterIP: clusterIP},
 		},
+		// The console presents the ops token; the proxy's felis-service-token sits
+		// beside it and must not be the one picked up.
 		&corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: bgControlNS},
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-ops-token", Namespace: bgControlNS},
+			Data:       map[string][]byte{"token": []byte(token)},
+		},
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-service-token", Namespace: bgControlNS},
+			Data:       map[string][]byte{"token": []byte("proxy-" + token)},
 		},
 	}
 }
@@ -114,16 +120,23 @@ func TestRequestBackup(t *testing.T) {
 	cases := []struct {
 		name   string
 		code   int
+		body   string // optional JSON error body
 		expect string
 	}{
-		{"409 not_stopped", http.StatusConflict, "must be stopped"},
-		{"503 backup_unavailable", http.StatusServiceUnavailable, "not configured"},
-		{"404 not found", http.StatusNotFound, "no such server"},
+		{"409 not_stopped", http.StatusConflict, "", "must be stopped"},
+		{"409 no_world_volume surfaces the API's own text", http.StatusConflict,
+			`{"error":{"code":"no_world_volume","message":"this server has no world volume yet — start it once to create it, then retry"}}`,
+			"no world volume yet"},
+		{"503 backup_unavailable", http.StatusServiceUnavailable, "", "not configured"},
+		{"404 not found", http.StatusNotFound, "", "no such server"},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(tc.code)
+				if tc.body != "" {
+					_, _ = io.WriteString(w, tc.body)
+				}
 			}))
 			defer srv.Close()
 			_, err := requestBackup(context.Background(), hc, srv.URL, "tok", "survival", "alice")
@@ -140,5 +153,31 @@ func TestRequestBackup(t *testing.T) {
 		if err == nil || !strings.Contains(err.Error(), "unreachable") {
 			t.Fatalf("err = %v, want an 'unreachable' transport error", err)
 		}
+		// backup-now ends its run on this error alone.
+		if !errors.Is(err, errBackupAPIUnreachable) {
+			t.Fatalf("err = %v, want it to wrap errBackupAPIUnreachable", err)
+		}
 	})
 }
+
+// The Sync picker must not offer system servers: the backup API validates names
+// and resolves a servers-table row, so a lobby/login pick can only die in
+// validation — a dead choice in an emergency console.
+func TestBackupPickable(t *testing.T) {
+	got := backupPickable([]haltableServer{
+		{name: "lobby", phase: "Running", system: true},
+		{name: "login", phase: "Running", system: true},
+		{name: "test-one", phase: "Stopped"},
+	})
+	if len(got) != 1 || got[0].name != "test-one" || got[0].system {
+		t.Fatalf("backupPickable = %+v, want only the user server", got)
+	}
+
+	// Survivors keep their input order (the picker's cursor math depends on it).
+	got = backupPickable([]haltableServer{
+		{name: "alpha"}, {name: "login", system: true}, {name: "beta"},
+	})
+	if len(got) != 2 || got[0].name != "alpha" || got[1].name != "beta" {
+		t.Fatalf("backupPickable order = %+v, want [alpha beta]", got)
+	}
+}

cmd/felis/breakglass.go

@@ -17,6 +17,7 @@ import (
 
 	"felis.lolicon.best/internal/api"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/store"
 
 	tea "github.com/charmbracelet/bubbletea"
@@ -33,13 +34,15 @@ import (
 //
 // Root is necessary but NOT sufficient for accountability: root is machine
 // authority, not a human identity, so the console additionally captures WHO is
-// breaking the glass. When a staff account already exists it asks the operator to
-// authenticate as an existing admin (the verified identity is the accountable
-// actor); when none exists yet it bootstraps the first Owner from the typed
-// credential and attributes the act to the OS user. The audit row records the
-// difference. This attribution is best-effort, not tamper-proof — whoever runs
-// this is root and can edit Postgres directly — but it produces an honest trail
-// for an honest operator, which is the point.
+// breaking the glass. When a staff account already exists the operator names one
+// and types the one-time code the console mails to its verified address
+// (breakglass_otp.go); that account is then the accountable actor. When no code can
+// be sent or proven, the typed OVERRIDE proceeds as the OS user and the audit row
+// says why. When no staff account exists yet it bootstraps the first Owner and
+// attributes the act to the OS user. The audit row records which of these
+// happened. This attribution is best-effort, not tamper-proof — whoever runs this
+// is root and can edit Postgres directly — but it produces an honest trail for an
+// honest operator, which is the point.
 //
 // When a staff account already exists the console opens on a thin top-level menu
 // (menuModel) so that operations are peers, not tails of one wizard. Two account
@@ -62,7 +65,7 @@ import (
 // suspension for the interactive `cloudflared tunnel login` browser consent.
 
 // breakGlassOverrideToken is the literal an operator must type to proceed when no
-// admin credential could be verified. Requiring an explicit, deliberate word (not a
+// admin could be verified by a mailed code. Requiring an explicit, deliberate word (not a
 // bare Enter) keeps the unverified root override from happening by reflex.
 const breakGlassOverrideToken = "OVERRIDE"
 
@@ -76,12 +79,19 @@ type ownerStore interface {
 	AdminExists(ctx context.Context) (bool, error)
 	// UserByUsername loads a staff login projection.
 	UserByUsername(ctx context.Context, username string) (*api.StaffUser, error)
+	// OwnerUsername names the single active Owner seat, or "" when none exists.
+	// provisionOwner refuses to re-target anything but this username: with the
+	// seat occupied, a fresh name would mint a second owner row (UpsertOwner's
+	// insert arm) while the existing — possibly compromised — seat stays live,
+	// and no supported path can delete an owner row afterwards.
+	OwnerUsername(ctx context.Context) (string, error)
 	UpsertOwner(ctx context.Context, id, username, email string) error
 	// InsertOperator mints a NEW Operator staff account. Unlike UpsertOwner it is
 	// insert-only: a username already taken is a conflict (api.ErrConflict), never a
 	// silent reset, so adding an Operator can never clobber the Owner or an existing
-	// Operator. The row is role=admin, identical in shape to the Owner — Felis has no
-	// separate operator DB role (migration 0003: staff = role=admin).
+	// Operator. The row is role=admin — an Operator is staff BELOW the single
+	// role=owner identity (migration 0011 adds that role); the two are the only
+	// staff roles.
 	InsertOperator(ctx context.Context, id, username, email string) error
 	// CompleteOwnerSetup atomically consumes the in-game link code, creates or
 	// promotes the bound Owner, enables local auth, and stores the one-time setup
@@ -135,7 +145,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 	repo := api.NewPGRepo(drv.DB())
 
 	// Decide bootstrap (no admin yet → typed credential mints the first Owner) vs
-	// recovery (an admin exists → the operator must authenticate as one) BEFORE the
+	// recovery (an admin exists → the operator proves one with a mailed code) BEFORE the
 	// alt-screen TUI takes over, so a database fault surfaces as a plain error.
 	adminExists, err := repo.AdminExists(ctx)
 	if err != nil {
@@ -143,7 +153,12 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	res, err := runBreakGlassTUI(ctx, repo, cfg.Database.URL, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists)
+	// Recovery mails its code through [smtp]; the relay is opened only if a code is
+	// asked for.
+	host, _ := os.Hostname()
+	recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, hostSMTPPasswordPath, platform.DefaultControlNamespace), host: host}
+
+	res, err := runBreakGlassTUI(ctx, repo, cfg.Database, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists, recovery)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis breakGlass: %v\n", err)
 		return 1
@@ -167,6 +182,9 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 			fmt.Fprintf(stdout, "\nfelis breakGlass: Owner account %q provisioned; local session sign-in is ENABLED.\n", res.username)
 		}
 		fmt.Fprintf(stdout, "Recorded as %q (mode: %s, os user: %s).\n", res.accountable, res.mode, res.osUser)
+		if res.mode == "root_override" {
+			fmt.Fprintln(stdout, "No admin was proven by an email code; the audit row records this run as an unverified root override and why.")
+		}
 		if res.setupTokenURL != "" {
 			fmt.Fprintf(stdout, "One-time setup URL (opens a lockdown session to verify email / enroll passkey):\n\n    %s\n\n", res.setupTokenURL)
 		}
@@ -251,35 +269,23 @@ func newOwnerID() string {
 	return "usr-" + hex.EncodeToString(b[:])
 }
 
-// authenticateAdmin resolves a typed admin username for recovery-mode attribution.
-// Password verification is gone (passwordless design); Phase 3 replaces this with
-// email-OTP recovery. For now it confirms the named admin exists.
-func authenticateAdmin(ctx context.Context, s ownerStore, username string) (matched string, ok bool, err error) {
-	username = strings.TrimSpace(username)
-	if username == "" {
-		return "", false, nil
-	}
-	u, err := s.UserByUsername(ctx, username)
-	if errors.Is(err, api.ErrNotFound) {
-		return "", false, nil
-	}
-	if err != nil {
-		return "", false, err
-	}
-	if u.Role != "admin" {
-		return "", false, nil
-	}
-	return u.Username, true, nil
-}
-
 // provisionOwner mints or resets the single Owner account direct-to-Postgres,
-// passwordless. The account is role=admin with no password — the Owner completes
+// passwordless. The account is role=owner with no password — the Owner completes
 // passwordless login setup via the web setup-token flow after `felis setup`.
+// With a seat already occupied the reset must name that seat (ownerSeatTakenError
+// otherwise): the upsert's insert arm would silently mint a SECOND owner, and
+// every owner row is undeletable through the panel, so the tier could never
+// converge back to one.
 func provisionOwner(ctx context.Context, s ownerStore, username, email string) error {
 	username = strings.TrimSpace(username)
 	if username == "" {
 		return errors.New("owner username is required")
 	}
+	if seat, err := s.OwnerUsername(ctx); err != nil {
+		return fmt.Errorf("check the owner seat: %w", err)
+	} else if seat != "" && seat != username {
+		return &ownerSeatTakenError{seat: seat}
+	}
 	id := newOwnerID()
 	if id == "" {
 		return errors.New("generate owner id: entropy source failed")
@@ -290,13 +296,26 @@ func provisionOwner(ctx context.Context, s ownerStore, username, email string) e
 	return nil
 }
 
-// provisionOperator mints a NEW Operator staff account direct-to-Postgres. Like the
-// Owner it is role=admin and passwordless — Felis has no separate operator DB role,
-// so an Operator is simply an additional staff admin (migration 0003). UNLIKE
-// provisionOwner, which upserts the single Owner and resets it on a username
-// conflict, this is insert-only: a username already taken returns api.ErrConflict
-// rather than overwriting a live account, so adding an Operator can never silently
-// clobber the Owner's or another Operator's account.
+// ownerSeatTakenError refuses an Owner reset that names anything but the
+// occupied seat, naming it so the operator can retype. Is reports
+// api.ErrConflict so the TUI's recoverable-error branch (shared with the
+// operator path's taken-name clash) routes back to the form instead of ending
+// the console.
+type ownerSeatTakenError struct{ seat string }
+
+func (e *ownerSeatTakenError) Error() string {
+	return fmt.Sprintf("an Owner already exists as %q — enter that username to reset the Owner", e.seat)
+}
+
+func (e *ownerSeatTakenError) Is(target error) bool { return target == api.ErrConflict }
+
+// provisionOperator mints a NEW Operator staff account direct-to-Postgres. It is
+// role=admin and passwordless — an additional staff admin below the single
+// role=owner identity (migrations 0003 + 0011). UNLIKE provisionOwner, which
+// upserts the single Owner and resets it on a username conflict, this is
+// insert-only: a username already taken returns api.ErrConflict rather than
+// overwriting a live account, so adding an Operator can never silently clobber
+// the Owner's or another Operator's account.
 func provisionOperator(ctx context.Context, s ownerStore, username, email string) error {
 	username = strings.TrimSpace(username)
 	if username == "" {
@@ -340,6 +359,10 @@ type breakGlassOp struct {
 	ownerUsername  string
 	ownerEmail     string
 	attemptedAdmin string // recovery / override: the admin username the operator typed
+	verifiedBy     string // recovery: how the admin was proven (verifiedByEmailOTP)
+	codeSentTo     string // recovery: the address the proving code went to
+	otpSkipped     string // root_override: why no code proved an admin (otpSkip*)
+	otpSkipDetail  string // root_override: what failed, when something did
 }
 
 // breakGlassOutcome is what performBreakGlass reports back to the TUI.
@@ -387,7 +410,7 @@ func newSetupToken() (raw, hash string, err error) {
 
 // performSetupMCBind is the `felis setup` Owner-establishment path: the operator
 // binds their Minecraft account via a one-time link code the login gate handed
-// them in-game, the bound user is promoted to role='admin' (passwordless Owner),
+// them in-game, the bound user is promoted to role='owner' (passwordless Owner),
 // local auth is enabled, and a one-time setup URL is minted for the first web
 // login where the Owner verifies email / enrolls a passkey. adminHostname is the
 // operator-console host the URL points at (op.console.<root>): the Owner is staff,
@@ -463,16 +486,7 @@ func auditSetupMCBind(ctx context.Context, s ownerStore, osUser, mcUUID, authSou
 // does not fail the recovery if this write fails — and intentionally honest: it
 // records attribution, it does not prove it (a malicious root can edit the row).
 func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
-	payload := map[string]any{
-		"mode":     op.mode,
-		"owner":    op.ownerUsername,
-		"os_user":  op.osUser,
-		"verified": op.mode == "recovery",
-	}
-	if op.attemptedAdmin != "" {
-		payload["admin_account"] = op.attemptedAdmin
-	}
-	blob, err := json.Marshal(payload)
+	blob, err := json.Marshal(breakGlassPayload(op, "owner"))
 	if err != nil {
 		return err
 	}
@@ -484,6 +498,33 @@ func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
 	})
 }
 
+// breakGlassPayload is the who/how both account audits carry, with the account the
+// run wrote under subjectKey. verified is true only for a run a mailed code proved;
+// such a run names the address the code went to, and an override names why no code
+// proved anyone.
+func breakGlassPayload(op breakGlassOp, subjectKey string) map[string]any {
+	payload := map[string]any{
+		"mode":     op.mode,
+		subjectKey: op.ownerUsername,
+		"os_user":  op.osUser,
+		"verified": op.verifiedBy != "",
+	}
+	if op.attemptedAdmin != "" {
+		payload["admin_account"] = op.attemptedAdmin
+	}
+	if op.verifiedBy != "" {
+		payload["verified_by"] = op.verifiedBy
+		payload["code_sent_to"] = op.codeSentTo
+	}
+	if op.otpSkipped != "" {
+		payload["otp_skipped"] = op.otpSkipped
+		if op.otpSkipDetail != "" {
+			payload["otp_skip_detail"] = op.otpSkipDetail
+		}
+	}
+	return payload
+}
+
 // performAddOperator mints a NEW Operator account and records a best-effort
 // accountability row. It mirrors performBreakGlass — passwordless — with two
 // deliberate differences. (1) It provisions insert-only (provisionOperator), so it
@@ -507,16 +548,7 @@ func performAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) (bre
 // break_glass.operator_create action, naming the new account under an "operator" key
 // rather than "owner".
 func auditAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) error {
-	payload := map[string]any{
-		"mode":     op.mode,
-		"operator": op.ownerUsername,
-		"os_user":  op.osUser,
-		"verified": op.mode == "recovery",
-	}
-	if op.attemptedAdmin != "" {
-		payload["admin_account"] = op.attemptedAdmin
-	}
-	blob, err := json.Marshal(payload)
+	blob, err := json.Marshal(breakGlassPayload(op, "operator"))
 	if err != nil {
 		return err
 	}
@@ -595,16 +627,29 @@ const (
 	cloudflareAPITokenDocsURL        = "https://developers.cloudflare.com/fundamentals/api/how-to/account-owned-token-template/"
 )
 
-func runBreakGlassTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
-	return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass)
+func runBreakGlassTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, recovery recoveryConfig) (breakGlassResult, error) {
+	return runConsoleTUI(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass, recovery)
 }
 
-func runSetupTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
-	return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup)
+// runSetupTUI never reaches recovery: setup with a staff account present lands on
+// the status screen, so it has no relay to hand over.
+func runSetupTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
+	return runConsoleTUI(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup, recoveryConfig{})
 }
 
-func runConsoleTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) (breakGlassResult, error) {
-	rm := newRootModel(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
+// newConsoleRoot is the console's root model as the host runs it: the summary
+// reads this host's alert route.
+func newConsoleRoot(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) *rootModel {
+	rm := newRootModel(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
+	rm.recovery = recovery
+	rm.alertRoute = func(ctx context.Context) alertRoute {
+		return hostAlertRoute(ctx, hostSetupConfigPath, db.URL, defaultHeartbeatFile)
+	}
+	return rm
+}
+
+func runConsoleTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) (breakGlassResult, error) {
+	rm := newConsoleRoot(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode, recovery)
 	final, err := tea.NewProgram(rm, tea.WithAltScreen()).Run()
 	if err != nil {
 		return breakGlassResult{}, err

cmd/felis/breakglass_otp.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/subtle"
+	"errors"
+	"fmt"
+	"math/big"
+	"os"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
+)
+
+// Recovery mode proves who is breaking the glass (#13). Naming a staff account is
+// where it starts: the console then mails a one-time code to that account's verified
+// address, and only that code, typed within recoveryCodeTTL, makes the run a
+// recovery attributed to the account. Every other ending — no such account, no
+// verified address, no relay, a send that fails, a wrong or late code, or the
+// operator giving up on the mail — leads to the typed OVERRIDE, which the audit row
+// records as an unverified root_override together with the reason (otp_skipped).
+// The code goes through the same [smtp] relay as the panel's login codes, so with
+// that relay down recovery still works, as an override that says why.
+
+const (
+	recoveryCodeTTL      = 10 * time.Minute
+	recoveryCodeAttempts = 5
+)
+
+// The reasons a run fell back to the override, recorded as otp_skipped.
+const (
+	otpSkipUnknownAdmin    = "unknown_admin"
+	otpSkipNoVerifiedEmail = "no_verified_email"
+	otpSkipNoRelay         = "no_relay"
+	otpSkipSendFailed      = "send_failed"
+	otpSkipCodeExpired     = "code_expired"
+	otpSkipCodeRejected    = "code_rejected"
+	otpSkipByOperator      = "operator_skipped"
+)
+
+// verifiedByEmailOTP is the audit's verified_by for a recovery the mailed code proved.
+const verifiedByEmailOTP = "email_otp"
+
+// recoveryMailer is the one relay call a recovery code needs; *mail.SMTP has it.
+type recoveryMailer interface {
+	SendNotice(ctx context.Context, email, subject, body string) error
+}
+
+// recoveryConfig is what the console needs to mail a recovery code. open resolves
+// the relay only when a code is about to go out, so a console used to halt a server
+// never touches [smtp] or the cluster; its error says why no relay is available.
+// host names this machine in the mail. The zero value has no relay.
+type recoveryConfig struct {
+	open func(ctx context.Context) (recoveryMailer, error)
+	host string
+	now  func() time.Time
+}
+
+func (r recoveryConfig) clock() time.Time {
+	if r.now != nil {
+		return r.now()
+	}
+	return time.Now()
+}
+
+// recoveryCode is one mailed code: its value, when it stops working, and how many
+// wrong codes were typed against it.
+type recoveryCode struct {
+	value    string
+	expires  time.Time
+	failures int
+}
+
+func newRecoveryCode(now time.Time) (*recoveryCode, error) {
+	n, err := rand.Int(rand.Reader, big.NewInt(1_000_000))
+	if err != nil {
+		return nil, fmt.Errorf("generate recovery code: %w", err)
+	}
+	return &recoveryCode{value: fmt.Sprintf("%06d", n.Int64()), expires: now.Add(recoveryCodeTTL)}, nil
+}
+
+type codeVerdict int
+
+const (
+	codeAccepted codeVerdict = iota
+	codeWrong
+	codeExpired
+	codeExhausted
+)
+
+// check compares a typed code in constant time. Each wrong code counts; the one
+// that reaches recoveryCodeAttempts exhausts the code, which then accepts nothing,
+// and neither does an expired one.
+func (c *recoveryCode) check(typed string, now time.Time) codeVerdict {
+	if c.failures >= recoveryCodeAttempts {
+		return codeExhausted
+	}
+	if !now.Before(c.expires) {
+		return codeExpired
+	}
+	if subtle.ConstantTimeCompare([]byte(strings.TrimSpace(typed)), []byte(c.value)) == 1 {
+		return codeAccepted
+	}
+	c.failures++
+	if c.failures >= recoveryCodeAttempts {
+		return codeExhausted
+	}
+	return codeWrong
+}
+
+func (c *recoveryCode) attemptsLeft() int { return recoveryCodeAttempts - c.failures }
+
+// recoveryStart is where naming an admin led: a code on its way to that admin, or
+// the reason the run has to fall back to the override.
+type recoveryStart struct {
+	admin  *api.StaffUser // the named staff account; nil when none matched
+	code   *recoveryCode  // set when the code went out
+	skip   string         // otpSkip* when it did not
+	detail string         // what failed, for the override screen and the audit row
+}
+
+// resolveAdmin loads the staff account (admin or owner) a typed username names, or
+// nil when there is none.
+func resolveAdmin(ctx context.Context, s ownerStore, username string) (*api.StaffUser, error) {
+	username = strings.TrimSpace(username)
+	if username == "" {
+		return nil, nil
+	}
+	u, err := s.UserByUsername(ctx, username)
+	if errors.Is(err, api.ErrNotFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	// Staff means admin or owner: the Owner is the primary break-glass identity.
+	if u.Role != "admin" && u.Role != "owner" {
+		return nil, nil
+	}
+	return u, nil
+}
+
+// beginRecovery resolves the named admin and mails it a recovery code. Only a
+// datastore or entropy fault is an error; every other way the code cannot go out is
+// a recoveryStart with skip set.
+func beginRecovery(ctx context.Context, s ownerStore, rc recoveryConfig, username, osUser string, op bgOperation) (recoveryStart, error) {
+	admin, err := resolveAdmin(ctx, s, username)
+	if err != nil {
+		return recoveryStart{}, err
+	}
+	if admin == nil {
+		return recoveryStart{skip: otpSkipUnknownAdmin}, nil
+	}
+	st := recoveryStart{admin: admin}
+	// An address nobody ever proved vouches for nobody.
+	email := strings.TrimSpace(admin.Email)
+	if email == "" || !admin.EmailVerified {
+		st.skip = otpSkipNoVerifiedEmail
+		return st, nil
+	}
+	if rc.open == nil {
+		st.skip, st.detail = otpSkipNoRelay, "this console has no mail relay"
+		return st, nil
+	}
+	relay, err := rc.open(ctx)
+	if err != nil {
+		st.skip, st.detail = otpSkipNoRelay, err.Error()
+		return st, nil
+	}
+	code, err := newRecoveryCode(rc.clock())
+	if err != nil {
+		return recoveryStart{}, err
+	}
+	sendCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	subject, body := recoveryMail(code.value, rc.host, osUser, admin.Username, op)
+	if err := relay.SendNotice(sendCtx, email, subject, body); err != nil {
+		st.skip, st.detail = otpSkipSendFailed, err.Error()
+		return st, nil
+	}
+	st.code = code
+	return st, nil
+}
+
+// recoveryMail words the code mail. It says where, by whom and for what the console
+// was opened, so an admin who did not ask for it learns that root on that machine is
+// in someone else's hands.
+func recoveryMail(code, host, osUser, admin string, op bgOperation) (subject, body string) {
+	what, whatZH := "reset the Owner account", "重置 Owner 账号"
+	if op == bgAddOperator {
+		what, whatZH = "add an Operator account", "添加 Operator 账号"
+	}
+	if host == "" {
+		host = "the Felis host"
+	}
+	minutes := int(recoveryCodeTTL / time.Minute)
+	subject = "Felis break-glass recovery code / 紧急恢复验证码"
+	body = fmt.Sprintf(`Someone with root on %[1]s (OS user %[2]s) opened felis breakGlass and named your staff account %[3]q to %[4]s.
+
+Recovery code: %[6]s
+It works for %[7]d minutes.
+
+If this was not you, root on that machine is in someone else's hands: change its credentials and read the audit log for break_glass entries.
+
+有人在 %[1]s 上以 root 身份（系统用户 %[2]s）打开了 felis breakGlass，指名你的管理员账号 %[3]q 来%[5]s。
+
+恢复验证码：%[6]s
+%[7]d 分钟内有效。
+
+如果不是你本人，这台机器的 root 已落入他人之手：请更换它的凭据，并查看审计日志中的 break_glass 记录。
+`, host, osUser, admin, what, whatZH, code, minutes)
+	return subject, body
+}
+
+// maskEmail keeps the first character of the local part and the domain, enough for
+// the operator to recognise the address without putting it on screen whole.
+func maskEmail(email string) string {
+	at := strings.LastIndex(email, "@")
+	if at <= 0 {
+		return "***"
+	}
+	return email[:1] + strings.Repeat("*", max(at-1, 3)) + email[at:]
+}
+
+// hostRecoveryMailer opens the [smtp] relay from the host the way the watchdog does:
+// the password is the env var password_ref names when that is set, else the host
+// copy at passwordPath, else the felis-smtp Secret, whose absence means a relay
+// without AUTH. The cluster is reached only when the host copy is missing, so a
+// break-glass on a host whose k3s is down still gets its code.
+func hostRecoveryMailer(c config.SMTPConfig, passwordPath, controlNS string) func(context.Context) (recoveryMailer, error) {
+	return func(ctx context.Context) (recoveryMailer, error) {
+		if strings.TrimSpace(c.Host) == "" {
+			return nil, errors.New("[smtp] is not configured in felis.toml")
+		}
+		if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" {
+			return smtpRelay(c, os.Getenv(ref)), nil
+		}
+		if password, ok, err := readHostCredential(passwordPath); err != nil {
+			return nil, fmt.Errorf("read the relay password: %w", err)
+		} else if ok {
+			return smtpRelay(c, password), nil
+		}
+		cl, err := buildSystemServerClient()
+		if err != nil {
+			return nil, fmt.Errorf("reach the cluster for the relay password: %w", err)
+		}
+		ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+		defer cancel()
+		password, err := smtpSecretPassword(ctx, cl, controlNS)
+		if err != nil {
+			return nil, fmt.Errorf("read the relay password from %s/%s: %w", controlNS, platform.SMTPSecretName, err)
+		}
+		return smtpRelay(c, password), nil
+	}
+}

cmd/felis/breakglass_otp_test.go

@@ -0,0 +1,498 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/platform"
+
+	tea "github.com/charmbracelet/bubbletea"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// These tests cover the recovery proof (#13): the mailed code's rules, where
+// naming an admin leads, what the mail says, how the console walks from a name to
+// a proven (or overridden) run, and what the audit row then records. No mail
+// leaves the process: the relay is a fake that keeps what it was handed.
+
+type sentMail struct{ to, subject, body string }
+
+type fakeRelay struct {
+	sent []sentMail
+	err  error
+}
+
+func (r *fakeRelay) SendNotice(_ context.Context, to, subject, body string) error {
+	if r.err != nil {
+		return r.err
+	}
+	r.sent = append(r.sent, sentMail{to, subject, body})
+	return nil
+}
+
+// sentCode pulls the code out of the one mail the relay carried.
+func (r *fakeRelay) sentCode(t *testing.T) string {
+	t.Helper()
+	if len(r.sent) != 1 {
+		t.Fatalf("relay carried %d mails, want 1", len(r.sent))
+	}
+	_, after, ok := strings.Cut(r.sent[0].body, "Recovery code: ")
+	if !ok || len(after) < 6 {
+		t.Fatalf("mail carries no recovery code:\n%s", r.sent[0].body)
+	}
+	return after[:6]
+}
+
+func relayConfig(r *fakeRelay, now func() time.Time) recoveryConfig {
+	return recoveryConfig{
+		open: func(context.Context) (recoveryMailer, error) { return r, nil },
+		host: "felis-host-1",
+		now:  now,
+	}
+}
+
+func verifiedAdmin(username, email string) *api.StaffUser {
+	return &api.StaffUser{ID: "usr-" + username, Username: username, Role: "owner", Email: email, EmailVerified: true}
+}
+
+func TestRecoveryCodeRules(t *testing.T) {
+	t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
+
+	t.Run("a fresh code is six digits and lives for the TTL", func(t *testing.T) {
+		seen := map[string]bool{}
+		for i := 0; i < 20; i++ {
+			c, err := newRecoveryCode(t0)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !isRecoveryCodeShape(c.value) {
+				t.Fatalf("code %q is not six digits", c.value)
+			}
+			if !c.expires.Equal(t0.Add(recoveryCodeTTL)) {
+				t.Fatalf("expires = %v, want %v", c.expires, t0.Add(recoveryCodeTTL))
+			}
+			seen[c.value] = true
+		}
+		if len(seen) < 2 {
+			t.Error("twenty codes were all the same")
+		}
+	})
+
+	t.Run("the right code is accepted, surrounding space ignored", func(t *testing.T) {
+		c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		if v := c.check(" 042917 ", t0); v != codeAccepted {
+			t.Errorf("check = %v, want accepted", v)
+		}
+	})
+
+	t.Run("wrong codes count down and the last one exhausts it", func(t *testing.T) {
+		c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		for i := 1; i < recoveryCodeAttempts; i++ {
+			if v := c.check("000000", t0); v != codeWrong {
+				t.Fatalf("wrong code %d: check = %v, want wrong", i, v)
+			}
+			if left := c.attemptsLeft(); left != recoveryCodeAttempts-i {
+				t.Fatalf("after %d wrong codes attemptsLeft = %d, want %d", i, left, recoveryCodeAttempts-i)
+			}
+		}
+		if v := c.check("000000", t0); v != codeExhausted {
+			t.Fatalf("wrong code %d: check = %v, want exhausted", recoveryCodeAttempts, v)
+		}
+		if v := c.check("042917", t0); v != codeExhausted {
+			t.Errorf("the right code after exhaustion: check = %v, want exhausted", v)
+		}
+	})
+
+	t.Run("an expired code accepts nothing", func(t *testing.T) {
+		c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		if v := c.check("042917", t0.Add(recoveryCodeTTL-time.Second)); v != codeAccepted {
+			t.Fatalf("a second before expiry: check = %v, want accepted", v)
+		}
+		c = &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		if v := c.check("042917", t0.Add(recoveryCodeTTL)); v != codeExpired {
+			t.Errorf("at expiry: check = %v, want expired", v)
+		}
+	})
+}
+
+func TestBeginRecovery(t *testing.T) {
+	ctx := context.Background()
+	t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
+	clock := func() time.Time { return t0 }
+
+	t.Run("mails a code to the named admin's verified address", func(t *testing.T) {
+		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
+		r := &fakeRelay{}
+		st, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgAddOperator)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if st.code == nil || st.skip != "" {
+			t.Fatalf("start = %+v, want a code and no skip", st)
+		}
+		if st.admin == nil || st.admin.Username != "root" {
+			t.Fatalf("start.admin = %+v, want root", st.admin)
+		}
+		if got := r.sentCode(t); got != st.code.value {
+			t.Errorf("mailed code %q, want the code the console checks (%q)", got, st.code.value)
+		}
+		m := r.sent[0]
+		if m.to != "[email protected]" {
+			t.Errorf("mail went to %q, want [email protected]", m.to)
+		}
+		for _, want := range []string{"felis-host-1", "OS user alice", `"root"`, "add an Operator account", "添加 Operator 账号", "10 minutes"} {
+			if !strings.Contains(m.body, want) {
+				t.Errorf("mail body lacks %q:\n%s", want, m.body)
+			}
+		}
+		if !st.code.expires.Equal(t0.Add(recoveryCodeTTL)) {
+			t.Errorf("code expires %v, want %v", st.code.expires, t0.Add(recoveryCodeTTL))
+		}
+	})
+
+	t.Run("the Owner reset is named as such", func(t *testing.T) {
+		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
+		r := &fakeRelay{}
+		if _, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgProvisionOwner); err != nil {
+			t.Fatal(err)
+		}
+		if body := r.sent[0].body; !strings.Contains(body, "reset the Owner account") || !strings.Contains(body, "重置 Owner 账号") {
+			t.Errorf("mail body does not name the Owner reset:\n%s", body)
+		}
+	})
+
+	// Each way the code cannot go out ends in a skip reason and no mail.
+	unverified := verifiedAdmin("root", "[email protected]")
+	unverified.EmailVerified = false
+	noEmail := verifiedAdmin("root", "")
+	cases := []struct {
+		name       string
+		user       *api.StaffUser
+		rc         func(r *fakeRelay) recoveryConfig
+		skip       string
+		detail     string
+		wantAdmin  bool
+		relayError error
+	}{
+		{name: "unknown admin", user: nil, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipUnknownAdmin},
+		{name: "unverified address", user: unverified, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
+		{name: "no address", user: noEmail, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
+		{name: "no relay wired", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig { return recoveryConfig{} }, skip: otpSkipNoRelay, detail: "this console has no mail relay", wantAdmin: true},
+		{name: "relay cannot open", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig {
+			return recoveryConfig{open: func(context.Context) (recoveryMailer, error) {
+				return nil, errors.New("[smtp] is not configured in felis.toml")
+			}}
+		}, skip: otpSkipNoRelay, detail: "[smtp] is not configured in felis.toml", wantAdmin: true},
+		{name: "send fails", user: verifiedAdmin("root", "[email protected]"), rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) },
+			skip: otpSkipSendFailed, detail: "554 relay refused", wantAdmin: true, relayError: errors.New("554 relay refused")},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := &fakeOwnerStore{users: map[string]*api.StaffUser{}}
+			if tc.user != nil {
+				f.users["root"] = tc.user
+			}
+			r := &fakeRelay{err: tc.relayError}
+			st, err := beginRecovery(ctx, f, tc.rc(r), "root", "alice", bgProvisionOwner)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if st.code != nil || st.skip != tc.skip || st.detail != tc.detail {
+				t.Errorf("start = {code:%v skip:%q detail:%q}, want no code, skip %q, detail %q", st.code, st.skip, st.detail, tc.skip, tc.detail)
+			}
+			if (st.admin != nil) != tc.wantAdmin {
+				t.Errorf("start.admin = %+v, want present=%v", st.admin, tc.wantAdmin)
+			}
+			if len(r.sent) != 0 {
+				t.Errorf("relay carried %d mails, want none", len(r.sent))
+			}
+		})
+	}
+
+	t.Run("a datastore fault is an error", func(t *testing.T) {
+		f := &fakeOwnerStore{userErr: errors.New("db down")}
+		if _, err := beginRecovery(ctx, f, relayConfig(&fakeRelay{}, clock), "root", "alice", bgProvisionOwner); err == nil {
+			t.Fatal("want the store fault")
+		}
+	})
+}
+
+func TestMaskEmail(t *testing.T) {
+	for in, want := range map[string]string{
+		"[email protected]": "a****@example.com",
+		"[email protected]":    "a***@example.com",
+		"@example.com":      "***",
+		"nonsense":          "***",
+	} {
+		if got := maskEmail(in); got != want {
+			t.Errorf("maskEmail(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestHostRecoveryMailer(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("no [smtp] host is no relay", func(t *testing.T) {
+		_, err := hostRecoveryMailer(config.SMTPConfig{}, hostSMTPPasswordPath, "felis")(ctx)
+		if err == nil || !strings.Contains(err.Error(), "[smtp]") {
+			t.Fatalf("err = %v, want it to name [smtp]", err)
+		}
+	})
+
+	t.Run("the password_ref env var supplies the password", func(t *testing.T) {
+		t.Setenv("FELIS_TEST_RELAY_PW", "from-env")
+		off := false
+		c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_RELAY_PW", RequireTLS: &off}
+		got, err := hostRecoveryMailer(c, hostSMTPPasswordPath, "felis")(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		relay, ok := got.(*mail.SMTP)
+		if !ok {
+			t.Fatalf("relay is %T, want *mail.SMTP", got)
+		}
+		if relay.Password != "from-env" || relay.Host != "mail.example.com" || relay.Port != 2525 || relay.From != "[email protected]" || relay.Username != "felis" || relay.RequireTLS {
+			t.Errorf("relay = %+v, want the [smtp] fields with the env password and TLS as configured", *relay)
+		}
+	})
+}
+
+func TestSMTPSecretPassword(t *testing.T) {
+	ctx := context.Background()
+	scheme := haltScheme(t)
+
+	t.Run("reads the password key", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
+			Data:       map[string][]byte{platform.SMTPSecretPasswordKey: []byte("s3cret")},
+		}).Build()
+		if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "s3cret" {
+			t.Errorf("smtpSecretPassword = (%q, %v), want (s3cret, nil)", pw, err)
+		}
+	})
+
+	t.Run("a missing Secret is a relay without AUTH", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
+		if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "" {
+			t.Errorf("smtpSecretPassword = (%q, %v), want (\"\", nil)", pw, err)
+		}
+	})
+
+	t.Run("any other read failure is an error", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{
+			Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
+				return apierrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, platform.SMTPSecretName, errors.New("rbac"))
+			},
+		}).Build()
+		if _, err := smtpSecretPassword(ctx, cl, "felis"); err == nil {
+			t.Fatal("want the read failure")
+		}
+	})
+}
+
+// recoveryModel is an Owner-reset console for admin "root" (verified address
+// [email protected]), run by OS user alice, with the fake relay behind it.
+func recoveryModel(t *testing.T, f *fakeOwnerStore, r *fakeRelay, now func() time.Time) *ownerModel {
+	t.Helper()
+	if f.users == nil {
+		f.users = map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}
+	}
+	return newOwnerModel(context.Background(), f, "alice", true).withRecovery(relayConfig(r, now))
+}
+
+// nameAdmin submits the admin-name form and feeds the result of the send back in.
+func nameAdmin(t *testing.T, m *ownerModel, name string) *ownerModel {
+	t.Helper()
+	m.authUser = name
+	_, cmd := m.onFormComplete()
+	if m.step != owWorking {
+		t.Fatalf("after naming the admin step = %v, want owWorking", m.step)
+	}
+	msg := findMsg[owAuthMsg](t, cmd)
+	next, _ := m.Update(msg)
+	return next.(*ownerModel)
+}
+
+// findMsg runs a (possibly batched) command and returns the first T it produces.
+func findMsg[T any](t *testing.T, cmd tea.Cmd) T {
+	t.Helper()
+	var zero T
+	if cmd == nil {
+		t.Fatalf("no command, want one producing %T", zero)
+	}
+	switch msg := cmd().(type) {
+	case T:
+		return msg
+	case tea.BatchMsg:
+		for _, c := range msg {
+			if c == nil {
+				continue
+			}
+			if got, ok := c().(T); ok {
+				return got
+			}
+		}
+	}
+	t.Fatalf("command produced no %T", zero)
+	return zero
+}
+
+// typeCode submits the code form with typed.
+func typeCode(t *testing.T, m *ownerModel, typed string) {
+	t.Helper()
+	if m.step != owCode {
+		t.Fatalf("step = %v, want owCode", m.step)
+	}
+	m.codeInput = typed
+	m.onFormComplete()
+}
+
+// provisionAudit finishes the run as an Owner reset and returns its audit payload.
+func provisionAudit(t *testing.T, m *ownerModel, f *fakeOwnerStore) (api.AuditEntry, map[string]any) {
+	t.Helper()
+	if m.step != owProvision {
+		t.Fatalf("step = %v, want owProvision", m.step)
+	}
+	m.username = "owner"
+	msg := m.provisionCmd()().(owProvisionMsg)
+	if msg.err != nil {
+		t.Fatalf("provision: %v", msg.err)
+	}
+	return auditOf(t, f)
+}
+
+func TestRecoveryConsoleFlow(t *testing.T) {
+	t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
+	fixed := func() time.Time { return t0 }
+
+	t.Run("the mailed code proves the admin and the audit says so", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		typeCode(t, m, r.sentCode(t))
+		if m.mode != "recovery" || m.accountable != "root" {
+			t.Fatalf("mode=%q accountable=%q, want recovery attributed to root", m.mode, m.accountable)
+		}
+		e, payload := provisionAudit(t, m, f)
+		if e.Actor != "root" || e.Action != "break_glass.recovery" {
+			t.Errorf("audit = %+v, want actor=root action=break_glass.recovery", e)
+		}
+		if payload["verified"] != true || payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" || payload["os_user"] != "alice" {
+			t.Errorf("payload = %v, want verified by email_otp to [email protected], os_user alice", payload)
+		}
+	})
+
+	t.Run("a wrong code asks again, and the last wrong one leads to the override", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		wrong := "000000"
+		if r.sentCode(t) == wrong {
+			wrong = "111111"
+		}
+		for i := 1; i < recoveryCodeAttempts; i++ {
+			typeCode(t, m, wrong)
+			if m.step != owCode || !strings.Contains(m.codeNote, "wrong") {
+				t.Fatalf("wrong code %d: step=%v note=%q, want the code form again with a note", i, m.step, m.codeNote)
+			}
+		}
+		typeCode(t, m, wrong)
+		if m.step != owOverride || m.skip != otpSkipCodeRejected {
+			t.Fatalf("after %d wrong codes step=%v skip=%q, want the override for code_rejected", recoveryCodeAttempts, m.step, m.skip)
+		}
+		m.onFormComplete() // OVERRIDE typed
+		e, payload := provisionAudit(t, m, f)
+		if e.Actor != "alice" || e.Action != "break_glass.root_override" {
+			t.Errorf("audit = %+v, want actor=alice action=break_glass.root_override", e)
+		}
+		if payload["verified"] != false || payload["otp_skipped"] != otpSkipCodeRejected || payload["admin_account"] != "root" {
+			t.Errorf("payload = %v, want unverified, otp_skipped=code_rejected, admin_account=root", payload)
+		}
+	})
+
+	t.Run("a late code leads to the override", func(t *testing.T) {
+		now := t0
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, func() time.Time { return now }), "root")
+		now = t0.Add(recoveryCodeTTL)
+		typeCode(t, m, r.sentCode(t))
+		if m.step != owOverride || m.skip != otpSkipCodeExpired {
+			t.Fatalf("step=%v skip=%q, want the override for code_expired", m.step, m.skip)
+		}
+	})
+
+	t.Run("OVERRIDE at the code prompt goes on unverified, saying the operator skipped", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		typeCode(t, m, breakGlassOverrideToken)
+		if m.mode != "root_override" || m.accountable != "alice" {
+			t.Fatalf("mode=%q accountable=%q, want root_override as alice", m.mode, m.accountable)
+		}
+		_, payload := provisionAudit(t, m, f)
+		if payload["verified"] != false || payload["otp_skipped"] != otpSkipByOperator {
+			t.Errorf("payload = %v, want unverified, otp_skipped=operator_skipped", payload)
+		}
+	})
+
+	t.Run("a relay failure leads to the override naming it", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{err: errors.New("dial tcp 10.0.0.9:587: connect: connection refused")}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		if m.step != owOverride || m.skip != otpSkipSendFailed {
+			t.Fatalf("step=%v skip=%q, want the override for send_failed", m.step, m.skip)
+		}
+		if reason := m.overrideReason(); !strings.Contains(reason, "connection refused") {
+			t.Errorf("override reason %q does not name the failure", reason)
+		}
+		m.onFormComplete()
+		_, payload := provisionAudit(t, m, f)
+		if payload["otp_skipped"] != otpSkipSendFailed || !strings.Contains(payload["otp_skip_detail"].(string), "connection refused") {
+			t.Errorf("payload = %v, want otp_skipped=send_failed with the relay's error", payload)
+		}
+	})
+
+	t.Run("esc at the code prompt starts over and forgets the code", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		code := r.sentCode(t)
+		next, _ := m.Update(key(tea.KeyEsc))
+		m = next.(*ownerModel)
+		if m.step != owAuth || m.code != nil || m.admin != nil {
+			t.Fatalf("after esc step=%v code=%v admin=%v, want owAuth with the attempt forgotten", m.step, m.code, m.admin)
+		}
+		// The old code cannot be replayed: the next name mails a new one.
+		m = nameAdmin(t, m, "root")
+		if len(r.sent) != 2 {
+			t.Fatalf("relay carried %d mails, want a second one for the new attempt", len(r.sent))
+		}
+		_, fresh, _ := strings.Cut(r.sent[1].body, "Recovery code: ")
+		if m.code.value != fresh[:6] {
+			t.Errorf("the console checks %q, want the newly mailed %q (old one was %q)", m.code.value, fresh[:6], code)
+		}
+	})
+}
+
+func TestRootHandsRecoveryToAccountOperations(t *testing.T) {
+	for _, op := range []bgOperation{bgProvisionOwner, bgAddOperator} {
+		m := newTestRoot(true, consoleModeBreakGlass, "")
+		m.recovery = recoveryConfig{host: "felis-host-1"}
+		m = drive(t, m, menuChoiceMsg{op: op})
+		om, ok := m.screen.(*ownerModel)
+		if !ok {
+			t.Fatalf("op %v: screen = %T, want *ownerModel", op, m.screen)
+		}
+		if om.recovery.host != "felis-host-1" {
+			t.Errorf("op %v: the account screen has no relay config; its codes could never go out", op)
+		}
+	}
+}

cmd/felis/breakglass_test.go

@@ -19,14 +19,15 @@ import (
 // terminal. The design is passwordless: accounts carry no credential, and the
 // Owner completes first-login through the setup-token web flow.
 type fakeOwnerStore struct {
-	upserts  []upsertCall
-	inserts  []upsertCall
-	settings map[string][]byte
-	audits   []api.AuditEntry
-	tokens   []setupTokenCall
-	redeems  []redeemCall
-	users    map[string]*api.StaffUser // keyed by username
-	admins   bool                      // AdminExists answer
+	upserts   []upsertCall
+	inserts   []upsertCall
+	settings  map[string][]byte
+	audits    []api.AuditEntry
+	tokens    []setupTokenCall
+	redeems   []redeemCall
+	users     map[string]*api.StaffUser // keyed by username
+	admins    bool                      // AdminExists answer
+	ownerSeat string                    // OwnerUsername answer: the occupied seat, "" when none
 
 	// CompleteOwnerSetup's success result. redeemUserID defaults to the fresh id
 	// the caller passes (the unlinked-UUID case) when left empty.
@@ -40,6 +41,7 @@ type fakeOwnerStore struct {
 	auditErr       error
 	userErr        error // non-not-found error from UserByUsername
 	adminErr       error
+	seatErr        error
 	redeemErr      error
 	createTokenErr error
 }
@@ -80,6 +82,15 @@ func (f *fakeOwnerStore) UserByUsername(_ context.Context, username string) (*ap
 	return nil, api.ErrNotFound
 }
 
+// OwnerUsername reports the single active Owner seat. Tests set ownerSeat; the
+// zero value models a fresh install where bootstrap is free to mint.
+func (f *fakeOwnerStore) OwnerUsername(_ context.Context) (string, error) {
+	if f.seatErr != nil {
+		return "", f.seatErr
+	}
+	return f.ownerSeat, nil
+}
+
 func (f *fakeOwnerStore) UpsertOwner(_ context.Context, id, username, email string) error {
 	if f.upsertErr != nil {
 		return f.upsertErr
@@ -201,6 +212,34 @@ func TestProvisionOwner(t *testing.T) {
 		}
 	})
 
+	t.Run("an occupied seat refuses any other username", func(t *testing.T) {
+		// The seat is the single owner row: upserting a fresh name would take the
+		// insert arm and mint a SECOND owner, while the existing seat — possibly the
+		// compromised account this reset was meant to replace — stays live, and no
+		// supported path can delete an owner row.
+		f := &fakeOwnerStore{ownerSeat: "seat-holder"}
+		err := provisionOwner(ctx, f, "someone-else", "")
+		if !errors.Is(err, api.ErrConflict) {
+			t.Fatalf("error = %v, want it to wrap api.ErrConflict so the TUI routes back to the form", err)
+		}
+		if !strings.Contains(err.Error(), `"seat-holder"`) {
+			t.Errorf("error = %q, want it to name the occupied seat", err)
+		}
+		if len(f.upserts) != 0 {
+			t.Errorf("want no write against an occupied seat, got %d", len(f.upserts))
+		}
+	})
+
+	t.Run("the occupied seat's own username still resets", func(t *testing.T) {
+		f := &fakeOwnerStore{ownerSeat: "seat-holder"}
+		if err := provisionOwner(ctx, f, "seat-holder", "[email protected]"); err != nil {
+			t.Fatalf("provisionOwner(reset): %v", err)
+		}
+		if len(f.upserts) != 1 || f.upserts[0].username != "seat-holder" || f.upserts[0].email != "[email protected]" {
+			t.Fatalf("want 1 reset upsert for the seat, got %+v", f.upserts)
+		}
+	})
+
 	t.Run("propagates a store error", func(t *testing.T) {
 		f := &fakeOwnerStore{upsertErr: errors.New("boom")}
 		if err := provisionOwner(ctx, f, "owner", ""); err == nil {
@@ -230,61 +269,57 @@ func TestEnableLocalAuth(t *testing.T) {
 	}
 }
 
-func TestAuthenticateAdmin(t *testing.T) {
+func TestResolveAdmin(t *testing.T) {
 	ctx := context.Background()
 
-	// Password verification is gone (passwordless design): authenticateAdmin now only
-	// resolves the named admin so recovery can attribute the audit to a real identity.
-	// The security boundary is the break-glass root gate, not a typed secret.
+	// resolveAdmin only finds the staff account a typed name points at; proving the
+	// operator holds it is the mailed code's job (beginRecovery).
 
-	t.Run("resolves an existing admin for attribution", func(t *testing.T) {
+	t.Run("resolves an existing admin", func(t *testing.T) {
 		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": mkAdmin("root")}}
-		matched, ok, err := authenticateAdmin(ctx, f, "root")
+		u, err := resolveAdmin(ctx, f, "  root ")
 		if err != nil {
-			t.Fatalf("authenticateAdmin: %v", err)
+			t.Fatalf("resolveAdmin: %v", err)
 		}
-		if !ok {
-			t.Fatal("ok = false, want true for an existing admin")
-		}
-		if matched != "root" {
-			t.Errorf("matched = %q, want root", matched)
+		if u == nil || u.Username != "root" {
+			t.Fatalf("resolveAdmin = %+v, want the root admin", u)
 		}
 	})
 
-	t.Run("a non-admin role can never attribute a break-glass", func(t *testing.T) {
+	t.Run("a non-admin role is no staff account", func(t *testing.T) {
 		player := mkAdmin("alice")
 		player.Role = "user" // a player row is not staff
 		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"alice": player}}
-		_, ok, err := authenticateAdmin(ctx, f, "alice")
-		if err != nil {
-			t.Fatalf("unexpected error: %v", err)
-		}
-		if ok {
-			t.Error("ok = true, want false for a non-admin role")
+		if u, err := resolveAdmin(ctx, f, "alice"); u != nil || err != nil {
+			t.Errorf("resolveAdmin(player) = (%+v, %v), want (nil, nil)", u, err)
 		}
 	})
 
-	t.Run("an unknown user is a non-match, not an error", func(t *testing.T) {
-		f := &fakeOwnerStore{}
-		_, ok, err := authenticateAdmin(ctx, f, "nobody")
-		if err != nil {
-			t.Fatalf("unexpected error: %v", err)
-		}
-		if ok {
-			t.Error("ok = true, want false for an unknown user")
+	t.Run("the owner role counts as staff", func(t *testing.T) {
+		owner := mkAdmin("root")
+		owner.Role = "owner" // the platform owner is staff too (migration 0011)
+		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": owner}}
+		if u, err := resolveAdmin(ctx, f, "root"); err != nil || u == nil {
+			t.Fatalf("resolveAdmin(owner) = (%+v, %v), want the owner", u, err)
 		}
 	})
 
-	t.Run("an empty username is a non-match with no store call", func(t *testing.T) {
+	t.Run("an unknown user is nil, not an error", func(t *testing.T) {
+		if u, err := resolveAdmin(ctx, &fakeOwnerStore{}, "nobody"); u != nil || err != nil {
+			t.Errorf("resolveAdmin(unknown) = (%+v, %v), want (nil, nil)", u, err)
+		}
+	})
+
+	t.Run("an empty username makes no store call", func(t *testing.T) {
 		f := &fakeOwnerStore{userErr: errors.New("must not be called")}
-		if _, ok, err := authenticateAdmin(ctx, f, ""); ok || err != nil {
-			t.Errorf("empty username: ok=%v err=%v, want false,nil", ok, err)
+		if u, err := resolveAdmin(ctx, f, " "); u != nil || err != nil {
+			t.Errorf("empty username: (%+v, %v), want (nil, nil)", u, err)
 		}
 	})
 
 	t.Run("a datastore fault is surfaced", func(t *testing.T) {
 		f := &fakeOwnerStore{userErr: errors.New("db down")}
-		if _, _, err := authenticateAdmin(ctx, f, "root"); err == nil {
+		if _, err := resolveAdmin(ctx, f, "root"); err == nil {
 			t.Fatal("want error when the store fails")
 		}
 	})
@@ -352,6 +387,8 @@ func TestPerformBreakGlass(t *testing.T) {
 			osUser:         "alice",
 			ownerUsername:  "owner",
 			attemptedAdmin: "root",
+			verifiedBy:     verifiedByEmailOTP,
+			codeSentTo:     "[email protected]",
 		}
 		out, err := performBreakGlass(ctx, f, op)
 		if err != nil {
@@ -376,6 +413,23 @@ func TestPerformBreakGlass(t *testing.T) {
 		if payload["admin_account"] != "root" {
 			t.Errorf("payload.admin_account = %v, want root", payload["admin_account"])
 		}
+		if payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" {
+			t.Errorf("payload = %v, want verified_by=email_otp [email protected]", payload)
+		}
+		if _, present := payload["otp_skipped"]; present {
+			t.Error("a proven recovery carries no otp_skipped")
+		}
+	})
+
+	t.Run("recovery without a proof is recorded unverified", func(t *testing.T) {
+		f := &fakeOwnerStore{}
+		op := breakGlassOp{mode: "recovery", accountable: "root", osUser: "alice", ownerUsername: "owner", attemptedAdmin: "root"}
+		if _, err := performBreakGlass(ctx, f, op); err != nil {
+			t.Fatalf("performBreakGlass: %v", err)
+		}
+		if _, payload := auditOf(t, f); payload["verified"] != false {
+			t.Errorf("payload.verified = %v, want false: only a mailed code verifies", payload["verified"])
+		}
 	})
 
 	t.Run("root override records an unverified row attributed to the OS user", func(t *testing.T) {
@@ -386,6 +440,8 @@ func TestPerformBreakGlass(t *testing.T) {
 			osUser:         "alice",
 			ownerUsername:  "owner",
 			attemptedAdmin: "typo-admin",
+			otpSkipped:     otpSkipSendFailed,
+			otpSkipDetail:  "dial tcp: connection refused",
 		}
 		if _, err := performBreakGlass(ctx, f, op); err != nil {
 			t.Fatalf("performBreakGlass: %v", err)
@@ -401,6 +457,13 @@ func TestPerformBreakGlass(t *testing.T) {
 		if payload["admin_account"] != "typo-admin" {
 			t.Errorf("payload.admin_account = %v, want typo-admin", payload["admin_account"])
 		}
+		// Why no code proved anyone is part of the record.
+		if payload["otp_skipped"] != otpSkipSendFailed || payload["otp_skip_detail"] != "dial tcp: connection refused" {
+			t.Errorf("payload = %v, want otp_skipped=send_failed with its detail", payload)
+		}
+		if _, present := payload["verified_by"]; present {
+			t.Error("an override carries no verified_by")
+		}
 	})
 
 	t.Run("an audit failure does not fail the recovery", func(t *testing.T) {
@@ -567,6 +630,8 @@ func TestPerformAddOperator(t *testing.T) {
 			ownerUsername:  "ops-jordan",
 			ownerEmail:     "[email protected]",
 			attemptedAdmin: "root",
+			verifiedBy:     verifiedByEmailOTP,
+			codeSentTo:     "[email protected]",
 		}
 		out, err := performAddOperator(ctx, f, op)
 		if err != nil {
@@ -593,14 +658,14 @@ func TestPerformAddOperator(t *testing.T) {
 		if _, present := payload["owner"]; present {
 			t.Error("payload.owner present, want the new account under the operator key")
 		}
-		if payload["verified"] != true || payload["admin_account"] != "root" {
-			t.Errorf("payload = %v, want verified=true admin_account=root", payload)
+		if payload["verified"] != true || payload["admin_account"] != "root" || payload["verified_by"] != verifiedByEmailOTP {
+			t.Errorf("payload = %v, want verified=true admin_account=root verified_by=email_otp", payload)
 		}
 	})
 
 	t.Run("root override records an unverified operator row", func(t *testing.T) {
 		f := &fakeOwnerStore{}
-		op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin"}
+		op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin", otpSkipped: otpSkipUnknownAdmin}
 		if _, err := performAddOperator(ctx, f, op); err != nil {
 			t.Fatalf("performAddOperator: %v", err)
 		}
@@ -608,8 +673,8 @@ func TestPerformAddOperator(t *testing.T) {
 			t.Fatalf("want 1 insert, got %d", len(f.inserts))
 		}
 		_, payload := auditOf(t, f)
-		if payload["verified"] != false {
-			t.Errorf("payload.verified = %v, want false for root_override", payload["verified"])
+		if payload["verified"] != false || payload["otp_skipped"] != otpSkipUnknownAdmin {
+			t.Errorf("payload = %v, want verified=false otp_skipped=unknown_admin", payload)
 		}
 	})
 

cmd/felis/conflict_test.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// racingClient lands one concurrent write (race) on the stored object just before
+// setup's first write of it goes out, the way the operator's status update or
+// felis-api's idle patch can, and counts setup's writes.
+func racingClient(t *testing.T, race func(ctx context.Context, c client.WithWatch), objs ...client.Object) (client.Client, *int) {
+	t.Helper()
+	writes := 0
+	before := func(ctx context.Context, c client.WithWatch) {
+		writes++
+		if writes == 1 {
+			race(ctx, c)
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
+		WithStatusSubresource(&v1alpha1.MinecraftServer{}).
+		WithInterceptorFuncs(interceptor.Funcs{
+			Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
+				before(ctx, c)
+				return c.Update(ctx, obj, opts...)
+			},
+			Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+				before(ctx, c)
+				return c.Patch(ctx, obj, patch, opts...)
+			},
+		}).Build()
+	return cl, &writes
+}
+
+func staleLoginGate(t *testing.T) *v1alpha1.MinecraftServer {
+	t.Helper()
+	ms, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://old.internal:8081", "203.0.113.10.nip.io", "console.203.0.113.10.nip.io")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ms
+}
+
+func getLogin(t *testing.T, cl client.Client) *v1alpha1.MinecraftServer {
+	t.Helper()
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	return &ms
+}
+
+func envMap(ms *v1alpha1.MinecraftServer) map[string]string {
+	m := map[string]string{}
+	for _, e := range ms.Spec.Env {
+		m[e.Name] = e.Value
+	}
+	return m
+}
+
+// A hand edit that lands while setup refreshes the console hostnames costs setup a
+// re-read and a second write; both the edit and the refresh survive.
+func TestRefreshDerivedEnvRetriesAConcurrentEdit(t *testing.T) {
+	ctx := context.Background()
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		ms := getLogin(t, c)
+		ms.Spec.Env = append(ms.Spec.Env, v1alpha1.EnvVar{Name: "HAND_TUNED", Value: "1"})
+		if err := c.Update(ctx, ms); err != nil {
+			t.Fatal(err)
+		}
+	}, staleLoginGate(t))
+
+	desired, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatal(err)
+	}
+	refreshed, err := refreshDerivedEnv(ctx, cl, getLogin(t, cl), desired)
+	if err != nil || !refreshed {
+		t.Fatalf("refreshed=%v err=%v, want a refresh after the retry", refreshed, err)
+	}
+	env := envMap(getLogin(t, cl))
+	if env[envPanelHostname] != "console.mc.example.net" || env[envRootDomain] != "mc.example.net" {
+		t.Errorf("env = %v, want the new hostnames", env)
+	}
+	if env["HAND_TUNED"] != "1" {
+		t.Errorf("env = %v: the concurrent hand edit was dropped", env)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2 (one conflict, one retry)", *writes)
+	}
+}
+
+// converge reports each fill once even when a status write forced a retry.
+func TestConvergeRetriesAConcurrentStatusWrite(t *testing.T) {
+	ctx := context.Background()
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatal(err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Status.Phase = v1alpha1.PhaseRunning
+		if err := c.Status().Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, lobby)
+
+	var got systemServerOutcome
+	for _, o := range convergeSystemServers(ctx, cl, "minecraft", "", "reg/lobby:1",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
+		if o.name == naming.SystemLobbyServer {
+			got = o
+		}
+	}
+	if got.err != nil || !got.updated || !slices.Equal(got.changes, []string{"spec.rcon"}) {
+		t.Fatalf("lobby outcome = %+v, want spec.rcon filled once", got)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if !ms.Spec.Rcon.Enabled || ms.Status.Phase != v1alpha1.PhaseRunning {
+		t.Errorf("rcon.enabled=%v phase=%q, want the fill and the status write both kept", ms.Spec.Rcon.Enabled, ms.Status.Phase)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+// A replica refresh racing another writer keeps that writer's key.
+func TestSecretReplicaRefreshRetriesAConcurrentWrite(t *testing.T) {
+	secret := func(ns, body string) *corev1.Secret {
+		return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
+			Data: map[string][]byte{"felis.toml": []byte(body)}}
+	}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var s corev1.Secret
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+			t.Fatal(err)
+		}
+		s.Data["extra"] = []byte("x")
+		if err := c.Update(ctx, &s); err != nil {
+			t.Fatal(err)
+		}
+	}, secret("felis", "current"), secret("minecraft", "stale"))
+
+	out := ensureSecretReplica(context.Background(), cl, "felis", "minecraft",
+		"felis-config", "felis.toml", "config", "minecraft ns", true)
+	if out.err != nil || !out.updated {
+		t.Fatalf("outcome = %+v, want refreshed", out)
+	}
+	var s corev1.Secret
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+		t.Fatal(err)
+	}
+	if string(s.Data["felis.toml"]) != "current" || string(s.Data["extra"]) != "x" {
+		t.Errorf("replica data = %q, want felis.toml=current and extra=x", s.Data)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+const (
+	sysOldDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
+	sysNewDigest = "sha256:4444444444444444444444444444444444444444444444444444444444444444"
+)
+
+func systemPinRegistry(t *testing.T) imagepin.Resolver {
+	t.Helper()
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v2/felis/limbo/manifests/demo", "/v2/felis/lobby/manifests/demo":
+			w.Header().Set("Docker-Content-Digest", sysNewDigest)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(reg.Close)
+	return imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+}
+
+func systemServer(name, image string) *v1alpha1.MinecraftServer {
+	ms := &v1alpha1.MinecraftServer{}
+	ms.Name, ms.Namespace = name, "minecraft"
+	ms.Labels = map[string]string{v1alpha1.LabelSystemRole: name}
+	ms.Spec.Image = image
+	return ms
+}
+
+// The installer's --system pin moves a system server onto the build its tag names
+// now, from the bare tag or from an earlier digest, and is a no-op the second time.
+func TestPinSystemServerImage(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	limbo := defaultRegistryURL + "/felis/limbo:demo"
+	lobby := defaultRegistryURL + "/felis/lobby:demo"
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		systemServer(naming.SystemLoginServer, limbo),
+		systemServer(naming.SystemLobbyServer, lobby+"@"+sysOldDigest),
+	).Build()
+	image := func(name string) string {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		return ms.Spec.Image
+	}
+
+	for name, want := range map[string]string{
+		naming.SystemLoginServer: limbo + "@" + sysNewDigest,
+		naming.SystemLobbyServer: lobby + "@" + sysNewDigest,
+	} {
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || o.err != nil || !o.updated {
+			t.Fatalf("%s: outcome=%+v err=%v, want pinned", name, o, err)
+		}
+		if got := image(name); got != want {
+			t.Errorf("%s image = %q, want %q", name, got, want)
+		}
+		again, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || again.updated || again.skipped != "already runs "+want {
+			t.Errorf("%s second pass = %+v, %v; want already runs %s", name, again, err, want)
+		}
+	}
+}
+
+func TestPinSystemServerImageLeavesOthersAlone(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	pin := func(t *testing.T, ms *v1alpha1.MinecraftServer) (systemServerOutcome, string) {
+		t.Helper()
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(ms).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(ms), &got); err != nil {
+			t.Fatal(err)
+		}
+		return o, got.Spec.Image
+	}
+
+	t.Run("external image", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, "docker.io/example/limbo:1.2"))
+		if o.err != nil || o.updated || img != "docker.io/example/limbo:1.2" ||
+			o.skipped != "runs docker.io/example/limbo:1.2, which names no platform registry tag to follow; left alone" {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("digest without a tag", func(t *testing.T) {
+		ref := defaultRegistryURL + "/felis/limbo@" + sysOldDigest
+		o, img := pin(t, systemServer(naming.SystemLoginServer, ref))
+		if o.err != nil || o.updated || img != ref {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("not a system server", func(t *testing.T) {
+		ms := systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo")
+		ms.Labels = nil
+		o, img := pin(t, ms)
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:demo" {
+			t.Fatalf("outcome=%+v image=%q, want refused", o, img)
+		}
+	})
+	t.Run("tag the registry lost", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:gone"))
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:gone" {
+			t.Fatalf("outcome=%+v image=%q, want an error the installer falls back on", o, img)
+		}
+	})
+	t.Run("absent", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.skipped != "not present yet; sudo felis setup creates it" {
+			t.Fatalf("outcome=%+v err=%v, want a skip", o, err)
+		}
+	})
+	t.Run("retargeted while pinning", func(t *testing.T) {
+		cl, _ := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+			ms := getLogin(t, c)
+			ms.Spec.Image = "docker.io/example/limbo:1.2"
+			if err := c.Update(ctx, ms); err != nil {
+				t.Fatal(err)
+			}
+		}, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo"))
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.updated {
+			t.Fatalf("outcome=%+v err=%v, want nothing written", o, err)
+		}
+		if img := getLogin(t, cl).Spec.Image; img != "docker.io/example/limbo:1.2" {
+			t.Errorf("image = %q, want the admin's retarget kept", img)
+		}
+	})
+}
+
+// --system names one of the two system servers; anything else is a usage error
+// before any cluster is touched.
+func TestPinImagesSystemFlagTakesOnlySystemServers(t *testing.T) {
+	var stdout, stderr strings.Builder
+	if code := cmdPinImages([]string{"--system", "survival"}, &stdout, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2", code)
+	}
+	if got := stderr.String(); got != "felis pin-images: --system takes login or lobby, not \"survival\"\n" {
+		t.Errorf("stderr = %q", got)
+	}
+}
+
+// An idle setting the panel saves while converge fills the default is kept.
+func TestConvergeIdleKeepsAConcurrentPanelEdit(t *testing.T) {
+	ctx := context.Background()
+	srv := &v1alpha1.MinecraftServer{}
+	srv.Name, srv.Namespace = "survival", "minecraft"
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Spec.Idle = v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}
+		if err := c.Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, srv)
+
+	if out := convergeUserServerIdle(ctx, cl, "minecraft"); len(out) != 0 {
+		t.Fatalf("outcomes = %+v, want none: the server has a setting by the time converge writes", out)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if want := (v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}); ms.Spec.Idle != want {
+		t.Errorf("idle = %+v, want the panel's %+v", ms.Spec.Idle, want)
+	}
+	if *writes != 1 {
+		t.Errorf("writes = %d, want 1 (the conflicted attempt; the retry sends nothing)", *writes)
+	}
+}

cmd/felis/context_max_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+)
+
+func TestContextMaxBytes(t *testing.T) {
+	cases := []struct {
+		name    string
+		reg     config.RegistryConfig
+		auth    config.AuthConfig
+		want    int64
+		wantErr bool
+	}{
+		{name: "direct install keeps the package default", want: 0},
+		// The panel uploads in parts under the edge's 100 MB body limit, so the
+		// Cloudflare edge keeps the full default.
+		{name: "the Cloudflare edge keeps the package default", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 0},
+		{name: "CF-Connecting-IP keeps the package default", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 0},
+		{name: "explicit value behind the edge", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
+		{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
+		{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
+		{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := contextMaxBytes(&config.Config{Registry: tc.reg, Auth: tc.auth})
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
+			}
+			if got != tc.want {
+				t.Fatalf("contextMaxBytes = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestUploadPartsDir(t *testing.T) {
+	if got := uploadPartsDir("/var/lib/felis/uploads"); got != "/var/lib/felis/uploads/.parts" {
+		t.Errorf("local store: parts dir = %q, want beside the contexts", got)
+	}
+	if got := uploadPartsDir("file:///srv/uploads"); got != "/srv/uploads/.parts" {
+		t.Errorf("file:// store: parts dir = %q, want /srv/uploads/.parts", got)
+	}
+	// This machine has no /var/lib/felis/uploads mount, so an s3:// store falls
+	// back to the temp dir.
+	if _, err := os.Stat("/var/lib/felis/uploads"); err == nil {
+		t.Skip("/var/lib/felis/uploads exists here")
+	}
+	if got := uploadPartsDir("s3://bucket/uploads"); got != filepath.Join(os.TempDir(), "felis-upload-parts") {
+		t.Errorf("s3 store without the uploads mount: parts dir = %q", got)
+	}
+}

cmd/felis/converge.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// cmdConverge is the explicit convergence pass over already-installed system
+// servers (#1), plus the idle-stop default for user servers that predate it, and
+// with -user-rcon their RCON block (#3). Provisioning is create-if-absent, so a field the desired spec
+// gained after an install (spec.rcon, spec.startup.healthHTTPPort, a derived env
+// key) never reaches the existing CR — and nothing says so. This command fills
+// exactly those zero-value fields; see convergeSystemServers for the full contract
+// and why it is a separate, operator-timed step rather than part of setup.
+//
+// It reads the same host config as setup (the control plane's felis.toml) and
+// talks to the cluster with the local kubeconfig, so it must run as root on the
+// control-plane host.
+func cmdConverge(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("converge", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	userRcon := fs.Bool("user-rcon", false, "also turn RCON on for user servers created before it was the default")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis converge: refused — converging needs the cluster credentials, so it must run as root (try: sudo felis converge)")
+		return 1
+	}
+
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis converge: %v\n", err)
+		fmt.Fprintln(stderr, "If this host was never installed, run `sudo felis setup` first.")
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis converge: %v\n", err)
+		return 1
+	}
+
+	controlNS := platform.DefaultControlNamespace
+	outcomes := convergeSystemServers(context.Background(), cl, cfg.K8s.Namespace,
+		cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage,
+		platform.InternalAPIBaseURL(controlNS), cfg.Server.RootDomain,
+		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
+
+	outcomes = append(outcomes, convergeUserServerIdle(context.Background(), cl, cfg.K8s.Namespace)...)
+	outcomes = append(outcomes, convergeUserServerRcon(context.Background(), cl, cfg.K8s.Namespace, *userRcon)...)
+
+	fmt.Fprintln(stdout, "felis converge: filling fields an installed server predates (operator-set values are never overwritten):")
+	exit := 0
+	for _, o := range outcomes {
+		switch {
+		case o.err != nil:
+			fmt.Fprintf(stdout, "  - %s: ERROR %v\n", o.name, o.err)
+			exit = 1
+		case len(o.changes) > 0:
+			fmt.Fprintf(stdout, "  - %s: updated (%s)\n", o.name, strings.Join(o.changes, ", "))
+		default:
+			fmt.Fprintf(stdout, "  - %s: %s\n", o.name, o.skipped)
+		}
+	}
+	return exit
+}
+
+// convergeUserServerIdle gives every user server that predates the idle default
+// (spec.idle entirely unset) the default idle stop. A server whose idle stop was
+// turned off keeps a duration on its spec, so it is not "unset" and is left
+// alone; system servers never idle out and are skipped. Servers that already
+// carry a value produce no line, so a converged fleet prints nothing here.
+func convergeUserServerIdle(ctx context.Context, cl client.Client, namespace string) []systemServerOutcome {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+			continue
+		}
+		// Re-checked on the copy each attempt reads: an idle setting the panel saved
+		// meanwhile is the user's, and the default must not land over it.
+		changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
+			if ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+				return false
+			}
+			ms.Spec.Idle = v1alpha1.DefaultIdle()
+			return true
+		})
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
+			continue
+		}
+		if !changed {
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{fmt.Sprintf("spec.idle (stop after %ds empty)", v1alpha1.DefaultEmptySecondsBeforeStop)}})
+	}
+	return out
+}
+
+// convergeUserServerRcon handles user servers created before RCON was part of every
+// new server (694e3cb): spec.rcon entirely unset. Such a server has a dead console,
+// reports nobody online, and never idles out, because all three ride RCON.
+//
+// Only with fill does it turn RCON on, with the same block CreateServer writes
+// today; without it each such server gets a line saying so. The fill is opt-in
+// because the operator gates readiness on the RCON probe: a server whose image does
+// not open the listener RCON_PASSWORD asks for would sit in Starting until it is
+// marked Failed. Felis's own paper and lobby images open it; an image a user brought
+// may not, and only the operator running this can tell. A server that already
+// carries any RCON setting (on or off) is left alone and produces no line.
+func convergeUserServerRcon(ctx context.Context, cl client.Client, namespace string, fill bool) []systemServerOutcome {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
+			continue
+		}
+		if !fill {
+			out = append(out, systemServerOutcome{name: ms.Name, available: true,
+				skipped: "no RCON (console, online count and idle stop are off); once its image serves RCON, sudo felis converge -user-rcon turns it on"})
+			continue
+		}
+		changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
+			if ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
+				return false
+			}
+			ms.Spec.Rcon = v1alpha1.RconSpec{
+				Enabled:   true,
+				SecretRef: v1alpha1.SecretKeyRef{Name: naming.RconSecretName(ms.Name), Key: naming.RconSecretKey},
+			}
+			return true
+		})
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
+			continue
+		}
+		if changed {
+			out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true, changes: []string{"spec.rcon"}})
+		}
+	}
+	return out
+}

cmd/felis/converge_test.go

@@ -0,0 +1,295 @@
+package main
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+// converge is the explicit pass over an installed system server whose CR predates
+// a field the desired spec has since gained (#1). It must fill exactly the
+// zero-valued whitelist fields and the derived env, and must not touch anything a
+// non-zero value already occupies — that is the operator's.
+func TestConvergeSystemServersFillsPredatedFields(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+
+	// An old install: the lobby CR was created before the desired spec began
+	// rendering spec.rcon, and the login CR before the HTTP readiness gate existed.
+	// One derived env key is absent entirely (as if it were added later), and one
+	// hand-added env var plus a non-whitelisted spec field must survive.
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatalf("build lobby: %v", err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{}
+	lobby.Spec.JavaMemory = "999Mi"
+
+	login, err := loginSystemServer("reg/limbo:1", "minecraft",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatalf("build login: %v", err)
+	}
+	login.Spec.Startup.HealthHTTPPort = 0
+	kept := login.Spec.Env
+	login.Spec.Env = nil
+	for _, e := range kept {
+		if e.Name != envPanelHostname {
+			login.Spec.Env = append(login.Spec.Env, e)
+		}
+	}
+	login.Spec.Env = append(login.Spec.Env, v1alpha1.EnvVar{Name: "OPERATOR_TUNING", Value: "keep-me"})
+
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(lobby, login).Build()
+	outcomes := convergeSystemServers(ctx, cl, "minecraft", "reg/limbo:1", "reg/lobby:1",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+
+	byName := map[string]systemServerOutcome{}
+	for _, o := range outcomes {
+		if o.err != nil {
+			t.Fatalf("%s: unexpected error: %v", o.name, o.err)
+		}
+		byName[o.name] = o
+	}
+	lobbyOut := byName[naming.SystemLobbyServer]
+	if len(lobbyOut.changes) != 1 || lobbyOut.changes[0] != "spec.rcon" {
+		t.Errorf("lobby changes = %v, want [spec.rcon] (only the zero-valued field)", lobbyOut.changes)
+	}
+	loginOut := byName[naming.SystemLoginServer]
+	if !slices.Contains(loginOut.changes, "spec.startup.healthHTTPPort") || !slices.Contains(loginOut.changes, "env "+envPanelHostname) {
+		t.Errorf("login changes = %v, want the health port plus the missing derived env key", loginOut.changes)
+	}
+
+	var gotLobby v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &gotLobby); err != nil {
+		t.Fatalf("get lobby: %v", err)
+	}
+	if !gotLobby.Spec.Rcon.Enabled ||
+		gotLobby.Spec.Rcon.SecretRef.Name != naming.RconSecretName(naming.SystemLobbyServer) ||
+		gotLobby.Spec.Rcon.SecretRef.Key != naming.RconSecretKey {
+		t.Errorf("lobby rcon = %+v, want the desired block with the %s secret",
+			gotLobby.Spec.Rcon, naming.RconSecretName(naming.SystemLobbyServer))
+	}
+	if gotLobby.Spec.JavaMemory != "999Mi" {
+		t.Errorf("lobby javaMemory = %q, want 999Mi — converge fills new fields, it does not rewrite the spec", gotLobby.Spec.JavaMemory)
+	}
+
+	var gotLogin v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &gotLogin); err != nil {
+		t.Fatalf("get login: %v", err)
+	}
+	if gotLogin.Spec.Startup.HealthHTTPPort != felisLimboHealthPort {
+		t.Errorf("login healthHTTPPort = %d, want %d", gotLogin.Spec.Startup.HealthHTTPPort, felisLimboHealthPort)
+	}
+	env := map[string]string{}
+	for _, e := range gotLogin.Spec.Env {
+		env[e.Name] = e.Value
+	}
+	if env[envPanelHostname] != "console.mc.example.net" {
+		t.Errorf("%s was not added back: %q", envPanelHostname, env[envPanelHostname])
+	}
+	if env["OPERATOR_TUNING"] != "keep-me" {
+		t.Error("a hand-added env var was dropped; converge only touches config-derived names")
+	}
+}
+
+// A field already holding a non-zero value belongs to the operator: converge must
+// report "already converged" and write nothing.
+func TestConvergeSystemServersLeavesNonZeroFieldsAlone(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatalf("build lobby: %v", err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{
+		Enabled:   true,
+		SecretRef: v1alpha1.SecretKeyRef{Name: "operator-rotated", Key: "password"},
+	}
+	login, err := loginSystemServer("reg/limbo:1", "minecraft",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatalf("build login: %v", err)
+	}
+
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(lobby, login).Build()
+	for _, o := range convergeSystemServers(ctx, cl, "minecraft", "reg/limbo:1", "reg/lobby:1",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
+		if o.err != nil {
+			t.Fatalf("%s: unexpected error: %v", o.name, o.err)
+		}
+		if len(o.changes) != 0 || o.skipped != "already converged" {
+			t.Errorf("%s outcome = %+v, want already converged with no writes", o.name, o)
+		}
+	}
+	var got v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &got); err != nil {
+		t.Fatalf("get lobby: %v", err)
+	}
+	if got.Spec.Rcon.SecretRef.Name != "operator-rotated" {
+		t.Errorf("lobby rcon secretRef = %q — converge overwrote a field the operator had already set",
+			got.Spec.Rcon.SecretRef.Name)
+	}
+}
+
+// Guards: an absent CR is reported (creation is setup's job), a foreign CR is
+// refused rather than adopted, and an unset image skips like the provisioner does.
+func TestConvergeSystemServersGuards(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	run := func(cl client.Client, loginImage, lobbyImage string) []systemServerOutcome {
+		return convergeSystemServers(ctx, cl, "minecraft", loginImage, lobbyImage,
+			"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	}
+
+	t.Run("absent CRs are reported, not created", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
+		for _, o := range run(cl, "reg/limbo:1", "reg/lobby:1") {
+			if o.err != nil {
+				t.Fatalf("%s: %v", o.name, o.err)
+			}
+			if o.created || !strings.Contains(o.skipped, "not present") {
+				t.Errorf("%s outcome = %+v, want a not-present skip", o.name, o)
+			}
+		}
+	})
+
+	t.Run("foreign CR is refused", func(t *testing.T) {
+		foreign := &v1alpha1.MinecraftServer{}
+		foreign.Name = naming.SystemLoginServer
+		foreign.Namespace = "minecraft"
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(foreign).Build()
+		out := run(cl, "reg/limbo:1", "")
+		if len(out) != 2 {
+			t.Fatalf("outcomes = %d, want 2", len(out))
+		}
+		if out[0].err == nil || !strings.Contains(out[0].err.Error(), "not marked") {
+			t.Fatalf("login error = %v, want an unmarked-name refusal", out[0].err)
+		}
+	})
+
+	t.Run("unset image skips", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
+		out := run(cl, "", "reg/lobby:1")
+		if out[0].skipped != "image not configured" {
+			t.Errorf("login skipped = %q, want %q", out[0].skipped, "image not configured")
+		}
+	})
+}
+
+// TestConvergeUserServerIdle fills the idle default only where spec.idle was
+// never set: a server whose idle stop was turned off (duration kept), one with
+// its own duration, and a system server all stay as they are.
+func TestConvergeUserServerIdle(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	mk := func(name string, idle v1alpha1.IdleSpec, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Idle = idle
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		mk("legacy", v1alpha1.IdleSpec{}, ""),
+		mk("off", v1alpha1.IdleSpec{EmptySecondsBeforeStop: 600}, ""),
+		mk("custom", v1alpha1.IdleSpec{AutoStopEnabled: true, EmptySecondsBeforeStop: 1800}, ""),
+		mk(naming.SystemLobbyServer, v1alpha1.IdleSpec{}, naming.SystemLobbyServer),
+	).Build()
+
+	outcomes := convergeUserServerIdle(ctx, cl, "minecraft")
+	if len(outcomes) != 1 || outcomes[0].name != "legacy" || outcomes[0].err != nil {
+		t.Fatalf("outcomes = %+v, want exactly one fill for legacy", outcomes)
+	}
+	want := map[string]v1alpha1.IdleSpec{
+		"legacy":                 v1alpha1.DefaultIdle(),
+		"off":                    {EmptySecondsBeforeStop: 600},
+		"custom":                 {AutoStopEnabled: true, EmptySecondsBeforeStop: 1800},
+		naming.SystemLobbyServer: {},
+	}
+	for name, idle := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Idle != idle {
+			t.Errorf("%s idle = %+v, want %+v", name, ms.Spec.Idle, idle)
+		}
+	}
+	if again := convergeUserServerIdle(ctx, cl, "minecraft"); len(again) != 0 {
+		t.Fatalf("second pass = %+v, want nothing to do", again)
+	}
+}
+
+// TestConvergeUserServerRcon reports a user server with no RCON block at all and
+// fills it only when asked, with the block CreateServer writes. RCON turned off on
+// purpose, a server with its own secret, and a system server stay as they are and
+// produce no line.
+func TestConvergeUserServerRcon(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	mk := func(name string, rcon v1alpha1.RconSpec, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Rcon = rcon
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	own := v1alpha1.RconSpec{Enabled: true, Port: 25580, SecretRef: v1alpha1.SecretKeyRef{Name: "own", Key: "pw"}}
+	off := v1alpha1.RconSpec{SecretRef: v1alpha1.SecretKeyRef{Name: "rcon-off", Key: naming.RconSecretKey}}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		mk("demo", v1alpha1.RconSpec{}, ""),
+		mk("off", off, ""),
+		mk("own", own, ""),
+		mk(naming.SystemLobbyServer, v1alpha1.RconSpec{}, naming.SystemLobbyServer),
+	).Build()
+	get := func(name string) v1alpha1.RconSpec {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		return ms.Spec.Rcon
+	}
+
+	report := convergeUserServerRcon(ctx, cl, "minecraft", false)
+	if len(report) != 1 || report[0].name != "demo" || report[0].updated || report[0].err != nil ||
+		!strings.Contains(report[0].skipped, "-user-rcon") {
+		t.Fatalf("report = %+v, want one skipped line for demo naming -user-rcon", report)
+	}
+	if got := get("demo"); got != (v1alpha1.RconSpec{}) {
+		t.Fatalf("the report-only pass wrote demo's rcon: %+v", got)
+	}
+
+	filled := convergeUserServerRcon(ctx, cl, "minecraft", true)
+	if len(filled) != 1 || filled[0].name != "demo" || !filled[0].updated || filled[0].err != nil {
+		t.Fatalf("fill = %+v, want exactly one update for demo", filled)
+	}
+	want := map[string]v1alpha1.RconSpec{
+		"demo": {Enabled: true, SecretRef: v1alpha1.SecretKeyRef{
+			Name: naming.RconSecretName("demo"), Key: naming.RconSecretKey}},
+		"off":                    off,
+		"own":                    own,
+		naming.SystemLobbyServer: {},
+	}
+	for name, rcon := range want {
+		if got := get(name); got != rcon {
+			t.Errorf("%s rcon = %+v, want %+v", name, got, rcon)
+		}
+	}
+	for _, fill := range []bool{false, true} {
+		if again := convergeUserServerRcon(ctx, cl, "minecraft", fill); len(again) != 0 {
+			t.Fatalf("second pass (fill=%v) = %+v, want nothing to do", fill, again)
+		}
+	}
+}

cmd/felis/db.go

@@ -0,0 +1,501 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	neturl "net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/retention"
+)
+
+const dbUsage = `usage:
+  felis db backup  [-config path] [-dir dir] [-label daily|manual|...] [-keep n] [-state-dir dir]
+                   [-no-servers] [-metrics-file path]
+  felis db restore [-config path] [-dir dir] [-yes] [-force] [-no-safety-backup] <bundle>
+  felis db verify  [-dir dir] <bundle>
+  felis db list    [-dir dir]
+  felis db check   [-dir dir] [-max-age 26h]
+  felis db audit-export [-config path] [-since date] [-until date] [-out file]
+`
+
+// defaultKeep is how many bundles of a label a backup leaves behind. Manual
+// bundles are the operator's own and are never pruned.
+var defaultKeep = map[string]int{
+	dbbackup.LabelDaily:      14,
+	dbbackup.LabelPreMigrate: 10,
+	dbbackup.LabelPreRestore: 5,
+	dbbackup.LabelOffsite:    1,
+}
+
+// cmdDB implements `felis db`: logical backups of the control-plane database
+// together with the host state a rebuild needs (internal/dbbackup). The verb
+// comes first for the same reason as `felis migrate up`.
+func cmdDB(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("db "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, dbUsage) }
+	dir := fs.String("dir", dbbackup.DefaultDir, "bundle directory")
+	switch verb {
+	case "backup":
+		return dbBackup(fs, dir, rest, stdout, stderr)
+	case "restore":
+		return dbRestore(fs, dir, rest, stdout, stderr)
+	case "verify":
+		return dbVerify(fs, dir, rest, stdout, stderr)
+	case "list":
+		return dbList(fs, dir, rest, stdout, stderr)
+	case "check":
+		return dbCheck(fs, dir, rest, stdout, stderr)
+	case "audit-export":
+		return dbAuditExport(fs, rest, stdout, stderr)
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, dbUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis db: unknown verb %q\n%s", verb, dbUsage)
+	return 2
+}
+
+// parseWithArg parses flags that may sit on either side of one positional
+// argument (`restore -yes x.tar` and `restore x.tar -yes` both work) and
+// returns that argument.
+func parseWithArg(fs *flag.FlagSet, args []string) (string, bool) {
+	if err := fs.Parse(args); err != nil {
+		return "", false
+	}
+	if fs.NArg() == 0 {
+		return "", true
+	}
+	arg := fs.Arg(0)
+	if err := fs.Parse(fs.Args()[1:]); err != nil {
+		return "", false
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprintf(fs.Output(), "felis db: unexpected argument %q\n", fs.Arg(0))
+		return "", false
+	}
+	return arg, true
+}
+
+func dbDatabaseURL(path string) (string, error) {
+	db, err := dbDatabase(path)
+	return db.URL, err
+}
+
+func dbDatabase(path string) (config.DatabaseConfig, error) {
+	cfg, err := config.Load(path)
+	if err != nil {
+		return config.DatabaseConfig{}, err
+	}
+	return cfg.Database, nil
+}
+
+// dbTools places pg_dump, pg_restore and psql. The installer's database runs in
+// k3s and the host has no PostgreSQL client, so when the host config names the
+// [database] deployment the tools run in its postgres container over the
+// container's socket, as the URL's role on the URL's database. Otherwise they
+// come from PATH and connect with the URL.
+func dbTools(db config.DatabaseConfig) (dbbackup.Tools, error) {
+	if db.Deployment == "" {
+		return dbbackup.Tools{}, nil
+	}
+	ns, name, _ := strings.Cut(db.Deployment, "/")
+	u, err := neturl.Parse(db.URL)
+	if err != nil || u.User == nil || u.User.Username() == "" || strings.TrimPrefix(u.Path, "/") == "" {
+		return dbbackup.Tools{}, errors.New("[database] url must name the role and the database to run the tools in the database's pod")
+	}
+	return dbbackup.Tools{
+		Exec: []string{"k3s", "kubectl", "exec", "-i", "-n", ns, "deploy/" + name, "-c", platform.PostgresContainer, "--"},
+		Conn: fmt.Sprintf("host=%s port=%d dbname=%s user=%s connect_timeout=15",
+			platform.PostgresSocketDir, platform.PostgresPort,
+			libpqQuote(strings.TrimPrefix(u.Path, "/")), libpqQuote(u.User.Username())),
+	}, nil
+}
+
+// libpqQuote renders v as a single-quoted libpq connection-string value.
+func libpqQuote(v string) string {
+	return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(v) + "'"
+}
+
+func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never")
+	keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, offsite 1, manual all)")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`)
+	noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle")
+	metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	db, err := dbDatabase(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	tools, err := dbTools(db)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	if *keep < 0 {
+		*keep = defaultKeep[*label]
+	}
+	o := dbbackup.BackupOptions{
+		DatabaseURL: db.URL, Tools: tools, Dir: *dir, Label: *label, Keep: *keep,
+		StateDir: *stateDir, Version: resolvedVersion(), Log: stderr,
+		MetricsFile: *metrics, Record: true,
+	}
+	if !*noServers {
+		o.ExportServers = exportMinecraftServers
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	path, err := dbbackup.Backup(ctx, o)
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// The bundle is on disk and holds the database; the exit status fails
+		// the timer's run so the gap shows in systemctl and the journal, and
+		// the panel and the watchdog read it from the record and the manifest.
+		fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+		fmt.Fprintf(stderr, "felis db backup: %v\n  a restore from %s brings back the database but no servers; check `k3s kubectl get minecraftservers -A`, then run `felis db backup` again\n", err, filepath.Base(path))
+		return 1
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+	return 0
+}
+
+// resolveBundle accepts a path, or a bare bundle name looked up in dir.
+func resolveBundle(dir, arg string) string {
+	if strings.ContainsRune(arg, os.PathSeparator) {
+		return arg
+	}
+	if _, err := os.Stat(arg); err == nil {
+		return arg
+	}
+	return filepath.Join(dir, arg)
+}
+
+func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	yes := fs.Bool("yes", false, "replace the database's contents (required)")
+	force := fs.Bool("force", false, "restore even while other clients are connected")
+	noSafety := fs.Bool("no-safety-backup", false, "skip the bundle of the current database taken first")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, "host state directory for the safety bundle")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	if !*yes {
+		fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d, holding %s).\n",
+			filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion, m.Counts.String())
+		fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.")
+		return 2
+	}
+	db, err := dbDatabase(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	tools, err := dbTools(db)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
+	defer cancel()
+	_, safety, err := dbbackup.Restore(ctx, dbbackup.RestoreOptions{
+		DatabaseURL: db.URL, Tools: tools, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
+		Safety: dbbackup.BackupOptions{Keep: defaultKeep[dbbackup.LabelPreRestore], StateDir: *stateDir,
+			Version: resolvedVersion(), ExportServers: exportMinecraftServers},
+		Log: stderr,
+	})
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		if errors.Is(err, dbbackup.ErrClientsConnected) {
+			fmt.Fprintln(stderr, "  kubectl -n felis scale deployment felis-api felis-operator --replicas=0")
+		}
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db restore: restored %s (schema %d)\n", filepath.Base(bundle), m.SchemaVersion)
+	if safety != "" {
+		fmt.Fprintf(stdout, "  the database as it was before is in %s\n", safety)
+	}
+	// Nothing migrates at startup, so a control plane newer than the bundle needs
+	// its migrations re-applied; rolling back to the release that wrote the bundle
+	// must skip that, or the rollback is undone.
+	fmt.Fprintf(stdout, "  next: felis migrate up -config %s (skip it when rolling back to felis %s, which wrote this bundle)\n", *cfgPath, orUnknown(m.FelisVersion))
+	fmt.Fprintln(stdout, "        kubectl -n felis scale deployment felis-api felis-operator --replicas=1")
+	return 0
+}
+
+func dbVerify(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db verify: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "%s: ok\n  taken   %s (%s)\n  felis   %s\n  schema  %d\n  holds   %s\n  %s\n",
+		filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion,
+		m.Counts.String(), orUnknown(m.PGDumpVersion))
+	for _, f := range m.Files {
+		if f.Link != "" {
+			fmt.Fprintf(stdout, "  %-40s -> %s\n", f.Name, f.Link)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %-40s %d bytes\n", f.Name, f.Size)
+	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  (no MinecraftServer objects: %s)\n", m.ServersError)
+	}
+	return 0
+}
+
+func orUnknown(s string) string {
+	if s == "" {
+		return "unknown"
+	}
+	return s
+}
+
+func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	all, err := dbbackup.List(*dir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db list: %v\n", err)
+		return 1
+	}
+	if len(all) == 0 {
+		fmt.Fprintf(stdout, "no database backups in %s\n", *dir)
+		return 0
+	}
+	now := time.Now()
+	for _, b := range all {
+		fmt.Fprintf(stdout, "%-50s %-12s %10s  %s ago\n", b.Name, b.Label, humanBytes(b.Size), dbbackup.Age(now.Sub(b.Created)))
+	}
+	return 0
+}
+
+// dbAuditExport writes audit rows to a file (or stdout) as JSON lines, so an
+// install can keep them past [audit] retention, after which felis-api deletes them.
+func dbAuditExport(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	sinceFlag := fs.String("since", "", "first day (or RFC 3339 instant) to export, inclusive; empty starts at the oldest row")
+	untilFlag := fs.String("until", "", "day (or RFC 3339 instant) to stop before, exclusive; empty runs to the newest row")
+	out := fs.String("out", "", "file to write (created 0600, never overwritten); empty writes to stdout")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	since, err := parseExportBound(*sinceFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -since: %v\n", err)
+		return 2
+	}
+	until, err := parseExportBound(*untilFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -until: %v\n", err)
+		return 2
+	}
+	if !since.IsZero() && !until.IsZero() && !until.After(since) {
+		fmt.Fprintf(stderr, "felis db audit-export: -until %s is not after -since %s\n", *untilFlag, *sinceFlag)
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+		return 1
+	}
+	w := stdout
+	var f *os.File
+	if *out != "" {
+		if f, err = os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600); err != nil {
+			fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+			return 1
+		}
+		w = f
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	n, err := exportAudit(ctx, url, since, until, w)
+	if f != nil {
+		if cerr := f.Close(); err == nil {
+			err = cerr
+		}
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v (%d rows written)\n", err, n)
+		return 1
+	}
+	fmt.Fprintf(stderr, "felis db audit-export: %d audit rows written\n", n)
+	return 0
+}
+
+func exportAudit(ctx context.Context, url string, since, until time.Time, w io.Writer) (int, error) {
+	drv, err := openStore(ctx, url, false)
+	if err != nil {
+		return 0, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	return retention.ExportAudit(ctx, drv.DB(), since, until, w)
+}
+
+// parseExportBound reads a -since/-until value: a day (midnight UTC) or an
+// RFC 3339 instant; empty is an open bound.
+func parseExportBound(v string) (time.Time, error) {
+	if v == "" {
+		return time.Time{}, nil
+	}
+	if t, err := time.Parse(time.DateOnly, v); err == nil {
+		return t, nil
+	}
+	if t, err := time.Parse(time.RFC3339, v); err == nil {
+		return t.UTC(), nil
+	}
+	return time.Time{}, fmt.Errorf("%q is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)", v)
+}
+
+func humanBytes(n int64) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%d B", n)
+	}
+	div, exp := int64(unit), 0
+	for m := n / unit; m >= unit; m /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
+}
+
+// dbCheck is the freshness probe: exit 1 when the newest daily bundle is
+// missing or older than -max-age, for a monitor or the break-glass console to
+// act on.
+func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest daily bundle")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	b, err := dbbackup.Check(*dir, *maxAge, time.Now())
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db check: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db check: ok, newest daily backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
+	return 0
+}
+
+// serverExportTries and serverExportRetry are how long a backup waits out a
+// cluster that is briefly away (an apiserver restart) before its bundle goes
+// without the MinecraftServer objects.
+const serverExportTries = 3
+
+var serverExportRetry = 10 * time.Second
+
+// exportMinecraftServers is dbbackup's ExportServers on the host: the
+// MinecraftServer objects through k3s kubectl, tried serverExportTries times.
+func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+	for try := 1; ; try++ {
+		out, err := getMinecraftServers(ctx)
+		if err == nil {
+			return out, nil
+		}
+		if try == serverExportTries {
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		}
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		case <-time.After(serverExportRetry):
+		}
+	}
+}
+
+// getMinecraftServers reads every MinecraftServer through the host's k3s
+// kubectl and strips what the API server owns, so the result can be fed back
+// with `kubectl apply -f` on a rebuilt cluster.
+func getMinecraftServers(ctx context.Context) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning
+	// on stderr must not end up inside the JSON.
+	cmd := exec.CommandContext(ctx, "k3s", "kubectl", "get", "minecraftservers.felis.lolicon.best", "-A", "-o", "json")
+	cmd.Env = append(os.Environ(), "KUBECONFIG="+hostBootstrapKubeconfigPath)
+	var errBuf strings.Builder
+	cmd.Stderr = &errBuf
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("k3s kubectl get minecraftservers: %w: %s", err, strings.TrimSpace(errBuf.String()))
+	}
+	return cleanServerList(out)
+}
+
+// cleanServerList drops status and the server-assigned metadata from a
+// `kubectl get -o json` List.
+func cleanServerList(raw []byte) ([]byte, error) {
+	var list struct {
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(raw, &list); err != nil {
+		return nil, fmt.Errorf("parse MinecraftServer list: %w", err)
+	}
+	for _, it := range list.Items {
+		delete(it, "status")
+		if md, ok := it["metadata"].(map[string]any); ok {
+			for _, k := range []string{"resourceVersion", "uid", "creationTimestamp", "generation", "managedFields", "selfLink"} {
+				delete(md, k)
+			}
+		}
+	}
+	if list.Items == nil {
+		list.Items = []map[string]any{}
+	}
+	return json.MarshalIndent(map[string]any{"apiVersion": "v1", "kind": "List", "items": list.Items}, "", "  ")
+}

cmd/felis/db_test.go

@@ -0,0 +1,567 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/store"
+)
+
+func TestDBUsage(t *testing.T) {
+	for _, args := range [][]string{{"db"}, {"db", "frobnicate"}, {"db", "restore"}, {"db", "verify"}, {"db", "backup", "extra"}} {
+		var out, errBuf bytes.Buffer
+		if code := run(args, &out, &errBuf); code != 2 {
+			t.Errorf("%v: exit %d, want 2", args, code)
+		}
+		if !strings.Contains(errBuf.String(), "felis db restore") {
+			t.Errorf("%v: no usage on stderr: %q", args, errBuf.String())
+		}
+	}
+}
+
+func TestDBVerifySaysWhatTheBundleHolds(t *testing.T) {
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	bundles := filepath.Join(dir, "bundles")
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
+		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	out.Reset()
+	if code := run([]string{"db", "verify", "-dir", bundles, filepath.Base(bundle)}, &out, &errBuf); code != 0 {
+		t.Fatalf("verify: exit %d: %s", code, errBuf.String())
+	}
+	for _, want := range []string{filepath.Base(bundle) + ": ok", "schema  3", "holds   4 accounts, 2 servers", "pg_dump (PostgreSQL) 18.6"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("verify output lacks %q:\n%s", want, out.String())
+		}
+	}
+}
+
+// TestDBRestoreNeedsYes: without -yes a restore describes the bundle and stops
+// before anything reaches the database, even with -force and
+// -no-safety-backup, which would otherwise let the replay run at once.
+func TestDBRestoreNeedsYes(t *testing.T) {
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	bundles := filepath.Join(dir, "bundles")
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
+		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	podRuns(t, dir)
+
+	out.Reset()
+	errBuf.Reset()
+	code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-force", "-no-safety-backup", filepath.Base(bundle)}, &out, &errBuf)
+	if code != 2 {
+		t.Fatalf("exit %d, want 2; stderr %q", code, errBuf.String())
+	}
+	if want := filepath.Base(bundle) + " (manual, taken "; !strings.Contains(errBuf.String(), want) || !strings.Contains(errBuf.String(), "schema 3, holding 4 accounts, 2 servers).") {
+		t.Errorf("stderr %q does not describe the bundle", errBuf.String())
+	}
+	if !strings.Contains(errBuf.String(), "re-run with -yes") {
+		t.Errorf("stderr %q does not say how to go on", errBuf.String())
+	}
+	if argv, err := os.ReadFile(filepath.Join(dir, "k3s.args")); err == nil {
+		t.Errorf("a restore without -yes ran in the database pod:\n%s", argv)
+	}
+
+	// A bundle that does not verify is refused before -yes is weighed.
+	errBuf.Reset()
+	if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "missing.tar"}, &out, &errBuf); code != 1 {
+		t.Errorf("missing bundle: exit %d, want 1; stderr %q", code, errBuf.String())
+	}
+	if _, err := os.Stat(filepath.Join(dir, "k3s.args")); err == nil {
+		t.Error("a missing bundle reached the database pod")
+	}
+}
+
+func TestParseWithArg(t *testing.T) {
+	for _, args := range [][]string{{"-yes", "b.tar"}, {"b.tar", "-yes"}} {
+		fs := flag.NewFlagSet("t", flag.ContinueOnError)
+		fs.SetOutput(io.Discard)
+		yes := fs.Bool("yes", false, "")
+		arg, ok := parseWithArg(fs, args)
+		if !ok || arg != "b.tar" || !*yes {
+			t.Errorf("%v -> %q ok=%v yes=%v", args, arg, ok, *yes)
+		}
+	}
+	fs := flag.NewFlagSet("t", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	if _, ok := parseWithArg(fs, []string{"a.tar", "b.tar"}); ok {
+		t.Error("two positional arguments accepted")
+	}
+}
+
+func TestResolveBundle(t *testing.T) {
+	if got := resolveBundle("/var/lib/felis/db-backups", "felis-db-x.tar"); got != "/var/lib/felis/db-backups/felis-db-x.tar" {
+		t.Errorf("bare name -> %s", got)
+	}
+	if got := resolveBundle("/var/lib/felis/db-backups", "/root/copy.tar"); got != "/root/copy.tar" {
+		t.Errorf("path -> %s", got)
+	}
+}
+
+func TestCleanServerList(t *testing.T) {
+	raw := `{"apiVersion":"v1","kind":"List","metadata":{"resourceVersion":""},"items":[{
+		"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer",
+		"metadata":{"name":"survival","namespace":"minecraft","uid":"u","resourceVersion":"42","generation":3,
+			"creationTimestamp":"2026-09-01T00:00:00Z","managedFields":[{}],"labels":{"a":"b"}},
+		"spec":{"desiredState":"Running"},"status":{"phase":"Running"}}]}`
+	out, err := cleanServerList([]byte(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Kind  string           `json:"kind"`
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatal(err)
+	}
+	if got.Kind != "List" || len(got.Items) != 1 {
+		t.Fatalf("got %s", out)
+	}
+	it := got.Items[0]
+	if _, ok := it["status"]; ok {
+		t.Error("status kept")
+	}
+	md := it["metadata"].(map[string]any)
+	for _, k := range []string{"uid", "resourceVersion", "generation", "creationTimestamp", "managedFields"} {
+		if _, ok := md[k]; ok {
+			t.Errorf("metadata.%s kept", k)
+		}
+	}
+	if md["name"] != "survival" || md["namespace"] != "minecraft" || md["labels"] == nil {
+		t.Errorf("identity lost: %v", md)
+	}
+	if it["spec"].(map[string]any)["desiredState"] != "Running" {
+		t.Error("spec lost")
+	}
+
+	empty, err := cleanServerList([]byte(`{"items":null}`))
+	if err != nil || !strings.Contains(string(empty), `"items": []`) {
+		t.Errorf("empty list -> %s, %v", empty, err)
+	}
+	if _, err := cleanServerList([]byte("Warning: x\n{")); err == nil {
+		t.Error("garbage parsed")
+	}
+}
+
+func TestHasPending(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}, {Version: 3}}
+	if hasPending(map[int]struct{}{1: {}, 2: {}, 3: {}}, ms) {
+		t.Error("fully applied reported pending")
+	}
+	if !hasPending(map[int]struct{}{1: {}, 2: {}}, ms) {
+		t.Error("missing 3 not reported")
+	}
+}
+
+type appliedDriver struct {
+	store.Driver
+	done map[int]struct{}
+}
+
+func (d appliedDriver) EnsureVersionTable(context.Context) error { return nil }
+func (d appliedDriver) AppliedVersions(context.Context) (map[int]struct{}, error) {
+	return d.done, nil
+}
+
+func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	// An unusable URL makes an attempted backup observable as an error without
+	// any PostgreSQL tooling.
+	const badURL = "not-a-url"
+	for _, tc := range []struct {
+		name    string
+		done    map[int]struct{}
+		attempt bool
+	}{
+		{"fresh database", map[int]struct{}{}, false},
+		{"up to date", map[int]struct{}{1: {}, 2: {}}, false},
+		{"pending on a populated database", map[int]struct{}{1: {}}, true},
+	} {
+		path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, config.DatabaseConfig{URL: badURL}, t.TempDir(), io.Discard)
+		if attempted := err != nil; attempted != tc.attempt {
+			t.Errorf("%s: attempted = %v (err %v), want %v", tc.name, attempted, err, tc.attempt)
+		}
+		if path != "" {
+			t.Errorf("%s: path = %q", tc.name, path)
+		}
+	}
+}
+
+// audit-export takes a day or an RFC 3339 instant for each bound, and refuses a
+// malformed or inverted window before it opens the config or the database.
+func TestAuditExportBounds(t *testing.T) {
+	for _, tc := range []struct{ in, want string }{
+		{"", "0001-01-01T00:00:00Z"},
+		{"2026-01-31", "2026-01-31T00:00:00Z"},
+		{"2026-01-31T12:30:00+08:00", "2026-01-31T04:30:00Z"},
+	} {
+		got, err := parseExportBound(tc.in)
+		if err != nil || got.Format(time.RFC3339) != tc.want {
+			t.Errorf("parseExportBound(%q) = %v, %v; want %s", tc.in, got, err, tc.want)
+		}
+	}
+	if _, err := parseExportBound("31/01/2026"); err == nil || err.Error() != `"31/01/2026" is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)` {
+		t.Errorf("parseExportBound(31/01/2026) err = %v", err)
+	}
+	for _, tc := range []struct {
+		args    []string
+		wantErr string
+	}{
+		{[]string{"db", "audit-export", "-since", "yesterday"}, `felis db audit-export: -since: "yesterday" is neither`},
+		{[]string{"db", "audit-export", "-until", "2026-13-01"}, `felis db audit-export: -until: "2026-13-01" is neither`},
+		{[]string{"db", "audit-export", "-since", "2026-02-01", "-until", "2026-02-01"}, "felis db audit-export: -until 2026-02-01 is not after -since 2026-02-01"},
+		{[]string{"db", "audit-export", "extra"}, "felis db audit-export [-config path]"},
+	} {
+		var out, errBuf bytes.Buffer
+		code := run(append(tc.args, "-config", "/nonexistent/felis.toml"), &out, &errBuf)
+		if code != 2 || !strings.Contains(errBuf.String(), tc.wantErr) {
+			t.Errorf("%v: exit %d, stderr %q; want 2 and %q", tc.args, code, errBuf.String(), tc.wantErr)
+		}
+	}
+}
+
+// podK3s stands in for `k3s kubectl exec ... --`: it logs its argv and runs the
+// command after -- from the "container" directory, which is the only place the
+// PostgreSQL tools exist, as on an installed host. `kubectl get` lists one
+// MinecraftServer, logged to k3s.get, and refuses its first N calls while
+// servers_fail holds N.
+const podK3s = `#!/bin/sh
+if [ "$1" = kubectl ] && [ "$2" = get ]; then
+  printf '%s\n' "$*" >> "$FAKE_DIR/k3s.get"
+  n=$(/usr/bin/wc -l < "$FAKE_DIR/k3s.get")
+  if [ -f "$FAKE_DIR/servers_fail" ] && [ "$n" -le "$(/bin/cat "$FAKE_DIR/servers_fail")" ]; then
+    echo "The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?" >&2
+    exit 1
+  fi
+  echo '{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer","metadata":{"name":"lobby","namespace":"felis-servers","uid":"u-1"},"spec":{"type":"PAPER"},"status":{"phase":"Running"}}]}'
+  exit 0
+fi
+printf '%s\n' "$*" >> "$FAKE_DIR/k3s.args"
+while [ $# -gt 0 ] && [ "$1" != "--" ]; do shift; done
+shift
+tool=$1; shift
+exec /usr/bin/env -i FAKE_DIR="$FAKE_DIR" PATH=/usr/bin:/bin "$FAKE_DIR/container/$tool" "$@"
+`
+
+var podTools = map[string]string{
+	"pg_dump": `#!/bin/sh
+case "$1" in --version) echo "pg_dump (PostgreSQL) 18.6"; exit 0 ;; esac
+printf 'PGDMP-fake-archive'
+`,
+	"pg_restore": `#!/bin/sh
+cat > /dev/null
+`,
+	"psql": `#!/bin/sh
+for a in "$@"; do case "$a" in *"FROM users"*) echo "4|2"; exit 0 ;; esac; done
+for a in "$@"; do [ "$a" = "-c" ] && { echo 3; exit 0; }; done
+cat > /dev/null
+`,
+}
+
+const podPassword = "pw-must-stay-on-the-host"
+
+// podDB is the host config's [database] on an installed host.
+var podDB = config.DatabaseConfig{
+	URL:        "postgres://felis:" + podPassword + "@127.0.0.1:15432/felis?sslmode=disable",
+	Deployment: "felis/felis-postgres",
+}
+
+const podExecPrefix = "kubectl exec -i -n felis deploy/felis-postgres -c postgres -- "
+
+// newPodRig puts the fake k3s on PATH, alone, and returns the directory its
+// k3s.args log lands in.
+func newPodRig(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	bin := filepath.Join(dir, "bin")
+	container := filepath.Join(dir, "container")
+	for _, d := range []string{bin, container} {
+		if err := os.Mkdir(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeTestFile(t, filepath.Join(bin, "k3s"), podK3s, 0o755)
+	for name, body := range podTools {
+		writeTestFile(t, filepath.Join(container, name), body, 0o755)
+	}
+	t.Setenv("PATH", bin)
+	t.Setenv("FAKE_DIR", dir)
+	return dir
+}
+
+// podRuns returns what the fake k3s ran since the last call, failing on any
+// run outside the database container or with the password on its command
+// line (visible to every local user in ps).
+func podRuns(t *testing.T, dir string) []string {
+	t.Helper()
+	log := filepath.Join(dir, "k3s.args")
+	argv, err := os.ReadFile(log)
+	if err != nil {
+		t.Fatalf("nothing ran through k3s: %v", err)
+	}
+	os.Remove(log)
+	var runs []string
+	for _, line := range strings.Split(strings.TrimSpace(string(argv)), "\n") {
+		if !strings.HasPrefix(line, podExecPrefix) {
+			t.Errorf("k3s ran %q, want everything under %q", line, podExecPrefix)
+		}
+		if strings.Contains(line, podPassword) {
+			t.Errorf("the password crossed into the pod on a command line: %q", line)
+		}
+		runs = append(runs, strings.TrimPrefix(line, podExecPrefix))
+	}
+	return runs
+}
+
+// podConfig writes an installed host's felis.toml, [database] pointing at the
+// pod, into dir.
+func podConfig(t *testing.T, dir string) string {
+	t.Helper()
+	toml := strings.Replace(installerTOML("example.com", "127.0.0.1"),
+		`url = "postgres://felis:[email protected]:5432/felis?sslmode=disable"`,
+		`url = "`+podDB.URL+`"
+deployment = "`+podDB.Deployment+`"`, 1)
+	cfg := filepath.Join(dir, "felis.toml")
+	writeTestFile(t, cfg, toml, 0o600)
+	return cfg
+}
+
+func ranIn(runs []string, prefix string) bool {
+	for _, r := range runs {
+		if strings.HasPrefix(r, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+// TestDBBackupAndRestoreRunTheToolsInTheDatabasePod: on an installed host the
+// database is a k3s Deployment and no PostgreSQL client exists outside it, so
+// `felis db backup` and `restore` must reach the tools through kubectl exec,
+// over the pod's socket, and without putting the role's password on a command
+// line.
+func TestDBBackupAndRestoreRunTheToolsInTheDatabasePod(t *testing.T) {
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+
+	var out, errBuf bytes.Buffer
+	bundles := filepath.Join(dir, "bundles")
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
+		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	if _, err := dbbackupVerify(bundle); err != nil {
+		t.Fatalf("the bundle does not verify: %v", err)
+	}
+	runs := podRuns(t, dir)
+	if !ranIn(runs, "pg_dump --format=custom --no-password --dbname=host=/var/run/postgresql port=5432 dbname='felis' user='felis'") {
+		t.Errorf("pg_dump did not dump over the pod's socket as felis on felis: %q", runs)
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "-force", "-no-safety-backup", bundle}, &out, &errBuf); code != 0 {
+		t.Fatalf("restore: exit %d: %s", code, errBuf.String())
+	}
+	runs = podRuns(t, dir)
+	if !ranIn(runs, "pg_restore --no-owner --no-privileges --file=-") || !ranIn(runs, "psql -X -q -w -v ON_ERROR_STOP=1 -d host=/var/run/postgresql") {
+		t.Errorf("the replay did not run in the pod: %q", runs)
+	}
+}
+
+// TestPreMigrateBackupRunsInTheDatabasePod: the snapshot in front of an upgrade
+// is the one taken most often, by bootstrap on every rerun.
+func TestPreMigrateBackupRunsInTheDatabasePod(t *testing.T) {
+	dir := newPodRig(t)
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	// The snapshot also bundles /etc/felis, which a test machine may lack; the
+	// dump runs first either way.
+	_, err := preMigrateBackup(context.Background(), appliedDriver{done: map[int]struct{}{1: {}}}, ms, podDB, filepath.Join(dir, "bundles"), io.Discard)
+	if err != nil && !strings.Contains(err.Error(), "read host state") {
+		t.Fatalf("snapshot: %v", err)
+	}
+	if runs := podRuns(t, dir); !ranIn(runs, "pg_dump --format=custom") {
+		t.Errorf("pg_dump did not run in the pod: %q", runs)
+	}
+}
+
+func TestDBToolsNeedTheRoleAndDatabase(t *testing.T) {
+	if tools, err := dbTools(config.DatabaseConfig{URL: "postgres://felis:pw@db:5432/felis"}); err != nil || len(tools.Exec) != 0 {
+		t.Errorf("no deployment: tools %+v err %v, want the PATH tools", tools, err)
+	}
+	for _, u := range []string{"postgres://db:5432/felis", "postgres://felis:pw@db:5432/"} {
+		if _, err := dbTools(config.DatabaseConfig{URL: u, Deployment: "felis/felis-postgres"}); err == nil {
+			t.Errorf("%s: no error, want a refusal (the pod connection needs the role and the database)", u)
+		}
+	}
+	tools, err := dbTools(config.DatabaseConfig{URL: `postgres://o%27brien@db/my%20db`, Deployment: "felis/felis-postgres"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(tools.Conn, `dbname='my db' user='o\'brien'`) {
+		t.Errorf("Conn = %q, want the values quoted for libpq", tools.Conn)
+	}
+}
+
+var dbbackupVerify = dbbackup.Verify
+
+func noServerExportWait(t *testing.T) {
+	t.Helper()
+	old := serverExportRetry
+	serverExportRetry = 0
+	t.Cleanup(func() { serverExportRetry = old })
+}
+
+// serverGets counts the `kubectl get` calls the fake k3s answered or refused.
+func serverGets(dir string) int {
+	b, _ := os.ReadFile(filepath.Join(dir, "k3s.get"))
+	return strings.Count(string(b), "\n")
+}
+
+// bundleServers returns the bundle's k8s/minecraftservers.json, or nil.
+func bundleServers(t *testing.T, bundle string) []byte {
+	t.Helper()
+	f, err := os.Open(bundle)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	tr := tar.NewReader(f)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		if h.Name == "k8s/minecraftservers.json" {
+			data, err := io.ReadAll(tr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return data
+		}
+	}
+}
+
+// TestDBBackupWithoutServersFails: when the cluster stays away the daily
+// bundle is still written, and `felis db backup` exits 1, so the timer's run
+// shows failed, saying a restore from the bundle brings back no servers.
+func TestDBBackupWithoutServersFails(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	var out, errBuf bytes.Buffer
+	code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", "", "-label", "daily"}, &out, &errBuf)
+	if code != 1 {
+		t.Fatalf("exit %d, want 1: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	m, err := dbbackupVerify(bundle)
+	if err != nil {
+		t.Fatalf("the database must still be bundled: %v", err)
+	}
+	if !strings.Contains(m.ServersError, "6443 was refused") || !strings.Contains(m.ServersError, "(tried 3 times)") || bundleServers(t, bundle) != nil {
+		t.Errorf("manifest servers error = %q", m.ServersError)
+	}
+	if n := serverGets(dir); n != serverExportTries {
+		t.Errorf("export tried %d times, want %d", n, serverExportTries)
+	}
+	if msg := errBuf.String(); !strings.Contains(msg, "a restore from "+filepath.Base(bundle)+" brings back the database but no servers") {
+		t.Errorf("stderr = %q", msg)
+	}
+}
+
+// TestDBBackupRetriesTheServerExport: a cluster back on the last try costs the
+// bundle nothing, and what it holds is ready for kubectl apply.
+func TestDBBackupRetriesTheServerExport(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "2", 0o600)
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", ""}, &out, &errBuf); code != 0 {
+		t.Fatalf("exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	servers := string(bundleServers(t, bundle))
+	if !strings.Contains(servers, `"name": "lobby"`) || strings.Contains(servers, "status") || strings.Contains(servers, "u-1") {
+		t.Errorf("k8s/minecraftservers.json = %s", servers)
+	}
+	if n := serverGets(dir); n != 3 {
+		t.Errorf("export tried %d times, want 3", n)
+	}
+}
+
+// TestPreMigrateBackupExportsServers: the snapshot every upgrade takes, often
+// the newest bundle, carries the MinecraftServer objects too; a cluster that
+// is away does not hold back the migration, whose rollback needs the database
+// alone.
+func TestPreMigrateBackupExportsServers(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	old := preMigrateStateDir
+	preMigrateStateDir = ""
+	t.Cleanup(func() { preMigrateStateDir = old })
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	bundles := filepath.Join(dir, "bundles")
+	pending := appliedDriver{done: map[int]struct{}{1: {}}}
+
+	path, err := preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil {
+		t.Fatalf("snapshot: %v", err)
+	}
+	if !strings.Contains(string(bundleServers(t, path)), `"name": "lobby"`) {
+		t.Errorf("%s holds no MinecraftServer objects", path)
+	}
+
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	path, err = preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil || path == "" {
+		t.Fatalf("snapshot with the cluster away = %q, %v; want the bundle and no error", path, err)
+	}
+	if m, err := dbbackupVerify(path); err != nil || m.ServersError == "" || bundleServers(t, path) != nil {
+		t.Errorf("snapshot with the cluster away: %+v, %v", m, err)
+	}
+}
+
+// TestServerExportStopsWaitingWithTheContext: a backup whose time is up stops
+// waiting for the cluster between tries.
+func TestServerExportStopsWaitingWithTheContext(t *testing.T) {
+	dir := newPodRig(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	// Long enough for the first try to run to its refusal: starting the fake
+	// k3s on a busy machine can take a few hundred ms. Still far below the
+	// 10s retry wait, so waiting it out would fail the check below.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	start := time.Now()
+	_, err := exportMinecraftServers(ctx)
+	if err == nil || !strings.Contains(err.Error(), "(tried 1 times)") || serverGets(dir) != 1 {
+		t.Fatalf("err = %v after %d tries, want the first failure alone", err, serverGets(dir))
+	}
+	if took := time.Since(start); took > 5*time.Second {
+		t.Errorf("took %s, want the context's deadline, not the %s retry wait", took, serverExportRetry)
+	}
+}

cmd/felis/doctor.go

@@ -0,0 +1,387 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/watchdog"
+)
+
+// systemdUnitDir is where the installer writes its units.
+const systemdUnitDir = "/etc/systemd/system"
+
+// hostCommand runs a host tool (systemctl, journalctl, k3s) and returns its
+// stdout. A tool that exits non-zero still returns what it printed:
+// `systemctl is-active` prints "inactive" and exits 3.
+func hostCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
+	return exec.CommandContext(ctx, name, args...).Output()
+}
+
+// hostServices are the long-running units a full install depends on, checked
+// when their unit file is present: k3s runs the cluster, felis-velocity is the
+// game proxy, felis-nano the single-binary host that runs without k3s.
+var hostServices = []string{"k3s.service", "felis-velocity.service", "felis-nano.service"}
+
+// cmdDoctor runs every check felis watchdog runs, with the settings
+// felis-watchdog.service gives it, plus what only the host shows (systemd
+// units that failed or stopped, timers that no longer fire, alerts that reach
+// no one), and prints them grouped by area with where to look next. It mails
+// nothing, pings no heartbeat and leaves the watchdog's state alone: it is
+// safe to run at any time, as often as wanted.
+func cmdDoctor(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("doctor", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis doctor: run as root (sudo felis doctor): the checks read root-only state under /etc/felis and /var/lib/felis")
+		return 1
+	}
+	host, _ := os.Hostname()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	return runDoctor(ctx, doctorEnv{unitDir: *unitDir, run: hostCommand, now: time.Now(), host: host}, stdout)
+}
+
+// doctorEnv is what one doctor run reads the host through.
+type doctorEnv struct {
+	unitDir string
+	run     func(ctx context.Context, name string, args ...string) ([]byte, error)
+	now     time.Time
+	host    string
+}
+
+// doctorAreas are the report's headings in order, by the area findingArea
+// puts a finding under.
+var doctorAreas = []struct{ key, title string }{
+	{key: "config", title: "configuration"},
+	{key: "cluster", title: "Kubernetes cluster"},
+	{key: "postgres", title: "PostgreSQL"},
+	{key: "proxy", title: "game proxy"},
+	{key: "db-backup", title: "database backups"},
+	{key: "offsite", title: "off-site copy"},
+	{key: "scan-db", title: "build scan database"},
+	{key: "disk", title: "disk space"},
+	{key: "memory", title: "memory"},
+	{key: "k3s-certs", title: "k3s certificates"},
+	{key: "host-address", title: "node address"},
+	{key: "clock", title: "clock"},
+	{key: "systemd", title: "systemd units and timers"},
+	{key: "alerts", title: "alerting"},
+}
+
+func runDoctor(ctx context.Context, env doctorEnv, stdout io.Writer) int {
+	unit := filepath.Join(env.unitDir, "felis-watchdog.service")
+	w, found, unitErr := watchdogUnitFlags(unit)
+	var report watchdog.Report
+	var notes []string
+	fmt.Fprintf(stdout, "felis doctor on %s at %s\n", env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
+	switch {
+	case unitErr != nil:
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "watchdog/unit", Severity: watchdog.Critical,
+			SummaryEN: fmt.Sprintf("cannot read the watchdog's settings: %v", unitErr),
+			Hint:      "rerun the installer (deploy/bootstrap.sh) to rewrite felis-watchdog.service",
+		})
+		fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
+	case !found:
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "watchdog/unit", Severity: watchdog.Critical,
+			SummaryEN: fmt.Sprintf("%s is not installed: nothing checks this host or mails anyone when it breaks", unit),
+			Hint:      "rerun the installer (deploy/bootstrap.sh), which installs felis-watchdog.timer",
+		})
+		fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
+	default:
+		fmt.Fprintf(stdout, "checks run as %s runs them (config %s)\n", unit, w.cfgPath)
+	}
+	fmt.Fprintln(stdout)
+
+	skip := map[string]string{}
+	cfg, cfgErr := config.Load(w.cfgPath)
+	if cfgErr != nil {
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "config", Severity: watchdog.Critical,
+			SummaryEN: cfgErr.Error(),
+			Hint:      "the installer writes it (deploy/bootstrap.sh); felis watchdog fails on every run until it loads",
+		})
+		// The host's units are read without it; whether alerts reach anyone
+		// is not known without its relay.
+		for _, a := range doctorAreas {
+			if a.key != "config" && a.key != "systemd" {
+				skip[a.key] = "the configuration did not load"
+			}
+		}
+	} else {
+		_, owners, ownersErr := watchdogProbes(ctx, w, cfg, env.now, &report)
+		report.Findings = append(report.Findings, alertReachFindings(cfg, owners, ownersErr)...)
+		if w.proxyAddr == "" {
+			skip["proxy"] = "no -proxy-addr"
+		}
+		if w.backupDir == "" {
+			skip["db-backup"] = "no -backup-dir"
+		}
+		if !cfg.Offsite.Enabled() {
+			skip["offsite"] = "not configured"
+		}
+		if !usesMirroredScanDB(cfg) {
+			skip["scan-db"] = "builds do not scan against the registry's copy"
+		}
+		if len(splitList(w.certDirs)) == 0 {
+			skip["k3s-certs"] = "no -k3s-cert-dirs"
+		}
+		if w.nodeIP == "" {
+			skip["host-address"] = "no -node-ip"
+		}
+	}
+	report.Findings = append(report.Findings, unitFindings(ctx, env)...)
+
+	switch url, err := readHeartbeatURL(w.heartbeatFile); {
+	case err != nil:
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "watchdog/heartbeat", Severity: watchdog.Warning,
+			SummaryEN: fmt.Sprintf("the heartbeat URL is unusable, so no run pings it: %v", err),
+			Hint:      "rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL set (docs/troubleshooting.md §14)",
+		})
+	case url == "":
+		notes = append(notes, "no heartbeat URL is set: a host that goes down entirely, or a watchdog that stops running, alerts no one. "+
+			"Rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL (docs/troubleshooting.md §14)")
+	}
+	if until := watchdog.QuietUntil(w.quietPath); env.now.Before(until) {
+		notes = append(notes, fmt.Sprintf("the watchdog mails nothing until %s (%s): the installer holds it while it restarts things on purpose, "+
+			"and a marker an installer killed mid-run left behind holds it until then",
+			until.UTC().Format("2006-01-02 15:04 UTC"), w.quietPath))
+	}
+	return printDoctorReport(stdout, report.Findings, skip, notes)
+}
+
+// printDoctorReport prints each area's findings under its heading, an area
+// with none as fine or, when skip says why, as not checked, then the notes
+// and the count. It returns the exit status: 1 when anything was found.
+func printDoctorReport(stdout io.Writer, findings []watchdog.Finding, skip map[string]string, notes []string) int {
+	byArea := map[string][]watchdog.Finding{}
+	for _, f := range findings {
+		a := findingArea(f.Key)
+		byArea[a] = append(byArea[a], f)
+	}
+	var critical, warning int
+	for _, a := range doctorAreas {
+		fs := byArea[a.key]
+		switch {
+		case len(fs) > 0:
+		case skip[a.key] != "":
+			fmt.Fprintf(stdout, "-  %s: not checked, %s\n", a.title, skip[a.key])
+			continue
+		default:
+			fmt.Fprintf(stdout, "✓  %s\n", a.title)
+			continue
+		}
+		mark := "!"
+		if slices.ContainsFunc(fs, func(f watchdog.Finding) bool { return f.Severity == watchdog.Critical }) {
+			mark = "✗"
+		}
+		fmt.Fprintf(stdout, "%s  %s\n", mark, a.title)
+		for _, f := range fs {
+			if f.Severity == watchdog.Critical {
+				critical++
+			} else {
+				warning++
+			}
+			fmt.Fprintf(stdout, "     %-8s %s: %s\n", f.Severity, f.Key, f.SummaryEN)
+			if f.Hint != "" {
+				fmt.Fprintf(stdout, "              → %s\n", f.Hint)
+			}
+		}
+	}
+	for _, n := range notes {
+		fmt.Fprintf(stdout, "\nnote: %s\n", n)
+	}
+	fmt.Fprintln(stdout)
+	if critical+warning == 0 {
+		fmt.Fprintln(stdout, "no problems found")
+		return 0
+	}
+	fmt.Fprintf(stdout, "%d problem(s): %d critical, %d warning(s)\n", critical+warning, critical, warning)
+	return 1
+}
+
+// findingArea is the report heading a finding key goes under.
+func findingArea(key string) string {
+	head, _, _ := strings.Cut(key, "/")
+	switch head {
+	case "kube-api", "deployment", "system-server", "server-failed", "job-failed", "reaper-stale", "node":
+		return "cluster"
+	case "db-backup", "db-backup-servers":
+		return "db-backup"
+	case "unit", "timer":
+		return "systemd"
+	case "watchdog":
+		return "alerts"
+	}
+	return head
+}
+
+// alertReachFindings is why the watchdog's alerts would reach no one, which
+// its own runs only log: no relay, or no owner with a verified address.
+func alertReachFindings(cfg *config.Config, owners []string, ownersErr error) []watchdog.Finding {
+	var out []watchdog.Finding
+	if cfg.SMTP.Host == "" {
+		out = append(out, watchdog.Finding{
+			Key: "alerts/relay", Severity: watchdog.Warning,
+			SummaryEN: "no [smtp] relay is configured: the watchdog logs its alerts to the journal and mails no one",
+			Hint:      "sudo felis setup, step SMTP",
+		})
+	}
+	if ownersErr == nil && len(owners) == 0 {
+		out = append(out, watchdog.Finding{
+			Key: "alerts/recipients", Severity: watchdog.Warning,
+			SummaryEN: "no owner account has a verified email: the watchdog's alerts reach no one",
+			Hint:      "an owner verifies an address in the panel's account settings",
+		})
+	}
+	return out
+}
+
+// unitFindings reports the installer's systemd units that failed, the
+// long-running ones that are not running, and timers that no longer fire.
+func unitFindings(ctx context.Context, env doctorEnv) []watchdog.Finding {
+	var out []watchdog.Finding
+	seen := map[string]bool{}
+	failed, err := env.run(ctx, "systemctl", "list-units", "--all", "--plain", "--no-legend", "--no-pager", "--state=failed", "felis-*", "k3s.service")
+	if err != nil && len(failed) == 0 {
+		return []watchdog.Finding{{
+			Key: "unit/systemctl", Severity: watchdog.Warning,
+			SummaryEN: fmt.Sprintf("systemctl list-units failed, so no unit was checked: %v", err),
+		}}
+	}
+	for _, line := range strings.Split(string(failed), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
+		}
+		name := fields[0]
+		seen[name] = true
+		out = append(out, watchdog.Finding{
+			Key: "unit/" + name, Severity: watchdog.Critical,
+			SummaryEN: name + " failed",
+			Hint:      fmt.Sprintf("journalctl -u %s -n 100 --no-pager; once fixed, sudo systemctl reset-failed %s (a timer's job clears on its next good run)", name, name),
+		})
+	}
+	for _, name := range hostServices {
+		if seen[name] {
+			continue
+		}
+		if _, err := os.Stat(filepath.Join(env.unitDir, name)); err != nil {
+			continue
+		}
+		if state := unitActiveState(ctx, env, name); state != "active" {
+			out = append(out, watchdog.Finding{
+				Key: "unit/" + name, Severity: watchdog.Critical,
+				SummaryEN: fmt.Sprintf("%s is %s", name, state),
+				Hint:      fmt.Sprintf("sudo systemctl start %s; journalctl -u %s -n 100 --no-pager", name, name),
+			})
+		}
+	}
+	timers, _ := filepath.Glob(filepath.Join(env.unitDir, "felis-*.timer"))
+	for _, path := range timers {
+		name := filepath.Base(path)
+		if state := unitActiveState(ctx, env, name); state != "active" {
+			out = append(out, watchdog.Finding{
+				Key: "timer/" + name, Severity: watchdog.Warning,
+				SummaryEN: fmt.Sprintf("%s is %s: the job it starts no longer runs", name, state),
+				Hint:      fmt.Sprintf("sudo systemctl enable --now %s", name),
+			})
+		}
+	}
+	return out
+}
+
+// unitActiveState is what `systemctl is-active` says of unit.
+func unitActiveState(ctx context.Context, env doctorEnv, unit string) string {
+	out, err := env.run(ctx, "systemctl", "is-active", unit)
+	if state := strings.TrimSpace(string(out)); state != "" {
+		return state
+	}
+	return fmt.Sprintf("in an unknown state (systemctl is-active: %v)", err)
+}
+
+// watchdogUnitFlags reads the flags felis-watchdog.service runs felis
+// watchdog with. found is false when there is no such unit; w is then the
+// watchdog's defaults.
+func watchdogUnitFlags(path string) (w watchdogFlags, found bool, err error) {
+	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	w.register(fs)
+	raw, err := os.ReadFile(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return w, false, nil
+	}
+	if err != nil {
+		return w, false, err
+	}
+	for _, line := range strings.Split(string(raw), "\n") {
+		cmd, ok := strings.CutPrefix(strings.TrimSpace(line), "ExecStart=")
+		if !ok {
+			continue
+		}
+		fields := execArgs(cmd)
+		i := slices.Index(fields, "watchdog")
+		if i < 0 {
+			continue
+		}
+		if err := fs.Parse(fields[i+1:]); err != nil {
+			return w, true, fmt.Errorf("%s: %w", path, err)
+		}
+		return w, true, nil
+	}
+	return w, true, fmt.Errorf("%s runs no `felis watchdog`", path)
+}
+
+// execArgs splits an ExecStart= command line into its words. A word may be
+// quoted with " or ', as systemd allows, which is how an empty value is
+// written.
+func execArgs(s string) []string {
+	var out []string
+	var cur strings.Builder
+	inWord := false
+	var quote rune
+	for _, r := range s {
+		switch {
+		case quote != 0:
+			if r == quote {
+				quote = 0
+			} else {
+				cur.WriteRune(r)
+			}
+		case r == '"' || r == '\'':
+			quote, inWord = r, true
+		case r == ' ' || r == '\t':
+			if inWord {
+				out = append(out, cur.String())
+				cur.Reset()
+				inWord = false
+			}
+		default:
+			cur.WriteRune(r)
+			inWord = true
+		}
+	}
+	if inWord {
+		out = append(out, cur.String())
+	}
+	return out
+}

cmd/felis/doctor_test.go

@@ -0,0 +1,423 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/watchdog"
+)
+
+// bootstrapWatchdogExecStart is the ExecStart= line deploy/bootstrap.sh writes
+// into felis-watchdog.service, with its variables filled in as an install
+// fills them.
+func bootstrapWatchdogExecStart(t *testing.T, vars map[string]string) string {
+	t.Helper()
+	raw, err := os.ReadFile("../../deploy/bootstrap.sh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var line string
+	for _, l := range strings.Split(string(raw), "\n") {
+		if strings.HasPrefix(l, "ExecStart=${HOST_BIN} watchdog -config") {
+			line = l
+		}
+	}
+	if line == "" {
+		t.Fatal("deploy/bootstrap.sh writes no `ExecStart=${HOST_BIN} watchdog -config` line")
+	}
+	line = strings.ReplaceAll(line, "${NODE_IP:+ -node-ip ${NODE_IP}}", " -node-ip "+vars["NODE_IP"])
+	line = regexp.MustCompile(`\$\{([A-Za-z_]+)\}`).ReplaceAllStringFunc(line, func(m string) string {
+		v, ok := vars[m[2:len(m)-1]]
+		if !ok {
+			t.Fatalf("bootstrap's watchdog ExecStart= uses %s, which this test does not fill in", m)
+		}
+		return v
+	})
+	return line
+}
+
+// felis doctor reads the watchdog's settings from the unit the installer
+// writes, so it checks the paths the timer's runs check.
+func TestWatchdogUnitFlagsReadsTheInstallersUnit(t *testing.T) {
+	dir := t.TempDir()
+	exec := bootstrapWatchdogExecStart(t, map[string]string{
+		"HOST_BIN": "/usr/local/bin/felis", "STATE_DIR": "/srv/felis-etc", "WATCHDOG_STATE": "/srv/watchdog/state.json",
+		"WATCHDOG_QUIET_FILE": "/srv/quiet-until", "FELIS_DB_BACKUP_DIR": "/srv/db-backups", "FELIS_GAME_PORT": "25577",
+		"disks": "/,/srv/data", "NODE_IP": "10.0.0.5",
+	})
+	unit := filepath.Join(dir, "felis-watchdog.service")
+	writeTestFile(t, unit, "[Unit]\nDescription=Felis watchdog\n\n[Service]\nType=oneshot\n"+exec+"\nTimeoutStartSec=3min\n", 0o644)
+
+	w, found, err := watchdogUnitFlags(unit)
+	if err != nil || !found {
+		t.Fatalf("found %v, err %v", found, err)
+	}
+	got := []string{w.cfgPath, w.statePath, w.quietPath, w.backupDir, w.proxyAddr, w.diskPaths, w.nodeIP, w.heartbeatFile, w.controlNS}
+	want := []string{"/srv/felis-etc/felis.host.toml", "/srv/watchdog/state.json", "/srv/quiet-until", "/srv/db-backups", "127.0.0.1:25577", "/,/srv/data", "10.0.0.5", defaultHeartbeatFile, "felis"}
+	if strings.Join(got, "|") != strings.Join(want, "|") {
+		t.Errorf("read\n %q\nwant\n %q", got, want)
+	}
+
+	w, found, err = watchdogUnitFlags(filepath.Join(dir, "missing.service"))
+	if err != nil || found || w.cfgPath != "/etc/felis/felis.toml" || w.backupDir != "/var/lib/felis/db-backups" {
+		t.Errorf("no unit: found %v, err %v, config %q, backups %q; want the watchdog's defaults", found, err, w.cfgPath, w.backupDir)
+	}
+
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis version\n", 0o644)
+	if _, found, err := watchdogUnitFlags(unit); !found || err == nil || !strings.Contains(err.Error(), "runs no `felis watchdog`") {
+		t.Errorf("a unit that runs something else: found %v, err %v", found, err)
+	}
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
+	if _, _, err := watchdogUnitFlags(unit); err == nil {
+		t.Error("a flag this binary does not know was accepted")
+	}
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -backup-dir \"\"\t-proxy-addr '127.0.0.1:1' -disk-paths \"/a b\"\n", 0o644)
+	if w, _, err := watchdogUnitFlags(unit); err != nil || w.backupDir != "" || w.proxyAddr != "127.0.0.1:1" || w.diskPaths != "/a b" {
+		t.Errorf("quoted words: backups %q, proxy %q, disks %q, err %v", w.backupDir, w.proxyAddr, w.diskPaths, err)
+	}
+}
+
+func TestFindingArea(t *testing.T) {
+	for key, want := range map[string]string{
+		"kube-api":                           "cluster",
+		"deployment/felis-api":               "cluster",
+		"system-server/lobby":                "cluster",
+		"server-failed/survival":             "cluster",
+		"job-failed/reaper-123":              "cluster",
+		"reaper-stale":                       "cluster",
+		"node/felis-1/NotReady":              "cluster",
+		"postgres":                           "postgres",
+		"proxy":                              "proxy",
+		"db-backup":                          "db-backup",
+		"db-backup-servers":                  "db-backup",
+		"offsite":                            "offsite",
+		"scan-db":                            "scan-db",
+		"disk//var/lib/felis":                "disk",
+		"memory":                             "memory",
+		"k3s-certs":                          "k3s-certs",
+		"host-address":                       "host-address",
+		"clock":                              "clock",
+		"config":                             "config",
+		"unit/felis-offsite.service":         "systemd",
+		"timer/felis-db-backup.timer":        "systemd",
+		"watchdog/unit":                      "alerts",
+		"watchdog/heartbeat":                 "alerts",
+		"alerts/relay":                       "alerts",
+		"alerts/recipients":                  "alerts",
+		"something-a-later-release-reported": "something-a-later-release-reported",
+	} {
+		if got := findingArea(key); got != want {
+			t.Errorf("findingArea(%q) = %q, want %q", key, got, want)
+		}
+	}
+}
+
+func TestAlertReachFindings(t *testing.T) {
+	relay := &config.Config{SMTP: config.SMTPConfig{Host: "smtp.example.com"}}
+	keys := func(fs []watchdog.Finding) string {
+		var k []string
+		for _, f := range fs {
+			k = append(k, f.Key)
+		}
+		return strings.Join(k, ",")
+	}
+	for _, tc := range []struct {
+		what      string
+		cfg       *config.Config
+		owners    []string
+		ownersErr error
+		want      string
+	}{
+		{"a relay and an owner", relay, []string{"[email protected]"}, nil, ""},
+		{"no relay", &config.Config{}, []string{"[email protected]"}, nil, "alerts/relay"},
+		{"no owner with an address", relay, nil, nil, "alerts/recipients"},
+		{"PostgreSQL down: its own finding says so", relay, nil, errors.New("refused"), ""},
+		{"neither", &config.Config{}, nil, nil, "alerts/relay,alerts/recipients"},
+	} {
+		if got := keys(alertReachFindings(tc.cfg, tc.owners, tc.ownersErr)); got != tc.want {
+			t.Errorf("%s: %q, want %q", tc.what, got, tc.want)
+		}
+	}
+}
+
+// fakeSystemctl answers list-units with failed and is-active from states;
+// a unit missing from states is "inactive", as systemctl says, and one whose
+// state is "" gets no answer.
+func fakeSystemctl(failed string, states map[string]string) func(ctx context.Context, name string, args ...string) ([]byte, error) {
+	return func(_ context.Context, name string, args ...string) ([]byte, error) {
+		if name != "systemctl" || len(args) == 0 {
+			return nil, errors.New("unexpected command " + name)
+		}
+		switch args[0] {
+		case "list-units":
+			return []byte(failed), nil
+		case "is-active":
+			if s, ok := states[args[1]]; ok && s == "" {
+				return nil, errors.New("signal: killed")
+			} else if ok {
+				return []byte(s + "\n"), nil
+			}
+			return []byte("inactive\n"), errors.New("exit status 3")
+		}
+		return nil, errors.New("unexpected systemctl " + args[0])
+	}
+}
+
+func TestUnitFindings(t *testing.T) {
+	dir := t.TempDir()
+	for _, f := range []string{"k3s.service", "felis-velocity.service", "felis-db-backup.timer", "felis-offsite.timer", "felis-offsite.service"} {
+		writeTestFile(t, filepath.Join(dir, f), "[Unit]\n", 0o644)
+	}
+	env := doctorEnv{unitDir: dir, run: fakeSystemctl(
+		"felis-offsite.service loaded failed failed Felis off-site copy\nfelis-velocity.service loaded failed failed Velocity\n",
+		map[string]string{"k3s.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "inactive"},
+	)}
+	var got []string
+	for _, f := range unitFindings(context.Background(), env) {
+		got = append(got, string(f.Severity)+" "+f.Key+": "+f.SummaryEN)
+	}
+	want := []string{
+		"critical unit/felis-offsite.service: felis-offsite.service failed",
+		"critical unit/felis-velocity.service: felis-velocity.service failed",
+		"warning timer/felis-offsite.timer: felis-offsite.timer is inactive: the job it starts no longer runs",
+	}
+	if strings.Join(got, "\n") != strings.Join(want, "\n") {
+		t.Errorf("findings:\n%s\nwant (felis-nano.service has no unit file here, and a failed unit is reported once):\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
+	}
+
+	env.run = fakeSystemctl("", map[string]string{"k3s.service": "activating", "felis-velocity.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "active"})
+	got = nil
+	for _, f := range unitFindings(context.Background(), env) {
+		got = append(got, f.Key+": "+f.SummaryEN)
+	}
+	if strings.Join(got, "\n") != "unit/k3s.service: k3s.service is activating" {
+		t.Errorf("findings %q, want only k3s.service, which is not active", got)
+	}
+
+	env.run = fakeSystemctl("", map[string]string{"k3s.service": "active", "felis-velocity.service": "active", "felis-db-backup.timer": "", "felis-offsite.timer": "active"})
+	got = nil
+	for _, f := range unitFindings(context.Background(), env) {
+		got = append(got, f.Key+": "+f.SummaryEN)
+	}
+	if want := "timer/felis-db-backup.timer: felis-db-backup.timer is in an unknown state (systemctl is-active: signal: killed): the job it starts no longer runs"; strings.Join(got, "\n") != want {
+		t.Errorf("findings %q, want %q", got, want)
+	}
+
+	env.run = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("no systemctl") }
+	if fs := unitFindings(context.Background(), env); len(fs) != 1 || fs[0].Key != "unit/systemctl" || fs[0].Severity != watchdog.Warning {
+		t.Errorf("without systemctl: %+v, want the one unit/systemctl warning", fs)
+	}
+}
+
+// quoteArgs writes args as an ExecStart= line does, each word in quotes so an
+// empty one survives.
+func quoteArgs(args []string) string {
+	q := make([]string, len(args))
+	for i, a := range args {
+		q[i] = `"` + a + `"`
+	}
+	return strings.Join(q, " ")
+}
+
+// doctorHost is a host with the installer's watchdog unit, whose API server
+// and PostgreSQL are down.
+func doctorHost(t *testing.T, cfg string, extraFlags string) (env doctorEnv, h *watchdogHost) {
+	t.Helper()
+	h = newWatchdogHost(t, cfg, nil)
+	unitDir := t.TempDir()
+	writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"),
+		"[Service]\nType=oneshot\nExecStart=/usr/local/bin/felis watchdog "+quoteArgs(h.args)+
+			// Off this machine's disk, whose free space is not the test's.
+			` -disk-paths "/nonexistent-felis-doctor-test"`+extraFlags+"\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-offsite.timer"), "[Unit]\n", 0o644)
+	return doctorEnv{
+		unitDir: unitDir, now: time.Now(), host: "felis-test",
+		run: fakeSystemctl("felis-db-backup.service loaded failed failed Felis database backup\n",
+			map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"}),
+	}, h
+}
+
+func TestDoctorReportsByArea(t *testing.T) {
+	env, _ := doctorHost(t, testWatchdogConfig, "")
+	var out bytes.Buffer
+	code := runDoctor(context.Background(), env, &out)
+	got := out.String()
+	if code != 1 {
+		t.Errorf("exit %d, want 1 with problems found", code)
+	}
+	for _, want := range []string{
+		"felis doctor on felis-test at ",
+		"checks run as " + filepath.Join(env.unitDir, "felis-watchdog.service") + " runs them (config ",
+		"✓  configuration\n",
+		"✗  Kubernetes cluster\n     critical kube-api: ",
+		"✗  PostgreSQL\n     critical postgres: ",
+		"-  game proxy: not checked, no -proxy-addr\n",
+		"-  database backups: not checked, no -backup-dir\n",
+		"-  off-site copy: not checked, not configured\n",
+		"-  build scan database: not checked, builds do not scan against the registry's copy\n",
+		"-  k3s certificates: not checked, no -k3s-cert-dirs\n",
+		"-  node address: not checked, no -node-ip\n",
+		"✗  systemd units and timers\n     critical unit/felis-db-backup.service: felis-db-backup.service failed\n" +
+			"              → journalctl -u felis-db-backup.service -n 100 --no-pager; once fixed, sudo systemctl reset-failed felis-db-backup.service",
+		"!  alerting\n     warning  alerts/relay: no [smtp] relay is configured",
+		"\n4 problem(s): 3 critical, 1 warning(s)\n",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("report lacks %q:\n%s", want, got)
+		}
+	}
+	if strings.Contains(got, "alerts/recipients") {
+		t.Errorf("reported no recipients although PostgreSQL, which names them, is down:\n%s", got)
+	}
+	if strings.Contains(got, "note: no heartbeat URL") {
+		t.Errorf("the host has a heartbeat URL:\n%s", got)
+	}
+}
+
+// A doctor run is only a look: whatever is due to be mailed stays due, no
+// heartbeat is pinged, and the watchdog's state is left as it was.
+func TestDoctorMailsPingsAndSavesNothing(t *testing.T) {
+	cfg := testWatchdogConfig + "[smtp]\nhost = \"smtp.example.com\"\nport = 587\nfrom = \"[email protected]\"\n"
+	env, h := doctorHost(t, cfg, "")
+	if err := watchdog.SaveState(h.statePath, duePostgres("cached-pw")); err != nil {
+		t.Fatal(err)
+	}
+	before, err := os.ReadFile(h.statePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	watchdogSender = h.rec.sender
+	defer func() { watchdogSender = smtpSender }()
+	var out bytes.Buffer
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "critical postgres: ") {
+		t.Fatalf("the due PostgreSQL alert was not seen:\n%s", out.String())
+	}
+	if len(h.rec.sent) != 0 || len(h.rec.relays) != 0 {
+		t.Errorf("mailed %v", h.rec.sent)
+	}
+	if n := len(h.pings.pings); n != 0 {
+		t.Errorf("pinged the heartbeat %d times", n)
+	}
+	after, err := os.ReadFile(h.statePath)
+	if err != nil || !bytes.Equal(before, after) {
+		t.Errorf("the watchdog state changed (err %v)", err)
+	}
+	if _, err := os.Stat(h.fallbackPath); !errors.Is(err, os.ErrNotExist) {
+		t.Errorf("a fallback state was written: %v", err)
+	}
+}
+
+func TestDoctorNotes(t *testing.T) {
+	env, h := doctorHost(t, testWatchdogConfig, "")
+	if err := os.Remove(filepath.Join(h.dir, "watchdog-heartbeat-url")); err != nil {
+		t.Fatal(err)
+	}
+	until := env.now.Add(10 * time.Minute).Unix()
+	writeTestFile(t, filepath.Join(h.dir, "quiet"), strconv.FormatInt(until, 10)+"\n", 0o644)
+	var out bytes.Buffer
+	runDoctor(context.Background(), env, &out)
+	for _, want := range []string{
+		"\nnote: no heartbeat URL is set: ",
+		"\nnote: the watchdog mails nothing until " + time.Unix(until, 0).UTC().Format("2006-01-02 15:04 UTC") + " (" + filepath.Join(h.dir, "quiet") + ")",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("report lacks %q:\n%s", want, out.String())
+		}
+	}
+
+	writeTestFile(t, filepath.Join(h.dir, "watchdog-heartbeat-url"), "not a url\n", 0o600)
+	out.Reset()
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "warning  watchdog/heartbeat: the heartbeat URL is unusable") {
+		t.Errorf("an unusable heartbeat URL is not reported:\n%s", out.String())
+	}
+}
+
+// Without the watchdog's unit the doctor still checks, with the watchdog's
+// defaults, and says the host has no watchdog; a configuration that does not
+// load leaves the checks that need it unchecked and the host's own checks on.
+func TestDoctorWithoutUnitOrConfig(t *testing.T) {
+	env, _ := doctorHost(t, testWatchdogConfig, "")
+	if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
+		t.Fatal(err)
+	}
+	writeTestFile(t, filepath.Join(env.unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+filepath.Join(t.TempDir(), "absent.toml")+"\n", 0o644)
+	env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"})
+	var out bytes.Buffer
+	if code := runDoctor(context.Background(), env, &out); code != 1 {
+		t.Errorf("exit %d, want 1", code)
+	}
+	for _, want := range []string{
+		"✗  configuration\n     critical config: ",
+		"-  Kubernetes cluster: not checked, the configuration did not load\n",
+		"-  PostgreSQL: not checked, the configuration did not load\n",
+		"-  disk space: not checked, the configuration did not load\n",
+		"✓  systemd units and timers\n",
+		"-  alerting: not checked, the configuration did not load\n",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("report lacks %q:\n%s", want, out.String())
+		}
+	}
+
+	if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
+		t.Fatal(err)
+	}
+	out.Reset()
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "critical watchdog/unit: "+filepath.Join(env.unitDir, "felis-watchdog.service")+" is not installed") ||
+		!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
+		t.Errorf("a host without the watchdog's unit:\n%s", out.String())
+	}
+
+	unit := filepath.Join(env.unitDir, "felis-watchdog.service")
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
+	out.Reset()
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "critical watchdog/unit: cannot read the watchdog's settings: "+unit+": flag provided but not defined: -no-such-flag\n") ||
+		!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
+		t.Errorf("a watchdog unit this binary cannot read:\n%s", out.String())
+	}
+}
+
+func TestPrintDoctorReport(t *testing.T) {
+	var out bytes.Buffer
+	if code := printDoctorReport(&out, nil, map[string]string{"proxy": "no -proxy-addr"}, nil); code != 0 {
+		t.Errorf("exit %d with nothing found, want 0", code)
+	}
+	want := "✓  configuration\n✓  Kubernetes cluster\n✓  PostgreSQL\n-  game proxy: not checked, no -proxy-addr\n✓  database backups\n✓  off-site copy\n" +
+		"✓  build scan database\n✓  disk space\n✓  memory\n✓  k3s certificates\n✓  node address\n✓  clock\n✓  systemd units and timers\n✓  alerting\n" +
+		"\nno problems found\n"
+	if out.String() != want {
+		t.Errorf("report:\n%s\nwant:\n%s", out.String(), want)
+	}
+
+	out.Reset()
+	code := printDoctorReport(&out, []watchdog.Finding{
+		{Key: "unit/systemctl", Severity: watchdog.Warning, SummaryEN: "systemctl list-units failed"},
+		{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "PostgreSQL is down", Hint: "kubectl -n felis get pods"},
+	}, map[string]string{"postgres": "a skip loses to what was found"}, []string{"a note"})
+	if code != 1 {
+		t.Errorf("exit %d with problems, want 1", code)
+	}
+	for _, want := range []string{
+		"✗  PostgreSQL\n     critical postgres: PostgreSQL is down\n              → kubectl -n felis get pods\n✓  game proxy\n",
+		"!  systemd units and timers\n     warning  unit/systemctl: systemctl list-units failed\n✓  alerting\n",
+		"✓  alerting\n\nnote: a note\n\n2 problem(s): 1 critical, 1 warning(s)\n",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("report lacks %q:\n%s", want, out.String())
+		}
+	}
+}

cmd/felis/domain_test.go

@@ -0,0 +1,892 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// installerTOML is felis.toml as deploy/bootstrap.sh write_felis_toml renders it,
+// comments included: the domain move has to leave all of it but three values alone.
+func installerTOML(root, dbHost string) string {
+	return `# Generated by deploy/bootstrap.sh; rerun the installer to regenerate. Hand edits are
+# overwritten, except [smtp], [[auth_source]], [offsite], and the operator-owned
+# [registry] / [archive] overrides, which carry forward.
+[server]
+listen = "0.0.0.0:8080"
+root_domain = "` + root + `"
+
+[database]
+url = "postgres://felis:pw@` + dbHost + `:5432/felis?sslmode=disable"
+
+[k8s]
+namespace = "minecraft"
+egress_mode = "nodeport"
+
+[velocity]
+# The two always-on system servers that felis setup provisions.
+login_image = "felis/limbo:1"
+lobby_image = "felis/lobby:1"
+game_port = 25565
+
+[registry]
+url = "registry.felis.svc:5000"
+build_namespace = "felis-build"
+
+[archive]
+store = "tarLocal"
+local_path = "/var/lib/felis/archives"
+
+[auth]
+admin_hostname = "op.console.` + root + `"
+panel_hostname = "console.` + root + `"
+access_jwt_aud = "aud123"
+
+# Third-party Yggdrasil sources federated by the hasJoined multiplexer.
+[[auth_source]]
+tag = "littleskin"
+prefix = "LS"
+url = "https://littleskin.cn/api/yggdrasil/sessionserver/session/minecraft/hasJoined"
+`
+}
+
+const linkPropsBody = `# Generated by deploy/bootstrap.sh — do not edit by hand; rerun the installer.
+api-base-url=http://10.43.0.9:8081
+service-token=TOKEN-NOT-TO-TOUCH
+root-domain=old.example
+panel-hostname=console.old.example
+admin-hostname=op.console.old.example
+login-server=login
+lobby-server=lobby
+`
+
+var oldNames = domainNames{root: "old.example", panel: "console.old.example", admin: "op.console.old.example"}
+var newNames = domainNames{root: "new.example", panel: "console.new.example", admin: "op.console.new.example"}
+
+// domainRig models the host: the files, the cluster, and a felis-api, proxy and
+// operator that pick up config the way the real ones do — the api serves what it
+// read at its last restart, the proxy runs since its last restart, and the login
+// pod carries the env its MinecraftServer had when it was last rolled.
+type domainRig struct {
+	h      domainHost
+	cl     client.Client
+	out    *bytes.Buffer
+	dir    string
+	events []string
+
+	served      domainNames
+	servedCert  *x509.Certificate
+	proxySince  time.Time
+	proxyLoaded bool
+	unresolved  map[string]bool
+	// The fake operator: the CR env the login pod was last rolled to, and how
+	// many looks at the pod since the CR moved on.
+	rolledTo string
+	pending  int
+}
+
+func (rig *domainRig) path(name string) string { return filepath.Join(rig.dir, name) }
+
+func newDomainRig(t *testing.T) *domainRig {
+	t.Helper()
+	rig := &domainRig{out: &bytes.Buffer{}, dir: t.TempDir(), proxyLoaded: true, unresolved: map[string]bool{}}
+	writeTestFile(t, rig.path("felis.host.toml"), installerTOML("old.example", "127.0.0.1"), 0o600)
+	writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
+	if err := os.Symlink(rig.path("felis.host.toml"), rig.path("felis.toml")); err != nil {
+		t.Fatal(err)
+	}
+	certPEM, keyPEM, err := issuePanelCert(oldNames, []net.IP{net.ParseIP("10.211.55.6")}, time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
+	writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
+	// The proxy started before its config was last written, which is how it
+	// stands after an install.
+	rig.proxySince = time.Now().Add(-time.Hour)
+
+	pod := []byte(installerTOML("old.example", "10.211.55.6"))
+	login, err := loginSystemServer("felis/limbo:1", "minecraft", platform.InternalAPIBaseURL("felis"), oldNames.root, oldNames.panel)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lobby, err := lobbySystemServer("felis/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatal(err)
+	}
+	loginPod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"},
+		Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "minecraft", Env: podEnv(login.Spec.Env)}}},
+		Status:     corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}},
+	}
+	rig.rolledTo = envKey(login.Spec.Env)
+	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
+		Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+			if key.Name == naming.SystemLoginServer+"-0" {
+				rig.operatorTick(t, c)
+			}
+			return c.Get(ctx, key, obj, opts...)
+		},
+	}).WithObjects(
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.ConfigSecretName},
+			Data: map[string][]byte{platform.ConfigSecretKey: pod}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: platform.ConfigSecretName},
+			Data: map[string][]byte{platform.ConfigSecretKey: pod}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.APITLSSecretName},
+			Type: corev1.SecretTypeTLS, Data: map[string][]byte{corev1.TLSCertKey: certPEM, corev1.TLSPrivateKeyKey: keyPEM}},
+		login, lobby, loginPod,
+	).Build()
+	rig.served = oldNames
+	rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
+
+	rig.h = domainHost{
+		paths: domainPaths{
+			hostTOML: rig.path("felis.host.toml"), podTOML: rig.path("felis.pod.toml"), defaultTOML: rig.path("felis.toml"),
+			cert: rig.path("panel-tls.crt"), key: rig.path("panel-tls.key"),
+			linkProps: rig.path("felis-link.properties"), tunnelConfig: rig.path("cloudflared.yml"),
+		},
+		cl:        rig.cl,
+		controlNS: "felis",
+		rollAPI: func(ctx context.Context) error {
+			rig.events = append(rig.events, "roll-api")
+			rig.restartAPI(t)
+			return nil
+		},
+		restartUnit: func(_ context.Context, unit string) error {
+			rig.events = append(rig.events, "restart "+unit)
+			rig.proxySince = time.Now().Add(time.Second)
+			return nil
+		},
+		unitState: func(context.Context, string) (unitStatus, error) {
+			return unitStatus{loaded: rig.proxyLoaded, active: rig.proxyLoaded, since: rig.proxySince}, nil
+		},
+		liveAPI: func(context.Context, string) (liveAPIView, error) {
+			return liveAPIView{names: rig.served, cert: rig.servedCert}, nil
+		},
+		lookupHost: func(_ context.Context, host string) ([]string, error) {
+			if rig.unresolved[host] {
+				return nil, errors.New("no such host")
+			}
+			return []string{"10.211.55.6"}, nil
+		},
+		passkeys:  func(context.Context) (int, int, error) { return 3, 2, nil },
+		now:       time.Now,
+		out:       rig.out,
+		loginWait: 50 * time.Millisecond,
+		pollEvery: time.Millisecond,
+	}
+	return rig
+}
+
+func podEnv(env []v1alpha1.EnvVar) []corev1.EnvVar {
+	out := make([]corev1.EnvVar, len(env))
+	for i, e := range env {
+		out[i] = corev1.EnvVar{Name: e.Name, Value: e.Value}
+	}
+	return out
+}
+
+// restartAPI makes the fake felis-api load the config and certificate its
+// Secrets hold now.
+func (rig *domainRig) restartAPI(t *testing.T) {
+	t.Helper()
+	var cfg, tlsSec corev1.Secret
+	ctx := context.Background()
+	if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.ConfigSecretName}, &cfg); err != nil {
+		t.Fatal(err)
+	}
+	if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.APITLSSecretName}, &tlsSec); err != nil {
+		t.Fatal(err)
+	}
+	names, err := tomlDomainNames(cfg.Data[platform.ConfigSecretKey])
+	if err != nil {
+		t.Fatal(err)
+	}
+	rig.served = names
+	rig.servedCert, err = x509.ParseCertificate(mustCertDER(tlsSec.Data[corev1.TLSCertKey]))
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// rollLoginPod is the operator restarting the login pod onto its CR's env.
+// operatorTick is the operator as the login pod is watched: once the CR's env
+// changes it takes operatorLag looks at the pod before the restarted pod
+// carries the new env, the way a real rollout lags the CR.
+func (rig *domainRig) operatorTick(t *testing.T, c client.Client) {
+	t.Helper()
+	ctx := context.Background()
+	var ms v1alpha1.MinecraftServer
+	if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if envKey(ms.Spec.Env) == rig.rolledTo {
+		return
+	}
+	if rig.pending++; rig.pending < operatorLag {
+		return
+	}
+	var pod corev1.Pod
+	if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"}, &pod); err != nil {
+		t.Fatal(err)
+	}
+	pod.Spec.Containers[0].Env = podEnv(ms.Spec.Env)
+	if err := c.Update(ctx, &pod); err != nil {
+		t.Fatal(err)
+	}
+	rig.rolledTo, rig.pending = envKey(ms.Spec.Env), 0
+}
+
+const operatorLag = 3
+
+func envKey(env []v1alpha1.EnvVar) string {
+	var b strings.Builder
+	for _, e := range env {
+		b.WriteString(e.Name + "=" + e.Value + "\n")
+	}
+	return b.String()
+}
+
+func (rig *domainRig) read(t *testing.T, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(rig.path(name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
+func (rig *domainRig) secret(t *testing.T, ns, name string) map[string][]byte {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		t.Fatal(err)
+	}
+	return s.Data
+}
+
+func (rig *domainRig) crEnv(t *testing.T, name string) map[string]string {
+	t.Helper()
+	var ms v1alpha1.MinecraftServer
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	env := map[string]string{}
+	for _, e := range ms.Spec.Env {
+		env[e.Name] = e.Value
+	}
+	return env
+}
+
+// snapshot is every byte `set` may touch, for proving a refused or dry run
+// touched none of it.
+func (rig *domainRig) snapshot(t *testing.T) string {
+	t.Helper()
+	var b strings.Builder
+	entries, _ := os.ReadDir(rig.dir)
+	for _, e := range entries {
+		b.WriteString(e.Name() + "\n" + rig.read(t, e.Name()) + "\n")
+	}
+	var lines []string
+	for _, s := range []struct{ ns, name string }{{"felis", platform.ConfigSecretName}, {"minecraft", platform.ConfigSecretName}, {"felis", platform.APITLSSecretName}} {
+		for k, v := range rig.secret(t, s.ns, s.name) {
+			lines = append(lines, s.ns+"/"+s.name+"/"+k+"\n"+string(v))
+		}
+	}
+	for k, v := range rig.crEnv(t, naming.SystemLoginServer) {
+		lines = append(lines, "env "+k+"="+v)
+	}
+	sort.Strings(lines)
+	b.WriteString(strings.Join(lines, "\n"))
+	return b.String()
+}
+
+func TestNormalizeRootDomain(t *testing.T) {
+	for in, want := range map[string]string{
+		"Example.COM.":          "example.com",
+		"  mc.example.org ":     "mc.example.org",
+		"10.211.55.6.nip.io":    "10.211.55.6.nip.io",
+		"xn--bcher-kva.example": "xn--bcher-kva.example",
+	} {
+		got, err := normalizeRootDomain(in)
+		if err != nil || got != want {
+			t.Errorf("normalizeRootDomain(%q) = %q, %v; want %q", in, got, err, want)
+		}
+	}
+	for _, in := range []string{"", "https://example.com", "example.com:443", "example.com/x", "10.0.0.1", "::1",
+		"localhost", "a_b.example", "-a.example", "a-.example", strings.Repeat("a", 64) + ".example",
+		strings.Repeat("abcdefghi.", 25) + "example"} {
+		if got, err := normalizeRootDomain(in); err == nil {
+			t.Errorf("normalizeRootDomain(%q) = %q, want an error", in, got)
+		}
+	}
+}
+
+func TestPlanDomainChangeMovesDefaultsAndKeepsHandSetNames(t *testing.T) {
+	p := planDomainChange(oldNames, "new.example")
+	if p.to != newNames || p.customPanel || p.customAdmin {
+		t.Fatalf("defaults: %+v", p)
+	}
+	p = planDomainChange(domainNames{root: "old.example", panel: "play.corp.net", admin: "op.console.old.example"}, "new.example")
+	if p.to.panel != "play.corp.net" || !p.customPanel || p.to.admin != "op.console.new.example" || p.customAdmin {
+		t.Fatalf("hand-set panel: %+v", p)
+	}
+	p = planDomainChange(domainNames{root: "old.example", panel: "console.old.example", admin: "admin.corp.net"}, "new.example")
+	if p.to.admin != "admin.corp.net" || !p.customAdmin || p.to.panel != "console.new.example" {
+		t.Fatalf("hand-set admin: %+v", p)
+	}
+}
+
+func TestEditTOMLStringsChangesOnlyTheDomainLines(t *testing.T) {
+	orig := installerTOML("old.example", "127.0.0.1")
+	out, err := editTOMLStrings([]byte(orig), domainTOMLEdits(newNames))
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, b := strings.Split(orig, "\n"), strings.Split(string(out), "\n")
+	if len(a) != len(b) {
+		t.Fatalf("line count %d → %d:\n%s", len(a), len(b), out)
+	}
+	changed := map[string]string{}
+	for i := range a {
+		if a[i] != b[i] {
+			changed[a[i]] = b[i]
+		}
+	}
+	want := map[string]string{
+		`root_domain = "old.example"`:               `root_domain = "new.example"`,
+		`admin_hostname = "op.console.old.example"`: `admin_hostname = "op.console.new.example"`,
+		`panel_hostname = "console.old.example"`:    `panel_hostname = "console.new.example"`,
+	}
+	if len(changed) != len(want) {
+		t.Fatalf("changed lines %v, want %v", changed, want)
+	}
+	for k, v := range want {
+		if changed[k] != v {
+			t.Errorf("%q → %q, want %q", k, changed[k], v)
+		}
+	}
+}
+
+func TestEditTOMLStringsAddsMissingKeysInTheirTable(t *testing.T) {
+	in := "[server]\nroot_domain = \"old.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\n\n[smtp]\nhost = \"relay\"\n"
+	out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "[server]\nroot_domain = \"new.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\npanel_hostname = \"console.new.example\"\nadmin_hostname = \"op.console.new.example\"\n\n[smtp]\nhost = \"relay\"\n"
+	if string(out) != want {
+		t.Fatalf("got:\n%s\nwant:\n%s", out, want)
+	}
+
+	out, err = editTOMLStrings([]byte("[server]\nroot_domain = \"old.example\"\n\n[[auth_source]]\ntag = \"ls\"\n"), domainTOMLEdits(newNames))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := tomlDomainNames(out)
+	if err != nil || got != newNames || !strings.Contains(string(out), "[[auth_source]]\ntag = \"ls\"\n") {
+		t.Fatalf("no [auth] table: %v %+v\n%s", err, got, out)
+	}
+}
+
+func TestEditTOMLStringsRefusesWhatItCannotEditExactly(t *testing.T) {
+	for name, in := range map[string]string{
+		"multi-line value": "[server]\nroot_domain = \"\"\"\nold.example\"\"\"\n[auth]\n",
+		// The key's line sits inside another value; the real key is absent.
+		"key inside a string": "[server]\nmotd = \"\"\"\nroot_domain = \"old.example\"\n\"\"\"\n[auth]\n",
+		"quoted header":       "[server]\nroot_domain = \"old.example\"\n[\"auth\"]\npanel_hostname = \"console.old.example\"\n",
+		"dotted key":          "server.root_domain = \"old.example\"\n",
+		"inline table":        "server = { root_domain = \"old.example\" }\n",
+	} {
+		if out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames)); err == nil {
+			t.Errorf("%s: edited instead of refusing:\n%s", name, out)
+		}
+	}
+}
+
+// caSignedCert is an operator's certificate from their own CA; it names
+// localhost too, so only the issuer tells it apart from the installer's.
+func caSignedCert(t *testing.T, hosts ...string) (certPEM, keyPEM []byte) {
+	t.Helper()
+	caKey, _ := rsa.GenerateKey(rand.Reader, 2048)
+	ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Corp CA"}, IsCA: true,
+		BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
+	caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	caCert, _ := x509.ParseCertificate(caDER)
+	key, _ := rsa.GenerateKey(rand.Reader, 2048)
+	leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: hosts[0]},
+		DNSNames: append(hosts, "localhost"), IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
+		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
+	der, err := x509.CreateCertificate(rand.Reader, leaf, caCert, &key.PublicKey, caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pk, _ := x509.MarshalPKCS8PrivateKey(key)
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pk})
+}
+
+func TestFelisIssuedCert(t *testing.T) {
+	mine, _, err := issuePanelCert(oldNames, nil, time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	c, _ := x509.ParseCertificate(mustCertDER(mine))
+	if !felisIssuedCert(c) {
+		t.Error("the installer's kind of certificate is not recognised as Felis-issued")
+	}
+	theirs, _ := caSignedCert(t, "console.old.example")
+	c, _ = x509.ParseCertificate(mustCertDER(theirs))
+	if felisIssuedCert(c) {
+		t.Error("a CA-signed certificate is taken for Felis-issued")
+	}
+
+	// Self-signed but without one of the installer's localhost names: someone
+	// else's.
+	key, _ := rsa.GenerateKey(rand.Reader, 2048)
+	for name, self := range map[string]*x509.Certificate{
+		"no localhost": {DNSNames: []string{"console.old.example"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}},
+		"no 127.0.0.1": {DNSNames: []string{"console.old.example", "localhost"}},
+	} {
+		self.SerialNumber, self.Subject = big.NewInt(3), pkix.Name{CommonName: "x"}
+		self.NotBefore, self.NotAfter = time.Now().Add(-time.Hour), time.Now().Add(time.Hour)
+		der, _ := x509.CreateCertificate(rand.Reader, self, self, &key.PublicKey, key)
+		c, _ = x509.ParseCertificate(der)
+		if felisIssuedCert(c) {
+			t.Errorf("%s: a self-signed certificate is taken for Felis-issued", name)
+		}
+	}
+}
+
+func TestIssuePanelCertIsTheInstallersShape(t *testing.T) {
+	now := time.Now()
+	certPEM, keyPEM, err := issuePanelCert(newNames, []net.IP{net.ParseIP("10.211.55.6"), net.IPv4(127, 0, 0, 1)}, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil {
+		t.Fatalf("key does not match the certificate: %v", err)
+	}
+	if b, _ := pem.Decode(keyPEM); b == nil || b.Type != "PRIVATE KEY" {
+		t.Fatalf("key is not PKCS#8 PEM like openssl writes")
+	}
+	c, _ := x509.ParseCertificate(mustCertDER(certPEM))
+	if !certCovers(c, newNames.panel, newNames.admin, "localhost") || !felisIssuedCert(c) {
+		t.Fatalf("names %v", c.DNSNames)
+	}
+	if len(c.IPAddresses) != 2 || !c.IPAddresses[1].Equal(net.ParseIP("10.211.55.6")) {
+		t.Fatalf("addresses %v, want 127.0.0.1 and the node address once each", c.IPAddresses)
+	}
+	if c.Subject.CommonName != newNames.admin || c.NotAfter.Sub(now) < 824*24*time.Hour || c.NotAfter.Sub(now) > 826*24*time.Hour {
+		t.Fatalf("CN %q, valid until %s", c.Subject.CommonName, c.NotAfter)
+	}
+	if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth || c.IsCA {
+		t.Fatalf("usage %v, CA %v", c.ExtKeyUsage, c.IsCA)
+	}
+}
+
+func TestDomainSetMovesEverySurface(t *testing.T) {
+	rig := newDomainRig(t)
+	hostBefore := rig.read(t, "felis.host.toml")
+	code, err := rig.h.set(context.Background(), "New.Example", true)
+	if err != nil || code != 0 {
+		t.Fatalf("set = %d, %v\n%s", code, err, rig.out)
+	}
+
+	// The configs: the three values moved, everything else — comments, the
+	// database host of each copy, the Access audience — is as it was.
+	host := rig.read(t, "felis.host.toml")
+	if want := strings.NewReplacer("old.example", "new.example").Replace(hostBefore); host != want {
+		t.Fatalf("host toml:\n%s", host)
+	}
+	pod := rig.read(t, "felis.pod.toml")
+	if got, _ := tomlDomainNames([]byte(pod)); got != newNames || !strings.Contains(pod, "@10.211.55.6:5432") {
+		t.Fatalf("pod toml:\n%s", pod)
+	}
+	if link, err := os.Readlink(rig.path("felis.toml")); err != nil || link != rig.path("felis.host.toml") {
+		t.Fatalf("felis.toml is no longer the link to the host copy: %q %v", link, err)
+	}
+	if st, _ := os.Stat(rig.path("felis.host.toml")); st.Mode().Perm() != 0o600 {
+		t.Fatalf("host toml mode %v", st.Mode().Perm())
+	}
+
+	// The certificate: reissued for the new names, the node address kept, the old
+	// pair beside it.
+	c, err := readCertFile(rig.path("panel-tls.crt"))
+	if err != nil || !certCovers(c, newNames.panel, newNames.admin) || !felisIssuedCert(c) {
+		t.Fatalf("certificate: %v %v", err, c.DNSNames)
+	}
+	if !c.IPAddresses[len(c.IPAddresses)-1].Equal(net.ParseIP("10.211.55.6")) {
+		t.Fatalf("addresses %v", c.IPAddresses)
+	}
+	if _, err := tls.LoadX509KeyPair(rig.path("panel-tls.crt"), rig.path("panel-tls.key")); err != nil {
+		t.Fatalf("new pair: %v", err)
+	}
+	if st, _ := os.Stat(rig.path("panel-tls.key")); st.Mode().Perm() != 0o600 {
+		t.Fatalf("key mode %v", st.Mode().Perm())
+	}
+	backups, _ := filepath.Glob(rig.path("panel-tls.*.pre-domain-*"))
+	if len(backups) != 2 {
+		t.Fatalf("old pair kept as %v", backups)
+	}
+	for _, b := range backups {
+		if st, _ := os.Stat(b); strings.Contains(b, ".key.") && st.Mode().Perm() != 0o600 {
+			t.Fatalf("kept key %s has mode %v", b, st.Mode().Perm())
+		}
+		old, _ := os.ReadFile(b)
+		if strings.Contains(b, ".crt.") {
+			oc, _ := x509.ParseCertificate(mustCertDER(old))
+			if oc == nil || !certCovers(oc, oldNames.panel) {
+				t.Fatalf("kept certificate is not the old one")
+			}
+		}
+	}
+
+	// The Secrets carry the files.
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secret(t, ns, platform.ConfigSecretName)[platform.ConfigSecretKey]; string(got) != pod {
+			t.Fatalf("%s/felis-config is not felis.pod.toml", ns)
+		}
+	}
+	tlsData := rig.secret(t, "felis", platform.APITLSSecretName)
+	if string(tlsData[corev1.TLSCertKey]) != rig.read(t, "panel-tls.crt") || string(tlsData[corev1.TLSPrivateKeyKey]) != rig.read(t, "panel-tls.key") {
+		t.Fatal("felis-api-tls does not hold the new pair")
+	}
+
+	// The login gate's env moved and nothing else did.
+	env := rig.crEnv(t, naming.SystemLoginServer)
+	if env[envRootDomain] != newNames.root || env[envPanelHostname] != newNames.panel || env[envAPIBaseURL] != platform.InternalAPIBaseURL("felis") {
+		t.Fatalf("login env %v", env)
+	}
+
+	// The proxy's file: the three keys moved, its token and mode did not.
+	props := rig.read(t, "felis-link.properties")
+	if want := strings.NewReplacer("old.example", "new.example").Replace(linkPropsBody); props != want {
+		t.Fatalf("felis-link.properties:\n%s", props)
+	}
+	if st, _ := os.Stat(rig.path("felis-link.properties")); st.Mode().Perm() != 0o640 {
+		t.Fatalf("felis-link.properties mode %v", st.Mode().Perm())
+	}
+
+	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+		t.Fatalf("events %v", rig.events)
+	}
+	if !strings.Contains(rig.out.String(), "Every surface is on new.example.") {
+		t.Fatalf("the closing check did not pass:\n%s", rig.out)
+	}
+}
+
+func TestDomainSetWithoutYesChangesNothing(t *testing.T) {
+	rig := newDomainRig(t)
+	before := rig.snapshot(t)
+	code, err := rig.h.set(context.Background(), "new.example", false)
+	if err != nil || code != 0 {
+		t.Fatalf("set = %d, %v", code, err)
+	}
+	if rig.snapshot(t) != before || len(rig.events) != 0 {
+		t.Fatalf("a dry run changed something (events %v)", rig.events)
+	}
+	out := rig.out.String()
+	for _, want := range []string{
+		"console.old.example → console.new.example",
+		"3 passkey(s) of 2 user(s) are bound to console.old.example",
+		"does not cover op.console.new.example",
+		"No [smtp] relay is configured",
+		"sudo felis domain set -yes new.example",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("plan lacks %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestDomainSetKeepsAHandSetPanelHostname(t *testing.T) {
+	rig := newDomainRig(t)
+	for _, f := range []string{"felis.host.toml", "felis.pod.toml"} {
+		writeTestFile(t, rig.path(f), strings.Replace(rig.read(t, f), `panel_hostname = "console.old.example"`, `panel_hostname = "play.corp.net"`, 1), 0o600)
+	}
+	code, err := rig.h.set(context.Background(), "new.example", true)
+	if err != nil {
+		t.Fatalf("set: %v\n%s", err, rig.out)
+	}
+	got, _ := tomlDomainNames([]byte(rig.read(t, "felis.host.toml")))
+	if got != (domainNames{root: "new.example", panel: "play.corp.net", admin: "op.console.new.example"}) {
+		t.Fatalf("names %+v", got)
+	}
+	c, _ := readCertFile(rig.path("panel-tls.crt"))
+	if !certCovers(c, "play.corp.net", "op.console.new.example") {
+		t.Fatalf("certificate names %v", c.DNSNames)
+	}
+	if env := rig.crEnv(t, naming.SystemLoginServer); env[envPanelHostname] != "play.corp.net" {
+		t.Fatalf("login env %v", env)
+	}
+	if code != 0 || !strings.Contains(rig.out.String(), "play.corp.net (set by hand, kept") {
+		t.Fatalf("code %d:\n%s", code, rig.out)
+	}
+}
+
+func TestDomainSetRefusesAnOperatorCertificateForOtherNames(t *testing.T) {
+	rig := newDomainRig(t)
+	certPEM, keyPEM := caSignedCert(t, "console.old.example", "op.console.old.example")
+	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
+	before := rig.snapshot(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "not issued by Felis") {
+		t.Fatalf("err = %v", err)
+	}
+	if rig.snapshot(t) != before || len(rig.events) != 0 {
+		t.Fatal("a refused move changed something")
+	}
+
+	// The operator's certificate for the new names is kept as it is.
+	certPEM, keyPEM = caSignedCert(t, "console.new.example", "op.console.new.example")
+	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatalf("set: %v", err)
+	}
+	if rig.read(t, "panel-tls.crt") != string(certPEM) {
+		t.Fatal("the operator's certificate was replaced")
+	}
+}
+
+func TestDomainSetRefusesAConfigItCannotEdit(t *testing.T) {
+	rig := newDomainRig(t)
+	writeTestFile(t, rig.path("felis.pod.toml"), strings.Replace(rig.read(t, "felis.pod.toml"), "[auth]", "[\"auth\"]", 1), 0o600)
+	before := rig.snapshot(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "by hand") {
+		t.Fatalf("err = %v", err)
+	}
+	if rig.snapshot(t) != before || len(rig.events) != 0 {
+		t.Fatal("a refused move changed something")
+	}
+}
+
+func TestDomainSetAgainOnlyConvergesWhatIsBehind(t *testing.T) {
+	rig := newDomainRig(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	rig.events, rig.out = nil, &bytes.Buffer{}
+	rig.h.out = rig.out
+	before := rig.snapshot(t)
+	code, err := rig.h.set(context.Background(), "new.example", true)
+	if err != nil || code != 0 {
+		t.Fatalf("second set = %d, %v\n%s", code, err, rig.out)
+	}
+	if len(rig.events) != 0 || rig.snapshot(t) != before {
+		t.Fatalf("a converged install was touched again: %v\n%s", rig.events, rig.out)
+	}
+
+	// A proxy that was not restarted after the move is restarted by a re-run, and
+	// an api still on the old config is rolled.
+	rig.proxySince = time.Now().Add(-time.Hour)
+	rig.served = oldNames
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+		t.Fatalf("events %v", rig.events)
+	}
+
+	// An api on the new names that still presents the old certificate is rolled.
+	rig.events = nil
+	oldCert, _, _ := issuePanelCert(oldNames, nil, time.Now())
+	rig.servedCert, _ = x509.ParseCertificate(mustCertDER(oldCert))
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll-api" {
+		t.Fatalf("events %v", rig.events)
+	}
+}
+
+func TestDomainSetRefusesALoginServerItDoesNotOwn(t *testing.T) {
+	rig := newDomainRig(t)
+	var ms v1alpha1.MinecraftServer
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	delete(ms.Labels, v1alpha1.LabelSystemRole)
+	if err := rig.cl.Update(context.Background(), &ms); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "system role") {
+		t.Fatalf("err = %v", err)
+	}
+	if env := rig.crEnv(t, naming.SystemLoginServer); env[envRootDomain] != oldNames.root {
+		t.Fatalf("a server not marked as the login gate was changed: %v", env)
+	}
+}
+
+func TestDomainCheckNamesTheSurfaceThatIsBehind(t *testing.T) {
+	cases := []struct {
+		surface string
+		breakIt func(t *testing.T, rig *domainRig)
+	}{
+		{"felis.pod.toml", func(t *testing.T, rig *domainRig) {
+			writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
+		}},
+		{"Secret minecraft/felis-config", func(t *testing.T, rig *domainRig) {
+			rig.putSecret(t, "minecraft", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
+		}},
+		{"Secret felis/felis-config", func(t *testing.T, rig *domainRig) {
+			rig.putSecret(t, "felis", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
+		}},
+		{"panel certificate", func(t *testing.T, rig *domainRig) {
+			// The Secret follows the file, so only the certificate's names are wrong.
+			certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
+			writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+			rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
+		}},
+		{"Secret felis/felis-api-tls", func(t *testing.T, rig *domainRig) {
+			certPEM, _, _ := issuePanelCert(newNames, nil, time.Now())
+			rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
+		}},
+		{"felis-api", func(t *testing.T, rig *domainRig) { rig.served = oldNames }},
+		{"felis-api", func(t *testing.T, rig *domainRig) {
+			certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
+			rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
+		}},
+		{"proxy", func(t *testing.T, rig *domainRig) {
+			writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
+			rig.proxySince = time.Now().Add(time.Hour)
+		}},
+		{"proxy", func(t *testing.T, rig *domainRig) { rig.proxySince = time.Now().Add(-time.Hour) }},
+		{"login gate", func(t *testing.T, rig *domainRig) {
+			var ms v1alpha1.MinecraftServer
+			_ = rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms)
+			for i := range ms.Spec.Env {
+				if ms.Spec.Env[i].Name == envRootDomain {
+					ms.Spec.Env[i].Value = "old.example"
+				}
+			}
+			if err := rig.cl.Update(context.Background(), &ms); err != nil {
+				t.Fatal(err)
+			}
+		}},
+		{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envRootDomain, "old.example") }},
+		{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envPanelHostname, "console.old.example") }},
+		{"Cloudflare tunnel", func(t *testing.T, rig *domainRig) {
+			writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n  service: https://127.0.0.1:30443\n- hostname: op.console.old.example\n  service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
+		}},
+	}
+	for _, tc := range cases {
+		rig := newDomainRig(t)
+		if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+			t.Fatal(err)
+		}
+		rig.out.Reset()
+		tc.breakIt(t, rig)
+		if code := rig.h.check(context.Background()); code != 1 {
+			t.Errorf("%s behind: check = %d\n%s", tc.surface, code, rig.out)
+			continue
+		}
+		var failed []string
+		for _, ln := range strings.Split(rig.out.String(), "\n") {
+			if strings.HasPrefix(ln, "  FAIL ") {
+				failed = append(failed, ln)
+			}
+		}
+		if len(failed) != 1 || !strings.Contains(failed[0], tc.surface) {
+			t.Errorf("%s behind: FAIL lines %q", tc.surface, failed)
+		}
+	}
+}
+
+func TestDomainCheckPassesAConvergedInstallAndWarnsOnDNS(t *testing.T) {
+	rig := newDomainRig(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n  service: https://127.0.0.1:30443\n- hostname: op.console.new.example\n  service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
+	rig.unresolved["op.console.new.example"] = true
+	rig.unresolved[dnsProbeLabel+".new.example"] = true
+	rig.out.Reset()
+	if code := rig.h.check(context.Background()); code != 0 {
+		t.Fatalf("check = %d\n%s", code, rig.out)
+	}
+	out := rig.out.String()
+	for _, want := range []string{"  ok   Cloudflare tunnel: routes both names", "  warn DNS: op.console.new.example, *.new.example do not resolve from this host\n"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("check lacks %q:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "TOKEN-NOT-TO-TOUCH") {
+		t.Fatal("check printed the proxy's service token")
+	}
+
+	// A zone with only the wildcard: the admin name alone is missing, and why is said.
+	delete(rig.unresolved, dnsProbeLabel+".new.example")
+	rig.out.Reset()
+	rig.h.check(context.Background())
+	if want := "  warn DNS: op.console.new.example does not resolve from this host: the *.new.example wildcard does not cover op.console.new.example, which needs its own record\n"; !strings.Contains(rig.out.String(), want) {
+		t.Errorf("check lacks %q:\n%s", want, rig.out)
+	}
+}
+
+func (rig *domainRig) setPodEnv(t *testing.T, name, value string) {
+	t.Helper()
+	var pod corev1.Pod
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+		t.Fatal(err)
+	}
+	for i, e := range pod.Spec.Containers[0].Env {
+		if e.Name == name {
+			pod.Spec.Containers[0].Env[i].Value = value
+		}
+	}
+	if err := rig.cl.Update(context.Background(), &pod); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (rig *domainRig) putSecret(t *testing.T, ns, name, key, val string) {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		t.Fatal(err)
+	}
+	s.Data[key] = []byte(val)
+	if err := rig.cl.Update(context.Background(), &s); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestParseUnitShow(t *testing.T) {
+	st := parseUnitShow("LoadState=loaded\nActiveState=active\nActiveEnterTimestamp=@1790000000\n")
+	if !st.loaded || !st.active || !st.since.Equal(time.Unix(1790000000, 0)) {
+		t.Fatalf("%+v", st)
+	}
+	st = parseUnitShow("LoadState=not-found\nActiveState=inactive\nActiveEnterTimestamp=\n")
+	if st.loaded || st.active || !st.since.IsZero() {
+		t.Fatalf("%+v", st)
+	}
+}

cmd/felis/egressgate.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"time"
+)
+
+// Vars so tests can shrink them. A dial that neither connects nor is refused
+// within egressDialTimeout counts as blocked: a policy that drops packets looks
+// exactly like that.
+var (
+	egressDialTimeout  = 500 * time.Millisecond
+	egressPollInterval = 200 * time.Millisecond
+)
+
+// cmdEgressGate is the first initContainer of every build pod and the last of
+// every game server pod. A pod's NetworkPolicy is programmed asynchronously after
+// the pod starts (live on k3s: a build-labelled pod reached the internet and the
+// Kubernetes API for its first ~0.7 s, a server-labelled one felis-api's internal
+// face on its first request), so the gate dials a destination the policy denies
+// until it stops answering, and only then lets the pod's next container, the
+// untrusted Dockerfile or server image, start.
+//
+// The default probe is the Kubernetes API Service, which the kubelet names in
+// every pod's environment and neither policy admits. A probe that still answers
+// after --wait means the policy is not enforced at all (a CNI without
+// NetworkPolicy support, or k3s run with --disable-network-policy), and the
+// build fails closed. A server passes --fail-open: an operator's
+// --server-egress-allow-cidr may cover the node the API Service leads to, so a
+// probe that keeps answering does not prove the fence is missing, and by then
+// the policy has had --wait to land.
+func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	probe := fs.String("probe", "", "host:port the pod's NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
+	wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the gate gives up")
+	failOpen := fs.Bool("fail-open", false, "when --wait runs out, warn and let the pod go on instead of refusing it")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *probe == "" {
+		host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
+		if host == "" || port == "" {
+			fmt.Fprintln(stderr, "felis egress-gate: no --probe and no KUBERNETES_SERVICE_HOST/PORT to default to")
+			return 2
+		}
+		*probe = net.JoinHostPort(host, port)
+	}
+
+	start := time.Now()
+	for {
+		conn, err := net.DialTimeout("tcp", *probe, egressDialTimeout)
+		if err != nil {
+			fmt.Fprintf(stdout, "felis egress-gate: %s is unreachable after %s (%v); the egress lock is in effect\n",
+				*probe, time.Since(start).Round(time.Millisecond), err)
+			return 0
+		}
+		_ = conn.Close()
+		if time.Since(start) >= *wait {
+			if *failOpen {
+				fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s; starting anyway. Either this namespace's "+
+					"NetworkPolicy is not enforced (a CNI without NetworkPolicy support, or k3s started with "+
+					"--disable-network-policy), or an allowed CIDR admits the address behind it\n", *probe, *wait)
+				return 0
+			}
+			fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s: the build namespace's NetworkPolicy is not enforced "+
+				"(a CNI without NetworkPolicy support, or k3s started with --disable-network-policy); refusing to run the build\n",
+				*probe, *wait)
+			return 1
+		}
+		time.Sleep(egressPollInterval)
+	}
+}

cmd/felis/egressgate_test.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"bytes"
+	"net"
+	"strings"
+	"testing"
+	"time"
+)
+
+func shrinkEgressGate(t *testing.T) {
+	t.Helper()
+	dial, poll := egressDialTimeout, egressPollInterval
+	egressDialTimeout, egressPollInterval = 200*time.Millisecond, 10*time.Millisecond
+	t.Cleanup(func() { egressDialTimeout, egressPollInterval = dial, poll })
+}
+
+// The gate holds while the probe answers and lets the pod go on once the policy
+// lands, which the test plays by closing the listener.
+func TestEgressGateWaitsForTheLock(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	accepted := make(chan struct{}, 100)
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+			accepted <- struct{}{}
+		}
+	}()
+	go func() {
+		for i := 0; i < 3; i++ {
+			<-accepted
+		}
+		_ = ln.Close()
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "10s"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if !strings.Contains(out.String(), "egress lock is in effect") {
+		t.Errorf("stdout = %q", out.String())
+	}
+}
+
+// A probe that keeps answering means no policy is enforced: the build must not run.
+func TestEgressGateRefusesAnOpenNetwork(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+		}
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "100ms"}, &out, &errb); code != 1 {
+		t.Fatalf("exit %d, want 1", code)
+	}
+	if !strings.Contains(errb.String(), "not enforced") {
+		t.Errorf("stderr = %q", errb.String())
+	}
+}
+
+// A server's gate waits out --wait all the same, then lets the pod start with a
+// warning in its log.
+func TestEgressGateFailOpenWaitsThenWarns(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+		}
+	}()
+	var out, errb bytes.Buffer
+	start := time.Now()
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "150ms", "--fail-open"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d, want 0: %s", code, errb.String())
+	}
+	if waited := time.Since(start); waited < 150*time.Millisecond {
+		t.Errorf("gave up after %s, before --wait ran out", waited)
+	}
+	if !strings.Contains(errb.String(), ln.Addr().String()+" still answers after 150ms; starting anyway") {
+		t.Errorf("stderr = %q", errb.String())
+	}
+	if out.Len() != 0 {
+		t.Errorf("stdout = %q, want nothing: the lock was never seen", out.String())
+	}
+}
+
+func TestEgressGateDefaultsToTheKubernetesService(t *testing.T) {
+	shrinkEgressGate(t)
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	t.Setenv("KUBERNETES_SERVICE_PORT", "")
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate(nil, &out, &errb); code != 2 {
+		t.Fatalf("exit %d without a probe, want 2", code)
+	}
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	host, port, _ := net.SplitHostPort(ln.Addr().String())
+	_ = ln.Close() // closed: the lock reads as in effect at once
+	t.Setenv("KUBERNETES_SERVICE_HOST", host)
+	t.Setenv("KUBERNETES_SERVICE_PORT", port)
+	out.Reset()
+	if code := cmdEgressGate(nil, &out, &errb); code != 0 || !strings.Contains(out.String(), ln.Addr().String()) {
+		t.Fatalf("exit %d, stdout %q", code, out.String())
+	}
+}

cmd/felis/export.go

@@ -0,0 +1,302 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"hash"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/backup"
+	"felis.lolicon.best/internal/fileedit"
+	"felis.lolicon.best/internal/worldexport"
+)
+
+// cmdExport is the in-Pod entrypoint the export Job runs. internal/worldexport
+// renders a Pod whose command is `/usr/local/bin/felis export`. It archives the
+// mounted world, re-streams one archive from the mounted backup store, or sends
+// one file or folder of the world, PUTs it to felis-api's internal face, and
+// exits once felis-api says the owner's browser got all of it. It is NOT a
+// user-facing command and is never invoked by hand.
+//
+// Like cmdRestore it holds no database credentials and never calls config.Load:
+// felis-api made every decision (who may download what, that the server is
+// stopped, which archive) before the Job existed. Its input is the flags below
+// plus the one-time upload token in the environment, which opens this one
+// export and nothing else.
+//
+// Whatever leaves goes through the same guards as the file editor
+// (fileedit.Guard): the proxy forwarding secret, which every server on the
+// install shares, never leaves, and server.properties leaves with its RCON
+// password redacted. A backup is stored with both, since a restore must bring
+// the world back whole, so it is filtered on the way out rather than handed
+// over as stored.
+//
+// Exit status: 0 once felis-api answers 204 (the download completed), 1 when
+// the export could not be read or handed over, a backup failed its digest
+// check, or felis-api refused it (the browser never came or left early), 2 on
+// bad flags. The last stderr line reaches the export's status and the jobs list.
+func cmdExport(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("export", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	mode := fs.String("mode", "", "what to export: world, backup or files")
+	server := fs.String("server", "", "server name being exported (for logging)")
+	target := fs.String("target-url", "", "felis-api URL to PUT the export to")
+	ref := fs.String("ref", "", "backup only: absolute path to the archive on the backup mount")
+	backupRoot := fs.String("backup-root", "/backups", "backup only: mount path of the backup PVC (the ref must resolve under it)")
+	sum := fs.String("sha256", "", "backup only: the sha256 recorded when the archive was written; a mismatch fails the export before its end is sent")
+	worldsRoot := fs.String("worlds-root", "/world", "world and files: mount path of the world PVC")
+	path := fs.String("path", "", "files only: the file or folder to send, relative to the world root")
+	dir := fs.Bool("dir", false, "files only: the path is a folder, sent as a zip")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	token := os.Getenv(worldexport.TokenEnv)
+	if *target == "" || token == "" {
+		fmt.Fprintf(stderr, "felis export: --target-url and %s are required\n", worldexport.TokenEnv)
+		return 2
+	}
+	limitHeapToCgroup()
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	var err error
+	switch *mode {
+	case worldexport.ModeBackup:
+		if *ref == "" {
+			fmt.Fprintln(stderr, "felis export: --ref is required for a backup")
+			return 2
+		}
+		err = exportBackup(ctx, *target, token, *ref, *backupRoot, *sum, stdout)
+	case worldexport.ModeWorld:
+		err = exportWorld(ctx, *target, token, *worldsRoot, stdout)
+	case worldexport.ModeFiles:
+		if *path == "" {
+			fmt.Fprintln(stderr, "felis export: --path is required for files")
+			return 2
+		}
+		err = exportFiles(ctx, *target, token, *worldsRoot, *path, *dir, stdout)
+	default:
+		fmt.Fprintf(stderr, "felis export: --mode must be %s, %s or %s\n", worldexport.ModeWorld, worldexport.ModeBackup, worldexport.ModeFiles)
+		return 2
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis export: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis export: server=%s mode=%s downloaded\n", *server, *mode)
+	return 0
+}
+
+// archiveType is the content type of a world or backup export.
+const archiveType = "application/gzip"
+
+// errBackupDigest fails a backup export whose stored archive no longer hashes
+// to what was recorded when it was written.
+var errBackupDigest = errors.New("the backup archive does not match the sha256 recorded when it was written")
+
+// exportBackup re-streams one stored archive through the export guards
+// (backup.FilterTarGz with archiveFilter). Its length changes on the way, so it
+// goes chunked. With want set, the stored bytes are hashed as they are read,
+// and FilterTarGz reads them to their end before it closes its own archive: a
+// mismatch aborts the upload while what felis-api has passed on still lacks
+// its end, so the browser never keeps a complete-looking corrupt file.
+func exportBackup(ctx context.Context, target, token, ref, root, want string, stdout io.Writer) error {
+	// Defense in depth, as in cmdRestore: the ref comes from felis-api, but this
+	// process opens it, so it confirms the ref stays on the backup mount.
+	if !refWithinRoot(ref, root) {
+		return fmt.Errorf("ref %q is not under backup root %q", ref, root)
+	}
+	f, err := os.Open(ref)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	var src io.Reader = f
+	if want != "" {
+		src = &digestReader{r: f, sum: sha256.New(), want: want}
+	}
+	var withheld []string
+	err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
+		var err error
+		withheld, err = backup.FilterTarGz(ctx, w, src, archiveFilter)
+		return err
+	})
+	if errors.Is(err, errBackupDigest) {
+		return errBackupDigest // the jobs list shows it as it is, not wrapped as a read error
+	}
+	reportWithheld(stdout, len(withheld))
+	return err
+}
+
+// exportWorld archives the world straight into the request body: nothing is
+// staged, so a world bigger than the Pod's memory or any scratch disk exports
+// the same.
+func exportWorld(ctx context.Context, target, token, root string, stdout io.Writer) error {
+	r, err := os.OpenRoot(root)
+	if err != nil {
+		return err
+	}
+	guard := fileedit.NewGuard(r)
+	r.Close()
+	var skipped, withheld []string
+	err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
+		var err error
+		skipped, withheld, err = backup.WriteTarGz(ctx, w, root, worldFilter(guard))
+		return err
+	})
+	if len(skipped) > 0 {
+		fmt.Fprintf(stdout, "felis export: left out %d entries a tar cannot hold (symbolic links, devices, sockets)\n", len(skipped))
+	}
+	reportWithheld(stdout, len(withheld))
+	return err
+}
+
+// exportFiles sends one file or folder of the world (fileedit.OpenDownload): a
+// file with its exact length, a folder as a zip made as it streams. dir is
+// what the owner saw at path when they asked.
+func exportFiles(ctx context.Context, target, token, root, path string, dir bool, stdout io.Writer) error {
+	d, err := fileedit.OpenDownload(root, path, dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	err = streamExport(ctx, target, token, d.ContentType, d.Size, func(w io.Writer) error { return d.WriteTo(ctx, w) })
+	if d.Skipped > 0 {
+		fmt.Fprintf(stdout, "felis export: left out %d entries a zip does not carry (symbolic links, devices, sockets)\n", d.Skipped)
+	}
+	reportWithheld(stdout, d.Withheld)
+	return err
+}
+
+func reportWithheld(stdout io.Writer, n int) {
+	if n > 0 {
+		fmt.Fprintf(stdout, "felis export: left out %d files that hold platform secrets\n", n)
+	}
+}
+
+// worldFilter guards a live world by file identity, so a link to a guarded
+// file under another name is caught as well.
+func worldFilter(g fileedit.Guard) backup.Filter {
+	return func(_ string, info fs.FileInfo) (bool, func([]byte) []byte) {
+		return guardAction(g.Rule(info))
+	}
+}
+
+// archiveFilter guards a stored archive, which has only names.
+func archiveFilter(name string, _ fs.FileInfo) (bool, func([]byte) []byte) {
+	return guardAction(fileedit.ArchiveRule(name))
+}
+
+func guardAction(withhold, redact bool) (bool, func([]byte) []byte) {
+	if redact {
+		return withhold, fileedit.RedactProps
+	}
+	return withhold, nil
+}
+
+// digestReader passes r through, hashing it, and turns r's EOF into
+// errBackupDigest when the bytes do not hash to want.
+type digestReader struct {
+	r    io.Reader
+	sum  hash.Hash
+	want string
+}
+
+func (d *digestReader) Read(p []byte) (int, error) {
+	n, err := d.r.Read(p)
+	d.sum.Write(p[:n])
+	if err == io.EOF && !strings.EqualFold(hex.EncodeToString(d.sum.Sum(nil)), d.want) {
+		return n, errBackupDigest
+	}
+	return n, err
+}
+
+// streamExport runs write straight into the body of the PUT, hashing it as it
+// goes. Once write has finished, the SHA-256 of all it wrote rides the
+// request's trailer (worldexport.DigestTrailer), and felis-api holds back the
+// last bytes from the browser until what it received hashes the same. An error
+// from write aborts the chunked body before the trailer, and felis-api then
+// cuts the browser's download off rather than end it; that error is the one
+// reported, since the PUT's own error only wraps it. When the PUT ends first,
+// write is stopped.
+func streamExport(ctx context.Context, target, token, contentType string, size int64, write func(io.Writer) error) error {
+	pr, pw := io.Pipe()
+	trailer := http.Header{worldexport.DigestTrailer: nil}
+	werr := make(chan error, 1)
+	go func() {
+		sum := sha256.New()
+		err := write(io.MultiWriter(pw, sum))
+		if err == nil {
+			// Set before the body ends: the transport reads the trailer once it
+			// has read the body to its end.
+			trailer.Set(worldexport.DigestTrailer, "sha-256=:"+base64.StdEncoding.EncodeToString(sum.Sum(nil))+":")
+		}
+		pw.CloseWithError(err)
+		werr <- err
+	}()
+	err := putExport(ctx, target, token, contentType, pr, size, trailer)
+	pr.CloseWithError(io.ErrClosedPipe)
+	if w := <-werr; w != nil && !errors.Is(w, io.ErrClosedPipe) {
+		return w
+	}
+	return err
+}
+
+// putExport PUTs the export to felis-api. There is no retry: the token opens
+// the export once, so a second attempt could only be refused. Redirects are
+// refused because the request carries the token and the internal face never
+// redirects. felis-api answers only after the whole download, which the Job's
+// activeDeadlineSeconds bounds, so the header timeout is a backstop for a
+// wedged endpoint and not the real limit.
+//
+// The body always goes chunked, which is what lets it end with a trailer; a
+// size the Job knows (-1 when it does not) goes as worldexport.LengthHeader in
+// place of Content-Length.
+func putExport(ctx context.Context, target, token, contentType string, body io.Reader, size int64, trailer http.Header) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body)
+	if err != nil {
+		return err
+	}
+	req.ContentLength = -1
+	req.Trailer = trailer
+	if size >= 0 {
+		req.Header.Set(worldexport.LengthHeader, strconv.FormatInt(size, 10))
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("Content-Type", contentType)
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: 10 * time.Minute},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNoContent {
+		return nil
+	}
+	var e struct {
+		Error struct {
+			Message string `json:"message"`
+		} `json:"error"`
+	}
+	if json.NewDecoder(io.LimitReader(resp.Body, 4<<10)).Decode(&e) == nil && e.Error.Message != "" {
+		return fmt.Errorf("felis-api answered %s: %s", resp.Status, e.Error.Message)
+	}
+	return fmt.Errorf("felis-api answered %s", resp.Status)
+}

cmd/felis/export_test.go

@@ -0,0 +1,532 @@
+package main
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"slices"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/worldexport"
+)
+
+// exportReceiver stands in for felis-api's internal upload route: it records
+// the PUT it gets (or the error reading it ended on) and answers with reply.
+type exportReceiver struct {
+	srv     *httptest.Server
+	hits    atomic.Int32
+	req     *http.Request
+	body    []byte
+	readErr error
+	served  chan struct{} // one send per request, once it is answered
+}
+
+func receiveExport(t *testing.T, reply func(w http.ResponseWriter)) *exportReceiver {
+	t.Helper()
+	rcv := &exportReceiver{served: make(chan struct{}, 1)}
+	rcv.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		rcv.hits.Add(1)
+		rcv.req = r
+		rcv.body, rcv.readErr = io.ReadAll(r.Body)
+		reply(w)
+		rcv.served <- struct{}{}
+	}))
+	t.Cleanup(rcv.srv.Close)
+	return rcv
+}
+
+func noContent(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) }
+
+// sentWhole fails unless the upload rcv got ended with the Content-Digest
+// trailer of its own bytes, and declared length as its size (-1: none).
+func sentWhole(t *testing.T, rcv *exportReceiver, length int64) {
+	t.Helper()
+	sum := sha256.Sum256(rcv.body)
+	want := "sha-256=:" + base64.StdEncoding.EncodeToString(sum[:]) + ":"
+	wantLength := ""
+	if length >= 0 {
+		wantLength = strconv.FormatInt(length, 10)
+	}
+	r := rcv.req
+	if rcv.readErr != nil || r.Trailer.Get(worldexport.DigestTrailer) != want || r.Header.Get(worldexport.LengthHeader) != wantLength ||
+		r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
+		t.Fatalf("upload read %v, trailer %v, %s %q, length %d, encoding %v; want trailer %q and %s %q, chunked",
+			rcv.readErr, r.Trailer, worldexport.LengthHeader, r.Header.Get(worldexport.LengthHeader), r.ContentLength, r.TransferEncoding,
+			want, worldexport.LengthHeader, wantLength)
+	}
+}
+
+func tarEntries(t *testing.T, archive []byte) map[string]string {
+	t.Helper()
+	gz, err := gzip.NewReader(bytes.NewReader(archive))
+	if err != nil {
+		t.Fatalf("not gzip: %v", err)
+	}
+	out := map[string]string{}
+	tr := tar.NewReader(gz)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return out
+		}
+		if err != nil {
+			t.Fatalf("tar: %v", err)
+		}
+		b, _ := io.ReadAll(tr)
+		out[h.Name] = string(b)
+	}
+}
+
+// writeTree writes name → body under root, making the folders on the way.
+func writeTree(t *testing.T, root string, files map[string]string) {
+	t.Helper()
+	for name, body := range files {
+		p := filepath.Join(root, name)
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// The two secrets a world holds, and what server.properties reads as once
+// redacted.
+const (
+	secretProps   = "motd=hi\nrcon.password=hunter2\n"
+	redactedProps = "motd=hi\nrcon.password=<redacted by felis>\n"
+	forwardingKey = "secret: aVeryRealForwardingKey\n"
+)
+
+// secretWorld is a world holding both secrets, with a hard link to the
+// forwarding secret under a name nothing would guard by.
+func secretWorld(t *testing.T) string {
+	t.Helper()
+	root := t.TempDir()
+	writeTree(t, root, map[string]string{
+		"server.properties":       secretProps,
+		"config/paper-global.yml": forwardingKey,
+		"world/region/r.0.0.mca":  "chunks",
+	})
+	if err := os.MkdirAll(filepath.Join(root, "plugins"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(root, "plugins/copy.yml")); err != nil {
+		t.Fatal(err)
+	}
+	return root
+}
+
+func TestCmdExportWorld(t *testing.T) {
+	root := secretWorld(t)
+	if err := os.Symlink("server.properties", filepath.Join(root, "props-link")); err != nil {
+		t.Fatal(err)
+	}
+	rcv := receiveExport(t, noContent)
+	t.Setenv(worldexport.TokenEnv, "tok")
+	var stdout, stderr bytes.Buffer
+	code := cmdExport([]string{"--mode", "world", "--server", "survival", "--target-url", rcv.srv.URL + "/api/v1/internal/exports/ab",
+		"--worlds-root", root}, &stdout, &stderr)
+	if code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, stderr.String())
+	}
+	r := rcv.req
+	if r.Method != http.MethodPut || r.URL.Path != "/api/v1/internal/exports/ab" || r.Header.Get("Authorization") != "Bearer tok" ||
+		r.Header.Get("Content-Type") != "application/gzip" || r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
+		t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
+	}
+	want := map[string]string{
+		"server.properties": redactedProps, "config/": "", "plugins/": "",
+		"world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks",
+	}
+	if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
+		t.Fatalf("archive holds %v\nwant %v", got, want)
+	}
+	sentWhole(t, rcv, -1)
+	want2 := "felis export: left out 1 entries a tar cannot hold (symbolic links, devices, sockets)\n" +
+		"felis export: left out 2 files that hold platform secrets\n" +
+		"felis export: server=survival mode=world downloaded\n"
+	if stdout.String() != want2 {
+		t.Errorf("stdout = %q, want %q", stdout.String(), want2)
+	}
+}
+
+// A world root that cannot be opened fails before anything reaches felis-api.
+func TestCmdExportWorldUnreadable(t *testing.T) {
+	rcv := receiveExport(t, noContent)
+	t.Setenv(worldexport.TokenEnv, "tok")
+	var stdout, stderr bytes.Buffer
+	code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
+	if code != 1 || rcv.hits.Load() != 0 {
+		t.Fatalf("exit %d with %d requests, want 1 and none", code, rcv.hits.Load())
+	}
+}
+
+// An export that fails part-way must never reach felis-api as a complete body:
+// the chunked upload is cut off, so felis-api aborts the browser's download,
+// and the failure itself is what the Job reports.
+func TestStreamExportWriteErrorAbortsTheUpload(t *testing.T) {
+	broken := errors.New("disk read failed")
+	rcv := receiveExport(t, noContent)
+	err := streamExport(context.Background(), rcv.srv.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
+		if _, err := w.Write(bytes.Repeat([]byte("x"), 100_000)); err != nil {
+			return err
+		}
+		return broken
+	})
+	if err != broken {
+		t.Fatalf("err = %v, want the write's own error, unwrapped", err)
+	}
+	select {
+	case <-rcv.served:
+		if rcv.readErr == nil {
+			t.Fatalf("felis-api read a complete %d-byte body from a failed export", len(rcv.body))
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("the request never reached felis-api")
+	}
+}
+
+// When felis-api refuses first, its reason is reported, not the closed pipe
+// that then stops the writer.
+func TestStreamExportRefusalStopsTheWriter(t *testing.T) {
+	refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer refusing.Close()
+	stopped := make(chan error, 1)
+	err := streamExport(context.Background(), refusing.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
+		for {
+			if _, err := w.Write(make([]byte, 32<<10)); err != nil {
+				stopped <- err
+				return err
+			}
+		}
+	})
+	if err == nil || err.Error() != "felis-api answered 404 Not Found" {
+		t.Fatalf("err = %v, want felis-api's answer", err)
+	}
+	if werr := <-stopped; !errors.Is(werr, io.ErrClosedPipe) {
+		t.Fatalf("the writer stopped on %v, want the closed pipe", werr)
+	}
+
+	// A PUT that never starts leaves no transport to close the body: the writer
+	// is still stopped, and the export fails rather than hangs.
+	done := make(chan error, 1)
+	go func() {
+		done <- streamExport(context.Background(), "http://[::1", "tok", "application/gzip", -1, func(w io.Writer) error {
+			_, err := w.Write([]byte("x"))
+			return err
+		})
+	}()
+	select {
+	case err := <-done:
+		if err == nil || !strings.Contains(err.Error(), "missing ']'") {
+			t.Fatalf("err = %v, want the bad URL", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("an export whose PUT never started hung")
+	}
+}
+
+// storedBackup writes, at path, a gzip+tar like one the backup store holds:
+// the world whole, both secrets included, and a region file that does not
+// compress. It returns the archive's sha256.
+func storedBackup(t *testing.T, path string) string {
+	t.Helper()
+	region := make([]byte, 64<<10)
+	if _, err := rand.Read(region); err != nil {
+		t.Fatal(err)
+	}
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zw)
+	for _, e := range []struct{ name, body string }{
+		{"server.properties", secretProps},
+		{"config/paper-global.yml", forwardingKey},
+		{"world/level.dat", "level"},
+		{"world/region/r.0.0.mca", string(region)},
+	} {
+		if err := tw.WriteHeader(&tar.Header{Name: e.name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(e.body))}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := io.WriteString(tw, e.body); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256(buf.Bytes())
+	return hex.EncodeToString(sum[:])
+}
+
+func TestCmdExportBackup(t *testing.T) {
+	root := t.TempDir()
+	ref := filepath.Join(root, "survival-1.tar.gz")
+	sum := storedBackup(t, ref)
+	stored, err := os.ReadFile(ref)
+	if err != nil {
+		t.Fatal(err)
+	}
+	region := tarEntries(t, stored)["world/region/r.0.0.mca"]
+	args := func(url, ref string) []string {
+		return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
+	}
+
+	for name, extra := range map[string][]string{
+		"no digest recorded":      nil,
+		"recorded digest matches": {"--sha256", sum},
+	} {
+		t.Run(name+": re-streamed through the guards", func(t *testing.T) {
+			rcv := receiveExport(t, noContent)
+			t.Setenv(worldexport.TokenEnv, "tok")
+			var stdout, stderr bytes.Buffer
+			if code := cmdExport(append(args(rcv.srv.URL, ref), extra...), &stdout, &stderr); code != 0 {
+				t.Fatalf("exit %d, stderr %q", code, stderr.String())
+			}
+			r := rcv.req
+			if r.ContentLength != -1 || r.Header.Get("Content-Type") != "application/gzip" || r.Header.Get("Authorization") != "Bearer tok" {
+				t.Fatalf("length %d, headers %v", r.ContentLength, r.Header)
+			}
+			want := map[string]string{"server.properties": redactedProps, "world/level.dat": "level", "world/region/r.0.0.mca": region}
+			if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
+				t.Fatalf("archive holds %d entries, want exactly the redacted properties, level.dat and the region file", len(got))
+			}
+			sentWhole(t, rcv, -1)
+			if want := "felis export: left out 1 files that hold platform secrets\nfelis export: server=survival mode=backup downloaded\n"; stdout.String() != want {
+				t.Errorf("stdout = %q, want %q", stdout.String(), want)
+			}
+		})
+	}
+
+	// The stored bytes are checked as they stream, and the archive the Job sends
+	// is only closed once they are all read: a mismatch cuts the upload off
+	// short of its end, so felis-api never passes on a complete-looking copy.
+	t.Run("a digest mismatch cuts the upload off before its end", func(t *testing.T) {
+		rcv := receiveExport(t, noContent)
+		t.Setenv(worldexport.TokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdExport(append(args(rcv.srv.URL, ref), "--sha256", strings.Repeat("ab", 32)), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if want := "felis export: the backup archive does not match the sha256 recorded when it was written\n"; stderr.String() != want {
+			t.Fatalf("stderr = %q, want %q", stderr.String(), want)
+		}
+		select {
+		case <-rcv.served:
+		case <-time.After(5 * time.Second):
+			t.Fatal("the upload never reached felis-api")
+		}
+		if rcv.readErr == nil {
+			t.Fatalf("felis-api read a complete %d-byte body", len(rcv.body))
+		}
+		zr, err := gzip.NewReader(bytes.NewReader(rcv.body))
+		if err == nil {
+			_, err = io.ReadAll(zr)
+		}
+		if err == nil {
+			t.Fatal("what felis-api got is a complete archive")
+		}
+	})
+
+	t.Run("a ref outside the backup root is refused before any request", func(t *testing.T) {
+		outside := filepath.Join(t.TempDir(), "secret.tar.gz")
+		if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		rcv := receiveExport(t, noContent)
+		t.Setenv(worldexport.TokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdExport(args(rcv.srv.URL, outside), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if n := rcv.hits.Load(); n != 0 {
+			t.Fatalf("felis-api got %d requests", n)
+		}
+	})
+
+	t.Run("a refusal exits 1 with felis-api's reason", func(t *testing.T) {
+		rcv := receiveExport(t, func(w http.ResponseWriter) {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusGone)
+			io.WriteString(w, `{"error":{"code":"export_expired","message":"nobody opened the download"}}`)
+		})
+		t.Setenv(worldexport.TokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if got := stderr.String(); got != "felis export: felis-api answered 410 Gone: nobody opened the download\n" {
+			t.Fatalf("stderr = %q", got)
+		}
+	})
+
+	// The request carries the token and the internal face never redirects, so a
+	// redirect is refused rather than followed with the token attached. A 302 or
+	// 303 is the one net/http would follow on its own (as a GET, and to the same
+	// host with the Authorization header still on it).
+	for _, status := range []int{http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
+		t.Run("a redirect is not followed: "+strconv.Itoa(status), func(t *testing.T) {
+			elsewhere := receiveExport(t, noContent)
+			redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.srv.URL, status))
+			defer redirecting.Close()
+			t.Setenv(worldexport.TokenEnv, "tok")
+			var stdout, stderr bytes.Buffer
+			if code := cmdExport(args(redirecting.URL, ref), &stdout, &stderr); code != 1 {
+				t.Fatalf("exit %d, want 1", code)
+			}
+			if n := elsewhere.hits.Load(); n != 0 {
+				t.Fatalf("the redirect target got %d requests", n)
+			}
+			if got, want := stderr.String(), "felis export: felis-api answered "+strconv.Itoa(status)+" "+http.StatusText(status)+"\n"; got != want {
+				t.Fatalf("stderr = %q, want %q", got, want)
+			}
+		})
+	}
+}
+
+func TestCmdExportFiles(t *testing.T) {
+	root := secretWorld(t)
+	writeTree(t, root, map[string]string{"plugins/Essentials/config.yml": "x: 1"})
+	if err := os.Symlink("config.yml", filepath.Join(root, "plugins/Essentials/link.yml")); err != nil {
+		t.Fatal(err)
+	}
+	export := func(t *testing.T, path string, dir bool) (*exportReceiver, int, string, string) {
+		t.Helper()
+		rcv := receiveExport(t, noContent)
+		t.Setenv(worldexport.TokenEnv, "tok")
+		args := []string{"--mode", "files", "--server", "survival", "--target-url", rcv.srv.URL, "--worlds-root", root, "--path", path}
+		if dir {
+			args = append(args, "--dir")
+		}
+		var stdout, stderr bytes.Buffer
+		code := cmdExport(args, &stdout, &stderr)
+		return rcv, code, stdout.String(), stderr.String()
+	}
+
+	for path, want := range map[string]string{
+		"world/region/r.0.0.mca": "chunks",
+		"server.properties":      redactedProps,
+	} {
+		t.Run("a file goes with its exact length: "+path, func(t *testing.T) {
+			rcv, code, stdout, stderr := export(t, path, false)
+			if code != 0 || stdout != "felis export: server=survival mode=files downloaded\n" {
+				t.Fatalf("exit %d, stdout %q, stderr %q", code, stdout, stderr)
+			}
+			if string(rcv.body) != want || rcv.req.Header.Get("Content-Type") != "application/octet-stream" {
+				t.Fatalf("body %q, type %q; want %q", rcv.body, rcv.req.Header.Get("Content-Type"), want)
+			}
+			sentWhole(t, rcv, int64(len(want)))
+		})
+	}
+
+	t.Run("a folder goes as a zip, guarded", func(t *testing.T) {
+		rcv, code, stdout, stderr := export(t, "plugins", true)
+		if code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr)
+		}
+		if rcv.req.ContentLength != -1 || rcv.req.Header.Get("Content-Type") != "application/zip" {
+			t.Fatalf("length %d, type %q", rcv.req.ContentLength, rcv.req.Header.Get("Content-Type"))
+		}
+		zr, err := zip.NewReader(bytes.NewReader(rcv.body), int64(len(rcv.body)))
+		if err != nil {
+			t.Fatal(err)
+		}
+		var names []string
+		for _, f := range zr.File {
+			names = append(names, f.Name)
+		}
+		if want := []string{"plugins/", "plugins/Essentials/", "plugins/Essentials/config.yml"}; !slices.Equal(names, want) {
+			t.Fatalf("zip holds %v, want %v", names, want)
+		}
+		want := "felis export: left out 1 entries a zip does not carry (symbolic links, devices, sockets)\n" +
+			"felis export: left out 1 files that hold platform secrets\n" +
+			"felis export: server=survival mode=files downloaded\n"
+		if stdout != want {
+			t.Errorf("stdout = %q, want %q", stdout, want)
+		}
+	})
+
+	for _, c := range []struct {
+		path string
+		dir  bool
+		want string
+	}{
+		{"config/paper-global.yml", false, "forwarding secret"},
+		{"plugins/copy.yml", false, "forwarding secret"},
+		{"plugins", false, "is a folder now"},
+		{"server.properties", true, "is not a folder now"},
+	} {
+		t.Run("refused before any request: "+c.path, func(t *testing.T) {
+			rcv, code, _, stderr := export(t, c.path, c.dir)
+			if code != 1 || rcv.hits.Load() != 0 || !strings.Contains(stderr, c.want) {
+				t.Fatalf("exit %d, %d requests, stderr %q; want 1, none, and %q", code, rcv.hits.Load(), stderr, c.want)
+			}
+		})
+	}
+}
+
+func TestCmdExportUsage(t *testing.T) {
+	for name, tc := range map[string]struct {
+		token string
+		args  []string
+	}{
+		"no token":           {"", []string{"--mode", "world", "--target-url", "http://api/x"}},
+		"no target":          {"tok", []string{"--mode", "world"}},
+		"unknown mode":       {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
+		"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
+		"files without path": {"tok", []string{"--mode", "files", "--target-url", "http://api/x"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Setenv(worldexport.TokenEnv, tc.token)
+			var stdout, stderr bytes.Buffer
+			if code := cmdExport(tc.args, &stdout, &stderr); code != 2 {
+				t.Fatalf("exit %d, want 2; stderr %q", code, stderr.String())
+			}
+		})
+	}
+}
+
+// TestCmdExportWiring: the Job's `felis export` reaches cmdExport, and
+// felis-api's executor mounts the backup store at the path the archives were
+// written under, since a backup's ref is an absolute path there.
+func TestCmdExportWiring(t *testing.T) {
+	t.Setenv(worldexport.TokenEnv, "")
+	var stdout, stderr bytes.Buffer
+	if code := run([]string{"export"}, &stdout, &stderr); code != 2 ||
+		stderr.String() != "felis export: --target-url and "+worldexport.TokenEnv+" are required\n" {
+		t.Fatalf("felis export = %d, stderr %q", code, stderr.String())
+	}
+	cfg := &config.Config{}
+	cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups"
+	want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"}
+	if got := exportConfig(cfg, "felis:1", "felis-backups"); got != want {
+		t.Fatalf("exportConfig = %+v, want %+v", got, want)
+	}
+}

cmd/felis/fetchcontext.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// cmdFetchContext is the in-Pod entrypoint the build Job's context-fetch
+// initContainer runs. It reads the blob the platform stored for a submission
+// from the felis-api INTERNAL face (with a bounded retry — see
+// fetchContextWithRetry) and extracts it into the shared emptyDir the Kaniko
+// container then builds from.
+//
+// Why this exists: the build Pod runs in the build namespace, where it can neither
+// mount the control-plane uploads PVC (a PVC does not cross namespaces) nor hold
+// object-store credentials, so the API that WROTE the blob is the transport. The
+// route is service-token-gated; the token arrives through a namespace-local Secret
+// mounted only into this initContainer, never into Kaniko's — so the untrusted
+// Dockerfile's build steps have no credential to read (their containers share no
+// environment, no PID namespace, and Kaniko itself mounts the context read-only).
+//
+// The extraction is deliberately paranoid: the tarball is attacker-controlled
+// input, so absolute paths, ".." escapes, links, and special files are refused
+// rather than sanitized. Kaniko treats the extracted tree as hostile regardless
+// (spec §16), but the pod's own filesystem still must not be written outside the
+// context directory it was given.
+func cmdFetchContext(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("fetch-context", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	url := fs.String("url", "", "internal-face URL of the submission's build-context tarball")
+	out := fs.String("out", "/context", "directory to extract the build context into")
+	want := fs.String("sha256", "", "refuse the context unless the tarball's sha256 is this lowercase hex digest")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *want != "" && !build.IsSHA256Hex(*want) {
+		fmt.Fprintf(stderr, "felis fetch-context: --sha256 %q is not a lowercase hex sha256\n", *want)
+		return 2
+	}
+	if *url == "" {
+		fmt.Fprintln(stderr, "felis fetch-context: --url is required")
+		return 2
+	}
+	token := os.Getenv("FELIS_SERVICE_TOKEN")
+	if token == "" {
+		fmt.Fprintln(stderr, "felis fetch-context: FELIS_SERVICE_TOKEN is empty — the internal face rejects anonymous reads")
+		return 2
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	// Validate the URL once up front: a bad one is a usage error (2), not
+	// something to sit in the retry loop.
+	if _, err := http.NewRequest(http.MethodGet, *url, nil); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: bad --url: %v\n", err)
+		return 2
+	}
+	// No overall client timeout: a legitimate modpack context can be large and the
+	// Job's activeDeadlineSeconds is the real bound. The header timeout catches a
+	// wedged endpoint without capping a healthy download.
+	// Redirects are refused: the request carries the service token, and the
+	// internal face never redirects, so a 3xx is someone steering the token.
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: time.Minute},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := fetchContextWithRetry(ctx, client, *url, token, stderr)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	defer resp.Body.Close()
+
+	h := sha256.New()
+	body := io.TeeReader(resp.Body, h)
+	if err := extractTarGz(body, *out); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	if *want == "" {
+		return 0
+	}
+	// The tar end marker comes before the gzip trailer and whatever follows it,
+	// so read to EOF: the digest must cover every byte the blob holds. The blob
+	// itself is size-capped at upload, which bounds this read.
+	if _, err := io.Copy(io.Discard, io.LimitReader(body, maxContextBytes)); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	if got := hex.EncodeToString(h.Sum(nil)); got != *want {
+		// The init container failing is what keeps Kaniko from ever starting on
+		// the extracted tree.
+		fmt.Fprintf(stderr, "felis fetch-context: the context's sha256 is %s, the approved digest is %s: it changed after approval; refusing to build\n", got, *want)
+		return 1
+	}
+	return 0
+}
+
+// fetchRetryInterval/fetchRetryWindow bound how long the fetch waits out a
+// control-plane blip before giving up. The api pod being replaced is a normal
+// event (rollout, eviction, a chaos drill), and without a retry one refused
+// dial turns it into a failed build: BackoffLimit=0 gives the Job no second
+// Pod, so the terminal verdict costs a manual re-approval — the live drill hit
+// exactly this (context-fetch exit 1 on `connect: connection refused` while
+// the api pod rolled; the new pod was serving 11 seconds later and the same
+// 198-byte blob). The window is tiny next to the Job's 30-minute
+// activeDeadline; a 4xx (missing blob, rejected token) still fails fast.
+//
+// Vars, not consts, so tests can shrink the window.
+var (
+	fetchRetryInterval = 3 * time.Second
+	fetchRetryWindow   = 45 * time.Second
+)
+
+// fetchContextWithRetry GETs the context tarball, retrying transport failures
+// and 5xx responses until fetchRetryWindow runs out. A 4xx is an answer, not a
+// blip — retrying it only delays the honest error.
+func fetchContextWithRetry(ctx context.Context, client *http.Client, url, token string, stderr io.Writer) (*http.Response, error) {
+	deadline := time.Now().Add(fetchRetryWindow)
+	for {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+		if err != nil {
+			return nil, fmt.Errorf("bad --url: %w", err)
+		}
+		req.Header.Set("Authorization", "Bearer "+token)
+
+		resp, err := client.Do(req)
+		if err == nil && resp.StatusCode == http.StatusOK {
+			return resp, nil
+		}
+		if err == nil {
+			status := resp.Status
+			_ = resp.Body.Close()
+			err = fmt.Errorf("GET returned %s", status)
+			if resp.StatusCode < 500 {
+				return nil, err
+			}
+		}
+		if ctx.Err() != nil {
+			return nil, fmt.Errorf("GET failed: %w", err)
+		}
+		if time.Now().After(deadline) {
+			return nil, fmt.Errorf("GET failed (retried for %s): %w", fetchRetryWindow, err)
+		}
+		fmt.Fprintf(stderr, "felis fetch-context: %v; retrying (the internal face may be restarting)\n", err)
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("GET failed: %w", err)
+		case <-time.After(fetchRetryInterval):
+		}
+	}
+}
+
+// maxContextBytes / maxContextEntries bound what one context may expand to. The
+// compressed upload is capped at 1 GiB, but gzip turns that into hundreds of GiB
+// or millions of empty files, and the emptyDir's 4 GiB sizeLimit is only
+// enforced by the kubelet's periodic sweep, after the disk has filled. The byte
+// cap matches that sizeLimit; the entry cap is far above any real modpack (a
+// large one is a few thousand files) and far below an inode exhaustion.
+//
+// Vars, not consts, so tests can shrink them.
+var (
+	maxContextBytes   int64 = 4 << 30
+	maxContextEntries       = 200_000
+)
+
+// extractTarGz streams a gzip'd tarball into root, creating directories as
+// needed. Every entry is vetted BEFORE anything is written: a path that is
+// absolute or escapes root (via ".."), a link (symlink or hardlink), or any
+// special file kind aborts the whole extraction. Refusing rather than skipping is
+// deliberate — a context that needs one of those constructs is not a context this
+// transport carries, and silently dropping entries would build from a corpus the
+// submitter did not upload. The whole extraction is also bounded by
+// maxContextBytes and maxContextEntries.
+func extractTarGz(r io.Reader, root string) error {
+	if err := os.MkdirAll(root, 0o755); err != nil {
+		return fmt.Errorf("create context dir: %w", err)
+	}
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return fmt.Errorf("context is not a valid gzip tarball: %w", err)
+	}
+	defer zr.Close()
+	tr := tar.NewReader(zr)
+	var written int64
+	entries := 0
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("read context tarball: %w", err)
+		}
+		if entries++; entries > maxContextEntries {
+			return fmt.Errorf("the build context has more than %d entries", maxContextEntries)
+		}
+		name := filepath.Clean(hdr.Name)
+		if name == "." {
+			continue
+		}
+		// The zip-slip guard: reject, never rewrite. filepath.Clean collapses any
+		// "a/../../b", so these two checks are sufficient once Clean has run.
+		if filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) {
+			return fmt.Errorf("context entry %q escapes the context directory", hdr.Name)
+		}
+		target := filepath.Join(root, name)
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0o755); err != nil {
+				return fmt.Errorf("create %q: %w", name, err)
+			}
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+				return fmt.Errorf("create parent of %q: %w", name, err)
+			}
+			mode := os.FileMode(0o644)
+			if hdr.FileInfo().Mode()&0o111 != 0 {
+				mode = 0o755 // preserve executability (entrypoint scripts), nothing else
+			}
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
+			if err != nil {
+				return fmt.Errorf("create %q: %w", name, err)
+			}
+			n, err := io.Copy(f, io.LimitReader(tr, maxContextBytes-written+1))
+			written += n
+			if err != nil {
+				_ = f.Close()
+				return fmt.Errorf("write %q: %w", name, err)
+			}
+			if written > maxContextBytes {
+				_ = f.Close()
+				return fmt.Errorf("the build context expands past %d bytes", maxContextBytes)
+			}
+			if err := f.Close(); err != nil {
+				return fmt.Errorf("close %q: %w", name, err)
+			}
+		default:
+			return fmt.Errorf("context entry %q has unsupported type %q (links and special files are refused)", hdr.Name, string(hdr.Typeflag))
+		}
+	}
+}

cmd/felis/fetchcontext_test.go

@@ -0,0 +1,404 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+type tarEntry struct {
+	name     string
+	body     string
+	mode     int64
+	typ      byte
+	linkname string
+}
+
+// tgzBody builds an in-memory .tar.gz from entries, preserving each entry's type
+// and mode so the tests can exercise the guards with exactly the bytes an
+// attacker could upload.
+func tgzBody(t *testing.T, entries ...tarEntry) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zw)
+	for _, e := range entries {
+		typ := e.typ
+		if typ == 0 {
+			typ = tar.TypeReg
+		}
+		mode := e.mode
+		if mode == 0 {
+			mode = 0o644
+		}
+		hdr := &tar.Header{Name: e.name, Typeflag: typ, Mode: mode, Size: int64(len(e.body))}
+		if typ == tar.TypeSymlink {
+			hdr.Linkname = e.linkname
+			hdr.Size = 0
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatalf("write header %q: %v", e.name, err)
+		}
+		if hdr.Size > 0 {
+			if _, err := tw.Write([]byte(e.body)); err != nil {
+				t.Fatalf("write body %q: %v", e.name, err)
+			}
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatalf("close tar: %v", err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatalf("close gzip: %v", err)
+	}
+	return buf.Bytes()
+}
+
+// A normal context extracts with its tree intact, and the executable bit that
+// modpack entrypoints rely on survives.
+func TestExtractTarGzRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	body := tgzBody(t,
+		tarEntry{name: "Dockerfile", body: "FROM scratch\n"},
+		tarEntry{name: "mods/example.jar", body: "jar-bytes"},
+		tarEntry{name: "start.sh", body: "#!/bin/sh\n", mode: 0o755},
+		tarEntry{name: "mods/", typ: tar.TypeDir, mode: 0o755},
+	)
+	if err := extractTarGz(bytes.NewReader(body), dir); err != nil {
+		t.Fatalf("extract: %v", err)
+	}
+	for name, want := range map[string]string{
+		"Dockerfile":       "FROM scratch\n",
+		"mods/example.jar": "jar-bytes",
+	} {
+		got, err := os.ReadFile(filepath.Join(dir, name))
+		if err != nil || string(got) != want {
+			t.Fatalf("%s = (%q, %v), want %q", name, got, err, want)
+		}
+	}
+	fi, err := os.Stat(filepath.Join(dir, "start.sh"))
+	if err != nil || fi.Mode()&0o111 == 0 {
+		t.Fatalf("entrypoint script lost its exec bit: %v (%v)", fi, err)
+	}
+}
+
+// The guards: "..", absolute paths, symlinks, and special files are refused whole
+// — nothing escapes, and nothing is silently skipped.
+func TestExtractTarGzRefusesEscapes(t *testing.T) {
+	cases := []struct {
+		name    string
+		entries []tarEntry
+	}{
+		{"dotdot", []tarEntry{{name: "../outside", body: "x"}}},
+		{"nested dotdot", []tarEntry{{name: "a/../../outside", body: "x"}}},
+		{"absolute", []tarEntry{{name: "/etc/outside", body: "x"}}},
+		{"symlink", []tarEntry{{name: "link", typ: tar.TypeSymlink, linkname: "/etc"}}},
+		{"hardlink", []tarEntry{{name: "hard", typ: tar.TypeLink, linkname: "somewhere"}}},
+		{"device", []tarEntry{{name: "dev", typ: tar.TypeChar}}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			if err := extractTarGz(bytes.NewReader(tgzBody(t, tc.entries...)), dir); err == nil {
+				t.Fatal("extract accepted a hostile entry, want an error")
+			}
+			// Nothing may have been written outside the target (or at all).
+			entries, _ := os.ReadDir(dir)
+			if len(entries) != 0 {
+				t.Fatalf("hostile archive left %d entries behind", len(entries))
+			}
+		})
+	}
+}
+
+// A context that expands past the byte or entry cap is refused, however small
+// it was compressed: gzip bombs and inode floods stop at the cap.
+func TestExtractTarGzCapsExpansion(t *testing.T) {
+	bytesCap, entriesCap := maxContextBytes, maxContextEntries
+	t.Cleanup(func() { maxContextBytes, maxContextEntries = bytesCap, entriesCap })
+	maxContextBytes, maxContextEntries = 1000, 5
+
+	fits := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 400)})
+	if err := extractTarGz(bytes.NewReader(fits), t.TempDir()); err != nil {
+		t.Fatalf("a context exactly at the byte cap: %v", err)
+	}
+	big := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 401)})
+	if err := extractTarGz(bytes.NewReader(big), t.TempDir()); err == nil || !strings.Contains(err.Error(), "expands past") {
+		t.Fatalf("one byte over the cap: err = %v", err)
+	}
+	var many []tarEntry
+	for i := 0; i < 6; i++ {
+		many = append(many, tarEntry{name: "d" + string(rune('0'+i)) + "/", typ: tar.TypeDir})
+	}
+	if err := extractTarGz(bytes.NewReader(tgzBody(t, many...)), t.TempDir()); err == nil || !strings.Contains(err.Error(), "entries") {
+		t.Fatalf("six entries over a cap of five: err = %v", err)
+	}
+}
+
+// The command end to end: it dials the URL with the bearer token from the
+// environment, and refuses to run without it (the internal face would 401
+// anyway; failing at parse time is the honest earlier error).
+func TestCmdFetchContextFetchAndExtract(t *testing.T) {
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer test-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		w.Header().Set("Content-Type", "application/gzip")
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + dir}, io.Discard, io.Discard); code != 0 {
+		t.Fatalf("cmdFetchContext exit = %d, want 0", code)
+	}
+	if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
+		t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
+	}
+
+	// No token: refuse before dialing.
+	t.Setenv("FELIS_SERVICE_TOKEN", "")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 2 {
+		t.Fatalf("missing token exit = %d, want 2 (stderr %q)", code, stderr.String())
+	}
+
+	// A non-200 answer (e.g. the route's 404 for a never-uploaded context) fails.
+	srv404 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv404.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	if code := cmdFetchContext([]string{"--url=" + srv404.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, io.Discard); code != 1 {
+		t.Fatalf("404 exit = %d, want 1", code)
+	}
+}
+
+// With --sha256 the fetch refuses any bytes but the approved ones, including
+// a tarball that extracts cleanly: that is exactly the context an uploader
+// swapped in after the review.
+func TestCmdFetchContextChecksDigest(t *testing.T) {
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	sum := sha256.Sum256(body)
+	good := hex.EncodeToString(sum[:])
+	args := func(digest string) []string {
+		return []string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir(), "--sha256=" + digest}
+	}
+
+	if code := cmdFetchContext(args(good), io.Discard, io.Discard); code != 0 {
+		t.Fatalf("matching digest exit = %d, want 0", code)
+	}
+	var stderr bytes.Buffer
+	other := strings.Repeat("0", 64)
+	if code := cmdFetchContext(args(other), io.Discard, &stderr); code != 1 || !strings.Contains(stderr.String(), "changed after approval") {
+		t.Fatalf("mismatched digest exit = %d, stderr %q; want 1 naming the change", code, stderr.String())
+	}
+	stderr.Reset()
+	if code := cmdFetchContext(args("ABC"), io.Discard, &stderr); code != 2 {
+		t.Fatalf("malformed digest exit = %d, want 2 (stderr %q)", code, stderr.String())
+	}
+
+	// Bytes after the tar end marker still count: appending to an approved blob
+	// must change what the fetch accepts.
+	padded := append(append([]byte{}, body...), "trailing"...)
+	srvPadded := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(padded)
+	}))
+	defer srvPadded.Close()
+	if code := cmdFetchContext([]string{"--url=" + srvPadded.URL + "/c", "--out=" + t.TempDir(), "--sha256=" + good}, io.Discard, io.Discard); code != 1 {
+		t.Fatalf("padded blob exit = %d, want 1", code)
+	}
+}
+
+// The request carries the service token, so a redirect is a failure: the token
+// never follows it to another host (build-supply-chain-13).
+func TestCmdFetchContextRefusesRedirects(t *testing.T) {
+	var leaked bool
+	elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		leaked = true
+	}))
+	defer elsewhere.Close()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, elsewhere.URL+"/steal", http.StatusFound)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/c", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("redirect exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if leaked {
+		t.Fatal("the fetch followed the redirect")
+	}
+}
+
+// A body that is not a gzip tarball must fail the extraction rather than produce
+// an empty (or partial) context Kaniko would then try to build.
+func TestExtractTarGzRejectsNonGzip(t *testing.T) {
+	dir := t.TempDir()
+	err := extractTarGz(strings.NewReader("not a tarball"), dir)
+	if err == nil || !strings.Contains(err.Error(), "gzip") {
+		t.Fatalf("err = %v, want a gzip complaint", err)
+	}
+}
+
+// shrinkFetchWindow swaps the retry knobs for a faster test and restores them
+// afterwards, so no test leaks a tiny window into another.
+func shrinkFetchWindow(t *testing.T, interval, window time.Duration) {
+	t.Helper()
+	oldInterval, oldWindow := fetchRetryInterval, fetchRetryWindow
+	fetchRetryInterval, fetchRetryWindow = interval, window
+	t.Cleanup(func() { fetchRetryInterval, fetchRetryWindow = oldInterval, oldWindow })
+}
+
+// A control-plane blip mid-fetch is survived: a 5xx on the first attempt is
+// retried and the second attempt's tarball extracts. This walks back the live
+// drill's failure, where the api pod rolled mid-fetch and the single attempt
+// died, failing the build Job.
+func TestFetchContextRetriesThroughBlip(t *testing.T) {
+	shrinkFetchWindow(t, 10*time.Millisecond, time.Second)
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			w.WriteHeader(http.StatusBadGateway) // the port is up, the API is not
+			return
+		}
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + dir}, io.Discard, &stderr); code != 0 {
+		t.Fatalf("exit = %d, want 0 (stderr %q)", code, stderr.String())
+	}
+	if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
+		t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
+	}
+	if !strings.Contains(stderr.String(), "retrying") {
+		t.Fatalf("stderr %q does not mention the retry", stderr.String())
+	}
+}
+
+// The live drill's exact shape: the dial itself is refused (the api pod is
+// gone and no endpoint answers). A refused dial is retried like any other
+// transport failure, and once the face is back the fetch completes.
+func TestFetchContextRetriesRefusedDial(t *testing.T) {
+	shrinkFetchWindow(t, 10*time.Millisecond, 5*time.Second)
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+
+	// Borrow a listen address, then close it: the first attempts dial into a
+	// refused connection, exactly like a restarting control plane.
+	probe := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+	addr := strings.TrimPrefix(probe.URL, "http://")
+	probe.Close()
+
+	dir := t.TempDir()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	// Start the fetch; while the retry loop burns refused dials, bring the same
+	// address back.
+	result := make(chan int, 1)
+	go func() {
+		result <- cmdFetchContext([]string{"--url=http://" + addr + "/sub-1/context", "--out=" + dir}, io.Discard, &stderr)
+	}()
+	time.Sleep(100 * time.Millisecond) // let a handful of dials be refused
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		t.Fatalf("rebind %s: %v", addr, err)
+	}
+	back := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer test-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		_, _ = w.Write(body)
+	})}
+	defer back.Close()
+	go func() { _ = back.Serve(ln) }()
+
+	code := <-result
+	if code != 0 {
+		t.Fatalf("exit = %d, want 0 (stderr %q)", code, stderr.String())
+	}
+	if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
+		t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
+	}
+	if !strings.Contains(stderr.String(), "retrying") {
+		t.Fatalf("stderr %q does not mention the retry", stderr.String())
+	}
+}
+
+// A 4xx is an answer, not a blip: a missing/never-uploaded context fails
+// immediately — no retry loop burns the build's deadline on a terminal error.
+func TestFetchContextDoesNotRetry4xx(t *testing.T) {
+	shrinkFetchWindow(t, 5*time.Millisecond, 200*time.Millisecond)
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if got := atomic.LoadInt32(&calls); got != 1 {
+		t.Fatalf("server saw %d attempts, want exactly 1", got)
+	}
+	if strings.Contains(stderr.String(), "retrying") {
+		t.Fatalf("stderr %q mentions a retry for a terminal 4xx", stderr.String())
+	}
+}
+
+// The retry is bounded: an internal face that stays down does not hang the
+// build pod; the window runs out and the fetch reports the exhausted retries.
+func TestFetchContextGivesUpAfterWindow(t *testing.T) {
+	shrinkFetchWindow(t, 5*time.Millisecond, 60*time.Millisecond)
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
+	defer srv.Close() // the face is up but never healthy: 503 forever
+
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	start := time.Now()
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if elapsed := time.Since(start); elapsed > 5*time.Second {
+		t.Fatalf("gave up after %v; the window is supposed to bound it", elapsed)
+	}
+	if got := atomic.LoadInt32(&calls); got < 2 {
+		t.Fatalf("server saw %d attempts, want at least one retry", got)
+	}
+	if !strings.Contains(stderr.String(), "retried for") {
+		t.Fatalf("stderr %q does not report the exhausted retry window", stderr.String())
+	}
+}

cmd/felis/files.go

@@ -1,11 +1,15 @@
 package main
 
 import (
-	"encoding/base64"
+	"context"
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
 	"felis.lolicon.best/internal/fileedit"
 )
@@ -19,32 +23,44 @@ import (
 // Like cmdRestore it deliberately holds NO database credentials and never calls
 // config.Load: felis-api made the authorization decision (the caller owns this
 // server, and the server is stopped so the RWO world volume is free); this process
-// is the unprivileged hands that touch bytes. Its entire input is the three flags
-// below plus, for a write, one environment variable. Every isolation guarantee
-// lives in the Pod spec (internal/fileedit/jobspec.go), and the path-containment
-// guarantee lives in fileedit.Execute, which resolves the path through os.Root and
-// therefore cannot be walked out of the world mount.
+// is the unprivileged hands that touch bytes. Its entire input is the flags
+// below plus, for a write, the content variables and, for an upload, one token.
+// Every isolation guarantee lives in the Pod spec (internal/fileedit/jobspec.go),
+// and the path-containment guarantee lives in fileedit.Execute, which resolves
+// every path through os.Root and therefore cannot be walked out of the world
+// mount.
 //
 // Exit status carries a specific meaning that felis-api depends on: a CALLER-fault
 // outcome — a path that escapes the root, a file that is missing or too large — is
 // a SUCCESSFUL run that prints a Result carrying an error code, so the API can map
 // it to a precise 4xx. A non-zero exit means the operation could not be attempted
-// at all (the world mount is unreadable, the result unprintable), which the API
-// reports as a 500.
+// at all (the world mount is unreadable, an upload's bytes could not be fetched
+// intact, the result unprintable), which the API reports as a 500.
 func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("files", flag.ContinueOnError)
 	fs.SetOutput(stderr)
-	op := fs.String("op", "", "operation: list, read, or write")
+	op := fs.String("op", "", "operation: list, read, write, mkdir, delete, rename, upload or unzip")
 	path := fs.String("path", "", "path to operate on, relative to the world root (empty = the root itself)")
 	worldsRoot := fs.String("worlds-root", "/data", "mount path of the world PVC; every path resolves under it")
+	expect := fs.String("expect-sha256", "", "write only: refuse unless the file's current SHA-256 (hex) is this")
+	createOnly := fs.Bool("create-only", false, "write only: refuse a path that already exists")
+	to := fs.String("to", "", "rename only: the destination path")
+	sourceURL := fs.String("source-url", "", "upload only: felis-api URL to fetch the bytes from")
+	size := fs.Int64("size", -1, "upload only: the byte count the fetched file must have")
+	sum := fs.String("sha256", "", "write and upload: the SHA-256 (hex) the content or the fetched file must have")
+	overwrite := fs.Bool("overwrite", false, "upload and unzip: replace files already there")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
 
 	if *op == "" {
-		fmt.Fprintln(stderr, "felis files: --op is required (list, read, or write)")
+		fmt.Fprintln(stderr, "felis files: --op is required")
 		return 2
 	}
+	limitHeapToCgroup()
+	req := fileedit.Request{
+		Op: *op, Path: *path, To: *to, Expect: *expect, CreateOnly: *createOnly, Overwrite: *overwrite,
+	}
 
 	// New content arrives base64-encoded in the environment rather than in argv:
 	// a process's arguments are world-readable on the node (/proc/<pid>/cmdline),
@@ -52,22 +68,46 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	// secrets — an RCON password in server.properties is the obvious case. The
 	// encoding is what lets arbitrary bytes (CRLF endings, a BOM, a NUL) survive a
 	// channel that must be a valid string.
-	var content []byte
-	if *op == fileedit.OpWrite {
-		raw, ok := os.LookupEnv(fileedit.ContentEnv)
-		if !ok {
-			fmt.Fprintf(stderr, "felis files: a write needs %s in the environment\n", fileedit.ContentEnv)
+	switch *op {
+	case fileedit.OpWrite:
+		// The content's SHA-256 comes with it, so bytes that changed on the way
+		// to this Job are refused rather than written (Request.ContentSHA256).
+		if *sum == "" {
+			fmt.Fprintln(stderr, "felis files: a write needs --sha256")
 			return 2
 		}
-		decoded, err := base64.StdEncoding.DecodeString(raw)
+		content, err := fileedit.ContentFromEnv(os.LookupEnv)
 		if err != nil {
-			fmt.Fprintf(stderr, "felis files: %s is not valid base64: %v\n", fileedit.ContentEnv, err)
+			fmt.Fprintf(stderr, "felis files: %v\n", err)
 			return 2
 		}
-		content = decoded
+		req.Content, req.ContentSHA256 = content, *sum
+	case fileedit.OpUpload:
+		token := os.Getenv(fileedit.UploadTokenEnv)
+		if *sourceURL == "" || token == "" {
+			fmt.Fprintf(stderr, "felis files: an upload needs --source-url and %s\n", fileedit.UploadTokenEnv)
+			return 2
+		}
+		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer stop()
+		req.Upload = &fileedit.Upload{
+			Size: *size, SHA256: *sum,
+			Open: func() (io.ReadCloser, error) { return fetchUpload(ctx, *sourceURL, token) },
+			Landed: func() {
+				if err := reportLanded(ctx, *sourceURL, token); err != nil {
+					// The file is in place; felis-api drops its copy when it
+					// has sat idle long enough, and the panel cancels it too.
+					fmt.Fprintf(stderr, "felis files: tell felis-api the upload landed: %v\n", err)
+				}
+			},
+		}
 	}
+	// An upload or an unzip (the only ops that report progress) can run long
+	// enough that felis-api does not wait on its Job, and the panel shows how far
+	// it has got from the latest of these lines (fileedit.K8sRunner.Ops).
+	req.Progress = fileedit.ThrottledProgress(stdout, time.Second, time.Now)
 
-	res, err := fileedit.Execute(*worldsRoot, *op, *path, content)
+	res, err := fileedit.Execute(*worldsRoot, req)
 	if err != nil {
 		// The operation could not be attempted — infrastructure, not caller fault.
 		fmt.Fprintf(stderr, "felis files: %v\n", err)
@@ -82,3 +122,57 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// fetchUpload opens the staged upload on felis-api's internal face. There is no
+// retry: the token opens the upload once (fileedit.Stage), so a second attempt
+// could only be refused, and the caller retries the failed Job whole (a file
+// sent in parts stays staged until its Job reports it landed, so that retry
+// does not send it again). Redirects are refused because the request carries the
+// token and the internal face never redirects; the header timeout catches a
+// wedged endpoint, and the Job's activeDeadlineSeconds bounds the body.
+func fetchUpload(ctx context.Context, url, token string) (io.ReadCloser, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: 30 * time.Second},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, fmt.Errorf("GET returned %s", resp.Status)
+	}
+	return resp.Body, nil
+}
+
+// reportLanded tells felis-api the upload's file is in place (DELETE on the URL
+// it was fetched from, with the same token), so it deletes the copy it staged.
+// One try: the file has landed whatever the answer, and a copy nobody deletes
+// is dropped once it has sat idle for fileedit.SessionIdle.
+func reportLanded(ctx context.Context, url, token string) error {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	client := &http.Client{
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		return fmt.Errorf("DELETE returned %s", resp.Status)
+	}
+	return nil
+}

cmd/felis/files_test.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"cmp"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"felis.lolicon.best/internal/fileedit"
+)
+
+// filesResult is the Result a `felis files` run printed on its marked line,
+// the last it prints.
+func filesResult(t *testing.T, stdout string) fileedit.Result {
+	t.Helper()
+	lines := strings.Split(strings.TrimSpace(stdout), "\n")
+	line, ok := strings.CutPrefix(lines[len(lines)-1], fileedit.ResultPrefix)
+	if !ok {
+		t.Fatalf("stdout has no result line: %q", stdout)
+	}
+	var res fileedit.Result
+	if err := json.Unmarshal([]byte(line), &res); err != nil {
+		t.Fatalf("result line %q: %v", line, err)
+	}
+	return res
+}
+
+// stagedSource is felis-api's internal face for one staged upload. It serves
+// body to a GET carrying Bearer token and 404 to any other, and answers the
+// DELETE that reports the file landed with landedCode (204 when unset),
+// redirecting to landedTo when that is a redirect. reports counts those
+// DELETEs, each with the token and at the path the bytes came from.
+type stagedSource struct {
+	*httptest.Server
+	reports, strays atomic.Int32
+	landedCode      int
+	landedTo        string
+}
+
+func stagedUpload(t *testing.T, token string, body []byte) *stagedSource {
+	t.Helper()
+	s := &stagedSource{}
+	s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer "+token || r.URL.Path != "/u" {
+			s.strays.Add(1)
+			http.Error(w, "no such upload", http.StatusNotFound)
+			return
+		}
+		switch r.Method {
+		case http.MethodGet:
+			w.Write(body)
+		case http.MethodDelete:
+			s.reports.Add(1)
+			if s.landedTo != "" {
+				w.Header().Set("Location", s.landedTo)
+			}
+			w.WriteHeader(cmp.Or(s.landedCode, http.StatusNoContent))
+		default:
+			s.strays.Add(1)
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		}
+	}))
+	t.Cleanup(s.Close)
+	return s
+}
+
+func uploadArgs(root, sourceURL string, body []byte) []string {
+	sum := sha256.Sum256(body)
+	return []string{
+		"--op", "upload", "--path", "plugins/a.jar", "--worlds-root", root,
+		"--source-url", sourceURL, "--size", "4", "--sha256", hex.EncodeToString(sum[:]),
+	}
+}
+
+// uploadRoot is a world with the plugins folder an upload lands in.
+func uploadRoot(t *testing.T) string {
+	t.Helper()
+	root := t.TempDir()
+	if err := os.Mkdir(filepath.Join(root, "plugins"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	return root
+}
+
+func TestCmdFilesUpload(t *testing.T) {
+	body := []byte("PK\x03\x04")
+
+	t.Run("fetches the staged bytes with its token and lands them", func(t *testing.T) {
+		root := uploadRoot(t)
+		srv := stagedUpload(t, "tok", body)
+		t.Setenv(fileedit.UploadTokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix+`{"done":4,"total":4}`+"\n") {
+			t.Fatalf("stdout %q does not start with the progress to the last byte", stdout.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != "" {
+			t.Fatalf("result = %+v", res)
+		}
+		got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar"))
+		if err != nil || !bytes.Equal(got, body) {
+			t.Fatalf("landed %q, %v", got, err)
+		}
+		if n, strays := srv.reports.Load(), srv.strays.Load(); n != 1 || strays != 0 || stderr.Len() != 0 {
+			t.Fatalf("reported landed %d times, %d stray requests, stderr %q; want once", n, strays, stderr.String())
+		}
+	})
+
+	t.Run("a file already there is a result, and nothing is reported landed", func(t *testing.T) {
+		root := uploadRoot(t)
+		if err := os.WriteFile(filepath.Join(root, "plugins", "a.jar"), []byte("old!"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		srv := stagedUpload(t, "tok", body)
+		t.Setenv(fileedit.UploadTokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeExists || srv.reports.Load() != 0 {
+			t.Fatalf("result = %+v, reported landed %d times", res, srv.reports.Load())
+		}
+	})
+
+	// The file is in place whatever felis-api answers, so the Job still succeeds
+	// and says why the staged copy may linger. A redirect is not followed, since
+	// the request carries the token.
+	for name, tc := range map[string]struct {
+		code   int
+		stderr string
+	}{
+		"refused":    {http.StatusNotFound, "felis files: tell felis-api the upload landed: DELETE returned 404 Not Found\n"},
+		"redirected": {http.StatusFound, "felis files: tell felis-api the upload landed: DELETE returned 302 Found\n"},
+	} {
+		t.Run("a landed report "+name+" still lands the file", func(t *testing.T) {
+			var elsewhere atomic.Int32
+			away := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { elsewhere.Add(1) }))
+			defer away.Close()
+			root := uploadRoot(t)
+			srv := stagedUpload(t, "tok", body)
+			srv.landedCode, srv.landedTo = tc.code, away.URL+"/u"
+			t.Setenv(fileedit.UploadTokenEnv, "tok")
+			var stdout, stderr bytes.Buffer
+			if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
+				t.Fatalf("exit %d, stderr %q", code, stderr.String())
+			}
+			if res := filesResult(t, stdout.String()); res.Code != "" || stderr.String() != tc.stderr || elsewhere.Load() != 0 {
+				t.Fatalf("result = %+v, stderr %q, redirect followed %d times", res, stderr.String(), elsewhere.Load())
+			}
+			if got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar")); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("landed %q, %v", got, err)
+			}
+		})
+	}
+
+	// A refused fetch is the Job failing, never a Result: the API answers it with a
+	// 500 the caller retries whole.
+	t.Run("a refused fetch exits 1 and lands nothing", func(t *testing.T) {
+		root := uploadRoot(t)
+		srv := stagedUpload(t, "tok", body)
+		t.Setenv(fileedit.UploadTokenEnv, "wrong")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1; stdout %q", code, stdout.String())
+		}
+		if !strings.Contains(stderr.String(), "404") {
+			t.Fatalf("stderr %q does not name the status", stderr.String())
+		}
+		if srv.reports.Load() != 0 {
+			t.Fatal("a refused fetch was reported landed")
+		}
+		if _, err := os.Lstat(filepath.Join(root, "plugins", "a.jar")); !os.IsNotExist(err) {
+			t.Fatalf("a refused fetch left a file: %v", err)
+		}
+	})
+
+	// The request carries the token, and the internal face never redirects, so a
+	// redirect is refused rather than followed with the token attached.
+	t.Run("a redirect is not followed", func(t *testing.T) {
+		root := uploadRoot(t)
+		var hits atomic.Int32
+		elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			w.Write(body)
+		}))
+		defer elsewhere.Close()
+		redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.URL+"/u", http.StatusFound))
+		defer redirecting.Close()
+		t.Setenv(fileedit.UploadTokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, redirecting.URL+"/u", body), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if n := hits.Load(); n != 0 {
+			t.Fatalf("the redirect target was fetched %d times", n)
+		}
+	})
+
+	for name, tc := range map[string]struct {
+		token string
+		drop  string
+	}{
+		"no token":      {"", ""},
+		"no source URL": {"tok", "--source-url"},
+	} {
+		t.Run(name+" exits 2", func(t *testing.T) {
+			srv := stagedUpload(t, "tok", body)
+			t.Setenv(fileedit.UploadTokenEnv, tc.token)
+			args := uploadArgs(uploadRoot(t), srv.URL+"/u", body)
+			if tc.drop != "" {
+				for i, a := range args {
+					if a == tc.drop {
+						args = append(args[:i:i], args[i+2:]...)
+						break
+					}
+				}
+			}
+			var stdout, stderr bytes.Buffer
+			if code := cmdFiles(args, &stdout, &stderr); code != 2 {
+				t.Fatalf("exit %d, want 2", code)
+			}
+		})
+	}
+}
+
+func TestCmdFilesWrite(t *testing.T) {
+	root := t.TempDir()
+	content := []byte("[]\r\n")
+	sum := sha256.Sum256(content)
+	args := []string{"--op", "write", "--path", "ops.json", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}
+
+	t.Run("reassembles the content parts", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "1")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(args, &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != "" {
+			t.Fatalf("result = %+v", res)
+		}
+		if got, err := os.ReadFile(filepath.Join(root, "ops.json")); err != nil || !bytes.Equal(got, content) {
+			t.Fatalf("wrote %q, %v", got, err)
+		}
+	})
+
+	// Writing what did arrive of an incomplete spec would truncate the file.
+	t.Run("an incomplete content spec exits 2 and writes nothing", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "2")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("x")))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 2 {
+			t.Fatalf("exit %d, want 2", code)
+		}
+		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
+			t.Fatalf("an incomplete spec wrote a file: %v", err)
+		}
+	})
+
+	// Without the content's SHA-256 the Job could not tell bytes changed on the
+	// way from the bytes felis-api sent.
+	t.Run("a write without its SHA-256 exits 2 and writes nothing", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "1")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root}, &stdout, &stderr); code != 2 {
+			t.Fatalf("exit %d, want 2", code)
+		}
+		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
+			t.Fatalf("a write without its SHA-256 wrote a file: %v", err)
+		}
+	})
+
+	t.Run("content that changed on the way is a result and writes nothing", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "1")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("[]\n")))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeDigestMismatch {
+			t.Fatalf("result = %+v, want %s", res, fileedit.CodeDigestMismatch)
+		}
+		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
+			t.Fatalf("changed content wrote a file: %v", err)
+		}
+	})
+}
+
+// A caller-fault outcome is a successful run carrying a code, so felis-api can
+// answer the precise 4xx instead of a 500.
+func TestCmdFilesCallerFaultIsAResult(t *testing.T) {
+	root := t.TempDir()
+	var stdout, stderr bytes.Buffer
+	if code := cmdFiles([]string{"--op", "mkdir", "--path", "../out", "--worlds-root", root}, &stdout, &stderr); code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, stderr.String())
+	}
+	if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeBadPath {
+		t.Fatalf("result = %+v, want code %s", res, fileedit.CodeBadPath)
+	}
+	stdout.Reset()
+	if code := cmdFiles([]string{"--worlds-root", root}, &stdout, &stderr); code != 2 {
+		t.Fatalf("no --op: exit %d, want 2", code)
+	}
+}
+
+// TestCmdFilesUnzip checks an unzip extracts next to the archive and reports its
+// progress before its result, the same way an upload does.
+func TestCmdFilesUnzip(t *testing.T) {
+	root := t.TempDir()
+	if err := os.Mkdir(filepath.Join(root, "maps"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	var zb bytes.Buffer
+	zw := zip.NewWriter(&zb)
+	for name, body := range map[string]string{"world/level.dat": "level", "world/region/r.0.0.mca": "region!"} {
+		w, err := zw.Create(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		io.WriteString(w, body)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "maps", "a.zip"), zb.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var stdout, stderr bytes.Buffer
+	if code := cmdFiles([]string{"--op", "unzip", "--path", "maps/a.zip", "--worlds-root", root}, &stdout, &stderr); code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, stderr.String())
+	}
+	if res := filesResult(t, stdout.String()); res.Code != "" || res.Files != 2 || res.Bytes != 12 {
+		t.Fatalf("result = %+v", res)
+	}
+	if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix) ||
+		!strings.Contains(stdout.String(), fileedit.ProgressPrefix+`{"done":12,"total":12}`+"\n") {
+		t.Fatalf("stdout %q does not report the progress to the last byte", stdout.String())
+	}
+	got, err := os.ReadFile(filepath.Join(root, "maps", "world", "region", "r.0.0.mca"))
+	if err != nil || string(got) != "region!" {
+		t.Fatalf("extracted %q, %v", got, err)
+	}
+}

cmd/felis/hostcreds.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// Host copies of the credentials `felis setup` takes at the keyboard: the [smtp]
+// relay password and the uploads bucket's keys. The cluster reads them from the
+// felis-smtp and felis-uploads-s3 Secrets, and a Secret lives in k3s's datastore,
+// which a reinstall (uninstall.sh keeps /etc/felis) or a host rebuilt from a
+// database bundle's state/ starts empty. Each file holds the bare value, mode
+// 0600, directly in /etc/felis beside secrets.env: every installer run applies
+// the Secrets from these files, and every database bundle, so the off-site copy
+// too, carries them.
+const (
+	hostSMTPPasswordPath       = "/etc/felis/smtp-password"
+	hostUploadsS3AccessKeyPath = "/etc/felis/uploads-s3-access-key"
+	hostUploadsS3SecretKeyPath = "/etc/felis/uploads-s3-secret-key"
+)
+
+// writeHostCredential replaces the file at path with value, mode 0600, through a
+// temporary file in the same directory, so a crash leaves the old value or the
+// new one and never a partial one.
+func writeHostCredential(path, value string) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	defer os.Remove(tmpPath)
+	// CreateTemp already makes the file 0600; the Chmod states it rather than
+	// leaning on that.
+	if err := tmp.Chmod(0o600); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if _, err := tmp.WriteString(value); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpPath, path)
+}
+
+// readHostCredential returns the value in path; ok is false when there is no
+// such file. An empty file is a value: the relay password of a relay without AUTH.
+func readHostCredential(path string) (value string, ok bool, err error) {
+	b, err := os.ReadFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return string(b), true, nil
+}
+
+// relayPassword is the [smtp] relay password as the host holds it: the copy
+// `felis setup` keeps at path, else, on an install from before that copy, the
+// felis-smtp Secret in ns, whose absence means a relay without AUTH. cl is only
+// used when the file is missing; a nil cl then reports errClusterUnreachable.
+func relayPassword(ctx context.Context, path string, cl client.Client, ns string) (string, error) {
+	if pw, ok, err := readHostCredential(path); err != nil || ok {
+		return pw, err
+	}
+	if cl == nil {
+		return "", errClusterUnreachable
+	}
+	return smtpSecretPassword(ctx, cl, ns)
+}
+
+var errClusterUnreachable = errors.New("the cluster did not answer")

cmd/felis/hostcreds_test.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/watchdog"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func TestHostCredentialFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "smtp-password")
+
+	if _, ok, err := readHostCredential(path); ok || err != nil {
+		t.Fatalf("a missing file read as ok=%v err=%v; want not there", ok, err)
+	}
+	// A copy an operator put there by hand, readable by everyone, is tightened.
+	if err := os.WriteFile(path, []byte("by hand"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range []string{"first secret", "a \"quoted\" $second\nsecret", ""} {
+		if err := writeHostCredential(path, v); err != nil {
+			t.Fatal(err)
+		}
+		got, ok, err := readHostCredential(path)
+		if err != nil || !ok || got != v {
+			t.Fatalf("read back (%q, %v, %v); want (%q, true, nil)", got, ok, err, v)
+		}
+		fi, err := os.Stat(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if mode := fi.Mode().Perm(); mode != 0o600 {
+			t.Fatalf("mode %v; want 0600", mode)
+		}
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 1 {
+		t.Fatalf("the directory holds %d entries; want the credential alone, no leftover temporary file", len(entries))
+	}
+
+	if _, _, err := readHostCredential(dir); err == nil {
+		t.Fatal("an unreadable credential read as fine")
+	}
+}
+
+func smtpSecretClient(t *testing.T, password string) client.Client {
+	t.Helper()
+	return fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
+		Data:       map[string][]byte{platform.SMTPSecretPasswordKey: []byte(password)},
+	}).Build()
+}
+
+func TestRelayPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	cl := smtpSecretClient(t, "from-secret")
+
+	if pw, err := relayPassword(ctx, host, cl, "felis"); err != nil || pw != "from-secret" {
+		t.Fatalf("without a host copy = (%q, %v); want the Secret's", pw, err)
+	}
+	if _, err := relayPassword(ctx, host, nil, "felis"); !errors.Is(err, errClusterUnreachable) {
+		t.Fatalf("without a host copy or a cluster err = %v; want errClusterUnreachable", err)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []client.Client{cl, nil} {
+		if pw, err := relayPassword(ctx, host, c, "felis"); err != nil || pw != "from-host" {
+			t.Fatalf("with a host copy (cluster %v) = (%q, %v); want the host copy", c != nil, pw, err)
+		}
+	}
+	if _, err := relayPassword(ctx, dir, cl, "felis"); err == nil {
+		t.Fatal("an unreadable host copy fell through to the Secret")
+	}
+}
+
+// The watchdog mails the most while the cluster is down: the host copy must
+// reach it then, and without one the password the last good run cached stays.
+func TestRefreshSMTPPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	var stderr bytes.Buffer
+
+	state := &watchdog.State{SMTPPassword: "cached"}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "cached" || stderr.Len() != 0 {
+		t.Fatalf("cluster down, no host copy: password %q, stderr %q; want the cached one kept quietly", state.SMTPPassword, stderr.String())
+	}
+	refreshSMTPPassword(ctx, host, smtpSecretClient(t, "from-secret"), "felis", state, &stderr)
+	if state.SMTPPassword != "from-secret" {
+		t.Fatalf("cluster up, no host copy: password %q; want the Secret's", state.SMTPPassword)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cluster down, host copy: password %q; want the host copy", state.SMTPPassword)
+	}
+	refreshSMTPPassword(ctx, dir, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" || !strings.Contains(stderr.String(), "keeping the cached one") {
+		t.Fatalf("unreadable host copy: password %q, stderr %q; want the cached one kept and the failure said", state.SMTPPassword, stderr.String())
+	}
+}
+
+func TestHostRecoveryMailerHostCopy(t *testing.T) {
+	ctx := context.Background()
+	// No kubeconfig anywhere: reaching for the cluster fails, so a pass proves
+	// the host copy was enough.
+	t.Setenv("KUBECONFIG", filepath.Join(t.TempDir(), "no-kubeconfig"))
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	off := false
+	c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_UNSET_RELAY_PW", RequireTLS: &off}
+
+	got, err := hostRecoveryMailer(c, host, "felis")(ctx)
+	if err != nil {
+		t.Fatalf("with the host copy: %v", err)
+	}
+	if relay, ok := got.(*mail.SMTP); !ok || relay.Password != "from-host" {
+		t.Fatalf("relay = %#v; want the host copy's password", got)
+	}
+	if _, err := hostRecoveryMailer(c, dir, "felis")(ctx); err == nil || !strings.Contains(err.Error(), "read the relay password") {
+		t.Fatalf("unreadable host copy: err = %v; want it named", err)
+	}
+	if _, err := hostRecoveryMailer(c, filepath.Join(dir, "none"), "felis")(ctx); err == nil || !strings.Contains(err.Error(), "reach the cluster") {
+		t.Fatalf("no host copy and no cluster: err = %v; want the cluster named", err)
+	}
+}
+
+// A whole watchdog run with the API server and PostgreSQL both down still
+// takes the relay password from the host copy, so the outage mail can
+// authenticate even when no earlier run cached it.
+func TestWatchdogReadsHostCopyWhileClusterDown(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
+	cfgPath := filepath.Join(dir, "felis.toml")
+	if err := os.WriteFile(cfgPath, []byte(`[database]
+url = "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
+[server]
+root_domain = "example.com"
+[archive]
+store = "tarLocal"
+[k8s]
+egress_mode = "nodeport"
+[smtp]
+host = "127.0.0.1"
+port = 1
+from = "[email protected]"
+username = "felis"
+password_ref = "FELIS_TEST_UNSET_RELAY_PW"
+`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	pwPath := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(pwPath, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	statePath := filepath.Join(dir, "state.json")
+	var stdout, stderr bytes.Buffer
+	cmdWatchdog([]string{
+		"-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"),
+		"-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", pwPath, "-heartbeat-file", filepath.Join(dir, "no-heartbeat"),
+	}, &stdout, &stderr)
+	if !strings.Contains(stdout.String(), "kube-api") {
+		t.Fatalf("the run found the API server up; the test needs it down (stdout %s)", stdout.String())
+	}
+	state, err := watchdog.LoadState(statePath)
+	if err != nil {
+		t.Fatalf("load state: %v (stderr %s)", err, stderr.String())
+	}
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cached relay password %q; want the host copy (stdout %s, stderr %s)", state.SMTPPassword, stdout.String(), stderr.String())
+	}
+}

cmd/felis/imagebundle.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"felis.lolicon.best/internal/imagepush"
+)
+
+// cmdImageBundle writes a release's image bundle: one OCI layout tar holding every
+// image an install runs, for one platform, plus its listing (one "role name
+// manifest-digest config-digest" line per image). deploy/build-release-artifacts.sh
+// runs it in CI; deploy/bootstrap.sh imports the tar into k3s's containerd and
+// pushes it into the platform registry with push-image --image.
+//
+//	--layout role=name=path   an image buildx wrote with --output type=oci
+//	--pull role=ref           a digest-pinned public image, named repository@digest
+//
+// The tar and the listing are written beside their final paths and renamed into
+// place, so a failed run leaves neither behind.
+func cmdImageBundle(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("image-bundle", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	platform := fs.String("platform", "", "os/arch the bundle is for, e.g. linux/arm64")
+	out := fs.String("out", "", "path of the bundle tar to write")
+	list := fs.String("list", "", "path of the listing to write")
+	var images []imagepush.BundleImage
+	fs.Func("layout", "role=name=path of an OCI layout tar (repeatable)", func(v string) error {
+		role, rest, ok := strings.Cut(v, "=")
+		name, path, ok2 := strings.Cut(rest, "=")
+		if !ok || !ok2 || role == "" || name == "" || path == "" {
+			return fmt.Errorf("want role=name=path, got %q", v)
+		}
+		images = append(images, imagepush.BundleImage{Role: role, Name: name, Layout: path})
+		return nil
+	})
+	fs.Func("pull", "role=ref of a digest-pinned public image (repeatable)", func(v string) error {
+		role, ref, ok := strings.Cut(v, "=")
+		if !ok || role == "" || !strings.Contains(ref, "@sha256:") {
+			return fmt.Errorf("want role=ref with ref pinned by digest, got %q", v)
+		}
+		images = append(images, imagepush.BundleImage{Role: role, Name: imagepush.PinnedName(ref), Source: ref})
+		return nil
+	})
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if *platform == "" || *out == "" || *list == "" || len(images) == 0 {
+		fmt.Fprintln(stderr, "felis image-bundle: --platform, --out, --list and at least one --layout or --pull are required")
+		return 2
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	if err := writeImageBundle(ctx, &imagepush.Source{Platform: *platform}, images, *out, *list); err != nil {
+		fmt.Fprintf(stderr, "felis image-bundle: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+func writeImageBundle(ctx context.Context, s *imagepush.Source, images []imagepush.BundleImage, out, list string) (err error) {
+	tmpOut, tmpList := out+".tmp", list+".tmp"
+	defer func() {
+		if err != nil {
+			os.Remove(tmpOut)
+			os.Remove(tmpList)
+		}
+	}()
+	f, err := os.Create(tmpOut)
+	if err != nil {
+		return err
+	}
+	w := bufio.NewWriterSize(f, 1<<20)
+	entries, err := imagepush.WriteBundle(ctx, s, images, w)
+	if err == nil {
+		err = w.Flush()
+	}
+	if err == nil {
+		err = f.Sync()
+	}
+	if cerr := f.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return err
+	}
+	var b strings.Builder
+	for _, e := range entries {
+		fmt.Fprintf(&b, "%s %s %s %s\n", e.Role, e.Name, e.Digest, e.Config)
+	}
+	if err := os.WriteFile(tmpList, []byte(b.String()), 0o644); err != nil {
+		return err
+	}
+	if err := os.Rename(tmpOut, out); err != nil {
+		return err
+	}
+	return os.Rename(tmpList, list)
+}

cmd/felis/imagebundle_test.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// writeTestLayout writes an OCI layout tar holding one linux/arch image with one
+// layer, the way buildx --output type=oci leaves a single-platform build, and
+// returns its path with the manifest and config digests.
+func writeTestLayout(t *testing.T, dir, arch, layer string) (path, manifestDigest, configDigest string) {
+	t.Helper()
+	blobs := map[string][]byte{}
+	add := func(b []byte) (string, int) {
+		sum := sha256.Sum256(b)
+		d := "sha256:" + hex.EncodeToString(sum[:])
+		blobs[d] = b
+		return d, len(b)
+	}
+	cfgDigest, cfgSize := add([]byte(`{"architecture":"` + arch + `","os":"linux","rootfs":{"type":"layers"}}`))
+	layerDigest, layerSize := add([]byte(layer))
+	manifest := fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",`+
+		`"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":%q,"size":%d},`+
+		`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":%q,"size":%d}]}`,
+		cfgDigest, cfgSize, layerDigest, layerSize)
+	mDigest, mSize := add([]byte(manifest))
+	index, _ := json.Marshal(map[string]any{
+		"schemaVersion": 2,
+		"manifests": []map[string]any{{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": mDigest, "size": mSize,
+		}},
+	})
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	put := func(name string, b []byte) {
+		tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(b)), Typeflag: tar.TypeReg})
+		tw.Write(b)
+	}
+	put("oci-layout", []byte(`{"imageLayoutVersion":"1.0.0"}`))
+	put("index.json", index)
+	for d, b := range blobs {
+		put("blobs/sha256/"+strings.TrimPrefix(d, "sha256:"), b)
+	}
+	tw.Close()
+	path = filepath.Join(dir, arch+"-"+layer+".tar")
+	if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return path, mDigest, cfgDigest
+}
+
+func TestImageBundleWritesTheListingTheInstallerReads(t *testing.T) {
+	dir := t.TempDir()
+	limbo, limboDigest, limboConfig := writeTestLayout(t, dir, "arm64", "limbo")
+	lobby, lobbyDigest, lobbyConfig := writeTestLayout(t, dir, "arm64", "lobby")
+	out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
+	var stderr bytes.Buffer
+	code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
+		"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo,
+		"--layout", "lobby=registry.felis.svc:5000/felis/lobby:demo=" + lobby,
+	}, nil, &stderr)
+	if code != 0 {
+		t.Fatalf("image-bundle = %d: %s", code, stderr.String())
+	}
+	// deploy/bootstrap.sh reads this with `read -r role name digest config`.
+	got, _ := os.ReadFile(list)
+	want := "limbo registry.felis.svc:5000/felis/limbo:demo " + limboDigest + " " + limboConfig + "\n" +
+		"lobby registry.felis.svc:5000/felis/lobby:demo " + lobbyDigest + " " + lobbyConfig + "\n"
+	if string(got) != want {
+		t.Errorf("listing:\n%s\nwant:\n%s", got, want)
+	}
+	if fi, err := os.Stat(out); err != nil || fi.Size() == 0 {
+		t.Errorf("bundle: %v", err)
+	}
+	if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
+		t.Errorf("left behind %v", leftovers)
+	}
+}
+
+func TestImageBundleLeavesNothingWhenAnImageIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	amd, _, _ := writeTestLayout(t, dir, "amd64", "limbo")
+	out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
+	// The pair an earlier run wrote stays as it was: a listing beside a bundle it
+	// does not describe would have the installer look for images that are not there.
+	os.WriteFile(out, []byte("old bundle"), 0o644)
+	os.WriteFile(list, []byte("old listing\n"), 0o644)
+	var stderr bytes.Buffer
+	code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
+		"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + amd}, nil, &stderr)
+	if code != 1 || !strings.Contains(stderr.String(), "is a linux/amd64 image") {
+		t.Fatalf("image-bundle = %d: %s", code, stderr.String())
+	}
+	if got, _ := os.ReadFile(out); string(got) != "old bundle" {
+		t.Errorf("bundle was replaced with %d bytes", len(got))
+	}
+	if got, _ := os.ReadFile(list); string(got) != "old listing\n" {
+		t.Errorf("listing was replaced with %q", got)
+	}
+	if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
+		t.Errorf("left behind %v", leftovers)
+	}
+}
+
+func TestPushImageReadsABundleByImageName(t *testing.T) {
+	dir := t.TempDir()
+	limbo, _, _ := writeTestLayout(t, dir, "arm64", "limbo")
+	out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
+	if code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
+		"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo}, nil, &bytes.Buffer{}); code != 0 {
+		t.Fatal("image-bundle failed")
+	}
+	t.Setenv("FELIS_REGISTRY_USERNAME", "platform")
+	t.Setenv("FELIS_REGISTRY_PASSWORD", "x")
+	// The name is looked up in the bundle's index before the registry is contacted,
+	// so a name the bundle lacks fails here with what it does hold.
+	var stderr bytes.Buffer
+	code := cmdPushImage([]string{"--tar", out, "--image", "registry.felis.svc:5000/felis/lobby:demo",
+		"--ref", "127.0.0.1:1/felis/lobby:demo"}, &bytes.Buffer{}, &stderr)
+	if code != 1 || !strings.Contains(stderr.String(), "holds no image named registry.felis.svc:5000/felis/lobby:demo (it holds: registry.felis.svc:5000/felis/limbo:demo") {
+		t.Fatalf("push-image = %d: %s", code, stderr.String())
+	}
+}

cmd/felis/initforwarding.go

@@ -20,18 +20,14 @@ const forwardingSecretEnv = "FELIS_FORWARDING_SECRET"
 // config/ and server.properties live under it.
 const defaultForwardingDataDir = "/data"
 
-// fwd*Mode make the written config readable AND rewritable by the main server
-// container, whose UID we do not control (an arbitrary user image). The
-// initContainer runs as root (see buildStatefulSet) so it can write into a data
-// volume of unknown ownership; 0666/0777 then let a non-root Paper rewrite the
-// same files on boot.
-//
-// This relies on the initContainer running as root to write into a volume of
-// unknown ownership; that is how the operator schedules it. If that ever changes,
-// give the server pod an fsGroup so the shared volume is group-writable instead.
+// fwd*Mode are the modes the written config lands with. The initContainer runs as
+// the same uid as the server container (naming.GameUID, pinned by the operator in
+// the pod securityContext) after the prepare-data initContainer has handed the
+// whole volume to that uid, so owner read/write is all the server needs to rewrite
+// these files on boot and nothing else on the node gets write access to them.
 const (
-	fwdFileMode os.FileMode = 0o666
-	fwdDirMode  os.FileMode = 0o777
+	fwdFileMode os.FileMode = 0o644
+	fwdDirMode  os.FileMode = 0o755
 )
 
 // cmdInitForwarding is the felis-image initContainer entrypoint that makes an
@@ -96,9 +92,8 @@ func writePaperGlobal(dataDir, secret string) error {
 	if err := os.MkdirAll(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("create %s: %w", dir, err)
 	}
-	// MkdirAll honours the process umask (root's is typically 022 → 0755); chmod
-	// does not, and a non-root main container must be able to place/replace the
-	// file in this directory on boot.
+	// MkdirAll honours the process umask; chmod does not, so a directory an older
+	// release left at 0777 is brought back to fwdDirMode here.
 	if err := os.Chmod(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("chmod %s: %w", dir, err)
 	}
@@ -188,8 +183,8 @@ func upsertProperty(content []byte, key, value string) []byte {
 }
 
 // writeFileMode writes data then forces the mode, since WriteFile honours the
-// umask (root's is typically 022 → 0644) but a non-root main container must be
-// able to rewrite these files on boot.
+// umask and leaves an existing file's mode alone: a file an older release wrote
+// world-writable (0666) is tightened back to fwdFileMode on the next boot.
 func writeFileMode(path string, data []byte) error {
 	if err := os.WriteFile(path, data, fwdFileMode); err != nil {
 		return fmt.Errorf("write %s: %w", path, err)

cmd/felis/initforwarding_test.go

@@ -164,8 +164,9 @@ func TestUpsertPropertyAppends(t *testing.T) {
 	}
 }
 
-// The written files must be group/world writable so a non-root main container can
-// rewrite them. chmod semantics are POSIX-only, so this asserts on non-Windows.
+// The written files land at fwdFileMode: owner-writable for the game uid the init
+// shares with the server container, and no longer world-writable. chmod semantics
+// are POSIX-only, so this asserts on non-Windows.
 func TestWriteForwardingFileModes(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("POSIX file modes not represented on Windows")

cmd/felis/initvolume.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"syscall"
+
+	"felis.lolicon.best/internal/naming"
+)
+
+// cmdInitVolume is the felis-image `prepare-data` initContainer entrypoint: it
+// hands every entry of a server's world volume to the game uid/gid before the
+// server container starts. The operator runs the server itself as naming.GameUID,
+// so a world written by an earlier release (whose server ran as root), a restore
+// Job (which extracts as root), or a storage provisioner that creates the volume
+// root-owned would otherwise leave files the server cannot write — a world that
+// boots and then fails every save.
+//
+// fsGroup covers only part of this: kubelet applies it to volume types that
+// support ownership management, and a k3s local-path PV is a hostPath underneath,
+// which it skips. A walk from inside the pod works for every volume type.
+//
+// Only mismatched entries are touched, so a volume already owned by the game uid
+// costs one lstat per entry and no writes. The walk runs inside an os.Root at the
+// data dir and uses lchown, so a symlink a plugin planted is re-owned as a link
+// and never followed out of the volume.
+//
+// A single entry that cannot be chowned is reported and skipped: failing the pod
+// over one odd file would keep the whole server down, while the server itself
+// reports the one file it cannot write. Only an unreadable data dir fails.
+func cmdInitVolume(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("init-volume", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	dataDir := fs.String("data", defaultForwardingDataDir, "world volume mount to hand to the game uid")
+	uid := fs.Int64("uid", naming.GameUID, "owner uid for every entry")
+	gid := fs.Int64("gid", naming.GameGID, "owner gid for every entry")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	root, err := os.OpenRoot(*dataDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: open %s: %v\n", *dataDir, err)
+		return 1
+	}
+	defer root.Close()
+	res, err := chownTree(root, int(*uid), int(*gid), root.Lchown)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: %v\n", err)
+		return 1
+	}
+	for _, f := range res.failures {
+		fmt.Fprintf(stderr, "felis init-volume: %s\n", f)
+	}
+	fmt.Fprintf(stdout, "felis init-volume: %d entries checked, %d handed to %d:%d, %d failed\n",
+		res.checked, res.changed, *uid, *gid, len(res.failures))
+	return 0
+}
+
+// chownResult tallies one walk; failures is capped so a volume of thousands of
+// unownable files cannot flood the pod log.
+type chownResult struct {
+	checked  int
+	changed  int
+	failures []string
+}
+
+const maxReportedChownFailures = 20
+
+// chownTree walks root and calls chown on every entry (the root dir included)
+// whose owner is not uid:gid. It returns an error only when the root itself
+// cannot be read; per-entry failures are collected in the result.
+func chownTree(root *os.Root, uid, gid int, chown func(name string, uid, gid int) error) (chownResult, error) {
+	var res chownResult
+	fail := func(name string, err error) {
+		if len(res.failures) < maxReportedChownFailures {
+			res.failures = append(res.failures, fmt.Sprintf("%s: %v", name, err))
+		} else if len(res.failures) == maxReportedChownFailures {
+			res.failures = append(res.failures, "further failures not listed")
+		}
+	}
+	err := fs.WalkDir(root.FS(), ".", func(name string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			if name == "." {
+				return walkErr
+			}
+			fail(name, walkErr)
+			// A directory that cannot be listed is skipped as a whole; a file
+			// error has nothing below it to skip.
+			if d != nil && d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		res.checked++
+		info, err := d.Info()
+		if err != nil {
+			fail(name, err)
+			return nil
+		}
+		if st, ok := info.Sys().(*syscall.Stat_t); ok && int(st.Uid) == uid && int(st.Gid) == gid {
+			return nil
+		}
+		if err := chown(name, uid, gid); err != nil {
+			if !errors.Is(err, fs.ErrNotExist) { // gone mid-walk: nothing left to own
+				fail(name, err)
+			}
+			return nil
+		}
+		res.changed++
+		return nil
+	})
+	return res, err
+}

cmd/felis/initvolume_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strconv"
+	"testing"
+)
+
+// openTree builds a small world under a temp dir: nested dirs, a file, and a
+// symlink pointing out of the volume that the walk must not follow.
+func openTree(t *testing.T) *os.Root {
+	t.Helper()
+	dir := t.TempDir()
+	for _, d := range []string{"world/region", "plugins"} {
+		if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, f := range []string{"level.dat", "world/region/r.0.0.mca"} {
+		if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	outside := t.TempDir()
+	if err := os.Symlink(outside, filepath.Join(dir, "plugins", "escape")); err != nil {
+		t.Fatal(err)
+	}
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { root.Close() })
+	return root
+}
+
+// Every entry owned by someone else is handed over, the root dir included, and a
+// symlink is re-owned as a link rather than walked into.
+func TestChownTreeHandsOverMismatchedEntries(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	var got []string
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		got = append(got, name)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	want := []string{".", "level.dat", "plugins", "plugins/escape", "world", "world/region", "world/region/r.0.0.mca"}
+	slices.Sort(got)
+	if !slices.Equal(got, want) {
+		t.Errorf("chowned %v, want %v", got, want)
+	}
+	if res.changed != len(want) || res.checked != len(want) || len(res.failures) != 0 {
+		t.Errorf("result = %+v, want %d checked and changed, no failures", res, len(want))
+	}
+}
+
+// A volume already owned by the game uid costs no chown at all: this is the steady
+// state every restart after the first one hits.
+func TestChownTreeSkipsMatchingOwner(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	calls := 0
+	res, err := chownTree(root, os.Getuid(), os.Getgid(), func(string, int, int) error {
+		calls++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if calls != 0 || res.changed != 0 {
+		t.Errorf("chown called %d times on an already-owned tree (result %+v)", calls, res)
+	}
+}
+
+// One entry that refuses the chown is reported and the walk carries on: a single
+// odd file must not keep the whole server from starting.
+func TestChownTreeContinuesPastFailures(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		if name == "level.dat" {
+			return os.ErrPermission
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if len(res.failures) != 1 || res.changed != 6 {
+		t.Errorf("result = %+v, want 1 failure and 6 changed", res)
+	}
+}
+
+// Against a real directory the owner already matches, so the command succeeds
+// without needing CAP_CHOWN — the path every test runner (non-root) can take.
+func TestCmdInitVolumeOwnedTree(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "server.properties"), []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	var out, errb bytes.Buffer
+	code := cmdInitVolume([]string{"--data", dir, "--uid", strconv.Itoa(os.Getuid()), "--gid", strconv.Itoa(os.Getgid())}, &out, &errb)
+	if code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, errb.String())
+	}
+	if code := cmdInitVolume([]string{"--data", filepath.Join(dir, "missing")}, &out, &errb); code != 1 {
+		t.Errorf("missing data dir exit = %d, want 1", code)
+	}
+}

cmd/felis/main.go

@@ -6,8 +6,32 @@ package main
 
 import (
 	"os"
+	"path/filepath"
 )
 
 func main() {
+	ensureHostBinDirOnPath()
 	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
 }
+
+// hostBinDir is where deploy/bootstrap.sh installs felis, k3s and cloudflared.
+const hostBinDir = "/usr/local/bin"
+
+// ensureHostBinDirOnPath appends hostBinDir to PATH when it is missing, so the
+// k3s and cloudflared this binary execs are found beside it. sudo's secure_path
+// on EL leaves /usr/local/bin out: `sudo /usr/local/bin/felis db backup` would
+// otherwise run with no k3s to reach the database's pod through. Appended, so a
+// PATH that names another k3s first keeps it.
+func ensureHostBinDirOnPath() {
+	path := os.Getenv("PATH")
+	for _, dir := range filepath.SplitList(path) {
+		if dir == hostBinDir {
+			return
+		}
+	}
+	if path == "" {
+		os.Setenv("PATH", hostBinDir)
+		return
+	}
+	os.Setenv("PATH", path+string(os.PathListSeparator)+hostBinDir)
+}

cmd/felis/main_test.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+	"regexp"
+	"testing"
+)
+
+func TestEnsureHostBinDirOnPath(t *testing.T) {
+	for _, c := range []struct{ in, want string }{
+		{"/usr/sbin:/usr/bin", "/usr/sbin:/usr/bin:/usr/local/bin"},
+		{"/usr/local/bin:/usr/bin", "/usr/local/bin:/usr/bin"},
+		{"/opt/k3s:/usr/bin:/usr/local/bin", "/opt/k3s:/usr/bin:/usr/local/bin"},
+		{"", "/usr/local/bin"},
+	} {
+		t.Setenv("PATH", c.in)
+		ensureHostBinDirOnPath()
+		if got := os.Getenv("PATH"); got != c.want {
+			t.Errorf("PATH %q became %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+// run is what the tests drive, so the PATH fix must sit in main, before it.
+func TestMainFixesPathBeforeRunning(t *testing.T) {
+	b, err := os.ReadFile("main.go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !regexp.MustCompile(`func main\(\) \{\n\tensureHostBinDirOnPath\(\)\n\tos\.Exit\(run\(`).Match(b) {
+		t.Fatal("main does not call ensureHostBinDirOnPath before run")
+	}
+}

cmd/felis/manifests.go

@@ -26,13 +26,20 @@ func (m *multiFlag) Set(v string) error {
 // felis-reaper identity only when the retention reaper is enabled, gated with
 // its CronJob), the weak build/restore Job SAs, the build/minecraft
 // NetworkPolicies, and the running control-plane workloads (felis-api/operator
-// Deployments + the in-cluster registry Deployment/Service/PVC) — as a single
-// multi-document YAML stream on stdout, ready for `kubectl apply -f -`.
+// Deployments + the in-cluster registry Deployment/Service/PVC + the
+// world-archive PVC that backs backup/restore, unless --backup-pvc is emptied)
+// — as a single multi-document YAML stream on stdout, ready for
+// `kubectl apply -f -`.
 //
 // It is a pure renderer: it never contacts a cluster and holds no credentials.
 // --velocity-cidr records the proxy host addresses allowed by the game NetworkPolicy.
 // Kubernetes permits resident-node traffic regardless, but remote proxy deployments
 // need an explicit CIDR, so the renderer refuses to guess.
+//
+// --only postgres renders just the control-plane database (platform.PostgresObjects),
+// which the installer brings up before migrations, before it has anything else
+// to render the full bundle with; it needs neither --felis-image nor
+// --velocity-cidr.
 func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("manifests", flag.ContinueOnError)
 	fs.SetOutput(stderr)
@@ -43,17 +50,39 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	registryPort := fs.Int("registry-port", 5000, "port the in-cluster registry listens on")
 	panelNodePort := fs.Int("panel-node-port", int(platform.DefaultPanelNodePort), "NodePort that exposes the built-in HTTPS panel/API origin")
 	felisImage := fs.String("felis-image", "", "container image the felis-api/operator Deployments run, also passed through as FELIS_IMAGE (REQUIRED)")
-	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry:2)")
-	backupPVC := fs.String("backup-pvc", "", "name of the backup PVC advertised to the restore executor via FELIS_BACKUP_PVC (default none = restore endpoint returns 503)")
-	worldsHostPath := fs.String("worlds-host-path", "", "node directory under which each world PVC is visible as <path>/<pvc>; enables the reaper CronJob (requires --backup-pvc and --archive-local-path)")
-	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
+	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry 2.8.3, pinned by digest)")
+	postgresImage := fs.String("postgres-image", "", "control-plane database image (default: PostgreSQL 18.6, pinned by digest)")
+	only := fs.String("only", "", `render one part of the bundle instead of all of it; "postgres" is the control-plane database`)
+	backupPVC := fs.String("backup-pvc", "felis-backups", "name of the world-archive PVC this bundle renders in the Minecraft namespace and advertises to the backup/restore executors via FELIS_BACKUP_PVC (default: felis-backups; pass an empty value to render none, leaving backup/restore answering 503)")
+	worldsHostPath := fs.String("worlds-host-path", "", "node directory the reaper reads worlds from: each world PVC resolves as <path>/<pvc>, or as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (k3s storage root: /var/lib/rancher/k3s/storage); enables the reaper CronJob (requires --archive-local-path and a non-empty --backup-pvc)")
+	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path. With the backup PVC alone it renders the retention-only CronJob, which deletes backups past their expiry and never touches a world")
+	registryStorage := fs.String("registry-storage", "", "capacity the registry PVC requests (default 10Gi; k3s local-path does not enforce it)")
+	uploadsStorage := fs.String("uploads-storage", "", "capacity the uploads PVC requests (default 5Gi; k3s local-path does not enforce it)")
+	backupStorage := fs.String("backup-storage", "", "capacity the world-archive PVC requests (default 10Gi; k3s local-path does not enforce it)")
+	reaperNode := fs.String("reaper-node", "", "node that holds --worlds-host-path: pins the reaper CronJob's pod there via nodeSelector kubernetes.io/hostname (multi-node clusters need this, or the reaper may schedule where the hostPath is empty)")
 	var velocityCIDRs multiFlag
 	fs.Var(&velocityCIDRs, "velocity-cidr", "CIDR of a Velocity proxy host allowed to reach game port 25565 (repeatable, REQUIRED)")
 	var packageCIDRs multiFlag
 	fs.Var(&packageCIDRs, "package-cidr", "CIDR of a package mirror build Pods may reach (repeatable; default none = no internet egress)")
+	var serverDenyCIDRs multiFlag
+	fs.Var(&serverDenyCIDRs, "server-egress-deny-cidr", "extra CIDR game server pods may never reach, e.g. the node's public address (repeatable)")
+	var serverAllowCIDRs multiFlag
+	fs.Var(&serverAllowCIDRs, "server-egress-allow-cidr", "private CIDR game server pods may reach despite the private-range block, e.g. a LAN database (repeatable)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
+	switch *only {
+	case "":
+	case "postgres":
+		return renderManifests(stdout, stderr, platform.PostgresObjects(platform.Params{
+			ControlNamespace:   *controlNS,
+			MinecraftNamespace: *minecraftNS,
+			PostgresImage:      *postgresImage,
+		}))
+	default:
+		fmt.Fprintf(stderr, "felis manifests: --only %q: the one part that renders alone is \"postgres\"\n", *only)
+		return 2
+	}
 
 	// Keep proxy placement explicit. This matters for remote proxies and documents
 	// the expected source even when Velocity runs on the resident node.
@@ -71,7 +100,9 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"(the felis-api/operator Deployments run it and it is passed through as FELIS_IMAGE, e.g. --felis-image registry.felis.svc:5000/felis:v1)")
 		return 2
 	}
-	for _, cidr := range append(append([]string{}, velocityCIDRs...), packageCIDRs...) {
+	allCIDRs := append(append([]string{}, velocityCIDRs...), packageCIDRs...)
+	allCIDRs = append(append(allCIDRs, serverDenyCIDRs...), serverAllowCIDRs...)
+	for _, cidr := range allCIDRs {
 		if _, _, err := net.ParseCIDR(cidr); err != nil {
 			fmt.Fprintf(stderr, "felis manifests: invalid CIDR %q: %v\n", cidr, err)
 			return 2
@@ -81,39 +112,70 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis manifests: --panel-node-port must be in Kubernetes NodePort range 30000-32767 (got %d)\n", *panelNodePort)
 		return 2
 	}
+	// The node pin exists only for the reaper's hostPath: naming a node without the
+	// worlds root would be silently dropped (no CronJob renders), so fail loud like
+	// the storage-trio check below.
+	if *reaperNode != "" && *worldsHostPath == "" {
+		fmt.Fprintln(stderr, "felis manifests: --reaper-node requires --worlds-host-path "+
+			"(it pins the reaper CronJob, which renders only with the retention storage trio)")
+		return 2
+	}
 
-	// Retention/reaper rendering is opt-in and needs all three storage coordinates
-	// together: where worlds live (to read+archive them), the backup PVC (to write
-	// archives into), and the path it is mounted at (which MUST equal felis.toml
-	// [archive] local_path so tarLocal's absolute archive refs resolve). A partial
-	// configuration is almost certainly an operator mistake, so fail loud rather than
-	// silently drop retention. Asking for it without the other two is rejected; an
-	// empty trio renders the bundle WITHOUT the reaper and says so.
+	// Retention/reaper rendering is opt-in and needs a storage topology together:
+	// where worlds live (to read+archive them), a backup PVC (to write archives
+	// into — rendered from --backup-pvc), and the path it is mounted at (which MUST
+	// equal felis.toml [archive] local_path so tarLocal's absolute archive refs
+	// resolve). A partial configuration is almost certainly an operator mistake, so
+	// fail loud rather than silently drop retention or render a reaper with nowhere
+	// to write. The backup PVC itself defaults to felis-backups (it is what makes a
+	// default install's backup endpoint work at all); retention additionally needs
+	// --worlds-host-path.
 	if *worldsHostPath != "" {
 		if *backupPVC == "" || *archiveLocalPath == "" {
 			fmt.Fprintln(stderr, "felis manifests: --worlds-host-path enables the reaper CronJob and requires "+
-				"--backup-pvc and --archive-local-path too (--archive-local-path must equal felis.toml [archive] local_path)")
+				"--archive-local-path (must equal felis.toml [archive] local_path) and a non-empty --backup-pvc "+
+				"(the archive store; default felis-backups)")
 			return 2
 		}
-		// The reaper WILL render. Two deployment preconditions this generator cannot
-		// check would SILENTLY turn retention into a no-op if unmet — surface them as
+		// The reaper WILL render. Two deployment facts this generator cannot check
+		// would silently turn retention into a no-op if unmet — surface them as
 		// loudly as the fail-closed cases above, so an operator is never left with a
-		// reaper that reaps an empty directory. (Both are also in the WorldsHostPath
-		// flag/field docs, but nobody deploying from stdout reads those.)
+		// reaper that reaps nothing. (Both are also in the WorldsHostPath flag/field
+		// docs, but nobody deploying from stdout reads those.)
+		pin := "the CronJob sets NO nodeSelector: a single-node starter pins it to the worlds implicitly, but on a " +
+			"multi-node cluster you MUST pass --reaper-node <name> (or add a nodeSelector) for the node holding the " +
+			"worlds, or the reaper may schedule where the hostPath is empty"
+		if *reaperNode != "" {
+			pin = fmt.Sprintf("the CronJob and its worlds-root PV are pinned to node %q via kubernetes.io/hostname — "+
+				"keep this pointed at the node that actually holds the world volumes", *reaperNode)
+		}
 		fmt.Fprintf(stderr, "felis manifests: note: rendering the retention reaper CronJob (worlds hostPath %q). "+
-			"Two preconditions are NOT verified here:\n"+
-			"  - each world PVC must be visible at %s/<pvc> on the node: a stock local-path-provisioner lays "+
-			"volumes under PV-name paths (.../pvc-<uuid>_<ns>_<pvc>/), so unless the worlds StorageClass is "+
-			"arranged to expose <path>/<pvc>, the reaper tars an empty directory;\n"+
-			"  - the CronJob sets NO nodeSelector: a single-node starter pins it to the worlds implicitly, but "+
-			"on a multi-node cluster you MUST add a nodeSelector for the node holding the worlds, or the reaper "+
-			"may schedule where the hostPath is empty.\n", *worldsHostPath, *worldsHostPath)
+			"These points are NOT verified here:\n"+
+			"  - the node's world volumes must actually live below %s: the reaper resolves a world as "+
+			"%s/<pvc>, then as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (what k3s "+
+			"writes under /var/lib/rancher/k3s/storage). Any other provisioner needs its volumes exposed as "+
+			"<path>/<pvc>, or each candidate's archive fails and the world is preserved;\n"+
+			"  - %s.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, pin)
 	} else {
-		fmt.Fprintln(stderr, "felis manifests: note: retention reaper CronJob not rendered "+
-			"(pass --worlds-host-path, --backup-pvc and --archive-local-path to enable it)")
+		switch {
+		case *archiveLocalPath != "" && *backupPVC == "":
+			fmt.Fprintln(stderr, "felis manifests: --archive-local-path names where the backup PVC is mounted, "+
+				"but --backup-pvc is empty (no archive store renders); drop one or the other")
+			return 2
+		case *archiveLocalPath != "":
+			fmt.Fprintln(stderr, "felis manifests: note: rendering the reaper CronJob retention-only: backups past "+
+				"their expiry are deleted daily, idle worlds are never archived or deleted "+
+				"(pass --worlds-host-path to reap them too)")
+		case *backupPVC != "":
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered: backups are never expired, so "+
+				"the archive store only grows until the disk fills (pass --archive-local-path, equal to felis.toml "+
+				"[archive] local_path, for the retention-only CronJob, and --worlds-host-path as well to reap idle worlds)")
+		default:
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered (backups are disabled)")
+		}
 	}
 
-	out, err := platform.RenderYAML(platform.Params{
+	params := platform.Params{
 		ControlNamespace:   *controlNS,
 		MinecraftNamespace: *minecraftNS,
 		BuildNamespace:     *buildNS,
@@ -122,12 +184,30 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 		PanelNodePort:      int32(*panelNodePort),
 		FelisImage:         *felisImage,
 		RegistryImage:      *registryImage,
+		PostgresImage:      *postgresImage,
 		BackupPVC:          *backupPVC,
 		WorldsHostPath:     *worldsHostPath,
+		ReaperNode:         *reaperNode,
 		ArchiveLocalPath:   *archiveLocalPath,
 		VelocityCIDRs:      []string(velocityCIDRs),
 		PackageSourceCIDRs: []string(packageCIDRs),
-	})
+
+		ServerEgressDenyCIDRs:  []string(serverDenyCIDRs),
+		ServerEgressAllowCIDRs: []string(serverAllowCIDRs),
+
+		RegistryStorage: *registryStorage,
+		UploadsStorage:  *uploadsStorage,
+		BackupStorage:   *backupStorage,
+	}
+	if err := params.Validate(); err != nil {
+		fmt.Fprintf(stderr, "felis manifests: %v\n", err)
+		return 2
+	}
+	return renderManifests(stdout, stderr, platform.Objects(params))
+}
+
+func renderManifests(stdout, stderr io.Writer, objs []platform.Object) int {
+	out, err := platform.RenderObjects(objs)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis manifests: render: %v\n", err)
 		return 1

cmd/felis/manifests_test.go

@@ -77,6 +77,10 @@ func TestManifestsRendersBundle(t *testing.T) {
 		"10.0.0.5/32",
 		// The felis image flows through to the Deployments.
 		"registry.felis.svc:5000/felis:v1",
+		// Backup works out of the box: the archive PVC renders and the api gets
+		// the env that wires the backup/restore executors to it.
+		"name: felis-backups",
+		"name: FELIS_BACKUP_PVC",
 	} {
 		if !strings.Contains(text, want) {
 			t.Errorf("rendered bundle missing %q", want)
@@ -86,24 +90,29 @@ func TestManifestsRendersBundle(t *testing.T) {
 		t.Error("rendered bundle must not contain ClusterRole/ClusterRoleBinding")
 	}
 	// Without the retention flags, the reaper CronJob is not rendered and the
-	// generator says so on stderr.
+	// generator says so on stderr, naming what that leaves: backups that never
+	// expire.
 	if strings.Contains(text, "kind: CronJob") {
-		t.Error("no reaper CronJob must render without --worlds-host-path")
+		t.Error("no reaper CronJob must render without --archive-local-path")
 	}
-	if !strings.Contains(errBuf.String(), "not rendered") {
-		t.Errorf("expected a 'reaper not rendered' notice on stderr, got %q", errBuf.String())
+	if !strings.Contains(errBuf.String(), "not rendered") || !strings.Contains(errBuf.String(), "never expired") {
+		t.Errorf("expected a 'reaper not rendered, backups never expired' notice on stderr, got %q", errBuf.String())
 	}
 }
 
-// TestManifestsReaperRequiresTrio proves --worlds-host-path is a fail-loud opt-in:
-// asking for the reaper without the backup PVC and its mount path (which must equal
-// [archive] local_path) is rejected rather than silently dropping retention.
-func TestManifestsReaperRequiresTrio(t *testing.T) {
+// TestManifestsReaperRequiresStorage proves --worlds-host-path is a fail-loud
+// opt-in: asking for the reaper without a writable archive store (the backup PVC,
+// which defaults to felis-backups but can be emptied) and its mount path (which
+// must equal [archive] local_path) is rejected rather than silently dropping
+// retention or deleting worlds it could not archive first.
+func TestManifestsReaperRequiresStorage(t *testing.T) {
 	base := []string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32", "--worlds-host-path", "/var/lib/felis/worlds"}
 	for _, extra := range [][]string{
-		{},                                   // neither backup-pvc nor archive-local-path
-		{"--backup-pvc", "felis-backups"},    // missing archive-local-path
-		{"--archive-local-path", "/backups"}, // missing backup-pvc
+		{},                        // missing archive-local-path (backup-pvc defaults)
+		{"--backup-pvc", "other"}, // still missing archive-local-path
+		// A reaper with no archive store would have nowhere to write the archive
+		// it must verify before deleting a world; emptying the PVC is rejected.
+		{"--archive-local-path", "/backups", "--backup-pvc="},
 	} {
 		var out, errBuf bytes.Buffer
 		code := run(append(append([]string{}, base...), extra...), &out, &errBuf)
@@ -119,6 +128,57 @@ func TestManifestsReaperRequiresTrio(t *testing.T) {
 	}
 }
 
+// TestManifestsBackupPVCOptOut proves --backup-pvc= renders a bundle with no
+// archive store at all: no PVC and no FELIS_BACKUP_PVC env, so backup/restore
+// answer 503 instead of pointing Jobs at a claim nobody provisions.
+func TestManifestsBackupPVCOptOut(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test",
+		"--velocity-cidr", "10.0.0.5/32", "--backup-pvc="}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	for _, absent := range []string{"felis-backups", "FELIS_BACKUP_PVC"} {
+		if strings.Contains(out.String(), absent) {
+			t.Errorf("--backup-pvc= bundle must not contain %q", absent)
+		}
+	}
+}
+
+// TestManifestsRendersRetentionOnly: the archive store without a worlds root
+// still gets the daily CronJob, retention-only, so backups past their expiry
+// leave the store on an install that never reaps a world; the operator is told
+// which of the two it got. An archive path with the store switched off is a
+// mistake and fails loud.
+func TestManifestsRendersRetentionOnly(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	text := out.String()
+	for _, want := range []string{"kind: CronJob", "name: felis-reaper", "--retention-only", "claimName: felis-backups"} {
+		if !strings.Contains(text, want) {
+			t.Errorf("retention-only bundle missing %q", want)
+		}
+	}
+	if strings.Contains(text, "kind: PersistentVolume\n") || strings.Contains(text, "--worlds-root") {
+		t.Error("a retention-only bundle must not reach for a worlds root")
+	}
+	if !strings.Contains(errBuf.String(), "retention-only") {
+		t.Errorf("stderr must say the CronJob is retention-only, got %q", errBuf.String())
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives", "--backup-pvc="}, &out, &errBuf)
+	if code != 2 || out.Len() != 0 || !strings.Contains(errBuf.String(), "--backup-pvc is empty") {
+		t.Errorf("archive path without an archive store: exit=%d out=%d bytes stderr=%q, want a fail-loud 2", code, out.Len(), errBuf.String())
+	}
+}
+
 // TestManifestsRendersReaper proves the happy path with the full retention trio:
 // a batch/v1 CronJob is emitted, named felis-reaper, mounting the backup PVC at the
 // supplied archive path.
@@ -150,9 +210,105 @@ func TestManifestsRendersReaper(t *testing.T) {
 	// this generator cannot verify (else a misarranged hostPath silently no-ops
 	// retention): the <path>/<pvc> arrangement-dependency and the multi-node
 	// nodeSelector hazard.
-	for _, want := range []string{"local-path-provisioner", "nodeSelector"} {
+	for _, want := range []string{"local-path", "nodeSelector"} {
 		if !strings.Contains(errBuf.String(), want) {
 			t.Errorf("reaper render must warn operators about %q on stderr, got %q", want, errBuf.String())
 		}
 	}
 }
+
+// TestManifestsReaperNodePin: --reaper-node pins the rendered CronJob's pod via
+// kubernetes.io/hostname and replaces the "no nodeSelector" hazard note with the
+// pin confirmation; using it without the worlds root is a fail-loud 2.
+func TestManifestsReaperNodePin(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{
+		"manifests",
+		"--felis-image", "registry.felis.svc:5000/felis:v1",
+		"--velocity-cidr", "10.0.0.5/32",
+		"--worlds-host-path", "/var/lib/felis/worlds",
+		"--archive-local-path", "/backups",
+		"--reaper-node", "node-a",
+	}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	for _, want := range []string{
+		"kubernetes.io/hostname: node-a",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("pinned render missing %q", want)
+		}
+	}
+	if !strings.Contains(errBuf.String(), "node-a") {
+		t.Errorf("stderr must confirm the pin, got %q", errBuf.String())
+	}
+
+	var out2, err2 bytes.Buffer
+	if code := run([]string{
+		"manifests",
+		"--felis-image", "registry.felis.svc:5000/felis:v1",
+		"--velocity-cidr", "10.0.0.5/32",
+		"--reaper-node", "node-a",
+	}, &out2, &err2); code != 2 {
+		t.Errorf("--reaper-node without --worlds-host-path: exit = %d, want 2", code)
+	}
+}
+
+// TestManifestsStorageSizes proves the PVC size flags reach the rendered claims
+// and a size the API server would reject fails before anything is applied.
+func TestManifestsStorageSizes(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "40Gi", "--uploads-storage", "8Gi"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, stderr = %s", code, errBuf.String())
+	}
+	for _, want := range []string{"storage: 40Gi", "storage: 8Gi"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("bundle lacks %q", want)
+		}
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "lots"}, &out, &errBuf)
+	if code == 0 || !strings.Contains(errBuf.String(), "registry storage") {
+		t.Errorf("--registry-storage lots: exit %d, stderr %q; want a refusal naming the flag", code, errBuf.String())
+	}
+}
+
+// TestManifestsOnlyPostgres: the installer renders the database before it has
+// an image or a proxy address for the rest of the bundle, so --only postgres
+// must render without them, and render the database and nothing else (a stray
+// Deployment in that apply would start without its identities).
+func TestManifestsOnlyPostgres(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--only", "postgres", "--control-namespace", "ctl", "--postgres-image", "example/pg:18@sha256:abc"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	var kinds []string
+	for _, doc := range strings.Split(out.String(), "\n---\n") {
+		for _, line := range strings.Split(doc, "\n") {
+			if strings.HasPrefix(line, "kind: ") {
+				kinds = append(kinds, strings.TrimPrefix(line, "kind: "))
+			}
+		}
+	}
+	if got, want := strings.Join(kinds, ","), "Namespace,ConfigMap,NetworkPolicy,Deployment,Service"; got != want {
+		t.Errorf("rendered kinds %s, want %s", got, want)
+	}
+	for _, want := range []string{"name: felis-postgres", "namespace: ctl", "image: example/pg:18@sha256:abc"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("rendered database missing %q", want)
+		}
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	if code := run([]string{"manifests", "--only", "registry"}, &out, &errBuf); code != 2 || out.Len() != 0 {
+		t.Errorf("--only registry: exit %d with %d bytes of YAML, want 2 and none", code, out.Len())
+	}
+}

cmd/felis/memlimit.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"os"
+	"runtime/debug"
+	"strconv"
+	"strings"
+)
+
+// cgroupMemoryFiles are where a container reads the memory it is allowed:
+// cgroup v2 first, then v1.
+var cgroupMemoryFiles = []string{"/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"}
+
+// limitHeapToCgroup sets the Go heap's soft limit from the container's memory
+// limit, so the collector works harder as a Job nears it and the kernel does not
+// kill the Job first. An extraction or a folder zipped for download keeps a few
+// hundred bytes per entry for as long as it runs; without the limit the heap
+// grows to twice that before a collection, and a 256 MiB Job was killed at
+// 400,000 entries whose live heap was 115 MB. GOMEMLIMIT set by hand wins.
+func limitHeapToCgroup() {
+	if os.Getenv("GOMEMLIMIT") != "" {
+		return
+	}
+	for _, f := range cgroupMemoryFiles {
+		b, err := os.ReadFile(f)
+		if err != nil {
+			continue
+		}
+		if n, ok := softMemoryLimit(string(b)); ok {
+			debug.SetMemoryLimit(n)
+		}
+		return
+	}
+}
+
+// softMemoryLimit answers three fifths of the limit a cgroup memory file holds,
+// or false for "max" (no limit) and anything unreadable. The rest is left for
+// what the kernel charges the container beyond the Go heap: the page cache of
+// the files it reads and writes, and the inodes it creates. Under a 256 MiB
+// limit, 400,000 extracted entries peaked at 184 MB resident with the heap held
+// to 150 MiB.
+func softMemoryLimit(content string) (int64, bool) {
+	n, err := strconv.ParseInt(strings.TrimSpace(content), 10, 64)
+	if err != nil || n <= 0 {
+		return 0, false
+	}
+	return n / 5 * 3, true
+}

cmd/felis/memlimit_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"math"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"testing"
+)
+
+func TestSoftMemoryLimit(t *testing.T) {
+	for _, c := range []struct {
+		in   string
+		want int64
+		ok   bool
+	}{
+		{"268435456\n", 161061273, true}, // 256 MiB, as memory.max holds it
+		{"max\n", 0, false},
+		{"0\n", 0, false},
+		{"-1", 0, false},
+		{"", 0, false},
+	} {
+		got, ok := softMemoryLimit(c.in)
+		if got != c.want || ok != c.ok {
+			t.Errorf("softMemoryLimit(%q) = %d %v, want %d %v", c.in, got, ok, c.want, c.ok)
+		}
+	}
+}
+
+// TestLimitHeapToCgroup checks the limit comes from the first cgroup file there
+// is, and that GOMEMLIMIT set by hand leaves the heap alone.
+func TestLimitHeapToCgroup(t *testing.T) {
+	prevFiles, prevLimit := cgroupMemoryFiles, debug.SetMemoryLimit(-1)
+	t.Cleanup(func() { cgroupMemoryFiles = prevFiles; debug.SetMemoryLimit(prevLimit) })
+	dir := t.TempDir()
+	v1, v1b := filepath.Join(dir, "v1"), filepath.Join(dir, "v1b")
+	if err := os.WriteFile(v1, []byte("268435456\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(v1b, []byte("536870912\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	cgroupMemoryFiles = []string{filepath.Join(dir, "missing"), v1, v1b}
+
+	t.Setenv("GOMEMLIMIT", "")
+	debug.SetMemoryLimit(math.MaxInt64)
+	limitHeapToCgroup()
+	if got := debug.SetMemoryLimit(-1); got != 161061273 {
+		t.Fatalf("limit = %d, want three fifths of 256 MiB", got)
+	}
+
+	t.Setenv("GOMEMLIMIT", "1GiB")
+	debug.SetMemoryLimit(math.MaxInt64)
+	limitHeapToCgroup()
+	if got := debug.SetMemoryLimit(-1); got != math.MaxInt64 {
+		t.Fatalf("limit = %d with GOMEMLIMIT set, want it left alone", got)
+	}
+}

cmd/felis/migrate.go

@@ -2,20 +2,32 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"time"
 
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
 // cmdMigrate implements `felis migrate up`: load config, open the database, and
 // apply every pending embedded migration under the advisory lock (spec §6).
+//
+// Migrations only roll forward, and some drop data (0017_drop_password), so a
+// database that already holds a schema and has migrations pending is bundled
+// first (internal/dbbackup, label pre-migrate). A failed snapshot stops the
+// upgrade; -no-backup is the explicit way past it, e.g. for an external
+// database (no [database] deployment, so the host's own pg_dump runs) whose
+// server is newer than that pg_dump.
 func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("migrate", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	backupDir := fs.String("backup-dir", dbbackup.DefaultDir, "where the pre-migration snapshot goes")
+	noBackup := fs.Bool("no-backup", false, "apply pending migrations without snapshotting the database first")
 	// The "up" verb precedes any flags (felis migrate up -config path). Go's
 	// flag.Parse stops at the first non-flag token and would never see a flag
 	// placed after "up", silently falling back to the default -config. Pull the
@@ -48,6 +60,18 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	if !*noBackup {
+		path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database, *backupDir, stderr)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis migrate: pre-migration backup failed, nothing applied: %v\n", err)
+			fmt.Fprintln(stderr, "  fix the backup, or re-run with -no-backup to migrate without one")
+			return 1
+		}
+		if path != "" {
+			fmt.Fprintf(stdout, "felis migrate: database snapshot %s\n", path)
+		}
+	}
+
 	applied, err := store.Up(ctx, drv, migrations)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis migrate: %v\n", err)
@@ -60,3 +84,98 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// preMigrateStateDir is the host state a pre-migrate bundle carries; tests
+// point it at a directory of their own.
+var preMigrateStateDir = dbbackup.DefaultStateDir
+
+// preMigrateBackup bundles the database when it already carries a schema and
+// some of migrations are not applied yet, and returns the bundle's path ("" when
+// there was nothing to protect: a fresh database, or nothing pending).
+func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, db config.DatabaseConfig, dir string, log io.Writer) (string, error) {
+	if err := drv.EnsureVersionTable(ctx); err != nil {
+		return "", fmt.Errorf("ensure version table: %w", err)
+	}
+	done, err := drv.AppliedVersions(ctx)
+	if err != nil {
+		return "", fmt.Errorf("read applied versions: %w", err)
+	}
+	if len(done) == 0 || !hasPending(done, migrations) {
+		return "", nil
+	}
+	tools, err := dbTools(db)
+	if err != nil {
+		return "", err
+	}
+	path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
+		DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelPreMigrate,
+		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: preMigrateStateDir,
+		Version: resolvedVersion(), ExportServers: exportMinecraftServers, Log: log, Record: true,
+	})
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// Rolling the migration back needs the database alone. Backup logged
+		// the gap, and the panel and the watchdog show it while this is the
+		// newest bundle.
+		return path, nil
+	}
+	return path, err
+}
+
+func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
+	for _, m := range migrations {
+		if _, ok := done[m.Version]; !ok {
+			return true
+		}
+	}
+	return false
+}
+
+// openStore opens the business database for a command that reads and writes its
+// tables, and refuses one whose schema this build was not written against: a newer
+// Felis migrated it (a rolled-back binary), or, unless allowPending, migrations this
+// build embeds have not run yet (a binary swapped in ahead of `felis migrate up`).
+func openStore(ctx context.Context, url string, allowPending bool) (*store.PostgresDriver, error) {
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	return checkSchema(ctx, drv, allowPending)
+}
+
+// podDBWindow and podDBInterval bound how long a pod that has just started retries its
+// first database dial while the network policy has yet to admit it
+// (store.OpenRetrying). A minute is far past the sync lag and far inside every Job's
+// deadline. Vars so a test can shrink them.
+var (
+	podDBWindow   = time.Minute
+	podDBInterval = time.Second
+)
+
+// openPodStore is openStore for felis-api and the reaper and backup Jobs, whose first
+// dial comes milliseconds after their pod starts.
+func openPodStore(ctx context.Context, url, prog string, stderr io.Writer) (*store.PostgresDriver, error) {
+	drv, err := store.OpenRetrying(ctx, url, podDBWindow, podDBInterval, func(err error) {
+		fmt.Fprintf(stderr, "felis %s: %v; retrying (a pod that has just started waits for the network policy to admit it)\n", prog, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return checkSchema(ctx, drv, false)
+}
+
+// checkSchema closes drv and fails when its schema is not the one this build was
+// written against (see openStore).
+func checkSchema(ctx context.Context, drv *store.PostgresDriver, allowPending bool) (*store.PostgresDriver, error) {
+	s, err := store.ReadSchema(ctx, drv)
+	if err == nil {
+		err = s.Err()
+		if allowPending {
+			err = s.Newer()
+		}
+	}
+	if err != nil {
+		drv.Close()
+		return nil, err
+	}
+	return drv, nil
+}

cmd/felis/migrate_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// openPodStore retries a refused first dial for podDBWindow, saying so on stderr
+// under the calling command's name each time.
+func TestOpenPodStoreRetriesARefusedDial(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := ln.Addr().String()
+	ln.Close()
+
+	window, interval := podDBWindow, podDBInterval
+	podDBWindow, podDBInterval = 200*time.Millisecond, 20*time.Millisecond
+	t.Cleanup(func() { podDBWindow, podDBInterval = window, interval })
+
+	var stderr bytes.Buffer
+	start := time.Now()
+	_, err = openPodStore(context.Background(), fmt.Sprintf("postgres://felis@%s/felis?sslmode=disable", addr), "reaper", &stderr)
+	if !errors.Is(err, syscall.ECONNREFUSED) {
+		t.Fatalf("err = %v, want a refused dial", err)
+	}
+	if elapsed := time.Since(start); elapsed < podDBWindow {
+		t.Fatalf("gave up after %s, inside the %s window", elapsed, podDBWindow)
+	}
+	lines := strings.Split(strings.TrimSuffix(stderr.String(), "\n"), "\n")
+	if len(lines) < 2 || len(lines) > 11 {
+		t.Fatalf("%d retry lines in a 200ms window at 20ms:\n%s", len(lines), stderr.String())
+	}
+	for _, l := range lines {
+		if !strings.HasPrefix(l, "felis reaper: failed to connect to `user=felis database=felis`: ") ||
+			!strings.HasSuffix(l, "; retrying (a pod that has just started waits for the network policy to admit it)") {
+			t.Fatalf("retry line %q", l)
+		}
+	}
+}

cmd/felis/mirrortools.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/build"
+	"felis.lolicon.best/internal/imagepush"
+)
+
+// defaultBuildToolsStatus is where mirror-build-tools records its last run; the
+// watchdog reads it to tell a vulnerability DB that stopped refreshing.
+const defaultBuildToolsStatus = "/var/lib/felis/build-tools/status.json"
+
+// cmdMirrorBuildTools copies the build lane's tools (build.Tools: the kaniko and
+// trivy images, Trivy's vulnerability and Java DBs) from upstream into the
+// platform registry, where build Jobs pull them. deploy/bootstrap.sh runs it at
+// install and from felis-build-tools.timer twice a day, which is what keeps the
+// DBs fresh; a root shell can run it the same way to refresh now.
+//
+// It writes as the platform principal through the node's loopback hostPort, the
+// same way the installer pushes, reading the token from the environment or from
+// /etc/felis/secrets.env.
+func cmdMirrorBuildTools(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("mirror-build-tools", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	endpoint := fs.String("endpoint", "127.0.0.1:5000", "host[:port] of the registry to write to (plain HTTP)")
+	only := fs.String("only", "", "comma-separated tool names to copy (default: all of "+toolNames()+")")
+	status := fs.String("status", defaultBuildToolsStatus, `file to record the run in ("" records nothing)`)
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	platformFlag := fs.String("platform", "", "os/arch of the images to copy (default: this machine's)")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	tools, err := selectTools(*only)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: "http", Username: user, Password: pass, Log: stdout}
+	src := &imagepush.Source{Platform: *platformFlag}
+	started := time.Now()
+	var failed []string
+	for _, t := range tools {
+		dst := strings.TrimSuffix(*endpoint, "/") + "/" + t.Mirror
+		if _, err := p.Mirror(ctx, src, t.Source, dst); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: %s: %v\n", t.Name, err)
+			failed = append(failed, t.Name+": "+err.Error())
+		}
+	}
+	if *status != "" {
+		st, err := imagepush.ReadMirrorStatus(*status)
+		if err != nil || st == nil {
+			st = &imagepush.MirrorStatus{}
+		}
+		st.LastAttempt = started
+		st.LastError = strings.Join(failed, "; ")
+		if len(failed) == 0 {
+			st.LastSuccess = started
+		}
+		if err := imagepush.WriteMirrorStatus(*status, *st); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: record %s: %v\n", *status, err)
+		}
+	}
+	if len(failed) > 0 {
+		return 1
+	}
+	return 0
+}
+
+func toolNames() string {
+	var names []string
+	for _, t := range build.Tools {
+		names = append(names, t.Name)
+	}
+	return strings.Join(names, ",")
+}
+
+func selectTools(only string) ([]build.Tool, error) {
+	if only == "" {
+		return build.Tools, nil
+	}
+	var out []build.Tool
+	for _, name := range strings.Split(only, ",") {
+		name = strings.TrimSpace(name)
+		found := false
+		for _, t := range build.Tools {
+			if t.Name == name {
+				out = append(out, t)
+				found = true
+			}
+		}
+		if !found {
+			return nil, fmt.Errorf("unknown tool %q (known: %s)", name, toolNames())
+		}
+	}
+	return out, nil
+}

cmd/felis/offsite.go

@@ -0,0 +1,946 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const offsiteUsage = `usage:
+  felis offsite sync         [-config path] [-archive-dir dir] [-db-dir dir] [-registry host:port|off]
+                             [-uploads-dir dir] [-status-file path]
+  felis offsite status       [-config path] [-status-file path]
+  felis offsite list         [-config path]
+  felis offsite fetch-db     [-config path | -endpoint url -bucket name [-region r] [-prefix p]]
+                             [-dir dir] latest|<bundle>
+  felis offsite fetch-worlds [-config path] [-archive-dir dir]
+  felis offsite fetch-images [-config path] [-registry host:port] [-at version]
+  felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
+  felis offsite check-key    [-config path]
+  felis offsite take-over    [-config path] [-status-file path] [-yes]
+  felis offsite keygen
+
+Every verb but keygen reads the bucket credentials and the encryption key from
+the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY,
+FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
+-env-file (default /etc/felis/offsite.env).
+
+check-key tells whether the key is the one the bucket's objects are sealed
+with, writing nothing; it exits 3 when they are sealed with another key, and
+sync then refuses to write or prune anything in the bucket.
+
+take-over names the host that writes the bucket, writing nothing; it exits 4
+when that is another host and this one never wrote it, and 5 when another
+host took the bucket over from this one. A host built from another host's
+backup (a rehearsal, or a rebuild) copies nothing into that host's bucket
+until -yes makes it the writer; the host it replaces then stops copying and
+says so.
+`
+
+// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
+// felis-offsite.service unit loads it as its EnvironmentFile.
+const defaultOffsiteEnvFile = "/etc/felis/offsite.env"
+
+// cmdOffsite implements `felis offsite`: the off-site copy of the world
+// archives, the database bundles, the registry's user images and the
+// submission uploads (internal/offsite). felis-offsite.timer runs `sync`
+// hourly on the host; the fetch verbs are the way back after the node is lost
+// (docs/troubleshooting.md §16).
+func cmdOffsite(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("offsite "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, offsiteUsage) }
+	switch verb {
+	case "sync":
+		return offsiteSync(fs, rest, stdout, stderr)
+	case "status":
+		return offsiteStatus(fs, rest, stdout, stderr)
+	case "list":
+		return offsiteList(fs, rest, stdout, stderr)
+	case "fetch-db":
+		return offsiteFetchDB(fs, rest, stdout, stderr)
+	case "fetch-worlds":
+		return offsiteFetchWorlds(fs, rest, stdout, stderr)
+	case "fetch-images":
+		return offsiteFetchImages(fs, rest, stdout, stderr)
+	case "fetch-uploads":
+		return offsiteFetchUploads(fs, rest, stdout, stderr)
+	case "check-key":
+		return offsiteCheckKey(fs, rest, stdout, stderr)
+	case "take-over":
+		return offsiteTakeOver(fs, rest, stdout, stderr)
+	case "keygen":
+		k, err := offsite.NewKey()
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite keygen: %v\n", err)
+			return 1
+		}
+		fmt.Fprintln(stdout, k)
+		return 0
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, offsiteUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis offsite: unknown verb %q\n%s", verb, offsiteUsage)
+	return 2
+}
+
+// offsiteEnv is the resolved [offsite] binding: the bucket and the key.
+type offsiteEnv struct {
+	cfg    config.OffsiteConfig
+	bucket *offsite.S3
+	key    []byte
+}
+
+// loadEnvFile sets each KEY=VALUE of path that is not already in the
+// environment, so a root shell runs a command the same way its unit does.
+// A missing file is not an error.
+func loadEnvFile(path string) error {
+	if path == "" {
+		return nil
+	}
+	f, err := os.Open(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		k = strings.TrimSpace(strings.TrimPrefix(k, "export "))
+		v = strings.TrimSpace(v)
+		if len(v) >= 2 && (v[0] == '"' || v[0] == '\'') && v[len(v)-1] == v[0] {
+			v = v[1 : len(v)-1]
+		}
+		if os.Getenv(k) == "" {
+			os.Setenv(k, v)
+		}
+	}
+	return sc.Err()
+}
+
+// resolveOffsite builds the bucket client and parses the key for c.
+func resolveOffsite(c config.OffsiteConfig) (*offsiteEnv, error) {
+	if !c.Enabled() {
+		return nil, errors.New("no [offsite] bucket is configured (docs/troubleshooting.md §16, \"Keep a copy somewhere else\")")
+	}
+	need := func(ref, what string) (string, error) {
+		v := os.Getenv(ref)
+		if v == "" {
+			return "", fmt.Errorf("%s: environment variable %s is empty (set it, or put it in %s)", what, ref, defaultOffsiteEnvFile)
+		}
+		return v, nil
+	}
+	ak, err := need(c.AccessKeyRef, "access key")
+	if err != nil {
+		return nil, err
+	}
+	sk, err := need(c.SecretKeyRef, "secret key")
+	if err != nil {
+		return nil, err
+	}
+	rawKey, err := need(c.KeyRef, "encryption key")
+	if err != nil {
+		return nil, err
+	}
+	key, err := offsite.ParseKey(rawKey)
+	if err != nil {
+		return nil, err
+	}
+	b, err := offsite.NewS3(offsite.S3Config{
+		Endpoint: c.Endpoint, Region: c.Region, Bucket: c.Bucket, Prefix: c.Prefix,
+		AccessKey: ak, SecretKey: sk,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &offsiteEnv{cfg: c, bucket: b, key: key}, nil
+}
+
+// loadOffsite loads felis.toml and the env file and resolves [offsite].
+func loadOffsite(cfgPath, envFile string) (*config.Config, *offsiteEnv, error) {
+	if err := loadEnvFile(envFile); err != nil {
+		return nil, nil, fmt.Errorf("read %s: %w", envFile, err)
+	}
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	env, err := resolveOffsite(cfg.Offsite)
+	if err != nil {
+		return cfg, nil, err
+	}
+	return cfg, env, nil
+}
+
+func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
+	dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory bundled into the database bundle taken after archives are copied ("" for none)`)
+	registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC through the cluster)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, `the submission uploads PVC, in the control-plane namespace ("" copies no uploads)`)
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "where the result of this run is recorded for the watchdog and `status`")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	st, lease := startRun(env.cfg, env.key, *statusFile, time.Now())
+	res, err := runOffsiteSync(cfg, env, offsiteSources{
+		archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir, stateDir: *stateDir,
+		registry:   offsiteRegistryEndpoint(*registry, cfg.Registry),
+		uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
+	}, &lease, stderr)
+	recordRun(&st, res, err, lease)
+	if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
+	}
+	return reportRun(res, err, stdout, stderr)
+}
+
+// reportRun prints one pass's outcome and its exit code. A run stopped before
+// it copied anything (a bucket that did not answer, another key's objects,
+// another host writing the bucket) prints no counts: its zeros would read as
+// an empty bucket.
+func reportRun(res offsite.Result, err error, stdout, stderr io.Writer) int {
+	if err == nil || len(res.Errors) > 0 {
+		fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n",
+			res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
+			res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
+			res.UploadsUploaded, res.UploadObjectsPruned,
+			res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes),
+			res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes))
+	}
+	for _, m := range res.WorldsMissing {
+		fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
+	}
+	for _, m := range res.ImagesIncomplete {
+		fmt.Fprintf(stderr, "felis offsite sync: registry image not whole: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// startRun begins a pass: its status record, in this release's format and
+// carrying the last success over, and this host's lease, read from the record
+// the last pass left.
+func startRun(cfg config.OffsiteConfig, key []byte, statusFile string, now time.Time) (offsite.Status, offsite.Lease) {
+	st := offsite.Status{
+		LastAttempt: now.UTC(), Endpoint: cfg.Endpoint, Bucket: cfg.Bucket,
+		Prefix: cfg.Prefix, KeyID: offsite.KeyID(key), Format: offsite.StatusFormat,
+	}
+	if prev, _ := offsite.ReadStatus(statusFile); prev != nil {
+		st.LastSuccess = prev.LastSuccess
+	}
+	return st, offsite.HostLease(statusFile)
+}
+
+// recordRun puts one pass's outcome into its status record. A host that
+// inherited the bucket from an older release keeps that until it has an id.
+func recordRun(st *offsite.Status, res offsite.Result, err error, lease offsite.Lease) {
+	st.Result = res
+	if err != nil {
+		st.LastError = err.Error()
+		st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch)
+		var we *offsite.WriterError
+		if errors.As(err, &we) {
+			st.Standby = errors.Is(err, offsite.ErrStandby)
+			st.Displaced = errors.Is(err, offsite.ErrDisplaced)
+			st.Writer = we.Writer
+		}
+	} else {
+		st.LastSuccess = st.LastAttempt
+	}
+	if lease.Inherited {
+		id, _ := lease.ID()
+		st.Inherited = id == ""
+	}
+}
+
+// offsiteSources is where one sync pass reads from: the world archive volume
+// (archiveDir, or the backupPVC's directory), the bundle directory (with the
+// host state the pass bundles, stateDir), the registry's loopback endpoint and
+// the uploads volume (uploadsDir, or the uploadsPVC's directory). An empty
+// source is skipped.
+type offsiteSources struct {
+	archiveDir, backupPVC  string
+	dbDir, stateDir        string
+	registry               string
+	uploadsDir, uploadsPVC string
+}
+
+// offsiteRunLimit backstops one sync pass. Each upload has its own deadline,
+// scaled to its size (internal/offsite), so a pass over a big archive may run
+// for hours; the timer starts no second pass while one runs, and the unit's
+// TimeoutStartSec sits above this.
+const offsiteRunLimit = 23 * time.Hour
+
+func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lease *offsite.Lease, log io.Writer) (offsite.Result, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit)
+	defer cancel()
+	checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
+	err := env.bucket.Check(checkCtx)
+	checkCancel()
+	if err != nil {
+		return offsite.Result{}, err
+	}
+	archiveDir, uploadsDir := src.archiveDir, src.uploadsDir
+	if archiveDir == "" && src.backupPVC != "" {
+		dir, err := resolveVolumeDir(ctx, cfg.K8s.Namespace, src.backupPVC, archiveVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		archiveDir = dir
+	}
+	// An s3:// uploads store is off the host already; only a local one, on
+	// the uploads PVC, needs the copy.
+	if uploadsDir == "" && src.uploadsPVC != "" && isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+		dir, err := resolveVolumeDir(ctx, platform.DefaultControlNamespace, src.uploadsPVC, uploadsVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		uploadsDir = dir
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		return offsite.Result{}, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	s := offsiteSyncer(cfg, env, src, archiveDir, uploadsDir, lease, log)
+	s.Catalog = offsite.PGCatalog{DB: drv.DB()}
+	if src.registry != "" {
+		s.Images = newRegistryImages(src.registry)
+		s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL)
+	}
+	return s.Run(ctx)
+}
+
+// offsiteSyncer is the pass runOffsiteSync runs over the resolved archive and
+// uploads directories, before its catalog and registry are attached. It
+// snapshots the database into the bundle directory after copying archives, and
+// sweeps world objects no backup records once they outlive every retention in
+// [archive]; a retention that does not parse sweeps none.
+func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, archiveDir, uploadsDir string, lease *offsite.Lease, log io.Writer) *offsite.Syncer {
+	s := &offsite.Syncer{
+		Bucket: env.bucket, Key: env.key,
+		ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log,
+	}
+	if src.dbDir != "" {
+		s.Snapshot = offsiteSnapshot(cfg.Database, src.dbDir, src.stateDir, log)
+	}
+	if rc, err := reaperConfig(cfg); err != nil {
+		fmt.Fprintf(log, "felis offsite: world objects no backup records are kept: %v\n", err)
+	} else {
+		s.OrphanAfter = max(rc.Retention, rc.ManualRetention, rc.ScheduledRetention)
+	}
+	return s
+}
+
+// offsiteSnapshot takes the bundle a pass sends after copying world archives
+// (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite,
+// with the newest one kept in dir. It is not recorded for the panel, whose
+// backup card watches felis-db-backup.timer: snapshots come only when archives
+// are copied, and would hide a daily timer that stopped. It requires the
+// MinecraftServer objects: it becomes the newest bundle in the bucket, which a
+// lost host restores from, and a pass that cannot take a whole one fails and
+// tries again next hour.
+func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error {
+	return func(ctx context.Context) error {
+		tools, err := dbTools(db)
+		if err != nil {
+			return err
+		}
+		ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+		defer cancel()
+		path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
+			DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite,
+			Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(),
+			ExportServers: exportMinecraftServers, RequireServers: true, Log: log,
+		})
+		if err == nil {
+			fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path))
+		}
+		return err
+	}
+}
+
+// volumeKind names a PVC the off-site copy reads or restores, for messages,
+// with the flag that bypasses finding it through the cluster.
+type volumeKind struct{ what, dirFlag, empty string }
+
+var (
+	archiveVolume = volumeKind{"archive volume", "-archive-dir", "no world has been archived"}
+	uploadsVolume = volumeKind{"uploads volume", "-uploads-dir", "no modpack has been uploaded"}
+)
+
+// resolveVolumeDir finds the host directory behind a PVC: a local-path volume
+// is a directory on this node. A PVC still waiting for its first consumer
+// holds nothing yet: without bind that is "" (nothing to copy), with bind it
+// is bound first, for a fetch to write into.
+func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind, bind bool, log io.Writer) (string, error) {
+	if ns == "" {
+		ns = platform.DefaultMinecraftNamespace
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		return "", fmt.Errorf("reach the cluster to find the %s (or pass %s): %w", kind.what, kind.dirFlag, err)
+	}
+	var pvc corev1.PersistentVolumeClaim
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+		return "", fmt.Errorf("%s %s/%s: %w", kind.what, ns, pvcName, err)
+	}
+	if pvc.Spec.VolumeName == "" {
+		if !bind {
+			fmt.Fprintf(log, "felis offsite: %s %s/%s is not bound yet; %s\n", kind.what, ns, pvcName, kind.empty)
+			return "", nil
+		}
+		if err := bindVolume(ctx, cl, ns, pvcName, kind, log); err != nil {
+			return "", err
+		}
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+			return "", err
+		}
+	}
+	var pv corev1.PersistentVolume
+	if err := cl.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil {
+		return "", fmt.Errorf("%s %s: %w", kind.what, pvc.Spec.VolumeName, err)
+	}
+	var dir string
+	switch {
+	case pv.Spec.Local != nil:
+		dir = pv.Spec.Local.Path
+	case pv.Spec.HostPath != nil:
+		dir = pv.Spec.HostPath.Path
+	default:
+		return "", fmt.Errorf("%s %s is not a directory on a node (local or hostPath); pass %s with where it is mounted on this host", kind.what, pv.Name, kind.dirFlag)
+	}
+	if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
+		return "", fmt.Errorf("%s %s is %s on its node, which is not a directory here; run this on the node that holds it, or pass %s", kind.what, pv.Name, dir, kind.dirFlag)
+	}
+	return dir, nil
+}
+
+// bindVolume runs a pod that mounts the PVC and exits, which is what makes a
+// WaitForFirstConsumer volume (k3s local-path) get provisioned. The pod uses
+// the control plane's own image, which every install already has.
+func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind volumeKind, log io.Writer) error {
+	var api appsv1.Deployment
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: platform.DefaultControlNamespace, Name: "felis-api"}, &api); err != nil {
+		return fmt.Errorf("find the felis image to bind the %s with: %w", kind.what, err)
+	}
+	if len(api.Spec.Template.Spec.Containers) == 0 {
+		return errors.New("felis-api has no container to take the image from")
+	}
+	image := api.Spec.Template.Spec.Containers[0].Image
+	pod := platform.VolumeBinderPod(ns, pvcName, image)
+	if err := cl.Create(ctx, pod); err != nil {
+		return fmt.Errorf("start a pod to bind the %s: %w", kind.what, err)
+	}
+	fmt.Fprintf(log, "felis offsite: binding the %s %s/%s (pod %s)\n", kind.what, ns, pvcName, pod.Name)
+	defer func() {
+		_ = cl.Delete(context.Background(), pod, client.PropagationPolicy(metav1.DeletePropagationBackground))
+	}()
+	deadline := time.Now().Add(3 * time.Minute)
+	for time.Now().Before(deadline) {
+		var pvc corev1.PersistentVolumeClaim
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err == nil && pvc.Spec.VolumeName != "" && pvc.Status.Phase == corev1.ClaimBound {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+	return fmt.Errorf("the %s %s/%s did not bind within 3 minutes; see kubectl -n %s describe pod %s", kind.what, ns, pvcName, ns, pod.Name)
+}
+
+func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if !cfg.Offsite.Enabled() {
+		fmt.Fprintln(stdout, "off-site copy: not configured. World archives, database bundles, user images and uploaded modpacks exist on this machine only.")
+		fmt.Fprintln(stdout, "See docs/troubleshooting.md §16, \"Keep a copy somewhere else\".")
+		return 1
+	}
+	o := cfg.Offsite
+	fmt.Fprintf(stdout, "bucket:       %s at %s", o.Bucket, o.Endpoint)
+	if o.Prefix != "" {
+		fmt.Fprintf(stdout, ", prefix %s", o.Prefix)
+	}
+	fmt.Fprintln(stdout)
+	st, err := offsite.ReadStatus(*statusFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if st == nil {
+		fmt.Fprintln(stdout, "last sync:    never (sudo systemctl start felis-offsite.service)")
+		return 1
+	}
+	now := time.Now()
+	fmt.Fprintf(stdout, "key id:       %s\n", st.KeyID)
+	fmt.Fprintf(stdout, "last attempt: %s (%s ago)\n", st.LastAttempt.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastAttempt)))
+	if st.LastSuccess.IsZero() {
+		fmt.Fprintln(stdout, "last success: never")
+	} else {
+		fmt.Fprintf(stdout, "last success: %s (%s ago)\n", st.LastSuccess.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastSuccess)))
+	}
+	if st.LastError != "" {
+		fmt.Fprintf(stdout, "last error:   %s\n", st.LastError)
+	}
+	if st.KeyMismatch {
+		fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile)
+		return 1
+	}
+	if st.Displaced && st.Writer != nil {
+		fmt.Fprintf(stdout, "\nThe last run was refused: %s took the bucket over (it last wrote it at %s), and this host copies nothing there any more. If that host is a rehearsal machine, take the bucket back: sudo felis offsite take-over -yes\n",
+			st.Writer, st.Writer.At.Local().Format(time.DateTime))
+		return 1
+	}
+	if st.Standby {
+		switch w := st.StandsBy(now); {
+		case w != nil:
+			fmt.Fprintf(stdout, "\nThis host stands by: %s writes the bucket (last at %s). This host was built from its backup, copies nothing into the bucket and, while that host keeps writing, mails no watchdog alert.", w, w.At.Local().Format(time.DateTime))
+		case st.Writer != nil:
+			fmt.Fprintf(stdout, "\nThis host copies nothing into the bucket: %s wrote it, last at %s, and this host was built from its backup.", st.Writer, st.Writer.At.Local().Format(time.DateTime))
+		default:
+			fmt.Fprint(stdout, "\nThis host copies nothing into the bucket: it holds copies this host did not write, and names no host writing it.")
+		}
+		fmt.Fprintln(stdout, " Once this host replaces that one for good: sudo felis offsite take-over -yes")
+		return 1
+	}
+	r := st.Result
+	fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
+		r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
+	if r.ImageIndex != "" {
+		fmt.Fprintf(stdout, "images:       %d in %d repositories (%s), registry index %s\n",
+			r.Images, r.ImageRepos, offsite.HumanBytes(r.RemoteImageBytes), r.ImageIndex)
+	} else {
+		fmt.Fprintln(stdout, "images:       not copied (no in-cluster registry, or no sync has reached it yet)")
+	}
+	if r.UploadIndex != "" {
+		fmt.Fprintf(stdout, "uploads:      %d submission contexts (%s), uploads index %s\n",
+			r.Uploads, offsite.HumanBytes(r.RemoteUploadBytes), r.UploadIndex)
+	} else {
+		fmt.Fprintln(stdout, "uploads:      not copied (an s3:// uploads store, or no sync has reached the volume yet)")
+	}
+	fmt.Fprintf(stdout, "waiting:      %d world archives not yet copied\n", r.WorldsPending)
+	for _, m := range r.WorldsMissing {
+		fmt.Fprintf(stdout, "missing:      %s is recorded but not on the volume\n", m)
+	}
+	for _, m := range r.ImagesIncomplete {
+		fmt.Fprintf(stdout, "not whole:    %s\n", m)
+	}
+	if st.LastSuccess.IsZero() || now.Sub(st.LastSuccess) > offsite.StaleAfter {
+		fmt.Fprintf(stdout, "\nThe last successful sync is older than %s: journalctl -u felis-offsite -n 50\n", dbbackup.Age(offsite.StaleAfter))
+		return 1
+	}
+	return 0
+}
+
+func orNone(s string) string {
+	if s == "" {
+		return "none"
+	}
+	return s
+}
+
+func offsiteList(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	return printOffsiteList(env, stdout, stderr)
+}
+
+func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	bundles, err := offsite.ListDB(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	worlds, err := env.bucket.List(ctx, "worlds/")
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	printDBBundles(ctx, env.bucket, env.key, bundles, stdout)
+	var total int64
+	for _, w := range worlds {
+		total += w.Size
+	}
+	fmt.Fprintf(stdout, "world archives: %d (%s)\n", len(worlds), offsite.HumanBytes(total))
+	versions, err := offsite.ImageIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "registry index versions (%d, newest first; restore one with fetch-images -at):\n", len(versions))
+	for i := len(versions) - 1; i >= 0; i-- {
+		x, err := offsite.LoadImageIndex(ctx, env.bucket, env.key, versions[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", versions[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d images in %d repositories\n", versions[i], x.Images(), len(x.Repositories))
+	}
+	uploads, err := offsite.UploadIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "uploads index versions (%d, newest first; restore one with fetch-uploads -at):\n", len(uploads))
+	for i := len(uploads) - 1; i >= 0; i-- {
+		x, err := offsite.LoadUploadIndex(ctx, env.bucket, env.key, uploads[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", uploads[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d submission contexts (%s)\n", uploads[i], len(x.Contexts), offsite.HumanBytes(x.Bytes()))
+	}
+	return 0
+}
+
+// printDBBundles lists the database bundles with what each one's database
+// held, read off the front of each, so a restore can pick one by its contents.
+func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles []offsite.Object, stdout io.Writer) {
+	fmt.Fprintf(stdout, "database bundles (%d, newest first; restore one with fetch-db):\n", len(bundles))
+	for _, o := range bundles {
+		m, err := offsite.PeekDB(ctx, b, key, o.Key)
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  %s  unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err)
+			continue
+		}
+		gap := ""
+		if m.ServersError != "" {
+			gap = ", no MinecraftServer objects"
+		}
+		fmt.Fprintf(stdout, "  %s  %s  %s%s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String(), gap)
+	}
+}
+
+func offsiteCheckKey(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	if err := env.bucket.Check(ctx); err != nil {
+		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
+		return 1
+	}
+	return checkKey(ctx, env.bucket, env.key, stdout, stderr)
+}
+
+// checkKey is check-key once the bucket is open: 0 when the key fits, 3 when
+// the bucket's objects are sealed with another one, 1 when it cannot tell.
+func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr io.Writer) int {
+	fit, err := offsite.CheckKey(ctx, b, key)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
+		if errors.Is(err, offsite.ErrKeyMismatch) {
+			return 3
+		}
+		return 1
+	}
+	id := offsite.KeyID(key)
+	switch fit {
+	case offsite.KeyRecorded:
+		fmt.Fprintf(stdout, "felis offsite check-key: the bucket records key id %s, this key's\n", id)
+	case offsite.KeyOpens:
+		fmt.Fprintf(stdout, "felis offsite check-key: the bucket's newest objects open with this key (key id %s); the next sync records it\n", id)
+	case offsite.KeyUnused:
+		fmt.Fprintf(stdout, "felis offsite check-key: the bucket holds no sealed object yet; the first sync records key id %s\n", id)
+	}
+	return 0
+}
+
+func offsiteTakeOver(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes; this host's id is kept next to it")
+	yes := fs.Bool("yes", false, "make this host the one that writes the bucket")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	if err := env.bucket.Check(ctx); err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	return takeOver(ctx, env.bucket, env.key, offsite.HostLease(*statusFile), *statusFile, *yes, time.Now(), stdout, stderr)
+}
+
+// takeOver is take-over once the bucket is open: without yes it says which
+// host writes the bucket, 0 for this one (or none yet), 4 for another and 5
+// for one that took the bucket over from this host; with
+// yes it records this host as the writer. A key the bucket's objects refuse
+// is 3, as in check-key: taking over a bucket this host cannot copy into
+// would only stop the host that can.
+func takeOver(ctx context.Context, b offsite.Bucket, key []byte, lease offsite.Lease, statusFile string, yes bool, now time.Time, stdout, stderr io.Writer) int {
+	fit, err := offsite.CheckKey(ctx, b, key)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		if errors.Is(err, offsite.ErrKeyMismatch) {
+			return 3
+		}
+		return 1
+	}
+	role, w, err := lease.Plan(ctx, b, fit == offsite.KeyUnused)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	switch role {
+	case offsite.RoleWrites:
+		id, _ := lease.ID()
+		fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket; nothing to take over\n", id)
+		return 0
+	case offsite.RoleClaims:
+		fmt.Fprintln(stdout, "felis offsite take-over: the bucket names no host writing it; this host's next sync records itself")
+		return 0
+	}
+	who := "another host"
+	if w != nil {
+		who = w.String()
+		fmt.Fprintf(stdout, "felis offsite take-over: %s writes the bucket, last at %s (%s ago)\n", w, w.At.Local().Format(time.DateTime), dbbackup.Age(now.Sub(w.At)))
+	} else {
+		fmt.Fprintln(stdout, "felis offsite take-over: the bucket holds copies this host did not write, and names no host writing it")
+	}
+	if !yes {
+		if role == offsite.RoleDisplaced {
+			fmt.Fprintf(stdout, "It took the bucket over from this host: this host copies nothing there any more, and its watchdog mails the owners about it. If that host is a rehearsal machine, take the bucket back:\n  sudo felis offsite take-over -yes\n")
+			return 5
+		}
+		fmt.Fprintf(stdout, "This host was built from its backup and copies nothing into the bucket.\n")
+		fmt.Fprintf(stdout, "Taking it over makes this host the one that copies into the bucket and prunes it; %s stops at its next copy and mails its owners. Do it once that host is gone for good, or is a rehearsal machine you are done with:\n  sudo felis offsite take-over -yes\n", who)
+		return 4
+	}
+	if _, err := lease.TakeOver(ctx, b, now); err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	// The refusal the last sync recorded is over: the watchdog mails again
+	// from now on, and status shows the next run's outcome.
+	if st, err := offsite.ReadStatus(statusFile); err == nil && st != nil && (st.Standby || st.Displaced) {
+		st.Standby, st.Displaced, st.Writer, st.LastError, st.Inherited = false, false, nil, "", false
+		if err := offsite.WriteStatus(statusFile, *st); err != nil {
+			fmt.Fprintf(stderr, "felis offsite take-over: record status: %v\n", err)
+		}
+	}
+	id, _ := lease.ID()
+	fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket now; %s stops at its next copy.\nStart the first copy: sudo systemctl start felis-offsite.service\n", id, who)
+	return 0
+}
+
+// keyHint explains an object the key cannot open when the bucket records
+// another key's id, "" otherwise.
+func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string {
+	if !errors.Is(err, offsite.ErrAuth) {
+		return ""
+	}
+	id, ierr := offsite.BucketKeyID(ctx, b)
+	if ierr != nil || id == "" || id == offsite.KeyID(key) {
+		return ""
+	}
+	return fmt.Sprintf("\n  the bucket records key id %s, and this key is %s: set FELIS_OFFSITE_KEY to the key the bucket was written with", id, offsite.KeyID(key))
+}
+
+func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	endpoint := fs.String("endpoint", "", "bucket endpoint, when there is no felis.toml")
+	bucket := fs.String("bucket", "", "bucket name, when there is no felis.toml")
+	region := fs.String("region", "", "bucket region, when there is no felis.toml")
+	prefix := fs.String("prefix", "", "key prefix, when there is no felis.toml")
+	dir := fs.String("dir", dbbackup.DefaultDir, "directory to write the bundle to")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	if err := loadEnvFile(*envFile); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: read %s: %v\n", *envFile, err)
+		return 1
+	}
+	var oc config.OffsiteConfig
+	if *bucket != "" {
+		oc = config.OffsiteConfig{
+			Endpoint: *endpoint, Bucket: *bucket, Region: *region, Prefix: *prefix,
+			AccessKeyRef: config.DefaultOffsiteAccessKeyEnv, SecretKeyRef: config.DefaultOffsiteSecretKeyEnv,
+			KeyRef: config.DefaultOffsiteKeyEnv,
+		}
+	} else {
+		cfg, err := config.Load(*cfgPath)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v (on a host with no install yet, pass -endpoint and -bucket)\n", err)
+			return 1
+		}
+		oc = cfg.Offsite
+	}
+	env, err := resolveOffsite(oc)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	return fetchDB(ctx, env.bucket, env.key, arg, *dir, time.Now(), stdout, stderr)
+}
+
+// fetchDB is fetch-db once the bucket is open: arg is a bundle name or latest.
+func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string, now time.Time, stdout, stderr io.Writer) int {
+	name := arg
+	if name == "latest" {
+		var err error
+		if name, _, err = offsite.ChooseDB(ctx, b, key); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
+			return 1
+		}
+	}
+	if _, _, ok := dbbackup.ParseBundleName(name); !ok {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %q is not a bundle name (felis-db-<stamp>-<label>.tar); see `felis offsite list`\n", name)
+		return 2
+	}
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	dst := filepath.Join(dir, name)
+	if err := offsite.FetchObject(ctx, b, key, offsite.DBKey(name), dst, 0o600); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
+		return 1
+	}
+	m, err := dbbackup.Verify(dst)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: fetched %s but it does not verify: %v\n", dst, err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-db: wrote %s (verified)\n", dst)
+	fmt.Fprintf(stdout, "  taken   %s (%s, %s ago)\n  felis   %s, schema %d\n  holds   %s\n",
+		m.CreatedAt.Format(time.RFC3339), m.Label, dbbackup.Age(now.Sub(m.CreatedAt)),
+		orUnknown(m.FelisVersion), m.SchemaVersion, m.Counts.String())
+	if m.Counts.Fresh() {
+		fmt.Fprintln(stdout, "  This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.")
+	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  This bundle lacks the MinecraftServer objects (%s): `felis db restore` brings back the database, and the servers come from k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap.\n", m.ServersError)
+	}
+	return 0
+}
+
+func offsiteFetchWorlds(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC, binding it if needed)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", "the world archive PVC, in the [k8s] namespace")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	dir := *archiveDir
+	if dir == "" {
+		if dir, err = resolveVolumeDir(ctx, cfg.K8s.Namespace, *backupPVC, archiveVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+			return 1
+		}
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: open database: %v\n", err)
+		return 1
+	}
+	defer drv.Close()
+	res, err := offsite.FetchWorlds(ctx, env.bucket, offsite.PGCatalog{DB: drv.DB()}, env.key, dir, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-worlds: %d recorded archives, %d fetched into %s, %d with no copy in the bucket\n",
+		res.Present, len(res.Fetched), dir, len(res.Missing))
+	for _, m := range res.Missing {
+		fmt.Fprintf(stdout, "  no off-site copy: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_images.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/registrygate"
+	"felis.lolicon.best/internal/registryprune"
+)
+
+// registryImages is the platform registry as the off-site copy sees it: read
+// anonymously through the gate (the catalog, its manifest index, manifests and
+// blobs) and, for a restore, written as the platform principal. Both go through
+// the node's loopback hostPort, the way the installer pushes.
+type registryImages struct {
+	host   string
+	index  *registryprune.Client
+	source *imagepush.Source
+	pusher *imagepush.Pusher
+}
+
+func newRegistryImages(endpoint string) *registryImages {
+	return &registryImages{
+		host:   endpoint,
+		index:  &registryprune.Client{Endpoint: "http://" + endpoint},
+		source: &imagepush.Source{Scheme: "http"},
+	}
+}
+
+func (r *registryImages) Repositories(ctx context.Context) ([]string, error) {
+	return r.index.Repositories(ctx)
+}
+
+func (r *registryImages) Revisions(ctx context.Context, repo string) ([]string, map[string]string, error) {
+	idx, err := r.index.Index(ctx, repo)
+	if err != nil {
+		return nil, nil, err
+	}
+	digests := make([]string, 0, len(idx.Revisions))
+	for _, rev := range idx.Revisions {
+		digests = append(digests, rev.Digest)
+	}
+	return digests, idx.Tags, nil
+}
+
+func (r *registryImages) Manifest(ctx context.Context, repo, digest string) ([]byte, string, error) {
+	body, mt, err := r.source.Manifest(ctx, r.host, repo, digest)
+	return body, mt, registryGone(err)
+}
+
+func (r *registryImages) Blob(ctx context.Context, repo, digest string) (io.ReadCloser, error) {
+	rc, err := r.source.Blob(ctx, r.host, repo, digest)
+	return rc, registryGone(err)
+}
+
+func (r *registryImages) PutBlob(ctx context.Context, repo, digest string, size int64, open func() (io.ReadCloser, error)) error {
+	return r.pusher.UploadBlob(ctx, r.host, repo, digest, size, open)
+}
+
+func (r *registryImages) PutManifest(ctx context.Context, repo, reference, mediaType string, body []byte) error {
+	_, err := r.pusher.PutManifest(ctx, r.host, repo, reference, mediaType, body)
+	return err
+}
+
+// imagePins lists, per repository of the registry refs spell as host, the
+// digests the MinecraftServers' specs and the image whitelist pin: what a
+// restored database and its servers will ask the registry for.
+func imagePins(db *sql.DB, host string) func(ctx context.Context) (map[string][]string, error) {
+	return func(ctx context.Context) (map[string][]string, error) {
+		cl, err := buildSystemServerClient()
+		if err != nil {
+			return nil, fmt.Errorf("reach the cluster: %w", err)
+		}
+		var servers v1alpha1.MinecraftServerList
+		if err := cl.List(ctx, &servers); err != nil {
+			return nil, fmt.Errorf("list MinecraftServers: %w", err)
+		}
+		refs := make([]string, 0, len(servers.Items))
+		for _, s := range servers.Items {
+			refs = append(refs, s.Spec.Image)
+		}
+		rows, err := db.QueryContext(ctx, `SELECT image_ref FROM image_whitelist`)
+		if err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var ref string
+			if err := rows.Scan(&ref); err != nil {
+				return nil, fmt.Errorf("read the image whitelist: %w", err)
+			}
+			refs = append(refs, ref)
+		}
+		if err := rows.Err(); err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		pins := map[string][]string{}
+		for _, ref := range refs {
+			repo, _, digest, ok := registryprune.ParseRef(ref, host)
+			if ok && digest != "" && !slices.Contains(pins[repo], digest) {
+				pins[repo] = append(pins[repo], digest)
+			}
+		}
+		return pins, nil
+	}
+}
+
+// registryGone marks a 404 as a manifest or blob the registry no longer holds.
+func registryGone(err error) error {
+	var se *imagepush.StatusError
+	if errors.As(err, &se) && se.Code == http.StatusNotFound {
+		return fmt.Errorf("%w: %v", offsite.ErrImageGone, err)
+	}
+	return err
+}
+
+// offsiteRegistryEndpoint is where the host reaches the registry whose images
+// the off-site copy covers: flag when given ("off" for none), otherwise the
+// loopback hostPort of the in-cluster registry [registry] url names. A
+// registry outside the cluster is not this install's to copy.
+func offsiteRegistryEndpoint(flag string, reg config.RegistryConfig) string {
+	switch flag {
+	case "off":
+		return ""
+	case "":
+	default:
+		return flag
+	}
+	host, _, _ := strings.Cut(reg.URL, "/")
+	name, _, _ := strings.Cut(host, ":")
+	if strings.HasSuffix(name, ".svc") || strings.HasSuffix(name, ".svc.cluster.local") {
+		return loopbackEndpoint(host)
+	}
+	return ""
+}
+
+// registryWriteCredential is the credential a host-side push uses:
+// FELIS_REGISTRY_USERNAME/PASSWORD when set, otherwise the platform principal
+// with REGISTRY_PLATFORM_TOKEN from the installer's secrets file.
+func registryWriteCredential(secrets string) (string, string, error) {
+	if err := loadEnvFile(secrets); err != nil {
+		return "", "", fmt.Errorf("read %s: %w", secrets, err)
+	}
+	user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if pass == "" {
+		user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
+	}
+	if pass == "" {
+		return "", "", errors.New("no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
+	}
+	return user, pass, nil
+}
+
+func offsiteFetchImages(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	registry := fs.String("registry", "", "host[:port] of the registry to push into (default: the loopback hostPort of the in-cluster registry)")
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	at := fs.String("at", "", "registry index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	endpoint := offsiteRegistryEndpoint(*registry, cfg.Registry)
+	if endpoint == "" {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: [registry] url %q is not the in-cluster registry; pass -registry host:port\n", cfg.Registry.URL)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 2
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseImageIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-images: restoring registry index %s (%d repositories, %d images) into %s\n",
+		stamp, len(idx.Repositories), idx.Images(), endpoint)
+	target := newRegistryImages(endpoint)
+	target.pusher = &imagepush.Pusher{Scheme: "http", Username: user, Password: pass}
+	res, err := offsite.FetchImages(ctx, env.bucket, env.key, idx, target, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-images: %d of %d repositories restored, %d images, %d tags, %d blobs pushed (%s)\n",
+		res.Repositories, len(idx.Repositories), res.Manifests, res.Tags, res.BlobsPushed, offsite.HumanBytes(res.BytesPushed))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v (a second run pushes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_test.go

@@ -0,0 +1,775 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+)
+
+func TestLoadOffsiteEnvFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "offsite.env")
+	body := `# written by bootstrap
+FELIS_OFFSITE_ACCESS_KEY=AKIA123
+export FELIS_OFFSITE_SECRET_KEY="se=cret"
+FELIS_OFFSITE_KEY='k'
+
+not a line
+`
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "from-the-shell")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "")
+	t.Setenv("FELIS_OFFSITE_KEY", "")
+	if err := loadEnvFile(path); err != nil {
+		t.Fatal(err)
+	}
+	for k, want := range map[string]string{
+		"FELIS_OFFSITE_ACCESS_KEY": "from-the-shell", // the environment wins
+		"FELIS_OFFSITE_SECRET_KEY": "se=cret",
+		"FELIS_OFFSITE_KEY":        "k",
+	} {
+		if got := os.Getenv(k); got != want {
+			t.Errorf("%s = %q, want %q", k, got, want)
+		}
+	}
+	if err := loadEnvFile(filepath.Join(t.TempDir(), "absent")); err != nil {
+		t.Errorf("a missing env file is not an error: %v", err)
+	}
+}
+
+func TestResolveOffsiteNamesTheMissingVariable(t *testing.T) {
+	c := config.OffsiteConfig{
+		Endpoint: "https://s3.example", Bucket: "b",
+		AccessKeyRef: "T_AK", SecretKeyRef: "T_SK", KeyRef: "T_KEY",
+	}
+	t.Setenv("T_AK", "ak")
+	t.Setenv("T_SK", "sk")
+	t.Setenv("T_KEY", "")
+	if _, err := resolveOffsite(c); err == nil || !strings.Contains(err.Error(), "T_KEY") {
+		t.Fatalf("err = %v, want it to name T_KEY", err)
+	}
+	t.Setenv("T_KEY", "not base64 at all")
+	if _, err := resolveOffsite(c); err == nil {
+		t.Fatal("a malformed key was accepted")
+	}
+	key, _ := offsite.NewKey()
+	t.Setenv("T_KEY", key)
+	env, err := resolveOffsite(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(env.key) != offsite.KeySize {
+		t.Fatalf("key is %d bytes", len(env.key))
+	}
+	if _, err := resolveOffsite(config.OffsiteConfig{}); err == nil {
+		t.Fatal("an unconfigured [offsite] resolved")
+	}
+}
+
+func TestOffsiteKeygen(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := cmdOffsite([]string{"keygen"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if _, err := offsite.ParseKey(strings.TrimSpace(out.String())); err != nil {
+		t.Fatalf("keygen printed %q: %v", out.String(), err)
+	}
+}
+
+func TestOffsiteFetchDBRejectsOddNames(t *testing.T) {
+	key, _ := offsite.NewKey()
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "ak")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "sk")
+	t.Setenv("FELIS_OFFSITE_KEY", key)
+	var out, errb bytes.Buffer
+	code := cmdOffsite([]string{"fetch-db", "-env-file", "", "-endpoint", "http://127.0.0.1:1", "-bucket", "b",
+		"-dir", t.TempDir(), "../../etc/shadow"}, &out, &errb)
+	if code != 2 || !strings.Contains(errb.String(), "not a bundle name") {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+}
+
+func TestOffsiteRegistryEndpoint(t *testing.T) {
+	for _, tc := range []struct{ flag, url, want string }{
+		{"", "registry.felis.svc:5000", "127.0.0.1:5000"},
+		{"", "registry.felis.svc.cluster.local:5001", "127.0.0.1:5001"},
+		{"", "ghcr.io/acme", ""},
+		{"", "", ""},
+		{"off", "registry.felis.svc:5000", ""},
+		{"10.0.0.5:5000", "ghcr.io/acme", "10.0.0.5:5000"},
+	} {
+		if got := offsiteRegistryEndpoint(tc.flag, config.RegistryConfig{URL: tc.url}); got != tc.want {
+			t.Errorf("offsiteRegistryEndpoint(%q, %q) = %q, want %q", tc.flag, tc.url, got, tc.want)
+		}
+	}
+}
+
+func TestRegistryGoneMarksNotFound(t *testing.T) {
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 404}); !errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("404 = %v, want ErrImageGone", err)
+	}
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 503}); errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("503 = %v, want it kept an ordinary failure", err)
+	}
+}
+
+// mapBucket is an in-memory offsite.Bucket.
+type mapBucket map[string][]byte
+
+func (b mapBucket) Put(_ context.Context, key string, r io.Reader, _ int64) error {
+	data, err := io.ReadAll(r)
+	b[key] = data
+	return err
+}
+
+func (b mapBucket) Get(_ context.Context, key string) (io.ReadCloser, error) {
+	data, ok := b[key]
+	if !ok {
+		return nil, offsite.ErrNotFound
+	}
+	return io.NopCloser(bytes.NewReader(data)), nil
+}
+
+func (b mapBucket) List(_ context.Context, prefix string) ([]offsite.Object, error) {
+	var out []offsite.Object
+	for k, v := range b {
+		if strings.HasPrefix(k, prefix) {
+			out = append(out, offsite.Object{Key: k, Size: int64(len(v))})
+		}
+	}
+	return out, nil
+}
+
+func (b mapBucket) Remove(_ context.Context, key string) error {
+	delete(b, key)
+	return nil
+}
+
+var fetchT0 = time.Date(2026, 9, 20, 3, 30, 0, 0, time.UTC)
+
+// putBundle seals a bundle that verifies, taken daysAgo days before fetchT0,
+// into b and returns its name.
+func putBundle(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts) string {
+	t.Helper()
+	return putBundleWith(t, b, key, daysAgo, counts, "")
+}
+
+// putBundleWith is putBundle for a bundle whose server export failed with
+// serversError, when that is not empty.
+func putBundleWith(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts, serversError string) string {
+	t.Helper()
+	created := fetchT0.AddDate(0, 0, -daysAgo)
+	dump := []byte("PGDMP " + created.String())
+	sum := sha256.Sum256(dump)
+	manifest, err := json.Marshal(dbbackup.Manifest{
+		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, ServersError: serversError,
+		Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var plain bytes.Buffer
+	tw := tar.NewWriter(&plain)
+	for _, f := range []struct {
+		name string
+		data []byte
+	}{{"MANIFEST.json", manifest}, {"db.dump", dump}} {
+		if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o600, Size: int64(len(f.data)), Typeflag: tar.TypeReg}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tw.Write(f.data); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	var sealed bytes.Buffer
+	if err := offsite.Encrypt(&sealed, &plain, key); err != nil {
+		t.Fatal(err)
+	}
+	name := dbbackup.BundleName(created, dbbackup.LabelDaily)
+	b[offsite.DBKey(name)] = sealed.Bytes()
+	return name
+}
+
+func TestOffsiteFetchDB(t *testing.T) {
+	rawKey, _ := offsite.NewKey()
+	key, _ := offsite.ParseKey(rawKey)
+	now := fetchT0.Add(2 * time.Hour)
+	fetch := func(b mapBucket, arg string) (dir string, code int, stdout, stderr string) {
+		dir = t.TempDir()
+		var out, errb bytes.Buffer
+		code = fetchDB(context.Background(), b, key, arg, dir, now, &out, &errb)
+		return dir, code, out.String(), errb.String()
+	}
+	fetched := func(t *testing.T, dir string) []string {
+		t.Helper()
+		var names []string
+		entries, _ := os.ReadDir(dir)
+		for _, e := range entries {
+			names = append(names, e.Name())
+		}
+		return names
+	}
+
+	t.Run("latest skips a rebuilt host's empty bundle", func(t *testing.T) {
+		b := mapBucket{}
+		full := putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
+		empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
+		dir, code, out, errb := fetch(b, "latest")
+		if code != 1 || !strings.Contains(errb, empty) || !strings.Contains(errb, full+" (5 accounts, 3 servers)") {
+			t.Fatalf("exit %d, stdout %q, stderr %q; want a refusal naming %s", code, out, errb, full)
+		}
+		if got := fetched(t, dir); len(got) != 0 {
+			t.Errorf("a refused fetch wrote %v", got)
+		}
+	})
+
+	t.Run("latest takes the newest bundle and says what it holds", func(t *testing.T) {
+		b := mapBucket{}
+		putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 2})
+		newest := putBundle(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3})
+		dir, code, out, errb := fetch(b, "latest")
+		if code != 0 {
+			t.Fatalf("exit %d: %s", code, errb)
+		}
+		if got := fetched(t, dir); !slices.Equal(got, []string{newest}) {
+			t.Errorf("wrote %v, want %s", got, newest)
+		}
+		for _, want := range []string{
+			"wrote " + filepath.Join(dir, newest) + " (verified)",
+			"taken   2026-09-19T03:30:00Z (daily, 26h0m ago)",
+			"felis   v1.2.3, schema 21",
+			"holds   5 accounts, 3 servers",
+		} {
+			if !strings.Contains(out, want) {
+				t.Errorf("stdout lacks %q:\n%s", want, out)
+			}
+		}
+		if strings.Contains(out, "new install") {
+			t.Errorf("a bundle with servers flagged as a new install's:\n%s", out)
+		}
+	})
+
+	t.Run("an empty bundle named outright is fetched with a warning", func(t *testing.T) {
+		b := mapBucket{}
+		putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
+		empty := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 1})
+		dir, code, out, errb := fetch(b, empty)
+		if code != 0 || !slices.Equal(fetched(t, dir), []string{empty}) {
+			t.Fatalf("exit %d, wrote %v: %s", code, fetched(t, dir), errb)
+		}
+		if !strings.Contains(out, "holds   1 account, 0 servers") || !strings.Contains(out, "like a new install's") {
+			t.Errorf("stdout = %s", out)
+		}
+	})
+
+	t.Run("a bundle without the servers says where they come from", func(t *testing.T) {
+		b := mapBucket{}
+		gapped := putBundleWith(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused (tried 3 times)")
+		_, code, out, errb := fetch(b, gapped)
+		if code != 0 || !strings.Contains(out, "This bundle lacks the MinecraftServer objects (connection refused (tried 3 times))") ||
+			!strings.Contains(out, "k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap") {
+			t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
+		}
+		whole := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+		if _, _, out, _ := fetch(b, whole); strings.Contains(out, "lacks the MinecraftServer objects") {
+			t.Errorf("a whole bundle flagged:\n%s", out)
+		}
+	})
+
+	t.Run("a bundle from before counts says so", func(t *testing.T) {
+		b := mapBucket{}
+		old := putBundle(t, b, key, 0, nil)
+		_, code, out, errb := fetch(b, "latest")
+		if code != 0 || !strings.Contains(out, old) || !strings.Contains(out, "holds   not recorded") || strings.Contains(out, "new install") {
+			t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
+		}
+	})
+}
+
+func TestOffsiteCheckKey(t *testing.T) {
+	newKey := func() []byte {
+		raw, _ := offsite.NewKey()
+		k, _ := offsite.ParseKey(raw)
+		return k
+	}
+	key, other := newKey(), newKey()
+	marked := func(k []byte) mapBucket { return mapBucket{"felis-key-id": []byte(offsite.KeyID(k) + "\n")} }
+	unmarked := func(k []byte) mapBucket {
+		b := mapBucket{}
+		putBundle(t, b, k, 0, nil)
+		return b
+	}
+	for _, tc := range []struct {
+		what   string
+		bucket mapBucket
+		code   int
+		says   []string
+	}{
+		{"the recorded key", marked(key), 0, []string{"records key id " + offsite.KeyID(key)}},
+		{"an unmarked bucket the key opens", unmarked(key), 0, []string{"open with this key", "the next sync records it"}},
+		{"an empty bucket", mapBucket{}, 0, []string{"no sealed object yet", "records key id " + offsite.KeyID(key)}},
+		{"another recorded key", marked(other), 3, []string{offsite.KeyID(other), offsite.KeyID(key), "FELIS_OFFSITE_KEY"}},
+		{"an unmarked bucket under another key", unmarked(other), 3, []string{"opens none of db/felis-db-"}},
+		{"a marker Felis did not write", mapBucket{"felis-key-id": []byte("hello")}, 1, []string{"not a key id"}},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			before := len(tc.bucket)
+			var out, errb bytes.Buffer
+			code := checkKey(context.Background(), tc.bucket, key, &out, &errb)
+			if code != tc.code {
+				t.Fatalf("exit %d, want %d; stdout %q, stderr %q", code, tc.code, out.String(), errb.String())
+			}
+			said := out.String() + errb.String()
+			for _, s := range tc.says {
+				if !strings.Contains(said, s) {
+					t.Errorf("output lacks %q: %s", s, said)
+				}
+			}
+			if len(tc.bucket) != before {
+				t.Errorf("check-key wrote to the bucket: %d objects, had %d", len(tc.bucket), before)
+			}
+		})
+	}
+
+	// fetch-db names both ids when the bucket records another key.
+	b := marked(other)
+	name := putBundle(t, b, other, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+	for _, arg := range []string{"latest", name} {
+		var out, errb bytes.Buffer
+		if code := fetchDB(context.Background(), b, key, arg, t.TempDir(), fetchT0, &out, &errb); code != 1 ||
+			!strings.Contains(errb.String(), "the bucket records key id "+offsite.KeyID(other)+", and this key is "+offsite.KeyID(key)) {
+			t.Errorf("fetch-db %s under another key: exit %d, stderr %q", arg, code, errb.String())
+		}
+	}
+	// A bundle the bucket lacks is not the key's fault.
+	var missOut, missErr bytes.Buffer
+	if code := fetchDB(context.Background(), b, key, "felis-db-20200101T000000Z-daily.tar", t.TempDir(), fetchT0, &missOut, &missErr); code != 1 || strings.Contains(missErr.String(), "records key id") {
+		t.Errorf("missing bundle: exit %d, stderr %q", code, missErr.String())
+	}
+	// A bundle damaged under the recorded key gets no such hint.
+	b = marked(key)
+	name = putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+	b[offsite.DBKey(name)][60] ^= 1
+	var out, errb bytes.Buffer
+	if code := fetchDB(context.Background(), b, key, name, t.TempDir(), fetchT0, &out, &errb); code != 1 || strings.Contains(errb.String(), "records key id") {
+		t.Errorf("damaged bundle: exit %d, stderr %q", code, errb.String())
+	}
+}
+
+func TestRecordRun(t *testing.T) {
+	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}
+	for _, tc := range []struct {
+		err                                   error
+		mismatch, standby, displaced, success bool
+	}{
+		{nil, false, false, false, true},
+		{errors.New("list worlds/ in the bucket: connection reset"), false, false, false, false},
+		{fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false, false, false},
+		{&offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, false, true, false, false},
+		{&offsite.WriterError{Kind: offsite.ErrDisplaced, Writer: w}, false, false, true, false},
+	} {
+		st := offsite.Status{LastAttempt: t0}
+		recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err, offsite.Lease{})
+		if st.KeyMismatch != tc.mismatch || st.Standby != tc.standby || st.Displaced != tc.displaced ||
+			(st.Writer != nil) != (tc.standby || tc.displaced) || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 {
+			t.Errorf("err %v: status %+v", tc.err, st)
+		}
+	}
+
+	// A host that copied before writers were recorded keeps that claim over
+	// failed runs until it has an id.
+	l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Inherited: true}
+	st := offsite.Status{}
+	recordRun(&st, offsite.Result{}, errors.New("cannot reach bucket"), l)
+	if !st.Inherited {
+		t.Error("a failed run on a host with no id dropped the older release's claim")
+	}
+	writeTestFile(t, l.IDFile, "aaaaaaaaaaaaaaaa\n", 0o600)
+	st = offsite.Status{Inherited: true}
+	recordRun(&st, offsite.Result{}, nil, l)
+	if st.Inherited {
+		t.Error("a host with an id still carries the older release's claim")
+	}
+}
+
+// A refused run has copied nothing and listed nothing: its zero counts would
+// tell the journal the bucket is empty. A pass that ran and failed a step
+// shows what it did get to.
+func TestReportRun(t *testing.T) {
+	w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)}
+	for _, tc := range []struct {
+		name   string
+		res    offsite.Result
+		err    error
+		code   int
+		counts bool
+	}{
+		{"a pass", offsite.Result{DBUploaded: 1, RemoteDB: 3}, nil, 0, true},
+		{"a pass with a failed step", offsite.Result{RemoteDB: 3, Errors: []string{"copy db/x: timeout"}}, errors.New("1 of this run's steps failed; first: copy db/x: timeout"), 1, true},
+		{"standing by", offsite.Result{}, &offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, 1, false},
+		{"another key", offsite.Result{}, fmt.Errorf("%w: the bucket records key id 1111111111111111", offsite.ErrKeyMismatch), 1, false},
+		{"no bucket", offsite.Result{}, errors.New("bucket: access denied"), 1, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var out, errOut bytes.Buffer
+			code := reportRun(tc.res, tc.err, &out, &errOut)
+			if code != tc.code {
+				t.Errorf("exit %d, want %d", code, tc.code)
+			}
+			if got := strings.Contains(out.String(), "bucket holds 0 worlds (0 B), 3 bundles"); got != tc.counts {
+				t.Errorf("counts shown = %v, want %v: %q", got, tc.counts, out.String())
+			}
+			if !tc.counts && out.Len() > 0 {
+				t.Errorf("a refused run printed %q", out.String())
+			}
+			if tc.err != nil && !strings.Contains(errOut.String(), "felis offsite sync: "+tc.err.Error()) {
+				t.Errorf("stderr %q lacks the error", errOut.String())
+			}
+		})
+	}
+}
+
+func TestOffsiteTakeOver(t *testing.T) {
+	newKey := func() []byte {
+		raw, _ := offsite.NewKey()
+		k, _ := offsite.ParseKey(raw)
+		return k
+	}
+	key, other := newKey(), newKey()
+	const mine, theirs = "aaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbb"
+	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	sealed := func(k []byte, writer string) mapBucket {
+		b := mapBucket{}
+		putBundle(t, b, k, 0, nil)
+		b["felis-key-id"] = []byte(offsite.KeyID(k) + "\n")
+		if writer != "" {
+			raw, _ := json.Marshal(offsite.Writer{HostID: writer, Host: "prod-1", At: t0.Add(-20 * time.Minute)})
+			b["felis-writer"] = raw
+		}
+		return b
+	}
+	lease := func(id string) offsite.Lease {
+		l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Host: "spare-1"}
+		if id != "" {
+			writeTestFile(t, l.IDFile, id+"\n", 0o600)
+		}
+		return l
+	}
+	run := func(b mapBucket, l offsite.Lease, statusFile string, yes bool) (int, string, string) {
+		t.Helper()
+		var out, errb bytes.Buffer
+		code := takeOver(context.Background(), b, key, l, statusFile, yes, t0, &out, &errb)
+		return code, out.String(), errb.String()
+	}
+	for _, tc := range []struct {
+		what   string
+		bucket mapBucket
+		id     string
+		code   int
+		says   []string
+	}{
+		{"this host writes the bucket", sealed(key, mine), mine, 0, []string{"this host (id " + mine + ") writes the bucket; nothing to take over"}},
+		{"an empty bucket", mapBucket{}, "", 0, []string{"names no host writing it; this host's next sync records itself"}},
+		{"a host built from the writer's backup", sealed(key, theirs), "", 4, []string{"host prod-1 (id " + theirs + ") writes the bucket, last at", "(20m ago)", "built from its backup", "sudo felis offsite take-over -yes"}},
+		{"another host's copies, no writer named", sealed(key, ""), "", 4, []string{"holds copies this host did not write, and names no host writing it", "take-over -yes"}},
+		{"a host another one took over from", sealed(key, theirs), mine, 5, []string{"took the bucket over from this host"}},
+		{"a key the bucket refuses", sealed(other, theirs), "", 3, []string{"sealed with another key"}},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			before := string(tc.bucket["felis-writer"])
+			l := lease(tc.id)
+			code, out, errb := run(tc.bucket, l, filepath.Join(t.TempDir(), "status.json"), false)
+			if code != tc.code {
+				t.Fatalf("exit %d, want %d\n%s%s", code, tc.code, out, errb)
+			}
+			for _, s := range tc.says {
+				if !strings.Contains(out+errb, s) {
+					t.Errorf("output lacks %q:\n%s%s", s, out, errb)
+				}
+			}
+			if string(tc.bucket["felis-writer"]) != before {
+				t.Error("take-over without -yes wrote the bucket's writer")
+			}
+			if tc.id == "" {
+				if _, err := os.Stat(l.IDFile); !errors.Is(err, os.ErrNotExist) {
+					t.Errorf("take-over without -yes made this host an id: %v", err)
+				}
+			}
+		})
+	}
+
+	// -yes on a standby host: the bucket names it, and the refusal the last
+	// sync recorded is cleared, so the watchdog mails again at once.
+	b := sealed(key, theirs)
+	l := lease("")
+	statusFile := filepath.Join(t.TempDir(), "status.json")
+	lastSuccess := t0.Add(-48 * time.Hour)
+	if err := offsite.WriteStatus(statusFile, offsite.Status{
+		LastAttempt: t0.Add(-time.Hour), LastSuccess: lastSuccess, LastError: "offsite: another host writes this bucket", Format: offsite.StatusFormat,
+		Standby: true, Writer: &offsite.Writer{HostID: theirs, Host: "prod-1", At: t0}, Inherited: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	code, out, errb := run(b, l, statusFile, true)
+	id, _ := l.ID()
+	if code != 0 || id == "" || !strings.Contains(out, "this host (id "+id+") writes the bucket now; host prod-1 (id "+theirs+") stops at its next copy") || !strings.Contains(out, "systemctl start felis-offsite.service") {
+		t.Fatalf("take-over -yes: exit %d, id %q\n%s%s", code, id, out, errb)
+	}
+	if w, err := offsite.BucketWriter(context.Background(), b); err != nil || w.HostID != id || w.Host != "spare-1" || !w.At.Equal(t0) {
+		t.Errorf("writer after take-over -yes = %+v, %v", w, err)
+	}
+	st, err := offsite.ReadStatus(statusFile)
+	if err != nil || st.Standby || st.Writer != nil || st.LastError != "" || st.Inherited || !st.LastSuccess.Equal(lastSuccess) {
+		t.Errorf("status after take-over -yes = %+v, %v; want the refusal cleared and the last success kept", st, err)
+	}
+
+	// -yes with a key the bucket refuses writes nothing.
+	b = sealed(other, theirs)
+	before := string(b["felis-writer"])
+	if code, _, _ := run(b, lease(""), filepath.Join(t.TempDir(), "status.json"), true); code != 3 || string(b["felis-writer"]) != before {
+		t.Errorf("take-over -yes under another key: exit %d, writer %s", code, b["felis-writer"])
+	}
+}
+
+func TestOffsiteStatusSaysWhoWritesTheBucket(t *testing.T) {
+	dir := t.TempDir()
+	cfg := filepath.Join(dir, "felis.toml")
+	writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
+	statusFile := filepath.Join(dir, "status.json")
+	now := time.Now()
+	w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-30 * time.Minute)}
+	stale := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-offsite.WriterLive - time.Hour)}
+	for _, tc := range []struct {
+		what string
+		st   offsite.Status
+		says []string
+		not  string
+	}{
+		{"standing by for a live writer", offsite.Status{Standby: true, Writer: w}, []string{"This host stands by: host prod-1 (id bbbbbbbbbbbbbbbb) writes the bucket", "mails no watchdog alert", "take-over -yes"}, "wrote it, last at"},
+		{"standing by for a writer gone quiet", offsite.Status{Standby: true, Writer: stale}, []string{"copies nothing into the bucket: host prod-1 (id bbbbbbbbbbbbbbbb) wrote it, last at", "take-over -yes"}, "mails no watchdog alert"},
+		{"standing by, no writer named", offsite.Status{Standby: true}, []string{"names no host writing it", "take-over -yes"}, "stands by:"},
+		{"displaced", offsite.Status{Displaced: true, Writer: w}, []string{"host prod-1 (id bbbbbbbbbbbbbbbb) took the bucket over", "rehearsal machine", "take-over -yes"}, "stands by"},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			tc.st.LastAttempt, tc.st.LastSuccess, tc.st.LastError = now.Add(-time.Minute), now.Add(-time.Hour), "offsite: another host writes this bucket"
+			if err := offsite.WriteStatus(statusFile, tc.st); err != nil {
+				t.Fatal(err)
+			}
+			var out, errb bytes.Buffer
+			code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
+			if code != 1 || strings.Contains(out.String(), "bucket holds:") || strings.Contains(out.String(), tc.not) {
+				t.Errorf("exit %d\n%s%s", code, out.String(), errb.String())
+			}
+			for _, s := range tc.says {
+				if !strings.Contains(out.String(), s) {
+					t.Errorf("output lacks %q:\n%s", s, out.String())
+				}
+			}
+		})
+	}
+}
+
+func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) {
+	dir := t.TempDir()
+	cfg := filepath.Join(dir, "felis.toml")
+	writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
+	statusFile := filepath.Join(dir, "status.json")
+	st := offsite.Status{LastAttempt: time.Now().Add(-time.Minute), LastSuccess: time.Now().Add(-time.Hour), KeyID: "0123456789abcdef"}
+	status := func() (int, string) {
+		t.Helper()
+		if err := offsite.WriteStatus(statusFile, st); err != nil {
+			t.Fatal(err)
+		}
+		var out, errb bytes.Buffer
+		code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
+		return code, out.String() + errb.String()
+	}
+	if code, out := status(); code != 0 || strings.Contains(out, "refused") {
+		t.Fatalf("a recent success: exit %d\n%s", code, out)
+	}
+	st.LastError, st.KeyMismatch = "offsite: the bucket's objects are sealed with another key", true
+	code, out := status()
+	if code != 1 || !strings.Contains(out, "The last run was refused") || !strings.Contains(out, "key id 0123456789abcdef") || strings.Contains(out, "bucket holds:") {
+		t.Fatalf("a refused run: exit %d\n%s", code, out)
+	}
+}
+
+func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
+	rawKey, _ := offsite.NewKey()
+	key, _ := offsite.ParseKey(rawKey)
+	otherRaw, _ := offsite.NewKey()
+	other, _ := offsite.ParseKey(otherRaw)
+	b := mapBucket{}
+	gapped := putBundleWith(t, b, key, 4, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused")
+	old := putBundle(t, b, key, 3, nil)
+	full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3})
+	sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3})
+	empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
+	bundles, err := offsite.ListDB(context.Background(), b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var out bytes.Buffer
+	printDBBundles(context.Background(), b, key, bundles, &out)
+	lines := strings.Split(strings.TrimSpace(out.String()), "\n")
+	want := []struct{ name, holds string }{
+		{empty, "0 accounts, 0 servers"},
+		{sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"},
+		{full, "5 accounts, 3 servers"},
+		{old, "not recorded"},
+		{gapped, "5 accounts, 3 servers, no MinecraftServer objects"},
+	}
+	if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (5, newest first") {
+		t.Fatalf("output:\n%s", out.String())
+	}
+	for i, w := range want {
+		if l := lines[i+1]; !strings.HasPrefix(l, "  "+w.name+"  ") || !strings.Contains(l, w.holds) || strings.Contains(l, "no MinecraftServer") != (w.name == gapped) {
+			t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds)
+		}
+	}
+}
+
+// TestRestoredHostKeepsStandingBy walks the status file across runs: a host
+// restored from the writer's backup stands by on its first run and on every
+// run after it, a host an older release left copying claims the bucket once,
+// and a failed first run after the upgrade keeps that claim.
+func TestRestoredHostKeepsStandingBy(t *testing.T) {
+	rawKey, _ := offsite.NewKey()
+	key, _ := offsite.ParseKey(rawKey)
+	cfg := config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-backups"}
+	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	standby := &offsite.WriterError{Kind: offsite.ErrStandby, Writer: &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}}
+	pass := func(statusFile string, at time.Time, err error) offsite.Lease {
+		t.Helper()
+		st, lease := startRun(cfg, key, statusFile, at)
+		recordRun(&st, offsite.Result{}, err, lease)
+		if werr := offsite.WriteStatus(statusFile, st); werr != nil {
+			t.Fatal(werr)
+		}
+		return lease
+	}
+
+	restored := filepath.Join(t.TempDir(), "status.json")
+	for i := range 3 {
+		if l := pass(restored, t0.Add(time.Duration(i)*time.Hour), standby); l.Inherited {
+			t.Fatalf("run %d of a restored host claims the bucket", i+1)
+		}
+	}
+	if st, _ := offsite.ReadStatus(restored); !st.Standby || st.Format != offsite.StatusFormat || st.KeyID != offsite.KeyID(key) || st.Bucket != "felis-backups" {
+		t.Errorf("restored host's status = %+v", st)
+	}
+
+	upgraded := filepath.Join(t.TempDir(), "status.json")
+	if err := offsite.WriteStatus(upgraded, offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-time.Hour)}); err != nil {
+		t.Fatal(err)
+	}
+	if l := pass(upgraded, t0, errors.New("cannot reach bucket")); !l.Inherited {
+		t.Fatal("the first run after the upgrade does not claim the bucket")
+	}
+	l := pass(upgraded, t0.Add(time.Hour), nil)
+	if !l.Inherited {
+		t.Fatal("a failed first run after the upgrade lost the claim")
+	}
+	if st, _ := offsite.ReadStatus(upgraded); !st.LastSuccess.Equal(t0.Add(time.Hour)) {
+		t.Errorf("upgraded host's status = %+v", st)
+	}
+}
+
+// TestOffsiteSyncerSnapshotsAndSweeps: the pass `offsite sync` runs takes its
+// snapshot the way `felis db backup` does, in the database pod, into the
+// bundle directory, keeping the newest one there; and it sweeps unrecorded
+// world objects only past the longest retention [archive] gives any backup.
+func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) {
+	dir := newPodRig(t)
+	bundles := filepath.Join(dir, "bundles")
+	env := &offsiteEnv{cfg: config.OffsiteConfig{DBKeep: 5}}
+	var log bytes.Buffer
+	cfg := &config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "120d"}}
+	s := offsiteSyncer(cfg, env, offsiteSources{dbDir: bundles}, "/archives", "/uploads", nil, &log)
+	if s.DBDir != bundles || s.DBKeep != 5 || s.ArchiveDir != "/archives" || s.UploadsDir != "/uploads" {
+		t.Fatalf("syncer = %+v", s)
+	}
+	if s.OrphanAfter != 120*24*time.Hour {
+		t.Errorf("OrphanAfter = %s, want the 120d retention", s.OrphanAfter)
+	}
+	if s.Snapshot == nil {
+		t.Fatal("the pass takes no snapshot after copying archives")
+	}
+	for i := 0; i < 2; i++ {
+		if err := s.Snapshot(context.Background()); err != nil {
+			t.Fatalf("snapshot %d: %v", i, err)
+		}
+	}
+	got, err := dbbackup.List(bundles)
+	if err != nil || len(got) != 1 || got[0].Label != dbbackup.LabelOffsite {
+		t.Fatalf("bundle directory = %+v, %v; want the newest offsite bundle alone", got, err)
+	}
+	if _, err := dbbackupVerify(got[0].Path); err != nil {
+		t.Fatalf("the snapshot does not verify: %v", err)
+	}
+	// The MinecraftServer objects are exported alongside, as in the daily bundle.
+	argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args"))
+	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") {
+		t.Errorf("k3s ran %q, want pg_dump in the pod", ran)
+	}
+	if !strings.Contains(string(bundleServers(t, got[0].Path)), `"name": "lobby"`) {
+		t.Errorf("the snapshot holds no MinecraftServer objects")
+	}
+	if !strings.Contains(log.String(), "took database bundle "+got[0].Name) {
+		t.Errorf("the snapshot is not logged:\n%s", log.String())
+	}
+	// It becomes the newest bundle in the bucket, so with the cluster away it
+	// fails, leaves no bundle, and the next pass tries again.
+	noServerExportWait(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	if err := s.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "export the MinecraftServer objects") {
+		t.Errorf("snapshot with the cluster away: %v, want a failure", err)
+	}
+	if again, _ := dbbackup.List(bundles); len(again) != 1 || again[0].Name != got[0].Name {
+		t.Errorf("bundle directory after the failed snapshot = %+v, want %s alone", again, got[0].Name)
+	}
+
+	for _, c := range []struct {
+		archive config.ArchiveConfig
+		want    time.Duration
+	}{
+		{config.ArchiveConfig{}, 90 * 24 * time.Hour},
+		{config.ArchiveConfig{ScheduledRetention: "200d"}, 200 * 24 * time.Hour},
+		{config.ArchiveConfig{ManualRetention: "150d", Retention: "30d", ScheduledRetention: "60d"}, 150 * 24 * time.Hour},
+	} {
+		s := offsiteSyncer(&config.Config{Database: podDB, Archive: c.archive}, env, offsiteSources{dbDir: bundles}, "", "", nil, io.Discard)
+		if s.OrphanAfter != c.want {
+			t.Errorf("%+v: OrphanAfter = %s, want %s", c.archive, s.OrphanAfter, c.want)
+		}
+	}
+	log.Reset()
+	s = offsiteSyncer(&config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "soon"}}, env, offsiteSources{}, "", "", nil, &log)
+	if s.OrphanAfter != 0 || !strings.Contains(log.String(), "world objects no backup records are kept") {
+		t.Errorf("a retention that does not parse: OrphanAfter %s, log %q; want no sweep, said", s.OrphanAfter, log.String())
+	}
+	if s.Snapshot != nil {
+		t.Error("a pass that copies no bundles takes a snapshot")
+	}
+}

cmd/felis/offsite_uploads.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+)
+
+func offsiteFetchUploads(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC, binding it if needed)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, "the submission uploads PVC, in the control-plane namespace")
+	at := fs.String("at", "", "uploads index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseUploadIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	dir := *uploadsDir
+	if dir == "" {
+		if !isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: [registry] user_uploads_context %q is not the uploads volume; pass -uploads-dir to restore into a directory anyway\n", cfg.Registry.UserUploadsContext)
+			return 2
+		}
+		if dir, err = resolveVolumeDir(ctx, platform.DefaultControlNamespace, *uploadsPVC, uploadsVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+			return 1
+		}
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: restoring uploads index %s (%d submission contexts, %s) into %s\n",
+		stamp, len(idx.Contexts), offsite.HumanBytes(idx.Bytes()), dir)
+	res, err := offsite.FetchUploads(ctx, env.bucket, env.key, idx, dir, platform.ControlPlaneUID, platform.ControlPlaneUID, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: %d written (%s), %d already in place, %d failed\n",
+		res.Written, offsite.HumanBytes(res.Bytes), res.Present, len(res.Failures))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v (a second run writes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/operator.go

@@ -1,19 +1,26 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"log/slog"
+	"net/http"
 	"os"
+	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	felismetrics "felis.lolicon.best/internal/metrics"
 	"felis.lolicon.best/internal/operator"
+	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 )
@@ -25,6 +32,11 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	fs := flag.NewFlagSet("operator", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	metricsAddr := fs.String("metrics-bind-address", ":8080", "address the metric endpoint binds to")
+	// healthAddr serves the manager's health endpoints (/healthz, /readyz) that the
+	// Deployment's probes dial. Without it the operator pod would carry no probe at
+	// all, and a wedged manager would keep its endpoint forever. It must differ from
+	// metricsAddr: the metrics server owns :8080.
+	healthAddr := fs.String("health-probe-bind-address", ":8081", "address the health probe endpoint binds to")
 	// namespace MUST equal the [k8s] namespace felis-api is configured with, and
 	// the deployment manifests (felis manifests) render both from one value. It
 	// scopes the manager's cache (informers) to a single namespace so the operator
@@ -42,9 +54,16 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(v1alpha1.AddToScheme(scheme))
 
+	// controller-runtime logs through its own logr sink; without one, its first
+	// reconcile prints "log.SetLogger(...) was never called" ATTACHED TO A FULL
+	// GOROUTINE STACK — pure noise, not signal. Route it to slog's default handler
+	// so its messages appear as ordinary stderr lines.
+	ctrl.SetLogger(logr.FromSlogHandler(slog.Default().Handler()))
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:  scheme,
-		Metrics: metricsserver.Options{BindAddress: *metricsAddr},
+		Scheme:                 scheme,
+		Metrics:                metricsserver.Options{BindAddress: *metricsAddr},
+		HealthProbeBindAddress: *healthAddr,
 		// Scope every informer to the single watched namespace. Without this the
 		// cached client (mgr.GetClient) would LIST/WATCH cluster-wide, which a
 		// namespaced Role cannot grant — the operator would fail closed at runtime
@@ -60,6 +79,23 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	fmt.Fprintf(stderr, "felis operator: watching namespace %q\n", *namespace)
 
+	// /healthz fails while a reconcile pass has been stuck past its limit, so the
+	// liveness probe restarts an operator whose workers are wedged (a Pod whose
+	// process answers but no server starts or stops). /readyz waits for the
+	// informer caches: until they sync the operator acts on nothing, and one that
+	// never syncs (lost RBAC, an unreachable API) never reports Available.
+	// A dependency hiccup fails neither: the caches ride through API blips, and
+	// each pass is bounded well inside the stuck limit.
+	watch := &operator.ReconcileWatch{}
+	if err := mgr.AddHealthzCheck("reconcile", watch.Check); err != nil {
+		fmt.Fprintf(stderr, "felis operator: register healthz check: %v\n", err)
+		return 1
+	}
+	if err := mgr.AddReadyzCheck("informers", cacheSynced(mgr.GetCache())); err != nil {
+		fmt.Fprintf(stderr, "felis operator: register readyz check: %v\n", err)
+		return 1
+	}
+
 	// Publish the named felis_* metrics (spec §23) on the endpoint the manager
 	// already serves (metricsAddr). controller-runtime's metrics server exposes
 	// its global Registry, so registering into it is all that is needed for
@@ -69,6 +105,7 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis operator: register metrics: %v\n", err)
 		return 1
 	}
+	felismetrics.SetBuildInfo("operator", resolvedVersion())
 
 	r := &operator.Reconciler{
 		Client: mgr.GetClient(),
@@ -78,6 +115,17 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		// injects into user servers. The Deployment passes it as FELIS_IMAGE (see
 		// platform.OperatorDeployment); absent, that injection is simply skipped.
 		FelisImage: os.Getenv("FELIS_IMAGE"),
+		// Uncached: the maintenance-lock check lists Jobs only when a server is
+		// about to start, which does not justify a namespace-wide Job informer.
+		Jobs: mgr.GetAPIReader(),
+		// Uncached too: RCON Secrets are read by name, so the Role grants
+		// secrets:get without the list/watch an informer would need.
+		Secrets: mgr.GetAPIReader(),
+		// And pods: pod-0 is read by name, so the Role grants pods:get and no
+		// namespace-wide pod informer runs.
+		Pods:     mgr.GetAPIReader(),
+		Recorder: mgr.GetEventRecorderFor("felis-operator"),
+		Watch:    watch,
 	}
 	if err := r.SetupWithManager(mgr); err != nil {
 		fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
@@ -99,3 +147,18 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// cacheSynced is a readyz check that passes once every informer the manager
+// started has synced. It waits at most a second, well inside the probe timeout.
+func cacheSynced(c interface {
+	WaitForCacheSync(ctx context.Context) bool
+}) healthz.Checker {
+	return func(req *http.Request) error {
+		ctx, cancel := context.WithTimeout(req.Context(), time.Second)
+		defer cancel()
+		if !c.WaitForCacheSync(ctx) {
+			return errors.New("informer caches not synced")
+		}
+		return nil
+	}
+}

cmd/felis/operator_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"context"
+	"net/http/httptest"
+	"testing"
+)
+
+type fakeCache bool
+
+func (f fakeCache) WaitForCacheSync(ctx context.Context) bool {
+	if !f {
+		<-ctx.Done()
+	}
+	return bool(f)
+}
+
+// TestCacheSynced: the operator reports ready only once its informers synced,
+// and a check against caches that never sync returns within its own deadline.
+func TestCacheSynced(t *testing.T) {
+	req := httptest.NewRequest("GET", "/readyz", nil)
+	if err := cacheSynced(fakeCache(true))(req); err != nil {
+		t.Errorf("synced: %v", err)
+	}
+	if err := cacheSynced(fakeCache(false))(req); err == nil {
+		t.Error("unsynced caches reported ready")
+	}
+}

cmd/felis/pinimages.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// defaultRegistryURL is the [registry] url every install uses; deploy/bootstrap.sh
+// spells the same value as REGISTRY_URL.
+const defaultRegistryURL = "registry.felis.svc:5000"
+
+// cmdPinImages pins every user server whose spec.image still names a mutable tag
+// in the platform registry to the digest that tag names now (internal/imagepin).
+// felis-api pins on create, so this covers the servers created before it did.
+//
+// deploy/bootstrap.sh runs it before it rebuilds the game images and pushes them
+// over the same tags: run after the push, it would pin those servers to the new
+// build, which is exactly the silent Minecraft upgrade pinning exists to stop.
+// It reaches the registry through the node's loopback hostPort, the same way the
+// installer pushes.
+func cmdPinImages(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("pin-images", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	namespace := fs.String("namespace", platform.DefaultMinecraftNamespace, "namespace the MinecraftServers live in")
+	registry := fs.String("registry", defaultRegistryURL, "registry host[:port] the image refs spell")
+	endpoint := fs.String("endpoint", "", "host[:port] to reach the registry at (default: 127.0.0.1 on the registry's port, its hostPort on this node)")
+	system := fs.String("system", "", "pin this system server ("+naming.SystemLoginServer+" or "+naming.SystemLobbyServer+") to the build its tag names now, instead of the user servers")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if *system != "" && *system != naming.SystemLoginServer && *system != naming.SystemLobbyServer {
+		fmt.Fprintf(stderr, "felis pin-images: --system takes %s or %s, not %q\n", naming.SystemLoginServer, naming.SystemLobbyServer, *system)
+		return 2
+	}
+	if *endpoint == "" {
+		*endpoint = loopbackEndpoint(*registry)
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	if *system != "" {
+		return reportSystemPin(ctx, cl, *namespace, *system, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint}, stdout, stderr)
+	}
+	outcomes, err := pinUserServerImages(ctx, cl, *namespace, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint})
+	if meta.IsNoMatchError(err) {
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	if len(outcomes) == 0 {
+		fmt.Fprintln(stdout, "felis pin-images: every user server already runs a pinned image")
+		return 0
+	}
+	fmt.Fprintln(stdout, "felis pin-images: pinning user servers to the build their tag names now:")
+	exit := 0
+	for _, o := range outcomes {
+		if o.err != nil {
+			fmt.Fprintf(stdout, "  - %s: ERROR %v\n", o.name, o.err)
+			exit = 1
+			continue
+		}
+		fmt.Fprintf(stdout, "  - %s: %s\n", o.name, strings.Join(o.changes, ", "))
+	}
+	return exit
+}
+
+// reportSystemPin runs pinSystemServerImage for `felis pin-images --system` and
+// prints what it did. Only a failed pin exits non-zero: the installer falls back
+// to restarting the pod on its tag then.
+func reportSystemPin(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver, stdout, stderr io.Writer) int {
+	o, err := pinSystemServerImage(ctx, cl, namespace, name, r)
+	switch {
+	case meta.IsNoMatchError(err):
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	case err == nil && o.err != nil:
+		err = o.err
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %s: %v\n", name, err)
+		return 1
+	}
+	switch {
+	case o.updated:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s; the operator rolls it onto that build\n", name, strings.Join(o.changes, ", "))
+	default:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s\n", name, o.skipped)
+	}
+	return 0
+}
+
+// pinSystemServerImage fixes a system server to the build its image tag names now,
+// replacing the digest of an earlier build. The installer runs it after pushing a
+// rebuilt login or lobby image, and the operator rolls the StatefulSet onto the new
+// ref, so the build a system server runs is written in its spec and moves only when
+// a build did. An image outside the platform registry, or one naming no tag to
+// follow, is the admin's choice and is left alone; so is a server whose image an
+// admin retargets while this runs.
+func pinSystemServerImage(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver) (systemServerOutcome, error) {
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &ms); err != nil {
+		if apierrors.IsNotFound(err) {
+			return systemServerOutcome{name: name, skipped: "not present yet; sudo felis setup creates it"}, nil
+		}
+		return systemServerOutcome{}, err
+	}
+	if ms.Labels[v1alpha1.LabelSystemRole] != name {
+		return systemServerOutcome{name: name, err: fmt.Errorf(
+			"MinecraftServer %s/%s is not marked as the Felis %q system server; left alone", namespace, name, name)}, nil
+	}
+	tagged := withoutDigest(ms.Spec.Image)
+	if !r.Covers(tagged) || !strings.Contains(tagged[strings.LastIndex(tagged, "/")+1:], ":") {
+		return systemServerOutcome{name: name, available: true, skipped: "runs " + ms.Spec.Image +
+			", which names no platform registry tag to follow; left alone"}, nil
+	}
+	pinned, err := r.Pin(ctx, tagged)
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("resolve %s: %w", tagged, err)}, nil
+	}
+	changed, err := patchOnConflictRetry(ctx, cl, &ms, func() bool {
+		if withoutDigest(ms.Spec.Image) != tagged || ms.Spec.Image == pinned {
+			return false
+		}
+		ms.Spec.Image = pinned
+		return true
+	})
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("patch %s: %w", name, err)}, nil
+	}
+	if !changed {
+		return systemServerOutcome{name: name, available: true, skipped: "already runs " + ms.Spec.Image}, nil
+	}
+	return systemServerOutcome{name: name, available: true, updated: true,
+		changes: []string{"spec.image pinned to " + pinned}}, nil
+}
+
+// withoutDigest drops the @sha256:… of a pinned ref, leaving the tag it came from.
+func withoutDigest(ref string) string {
+	if i := strings.Index(ref, "@"); i >= 0 {
+		return ref[:i]
+	}
+	return ref
+}
+
+// loopbackEndpoint is the registry's port on 127.0.0.1: the registry Deployment
+// binds it as a hostPort, and containerd's mirror and the installer's pushes use
+// the same address.
+func loopbackEndpoint(registry string) string {
+	if i := strings.LastIndex(registry, ":"); i >= 0 {
+		return "127.0.0.1" + registry[i:]
+	}
+	return "127.0.0.1"
+}
+
+// pinUserServerImages patches spec.image of every user server whose image the
+// resolver covers and is not yet pinned. System servers are the installer's to move:
+// it re-pins them with --system when it rolls them onto a new build
+// (restart_existing_system_servers).
+// A server that is already pinned, or runs an image from elsewhere, produces no
+// outcome, so a pinned fleet reports nothing. A running server restarts once as
+// the operator rolls its StatefulSet onto the pinned ref, which is the build it
+// already runs.
+func pinUserServerImages(ctx context.Context, cl client.Client, namespace string, r imagepin.Resolver) ([]systemServerOutcome, error) {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return nil, fmt.Errorf("list servers: %w", err)
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || imagepin.Pinned(ms.Spec.Image) || !r.Covers(ms.Spec.Image) {
+			continue
+		}
+		pinned, err := r.Pin(ctx, ms.Spec.Image)
+		if errors.Is(err, imagepin.ErrNotFound) {
+			err = fmt.Errorf("%s is not in the registry, so there is no build to pin it to; left unpinned: %w", ms.Spec.Image, err)
+		}
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: err})
+			continue
+		}
+		patch := client.MergeFrom(ms.DeepCopy())
+		ms.Spec.Image = pinned
+		if err := cl.Patch(ctx, ms, patch); err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("patch %s: %w", ms.Name, err)})
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{"spec.image pinned to " + pinned}})
+	}
+	return out, nil
+}

cmd/felis/pinimages_test.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+const pinTestDigest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
+
+// TestPinUserServerImages pins exactly the user servers still on a platform tag,
+// reports a tag the registry lost as an error without touching that server, and
+// has nothing left to do on a second pass.
+func TestPinUserServerImages(t *testing.T) {
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v2/felis/paper/manifests/demo" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Docker-Content-Digest", pinTestDigest)
+	}))
+	defer reg.Close()
+	res := imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+
+	paper := defaultRegistryURL + "/felis/paper:demo"
+	mk := func(name, image, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Image = image
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		mk("legacy", paper, ""),
+		mk("pinned", paper+"@sha256:"+strings.Repeat("3", 64), ""),
+		mk("external", "docker.io/itzg/minecraft-server:java21", ""),
+		mk("gone", defaultRegistryURL+"/felis/paper:old", ""),
+		mk(naming.SystemLobbyServer, defaultRegistryURL+"/felis/felis-lobby:demo", naming.SystemLobbyServer),
+	).Build()
+	ctx := context.Background()
+
+	outcomes, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil {
+		t.Fatalf("pinUserServerImages: %v", err)
+	}
+	byName := map[string]systemServerOutcome{}
+	for _, o := range outcomes {
+		byName[o.name] = o
+	}
+	if len(outcomes) != 2 || byName["legacy"].err != nil || byName["gone"].err == nil {
+		t.Fatalf("outcomes = %+v, want legacy pinned and gone reported", outcomes)
+	}
+
+	want := map[string]string{
+		"legacy":                 paper + "@" + pinTestDigest,
+		"pinned":                 paper + "@sha256:" + strings.Repeat("3", 64),
+		"external":               "docker.io/itzg/minecraft-server:java21",
+		"gone":                   defaultRegistryURL + "/felis/paper:old",
+		naming.SystemLobbyServer: defaultRegistryURL + "/felis/felis-lobby:demo",
+	}
+	for name, image := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Image != image {
+			t.Errorf("%s image = %q, want %q", name, ms.Spec.Image, image)
+		}
+	}
+
+	again, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil || len(again) != 1 || again[0].name != "gone" {
+		t.Fatalf("second pass = %+v, %v; want only the unresolvable server again", again, err)
+	}
+}
+
+// A fresh install has no CRD yet; the command must read that as nothing to pin.
+func TestPinUserServerImagesNoCRD(t *testing.T) {
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
+		List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error {
+			return &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "felis.lolicon.best", Kind: "MinecraftServer"}}
+		},
+	}).Build()
+	_, err := pinUserServerImages(context.Background(), cl, "minecraft", imagepin.Resolver{Registry: defaultRegistryURL})
+	if !meta.IsNoMatchError(err) {
+		t.Fatalf("err = %v, want a NoMatch error the command can recognise", err)
+	}
+}
+
+func TestLoopbackEndpoint(t *testing.T) {
+	for in, want := range map[string]string{
+		"registry.felis.svc:5000": "127.0.0.1:5000",
+		"registry.example":        "127.0.0.1",
+	} {
+		if got := loopbackEndpoint(in); got != want {
+			t.Errorf("loopbackEndpoint(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

cmd/felis/reaper.go

@@ -1,9 +1,13 @@
 package main
 
 import (
+	"context"
+	"database/sql"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -12,8 +16,9 @@ import (
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/backup"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/reaper"
-	"felis.lolicon.best/internal/store"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -26,11 +31,17 @@ import (
 // cadence, and RunOnce is idempotent and restart-safe, so a missed or retried
 // run simply converges. Only the tarLocal archive backend is wired in this
 // build; the snapshot backends (§19) are a later integration.
+//
+// --retention-only runs the archive-store half alone (reaper.RunRetention):
+// the CronJob an install without a worlds root gets, so backups past their
+// expiry still leave the store there. It builds no Kubernetes client and reads
+// no world.
 func cmdReaper(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("reaper", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
-	worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>)")
+	worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>, else the stock local-path <root>/<pv-name>_<ns>_<pvc-name>)")
+	retentionOnly := fs.Bool("retention-only", false, "only expire, read back and sweep the archive store: no server is evaluated and no world is read or deleted, so neither the worlds root nor the Kubernetes API is needed")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -47,50 +58,150 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	archiver, err := buildArchiver(cfg, *worldsRoot)
+	ctx := ctrl.SetupSignalHandler()
+
+	var cl client.Client
+	if !*retentionOnly {
+		scheme := runtime.NewScheme()
+		utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+		utilruntime.Must(v1alpha1.AddToScheme(scheme))
+		cl, err = client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
+		if err != nil {
+			fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
+			return 1
+		}
+	}
+
+	archiver, err := buildArchiver(ctx, cfg, *worldsRoot, cl)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: %v\n", err)
 		return 1
 	}
 
-	ctx := ctrl.SetupSignalHandler()
-
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	drv, err := openPodStore(ctx, cfg.Database.URL, "reaper", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
 		return 1
 	}
 	defer drv.Close()
 
-	scheme := runtime.NewScheme()
-	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-	utilruntime.Must(v1alpha1.AddToScheme(scheme))
-	cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
-	if err != nil {
-		fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
-		return 1
-	}
-
 	r := &reaper.Reaper{
 		Cfg:      rcfg,
 		Store:    reaper.NewPGStore(drv.DB()),
-		Cluster:  reaper.NewK8sCluster(cl, cfg.K8s.Namespace),
 		Archiver: archiver,
 	}
+	if *retentionOnly {
+		fmt.Fprintln(stderr, "felis reaper: retention only — no worlds root is configured, so idle worlds are neither archived nor released")
+		return reportReaperRun(r.RunRetention(ctx), stdout, stderr)
+	}
+	r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace)
+
+	// Pre-reap warnings go out by email when [smtp] is configured (the same
+	// relay and password_ref convention felis-api uses); without it the channel
+	// stays nil and the reaper logs each suppressed warning instead of stamping
+	// it, so a later SMTP setup still gets to warn. The owner must have a
+	// VERIFIED address — that flag is what proves the mailbox.
+	if cfg.SMTP.Host != "" {
+		passRef := cfg.SMTP.PasswordRef
+		if passRef == "" {
+			passRef = platform.SMTPPasswordEnv
+		}
+		password := os.Getenv(passRef)
+		if cfg.SMTP.Username != "" && password == "" {
+			fmt.Fprintf(stderr, "felis reaper: warning: [smtp] username is set but credentials env %s is empty — warning emails will fail AUTH\n", passRef)
+		}
+		db := drv.DB()
+		r.Warner = &mailWarner{
+			lookupEmail: func(ctx context.Context, ownerID string) (string, error) {
+				var email string
+				switch err := db.QueryRowContext(ctx,
+					`SELECT email FROM users
+					 WHERE id = $1 AND email_verified = true AND COALESCE(email, '') <> ''`,
+					ownerID).Scan(&email); {
+				case errors.Is(err, sql.ErrNoRows):
+					return "", fmt.Errorf("owner %s has no verified email", ownerID)
+				case err != nil:
+					return "", err
+				}
+				return email, nil
+			},
+			notifier: smtpRelay(cfg.SMTP, password),
+		}
+	} else {
+		fmt.Fprintln(stderr, "felis reaper: [smtp] not configured — pre-reap warnings are logged and NOT marked sent")
+	}
 
 	sum, err := r.RunOnce(ctx)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: %v\n", err)
 		return 1
 	}
-	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d warned=%d skipped=%d evicted=%d expired=%d\n",
-		sum.Evaluated, sum.WorldsReaped, sum.Warned, sum.Skipped, sum.EvictedEarly, sum.BackupsExpired)
-	return 0
+	return reportReaperRun(sum, stdout, stderr)
+}
+
+// reportReaperRun prints the run's tally and turns a run that left work undone
+// into exit 1, so the Job fails and the watchdog's job-failed check (and the
+// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
+// is kept, and without this nobody would learn that it is never reaped.
+func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
+	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d released=%d deleted=%d awaiting_offsite=%d awaiting_stop=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
+		sum.Evaluated, sum.WorldsReaped, sum.Released, sum.ServersDeleted, sum.AwaitingOffsite, sum.AwaitingStop, sum.Warned, sum.Skipped, sum.StoreFull,
+		sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
+		sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
+	if !sum.Failed() {
+		return 0
+	}
+	if sum.Skipped > 0 || sum.ExpireFailed > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
+			sum.Skipped, sum.StoreFull, sum.ExpireFailed)
+	}
+	if sum.Corrupt > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)
+	}
+	if sum.VerifyFailed > 0 || sum.SweepFailed {
+		fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",
+			sum.VerifyFailed, !sum.SweepFailed)
+	}
+	return 1
+}
+
+// mailWarner delivers a pre-reap notice to the owner's verified email — the
+// only channel this build can reach. Unowned owners and owners who never proved
+// a mailbox yield an error; the reaper retries such notices on its next run and
+// never lets them block the reap (red line ⑤).
+type mailWarner struct {
+	lookupEmail func(ctx context.Context, ownerID string) (string, error)
+	notifier    noticeNotifier
+}
+
+// noticeNotifier is the slice of mail.SMTP the warner needs (injected in tests).
+type noticeNotifier interface {
+	SendNotice(ctx context.Context, email, subject, body string) error
+}
+
+func (w *mailWarner) Warn(ctx context.Context, ownerID, server, remaining string) error {
+	email, err := w.lookupEmail(ctx, ownerID)
+	if err != nil {
+		return fmt.Errorf("resolve owner email: %w", err)
+	}
+	subject := fmt.Sprintf("Felis: 服务器 %s 将在 %s 后回收 · server reaped in %s", server, remaining, remaining)
+	body := fmt.Sprintf(
+		"Felis 世界回收提醒 / world-reaper notice\r\n"+
+			"\r\n"+
+			"服务器 / Server: %s\r\n"+
+			"距回收 / Time left: %s\r\n"+
+			"\r\n"+
+			"闲置的服务器会先自动备份，再释放世界；有人加入游戏即可重置倒计时。\r\n"+
+			"Idle servers are backed up and then released; any join resets the countdown.\r\n",
+		server, remaining)
+	return w.notifier.SendNotice(ctx, email, subject, body)
 }
 
 // reaperConfig derives the reaper's retention windows from felis.toml. The 15d
-// idle deadline is fixed by §18; only the warning offsets, retention, and the
-// store soft-cap are configurable (§24).
+// idle deadline is fixed by §18; only the warning offsets, retention, the
+// store soft-cap, the on-demand backup bounds and the scheduled restore points
+// are configurable (§24). The backup Job and felis-api read the manual_* and
+// scheduled_* keys through it too.
 func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 	rc := reaper.DefaultConfig()
 	if v := cfg.Archive.Retention; v != "" {
@@ -118,48 +229,112 @@ func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 		}
 		rc.MaxLocalBytes = b
 	}
+	if v := cfg.Archive.ManualRetention; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d <= 0 {
+			return rc, fmt.Errorf("[archive] manual_retention %q: want a positive span such as 30d", v)
+		}
+		rc.ManualRetention = d
+	}
+	switch n := cfg.Archive.ManualKeep; {
+	case n < 0:
+		return rc, fmt.Errorf("[archive] manual_keep %d: want 1 or more", n)
+	case n > 0:
+		rc.ManualKeep = n
+	}
+	if v := cfg.Archive.ManualCooldown; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d < 0 {
+			return rc, fmt.Errorf("[archive] manual_cooldown %q: want a span such as 10m (0s for none)", v)
+		}
+		rc.ManualCooldown = d
+	}
+	if v := cfg.Archive.ScheduledEvery; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d < 0 {
+			return rc, fmt.Errorf("[archive] scheduled_every %q: want a span such as 1d (0s for none)", v)
+		}
+		rc.ScheduledEvery = d
+	}
+	switch n := cfg.Archive.ScheduledKeep; {
+	case n < 0:
+		return rc, fmt.Errorf("[archive] scheduled_keep %d: want 1 or more", n)
+	case n > 0:
+		rc.ScheduledKeep = n
+	}
+	if v := cfg.Archive.ScheduledRetention; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d <= 0 {
+			return rc, fmt.Errorf("[archive] scheduled_retention %q: want a positive span such as 90d", v)
+		}
+		rc.ScheduledRetention = d
+	}
+	rc.RequireOffsite = cfg.Offsite.Enabled()
 	return rc, nil
 }
 
 // buildArchiver constructs the WorldArchiver. Only tarLocal is implemented in
-// this build; the resolver maps each world PVC to <worldsRoot>/<pvc>, the mount
-// convention the reaper Job is deployed with.
-func buildArchiver(cfg *config.Config, worldsRoot string) (backup.WorldArchiver, error) {
+// this build; the resolver maps each world PVC to its directory under worldsRoot
+// (resolveWorldDir). A nil cl is the retention-only run, which never archives a
+// world, so its archiver resolves none.
+func buildArchiver(ctx context.Context, cfg *config.Config, worldsRoot string, cl client.Client) (backup.WorldArchiver, error) {
 	switch cfg.Archive.Store {
 	case "tarLocal":
-		return &backup.TarLocal{
-			BackupRoot: cfg.Archive.LocalPath,
-			Resolve: func(pvc string) (string, error) {
-				return filepath.Join(worldsRoot, pvc), nil
-			},
-		}, nil
+		t := &backup.TarLocal{BackupRoot: cfg.Archive.LocalPath}
+		if cl != nil {
+			t.Resolve = resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot)
+		} else {
+			t.Resolve = func(pvc string) (string, error) {
+				return "", fmt.Errorf("resolve world PVC %s: this run has no worlds root (retention only)", pvc)
+			}
+		}
+		return t, nil
 	default:
 		return nil, fmt.Errorf("[archive] store %q is not implemented in this build (only tarLocal)", cfg.Archive.Store)
 	}
 }
 
-// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
-// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
-func parseSpanDuration(s string) (time.Duration, error) {
-	s = strings.TrimSpace(s)
-	switch {
-	case strings.HasSuffix(s, "mo"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
-		if err != nil {
-			return 0, err
+// resolveWorldDir maps a world PVC to its directory under worldsRoot, supporting
+// the two layouts a Felis host actually has:
+//
+//  1. <root>/<pvc> — the reaper's documented arrangement (worlds exposed by PVC
+//     name, e.g. via mounting each volume or a crafted storage class).
+//  2. <root>/<pv-name>_<namespace>_<pvc-name> — what a stock k3s install gets:
+//     local-path-provisioner stores every volume under its storage root as that
+//     exact directory name. Without this arm, retention on a default install could
+//     only ever fail to find a world (a no-op reaper, or worse an operator
+//     arranging paths by hand).
+//
+// The second path is derived EXACTLY from the live PVC's spec.volumeName, never
+// from a glob: a leftover directory of an old, deleted PV must never be mistaken
+// for the world the PVC currently binds, because the reaper archives the resolved
+// directory and then deletes that PVC — archiving stale bytes and deleting the
+// real world would be data loss. When neither path exists the first is returned,
+// so the archive walk fails loudly against the documented path.
+func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoot string) backup.PVCResolver {
+	return func(pvc string) (string, error) {
+		direct := filepath.Join(worldsRoot, pvc)
+		if _, err := os.Stat(direct); err == nil {
+			return direct, nil
 		}
-		return time.Duration(n) * 30 * 24 * time.Hour, nil
-	case strings.HasSuffix(s, "d"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
-		if err != nil {
-			return 0, err
+		var claim corev1.PersistentVolumeClaim
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: pvc}, &claim); err != nil {
+			return "", fmt.Errorf("resolve world PVC %s: %w", pvc, err)
 		}
-		return time.Duration(n) * 24 * time.Hour, nil
-	default:
-		return time.ParseDuration(s)
+		if pv := claim.Spec.VolumeName; pv != "" {
+			volDir := filepath.Join(worldsRoot, fmt.Sprintf("%s_%s_%s", pv, claim.Namespace, claim.Name))
+			if _, err := os.Stat(volDir); err == nil {
+				return volDir, nil
+			}
+		}
+		return direct, nil
 	}
 }
 
+// parseSpanDuration parses the human spans used in felis.toml's [archive] table
+// (config.ParseSpan).
+func parseSpanDuration(s string) (time.Duration, error) { return config.ParseSpan(s) }
+
 // parseByteSize parses a Kubernetes-style quantity ("200Gi", "10G") into bytes.
 // An empty string means unlimited (0).
 func parseByteSize(s string) (int64, error) {

cmd/felis/reaper_test.go

@@ -0,0 +1,230 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/reaper"
+)
+
+// TestReportReaperRunFailsTheJob: a run that could not process a server, or
+// could not remove an expired backup, exits 1 so the Job shows as failed.
+func TestReportReaperRunFailsTheJob(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		sum  reaper.Summary
+		want int
+	}{
+		{"clean", reaper.Summary{Evaluated: 3, WorldsReaped: 1, AwaitingOffsite: 1}, 0},
+		{"retirements", reaper.Summary{Evaluated: 5, WorldsReaped: 3, Released: 2, ServersDeleted: 1}, 0},
+		{"waiting for a stop", reaper.Summary{Evaluated: 3, AwaitingStop: 1}, 0},
+		{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
+		{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
+		{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},
+		{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},
+		{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},
+		{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},
+		{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},
+	} {
+		var out, errb bytes.Buffer
+		if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {
+			t.Errorf("%s: exit %d, want %d", tc.name, got, tc.want)
+		}
+		if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") ||
+			!strings.Contains(out.String(), fmt.Sprintf(" released=%d deleted=%d ", tc.sum.Released, tc.sum.ServersDeleted)) {
+			t.Errorf("%s: summary line = %q", tc.name, out.String())
+		}
+		if (tc.want == 1) != (errb.Len() > 0) {
+			t.Errorf("%s: stderr = %q", tc.name, errb.String())
+		}
+	}
+}
+
+// TestResolveWorldDir pins the two world layouts the reaper must find, and the
+// fail-closed miss. The stock local-path arm is derived from the live PVC's
+// volumeName — a name-based guess (glob) could tar a stale deleted PV's bytes and
+// then delete the current world, which is why it is read from the API instead.
+// TestReaperConfigManualKeys: the on-demand backup keys default to 30 days,
+// five per server and a ten-minute cooldown, accept overrides, and refuse
+// values that would keep nothing or throttle backwards.
+func TestReaperConfigManualKeys(t *testing.T) {
+	rc, err := reaperConfig(&config.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 30*reaper.Day || rc.ManualKeep != 5 || rc.ManualCooldown != 10*time.Minute {
+		t.Fatalf("defaults = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
+		ManualRetention: "7d", ManualKeep: 2, ManualCooldown: "0s"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 7*reaper.Day || rc.ManualKeep != 2 || rc.ManualCooldown != 0 {
+		t.Fatalf("overrides = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	for _, bad := range []config.ArchiveConfig{
+		{ManualRetention: "0d"},
+		{ManualRetention: "soon"},
+		{ManualKeep: -1},
+		{ManualCooldown: "-5m"},
+		{ManualCooldown: "often"},
+	} {
+		if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
+			t.Errorf("%+v was accepted", bad)
+		}
+	}
+}
+
+// TestReaperConfigScheduledKeys: the scheduled restore points default to one a
+// day, seven per server and the reaper's 90 days, accept overrides ("0s" turns
+// them off), and refuse values that would keep nothing or run backwards.
+func TestReaperConfigScheduledKeys(t *testing.T) {
+	rc, err := reaperConfig(&config.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ScheduledEvery != reaper.Day || rc.ScheduledKeep != 7 || rc.ScheduledRetention != 90*reaper.Day {
+		t.Fatalf("defaults = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
+		ScheduledEvery: "12h", ScheduledKeep: 3, ScheduledRetention: "14d"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ScheduledEvery != 12*time.Hour || rc.ScheduledKeep != 3 || rc.ScheduledRetention != 14*reaper.Day {
+		t.Fatalf("overrides = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{ScheduledEvery: "0s"}})
+	if err != nil || rc.ScheduledEvery != 0 {
+		t.Fatalf("scheduled_every 0s = %v, %v; want off", rc.ScheduledEvery, err)
+	}
+	for _, bad := range []config.ArchiveConfig{
+		{ScheduledEvery: "-1h"},
+		{ScheduledEvery: "daily"},
+		{ScheduledKeep: -1},
+		{ScheduledRetention: "0d"},
+		{ScheduledRetention: "forever"},
+	} {
+		if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
+			t.Errorf("%+v was accepted", bad)
+		}
+	}
+}
+
+func TestResolveWorldDir(t *testing.T) {
+	ctx := context.Background()
+	root := t.TempDir()
+
+	// Arrange a world under the documented <root>/<pvc> layout.
+	named := filepath.Join(root, "world-named-0")
+	if err := os.MkdirAll(named, 0o750); err != nil {
+		t.Fatal(err)
+	}
+	// Arrange a second world the way k3s local-path stores it.
+	pvDir := filepath.Join(root, "pvc-11111111-2222-3333-4444-555555555555_minecraft_world-live-0")
+	if err := os.MkdirAll(pvDir, 0o750); err != nil {
+		t.Fatal(err)
+	}
+
+	claim := &corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{Name: "world-live-0", Namespace: "minecraft"},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			VolumeName: "pvc-11111111-2222-3333-4444-555555555555",
+		},
+	}
+	cl := fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(claim).Build()
+	resolve := resolveWorldDir(ctx, cl, "minecraft", root)
+
+	t.Run("documented name layout wins", func(t *testing.T) {
+		got, err := resolve("world-named-0")
+		if err != nil || got != named {
+			t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, named)
+		}
+	})
+
+	t.Run("stock local-path layout resolves exactly", func(t *testing.T) {
+		got, err := resolve("world-live-0")
+		if err != nil || got != pvDir {
+			t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, pvDir)
+		}
+	})
+
+	t.Run("neither layout present falls back to the documented path", func(t *testing.T) {
+		// The claim exists but its directory does not: return the documented path so
+		// the archive walk fails there, and the reaper preserves the world.
+		missing := &corev1.PersistentVolumeClaim{
+			ObjectMeta: metav1.ObjectMeta{Name: "world-gone-0", Namespace: "minecraft"},
+			Spec:       corev1.PersistentVolumeClaimSpec{VolumeName: "pvc-99999999-0000-0000-0000-000000000000"},
+		}
+		cl := fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(missing).Build()
+		got, err := resolveWorldDir(ctx, cl, "minecraft", root)("world-gone-0")
+		if err != nil || got != filepath.Join(root, "world-gone-0") {
+			t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, filepath.Join(root, "world-gone-0"))
+		}
+	})
+
+	t.Run("unknown pvc is an error, not a guess", func(t *testing.T) {
+		_, err := resolve("world-unknown-0")
+		if err == nil || !strings.Contains(err.Error(), "resolve world PVC world-unknown-0") {
+			t.Fatalf("err = %v, want a resolve-world-PVC error", err)
+		}
+	})
+}
+
+// The pre-reap warner resolves the owner's VERIFIED email and hands the notice
+// to the mailer. Every failure (no verified address, relay refusal) returns an
+// error so the reaper retries on its next run instead of stamping a notice
+// nobody received.
+func TestMailWarner(t *testing.T) {
+	lookup := func(email string, err error) func(context.Context, string) (string, error) {
+		return func(context.Context, string) (string, error) { return email, err }
+	}
+
+	n := &captureNotifier{}
+	w := &mailWarner{lookupEmail: lookup("[email protected]", nil), notifier: n}
+	if err := w.Warn(context.Background(), "u1", "survival", "3d"); err != nil {
+		t.Fatalf("Warn: %v", err)
+	}
+	if n.email != "[email protected]" || !strings.Contains(n.subject, "survival") || !strings.Contains(n.subject, "3d") {
+		t.Fatalf("notice envelope = (%q, %q)", n.email, n.subject)
+	}
+	if !strings.Contains(n.body, "survival") || !strings.Contains(n.body, "3d") {
+		t.Fatalf("body missing server/remaining:\n%s", n.body)
+	}
+
+	w = &mailWarner{lookupEmail: lookup("", errors.New("owner u2 has no verified email")), notifier: n}
+	if err := w.Warn(context.Background(), "u2", "survival", "3d"); err == nil || !strings.Contains(err.Error(), "verified email") {
+		t.Fatalf("unverified owner = %v, want the lookup error surfaced", err)
+	}
+
+	w = &mailWarner{lookupEmail: lookup("[email protected]", nil), notifier: &captureNotifier{err: errors.New("relay down")}}
+	if err := w.Warn(context.Background(), "u1", "survival", "3d"); err == nil || !strings.Contains(err.Error(), "relay down") {
+		t.Fatalf("relay failure = %v, want it surfaced", err)
+	}
+}
+
+type captureNotifier struct {
+	email, subject, body string
+	err                  error
+}
+
+func (n *captureNotifier) SendNotice(_ context.Context, email, subject, body string) error {
+	if n.err != nil {
+		return n.err
+	}
+	n.email, n.subject, n.body = email, subject, body
+	return nil
+}

cmd/felis/registrygate.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/registrygate"
+)
+
+// cmdRegistryGate is the sidecar entrypoint in the registry pod: it owns the
+// registry port (and the loopback hostPort containerd pulls through), lets reads
+// through anonymously, and forwards writes to the loopback-only registry:2 only
+// for an authenticated principal allowed to write that repository. See
+// internal/registrygate for the policy.
+//
+// Tokens are files under --auth-dir, one per principal (platform, build, prune),
+// mounted from the registry-auth Secret. A missing file disables that principal:
+// writes fail closed while every pull keeps working, which is the right way round
+// for a registry the running workloads depend on.
+//
+// --maint-listen is the GC sidecar's read-only handshake (registrygate.MaintHandler).
+// It has no authentication, so it must name a loopback address; --maint-dir keeps
+// an open window across a gate restart.
+func cmdRegistryGate(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("registry-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	listen := fs.String("listen", ":5000", "address the gate serves the registry API on")
+	upstream := fs.String("upstream", "http://127.0.0.1:5001", "the loopback registry the gate forwards to")
+	authDir := fs.String("auth-dir", "/etc/felis-registry-auth", "directory holding one token file per principal")
+	maintListen := fs.String("maint-listen", "", "loopback address for the GC sidecar's read-only handshake (empty disables it)")
+	maintDir := fs.String("maint-dir", "", "directory that keeps an open read-only window across a gate restart")
+	quiet := fs.Duration("maint-quiet", registrygate.DefaultQuiet, "how long writes must be idle before a read-only window is granted")
+	dataDir := fs.String("data-dir", "", "the registry's storage root, mounted read-only, for the manifest index (empty disables it)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *maintListen != "" && !loopbackAddr(*maintListen) {
+		fmt.Fprintf(stderr, "felis registry-gate: --maint-listen %q must be a loopback address: the handshake has no authentication\n", *maintListen)
+		return 2
+	}
+	target, err := url.Parse(*upstream)
+	if err != nil || target.Scheme == "" || target.Host == "" {
+		fmt.Fprintf(stderr, "felis registry-gate: bad --upstream %q\n", *upstream)
+		return 2
+	}
+	log := slog.New(slog.NewTextHandler(stderr, nil))
+	tokens := map[string]string{}
+	for _, p := range registrygate.Principals {
+		b, err := os.ReadFile(filepath.Join(*authDir, p))
+		tok := strings.TrimSpace(string(b))
+		if err != nil || tok == "" {
+			log.Warn("registry principal disabled: no token", "principal", p, "dir", *authDir)
+			continue
+		}
+		tokens[p] = tok
+	}
+
+	gate := registrygate.New(target, tokens, log)
+	gate.SetQuiet(*quiet)
+	gate.DataDir = *dataDir
+	if *maintDir != "" {
+		if err := gate.SetMaintenanceState(registrygate.MaintStatePath(*maintDir)); err != nil {
+			// A corrupt file must not keep the registry from serving pulls.
+			log.Warn("ignoring the saved read-only window", "err", err)
+		}
+	}
+	srv := &http.Server{
+		Addr:              *listen,
+		Handler:           gate,
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	var maint *http.Server
+	if *maintListen != "" {
+		maint = &http.Server{Addr: *maintListen, Handler: gate.MaintHandler(), ReadHeaderTimeout: 10 * time.Second}
+		go func() {
+			if err := maint.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				log.Error("maintenance listener stopped; garbage collection cannot get a read-only window", "err", err)
+			}
+		}()
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	go func() {
+		<-ctx.Done()
+		shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(shutdown)
+		if maint != nil {
+			_ = maint.Shutdown(shutdown)
+		}
+	}()
+	log.Info("registry gate listening", "addr", *listen, "upstream", target.String(), "principals", len(tokens))
+	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintf(stderr, "felis registry-gate: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// loopbackAddr reports whether a host:port listen address binds loopback only.
+func loopbackAddr(addr string) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return false
+	}
+	if host == "localhost" {
+		return true
+	}
+	ip := net.ParseIP(host)
+	return ip != nil && ip.IsLoopback()
+}
+
+// cmdPushImage is the build Job's publish step. It runs after Kaniko built the
+// image into a tarball (--no-push) and Trivy passed that tarball, and it is the
+// only container of the build pod that holds the registry credential — the one
+// executing the untrusted Dockerfile never sees it.
+func cmdPushImage(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("push-image", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path, or with --image an OCI layout tar")
+	image := fs.String("image", "", "push the image this name (io.containerd.image.name) marks in the OCI layout tar --tar, e.g. a release's image bundle")
+	ref := fs.String("ref", "", "host/repository:tag to publish it as")
+	scheme := fs.String("scheme", "http", "registry scheme: http for the in-cluster registry, https otherwise")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *tarPath == "" || *ref == "" {
+		fmt.Fprintln(stderr, "felis push-image: --tar and --ref are required")
+		return 2
+	}
+	if *scheme != "http" && *scheme != "https" {
+		fmt.Fprintf(stderr, "felis push-image: bad --scheme %q\n", *scheme)
+		return 2
+	}
+	user := os.Getenv("FELIS_REGISTRY_USERNAME")
+	pass := os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if user == "" || pass == "" {
+		fmt.Fprintln(stderr, "felis push-image: FELIS_REGISTRY_USERNAME/FELIS_REGISTRY_PASSWORD are empty — the registry refuses anonymous writes")
+		return 2
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: *scheme, Username: user, Password: pass, Log: stderr}
+	var digest string
+	var err error
+	if *image != "" {
+		digest, err = p.PushLayout(ctx, *tarPath, *image, *ref)
+	} else {
+		digest, err = p.Push(ctx, *tarPath, *ref)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis push-image: %v\n", err)
+		return 1
+	}
+	fmt.Fprintln(stdout, digest)
+	return 0
+}

cmd/felis/restore_test.go

@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup
 		BackupRoot: backupRoot,
 		Resolve:    func(string) (string, error) { return srcDir, nil },
 	}
-	got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0")
+	got, err := ar.Archive(context.Background(), "survival", "world-survival-0")
 	if err != nil {
 		t.Fatalf("Archive: %v", err)
 	}
-	return string(got), backupRoot
+	return string(got.Ref), backupRoot
 }
 
 // The restore subcommand must extract the archived world into the target world

cmd/felis/rotatetoken.go

@@ -0,0 +1,854 @@
+package main
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/pbkdf2"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	neturl "net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/maintenance"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/operator"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/registrygate"
+
+	"github.com/BurntSushi/toml"
+	"github.com/jackc/pgx/v5"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// rotate-token replaces one credential the installer generated: an internal
+// caller's token (naming.CallerTokens), the registry's write tokens, the
+// Velocity forwarding secret or the database password. The new value goes into
+// the installer's record first, so that whatever fails later a re-run of the
+// installer puts it everywhere; then into the Secrets and files that carry it;
+// then whatever read the old value at start restarts. Without -yes it prints
+// what it would change and what that interrupts, and changes nothing.
+
+const (
+	defaultSecretsEnvPath = "/etc/felis/secrets.env"
+	defaultLinkPropsPath  = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
+	defaultForwardingPath = "/opt/felis/velocity/forwarding.secret"
+	velocityUnit          = "felis-velocity"
+	apiDeployment         = "felis-api"
+	registryDeployment    = "registry"
+
+	kindRegistry   = "registry"
+	kindForwarding = "forwarding"
+	kindDB         = "db"
+
+	// proxyReloadWait is how long the host proxy gets, once felis-api has rolled,
+	// to show it re-read its token: it reads the file again on its next call to
+	// felis-api, and it calls every 15 seconds.
+	proxyReloadWait = 60 * time.Second
+
+	// apiRestartNote is in every plan: felis-api runs as one replica replaced in
+	// place, so its restart is a short outage of everything that talks to it.
+	apiRestartNote = "felis-api restarts (a single replica): the panel, sign-in and the proxy's calls are unavailable for the few seconds that takes"
+)
+
+// installerTokenKeys names each caller's token in the installer's secrets.env
+// (deploy/bootstrap.sh load_or_make_secrets). A re-run of the installer applies
+// these values to the Secrets, so a rotation that skipped the file would be
+// undone by the next upgrade.
+var installerTokenKeys = map[string]string{
+	"velocity": "SERVICE_TOKEN",
+	"limbo":    "LIMBO_TOKEN",
+	"build":    "BUILD_TOKEN",
+	"ops":      "OPS_TOKEN",
+}
+
+// installerRegistryKeys names the registry principals' tokens in secrets.env,
+// in registrygate.Principals order.
+var installerRegistryKeys = map[string]string{
+	registrygate.PrincipalPlatform: "REGISTRY_PLATFORM_TOKEN",
+	registrygate.PrincipalBuild:    "REGISTRY_BUILD_TOKEN",
+	registrygate.PrincipalPrune:    "REGISTRY_PRUNE_TOKEN",
+}
+
+// installerForwardingKey and installerDBKey name the forwarding secret and the
+// database password in secrets.env.
+const (
+	installerForwardingKey = "FORWARDING_SECRET"
+	installerDBKey         = "DB_PASSWORD"
+)
+
+type tokenRotator struct {
+	cl          client.Client
+	controlNS   string
+	minecraftNS string
+	buildNS     string
+	// secretsEnv, linkProps and forwardingFile are the installer's record and the
+	// host proxy's felis-link.properties and forwarding.secret; a missing file is
+	// reported and skipped.
+	secretsEnv     string
+	linkProps      string
+	forwardingFile string
+	// hostTOML, podTOML and defaultTOML are the config copies that carry the
+	// database URL (defaultTOML only when it is a file of its own).
+	hostTOML    string
+	podTOML     string
+	defaultTOML string
+	newToken    func() (string, error)
+	// rollout restarts a control-namespace Deployment and waits for it.
+	rollout func(ctx context.Context, deployment string) error
+	// restartUnit restarts a systemd unit on this host.
+	restartUnit func(ctx context.Context, unit string) error
+	// proxyLog is what the host proxy has logged since a moment, as far as it
+	// can be read.
+	proxyLog func(ctx context.Context, since time.Time) string
+	// alterRole stores a password verifier for a role of the database that runs
+	// as deployment ("namespace/name"); verifyDB connects with a URL.
+	alterRole func(ctx context.Context, deployment, role, verifier string) error
+	verifyDB  func(ctx context.Context, url string) error
+	now       func() time.Time
+	// reloadWait bounds the wait for the proxy to re-read its token, polled
+	// every pollEvery.
+	reloadWait time.Duration
+	pollEvery  time.Duration
+	out        io.Writer
+}
+
+func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("rotate-token", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
+	linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
+	forwarding := fs.String("forwarding-secret", defaultForwardingPath, "the host proxy's forwarding secret file (forwarding only)")
+	yes := fs.Bool("yes", false, "rotate; without it the plan is printed and nothing changes")
+	fs.Usage = func() {
+		fmt.Fprintf(stderr, "Usage: felis rotate-token [-yes] [flags] <%s>\n\n", strings.Join(rotationKinds(), "|"))
+		fmt.Fprintln(stderr, "Replaces one generated credential: the installer's record, the Secrets and files that carry it, then what reads it.")
+		fmt.Fprintln(stderr, "Without -yes it prints what would change and what that interrupts.")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if fs.NArg() != 1 {
+		fs.Usage()
+		return 2
+	}
+	kind := fs.Arg(0)
+	if !knownRotation(kind) {
+		fmt.Fprintf(stderr, "felis rotate-token: unknown credential %q (one of %s)\n", kind, strings.Join(rotationKinds(), ", "))
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+kind+")")
+		return 1
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	buildNS := cfg.Registry.BuildNamespace
+	if buildNS == "" {
+		buildNS = platform.DefaultBuildNamespace
+	}
+	controlNS := platform.DefaultControlNamespace
+	r := tokenRotator{
+		cl:             cl,
+		controlNS:      controlNS,
+		minecraftNS:    cfg.K8s.Namespace,
+		buildNS:        buildNS,
+		secretsEnv:     *secretsEnv,
+		linkProps:      *linkProps,
+		forwardingFile: *forwarding,
+		hostTOML:       hostSetupConfigPath,
+		podTOML:        podSetupConfigPath,
+		defaultTOML:    defaultSetupConfigPath,
+		newToken:       randomToken,
+		rollout: func(ctx context.Context, deployment string) error {
+			if err := kubectl(ctx, "-n", controlNS, "rollout", "restart", "deployment/"+deployment); err != nil {
+				return err
+			}
+			return kubectl(ctx, "-n", controlNS, "rollout", "status", "deployment/"+deployment, "--timeout=180s")
+		},
+		restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
+		proxyLog:    journalSince,
+		alterRole:   alterRoleInPod,
+		verifyDB: func(ctx context.Context, url string) error {
+			ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+			defer cancel()
+			conn, err := pgx.Connect(ctx, url)
+			if err != nil {
+				return err
+			}
+			return conn.Close(ctx)
+		},
+		now:        time.Now,
+		reloadWait: proxyReloadWait,
+		pollEvery:  3 * time.Second,
+		out:        stdout,
+	}
+	if err := r.rotate(context.Background(), kind, *yes); err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+func callerNames() []string {
+	names := make([]string, 0, len(naming.CallerTokens))
+	for _, ct := range naming.CallerTokens {
+		names = append(names, ct.Caller)
+	}
+	return names
+}
+
+func callerToken(name string) (naming.CallerToken, bool) {
+	for _, ct := range naming.CallerTokens {
+		if ct.Caller == name {
+			return ct, true
+		}
+	}
+	return naming.CallerToken{}, false
+}
+
+// rotationKinds is every credential rotate-token replaces: the callers' tokens
+// and the installer's other generated secrets.
+func rotationKinds() []string {
+	return append(callerNames(), kindRegistry, kindForwarding, kindDB)
+}
+
+func knownRotation(kind string) bool {
+	for _, k := range rotationKinds() {
+		if k == kind {
+			return true
+		}
+	}
+	return false
+}
+
+// randomToken is 32 random bytes in hex, the shape the installer generates.
+func randomToken() (string, error) {
+	b := make([]byte, 32)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(b), nil
+}
+
+// tokenFingerprint is how the proxy names the token it reloaded in its log
+// (plugins/shared FileToken.fingerprint): the first twelve hex digits of its
+// SHA-256, enough to tell tokens apart and useless for finding one.
+func tokenFingerprint(token string) string {
+	sum := sha256.Sum256([]byte(token))
+	return hex.EncodeToString(sum[:])[:12]
+}
+
+func (r tokenRotator) rotate(ctx context.Context, kind string, apply bool) error {
+	switch kind {
+	case kindRegistry:
+		return r.rotateRegistry(ctx, apply)
+	case kindForwarding:
+		return r.rotateForwarding(ctx, apply)
+	case kindDB:
+		return r.rotateDB(ctx, apply)
+	}
+	ct, ok := callerToken(kind)
+	if !ok {
+		return fmt.Errorf("unknown credential %q (one of %s)", kind, strings.Join(rotationKinds(), ", "))
+	}
+	return r.rotateCaller(ctx, ct, apply)
+}
+
+// confirm ends the plan: without apply it says nothing changed and how to go
+// ahead, and reports false.
+func (r tokenRotator) confirm(kind string, apply bool) bool {
+	if !apply {
+		fmt.Fprintf(r.out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes %s\n", kind)
+		return false
+	}
+	fmt.Fprintln(r.out, "\nRotating:")
+	return true
+}
+
+// record writes new values into the installer's secrets.env. It comes first in
+// every rotation: from then on, whatever fails, a re-run of the installer puts
+// the new values everywhere.
+func (r tokenRotator) record(kv ...[2]string) error {
+	keys := make([]string, len(kv))
+	for i, p := range kv {
+		keys[i] = p[0]
+	}
+	switch err := setKeyValueLines(r.secretsEnv, "=", kv); {
+	case errors.Is(err, fs.ErrNotExist):
+		fmt.Fprintf(r.out, "  - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
+	case err != nil:
+		return fmt.Errorf("record the new value in %s: %w", r.secretsEnv, err)
+	default:
+		fmt.Fprintf(r.out, "  - %s: %s updated\n", r.secretsEnv, strings.Join(keys, ", "))
+	}
+	return nil
+}
+
+func (r tokenRotator) putSecret(ctx context.Context, ns, name string, data map[string][]byte) error {
+	if _, err := putSecretKeys(ctx, r.cl, ns, name, corev1.SecretTypeOpaque, data); err != nil {
+		return fmt.Errorf("write Secret %s/%s: %w", ns, name, err)
+	}
+	fmt.Fprintf(r.out, "  - Secret %s/%s: updated\n", ns, name)
+	return nil
+}
+
+func (r tokenRotator) rollAPI(ctx context.Context) error {
+	if err := r.rollout(ctx, apiDeployment); err != nil {
+		return fmt.Errorf("roll felis-api: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - felis-api: rolled out on the new value")
+	return nil
+}
+
+// withMinecraft is the control namespace plus the minecraft namespace when that
+// is a different one: where the Secrets game pods and Jobs mount are mirrored.
+func (r tokenRotator) withMinecraft() []string {
+	if r.minecraftNS == "" || r.minecraftNS == r.controlNS {
+		return []string{r.controlNS}
+	}
+	return []string{r.controlNS, r.minecraftNS}
+}
+
+func (r tokenRotator) rotateCaller(ctx context.Context, ct naming.CallerToken, apply bool) error {
+	namespaces := []string{r.controlNS}
+	replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
+	if replica != "" && replica != r.controlNS {
+		namespaces = append(namespaces, replica)
+	}
+	key := installerTokenKeys[ct.Caller]
+
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new internal token for the %s caller\n", ct.Caller, ct.Caller)
+	secrets := make([]string, len(namespaces))
+	for i, ns := range namespaces {
+		secrets[i] = "Secret " + ns + "/" + ct.Secret
+	}
+	writes := append([]string{r.secretsEnv + " (" + key + ")"}, secrets...)
+	hostProxy := ct.Caller == "velocity" && fileExists(r.linkProps)
+	if hostProxy {
+		writes = append(writes, r.linkProps+" (service-token)")
+	}
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
+	switch ct.Caller {
+	case "velocity":
+		if hostProxy {
+			fmt.Fprintf(r.out, "  - the proxy re-reads its token from the file and keeps its players; if it has not within %s of felis-api's restart, it is restarted, which disconnects everyone online\n", r.reloadWait)
+		} else {
+			fmt.Fprintf(r.out, "  - no proxy on this host (%s): set service-token in your proxy's felis-link.properties to the value in Secret %s/%s afterwards; it re-reads the file without a restart\n",
+				r.linkProps, r.controlNS, ct.Secret)
+		}
+	case "limbo":
+		fmt.Fprintln(r.out, "  - the login gate's pod restarts: a player signing in at that moment reconnects")
+	case "build":
+		fmt.Fprintln(r.out, "  - a build fetching its context at that moment fails and can be submitted again")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now presents the new token on its next run")
+	}
+	if !r.confirm(ct.Caller, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a token: %w", err)
+	}
+	if err := r.record([2]string{key, tok}); err != nil {
+		return err
+	}
+	for _, ns := range namespaces {
+		if err := r.putSecret(ctx, ns, ct.Secret, map[string][]byte{naming.ServiceTokenSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+
+	// The proxy's file changes before felis-api rolls, so the file never falls
+	// behind the Secret; the proxy's log is read from just before the write,
+	// since it may pick the new token up before the rollout ends.
+	since := r.now()
+	if hostProxy {
+		if err := setProxyToken(r.linkProps, tok); err != nil {
+			return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: service-token updated\n", r.linkProps)
+	}
+
+	if err := r.rollAPI(ctx); err != nil {
+		return err
+	}
+
+	switch ct.Caller {
+	case "velocity":
+		if !hostProxy {
+			break
+		}
+		if r.proxyReloaded(ctx, since, tokenFingerprint(tok)) {
+			fmt.Fprintf(r.out, "  - %s: took the new token from its properties; players stayed connected\n", velocityUnit)
+			break
+		}
+		if err := r.restartUnit(ctx, velocityUnit); err != nil {
+			return fmt.Errorf("restart %s: %w", velocityUnit, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: had not taken the new token within %s, restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit, r.reloadWait)
+	case "limbo":
+		// Only the operator's server pods carry its managed-by label; a backup or
+		// restore Job's pod carries the server label too, and app.kubernetes.io ones.
+		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), client.MatchingLabels{
+			v1alpha1.LabelServer:    naming.SystemLoginServer,
+			v1alpha1.LabelManagedBy: operator.ManagedByValue,
+		}); err != nil {
+			return fmt.Errorf("restart the login gate: %w", err)
+		}
+		fmt.Fprintln(r.out, "  - login gate: pod restarted to read the new token")
+	case "build":
+		fmt.Fprintln(r.out, "  - builds: the next build Job reads the new token")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now reads the new token on its next run")
+	}
+	return nil
+}
+
+// proxyReloaded waits up to reloadWait for the host proxy to log that it
+// reloaded the token with this fingerprint (plugins/shared FileToken).
+func (r tokenRotator) proxyReloaded(ctx context.Context, since time.Time, fingerprint string) bool {
+	want := "(fingerprint " + fingerprint + ")"
+	deadline := r.now().Add(r.reloadWait)
+	for {
+		if strings.Contains(r.proxyLog(ctx, since), want) {
+			return true
+		}
+		if !r.now().Before(deadline) {
+			return false
+		}
+		select {
+		case <-ctx.Done():
+			return false
+		case <-time.After(r.pollEvery):
+		}
+	}
+}
+
+// journalSince is the proxy unit's journal from the second since falls in. A
+// journal that cannot be read reads as one without the line, which ends in the
+// restart a rotation made before the proxy could reload.
+func journalSince(ctx context.Context, since time.Time) string {
+	out, _ := exec.CommandContext(ctx, "journalctl", "-u", velocityUnit, "--since", "@"+strconv.FormatInt(since.Unix(), 10),
+		"-o", "cat", "--no-pager", "-q").Output()
+	return string(out)
+}
+
+// setProxyToken writes the proxy's token into felis-link.properties and puts
+// the file's modification time back. The proxy re-reads its token by itself,
+// while `felis domain check` and `felis domain set` read a file newer than the
+// proxy's start as config it has not loaded (the installer's
+// install_if_changed keeps the time for the same reason).
+func setProxyToken(path, token string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	if err := setKeyValueLine(path, "service-token", "=", token); err != nil {
+		return err
+	}
+	return os.Chtimes(path, time.Time{}, info.ModTime())
+}
+
+func (r tokenRotator) rotateRegistry(ctx context.Context, apply bool) error {
+	keys := make([]string, len(registrygate.Principals))
+	for i, p := range registrygate.Principals {
+		keys[i] = installerRegistryKeys[p]
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: new write tokens for the image registry's principals (%s)\n", kindRegistry, strings.Join(registrygate.Principals, ", "))
+	fmt.Fprintf(r.out, "  - writes %s (%s), Secret %s/%s, Secret %s/%s\n", r.secretsEnv, strings.Join(keys, ", "),
+		r.controlNS, naming.RegistryAuthSecretName, r.buildNS, naming.RegistryPushSecretName)
+	fmt.Fprintln(r.out, "  - the registry restarts to load them: a build pushing its image at that moment fails and can be submitted again, and an image pull in that moment retries")
+	fmt.Fprintf(r.out, "  - %s (it presents the prune token)\n", apiRestartNote)
+	if !r.confirm(kindRegistry, apply) {
+		return nil
+	}
+
+	tokens := map[string][]byte{}
+	kv := make([][2]string, 0, len(registrygate.Principals))
+	for _, p := range registrygate.Principals {
+		tok, err := r.newToken()
+		if err != nil {
+			return fmt.Errorf("generate a token: %w", err)
+		}
+		tokens[p] = []byte(tok)
+		kv = append(kv, [2]string{installerRegistryKeys[p], tok})
+	}
+	if err := r.record(kv...); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.controlNS, naming.RegistryAuthSecretName, tokens); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.buildNS, naming.RegistryPushSecretName, map[string][]byte{
+		naming.RegistryPushUsernameKey: []byte(registrygate.PrincipalBuild),
+		naming.RegistryPushPasswordKey: tokens[registrygate.PrincipalBuild],
+	}); err != nil {
+		return err
+	}
+	if err := r.rollout(ctx, registryDeployment); err != nil {
+		return fmt.Errorf("restart the registry: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - registry: restarted on the new tokens")
+	return r.rollAPI(ctx)
+}
+
+func (r tokenRotator) rotateForwarding(ctx context.Context, apply bool) error {
+	restart, held, err := r.gamePods(ctx)
+	if err != nil {
+		return err
+	}
+	hostProxy := fileExists(r.forwardingFile)
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new Velocity forwarding secret, the key a server checks each player's identity with\n", kindForwarding)
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, "Secret "+ns+"/"+naming.ForwardingSecretName)
+	}
+	writes := append([]string{r.secretsEnv + " (" + installerForwardingKey + ")"}, secrets...)
+	if hostProxy {
+		writes = append(writes, r.forwardingFile)
+	}
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - every running server restarts to read it (%d now), saving its world on the way down, and the proxy restarts: everyone online is disconnected and can rejoin once their server is back\n", len(restart))
+	if len(held) > 0 {
+		fmt.Fprintf(r.out, "  - left running, because a backup, restore or file write holds its world: %s. Players cannot join it until it restarts: stop and start it from the panel once that finishes\n", strings.Join(held, ", "))
+	}
+	if !hostProxy {
+		fmt.Fprintf(r.out, "  - no proxy on this host (%s): put the value in Secret %s/%s into your proxy's forwarding secret file and restart it\n",
+			r.forwardingFile, r.controlNS, naming.ForwardingSecretName)
+	}
+	if !r.confirm(kindForwarding, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a secret: %w", err)
+	}
+	if err := r.record([2]string{installerForwardingKey, tok}); err != nil {
+		return err
+	}
+	if hostProxy {
+		info, err := os.Stat(r.forwardingFile)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(r.forwardingFile, info, []byte(tok)); err != nil {
+			return fmt.Errorf("write %s: %w", r.forwardingFile, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: updated\n", r.forwardingFile)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, naming.ForwardingSecretName, map[string][]byte{naming.ForwardingSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+	// The servers go first: their pods read the Secret as they are recreated,
+	// and a player who rejoins through the restarted proxy meets a server on the
+	// new secret, or one still starting.
+	for i := range restart {
+		p := &restart[i]
+		if err := r.cl.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) {
+			return fmt.Errorf("restart %s: %w", p.Labels[v1alpha1.LabelServer], err)
+		}
+	}
+	fmt.Fprintf(r.out, "  - servers: %d restarting on the new secret\n", len(restart))
+	if hostProxy {
+		if err := r.restartUnit(ctx, velocityUnit); err != nil {
+			return fmt.Errorf("restart %s: %w", velocityUnit, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: restarted on the new secret\n", velocityUnit)
+	}
+	return nil
+}
+
+// gamePods lists the running game server pods: those a rotation may restart,
+// and the servers left alone because a backup, restore or file write holds
+// their world (internal/maintenance), which a restart in the middle of would
+// break.
+func (r tokenRotator) gamePods(ctx context.Context) ([]corev1.Pod, []string, error) {
+	var pods corev1.PodList
+	// The operator's managed-by label is on its server pods alone (see rotateCaller).
+	if err := r.cl.List(ctx, &pods, client.InNamespace(r.minecraftNS), client.MatchingLabels{v1alpha1.LabelManagedBy: operator.ManagedByValue}); err != nil {
+		return nil, nil, fmt.Errorf("list the servers' pods: %w", err)
+	}
+	var jobs batchv1.JobList
+	if err := r.cl.List(ctx, &jobs, client.InNamespace(r.minecraftNS)); err != nil {
+		return nil, nil, fmt.Errorf("list the maintenance Jobs: %w", err)
+	}
+	var restart []corev1.Pod
+	var held []string
+	for _, p := range pods.Items {
+		if p.DeletionTimestamp != nil {
+			continue
+		}
+		server := p.Labels[v1alpha1.LabelServer]
+		var ms v1alpha1.MinecraftServer
+		if err := r.cl.Get(ctx, client.ObjectKey{Namespace: r.minecraftNS, Name: server}, &ms); client.IgnoreNotFound(err) != nil {
+			return nil, nil, fmt.Errorf("read server %s: %w", server, err)
+		}
+		if kind, ok := maintenance.Holder(server, ms.Annotations, jobs.Items, r.now()); ok {
+			held = append(held, server+" ("+kind+")")
+			continue
+		}
+		restart = append(restart, p)
+	}
+	return restart, held, nil
+}
+
+func (r tokenRotator) rotateDB(ctx context.Context, apply bool) error {
+	cfg, err := config.Load(r.hostTOML)
+	if err != nil {
+		return err
+	}
+	if cfg.Database.Deployment == "" {
+		return fmt.Errorf("[database] deployment is unset in %s, so the database is not the installer's felis-postgres: change the role's password where it runs, then in [database] url of each config copy", r.hostTOML)
+	}
+	u, err := neturl.Parse(cfg.Database.URL)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return fmt.Errorf("[database] url in %s names no role", r.hostTOML)
+	}
+	role := u.User.Username()
+	targets, err := tomlTargetsOf(r.hostTOML, r.podTOML, r.defaultTOML)
+	if err != nil {
+		return err
+	}
+	paths := make([]string, len(targets))
+	for i, t := range targets {
+		paths[i] = t.path
+	}
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, ns+"/"+platform.ConfigSecretName)
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new password for the database role %q\n", kindDB, role)
+	fmt.Fprintf(r.out, "  - writes %s (%s), the role in %s, [database] url in %s, Secret %s\n",
+		r.secretsEnv, installerDBKey, cfg.Database.Deployment, strings.Join(paths, " and "), strings.Join(secrets, " and "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
+	fmt.Fprintln(r.out, "  - a backup, restore or file Job that connects in the seconds between the password change and felis-api's restart fails and can be run again; the host's timers read the new config on their next run")
+	if !r.confirm(kindDB, apply) {
+		return nil
+	}
+
+	password, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a password: %w", err)
+	}
+	// Every config copy is edited in memory first, so one this cannot edit stops
+	// the rotation before the role's password changes.
+	edited := make([][]byte, len(targets))
+	var hostURL string
+	var podConfig []byte
+	for i, t := range targets {
+		raw, err := os.ReadFile(t.real)
+		if err != nil {
+			return err
+		}
+		var doc struct {
+			Database struct {
+				URL string `toml:"url"`
+			} `toml:"database"`
+		}
+		if _, err := toml.Decode(string(raw), &doc); err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		next, err := withPassword(doc.Database.URL, password)
+		if err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		if edited[i], err = editTOMLStrings(raw, []tomlStringEdit{{"database", "url", next}}); err != nil {
+			return fmt.Errorf("%s: %w; set the password in its [database] url by hand", t.path, err)
+		}
+		switch t.path {
+		case r.hostTOML:
+			hostURL = next
+		case r.podTOML:
+			podConfig = edited[i]
+		}
+	}
+	if podConfig == nil {
+		return fmt.Errorf("%s resolves to the same file as %s; the pods reach the database at another address and need a copy of their own", r.podTOML, r.hostTOML)
+	}
+
+	if err := r.record([2]string{installerDBKey, password}); err != nil {
+		return err
+	}
+	salt := make([]byte, 16)
+	if _, err := rand.Read(salt); err != nil {
+		return err
+	}
+	verifier, err := scramVerifier(password, salt, scramIterations)
+	if err != nil {
+		return err
+	}
+	if err := r.alterRole(ctx, cfg.Database.Deployment, role, verifier); err != nil {
+		return fmt.Errorf("set the role's password: %w", err)
+	}
+	if err := r.verifyDB(ctx, hostURL); err != nil {
+		return fmt.Errorf("the database does not accept the new password (%v); the config copies still hold the old one: run the installer again (sudo bash deploy/bootstrap.sh), which sets the password in %s everywhere", err, r.secretsEnv)
+	}
+	fmt.Fprintf(r.out, "  - role %s: password changed, and the database accepts it\n", role)
+	for i, t := range targets {
+		info, err := os.Stat(t.real)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(t.real, info, edited[i]); err != nil {
+			return fmt.Errorf("write %s: %w", t.path, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: [database] url updated\n", t.path)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, platform.ConfigSecretName, map[string][]byte{platform.ConfigSecretKey: podConfig}); err != nil {
+			return err
+		}
+	}
+	return r.rollAPI(ctx)
+}
+
+// withPassword is a database URL with its password replaced.
+func withPassword(raw, password string) (string, error) {
+	u, err := neturl.Parse(raw)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return "", errors.New("its [database] url names no role")
+	}
+	u.User = neturl.UserPassword(u.User.Username(), password)
+	return u.String(), nil
+}
+
+// scramIterations is PostgreSQL's default scram_iterations.
+const scramIterations = 4096
+
+// scramVerifier is the SCRAM-SHA-256 verifier PostgreSQL stores for a password
+// (RFC 5802 and RFC 7677, in the form libpq's PQencryptPasswordConn makes).
+// ALTER ROLE stores a verifier as it is given, so the password itself never
+// reaches the server, where a failing statement is logged with its text.
+func scramVerifier(password string, salt []byte, iterations int) (string, error) {
+	salted, err := pbkdf2.Key(sha256.New, password, salt, iterations, sha256.Size)
+	if err != nil {
+		return "", err
+	}
+	mac := func(msg string) []byte {
+		h := hmac.New(sha256.New, salted)
+		h.Write([]byte(msg))
+		return h.Sum(nil)
+	}
+	stored := sha256.Sum256(mac("Client Key"))
+	b64 := base64.StdEncoding.EncodeToString
+	return fmt.Sprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, b64(salt), b64(stored[:]), b64(mac("Server Key"))), nil
+}
+
+// alterRoleInPod runs ALTER ROLE as the superuser inside the database's
+// container, over its socket, with the statement on stdin.
+func alterRoleInPod(ctx context.Context, deployment, role, verifier string) error {
+	ns, name, _ := strings.Cut(deployment, "/")
+	return kubectlWithInput(ctx, []byte(alterRoleSQL(role, verifier)), "-n", ns, "exec", "-i", "deploy/"+name, "-c", platform.PostgresContainer, "--",
+		"psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres")
+}
+
+// alterRoleSQL sets role's password to a SCRAM verifier, which holds no quote.
+func alterRoleSQL(role, verifier string) string {
+	return "ALTER ROLE \"" + strings.ReplaceAll(role, `"`, `""`) + "\" WITH PASSWORD '" + verifier + "';\n"
+}
+
+// setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file
+// (secrets.env, a .properties file), appending one when the key is absent. The
+// file is replaced atomically and keeps its mode and owner: felis-link.properties
+// is root:felis-velocity 0640, and the proxy must still be able to read it.
+func setKeyValueLine(path, key, sep, value string) error {
+	return setKeyValueLines(path, sep, [][2]string{{key, value}})
+}
+
+// setKeyValueLines is setKeyValueLine for several keys in one rewrite.
+func setKeyValueLines(path, sep string, kv [][2]string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
+	for _, p := range kv {
+		key, value := p[0], p[1]
+		found := false
+		for i, ln := range lines {
+			k, _, ok := strings.Cut(ln, sep)
+			if ok && strings.TrimSpace(k) == key {
+				lines[i] = key + sep + value
+				found = true
+			}
+		}
+		if !found {
+			lines = append(lines, key+sep+value)
+		}
+	}
+	return replaceFileKeepingMode(path, info, []byte(strings.Join(lines, "\n")+"\n"))
+}
+
+// replaceFileKeepingMode atomically replaces path with data, keeping the mode and
+// owner info describes: these files are read by other users (the proxy's) and
+// some hold credentials, so a rewrite must not widen or narrow who can read them.
+func replaceFileKeepingMode(path string, info os.FileInfo, data []byte) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	if err := tmp.Chmod(info.Mode().Perm()); err != nil {
+		tmp.Close()
+		return err
+	}
+	if st, ok := info.Sys().(*syscall.Stat_t); ok {
+		if err := tmp.Chown(int(st.Uid), int(st.Gid)); err != nil {
+			tmp.Close()
+			return err
+		}
+	}
+	if _, err := tmp.Write(data); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
+}

cmd/felis/rotatetoken_test.go

@@ -0,0 +1,883 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/maintenance"
+
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+type rotationRig struct {
+	r      tokenRotator
+	cl     client.Client
+	out    *bytes.Buffer
+	events []string
+	dir    string
+	clock  time.Time
+}
+
+func tokenSecret(ns, name, val string) *corev1.Secret {
+	return keySecret(ns, name, "token", val)
+}
+
+func keySecret(ns, name, key, val string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name},
+		Data:       map[string][]byte{key: []byte(val)},
+	}
+}
+
+// serverPod is a game server's pod as the operator labels it.
+func serverPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{
+			"felis.lolicon.best/server":     server,
+			"felis.lolicon.best/managed-by": "felis-operator",
+			"felis.lolicon.best/component":  "server",
+		}}}
+}
+
+// backupPod is a backup Job's pod as internal/backupjob labels it: it carries
+// the server's label too, and no rotation may take it for the server's own.
+func backupPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{
+			"felis.lolicon.best/server":    server,
+			"app.kubernetes.io/managed-by": "felis-backup",
+			"app.kubernetes.io/component":  "world-backup",
+		}}}
+}
+
+func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
+	t.Helper()
+	rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir(), clock: time.Unix(1_800_000_000, 0)}
+	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
+	rig.r = tokenRotator{
+		cl:             rig.cl,
+		controlNS:      "felis",
+		minecraftNS:    "minecraft",
+		buildNS:        "felis-build",
+		secretsEnv:     filepath.Join(rig.dir, "secrets.env"),
+		linkProps:      filepath.Join(rig.dir, "felis-link.properties"),
+		forwardingFile: filepath.Join(rig.dir, "forwarding.secret"),
+		hostTOML:       filepath.Join(rig.dir, "felis.host.toml"),
+		podTOML:        filepath.Join(rig.dir, "felis.pod.toml"),
+		defaultTOML:    filepath.Join(rig.dir, "felis.toml"),
+		newToken:       func() (string, error) { return "NEWTOKEN", nil },
+		rollout: func(_ context.Context, deployment string) error {
+			rig.events = append(rig.events, "roll "+deployment)
+			return nil
+		},
+		restartUnit: func(_ context.Context, unit string) error {
+			rig.events = append(rig.events, "restart "+unit)
+			return nil
+		},
+		proxyLog: func(context.Context, time.Time) string { return "" },
+		alterRole: func(context.Context, string, string, string) error {
+			rig.events = append(rig.events, "alter-role")
+			return nil
+		},
+		verifyDB: func(context.Context, string) error {
+			rig.events = append(rig.events, "verify-db")
+			return nil
+		},
+		// Every look at the clock moves it a second on, so a wait measured with it
+		// ends after a known number of looks.
+		now: func() time.Time {
+			rig.clock = rig.clock.Add(time.Second)
+			return rig.clock
+		},
+		reloadWait: 5 * time.Second,
+		out:        rig.out,
+	}
+	return rig
+}
+
+func (rig *rotationRig) secretKey(t *testing.T, ns, name, key string) string {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		return "<missing>"
+	}
+	return string(s.Data[key])
+}
+
+func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
+	t.Helper()
+	return rig.secretKey(t, ns, name, "token")
+}
+
+func (rig *rotationRig) pods(t *testing.T) string {
+	t.Helper()
+	var pods corev1.PodList
+	if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
+		t.Fatal(err)
+	}
+	names := make([]string, len(pods.Items))
+	for i, p := range pods.Items {
+		names[i] = p.Name
+	}
+	return strings.Join(names, ",")
+}
+
+func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(body), mode); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(path, mode); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func readTestFile(t *testing.T, path string) string {
+	t.Helper()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(raw)
+}
+
+func TestRotateLimboToken(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-service-token", "proxy"),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+		backupPod("minecraft", "login-backup-x", "login"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600)
+
+	// felis-api must roll only after both copies hold the new value, and the login
+	// pod must still be there then: restarting it earlier would have it present
+	// the new token to an api that does not know it yet.
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the control Secret held %q", got)
+		}
+		if got := rig.secret(t, "minecraft", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the minecraft replica held %q", got)
+		}
+		var pod corev1.Pod
+		if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+			t.Errorf("the login pod was restarted before the api rolled")
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), "limbo", true); err != nil {
+		t.Fatal(err)
+	}
+
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 {
+		t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm())
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" {
+		t.Errorf("the proxy's token changed to %q", got)
+	}
+	// The login pod restarted; a user server and the backup Job's pod, which
+	// carries the login server's label too, are untouched.
+	if got := rig.pods(t); got != "login-backup-x,survival-0" {
+		t.Errorf("pods left = %s, want login-backup-x,survival-0", got)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new token was printed: %s", rig.out.String())
+	}
+}
+
+const testLinkProps = "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n"
+
+func reloadLine(token string) string {
+	return "[12:00:00 INFO] [felis-link]: Felis: service-token reloaded from /x/felis-link.properties (fingerprint " + tokenFingerprint(token) + ")\n"
+}
+
+// The host proxy re-reads its token: the rotation waits for it to say so and
+// leaves it running.
+func TestRotateVelocityTokenReloadsTheHostProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	if err := os.Chtimes(rig.r.linkProps, written, written); err != nil {
+		t.Fatal(err)
+	}
+
+	// The proxy logs its reload on its next call to felis-api, which may be
+	// while the api rolls: the log must be read from before that.
+	var reloadedAt time.Time
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := readTestFile(t, rig.r.linkProps); !strings.Contains(got, "service-token=NEWTOKEN\n") {
+			t.Errorf("api rolled before the proxy's file held the new token: %q", got)
+		}
+		reloadedAt = rig.clock
+		return nil
+	}
+	looks := 0
+	rig.r.proxyLog = func(_ context.Context, since time.Time) string {
+		looks++
+		log := reloadLine("old") // an earlier rotation's
+		if looks >= 3 && !since.After(reloadedAt) {
+			log += reloadLine("NEWTOKEN")
+		}
+		return log
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.linkProps); got != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
+		t.Errorf("felis-link.properties = %q", got)
+	}
+	info, _ := os.Stat(rig.r.linkProps)
+	if info.Mode().Perm() != 0o640 {
+		t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm())
+	}
+	if !info.ModTime().Equal(written) {
+		t.Errorf("properties mtime = %v, want it kept at %v (felis domain check would read the proxy as stale)", info.ModTime(), written)
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	// The proxy's token has no replica: it must not appear in a workload namespace.
+	if got := rig.secret(t, "minecraft", "felis-service-token"); got != "<missing>" {
+		t.Errorf("rotation copied the proxy token into minecraft (%q)", got)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want the api roll and no proxy restart", rig.events)
+	}
+	if looks != 3 {
+		t.Errorf("the log was read %d times, want 3 (until the line appeared)", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: took the new token from its properties; players stayed connected") {
+		t.Errorf("output does not say the players stayed: %s", out)
+	}
+}
+
+// A proxy that never logs the new fingerprint (an older plugin, a stuck proxy)
+// is restarted once the wait is over.
+func TestRotateVelocityTokenRestartsAProxyThatDoesNotReload(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	looks := 0
+	rig.r.proxyLog = func(context.Context, time.Time) string {
+		looks++
+		return reloadLine("old")
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api,restart felis-velocity" {
+		t.Errorf("events = %v, want the api roll then the proxy restart", rig.events)
+	}
+	// reloadWait is 5s and each look moves the clock a second.
+	if looks != 5 {
+		t.Errorf("the log was read %d times, want 5", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: had not taken the new token within 5s, restarted") {
+		t.Errorf("output does not explain the restart: %s", out)
+	}
+}
+
+// The proxy and the Java plugin name a token by the same fingerprint
+// (plugins/shared LinkConfigLoaderTest fileTokenFollowsTheFile).
+func TestTokenFingerprintMatchesThePlugin(t *testing.T) {
+	if got := tokenFingerprint("new-token"); got != "348e9df2a42b" {
+		t.Errorf("tokenFingerprint(new-token) = %q, want 348e9df2a42b", got)
+	}
+}
+
+// An external proxy has no felis-link.properties here: the Secret still rotates,
+// nothing is restarted on this host, and the output says where the value is
+// without printing it.
+func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want no proxy restart", rig.events)
+	}
+	out := rig.out.String()
+	if !strings.Contains(out, "felis/felis-service-token") || strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("output should point at the Secret without the value: %s", out)
+	}
+}
+
+func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old"))
+	// An install from before per-caller tokens has no BUILD_TOKEN line yet.
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600)
+	if err := rig.r.rotate(context.Background(), "build", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("build replica = %q, want NEWTOKEN (created when absent)", got)
+	}
+	if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+}
+
+// A failed api rollout stops the rotation before the caller restarts: the login
+// gate keeps running on the old value the old api pods still accept.
+func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		serverPod("minecraft", "login-0", "login"),
+	)
+	rig.r.rollout = func(context.Context, string) error { return errors.New("rollout timed out") }
+	err := rig.r.rotate(context.Background(), "limbo", true)
+	if err == nil || !strings.Contains(err.Error(), "rollout timed out") {
+		t.Fatalf("err = %v, want the rollout failure", err)
+	}
+	if got := rig.pods(t); got != "login-0" {
+		t.Errorf("pods left = %s: the login pod was restarted although the api never rolled", got)
+	}
+}
+
+func TestRotateRefusesAnUnknownCredential(t *testing.T) {
+	rig := newRotationRig(t)
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600)
+	if err := rig.r.rotate(context.Background(), "admin", true); err == nil {
+		t.Fatal("rotated an unknown credential")
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\n" {
+		t.Errorf("secrets.env = %q, want it untouched", got)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want nothing touched", rig.events)
+	}
+}
+
+const (
+	testHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	testPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+// installedRig is a host as the installer leaves it, with every credential in
+// place, for the plan test.
+func installedRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-service-token", "old"),
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-build-token", "old"),
+		tokenSecret("felis-build", "felis-build-token", "old"),
+		tokenSecret("felis", "felis-ops-token", "old"),
+		keySecret("felis", "felis-registry-auth", "platform", "old"),
+		keySecret("felis-build", "felis-registry-push", "password", "old"),
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	return rig
+}
+
+// Without -yes every rotation prints its plan and changes nothing.
+func TestRotatePlanChangesNothing(t *testing.T) {
+	for _, kind := range rotationKinds() {
+		t.Run(kind, func(t *testing.T) {
+			rig := installedRig(t)
+			files := map[string]string{}
+			for _, p := range []string{rig.r.secretsEnv, rig.r.linkProps, rig.r.forwardingFile, rig.r.hostTOML, rig.r.podTOML} {
+				files[p] = readTestFile(t, p)
+			}
+			rig.r.newToken = func() (string, error) {
+				t.Error("a plan generated a token")
+				return "NEWTOKEN", nil
+			}
+			if err := rig.r.rotate(context.Background(), kind, false); err != nil {
+				t.Fatal(err)
+			}
+			for p, before := range files {
+				if got := readTestFile(t, p); got != before {
+					t.Errorf("%s changed to %q", filepath.Base(p), got)
+				}
+			}
+			for _, s := range [][3]string{
+				{"felis", "felis-service-token", "token"}, {"felis", "felis-limbo-token", "token"},
+				{"felis-build", "felis-build-token", "token"}, {"felis", "felis-ops-token", "token"},
+				{"felis", "felis-registry-auth", "platform"}, {"felis-build", "felis-registry-push", "password"},
+				{"minecraft", "felis-forwarding-secret", "secret"},
+			} {
+				if got := rig.secretKey(t, s[0], s[1], s[2]); got != "old" {
+					t.Errorf("Secret %s/%s changed to %q", s[0], s[1], got)
+				}
+			}
+			if got := rig.secretKey(t, "minecraft", "felis-config", "felis.toml"); got != testPodTOML {
+				t.Errorf("felis-config changed to %q", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want none", rig.events)
+			}
+			if got := rig.pods(t); got != "login-0,survival-0" {
+				t.Errorf("pods left = %s, want all", got)
+			}
+			out := rig.out.String()
+			if !strings.HasSuffix(out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes "+kind+"\n") {
+				t.Errorf("the plan does not end with how to go ahead: %s", out)
+			}
+			// Each plan names the interruption it causes.
+			want := apiRestartNote
+			if kind == kindForwarding {
+				want = "everyone online is disconnected"
+			}
+			if !strings.Contains(out, want) {
+				t.Errorf("the plan does not say %q: %s", want, out)
+			}
+		})
+	}
+}
+
+func TestRotateRegistryTokens(t *testing.T) {
+	rig := newRotationRig(t,
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: "felis-registry-auth"},
+			Data: map[string][]byte{"platform": []byte("a"), "build": []byte("b"), "prune": []byte("c")}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis-build", Name: "felis-registry-push"},
+			Data: map[string][]byte{"username": []byte("build"), "password": []byte("b")}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=a\nREGISTRY_BUILD_TOKEN=b\nREGISTRY_PRUNE_TOKEN=c\n", 0o600)
+	n := 0
+	rig.r.newToken = func() (string, error) {
+		n++
+		return fmt.Sprintf("NEWTOKEN%d", n), nil
+	}
+	// The registry reads its tokens as it starts: it restarts after the Secret
+	// holds them, and felis-api (the prune token) after that.
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", "prune"); got != "NEWTOKEN3" {
+			t.Errorf("%s rolled while the prune token was %q", deployment, got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindRegistry, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=NEWTOKEN1\nREGISTRY_BUILD_TOKEN=NEWTOKEN2\nREGISTRY_PRUNE_TOKEN=NEWTOKEN3\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	for key, want := range map[string]string{"platform": "NEWTOKEN1", "build": "NEWTOKEN2", "prune": "NEWTOKEN3"} {
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", key); got != want {
+			t.Errorf("felis-registry-auth %s = %q, want %s", key, got, want)
+		}
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "password"); got != "NEWTOKEN2" {
+		t.Errorf("the build Jobs' push password = %q, want the build token", got)
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "username"); got != "build" {
+		t.Errorf("the build Jobs' push username = %q, want build", got)
+	}
+	if strings.Join(rig.events, ",") != "roll registry,roll felis-api" {
+		t.Errorf("events = %v, want the registry then felis-api", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("a new token was printed: %s", rig.out.String())
+	}
+}
+
+func TestRotateForwardingSecret(t *testing.T) {
+	lock := time.Unix(1_800_000_000, 0)
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		serverPod("minecraft", "survival-0", "survival"),
+		serverPod("minecraft", "lobby-0", "lobby"),
+		// creative is being backed up: a running Job holds its world.
+		serverPod("minecraft", "creative-0", "creative"),
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "creative-backup",
+			Labels: map[string]string{maintenance.LabelServer: "creative", maintenance.LabelManagedBy: "felis-backup"}}},
+		// survival's last backup is over; its finished pod stays until the Job's TTL.
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival-backup",
+			Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: "felis-backup"}},
+			Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}},
+		backupPod("minecraft", "survival-backup-x", "survival"),
+		// A pod already on its way out is neither counted nor deleted again.
+		func() *corev1.Pod {
+			p := serverPod("minecraft", "lobby-old", "lobby")
+			p.DeletionTimestamp = &metav1.Time{Time: lock}
+			p.Finalizers = []string{"felis.lolicon.best/test"}
+			return p
+		}(),
+		// skyblock's restore has just been admitted, its Job not created yet.
+		serverPod("minecraft", "skyblock-0", "skyblock"),
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "skyblock",
+			Annotations: map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lock)}}},
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival"}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nFORWARDING_SECRET=old\n", 0o600)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	// The proxy restarts after the servers were told to: a player who rejoins
+	// must not meet a server still on the old secret.
+	rig.r.restartUnit = func(_ context.Context, unit string) error {
+		rig.events = append(rig.events, "restart "+unit)
+		if got := rig.pods(t); strings.Contains(got, "survival-0") {
+			t.Errorf("the proxy restarted before the servers (pods %s)", got)
+		}
+		if got := readTestFile(t, rig.r.forwardingFile); got != "NEWTOKEN" {
+			t.Errorf("the proxy restarted on forwarding.secret %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nFORWARDING_SECRET=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if info, _ := os.Stat(rig.r.forwardingFile); info.Mode().Perm() != 0o640 {
+		t.Errorf("forwarding.secret mode = %v, want 0640", info.Mode().Perm())
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+			t.Errorf("Secret %s/felis-forwarding-secret = %q, want NEWTOKEN", ns, got)
+		}
+	}
+	if got := rig.pods(t); got != "creative-0,lobby-old,skyblock-0,survival-backup-x" {
+		t.Errorf("pods left = %s, want the held servers, the terminating pod and the backup's pod", got)
+	}
+	if strings.Join(rig.events, ",") != "restart felis-velocity" {
+		t.Errorf("events = %v, want only the proxy restart (felis-api does not hold this secret)", rig.events)
+	}
+	out := rig.out.String()
+	for _, want := range []string{
+		"every running server restarts to read it (2 now)",
+		"left running, because a backup, restore or file write holds its world: creative (backup), skyblock (restore).",
+		"  - servers: 2 restarting on the new secret\n",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output lacks %q: %s", want, out)
+		}
+	}
+	if strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("the new secret was printed: %s", out)
+	}
+}
+
+func TestRotateForwardingSecretForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, keySecret("felis", "felis-forwarding-secret", "secret", "old"))
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secretKey(t, "felis", "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want no proxy restart on this host", rig.events)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "put the value in Secret felis/felis-forwarding-secret into your proxy's forwarding secret file") {
+		t.Errorf("output does not say where the value is: %s", out)
+	}
+}
+
+// dbRig is an installed host for the database rotation: felis.toml links to the
+// host copy, as the installer makes it.
+func dbRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n", 0o600)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	if err := os.Symlink("felis.host.toml", rig.r.defaultTOML); err != nil {
+		t.Fatal(err)
+	}
+	return rig
+}
+
+const (
+	wantHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	wantPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+func TestRotateDatabasePassword(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.alterRole = func(_ context.Context, deployment, role, verifier string) error {
+		rig.events = append(rig.events, "alter-role")
+		if deployment != "felis/felis-postgres" || role != "felis" {
+			t.Errorf("alterRole(%q, %q), want felis/felis-postgres and felis", deployment, role)
+		}
+		if !strings.Contains(readTestFile(t, rig.r.secretsEnv), "DB_PASSWORD=NEWTOKEN\n") {
+			t.Error("the role changed before secrets.env recorded the new password")
+		}
+		// The verifier is the new password's, under the salt it carries.
+		m := regexp.MustCompile(`^SCRAM-SHA-256\$4096:([^$]+)\$`).FindStringSubmatch(verifier)
+		if m == nil {
+			t.Fatalf("verifier %q is not a SCRAM-SHA-256 verifier", verifier)
+		}
+		salt, err := base64.StdEncoding.DecodeString(m[1])
+		if err != nil || len(salt) != 16 {
+			t.Fatalf("verifier salt %q: %v", m[1], err)
+		}
+		if want, _ := scramVerifier("NEWTOKEN", salt, 4096); verifier != want {
+			t.Errorf("verifier = %q, want %q", verifier, want)
+		}
+		return nil
+	}
+	// The configs change only once the database accepts the new password.
+	rig.r.verifyDB = func(_ context.Context, url string) error {
+		rig.events = append(rig.events, "verify-db")
+		if url != "postgres://felis:[email protected]:30432/felis?sslmode=disable" {
+			t.Errorf("verified with %q, want the host URL with the new password", url)
+		}
+		if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+			t.Errorf("the host config changed before the database accepted the password: %q", got)
+		}
+		return nil
+	}
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("felis-api rolled on felis-config %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindDB, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=NEWTOKEN\nSERVICE_TOKEN=s\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != wantHostTOML {
+		t.Errorf("host config = %q", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != wantPodTOML {
+		t.Errorf("pod config = %q", got)
+	}
+	if info, err := os.Lstat(rig.r.defaultTOML); err != nil || info.Mode()&os.ModeSymlink == 0 {
+		t.Errorf("felis.toml is no longer the link to the host copy (%v)", err)
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("Secret %s/felis-config = %q", ns, got)
+		}
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db,roll felis-api" {
+		t.Errorf("events = %v", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new password was printed: %s", rig.out.String())
+	}
+}
+
+// A password the database does not accept leaves the configs on the old one
+// and felis-api running: the installer's record, already new, is the way back.
+func TestRotateDatabasePasswordStopsWhenTheDatabaseRefuses(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.verifyDB = func(context.Context, string) error {
+		rig.events = append(rig.events, "verify-db")
+		return errors.New("password authentication failed")
+	}
+	err := rig.r.rotate(context.Background(), kindDB, true)
+	if err == nil || !strings.Contains(err.Error(), "password authentication failed") || !strings.Contains(err.Error(), "run the installer again") {
+		t.Fatalf("err = %v, want the refusal and the way back", err)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+		t.Errorf("host config = %q, want it unchanged", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != testPodTOML {
+		t.Errorf("pod config = %q, want it unchanged", got)
+	}
+	if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != testPodTOML {
+		t.Errorf("felis-config = %q, want it unchanged", got)
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db" {
+		t.Errorf("events = %v, want no api roll", rig.events)
+	}
+}
+
+// Everything that can refuse does so before the installer's record or the
+// role change; what the host config alone shows is refused by the plan too.
+func TestRotateDatabasePasswordRefusesEarly(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		apply bool
+		setup func(t *testing.T, rig *rotationRig)
+		want  string
+	}{
+		{"a database the installer does not run", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "deployment = \"felis/felis-postgres\"\n", "", 1), 0o600)
+		}, "[database] deployment is unset"},
+		{"a database url without a role", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "felis:oldpw@", "", 1), 0o600)
+		}, "names no role"},
+		{"a config copy the line editor cannot edit", true, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.podTOML, "[database]\nurl = \"\"\"\npostgres://felis:[email protected]:5432/felis\"\"\"\n", 0o600)
+		}, "set the password in its [database] url by hand"},
+		{"a pod copy that is the host copy", true, func(t *testing.T, rig *rotationRig) {
+			if err := os.Remove(rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.Symlink("felis.host.toml", rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+		}, "resolves to the same file as"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			rig := dbRig(t)
+			tc.setup(t, rig)
+			err := rig.r.rotate(context.Background(), kindDB, tc.apply)
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err = %v, want %q", err, tc.want)
+			}
+			if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n" {
+				t.Errorf("secrets.env = %q, want it untouched", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want nothing done", rig.events)
+			}
+		})
+	}
+}
+
+// The RFC 7677 example (user "user", password "pencil"), with the stored and
+// server keys computed by openssl's PBKDF2 and HMAC rather than this code.
+func TestScramVerifier(t *testing.T) {
+	salt, _ := base64.StdEncoding.DecodeString("W22ZaJ0SNY7soEsUEjb6gQ==")
+	got, err := scramVerifier("pencil", salt, 4096)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := "SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==$WG5d8oPm3OtcPnkdi4Uo7BkeZkBFzpcXkuLmtbsT4qY=:wfPLwcE6nTWhTAmQ7tl2KeoiWGPlZqQxSrmfPwDl2dU="; got != want {
+		t.Errorf("scramVerifier = %q\nwant            %q", got, want)
+	}
+}
+
+func TestAlterRoleSQL(t *testing.T) {
+	if got := alterRoleSQL(`fe"lis`, "SCRAM-SHA-256$4096:c2FsdA==$a:b"); got != "ALTER ROLE \"fe\"\"lis\" WITH PASSWORD 'SCRAM-SHA-256$4096:c2FsdA==$a:b';\n" {
+		t.Errorf("alterRoleSQL = %q", got)
+	}
+}
+
+// installerSecretKeys is every secrets.env key a rotation writes.
+func installerSecretKeys() map[string]bool {
+	keys := map[string]bool{installerForwardingKey: true, installerDBKey: true}
+	for _, k := range installerTokenKeys {
+		keys[k] = true
+	}
+	for _, k := range installerRegistryKeys {
+		keys[k] = true
+	}
+	return keys
+}
+
+// The installer and rotate-token must agree on where each caller's token lives:
+// rotate-token writes installerTokenKeys into secrets.env, and the installer
+// applies those same keys to the Secrets on its next run. A key the installer
+// does not read would be silently reverted by the next upgrade.
+func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	for caller, key := range installerTokenKeys {
+		ct, ok := callerToken(caller)
+		if !ok {
+			t.Fatalf("installerTokenKeys names unknown caller %q", caller)
+		}
+		apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`)
+		if !apply.MatchString(script) {
+			t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key)
+		}
+	}
+	// The replicas the installer applies straight into the workload namespaces, so an
+	// upgrade has them before the new operator and build Jobs reference them.
+	for _, line := range []string{
+		`apply_literal_secret "$MINECRAFT_NS" felis-limbo-token token "$LIMBO_TOKEN"`,
+		`apply_literal_secret "$BUILD_NS" felis-build-token token "$BUILD_TOKEN"`,
+		`kube -n "$MINECRAFT_NS" delete secret felis-service-token --ignore-not-found`,
+		`kube -n "$BUILD_NS" delete secret felis-service-token --ignore-not-found`,
+	} {
+		if !strings.Contains(script, line) {
+			t.Errorf("bootstrap.sh lacks %s", line)
+		}
+	}
+	for _, ns := range []string{"MINECRAFT_NS", "BUILD_NS"} {
+		if strings.Contains(script, `apply_literal_secret "$`+ns+`" felis-service-token`) {
+			t.Errorf("bootstrap.sh still copies the proxy's token into %s", ns)
+		}
+	}
+	if len(installerTokenKeys) != len(callerNames()) {
+		t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames()))
+	}
+}
+
+// Every value the installer keeps in secrets.env has a rotation that rewrites
+// that very key, and every key a rotation writes is one the installer
+// generates, keeps and so re-applies on its next run.
+func TestInstallerSecretsAreAllRotatable(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	m := regexp.MustCompile(`(?s)write_file_atomic "\$SECRETS_ENV" 0600 <<EOF\n(.*?)\nEOF\n`).FindStringSubmatch(script)
+	if m == nil {
+		t.Fatal("bootstrap.sh: no secrets.env heredoc found")
+	}
+	persisted := map[string]bool{}
+	for _, line := range strings.Split(m[1], "\n") {
+		key, value, _ := strings.Cut(line, "=")
+		if value != "${"+key+"}" {
+			t.Errorf("secrets.env line %q is not KEY=${KEY}", line)
+		}
+		persisted[key] = true
+	}
+	rotated := installerSecretKeys()
+	for key := range persisted {
+		if !rotated[key] {
+			t.Errorf("secrets.env keeps %s, which no rotation replaces", key)
+		}
+	}
+	for key := range rotated {
+		if !persisted[key] {
+			t.Errorf("a rotation writes %s, which the installer does not keep in secrets.env", key)
+		}
+		if !regexp.MustCompile(`\n  ` + key + `="\$\{` + key + `:-\$\(openssl rand -hex [0-9]+\)\}"\n`).MatchString(script) {
+			t.Errorf("bootstrap.sh does not generate %s when secrets.env lacks it", key)
+		}
+	}
+}

cmd/felis/run.go

@@ -11,17 +11,34 @@ Usage:
   felis <command> [flags]
 
 Commands:
-  migrate up        Apply embedded database migrations under an advisory lock
+  migrate up        Apply embedded database migrations under an advisory lock (snapshots the database first)
+  db                Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check)
+  offsite           Copy world archives, database bundles, user images and uploads to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|fetch-images|fetch-uploads|keygen)
   operator          Run the MinecraftServer controller-manager
   api               Run the felis-api HTTP server
   nano              Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)
   reaper            Run the world reaper / backup batch
   restore           Extract a world archive into a world volume (internal Job entrypoint)
   backup            Archive a world into the backup store and record it (internal Job entrypoint)
-  files             List/read/write one file in a stopped server's world (internal Job entrypoint)
+  backup-now        Archive every user server's world now, one at a time (or the named ones; -stop stops running ones first; prints the plan, -yes applies; requires root/sudo)
+  files             List, read, write, mkdir, delete, rename, upload or unzip one path in a stopped server's world (internal Job entrypoint)
+  export            Archive a stopped server's world, or read one of its backups, and hand it to felis-api for download (internal Job entrypoint)
+  egress-gate       Hold a build or game server pod until its egress NetworkPolicy is enforced (internal init container entrypoint)
+  fetch-context     Fetch and extract a submission's build context (internal Job entrypoint)
+  scan-gate         Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
+  push-image        Push a scanned image tarball to the registry (internal Job entrypoint)
+  mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
+  registry-gate     Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
   manifests         Render the control-plane RBAC + NetworkPolicy install bundle as YAML
   apply             Create a MinecraftServer CRD (direct K8s write; use -f server.json)
   setup             Run host bootstrap + first-run setup console (TUI; requires root/sudo)
+  converge          Fill in fields a newer desired spec added to already-installed system servers
+  rotate-token      Replace a generated credential and restart what reads it (velocity|limbo|build|ops|registry|forwarding|db; prints the plan, -yes applies; requires root/sudo)
+  domain            Move the install to a new root domain on every surface that carries it, or check each one (set|check; requires root/sudo)
+  status            Print the platform at a glance: node, control plane, proxy, servers, backups, host, open alerts (requires root/sudo)
+  doctor            Run every health check once, grouped by area, with where to look next; mails nothing (requires root/sudo)
+  support-bundle    Collect status, doctor, logs and cluster state into one redacted tar.gz to share when asking for help (requires root/sudo)
+  watchdog          Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
   version           Print the build stamp of this binary
   update            Report which platform components have updates available
   breakGlass        Open the local break-glass emergency console (TUI; requires root/sudo)
@@ -39,22 +56,42 @@ Run "felis <command> -h" for command-specific flags.
 // The help aliases are deliberately NOT entries: they print usage rather than run a
 // subcommand, and listing them would make the table disagree with the command list.
 var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
-	"migrate":          cmdMigrate,
-	"operator":         cmdOperator,
-	"api":              cmdAPI,
-	"nano":             cmdNano,
-	"reaper":           cmdReaper,
-	"restore":          cmdRestore,
-	"backup":           cmdBackup,
-	"files":            cmdFiles,
-	"manifests":        cmdManifests,
-	"apply":            cmdApply,
-	"setup":            cmdSetup,
-	"breakGlass":       cmdBreakGlass,
-	"bootstrap-assets": cmdBootstrapAssets,
-	"init-forwarding":  cmdInitForwarding,
-	"version":          cmdVersion,
-	"update":           cmdUpdate,
+	"migrate":            cmdMigrate,
+	"db":                 cmdDB,
+	"offsite":            cmdOffsite,
+	"operator":           cmdOperator,
+	"api":                cmdAPI,
+	"nano":               cmdNano,
+	"reaper":             cmdReaper,
+	"restore":            cmdRestore,
+	"backup":             cmdBackup,
+	"backup-now":         cmdBackupNow,
+	"files":              cmdFiles,
+	"export":             cmdExport,
+	"egress-gate":        cmdEgressGate,
+	"fetch-context":      cmdFetchContext,
+	"scan-gate":          cmdScanGate,
+	"push-image":         cmdPushImage,
+	"mirror-build-tools": cmdMirrorBuildTools,
+	"registry-gate":      cmdRegistryGate,
+	"manifests":          cmdManifests,
+	"apply":              cmdApply,
+	"setup":              cmdSetup,
+	"converge":           cmdConverge,
+	"rotate-token":       cmdRotateToken,
+	"domain":             cmdDomain,
+	"breakGlass":         cmdBreakGlass,
+	"bootstrap-assets":   cmdBootstrapAssets,
+	"init-forwarding":    cmdInitForwarding,
+	"init-volume":        cmdInitVolume,
+	"pin-images":         cmdPinImages,
+	"image-bundle":       cmdImageBundle,
+	"version":            cmdVersion,
+	"update":             cmdUpdate,
+	"watchdog":           cmdWatchdog,
+	"status":             cmdStatus,
+	"doctor":             cmdDoctor,
+	"support-bundle":     cmdSupportBundle,
 }
 
 // run dispatches a subcommand. It is separate from main so the router is

cmd/felis/run_test.go

@@ -38,9 +38,13 @@ func TestRunUnknownCommand(t *testing.T) {
 }
 
 // undocumentedCommands are routable on purpose but kept out of the usage text: they
-// are called by deploy/bootstrap.sh, not by a human at a prompt. Listing them here is
-// what makes their absence from usage a deliberate decision rather than an oversight.
-var undocumentedCommands = map[string]bool{"bootstrap-assets": true, "init-forwarding": true}
+// are called by deploy/bootstrap.sh or the operator's initContainers, not by a human
+// at a prompt. Listing them here is what makes their absence from usage a deliberate
+// decision rather than an oversight.
+var undocumentedCommands = map[string]bool{
+	"bootstrap-assets": true, "init-forwarding": true, "init-volume": true,
+	"pin-images": true, "image-bundle": true,
+}
 
 // The usage text and the dispatch table must describe the same set of commands.
 //

cmd/felis/scangate.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"strings"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// maxScanDocument bounds each document scan-gate reads: a modpack report lists a
+// few thousand packages, far below this. Tests shrink it.
+var maxScanDocument int64 = 64 << 20
+
+// cmdScanGate is the build pod's verdict step, after trivy wrote its full JSON
+// report and trivy convert wrote the CycloneDX SBOM. It applies the scan policy
+// to the report, prints the verdict and every blocking finding, then appends the
+// verdict, the report and the SBOM to its log as the envelope felis-api keeps on
+// the build (build.WriteScanEnvelope). It exits 1 when a finding blocks, which
+// fails the pod before the push step runs, and 2 when the report cannot be read,
+// so a scan that produced nothing usable never admits an image.
+func cmdScanGate(args []string, stdout, stderr io.Writer) int {
+	fset := flag.NewFlagSet("scan-gate", flag.ContinueOnError)
+	fset.SetOutput(stderr)
+	reportPath := fset.String("report", "", "trivy JSON report (required)")
+	sbomPath := fset.String("sbom", "", "CycloneDX SBOM to keep with the report")
+	failOn := fset.String("fail-on", strings.Join(build.DefaultScanFailOn, ","), "comma-separated severities that block the image")
+	failUnfixed := fset.Bool("fail-unfixed", false, "block on vulnerabilities that have no fixed release too")
+	accept := fset.String("accept", "", "comma-separated vulnerability ids and secret rule ids that never block")
+	termLog := fset.String("termination-log", "/dev/termination-log", "where the one-line verdict goes for the pod status")
+	if err := fset.Parse(args); err != nil {
+		return 2
+	}
+	// A stray argument is a policy the gate would otherwise drop without a word
+	// (a comma split out of --fail-on, say).
+	if fset.NArg() > 0 {
+		fmt.Fprintf(stderr, "felis scan-gate: unexpected argument %q\n", fset.Arg(0))
+		return 2
+	}
+	sevs, err := build.ParseSeverities(*failOn)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --fail-on: %v\n", err)
+		return 2
+	}
+	accepted, err := build.ParseScanAccept(*accept)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --accept: %v\n", err)
+		return 2
+	}
+	if *reportPath == "" {
+		fmt.Fprintln(stderr, "felis scan-gate: --report is required")
+		return 2
+	}
+	fail := func(msg string) int {
+		fmt.Fprintln(stderr, "felis scan-gate: "+msg)
+		writeTerminationLog(*termLog, msg)
+		return 2
+	}
+	report, err := readScanDocument(*reportPath)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	policy := build.ScanPolicy{FailOn: sevs, FailUnfixed: *failUnfixed, Accept: accepted}
+	summary, err := build.Summarize(report, policy)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	env := build.ScanEnvelope{Summary: summary, Report: report}
+	if *sbomPath != "" {
+		switch sbom, err := readScanDocument(*sbomPath); {
+		case err == nil:
+			env.SBOM = sbom
+		case errors.Is(err, fs.ErrNotExist):
+			fmt.Fprintf(stdout, "felis scan-gate: no SBOM at %s; keeping the report alone\n", *sbomPath)
+		default:
+			return fail("the SBOM is unreadable: " + err.Error())
+		}
+	}
+
+	printScanVerdict(stdout, summary)
+	written, err := build.WriteScanEnvelope(stdout, env)
+	if err != nil {
+		return fail("could not write the scan envelope: " + err.Error())
+	}
+	for _, doc := range written.Summary.Omitted {
+		fmt.Fprintf(stderr, "felis scan-gate: the %s is too large to keep with the build and was left out\n", doc)
+	}
+	if summary.Blocked {
+		writeTerminationLog(*termLog, summary.Reason())
+		return 1
+	}
+	writeTerminationLog(*termLog, "the scan passed")
+	return 0
+}
+
+// printScanVerdict writes the human half of scan-gate's log.
+func printScanVerdict(w io.Writer, s build.ScanSummary) {
+	var counts []string
+	for _, sev := range build.Severities {
+		counts = append(counts, fmt.Sprintf("%s %d", sev, s.Counts[sev]))
+	}
+	fmt.Fprintf(w, "felis scan-gate: %d packages; findings: %s\n", s.Packages, strings.Join(counts, ", "))
+	unfixed := "vulnerabilities with no fixed release do not block"
+	if s.Policy.FailUnfixed {
+		unfixed = "vulnerabilities with no fixed release block too"
+	}
+	fmt.Fprintf(w, "felis scan-gate: blocking on %s (%s)\n", strings.Join(s.Policy.FailOn, ", "), unfixed)
+	if len(s.Policy.Accept) > 0 {
+		matched := 0
+		for _, f := range s.Findings {
+			if f.Accepted {
+				matched++
+			}
+		}
+		fmt.Fprintf(w, "felis scan-gate: accepted ids, never blocking: %s; listed findings under them: %d\n", strings.Join(s.Policy.Accept, ", "), matched)
+	}
+	if !s.Blocked {
+		fmt.Fprintln(w, "felis scan-gate: nothing blocks this image")
+		return
+	}
+	fmt.Fprintln(w, "felis scan-gate: "+build.Printable(s.Reason()))
+	for _, f := range s.Findings {
+		if !f.Blocking {
+			break
+		}
+		fix := f.Fixed
+		if fix == "" {
+			fix = "no fix"
+		}
+		if f.Kind == build.FindingSecret {
+			fmt.Fprintf(w, "  %s %s secret in %s: %s\n", build.Printable(f.ID), f.Severity, build.Printable(f.Target), build.Printable(f.Title))
+			continue
+		}
+		fmt.Fprintf(w, "  %s %s %s %s -> %s (%s)\n", build.Printable(f.ID), f.Severity,
+			build.Printable(f.Package), build.Printable(f.Installed), build.Printable(fix), build.Printable(f.Target))
+	}
+}
+
+func readScanDocument(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	b, err := io.ReadAll(io.LimitReader(f, maxScanDocument+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(b)) > maxScanDocument {
+		return nil, fmt.Errorf("%s exceeds %d bytes", path, maxScanDocument)
+	}
+	return b, nil
+}
+
+// writeTerminationLog leaves msg where the kubelet copies it into the container
+// status. It is best effort: the log carries the same verdict.
+func writeTerminationLog(path, msg string) {
+	if path == "" {
+		return
+	}
+	_ = os.WriteFile(path, []byte(build.Printable(msg)), 0o644)
+}

cmd/felis/scangate_test.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// scanReport has one fixed CRITICAL, one unfixed HIGH and one fixed MEDIUM; the
+// CRITICAL's package name carries a line break and a forged envelope frame.
+const scanReport = `{"SchemaVersion":2,"Results":[{"Target":"data/mods/core.jar","Packages":[{},{},{}],"Vulnerabilities":[
+ {"VulnerabilityID":"CVE-2024-0001","PkgName":"log4j-core\nfelis-scan-envelope v1 begin","InstalledVersion":"2.14.1","FixedVersion":"2.17.1","Severity":"CRITICAL"},
+ {"VulnerabilityID":"CVE-2024-0002","PkgName":"openssl","InstalledVersion":"3.0.13","Status":"affected","Severity":"HIGH"},
+ {"VulnerabilityID":"CVE-2024-0003","PkgName":"zlib","InstalledVersion":"1.3","FixedVersion":"1.3.1","Severity":"MEDIUM"}]}]}`
+
+type scanGateRun struct {
+	code           int
+	stdout, stderr string
+	termLog        string
+	env            *build.ScanEnvelope
+}
+
+func runScanGate(t *testing.T, report, sbom string, extra ...string) scanGateRun {
+	t.Helper()
+	dir := t.TempDir()
+	reportPath := filepath.Join(dir, "trivy.json")
+	if report != "" {
+		if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	sbomPath := filepath.Join(dir, "sbom.cdx.json")
+	if sbom != "" {
+		if err := os.WriteFile(sbomPath, []byte(sbom), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	termPath := filepath.Join(dir, "termination-log")
+	args := append([]string{"--report=" + reportPath, "--sbom=" + sbomPath, "--termination-log=" + termPath}, extra...)
+	var stdout, stderr bytes.Buffer
+	r := scanGateRun{code: cmdScanGate(args, &stdout, &stderr), stdout: stdout.String(), stderr: stderr.String()}
+	if b, err := os.ReadFile(termPath); err == nil {
+		r.termLog = string(b)
+	}
+	if env, err := build.ReadScanEnvelope(strings.NewReader(r.stdout)); err == nil {
+		r.env = env
+	}
+	return r
+}
+
+func TestScanGateBlocksAndHandsOverTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`)
+	if r.code != 1 {
+		t.Fatalf("exit %d, want 1; stderr %s", r.code, r.stderr)
+	}
+	if r.termLog != "the scan blocked the image: 1 CRITICAL (CVE-2024-0001)" {
+		t.Errorf("termination log = %q", r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: 3 packages; findings: CRITICAL 1, HIGH 1, MEDIUM 1, LOW 0, UNKNOWN 0\n",
+		"felis scan-gate: blocking on CRITICAL (vulnerabilities with no fixed release do not block)\n",
+		"felis scan-gate: the scan blocked the image: 1 CRITICAL (CVE-2024-0001)\n",
+		"  CVE-2024-0001 CRITICAL log4j-core?felis-scan-envelope v1 begin 2.14.1 -> 2.17.1 (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	if strings.Contains(r.stdout, "  CVE-2024-0002") {
+		t.Errorf("an unfixed HIGH was listed as blocking:\n%s", r.stdout)
+	}
+	if strings.Count(r.stdout, "\nfelis-scan-envelope v1 begin\n") != 1 {
+		t.Errorf("stdout must hold exactly one frame start on a line of its own:\n%s", r.stdout)
+	}
+	if r.env == nil {
+		t.Fatal("no envelope in stdout")
+	}
+	if !r.env.Summary.Blocked || string(r.env.SBOM) != `{"bomFormat":"CycloneDX"}` || !strings.Contains(string(r.env.Report), "CVE-2024-0003") {
+		t.Errorf("envelope = blocked %t, sbom %s, report %d bytes", r.env.Summary.Blocked, r.env.SBOM, len(r.env.Report))
+	}
+}
+
+func TestScanGatePolicyFlags(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=high", "--fail-unfixed")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 HIGH (CVE-2024-0002)" {
+		t.Errorf("HIGH + unfixed: exit %d, termination log %q", r.code, r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: blocking on HIGH (vulnerabilities with no fixed release block too)\n",
+		"  CVE-2024-0002 HIGH openssl 3.0.13 -> no fix (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	r = runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`, "--fail-on=LOW")
+	if r.code != 0 || r.termLog != "the scan passed" || !strings.Contains(r.stdout, "felis scan-gate: nothing blocks this image\n") {
+		t.Errorf("LOW only: exit %d, termination log %q, stdout:\n%s", r.code, r.termLog, r.stdout)
+	}
+	if r.env == nil || r.env.Summary.Blocked || r.env.SBOM == nil {
+		t.Errorf("a passing scan must still hand over its report and SBOM: %+v", r.env)
+	}
+}
+
+func TestScanGateAcceptedIDsNeverBlock(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=CRITICAL,MEDIUM", "--accept=CVE-2024-0001, CVE-2024-0002,CVE-2099-0001")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 MEDIUM (CVE-2024-0003)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "felis scan-gate: accepted ids, never blocking: CVE-2024-0001, CVE-2024-0002, CVE-2099-0001; listed findings under them: 2\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+	if r.env == nil || strings.Join(r.env.Summary.Policy.Accept, ",") != "CVE-2024-0001,CVE-2024-0002,CVE-2099-0001" {
+		t.Fatalf("envelope = %+v", r.env)
+	}
+	for _, f := range r.env.Summary.Findings {
+		if f.ID == "CVE-2024-0001" && (f.Blocking || !f.Accepted) {
+			t.Errorf("accepted finding = %+v", f)
+		}
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001")
+	if r.code != 0 || r.termLog != "the scan passed" {
+		t.Errorf("only an accepted CRITICAL: exit %d, termination log %q", r.code, r.termLog)
+	}
+}
+
+func TestScanGateWithoutAnSBOMKeepsTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=LOW")
+	if r.code != 0 || !strings.Contains(r.stdout, "felis scan-gate: no SBOM at ") {
+		t.Errorf("exit %d, stdout:\n%s", r.code, r.stdout)
+	}
+	if r.env == nil || r.env.SBOM != nil || r.env.Report == nil {
+		t.Errorf("envelope = %+v", r.env)
+	}
+}
+
+func TestScanGateListsABlockingSecret(t *testing.T) {
+	r := runScanGate(t, `{"SchemaVersion":2,"Results":[{"Target":"config/keys.txt","Secrets":[
+ {"RuleID":"aws-access-key-id","Severity":"CRITICAL","Title":"AWS Access\nKey ID"}]}]}`, "")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 CRITICAL (aws-access-key-id)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "  aws-access-key-id CRITICAL secret in config/keys.txt: AWS Access?Key ID\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+}
+
+func TestScanGateFailsClosed(t *testing.T) {
+	r := runScanGate(t, "", "")
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the scan report is unreadable: open ") || r.env != nil {
+		t.Errorf("missing report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	r = runScanGate(t, `{"SchemaVersion":1}`, "")
+	if r.code != 2 || r.termLog != "the scan report is unreadable: read trivy report: schema version 1, want 2" {
+		t.Errorf("old schema: exit %d, termination log %q", r.code, r.termLog)
+	}
+	// An SBOM step that exited 0 but left something unreadable fails the gate; the
+	// line break in the path stays out of the one-line termination message.
+	sbomDir := filepath.Join(t.TempDir(), "sb\nom")
+	if err := os.Mkdir(sbomDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	r = runScanGate(t, scanReport, "", "--sbom="+sbomDir)
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the SBOM is unreadable: read ") ||
+		!strings.HasSuffix(r.termLog, "/sb?om: is a directory") || r.env != nil {
+		t.Errorf("unreadable SBOM: exit %d, termination log %q", r.code, r.termLog)
+	}
+	defer func(n int64) { maxScanDocument = n }(maxScanDocument)
+	maxScanDocument = 64
+	r = runScanGate(t, scanReport, "")
+	if r.code != 2 || !strings.HasSuffix(r.termLog, "/trivy.json exceeds 64 bytes") || r.env != nil {
+		t.Errorf("oversized report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	maxScanDocument = 64 << 20
+	r = runScanGate(t, scanReport, "", "--fail-on=SEVERE")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --fail-on: unknown severity "SEVERE"`) {
+		t.Errorf("bad severity: exit %d, stderr %q", r.code, r.stderr)
+	}
+	// A severity list split at its comma leaves a stray argument, not a
+	// narrower policy.
+	r = runScanGate(t, scanReport, "", "--fail-on=LOW", "CRITICAL")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: unexpected argument "CRITICAL"`) || r.env != nil {
+		t.Errorf("stray argument: exit %d, stderr %q", r.code, r.stderr)
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001 CVE-2024-0003")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --accept: "CVE-2024-0001 CVE-2024-0003" is not a vulnerability id or secret rule id`) {
+		t.Errorf("bad accept list: exit %d, stderr %q", r.code, r.stderr)
+	}
+	var stdout, stderr bytes.Buffer
+	if code := cmdScanGate(nil, &stdout, &stderr); code != 2 || !strings.Contains(stderr.String(), "--report is required") {
+		t.Errorf("no report flag: exit %d, stderr %q", code, stderr.String())
+	}
+}

cmd/felis/setup.go

@@ -13,7 +13,6 @@ import (
 	"felis.lolicon.best/internal/api"
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/store"
 )
@@ -113,7 +112,7 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database.URL, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
+	res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis setup: %v\n", err)
 		return 1
@@ -216,31 +215,30 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 			"Re-run `sudo felis setup` on the control-plane host once the cluster is reachable", err)
 	}
 	// The login limbo authenticates to the felis-api INTERNAL face, so it needs the
-	// internal base URL, the root domain (to link players at the console), and the
-	// service token. The first two are plain env baked into the pod here; the token
-	// is a Secret the operator injects by reference — but a secretKeyRef is
-	// namespace-local, so first replicate the token Secret from the control namespace
-	// into the minecraft namespace where the login pod runs. The control namespace is
+	// internal base URL, the root domain (to link players at the console), and its
+	// own token (felis-limbo-token). The first two are plain env baked into the pod
+	// here; the token is a Secret the operator injects by reference — but a
+	// secretKeyRef is namespace-local, so first replicate the token Secret from the
+	// control namespace into the minecraft namespace where the login pod runs. The control namespace is
 	// the platform default (there is no felis.toml override for it); a deployment that
 	// renamed it must replicate the Secret by hand.
 	controlNS := platform.DefaultControlNamespace
 	apiBaseURL := platform.InternalAPIBaseURL(controlNS)
 	// These Secrets must land in the minecraft namespace before the pods that
-	// mount them are created: the service token (login authenticates to felis-api
-	// with it), the Velocity forwarding secret (every backend verifies the proxy's
+	// mount them are created: the login gate's token (login authenticates to
+	// felis-api with it), the Velocity forwarding secret (every backend verifies the proxy's
 	// signed handshake with it — without it the login gate would derive an OFFLINE
 	// UUID and the Owner would bind the wrong Minecraft identity), and felis-config
 	// (the on-demand BACKUP Job runs in the minecraft namespace and mounts it to
 	// self-record its world_backups row; without the replica the Job's volume
 	// mount fails and every backup request strands in the cluster).
-	secretOutcomes := []systemServerOutcome{
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token"),
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret"),
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			"felis-config", "felis.toml", "config"),
+	// An empty build_namespace means the build system's compiled-in default; the
+	// replica must target the namespace the Jobs actually run in.
+	buildNS := cfg.Registry.BuildNamespace
+	if buildNS == "" {
+		buildNS = platform.DefaultBuildNamespace
 	}
+	secretOutcomes := provisionSecretReplicas(ctx, cl, controlNS, cfg.K8s.Namespace, buildNS)
 	outcomes := ensureSystemServers(ctx, cl, cfg.K8s.Namespace, cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, apiBaseURL, cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
 	outcomes = append(secretOutcomes, outcomes...)
 	fmt.Fprintln(out, "\nfelis setup: login/lobby system servers (always-on, reaper-exempt):")
@@ -250,6 +248,8 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 			fmt.Fprintf(out, "  - %s: ERROR %v\n", o.name, o.err)
 		case o.created:
 			fmt.Fprintf(out, "  - %s: created (DesiredState=Running)\n", o.name)
+		case o.updated:
+			fmt.Fprintf(out, "  - %s: refreshed from the control namespace\n", o.name)
 		default:
 			fmt.Fprintf(out, "  - %s: skipped (%s)\n", o.name, o.skipped)
 		}
@@ -320,7 +320,9 @@ func openConfiguredSetup(ctx context.Context, cfgPath string) (*configuredSetup,
 	if err != nil {
 		return nil, &setupOpenError{stage: "load config", err: err}
 	}
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Pending migrations are the preflight's to apply; a newer schema is a rolled-back
+	// binary, and nothing this console writes would match it.
+	drv, err := openStore(ctx, cfg.Database.URL, true)
 	if err != nil {
 		return nil, &setupOpenError{stage: "open database", err: err}
 	}

cmd/felis/status.go

@@ -0,0 +1,438 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+	"text/tabwriter"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// cmdStatus prints the platform at a glance: the release and the node, the
+// control plane's workloads, the game proxy, every server with its players
+// and newest world backup, the database backups and the off-site copy, the
+// host's disks and memory, and what the watchdog has open. It changes
+// nothing, and a part that is down (the cluster, PostgreSQL) reads as such
+// while the rest still prints. felis doctor says what is wrong and where to
+// look.
+func cmdStatus(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("status", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis status: run as root (sudo felis status): it reads root-only state under /etc/felis and /var/lib/felis")
+		return 1
+	}
+	env, err := hostStatusEnv(*unitDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis status: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	printStatus(ctx, env, stdout)
+	return 0
+}
+
+// statusEnv is what one status report reads the host through.
+type statusEnv struct {
+	cfg *config.Config
+	// w is how felis-watchdog.service runs the watchdog: where the backups,
+	// the off-site record and the proxy are.
+	w       watchdogFlags
+	cl      client.Client // nil while the API server is unreachable
+	clErr   error
+	backups func(ctx context.Context) (map[string]time.Time, error)
+	run     func(ctx context.Context, name string, args ...string) ([]byte, error)
+	unitDir string
+	meminfo string
+	host    string
+	now     time.Time
+}
+
+// hostStatusEnv reads this host: the watchdog's settings, the configuration
+// they name, and the cluster.
+func hostStatusEnv(unitDir string) (statusEnv, error) {
+	w, _, err := watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
+	if err != nil {
+		return statusEnv{}, err
+	}
+	cfg, err := config.Load(w.cfgPath)
+	if err != nil {
+		return statusEnv{}, err
+	}
+	cl, clErr := buildSystemServerClient()
+	if clErr != nil {
+		cl = nil
+	}
+	host, _ := os.Hostname()
+	return statusEnv{
+		cfg: cfg, w: w, cl: cl, clErr: clErr,
+		backups: func(ctx context.Context) (map[string]time.Time, error) {
+			return newestWorldBackups(ctx, cfg.Database.URL)
+		},
+		run: hostCommand, unitDir: unitDir, meminfo: "/proc/meminfo", host: host, now: time.Now(),
+	}, nil
+}
+
+func printStatus(ctx context.Context, env statusEnv, out io.Writer) {
+	fmt.Fprintf(out, "felis %s on %s at %s\n", resolvedVersion(), env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
+	if env.cl == nil {
+		fmt.Fprintf(out, "cluster:  unreachable (%v)\n", env.clErr)
+	} else {
+		statusCluster(ctx, env, out)
+	}
+	statusProxy(ctx, env, out)
+	statusServers(ctx, env, out)
+	statusBackups(env, out)
+	statusHost(env, out)
+	statusWatchdog(ctx, env, out)
+}
+
+func statusCluster(ctx context.Context, env statusEnv, out io.Writer) {
+	var nodes corev1.NodeList
+	if err := env.cl.List(ctx, &nodes); err != nil {
+		fmt.Fprintf(out, "cluster:  unreachable (%v)\n", err)
+		return
+	}
+	for _, n := range nodes.Items {
+		ready := "NotReady"
+		for _, c := range n.Status.Conditions {
+			if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue {
+				ready = "Ready"
+			}
+		}
+		info := n.Status.NodeInfo
+		fmt.Fprintf(out, "node:     %s %s, k3s %s, %s, kernel %s\n", n.Name, ready, info.KubeletVersion, info.OSImage, info.KernelVersion)
+	}
+
+	ns := env.w.controlNS
+	var deps appsv1.DeploymentList
+	var pods corev1.PodList
+	err := env.cl.List(ctx, &deps, client.InNamespace(ns))
+	if err == nil {
+		err = env.cl.List(ctx, &pods, client.InNamespace(ns))
+	}
+	fmt.Fprintf(out, "\ncontrol plane (namespace %s):\n", ns)
+	if err != nil {
+		fmt.Fprintf(out, "  cannot list it: %v\n", err)
+		return
+	}
+	sort.Slice(deps.Items, func(i, j int) bool { return deps.Items[i].Name < deps.Items[j].Name })
+	tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	for _, d := range deps.Items {
+		want := int32(1)
+		if d.Spec.Replicas != nil {
+			want = *d.Spec.Replicas
+		}
+		image := "-"
+		if cs := d.Spec.Template.Spec.Containers; len(cs) > 0 {
+			image = shortImage(cs[0].Image)
+		}
+		fmt.Fprintf(tw, "  %s\t%d/%d ready\t%s\trestarts %d\n", d.Name, d.Status.ReadyReplicas, want, image, podRestarts(d.Spec.Selector, pods.Items))
+	}
+	tw.Flush()
+}
+
+// podRestarts adds up the container restarts of the pods selector picks (the
+// API server refuses a Deployment whose selector is empty).
+func podRestarts(selector *metav1.LabelSelector, pods []corev1.Pod) int32 {
+	sel, err := metav1.LabelSelectorAsSelector(selector)
+	if err != nil {
+		return 0
+	}
+	var n int32
+	for _, p := range pods {
+		if !sel.Matches(labels.Set(p.Labels)) {
+			continue
+		}
+		for _, cs := range p.Status.ContainerStatuses {
+			n += cs.RestartCount
+		}
+	}
+	return n
+}
+
+// shortImage is an image reference without its registry and repository path,
+// and with its digest cut to 12 characters.
+func shortImage(ref string) string {
+	if i := strings.LastIndex(ref, "/"); i >= 0 {
+		ref = ref[i+1:]
+	}
+	if name, digest, ok := strings.Cut(ref, "@sha256:"); ok && len(digest) > 12 {
+		ref = name + "@" + digest[:12]
+	}
+	return ref
+}
+
+func statusProxy(ctx context.Context, env statusEnv, out io.Writer) {
+	var parts []string
+	if _, err := os.Stat(filepath.Join(env.unitDir, "felis-velocity.service")); err == nil {
+		parts = append(parts, "felis-velocity "+unitActiveState(ctx, doctorEnv{run: env.run}, "felis-velocity.service"))
+	}
+	if env.w.proxyAddr != "" {
+		d := net.Dialer{Timeout: 3 * time.Second}
+		if conn, err := d.DialContext(ctx, "tcp", env.w.proxyAddr); err != nil {
+			parts = append(parts, fmt.Sprintf("%s refuses connections (%v)", env.w.proxyAddr, err))
+		} else {
+			conn.Close()
+			parts = append(parts, env.w.proxyAddr+" accepts connections")
+		}
+	}
+	if len(parts) == 0 {
+		parts = append(parts, "not on this host")
+	}
+	fmt.Fprintf(out, "\nproxy:    %s\n", strings.Join(parts, ", "))
+}
+
+func statusServers(ctx context.Context, env statusEnv, out io.Writer) {
+	ns := env.cfg.K8s.Namespace
+	if ns == "" {
+		ns = platform.DefaultMinecraftNamespace
+	}
+	if env.cl == nil {
+		fmt.Fprintf(out, "\nservers (namespace %s): unknown while the cluster is unreachable\n", ns)
+		return
+	}
+	var list v1alpha1.MinecraftServerList
+	if err := env.cl.List(ctx, &list, client.InNamespace(ns)); err != nil {
+		fmt.Fprintf(out, "\nservers (namespace %s): cannot list them: %v\n", ns, err)
+		return
+	}
+	newest, backupErr := env.backups(ctx)
+	running, online := 0, int32(0)
+	for _, ms := range list.Items {
+		if ms.Status.Phase == v1alpha1.PhaseRunning {
+			running++
+			online += ms.Status.Players.Online
+		}
+	}
+	fmt.Fprintf(out, "\nservers (namespace %s): %d, %d running, %d players online\n", ns, len(list.Items), running, online)
+	if len(list.Items) == 0 {
+		return
+	}
+	sort.Slice(list.Items, func(i, j int) bool { return list.Items[i].Name < list.Items[j].Name })
+	tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "  NAME\tROLE\tDESIRED\tPHASE\tPLAYERS\tNEWEST WORLD BACKUP")
+	for _, ms := range list.Items {
+		role := ms.Labels[v1alpha1.LabelSystemRole]
+		if role == "" {
+			role = "-"
+		}
+		phase := string(ms.Status.Phase)
+		if phase == "" {
+			phase = "-"
+		}
+		players := "-"
+		if ms.Status.Phase == v1alpha1.PhaseRunning {
+			players = fmt.Sprintf("%d/%d", ms.Status.Players.Online, ms.Status.Players.Max)
+		}
+		backup := "none"
+		switch at, ok := newest[ms.Name]; {
+		case backupErr != nil:
+			backup = "?"
+		case ok:
+			backup = dbbackup.Age(env.now.Sub(at)) + " ago"
+		}
+		fmt.Fprintf(tw, "  %s\t%s\t%s\t%s\t%s\t%s\n", ms.Name, role, orDash(string(ms.Spec.DesiredState)), phase, players, backup)
+	}
+	tw.Flush()
+	if backupErr != nil {
+		fmt.Fprintf(out, "  world backups unknown: %v\n", backupErr)
+	}
+}
+
+func orDash(s string) string {
+	if s == "" {
+		return "-"
+	}
+	return s
+}
+
+// newestWorldBackups is when each server's newest world backup that a restore
+// can use was taken.
+func newestWorldBackups(ctx context.Context, url string) (map[string]time.Time, error) {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	defer drv.Close()
+	rows, err := drv.DB().QueryContext(ctx, `SELECT server_name, max(created_at) FROM world_backups
+		WHERE status = 'present' AND corrupt_at IS NULL GROUP BY server_name`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]time.Time{}
+	for rows.Next() {
+		var name string
+		var at time.Time
+		if err := rows.Scan(&name, &at); err != nil {
+			return nil, err
+		}
+		out[name] = at
+	}
+	return out, rows.Err()
+}
+
+func statusBackups(env statusEnv, out io.Writer) {
+	fmt.Fprintln(out, "\nbackups:")
+	switch bundles, err := dbbackup.List(env.w.backupDir); {
+	case env.w.backupDir == "":
+		fmt.Fprintln(out, "  database: not checked (felis-watchdog.service names no -backup-dir)")
+	case err != nil:
+		fmt.Fprintf(out, "  database: cannot read %s: %v\n", env.w.backupDir, err)
+	case len(bundles) == 0:
+		fmt.Fprintf(out, "  database: none in %s\n", env.w.backupDir)
+	default:
+		fmt.Fprintf(out, "  database: newest %s, %s ago; %d bundles in %s\n",
+			bundles[0].Name, dbbackup.Age(env.now.Sub(bundles[0].Created)), len(bundles), env.w.backupDir)
+	}
+	if !env.cfg.Offsite.Enabled() {
+		fmt.Fprintln(out, "  off-site: not configured, every backup is on this machine only")
+		return
+	}
+	switch st, err := offsite.ReadStatus(env.w.offsiteStatus); {
+	case err != nil:
+		fmt.Fprintf(out, "  off-site: %v\n", err)
+	case st == nil:
+		fmt.Fprintln(out, "  off-site: never synced")
+	case st.LastSuccess.IsZero():
+		fmt.Fprintf(out, "  off-site: never succeeded; last attempt %s ago: %s\n", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
+	default:
+		line := fmt.Sprintf("  off-site: last good sync %s ago to %s", dbbackup.Age(env.now.Sub(st.LastSuccess)), st.Bucket)
+		if st.LastAttempt.After(st.LastSuccess) { // a failed run records no success
+			line += fmt.Sprintf("; the last attempt, %s ago, failed: %s", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
+		}
+		fmt.Fprintln(out, line)
+	}
+}
+
+func statusHost(env statusEnv, out io.Writer) {
+	fmt.Fprintln(out, "\nhost:")
+	seen := map[uint64]bool{}
+	for _, p := range splitList(env.w.diskPaths) {
+		var st syscall.Stat_t
+		if err := syscall.Stat(p, &st); err != nil {
+			continue
+		}
+		dev := uint64(st.Dev) // int32 on darwin
+		if seen[dev] {
+			continue
+		}
+		seen[dev] = true
+		var fs syscall.Statfs_t
+		if err := syscall.Statfs(p, &fs); err != nil || fs.Blocks == 0 {
+			continue
+		}
+		bsize := uint64(fs.Bsize) // uint32 on darwin
+		total, avail := uint64(fs.Blocks)*bsize, uint64(fs.Bavail)*bsize
+		fmt.Fprintf(out, "  disk %s: %s free of %s (%.0f%% free)\n", p, offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)), float64(fs.Bavail)/float64(fs.Blocks)*100)
+	}
+	if total, avail, ok := readMeminfo(env.meminfo); ok {
+		fmt.Fprintf(out, "  memory: %s available of %s\n", offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)))
+	}
+}
+
+// readMeminfo reads MemTotal and MemAvailable, in bytes, from a /proc/meminfo
+// style file.
+func readMeminfo(path string) (total, avail uint64, ok bool) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, 0, false
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		fields := strings.Fields(sc.Text())
+		if len(fields) < 2 {
+			continue
+		}
+		v, _ := strconv.ParseUint(fields[1], 10, 64) // the kernel writes numbers
+		switch fields[0] {
+		case "MemTotal:":
+			total = v * 1024
+		case "MemAvailable:":
+			avail = v * 1024
+		}
+	}
+	return total, avail, total > 0
+}
+
+func statusWatchdog(ctx context.Context, env statusEnv, out io.Writer) {
+	fmt.Fprintln(out, "\nwatchdog:")
+	if _, err := os.Stat(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
+		fmt.Fprintln(out, "  not installed: nothing checks this host")
+		return
+	}
+	line := "  timer " + unitActiveState(ctx, doctorEnv{run: env.run}, "felis-watchdog.timer")
+	show, _ := env.run(ctx, "systemctl", "show", "--timestamp=unix", "-p", "Result", "-p", "ExecMainExitTimestamp", "felis-watchdog.service")
+	props := map[string]string{}
+	for _, ln := range strings.Split(string(show), "\n") {
+		if k, v, ok := strings.Cut(strings.TrimSpace(ln), "="); ok {
+			props[k] = v
+		}
+	}
+	// ExecMainExitTimestamp is empty until the service has run.
+	if sec, _ := strconv.ParseInt(strings.TrimPrefix(props["ExecMainExitTimestamp"], "@"), 10, 64); sec > 0 {
+		line += fmt.Sprintf(", last run %s ago (%s)", dbbackup.Age(env.now.Sub(time.Unix(sec, 0))), orDash(props["Result"]))
+	}
+	fmt.Fprintln(out, line)
+
+	state, err := watchdog.LoadState(watchdog.NewestState(env.w.statePath, env.w.fallbackState))
+	if err != nil {
+		fmt.Fprintf(out, "  alerts: unknown (%v)\n", err)
+		return
+	}
+	var open []string
+	for key, a := range state.Alerts {
+		if !a.ClearedAt.IsZero() {
+			continue
+		}
+		if a.Notified.IsZero() {
+			open = append(open, fmt.Sprintf("%s (%s, seen %s ago, not mailed yet)", key, a.Severity, dbbackup.Age(env.now.Sub(a.FirstSeen))))
+		} else {
+			open = append(open, fmt.Sprintf("%s (%s, mailed %s ago)", key, a.Severity, dbbackup.Age(env.now.Sub(a.Notified))))
+		}
+	}
+	if len(open) == 0 {
+		fmt.Fprintln(out, "  alerts: none open")
+		return
+	}
+	sort.Strings(open)
+	fmt.Fprintf(out, "  alerts: %d open (sudo felis doctor says where to look)\n", len(open))
+	for _, o := range open {
+		fmt.Fprintf(out, "    %s\n", o)
+	}
+}

cmd/felis/status_test.go

@@ -0,0 +1,490 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// statusCluster is a one-node install: felis-api with a restarted pod, three
+// servers (one of them the lobby), and a pod of another app whose restarts
+// are not felis-api's.
+func statusClusterObjects() []client.Object {
+	replicas := int32(1)
+	apiLabels := map[string]string{"app": "felis-api"}
+	return []client.Object{
+		&corev1.Node{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-1"},
+			Status: corev1.NodeStatus{
+				Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}},
+				NodeInfo:   corev1.NodeSystemInfo{KubeletVersion: "v1.36.4+k3s1", OSImage: "CentOS Stream 9", KernelVersion: "5.14.0-630.el9.aarch64"},
+			},
+		},
+		&appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
+			Spec: appsv1.DeploymentSpec{
+				Replicas: &replicas,
+				Selector: &metav1.LabelSelector{MatchLabels: apiLabels},
+				Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{
+					Name: "api", Image: "registry.felis.svc:5000/felis/felis-api:v1.4.0@sha256:0123456789abcdef0123456789abcdef",
+				}}}},
+			},
+			Status: appsv1.DeploymentStatus{ReadyReplicas: 1},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-api-7d9", Namespace: "felis", Labels: apiLabels},
+			Status:     corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 2}}},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "other-1", Namespace: "felis", Labels: map[string]string{"app": "other"}},
+			Status:     corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "x", RestartCount: 5}}},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
+			Spec:       v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
+			Status:     v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseRunning, Players: v1alpha1.PlayersStatus{Online: 2, Max: 20}},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "creative", Namespace: "minecraft"},
+			Spec:       v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped},
+			Status:     v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "lobby", Namespace: "minecraft", Labels: map[string]string{v1alpha1.LabelSystemRole: "lobby"}},
+			Spec:       v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
+			Status:     v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStarting},
+		},
+		// Another namespace's server is not this install's.
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "elsewhere", Namespace: "other"}},
+	}
+}
+
+// statusTestEnv is a host with the cluster above, a database backup nine hours
+// old, an off-site copy last good two hours ago, and one alert open.
+func statusTestEnv(t *testing.T) statusEnv {
+	t.Helper()
+	now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	dir := t.TempDir()
+	var w watchdogFlags
+	w.controlNS = "felis"
+	w.backupDir = filepath.Join(dir, "db-backups")
+	w.offsiteStatus = filepath.Join(dir, "offsite-status.json")
+	w.statePath = filepath.Join(dir, "state.json")
+	w.fallbackState = filepath.Join(dir, "fallback.json")
+	w.diskPaths = "/nonexistent-felis-status-test"
+	unitDir := filepath.Join(dir, "systemd")
+	for _, d := range []string{w.backupDir, unitDir} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-9*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
+	writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-33*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
+	if err := offsite.WriteStatus(w.offsiteStatus, offsite.Status{
+		LastAttempt: now.Add(-2 * time.Hour), LastSuccess: now.Add(-2 * time.Hour), Bucket: "felis-dr", Endpoint: "https://s3.example.com",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	state := &watchdog.State{Alerts: map[string]*watchdog.Alert{
+		"db-backup-servers": {Finding: watchdog.Finding{Key: "db-backup-servers", Severity: watchdog.Warning}, FirstSeen: now.Add(-3 * time.Hour), Notified: now.Add(-90 * time.Minute)},
+		"proxy":             {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: now.Add(-time.Hour), Notified: now.Add(-time.Hour), ClearedAt: now.Add(-10 * time.Minute)},
+		"disk//var":         {Finding: watchdog.Finding{Key: "disk//var", Severity: watchdog.Critical}, FirstSeen: now.Add(-4 * time.Minute)},
+	}}
+	if err := watchdog.SaveState(w.statePath, state); err != nil {
+		t.Fatal(err)
+	}
+	meminfo := filepath.Join(dir, "meminfo")
+	writeTestFile(t, meminfo, "MemTotal:        8000000 kB\nMemFree:          100000 kB\nMemAvailable:    2000000 kB\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.timer"), "[Unit]\n", 0o644)
+	cfg := &config.Config{Offsite: config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-dr"}}
+	return statusEnv{
+		cfg: cfg, w: w, cl: fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).Build(),
+		backups: func(context.Context) (map[string]time.Time, error) {
+			return map[string]time.Time{"survival": now.Add(-3 * time.Hour)}, nil
+		},
+		run: func(_ context.Context, name string, args ...string) ([]byte, error) {
+			switch strings.Join(append([]string{name}, args...), " ") {
+			case "systemctl is-active felis-watchdog.timer":
+				return []byte("active\n"), nil
+			case "systemctl show --timestamp=unix -p Result -p ExecMainExitTimestamp felis-watchdog.service":
+				return []byte("Result=success\nExecMainExitTimestamp=@" + strconv.FormatInt(now.Add(-70*time.Second).Unix(), 10) + "\n"), nil
+			}
+			return nil, errors.New("unexpected")
+		},
+		unitDir: unitDir, meminfo: meminfo, host: "felis-test", now: now,
+	}
+}
+
+// lineFields is the words of the first line of out that starts with prefix
+// once trimmed.
+func lineFields(out, prefix string) []string {
+	for _, l := range strings.Split(out, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(l), prefix) {
+			return strings.Fields(l)
+		}
+	}
+	return nil
+}
+
+func TestStatusReport(t *testing.T) {
+	env := statusTestEnv(t)
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	got := out.String()
+	for _, want := range []string{
+		"on felis-test at 2026-09-27 12:00 UTC\n",
+		"node:     felis-1 Ready, k3s v1.36.4+k3s1, CentOS Stream 9, kernel 5.14.0-630.el9.aarch64\n",
+		"\ncontrol plane (namespace felis):\n",
+		"\nproxy:    not on this host\n",
+		"\nservers (namespace minecraft): 3, 1 running, 2 players online\n",
+		"  database: newest " + dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily) + ", 9h0m ago; 2 bundles in " + env.w.backupDir + "\n",
+		"  off-site: last good sync 2h0m ago to felis-dr\n",
+		"  memory: 1.9 GiB available of 7.6 GiB\n",
+		"  timer active, last run 1m ago (success)\n",
+		"  alerts: 2 open (sudo felis doctor says where to look)\n" +
+			"    db-backup-servers (warning, mailed 1h30m ago)\n" +
+			"    disk//var (critical, seen 4m ago, not mailed yet)\n",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("status lacks %q:\n%s", want, got)
+		}
+	}
+	for prefix, want := range map[string]string{
+		"felis-api": "felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
+		"NAME":      "NAME ROLE DESIRED PHASE PLAYERS NEWEST WORLD BACKUP",
+		"creative":  "creative - Stopped Stopped - none",
+		"lobby":     "lobby lobby Running Starting - none",
+		"survival":  "survival - Running Running 2/20 3h0m ago",
+	} {
+		if f := strings.Join(lineFields(got, prefix), " "); f != want {
+			t.Errorf("row %q = %q, want %q\n%s", prefix, f, want, got)
+		}
+	}
+	if strings.Contains(got, "elsewhere") || strings.Contains(got, "other-1") {
+		t.Errorf("status shows what is not this install's:\n%s", got)
+	}
+}
+
+// Whatever is down reads as down, and the rest of the report still prints.
+func TestStatusWithPartsDown(t *testing.T) {
+	env := statusTestEnv(t)
+	env.cl, env.clErr = nil, errors.New("connection refused")
+	env.cfg = &config.Config{}
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	for _, want := range []string{
+		"cluster:  unreachable (connection refused)\n",
+		"\nservers (namespace minecraft): unknown while the cluster is unreachable\n",
+		"  off-site: not configured, every backup is on this machine only\n",
+		"  timer active, last run",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("status lacks %q:\n%s", want, out.String())
+		}
+	}
+
+	env = statusTestEnv(t)
+	env.backups = func(context.Context) (map[string]time.Time, error) { return nil, errors.New("postgres is down") }
+	out.Reset()
+	printStatus(context.Background(), env, &out)
+	if f := strings.Join(lineFields(out.String(), "survival"), " "); f != "survival - Running Running 2/20 ?" {
+		t.Errorf("survival with PostgreSQL down = %q", f)
+	}
+	if !strings.Contains(out.String(), "  world backups unknown: postgres is down\n") {
+		t.Errorf("status does not say why the backups are unknown:\n%s", out.String())
+	}
+}
+
+func TestShortImage(t *testing.T) {
+	for in, want := range map[string]string{
+		"registry.felis.svc:5000/felis/felis-api:v1.4.0":                                       "felis-api:v1.4.0",
+		"docker.io/library/postgres:17@sha256:0123456789abcdef0123":                            "postgres:17@0123456789ab",
+		"felis-operator@sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba98765432": "felis-operator@fedcba987654",
+		"busybox": "busybox",
+	} {
+		if got := shortImage(in); got != want {
+			t.Errorf("shortImage(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+// A node that is not ready, a Deployment without replicas or containers, and
+// lists the API server refuses each read as such.
+func TestStatusClusterEdges(t *testing.T) {
+	env := statusTestEnv(t)
+	env.cfg = &config.Config{K8s: config.K8sConfig{Namespace: "games"}}
+	env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}, Status: corev1.NodeStatus{
+			Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionFalse}, {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionTrue}}}},
+		&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-bare", Namespace: "felis"}},
+		&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "felis"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 4}}}},
+	).Build()
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	got := out.String()
+	for _, want := range []string{
+		"node:     felis-1 NotReady, k3s , , kernel \n",
+		"\nservers (namespace games): 0, 0 running, 0 players online\n\nbackups:\n",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("status lacks %q:\n%s", want, got)
+		}
+	}
+	if f := strings.Join(lineFields(got, "felis-bare"), " "); f != "felis-bare 0/1 ready - restarts 0" {
+		t.Errorf("row felis-bare = %q, want its one replica wanted, no image, and no pod of its own:\n%s", f, got)
+	}
+
+	failing := func(what client.ObjectList) {
+		env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).
+			WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
+				if fmt.Sprintf("%T", list) == fmt.Sprintf("%T", what) {
+					return errors.New("forbidden")
+				}
+				return c.List(ctx, list, opts...)
+			}}).Build()
+		out.Reset()
+		printStatus(context.Background(), env, &out)
+	}
+	for _, tc := range []struct {
+		list client.ObjectList
+		want string
+	}{
+		{&corev1.NodeList{}, "cluster:  unreachable (forbidden)\n\nproxy:"},
+		{&appsv1.DeploymentList{}, "\ncontrol plane (namespace felis):\n  cannot list it: forbidden\n\nproxy:"},
+		{&corev1.PodList{}, "\ncontrol plane (namespace felis):\n  cannot list it: forbidden\n\nproxy:"},
+		{&v1alpha1.MinecraftServerList{}, "\nservers (namespace games): cannot list them: forbidden\n\nbackups:"},
+	} {
+		failing(tc.list)
+		if !strings.Contains(out.String(), tc.want) {
+			t.Errorf("listing %T refused: status lacks %q:\n%s", tc.list, tc.want, out.String())
+		}
+	}
+}
+
+func TestStatusProxy(t *testing.T) {
+	env := statusTestEnv(t)
+	writeTestFile(t, filepath.Join(env.unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
+	env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active"})
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	env.w.proxyAddr = ln.Addr().String()
+	var out bytes.Buffer
+	statusProxy(context.Background(), env, &out)
+	if want := "\nproxy:    felis-velocity active, " + env.w.proxyAddr + " accepts connections\n"; out.String() != want {
+		t.Errorf("proxy listening: %q, want %q", out.String(), want)
+	}
+	ln.Close()
+	out.Reset()
+	statusProxy(context.Background(), env, &out)
+	if want := "\nproxy:    felis-velocity active, " + env.w.proxyAddr + " refuses connections (dial tcp " + env.w.proxyAddr + ": "; !strings.HasPrefix(out.String(), want) {
+		t.Errorf("proxy gone: %q, want it to start %q", out.String(), want)
+	}
+}
+
+func TestStatusBackups(t *testing.T) {
+	env := statusTestEnv(t)
+	status := func() string {
+		var out bytes.Buffer
+		statusBackups(env, &out)
+		return out.String()
+	}
+	for _, tc := range []struct {
+		what, dir, want string
+	}{
+		{"no -backup-dir", "", "  database: not checked (felis-watchdog.service names no -backup-dir)\n"},
+		{"an empty directory", t.TempDir(), "  database: none in %s\n"},
+		{"a file where the directory should be", filepath.Join(env.w.backupDir, dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily)), "  database: cannot read %s: "},
+	} {
+		env.w.backupDir = tc.dir
+		want := tc.want
+		if strings.Contains(want, "%s") {
+			want = fmt.Sprintf(want, tc.dir)
+		}
+		if got := status(); !strings.Contains(got, want) {
+			t.Errorf("%s: %q, want %q", tc.what, got, want)
+		}
+	}
+
+	for _, tc := range []struct {
+		what   string
+		status *offsite.Status
+		raw    string
+		want   string
+	}{
+		{"never synced", nil, "", "  off-site: never synced\n"},
+		{"never succeeded", &offsite.Status{LastAttempt: env.now.Add(-time.Hour), LastError: "403 Forbidden"}, "", "  off-site: never succeeded; last attempt 1h0m ago: 403 Forbidden\n"},
+		{"the last attempt failed", &offsite.Status{LastAttempt: env.now.Add(-30 * time.Minute), LastSuccess: env.now.Add(-26 * time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
+			"  off-site: last good sync 26h0m ago to felis-dr; the last attempt, 30m ago, failed: timeout\n"},
+		{"an error an earlier attempt left", &offsite.Status{LastAttempt: env.now.Add(-2 * time.Hour), LastSuccess: env.now.Add(-time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
+			"  off-site: last good sync 1h0m ago to felis-dr\n"},
+		{"an unreadable record", nil, "{", "  off-site: unexpected end of JSON input\n"},
+	} {
+		os.Remove(env.w.offsiteStatus)
+		switch {
+		case tc.status != nil:
+			if err := offsite.WriteStatus(env.w.offsiteStatus, *tc.status); err != nil {
+				t.Fatal(err)
+			}
+		case tc.raw != "":
+			writeTestFile(t, env.w.offsiteStatus, tc.raw, 0o600)
+		}
+		if got := status(); !strings.HasSuffix(got, tc.want) {
+			t.Errorf("%s: %q, want it to end %q", tc.what, got, tc.want)
+		}
+	}
+}
+
+func TestStatusHost(t *testing.T) {
+	env := statusTestEnv(t)
+	dir := t.TempDir()
+	env.w.diskPaths = "/nonexistent-felis-status-test," + dir + "," + dir
+	env.meminfo = filepath.Join(dir, "meminfo")
+	var out bytes.Buffer
+	statusHost(env, &out)
+	lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
+	if len(lines) != 3 || lines[0] != "" || lines[1] != "host:" || !strings.HasPrefix(lines[2], "  disk "+dir+": ") || !strings.HasSuffix(lines[2], "% free)") {
+		t.Errorf("host: %q, want one line for %s (a path on a disk already shown, and one that is not there, print none) and no memory line", lines, dir)
+	}
+
+	writeTestFile(t, env.meminfo, "MemTotal: 4096 kB\ngarbage\nMemAvailable: 1024 kB\n", 0o644)
+	if total, avail, ok := readMeminfo(env.meminfo); !ok || total != 4096*1024 || avail != 1024*1024 {
+		t.Errorf("readMeminfo = %d, %d, %v", total, avail, ok)
+	}
+	writeTestFile(t, env.meminfo, "MemAvailable: 1024 kB\n", 0o644)
+	if _, _, ok := readMeminfo(env.meminfo); ok {
+		t.Error("readMeminfo without MemTotal: ok")
+	}
+}
+
+func TestStatusWatchdog(t *testing.T) {
+	env := statusTestEnv(t)
+	status := func() string {
+		var out bytes.Buffer
+		statusWatchdog(context.Background(), env, &out)
+		return out.String()
+	}
+	run := env.run
+	env.run = func(ctx context.Context, name string, args ...string) ([]byte, error) {
+		if len(args) > 0 && args[0] == "show" {
+			return []byte("Result=success\nExecMainExitTimestamp=\n"), nil
+		}
+		return run(ctx, name, args...)
+	}
+	if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: map[string]*watchdog.Alert{
+		"proxy": {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: env.now.Add(-time.Hour), ClearedAt: env.now.Add(-time.Minute)},
+	}}); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := status(), "\nwatchdog:\n  timer active\n  alerts: none open\n"; got != want {
+		t.Errorf("a watchdog that has not run and has nothing open: %q, want %q", got, want)
+	}
+
+	writeTestFile(t, env.w.statePath, "{", 0o600)
+	if got := status(); !strings.Contains(got, "\n  alerts: unknown (") {
+		t.Errorf("an unreadable state: %q", got)
+	}
+
+	if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := status(), "\nwatchdog:\n  not installed: nothing checks this host\n"; got != want {
+		t.Errorf("no watchdog timer: %q, want %q", got, want)
+	}
+}
+
+// Rows print by name in whatever order the API server lists them, a server
+// the operator has not reconciled yet reads as dashes, and open alerts print
+// by key in whatever order the state's map yields them.
+func TestStatusOrder(t *testing.T) {
+	env := statusTestEnv(t)
+	objs := append(statusClusterObjects(),
+		&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-operator", Namespace: "felis"}},
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "fresh", Namespace: "minecraft"}})
+	env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
+		WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
+			// The fake lists by name; the other way round, then.
+			if err := c.List(ctx, list, opts...); err != nil {
+				return err
+			}
+			items, err := meta.ExtractList(list)
+			if err != nil {
+				return err
+			}
+			slices.Reverse(items)
+			return meta.SetList(list, items)
+		}}).Build()
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	var rows []string
+	for _, l := range strings.Split(out.String(), "\n") {
+		if f := strings.Fields(l); len(f) > 0 && (strings.HasPrefix(f[0], "felis-") || slices.Contains([]string{"creative", "fresh", "lobby", "survival"}, f[0])) {
+			rows = append(rows, strings.Join(f, " "))
+		}
+	}
+	want := []string{
+		"felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
+		"felis-operator 0/1 ready - restarts 0",
+		"creative - Stopped Stopped - none",
+		"fresh - - - - none",
+		"lobby lobby Running Starting - none",
+		"survival - Running Running 2/20 3h0m ago",
+	}
+	if !slices.Equal(rows, want) {
+		t.Errorf("rows %q, want %q:\n%s", rows, want, out.String())
+	}
+
+	alerts := map[string]*watchdog.Alert{}
+	for i, key := range []string{"a", "b", "c", "d"} {
+		alerts[key] = &watchdog.Alert{Finding: watchdog.Finding{Key: key, Severity: watchdog.Warning}, FirstSeen: env.now.Add(-time.Duration(i+1) * time.Minute)}
+	}
+	if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: alerts}); err != nil {
+		t.Fatal(err)
+	}
+	wantAlerts := "  alerts: 4 open (sudo felis doctor says where to look)\n" +
+		"    a (warning, seen 1m ago, not mailed yet)\n    b (warning, seen 2m ago, not mailed yet)\n" +
+		"    c (warning, seen 3m ago, not mailed yet)\n    d (warning, seen 4m ago, not mailed yet)\n"
+	// A map starts its walk at random: fifty walks all in order by chance
+	// is out of the question.
+	for range 50 {
+		out.Reset()
+		statusWatchdog(context.Background(), env, &out)
+		if !strings.HasSuffix(out.String(), wantAlerts) {
+			t.Fatalf("alerts %q, want them to end %q", out.String(), wantAlerts)
+		}
+	}
+}
+
+// A selector the API server would have refused picks no pods.
+func TestPodRestartsBadSelector(t *testing.T) {
+	pods := []corev1.Pod{{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 3}}}}}
+	if n := podRestarts(&metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "app", Operator: "Near"}}}, pods); n != 0 {
+		t.Errorf("podRestarts with a bad selector = %d, want 0", n)
+	}
+	if n := podRestarts(&metav1.LabelSelector{}, pods); n != 3 {
+		t.Errorf("podRestarts with a selector that picks all = %d, want 3", n)
+	}
+}

cmd/felis/supportbundle.go

@@ -0,0 +1,862 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+)
+
+// supportBundleDir is where felis support-bundle writes unless told otherwise.
+const supportBundleDir = "/var/lib/felis/support"
+
+// redacted stands in for every value the bundle leaves out.
+const redacted = "<redacted>"
+
+// minScrubLen is the shortest known secret value the bundle searches for: a
+// shorter one would blank ordinary words, and every secret the installer
+// generates is far longer.
+const minScrubLen = 8
+
+// hostSecretSources are the files on a Felis host that hold secrets; the
+// bundle reads them only to take each value out of what it collects.
+var hostSecretSources = secretSources{
+	envFiles:   []string{"/etc/felis/secrets.env", defaultOffsiteEnvFile},
+	valueFiles: []string{hostSMTPPasswordPath, hostUploadsS3AccessKeyPath, hostUploadsS3SecretKeyPath, defaultHeartbeatFile, "/opt/felis/velocity/forwarding.secret"},
+	propsFiles: []string{"/opt/felis/velocity/plugins/felis-link/felis-link.properties"},
+	tokenFiles: []string{"/var/lib/rancher/k3s/server/token", "/var/lib/rancher/k3s/server/agent-token"},
+}
+
+// cmdSupportBundle collects what someone helping with this host needs into
+// one tar.gz: felis status and felis doctor, the logs of the control plane,
+// the builds and the systemd units, the cluster's workloads and events, and a
+// summary of the configuration. It never collects a Secret, a ConfigMap, the
+// contents of a configuration file, the database or a world, and takes every
+// value of the host's secret files out of what it does collect. Game server
+// logs, which carry player names, addresses and chat, only with -server-logs.
+func cmdSupportBundle(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("support-bundle", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	outDir := fs.String("o", supportBundleDir, "directory to write the bundle to (created mode 0700 when missing)")
+	logLines := fs.Int64("log-lines", 2000, "lines kept from the end of each pod log and each unit's journal")
+	since := fs.Duration("since", 48*time.Hour, "how far back each unit's journal is read")
+	serverLogs := fs.Bool("server-logs", false, "also collect the game servers' own logs, which carry player names, IP addresses and chat")
+	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis support-bundle: run as root (sudo felis support-bundle): it reads root-only logs and state")
+		return 1
+	}
+	b := hostSupportBundle(*unitDir, *logLines, *since, *serverLogs)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	path, err := b.write(ctx, *outDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis support-bundle: %v\n", err)
+		return 1
+	}
+	size := int64(0)
+	if st, err := os.Stat(path); err == nil {
+		size = st.Size()
+	}
+	fmt.Fprintf(stdout, "wrote %s (%s, mode 0600)\n", path, humanSize(size))
+	fmt.Fprintln(stdout, "MANIFEST.txt inside says what it holds and what was taken out. Read it through before you send it anywhere:")
+	fmt.Fprintln(stdout, "redaction finds this host's known secrets and the common ways a secret is logged, and a secret logged another way stays in.")
+	if !*serverLogs {
+		fmt.Fprintln(stdout, "Game server logs are left out; -server-logs adds them (player names, IP addresses, chat).")
+	}
+	return 0
+}
+
+func humanSize(n int64) string {
+	switch {
+	case n >= 1<<20:
+		return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
+	case n >= 1<<10:
+		return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10))
+	}
+	return fmt.Sprintf("%d B", n)
+}
+
+// shortDuration is d as Duration.String writes it, less the zero minutes and
+// seconds after whole hours or minutes: 48h, and 90m as 1h30m.
+func shortDuration(d time.Duration) string {
+	s := d.String()
+	if strings.HasSuffix(s, "m0s") {
+		s = strings.TrimSuffix(s, "0s")
+	}
+	if strings.HasSuffix(s, "h0m") {
+		s = strings.TrimSuffix(s, "0m")
+	}
+	return s
+}
+
+// supportBundle is one collection and what it reads the host through.
+type supportBundle struct {
+	host    string
+	now     time.Time
+	unitDir string
+	// w is how felis-watchdog.service runs the watchdog, wErr why it could
+	// not be read (w then holds the defaults).
+	w          watchdogFlags
+	wErr       error
+	cfg        *config.Config // nil when cfgErr
+	cfgErr     error
+	cl         client.Client // nil when clErr
+	clErr      error
+	logs       func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error)
+	run        func(ctx context.Context, name string, args ...string) ([]byte, error)
+	backups    func(ctx context.Context) (map[string]time.Time, error)
+	doctor     func(ctx context.Context, out io.Writer)
+	secrets    secretSources
+	logLines   int64
+	since      time.Duration
+	serverLogs bool
+	// Host files read whole, and the directory whose listing is kept.
+	meminfo, osRelease, procVersion, stateDir string
+}
+
+func hostSupportBundle(unitDir string, logLines int64, since time.Duration, serverLogs bool) *supportBundle {
+	host, _ := os.Hostname()
+	b := &supportBundle{
+		host: host, now: time.Now(), unitDir: unitDir, run: hostCommand, secrets: hostSecretSources,
+		logLines: logLines, since: since, serverLogs: serverLogs,
+		meminfo: "/proc/meminfo", osRelease: "/etc/os-release", procVersion: "/proc/version", stateDir: "/etc/felis",
+	}
+	var found bool
+	b.w, found, b.wErr = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
+	if b.wErr == nil && !found {
+		b.wErr = fmt.Errorf("%s is not installed; read the watchdog's defaults", filepath.Join(unitDir, "felis-watchdog.service"))
+	}
+	if b.cfg, b.cfgErr = config.Load(b.w.cfgPath); b.cfgErr == nil {
+		cfg := b.cfg
+		b.backups = func(ctx context.Context) (map[string]time.Time, error) {
+			return newestWorldBackups(ctx, cfg.Database.URL)
+		}
+	} else {
+		b.cfg = nil
+	}
+	if b.cl, b.clErr = buildSystemServerClient(); b.clErr != nil {
+		b.cl = nil
+	} else if rc, err := hostRESTConfig(); err != nil {
+		b.clErr = err
+		b.cl = nil
+	} else if cs, err := kubernetes.NewForConfig(rc); err != nil {
+		b.clErr = err
+		b.cl = nil
+	} else {
+		limit := int64(8 << 20)
+		b.logs = func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error) {
+			return cs.CoreV1().Pods(ns).GetLogs(pod, &corev1.PodLogOptions{
+				Container: container, Previous: previous, Timestamps: true, TailLines: &logLines, LimitBytes: &limit,
+			}).DoRaw(ctx)
+		}
+	}
+	b.doctor = func(ctx context.Context, out io.Writer) {
+		runDoctor(ctx, doctorEnv{unitDir: unitDir, run: hostCommand, now: b.now, host: host}, out)
+	}
+	return b
+}
+
+// bundleWriter streams scrubbed files into the archive and keeps what went
+// wrong while collecting, for MANIFEST.txt.
+type bundleWriter struct {
+	tw     *tar.Writer
+	prefix string
+	now    time.Time
+	scrub  *scrubber
+	errs   []string
+}
+
+// logText adds a log, or a report that quotes errors: known secrets and
+// anything logged as one come out.
+func (bw *bundleWriter) logText(name string, data []byte) error {
+	return bw.add(name, bw.scrub.text(data))
+}
+
+// plain adds a file whose secrets were already taken out by structure (the
+// cluster's objects, the configuration summary) or that names none (the
+// release, disk use, addresses): known secret values still come out, and
+// nothing else is rewritten.
+func (bw *bundleWriter) plain(name string, data []byte) error {
+	return bw.add(name, bw.scrub.values(data))
+}
+
+func (bw *bundleWriter) add(name string, data []byte) error {
+	hdr := &tar.Header{Name: path.Join(bw.prefix, name), Mode: 0o600, Size: int64(len(data)), ModTime: bw.now, Typeflag: tar.TypeReg}
+	if err := bw.tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+	_, err := bw.tw.Write(data)
+	return err
+}
+
+func (bw *bundleWriter) failed(what string, err error) {
+	bw.errs = append(bw.errs, string(bw.scrub.text([]byte(fmt.Sprintf("%s: %v", what, err)))))
+}
+
+// write collects the bundle into dir and returns its path.
+func (b *supportBundle) write(ctx context.Context, dir string) (string, error) {
+	if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {
+		if err := os.MkdirAll(dir, 0o700); err != nil {
+			return "", err
+		}
+	}
+	stamp := b.now.UTC().Format("20060102T150405Z")
+	base := fmt.Sprintf("felis-support-%s-%s", safeName(b.host), stamp)
+	final := filepath.Join(dir, base+".tar.gz")
+	f, err := os.CreateTemp(dir, "."+base+".*.partial") // mode 0600
+	if err != nil {
+		return "", err
+	}
+	keep := false
+	defer func() {
+		if !keep {
+			f.Close()
+			os.Remove(f.Name())
+		}
+	}()
+	gz := gzip.NewWriter(f)
+	bw := &bundleWriter{tw: tar.NewWriter(gz), prefix: base, now: b.now, scrub: b.scrubber()}
+	if err := b.collect(ctx, bw); err != nil {
+		return "", err
+	}
+	if err := bw.tw.Close(); err != nil {
+		return "", err
+	}
+	if err := gz.Close(); err != nil {
+		return "", err
+	}
+	if err := f.Sync(); err != nil {
+		return "", err
+	}
+	if err := f.Close(); err != nil {
+		return "", err
+	}
+	if err := os.Rename(f.Name(), final); err != nil {
+		return "", err
+	}
+	keep = true
+	return final, nil
+}
+
+// safeName keeps a host name usable in a file name.
+func safeName(s string) string {
+	s = strings.Map(func(r rune) rune {
+		if r == '-' || r == '.' || r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
+			return r
+		}
+		return '_'
+	}, s)
+	if s == "" {
+		return "host"
+	}
+	return s
+}
+
+func (b *supportBundle) collect(ctx context.Context, bw *bundleWriter) error {
+	var buf bytes.Buffer
+	cmdVersion(nil, &buf, io.Discard)
+	for _, p := range []string{b.osRelease, b.procVersion} {
+		if raw, err := os.ReadFile(p); err == nil {
+			fmt.Fprintf(&buf, "\n# %s\n%s", p, raw)
+		}
+	}
+	if err := bw.plain("version.txt", buf.Bytes()); err != nil {
+		return err
+	}
+
+	buf.Reset()
+	switch {
+	case b.cfgErr != nil:
+		fmt.Fprintf(&buf, "felis status: the configuration did not load: %v\n", b.cfgErr)
+	default:
+		printStatus(ctx, statusEnv{
+			cfg: b.cfg, w: b.w, cl: b.cl, clErr: b.clErr, backups: b.backups, run: b.run,
+			unitDir: b.unitDir, meminfo: b.meminfo, host: b.host, now: b.now,
+		}, &buf)
+	}
+	if err := bw.logText("status.txt", buf.Bytes()); err != nil {
+		return err
+	}
+	buf.Reset()
+	b.doctor(ctx, &buf)
+	if err := bw.logText("doctor.txt", buf.Bytes()); err != nil {
+		return err
+	}
+	if b.cfg != nil {
+		if err := bw.plain("config.txt", configSummary(b.cfg, b.w.cfgPath)); err != nil {
+			return err
+		}
+	}
+	if err := b.collectHost(ctx, bw); err != nil {
+		return err
+	}
+	if err := b.collectJournal(ctx, bw); err != nil {
+		return err
+	}
+	if b.cl == nil {
+		bw.failed("cluster", b.clErr)
+	} else if err := b.collectCluster(ctx, bw); err != nil {
+		return err
+	}
+	return bw.add("MANIFEST.txt", b.manifest(bw))
+}
+
+func (b *supportBundle) collectHost(ctx context.Context, bw *bundleWriter) error {
+	commands := []struct {
+		name string
+		argv []string
+	}{
+		{"host/systemd-units.txt", []string{"systemctl", "list-units", "--all", "--no-pager", "--plain", "felis-*", "k3s.service"}},
+		{"host/systemd-timers.txt", []string{"systemctl", "list-timers", "--all", "--no-pager", "felis-*"}},
+		{"host/df.txt", []string{"df", "-h"}},
+		{"host/addresses.txt", []string{"ip", "-brief", "address"}},
+	}
+	for _, c := range commands {
+		out, err := b.run(ctx, c.argv[0], c.argv[1:]...)
+		if err != nil && len(out) == 0 {
+			bw.failed(strings.Join(c.argv, " "), err)
+			continue
+		}
+		if err := bw.plain(c.name, out); err != nil {
+			return err
+		}
+	}
+	if raw, err := os.ReadFile(b.meminfo); err == nil {
+		if err := bw.plain("host/meminfo.txt", raw); err != nil {
+			return err
+		}
+	}
+	listing, err := dirListing(b.stateDir)
+	if err != nil {
+		bw.failed("list "+b.stateDir, err)
+		return nil
+	}
+	return bw.plain("host/etc-felis.txt", listing)
+}
+
+// dirListing names every file under dir with its mode, size and time, and
+// holds nothing of what is in them.
+func dirListing(dir string) ([]byte, error) {
+	var buf bytes.Buffer
+	tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
+	fmt.Fprintf(tw, "# %s: names, modes, sizes and times only; no contents\n", dir)
+	err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(tw, "%s\t%d\t%s\t%s\n", info.Mode(), info.Size(), info.ModTime().UTC().Format(time.RFC3339), p)
+		return nil
+	})
+	tw.Flush()
+	return buf.Bytes(), err
+}
+
+func (b *supportBundle) collectJournal(ctx context.Context, bw *bundleWriter) error {
+	units, _ := filepath.Glob(filepath.Join(b.unitDir, "felis-*.service"))
+	if _, err := os.Stat(filepath.Join(b.unitDir, "k3s.service")); err == nil {
+		units = append(units, filepath.Join(b.unitDir, "k3s.service"))
+	}
+	since := "@" + strconv.FormatInt(b.now.Add(-b.since).Unix(), 10)
+	for _, u := range units {
+		unit := filepath.Base(u)
+		out, err := b.run(ctx, "journalctl", "-u", unit, "--since", since, "-n", strconv.FormatInt(b.logLines, 10), "--no-pager", "-o", "short-iso")
+		if err != nil && len(out) == 0 {
+			bw.failed("journalctl -u "+unit, err)
+			continue
+		}
+		if err := bw.logText("journal/"+strings.TrimSuffix(unit, ".service")+".log", out); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// bundleNamespaces are the namespaces whose objects and logs the bundle
+// collects: the control plane, the builds and the game servers.
+func (b *supportBundle) bundleNamespaces() (control, build, minecraft string) {
+	control, build, minecraft = b.w.controlNS, platform.DefaultBuildNamespace, platform.DefaultMinecraftNamespace
+	if b.cfg != nil {
+		if b.cfg.Registry.BuildNamespace != "" {
+			build = b.cfg.Registry.BuildNamespace
+		}
+		if b.cfg.K8s.Namespace != "" {
+			minecraft = b.cfg.K8s.Namespace
+		}
+	}
+	return control, build, minecraft
+}
+
+func (b *supportBundle) collectCluster(ctx context.Context, bw *bundleWriter) error {
+	control, build, minecraft := b.bundleNamespaces()
+	dump := func(name string, list client.ObjectList, opts ...client.ListOption) error {
+		if err := b.cl.List(ctx, list, opts...); err != nil {
+			bw.failed("list "+name, err)
+			return nil
+		}
+		redactList(list)
+		out, err := yaml.Marshal(list)
+		if err != nil {
+			bw.failed("encode "+name, err)
+			return nil
+		}
+		return bw.plain(name, out)
+	}
+	if err := dump("cluster/nodes.yaml", &corev1.NodeList{}); err != nil {
+		return err
+	}
+	if err := dump("cluster/persistentvolumes.yaml", &corev1.PersistentVolumeList{}); err != nil {
+		return err
+	}
+	if err := dump("cluster/minecraftservers.yaml", &v1alpha1.MinecraftServerList{}, client.InNamespace(minecraft)); err != nil {
+		return err
+	}
+	for _, ns := range []string{control, build, minecraft} {
+		for _, k := range []struct {
+			name string
+			list client.ObjectList
+		}{
+			{"pods", &corev1.PodList{}},
+			{"deployments", &appsv1.DeploymentList{}},
+			{"statefulsets", &appsv1.StatefulSetList{}},
+			{"jobs", &batchv1.JobList{}},
+			{"cronjobs", &batchv1.CronJobList{}},
+			{"services", &corev1.ServiceList{}},
+			{"persistentvolumeclaims", &corev1.PersistentVolumeClaimList{}},
+			{"networkpolicies", &networkingv1.NetworkPolicyList{}},
+			{"events", &corev1.EventList{}},
+		} {
+			if err := dump("cluster/"+ns+"/"+k.name+".yaml", k.list, client.InNamespace(ns)); err != nil {
+				return err
+			}
+		}
+	}
+
+	var all corev1.PodList
+	if err := b.cl.List(ctx, &all); err != nil {
+		bw.failed("list every pod", err)
+	} else if err := bw.plain("cluster/pods-all-namespaces.txt", podTable(all.Items, b.now)); err != nil {
+		return err
+	}
+
+	for _, ns := range []string{control, build, minecraft} {
+		var pods corev1.PodList
+		if err := b.cl.List(ctx, &pods, client.InNamespace(ns)); err != nil {
+			continue // already recorded by the dump above
+		}
+		for _, p := range pods.Items {
+			if err := b.collectPodLogs(ctx, bw, p, ns == minecraft && !b.serverLogs); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// collectPodLogs keeps the tail of each container's log, and of its previous
+// run when it restarted. initOnly keeps only the init containers: a game
+// server's own log carries player names, addresses and chat.
+func (b *supportBundle) collectPodLogs(ctx context.Context, bw *bundleWriter, p corev1.Pod, initOnly bool) error {
+	type ctr struct {
+		name     string
+		restarts int32
+	}
+	var ctrs []ctr
+	restarts := map[string]int32{}
+	for _, cs := range append(append([]corev1.ContainerStatus(nil), p.Status.InitContainerStatuses...), p.Status.ContainerStatuses...) {
+		restarts[cs.Name] = cs.RestartCount
+	}
+	for _, c := range p.Spec.InitContainers {
+		ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
+	}
+	if !initOnly {
+		for _, c := range p.Spec.Containers {
+			ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
+		}
+	}
+	for _, c := range ctrs {
+		for _, previous := range []bool{false, true} {
+			if previous && c.restarts == 0 {
+				continue
+			}
+			name := fmt.Sprintf("logs/%s/%s/%s.log", p.Namespace, p.Name, c.name)
+			if previous {
+				name = fmt.Sprintf("logs/%s/%s/%s.previous.log", p.Namespace, p.Name, c.name)
+			}
+			out, err := b.logs(ctx, p.Namespace, p.Name, c.name, previous)
+			if err != nil {
+				bw.failed(name, err)
+				continue
+			}
+			if err := bw.logText(name, out); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// podTable is every pod on the node, one line each.
+func podTable(pods []corev1.Pod, now time.Time) []byte {
+	sort.Slice(pods, func(i, j int) bool {
+		if pods[i].Namespace != pods[j].Namespace {
+			return pods[i].Namespace < pods[j].Namespace
+		}
+		return pods[i].Name < pods[j].Name
+	})
+	var buf bytes.Buffer
+	tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "NAMESPACE\tNAME\tPHASE\tREADY\tRESTARTS\tAGE")
+	for _, p := range pods {
+		var ready, restarts int32
+		for _, cs := range p.Status.ContainerStatuses {
+			if cs.Ready {
+				ready++
+			}
+			restarts += cs.RestartCount
+		}
+		age := "-"
+		if !p.CreationTimestamp.IsZero() {
+			age = now.Sub(p.CreationTimestamp.Time).Round(time.Minute).String()
+		}
+		fmt.Fprintf(tw, "%s\t%s\t%s\t%d/%d\t%d\t%s\n", p.Namespace, p.Name, p.Status.Phase, ready, len(p.Spec.Containers), restarts, age)
+	}
+	tw.Flush()
+	return buf.Bytes()
+}
+
+// redactList takes out of every object what the bundle must not carry: the
+// literal env values of pod specs and of MinecraftServers (valueFrom
+// references stay, naming the Secret without its contents), the
+// last-applied-configuration annotation that repeats them, and managedFields.
+func redactList(list client.ObjectList) {
+	items, err := meta.ExtractList(list)
+	if err != nil {
+		return
+	}
+	for _, it := range items {
+		if acc, err := meta.Accessor(it); err == nil {
+			acc.SetManagedFields(nil)
+			if ann := acc.GetAnnotations(); ann != nil {
+				delete(ann, corev1.LastAppliedConfigAnnotation)
+				acc.SetAnnotations(ann)
+			}
+		}
+		switch o := it.(type) {
+		case *corev1.Pod:
+			redactPodSpec(&o.Spec)
+		case *appsv1.Deployment:
+			redactPodSpec(&o.Spec.Template.Spec)
+		case *appsv1.StatefulSet:
+			redactPodSpec(&o.Spec.Template.Spec)
+		case *batchv1.Job:
+			redactPodSpec(&o.Spec.Template.Spec)
+		case *batchv1.CronJob:
+			redactPodSpec(&o.Spec.JobTemplate.Spec.Template.Spec)
+		case *v1alpha1.MinecraftServer:
+			for i := range o.Spec.Env {
+				if o.Spec.Env[i].Value != "" {
+					o.Spec.Env[i].Value = redacted
+				}
+			}
+		}
+	}
+}
+
+func redactPodSpec(s *corev1.PodSpec) {
+	blank := func(cs []corev1.Container) {
+		for i := range cs {
+			for j := range cs[i].Env {
+				if cs[i].Env[j].Value != "" {
+					cs[i].Env[j].Value = redacted
+				}
+			}
+		}
+	}
+	blank(s.InitContainers)
+	blank(s.Containers)
+	for i := range s.EphemeralContainers {
+		for j := range s.EphemeralContainers[i].Env {
+			if s.EphemeralContainers[i].Env[j].Value != "" {
+				s.EphemeralContainers[i].Env[j].Value = redacted
+			}
+		}
+	}
+}
+
+// configSummary is felis.toml without a single credential: the hostnames,
+// namespaces and which features are on. Fields are picked one by one, so a
+// field added later stays out until someone decides it is safe.
+func configSummary(c *config.Config, path string) []byte {
+	var buf bytes.Buffer
+	line := func(k string, v any) { fmt.Fprintf(&buf, "%-34s %v\n", k, v) }
+	fmt.Fprintf(&buf, "# a summary of %s; no password, key or token is in it\n", path)
+	line("server.root_domain", c.Server.RootDomain)
+	line("server.listen", c.Server.Listen)
+	db := "(unparsable)"
+	if u, err := url.Parse(c.Database.URL); err == nil {
+		db = u.Scheme + "://" + u.User.Username() + "@" + u.Host + u.Path
+	}
+	line("database.url (no password)", db)
+	line("database.deployment", c.Database.Deployment)
+	line("velocity.public_ip", c.Velocity.PublicIP)
+	line("velocity.game_port", c.Velocity.GamePort)
+	line("velocity.login_image", c.Velocity.LoginImage)
+	line("velocity.lobby_image", c.Velocity.LobbyImage)
+	line("auth.panel_hostname", c.Auth.PanelHostname)
+	line("auth.admin_hostname", c.Auth.AdminHostname)
+	line("auth.client_ip_header", c.Auth.ClientIPHeader)
+	line("k8s.namespace", c.K8s.Namespace)
+	line("k8s.egress_mode", c.K8s.EgressMode)
+	line("k8s.metallb_pool", c.K8s.MetalLBPool)
+	line("registry.url", c.Registry.URL)
+	line("registry.build_namespace", c.Registry.BuildNamespace)
+	line("registry.trivy_db_repository", c.Registry.TrivyDBRepository)
+	line("registry.build_user_namespaces", c.Registry.BuildUserNamespaces)
+	line("registry.build_runtime_class", c.Registry.BuildRuntimeClass)
+	line("registry.max_concurrent_builds", c.Registry.MaxConcurrentBuilds)
+	line("registry.user_uploads_context", c.Registry.UserUploadsContext)
+	line("archive.store", c.Archive.Store)
+	line("archive.local_path", c.Archive.LocalPath)
+	line("archive.retention", c.Archive.Retention)
+	line("archive.scheduled_every", c.Archive.ScheduledEvery)
+	line("archive.scheduled_keep", c.Archive.ScheduledKeep)
+	line("offsite (configured)", c.Offsite.Enabled())
+	if c.Offsite.Enabled() {
+		line("offsite.endpoint", c.Offsite.Endpoint)
+		line("offsite.bucket", c.Offsite.Bucket)
+		line("offsite.prefix", c.Offsite.Prefix)
+	}
+	line("smtp.host", c.SMTP.Host)
+	if c.SMTP.Host != "" {
+		line("smtp.port", c.SMTP.Port)
+		line("smtp.require_tls", c.SMTP.TLSRequired())
+		line("smtp.max_per_hour", c.SMTP.MaxPerHour)
+	}
+	for i, s := range c.AuthSources {
+		line(fmt.Sprintf("auth_source[%d]", i), s.Tag+" "+s.Prefix+" "+s.URL)
+	}
+	return buf.Bytes()
+}
+
+func (b *supportBundle) manifest(bw *bundleWriter) []byte {
+	control, build, minecraft := b.bundleNamespaces()
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "Felis support bundle\nhost %s, collected %s, felis %s\n\n", b.host, b.now.UTC().Format(time.RFC3339), resolvedVersion())
+	fmt.Fprintf(&buf, `What it holds:
+  status.txt, doctor.txt   felis status and felis doctor at collection time
+  version.txt              the release, the OS and the kernel
+  config.txt               a summary of felis.toml: hostnames, namespaces, which features are on
+  host/                    systemd units and timers, disk use, memory, addresses, and the names,
+                           modes, sizes and times of the files under %s (not what is in them)
+  journal/                 up to %d lines per Felis unit and k3s, from the last %s
+  logs/                    up to %d lines of each container of the pods in %s and %s, and of the
+                           init containers of the game server pods in %s; the previous run too
+                           where a container restarted
+  cluster/                 nodes, volumes, the MinecraftServers, and the pods, workloads, services,
+                           volume claims, network policies and events of %s, %s and %s
+`, b.stateDir, b.logLines, shortDuration(b.since), b.logLines, control, build, minecraft, control, build, minecraft)
+	if b.serverLogs {
+		fmt.Fprintln(&buf, "\nThe game servers' own logs are in logs/ (-server-logs): they carry player names, IP addresses and chat.")
+	} else {
+		fmt.Fprintln(&buf, "\nThe game servers' own logs are left out (they carry player names, IP addresses and chat; -server-logs adds them).")
+	}
+	fmt.Fprint(&buf, `
+Never collected: Kubernetes Secrets and ConfigMaps, what is in /etc/felis or any configuration
+file, the database, worlds, uploads.
+
+Taken out:
+`)
+	if len(bw.scrub.sources) > 0 {
+		fmt.Fprintf(&buf, "  - %d secret values, wherever they appear, read from:\n", len(bw.scrub.vals))
+		for _, s := range bw.scrub.sources {
+			fmt.Fprintf(&buf, "      %s\n", s)
+		}
+	} else {
+		fmt.Fprintln(&buf, "  - no secret file was found on this host to take values from")
+	}
+	fmt.Fprint(&buf, `  - passwords in URLs, private keys, Bearer and Basic credentials
+  - in logs and command output, whatever follows password=, secret=, token=, api_key=,
+    access_key=, private_key= or credentials= (and the same with a colon)
+  - every literal env value in pod specs and MinecraftServers (valueFrom references stay)
+
+Read it through before you send it anywhere: redaction finds this host's known secrets and the
+common ways a secret is logged, and a secret logged another way stays in.
+`)
+	if b.wErr != nil {
+		fmt.Fprintf(&buf, "\nThe watchdog's settings: %v\n", b.wErr)
+	}
+	if len(bw.errs) > 0 {
+		fmt.Fprintln(&buf, "\nNot collected:")
+		for _, e := range bw.errs {
+			fmt.Fprintf(&buf, "  - %s\n", e)
+		}
+	}
+	// Its own words name what is redacted, and would be redacted themselves;
+	// what it quotes was scrubbed as it was recorded.
+	return buf.Bytes()
+}
+
+// secretSources are files that hold secrets, by how each is laid out.
+type secretSources struct {
+	envFiles   []string // KEY=VALUE lines, every value a secret (a commented-out one too)
+	valueFiles []string // one secret, the whole file
+	propsFiles []string // key=value lines; the keys naming a token, secret, password or key hold one
+	tokenFiles []string // k3s join tokens: the whole token and its secret part after the last ':'
+}
+
+// scrubber takes secrets out of what the bundle collects.
+type scrubber struct {
+	vals    []string // longest first, so a secret that contains another goes whole
+	sources []string // the files vals came from
+}
+
+var (
+	pemPrivateKey = regexp.MustCompile(`(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?-----END [A-Z0-9 ]*PRIVATE KEY-----`)
+	urlUserinfo   = regexp.MustCompile(`([A-Za-z][A-Za-z0-9+.-]*://[^/\s:@]*:)[^/\s@]+@`)
+	authScheme    = regexp.MustCompile(`(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}`)
+	secretAssign  = regexp.MustCompile(`(?i)((?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|credentials?)[A-Za-z0-9_.-]*"?[ \t]*[=:][ \t]*"?)([^\s"',;&]{4,})`)
+	secretPropKey = regexp.MustCompile(`(?i)token|secret|password|key`)
+)
+
+func (b *supportBundle) scrubber() *scrubber {
+	s := &scrubber{}
+	s.readSources(b.secrets)
+	if b.cfg != nil {
+		if u, err := url.Parse(b.cfg.Database.URL); err == nil {
+			if pw, ok := u.User.Password(); ok {
+				s.add(pw)
+			}
+		}
+		for _, ref := range []string{
+			b.cfg.Velocity.ServiceTokenRef, b.cfg.SMTP.PasswordRef,
+			b.cfg.Offsite.AccessKeyRef, b.cfg.Offsite.SecretKeyRef, b.cfg.Offsite.KeyRef,
+			b.cfg.Registry.S3.AccessKeyRef, b.cfg.Registry.S3.SecretKeyRef,
+			b.cfg.Archive.S3.AccessKeyRef, b.cfg.Archive.S3.SecretKeyRef,
+		} {
+			if ref != "" {
+				s.add(os.Getenv(ref))
+			}
+		}
+	}
+	if st, err := watchdog.LoadState(watchdog.NewestState(b.w.statePath, b.w.fallbackState)); err == nil {
+		s.add(st.SMTPPassword)
+	}
+	return s
+}
+
+func (s *scrubber) add(v string) bool {
+	v = strings.TrimSpace(v)
+	if len(v) < minScrubLen {
+		return false
+	}
+	for _, have := range s.vals {
+		if have == v {
+			return true
+		}
+	}
+	s.vals = append(s.vals, v)
+	sort.SliceStable(s.vals, func(i, j int) bool { return len(s.vals[i]) > len(s.vals[j]) })
+	return true
+}
+
+func (s *scrubber) readSources(src secretSources) {
+	read := func(p string, take func(content string) bool) {
+		raw, err := os.ReadFile(p)
+		if err != nil {
+			return
+		}
+		if take(string(raw)) {
+			s.sources = append(s.sources, p)
+		}
+	}
+	keyValues := func(content string, keep func(key string) bool) bool {
+		found := false
+		for _, line := range strings.Split(content, "\n") {
+			k, v, ok := strings.Cut(line, "=")
+			if !ok || !keep(strings.TrimSpace(k)) {
+				continue
+			}
+			v = strings.TrimSpace(v)
+			if len(v) >= 2 && (v[0] == '\'' || v[0] == '"') && v[len(v)-1] == v[0] {
+				v = v[1 : len(v)-1]
+			}
+			found = s.add(v) || found
+		}
+		return found
+	}
+	for _, p := range src.envFiles {
+		read(p, func(c string) bool { return keyValues(c, func(string) bool { return true }) })
+	}
+	for _, p := range src.propsFiles {
+		read(p, func(c string) bool { return keyValues(c, secretPropKey.MatchString) })
+	}
+	for _, p := range src.valueFiles {
+		read(p, func(c string) bool { return s.add(c) })
+	}
+	for _, p := range src.tokenFiles {
+		read(p, func(c string) bool {
+			c = strings.TrimSpace(c)
+			whole := s.add(c)
+			part := false
+			if i := strings.LastIndex(c, ":"); i >= 0 {
+				part = s.add(c[i+1:])
+			}
+			return whole || part
+		})
+	}
+}
+
+// values takes every known secret value out of data.
+func (s *scrubber) values(data []byte) []byte {
+	t := pemPrivateKey.ReplaceAllString(string(data), "<redacted private key>")
+	for _, v := range s.vals {
+		t = strings.ReplaceAll(t, v, redacted)
+	}
+	t = urlUserinfo.ReplaceAllString(t, "${1}"+redacted+"@")
+	t = authScheme.ReplaceAllString(t, "${1} "+redacted)
+	return []byte(t)
+}
+
+// text is values, and whatever a log names as a secret as well.
+func (s *scrubber) text(data []byte) []byte {
+	return secretAssign.ReplaceAll(s.values(data), []byte("${1}"+redacted))
+}

cmd/felis/supportbundle_test.go

@@ -0,0 +1,433 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+// The secrets a bundle host holds, each of which must be nowhere in the bundle.
+var plantedSecrets = map[string]string{
+	"secrets.env SERVICE_TOKEN":     "svc-token-planted-0a1b2c3d",
+	"secrets.env DB_PASSWORD":       "db-pass-planted-4e5f6a7b",
+	"offsite.env FELIS_OFFSITE_KEY": "offsite-key-planted+8c9d/0e1f=",
+	"smtp-password":                 "smtp-pass-planted-2a3b",
+	"felis-link.properties token":   "props-token-planted-4c5d",
+	"k3s token secret part":         "k3s-secret-part-planted-6e7f",
+	"watchdog state relay password": "cached-relay-pw-planted-8a9b",
+	"pod env literal":               "env-literal-planted-0c1d",
+	"deployment env literal":        "deploy-env-planted-2e3f",
+	"MinecraftServer env literal":   "cr-env-planted-4a5b",
+	"last-applied annotation":       "annotation-planted-6c7d",
+	"logged password":               "hunter2-planted-8e9f",
+	"logged bearer":                 "bearerplanted0a1b2c3d",
+	"URL password":                  "urlpass-planted-4e5f",
+	"token in an error":             "errtoken-planted-0f1e",
+	"felis.host.toml DB password":   "cfg-db-pass-planted-1c2d",
+	"service token by its env ref":  "env-ref-token-planted-3e4f",
+	"init container env literal":    "init-env-planted-5a6b",
+	"ephemeral container env":       "ephemeral-env-planted-7c8d",
+	"statefulset env literal":       "sts-env-planted-9e0f",
+	"job env literal":               "job-env-planted-1a2b",
+	"cronjob env literal":           "cronjob-env-planted-3c4d",
+	"commented-out offsite key":     "old-offsite-key-planted-5e6f",
+	"doctor quoting a password":     "doctor-pw-planted-7a8b",
+	"status quoting a password":     "status-pw-planted-9c0d",
+	"journal quoting a password":    "journal-pw-planted-1e2f",
+}
+
+// bundleHost is a Felis host for felis support-bundle: its secret files, its
+// watchdog unit and state, a cluster with a control-plane pod that restarted
+// and a game server, and logs and a journal that name secrets.
+func bundleHost(t *testing.T) *supportBundle {
+	t.Helper()
+	p := plantedSecrets
+	dir := t.TempDir()
+	etc := filepath.Join(dir, "etc-felis")
+	for _, d := range []string{etc, filepath.Join(dir, "systemd"), filepath.Join(dir, "k3s")} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeTestFile(t, filepath.Join(etc, "secrets.env"), "SERVICE_TOKEN="+p["secrets.env SERVICE_TOKEN"]+"\nDB_PASSWORD="+p["secrets.env DB_PASSWORD"]+"\nSHORT=abc\n", 0o600)
+	writeTestFile(t, filepath.Join(etc, "offsite.env"), "# before the rotation\n# FELIS_OFFSITE_KEY="+p["commented-out offsite key"]+"\nFELIS_OFFSITE_KEY='"+p["offsite.env FELIS_OFFSITE_KEY"]+"'\n", 0o600)
+	writeTestFile(t, filepath.Join(etc, "smtp-password"), p["smtp-password"]+"\n", 0o600)
+	writeTestFile(t, filepath.Join(etc, "felis-link.properties"), "api-base-url=http://10.43.0.10:8081\nservice-token="+p["felis-link.properties token"]+"\nroot-domain=games.example.org\n", 0o640)
+	writeTestFile(t, filepath.Join(dir, "k3s", "token"), "K10deadbeefcafe::server:"+p["k3s token secret part"]+"\n", 0o600)
+	cfgPath := filepath.Join(etc, "felis.host.toml")
+	writeTestFile(t, cfgPath, "[database]\nurl = \"postgres://felis:"+p["felis.host.toml DB password"]+"@127.0.0.1:1/felis?sslmode=disable&connect_timeout=1\"\n"+
+		"[server]\nroot_domain = \"games.example.org\"\n[archive]\nstore = \"tarLocal\"\n[k8s]\negress_mode = \"nodeport\"\n"+
+		"[velocity]\nservice_token_ref = \"FELIS_BUNDLE_TEST_SERVICE_TOKEN\"\n", 0o600)
+	t.Setenv("FELIS_BUNDLE_TEST_SERVICE_TOKEN", p["service token by its env ref"])
+	statePath := filepath.Join(dir, "state.json")
+	if err := watchdog.SaveState(statePath, &watchdog.State{SMTPPassword: p["watchdog state relay password"]}); err != nil {
+		t.Fatal(err)
+	}
+	unitDir := filepath.Join(dir, "systemd")
+	writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+cfgPath+" -state "+statePath+"\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-offsite.service"), "[Unit]\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "k3s.service"), "[Unit]\n", 0o644)
+	meminfo := filepath.Join(dir, "meminfo")
+	writeTestFile(t, meminfo, "MemTotal: 8000000 kB\nMemAvailable: 2000000 kB\n", 0o644)
+
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var w watchdogFlags
+	w, _, err = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	w.diskPaths = "/nonexistent-felis-bundle-test"
+
+	secretEnv := corev1.EnvVar{Name: "FELIS_SMTP_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
+		LocalObjectReference: corev1.LocalObjectReference{Name: "felis-smtp"}, Key: "password"}}}
+	replicas := int32(1)
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "felis-api-7d9", Namespace: "felis",
+				Annotations:   map[string]string{corev1.LastAppliedConfigAnnotation: `{"env":"` + p["last-applied annotation"] + `"}`, "felis.lolicon.best/kept": "yes"},
+				ManagedFields: []metav1.ManagedFieldsEntry{{Manager: "kubectl-client-side-apply"}},
+			},
+			Spec: corev1.PodSpec{
+				InitContainers: []corev1.Container{{Name: "wait-db", Image: "busybox", Env: []corev1.EnvVar{{Name: "FELIS_INIT_SETTING", Value: p["init container env literal"]}}}},
+				Containers: []corev1.Container{{Name: "api", Image: "felis-api:v1", Env: []corev1.EnvVar{
+					{Name: "FELIS_PLAIN_SETTING", Value: p["pod env literal"]}, secretEnv,
+				}}},
+				EphemeralContainers: []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{
+					Name: "debug", Env: []corev1.EnvVar{{Name: "FELIS_DEBUG_SETTING", Value: p["ephemeral container env"]}}}}},
+			},
+			Status: corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 1, Ready: true}}},
+		},
+		&appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
+			Spec: appsv1.DeploymentSpec{Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "felis-api"}},
+				Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "api", Env: []corev1.EnvVar{
+					{Name: "FELIS_DEPLOY_SETTING", Value: p["deployment env literal"]},
+				}}}}}},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "survival-0", Namespace: "minecraft"},
+			Spec: corev1.PodSpec{
+				InitContainers: []corev1.Container{{Name: "prepare-data"}, {Name: "egress-gate"}},
+				Containers:     []corev1.Container{{Name: "server"}},
+			},
+			Status: corev1.PodStatus{Phase: corev1.PodRunning},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
+			Spec:       v1alpha1.MinecraftServerSpec{Env: []v1alpha1.EnvVar{{Name: "DISCORD_WEBHOOK", Value: p["MinecraftServer env literal"]}}},
+		},
+		&appsv1.StatefulSet{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-postgres", Namespace: "felis"},
+			Spec:       appsv1.StatefulSetSpec{Template: envTemplate("FELIS_STS_SETTING", p["statefulset env literal"])},
+		},
+		&batchv1.Job{
+			ObjectMeta: metav1.ObjectMeta{Name: "build-1", Namespace: "felis-build"},
+			Spec:       batchv1.JobSpec{Template: envTemplate("FELIS_JOB_SETTING", p["job env literal"])},
+		},
+		&batchv1.CronJob{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-reaper", Namespace: "felis"},
+			Spec:       batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: envTemplate("FELIS_CRON_SETTING", p["cronjob env literal"])}}},
+		},
+		&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}},
+	).Build()
+
+	secretLog := fmt.Sprintf("started with SERVICE_TOKEN=%s\nlogin password=%s ok\nAuthorization: Bearer %s\ndial postgres://felis:%s@db:5432/felis\n",
+		p["secrets.env SERVICE_TOKEN"], p["logged password"], p["logged bearer"], p["URL password"])
+	return &supportBundle{
+		host: "felis-test", now: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC), unitDir: unitDir,
+		w: w, cfg: cfg, cl: cl,
+		logs: func(_ context.Context, ns, pod, container string, previous bool) ([]byte, error) {
+			if container == "wait-db" {
+				return nil, errors.New("container \"wait-db\" is waiting to start; token=" + p["token in an error"])
+			}
+			return []byte(fmt.Sprintf("log of %s/%s/%s previous=%v\n%s", ns, pod, container, previous, secretLog)), nil
+		},
+		run: func(_ context.Context, name string, args ...string) ([]byte, error) {
+			switch name {
+			case "journalctl":
+				return []byte("journal of " + args[1] + "\nFELIS_OFFSITE_KEY=" + p["offsite.env FELIS_OFFSITE_KEY"] + "\nrelay " + p["smtp-password"] + " refused\n" +
+					"relay login with the cached " + p["watchdog state relay password"] + "\ndatabase auth failed for " + p["felis.host.toml DB password"] +
+					"\nservice token " + p["service token by its env ref"] + " rejected\nnode joined with " + p["k3s token secret part"] + "\n" +
+					"old copies sealed with " + p["commented-out offsite key"] + "\nretrying with password=" + p["journal quoting a password"] + "\n"), nil
+			case "systemctl", "df", "ip":
+				return []byte(name + " output\n"), nil
+			}
+			return nil, errors.New("unexpected " + name)
+		},
+		backups: func(context.Context) (map[string]time.Time, error) {
+			return nil, errors.New("connect: password=" + p["status quoting a password"])
+		},
+		doctor: func(_ context.Context, out io.Writer) {
+			fmt.Fprintf(out, "doctor report; the k3s token K10deadbeefcafe::server:%s leaked here\nprobe said password=%s\n", p["k3s token secret part"], p["doctor quoting a password"])
+		},
+		secrets: secretSources{
+			envFiles:   []string{filepath.Join(etc, "secrets.env"), filepath.Join(etc, "offsite.env"), filepath.Join(etc, "absent.env")},
+			valueFiles: []string{filepath.Join(etc, "smtp-password"), filepath.Join(etc, "uploads-s3-secret-key")},
+			propsFiles: []string{filepath.Join(etc, "felis-link.properties")},
+			tokenFiles: []string{filepath.Join(dir, "k3s", "token")},
+		},
+		logLines: 500, since: 48 * time.Hour,
+		meminfo: meminfo, osRelease: filepath.Join(dir, "os-release"), procVersion: filepath.Join(dir, "version"), stateDir: etc,
+	}
+}
+
+// envTemplate is a pod template whose one container sets name to a literal.
+func envTemplate(name, value string) corev1.PodTemplateSpec {
+	return corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Env: []corev1.EnvVar{{Name: name, Value: value}}}}}}
+}
+
+// readBundle is every file in the bundle at path, by its name inside the
+// bundle's directory, and each one's mode.
+func readBundle(t *testing.T, path string) (map[string]string, map[string]int64) {
+	t.Helper()
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr := tar.NewReader(gz)
+	files, modes := map[string]string{}, map[string]int64{}
+	prefix := strings.TrimSuffix(filepath.Base(path), ".tar.gz") + "/"
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		name, ok := strings.CutPrefix(hdr.Name, prefix)
+		if !ok {
+			t.Errorf("%s is outside the bundle's directory %s", hdr.Name, prefix)
+		}
+		raw, err := io.ReadAll(tr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		files[name], modes[name] = string(raw), hdr.Mode
+	}
+	return files, modes
+}
+
+func TestSupportBundle(t *testing.T) {
+	b := bundleHost(t)
+	out := filepath.Join(t.TempDir(), "support")
+	path, err := b.write(context.Background(), out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := filepath.Join(out, "felis-support-felis-test-20260927T120000Z.tar.gz"); path != want {
+		t.Errorf("path %s, want %s", path, want)
+	}
+	for p, want := range map[string]os.FileMode{out: 0o700 | os.ModeDir, path: 0o600} {
+		if st, err := os.Stat(p); err != nil || st.Mode() != want {
+			t.Errorf("%s: mode %v (err %v), want %v", p, st.Mode(), err, want)
+		}
+	}
+	if left, _ := filepath.Glob(filepath.Join(out, ".*partial")); len(left) != 0 {
+		t.Errorf("left behind %v", left)
+	}
+
+	files, modes := readBundle(t, path)
+	var names []string
+	for n := range files {
+		names = append(names, n)
+		if modes[n] != 0o600 {
+			t.Errorf("%s: mode %o in the archive, want 600", n, modes[n])
+		}
+	}
+	sort.Strings(names)
+	want := []string{
+		"MANIFEST.txt",
+		"cluster/felis-build/cronjobs.yaml", "cluster/felis-build/deployments.yaml", "cluster/felis-build/events.yaml", "cluster/felis-build/jobs.yaml",
+		"cluster/felis-build/networkpolicies.yaml", "cluster/felis-build/persistentvolumeclaims.yaml", "cluster/felis-build/pods.yaml",
+		"cluster/felis-build/services.yaml", "cluster/felis-build/statefulsets.yaml",
+		"cluster/felis/cronjobs.yaml", "cluster/felis/deployments.yaml", "cluster/felis/events.yaml", "cluster/felis/jobs.yaml",
+		"cluster/felis/networkpolicies.yaml", "cluster/felis/persistentvolumeclaims.yaml", "cluster/felis/pods.yaml",
+		"cluster/felis/services.yaml", "cluster/felis/statefulsets.yaml",
+		"cluster/minecraft/cronjobs.yaml", "cluster/minecraft/deployments.yaml", "cluster/minecraft/events.yaml", "cluster/minecraft/jobs.yaml",
+		"cluster/minecraft/networkpolicies.yaml", "cluster/minecraft/persistentvolumeclaims.yaml", "cluster/minecraft/pods.yaml",
+		"cluster/minecraft/services.yaml", "cluster/minecraft/statefulsets.yaml",
+		"cluster/minecraftservers.yaml", "cluster/nodes.yaml", "cluster/persistentvolumes.yaml", "cluster/pods-all-namespaces.txt",
+		"config.txt", "doctor.txt",
+		"host/addresses.txt", "host/df.txt", "host/etc-felis.txt", "host/meminfo.txt", "host/systemd-timers.txt", "host/systemd-units.txt",
+		"journal/felis-offsite.log", "journal/felis-watchdog.log", "journal/k3s.log",
+		"logs/felis/felis-api-7d9/api.log", "logs/felis/felis-api-7d9/api.previous.log",
+		"logs/minecraft/survival-0/egress-gate.log", "logs/minecraft/survival-0/prepare-data.log",
+		"status.txt", "version.txt",
+	}
+	if strings.Join(names, "\n") != strings.Join(want, "\n") {
+		t.Errorf("bundle holds\n  %s\nwant\n  %s", strings.Join(names, "\n  "), strings.Join(want, "\n  "))
+	}
+
+	for what, secret := range plantedSecrets {
+		for n, body := range files {
+			if strings.Contains(body, secret) {
+				t.Errorf("%s (%s) is in %s:\n%s", what, secret, n, body)
+			}
+		}
+	}
+
+	// What was taken out leaves what a reader needs around it.
+	for name, wants := range map[string][]string{
+		"logs/felis/felis-api-7d9/api.previous.log": {"log of felis/felis-api-7d9/api previous=true\n", "SERVICE_TOKEN=<redacted>\n", "login password=<redacted> ok\n", "Authorization: Bearer <redacted>\n", "postgres://felis:<redacted>@db:5432/felis\n"},
+		"journal/k3s.log": {"journal of k3s.service\n", "FELIS_OFFSITE_KEY=<redacted>\n", "relay <redacted> refused\n",
+			"relay login with the cached <redacted>\n", "database auth failed for <redacted>\n", "service token <redacted> rejected\n", "node joined with <redacted>\n"},
+		"cluster/felis/statefulsets.yaml": {"name: FELIS_STS_SETTING\n"},
+		"cluster/felis/cronjobs.yaml":     {"name: FELIS_CRON_SETTING\n"},
+		"cluster/felis-build/jobs.yaml":   {"name: FELIS_JOB_SETTING\n"},
+		"cluster/felis/pods.yaml":         {"name: FELIS_PLAIN_SETTING\n        value: <redacted>\n", "name: FELIS_SMTP_PASSWORD\n        valueFrom:\n          secretKeyRef:\n            key: password\n            name: felis-smtp\n", "felis.lolicon.best/kept: \"yes\"", "name: FELIS_INIT_SETTING\n", "name: FELIS_DEBUG_SETTING\n"},
+		"cluster/felis/deployments.yaml":  {"name: FELIS_DEPLOY_SETTING\n            value: <redacted>\n"},
+		"cluster/minecraftservers.yaml":   {"name: DISCORD_WEBHOOK\n      value: <redacted>\n"},
+		"config.txt":                      {fmt.Sprintf("%-34s %s\n", "server.root_domain", "games.example.org"), fmt.Sprintf("%-34s %s\n", "database.url (no password)", "postgres://[email protected]:1/felis")},
+		"doctor.txt":                      {"the k3s token <redacted> leaked here\nprobe said password=<redacted>\n"},
+		"status.txt":                      {"  world backups unknown: connect: password=<redacted>\n"},
+		"host/etc-felis.txt":              {"secrets.env\n", "smtp-password\n", "felis-link.properties\n"},
+		"MANIFEST.txt": {
+			"The game servers' own logs are left out",
+			"  journal/                 up to 500 lines per Felis unit and k3s, from the last 48h\n",
+			"  - 11 secret values, wherever they appear, read from:\n",
+			"secrets.env\n", "offsite.env\n", "smtp-password\n", "felis-link.properties\n", "token\n",
+			"logs/felis/felis-api-7d9/wait-db.log: container \"wait-db\" is waiting to start; token=<redacted>\n",
+			// Its own words about what is redacted come through whole.
+			"  - passwords in URLs, private keys, Bearer and Basic credentials\n",
+			"access_key=, private_key= or credentials= (and the same with a colon)\n",
+			"Read it through before you send it anywhere",
+		},
+	} {
+		for _, w := range wants {
+			if !strings.Contains(files[name], w) {
+				t.Errorf("%s lacks %q:\n%s", name, w, files[name])
+			}
+		}
+	}
+	for _, gone := range []string{"managedFields", "last-applied-configuration"} {
+		if strings.Contains(files["cluster/felis/pods.yaml"], gone) {
+			t.Errorf("pods.yaml keeps %s:\n%s", gone, files["cluster/felis/pods.yaml"])
+		}
+	}
+	if strings.Contains(files["MANIFEST.txt"], "absent.env") || strings.Contains(files["MANIFEST.txt"], "uploads-s3-secret-key") {
+		t.Errorf("MANIFEST names files this host does not have:\n%s", files["MANIFEST.txt"])
+	}
+}
+
+// -server-logs adds the game servers' own logs, and says so.
+func TestSupportBundleServerLogs(t *testing.T) {
+	b := bundleHost(t)
+	b.serverLogs = true
+	path, err := b.write(context.Background(), t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	files, _ := readBundle(t, path)
+	if _, ok := files["logs/minecraft/survival-0/server.log"]; !ok {
+		t.Error("no server.log with -server-logs")
+	}
+	if !strings.Contains(files["MANIFEST.txt"], "The game servers' own logs are in logs/ (-server-logs)") {
+		t.Errorf("MANIFEST does not say server logs are in:\n%s", files["MANIFEST.txt"])
+	}
+}
+
+// With the cluster and the configuration gone the bundle still holds what the
+// host shows, and MANIFEST says what it could not collect.
+func TestSupportBundleWithTheClusterDown(t *testing.T) {
+	b := bundleHost(t)
+	b.cl, b.clErr = nil, errors.New("connection refused")
+	b.cfg, b.cfgErr = nil, errors.New("felis.host.toml: no such file")
+	b.wErr = errors.New("felis-watchdog.service is not installed; read the watchdog's defaults")
+	path, err := b.write(context.Background(), t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	files, _ := readBundle(t, path)
+	for _, n := range []string{"status.txt", "doctor.txt", "host/etc-felis.txt", "journal/k3s.log"} {
+		if _, ok := files[n]; !ok {
+			t.Errorf("no %s", n)
+		}
+	}
+	for n := range files {
+		if strings.HasPrefix(n, "cluster/") || strings.HasPrefix(n, "logs/") || n == "config.txt" {
+			t.Errorf("%s with no cluster and no configuration", n)
+		}
+	}
+	if !strings.Contains(files["status.txt"], "felis status: the configuration did not load: felis.host.toml: no such file") {
+		t.Errorf("status.txt:\n%s", files["status.txt"])
+	}
+	if !strings.Contains(files["MANIFEST.txt"], "\nThe watchdog's settings: felis-watchdog.service is not installed; read the watchdog's defaults\n") ||
+		!strings.Contains(files["MANIFEST.txt"], "  - cluster: connection refused\n") {
+		t.Errorf("MANIFEST.txt:\n%s", files["MANIFEST.txt"])
+	}
+}
+
+func TestShortDuration(t *testing.T) {
+	for d, want := range map[time.Duration]string{
+		48 * time.Hour:             "48h",
+		90 * time.Minute:           "1h30m",
+		45 * time.Minute:           "45m",
+		30 * time.Second:           "30s",
+		time.Hour + 30*time.Second: "1h0m30s",
+	} {
+		if got := shortDuration(d); got != want {
+			t.Errorf("shortDuration(%v) = %q, want %q", d, got, want)
+		}
+	}
+}
+
+func TestScrubber(t *testing.T) {
+	s := &scrubber{}
+	for _, v := range []string{"known-secret-value", "known-secret-value-longer", "short", "known-secret-value"} {
+		s.add(v)
+	}
+	if got := strings.Join(s.vals, ","); got != "known-secret-value-longer,known-secret-value" {
+		t.Errorf("kept %q; want each value once, longest first, none under %d characters", s.vals, minScrubLen)
+	}
+	for in, want := range map[string]string{
+		"a known-secret-value-longer b":                                     "a <redacted> b",
+		"a known-secret-value b":                                            "a <redacted> b",
+		"password=abcd1234 next":                                            "password=<redacted> next",
+		`{"token": "abcd1234"}`:                                             `{"token": "<redacted>"}`,
+		"SMTP_PASSWORD: s3cr3t!x":                                           "SMTP_PASSWORD: <redacted>",
+		"x-api-key=zzzz9999&q=1":                                            "x-api-key=<redacted>&q=1",
+		"Authorization: Basic dXNlcjpwYXNzd29yZA==":                         "Authorization: Basic <redacted>",
+		"s3://AKIA:secretpart123@bucket/key":                                "s3://AKIA:<redacted>@bucket/key",
+		"https://user@host/path":                                            "https://user@host/path",
+		"-----BEGIN EC PRIVATE KEY-----\nMHc\n-----END EC PRIVATE KEY-----": "<redacted private key>",
+		"tokens: 3": "tokens: 3",
+		"read the relay password (keeping the cached one)": "read the relay password (keeping the cached one)",
+		`secrets "felis-smtp" not found`:                   `secrets "felis-smtp" not found`,
+	} {
+		if got := string(s.text([]byte(in))); got != want {
+			t.Errorf("text(%q) = %q, want %q", in, got, want)
+		}
+	}
+	// Structured files keep what only looks like a secret.
+	if got := string(s.values([]byte("secretName: felis-forwarding-secret and known-secret-value"))); got != "secretName: felis-forwarding-secret and <redacted>" {
+		t.Errorf("values() = %q", got)
+	}
+}

cmd/felis/systemservers.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"time"
@@ -14,7 +15,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -95,7 +98,7 @@ const felisLimboHealthPort int32 = 8080
 // fail-safes to readiness-only, so a login pod that has the URL/domain but not yet
 // the token is safe (it simply does not authenticate) rather than broken.
 const (
-	envAPIBaseURL    = "FELIS_API_BASE_URL"
+	envAPIBaseURL    = naming.EnvAPIBaseURL
 	envRootDomain    = "FELIS_ROOT_DOMAIN"
 	envPanelHostname = "FELIS_PANEL_HOSTNAME"
 	envLobbyServer   = "FELIS_LOBBY_SERVER"
@@ -241,6 +244,15 @@ func buildSystemServerClient() (client.Client, error) {
 	if err := v1alpha1.AddToScheme(scheme); err != nil {
 		return nil, err
 	}
+	cfg, err := hostRESTConfig()
+	if err != nil {
+		return nil, err
+	}
+	return client.New(cfg, client.Options{Scheme: scheme})
+}
+
+// hostRESTConfig is the cluster connection buildSystemServerClient describes.
+func hostRESTConfig() (*rest.Config, error) {
 	cfg, err := ctrl.GetConfig()
 	if err != nil {
 		cfg, err = clientcmd.BuildConfigFromFlags("", hostBootstrapKubeconfigPath)
@@ -248,17 +260,38 @@ func buildSystemServerClient() (client.Client, error) {
 			return nil, fmt.Errorf("no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and %s): %w", hostBootstrapKubeconfigPath, err)
 		}
 	}
-	return client.New(cfg, client.Options{Scheme: scheme})
+	return cfg, nil
 }
 
 // systemServerOutcome records what ensureSystemServers did with one service so
 // setup can report it without the provisioner deciding on the output format.
 type systemServerOutcome struct {
 	name      string
-	created   bool   // true = we created it this run
-	available bool   // true = the required object now exists
-	skipped   string // non-empty = why it was skipped (image unset / already exists)
-	err       error  // non-nil = create failed
+	created   bool     // true = we created it this run
+	updated   bool     // true = we refreshed an existing replica from the source
+	available bool     // true = the required object now exists
+	skipped   string   // non-empty = why it was skipped (image unset / already exists)
+	err       error    // non-nil = create failed
+	changes   []string // converge only: the fields this pass filled
+}
+
+// systemServerPlan is one system service in the provisioner's table: its name,
+// the image config gives it, and the pure builder for its desired CR.
+type systemServerPlan struct {
+	name  string
+	image string
+	build func(image, namespace string) (*v1alpha1.MinecraftServer, error)
+}
+
+// systemServerPlans is the single description of the login+lobby pair, shared by
+// ensureSystemServers (create-if-absent) and convergeSystemServers (field fill).
+func systemServerPlans(loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname string) []systemServerPlan {
+	return []systemServerPlan{
+		{name: naming.SystemLoginServer, image: loginImage, build: func(image, ns string) (*v1alpha1.MinecraftServer, error) {
+			return loginSystemServer(image, ns, apiBaseURL, rootDomain, panelHostname)
+		}},
+		{name: naming.SystemLobbyServer, image: lobbyImage, build: lobbySystemServer},
+	}
 }
 
 // ensureSystemServers idempotently creates the login and lobby system services.
@@ -269,17 +302,7 @@ type systemServerOutcome struct {
 // K8s client and namespace; this function performs no signal-handler or client
 // setup of its own.
 func ensureSystemServers(ctx context.Context, cl client.Client, namespace, loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname string) []systemServerOutcome {
-	type plan struct {
-		name  string
-		image string
-		build func(image, namespace string) (*v1alpha1.MinecraftServer, error)
-	}
-	plans := []plan{
-		{name: naming.SystemLoginServer, image: loginImage, build: func(image, ns string) (*v1alpha1.MinecraftServer, error) {
-			return loginSystemServer(image, ns, apiBaseURL, rootDomain, panelHostname)
-		}},
-		{name: naming.SystemLobbyServer, image: lobbyImage, build: lobbySystemServer},
-	}
+	plans := systemServerPlans(loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname)
 
 	outcomes := make([]systemServerOutcome, 0, len(plans))
 	for _, p := range plans {
@@ -379,27 +402,162 @@ var derivedSystemEnv = map[string]bool{
 // deliberate removal is indistinguishable from drift and re-adding it would fight the
 // operator every run.
 func refreshDerivedEnv(ctx context.Context, cl client.Client, existing, desired *v1alpha1.MinecraftServer) (bool, error) {
+	want := derivedEnvWanted(desired)
+	changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
+		changed := false
+		for i, e := range existing.Spec.Env {
+			if v, ok := want[e.Name]; ok && v != e.Value {
+				existing.Spec.Env[i].Value = v
+				changed = true
+			}
+		}
+		return changed
+	})
+	if err != nil {
+		return false, fmt.Errorf("refresh %s env: %w", existing.Name, err)
+	}
+	return changed, nil
+}
+
+// patchOnConflictRetry applies mutate to obj and sends only the difference, as a
+// merge patch that carries the resourceVersion obj was read at. The operator writes
+// status and felis-api patches spec.idle on these same objects, so a write can land
+// between setup's read and its patch: the apiserver then answers 409, and this
+// re-reads obj and runs mutate again on the fresh copy, up to retry.DefaultRetry's
+// five attempts. The pinned resourceVersion is what keeps a list field such as
+// spec.env safe — a merge patch replaces a list whole, and without the lock an
+// entry added concurrently would be dropped. mutate reports whether it changed
+// anything; nothing is sent when it did not. obj holds the stored object after.
+func patchOnConflictRetry(ctx context.Context, cl client.Client, obj client.Object, mutate func() bool) (bool, error) {
+	key := client.ObjectKeyFromObject(obj)
+	changed, reread := false, false
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		if reread {
+			if err := cl.Get(ctx, key, obj); err != nil {
+				return err
+			}
+		}
+		reread = true
+		base := obj.DeepCopyObject().(client.Object)
+		if changed = mutate(); !changed {
+			return nil
+		}
+		return cl.Patch(ctx, obj, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}))
+	})
+	return changed, err
+}
+
+// derivedEnvWanted maps the derived env keys of desired onto their values.
+func derivedEnvWanted(desired *v1alpha1.MinecraftServer) map[string]string {
 	want := make(map[string]string, len(derivedSystemEnv))
 	for _, e := range desired.Spec.Env {
 		if derivedSystemEnv[e.Name] {
 			want[e.Name] = e.Value
 		}
 	}
+	return want
+}
 
-	changed := false
-	for i, e := range existing.Spec.Env {
+// convergeSystemServers is the explicit convergence pass over already-installed
+// system servers (#1). ensureSystemServers is create-if-absent by design — an
+// existing CR is left alone so a re-run cannot clobber an operator's edits — and
+// that leaves no path for a field the DESIRED spec gained after the install:
+// spec.rcon (the lobby's write channel), spec.startup.healthHTTPPort (the login
+// gate's readiness probe), or a config-derived env key that did not exist yet.
+// Such fields sit at their zero value forever while re-running setup reports
+// success, which is exactly the reported "configuration updates never reach an
+// installed deployment" symptom.
+//
+// This pass fills exactly those zero-value fields and the config-derived env keys,
+// and nothing else: a field already holding a non-zero value is the operator's and
+// is never overwritten. It is an explicit command rather than an implicit step of
+// setup because some fills need an ordering only the operator knows — enabling
+// RCON or the HTTP readiness gate on a server whose image predates the listener
+// would hold that server in Starting until it was marked Failed. Rebuild (or
+// upgrade) the images first, then run this.
+func convergeSystemServers(ctx context.Context, cl client.Client, namespace, loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname string) []systemServerOutcome {
+	outcomes := make([]systemServerOutcome, 0, 2)
+	for _, p := range systemServerPlans(loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname) {
+		if p.image == "" {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, skipped: "image not configured"})
+			continue
+		}
+		desired, err := p.build(p.image, namespace)
+		if err != nil {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: err})
+			continue
+		}
+
+		var existing v1alpha1.MinecraftServer
+		switch err := cl.Get(ctx, client.ObjectKeyFromObject(desired), &existing); {
+		case apierrors.IsNotFound(err):
+			outcomes = append(outcomes, systemServerOutcome{name: p.name,
+				skipped: "not present — run `sudo felis setup` first"})
+			continue
+		case err != nil:
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: err})
+			continue
+		}
+		if existing.Labels[v1alpha1.LabelSystemRole] != p.name {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf(
+				"existing MinecraftServer %s/%s is not marked as the Felis %q system role; refusing to converge it",
+				namespace, p.name, p.name,
+			)})
+			continue
+		}
+
+		var changes []string
+		changed, err := patchOnConflictRetry(ctx, cl, &existing, func() bool {
+			changes = nil
+			if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
+				existing.Spec.Rcon = desired.Spec.Rcon
+				changes = append(changes, "spec.rcon")
+			}
+			if existing.Spec.Startup.HealthHTTPPort == 0 && desired.Spec.Startup.HealthHTTPPort != 0 {
+				existing.Spec.Startup.HealthHTTPPort = desired.Spec.Startup.HealthHTTPPort
+				changes = append(changes, "spec.startup.healthHTTPPort")
+			}
+			changes = append(changes, convergeDerivedEnv(&existing, desired)...)
+			return len(changes) > 0
+		})
+		if err != nil {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
+			continue
+		}
+		if !changed {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
+			continue
+		}
+		outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, updated: true, changes: changes})
+	}
+	return outcomes
+}
+
+// convergeDerivedEnv makes the config-derived env match the desired values: a key
+// whose value drifted is overwritten, and a key missing entirely is added. This is
+// the wider half of the same explicit pass — refreshDerivedEnv's present-only loop
+// can never introduce a NEW key, which is how a derived key added after an install
+// never reached it at all.
+func convergeDerivedEnv(existing, desired *v1alpha1.MinecraftServer) []string {
+	want := derivedEnvWanted(desired)
+	var changes []string
+	present := make(map[string]bool, len(existing.Spec.Env))
+	for i := range existing.Spec.Env {
+		e := &existing.Spec.Env[i]
+		present[e.Name] = true
 		if v, ok := want[e.Name]; ok && v != e.Value {
-			existing.Spec.Env[i].Value = v
-			changed = true
+			e.Value = v
+			changes = append(changes, "env "+e.Name)
 		}
 	}
-	if !changed {
-		return false, nil
+	for _, e := range desired.Spec.Env {
+		if !derivedSystemEnv[e.Name] || present[e.Name] {
+			continue
+		}
+		existing.Spec.Env = append(existing.Spec.Env, e)
+		changes = append(changes, "env "+e.Name)
 	}
-	if err := cl.Update(ctx, existing); err != nil {
-		return false, fmt.Errorf("refresh %s env: %w", existing.Name, err)
-	}
-	return true, nil
+	return changes
 }
 
 // The login gate is a hard prerequisite of the Owner bind, so setup waits for it
@@ -471,26 +629,73 @@ func phaseOrPending(p v1alpha1.Phase) string {
 	return string(p)
 }
 
-// ensureSecretReplica copies one Secret from the control namespace into the minecraft
-// namespace so a backend pod can mount it via secretKeyRef. A secretKeyRef is
-// namespace-local, but the backends run in the minecraft namespace while the sources
-// of truth live beside the control plane — so without this replica the operator's
-// injected secretKeyRef would dangle and wedge the pod in CreateContainerConfigError.
+// provisionSecretReplicas copies the Secrets workload pods mount from the control
+// namespace into the namespaces those pods run in. The proxy's felis-service-token
+// is not among them: it lives in the control namespace and on the host, and a copy
+// anywhere else would open every game route to whoever reads that namespace.
+func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, minecraftNS, buildNS string) []systemServerOutcome {
+	return []systemServerOutcome{
+		// refresh=true for both caller tokens: the control namespace holds the
+		// current value and `felis rotate-token` replaces it there, so a replica
+		// that differs is stale and the login gate would be turned away with it.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.LimboTokenSecretName, naming.ServiceTokenSecretKey, "limbo-token", "minecraft ns", true),
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
+		// refresh=true: felis-config is the rendered config, not a credential. The
+		// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
+		// must update it when the control plane's render has moved on (a stale copy
+		// e.g. keeps an old database URL after a credential rotation).
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-config", "felis.toml", "config", "minecraft ns", true),
+		// The reaper's pre-reap warning emails authenticate with the same relay
+		// password felis-api uses; the reaper pod runs in the minecraft namespace,
+		// where a secretKeyRef resolves only against a local mirror. Skipped while
+		// the relay is not configured yet — the "configure email" screen refreshes
+		// both mirrors when it applies.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-smtp", "password", "smtp", "minecraft ns", false),
+		// The build namespace needs the build token: the build Job's fetch
+		// initContainer reads the submission context from the internal face, and
+		// that is all this token opens. Best effort — a deployment that only
+		// installs the control plane simply never builds a user submission.
+		ensureSecretReplica(ctx, cl, controlNS, buildNS,
+			naming.BuildTokenSecretName, naming.ServiceTokenSecretKey, "build-token", "felis-build ns", true),
+	}
+}
+
+// ensureSecretReplica copies one Secret from the control namespace into a workload
+// namespace (minecraft — or the build namespace, whose fetch initContainer reads the
+// context from the felis-api internal face with the build token) so a pod can mount it
+// via secretKeyRef. A secretKeyRef is namespace-local, but those workloads do not run
+// beside the control plane — so without this replica the secretKeyRef would dangle and
+// wedge the pod in CreateContainerConfigError.
 //
-// Two Secrets need it, for different reasons: the service token (login only — it
-// authenticates the limbo plugin to the felis-api internal face) and the Velocity
-// modern-forwarding secret (every backend — it is how a backend knows a login really
-// came from the proxy, and so that the player's UUID is Mojang-verified rather than
-// offline-derived).
+// Several Secrets need it, for different reasons: the caller tokens of the login
+// limbo and the build Pod's context fetch (both authenticate to the felis-api
+// internal face, each with its own token),
+// the Velocity modern-forwarding secret (every backend — it is how a backend knows
+// a login really came from the proxy, and so that the player's UUID is Mojang-verified
+// rather than offline-derived), and the SMTP relay password (the reaper's pre-reap
+// warning emails; the felis-config mirror is what carries [smtp] into its pod).
 //
 // It is create-if-absent: an existing replica is left untouched so a hand-rotated
-// value in the minecraft namespace is never clobbered (to rotate, delete the replica
+// value in the workload namespace is never clobbered (to rotate, delete the replica
 // and re-run setup). Best-effort like the rest of the provisioner: a missing source or
 // a create failure degrades to a reported outcome, never a hard setup failure. It
 // copies only Type and Data — never labels/annotations/ownerRefs — so the replica
 // carries no accidental GC owner or managed-by lineage.
-func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label string) systemServerOutcome {
-	name := label + " (minecraft ns)"
+//
+// refreshExisting switches a replica to refresh-in-place from the control namespace.
+// The felis-config mirror uses it because that Secret is a rendered config and the
+// workload Jobs that mount it (backup/restore/fileedit) plus the reaper silently
+// misbehave on a stale copy — e.g. after a database credential rotation the control
+// plane moves on while every backup Job keeps failing auth. The caller tokens use it
+// because `felis rotate-token` replaces them in the control namespace, which makes a
+// differing replica stale by definition. The forwarding and SMTP Secrets keep the
+// never-overwrite rule.
+func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label, where string, refreshExisting bool) systemServerOutcome {
+	name := label + " (" + where + ")"
 	validate := func(secret *corev1.Secret, location, skipped string) systemServerOutcome {
 		if len(secret.Data[secretKey]) == 0 {
 			return systemServerOutcome{name: name, skipped: fmt.Sprintf(
@@ -498,6 +703,39 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 		}
 		return systemServerOutcome{name: name, available: true, skipped: skipped}
 	}
+	// refreshFromControl updates an existing replica from the control-namespace source
+	// when the rendered key differs. Only the felis-config mirror opts in.
+	refreshFromControl := func(existing *corev1.Secret) systemServerOutcome {
+		var src corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: controlNamespace, Name: secretName}, &src); err != nil {
+			if apierrors.IsNotFound(err) {
+				return systemServerOutcome{name: name, skipped: fmt.Sprintf(
+					"source Secret %s/%s not found — provision it (deploy/bootstrap.sh), then re-run setup",
+					controlNamespace, secretName)}
+			}
+			return systemServerOutcome{name: name, err: err}
+		}
+		if out := validate(&src, controlNamespace, ""); !out.available {
+			return out
+		}
+		changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
+			if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
+				return false
+			}
+			if existing.Data == nil {
+				existing.Data = map[string][]byte{}
+			}
+			existing.Data[secretKey] = src.Data[secretKey]
+			return true
+		})
+		if err != nil {
+			return systemServerOutcome{name: name, err: err}
+		}
+		if !changed {
+			return validate(existing, minecraftNamespace, "already current")
+		}
+		return systemServerOutcome{name: name, updated: true, available: true}
+	}
 	if controlNamespace == minecraftNamespace {
 		// Same namespace needs no replica, but the source still has to exist.
 		var existing corev1.Secret
@@ -516,6 +754,9 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 	var existing corev1.Secret
 	getErr := cl.Get(ctx, client.ObjectKey{Namespace: minecraftNamespace, Name: secretName}, &existing)
 	if getErr == nil {
+		if refreshExisting {
+			return refreshFromControl(&existing)
+		}
 		return validate(&existing, minecraftNamespace, "already exists")
 	}
 	if !apierrors.IsNotFound(getErr) {
@@ -544,6 +785,9 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 			if getErr := cl.Get(ctx, client.ObjectKey{Namespace: minecraftNamespace, Name: secretName}, &existing); getErr != nil {
 				return systemServerOutcome{name: name, err: getErr}
 			}
+			if refreshExisting {
+				return refreshFromControl(&existing)
+			}
 			return validate(&existing, minecraftNamespace, "already exists")
 		}
 		return systemServerOutcome{name: name, err: err}
@@ -553,7 +797,7 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 
 func requiredProvisioningError(outcomes []systemServerOutcome) error {
 	required := map[string]struct{}{
-		"service-token (minecraft ns)":     {},
+		"limbo-token (minecraft ns)":       {},
 		"forwarding-secret (minecraft ns)": {},
 		naming.SystemLoginServer:           {},
 	}

cmd/felis/systemservers_test.go

@@ -142,21 +142,21 @@ func TestLoginSystemServerEnv(t *testing.T) {
 // namespace (create-if-absent), so the operator's secretKeyRef on the backend pod
 // resolves. It must not overwrite an existing replica, and must degrade gracefully
 // when the source is missing or the namespaces coincide. Exercised here with the
-// service token; setup runs it a second time for the Velocity forwarding secret.
+// Velocity forwarding secret, which setup replicates in this never-overwrite mode.
 func TestEnsureSecretReplica(t *testing.T) {
 	scheme := newSystemServerScheme(t)
 	ctx := context.Background()
 
 	srcSecret := func() *corev1.Secret {
 		return &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "felis"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "felis"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("s3cr3t")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("s3cr3t")},
 		}
 	}
 	replicate := func(cl client.Client, controlNS, mcNS string) systemServerOutcome {
 		return ensureSecretReplica(ctx, cl, controlNS, mcNS,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token")
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false)
 	}
 
 	t.Run("replicates when absent", func(t *testing.T) {
@@ -166,19 +166,19 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want created", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "s3cr3t" {
-			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ServiceTokenSecretKey])
+		if string(replica.Data[naming.ForwardingSecretKey]) != "s3cr3t" {
+			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ForwardingSecretKey])
 		}
 	})
 
 	t.Run("does not overwrite existing replica", func(t *testing.T) {
 		existing := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("rotated")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("rotated")},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), existing).Build()
 		out := replicate(cl, "felis", "minecraft")
@@ -186,11 +186,11 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want skipped (not clobbered)", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "rotated" {
-			t.Error("existing replica was overwritten — a rotated token must survive")
+		if string(replica.Data[naming.ForwardingSecretKey]) != "rotated" {
+			t.Error("existing replica was overwritten — a hand-set value must survive")
 		}
 	})
 
@@ -204,22 +204,22 @@ func TestEnsureSecretReplica(t *testing.T) {
 
 	t.Run("rejects a source with an empty required key", func(t *testing.T) {
 		bad := srcSecret()
-		bad.Data[naming.ServiceTokenSecretKey] = nil
+		bad.Data[naming.ForwardingSecretKey] = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
 
 	t.Run("rejects an existing replica with an empty required key", func(t *testing.T) {
 		bad := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Data:       map[string][]byte{},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable existing replica", out)
 		}
 	})
@@ -245,15 +245,96 @@ func TestEnsureSecretReplica(t *testing.T) {
 		bad.Data = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "felis")
-		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
 }
 
+// The felis-config mirror is the one replica that must refresh: it is a rendered
+// config, and a stale workload-side copy (backup/restore/fileedit Jobs, the reaper)
+// misbehaves silently — a rotated database credential keeps the control plane moving
+// while every backup Job keeps failing auth. Credential Secrets keep create-if-absent.
+func TestEnsureSecretReplicaRefresh(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	configSecret := func(ns, body string) *corev1.Secret {
+		return &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{"felis.toml": []byte(body)},
+		}
+	}
+	refresh := func(cl client.Client) systemServerOutcome {
+		return ensureSecretReplica(ctx, cl, "felis", "minecraft",
+			"felis-config", "felis.toml", "config", "minecraft ns", true)
+	}
+	replicaBody := func(t *testing.T, cl client.Client) string {
+		t.Helper()
+		var got corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &got); err != nil {
+			t.Fatalf("get replica: %v", err)
+		}
+		return string(got.Data["felis.toml"])
+	}
+
+	t.Run("refreshes a stale config replica", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("felis", "current"),
+			configSecret("minecraft", "stale"),
+		).Build()
+		out := refresh(cl)
+		if out.err != nil || !out.updated || !out.available {
+			t.Fatalf("outcome = %+v, want refreshed", out)
+		}
+		if got := replicaBody(t, cl); got != "current" {
+			t.Errorf("replica = %q, want current", got)
+		}
+	})
+
+	t.Run("leaves a current config replica alone", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("felis", "same"),
+			configSecret("minecraft", "same"),
+		).Build()
+		out := refresh(cl)
+		if out.err != nil || out.updated || !out.available || out.skipped != "already current" {
+			t.Fatalf("outcome = %+v, want already current", out)
+		}
+	})
+
+	t.Run("fills an empty-key replica", func(t *testing.T) {
+		empty := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: "minecraft"},
+			Data:       map[string][]byte{},
+		}
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("felis", "current"), empty).Build()
+		out := refresh(cl)
+		if out.err != nil || !out.updated {
+			t.Fatalf("outcome = %+v, want refreshed", out)
+		}
+		if got := replicaBody(t, cl); got != "current" {
+			t.Errorf("replica = %q, want current", got)
+		}
+	})
+
+	t.Run("missing source degrades to a skip", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("minecraft", "stale")).Build()
+		out := refresh(cl)
+		if out.err != nil || out.updated || out.available || out.skipped == "" {
+			t.Fatalf("outcome = %+v, want skipped (source missing)", out)
+		}
+		if got := replicaBody(t, cl); got != "stale" {
+			t.Errorf("replica = %q, want untouched stale", got)
+		}
+	})
+}
+
 func TestRequiredProvisioningError(t *testing.T) {
 	ready := []systemServerOutcome{
-		{name: "service-token (minecraft ns)", available: true},
+		{name: "limbo-token (minecraft ns)", available: true},
 		{name: "forwarding-secret (minecraft ns)", available: true},
 		{name: naming.SystemLoginServer, available: true},
 		{name: naming.SystemLobbyServer, skipped: "image not configured"},
@@ -268,6 +349,14 @@ func TestRequiredProvisioningError(t *testing.T) {
 		t.Fatalf("missing forwarding secret = %v, want named error", err)
 	}
 
+	// The login gate cannot reach felis-api without its token, so setup must not
+	// report success while that replica is missing.
+	noToken := append([]systemServerOutcome(nil), ready...)
+	noToken[0] = systemServerOutcome{name: "limbo-token (minecraft ns)", skipped: "source missing"}
+	if err := requiredProvisioningError(noToken); err == nil || !strings.Contains(err.Error(), "limbo-token") {
+		t.Fatalf("missing limbo token = %v, want named error", err)
+	}
+
 	failed := append([]systemServerOutcome(nil), ready...)
 	failed[3] = systemServerOutcome{name: naming.SystemLobbyServer, err: context.DeadlineExceeded}
 	if err := requiredProvisioningError(failed); err == nil || !strings.Contains(err.Error(), naming.SystemLobbyServer) {
@@ -581,3 +670,62 @@ func TestEnsureSystemServersRefreshesDerivedEnv(t *testing.T) {
 		}
 	})
 }
+
+// Setup's replicas carry each workload its own caller token and nothing more: the
+// login gate gets felis-limbo-token in the minecraft namespace, the build Jobs get
+// felis-build-token in the build namespace, a replica left stale by a rotation is
+// brought up to date, and the proxy's felis-service-token is copied nowhere.
+func TestProvisionSecretReplicasCarryCallerTokens(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	secret := func(ns, name, key, val string) *corev1.Secret {
+		return &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{key: []byte(val)},
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		secret("felis", "felis-service-token", "token", "proxy-tok"),
+		secret("felis", "felis-limbo-token", "token", "limbo-new"),
+		secret("felis", "felis-build-token", "token", "build-tok"),
+		secret("felis", "felis-ops-token", "token", "ops-tok"),
+		secret("felis", naming.ForwardingSecretName, naming.ForwardingSecretKey, "fwd"),
+		// What a rotation leaves behind before setup runs again.
+		secret("minecraft", "felis-limbo-token", "token", "limbo-old"),
+	).Build()
+
+	outcomes := provisionSecretReplicas(ctx, cl, "felis", "minecraft", "felis-build")
+	for _, o := range outcomes {
+		if o.err != nil {
+			t.Fatalf("%s: %v", o.name, o.err)
+		}
+	}
+
+	read := func(ns, name string) (string, bool) {
+		var s corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+			return "", false
+		}
+		return string(s.Data["token"]), true
+	}
+	if got, _ := read("minecraft", "felis-limbo-token"); got != "limbo-new" {
+		t.Errorf("minecraft/felis-limbo-token = %q, want the rotated limbo-new", got)
+	}
+	if got, _ := read("felis-build", "felis-build-token"); got != "build-tok" {
+		t.Errorf("felis-build/felis-build-token = %q, want build-tok", got)
+	}
+	for _, ns := range []string{"minecraft", "felis-build"} {
+		for _, name := range []string{"felis-service-token", "felis-ops-token"} {
+			if _, ok := read(ns, name); ok {
+				t.Errorf("%s/%s was replicated; only the control namespace holds it", ns, name)
+			}
+		}
+	}
+	if _, ok := read("minecraft", "felis-build-token"); ok {
+		t.Error("the build token was copied into the minecraft namespace")
+	}
+	if _, ok := read("felis-build", "felis-limbo-token"); ok {
+		t.Error("the limbo token was copied into the build namespace")
+	}
+}

cmd/felis/tui_alerts_test.go

@@ -0,0 +1,202 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/charmbracelet/lipgloss"
+
+	"felis.lolicon.best/internal/config"
+)
+
+// TestAlertRouteLines: the summary's alert rows say where the watchdog's
+// alerts go, and flag every route that reaches no one.
+func TestAlertRouteLines(t *testing.T) {
+	cases := []struct {
+		name     string
+		r        alertRoute
+		alerts   string
+		alertsOK bool
+		beat     string
+		beatOK   bool
+	}{
+		{
+			name:   "fresh install",
+			alerts: "only logged: no email relay (press e)",
+			beat:   "none: no outside check (troubleshooting.md §14)",
+		},
+		{
+			name:   "relay, no verified owner",
+			r:      alertRoute{relay: "smtp.example.com:587", heartbeat: "https://hc-ping.com/..."},
+			alerts: "only logged: no verified Owner email (panel → Account)",
+			beat:   "https://hc-ping.com/... every 2 minutes", beatOK: true,
+		},
+		{
+			name:   "relay, owners unreadable",
+			r:      alertRoute{relay: "smtp.example.com:587", lookupErr: errors.New("connection refused")},
+			alerts: "via smtp.example.com:587; could not read the Owner addresses",
+			beat:   "none: no outside check (troubleshooting.md §14)",
+		},
+		{
+			name:   "owners but no relay",
+			r:      alertRoute{recipients: []string{"[email protected]"}},
+			alerts: "only logged: no email relay (press e)",
+			beat:   "none: no outside check (troubleshooting.md §14)",
+		},
+		{
+			name:   "mailed",
+			r:      alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]", "[email protected]"}, heartbeatErr: errors.New("/etc/felis/watchdog-heartbeat-url: the heartbeat URL is not an http:// or https:// URL")},
+			alerts: "mailed to [email protected], [email protected] via smtp.example.com:587", alertsOK: true,
+			beat: "unreadable: /etc/felis/watchdog-heartbeat-url: the heartbeat URL is not an http:// or https:// URL",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if line, ok := c.r.alertsLine(); line != c.alerts || ok != c.alertsOK {
+				t.Errorf("alerts = %q, %v; want %q, %v", line, ok, c.alerts, c.alertsOK)
+			}
+			if line, ok := c.r.heartbeatLine(); line != c.beat || ok != c.beatOK {
+				t.Errorf("heartbeat = %q, %v; want %q, %v", line, ok, c.beat, c.beatOK)
+			}
+		})
+	}
+}
+
+// TestSummaryAlertRows: the rows show on the card, a route that needs action
+// marked so it reads without colour, and stay off a summary with no route.
+func TestSummaryAlertRows(t *testing.T) {
+	m := &summaryModel{ownerUsername: "owner", alerts: &alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]"}}}
+	v := m.View()
+	for _, want := range []string{"alerts    mailed to [email protected] via smtp.example.com:587", "heartbeat ⚠ none: no outside check"} {
+		if !strings.Contains(v, want) {
+			t.Errorf("summary lacks %q:\n%s", want, v)
+		}
+	}
+	if strings.Contains(v, "⚠ mailed") {
+		t.Errorf("a route that reaches the owners is marked:\n%s", v)
+	}
+	m.alerts = &alertRoute{heartbeat: "https://hc-ping.com/..."}
+	v = m.View()
+	for _, want := range []string{"alerts    ⚠ only logged: no email relay (press e)", "heartbeat https://hc-ping.com/... every 2 minutes"} {
+		if !strings.Contains(v, want) {
+			t.Errorf("summary lacks %q:\n%s", want, v)
+		}
+	}
+	m.alerts = nil
+	if v = m.View(); strings.Contains(v, "alerts") || strings.Contains(v, "heartbeat") {
+		t.Errorf("a summary with no route shows alert rows:\n%s", v)
+	}
+}
+
+// TestRootSummaryReadsAlertRoute: the summary and the re-run status screen
+// read the route each time they open, so a relay configured with e shows at
+// once.
+func TestRootSummaryReadsAlertRoute(t *testing.T) {
+	m := newTestRoot(false, consoleModeSetup, "")
+	reads := 0
+	route := alertRoute{}
+	m.alertRoute = func(context.Context) alertRoute {
+		reads++
+		return route
+	}
+	m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk"})
+	sum, ok := m.screen.(*summaryModel)
+	if !ok || sum.alerts == nil || sum.alerts.relay != "" {
+		t.Fatalf("summary after storage: %T %+v", m.screen, sum)
+	}
+	route = alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]"}}
+	m = drive(t, m, smtpResultMsg{configured: true})
+	sum, ok = m.screen.(*summaryModel)
+	if !ok || sum.alerts == nil || sum.alerts.relay != "smtp.example.com:587" {
+		t.Fatalf("summary after configuring email: %T %+v", m.screen, sum)
+	}
+	if reads != 2 {
+		t.Errorf("route read %d times, want 2", reads)
+	}
+
+	status := newTestRoot(true, consoleModeSetup, "")
+	status.alertRoute = m.alertRoute
+	status.showStatus()
+	if sum, ok := status.screen.(*summaryModel); !ok || sum.alerts == nil || sum.alerts.relay != "smtp.example.com:587" {
+		t.Fatalf("status screen: %T %+v", status.screen, status.screen)
+	}
+}
+
+// TestHostAlertRoute reads the relay from the host config and the heartbeat
+// file, shows the heartbeat by its host only, and reports a database it cannot
+// reach.
+func TestHostAlertRoute(t *testing.T) {
+	dir := t.TempDir()
+	cfg := filepath.Join(dir, "felis.host.toml")
+	writeTestFile(t, cfg, testWatchdogConfig+testWatchdogSMTP, 0o600)
+	beat := filepath.Join(dir, "watchdog-heartbeat-url")
+	writeTestFile(t, beat, "https://hc-ping.com/check-key\n", 0o600)
+	dbURL := "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
+
+	r := hostAlertRoute(context.Background(), cfg, dbURL, beat)
+	if r.relay != "smtp.config.example:2525" {
+		t.Errorf("relay = %q", r.relay)
+	}
+	if r.heartbeat != "https://hc-ping.com/..." || r.heartbeatErr != nil {
+		t.Errorf("heartbeat = %q, %v", r.heartbeat, r.heartbeatErr)
+	}
+	if r.lookupErr == nil {
+		t.Errorf("an unreachable database reads as %v", r.recipients)
+	}
+
+	noRelay := filepath.Join(dir, "no-relay.toml")
+	writeTestFile(t, noRelay, testWatchdogConfig, 0o600)
+	writeTestFile(t, beat, "hc-ping.com/check-key\n", 0o600)
+	r = hostAlertRoute(context.Background(), noRelay, dbURL, beat)
+	if r.relay != "" {
+		t.Errorf("no [smtp]: relay = %q", r.relay)
+	}
+	if r.heartbeatErr == nil || strings.Contains(r.heartbeatErr.Error(), "check-key") {
+		t.Errorf("a bad heartbeat file: %v", r.heartbeatErr)
+	}
+	r = hostAlertRoute(context.Background(), noRelay, dbURL, filepath.Join(dir, "none"))
+	if r.heartbeat != "" || r.heartbeatErr != nil {
+		t.Errorf("no heartbeat file: %q, %v", r.heartbeat, r.heartbeatErr)
+	}
+}
+
+// TestSummaryWithAlertsFitsTerminal: the two rows keep the summary inside the
+// terminal, with the longest route lines.
+func TestSummaryWithAlertsFitsTerminal(t *testing.T) {
+	for _, w := range []int{60, 80, 90} {
+		for _, h := range []int{24, 30, 45} {
+			m := newTestRoot(false, consoleModeSetup, "")
+			m.alertRoute = func(context.Context) alertRoute {
+				return alertRoute{relay: "smtp.example.com:587", lookupErr: errors.New("dial tcp 127.0.0.1:5432: connect: connection refused")}
+			}
+			m = drive(t, m, tea.WindowSizeMsg{Width: w, Height: h})
+			m = drive(t, m, preflightDoneMsg{})
+			m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
+			m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.example.com"})
+			m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"})
+			if _, ok := m.screen.(*summaryModel); !ok {
+				t.Fatalf("screen = %T, want the summary", m.screen)
+			}
+			if got := lipgloss.Height(m.View()); got > h {
+				t.Errorf("terminal %dx%d: summary with alert rows = %d rows (exceeds height)", w, h, got)
+			}
+		}
+	}
+}
+
+// TestConsoleRootReadsHostAlertRoute: the console the host runs gives its
+// summary this host's route, read against its database.
+func TestConsoleRootReadsHostAlertRoute(t *testing.T) {
+	db := config.DatabaseConfig{URL: "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"}
+	rm := newConsoleRoot(context.Background(), &fakeOwnerStore{}, db, "felis.example.com", "admin.felis.example.com", "panel.felis.example.com", "", "minecraft", "root", false, consoleModeSetup, recoveryConfig{})
+	if rm.alertRoute == nil {
+		t.Fatal("the console's summary reads no alert route")
+	}
+	if r := rm.alertRoute(context.Background()); r.lookupErr == nil {
+		t.Errorf("the route did not query the console's database: %+v", r)
+	}
+}

cmd/felis/tui_backupnow.go

@@ -86,7 +86,7 @@ func (m *backupModel) loadCmd() tea.Cmd {
 		if err != nil {
 			return backupListMsg{err: fmt.Errorf("list servers: %w", err)}
 		}
-		return backupListMsg{cl: cl, servers: servers}
+		return backupListMsg{cl: cl, servers: backupPickable(servers)}
 	}
 }
 

cmd/felis/tui_edge.go

@@ -437,6 +437,10 @@ func (m *edgeModel) errorView() string {
 	if m.lastErr != nil {
 		b.WriteString(tuiHint.Render(m.lastErr.Error()) + "\n")
 	}
+	// Nothing done above is rolled back, and nothing needs to be: every step finds what an
+	// earlier attempt created (the tunnel, its DNS route, the Access app and policy) and
+	// carries on from it.
+	b.WriteString("\n" + tuiHint.Render("Retrying is safe: it reuses the tunnel, DNS record and Access app created so far instead of making duplicates.") + "\n")
 	b.WriteString("\n" + tuiAction("enter", "retry", "esc", "edit"))
 	return b.String()
 }

cmd/felis/tui_edge_apply.go

@@ -32,7 +32,9 @@ func applyCloudflareEdge(ctx context.Context, result *cfsetup.Result, panelHost,
 	if adminHost == "" {
 		return fmt.Errorf("admin hostname is required")
 	}
-	if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud); err != nil {
+	// cloudflared is the only way in once the NodePort is fenced, so the
+	// visitor address it writes can key the sign-in rate limit.
+	if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud, "CF-Connecting-IP"); err != nil {
 		return err
 	}
 	if err := applyFelisConfigSecret(ctx); err != nil {
@@ -78,7 +80,8 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
 	if adminHost == "" {
 		return fmt.Errorf("admin hostname is required")
 	}
-	if err := writeConnectionConfig(panelHost, adminHost, ""); err != nil {
+	// Caddy, nginx and Traefik all append the peer they saw to X-Forwarded-For.
+	if err := writeConnectionConfig(panelHost, adminHost, "", "X-Forwarded-For"); err != nil {
 		return err
 	}
 	if err := applyFelisConfigSecret(ctx); err != nil {
@@ -93,16 +96,17 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
 // writeConnectionConfig stamps the chosen hostnames (and optional Access audience)
 // into both the host and pod config files. An empty aud clears any prior
 // Cloudflare audience, which is correct when switching to a non-Access front.
-func writeConnectionConfig(panelHost, adminHost, aud string) error {
+// clientIPHeader is the header that front writes the visitor address into.
+func writeConnectionConfig(panelHost, adminHost, aud, clientIPHeader string) error {
 	for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} {
-		if err := updateAuthConfig(path, panelHost, adminHost, aud); err != nil {
+		if err := updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func updateAuthConfig(path, panelHost, adminHost, aud string) error {
+func updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader string) error {
 	cfg, err := config.Load(path)
 	if err != nil {
 		return err
@@ -112,6 +116,7 @@ func updateAuthConfig(path, panelHost, adminHost, aud string) error {
 	}
 	cfg.Auth.AdminHostname = adminHost
 	cfg.Auth.AccessJWTAud = aud
+	cfg.Auth.ClientIPHeader = clientIPHeader
 	return writeConfig(path, cfg)
 }
 
@@ -133,6 +138,11 @@ func writeConfig(path string, cfg *config.Config) error {
 	return os.Rename(tmpPath, path)
 }
 
+// applyFelisConfigSecret applies the rendered config to the control namespace and
+// then converges the workload-namespace mirror best-effort. The mirror feeds the
+// backup/restore/fileedit Jobs and the reaper; without this refresh a reconfigure
+// here would leave those readers on the previous render until the next `felis
+// setup` run (startup pass) or installer re-run.
 func applyFelisConfigSecret(ctx context.Context) error {
 	out, err := kubectlOutput(ctx,
 		"-n", "felis", "create", "secret", "generic", "felis-config",
@@ -142,7 +152,41 @@ func applyFelisConfigSecret(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	return kubectlWithInput(ctx, out, "apply", "-f", "-")
+	if err := kubectlWithInput(ctx, out, "apply", "-f", "-"); err != nil {
+		return err
+	}
+	if err := replicateFelisConfigToWorkloadNamespace(ctx); err != nil {
+		fmt.Fprintf(os.Stderr, "felis setup: warning: the control-plane config is applied, but the workload-namespace mirror could not be refreshed (%v); re-run felis setup once that is fixed\n", err)
+	}
+	return nil
+}
+
+// replicateFelisConfigToWorkloadNamespace overwrites the workload-namespace
+// felis-config mirror with the freshly rendered pod config. Deliberately a full
+// replace, not create-if-absent: a stale mirror is exactly what silently hands
+// the Jobs that mount it old settings after a reconfigure. No-op when the
+// workload namespace is unset or is the control namespace itself.
+func replicateFelisConfigToWorkloadNamespace(ctx context.Context) error {
+	cfg, err := config.Load(hostSetupConfigPath)
+	if err != nil {
+		return err
+	}
+	ns := cfg.K8s.Namespace
+	if ns == "" || ns == "felis" {
+		return nil
+	}
+	manifest, err := kubectlOutput(ctx,
+		"-n", ns, "create", "secret", "generic", "felis-config",
+		"--from-file=felis.toml="+podSetupConfigPath,
+		"--dry-run=client", "-o", "yaml",
+	)
+	if err != nil {
+		return fmt.Errorf("render felis-config for %s: %w", ns, err)
+	}
+	if err := kubectlWithInput(ctx, manifest, "-n", ns, "apply", "-f", "-"); err != nil {
+		return fmt.Errorf("replicate felis-config to %s: %w", ns, err)
+	}
+	return nil
 }
 
 func installCloudflaredService(ctx context.Context, cloudflaredBin, configPath string) error {

cmd/felis/tui_mc_bind.go

@@ -207,7 +207,11 @@ func (m *mcBindModel) doneView() string {
 		if box.Len() > 0 {
 			box.WriteString("\n")
 		}
-		box.WriteString(tuiLabel.Render("setup URL  ") + "\n" + tuiPassword.Render(m.setupTokenURL) + "\n\n")
+		box.WriteString(tuiLabel.Render("setup URL  ") + "\n")
+		for _, line := range wrapDisplayURL(m.setupTokenURL, 70) {
+			box.WriteString(tuiPassword.Render(line) + "\n")
+		}
+		box.WriteString("\n")
 		box.WriteString(tuiWarn.Render("Open this URL to complete passwordless login setup.\nIt is shown only once."))
 	}
 	if m.auditWarning != "" {

cmd/felis/tui_menu_test.go

@@ -151,19 +151,46 @@ func TestOwnerModelProvisionErrorRouting(t *testing.T) {
 		}
 	})
 
-	t.Run("a conflict on the Owner path is not a retry", func(t *testing.T) {
-		// Defensive: the Owner upserts and so never conflicts, but were one ever to
-		// surface it must end the session rather than loop the form — only the
-		// insert-only operator path is retryable.
+	t.Run("the Owner seat refusal returns to the form naming the seat", func(t *testing.T) {
+		// Upserting a fresh username while a seat is occupied would mint a second
+		// owner, so provisionOwner refuses with ownerSeatTakenError (Is
+		// api.ErrConflict) and the console must route back for a retype — the same
+		// recoverable contract as the operator clash, and the only Owner-path
+		// conflict there is.
 		m := newOwnerModel(ctx, &fakeOwnerStore{}, "root", true)
+		seatErr := &ownerSeatTakenError{seat: "seat-holder"}
 
-		next, cmd := m.Update(owProvisionMsg{err: conflict})
+		next, cmd := m.Update(owProvisionMsg{err: seatErr})
+		om := next.(*ownerModel)
+		if om.step != owProvision {
+			t.Fatalf("step = %v, want owProvision — the seat refusal is recoverable", om.step)
+		}
+		if om.provisionErr == nil || !errors.Is(om.provisionErr, api.ErrConflict) || !strings.Contains(om.provisionErr.Error(), "seat-holder") {
+			t.Errorf("provisionErr = %v, want the seat refusal naming the seat", om.provisionErr)
+		}
+		// Feed the rebuilt form's init message back through so its view renders;
+		// then the note must carry the seat name (the operator's retype cue).
+		if cmd != nil {
+			if msg := cmd(); msg != nil {
+				if n2, _ := om.Update(msg); n2 != nil {
+					om = n2.(*ownerModel)
+				}
+			}
+		}
+		if view := om.form.View(); !strings.Contains(view, "seat-holder") {
+			t.Errorf("the provision form must surface the seat refusal:\n%s", view)
+		}
+	})
+
+	t.Run("a generic Owner-path fault still tears the console down", func(t *testing.T) {
+		m := newOwnerModel(ctx, &fakeOwnerStore{}, "root", true)
+		next, cmd := m.Update(owProvisionMsg{err: errors.New("boom")})
 		om := next.(*ownerModel)
 		if om.provisionErr != nil {
-			t.Error("the Owner path recorded a retryable conflict; only the operator path retries")
+			t.Error("a generic fault must not be treated as a retryable refusal")
 		}
 		if res, ok := cmd().(ownerResultMsg); !ok || res.err == nil {
-			t.Error("an Owner-path conflict should tear down via an error result")
+			t.Error("a generic Owner-path fault should tear down via an error result")
 		}
 	})
 }

cmd/felis/tui_migration.go

@@ -3,18 +3,21 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"time"
 
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
 // applyMigrations runs any pending schema migrations and returns the resulting
 // applied count. Used by the preflight stage to self-heal a freshly bootstrapped
 // (or upgraded) database.
-func applyMigrations(dbURL string) (int, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+func applyMigrations(db config.DatabaseConfig) (int, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
-	drv, err := store.Open(ctx, dbURL)
+	drv, err := store.Open(ctx, db.URL)
 	if err != nil {
 		return 0, fmt.Errorf("open db: %w", err)
 	}
@@ -23,20 +26,17 @@ func applyMigrations(dbURL string) (int, error) {
 	if err != nil {
 		return 0, err
 	}
+	// Same guard as `felis migrate up`: never roll a populated database forward
+	// without a snapshot to roll back to.
+	if _, err := preMigrateBackup(ctx, drv, migrations, db, dbbackup.DefaultDir, io.Discard); err != nil {
+		return 0, fmt.Errorf("pre-migration backup: %w", err)
+	}
 	if _, err := store.Up(ctx, drv, migrations); err != nil {
 		return 0, err
 	}
-	applied, err := drv.AppliedVersions(ctx)
+	s, err := store.ReadSchema(ctx, drv)
 	if err != nil {
 		return 0, err
 	}
-	return len(applied), nil
-}
-
-func totalMigrations() (int, error) {
-	migrations, err := store.LoadMigrations()
-	if err != nil {
-		return 0, err
-	}
-	return len(migrations), nil
+	return s.Applied, nil
 }

cmd/felis/tui_owner.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"felis.lolicon.best/internal/api"
 
@@ -14,9 +15,8 @@ import (
 )
 
 type owAuthMsg struct {
-	matched string
-	ok      bool
-	err     error
+	start recoveryStart
+	err   error
 }
 
 type owProvisionMsg struct {
@@ -29,14 +29,15 @@ type owStep int
 const (
 	owAuth owStep = iota
 	owOverride
+	owCode // typing the recovery code mailed to the named admin
 	owProvision
 	owWorking
 	owDone
 	owError
 )
 
-// ownerModel collects the owner account. The input phases (admin auth, root
-// override, owner details) are huh forms; the async phases (verifying,
+// ownerModel collects the owner account. The input phases (admin name, recovery
+// code, root override, owner details) are huh forms; the async phases (verifying,
 // provisioning) show a spinner; the done phase shows the credential card. The
 // outward contract is unchanged: it emits an ownerResultMsg when finished.
 //
@@ -55,6 +56,18 @@ type ownerModel struct {
 	mode        string // "bootstrap", "recovery", "root_override"
 	accountable string
 	attempt     string
+	recovery    recoveryConfig
+
+	// Recovery proof: the admin the typed name resolved to, the code mailed to it,
+	// and once settled either how it was proven or why the run fell back to the
+	// override.
+	admin      *api.StaffUser
+	code       *recoveryCode
+	codeNote   string // "wrong code" line shown above a rebuilt code form
+	verifiedBy string
+	codeSentTo string
+	skip       string // otpSkip*
+	skipDetail string
 
 	step         owStep
 	form         *huh.Form
@@ -66,6 +79,7 @@ type ownerModel struct {
 
 	// huh-bound form values
 	authUser    string
+	codeInput   string
 	overrideTok string
 	ownerUser   string
 	ownerEmail  string
@@ -124,6 +138,12 @@ func newOperatorModel(ctx context.Context, store ownerStore, osUser string) *own
 	return m
 }
 
+// withRecovery hands the model the relay its recovery codes go through.
+func (m *ownerModel) withRecovery(r recoveryConfig) *ownerModel {
+	m.recovery = r
+	return m
+}
+
 func (m *ownerModel) Init() tea.Cmd { return m.form.Init() }
 
 // subject is the human label for the account being provisioned, branching every
@@ -152,7 +172,7 @@ func (m *ownerModel) sized(f *huh.Form) *huh.Form {
 }
 
 func (m *ownerModel) isFormStep() bool {
-	return m.step == owAuth || m.step == owOverride || m.step == owProvision
+	return m.step == owAuth || m.step == owOverride || m.step == owCode || m.step == owProvision
 }
 
 func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
@@ -161,25 +181,25 @@ func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 		if msg.err != nil {
 			return m, m.failCmd(msg.err)
 		}
-		if msg.ok {
-			m.mode = "recovery"
-			m.accountable = msg.matched
-			m.step = owProvision
-			m.form = m.sized(m.buildProvisionForm())
+		m.admin = msg.start.admin
+		if msg.start.code != nil {
+			m.code = msg.start.code
+			m.codeNote = ""
+			m.step = owCode
+			m.form = m.sized(m.buildCodeForm())
 			return m, m.form.Init()
 		}
-		m.step = owOverride
-		m.form = m.sized(m.buildOverrideForm())
-		return m, m.form.Init()
+		return m.toOverride(msg.start.skip, msg.start.detail)
 
 	case owProvisionMsg:
 		if msg.err != nil {
-			// A taken Operator username is the expected, recoverable outcome of the
-			// insert-only operator path (refusing the clash is the whole reason it is
-			// insert-only, not an upsert). Route back to the form with a note so the
-			// operator can pick another name, rather than tearing down the console —
-			// any other error is a genuine fault and still ends the session.
-			if m.operation == bgAddOperator && errors.Is(msg.err, api.ErrConflict) {
+			// api.ErrConflict marks the two recoverable refusals: a taken Operator
+			// username (insert-only clash) and an Owner reset naming anything but the
+			// occupied seat (ownerSeatTakenError Is ErrConflict). Route back to the
+			// form with a note so the operator can retype, rather than tearing down
+			// the console — any other error is a genuine fault and still ends the
+			// session.
+			if errors.Is(msg.err, api.ErrConflict) {
 				m.provisionErr = msg.err
 				m.step = owProvision
 				m.form = m.sized(m.buildProvisionForm())
@@ -220,7 +240,9 @@ func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			case "ctrl+c":
 				return m, tea.Quit
 			case "esc":
-				if m.step == owOverride {
+				if m.step == owOverride || m.step == owCode {
+					// Start over: a code mailed for the old attempt dies with it.
+					m.admin, m.code, m.skip, m.skipDetail = nil, nil, "", ""
 					m.step = owAuth
 					m.form = m.sized(m.buildAuthForm())
 					return m, m.form.Init()
@@ -252,18 +274,41 @@ func (m *ownerModel) onFormComplete() (tea.Model, tea.Cmd) {
 	case owAuth:
 		m.attempt = strings.TrimSpace(m.authUser)
 		m.step = owWorking
-		m.working = "Verifying admin…"
-		user := m.authUser
+		m.working = "Sending a recovery code…"
+		user, rc, osUser, op := m.authUser, m.recovery, m.osUser, m.operation
 		return m, tea.Batch(m.sp.Tick, func() tea.Msg {
-			matched, ok, err := authenticateAdmin(m.ctx, m.store, user)
-			return owAuthMsg{matched: matched, ok: ok, err: err}
+			start, err := beginRecovery(m.ctx, m.store, rc, user, osUser, op)
+			return owAuthMsg{start: start, err: err}
 		})
+	case owCode:
+		typed := strings.TrimSpace(m.codeInput)
+		m.codeInput = ""
+		if typed == breakGlassOverrideToken {
+			m.skip, m.skipDetail = otpSkipByOperator, ""
+			m.code = nil
+			return m.proceedAsRoot()
+		}
+		switch m.code.check(typed, m.recovery.clock()) {
+		case codeAccepted:
+			m.mode = "recovery"
+			m.accountable = m.admin.Username
+			m.verifiedBy = verifiedByEmailOTP
+			m.codeSentTo = m.admin.Email
+			m.code = nil
+			m.step = owProvision
+			m.form = m.sized(m.buildProvisionForm())
+			return m, m.form.Init()
+		case codeWrong:
+			m.codeNote = fmt.Sprintf("That code is wrong. %d attempts left.", m.code.attemptsLeft())
+			m.form = m.sized(m.buildCodeForm())
+			return m, m.form.Init()
+		case codeExpired:
+			return m.toOverride(otpSkipCodeExpired, "")
+		default:
+			return m.toOverride(otpSkipCodeRejected, fmt.Sprintf("%d wrong codes", recoveryCodeAttempts))
+		}
 	case owOverride:
-		m.mode = "root_override"
-		m.accountable = m.osUser
-		m.step = owProvision
-		m.form = m.sized(m.buildProvisionForm())
-		return m, m.form.Init()
+		return m.proceedAsRoot()
 	case owProvision:
 		m.username = strings.TrimSpace(m.ownerUser)
 		m.step = owWorking
@@ -273,6 +318,24 @@ func (m *ownerModel) onFormComplete() (tea.Model, tea.Cmd) {
 	return m, nil
 }
 
+// toOverride records why no code proved an admin and asks for the typed OVERRIDE.
+func (m *ownerModel) toOverride(skip, detail string) (tea.Model, tea.Cmd) {
+	m.skip, m.skipDetail = skip, detail
+	m.code = nil
+	m.step = owOverride
+	m.form = m.sized(m.buildOverrideForm())
+	return m, m.form.Init()
+}
+
+// proceedAsRoot is the typed OVERRIDE: the run goes on as the OS user, unverified.
+func (m *ownerModel) proceedAsRoot() (tea.Model, tea.Cmd) {
+	m.mode = "root_override"
+	m.accountable = m.osUser
+	m.step = owProvision
+	m.form = m.sized(m.buildProvisionForm())
+	return m, m.form.Init()
+}
+
 func (m *ownerModel) provisionCmd() tea.Cmd {
 	op := breakGlassOp{
 		mode:           m.mode,
@@ -281,6 +344,10 @@ func (m *ownerModel) provisionCmd() tea.Cmd {
 		ownerUsername:  m.username,
 		ownerEmail:     m.ownerEmail,
 		attemptedAdmin: m.attempt,
+		verifiedBy:     m.verifiedBy,
+		codeSentTo:     m.codeSentTo,
+		otpSkipped:     m.skip,
+		otpSkipDetail:  m.skipDetail,
 	}
 	// performAddOperator and performBreakGlass share a signature; the operation
 	// discriminator selects which one runs. The operator path is insert-only and
@@ -319,7 +386,7 @@ func (m *ownerModel) buildAuthForm() *huh.Form {
 	return m.sized(newFelisForm(huh.NewGroup(
 		huh.NewNote().
 			Title("Admin authentication").
-			Description("A staff account already exists. Identify yourself to continue."),
+			Description("A staff account already exists. Name yours: a one-time code goes to its verified email address."),
 		huh.NewInput().
 			Title("Admin username").
 			Value(&m.authUser).
@@ -327,11 +394,64 @@ func (m *ownerModel) buildAuthForm() *huh.Form {
 	)))
 }
 
+func (m *ownerModel) buildCodeForm() *huh.Form {
+	desc := fmt.Sprintf("A recovery code went to %s, the verified address of %q. It works for %d minutes.\n\n"+
+		"No mail? Type %s to go on as OS user %q with root authority; the audit log records that as an unverified override. Esc starts over.",
+		maskEmail(m.admin.Email), m.admin.Username, int(recoveryCodeTTL/time.Minute), breakGlassOverrideToken, m.osUser)
+	if m.codeNote != "" {
+		desc = m.codeNote + "\n\n" + desc
+	}
+	return m.sized(newFelisForm(huh.NewGroup(
+		huh.NewNote().Title("Email verification").Description(desc),
+		huh.NewInput().
+			Title("Recovery code").
+			Value(&m.codeInput).
+			Validate(func(s string) error {
+				s = strings.TrimSpace(s)
+				if s == breakGlassOverrideToken || isRecoveryCodeShape(s) {
+					return nil
+				}
+				return errors.New("enter the 6-digit code, or " + breakGlassOverrideToken)
+			}),
+	)))
+}
+
+func isRecoveryCodeShape(s string) bool {
+	if len(s) != 6 {
+		return false
+	}
+	for _, c := range s {
+		if c < '0' || c > '9' {
+			return false
+		}
+	}
+	return true
+}
+
+// overrideReason says why no code proved an admin, first line of the override form.
+func (m *ownerModel) overrideReason() string {
+	switch m.skip {
+	case otpSkipUnknownAdmin:
+		return fmt.Sprintf("No staff account is named %q.", m.attempt)
+	case otpSkipNoVerifiedEmail:
+		return fmt.Sprintf("%q has no verified email address, so no recovery code can reach it.", m.admin.Username)
+	case otpSkipNoRelay:
+		return "No mail relay can send a recovery code: " + m.skipDetail + "."
+	case otpSkipSendFailed:
+		return "The recovery code could not be sent: " + m.skipDetail + "."
+	case otpSkipCodeExpired:
+		return "The recovery code expired."
+	case otpSkipCodeRejected:
+		return fmt.Sprintf("%d wrong codes; that code no longer works.", recoveryCodeAttempts)
+	}
+	return "No admin was verified."
+}
+
 func (m *ownerModel) buildOverrideForm() *huh.Form {
 	return m.sized(newFelisForm(huh.NewGroup(
 		huh.NewNote().
 			Title("Root override").
-			Description(fmt.Sprintf("That credential did not match. Proceed as OS user %q with root authority by typing the confirmation token.", m.osUser)),
+			Description(m.overrideReason()+"\n\n"+fmt.Sprintf("Proceed as OS user %q with root authority by typing the confirmation token. The audit log records this run as an unverified root override and the reason above. Esc starts over.", m.osUser)),
 		huh.NewInput().
 			Title("Type "+breakGlassOverrideToken+" to confirm").
 			Value(&m.overrideTok).
@@ -351,22 +471,28 @@ func (m *ownerModel) buildProvisionForm() *huh.Form {
 	desc := fmt.Sprintf("Create the first Owner — recorded as OS user %q.", m.osUser)
 	switch m.mode {
 	case "recovery":
-		desc = fmt.Sprintf("Authenticated as %q.", m.accountable)
+		desc = fmt.Sprintf("Verified as %q by an email code.", m.accountable)
 	case "root_override":
 		desc = "Root override — the Owner will be reset."
 	}
 	if m.operation == bgAddOperator {
 		switch m.mode {
 		case "recovery":
-			desc = fmt.Sprintf("Add an Operator — authenticated as %q.", m.accountable)
+			desc = fmt.Sprintf("Add an Operator — verified as %q by an email code.", m.accountable)
 		case "root_override":
 			desc = "Add an Operator (root override)."
 		}
 	}
 	if m.provisionErr != nil {
-		// The only error routed back to this form is a username clash on the insert-only
-		// operator path; show a concrete prompt to choose another name.
-		desc = "That username is already taken — choose a different one.\n\n" + desc
+		// Recoverable refusals routed back here: the seat refusal already names the
+		// username to enter, so show it verbatim; the operator-name clash gets the
+		// generic retry prompt.
+		note := "That username is already taken — choose a different one."
+		var seatErr *ownerSeatTakenError
+		if errors.As(m.provisionErr, &seatErr) {
+			note = seatErr.Error()
+		}
+		desc = note + "\n\n" + desc
 	}
 
 	fields := []huh.Field{
@@ -420,7 +546,11 @@ func (m *ownerModel) doneView() string {
 	var box strings.Builder
 	box.WriteString(tuiLabel.Render("username  ") + m.username + "\n")
 	if m.setupTokenURL != "" {
-		box.WriteString("\n" + tuiLabel.Render("setup URL  ") + "\n" + tuiPassword.Render(m.setupTokenURL) + "\n\n")
+		box.WriteString("\n" + tuiLabel.Render("setup URL  ") + "\n")
+		for _, line := range wrapDisplayURL(m.setupTokenURL, 70) {
+			box.WriteString(tuiPassword.Render(line) + "\n")
+		}
+		box.WriteString("\n")
 		box.WriteString(tuiWarn.Render("Open this URL to complete passwordless login setup. It is shown only once."))
 	}
 	if m.auditWarning != "" {

cmd/felis/tui_postgres.go

@@ -72,20 +72,14 @@ func parseDBURL(url string) (dbCfg, error) {
 	}, nil
 }
 
-func countMigrations(dbURL string) (int, error) {
+// readSchema lines the database's recorded migrations up with this build's.
+func readSchema(dbURL string) (store.Schema, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	drv, err := store.Open(ctx, dbURL)
 	if err != nil {
-		return 0, err
+		return store.Schema{}, err
 	}
 	defer drv.Close()
-	if err := drv.EnsureVersionTable(ctx); err != nil {
-		return 0, err
-	}
-	applied, err := drv.AppliedVersions(ctx)
-	if err != nil {
-		return 0, err
-	}
-	return len(applied), nil
+	return store.ReadSchema(ctx, drv)
 }

cmd/felis/tui_preflight.go

@@ -3,6 +3,8 @@ package main
 import (
 	"fmt"
 
+	"felis.lolicon.best/internal/config"
+
 	"github.com/charmbracelet/bubbles/spinner"
 	tea "github.com/charmbracelet/bubbletea"
 )
@@ -14,7 +16,7 @@ import (
 // of the wizard can assume a healthy backend. It is non-interactive — it runs
 // to completion and reports back to the root via preflightDoneMsg.
 type preflightModel struct {
-	dbURL      string
+	db         config.DatabaseConfig
 	rootDomain string
 	adminHost  string
 
@@ -46,6 +48,7 @@ type pfDBMsg struct{ err error }
 type pfMigCheckMsg struct {
 	applied int
 	total   int
+	pending bool
 	err     error
 }
 
@@ -56,11 +59,11 @@ type pfMigApplyMsg struct {
 
 type pfPanelMsg struct{ err error }
 
-func newPreflightModel(dbURL, rootDomain, adminHostname string) *preflightModel {
+func newPreflightModel(db config.DatabaseConfig, rootDomain, adminHostname string) *preflightModel {
 	sp := spinner.New()
 	sp.Spinner = spinner.Dot
 	sp.Style = tuiLabel
-	return &preflightModel{dbURL: dbURL, rootDomain: rootDomain, adminHost: adminHostname, sp: sp, state: pfCheckDB}
+	return &preflightModel{db: db, rootDomain: rootDomain, adminHost: adminHostname, sp: sp, state: pfCheckDB}
 }
 
 func (m *preflightModel) Init() tea.Cmd {
@@ -84,7 +87,7 @@ func (m *preflightModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			return m, nil
 		}
 		m.applied, m.total = msg.applied, msg.total
-		if msg.applied < msg.total {
+		if msg.pending {
 			m.state = pfApplyMig
 			return m, m.applyMigrations()
 		}
@@ -199,23 +202,25 @@ func (m *preflightModel) errText() string {
 }
 
 func (m *preflightModel) checkDB() tea.Cmd {
-	return func() tea.Msg { return pfDBMsg{err: checkPostgres(m.dbURL)} }
+	return func() tea.Msg { return pfDBMsg{err: checkPostgres(m.db.URL)} }
 }
 
 func (m *preflightModel) checkMigrations() tea.Cmd {
 	return func() tea.Msg {
-		applied, err := countMigrations(m.dbURL)
-		if err != nil {
-			return pfMigCheckMsg{err: err}
+		// The sets, not their sizes: a database a newer release migrated can hold as
+		// many rows as this build has migrations, and must stop here rather than be
+		// "healed" by an older binary.
+		s, err := readSchema(m.db.URL)
+		if err == nil {
+			err = s.Newer()
 		}
-		total, err := totalMigrations()
-		return pfMigCheckMsg{applied: applied, total: total, err: err}
+		return pfMigCheckMsg{applied: s.Applied, total: s.Total, pending: len(s.Pending) > 0, err: err}
 	}
 }
 
 func (m *preflightModel) applyMigrations() tea.Cmd {
 	return func() tea.Msg {
-		applied, err := applyMigrations(m.dbURL)
+		applied, err := applyMigrations(m.db)
 		return pfMigApplyMsg{applied: applied, err: err}
 	}
 }

cmd/felis/tui_root.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"felis.lolicon.best/internal/cfsetup"
+	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/platform"
 
 	tea "github.com/charmbracelet/bubbletea"
@@ -133,7 +134,7 @@ type rootModel struct {
 	err    error
 
 	mode        consoleMode
-	dbURL       string
+	db          config.DatabaseConfig
 	store       ownerStore
 	osUser      string
 	rootDomain  string
@@ -142,13 +143,17 @@ type rootModel struct {
 	accessAud   string
 	namespace   string // minecraft workload namespace (cfg.K8s.Namespace); target of the halt op
 	adminExists bool
+	recovery    recoveryConfig // how the account operations mail a recovery code
+	// alertRoute reads where the watchdog's alerts go for the summary; nil
+	// leaves those rows out.
+	alertRoute func(context.Context) alertRoute
 }
 
-func newRootModel(ctx context.Context, store ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) *rootModel {
+func newRootModel(ctx context.Context, store ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) *rootModel {
 	rm := &rootModel{
 		ctx:         ctx,
 		reviewing:   -1,
-		dbURL:       dbURL,
+		db:          db,
 		store:       store,
 		osUser:      osUser,
 		rootDomain:  rootDomain,
@@ -179,7 +184,7 @@ func newRootModel(ctx context.Context, store ownerStore, dbURL, rootDomain, admi
 		}
 	} else {
 		rm.stage = stagePreflight
-		rm.screen = newPreflightModel(dbURL, rootDomain, adminHostname)
+		rm.screen = newPreflightModel(db, rootDomain, adminHostname)
 	}
 	return rm
 }
@@ -221,7 +226,7 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 		m.stage = stageOwner
 		switch msg.op {
 		case bgAddOperator:
-			return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser))
+			return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser).withRecovery(m.recovery))
 		case bgHaltServer:
 			return m.adopt(newHaltModel(m.ctx, m.store, m.namespace, m.osUser))
 		case bgSyncBackup:
@@ -229,7 +234,7 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			// Service + service token the peer dials live in the control namespace.
 			return m.adopt(newBackupModel(m.ctx, m.namespace, platform.DefaultControlNamespace, m.osUser))
 		default:
-			return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists))
+			return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists).withRecovery(m.recovery))
 		}
 
 	case haltResultMsg:
@@ -553,6 +558,8 @@ func (m *rootModel) showSummary() (tea.Model, tea.Cmd) {
 		storageLabel:  m.result.storageDetail,
 		routedHosts:   routed,
 		localHint:     m.result.connectMethod == connectLocal,
+		alreadySetUp:  m.result.alreadySetUp,
+		alerts:        m.readAlertRoute(),
 	})
 }
 
@@ -573,9 +580,20 @@ func (m *rootModel) showStatus() (tea.Model, tea.Cmd) {
 		accessLabel:  accessLabel,
 		alreadySetUp: true,
 		localHint:    m.accessAud == "" && rootDomainEmbeddedIP(m.rootDomain) != "",
+		alerts:       m.readAlertRoute(),
 	})
 }
 
+// readAlertRoute reads the alert route afresh, so the summary shows a relay the
+// Owner just configured with e.
+func (m *rootModel) readAlertRoute() *alertRoute {
+	if m.alertRoute == nil {
+		return nil
+	}
+	r := m.alertRoute(m.ctx)
+	return &r
+}
+
 func panelURLFor(method connectMethod, panelHostname, rootDomain, adminHostname string) string {
 	if method != connectLocal && panelHostname != "" {
 		return "https://" + panelHostname

cmd/felis/tui_root_test.go

@@ -5,6 +5,8 @@ import (
 	"strings"
 	"testing"
 
+	"felis.lolicon.best/internal/config"
+
 	tea "github.com/charmbracelet/bubbletea"
 )
 
@@ -30,7 +32,7 @@ func newTestRoot(adminExists bool, mode consoleMode, accessAud string) *rootMode
 	return newRootModel(
 		context.Background(),
 		&fakeOwnerStore{},
-		"postgres://localhost/felis",
+		config.DatabaseConfig{URL: "postgres://localhost/felis"},
 		"felis.example.com",
 		"admin.felis.example.com",
 		"panel.felis.example.com",
@@ -249,6 +251,34 @@ func TestRootReconfigureSMTP(t *testing.T) {
 	}
 }
 
+// TestRootReconfigureStorageKeepsStatusFraming locks the same rule for the
+// "change storage" path: on a re-run, completing it must land back on the
+// alreadySetUp status framing (with the updated recap), not "Setup complete."
+func TestRootReconfigureStorageKeepsStatusFraming(t *testing.T) {
+	m := newTestRoot(true, consoleModeSetup, "")
+	m = drive(t, m, preflightDoneMsg{})
+	if _, ok := m.screen.(*summaryModel); !ok {
+		t.Fatalf("re-run after preflight, screen = %T, want *summaryModel", m.screen)
+	}
+
+	m = drive(t, m, reconfigureStorageMsg{})
+	if _, ok := m.screen.(*storageChooserModel); !ok {
+		t.Fatalf("reconfigure-storage screen = %T, want *storageChooserModel", m.screen)
+	}
+
+	m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"})
+	sum, ok := m.screen.(*summaryModel)
+	if !ok {
+		t.Fatalf("after reconfigure-storage, screen = %T, want *summaryModel", m.screen)
+	}
+	if !sum.alreadySetUp {
+		t.Fatalf("after reconfigure-storage, summary should keep the alreadySetUp framing")
+	}
+	if sum.storageLabel != "local disk · /var/lib/felis/uploads" {
+		t.Fatalf("storageLabel = %q, want the updated recap", sum.storageLabel)
+	}
+}
+
 func TestRootRerunLandsOnStatus(t *testing.T) {
 	// adminExists at start of a setup run = re-run: preflight should skip straight
 	// to the "manage in panel" status screen, never touching owner/connect.

cmd/felis/tui_smtp.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/mail"
 	"felis.lolicon.best/internal/platform"
 
 	"github.com/charmbracelet/bubbles/spinner"
@@ -276,11 +276,14 @@ func validateSMTPFrom(s string) error {
 }
 
 // currentSMTPInputs reads the relay already recorded in felis.toml so the form
-// pre-fills the non-secret fields. The password lives only in the felis-smtp
-// Secret and is deliberately never read back — it must be re-entered to change.
+// pre-fills the non-secret fields. The password (the felis-smtp Secret and its
+// host copy) is deliberately never read back — it must be re-entered to change.
 // Any read error falls back to a blank form rather than blocking reconfig.
-func currentSMTPInputs() smtpInputs {
-	cfg, err := config.Load(hostSetupConfigPath)
+func currentSMTPInputs() smtpInputs { return smtpInputsFrom(hostSetupConfigPath) }
+
+// smtpInputsFrom is currentSMTPInputs for the config at path.
+func smtpInputsFrom(path string) smtpInputs {
+	cfg, err := config.Load(path)
 	if err != nil || cfg.SMTP.Host == "" {
 		return smtpInputs{}
 	}
@@ -295,7 +298,8 @@ func currentSMTPInputs() smtpInputs {
 // applySMTPConfig proves the relay works, then persists it and rolls felis-api:
 // Ping (a full transaction — connect/STARTTLS/AUTH/MAIL FROM/RCPT/DATA, which
 // delivers one self-test message to the From address) → [smtp] into both config
-// files → the felis-smtp Secret → the config Secret → rollout. A failed Ping
+// files → the password into /etc/felis/smtp-password (hostcreds.go) → the
+// felis-smtp Secret → the config Secret → rollout. A failed Ping
 // leaves the install untouched, so a bad relay dies at the keyboard, not at a
 // player's OTP.
 //
@@ -310,48 +314,77 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
 	if err != nil {
 		return fmt.Errorf("port %q is not a number", in.port)
 	}
-	relay := &mail.SMTP{Host: in.host, Port: port, From: in.from, Username: in.username, Password: in.password}
-	if err := relay.Ping(ctx); err != nil {
+	var prev config.SMTPConfig
+	if cur, err := config.Load(hostSetupConfigPath); err == nil {
+		prev = cur.SMTP
+	}
+	// Ping under the posture felis api will send with, so a relay without
+	// STARTTLS is turned down here rather than at a player's first code.
+	if err := smtpRelay(setupSMTPConfig(in, port, prev), in.password).Ping(ctx); err != nil {
 		return err
 	}
 
-	smtpCfg := config.SMTPConfig{
-		Host:        in.host,
-		Port:        port,
-		From:        in.from,
-		Username:    in.username,
-		PasswordRef: platform.SMTPPasswordEnv,
-	}
 	for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} {
 		cfg, err := config.Load(path)
 		if err != nil {
 			return err
 		}
-		cfg.SMTP = smtpCfg
+		cfg.SMTP = setupSMTPConfig(in, port, cfg.SMTP)
 		if err := writeConfig(path, cfg); err != nil {
 			return err
 		}
 	}
+	// The host copy first: should the Secret fail, the next installer run applies
+	// it from this file.
+	if err := writeHostCredential(hostSMTPPasswordPath, in.password); err != nil {
+		return fmt.Errorf("keep the relay password in %s: %w", hostSMTPPasswordPath, err)
+	}
 	if err := applySMTPSecret(ctx, in.password); err != nil {
 		return err
 	}
 	if err := applyFelisConfigSecret(ctx); err != nil {
 		return err
 	}
+	// Refresh the workload-namespace copies too (the reaper's warning path): the
+	// OTP path is already live in the control namespace, so a replica miss is
+	// reported but not fatal.
+	if err := replicateSMTPToWorkloadNamespace(ctx, in.password); err != nil {
+		fmt.Fprintf(os.Stderr, "felis setup: warning: email is configured, but refreshing the workload copies failed (pre-reap warning emails may stay suppressed): %v\n", err)
+	}
 	if err := kubectl(ctx, "-n", "felis", "rollout", "restart", "deployment/felis-api"); err != nil {
 		return err
 	}
 	return kubectl(ctx, "-n", "felis", "rollout", "status", "deployment/felis-api", "--timeout=180s")
 }
 
-// applySMTPSecret creates (or replaces) the felis-smtp Secret the felis-api
-// Deployment injects the relay password from. Rendered in-process and piped to
+// setupSMTPConfig is the [smtp] block this screen writes: the relay it just
+// proved, plus the keys only an operator sets by hand (require_tls,
+// max_per_hour), carried over from the block it replaces so reconfiguring the
+// relay does not quietly reset them.
+func setupSMTPConfig(in smtpInputs, port int, prev config.SMTPConfig) config.SMTPConfig {
+	return config.SMTPConfig{
+		Host:        in.host,
+		Port:        port,
+		From:        in.from,
+		Username:    in.username,
+		PasswordRef: platform.SMTPPasswordEnv,
+		MaxPerHour:  prev.MaxPerHour,
+		RequireTLS:  prev.RequireTLS,
+	}
+}
+
+// smtpSecretManifest renders the felis-smtp Secret for the given namespace, the
+// one the receiving Deployment/CronJob resolves its secretKeyRef against (felis
+// for felis-api, the workload namespace for the reaper's mirror). The namespace
+// must be IN the manifest: kubectl rejects a manifest whose namespace conflicts
+// with -n, so leaving the control namespace hardcoded made every workload-ns
+// replica fail before it started. Rendered in-process and piped to
 // `kubectl apply` — the password is never a command-line arg, so it never
 // appears in the host process table.
-func applySMTPSecret(ctx context.Context, password string) error {
+func smtpSecretManifest(password, namespace string) ([]byte, error) {
 	secret := &corev1.Secret{
 		TypeMeta:   metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"},
-		ObjectMeta: metav1.ObjectMeta{Name: platform.SMTPSecretName, Namespace: "felis"},
+		ObjectMeta: metav1.ObjectMeta{Name: platform.SMTPSecretName, Namespace: namespace},
 		Type:       corev1.SecretTypeOpaque,
 		StringData: map[string]string{
 			platform.SMTPSecretPasswordKey: password,
@@ -359,7 +392,42 @@ func applySMTPSecret(ctx context.Context, password string) error {
 	}
 	manifest, err := yaml.Marshal(secret)
 	if err != nil {
-		return fmt.Errorf("render smtp secret: %w", err)
+		return nil, fmt.Errorf("render smtp secret: %w", err)
+	}
+	return manifest, nil
+}
+
+func applySMTPSecret(ctx context.Context, password string) error {
+	manifest, err := smtpSecretManifest(password, "felis")
+	if err != nil {
+		return err
 	}
 	return kubectlWithInput(ctx, manifest, "apply", "-f", "-")
 }
+
+// replicateSMTPToWorkloadNamespace refreshes the workload-namespace (minecraft)
+// copy of felis-smtp after email is reconfigured. The reaper's CronJob runs
+// there and resolves the password by local reference — a secretKeyRef is
+// namespace-local — so without this refresh a later SMTP change would never
+// reach the pre-reap warning emails. Deliberately OVERWRITES: this is a mirror
+// of the control-namespace source, and a stale mirror is exactly the failure
+// this closes. The felis-config mirror rides along in applyFelisConfigSecret,
+// which every apply path refreshes.
+func replicateSMTPToWorkloadNamespace(ctx context.Context, password string) error {
+	cfg, err := config.Load(hostSetupConfigPath)
+	if err != nil {
+		return err
+	}
+	ns := cfg.K8s.Namespace
+	if ns == "" || ns == "felis" {
+		return nil
+	}
+	smtpManifest, err := smtpSecretManifest(password, ns)
+	if err != nil {
+		return err
+	}
+	if err := kubectlWithInput(ctx, smtpManifest, "-n", ns, "apply", "-f", "-"); err != nil {
+		return fmt.Errorf("replicate %s to %s: %w", platform.SMTPSecretName, ns, err)
+	}
+	return nil
+}

cmd/felis/tui_smtp_test.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"sigs.k8s.io/yaml"
+)
+
+// TestSMTPSecretManifestCarriesTargetNamespace pins the fix for the
+// workload-namespace replica: kubectl refuses a manifest whose namespace
+// conflicts with -n ("the namespace from the provided object ... does not
+// match"), so the mirror must render felis-smtp with the TARGET namespace —
+// otherwise the "configure email" refresh fails on the first apply and the
+// felis-config mirror never runs at all.
+func TestSMTPSecretManifestCarriesTargetNamespace(t *testing.T) {
+	for _, ns := range []string{"felis", "minecraft"} {
+		b, err := smtpSecretManifest("pw", ns)
+		if err != nil {
+			t.Fatalf("render for %s: %v", ns, err)
+		}
+		var got struct {
+			Metadata struct {
+				Namespace string `json:"namespace"`
+			} `json:"metadata"`
+		}
+		if err := yaml.Unmarshal(b, &got); err != nil {
+			t.Fatalf("unmarshal for %s: %v", ns, err)
+		}
+		if got.Metadata.Namespace != ns {
+			t.Fatalf("manifest namespace = %q, want %q", got.Metadata.Namespace, ns)
+		}
+		if !strings.Contains(string(b), "name: felis-smtp") {
+			t.Fatalf("manifest must still name felis-smtp: %s", b)
+		}
+	}
+}
+
+// TestSMTPRelayCarriesTLSPosture: the relay felis api, the reaper and the
+// watchdog send through refuses plaintext for a remote host and allows it for
+// one on this host, as [smtp] says.
+func TestSMTPRelayCarriesTLSPosture(t *testing.T) {
+	remote := smtpRelay(config.SMTPConfig{Host: "smtp.example.net", Port: 587, From: "[email protected]", Username: "felis"}, "pw")
+	want := &mail.SMTP{Host: "smtp.example.net", Port: 587, From: "[email protected]", Username: "felis", Password: "pw", RequireTLS: true}
+	if !reflect.DeepEqual(remote, want) {
+		t.Errorf("remote relay = %+v, want %+v", remote, want)
+	}
+	if local := smtpRelay(config.SMTPConfig{Host: "127.0.0.1", Port: 25, From: "[email protected]"}, ""); local.RequireTLS {
+		t.Error("a relay on this host must not require TLS by default")
+	}
+}
+
+// TestSetupSMTPConfigKeepsHandSetKeys: re-running the email screen replaces the
+// relay but keeps require_tls and max_per_hour, which only an operator sets.
+func TestSetupSMTPConfigKeepsHandSetKeys(t *testing.T) {
+	off := false
+	prev := config.SMTPConfig{Host: "old.example.net", Port: 25, From: "[email protected]", MaxPerHour: 500, RequireTLS: &off}
+	in := smtpInputs{host: "smtp.example.net", from: "[email protected]", username: "felis"}
+	got := setupSMTPConfig(in, 465, prev)
+	if got.Host != "smtp.example.net" || got.Port != 465 || got.From != "[email protected]" ||
+		got.Username != "felis" || got.PasswordRef != "FELIS_SMTP_PASSWORD" {
+		t.Errorf("relay fields = %+v", got)
+	}
+	if got.MaxPerHour != 500 {
+		t.Errorf("max_per_hour = %d, want 500", got.MaxPerHour)
+	}
+	if got.RequireTLS == nil || *got.RequireTLS {
+		t.Errorf("require_tls = %v, want the operator's false", got.RequireTLS)
+	}
+	if fresh := setupSMTPConfig(in, 587, config.SMTPConfig{}); fresh.RequireTLS != nil || fresh.MaxPerHour != 0 {
+		t.Errorf("first setup = %+v, want require_tls and max_per_hour unset", fresh)
+	}
+}