style: unify panel cards and form controls

Manage Yggdrasil providers from the panel using durable platform settings, protected identity namespaces and atomic revisions. Apply changes to subsequent logins and profile lookups without restarting. Return operator-host logouts to the login method selection page.

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# The workflows pin every action to a commit SHA, and every Dockerfile pins its base images
+# by digest. This keeps those pins moving: Dependabot reads the "# vX.Y.Z" comment next to
+# each action SHA, and the tag in front of each image digest, and opens a PR that bumps both
+# together.
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directories:
+      - /
+      - /deploy/limbo
+      - /deploy/lobby
+      - /deploy/paper
+    schedule:
+      interval: weekly
+    # A new major is a runtime change (Paper 26.x needs Java 25, Limbo's jar is Java 21
+    # bytecode), so only digests and minors are proposed; majors move by hand.
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/workflows/ci.yml

@@ -9,15 +9,23 @@
 # The push trigger is limited to main rather than every branch, for the reason release.yml is
 # not repeated here: a branch with an open PR would otherwise run the whole suite twice per
 # push, once for refs/heads/<branch> and once for refs/pull/N/merge. Those are different
-# concurrency groups, so neither cancels the other, and this repository is private and billed
-# for both. A branch with no PR open yet is the one case that loses coverage, and opening the
+# concurrency groups, so neither cancels the other, and both would occupy runners for the
+# same commit. A branch with no PR open yet is the one case that loses coverage, and opening the
 # PR is what asks for the answer.
 name: ci
 
+#
+# release.yml calls this workflow (workflow_call) before it builds anything, so a tag passes
+# exactly these gates and there is one list of them.
+#
+# workflow_dispatch reruns the suite on a commit whose push produced no run, such as one
+# pushed while Actions was unavailable.
 on:
   push:
     branches: [main]
   pull_request:
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -30,9 +38,9 @@ jobs:
   go:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
           go-version-file: go.mod
 
@@ -43,12 +51,70 @@ jobs:
             echo "gofmt needed on:"; echo "$unformatted"; exit 1
           fi
       - run: go vet ./...
-      - run: go test ./...
+      # -race: felis-api and the operator are mostly goroutines (watchers, the
+      # registry pruner, the backup scheduler, the rate limiters).
+      - run: go test -race ./...
+      # The version is pinned here and bumped by hand; Dependabot does not read `go run`.
+      - name: staticcheck
+        run: go run honnef.co/go/tools/cmd/[email protected] ./...
+
+  # Separate from the go job so a newly published advisory reads as what it is. govulncheck
+  # exits non-zero only for vulnerable code this module can actually reach, standard
+  # library included: setup-go installs the newest patch of go.mod's Go line, so a finding
+  # there means the Dockerfile's golang digest (which ships the release) needs a bump too.
+  vuln:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go run golang.org/x/vuln/cmd/[email protected] ./...
+
+  # The business stores' SQL against a real PostgreSQL (internal/pgint): the unit suites run
+  # on fakes, and PGRepo drifted from them three times while those stayed green. 13 is the
+  # oldest server a supported distribution installs (EL9), 18 the newest (Arch).
+  # `felis db backup` and `restore` run there too, with the tools inside the service
+  # container, as production runs them inside felis-postgres: the runner's own client is one
+  # major version, and pg_dump refuses a newer server.
+  pgint:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        postgres: ['13', '18']
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres }}
+        env:
+          POSTGRES_USER: felis
+          POSTGRES_PASSWORD: pgint
+          POSTGRES_DB: felis_pgint
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U felis -d felis_pgint"
+          --health-interval 2s
+          --health-timeout 5s
+          --health-retries 30
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go test -race -tags pgint -count=1 ./internal/pgint/
+        env:
+          FELIS_TEST_PG_URL: postgres://felis:pgint@localhost:5432/felis_pgint?sslmode=disable
+          FELIS_TEST_PG_EXEC: docker exec -i ${{ job.services.postgres.id }}
 
   shell:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # bootstrap.sh is the only thing that ever runs on a fresh host, and nothing here can
       # run it — it wants root, a package manager and k3s. Syntax plus the extracted-block
@@ -66,12 +132,50 @@ jobs:
             esac
           done
 
+      # A pinned release rather than the runner image's copy, so a runner update cannot
+      # change what fails. Warnings and errors fail the job; style notes (info) do not.
+      - name: shellcheck
+        run: |
+          curl -fsSL -o shellcheck.tar.xz \
+            https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz
+          echo "8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198  shellcheck.tar.xz" | sha256sum -c
+          tar -xJf shellcheck.tar.xz
+          ./shellcheck-v0.11.0/shellcheck -S warning $(git ls-files '*.sh')
+
+      # An exit status is 8 bits, so `exit "$fails"` reads 256 failures as a pass. The suites
+      # exit 1 on any failure; this keeps the next one from carrying its count out.
+      - name: No script exits with its failure count
+        run: |
+          if git grep -nE 'exit +"?\$\{?[a-z_]*fail[a-z_]*\}?"?[[:space:]]*$' -- '*.sh'; then exit 1; fi
+
       - run: sh deploy/bootstrap_test.sh
+      - run: sh deploy/uninstall_test.sh
+      - run: bash deploy/e2e_release_test.sh
+      - run: bash deploy/e2e_upstream_test.sh
+
+  # The shipped alert rules (deploy/alerts): promtool parses them and runs their unit tests,
+  # which pin when each alert fires and that it stays quiet before. internal/metrics'
+  # alerts_test.go pins what promtool cannot see from there: the PrometheusRule twin and the
+  # metric names the rules read.
+  alerts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # A pinned release, like shellcheck's, so a new Prometheus cannot change what fails.
+      - name: promtool
+        run: |
+          curl -fsSL -o prometheus.tar.gz \
+            https://github.com/prometheus/prometheus/releases/download/v3.15.0/prometheus-3.15.0.linux-amd64.tar.gz
+          echo "2a542df32eac02ee17b9d844fb2aa1de00dafa5476579ba8a3ba862e9d572ea0  prometheus.tar.gz" | sha256sum -c
+          tar -xzf prometheus.tar.gz --strip-components=1 prometheus-3.15.0.linux-amd64/promtool
+          ./promtool check rules deploy/alerts/felis-alerts.yaml
+          ./promtool test rules deploy/alerts/felis-alerts_test.yml
 
   panel:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # The Dockerfile's `FROM node:<major>` is the only place the panel's Node version is
       # declared — there is no .nvmrc and no engines field. Reading it here rather than
@@ -84,7 +188,7 @@ jobs:
           [ -n "$version" ] || { echo "Dockerfile has no 'FROM ... node:<major>' line"; exit 1; }
           echo "version=${version}" >> "$GITHUB_OUTPUT"
 
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: ${{ steps.node.outputs.version }}
           cache: npm
@@ -96,44 +200,71 @@ jobs:
       - run: npm test
         working-directory: panel
 
+      # `tsc -b`: tsconfig.json is a solution file (files: [] plus references), so a plain
+      # `tsc --noEmit` checked nothing and passed with type errors in the tree.
       - run: npm run typecheck
         working-directory: panel
 
+      # Rules of hooks and effect dependency lists, with --max-warnings 0.
+      - run: npm run lint
+        working-directory: panel
+
+      # openapi.gen.ts is generated from docs/openapi.yaml and checked in, so the
+      # compile-time parity in src/lib/types.parity.ts needs no generator in the build;
+      # a schema edit that was not regenerated fails here.
+      - name: openapi.gen.ts matches docs/openapi.yaml
+        working-directory: panel
+        run: |
+          npm run gen:api
+          git diff --exit-code -- src/lib/openapi.gen.ts
+
+      # Browser smoke over the mock-mode dev server, in the runner's installed Chrome
+      # (playwright.config.ts sets channel: chrome, so nothing is downloaded).
+      - run: npm run test:e2e
+        working-directory: panel
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: panel-playwright-report
+          path: |
+            panel/playwright-report
+            panel/test-results
+          retention-days: 7
+
   plugins:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # The other jobs never touch the Java layer: the plugin jars were only ever
-      # compiled by bootstrap on a live host, and the three test mains under
-      # plugins/*/test were run by hand. JDK 21 plus the Gradle major the plugin
-      # Dockerfiles pin (8.14) is that same toolchain, in CI.
-      - uses: actions/setup-java@v4
+      # compiled by bootstrap on a live host, and the test mains under plugins/*/test
+      # were run by hand. JDK 25 is what the plugin build image runs (paper-api 26.x
+      # needs it); each module's wrapper brings the Gradle the image pins.
+      - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
         with:
           distribution: temurin
-          java-version: '21'
+          java-version: '25'
 
-      - uses: gradle/actions/setup-gradle@v4
-        with:
-          gradle-version: '8.14'
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
 
       - run: bash plugins/test.sh
 
   mods:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # The three loader mods (Minecraft 1.20.1 / 1.20.4, Java-17 lines) compile
       # through their vendored Gradle wrappers, which fetch their own Gradle. Until
       # this job nothing ever built them: no install path touches them, and their
       # gradlew scripts were committed without the exec bit, so the README's
       # one-liners failed on a fresh clone.
-      - uses: actions/setup-java@v4
+      - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
         with:
           distribution: temurin
           java-version: '17'
 
-      - uses: gradle/actions/setup-gradle@v4
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
 
       - run: bash plugins/test-mods.sh

.github/workflows/e2e.yml

@@ -0,0 +1,343 @@
+# deploy/bootstrap.sh end to end on a fresh Ubuntu 24.04 x86_64 runner: the paths every
+# host goes through, run for real instead of by hand on a VM.
+#
+#   artifacts  this commit's release assets, built by deploy/build-release-artifacts.sh the
+#              way release.yml builds a tag's (amd64 only, the runners' architecture)
+#   install    a full install from those assets, the way a release installs, on a host with
+#              ufw enabled, then the same assets again (a rerun must converge without
+#              restarting what did not change, importing or uploading an image again, or
+#              touching Docker)
+#   readme     the README's one-line install as a new host runs it today: this commit's
+#              installer on its default channel, which installs the newest published
+#              release's binary, images and plugin from that release's assets; skipped until
+#              a release exists
+#   upgrade    the newest published release through its own installer and its own assets,
+#              rows of every kind seeded into its database, then this commit's assets on
+#              top of it; skipped until a release exists
+#   source     a full install built on the host from this checkout (FELIS_SKIP_FETCH):
+#              the fallback a release without assets, or an unsupported one, takes. It builds
+#              everything with Docker, so it runs by hand and weekly only
+#
+# This runs on pushes that touch what gets installed, by hand, and weekly (a moving
+# upstream: apt mirrors, k3s's install script and release assets, Adoptium).
+# deploy/e2e_check.sh holds the assertions, deploy/e2e_seed.sh the upgrade's seed and its check.
+#
+# An installer that stops on an upstream download refused or dropped (a GitHub 403, a 5xx, a
+# timeout) ends its job green, with a warning on the run: each install step hands a failed
+# run's log to deploy/e2e_upstream.sh, which sets E2E_UPSTREAM_SKIP for the job's later
+# steps. Every other installer failure, a 404 included, fails the job.
+name: e2e
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'deploy/**'
+      - 'cmd/**'
+      - 'internal/**'
+      - 'panel/**'
+      - 'plugins/**'
+      - 'Dockerfile'
+      - 'go.mod'
+      - 'go.sum'
+      - '.github/workflows/e2e.yml'
+  workflow_dispatch:
+  schedule:
+    - cron: '23 4 * * 1'
+
+permissions:
+  contents: read
+
+# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` carries the
+# installer's status to deploy/e2e_upstream.sh; the default shell reports tee's status.
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  artifacts:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0 # the newest tag is the version's base, as on the dev channel
+
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      # Stamped the way bootstrap stamps a build of main: "<newest tag>+g<sha>".
+      - name: Build the release assets
+        run: |
+          base="$(git describe --tags --abbrev=0 --match 'v*' 2>/dev/null || echo v0.0.0)"
+          FELIS_RELEASE_ARCHES=amd64 deploy/build-release-artifacts.sh "${base}+g$(git rev-parse --short HEAD)" dist
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: e2e-assets
+          path: dist/
+          if-no-files-found: error
+          retention-days: 1
+
+  install:
+    needs: artifacts
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The images, k3s and the JRE need more room than a stock runner leaves free.
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: e2e-assets
+          path: dist
+
+      # The runner has Docker preinstalled, so its absence proves nothing: the log shows
+      # whether bootstrap reached for it. From FELIS_ARTIFACT_DIR every image and the plugin
+      # come out of the assets, so it must not have. (`! grep` is spelled `if grep ... exit 1`
+      # below because bash -e ignores a failing `!` command.)
+      # ufw, enabled on many Ubuntu hosts, drops every inbound packet no rule admits, the
+      # pods' traffic to the API server among them; the runner ships it disabled. The
+      # runner's own traffic is outbound, which ufw lets through.
+      - name: Enable ufw
+        run: sudo ufw --force enable
+
+      - name: Install
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log || bash deploy/e2e_upstream.sh install.log $?
+
+      - name: Check the install
+        if: env.E2E_UPSTREAM_SKIP != '1'
+        run: |
+          sudo bash deploy/e2e_check.sh install
+          for tag in felis-k3s-pods felis-k3s-services felis-panel felis-proxy; do
+            sudo ufw status | grep -qE "# ${tag} *$"
+          done
+          if grep -E 'docker already installed|installing docker|docker running' install.log; then exit 1; fi
+          for role in felis limbo lobby paper; do
+            grep -q "felis/${role}:[^ ]* is the release's" install.log
+          done
+          grep -q "docker.io/library/registry@sha256:[0-9a-f]* is the release's" install.log
+          grep -q "docker.io/library/postgres@sha256:[0-9a-f]* is the release's" install.log
+          grep -q "felis-velocity.jar is the release's" install.log
+
+      - name: Rerun the same assets
+        if: env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log || bash deploy/e2e_upstream.sh rerun.log $?
+
+      # containerd and the registry already hold every image under the digest the listing
+      # names, so nothing is imported or uploaded twice.
+      - name: Check the rerun
+        if: env.E2E_UPSTREAM_SKIP != '1'
+        run: |
+          sudo bash deploy/e2e_check.sh rerun
+          grep -q 'felis-velocity unchanged; left running' rerun.log
+          if grep -E 'docker already installed|installing docker|docker running' rerun.log; then exit 1; fi
+          if grep -E 'importing felis-image-|mirroring .* into the internal registry' rerun.log; then exit 1; fi
+          grep -q 'is already in the internal registry' rerun.log
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          k() { sudo -E /usr/local/bin/k3s kubectl "$@"; }
+          k get pods -A -o wide || true
+          k get events -A --sort-by=.lastTimestamp | tail -n 60 || true
+          for p in $(k get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" {print $1 "/" $2}'); do
+            k -n "${p%%/*}" describe pod "${p#*/}" | tail -n 40 || true
+            k -n "${p%%/*}" logs "${p#*/}" --all-containers --tail=60 || true
+          done
+          k -n felis logs deploy/felis-postgres --tail=60 || true
+          sudo journalctl -u k3s -u felis-velocity --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-install-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  # The README's command on a fresh host: this commit's installer, as main serves it, on its
+  # default channel with nothing pinned. It resolves the newest release and installs that
+  # release's binary, images and plugin from its assets, each checked against its
+  # SHA256SUMS. The workflow's token is the only thing added: it lifts GitHub's limit of 60 API
+  # calls an hour for an address without one. FELIS_INSTALL_MODE picks the mode the setup
+  # console would ask for.
+  readme:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - name: Find the newest release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: bash deploy/e2e_release.sh find
+
+      - name: Install as the README does
+        if: steps.release.outputs.tag != ''
+        env:
+          TOKEN: ${{ github.token }}
+        run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log || bash deploy/e2e_upstream.sh readme.log $?
+
+      # The binary is the release's, so the phase is `release`: its database backup and
+      # timers are the release's to have or lack.
+      - name: Check the install
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          BINARY: ${{ steps.release.outputs.binary }}
+          SUMS: ${{ steps.release.outputs.sums }}
+        run: |
+          sudo bash deploy/e2e_check.sh release
+          sudo /usr/local/bin/felis version | grep -qx "felis ${TAG}"
+          bash deploy/e2e_release.sh check-readme readme.log
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
+          sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-readme-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  upgrade:
+    needs: artifacts
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: e2e-assets
+          path: dist
+
+      - name: Find the newest release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: bash deploy/e2e_release.sh find
+
+      # The release's own installer on its default channel, fetching the release's own
+      # binary: what a host that installed that release is running today. FELIS_REF would
+      # build the tag from source instead, a path no host takes by default. FELIS_RELEASE pins
+      # the tag found above; an installer older than FELIS_RELEASE ignores it and resolves
+      # the newest release, the same tag.
+      - name: Install the newest release
+        if: steps.release.outputs.tag != ''
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          TOKEN: ${{ github.token }}
+        run: |
+          git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh
+          sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log || bash deploy/e2e_upstream.sh release.log $?
+
+      - name: Check the release install
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          BINARY: ${{ steps.release.outputs.binary }}
+        run: |
+          sudo bash deploy/e2e_check.sh release
+          bash deploy/e2e_release.sh check-own release.log
+
+      # A fresh install's database holds only what its migrations wrote: the seed gives the
+      # upgrade's move and its pending migrations existing rows to carry.
+      - name: Seed the release's database
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo bash deploy/e2e_seed.sh seed
+
+      - name: Upgrade to this commit
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log || bash deploy/e2e_upstream.sh upgrade.log $?
+
+      # Both checks run and report: the seeded rows go last, after the restore drill has
+      # also put them back from a bundle.
+      - name: Check the upgrade
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        run: |
+          rc=0
+          sudo bash deploy/e2e_check.sh upgrade || rc=1
+          sudo bash deploy/e2e_seed.sh check || rc=1
+          exit "$rc"
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
+          # The release ran the database on the host; the upgrade moves it into k3s.
+          sudo journalctl -u k3s -u felis-velocity -u postgresql --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-upgrade-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  source:
+    if: github.event_name != 'push'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      # FELIS_SKIP_FETCH builds whatever sits in /opt/felis/src; the .git directory is what
+      # stamps the build. Root owns it so git, run as root by the installer, does not refuse
+      # it as dubious.
+      - name: Stage this commit as the installer's source
+        run: |
+          sudo mkdir -p /opt/felis
+          sudo cp -a "$GITHUB_WORKSPACE" /opt/felis/src
+          sudo chown -R root:root /opt/felis/src
+
+      - name: Install
+        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log || bash deploy/e2e_upstream.sh source.log $?
+
+      - name: Check the install
+        if: env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo bash deploy/e2e_check.sh install
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-source-logs
+          path: '*.log'
+          if-no-files-found: ignore

.github/workflows/release.yml

@@ -2,14 +2,15 @@
 #
 # Two things depend on this job. /repos/{repo}/releases/latest must ANSWER — that endpoint is
 # what `felis update` polls (internal/updater/github.go) and what deploy/bootstrap.sh's default
-# "release" channel resolves its ref from. And the binaries below are what that channel then
-# INSTALLS: bootstrap downloads felis-linux-<arch> instead of compiling on the target host, so
-# these are the shipped artifact, not a convenience.
+# "release" channel resolves its ref from. And the assets below are what that channel then
+# INSTALLS: bootstrap downloads the felis binary, the image tars, their listing and the Velocity
+# plugin instead of building anything on the target host, so these are the shipped artifact,
+# not a convenience. A host installing a release needs neither Docker, Gradle, Go nor Docker Hub.
 #
-# The asset NAME is a contract with deploy/bootstrap.sh (download_release_binary builds
-# "felis-linux-${arch}"). It is deliberately a plain literal on both sides: a GitHub Actions
-# YAML and a go:embed'ed shell script have no honest way to share a constant, and the failure
-# mode is benign — bootstrap warns and falls back to a source build of the same tag.
+# deploy/build-release-artifacts.sh builds every asset in one run and documents each name; the
+# names are a contract with deploy/bootstrap.sh. They are plain literals on both sides: a YAML
+# workflow and a go:embed'ed shell script have no honest way to share a constant, and the
+# failure mode is benign — bootstrap warns and falls back to building on the host.
 #
 # The binary is built through the repo Dockerfile rather than a plain `go build`.
 # internal/panel/static holds a tracked PLACEHOLDER index.html so the //go:embed
@@ -17,6 +18,15 @@
 # quietly ships a release whose panel is that placeholder. The Dockerfile runs the npm
 # build first, and is the same recipe bootstrap uses, so there is one way to build felis
 # rather than two that can drift.
+#
+# SHA256SUMS is a contract with bootstrap too: it refuses any asset whose hash is not listed
+# there, BEFORE it runs or imports it.
+#
+# The write token never meets the test suite: `gates` (ci.yml) and `build` run the tests,
+# Gradle and the Docker builds (each of which executes third-party code) with a read-only
+# token, and `build` hands the assets over as a workflow artifact; `publish` holds contents:write and runs only
+# pinned actions and gh. Every action is pinned to a commit SHA (the tag in the trailing
+# comment is for humans); .github/dependabot.yml proposes the bumps.
 name: release
 
 on:
@@ -24,84 +34,99 @@ on:
     tags: ['v*']
 
 permissions:
-  contents: write # gh release create/upload
+  contents: read
 
 jobs:
-  release:
+  # A tag that ships red is worse than a tag that fails to ship. These are ci.yml's gates,
+  # called rather than copied: Go (race, vet, staticcheck), govulncheck, the PostgreSQL
+  # contract suite, shellcheck and the bootstrap tests, promtool on the alert rules, the
+  # panel, and the Java layer the binary EMBEDS (bootstrap_asset.go ships the plugin
+  # sources, so a tag whose plugins do not compile turns every install of that release into
+  # a failed bootstrap).
+  gates:
+    uses: ./.github/workflows/ci.yml
+
+  build:
+    needs: gates
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-
-      # A tag that ships red is worse than a tag that fails to ship.
-      - run: go vet ./...
-      - run: go test ./...
-
-      # The same reason, for the Java layer the binary EMBEDS: the release asset is
-      # the tree's plugin sources (bootstrap_asset.go), and a tag whose plugins don't
-      # compile turns every install of that release into a failed bootstrap. JDK 21
-      # gates the install-time plugins + codec/invite tests; JDK 17 gates the loader
-      # mods (their vendored wrappers fetch their own Gradle).
-      - uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: '21'
-
-      - uses: gradle/actions/setup-gradle@v4
-        with:
-          gradle-version: '8.14'
-
-      - run: bash plugins/test.sh
-
-      - uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: '17'
-
-      - uses: gradle/actions/setup-gradle@v4
-
-      - run: bash plugins/test-mods.sh
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # Both architectures, because bootstrap's default release channel DOWNLOADS these
-      # rather than compiling on the target host — an arm64 host with no asset silently
-      # falls back to a slow source build. Neither stage is emulated: the Dockerfile pins
-      # both build stages to $BUILDPLATFORM and the Go stage cross-compiles via TARGETARCH,
-      # so the second architecture costs about a minute.
-      - uses: docker/setup-buildx-action@v3
+      # rather than building on the target host — an arm64 host with no asset falls back to
+      # a slow on-host build. The felis binary and the plugin jars cross-compile on the
+      # runner (their build stages are pinned to $BUILDPLATFORM); the game images' runtime
+      # stages run apt-get for the target platform, which for arm64 takes QEMU.
+      - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
 
-      - name: Build the stamped binaries
-        run: |
-          docker buildx build --platform linux/amd64,linux/arm64 \
-            --build-arg FELIS_VERSION="${GITHUB_REF_NAME}" \
-            --output type=local,dest=out .
-          mv out/linux_amd64/usr/local/bin/felis ./felis-linux-amd64
-          mv out/linux_arm64/usr/local/bin/felis ./felis-linux-arm64
-          chmod +x ./felis-linux-amd64 ./felis-linux-arm64
+      # Two architectures of five images plus BuildKit's cache outgrow the runner's free disk
+      # with its preinstalled SDKs in place; none of them is used here.
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
 
-      # The stamp is the whole point and it fails silently: an unstamped binary reports
-      # "dev", which `felis update` refuses to compare, disabling update reporting for
-      # every install built from it. Assert it end to end instead of trusting the ARG
-      # reached the linker.
-      - name: Verify the version stamp
-        run: |
-          got="$(./felis-linux-amd64 version | head -1)"
-          echo "reported: ${got}"
-          [ "$got" = "felis ${GITHUB_REF_NAME}" ] \
-            || { echo "expected 'felis ${GITHUB_REF_NAME}' — the -X main.version stamp did not reach the binary"; exit 1; }
-          # The arm64 binary is checked by ELF machine type, NOT by running it. Runners have
-          # binfmt/QEMU registered, so `./felis-linux-arm64 version` would happily succeed on
-          # an amd64 binary misnamed arm64 — which is exactly the failure the Dockerfile's
-          # ${TARGETARCH:-$(go env GOARCH)} fallback produces if buildx did not take. Both
-          # binaries come out of one RUN with one -ldflags string, so the stamp is checked once.
-          file ./felis-linux-arm64
-          file ./felis-linux-arm64 | grep -q 'ARM aarch64' \
-            || { echo "felis-linux-arm64 is not an arm64 ELF — TARGETARCH did not reach the go build"; exit 1; }
+      # The script checks what the old inline steps did: the host binary reports exactly
+      # "felis <tag>" (unstamped, `felis update` refuses to compare), and the other one is an
+      # ELF of its architecture, read with file(1) — binfmt would run a misnamed binary happily.
+      - name: Build the release assets
+        run: deploy/build-release-artifacts.sh "${GITHUB_REF_NAME}" dist
 
-      # --verify-tag refuses to invent a release for a tag that is not pushed. The upload
-      # fallback makes a re-run converge rather than failing on an existing release.
+      # A CycloneDX SBOM per binary: the Go modules (and versions) linked into it, read
+      # from the build info the linker embeds. Written after SHA256SUMS, so beside it in the
+      # release but outside it: that lists what bootstrap installs. Straight into dist/,
+      # because the action does not create a missing output directory.
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: dist/felis-linux-amd64
+          format: cyclonedx-json
+          output-file: dist/felis-linux-amd64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: dist/felis-linux-arm64
+          format: cyclonedx-json
+          output-file: dist/felis-linux-arm64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: release-assets
+          path: dist/
+          if-no-files-found: error
+          retention-days: 7
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # gh release create/upload
+      id-token: write # the Sigstore certificate behind the provenance attestation
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: release-assets
+
+      # The artifact store sits between the two jobs, so check the handover too.
+      - run: sha256sum -c SHA256SUMS
+
+      # Signed SLSA provenance: which workflow run, commit and repository produced each
+      # asset. Check one with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`.
+      # GitHub only stores attestations for private repositories on Enterprise Cloud, and a
+      # failure here would block the release, so a private repository skips the step and
+      # relies on SHA256SUMS alone.
+      - name: Attest build provenance
+        if: ${{ !github.event.repository.private }}
+        uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0
+        with:
+          subject-path: |
+            felis-linux-*
+            felis-image-*.tar
+            felis-velocity.jar
+
+      # --verify-tag refuses to invent a release for a tag that is not pushed.
       #
       # The prerelease flag has to be passed explicitly: the trigger glob is v*, so v1.2.3-rc1
       # lands here too, and gh does not read semver out of the tag name. Published as a full
@@ -109,13 +134,35 @@ jobs:
       # channel installs from and `felis update` polls — so every fresh install would get the
       # RC binary and every deployed felis-api would error on the felis component until a
       # stable tag was cut. Flagged, GitHub keeps latest pointing at the last stable release.
+      #
+      # A re-run (the release already exists) uploads only what is missing and never
+      # replaces a published asset: hosts may already have installed it, and their
+      # SHA256SUMS check would start failing against a swapped file. An asset that is
+      # there with different bytes stops the job; cut a new tag instead.
       - name: Publish the release
         env:
           GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
         run: |
+          # Every file the build handed over: SHA256SUMS names the installable ones, and the
+          # SBOMs ride along.
+          assets="$(ls)"
           flags=""
           case "$GITHUB_REF_NAME" in *-*) flags="--prerelease" ;; esac
-          gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags \
-              ./felis-linux-amd64 ./felis-linux-arm64 \
-            || gh release upload "$GITHUB_REF_NAME" \
-                 ./felis-linux-amd64 ./felis-linux-arm64 --clobber
+          if ! gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
+            # shellcheck disable=SC2086 # word-splitting the list is the point
+            gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags $assets
+            exit 0
+          fi
+          # The REST payload's per-asset "digest" is GitHub's own sha256 of the stored file.
+          published="$(gh api "repos/${GH_REPO}/releases/tags/${GITHUB_REF_NAME}" --jq '.assets[] | "\(.name) \(.digest)"')"
+          for a in $assets; do
+            have="$(printf '%s\n' "$published" | awk -v n="$a" '$1 == n { print $2 }')"
+            want="sha256:$(sha256sum < "$a" | cut -d' ' -f1)"
+            if [ -z "$have" ]; then
+              gh release upload "$GITHUB_REF_NAME" "$a"
+            elif [ "$have" != "$want" ]; then
+              echo "::error::$a is already published with $have; this run built $want. Published assets are never replaced."
+              exit 1
+            fi
+          done

.gitignore

@@ -51,3 +51,7 @@ AGENTS.md
 docs/Felis-Spec-V4.1.md
 panel/DESIGN.md
 panel/DESIGN-WEB-3SIDES.md
+
+# Audit ledgers and readiness reviews stay on the maintainer's disk.
+/AUDIT-*.md
+/READINESS-*.md

AUDIT-2026-09-22.md

@@ -1,927 +0,0 @@
-# Felis 生产就绪审计 — 2026-09-22（真机 E2E + 混沌）
-
-分支：全部修复**逐 commit 直接推 `main`**（不再用功能分支）。remote = `[email protected]:FelisMC/Felis.git`（2026-09-23 起；旧 `MliroLirrorsIngenuity/Felis` 仅靠 GitHub 301 兜住）。环境：CentOS Stream 9 / aarch64 / k3s v1.36.4，
-IPv6-only 接入（`ssh -6 -i ~/.ssh/id_ed25519 root@fdb2:2c26:f4e4:0:21c:42ff:fede:69ec`），
-面板经 `socat TCP6:443 → 127.0.0.1:30443` 中继（手动启动，重启 VM 后需重开）。
-
-## 已修复并验证（分支内）
-
-| # | 缺陷 | 证据 | 修复 |
-|---|------|------|------|
-| 1 | **失败恢复后重试被静默吞掉**：restore Job 固定名 + `ErrAlreadyExists` 一律当"幂等成功"；失败 Job 占名 10 分钟（TTL），期间重试返回 202 `restoring` 但什么都不跑 | 真机：坏 ref 制造失败 → 立刻合法重试 → Job 原地不动、无新 Pod | `k8sjobs.go`：撞名时检查已完成（成功/失败）→ 删除+**等 finalizer 释放**（有界 10s）+ 重建；进行中仍幂等吸收。单测 3 个。**真机复验：重试 4s 完成恢复** |
-| 2 | **备份 Job 全部 FailedMount**：Job 在 minecraft 命名空间挂 `felis-config`，而 bootstrap 只在 felis 命名空间创建该 Secret | 事件：`MountVolume.SetUp failed: secret "felis-config" not found` | `felis setup` 用既有 `ensureSecretReplica` 把 felis-config(`felis.toml`) 复制到 minecraft；VM 上手工复制后备份成功（167MB 归档） |
-| 3 | RBAC 缺 `jobs:get/delete`（修复 #1 需要） | Role 检查 | `APIMinecraftRole` jobs → create/get/delete，测试同步更新 |
-| 4 | gofmt 9 文件漂移 + CI 无 gofmt 门禁；staticcheck 9 处 | 基线扫描 | 全部修复；CI 加 gofmt job |
-| 5 | 依赖漏洞：pgx v5.7.1（GO-2026-5004，可达 pgrepo.go）、x/net、x/text | govulncheck | 升 pgx v5.9.2 / x/net v0.55.0 / x/text v0.39.0 |
-| 16 | **`ConsumeLoginEmailOTP` PG 实现与接口契约漂移**：契约/`fakeRepo` 说「同 VerifyEmailOTP 生命周期（扣尝试/锁定/ErrOTPInvalid/ErrOTPLocked）」，PG 却是单条 UPDATE+`ErrNotFound` → 错码/重放/过期在 login 门、op-login finish、migration confirm 三个入口全部 500；且尝试次数永不累计、`otpMaxAttempts` 锁定失效 | 真机：login 门重放正确码 → **HTTP 500**；修复前错码不扣次。修复后复测：5 次错码 400 且 attempts=5（正确码因锁定也 400、码未消费）、新码可用、重放 400 | `pgrepo.go` 改置为 VerifyEmailOTP 同构事务（FOR UPDATE、先锁后比、mismatch 扣次、match 消费），无 users 写副作用。提交 `52549f7` |
-| 17 | **`ListPendingOpLogins` PG 少列**：接口注释承诺「joined to its staff username」，handler 输出 `username`/`created_at`，fake 正确填充；PG SQL 未 JOIN 也未取 `created_at` → 真机 pending 列表 username 为空、created_at 为 `0001-01-01` | 真机 internal `/op-login/pending` 响应 | `pgrepo.go` SQL 改为 JOIN users + 取 created_at。提交见分支 |
-| 18 | **绑定码并发兑换 500**：`RedeemPlayerBindCode` 无 `FOR UPDATE`（同文件 `VerifyLinkCode` 有），且裸 INSERT。6 路并发同码兑换 → 3×HTTP 500（`users_username_key` 唯一冲突）+ 1×400 + 2×200；跨码并发同 UUID 同样会撞 | 真机四组并发测试 + API 日志 `unmapped error ... duplicate key` | 同码：码行 `FOR UPDATE`（输家干净地 400 invalid_code）；跨码：`INSERT users ... ON CONFLICT (username) DO NOTHING`+重读、`account_links ON CONFLICT (mc_uuid) DO NOTHING`（两路汇聚同一 user）。复测：同码×6=1×200+5×400、双码×2=2×200 同 ID、DB 干净、日志零 unmapped |
-| 19 | **reaper CronJob 渲染位置错误，永远无法调度**：`felis manifests` 把 CronJob 渲染到控制 ns，却引用 minecraft ns 的备份 PVC（Pod 不能跨 ns 挂 PVC：真机 `FailedScheduling: persistentvolumeclaim "felis-backups" not found`）；修正 ns 后又发现 ServerAccount 也不能跨 ns 使用（`serviceaccount "felis-reaper" not found`）。而 reaper 的 Role/RoleBinding 本就在 minecraft ns | 真机三层取证（PVC/SA/调度）| CronJob 与 SA、RoleBinding subject 全部移到 `MinecraftNamespace`（提交 `e4f2cff`+`c839454`）。**修复后完整演练通过**（见下） |
-| 6 | **默认安装无备份能力 + 回收链路不可达/不可用**：(a) 没有任何环节渲染归档 PVC，`FELIS_BACKUP_PVC` 永远为空 → backup/restore 恒 503；(b) bootstrap 从不下传 reaper 三旗标 → 官方安装路径根本无法启用回收；(c) 即便启用，stock k3s 的 `<root>/<pvc>` 布局不存在，且 k3s storage root 是 `0700 root:root`、reaper pod 以 uid 1000 运行 → 真机 `lstat /worlds/…: permission denied`（fail-closed 跳过，但纯空转）；(d) README 口径（“定时备份”）不实 | 真机全链路：默认渲染 PVC+env → 建服→marker→备份 202→jobs 端点 running→succeeded→改 marker→restore→读回原值 ✅；再建 resolvecheck 世界（20d idle，marker）→ CronJob 手动 Job：`evaluated=2 reaped=1`，归档含 marker、PVC+宿主目录回收、`world_backups` 得 `inactive_15d` 行、servers 行/CR 保留 ✅ | `platform/workloads.go` 渲染归档 PVC（minecraft ns、RWO 10Gi、默认 SC），`felis manifests --backup-pvc` 默认 `felis-backups`（`=` 空为显式关闭），bootstrap 统一下传 env/旗标并写 [archive] local_path；`resolveWorldDir` 新增精确 local-path 目录解析（读 PVC `spec.volumeName`，非 glob，杜绝陈旧 PV 目录顶替）；ReaperRole 增 `pvc:get`；bootstrap 设 `FELIS_WORLDS_HOST_PATH` 时给 uid 1000 授 traverse（setfacl/o+x）；README/故障手册改写。提交 `fd33fd0`+`2b87a5a` |
-| 8 | **磁盘打满灾难链**：DiskPressure → kubelet 驱逐控制面（无 PriorityClass 保护）→ 镜像被 GC（无外网、registry 空）→ 全部 ImagePullBackOff；释放后数分钟才恢复调度。恢复依赖人工 `docker save | k3s ctr images import -` | 三轮真机 drill（原缺陷复现 + 两轮修复验证）：①自定义 1e6 类：游戏 pod 先走、控制面随后仍被驱逐（kubelet 日志逐条列出 ranked/evicted），随后镜像被 GC → ErrImagePull；②内建 `system-cluster-critical`（2e9）：填盘至 1.7G free，kubelet 对 felis-api/operator/registry 全部报 **“cannot evict a critical pod”**，三者在整个 DiskPressure 期间保持 Running；游戏 pod（0）被驱逐；③释放磁盘后：`DiskPressure` 经 ~5 分钟（`eviction-pressure-transition-period`）转 False——即“恢复慢”的主因；被 GC 的游戏镜像按 runbook 重导入后 25s 恢复 ✅ | 控制面四类 pod 挂内建 `system-cluster-critical`（用户自定义类值上限 1e9，达不到 2e9 临界阈值；preemption 保留=管理面可调度，已记录权衡）；troubleshooting §13b 固化事件链、5 分钟条件过渡与镜像恢复路径。提交见下 |
-| 9 | **升级策略 Recreate**：单副本 + Recreate：任何控制面升级=停机；坏升级（错 tag）中断约 95s 且需人工 `rollout undo`（无自动回滚） | 复核部署模板（Recreate + 单副本 + 无 leader election 的注释理由成立） | 不改策略（Recreate 是对无 leader election 的正确取舍），改为固化 runbook：troubleshooting §15 = 升级即重跑安装器；停机窗口=rollout 时长；坏镜像在安装器 180s 等待内以 `kubectl describe` 诊断呈现；回滚 `kubectl rollout undo`（镜像被 GC 时先按 §13b 重导入）。提交见下 |
-| 10 | **备份语义**：归档包含整个 /data（jar、libraries、cache），167MB；是否符合 world backup 定位待评估 | 真机 tar 清单（cache/、config/、eula.txt、server.properties、jar）+ 恢复语义（overlay+prune） | 评估结论：**整卷备份是正确语义**（回滚=整服状态回滚，含配置/插件），保留行为；文档化：troubleshooting §10「backup contains」+ OpenAPI/README 口径改为“整服数据卷”。提交见下 |
-| 20 | **验证邮箱唯一性只存在于注释里**：errors.go/repo.go 都声称迁移 0010 建了 `users_verified_email_unique` 部分唯一索引、`VerifyEmailOTP` 会返回 `ErrEmailTaken`；实际上**索引从未创建**、`ErrEmailTaken` 全仓库从未被返回 → 两个账号可同时验证一个邮箱，而登录门正是按 verified email 解析账号 → 同一邮箱的登录码归谁由数据库任意决定 | pgint 首跑即红：第二账号验证同邮箱返回 `nil`；`grep users_verified_email_unique internal/store/migrations/*.sql` 零命中。真机复验（auditfix17 邮箱 409 演练）：同码二次 verify 仍 409（码未被消费） | 新增迁移 `0020_verified_email_unique.sql`（`lower(email) WHERE email_verified`）；`VerifyEmailOTP` 在消费码**前**查「他人已验证」→ `ErrEmailTaken`（不消费码、不扣次数），并把并发唯一冲突映射为同一答案；handler 新增 409 `email_taken`。提交 `b6ef27c`；PG 契约测试基建 `2a55a0d`（`-tags pgint`，见 CONTRIBUTING） |
-| 21 | **admin 改邮箱不清 verified**：`UpdateUser` 写入新地址但保留 `email_verified=true` → 面板改错一个字符就能让登录码寄到别人邮箱（该 flag 正是邮箱登录解析/寄码的依据）；fake 同错 | pgint 契约测试 | 改地址时同写 `email_verified = email_verified AND email IS NOT DISTINCT FROM 新值`（同值 no-op 保留证明；换值即清除）；fake 同步。提交 `d1ec40f`。真机验证：PATCH→`f`、PATCH 回原值仍 `f`、玩家重验证→`t` |
-| 22 | **`role='owner'` 是死信**：迁移 0011 加了 owner 角色并把全部用户管理路由压在 `IsOwner()` 上，但**没有任何代码写过 `owner`**——breakGlass(`UpsertOwner`)、setup MC 绑定(`CompleteOwnerSetup`)、重置路径一律写 `admin` → 全新安装的整个 owner 层（用户列表/创建/编辑/禁用/删除/配额/会话）不可达；且新角色没进各处 staff 判定（op-login 只认 admin、玩家邮箱门只拒 admin、reclaim 保护与 AdminExists 只认 admin）；面板一旦有 owner 行还可被降级/删除/禁用 | 真机：owner 会话加载 `/api/v1/users` 403；promote 后 200。op-login：修复前 owner start 中立不发码，修复后铸请求+finish 得 `role=owner` 会话 | `UpsertOwner`/`CompleteOwnerSetup` 改写 `role='owner'`（冲突臂重断言，即 0011 文档的 promote 路径）；op-login 双端改 `staffRole`；玩家邮箱门改 `role != 'user'`；`IsProtectedAdminLink`/`AdminExists` 计入 owner；面板新增守卫：owner 行不可降级/删除/禁用（用户名/邮箱编辑仍可）。提交 `e0d2378` |
-| 23 | **配额门非原子 + storage 缓存被清零**：(a) audit #4：`QuotaCheck` 与 `ClaimServer` 两条语句，同一用户并发认领两台无主服可双双通过 `max_servers`（deferred-seams 曾把这条挂为“只能在真 PG 上关闭”）；(b) 更隐蔽：PATCH 资源时 `UpdateServerResources(..., 0)` 把本不能改的 storage 缓存写 0，而缓存列是配额聚合的**唯一**输入 → 此后该服的 storage 维度在配额里凭空消失 | (a) 新增 pgint 并发测试：修复前两台全赢；修复后恰 1 赢 + 1 `ErrQuotaExceeded`，DB 只 1 行 owned；(b) hermetic 测试 `TestPatchServerPreservesStorageCache`（修复前 `resourceUpdates` 里 storage=0） | (a) 门槛进 `ClaimServer`：同一事务内 `pg_advisory_xact_lock(hashtext(user_id))` + 四维重查（与 `QuotaCheck` 共用 `quotaAllows` 防漂移）+ 行 `FOR UPDATE`，两个 claim handler 把 `ErrQuotaExceeded` 映射为与串行一致的 403；(b) resize 前读取现值并透传 storage。提交 `bb68fef`；deferred-seams 对应条目核销 |
-| 24 | **reaper 警告信从不真正投递**：`felis reaper` 从不装配任何 Warner（`r.Warner` 恒 nil），而 `maybeWarn` 对 nil warner / 投递失败一律照样 `MarkWarned` + `warned++` → 每位有主的服都在**无人收到提醒**的情况下 15 天后被静默回收，跑批日志还谎报"已警告 N 台"。红线⑤的"best-effort 不阻塞回收"被误读成了"失败也要记成已通知" | hermetic 单测 3 例（坏 notifier / nil warner → 不盖章、成功才盖章）。真机 drill（auditfix20）：13d idle 实收 1 封（收件人=owner 的验证邮箱、subject 含 3d）；重跑不重发；14.5d 补发 1d（elif 档位）；SMTP 端口打坏 → 日志 `warn delivery failed; will retry next run` 且**不盖章**，恢复后补发；邮箱未验证 → `has no verified email` 不盖章；owner NULL → 静默跳过；边界 11d23h 不发 / 12d1m 发；全程 `world_backups` 保持 4 条不动（纯警告零备份副作用） | `mail.SendNotice` + `mailWarner`：查 owner 的 verified email → `SendNotice`；nil/失败不盖章、下次重试；reaper pod 模板加 optional `FELIS_SMTP_PASSWORD` env；`felis setup` 复制 felis-smtp 镜像并在「configure email」刷新 minecraft ns 的 felis-smtp+felis-config 镜像；docs/troubleshooting §10 更新。提交 `8e7c7bb` |
-| 25 | **idle auto-stop 从未触发（三层复合缺陷）**：条件齐备的空载服永远不停。① CRD status schema 未声明 `emptySince`，apiserver **pruning** 掉计时戳（`unknown field "status.emptySince"`），计时器每次读回都是 nil；② 即使戳幸存，Running 空载稳态**没有任何 watch 事件**（玩家进出不碰 CRD、RCON 只在 Reconcile 内探），盖章一次后 reconcile 链停摆，无人叫醒；③ auto-stop 用整对象 `Update` 写 spec 且 OperatorRole 从未有 `minecraftservers:patch/update` → 403 `cannot update resource`。三层任一都让功能永久失效，而单测（fake client 不剪 schema、手动驱动、无 RBAC）全部覆盖不到 | 真机逐层实锤：schema 修复后 `empty=2026-09-22T20:02:13Z` 首次成功持久化；静置 88s+ 无动作、operator 日志 90s 零行（②实锤）；修复前日志 5 条 403（③实锤）；全修后场景 1：超时戳触发即 Stopped；场景 2：起服 → 20:11:43 盖章 → **全程无干预** → 20:12:17 自动 Stopped；翻转 Running→Stopped→Running 收敛 Running；`idle=null` 后不再自停 | ① schema 补 `emptySince`（`c04a3f0`）；② `reconcileRunning` 尾部返回 RequeueAfter——空载=到点精确唤醒、有人=30s 探针周期，+3 个单测（`1c89a5e`）；③ auto-stop 改 merge-patch（防 status clobber，与 reaper 的 Stop 同型）+ OperatorRole 补 `patch` + rbac 测试锚点（`f650bf8`）。真机 auditfix22 全通；troubleshooting §11 增自检三连 |
-| 26 | **RCON Secret 被删 → 服务器永久锁死**：删掉 per-server RCON 密码 Secret 后，(a) operator 没有任何 watch 能看到删除（Running 稳态零事件），secret 一直不重建；(b) 一旦被任意事件带到 reconcile：新密码铸造成功，但运行中的 pod 仍持有旧密码、sts 模板无变化 → pod 永不重启 → 探针用新密码连旧密码 pod，**永久 `RconNotReachable`** 直至 300s `ReadinessTimeout`；恢复只有人工删 pod。`ensureRconSecret` 的注释却声称"heals on the next pass" | 真机：删 secret 后静置 60s 无重建、无日志；annotate 触发后 96s+ 持续 RconNotReachable（新密码 `91dd62…` vs pod 旧密码）；删 pod 后 17s 恢复 Running——根因=密码漂移实锤 | ① `SetupWithManager` 增 `Owns(&corev1.Secret{})`（controller-owned，删除事件映射回 CR）——真机日志见 `source="kind source: *v1.Secret"`；② pod template 新增 `RconSecretAnnotation` = 当前密码 SHA-256 指纹（64-bit）：secret 重建指纹变 → sts 自动 rollout 用上新密码，未重建则恒定不抖动；测试 2 例（跨 reconcile 稳定 / 重建必变）。提交 `56f3abd`。真机复测：删 secret 后**同秒**察觉+重建+换戳，31s 全自动恢复 Running |
-| 27 | **陈旧启动锚点 → 已恢复的服务器被误判 StartupTimeout + Provisioned 永久 False**：`status.startRequestedAt` 只在 Stop 时清，成功（markRunningReady）不清——一旦服务器曾经历一次长 Starting，其 300s 预算就悬在健康运行中；此后任意一次 pod 波动（rollout/崩溃）都会立即套用旧戳判 `Failed(StartupTimeout)`。且 `markFailed` 写入的 `ConditionProvisioned=False` 无人复位，恢复后仍永久挂着失败标记 | 真机：20:19 已恢复 Running 的 test-one，20:21 因一次 stamp 注入引发的 pod rollout 被标记 `Failed StartupTimeout`（锚点残留自 20:16）；恢复后 status.conditions 里 `Provisioned=False … StartupTimeout` 持续存在 | `markRunningReady`：清 `StartRequestedAt`（每次启动/恢复尝试各有独立预算）+ 复位 `ConditionProvisioned=True`；测试 2 例（Ready 清锚点、Failed→Running 后 Provisioned 恢复）。提交 `82b5a60`。真机复测：清戳生效（`startRequestedAt: None`）、pod blip 删→33s 恢复全程无 Failed、锚点重新盖章后成功清除 |
-| 28 | **cfsetup 把 "policy already exists" 当成功吞掉 → fail-closed 保证可被旧策略顶替**：Cloudflare Access 的 `CreateAccessPolicy` 在收到 already-exists 时直接返回 nil；若该 app 上已有一条更宽松的旧策略（改 identity 后重跑、或此前手工配置），守卫 op.console 的仍是旧策略，而 Setup 报告成功——`validateFailClosed` 只校验过"我们构建的策略"，从未校验证留在线上的那条 | 代码审查（集成侧无真实 CF 账号，无法真机）：httptest 3 例复刻——已存在同名策略、缺席、POST 竞态。修复前第 1 例吞错返回 nil 且不发 PUT | 改为按名 upsert：lookup → PUT 覆盖守卫体 → 缺席才 POST（POST 撞 already-exists → 重查后 PUT，绝不吞）。`apiPost/apiPut` 共用一个 `apiWrite`。提交 `30857df` |
-| 29 | **"configure email" 的工作负载镜像复制从不生效**：`replicateSMTPToWorkloadNamespace` 把硬编码 `namespace: felis` 的 felis-smtp manifest 用 `-n minecraft apply` 发出——kubectl 拒绝 namespace 冲突（`the namespace from the provided object ... does not match`）→ 第一段直接 return err，**第二段 felis-config 的镜像复制根本不会执行**。于是任何"装好后再改 SMTP"的部署，reaper 的 pre-reap 警告永远拿不到新配置（这正是 8e7c7bb 添加该刷新要解决的事），且失败仅 warning 不中止 | 真机 kubectl 行为实验：`-n minecraft apply` 带 `namespace: felis` 的 manifest → `error: … does not match …`；修复后（manifest namespace=minecraft）→ `accepted-namespace=minecraft`（server dry-run）；felis-config 渲染+apply 序列同为 accepted | `smtpSecretManifest(password, namespace)` 显式参数（控制面调用传 "felis"，镜像传工作负载 ns）；回归测试 `TestSMTPSecretManifestCarriesTargetNamespace`。提交 `ed722d5` |
-
-## 待决策台账（未修）
-
-前两日台账的 #7、#11–#15 已全部修复（本批核销，证据见下），不再挂在"未修"里：
-
-| # | 主题 | 状态 |
-|---|------|------|
-| 7 | 异步失败不可感知 | ✅ 修复：`GET /api/v1/servers/{name}/jobs`（`ff7c57c`，含 RBAC `jobs:list` 与 OpenAPI）；真机 drill 用它看到 running→succeeded |
-| 11 | PG 断连报 401 | ✅ 修复：会话存储故障改为 503（`2a8f897`） |
-| 12 | ready 门滞后 | ✅ 修复：operator 启动期内每 2s 重探 RCON（`a2df2f2`） |
-| 13 | setup URL 截断 | ✅ 修复：TUI 折行（`abce381`） |
-| 14 | controller-runtime 日志噪音 | ✅ 修复：SetLogger 接 slog（`a415246`） |
-| 15 | reaper 启用未演练 | ✅ 已演练（见"第二日"节；CronJob 仍 `suspend=true` 防误删） |
-
-## 可达性分级（#1–#74；#62 立案后剔除）——这些缺陷真实使用中到底谁能踩到
-
-回应质疑"是不是全在测边界条件 / 只有内部 hook 才能触发"：
-
-- **hook 的边界**：演练中使用的内部手段（读服务端日志取 OTP 码、service token 打内部面代 approve、本地 SMTP sink、tmux 驱动 TUI）只是**测试仪表**——代替"真实收邮件 / 键盘敲击 / 操作台点击"；触发路径本身是公开路径，真实用户做同一动作走同一段代码。真正**本环境不可复现**的只有 #28（无真实 CF 账号，仅 httptest 复刻），另有 #4/#5/#14 属工具/卫生级（无运行时触发面）。
-- **分级口径**：① 日常＝普通使用或默认安装下即会踩到；② 运维＝升级/重启/故障恢复/磁盘满/删资源/闲置回收等真实运维动作会踩到；③ 窗口＝需要并发或特定状态时序，真实但概率低；④ 工具＝仅工具判定/审查；决策＝非缺陷（评估/演练项）。
-
-| # | 真实触发路径（一句话） | 分级 |
-|---|------------------------|------|
-| 1 | 恢复失败后 10 分钟内重试（必然的恢复动作）→ 202 但 Job 不动 | ② |
-| 2 | 默认安装下第一次点「备份」→ Job FailedMount（felis-config 缺在 minecraft ns） | ① |
-| 3 | #1 修复所需 RBAC（单独不产生用户可见行为） | ② |
-| 4 | gofmt/staticcheck 漂移 + CI 门禁缺失 | ④ |
-| 5 | 依赖漏洞（govulncheck 判定可达）：需特定输入 | ④ |
-| 6 | 默认安装即踩：备份/恢复恒 503、回收链无旗标不可启用 | ① |
-| 7 | 任何异步操作（备份/构建）想查结果：无端点（功能缺口） | ① |
-| 8 | 磁盘打满事故链（控制面被逐 → 镜像 GC → 全体 ImagePullBackOff） | ② |
-| 9 | 非缺陷：升级策略评估 → runbook | 决策 |
-| 10 | 非缺陷：备份语义评估 → 文档化 | 决策 |
-| 11 | PG 掉线期间任何会话校验 → 401 误导 | ② |
-| 12 | 每次起服：就绪感知滞后 | ① |
-| 13 | 首次安装向导：长 URL 截断 | ① |
-| 14 | operator 日志噪音（卫生项） | ④ |
-| 15 | 流程项：reaper 启用补演练 | 决策 |
-| 16 | 邮箱登录主路径：错码/重放 → 500、5 次锁定失效 | ① |
-| 17 | 面板 op-login pending 列表字段缺失 | ① |
-| 18 | 绑定码并发兑换（双击/双端即可）→ 部分 500 | ③ |
-| 19 | 按文档启用回收 → CronJob 永远无法调度 | ② |
-| 20 | 两账号验证同一邮箱 → 登录码归属不定（安全） | ③ |
-| 21 | 面板改任一用户邮箱 → verified 未清、码寄旧地址 | ① |
-| 22 | 全新安装 owner 层不可达（用户管理全路由） | ① |
-| 23 | (a) 并发认领两服可双双过配额；(b) PATCH 任意服资源 → storage 缓存清零 | ①（a=③） |
-| 24 | 有主服闲置 12–15d：警告信从不投递、静默回收 | ② |
-| 25 | 启用空载停机后等超时：三层复合缺陷永不生效 | ② |
-| 26 | 运维删/轮换 RCON Secret → 永久锁死至人工删 pod | ② |
-| 27 | 曾长启动的服 + 任意 pod 波动 → 误判 StartupTimeout | ② |
-| 28 | 重跑 cfsetup 且线上有旧策略 → fail-closed 被顶替；⚠无真实 CF 账号，仅 httptest | ④ |
-| 29 | 装后改 SMTP（configure email）→ 复制链从未生效 | ② |
-| 30 | 对不存在/刚被删的用户 id 操作 → 500 而非 404 | ③ |
-| 31 | 真实 owner 打开 /admin/builds 被 mock 残留误判 | ① |
-| 32 | 新浏览器首访 /admin/builds 播种两个假 404 | ①（轻） |
-| 33 | 禁用/已删账号重走邮箱登录门 → 可复活（安全） | ① |
-| 34 | 同族：死账号游戏内身份面仍存活 | ① |
-| 35 | 任何真实服务器保存过一次后：所有备份/fileread 必败 | ① |
-| 36 | 任何构建失败的提交者永远看不到结果 | ① |
-| 37 | 管理端 fleet：login/lobby 行全是死操作 | ①（轻） |
-| 38 | felis manifests 输出残留过期指导 | ②（文档） |
-| 39 | 断玻璃 reset owner 用非在位名 → 铸第二 owner 永不可清 | ② |
-| 40 | 断玻璃 add operator 撞名 → 裸 SQLSTATE | ② |
-| 41 | 断玻璃 Sync 选系统服 → 裸内部错误 | ② |
-| 42 | 对无世界盘服备份/恢复 → 202 后静默卡 30 分钟 | ② |
-| 43 | #42 配套：409 一刀切文案 | ②（轻） |
-| 44 | 重跑 `felis setup` 完成 s/c reconfigure → 状态框退回“Setup complete”（信息一致性） | ①（轻） |
-| 45 | 评审看不到将被执行的 recipe（执行的是上传 blob 里的 Dockerfile）→ 盲批 | ① |
-| 46 | 大层推送打死 registry（256Mi 模板被 OOM kill；实测 475MB 层）——用户构建/安装器入仓即触发 | ① |
-| 47 | Mac 打包树构建：`._*.sql` 旁文件混入 embed → `felis migrate` 全挂 | ② |
-| 48 | 安装器重跑：每镜像 start/stop docker 触发 systemd 限流 → 批量入仓中途断 | ② |
-| 49 | 安装器重跑：`[registry]`/`[archive]` 运维配置静默回退（构建/S3/reaper 行为回默认） | ② |
-| 50 | 配过 SMTP 的安装按文档重跑安装器 → 生成的注释块被 `[smtp]` carry 吞并并每次 +1（纯膨胀） | ② |
-| 51 | 改配置/轮换 DB 凭据后：工作负载 ns 的 `felis-config` 副本永不刷新 → backup/reaper 静默读旧配置 | ② |
-| 52 | 向导内 `s`/`c` 重配置后：工作负载 ns 的 `felis-config` 镜像停在上一次运行快照，直到下次 `felis setup`/安装器才收敛 | ② |
-| 53 | 默认安装或 `felis update` 的 felis-api 检查：坐标仍指向已迁移的旧仓库，现仅靠 GitHub 301 兜住——redirect 一退休，安装器默认 URL 与更新检查全挂 | ② |
-| 54 | 完成态安装上照 `felis update` 指引升级：`sudo felis setup` 只开配置控制台，任何组件都不会动（三处 run 指引 + trailer 全假） | ②（文档） |
-| 55 | 配置的源回 200 但档案字段不可用（马虎自建/三方 Yggdrasil）：坏源在前 → 经梯子的登录被静默吞掉（零日志、后续源不被询问） | ② |
-| 56 | 给"还没进过服"的玩家加白名单/封禁/踢出（面板说成功、vanilla 实际拒绝）——任何新玩家第一次被管理就撞 | ① |
-| 57 | 控制台发任何命令后无反馈（回复被丢弃、pod 日志也不回显） | ① |
-| 58 | 对从未启动过的服务器点"文件"页（90s 卡死 → 误导性 504） | ① |
-| 59 | 装了 LuckPerms 的服打开 LP 页：读不到被报成"未分配任何权限"（假事实），且历史伪造 `[RCON]` 输出 | ① |
-| 60 | 新建服务器输入非法名/子域名（前端预检比后端弱）→ 对话框直出 Go 英文错误；账户验证撞已占邮箱同理 | ① |
-| 61 | 给"还没进过服"的玩家预授权限：面板提示"仍然会实际生效"（过度承诺），实际 LuckPerms 静默丢弃（未进服的名字无法解析） | ① |
-
-统计：① 26 ｜ ② 25 ｜ ③ 3＋#23(a) ｜ ④ 4 ｜ 决策 3 ＝ 61。第三批构建链另有 3 处未编号修复（Job requests 超小节点上限 → 永远 Pending、Kaniko chown、Trivy DB egress 被锁）——均属 ①/②。
-
-第四十批追加：#63（demo-up 单起点化——旧镜像臂能产出"起了但无处路由"的演示机；dev/demo 路径）②；#62 经立案复核**不成立**（全部安装路径都构建 felis-velocity.jar，证据见该批节），已剔除、不计入。
-
-第四十一批追加：#64（三个 vendored `gradlew` 以 100644 提交、无可执行位——README 教的构建命令在全新 clone 上直接 `Permission denied`；无任何 CI/安装路径跑过这三个模块）①；#65（三个装载器 mod 的 `license` 仍写 MIT、Forge/NeoForge 的 `issueTrackerURL` 是 `example.invalid` 占位，与仓库 AGPL-3.0-only 相悖——fabric loader 启动时会打印该字段）②（低）。
-
-第四十二批追加：#66（英文 README 缺中文版"使用方式"里的私有仓库安装 workaround 与重跑升级说明——英文读者照文档第一步即 404、无任何指引）①（轻）。
-
-第四十三批追加：#67（build Job 从不设 `ttlSecondsAfterFinished`——每构建一次就永久留下一个完成 Job+Pod，完成 Pod 计入节点 pod 预算（stock k3s 110），构建量上来后新构建全 Pending）②（时间维度；随构建量从②滑向①）。
-
-第四十三批追加（二）：#68（构建 context 拉取单次尝试——控制面滚动/重启/驱逐恰好撞上构建窗口时，fetch initContainer 一次 `connection refused` 直接打成终态 `Failed`；`BackoffLimit=0` 无第二次 Pod，代价 = 人工重审重提）②（运维：升级/重启/故障恢复撞上正在进行的构建；构建量大时概率上升）。
-
-第四十五批追加：#69（README 中英"审批通过后自动构建**并部署**"过度承诺——数据模型无目标服务器、部署实为"选用该镜像"；① 轻）；#70（kaniko 拉内建底座默认 HTTPS——`--insecure` 只覆盖推，任何 `FROM registry.felis.svc:5000/…` 构建必败 ①）；#71（kaniko 以 drop-ALL 解包底座层，chown 必败——任何非 scratch 底座必败 ①）；#72（trivy 扫 jar 必拉 Java DB、被 egress 锁拒绝——含 jar 即所有真实模组包的构建必败 ①）；#73（bootstrap 测试在跑 k3s 的主机上必假失败 ④ 工具）；#74（console 断连测试读写竞态 flaky ④ 工具）。
-
-## 已验证事实（正向清单）
-
-- 安装→hook 发码→Owner 绑定→passkey（虚拟认证器）→面板管理员全链路 ✅
-- 建服（POST /servers）→ 唤醒（operator 拉 StatefulSet pod）→ RCON `list` → SSE 控制台 → 停止 ✅
-- 文件编辑：列目录/读/写（wire 为 base64）/256KiB 413/路径穿越 5 变体全拦截/运行中 409 ✅
-- **备份→篡改→恢复数据演练**：v1→备份→v2→恢复→读回 v1 ✅（G2 数据可恢复）
-- **毒档案 fail-closed**：穿越/绝对路径/符号链接条目 → `archive entry escapes target` 退出码 1，零写入 ✅
-- 混沌：PG 掉线（healthz 仍 200、恢复后连接池自愈）、API pod 击杀（~2s 中断）、整机重启（32s 回归、会话/CRD/停止态全保留）✅
-- `felis update` 报告（k3s/velocity 有更新、私有仓库 404 优雅处理）✅
-- 账户全套真机 E2E：绑定码新玩家注册（幂等/并发见 #18）、邮箱验证（onboarding 门）、邮箱 OTP 登录（错码扣次/5 次锁定后正确码也 400、重放 400、staff 账号 403 拒绝）、Passkey 注册+discoverable 登录+邮箱优先登录（虚拟认证器；Chrome 要求 `Page.bringToFront` 才能过 focus 检查）、登出吊销会话（旧 cookie 401）
-- op-login 全状态机（start→status→approve→finish；早 finish 不烧码、错码扣次且请求保留、重放/重复批准/非管理员批准/未知 handle 全部按契约返回）✅
-- 并发：OTP 风暴 8 路 = 1×202 + 7×429 且仅铸 1 码；绑定码并发（同码/双码）见 #18 ✅
-- **reaper 全链路真机演练**（修复 #19 后）：绑定挂载假世界 → `felis reaper` 一 pass：归档 tar 落盘（内容含标记文件）、`world_backups` 行 `inactive_15d`+90d 过期、**PVC 删除且宿主目录回收**、servers 行保留且 activity 时钟重置（红线②）、CRD 保留；第二 pass：伪造过期归档被驱逐（`expired=1`，文件删、行转 `deleted`），合法归档未动；`evaluated=2 reaped=1` 只动到期的世界 ✅
-- **S3 存储向导全链**（第三十二批）：错误凭据预检零副作用、正例落地（Secret + 两 toml + API 滚 + 状态行）、上传 → MinIO 桶 → 内部面取件、UI 回滚归位、控制面/工作负载镜像对齐 ✅
-- **NetworkPolicy 栅栏真机**（第三十五批）：pod 源"身份×端口"矩阵全对（允许行与拒绝行同源对照）、转发型外部源白名单闭环（拒→放→拒，ipset 与 spec 双幂等）、host/ClusterIP 路径全通、Mac→NodePort 面板 200；两条归因注记见该批 ✅
-
-### 本轮新增真机证据（第二日）
-
-- **默认安装的备份闭环**：apply 新 bundle → 归档 PVC 自动创建（WaitForFirstConsumer，被首个 backup Job 拉绑定）→ `POST /backup` 202 → `GET /jobs` running→succeeded → 漂移 marker → `restore-backup` → 读回备份前内容 ✅
-- **回收闭环（真实 k3s 布局 + 权限）**：resolvecheck（20d idle）→ CronJob 派 Job：`evaluated=2 reaped=1`；归档含 marker、PVC+宿主目录回收、`world_backups` 记 `inactive_15d`；期间先经历 `lstat /worlds/…: permission denied`（k3s storage root 0700）→ 修 ACL/root 权限后通过 ✅
-- **磁盘压力三连 drill**：①1e6 自定义类：游戏 pod 先走、控制面随后仍被逐（kubelet ranked/evicted 日志）→ 镜像 GC → ErrImagePull；②改内建 `system-cluster-critical` 后：`cannot evict a critical pod` × felis-api/operator/registry，全部保持 Running；③恢复阶段实测 `DiskPressure` 条件 ~5min（eviction-pressure-transition-period）转 False，被 GC 的游戏镜像按 runbook 重导入后 25s 恢复 ✅
-- **混沌**：SIGKILL k3s 主进程 → kubectl 与 felis-api ready 均在 **10s** 内恢复（api pod 本身未被重启）✅；内存压力（2.6G tmpfs 满写）：无 OOM kill、无人被逐（swap 5.9G 吸收 + 内核回收），控制面不受影响 ✅
-- **构建链路定位（当时未修）**：Kaniko/Trivy 默认镜像是外部的 → 已加 `[registry] kaniko_image/trivy_image/build_cpu_limit/build_mem_limit` 覆写；上下文跨 ns 不可挂载（PVC 不能跨 ns）是设计级缺口，s3 lane 也缺凭据注入 ✅（证据：`internal/submit/blobstore.go:40`、`cmd/felis/api.go` contextBase 分支、jobspec 无 volumes/env）→ **第三批已修复，见下**
-
-### 本轮新增真机证据（第三批：探针 + 构建链路全通）
-
-- **控制面探针（`0c8e29b`）**：felis-api / felis-operator / registry 三个 Deployment 此前**完全没有探针**。修复后真机验证：api、operator `/readyz`+`/healthz`（internal 8081），registry `/v2/`（含命名端口解析）；三个 Pod `Ready=true`、`restartCount=0`、无 `Unhealthy` 事件；operator 新增 `--health-probe-bind-address`（8081，与 metrics 8080 分离，零检查也 404 → 已注册 ping）
-- **构建链路全通（`f79e5eb` + `02fd2de`）**：
-  - 传输：context_ref 改为 internal-face URL；`felis fetch-context` initContainer 走 service token（对象来自 `felis-build` 里按 `bootstrap`/`felis setup` 同款 Secret 复制机制落地的 `felis-service-token`；netpol 只放行控制 ns:8081）+ zip-slip 安全解包到限容 emptyDir；Kaniko `--context=/context` 只读挂载
-  - **真机 E2E**：玩家 OTP 登录 → 建 submission → 上传 gzip context（marker `felis-e2e-build-marker`）→ Owner approve → Job：fetch ✅ → Kaniko 构建并 push `registry.felis.svc:5000/user-uploads/<sub>:latest` ✅ → Trivy 扫描（内建镜像 DB）✅ → Job `Complete`；**从 registry 拉回镜像校验 `/hello.txt` 内容 = 上传的 marker** ✅
-  - 演练中真机抓到并修复 3 个单测看不到的缺陷：① Job requests=limits（2C/4Gi）在 4C/5.5G starter 上**永远 Pending**（`FailedScheduling/Insufficient memory`）→ requests 改为地板值（250m/512Mi，不超上限）；② Kaniko 重拷 Dockerfile 时 chown/chmod 到源文件 owner（distroless uid 65532）在 drop-ALL 能力下失败（`copying dockerfile: chown … operation not permitted`）→ fetch 容器以 root 解包（= Kaniko 自身 uid）；③ Trivy DB 默认 `mirror.gcr.io` 正被 egress 锁拒绝（fail-closed）→ 新配置 `[registry] trivy_db_repository` + 文档 §8e 固化镜像配方（`docker pull/tag/push aquasec/trivy-db:2` 进内建 registry；`--insecure` 已覆盖明文 HTTP）
-
-### 本轮新增真机证据（第四批：PG 契约测试 + 验证邮箱唯一性 + owner 角色）
-
-- **PG 契约测试首跑抓到 #20**：`internal/pgint` 对着真实 PG（`felis_pgint`，drop schema + 重放迁移）跑会话/OTP/op-login/绑定码/submission/build 全契约；`TestOnboardingEmailOTPRejectsTakenEmail` 当场变红 → 挖出「索引从未创建 + `ErrEmailTaken` 从未返回」。
-- **迁移 0020 真机应用**：`felis migrate up` → `applied 1 migration(s): [20]`；`schema_migrations` max=20；`pg_indexes` 出现 `users_verified_email_unique`。
-- **邮箱唯一性真机 409 演练（auditfix17）**：owner 已验地址被玩家 onboarding 流程二次验证 → 第一次 409 `email_taken`；**同码重放仍是 409**（证明码未被消费，符合契约；若被消费会是 400）。
-- **admin 改邮箱清 verified 真机演练（auditfix18）**：PATCH player.test → `[email protected]` 后 `email_verified=f`；PATCH 回原地址仍 `f`；玩家走 onboarding 重验证 → `t`。
-- **owner 角色真机闭环（auditfix18）**：owner 行按 0011 文档升级路径置 `role='owner'` → `GET /api/v1/users` 200（修复前 403）；owner op-login start 铸请求+发码、finish 得 `role=owner` 会话；PATCH owner role / DELETE owner / DISABLE owner 全部 403；玩家邮箱登录回归 200。
-- **配额原子门（第五批，`bb68fef`，auditfix19 已上线）**：pgint 并发实证（真实 PG 上两路并发认领：恰 1 赢 + 1 `ErrQuotaExceeded`；修复前两路全赢）；`docs/deferred-seams.md` 的 audit #4 条目核销。
-
-### 本轮新增真机证据（第六批：reaper 警告链路端到端演练，auditfix20）
-
-- **演练装置**：独立 hostNetwork Pod（`felis:auditfix20`，SA/卷结构复刻 live CronJob）+ 宿主本地零依赖 SMTP sink（捕获完整 RFC5322 原文）+ 专用 `felis-config-drill` Secret（SMTP 指向 `127.0.0.1:1025`）；场景服 `warntest`（独立 CRD，desiredState=Stopped，不占资源），期间 operator 暂停防 reconcile。
-- **投递**：13d idle → 恰 1 封，`To: [email protected]`（owner 的**验证**邮箱）、subject 含 `3d`、正文含 server/remaining；`warned_3d_at` 盖章、`warned_1d_at` 留空；`world_backups` 保持 4 条（纯警告零备份）。
-- **去重与档位**：重跑 0 封（已盖章不重发）；14.5d idle → 补发第 2 封 = `1d` 档（elif 顺序正确，不是重复 3d），`warned_1d_at` 盖章。
-- **失败语义（三连）**：SMTP 端口打坏 → 日志 `warn delivery failed; will retry next run`（err=`connection refused`）、**不盖章**，恢复端口后同一 run 补发成功；邮箱 `email_verified=false` → `resolve owner email: … has no verified email`、不盖章，恢复后补发；owner NULL → 静默跳过（无日志无警告）。
-- **边界**：idle=11d23h（< 12d 阈值）不发不盖章；idle=12d1m 发。`http_code` 冒烟：升级 auditfix20 后 panel 200 / op-login 200。
-- **环境还原**：warntest CRD+行删除、drill Secret/Pod 删除、operator 恢复；`servers` 表回到 resolvecheck+test-one，CRD 回到 lobby/login/resolvecheck/test-one。
-
-### 本轮新增真机证据（第七批：idle auto-stop 三层修复，auditfix21/22）
-
-- **发现路径**：operator 深挖时先怀疑"计时器无驱动"，真机实验立刻抓到 pruning（操作日志 `unknown field "status.emptySince"`）+ 触发后 88s 无动作 + RBAC 403，三层各自独立、各自足以致死。
-- **场景 1（超时点唤醒 + 写权限）**：EmptySince 已超时 11 分钟 → annotate 触发一次 → 秒级内 desiredState=Stopped、sts 0/replicas、EmptySince 清、phase=Stopped。
-- **场景 2（完整自驱，决定性）**：desired=Running → pod 起 → 20:11:43 盖章 → 静置无干预 → **20:12:17 自动 Stopped**（30s 到点后 ~4s 完成 stop+scaledown+markStopped）。
-- **翻转混沌**：3 秒内 Running→Stopped→Running，最终收敛 Running ready（期间 409 竞争为控制器正常噪音，controller-runtime 重试自愈）。
-- **收尾**：`spec.idle` 删除后再无自停；test-one 回 Stopped、空 sts、无 EmptySince；VM 镜像 auditfix22 = `f650bf8`。
-
-### 本轮新增真机证据（第八批：operator 自愈深挖，auditfix23/24）
-
-- **RCON Secret 删除自愈（`56f3abd`）**：修复前——删 secret 静置 60s 零察觉、触发后 96s+ 永久 `RconNotReachable`、仅人工删 pod 可恢复；修复后——删 secret **同秒**（20:22:16）察觉（Secret watch 日志）→ 新密码 + stamp 换值（`3b24164f…`→`c465b627…`）→ rollout → **20:22:47 全自动恢复 Running**（31s）。
-- **陈旧锚点（`82b5a60`）**：修复前——已恢复服务器因锚点残留被 `Failed(StartupTimeout)`、`Provisioned=False` 永挂；修复后——Ready 即清锚（`startRequestedAt: None`）、`Provisioned: True`；pod blip 演练：删除→20:24:48 重锚→20:25:19 恢复→锚清空，**全程无 Failed**。
-- **StatefulSet 误删自愈**：删除 sts → **秒级**重建（PVC `world-test-one-0` 保持 Bound，Retain 策略）、29s 后 Running、stamp 保持 `c465b627…`（未触发多余 rollout）。
-- **收尾**：期间 7 条 operator ERROR 全为 CR/sts 409 竞争噪音（自愈）；test-one 回 Stopped。
-
-### 本轮新增真机证据（第九批：未演练端点地毯覆盖 — 迁移 / access / window / credentials）
-
-- **account/migrate 四步状态机全链路（首个端到端演练，0 缺陷）**：造 user2 目标账户（绑定码 `FDTGKYAQ`）→ player.test claim test-one → 内部 start（未联动 UUID=404 not_linked；源=201 initiated）→ `passkey_required 409`（有 passkey 强制强因子）→ **passkey credentials list + DELETE 204**（顺带覆盖管理端点；删后 OTP 门自动解除）→ OTP start 202 / 错码 400 / 真码 confirmed → issue-code（self=400 / missing=400 / 真目标=201）→ redeem 负例×2（源持码=400；target 错码=403 setup_required——**onboarding 门正确拦截未验证账户**；验证邮箱后重试）→ **redeem 成功**（小写+空格码兼容，`servers_moved:1`）。
-- **retire 断言全对**：servers.owner→user2；migration=redeemed（含时间戳）；源 disabled+软删；源活跃 session=0；源旧 cookie 401；double-redeem 400；源再加迁移 409 account_retired。**环境复原**：源复活 + 新 OTP 会话、test-one 归还无主。
-- **access 玩家管理全组（0 缺陷）**：players/whitelist/banlist 三个读 projector 解析正确（`There are 0 of a max of 20 players online: `→[]）；kick/permission/group/luckperms-info 调用全通（demo 服无 LuckPerms → 原样透传 `Unknown or incomplete command`，API 不掩饰）；输入负例 4 连 400（bad player/bad action/bad node/bad world）；`whitelist add/remove` 在空档案服上得到 vanilla `That player does not exist`（fail-closed egress 无法解析 Mojang profile——**vanilla 约束非 API 缺陷**，命令已如实送达）；手写 whitelist.json + `command whitelist reload` 验证成功路径解析（`players:["E2E_Tester"]`）；**audit 7 条 access.\* 全记录**。
-- **updates/window**：unset→null；PUT 正例+读回；半设 400；倒置 400；clear→null 全对。
-- **/fleet**：4 服全量（含运行中 lobby/login 的 endpoint 地址）一次读全。
-- **bootstrap-assets crd**：嵌入式资产含 `emptySince`（auditfix24 二进制核对）。
-
-### 本轮新增真机证据（第十批：configure email 复制链修复，auditfix25）
-
-- **发现路径**：复核 `8e7c7bb` 新增的 replicate 逻辑时怀疑 manifest/-n namespace 冲突 → kubectl 真机实验裁决（hardcoded `namespace: felis` 版直接 `error: … does not match …`，证实第一段必错、第二段被跳过）。
-- **修复后行为验证**：两条命令序列（felis-smtp manifest 携带目标 ns + `felis-config` create--dry-run|apply）均被 server 接受且落点 `minecraft`。
-- **live 收敛项（随下次 `felis install/setup` 重跑）**：live CronJob 仍是旧模板（无 `FELIS_SMTP_PASSWORD` env）；`felis-smtp`/镜像两 ns 均未创建（SMTP 未配置，属正常）；与 operator Role 手动补 patch 同批处理。
-- **部署**：`felis:auditfix25` = `ed722d5`（api/operator/reaper 三处 set；api/operator rollout 完成，panel 200）。
-
-### 本轮新增真机证据（第十一批：submission 全路由负例矩阵 + 内部面取件 + owner 会话重铸）
-
-- **session 重铸路径（hook）**：owner 旧 cookie 过期（401）→ 临时插 `account_links`（owner↔假 UUID）→ op-login 三件套：start 202 → 日志取码 → 内部面 `op-login/{id}/approve`（hook，经 service token）→ status `approved:true` → finish 200 `role=owner` → op 域新 cookie 生效（`/fleet` 200）；**演习后已删除假链接行**。顺带复证：approve 失败时 status=false、finish 400 `op_login_invalid`（码不烧，复用同码二次 finish 成功）。
-- **submission 车道负例矩阵（23/23 PASS，0 缺陷）**：create 未知字段/空名/非法字符/超长 → 全 400；非 gzip 上传 400、未知 id 404；跨用户上传 → **404 不可见**（非 403）；玩家打 admin 三路由（queue/approve/reject）全 403；reject 空理由/1001 字/夹带字段 → 400，成功 200（`reject_reason`/`reviewed_by`=验主邮箱/`status=rejected` 全对）；重复 reject 409、reject 后 approve 409；**无 blob approve → 400 且行保持 pending_review（未 stranded）**；玩家列表严格只见自己。
-- **内部面取件**：`GET /api/v1/internal/submissions/{id}/context`（service token）→ 200 + gzip magic `1f8b` + 解包内容 = 上传 marker；无 token 401；未知 id 404。
-- **演习清理**：E2E-Lane-\* 8 行已删、假链接已删；队列只留历史 "E2E *"（approved）行。
-- 备注：port-forward 再次因 API pod 重建悬死（空响应）→ 按速查重启即恢复；本轮无需新镜像（纯验证）。
-
-### 本轮新增真机证据（第十二批：users 管理面全量矩阵 → 抓到并修复 #30）
-
-- **/users 管理面矩阵（修复前，39 PASS / 2 NOTE）**：create/get/patch/list 权限与负例全对（未知字段/非法用户名/owner 角色/未知 id → 400/404；owner 保护 403 复证）；sessions 单条吊销→旧 cookie 立即 401→重登恢复、revoke-all 同效（重登两次含 61s 冷却等待，全绿）；passkeys 无凭证 200 no-op；links 幂等/跨用户 409/非法 auth_source 400/解链 404 全对；disable/enable/delete/double-delete 全对。
-- **缺陷 #30（两处 FK 500 + 一处假 200）**：`PUT /users/{unknown}/quotas` 与 `POST /users/{unknown}/links` 触碰 user_id 外键 → **500 internal**（契约要求 404）；`GET /users/{unknown}/quotas` 回**零值视图（=unlimited）**，像 id 存在。修复：`PGRepo.requireLiveUser`（live 行 + `deleted_at IS NULL`）守卫三个方法，`handleGetQuotas`/`handleLinkAccount` 补 ErrNotFound→404 映射；单测 `TestAdminSubresourcesRequireLiveUser`（fake 同步 liveUserExists 契约）+ pgint 真库契约（ghost→ErrNotFound、live 对照、软删后→ErrNotFound）。commit `1918da2`。
-- **auditfix26 真机复验**：ghost 三连 → **404 not_found**；活用户对照 → PUT/GET quotas 200（读回 2）、POST link 200、delete 200；panel 200，api/operator 滚动完成（新 pod 24s Ready）。
-- 契约备注（非缺陷）：软删用户 `GET /users/{id}` 仍 200（带 `disabled=true` + `deleted_at`，列表已过滤）；`DELETE .../sessions/deadbeef` 幂等 200；revoke-all 对未知用户 200 no-op。
-
-### 本轮新增真机证据（第十三批：CLI / 镜像面 / 直接构建 / 面板 CDP 全路由 + #31/#32）
-
-- **CLI 面 19/19（VM，`felis-auditfix26`）**：`version`/`-h`/无参=exit2/未知命令=exit2；`manifests` 缺 `--velocity-cidr`、缺 `--felis-image`、坏 CIDR、坏 NodePort 全 fail-loud exit2，正例渲染 25 文档且 `--worlds-host-path` 分支出 CronJob+三条前置警示——整包 `kubectl apply --dry-run=server` **全部 `configured`**；`apply` 缺 `-f`/文件不存在/坏 JSON 各自 exit2/1/1；`migrate up` 二次幂等（`database already up to date`）；`bootstrap-assets crd`（含 `emptySince`）与 `game-stack`（tar）正常；`update` 正常。
-- **镜像面负例 + builds 负例**：`/images` 列表 200、空体 400、合法 201、重复幂等 201、无 `?ref` 400、删除 **204**（契约；我的 200 预期作废）、再删 404、坏 ref 400；`/images/build/unknown` GET/cancel 均 404；空体提交 400；玩家打 `/images*` 全 403。
-- **`/images/build` 正路径（直接构建）**：借遗留 submission blob 作 context，`FROM scratch` 内联 Dockerfile → **202 → 12s succeeded**；`/logs` SSE 实收 Kaniko 流；真机 registry `tags/list` 出现 `e2e/direct-probe:latest`（镜像确实被推入）。留档：该 drill 镜像保留在 registry。
-- **`auth/options`**：未知邮箱 → `{"methods":[]}`；玩家 → `["email_otp"]`（无 passkey 时）；坏邮箱 400。
-- **`PATCH /servers/{name}`**：空体/storage/空 policy/未知字段/坏名 → 400；未知服 404；displayName 设置+还原 200。**契约备注**：`displayName:""` 因 `omitempty`+merge-null 语义 = **清除该字段**（本次 drill 先误读为设空串，比对 patch 前 fleet 快照确认 test-one 原无 displayName，无数据损失）。
-- **面板 CDP 全路由巡检（owner 会话，14 路由）**：全部有标题/有内容/无崩溃；唯二发现 = #31/#32（见下）；`luckperms` 页对 stopped 服 409 → **优雅降级**为「服务器已休眠 + 启动」态（browser 级 409 日志属正常资源日志，非缺陷）。玩家会话 4 路由零错误且**导航分级正确**（不见管理/平台组）；未登录 `/login` 渲染正常（`/me` 401 为预期探测）、`/account` 未登录重定向 `/login`。
-- **缺陷 #31（面板 mock 残留·owner 判定）**：`ImageBuildPage` 用 `email==="[email protected]" || startsWith("owner@")` 猜 owner——真实 owner（`[email protected]`）**不被识别**（只能看 approved 提交），而任何 `owner@x` 邮箱都冒充 owner。改为消费 TierProvider 服务端 `isOwner`。commit `6907961`。
-- **缺陷 #32（面板 mock 残留·假构建种子）**：首次访问 `/admin/builds` 向 localStorage 播种 `["bld-1","bld-2"]` → 每个新浏览器打两个必然 404 的请求（页面注释自认“in production will 404”）。移除播种。同 commit。
-- **auditfix27 复验**：清掉该 localStorage 键后重访 → **只发 `/me`**、零错误、空态正确；面板 111 单测 + typecheck 全绿。
-- **passkey 全流程复验（CDP 虚拟认证器，面板 UI 实操）**：注册（对话框输入→仪式→列表出现 `E2E-Key`）→ API logout → 邮箱优先 passkey 登录（`/me` 回 `[email protected] user`、落回 `/`）→ `DELETE credentials/{id}` 204 → 列表清空（环境复原）。
-- **环境**：api/operator/reaper 镜像 = `felis:auditfix28`（= `6907961`）；面板 200。**教训**：`pkill -f "port-forward …"` 会匹配到**执行该命令的 ssh 自身 cmdline**（命令里同时含明文 `port-forward svc/…`）→ 自杀式断连；改为 `ss -ltnp` 取 PID kill，另起一条 ssh 启动转发。
-
-### 本轮新增真机证据（第十四批：#33 死账号复活漏洞 —— 全登录门闭环 + 资产切断）
-
-- **发现路径**：users 管理面演习收尾核对时发现软删用户仍留 `account_links`/`quotas` 孤儿行 → 顺藤摸瓜确认 `UserByEmail` 与 `SessionUser` 均不过滤 `disabled`/`deleted_at`。
-- **红证据（auditfix27，真机）**：禁用用户的邮箱门重登录 `verify=200`、`/me=200`（**禁用锁死可绕过**）；**已删号**用户 `start=202`（真实发码）、`verify=200`、`/me=200`（**删号可复活**）。
-- **修复（`5853589`，7 文件 / +306 行）**：① `UserByEmail` 只解析 live 账号（覆盖 email/passkey 邮箱优先/op-login/auth options 全部前置门）；② `SessionUser` 同样过滤（皮带：任何门铸出的死号会话都立即失效）；③ `RedeemPlayerBindCode` 两个解析臂拒绝死账号（新哨兵 `ErrPlayerAccountRetired` → 403 `account_retired`，**不消费码**，可逆重试）；④ `DeleteUser` 事务内 `DELETE webauthn_credentials` + `account_links`（凭证与 `UNIQUE(mc_uuid)` 占用不再外泄）；⑤ discoverable passkey resolve 补 liveness 检查。单测 `TestDeadAccountsCannotLogInOrKeepSessions` + pgint `TestDeadAccountsAreLockedOutInPG`（含"删后新铸会话不验证"与资产清点）。
-- **绿证据（auditfix28，真机）**：红期为死号铸出的会话 → **401**（皮带生效）；死号 `start=202` 中立且 **90s 日志窗口 0 条码**、`verify=400 invalid_code`；禁用中 `start` 中立（日志码数 1→1 不变）→ `verify=400`；**恢复启用后 `verify=200` + `/me=200`（可逆）**；bind 门死账号 → **403 `account_retired`** 且码保留 `count=1`（retryable）；删号后 `account_links`/`webauthn_credentials` 行数 = 0。
-- **演习清理**：红演练遗留（2 个测试账号的 quotas 行、1 条旧链接、2 条死号会话）已清除；`/fleet` 与 panel 冒烟保持 200。
-
-### 本轮新增真机证据（第十五批：#34 死账号同族收尾 —— in-game 身份解析与链接接管，auditfix29）
-
-- **发现路径**：#33 修完 Web 登录门后做同族复查——**in-game 面是 UUID 驱动、不走 Web 会话，因此 #33 的会话皮带盖不住它**。两处：① `UserByMCUUID`（claim / menu / wake 授权 / op-login vouch / QR link-status 五处共享解析）直接读 `account_links`、不过滤 users 的 liveness；② `VerifyLinkCode` 对「holder 已死」的接管语义未定义——一刀切 409 会把迁移退役这类真实场景锁死。
-- **修复（`bb9798e`，5 文件 / +182−7）**：① `UserByMCUUID` JOIN users 过滤 `disabled`+`deleted_at`——死账号的链接在游戏面**读作未链接**（claim 412、wake 落回 policy 门、vouch 403、status `linked:false`），绝不作为遗留身份存活；② `VerifyLinkCode`：**软删** holder 的链接可由新账号凭新 mint 码**接管**（账号已亡，码证明调用者仍持有该 UUID），**禁用** holder 仍 409（接管= 绕过锁死，不允许）、任何失败都不消费码。fake 单测 `TestLinkVerifyTakesOverDeletedLinkOnly`；pgint `TestVerifyLinkCodeTakesOverDeletedLink` + `TestDeadAccountsAreLockedOutInPG` 增「禁用链接无资格 / 复启用恢复」断言。
-- **绿证据（auditfix29 真机，三面六点）**：
-  1. `link/status`：活 `{"linked":true}` → 禁用 `{"linked":false}` → 恢复 `{"linked":true}`；
-  2. `claim`：活链 404（身份已解析、ghost 服不存在）→ 禁用 **412 not_linked** → 恢复 404；未知 UUID 对照 412；
-  3. **接管正例**：临时软删 holder（`de49df52`）→ 新 mint 码 `X3G3RZZM` 由 player.test verify → **200 linked:true**；链接行迁移到 player.test（`auth_source=mojang`）、码被消费（`account_link_codes` 0 行）；演练后 holder 与链接全部还原；
-  4. **接管负例**：临时禁用 holder（user2）→ 新码 `X5W94S42` verify → **409 already_linked**；同码重试仍 409（**不消费**）；恢复启用后**同码**由 holder verify → **200**（码保留、可逆）；
-  5. **wake 授权**：活 owner **202**（真实启动）→ 禁用 **403 forbidden** → 恢复 **202**；test-one 已停回 Stopped 且所有权释放；
-  6. **op-login vouch**：禁用 owner UUID → **403 not_admin**（与未链接/非 staff 同一个拒绝面）；活 owner UUID → 404 op_login_not_found（vouch 已解析、只是请求不存在）。
-- **环境刷新（部署态）**：热升级遗留 `FELIS_IMAGE=felis:auditfix16`（job 侧执行器 backup/restore/fileedit/forwarding-init 引用的镜像）→ 已 `set env` api/operator 双 Deployment 至 `felis:auditfix29` 并滚动完成；api/operator/reaper 三镜像一致 = auditfix29；port-forward 重启后内部面绿；panel 200、op 面 `/me` 200。
-- **运维备注**：`account_link_codes` 会积累过期行（"失败不消费"的另一面），巡检顺手 `DELETE FROM account_link_codes WHERE expires_at < now()`。
-
-### 本轮新增真机证据（第十六批：内部面残面地毯补测 —— ready / join-event / status / reclaim / blacklist / backup 负例）
-
-- **背景**：对内部面做覆盖盘点后发现五个端点此前无真机证据：`ready`、`join-event`、`status`、`player/reclaim`、`player/blacklist`，以及 `internal/servers/{name}/backup` 的负例臂。本批全部补测（auditfix29），**0 新缺陷**。
-- **`ready`**：test-one → **204**（advisory 语义）；坏名 → 400。
-- **`join-event`**（临时 UUID `3333…`）：首次 **204**、重复 **204**（幂等）；DB 实锤：`test-one.last_active_at` 15:02→05:53（重置）、`server_allowlist` 恰 1 行；缺 uuid → 400；未知服 → 404。演习后 allowlist 行删除、`last_active_at` 复原。
-- **`status`**：test-one → 200 全字段（phase/desiredState/endpointMode=fallback/…）；未知服 → 404。
-- **`player/reclaim`**：新建（临时 UUID `4444…`）→ 200 `hold_expires_at`=+30d；**幂等重试返回同一时间戳**（首窗保留，与存储行 21:53:51.726017Z 完全一致）；缺字段 → 400；**protected-admin 负例**：临时插 owner↔thirdparty 链接 → **409 protected_admin**（不 bar、不 stash）。清理后 `username_blacklist`/`player_data_holds`/`server_allowlist`/临时链接 = 0/0/0/0。
-- **`player/blacklist`**：bar 后命中 `{"blacklisted":true}`；陌生 UUID → `false`。
-- **`internal backup` 负例**：未知服 → 404；坏名 → 400；保留名（login）→ 400 `bad_name`（在入队前拒绝，RWO 闸门单测已覆盖）。
-- **探针**：`/healthz` 200、`/readyz` `{"status":"ready"}`。
-
-### 本轮新增真机证据（第十七批：hasJoined 多源会话校验器 —— 假 Yggdrasil 全链路 + 三方身份重写/改名）
-
-- **背景**：`hasJoined`（velocity 指向的 vanilla sessionserver 协议面，`handleHasJoined`）此前零真机覆盖。本批用「VM 主机假 Yggdrasil + 临时将 `[[auth_source]]` 换向」的方式把正/负路径全部打通。
-- **装置**：主机 python 假源（`:18099`，按 username 分流 `ftok`/`ftnotch`/`ftbadname`/`ftdown`/其余 204）；`felis-config` 的 littleskin 源临时改指 `http://10.42.0.1:18099/fake`（tag=`faketest`/prefix=`FT`），api 滚动后逐项打靶。**演练后配置已还原 littleskin、装置已清理**。
-- **结果（全绿，0 缺陷）**：
-  1. **三方身份重写**：`ftok` → 200 `{"id":"74409c3bbae93acabd2176e517b4f2a0",…}`，与本地按 `uuid.NewMD5(felisAuthNS, "faketest:native-123")` 的预算值**逐位一致**；
-  2. **保费名冲突改名**：`ftnotch` → `47c5527a18b03fe5a49ec50c11154dcd` + `name="FT_Notch"`（Mojang 实查 Notch=200 premium）；`ftok` 的 `FtPlayer` 恰也是真实 Mojang 名（`640c1672…`）→ `FT_FtPlayer`，改名按设计触发（非保费名保持原名）；
-  3. **敌意插件名拒绝**：假源返回 `§4admin` → **204**（不落 proxy 玩家列表）；
-  4. **源故障不静默**：假源 500 → **503**（velocity 报 auth servers down），对照未知玩家 → 204；
-  5. **形状负例**：缺参 / 超长参 → 204；声明 body → **400 + `Connection: close`**（防 drain 挂连接）。
-- **顺带覆盖**：`isPremiumName` 的 Mojang 实查（`api.mojang.com`，超时/错误 fail-closed=改名）——404→非保费、200→保费判别实测成立。
-
-### 本轮新增真机证据（第十八批：缺陷 #35 —— world 执行器 uid 1000 读不了游戏服写的世界 + 面板备份模块补全）
-
-- **发现路径**：给备份页新增「立即备份 + 最近操作」后做第一次真机 drill——面板链路全对（按钮/成功消息/运行态→终态、零坏请求），**但 backup Job 真的失败了**：`Job has reached the specified backoff limit`，pod 日志 `felis backup: archive: backup: tar walk: open /world/world/level.dat: permission denied`。
-- **根因（缺陷 #35，跨模块）**：世界卷的属主是**游戏镜像自己的 UID**（我们发布的 Paper 镜像都是 root），而 Paper 保存 `level.dat` 用的是 **0600**（Files.createTempFile 默认权限）→ 固定 uid 1000 的 **backup / restore / fileedit / reaper** 四类执行器：读不了（归档 `permission denied`）、也覆盖不了（restore 写不进 600-root 的 level.dat）。此前 drill 侥幸全绿，是因为当时的世界文件由 uid-1000 工具（restore/夹具）写的；**服务器真实启动保存过一次之后**，所有备份从此必死。测试全部是 shape 断言（无集群），这条只能真机抓。
-- **修复（`2010961`）**：四类执行器统一改**以 root 运行**（`runAsNonRoot:false`，省略 fsGroup 防误 chgrp），容器保持除 `DAC_OVERRIDE` 外 drop-ALL——与 operator `init-forwarding` 容器的既定先例同源（"只有 root 能可靠读写这些文件"）；DAC_OVERRIDE 兜住"游戏镜像是非 root UID"的任意镜像场景。四份 shape 测试同步改断言。troubleshooting §10「Permissions」段落重写（uid-1000 + setfacl 时代结束）。
-- **绿证据（auditfix31，真机四联 drill）**：
-  1. **backup**（面板 CDP 实操，红→绿同场景）：点击「立即备份」→ `进行中` → **`成功`**（同页面 reload 后仍在）；Job pod `runAsUser:0` + DAC_OVERRIDE；新归档 `test-one-1790115428472416736.tar.gz` 实测含 `world/level.dat`（471B）与 server.properties，共 499 条。
-  2. **restore**：`POST restore-backup`（bk-9df0…）→ 202 → Job **Succeeded**（root 写路径过关），日志 `restored from … into /world`。
-  3. **fileedit**：`GET /servers/test-one/file?path=world/level.dat` → **200**（base64-gzip 内容 628B）——修复前该请求必然 permission denied。
-  4. **reaper**（从 live CronJob 派生一次性 Job + 复刻钻取世界：1Gi PV/PVC + `world/level.dat` 512B **0600 root** + marker + 20d idle 行）：pod **root+DAC_OVERRIDE**；`world reaped server=reapdrill2 …`、`evaluated=3 reaped=1`；归档 711B 含 marker 与 level.dat；PVC 删除；`world_backups` 得 `inactive_15d` 行；servers 行/CRD 保留（红线②）。**钻取现场全部清理**（job/CRD/行/PV/宿主目录/临时归档）。
-- **面板补全（`97a64c8`）**：备份页新增「立即备份」（仅 Stopped 可用，409 原文呈现）与「最近操作」卡（GET `/servers/{name}/jobs`，running 每 5s 自刷新，failed 显示 Job 失败文本——正是它把上面这次失败暴露出来的）。面板 113 单测 + typecheck 全绿。
-- **部署注意**：backup/restore/fileedit 的 Job spec 是 api **运行时渲染**，随镜像即生效；**reaper CronJob 的 pod 模板是安装期静态渲染**——本轮已按新形状热补丁 live 对象（root + DAC_OVERRIDE），`felis install/setup` 重渲染时收敛。
-
-### 本轮新增真机证据（第十九批：面板补齐 owner 解绑通行密钥入口）
-
-- **背景**：`DELETE /users/{id}/passkeys`（owner-tier 凭据补救：密钥丢失/被盗时切断登录脚架，且不锁死账号——邮箱码/游戏内审批仍可用）后端早已实现并审计，但面板无入口，owner 只能靠 API。属"功能缺口"而非缺陷。
-- **修复（`11ac4f5`）**：用户详情页危险操作区新增「解绑通行密钥」行 + 确认对话框（`api.unbindUserPasskeys` 客户端方法 + 双语 i18n + wire-shape 测试）。
-- **绿证据（auditfix32，CDP 真机全链）**：player.test 注册虚拟认证器密钥 `E2E-Unbind` → `/auth/options` 从 `["email_otp"]` 变为 `["passkey","email_otp"]` → owner 在 `/admin/users/<id>` 点「解绑通行密钥」→ 对话框 → 确认 → **凭据列表清空、options 回落 `["email_otp"]`**（passkey 门确实关闭）；审计 `user.unbind_passkeys` 落账（同批还可见 `account.passkey.registered`/`backup.create`/`backup.restore`/`reap_world` 各审计行）。面板 114 单测全绿。
-
-### 本轮新增真机证据（第二十批：缺陷 #36 —— 提交者看不到构建结果）
-
-- **缺陷 #36（提交/构建模块·结果不可见）**：`/me/submissions` 只回审核状态；构建的成功/失败（含失败原因）只有 admin-tier `/images/build/{id}` 看得到 → **提交者永远不知道自己的包构建死了**。修复 `72c4aa3`：两个列表路由（玩家 `/me/submissions` + 管理 `/submissions`）对 `build_id` 非空的行附 `build_status`/`build_error`，来源是只读 `Builder.Get`（**绝不调 Sync**——状态推进归 15s reconcile 循环，列表渲染不碰集群）；构建行已消失（ErrNotFound）→ 字段省略；其他存储错误照常 500，绝不静默吞。OpenAPI 的 Submission schema 同步。
-- **绿证据（auditfix33，真机双例）**：① 失败例（上下文 Dockerfile `COPY does-not-exist`）→ 提交 → owner approve → `/me/submissions` 返回 `build_status:"failed"` + `build_error:"build job failed or scan found a CRITICAL CVE"`；② 成功例（`FROM scratch`+LABEL）→ `build_status:"succeeded"`。面板 CDP（玩家会话）：展开行显示「构建状态」徽章（构建失败/构建成功）+ 失败原因 + build_id，全程零 4xx。**演习残留已清**（2 行 submission+build、2 个 context blob、1 条 whitelist 条目；`sub-bf7dc1…` 那条是更早 E2E 遗留，未动）。
-
-### 本轮新增真机证据（第二十一批：面板文件编辑器补齐 —— 缺口而非缺陷）
-
-- **背景**：`GET /files`、`GET/PUT /file` 后端早已全绿（可读写 `level.dat`），但面板无入口——「一行 server.properties 写错导致起不来」的修复路径只有 API。属功能缺口。
-- **修复（`0a36b3f`）**：新增 `/servers/:name/files` 页：面包屑目录浏览、编辑器对话框（[]byte ↔ base64 编解码）、二进制文件打开即只读（NUL/非 UTF-8 拒绝 round-trip）、>256KiB 禁用保存；**停服门前置**（世界卷 RWO，未停服时整页显示「服务器正在运行」+ 停止动作，而不是让每个调用 409）；控制台右侧新增门口卡。i18n `files` 命名空间（en/zh）+ 3 条 wire-shape 测试。
-- **绿证据（auditfix34，CDP owner 全链）**：根目录 → `world/` 导航；`felis-e2e-marker.txt`（原 `v1\n`）打开 → 追加 → 保存「已保存 …」→ **API 读回一致** → 重开一致 → 还原原始字节 → 落盘复核一致（零残留）；`level.dat` 打开为只读 + 二进制提示；控制台门口卡存在；`audit_logs` 两条 `file.write`（edit/restore 各一）；全程零 API 4xx/5xx。面板 117 单测 + typecheck 全绿。
-
-### 本轮新增真机证据（第二十二批：缺陷 #37 —— 系统服务在面板里全是死操作）
-
-- **缺陷 #37（面板·fleet 死操作）**：`login`/`lobby` 是平台自建系统服务、名字在保留名单里，于是**每个 per-server 路由都用 `ValidateServerName` 拒绝**（400 `bad_name`）——但管理端 fleet 表格给这两行渲染 认领/停止/唤醒/控制台，全是死操作（控制台链接点进去也是一页 `invalid server name: reserved`）。修复 `2f90851`：`naming.IsSystemServer` 作为唯一事实源；fleet 行附 `system:true`；面板把这两行渲染为「系统服务」纯标签（owner 列 + 操作列），不再给任何动作。玩家侧不受影响（`/me/servers` 本就不含系统服务）。
-- **绿证据（auditfix35，真机）**：`GET /fleet`（owner）→ `lobby system:true`、`login system:true`、`test-one/resolvecheck` 无 flag；面板 CDP：两行「系统服务」、**0 按钮 0 链接**；`test-one` 行照常 认领/控制台、无系统标记；零 4xx。Go 侧新增 `TestIsSystemServer` + `TestFleetAdminRead` 的 system 子测试。
-
-### 本轮新增真机证据（第二十三批：缺陷 #38 + 多节点回收缺口 —— reaper 提示过期 / 钉节点）
-
-- **缺陷 #38（CLI·提示过期）**：`felis manifests` 渲染 reaper 时的 stderr 提示还在教“uid 1000 需要 `setfacl -m u:1000:x` 才能遍历存储根”——#35 之后 reaper 已改为 **root + DAC_OVERRIDE**，这条指导已失效且会误导运维（照做无害但白做，真问题被掩盖）。同批落 **多节点回收缺口**（结论第 5 条）：新增 `--reaper-node`，reaper CronJob 的 pod 渲染 `nodeSelector kubernetes.io/hostname=<node>`；多节点集群必须钉在存世界的节点，否则可能调度到 hostPath 为空的节点。`--reaper-node` 无 `--worlds-host-path` 时 fail-loud exit 2。修复 `daf7602`（双测：`TestReaperCronJob_NodePin`、`TestManifestsReaperNodePin`）。
-- **绿证据（auditfix36，真机 render + dry-run + 收敛 diff）**：
-  1. **固定渲染**：`felis manifests --felis-image felis:auditfix36 --velocity-cidr 10.211.55.6/32 --panel-node-port 30443 --worlds-host-path /var/lib/rancher/k3s/storage --archive-local-path /var/lib/felis/archives --reaper-node localhost.localdomain` → exit 0；bundle 内 `kubernetes.io/hostname: localhost.localdomain`；stderr **0 处** uid 1000 / setfacl，改为「已钉到节点 …」；
-  2. **负例**：`--reaper-node` 无 `--worlds-host-path` → exit 2 + 原文；不传 node 的渲染 stderr 仍完整保留「NO nodeSelector … 多节点必须传 --reaper-node」警示，bundle 内 0 个 selector；
-  3. **`kubectl apply --dry-run=server -f -`**：整包 **全部 configured**（含新 nodeSelector 的 CronJob）；
-  4. **再安装收敛性 diff**（`kubectl diff`，本批新增的收敛证据）：与 live 对比只剩 **一个对象**（reaper CronJob）两处实质增量——`+env FELIS_SMTP_PASSWORD`（热升级期未补的模板字段）与本次显式传入的 `+nodeSelector`；其余 24 份文档（api/operator Deployment、RBAC、NetworkPolicy、PVC、registry）**零差异**——即“热补丁过的 live 对象”与“当前代码重渲染”已收敛（reaper 安全上下文 root+DAC_OVERRIDE 两侧一致，无 diff）。
-
-### 本轮新增真机证据（第二十四批：S3 上传通道真机演练 —— 此前只有单测的暗路径）
-
-- **背景**：`internal/submit/s3store.go`（S3ContextStore）此前只有单测，本装是 local 路径（`user_uploads_context = "/var/lib/felis/uploads"`）从未激活。本批用「VM 宿主 MinIO（quay.io 镜像，:9000）+ 临时把 felis-config 切到 `s3://felis-user-uploads` + `[registry.s3] endpoint=http://10.211.55.6:9000` + 创建 `felis-uploads-s3` Secret」把整条通道打通，**全程 0 缺陷**、演练后还原并逐项复核。
-- **绿证据（auditfix36，真机全链）**：
-  1. 切 S3 后 api 滚动启动 **无** “S3 user-uploads store not configured” 告警（凭据解析成功）；pod 内 busybox 探针实测可达 `http://10.211.55.6:9000/minio/health/live`（rc=0）；
-  2. 玩家提交 + 上传 → **200**，对象实测落桶：`sub-12c953e12b0cd144/context.tar.gz`（197B，mc ls 实见）；
-  3. owner approve → 构建 Job `build-bld-1790118222183394193` **status.succeeded=1**（fetch-context 从内部面流式取件 = api 自 S3 读回成功）；`/me/submissions` 的 `build_status` 收敛为 `succeeded`；
-  4. **回滚**：felis-config 还原（sha256 与演练前备份**逐字节一致**）、删除 `felis-uploads-s3`、api 滚动；再演练一次本地路径：新提交上传 **200** 且 blob 实测落在 uploads PVC（context.tar.gz 197B）；启动日志仅剩 smtp/jwks 两条既有提示；
-  5. **清理**：2 行 submission + 1 行 build 删除、回滚演练 blob 删除、S3 构建产物从 whitelist 摘除（204）、MinIO 容器 + 两个镜像移除；`/fleet` 200。
-- **遗留观察（非缺陷）**：registry 里保留本次推送的 `user-uploads/sub-12c953e12b0cd144:latest` 层数据（与早前 direct-probe 同类，filesystem registry 无删除接口）；`felis setup` 的「S3 存储」向导屏本身未演练（列入 TUI 逐屏待办）。
-
-### 本轮新增真机证据（第二十五批：breakGlass 控制台逐屏全量 + 备份/恢复门禁 —— 缺陷 #39–#43）
-
-- **背景**：breakGlass 此前只验过 Owner 首装与 Add Operator happy path。本批把菜单四操作（Owner reset / Add Operator / Halt / Sync）+ 首装（bootstrap）分支全部逐屏真机走完，并打穿 Sync/恢复背后的 API 门禁；共抓 5 个缺陷、全部修复复验。驱动方式：VM tmux（`remain-on-exit on` 才能读回 alt-screen 撕掉后的 durable summary）。
-- **先落的正向证据（无缺陷）**：Halt——`test-one` Running → 选中 → 卡「is stopping」→ CRD `desiredState=Stopped`、pod 收敛消失、审计 `break_glass.halt`；面板 `wake`/`stop` 两个恢复杠杆均 202（复验后还原）。Sync 正路径——test-one（Stopped）→ 卡 backup started → Job 6s Complete → 归档落 `felis-backups` PVC（167MB）→ `world_backups` 行 `present` → `/api/v1/backups` 可见。Sync 负路径——对 Running 选 → 友好 409 卡（不误烧冷却）。
-- **缺陷 #39（owner 席位可被静默复制，且不可清理）**：恢复模式下用非在位席位名做「reset」→ `UpsertOwner` insert 臂**铸出第二个 owner 行**、原席位继续存活；面板对任何 owner 行都删/降/禁 403 → 永久无法收敛回单席。修复 `55d515d`：`provisionOwner` 先查在位席位（新 `PGRepo.OwnerUsername`），非席位名 → `ownerSeatTakenError`（`Is api.ErrConflict` → TUI 路由回表单并**指名**应输入的用户名）；bootstrap（无席位）与同席位名复位原样。真机双验：新名被拒（表单原位显示 `an Owner already exists as "08595879-…" — enter that username to reset the Owner`）；改席位名复位成功（owner 恒 1 行、id/邮箱不变）；面板删除保护同步实证（对新 owner 行 DELETE → 403）。
-- **缺陷 #40（operator 撞名 = 裸 SQLSTATE，「换名重试」分支在真机从未生效）**：`InsertOperator` 冲突返回原始驱动错误（23505），TUI 却按 `api.ErrConflict` 判定「可恢复、换名重试」——fake 与 PG 漂移。真机复现：输入已存在用户名 → 控制台 exit 1 + 裸错误。修复 `55d515d`：`isUniqueViolation → ErrConflict` + pgint 契约断言。复验：撞名**回到表单**提示换名 → `drill-op-2` 成功（审计 `break_glass.operator_create` 落账，演练行已清）。
-- **缺陷 #41（Sync picker 死选项）**：picker 列出 login/lobby，而备份 API 对它们**永远失败**（保留名 + 无 servers 行）；真机选 lobby → 裸内部错误 + exit 1。修复 `ac3a557`：`backupPickable` 过滤系统服（halt picker 保留它们，断玻璃完整权力）。
-- **缺陷 #42（缺失世界盘 → 202 后静默卡死 30 分钟）**：对「从未启动/已被回收」的服备份或恢复：202 → Job → Pod `persistentvolumeclaim "world-<name>-0" not found` **Pending 至 deadline**，全程零失败记录。真机用已回收的 `resolvecheck` 复现（留证后删除）。修复 `508a1c0`：`Cluster.WorldVolumeExists`（直接 Get 与 Job 挂载**同名**的 PVC）+ 两 handler 409 `no_world_volume`（"start it once to create it, then retry"）。**修复首跑翻出配套 RBAC 洞**：felis-api SA 无 `persistentvolumeclaims:get`（403 被吞成 500「internal error」）→ `APIMinecraftRole` 补 get-only 规则 + rbac 测试锚点。复验（auditfix38）：resolvecheck 两面 409 + 友好文案；test-one 照常 202 → Job 10s Complete → 新行落库。
-- **缺陷 #43（409 一刀切文案）**：TUI 把所有 409 当停服门 → 世界盘拒绝会展示错误原因。修复 `ac3a557`：按 body 的 `error.code` 分流（无 code 的旧体仍按停服门）。复验：无盘 pick 显示 API 原文；停服门文案不变。
-- **同步完成**：① bootstrap 分支 scratch 库演练（`migrate up` 20 迁移 → 无菜单/无认证直接铸 owner；`local_auth_enabled=true`；审计 `break_glass.bootstrap`；exit 0；库/hba 规则/临时配置即测即清）；② 台面收敛：reaper CronJob `suspend=true/auditfix36/无 pin` → `suspend=false / felis:auditfix38 / nodeSelector=localhost.localdomain`；③ 镜像升级 `auditfix37→38`（api/operator + `FELIS_IMAGE`）。
-- **流程修正（教训）**：pgint 一度误用**本机 Docker Desktop**（启动 daemon + 临时 PG 容器）——已完全清理（容器/镜像删除、daemon 退出），并改为**经 ssh 隧道用 VM 的 postgres** 运行（`ssh -L 15433:127.0.0.1:5432` → `postgres://felis:***@localhost:15433/felis_pgint?sslmode=disable`）。勿再在本机跑容器。
-
-### 本轮新增真机证据（第二十六批：`felis setup` 重跑向导逐屏 + 构建链 pin 回验 —— 缺陷 #44）
-
-- **范围**：本机已装机，故覆盖"重跑状态屏 + c/s/e 三条 reconfigure 流"；首装屏（postgres/owner/connect/edge/storage/smtp/mc-bind/migration/preflight/summary）此前各批已有定点真机证据（安装闭环 / 第二~三批 / 第九批 / 第二十五批），本轮不重复。
-- **逐屏走查（auditfix38→39，VM tmux）**：
-  1. 重跑 → 直落状态屏「✓ Felis is already set up.」（owner/connect 不触碰；host bootstrap 已就绪跳过）✓
-  2. `c` → 三选一 chooser（Local / Cloudflare+Access / Reverse proxy + 警示语）渲染 ✓，esc 无损返回。
-  3. `s` → chooser 预选当前后端；S3 分支表单（Endpoint/Bucket/Region/AK/SK + 提示）渲染 ✓。**观察：reconfigure 非只读**——选「Local disk」即 apply（写 /etc 两文件 + 重渲染 felis-config + 滚 API）；从 S3 表单 esc 退回会把选择重置为 Local 预选。
-  4. `e` → SMTP 表单渲染 ✓；esc 直接回状态屏、零副作用（felis-smtp 未创建）✓。
-- **缺陷 #44（CLI·重跑框脱落）**：重跑后完成 `s`/`c` reconfigure，落回首装 summary「✓ Setup complete.」——丢了 alreadySetUp 框（smtp 路径有专门分支，storage/connect 漏）。修复 `abb5910`（`showSummary` 透传 `m.result.alreadySetUp`）+ 回归测试 `TestRootReconfigureStorageKeepsStatusFraming`。真机复验（auditfix39）：storage reconfigure 完成 → 「✓ Felis is already set up.」+ storage recap ✓。
-- **演练事故（自曝；环境 drift，非产品缺陷）**：首次 storage 走查意外触发 Local apply——它从 `/etc/felis/felis.pod.toml` 重渲染 Secret，而构建链 pin 值当初**只热补在 live Secret、不在 /etc 文件** → 重渲染清空 pin（放任则下次构建死在 `:latest` 拉取 + trivy DB egress）。当场修复：pin 值写回 `/etc/felis/felis.host.toml` + `/etc/felis/felis.pod.toml` → 重渲染 Secret → 滚 API；再做第二次 storage apply，重渲染后 pin 仍在（drift 修复耐久）。
-- **构建链回验（direct build，真机）**：Job 规格实证 `kaniko=gcr.io/kaniko-project/executor:v1.24.0`、`trivy=aquasec/trivy:0.74.0`、`--db-repository registry.felis.svc:5000/mirror/trivy-db:2`（registry 仍有 `mirror/trivy-db`）；`POST /images/build`（context=遗留 `sub-bf7dc18…` blob，内部面取件）→ 202 → **succeeded**；registry `e2e/pins-check` 落位；whitelist 条目已摘除（204）、Job 已清。
-- **文档修正（`ae6e925`）**：§8e 原「编辑 felis.toml 后重启 felis-api」不完整（API 挂的是 Secret）→ 改为「写进 /etc 两文件 → 重渲染 Secret → roll」，并写明三种无效/易损做法（只 restart / 只改 host 文件 / 只热补 live Secret——后者会在下次 reconfigure 被冲掉）。
-- 收尾：api 1/1、panel 200、tmux 全清。
-
-### 本轮新增真机证据（第二十七批：告警模块落地 —— 内部面 /metrics + 规则集 + 真实构建失败实弹演练）
-
-- **范围**：把"指标 → 规则 → 告警"链路从零补到可交付：API 内部面 `/metrics`（`94f71ee`）、`deploy/alerts/` 规则与 promtool 单测（`43df08b`）、真机实弹演练（本批）。
-- **/metrics（`94f71ee`）**：`felis_image_build_failures_total` 此前只在进程内存里、无任何 scrape 出口。修复：internal face（8081）新增 `GET /metrics`（服务面 Public 路由，语义同 /healthz）；单测断言外部面 404。真机：port-forward `svc/felis-api-internal 18081:8081` → 200 且含 `felis_image_build_failures_total`；operator `:8080` 提供 `felis_servers_total` / `felis_start_duration_seconds_*`。
-- **规则集（`43df08b`）**：`deploy/alerts/felis-alerts.yaml`（plain Prometheus）5 条——构建失败 increase>0 / 起服 p90>300s / 磁盘可用<15% / DiskPressure / 内存可用<10%；`felis-prometheusrule.yaml` 为 prometheus-operator twin（脚本比对两文件 groups 一致）；`felis-alerts_test.yml` 为 promtool 单测。VM 上 promtool 3.14.0 实跑：`check rules` + `test rules` 双 SUCCESS。
-- **实弹演练（真实构建失败 → pending → firing）**：
-  1. 打包含 `COPY does-not-exist` 的 Dockerfile 上传到 uploads PVC `sub-alertdrill` → `POST /images/build`（`e2e/alert-drill2:latest`）；
-  2. Kaniko `failed to get fileinfo for /context/does-not-exist` → Job Failed、build 行 `failed`；
-  3. 真实 Prometheus（宿主 `:19090`）scrape `127.0.0.1:18081`(api) 与 `:18080`(operator) 双 target up；`felis_image_build_failures_total{job="felis-api"}=1`；
-  4. `FelisImageBuildFailures` pending（activeAt 08:28:14Z）→ **08:33:14Z 准时 firing**（`for: 5m` 精确到期），labels/annotations 完整。
-- **清尾（残留全清）**：whitelist `e2e/alert-drill` 摘除（204）；`sub-alertdrill` 目录、`/tmp/drillctx`、两个演练 Job、tmux `prom`/`fwd`/`alertpoll`、`/root/prom-drill`（promtool+prometheus 二进制）全删；DB `%alert-drill%` 行删除（whitelist/build 复核 count=0）；宿主无残留监听/进程。演练期间 live api 进程内计数器=1（重启归零，属演练事实）。
-- **可达性追加**：#44 定级 ①（轻）——已装机环境重跑 `felis setup` 完成 storage/connect reconfigure 即触发。
-
-### 本轮新增真机证据（第二十八批：缺陷 #45 —— 审核门"盲批"：评审看不到将被执行的 recipe）
-
-- **缺陷 #45（构建 lane·审核语义）**：被执行的 Dockerfile 永远来自**上传上下文压缩包根目录的 `Dockerfile`**（`build/jobspec.go` 钉死 `--dockerfile=Dockerfile`），API 的 `dockerfile` 字段**仅审计存档**（`submit.auditDockerfile`、`build.Request` 注释均已声明）——但审核者没有任何路径能看到它：`GET /api/v1/submissions` 不含 blob 内容、面板只显示 `context_ref` 文本、内部面取件路由是 service-token（评审用不了）→ "人工审核是门禁"事实上是**盲批**。同批口径缺口：`POST /api/v1/images/build` 的 `dockerfile` 字段在 OpenAPI 里无任何说明（易被当成"将被执行"），面板表单也把该框呈现为"Dockerfile 内容 *"。
-- **修复（`168a375`）**：
-  - 新增 admin-tier `GET /api/v1/submissions/{id}/context`：评审下载与构建 Pod 同源同字节的 `context.tar.gz`；`Content-Disposition: attachment` + `nosniff`（攻击者提供的归档只下载、不渲染）；审计 `submission.context.download`（actor=评审者、target=submission id）。
-  - 内部面取件 handler 共享 `openSubmissionContext`/`streamSubmissionContext`（行为不变，原测试锁定）。
-  - OpenAPI：新路由 + `/api/v1/images/build` 字段描述补全（明说"执行的是 context 根目录的 Dockerfile；`dockerfile` 仅审计"）。
-  - 面板：SubmissionsPage 展开区新增「下载上下文」按钮（spinner/错误呈现，i18n en/zh）；ImageBuildPage 表单补审计说明行。
-  - 测试：`TestAdminSubmissionContextRoute`（流式/404/503）+ admin-only 矩阵加该路由 + OpenAPI parity 强制文档；面板 117 单测 + 构建、go vet/go test 全绿。
-- **真机验证（auditfix41 已部署；owner 会话经 op-login + 内部面代 approve 重铸）**：
-  - admin 下载 `sub-bf7dc18e9dd97dc2` → **200**，`attachment; filename="context.tar.gz"`、`application/gzip`、`nosniff`；sha256 `205496f2…` **与 uploads PVC blob 逐字节一致**；
-  - 内部面（Bearer=felis-service-token）同 blob → 200 + 同 sha256（重构未破坏构建取件路径）；无 token → 401；
-  - 有效玩家会话（console host 隔离，仅 adminOnly 生效）→ **403**；匿名 → 401；不存在 id（admin）→ 404；
-  - 审计落账：`audit_logs` = `[email protected] | submission.context.download | sub-bf7dc18e9dd97dc2`；
-  - 面板产物：服务端 index.html 引用新构建 `index-CwFSKpgW.js`，bundle 内含新按钮逻辑（grep 命中 3 处）。
-- **可达性追加**：#45 定级 ①——每一次真实的"用户提交 → 管理员审核"都会踩到（审核者此前无法查看将被执行的内容）。
-- **hook 链补齐（可复用）**：staff 账号走邮件登录门会被设计拒绝（refuse staff）→ owner 会话铸法：`op-login/start`（email=felis-owner@example.com）→ VM 日志 grep `email-otp` 取码（no-Mailer fallback）→ 内部面 `op-login/{id}/approve`（Bearer=felis-service-token；body `approver_uuid`=owner 的 mc_uuid）→ `op-login/finish`（curl -c 存 cookie）。
-
-### 本轮新增真机证据（第二十九批：镜像耐久落地 —— registry 托管 + 回环拉取路径；#46–#49）
-
-- **背景（结论清单第 2 条）**：磁盘压力演练证明 kubelet 会 GC 掉"当前无人使用"的镜像 → ImagePullBackOff，恢复依赖人工重导入。本批让镜像自愈：自建镜像全部托管进内建 registry，节点侧 pull 经回环 hostPort（节点 containerd 到 Service VIP 是死路，实测 "Empty reply"）。
-- **平台侧 `a9b275a`**：registry 容器端口加 `hostPort 127.0.0.1:5000`；同提交修 **#46** —— registry 独立资源模板（1 CPU / 2Gi）：旧模板 256Mi 在实测推 475MB 层时被 OOM kill（dmesg `oom-kill … registry, oom_score_adj=989`，上传中断），2Gi 下同一推送 2 秒完成。
-- **安装器侧 `13d64e0` + 文档 `fa0e8d7`**：自建镜像规范 ref = `registry.felis.svc:5000/felis/{felis,limbo,lobby,paper}:demo`；import 进 containerd 就用该名（首启命中本地，免 registry round-trip），`deploy_bundle` 之后统一 `push_images_to_registry` 入仓（推 `127.0.0.1:5000`；registry 只认主机名之后的路径 —— 推/拉落点一致）。`configure_registry_mirror` 写 `registries.yaml`（`registry.felis.svc:5000 → http://127.0.0.1:5000`），内容不变不重启 k3s；`import_registry_image` 预缓存 registry:2（重跑走跳过分支）。迁移 **0021** 把 recommended 白名单重指到 registry ref（线上实查两行已落）。
-- **真机三次重跑**（`/opt/felis/src` = 765a892 快照；`FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full FELIS_IMAGE=registry.felis.svc:5000/felis/felis:auditfix42`）：
-  - run1 失败 = **#47**：Mac tar 的 `._*` 旁文件混入构建上下文，`._0004_*.sql` 被 //go:embed → 新二进制的 `felis migrate` 报 `non-numeric version "."`，安装器停在 run_migrations。修复 `5fa8b74`（.dockerignore 排除 `._*`/`.DS_Store`）。复验方式：故意在暂存树留 `._zz_probe_junk.sql` → 重建后 `migrations applied`（过滤生效）。
-  - run2 失败 = **#48**：每个镜像一对 `systemctl start/stop docker` 触发 systemd 限流（`Start request repeated too quickly / start-limit-hit`），第 4 个镜像（paper）未入仓。修复 `c7e585e`（整批一次 start/stop，单测锚定）。run3 零 `[fail]`：4 镜像全部入仓（push digest ×4 实收）。
-  - run3 收敛实查：api/operator/reaper = `registry.felis.svc:5000/felis/felis:auditfix42`（registry ref 首滚命中本地 import）；registry 模板 1 CPU/2Gi 生效；reaper CronJob 同步换 ref 且 nodeSelector 保留；login/lobby Running 于 registry ref；迁移 applied。
-- **GC 演练（本批验收本体）**：
-  - A 控制面：`ctr images rm …/felis/felis:auditfix42` + `ctr content prune references` → 本地 ref 消失 → `rollout restart felis-api` → 事件 `Pulling` → `Pulled … Successfully pulled image … in 25ms`，ref 恢复、pod Running。
-  - B 游戏：rm `…/felis/lobby:demo` → 删 `lobby-0` → `Successfully pulled … in 10ms … Image size: 182357358 bytes`，Running。
-  - #46 复验：整轮重建 + 4 推送期间 dmesg `oom-kill` 计数不变（仍 2，历史）。
-- **构建 lane 复验**：`POST /images/build`（context=遗留 `sub-bf7dc18…`；ref `registry.felis.svc:5000/e2e/durable-check2:latest`）→ 202 → succeeded；registry `e2e/durable-check2` tags 落位；Job 事件见 felis/kaniko/trivy 三镜像就绪；白名单条目摘除（204）、Job 清。
-- **#49（同一条升级路径的第二类静默回退）**：`write_felis_toml` 重写整表（[registry] 仅 url+build_namespace、[archive] 仅 store+local_path）→ §8e 的 kaniko/trivy pin、构建上限、uploads 后端、[registry.s3]、reaper 的 retention/warn_before/max_local_bytes 在重跑时全部丢失（S3 安装切回 local、构建回退被 egress 拒绝的上游 executor）。修复 `765a892`+`b8e554d`（沿用 [smtp]/[[auth_source]] 的 carry 模式；url/build_namespace/store/local_path 保持安装器所有）。真机复验：预置 `retention = "30d"`，run3 后 host toml / pod toml / felis-config Secret 三层都在，且构建 lane 直接用 carry 的配置跑通（上条）。
-- **可达性**：#46 ①（用户构建大层或安装器入仓即触发；实测 475MB 层）；#47 ②（Mac 打包树构建路径，真机踩中）；#48 ②（安装器重跑，真机踩中）；#49 ②（升级=重跑安装器，真机踩中）。
-- **口径/遗留**：`demo-up.sh` 未改（dev/demo 路径，本地 tag 导入维持原状）；kaniko/trivy 编译默认值未动（§8e 改为"镜像进 registry"配方）；registry 2Gi 为渲染常量（暂未开 flag）；drill 残留：registry 里 `e2e/*` 小镜像留档，`durable-check2` 白名单条目已摘。
-
-### 本轮新增真机证据（第三十批：缺陷 #50 —— `[smtp]` carry 吞掉下一节的注释块，每次重跑 +1）
-
-- **发现路径**：为 `felis setup` 首装连续走查做前置盘点时读 live 配置——`/etc/felis/felis.host.toml` 已经堆了 **3 份**、`felis.pod.toml` **4 份**重复的 Yggdrasil 注释块（同一段文案逐次叠加）；用脚本自带的提取器实测：一次重跑会把 3 份全部当作 `[smtp]` 内容带走，再叠一份模板注释 → **每次重跑 +1、无上界**（host/pod 增速不同步，现场 3 vs 4 即历史残迹）。
-- **根因**：`persisted_smtp_block` 打印"`[smtp]` 到下一个 section header 之间"的**所有行**；generated 注释块正好落在这段区间里 → 被吞并。同文件的自称"cached on first call"缓存因写方是命令替换（子 shell）从未生效，pod 写实际上重复抽取刚被重写的 host，加剧了两文件的不同步。纯注释膨胀、无功能损失，但属 #49 同族 carry 语义缺陷（把不属于自己的内容也带走了）。
-- **修复 `4d3c85f`**：carry 改为白名单（section header + 键行），与 #49 的 `[registry]`/`[archive]` 提取同型；删掉失效缓存说明。`deploy/bootstrap_test.sh` 新增用例：配置值被携带 / 不吞注释行 / 不越节 / 写回后二次抽取**字节稳定**（幂等）。
-- **真机验证（auditfix43，两次连续全量重跑）**：
-  - run1（`bootstrap-auditfix43.log`）：零 `[fail]`；host 注释块 **3→1**、pod **4→1**；与 run 前快照 diff 恰为 21/31 行（全部是被删掉的重复注释）——值零漂移（kaniko/trivy pin、`trivy_db_repository`、uploads local、`[registry.s3]` 空、`retention="30d"`、auth_source 原样、空 `[smtp]` 节保留；两文件 db host 仍分别为 127.0.0.1 / 10.211.55.6）。
-  - run2（`bootstrap-auditfix43b.log`，紧接再跑）：零 `[fail]`，仍 **1/1** —— 收敛证明（旧代码此处会 1→2 继续增长）。
-  - 部署同步：api/operator = `registry.felis.svc:5000/felis/felis:auditfix43`；reaper CronJob 同 ref 且 nodeSelector 保留；registry `felis/felis` tags = auditfix41/42/43；panel 200；host 二进制已从新镜像提取。
-- **可达性**：#50 ②——配置过 email（存在 `[smtp]` 节）的安装，按文档升级=重跑安装器即触发；危害=配置注释无限膨胀（每次 +1），无功能损失。
-
-### 本轮新增真机证据（第三十一批：首装连续走查 + 缺陷 #51 —— 工作负载 `felis-config` 副本永不刷新）
-
-**一、`felis setup` 首装单次连续走查（队列第 1 项，完成；0 缺陷）**
-
-- **装置**：scratch 库 `felis_scratch`（新建 + `felis migrate up` 21 条迁移）+ scratch 配置（真实 pod toml 副本，仅换库名；root 0600）；hook 直插 `account_link_codes` 一枚绑定码（等价 `/link` 内网端点写入，代替"进服拿码"）；tmux 驱动 `felis setup -config`；k8s 只读复用（登录门 Ready 等待通过）。
-- **连续走查（一条会话走完）**：Preflight（自动：PG✓/迁移 21 applied/面板✓）→ **MC 绑定**（输码 → working →「✓ Owner account is ready.」+ 一次性 setup URL）→ **连接 chooser**（Local）→ **存储 chooser**（Local → working → ~20s 后「✓ Local storage configured.」，含 Secret 重渲染 + API rollout）→ **首装 Summary**（「✓ Setup complete.」+ owner/setup URL/access/storage/panel 卡片 + c/s/e 提示）→ **轨道回顾**（← 依次只读 recap Storage→Connection→Owner→Preflight，→/esc 回到前台）→ Enter 退出 → stdout 汇总（`Owner account … provisioned (passwordless)`、`Recorded as "root"`、setup URL、Admin console）→ **EXIT=0**。
-- **结果**：移动端/文案/切换全部符合设计，**0 新缺陷**；scratch 库侧复核：owner(role=owner) 1 行、绑定码已消费(0)、setup_tokens 1、`local_auth_enabled=true`。
-- **副作用与还原（如实记录）**：存储 apply 会把 `/etc/felis` 两文件重写为 Go encoder 形态（无注释、含空值键如 `[archive.s3]`，**值零漂移**：kaniko/trivy pin、uploads local、retention 30d、auth_source 原样），并重渲染控制面 Secret + 滚 API——这是该向导的既定行为；演练后按 pre 快照整文件还原（sha256 逐字节一致），两 ns Secret 重渲染复核一致，scratch 库/配置/hba 行/tmux 全部清理。
-
-**二、缺陷 #51（工作负载 `felis-config` 副本永不刷新）**
-
-- **发现路径**：上条还原核对时发现 minecraft ns 的 `felis-config` 是**旧形态**（encoder 式）而 felis ns 已是模板式 → 挖出 `ensureSecretReplica` 的「绝不覆盖既有副本」（凭据语义：防冲掉手工轮换值）把 **felis-config 也纳入只建不更**，而 bootstrap 只 apply 控制 ns。后果：改配置后（DB 凭据轮换、[archive] 保留策略调整、root domain 等）backup/restore/fileedit Job 与 reaper 永远读旧副本 → 静默失效（如备份认证失败）。
-- **红证据（auditfix43，真机）**：向 minecraft 副本注入 `# drill-51-stale-marker` → 运行 `felis setup` → 输出 `- config (minecraft ns): skipped (already exists)`；副本 sha `5ec2ff6e…` 保持，控制面 `e4791fe1…` 不同（陈旧坐实）。
-- **修复 `328e570`**：① setup 侧：`ensureSecretReplica` 增 `refreshExisting`（仅 felis-config 传 true）——源缺失降级 skip、内容一致 skip(`already current`)、不同则原地 Update；凭据类保持 create-if-absent；新增 `updated` 结果与「refreshed from the control namespace」文案。② bootstrap 侧：新增 `apply_felis_config_secrets()`，每次运行同时 apply 控制 ns + 工作负载 ns 两份（同渲染自最新 pod toml）。单测：Go 新增 4 例（陈旧刷新/一致跳过/空键补写/源缺失降级）；`bootstrap_test.sh` 新增 4 断言（两 ns apply、同一 pod toml 渲染、恰 2 次 apply）。
-- **绿证据（auditfix44，真机）**：A) 安装器重跑（标记仍在副本中）→ 日志出现 `secret/felis-config configured`（工作负载 ns 被刷新）→ 副本 sha 与控制面一致、标记 0；B) 再注入标记 → 运行**新** `felis setup` → 输出 `- config (minecraft ns): refreshed from the control namespace`、凭据仍 `skipped (already exists)`、副本 sha 一致、`EXIT=0`。部署=auditfix44（api/operator/reaper），panel 200，host 二进制随镜像 `docker cp` 刷新。
-- **可达性**：#51 ②——升级=重跑安装器、或改完配置跑 setup 即触发；旧行为下 backup/reaper 静默使用旧配置（DB 轮换后备份全挂）。
-
-**三、环境修复（非产品）**：现场 pg_hba 缺 `felis_pgint` 规则（按文档走 ssh 隧道跑 pgint 会 ident 失败）——补回 `host felis_pgint felis 127.0.0.1/32 scram-sha-256` 并复测连接成功。
-
-### 本轮新增真机证据（第三十二批：S3 存储向导逐屏走查收尾 + 缺陷 #52 —— 向导 apply 不刷新工作负载 `felis-config` 镜像）
-
-**一、S3 存储向导屏走查（队列第 2 项，完成；0 功能缺陷，走查自身暴露镜像缺口 → #52）**
-
-- **装置**：VM 宿主 MinIO 容器（`quay.io/minio/minio`，`felis`/`felis-drill-9000`，:9000）+ bucket `felis-wizard-uploads`；tmux 驱动 `felis setup` 重跑向导；行动前先留 preS3 快照（两 toml + 两 ns Secret + sha256）。
-- **负例**：错误凭据 → `✗ Could not save storage settings.` + `submit: s3 credentials rejected: The Access Key Id you provided does not exist`（`CheckS3Access` 预检先于一切写入）；**零副作用**（无 Secret、两 toml sha 不变）；`esc` 返回编辑时已填值保留（密钥掩码）✓
-- **正例**：修正凭据 → ~10s working → `✓ Object storage configured.` → Enter → 状态屏 `storage  s3://felis-wizard-uploads · http://10.211.55.6:9000` ✓
-- **落地核对**：`felis-uploads-s3` Secret（access_key_id/secret_access_key）；两 toml `user_uploads_context` + `[registry.s3]` endpoint/refs；API 滚动；启动日志仅既有的 smtp/jwks 警告 ✓
-- **功能链（batch24 同款）**：player.test 邮箱 OTP 登录（hook 取码）→ `POST /api/v1/me/submissions` 201 → context 上传 200 → MinIO 桶实见 `sub-853a4e2ba4ba6443/context.tar.gz` → 内部面取回 200 + tar 内容正确 ✓
-- **UI 回滚（`s` → Local）**：两 toml 归位（`/var/lib/felis/uploads`、`[registry.s3]` 归空）、控制面 Secret 更新、API 滚毕；与 preS3 快照的差异仅「s3 字段回环 + encoder 形态」（注释丢失属该向导既定行为）——**值零漂移** ✓
-- **观察（不计缺陷）**：切回 Local 后 `felis-uploads-s3` Secret 残留（演练按清理流程删除；是否自动清理属产品取舍）。
-
-**二、缺陷 #52（向导内 apply 只刷新控制面，工作负载镜像滞后到下一次运行）**
-
-- **发现路径**：回滚后按计划复核「两 ns 重渲染」——minecraft 副本仍为 S3 内容（`4fb80ed8…`），而控制面与两 toml 已 local（`df206074…`）。
-- **红证据（auditfix44）**：① S3 方向：S3 apply（20:18:44）后副本停在 local 内容（20:22:53 快照 = `e4791fe1…`）达 4 分钟；② Local 方向：回滚 apply（20:23:41）后副本停在 S3 内容。副本 managedFields 两笔写入（12:13:58Z `kubectl-client-side-apply` = 安装器 rerun 的双 ns apply；12:23:06Z manager `felis` = 下一次 setup 启动的 refresh）都不是 apply 时刻——**apply 本身不碰副本**。
-- **根因**：`applyFelisConfigSecret`（storage/connection/edge 三条 apply 的公共出口）只 apply 控制 ns；镜像刷新只存在于 `felis setup` 启动（#51）与安装器。email 路径早有显式镜像刷新，storage/connection 是漏网的两条。
-- **修复 `de7fb2c`**：镜像刷新移入 `applyFelisConfigSecret`（best-effort + stderr 警告；控制面-only 安装无工作负载 ns 时降级不阻塞）；smtp helper 去掉重复块。门禁：`gofmt`/`go vet`/`go test ./...`/`bootstrap_test.sh` 全绿。
-- **绿证据（`v0.0.0+fix52`，宿主二进制 sha `0bd49467…` 已装 `/usr/local/bin/felis`，旧版留 `/root/felis-auditfix44.bin`；真机双向）**：Run1 从 Local `s`→S3：apply 后 `control = mirror = podtoml = 4fb80ed8…`（S3 渲染；旧代码此刻镜像会停在 local）；Run2 `s`→Local：`control = mirror = podtoml = df206074…`；副本 managedFields 写者 = `kubectl-client-side-apply` @ 12:33:40Z / 12:34:59Z（正是 apply 时刻）。Run1 启动块另见 `config (minecraft ns): refreshed from the control namespace`（#51 机制照常先收敛一次旧账）。
-- **可达性**：#52 ②——任何 `s`/`c` 重配置即触发；危害等级低（镜像消费者 backup/restore/fileedit/reaper 当前不读被改动字段——`UserUploadsContext` 仅 `api.go` 消费——但「配置动了、镜像没动」正是 #51 要消灭的静默滞后类，且与 email 路径的既定行为不一致）。
-
-**三、清理与还原**：MinIO 容器/卷/两镜像、`felis-uploads-s3` Secret、DB 行（`image_submissions` `sub-853a4e2ba4ba6443`）、/tmp 残留（player-cookies/drill-ctx/svc-tok 等）全清；docker 停；收尾 `control = mirror = df206074`（ALIGNED）、API 滚毕 Running、panel/healthz 200 ✓。
-
-### 本轮新增真机证据（第三十三批：`/updates` 维护窗口 API+UI 走查全绿；#53 跟随仓库迁移；#54 update 升级指引全假）
-
-**零、仓库迁移（背景）**：remote 已改 `[email protected]:FelisMC/Felis.git`（本批两个修复随 `2c6739a`、`397a400` 直推 main）。带 token 实测：旧 `MliroLirrorsIngenuity/Felis` 路径 301（GitHub rename redirect）、新路径 200——旧坐标当前仍能工作但全靠 redirect。新仓库**尚无 stable release**（`releases/latest` 带 token 也 404）：felis-api 更新检查的 404 属发布流程事实，非代码缺陷。
-
-**一、`/updates` 维护窗口 API 走查（完成；全绿）**
-
-- 读：GET 未设置 → `{"not_before":null,"not_after":null}`。
-- 负例全按预期拒绝：半设 / 倒序 / 相等 / 坏 JSON / 未知字段 → 400；缺 `Content-Type` → 415。
-- 正例：写入 → 回读一致 → **API pod 重启后仍在**（已落 DB，非内存态）。
-- 鉴权：无 cookie → 401；player 会话 → 403（新铸 player 会话，留 `/tmp/player-cookies.txt`）。
-- 收尾：DB 回到 `{null,null}`。
-
-**二、`/updates` 面板 UI 走查（CDP，完成；全绿零 console 错误）**
-
-- 状态流转逐屏：生效中 → 过期 → 计划 → 未设置（截图 `/tmp/updates-{1..5}-*.png`）。
-- 倒序提交 → 校验文案正确；清除后表单清空 + 「未设置」提示。
-- 核账：DB 收尾 `{null,null}`；审计 `updates.window_set` 恰 3 笔（20:46:18 / :20 / :21）。
-- 驱动：`/tmp/cdp-updates2.js`（bun + 原生 WebSocket；须 `Network.setCookie` 注入 owner 会话，否则新标签页 401 跳登录）。
-- 观察（不计缺陷）：窗口自身存取已验证；「窗口被 runner 消费」的端到端链路（Applier/Notifier）仍属 INTEGRATION-ONLY 设计（deferred-seams），不要当缺陷重复修。
-
-**三、#53（旧仓库坐标残留）—— 提交 `2c6739a`**
-
-- 红（`v0.0.0+fix52` 实机）：`felis update` → `github: MliroLirrorsIngenuity/Felis releases/latest returned HTTP 404 — …`。
-- 绿（`v0.0.0+fix53` 实机）：同一命令 → `github: FelisMC/Felis releases/latest returned HTTP 404 — …`。
-- 范围：`internal/updater/topology.go`、`deploy/bootstrap.sh`（默认 `FELIS_REPO_URL` + 两处 UA）、README ×2、两个测试夹具（6 文件 9 处）；门禁四件套全绿。
-- 可达性：②——默认安装 / 每次 `felis update` 都读该坐标；危害在 redirect 退休时兑现（安装与更新一起挂）。
-
-**四、#54（`felis update` 的 apply 指引在既有安装上全是错的）—— 提交 `397a400`**
-
-- 红（`v0.0.0+fix52` 实机文本）：`--panel --force` → `run: sudo felis setup` + “felis setup is idempotent and re-runs the installer…” + felis-api「例外」块。
-- 决定性证据（完成态安装）：`felis setup` 跑 8s 退出，`/opt/felis/velocity/velocity.jar` mtime/hash 不变、零 bootstrap 输出；同刻安装器重跑日志有 `resolving the newest Velocity 3.5.1 build`。代码侧 `shouldRunHostBootstrapBeforeConfig` 仅在 4 个 marker 不全时进 bootstrap——既有安装上 `felis setup` 只开配置 TUI。
-- 绿（`v0.0.0+fix54` 实机；宿主二进制已换 `/usr/local/bin/felis`，fix52 留档 `/root/felis-fix52.bin` sha `0bd49467…`）：
-  - `--panel --force` / `--velocity --force` → `run: curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash` + 单条 trailer（channel 未持久化 caveat、私有仓库 token'd form、`felis setup is not this path`）；
-  - `--mc` 无命令无 trailer；`--all` trailer 恰一次；`--force`/`LatestKnown` 语义不变。
-- 文案同步：`docs/troubleshooting.md §15` 删掉 “or `sudo felis setup`”、补 channel caveat；测试改为 `TestApplyGuidancePointsEveryComponentAtTheInstaller`（钉安装器路径 + 禁 `run: sudo felis setup`）。
-- 可达性：②（文档/指引；同 #38 型）。
-
-### 本轮新增真机证据（第三十四批：`felis nano` 全链走查 60/60；缺陷 #55 —— 坏源静默挡住验证梯子）
-
-**零、装置（全部在 VM，不进产品环境）**：`/srv/nanotest/nano-stub.py`＝可控 Yggdrasil 源，路径即行为——`/ok/<name>`、`/okid/<name>/<id>`、`/props/<name>`、`/badname/<case>`（tooshort/toolong/badchar/section/noname）、`/none`=204、`/boom`=500、`/redir`=302、`/garbage`、`/emptyid`、`/slow/<secs>`；跑在 127.0.0.1:9900，临时单元 `nano-stub`，请求日志 `/srv/nanotest/stub.log`。驱动脚本：`nano-matrix.sh`（HTTP 矩阵 S1–S7）、`nano-service.sh`（systemd 层）、`nano-config.sh`（配置校验）、`nano-fw.sh`（firewalld）；`nano-harness.sh` 从 HEAD 版 `bootstrap.sh` 提取产品函数在 scratch 路径跑（`felis-nano-test` 改名件，不碰产品单元）。
-
-**一、HTTP 矩阵 S1–S7（60 项断言）**
-- 红（修复前 `v0.0.0+audit-nano1`）：`SUMMARY pass=47 fail=13`——13 红全在 #55 语义内：5 个坏名用例「应 503+日志、实得静默 204」（状态+日志 ×2＝10），failover 3（坏源在前登录不落第二源 ×2、无日志 ×1）。留档 `matrix-red.r2.log`。
-- 绿（fix55，同装置重跑）：`SUMMARY pass=60 fail=0`。留档 `matrix-fix55.r2.log` 与 `matrix-fix55/`（更早原跑在 `matrix/`）。
-- 覆盖面：S1 输入校验（缺参/超长→204、POST→405、未知路径→404、带体 GET→400、HEAD→204、400 带 `Connection: close`）、S2 三方登录全链（前缀 + felisAuthNS UUID 确定性、ip/serverId 转发保真）、S3 premium 撞名改名（`LS_`）与免费名不动、S4 失败模式（不可达/500/302/garbage/204 → 跳过、记日志、503）、S5 多源优先级 + 坏源后 failover、S6 日志纪律（长 URI 截断、控制字节转义）、S7 properties 转发。
-
-**二、systemd 层（`nano-service.sh`，真 unit + 真 DynamicUser；重跑落盘 `nano-service.r2.log`，0 FAIL）**：装服务→起服务；重跑语义（配置字节不动、`resolve_nano_listen` 从 unit 读回端点）；drain（stop 期间 in-flight 登录照样答完，实测 stop 等待 5335ms）；SIGKILL 后自动重启；端点迁移 8081→8099 后新址应答、旧址关闭；三种启动文案（loopback「Bound to loopback」/ CIDR「admits」/ 无 CIDR「WARNING」）。
-
-**三、配置层（`nano-config.sh`，LoadNano 16/16，落盘 `nano-config.r2.log`）**：未知键、空/重复/保留 tag、colon、空白、坏 prefix、重复 prefix（大小写不敏感）、非 http scheme、URL 带 query、明文公网 http、缺文件、`[server] listen` 忽略警告、Mojang-only 可服务。
-
-**四、firewalld（`nano-fw.sh`，3/3，落盘 `nano-fw.r2.log`）**：关旧「开全源」8081/tcp、仅对 CIDR 开 rich rule、loopback 不开洞、无 CIDR 警告、快照清理还原。
-
-**五、缺陷 #55（提交 `9dad61f`）**
-- 红（fix54 实机）：配置源回 200 但档案字段不可用（三方源名字进不了 MC 字符集；身份源 UUID 不解析）→ 静默 204、零日志；且筛查在 `handleHasJoined` 直接 `return` → [坏源, 好源] 顺序下 204，好源根本没被询问。
-- 根因：两个 200-后筛查在 handler 层（200 已赢下梯子之后），而同类坏答案（200 无档案/非 200/不可达）在 `resolveHasJoined` 里是「记日志 + skip + failed」——一致性缺口。
-- 修复：筛查移入 `resolveHasJoined`（身份源 `uuid.Parse`、三方源 `mcUsernameRe`）→ 命中记日志（`unusable profile name` / `unparseable profile id`）+ `failed=true` + `continue`；handler 守卫保留为最后防线（注释更新）。单测 +3（含 `log.Writer()` 捕获断言）。
-- 可达性：#55 ②——需要配置了一个「回 200 但字段不可用」的源（马虎的自建/三方 Yggdrasil）；坏源在前时经梯子的登录全部被吞（静默、零日志），安全侧（坏名不落玩家列表）保留。
-
-**六、观察（不计缺陷）**：premium 冷查询 fail-closed 抖动——api.mojang.com 从 VM 首查偶发接近 2s 超时 → 偶按「疑似 premium」给免费名加前缀（`FelisNanoStub1` → `LS_FelisNanoStub`）；符合 `isPremiumName` 既定取舍（误加前缀=外观代价，误放行=抢名），记观察不修。
-
-### 本轮新增真机证据（第三十五批：NetworkPolicy 真机强制矩阵全绿 —— 端到端白名单闭环 + felis-velocity 刷新存活验证）
-
-**零、装置与现场**：三张策略 live 于 `minecraft` ns（`felis-default-deny-ingress` / `felis-allow-rcon-from-control-plane` / `felis-allow-game-from-velocity`；apply 于 09-22T05:37:24Z，与渲染收敛 diff 零漂移）；k3s 参数无 `--disable-network-policy`，**强制执行实测生效**；kube-router 机制实证：per-pod `KUBE-POD-FW-*` 链、未标记流量 `REJECT --reject-with icmp-port-unreachable`、每链首条 `--src-type LOCAL -j ACCEPT`（本节点豁免）、ipBlock 落 ipset `KUBE-SRC-*`（白名单成员可直接查）。探测法：**nsenter 进真实 pod 网络命名空间（源 IP=真实 pod IP）+ netns/veth 合成「转发型外部源」10.99.0.2（模拟 velocity 在另一台机器）**；全程不改产品代码。
-
-**一、pod 源矩阵（源=真实 pod netns）**
-- api（felis，命中 RCON 白名单标签）→ lobby:25575 = **OPEN**；同源 → lobby:25565 = **REFUSED**（端口级区分 ✓）
-- registry（felis，非匹配）→ lobby:25575/25565 = **REFUSED**；同 pod → api:8081 = OPEN（对照：无策略命名空间不受限）
-- coredns（kube-system）→ login:25565 = **REFUSED**
-- 收尾复跑全矩阵与首轮逐行一致（`np-matrix-run1/2.log`）。
-
-**二、host（=velocity 同机侧）**：→ login/lobby 的 ClusterIP 与 podIP :25565 **全 OPEN**（velocity 注册的后端路径实际可用）；→ api-internal:8081 OPEN；→ lobby:25575 亦 OPEN——归因：kube-router 每 pod 链的 `--src-type LOCAL -j ACCEPT`（**本节点流量豁免，kube-router 设计行为**，kubelet 探针等依赖它；netpol 语义无法对节点自身收口，节点 root 本在 TCB 内）。**注记①：node-local 豁免。**
-
-**三、转发型外部源（最严苛模拟）**
-- 基线：netns(10.99.0.2) → 全部服务器端口（podIP 与 ClusterIP、25565/25575）= REFUSED，包级可见 netpol 的 icmp-port-unreachable（`td3.log`）。
-- **白名单闭环**：临时把 10.99.0.0/24 加入 allow-game → ipset 即时生效（members：`10.99.0.0/24` + `10.211.55.6`）→ **lobby-svc:25565 与 login-svc:25565 = OPEN**（ClusterIP=velocity 实际拨号形态）；25575 仍 REFUSED（端口维度不破）→ 回滚 → spec 哈希逐字节一致（md5 `e6b07440…`）、ipset 复原、复测全部 REFUSED。
-- 层间归因：firewalld 规则含 `ct status dnat accept`（**DNAT 后的服务流量放行**；非 DNAT 转发走 forward policy 的 `admin-prohibited` 拒绝）——受支持拓扑（velocity 同机=LOCAL、ClusterIP、Mac→NodePort 面板 200）全部实测可用；「外部未经服务直连 podIP」不属于任何产品流。**注记②：firewalld 只放 DNAT 服务流。**
-- 装置保养：veth 未归区时 firewalld 会拒其转发属装置噪声（已归因）；public 区临时挂载已摘、netns/veth 已删、策略零残留（spec diff 为空）。
-
-**四、felis-velocity 刷新循环存活（顺带验证）**：20:18–20:42 的 `server list refresh failed` 全部落在 #52 演练的 API rollout 窗口（成功不打日志属设计）；tcpdump 450s 窗口抓到 52 条 `GET /api/v1/servers` 载荷行（成对出现=双抓包点看到同一请求，折算约 26 次 ≈ 每 15–17s 一次，与 `REGISTRATION_REFRESH=15s` 常量吻合）及对应 200 响应；"keeping current registrations" 为设计降级。非缺陷。
-
-**五、留档（VM `/srv/npdrill/`）**：`np-matrix.sh`+`np-matrix-run1/2.log`、`np-netns.sh`、`netns-probe.py`、`np-cidr-test.sh/.log`、`spec-before/after.json`、`td3.log`（包级归因）、`tcpdump-8081.log`（刷新存活）、`iptables-save.txt` 与 `nft-rules.txt`（现场快照）。
-
-### 本轮新增真机证据（第三十六批：面板错误文案全量本地化（#60）+ Run4c 管理面写操作复核 0 缺陷）
-
-**零、现场**：三个缺陷批次（#56–#59）已按「一缺陷一 commit」推 main 并部署 `auditfix59`；本批完成 #60 后部署 `auditfix60`（api/operator/reaper 三处 + 宿主 CLI，`felis version` = v0.0.0+fix60）。面板门禁：typecheck 0 错、vitest 117/117、vite build 通过（注意项目测试是 `bun run test`＝vitest run；裸 `bun test` 是 Bun 内置 runner，解析不了 `@/` 别名，41 个 fail 系假象，勿误报）。
-
-**一、Run4c 管理面写操作复核（4 段全真机，最终 0 缺陷；3 处为测试脚本自身误报，均已自纠）**
-1. **镜像添加+删除**：添加 `registry.felis.svc:5000/e2e/probe:1` → 列表出现、API 落库（`added_at=14:50:53Z`）✅。删除初测"未生效"系脚本用 `tr` 找行——该表是 `div` 网格，`tr` 选择器命中 0 → 从未点到按钮。换 `div[class*=grid-cols-12] → button[title]` 重测：confirm（"确定删除此镜像吗？"）自动接受 → 页面 1ms 内刷新、**API 侧同 ref 从 5 条降至 4 条**——删除通道无缺陷。
-2. **构建负例**：外部 registry（`docker.io/...`）+ 不存在 context → 对话框红字拒绝 `build: invalid request: image reference … must target the internal registry "registry.felis.svc:5000"`（截图 `run4c-3`）。脚本"未捕获"系其正则先命中了侧边栏"镜像"二字——误报。
-3. **创建重复用户**：对话框内正确显示 **"该用户名已被使用。"**、对话框保持打开（截图 `run6-1`）。此前 run4c 的"dialog-closed"系脚本点错页面（进到了用户详情页）；且该轮实际是一次**正例**：真管理员的 username 是 UUID 串（`08595879-…`，role=`owner` 是角色名），字面用户名 `owner` 当时空闲、创建确实成功——测试件已删（DELETE→200，列表 7→6）。
-4. **创建重复子域名**：正确拒绝 **"该子域名已被占用。"**（截图 `run4c-5`）✅
-
-**二、缺陷 #60（提交 `c9af548`）：37 个用户可达错误码显示生英文**
-- 发现路径：run4c 复核 `email_taken` 时做了一次系统性对照——后端 `newError` 共 **71 个错误码**，面板 `humanizeError` 只映射 **23 个**；其余走 `default` 分支直接透传 `err.message`（多为 Go 包装嵌套的英文，如 `invalid server name: naming: invalid server name "BadName": must match ^[a-z0-9-]{3,32}$`）。
-- 三个代表真机验证（修复前→修复后）：
-  - `bad_name`：新建服务器"名称"填 `BadName`（前端只查非空）→ 前：对话框直出上述嵌套英文；后：**"服务器名称不合法：需为 3–32 位小写字母、数字或连字符，且不能使用保留名。"**（`run8-1`）
-  - `bad_subdomain`：子域名填 `ab`（前端 RE 允许 1–2 字符，后端要求 ≥3）→ 后：**"子域名不合法：…"**（`run8-2`）
-  - `email_taken`：DB hook 造"他人已验证邮箱" → 账户页真发码（no-mailer 日志取码）→ verify 409 → 后：**"该邮箱已在其他账户上完成验证；请直接用该邮箱登录，或换一个地址。"**（`run9-1`）
-- 修复：api.ts +37 case、zh/en errors.json 各 +37 键（纯映射，无逻辑变更）。**剩 11 个故意不补**：`bad_request`/`conflict`/`internal`/`panic`/`not_found`（通用兜底）、`forbidden`/`not_admin`/`unauthorized`（状态码分支已在 default 覆盖）、`not_ready`（内部面）、`setup_required`（bootstrap 期 SPA 流控信号，403 文案可用）、`unsupported_media_type`（CSRF 门，面板永不可达）。
-- 遗留观察（不计缺陷）：构建负例的英文前缀 `build: invalid request:` 仍会透传——该错误无独立码、message 即最终文案；对管理员可读，记观察。
-
-**三、观察（不计缺陷）**
-- UserDetailPage 对 owner/对自己都显示删除按钮；真的点了会得 403 + 具体 message（"the owner account cannot be deleted from the panel"），但前端 403 分支统一显示"你无权执行此操作"——笼统但不算错，记观察。
-- 字面用户名 `owner` 可注册（用户名无保留名单；角色由服务端管理、无提权路径），记观察。
-
-**四、留档（Mac）**：脚本 `/tmp/cdp-run5.js`（镜像删除复测）、`run6.js`（重复用户复测）、`run7/run8.js`（bad_name/bad_subdomain 前后对照）、`run9b.js`（email_taken，内置 ssh 取码）；截图 `run5-1`/`run6-1`/`run7-1/2`/`run8-1/2`/`run9-1`。VM `/opt/felis/src` = `c9af548` 快照。
-
-### 本轮新增真机证据（第三十七批：管理面交互收尾 0 缺陷 + 缺陷 #61 —— LuckPerms 写操作的过度承诺）
-
-**零、现场**：`auditfix61`（api/operator/reaper 三处 + 宿主 CLI）。面板门禁 typecheck/vitest 117/build 全绿后部署。
-
-**一、管理面交互收尾（3 项全真机，0 缺陷）**
-1. **submissions approve/reject**：player 会话（邮箱 OTP 铸造，no-mailer 日志取码）现场造两条 pending（`sub-489eda47fe4bc12d` / `sub-b1dcc4b4a2cc5f16`，各上传 tar.gz context）→ 面板「通过」→ 状态变「审核通过」且**自动构建 `bld-1790176562566802097` 到 succeeded**（点击到构建完成全链闭环）；「驳回」→ 对话框必填原因 → 状态变「已拒绝」。全程 UI 零报错。
-2. **用户会话撤销**：player.test 铸 2 条新会话（共 4 条）→ UI「单独撤销」首条 → **精确生效**（cookie5 → 401、cookie4 → 200、列表 4→3）；「全部撤销」→ 提示「所有会话已撤销。」、列表清空、cookie4 也 401。
-3. **ServerCard 启动交互**：面板点 test-one「启动」→ `Starting` → `Running/ready`。
-
-**二、缺陷 #61（提交 `b4ef42d`）：LuckPerms 页对写操作"过度承诺"**
-- 发现路径：LP 页写操作交互测试（给 `E2E_Tester` 写权限）→ 历史卡显示绿色 success +「（服务器未返回输出）」→ **落盘取证发现根本没写入**。
-- 真机实验矩阵（test-one，LP 5.5.85 / H2 存储）：
-  | 操作 | RCON 回包 | 落盘（`lp export` 实测） |
-  |---|---|---|
-  | 面板 UI：`E2E_Tester permission set e2e.ui.write.test` | 空 | ❌ |
-  | 面板控制台：`E2E_Tester … set probe.test true` | 空 | ❌ |
-  | 面板控制台：`<UUID> … set probe2.test true` | 空 | ✅ |
-  | **独立 Python RCON 客户端**（绕开 felis）：`E2E_Tester … set probe3.test` | 空 | ❌ |
-  | 独立客户端：`<UUID> … set probe4.test` | "Another command…"（异步提示） | ✅ |
-  - 结论：**felis 只如实转发空回包；名字写入的静默失败是 LuckPerms 自身行为**（独立客户端 1:1 复现）。
-- 根因（LP config.yml 官方注释背书）：`use-server-uuid-cache: false`（LP 默认）→ "commands using a player's username will not work **unless the player has joined since LuckPerms was first installed**"——**未进过服的玩家名永远无法解析**（与有无外网无关）。
-- 面板问题：旧文案承诺"授予与撤销操作仍然会实际生效"（#59 遗留半句）→ 对"给还没来过的玩家预授权"场景是不成立的承诺。
-- 修复（纯文案，双语）：`luckperms_no_reply` → "……按玩家名的操作只对「自 LuckPerms 安装以来进过本服」的玩家可靠，对没进过服的玩家名可能静默不生效"；`luckperms_no_output` → "（服务器未返回输出，无法确认结果）"。真机复验（`run16b`）：读提示与写占位均按新文案显示。
-- 可达性：#61 ①——给未进服的玩家预授权是日常动作，此前面板显示绿色成功构成误导。
-- 局限（记观察）：读提示块只在"读取为空"时渲染；已有部分玩家数据的服上不会露出这段说明（后续增强候选：常驻说明或写前对未解析名字的提示）。
-
-**三、观察（不计缺陷）**
-- LP 命令是**串行异步**执行：快速连发第二条会得到 `§7[§b§lL§3§lP§7]§r §7Another command is being executed, waiting for it to finish...`（带颜色代码原文，面板如实显示）；`lp export` 回包时有时无（响应与执行解耦）。felis 层无责。
-- submissions 徽章（`submissions.json`："审核通过/已拒绝"）与过滤标签（`admin.json`："已通过/已驳回"）两套词，语义均可，记观察。
-- 控制台对空回包命令只显示 echo、无占位提示（终端风格）；LP 页有占位文案。记观察。
-
-**四、留档（Mac）**：`/tmp/cdp-run10.js`（approve/reject）、`run11a/11b.js`（会话撤销）、`run12a.js`（启动交互）、`run13/14/15/15b/16/16b.js`（LP 全链）。VM 独立探针：`/tmp/rcon_probe.py`、`/tmp/rcon_one.py`（+ `/tmp/rcon_pw.txt`）；LP 导出样本在 `/data/plugins/LuckPerms/luckperms-2026-09-23-15-*.json.gz`。测试数据：两条 pending 提交（一 approved 一 rejected，构建 succeeded/failed 各一）。VM `/opt/felis/src` = `b4ef42d` 快照。
-
-### 本轮新增真机证据（第三十八批：#56–#59 证据回填 + 服务器详情/运维面收尾 0 缺陷）
-
-**零、现场**：`auditfix61`（api/operator/reaper 三处 + 宿主 CLI，`felis version` = v0.0.0+fix61）。本批两部分：把 #56–#59 四项修复的真机证据落节（此前只存在于 commit message），并把队列里剩余的服务器详情页/运维面交互复跑做完（0 缺陷）。
-
-**一、缺陷 #56–#59 证据回填（部署 `auditfix59`；四项均 = 修复前真机触发 + 修复后复验）**
-
-1. **#56（`0790f8d`）玩家管理操作把 RCON 回包扔掉、只报 canned 成功。** 触发路径：给"还没进过服"的玩家加白名单/封禁——vanilla 对没见过的名字回 `That player does not exist` 并拒绝，而旧面板无论如何都显示成功。修复后回包逐字上屏：`whitelist add E2E_Bad` → `Added E2E_Bad to the whitelist`（磁盘同步真写入）；负例 `NoSuchPlayerXYZ` → `That player does not exist`（不再伪装成功）；静默服务器回退本地化文案。
-2. **#57（`70c988e`）控制台命令的回复无处显示。** 触发路径：控制台发任何命令——`sendCommand` 拿得到 RCON 回包但被丢弃、pod 日志也不回显命令输出，等于零反馈。修复后 echo + 回包以终端样式渲染在提示符上方（实测 `list`）。
-3. **#58（`4d4cdd6`）无世界盘服务器的文件页 90s 卡死。** 触发路径：对从未启动/已回收的服点"文件"页——旧行为建 Job → Pod `FailedScheduling (pvc not found)` Pending 到 90s 超时 → 误导性 504 `files_timeout`。修复：file 路由补上与 backup/restore 同款 `WorldVolumeExists` 门 → 快速 409 `no_world_volume` + 面板双语文案。本批现场复查：`resolvecheck`（无 world PVC）→ 409 `no_world_volume`，0.03s；`test-one`（有盘）→ 200（~2s）。
-4. **#59（`a2ff2a1`）LuckPerms 页把"读不到"报成"没有"。** 触发路径：打开装了 LP 的服的管理页——LP 5.5.85 的 `lp` 命令 RCON 回包全为空（独立 RCON 客户端 1:1 复现），读投影永远为空，旧页面却断言"没有父组/没有显式节点"（假事实）、写历史伪造 `[RCON]` 行。修复：原始回包随 rosters 同款披露渲染；空回包显式提示、不再假断言；历史占位不再伪造输出。
-
-**二、服务器详情页交互复跑（5 项全真机，0 缺陷）**
-
-1. **ServerCard 停止**：test-one `Stopping` → `Stopped`（启动在第三十七批；收尾复查 `desiredState: Stopped`）。
-2. **ServerFiles 写流程**：编辑 motd → 保存「已保存」→ 重开读回一致（`motd=Felis E2E files drill`）→ 还原默认 `A Minecraft Server`（收尾复查确认）。
-3. **备份/恢复**：立即备份 → Job `succeeded`；恢复 → 确认对话框（破坏性警告原文）→ Job `restore-test-one` `Complete`（5s）→ UI「恢复 27秒钟前 成功」→ 唤醒 `Running/ready`（恢复后的世界可加载）。
-4. **白名单**：加/负例/移除全链复跑（证据见 #56）——终态磁盘只剩 `E2E_Tester`（88 字节）。
-5. **封禁/解封**：封禁（内联确认）→ `Banned E2E_Bad: Banned by an operator.` + 磁盘写入；解封 → `Unbanned E2E_Bad`；`banned-players.json` 终态 `[]`。
-
-**三、运维面复查（3 项，0 缺陷）**
-
-1. **`/admin/updates` 复跑**（队列"深挖"项）：设置窗口 → 「维护窗口更新成功。」+ 状态卡更新；负例 end<start → 前端「结束时间必须在开始时间之后。」、后端 400 `end must be after start`；清除 → 「当前未设置维护窗口」。
-2. **metrics 端点**：内部面 `/metrics` 正常——`felis_*` 样本 18 条 / 3 个指标族（build 失败计数、回收计数、起服时长直方图）。
-3. **CLI 覆盖核销**：`run` 分发表 17 项（15 个用户面命令 + `bootstrap-assets`/`init-forwarding` 两个容器内部入口）在历批演练中均已有真机记录，本批逐项核销无遗漏。
-
-**四、接口语义注记与收尾**
-
-- 文件 API 的 `path` 是**相对路径**：空串 / `.` = 根（正常列出）、`config` 下钻正常；字面 `/` 被路径约束拒绝（400 `bad_path: path escapes from parent`）——面板从不发绝对路径（`joinPath` 只拼相对段），无用户面影响；本批复查脚本初次误用 `/` 时曾见 13s 延迟，属首次 fileedit Job 冷启动，非卡死。
-- 收尾静止态：test-one `Stopped`；三个名单 = whitelist `E2E_Tester` / banned `[]` / ops `[]`；`E2E_Bad` 仅余 latest.log 与 usercache.json（日志与 Mojang 缓存）。
-- 留档（Mac）：脚本 `/tmp/cdp-run17a/17b/17c`（停止、写文件、还原）、`run18/18b/19`（备份、恢复对话框、完成等待）、`run20a/20b`（白名单加/减）、`run21/21b/22`（封禁重试、封禁、解封）、`run23`（维护窗口）＋同名 `-out.json`；截图 `run17b-1`、`run18-1..4`、`run18b-1,2`、`run19-1`、`run20a-1,2`、`run20b-1`、`run21b-1`、`run23-1..3`。
-
-### 本轮新增真机证据（第三十九批：reaper 多节点实机 —— VM 克隆双节点验证）
-
-**零、装置**：`prlctl clone "CentOS Linux 9 Stream" --linked --name felis-node2`（linked 克隆，初始 1.4M）→ node2 改 host 名 `felis-node2`，停用/禁用 `felis-velocity`、`k3s.service`（server 形态）、docker，以 k3s-agent 加入同一集群（`https://10.211.55.6:6443`，复用 node1 node-token）。克隆副作用 = node2 自带 node1 storage 副本（6 项 / 3.1G）——已移开，模拟真实新节点。两节点均 Ready（v1.36.4+k3s1）；为克隆，node1 经历一次正常重启，重启后组件/服务全回归（docker 随自启后又停回 `inactive`）。
-
-**一、pin 正向**：`kubectl create job reaper-b39-ok --from=cronjob/felis-reaper`（live 模板原样，nodeSelector=`localhost.localdomain`；node2 无 taint、是合法调度候选）→ pod 落在 `localhost.localdomain`（nodeSelector 命中；backups PVC 的 affinity 亦指向同节点——两者本就该同节点），4s 跑通：`evaluated=2 reaped=0 warned=0 skipped=0 evicted=0 expired=0`、Job `Complete`。即：渲染出的 pin 在真实双节点集群把 reaper 钉在持盘节点。
-
-**二、错位 pin 反向对照**：同模板把 nodeSelector 改成 `felis-node2` → pod 停在 **Pending**，scheduler 事件原文：`0/2 nodes are available: 1 node(s) didn't match PersistentVolume's node affinity, 1 node(s) didn't match Pod's node affinity/selector`——node1 被错位 selector 拒绝、node2 被 `felis-backups` PVC 的 volume node affinity 拒绝（PV `pvc-0b4fbda6-…`，nodeAffinity=`localhost.localdomain`、hostPath=`/var/lib/rancher/k3s/storage/pvc-0b4fbda6-…_minecraft_felis-backups`）。**结论：本部署形态下 pin 写错是 fail-closed（卡住 + 明确调度事件），不会在无世界节点上静默执行**；真正要防的是首跑顺序（全新多节点安装时，第一次 reaper 运行会把 backups PV 落在其所在节点）——这正是渲染默认要求 `--reaper-node` 并 stderr 警告的原因，维持现状不修。
-
-**三、装置回收**：drill job ×2 删除（minecraft ns 无残留）；node2 关机 → `kubectl delete node felis-node2` → `prlctl delete felis-node2`（VM+克隆件删除）。收尾 `get nodes` = 单节点 `localhost.localdomain`，felis/游戏 pod 全 Running，CronJob `suspend=false` + nodeSelector 原样，docker `inactive`。重启副作用按既有说明处理：Mac 侧 443 入面板依赖的手工 `socat` 中继（本台账开头注记「重启 VM 后需重开」）随重启消失——已按原样重开（`TCP6-LISTEN:443,ipv6only=0,reuseaddr,fork TCP:127.0.0.1:30443`）并复核 op.console 面板 / api-me / player 面板皆 200。
-
-**四、留档**：node2 agent 上线日志（`k3s agent is up and running`、VXLAN subnet event 来自 10.211.55.6）；两向 job 的 pod/调度事件原文（见上）；`/root/node2-storage-copy/`（3.1G，随 VM 删除）。
-
-### 本轮新增真机证据（第四十批：Java 插件层收口 —— #63 demo-up 单起点、CI Java 门禁、插件面真机 E2E）
-
-**零、现场**：本批不改 Go/面板（控制面维持 `auditfix61`）；产出 = `deploy/demo-up.sh` 重写（`f5a76cf`）＋ `plugins/test.sh` 与 CI `plugins` 作业（`c59b387`）。三组真机验证（T1/T2/T3）在 VM 上针对该两文件跑完；CI 首跑即绿。
-
-**一、#62 复核：不成立（立案后剔除，未计为缺陷）**
-- 主张："fresh 非 demo 安装跳过 Velocity 插件构建 → proxy 静默不路由"。复核三条路径，**每条都构建 felis-velocity.jar**：
-  1. 源码臂：`felis-install.log`（首装）L1530 `building felis-velocity.jar`、L1553 `staged`；`bootstrap-auditfix{42,42b,42c,43,43b,44}.log` 每次重跑同两行。
-  2. 嵌入 tar 臂（fresh release/TUI 路径；本 VM 从未走过）：`felis bootstrap-assets game-stack | tar -x` → 37 文件（含 `plugins/velocity`、`plugins/shared`，构建输入齐全）→ `docker run --rm -v /tmp/gs62:/src:z -w /src/plugins/velocity gradle:8.14-jdk21 gradle --no-daemon clean build` → **BUILD SUCCESSFUL in 35s**，产出唯一 `felis-velocity-0.1.0.jar`（71117B，与现装同尺寸）。
-  3. 调用图：`build_velocity_plugin` 唯一调用点在 `build_game_stack` 末尾；`build_game_stack` 在 full 模式主链路（L2868）必经，nano 早返回，不存在可绕开的"demo 分支"。
-- 结论：代码阅读误判，剔除。真正会跳过插件构建的是 demo-up.sh 的旧镜像臂 → 即 #63（本批修复）。
-
-**二、#63（`f5a76cf`）：demo-up.sh 双起源 → 单起点（129 → 63 行）**
-- 红证据（读取 + 真机口径核对）：① 版本分叉——demo-up 硬编码 `PAPER_MC_VERSION:=1.21.8`，bootstrap `resolve_game_jars` 从 Limbo CI 产物名推导（T3 实测当日 Limbo 2026.0.3-ALPHA / MC 26.3；同一登录两跳必须同协议）；② 其镜像臂从不构建 felis-velocity.jar，导入的是本地 tag（`felis-limbo:demo` 等），与 bootstrap 写入的 registry refs 不一致 → 死件、旧基座上可致"起了但无处路由"；③ 起 docker 后从不停（违反 13d64e0 起"构建方以 docker 停收尾"的约定）；wiring 段在现代基座上只是 no-op。
-- 修复后 = bootstrap（未 SKIP 时）＋ 三项硬校验（`felis.host.toml`、`[velocity]` 段、`felis-velocity.jar`）＋ `felis setup` 交棒；镜像逻辑全删。
-- 真机验证：**T1** 健康基座（`SKIP_BOOTSTRAP=1 SKIP_SETUP=1`）→ 通过、exit 0；**T2** 移走 jar → `ERROR: /opt/felis/velocity/plugins/felis-velocity.jar missing — … silently routes nothing …`、exit 1（随即还原 71117B root:root 0644）；**T3** 默认臂全量重跑（`FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full FELIS_IMAGE=…:auditfix61 FELIS_WORLDS_HOST_PATH=… SKIP_SETUP=1`，日志 `/root/demo-up-b40.log`）：L733 插件构建、L752 `staged`、L941 交棒行、`[fail]`=0；收尾态 `felis-velocity active / k3s active / docker inactive`。
-- 附带实录（同一日志窗）：重跑期间 felis-api 滚动，velocity 00:22:25/00:22:40 两次 `refresh failed … keeping current registrations.`（失败保旧——规格 §11 承诺的真机上演）→ 00:22:55 恢复注册 → 00:22:58 服务随 `install_velocity` 重启并重载插件（新 pid）→ 00:22:59 `routing ready` → 00:23:29 收敛到 pod IP。
-
-**三、Java 层 CI 门禁（`c59b387`）：三个手工测试 + 三个装机 jar 首进 CI**
-- 缺口：`plugins/*/test` 三个 main 测试从未被任何 build/CI 运行；velocity/paper/limbo 三个"装机即用"jar 只在 bootstrap/Dockerfile 编译（Go CI 从不碰 Java）。
-- 产出：`plugins/test.sh`（Maven Central 取 adventure 三 jar，pinned+sha256；三测试 javac+java；三生产编译，limbo 按 bootstrap 同源解析当日版本）+ `ci.yml` 新增 `plugins` 作业（temurin 21 + Gradle 8.14）。
-- 首跑两次真跑抓出两处"从未被跑过"的假设错误：① `InviteCardTest` 一行式缺 examination-api（adventure-api 4.26.1 的 `Component` 签名引用 `Examinable`，javac 编译期即需）；② limbo 的 `com.loohp:Limbo:+` **永远不可解析**（LOOHP 仓无 maven-metadata，404 实查）→ 裸 `gradle -p plugins/limbo build`（plugins/README 原文）从来不可行；二者均已修（脚本 + 该测试 javadoc + README）。
-- 验证：VM 容器三测试 `OK (32/36/48 checks)`；velocity 21s / paper 29s / limbo 13s（2026.0.3-ALPHA）全 `BUILD SUCCESSFUL`。GitHub Actions run `35888009965`（c59b387）= go/shell/panel/plugins **4/4 success**（`plugins` 作业首跑即绿）。
-
-**四、插件层真机 E2E（不依赖真实客户端的面）**
-- 自建纯 socket 探针 `/tmp/mcprobe.py`（status = 服务器列表 ping；login = 登录首包），重装前与重装后各跑一遍，结果一致：
-  - 子域 MOTD（§11 只读缓存 + 相位）：`test-one` → `« test-one » 休眠中，加入即唤醒 / sleeping — join to wake`；`lobby`/`login` → `« … » 在线 / online`；`resolvecheck` → 休眠中；`nosuchxyz.<root>` → 回落 `A Felis server`（非 felis 子域不被劫持）。
-  - 登录边界：`LoginStart`（含现代协议 UUID 字段）→ 服务端首包 `0x01 EncryptionRequest` ⇒ 边缘 online-mode 强制成立。
-  - 加载面：`Loaded plugin felis-link 0.2.0`（共 5 插件）。
-- 口径：真实账号进服（limbo 门 → 菜单 → 转服）仍按"进服跳过"决定不演练；以上为不依赖客户端的最大真机面。（首次探测因探针漏发 UUID 字段被静默关闭——探针缺陷，非服务端问题；补齐后一次通过。）
-
-**五、观察（不计缺陷）**
-- ViaVersion 自报有 5.12.0（当前 5.11.0）：bootstrap 的 pin 是 FL-007 实测版本，属刻意，不随提示升级。
-- limbo 插件编译期一条 deprecated API 提示（`FelisLimboPlugin` 用/覆写已弃用 API）：门禁下可见、不阻塞，留观察。
-
-### 本轮新增真机证据（第四十一批：装载器 mod 层收口 —— #64 exec 位、#65 元数据、编译+起服双门禁）
-
-**零、现场**：控制面与面板不动；产出三个 commit：`f6048f2`（#64 修复）、`c2fe6a6`（#65 元数据）、`aa5abfa`（`plugins/test-mods.sh` + CI `mods` 作业 + release 双门禁 + README 记录）——这是最后一块"从未被任何自动化碰过"的模块面（fabric / forge / neoforge 三个装载器 mod）。
-
-**一、#64（`f6048f2`）：三个 vendored `gradlew` 缺可执行位——文档正路第一步即失败**
-- 红证据：`git ls-tree origin/main` 三个 `gradlew` 全为 `100644`（blob `b9bb139f…`）；`plugins/README.md` 第 212–214 行原文教 `plugins/fabric/gradlew -p plugins/fabric build`（forge/neoforge 同式）；VM 全新 clone 实跑 → `bash: line 1: ./gradlew: Permission denied`，exit 126。
-- 为何从未暴露：无 CI、无安装路径，README 是唯一入口——直到本批第一次真跑才现形。
-- 修复：`chmod +x` + `git update-index --chmod=+x`（mode 100644→100755 ×3）。
-- 可达性：①（全新用户照文档走的第一步）。
-
-**二、#65（`c2fe6a6`）：mod 元数据与仓库 LICENSE 相悖**
-- 三处 `license = "MIT"`（`fabric.mod.json`、forge/neoforge 的 `mods.toml`）vs 仓库 `README.md:73` 的 `AGPL-3.0-only` + LICENSE 全文。时间线：mods 2026-06-26 加入（`93f143f`）、LICENSE 2026-07-12 才落地（`037eb24`）——陈旧残留。
-- 另两处 `issueTrackerURL = "https://example.invalid/felis"`（占位域名）→ `https://github.com/FelisMC/Felis/issues`。
-- 用户面：fabric loader 启动打印 license 字段；`mods.toml` 被两个 loader 解析。
-- 可达性：②（低；元数据/合规面，非功能）。
-
-**三、双门禁落地（`aa5abfa`）**
-- `plugins/test-mods.sh`：JDK 17 下依次跑三个模块的 vendored wrapper（`./gradlew --no-daemon build`）；java 大版本非 17 直接 fail-loud（三模块目标是 Java-17 的 Minecraft 线）。
-- `ci.yml` 新增 `mods` 作业（temurin 17 + `gradle/actions/setup-gradle@v4`，版本由各模块 wrapper 自管）；`release.yml` 发布前加两道 Java 门禁（JDK21 `plugins/test.sh` + JDK17 `plugins/test-mods.sh`）。
-- README：Status 更新为 compile + boot 双验证；Building 段补 limbo `-PlimboVersion=<release>` 要求与两个门禁说明。
-- CI：run `35892544563`（`aa5abfa`）——`mods` 作业首跑即绿，五作业全过（shell / mods / panel / go / plugins 5/5）。
-
-**四、三个模块的真机编译 + 起服 E2E（本批核心证据）**
-- 编译（VM 容器 `eclipse-temurin:17-jdk` + 持久 gradle 缓存）：三模块 `BUILD SUCCESSFUL`（首跑 4–5 分钟级/模块），产物 `felis-{fabric,forge,neoforge}-0.1.0.jar` = 29276 / 28855 / 28562 B；留档 `/root/mods-build-b41.log`。
-- 起服（三个真实专用服，控制台直驱 `/link`，随后 `linkx` 未知命令对照、`stop` 正常停服）：
-  - **fabric** 1.20.1 + loader 0.19.5 + fabric-api：`Felis link ready; /link is registered.` → `Done (10.670s)!` → `/link 只能由玩家执行 / /link can only be run by a player.` → `Unknown or incomplete command`（linkx）→ `Stopping server`；`EXIT_fabric_boot=0`。
-  - **forge** 1.20.1-47.3.0：`Felis link ready; /link will be registered.`（modloading-worker）→ `Done (11.751s)!` → 同四段证据；`EXIT_forge_boot=0`。
-  - **neoforge** 20.4.251（1.20.4）：`Done (16.094s)!` → 同四段证据；`EXIT_neoforge_boot=0`。
-- 口径：三服均为"装好 mod 后真启动"的场景，同时验证 loader 真实解析修改后的元数据（license=AGPL、tracker URL）后正常装载；完整取码链（玩家在游戏内 `/link`）按"进服跳过"决定不演练，装载/注册/拒绝面已全部真机成立。
-- 留档：`/root/mods-e2e-b41.log`、`/root/mods-e2e-b41-resume{2}.log`、`/opt/felis/mods-e2e/`（496M，三服目录保留）；收尾态：docker `inactive`、k3s/felis-velocity `active`、磁盘 17G free。
-
-**五、演习装置自身两次修正（非产品缺陷，如实记录）**
-- 控制台驱动脚本 `/root/modserver-drive.sh` 初版把 FIFO 写端先开（`exec 3> console`）→ 自我死锁（内核 `wait_for_partner`、`State: S`），java 从未启动（无 `server.out` 实证）；改 `exec 3<> console`（RDWR 打开不阻塞）后三服全通。
-- forge 安装器首跑：`libraries.minecraft.net` 连接失败 → `com.google.code.findbugs:jsr305:3.0.2` 下载失败、`There was an error during installation`、无 `run.sh`；原样重试即 `The server installed successfully`（VM 网络抖动，非 forge/产品问题）。
-
-**六、观察（不计缺陷）**
-- 无新增。
-
-### 本轮新增真机证据（第四十二批：覆盖面对账 + 文档/工具收尾 —— #66 README_EN、§28 图对齐、sync.sh 本地修复）
-
-**零、现场**：本批不碰产品代码；对"所有模块均已演练"的结论做**独立对账**（枚举仓库全部模块/交付物 × 台账覆盖），只挖到文档与本地工具层面的残留。commits：`62c5a8a`（#66 README_EN）、`62e8c87`（§28 序列图对齐）。
-
-**一、覆盖面对账（本批核心，逐项）**
-- 仓库卫生：`git ls-files` 全量扫描无 `.DS_Store`/`*.log`/`node_modules`/构建产物误入库；顶层 `felis` 二进制（83MB）被 `.gitignore` 正确忽略；旧仓库名 `MliroLirrorsIngenuity` 残留仅存在于台账（历史记录）与本地未跟踪二进制。
-- 未竟工作标记：`TODO|FIXME|XXX|HACK` 全仓（go/ts/sh/md/yml）仅 1 处命中——`internal/api/api_test.go:263` 的 `context.TODO()`（合法测试写法），无未完成实现标记。
-- CLI 派发面：`cmd/felis/run.go` 的 usage ↔ dispatch 表有双向测试强约束（`run_test.go`，含显式 `undocumentedCommands` 允许表：`bootstrap-assets`/`init-forwarding` 等 in-Pod 入口），机制已核实成立。
-- CI 覆盖：`shell` 作业按 shebang 对**全部 tracked `*.sh`** 做语法检查（`git ls-files '*.sh'` 逐文件 `bash -n`/`sh -n`）并运行 `deploy/bootstrap_test.sh`；`plugins`/`mods` 各自作业。无"存在但从无入口运行"的脚本。
-- `internal/*` 22 个包逐一对账：0 提及者（`internal/apis`）为 CRD 类型包（被 40+ 站点引用、随 operator/manifests 演练间接全覆盖）；`deploy/{limbo,lobby,paper}` 为三张游戏镜像定义，随每次起服演练执行。真盲区＝无。
-- `panel` 全部 17 条页面路由 × 既有批次（CDP 巡检、深挖、写流程）覆盖；`scripts/` 三脚本为**本地私有**（`.git/info/exclude`，不入库）。
-- `docs/` 四件：troubleshooting（多批引用）、openapi（含 parity 强制测试）、deferred-seams（内容为现行状态，含 CLOSED/verified-live 注记，抽查与代码一致）、sequence-diagrams（发现漂移 → 修复，见第三节）。
-
-**二、#66（`62c5a8a`）：README_EN 与中文 README 漂移**
-- 红证据：`README.md`（中文）"使用方式"含**必需**的私有仓库 workaround（仓库私有 → 裸 raw 命令 404；token 经 `curl --config -` 由 stdin 下发、不进 argv；`sudo -E` 传给安装器）与"重跑=升级 felis-api、通道不继承（跟 main 需 `FELIS_VERSION_BOOTSTRAP=dev`）"两段；`README_EN.md` 是 `README.md:6` 链接的英文入口，这两段**完全没有** → 英文读者照文档第一步就 404 且无指引。
-- 修复：两段译入 EN（保留原命令与语义；`grep` 校验 token/config/dev 三关键串在位）。可达性：①（轻）——英文首装路径第一步。
-
-**三、§28 序列图对齐（`62e8c87`）**
-- 漂移点（两处，与修复后代码实读对照）：① Claim Transaction 图注仍是 audit #4 前形态（`SELECT EXISTS` + 事务外 pre-check）→ 更新为现行实现：`pg_advisory_xact_lock(user_id)` + 行 `FOR UPDATE` + **事务内**四维配额门（图上 `QuotaAvailable` 只是 fast path）；② Link Flow 更新为：取码 `FOR UPDATE`、同用户重验幂等、退役（soft-deleted）账户链接被当场接管——409 仅对"其他活跃用户"。均以 `internal/api/pgrepo.go` 实读 + handler 映射核对。
-- 惯例依据：该文件有维护史（`676407d docs(diagrams): align §28 sequence diagrams with implemented routes`）。
-
-**四、本地工具修复（私有脚本，不入库）：`scripts/sync.sh` 排除 `plugins/` 与 `go:embed` 冲突**
-- 红证据（本机复现）：以 sync.sh 原排除清单 rsync 一个 fresh 目标 → `go build` 立即失败：`bootstrap_asset.go:30:12: pattern plugins/limbo/build.gradle: no matching files found`（`bootstrap_asset.go` embed 了 plugins/{limbo,paper,velocity,shared} 源码；排除清单成文于 embed 引入之前）。存量目标靠 rsync 对排除路径的"保护"而看似正常，fresh 目标必炸。
-- 修复：排除项从 `plugins/` 改为 `plugins/*/build|.gradle|bin/`（与 `.gitignore` 同义）；`watch.sh` 同步去掉 plugins 的忽略。验证：新清单 rsync fresh 目标 → `go build` 绿（83.7MB 二进制产出）；两脚本 `bash -n` 通过。因脚本被 `.git/info/exclude` 私有，修复留在工作机、无 commit。
-
-**五、release.yml 静态复核（该流水线从未执行过——仓库当前无任何 tag/release）**
-- 三处输入/输出契约实核：① 发布资产名 `felis-linux-amd64/arm64` ↔ `deploy/bootstrap.sh` 的 `asset="felis-linux-${arch}"` 及收敛检查 `felis ${FELIS_REF}`（版本首行契约两侧一致）；② `out/linux_amd64/usr/local/bin/felis` ↔ Dockerfile 最终层 `COPY /out/felis /usr/local/bin/felis`；③ 版本断言 `felis ${GITHUB_REF_NAME}` ↔ `cmdVersion` 的 `felis %s` 首行。
-- 唯一无法在本机完全复刻的 `file(1)` 机器类型断言：用交叉编译的 linux/arm64 二进制实测输出 `ELF 64-bit LSB executable, ARM aarch64, …`，断言串 `ARM aarch64` 匹配（BSD/Ubuntu file 同源 magic）。
-- 结论：整条流水线未跑属"尚未切 tag"的发布流程事实（第四十批已注记）；切首个 tag 前无待修项。
-
-**六、观察（不计缺陷）**
-- `internal/api/handlers_account.go:169` 的 reclaim 选择仍为 Java/Velocity 侧 CODE-ONLY（deferred-seams 已记 accepting）——维持。
-
-### 本轮新增真机证据（第四十三批：构建 Job 收尸 #67 + #68 上下文拉取重试 + 生命周期/故障注入压测）
-
-**零、现场**：控制面从 auditfix61 升到 **auditfix62**（含 #67；自建镜像推入内建 registry，api/operator/reaper 三处滚动，`felis version` = `v0.0.0+fix62`）。本批证据 = 6 轮起停循环 + 3 处故障注入 + PG 断连语义 + 并发风暴 + 持续轮询。
-
-**一、#67（`2755e41`）：build Job 从不回收——每构建一个、完成 Pod 永久堆积**
-- 红证据（真机）：`felis-build` 里 5 个完成 Job/Pod 最长 26h 无人回收；全仓 TTL 对照——fileedit 2m / backup 10m / restore 10m / reaper CronJob 3+3 历史，**唯独 build 没有 `ttlSecondsAfterFinished`**；且 `build.go:69` 的注释写着 "(e.g. GC'd); treated as failed"——预期的 GC 从未存在。危害：完成 Pod 计入节点 pod 预算（stock k3s 110），构建量一上来先把节点塞满，新构建全部 Pending；etcd/磁盘同步膨胀。
-- 修复：`buildJobTTL = 7 * 24h`（终态后计时；日志路由是失败分诊面故取长窗；`Sync` 对终态幂等、`JobUnknown` 只影响非终态 → 删除后的唯一代价是日志 404）。单测 `TestBuildJobIsReapedAfterCompletion`。
-- 真机验证（auditfix62）：① 真实路径触发构建（owner 会话 → `POST /api/v1/images/build`，借遗留 blob 作 context）→ 新 Job `ttlSecondsAfterFinished=604800`、构建 25s `succeeded`；② 对旧 Job 打 `ttl=30s` → 40s 内 Job+Pod 被 TTL 控制器收走（机制实证）；③ 旧堆积 5 个里 1 个已收、4 个留存对照。
-
-**二、生命周期 ×6 + 三处故障注入（operator / api / postgresql）**
-- 每轮：CR `desiredState` Running → 等 Running → Stopped → 等 pod 消失。结果：**6/6 全收敛**——Running 23–29s（注入轮与无注入轮无差）、Stopped 3s、pod 每轮如期消失；收尾 conditions 无 Failed 残留（Ready=False / RconReached=False = Stopped 正常；Provisioned=True）。
-- 注入 1（cycle 2，删 operator pod @t+2s）：Running 仍 23s 达成（新 operator 立即接管 reconcile）。
-- 注入 2（cycle 4，`systemctl restart postgresql`）：CR 路径无感；另做定点验证：PG 停机窗口 `/me` = **503**（非 401，#11 语义保持）、`/healthz` = 200（存活探针独立）、内部 status = 200（纯集群读）；PG 恢复后 `/me` = 200。
-- 注入 3（cycle 5，删 api pod @t+2s）：内部轮询出现 2 次 `http=000`（约 6s 窗口）后自愈；Running 28s 达成。
-- 观察（不计缺陷）：高频翻杆期间 operator 报 9 条 `Operation cannot be fulfilled ... object has been modified`（乐观锁冲突 → 重排队自愈），目标时间无差；controller-runtime 正常重试语义，生产低频操作下更罕见。
-
-**三、并发风暴 + 拒绝面 + 持续轮询**
-- 20 路并发 internal status + 10 路 `/me`：30/30 全 200。
-- 无主 + ownerOnly 服 wake：403 `forbidden`（正确拒绝面，非 5xx）。
-- 持续轮询（status + healthz + me，5s 一轮 × 300 轮 ≈ 25 分钟，900 样本）：收盘 `LONG POLL DONE fails=27`——27 个失败样本**全部**落在三个自导演练窗口内（02:24:15/20 红证据杀 api〔6〕、02:29:48/53 auditfix63 滚动升级〔6〕、02:30:53–02:31:13 #68 确定性复现 scale api→0〔15〕），窗口外 873 样本全 200、零自发失败；每个窗口在动作结束后 ≤1 个探测周期（~5s）内恢复。
-
-**四、留档与收尾**
-- 留档：`/root/soak43.sh` + `soak43.log`、`/root/poll43-long.sh` + `poll43-long.log`、`/root/mint-owner-43.sh`（owner 会话重铸）、`/root/felis-image-build43.log`（镜像构建）、`/tmp/owner-jar43.txt`（owner cookie）；镜像 `10.43.182.43:5000/felis/felis:auditfix62` 已入 registry（`e2e/ttl-probe:latest` = 本批 drill 镜像，保留）。
-- VM 状态：docker 用毕已停（inactive）、k3s/velocity active；控制面三处 = auditfix62。
-
-**五、#68（`b76d0ac`）：构建 context 拉取对控制面重启零容忍——一次拒连即终态失败**
-- 红证据（真机）：删 api pod 后触发构建 `bld-1790187851749257160` → Job `Failed`；pod 时间线 `18:24:12Z` 起、context-fetch `18:24:13Z` exit 1，日志原文 `dial tcp 10.43.237.249:8081: connect: connection refused`；Kaniko 从未启动（PodInitializing）；新 api pod 11s 后就绪、同 blob 前后各一次构建均 25s succeeded——纯粹"单次尝试"造成的无谓失败。
-- 修复（`b76d0ac`）：`fetchContextWithRetry`——传输错误/5xx 重试至 45s 窗口（3s 间隔），4xx 快速失败（是答案不是抖动）；URL 校验提前为 usage error(2)；保留原错误文案前缀。窗口/间隔为 vars（测试可缩窗）；4 例单测：5xx 后恢复、拒连后恢复、404 不重试、窗口耗尽（全绿，含 `-race`）。
-- 真机验证（auditfix63，确定性优先，不复刻竞态）：`scale api→0` → 以复刻 Job 直建（由红证据 Job 派生：换名 `build-retry68`、剥 controller 标签与 selector、镜像改 auditfix63，复用 `felis-service-token` Secret）→ fetch 连续 8 次 `connection refused; retrying`（实捕日志）→ `scale api→1` → fetch 恢复、Kaniko 构建推送、trivy 干净 → **Job Complete**（`18:30:52Z→18:31:24Z`，32s）；同形场景对照旧版 = 终态 Failed。正常路径回归：api 在线直构 `bld-1790188306964162960` → `succeeded`（10s），fetch 日志**零重试行**（重试不引入正常路径开销）。
-- 附带发现（运维）：本批升级沿用 `set image` 后 `FELIS_IMAGE` env 仍停 `auditfix61`（两代落后）——构建 Job 的 fetch 容器实际一直在用旧镜像；已把 api/operator 的 `FELIS_IMAGE` 与 api/operator/reaper 三处镜像全部对齐 `auditfix63`。真实安装器路径随 manifests 重渲注入该 env，手工升级须成对改（升级清单事项）。
-
-### 本轮新增真机证据（第四十四批：modpack 规模上下文全链 + 并发构建）
-
-**零、现场**：控制面 = auditfix63（含 #67/#68；api/operator `FELIS_IMAGE` 同版本），VM docker 停。本批目标 = 把"真实模组包大小"这条链压到生产尺度：此前所有提交/构建演练的上下文都是 KB 级。
-
-**一、200 MiB 上下文全链（零缺陷）**
-- 构造：5×40 MiB `/dev/urandom` 载荷 + Dockerfile（随机数据不可压缩 = 最坏情形），tar.gz = 209,750,275 B。
-- 上传（真实 app 路由）：`POST /api/v1/me/submissions/{id}/context`（owner 会话）→ 200，**0.44s / ~481 MB/s**；宿主核对 PVC 落盘 `sub-cc160b3ad19080f9/context.tar.gz`。服务端读写超时设计（ReadTimeout/WriteTimeout 有意不设、仅 ReadHeaderTimeout）与流式落盘（io.Copy → cappedReader）均按预期，无内存尖峰。
-- 批准 → 构建（`bld-1790189227767686191`）：fetch init **1s**（200MB 内部面拉取 + 解压，无重试行）、kaniko **4s**（unpack/COPY/snapshot/push gzip 层 **209,738,853 B**）、trivy **4s**（0 findings）；Job Complete，全链 13s（18:47:07Z→18:47:20Z）。registry manifest 实查层大小一致（推送非虚）。
-- 覆盖点随验：提交构建 Job `ttlSecondsAfterFinished=604800`（#67 对真实提交路同样生效）；`/me/submissions` `build_status=succeeded`（#36 面在 200MB 规模下仍成立）。
-- 资源采样（2s 粒度，构建太快可能漏尖峰，标注局限）：无 OOM——api 17.9MB / kaniko 33.4MB / registry 34.1MB / trivy 35.9MB；emptyDir+快照为临时占用，随 Job TTL 回收。
-- 1 GiB 帽的单测已有覆盖（`internal/submit/submit_test.go` oversize→ErrInvalid）；帽下真机 = 本条（帽上真机不划算，不做）。
-
-**二、并发构建 ×3（零缺陷）**
-- 三路同时 `POST /api/v1/images/build`（各自 ref）→ 202×3；三个 Job 均 **11s Complete**，无串扰：每个 Job destination 与 tag 一一对应（conc-b44-1/2/3 各落位）、`ttl=604800` 各自在。
-
-**三、磁盘运维注记（非缺陷）**
-- 连续镜像构建（docker build）把 VM 根盘从 11G free 压到 4.3G；`docker builder prune -af` 一键回收 **6.5G**（回 11G free）。即每轮全量重建约需 5–7G 构建缓存空间；生产主机根盘建议 ≥40G 并定期 prune（如需可在部署文档补一行，本批未改码）。
-- 留档：`/root/bigctx.tar.gz`（200MiB 上下文原件）、`/root/bigctx/`、`/root/stats44.log`、`/root/stats44-build.log`；提交 `sub-cc160b3ad19080f9`（blob 与 `user-uploads` 镜像保留作尺寸样本）；镜像 `e2e/conc-b44-{1,2,3}:latest`。
-
-### 本轮新增真机证据（第四十五批：模组构建链三连修 #70/#71/#72 + 提交→装服全链闭环 + 文档/测试修 #69/#73/#74）
-
-**零、现场**：控制面 auditfix63→66（每修一版真机重演）；本批目标 = 真实用户会写的那种模组包 Dockerfile（`FROM <平台底座>` + 内容层）。此前所有构建演练都是 `FROM scratch`——用户真实形态的第一步从未跑过，本批连撞三个缺陷。
-
-**一、#70（`ac403b9`）：kaniko 拉底座缺 pull 侧 insecure 标志**
-- 红证据（真机）：提交 `FROM registry.felis.svc:5000/felis/paper:demo`（`bld-1790189685537480076`）→ kaniko `Retrieving image manifest …` 后即败：`Get "https://registry.felis.svc:5000/v2/": http: server gave HTTP response to HTTPS client`。根因：Job 只给了 `--insecure --skip-tls-verify`——kaniko v1.24 `--help` 实测二者均只覆盖 **push**，拉取默认 HTTPS，撞上明文 registry。
-- 修复：补 `--insecure-pull` / `--skip-tls-verify-pull`（对称补齐；构建命名空间 egress 本就只许 DNS/内建 registry，不扩大可达面）。单测断言两 flag 在参。
-- 真机复验：下一次构建 `Retrieving image manifest` 不再报错，进入解包阶段（随即暴露 #71）。
-
-**二、#71（`6e47730`）：kaniko 解包底座层被 drop-ALL 卡死**
-- 红证据（真机）：`error building image: error building stage: failed to get filesystem from image: chown /etc/gshadow: operation not permitted`——kaniko 以 root 解包 tar 层要把文件 chown 到层里记录的属主（root:shadow 等），drop-ALL 后 CAP_CHOWN/CAP_FOWNER/DAC_OVERRIDE 全无。`FROM scratch` 全用 COPY（属主= kaniko 自己）所以从未暴露；**任何真实底座**必触。
-- 修复：仅 kaniko 容器补回最小能力集 `CHOWN + DAC_OVERRIDE + FOWNER`（fetch/trivy 保持 drop-ALL 基线；单测断言"恰好三枚"防漂移）。
-- 真机复验：解包通过、COPY、推送成功（进入 trivy 阶段，撞上 #72）。
-
-**三、#72（`b14bacb`）：trivy 扫 jar 必拉 Java DB——egress 锁下必败**
-- 红证据（真机）：kaniko 全绿后 trivy `FATAL … Unable to initialize the Java DB … failed to download artifact from mirror.gcr.io/aquasec/trivy-java-db:1: connection refused`。Java DB 按需下载：镜像一有 jar 就触发——模组包 = jar 集合，故所有真实用户构建都会倒在扫描门；旧构建全 scratch（无 jar）从未触发。
-- 修复：新增 `[registry] trivy_java_db_repository`（→ `--java-db-repository`，与 vuln-DB 旋钮对称）；installer carry 白名单收录；§8e 配方补 Java DB 镜像步骤；deferred-seams 更新。
-- 真机复验（第 4 次尝试，auditfix66）：Job spec 实测含 `--db-repository registry.felis.svc:5000/mirror/trivy-db:2 --java-db-repository registry.felis.svc:5000/mirror/trivy-java-db:1`；扫描 `ubuntu 26.04` + `paper/paper.jar` + `pebble` 全 0 漏洞；**Job Complete**（全链 27s，`bld-1790190788310114143`）。
-
-**四、capstone：提交 → 装服 → 起服，全链闭环（零缺陷）**
-- 第 4 次尝试产物 `user-uploads/sub-c006cbd633317ccb:latest`（= `felis/paper:demo` + 用户 marker，经完整提交管道构建）：
-  - 白名单自动收录（`source=built`）→ 产品路由 `PATCH /api/v1/servers/test-one {"image": …}` **200**（`image_not_whitelisted` 门通过）；
-  - `POST /servers/test-one/wake` 202 → pod `test-one-0` Running 1/1（25s 内）；
-  - `kubectl -n minecraft exec test-one-0 -- cat /felis-probe-marker.txt` → **`felis-probe44-ok`**（用户构建上下文的内容确在运行中的服务器里）；
-  - 服务器日志 `Done (4.678s)!`（paper 完整启动）；RCON 线程应答平台就绪探针；
-  - 收尾：stop 202、pod 消失、镜像回滚 `felis/paper:demo`、desiredState=Stopped。
-- 口径：这是"玩家上传模组包 → 服主审批 → 自动构建 → 选用为该服镜像 → 起服"的机制全链（真实客户端进服仍按决定跳过）。
-
-**五、#69（`5cbfa89`）：README 过度承诺"自动部署"**
-- 复核（读全）：提交数据模型无目标服务器字段；approve→build→白名单是唯一自动化；部署 = 服务器编辑框选镜像（产品路由与 UI 均在）；面板文案本身只承诺"自动触发安全构建"。
-- 修复：README 中英两行改为"自动构建；产物进入镜像白名单，可直接选用为服务器镜像完成部署"。可达性 ①（轻）。
-
-**六、#73（`2961beb`）：bootstrap 测试在跑 k3s 的主机上必假失败（工具级）**
-- 红证据：VM（真实 k3s 主机）跑 `bootstrap_test.sh`（**原版同现**）→ `FAIL a missing worlds root is warned about`：用例拿真实路径 `/var/lib/rancher/k3s/storage` 期望 WARN，而该目录在"安装器真正要服务的机器"上必然存在 → 假红；CI 从未见到（runner 无此路径）。
-- 修复：warn 情形改用保证不存在的路径；并把 `trivy_db_repository` / `trivy_java_db_repository` carry 断言补齐。VM 复跑 **140 PASS / 0 FAIL**。
-
-**七、#74（`96aa817`）：CI flaky——console 断连测试读写竞态（工具级）**
-- 红证据：run `35907662213`（纯 README 提交）go 作业红：`TestServerConsoleDisconnectTeardown: expected the first event before disconnect, got ""`；`gh run rerun --failed` 即绿 → 竞态确认。
-- 根因：测试只等"源读到第一行"就 cancel，"中继把事件写入响应"尚未发生；cancel 落进缝里时 body 为空。
-- 修复：加 `firstDataWriter` 信号（写完成后再 cancel，写→读有 happens-before）；本地 `-count=60` 与 `-race ×5` 全绿；CI 复跑 `5cbfa89` = success。
-
-**八、运维注记（非缺陷）**
-- 升级滚动撞 kubelet **ephemeral-storage 压力**：auditfix64 滚动时新 api/operator pod `Pending 4m46s`（事件 `untolerated taint(s)`），02:58:12 kubelet eviction manager 回收后自动调度成功。诱因 = 构建把根盘压到 89%（docker 构建缓存 + kaniko/emptyDir 临时层）；`docker builder prune -af` 两清共回收 ~11G，余 11G free。生产清单：根盘 ≥40G + 升级前清构建缓存（与批 44 注记合并）。
-- 中间失败构建留下的 registry 镜像（`user-uploads/sub-83f6…`、`sub-171e…` = 已推未过门；`sub-c006…` = capstone 产物）留档；测试服 `test-one` 已回 `felis/paper:demo` + Stopped；docker 停。
-
-## 结论：离"生产可用"还差什么（按优先级）
-
-1. ~~构建链路的上下文通道~~ ✅ **已修**（`f79e5eb`/`02fd2de`，真机全链路含拉回校验；Trivy DB 需按 §8e 镜像一次）。
-2. ~~镜像耐久~~ ✅ **已完成**（第二十九批：`a9b275a`/`13d64e0`/`fa0e8d7` + `c7e585e`；三次真机重跑 + GC 两演练——控制面与游戏镜像被 GC 后自动回拉；同批修复并复验 #46/#47/#48/#49）。
-3. ~~PG 级契约测试~~ ✅ **已落地**（`2a55a0d`）：`internal/pgint`（`-tags pgint`，需 `FELIS_TEST_PG_URL` 指向名字含 `pgint` 的库，harness 会 drop schema + 重放真实迁移）已覆盖会话/OTP/op-login/绑定码/submission/build/owner 角色；**首跑即抓到 #20**（索引与 ErrEmailTaken 从未存在）。运行方式见 CONTRIBUTING.md。
-4. ~~面板把 /jobs 显示出来~~ ✅ **已完成**（`97a64c8`，备份页「最近操作」卡，正是它把 #35 暴露出来的）。
-5. ~~多节点回收~~ ✅ **已修**（`daf7602`：`--reaper-node` → CronJob pod `kubernetes.io/hostname` nodeSelector，真机 render/dry-run/收敛 diff 三连；单节点部署不传即维持原状）。附带核销缺陷 #38（渲染提示里过期的 uid-1000/setfacl 指导）。
-6. ~~告警~~ ✅ **已完成**（`94f71ee` + `43df08b` + 第二十七批实弹演练：真实构建失败 → pending → 08:33:14Z firing；规则随 `deploy/alerts/` 交付）。
-7. ~~玩家可见的构建结果~~ ✅ **已修**（`72c4aa3`，缺陷 #36：列表路由附 `build_status`/`build_error`，面板「我的提交」展开行呈现，真机双例验证）。
-8. ~~面板文件编辑器入口~~ ✅ **已补**（`0a36b3f`，缺口补齐，真机 CDP 全链）。
-9. ~~fleet 系统服务死操作~~ ✅ **已修**（`2f90851`，缺陷 #37）。
-10. ~~S3 上传通道演练 + 存储向导逐屏~~ ✅ **已演练**（第二十四批：上传通道全链、0 缺陷；第三十二批：向导屏逐屏 + 上传取件 + UI 回滚，0 功能缺陷并修 #52）。
-11. ~~breakGlass 控制台逐屏~~ ✅ **已演练**（第二十五批：菜单 4 操作 + bootstrap 分支，缺陷 #39–#43 全修全验）。~~`felis setup` 全屏向导逐屏~~ ✅ **重跑侧已逐屏**（第二十六批），**首装侧连续走查也已完成**（第三十一批：scratch 库单次连续走完全部屏幕 + 轨道回顾，0 缺陷）。
-12. ~~`felis nano` 全链~~ ✅ **已走查**（第三十四批：HTTP 矩阵红 13→绿 60/60；systemd/配置/firewalld 三层重跑全绿；同批修复 #55——坏源静默挡住验证梯子）。
-13. ~~NetworkPolicy 真机强制矩阵~~ ✅ **已演练**（第三十五批：pod 源端口级矩阵全绿、转发型外部源「拒→放→拒」白名单闭环、host/ClusterIP/NodePort 路径全通；两条已归因注记——node-local 流量豁免、firewalld 仅放 DNAT 服务流）。
-14. ~~面板错误文案全量本地化~~ ✅ **已修**（第三十六批：#60，37 个用户可达码补齐双语映射，三类真机验证；管理面写操作 Run4c 复核 0 缺陷）。
-15. ~~管理面交互级收尾（submissions / 会话撤销 / 启动）~~ ✅ **已演练**（第三十七批：点击→构建→succeeded 全链、单条/全部撤销精确验证；同批修 #61——LP 写承诺按 LP 官方语义收窄，真机复验）。
-16. ~~缺陷 #56–#59（玩家管理回包 / 控制台回显 / 文件页 fail-fast / LP 诚实化）~~ ✅ **已修并归档**（第三十八批回填真机证据：修复前触发 + 修复后复验，四项全绿）。
-17. ~~服务器详情与运维面收尾~~ ✅ **已演练**（第三十八批：ServerFiles 写流程、备份/恢复全链、白名单/封禁闭环、ServerCard 停止、`/admin/updates` 复跑、metrics、CLI 核销——0 缺陷）。
-18. ~~reaper 多节点实机~~ ✅ **已演练**（第三十九批：VM 克隆双节点 k3s——pin 命中持盘节点并跑通、错位 pin fail-closed Pending、装置全回收；`felis-backups` PVC 的 volume node affinity 构成第二层保险）。
-19. ~~Java 插件层自动化缺口~~ ✅ **已补**（第四十批：`plugins/test.sh` + CI `plugins` 作业——三个手工测试与三个装机 jar 全进门禁，CI 4/4 绿；首跑抓出两处"没跑过"的错误：InviteCardTest 编译类路径缺 examination-api、limbo `+` 版本不可解析。同批关闭 demo-up 单起点化 #63、并复核剔除 #62）。
-20. ~~装载器 mod 层（fabric/forge/neoforge）~~ ✅ **已收口**（第四十一批：#64 三个 `gradlew` 补可执行位、#65 元数据对齐 AGPL-3.0-only；`plugins/test-mods.sh` + CI `mods` 作业 + release 双门禁；三真实专用服起服 E2E——mod 装载、`/link` 注册、控制台拒绝全绿。CI run `35892544563` = 5/5，`mods` 作业首跑即绿）。
-21. ~~全仓覆盖面对账 + 文档收尾~~ ✅ **已完成**（第四十二批：22 个 internal 包 × 面板 17 条页面路由 × 全部 tracked 脚本 × 仓库卫生逐项对账，无盲区；#66 README_EN 修复；§28 两张序列图对齐现行实现；本地 `scripts/sync.sh` 排除清单与 `go:embed` 冲突——修复+复现验证〔脚本私有，不入库〕）。
-22. ~~构建层资源回收~~ ✅ **已修**（第四十三批：#67 build Job TTL——完成 Job/Pod 不再无限堆积；真机：新构建 Job `ttl=604800` + 旧 Job 打 30s TTL 实测被 TTL 控制器收走。同批：6 轮生命周期循环 + operator/api/pg 三处注入全绿、PG 503 语义复验、30 路并发全 200）。
-23. ~~构建上下文拉取的短暂断连~~ ✅ **已修**（第四十三批追加：#68——fetch-context 有界重试〔45s 窗口 / 3s 间隔；4xx 不重试〕；真机：api 停机期连续 8 次拒连全部重试、恢复后构建 32s Complete；正常路径无重试开销。附：演练升级暴露的 `FELIS_IMAGE` 两代错位已对齐 auditfix63）。
-24. ~~modpack 规模上下文全链 + 并发构建~~ ✅ **已演练**（第四十四批：200MiB 上下文上传 0.44s → 批准 → 构建全链 13s、三路并发构建全绿；TTL 与玩家可见面在规模下复验，零缺陷。见该批节）。
-25. ~~内建底座构建（`FROM registry.felis.svc:5000/…`）~~ ✅ **已修并真机闭环**（第四十五批三连修：#70 拉取缺 `--insecure-pull`、#71 drop-ALL 卡解包、#72 trivy Java DB 未镜像——"真实模组包"形态的三个必踩点；修复后 paper 底座构建 27s Complete、`paper.jar` 扫描 0 漏洞）。
-26. ~~提交 → 装服 → 起服 闭环~~ ✅ **已演练**（第四十五批 capstone：构建产物 PATCH 为服务器镜像〔白名单门通过〕→ wake → Running 1/1 → 用户 marker 可读 + `Done (4.678s)!`；test-one 已复原为 paper:demo/Stopped）。
-27. ~~文档与测试工具面~~ ✅ **已修**（第四十五批：#69 README 部署承诺对齐实现；#73 bootstrap 测试密闭化〔VM 140 PASS〕；#74 console 断连测试消抖；CI 全绿）。
-
-## 剩余待演练队列（截至第四十三批）
-
-- ~~reaper 多节点实机~~ ✅ 已完结（第三十九批：临时克隆第二节点组双节点 k3s 实机——pin 命中持盘节点并跑通、错位 pin fail-closed Pending；装置已回收）。
-- ~~面板交互级收尾~~ ✅ 已完结（第三十六–三十八批：管理面写操作、submissions/会话、服务器详情页全链，0 缺陷）。
-- ~~`demo-up.sh`~~ ✅ 已完结（第四十批：单起点化 #63——bootstrap 装机 + 三项硬校验 + `felis setup` 交棒；T1/T2/T3 真机全绿）。
-- ~~Java 插件层自动化~~ ✅ 已补（第四十批：`plugins/test.sh` + CI `plugins` 作业；三个手工测试与三个装机 jar 首进 CI）。
-- ~~装载器 mod 层（fabric/forge/neoforge）~~ ✅ 已收口（第四十一批：#64/#65 修复 + `plugins/test-mods.sh`/CI `mods`/release 双门禁 + 三真实专用服起服 E2E）。
-- ~~全仓覆盖面对账~~ ✅ 已复核（第四十二批：模块×台账逐项对账无盲区；批次产出为文档/本地工具层面的修复，见该批节）。
-- ~~长稳/故障注入（首轮）~~ ✅ 已完成（第四十三批：6 轮起停循环 + 3 处注入 + PG 断连 503 + 30 路并发 + 25 分钟持续轮询；见该批节）。
-- ~~modpack 规模上下文 + 并发构建~~ ✅ 已完成（第四十四批：200MiB 上下文全链 13s + 并发 ×3 全绿；见该批节）。
-- ~~提交 → 装服 全链（含平台底座 FROM）~~ ✅ 已完成（第四十五批：三连修 #70/#71/#72 + capstone 提交→装服→起服；见该批节）。
-
-**队列现已清空。**（真实游戏客户端进服已按决定跳过，用内部 mint/approve hook 链替代；不依赖客户端的最大真机面见第四十批节 + 第四十一批节〔装载器三服起服〕；第四十二批为覆盖面对账复核。）
-
-## 系统性观察
-
-- **PGRepo 与接口契约/fake 漂移**（#16/#17/#18）：`fakeRepo` 与接口注释是对的、PG 实现在细节上落后，单测全绿也发现不了。建议后续引入 PG 级契约测试（testcontainers 或针对关键写路径的集成测试），重点覆盖「接口注释承诺了字段/错误码/生命周期」的方法。
-- 部署知识：交叉编译必须先把 `panel/dist` 拷进 `internal/panel/static` 再 `go build`，否则镜像内 SPA 缺失（页面显示 “assets were not built”）。本批镜像为 `felis:auditfix7`。
-
-## 复现入口速查
-
-- 面板会话 cookie：`/tmp/owner-cookies2.txt`（2026-09-23 铸；auditfix42 部署后仍有效；staff 邮件门被设计拒绝 → 走第二十八批的 op-login + 内部面 approve hook 链）；API 助手：`/tmp/fcurl.sh`
-- 测试服：`test-one`（minecraft ns，stopped）；合法备份 `bk-47ee2e7e96e5a4ca9d0e51b805518bac`
-- CDP 调试口：Mac `127.0.0.1:9333`（独立 Chrome，profile `/tmp/felis-chrome2`）；WebAuthn 虚拟认证器需在**同一 CDP 会话**内完成仪式，且先 `Page.bringToFront`（否则 NotAllowedError: page does not have focus）
-- 已部署到 VM：控制面（api/operator/reaper + `FELIS_IMAGE`）= `registry.felis.svc:5000/felis/felis:auditfix61`（含至 #61 的全部修复；镜像托管于内建 registry）；系统服/推荐 ref = `registry.felis.svc:5000/felis/{limbo,lobby,paper}:demo`；源码快照 `/opt/felis/src`（`b4ef42d`；上一版在 `/opt/felis/src.bak`，更早 `src.old43`）；host 二进制 = `/usr/local/bin/felis` = `v0.0.0+fix61`（含 #53–#61；旧件留档 `/root/felis-fix54.bin`（sha `a0b29153…`）、`/root/felis-fix52.bin`、`/root/felis-auditfix44.bin`）；**docker 守护进程在第三十七批构建后已停（用前 `sudo systemctl start docker`）**；**world 执行器以 root+DAC_OVERRIDE 运行**；reaper CronJob（minecraft ns）`suspend=false / …auditfix61 / nodeSelector=localhost.localdomain`；迁移 `schema_migrations` max=21；SMTP 密码 env 待「configure email」刷新时落地
-- 安装器重跑配方（第三十批复用；先同步 `/opt/felis/src` 再跑）：`cd /root && FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full FELIS_IMAGE=registry.felis.svc:5000/felis/felis:<tag> FELIS_WORLDS_HOST_PATH=/var/lib/rancher/k3s/storage nohup bash /opt/felis/src/deploy/bootstrap.sh > /root/bootstrap-<tag>.log 2>&1 &`；完成后 `grep -c '\[fail\]'` 应为 0
-- 第三十一批 drill 留档：`/root/preTUI43/`（首装走查前快照：两 toml + 两 ns Secret + sha256、走查后 Secret 快照）、`/root/pre51-replica.toml`/`post51-replica.toml`/`post51b-replica.toml`（#51 红/绿副本证据）；走查残留（scratch 库、scratch 配置、hba 行、tmux 会话）均已清理
-- 第三十二批 drill 留档：`/root/preS3/`（S3 走查前快照：两 toml + 两 ns Secret + sha256 + deploys.txt）、`/root/s3wiz/`（posts3 快照、postroll-sha256、fix52-red.txt、fix52-green.txt）、`/root/felis-fix52`（含 #52 的宿主二进制）；MinIO 容器+卷+两镜像、`felis-uploads-s3` Secret、DB 行（`image_submissions` `sub-853a4e2ba4ba6443`）、/tmp 残留均已清理，docker 已停
-- 第三十三批 drill 留档（VM）：`/tmp/felis-fix53`、`/tmp/felis-fix54`（宿主二进制；sha `9afd141d…` / `a0b29153…`）、`/root/felis-fix52.bin`；`/tmp/player-cookies.txt`（player 会话）；面板 CDP 驱动脚本 `/tmp/cdp-updates2.js`（Mac 侧）
-- 第三十四批 drill 留档（VM）：`/srv/nanotest/`（nano 装置：`nano-stub.py` + 5 驱动脚本 + `matrix/` 红原跑 / `matrix-fix55/` 绿原跑）与四层重跑落盘 `matrix-red.r2.log`（`pass=47 fail=13`）/ `matrix-fix55.r2.log`（`pass=60 fail=0`）/ `nano-service.r2.log` / `nano-config.r2.log` / `nano-fw.r2.log`；`nano-stub` 临时单元仍在跑（收尾 `systemctl stop nano-stub`）；宿主二进制 `/usr/local/bin/felis-nano-test` = fix55 改名件（sha `2c8c34fd…`，与 `/usr/local/bin/felis` 同物）
-- 第三十五批 drill 留档（VM）：`/srv/npdrill/`（netpol 装置全套：`np-matrix.sh` + `np-matrix-run1/2.log`、`np-netns.sh`、`netns-probe.py`、`np-cidr-test.sh/.log`、`spec-before/after.json`、`td3.log`（包级归因）、`tcpdump-8081.log`（刷新存活）、`iptables-save.txt` 与 `nft-rules.txt`）；netns/veth、firewalld 临时挂载、策略改动全部已清理/复原（spec md5 `e6b07440…` 前后一致）
-- pgint 正确跑法（走 VM 的 PG，勿在本机起容器）：Mac 侧短隧道 `ssh -6 -i ~/.ssh/id_ed25519 -N -L 15433:127.0.0.1:5432 root@…` → `FELIS_TEST_PG_URL='postgres://felis:<pw>@localhost:15433/felis_pgint?sslmode=disable' go test -tags pgint ./internal/pgint/ -count=1`（pg_hba 需 `host felis_pgint felis 127.0.0.1/32 scram-sha-256`；第三十一批把现场丢失的这条补回；隧道用完即 kill）
-- breakGlass TUI 驱动法：VM tmux `new-session -d -s bg -x 160 -y 45 "/root/felis-auditfixNN breakGlass"` + `set-window-option -t bg remain-on-exit on`（否则退出摘要读不到）；send-keys/capture-pane 驱动；退出码 1 = 操作失败（卡上会显示原因）
-- 构建链路 drill 现成条件：`felis-build` 里有 `felis-service-token`（Secret 复制）；`felis-config` 的 kaniko/trivy pin 指向内建 registry 的 mirror 路径（`registry.felis.svc:5000/mirror/...`，见 §8e）——本批后安装器重跑会 **carry** 这些 pin（不再被写盘冲掉；#49 复验点）；直构一行：`POST /api/v1/images/build`（context=`http://felis-api-internal.felis.svc.cluster.local:8081/api/v1/internal/submissions/sub-bf7dc18e9dd97dc2/context`）
-- registry 工具：`curl -s http://127.0.0.1:5000/v2/_catalog`、`/v2/<repo>/tags/list`；**取 manifest 必须带 `Accept:`（OCI index/manifest list；缺 Accept 的 GET 会 404，别误判没推上）**；GC 演练配方：`k3s ctr images rm <ref>`（要清到 blob 级再加 `k3s ctr content prune references`）→ 删 pod / `rollout restart` → `kubectl describe` 看 `Pulled … in …ms`
-- setup 重跑向导（第二十六批）：状态屏 `c/s/e` 三流已走查；**reconfigure 非只读**——storage 选 Local / 提交 S3 即 apply + 滚 API（幂等），email 表单 esc 无副作用；从 S3 表单 esc 退回会重置为 Local 预选；存储屏（第三十二批补充）：chooser 预选当前值（↑↓ 切换），Local 直接 enter 提交、S3 五字段 `enter next`/末字段 `enter submit`，提交后 10–40s（含 API rollout status 180s 上限）
-- VM 内部面：`k3s kubectl -n felis port-forward svc/felis-api-internal 18081:8081`（Pod 重建后转发会悬死，需重启）；reaper CronJob（minecraft ns）已应用且已收敛（`suspend=false`）
-- 第三十八批收尾复查：docker `inactive`；控制面三处 = `auditfix61`；`/opt/felis/src` = `b4ef42d` tarball 展开（无 `.git`）；reaper `suspend=false` + `nodeSelector=localhost.localdomain`；test-one `Stopped`；`resolvecheck` 文件接口 409 `no_world_volume`（0.03s）
-- 第三十九批 drill（已回收）：临时 VM `felis-node2`（`prlctl clone --linked` 克隆 node1；node2 停用 felis-velocity/k3s-server/docker 后以 agent 加入）——流程与两向对照见该批节；node1 为克隆经历过一次正常重启（组件全回归、docker 停回 `inactive`）；收尾 job/节点对象/VM 全部删除，集群回到单节点
-- 第四十批工具与留档：VM `/tmp/mcprobe.py`（纯 socket MC 探针：`python3 /tmp/mcprobe.py status <sub>.<root>`；`login <sub>.<root> <name>` 打登录首包）；插件门禁跑法 = `docker run --rm -v /opt/felis/src:/src:z -w /src gradle:8.14-jdk21 bash plugins/test.sh`（或宿主 `bash plugins/test.sh`，需 JDK 21 + Gradle 8.14；日志 `/root/plugins-test-b40.log`）；T3（demo-up 默认臂全量重跑）日志 `/root/demo-up-b40.log`；`/opt/felis/src` = `c59b387` 快照（上一版 `src.bak40`；此后全量重跑用 `sudo bash /opt/felis/src/deploy/demo-up.sh` 即可，SKIP 开关见脚本头）；CI run `35888009965`（含新 `plugins` 作业首跑绿）
-- 第四十三批工具与留档：控制面三处 = `registry.felis.svc:5000/felis/felis:auditfix62`（含 #67；镜像源 `10.43.182.43:5000/felis/felis:auditfix62`，宿主 docker build 后 `docker push 10.43.182.43:5000/...`）；`/opt/felis/src` = `2755e41` 快照（上一版 `src.bak42`）；owner 会话重铸 = `bash /root/mint-owner-43.sh`（cookie 落 `/tmp/owner-jar43.txt`）；直构触发（含 TTL 检查）= `curl -sk -b /tmp/owner-jar43.txt -X POST https://127.0.0.1:30443/api/v1/images/build -H 'Content-Type: application/json' -d '{"image_ref":"registry.felis.svc:5000/e2e/ttl-probe:latest","dockerfile":"FROM scratch\nLABEL felis=x\n","context_ref":"http://felis-api-internal.felis.svc.cluster.local:8081/api/v1/internal/submissions/sub-bf7dc18e9dd97dc2/context"}'` → `kubectl -n felis-build get job build-bld-<id> -o jsonpath='{.spec.ttlSecondsAfterFinished}'` 应为 `604800`；压测编排 `/root/soak43.sh`（日志 `soak43.log`，含 6 轮循环 + 3 注入）、长轮询 `/root/poll43-long.sh`（日志 `poll43-long.log`）；镜像构建日志 `/root/felis-image-build43.log`。注意：`lobby`/`login` 是保留名，internal per-server 路由（status/wake 等）对它们**按设计**返回 400，勿作缺陷误报。
-- 第四十三批追加（#68）留档：控制面 api/operator/reaper 镜像 + api/operator `FELIS_IMAGE` = `registry.felis.svc:5000/felis/felis:auditfix63`（`felis version` = `v0.0.0+fix63`；升级时 `set image` 与 `FELIS_IMAGE` env 必须成对改，否则构建 Job 的 fetch 容器会悄悄用旧镜像）；`/opt/felis/src` = `b76d0ac` 快照（上一版 `src.bak43` = `2755e41`）；复刻 Job `/root/job68-new.json`（由红证据 Job `build-bld-1790187851749257160` 派生：换名 `build-retry68`、剥 controller uid 标签与 selector、镜像改 auditfix63、复用 `felis-service-token` Secret）；重演验证法：`kubectl -n felis scale deploy/felis-api --replicas=0` → `kubectl -n felis-build create -f /root/job68-new.json` → `kubectl -n felis-build logs <pod> -c context-fetch`（应见 `connection refused; retrying`）→ `scale --replicas=1` → Job 收敛 `Complete`；对照：api 在线直构成功且 fetch 日志无重试行。
-- 第四十四批工具与留档：200MiB 上下文构造 = 5×40MiB `head -c 41943040 /dev/urandom` + `Dockerfile`（`FROM scratch` + `COPY payload /payload`），`tar -czf /root/bigctx.tar.gz Dockerfile payload`；上传 = `curl -sk -b /tmp/owner-jar43.txt -X POST -H "Content-Type: application/gzip" --data-binary @/root/bigctx.tar.gz https://127.0.0.1:30443/api/v1/me/submissions/<id>/context`；采样 = `k3s crictl stats`（此版 **无 `--no-trunc`**，列解析取 `$2=NAME $4=MEM`）；docker 缓存清理 = `systemctl start docker && docker builder prune -af && systemctl stop docker`；留档 `/root/stats44.log`、`/root/stats44-build.log`、提交 `sub-cc160b3ad19080f9`、镜像 `e2e/conc-b44-{1,2,3}`。
-- 第四十五批工具与留档：从平台底座构建的配方 = 提交上下文含 `Dockerfile`（`FROM registry.felis.svc:5000/felis/paper:demo` + `COPY marker.txt /felis-probe-marker.txt`）；装服验证 = `PATCH /api/v1/servers/test-one {"image":"registry.felis.svc:5000/user-uploads/<sub>:latest"}` → `POST /servers/test-one/wake` → `kubectl -n minecraft exec test-one-0 -- cat /felis-probe-marker.txt`；trivy 双 DB 镜像 = `mirror/trivy-db:2` + `mirror/trivy-java-db:1`（Java DB digest `5766dfbb…`；两键在 `/etc/felis/felis.{host,pod}.toml` 与 felis-config Secret 双副本）；`bootstrap_test.sh` 需在 Linux 跑（VM 留档 `/root/btest72/`，应 140 PASS）；本批提交样本 `sub-{b45d416f4af811ef(无镜像),83f677e41acd80c3,171e8f967f0de3bc,c006cbd633317ccb,cc160b3ad19080f9}`；留档 `/root/probe44/`、`/root/probe44.tar.gz`、`/root/probe44*.sid`。

CONTRIBUTING.md

@@ -23,6 +23,28 @@ The codebase is intentionally split by responsibility. Prefer changing the
 smallest owning module instead of adding broad abstractions or rebuilding nearby
 code.
 
+## Interface Copy
+
+Interface copy must use a formal documentation style: objective, concise, and
+precise. State the current condition, its cause or impact, and the available
+action. Describe confirmed facts only; avoid conversational phrasing,
+personification, and unsupported duration estimates. Chinese instructions should
+use explicit verbs such as “执行”, “选择”, “查看”, and “配置”, with operation names
+matching the actual UI labels. English and Chinese copy must retain the same
+meaning and degree of formality. Update existing translation entries rather than
+adding duplicate messages or components.
+
+## Panel Visual Style
+
+Reuse `PageHeader`, the `Card` family, and the shared form and button components.
+Card titles, borders, corner radii, spacing, and save footers follow their shared
+definitions; avoid redefining these styles in individual pages. Use white card
+backgrounds in the light theme, restrained semantic colors, and 16px functional
+icons. Use `CardFooter` for save actions and `Button` for interactive controls.
+Tables, consoles, compact statistics, and status messages may retain spacing and
+colors suited to their content. Preserve the shared page margins and bottom
+spacing.
+
 ## Local Development
 
 You can do most day-to-day development on macOS or Linux without a full cluster.
@@ -78,8 +100,15 @@ FELIS_TEST_PG_URL='postgres://felis:***@127.0.0.1:5432/felis_pgint?sslmode=disab
 ```
 
 Run it after touching anything under `internal/api/pgrepo.go`, `internal/submit`,
-or `internal/build` that speaks SQL: the fakes encode the contract, and this
-suite exists to catch the drift between the fakes and the real queries.
+`internal/build` or `internal/dbbackup` that speaks SQL: the fakes encode the
+contract, and this suite exists to catch the drift between the fakes and the real
+queries. The `felis db backup` and `restore` tests also run `pg_dump`, `pg_restore`
+and `psql`, which must be the server's major version. For a server in a container,
+run them in it, as production does in felis-postgres:
+
+```bash
+FELIS_TEST_PG_EXEC='docker exec -i <container>' FELIS_TEST_PG_URL=... go test -tags pgint ./internal/pgint/
+```
 
 Build the CLI:
 
@@ -205,13 +234,13 @@ export FELIS_IMAGE=felis:dev
 export FELIS_ROOT_DOMAIN=<node-ip>.nip.io
 ```
 
-By default the installer builds the newest **published GitHub release**. While this
-repository is private that lookup — and the clone itself — needs a token, and building
-the development tip needs an opt-in:
+By default the installer builds the newest **published GitHub release**. Building the
+development tip needs an opt-in, and installing from a private fork additionally needs a
+token for the release lookup and the clone:
 
 ```bash
-export FELIS_GITHUB_TOKEN=<token with read access to the repo>
 export FELIS_VERSION_BOOTSTRAP=dev      # build main instead of the newest release
+export FELIS_GITHUB_TOKEN=<token>       # private forks only: read access to the fork
 ```
 
 `dev` is also the escape hatch before the first `vX.Y.Z` tag exists: with no published

Dockerfile

@@ -21,14 +21,18 @@
 # minutes. The FINAL stage is deliberately NOT pinned — it must stay on the target platform
 # or the published arm64 image would carry amd64 layers. It contains only COPY, which
 # BuildKit performs itself, so it needs no QEMU either; adding a RUN there would.
-FROM --platform=$BUILDPLATFORM node:22-bookworm AS panel
+#
+# Every base image here and in deploy/{limbo,lobby,paper} is pinned by digest, so a rebuild
+# of one release uses the same bytes; .github/dependabot.yml proposes the bumps (tag and
+# digest together).
+FROM --platform=$BUILDPLATFORM node:22-bookworm@sha256:363e1587494626837fa7f9a23bdb453d13b0ff3c67c705c2805cfc69c2d2fad7 AS panel
 WORKDIR /panel
 COPY panel/package*.json ./
 RUN npm ci
 COPY panel/ ./
 RUN npm run build
 
-FROM --platform=$BUILDPLATFORM golang:1.26 AS build
+FROM --platform=$BUILDPLATFORM golang:1.26@sha256:6c2a5538f964f1c82f97ad14988bf05de100d922d159d0e398b54c7b0ca0c6c9 AS build
 WORKDIR /src
 ARG TARGETOS=linux
 ARG TARGETARCH
@@ -55,7 +59,7 @@ ARG FELIS_VERSION=dev
 RUN CGO_ENABLED=0 GOOS="$TARGETOS" GOARCH="${TARGETARCH:-$(go env GOARCH)}" \
     go build -trimpath -ldflags="-s -w -X main.version=${FELIS_VERSION}" -o /out/felis ./cmd/felis
 
-FROM gcr.io/distroless/static-debian12:nonroot
+FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
 ENV PATH=/usr/local/bin:/usr/bin:/bin
 COPY --chmod=0755 --from=build /out/felis /usr/local/bin/felis
 # distroless "nonroot" is uid 65532; the rendered PodSecurityContext pins

README.md

@@ -1,11 +1,25 @@
-# Felis
+<div align="center">
+  <h1 align="center">
+    <img src="docs/assets/felis-logo.png" alt="Felis logo" width="270"><br>
+    Felis
+  </h1>
+  <p align="center">
+    基于 Kubernetes 的 Minecraft 服务器托管平台<br>
+    单条命令完成部署，自动管理服务器生命周期、备份与安全
+    <br><br>
+    <a href="README.md">简体中文</a> | <a href="README_EN.md">English</a>
+    <br>
+    <a href="https://felismc.com/">官网</a> | <a href="https://docs.felismc.com/">文档</a>
+  </p>
+</div>
 
-一款 Kubernetes 驱动的 Minecraft 服务器托管平台，一行命令部署，自动管理生命周期与安全。  
-A Kubernetes-driven Minecraft server hosting platform — one command to deploy, automatic lifecycle, backup, and security.
+> [!CAUTION]
+> **当前仅提供开发快照，已撤下公开 Release。仅供开发者在隔离的测试环境中验证，不建议普通用户部署。**
+> **此项目仍处于早期开发阶段，您不该在任何生产环境使用该项目。若产生任何问题，贵用户的使用行为与 FelisMC 团队无任何民事刑事法律关系。**<br>
+> **THIS PROJECT IS STILL WIP, YOU SHOULD DO NOT USE THIS PROJECT IN ANY PRODUCTION USAGE. WE ARE NOT RESPOND FOR ANY LEGAL OR HUMANLY PROBLEM.**
 
-[简体中文](README.md) | [English](README_EN.md)
-
-目录
+<details>
+<summary>目录</summary>
 
 - [特性](#特性)
 - [使用方式](#使用方式)
@@ -13,47 +27,79 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy,
 - [开源协议](#开源协议)
 - [致谢](#致谢)
 
+</details>
+
 ## 特性
 
-- **即开即玩**：玩家尝试连接时自动唤醒服务器，空闲后自动休眠，像游戏主机一样省资源。
-- **Web 控制面板**：浏览器中查看服务器状态、在线玩家与资源用量，管理备份与恢复。
-- **备份与恢复**：一键把整服数据（世界、配置、插件/模组，即整个 /data 卷）打包进集群内的归档库，支持从任意备份点回滚；默认安装就已启用（归档 PVC 与路径由安装器一并生成）。
-- **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。
-- **多核心支持**：兼容 Paper、Fabric、Forge、NeoForge，经由 Velocity 代理统一入口。
-- **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建；构建产物进入镜像白名单，可直接选用为服务器镜像完成部署。
-- **Passkey 登录**：支持指纹、面容、硬件密钥等无密码认证方式。
-- **零信任安全**：面板流量由 Cloudflare Access 保护，集群内 API 不暴露到公网。
+* **按需启停**：玩家连接代理时自动启动目标服务器，启动期间玩家进入等待队列，服务器就绪后自动传送；服务器空闲后自动停止，释放内存。
+
+* **Web 控制面板**：在浏览器中查看服务器状态、在线玩家与资源用量。
+  * 控制台（RCON）、白名单、封禁、OP 与 LuckPerms 权限管理
+  * 文件管理：新建、删除、重命名、分片上传、下载，以及停服状态下解压 zip，可用于导入世界
+  * 计划任务：按星期与时区定时执行命令、重启、停止、启动或备份，执行前在游戏内向玩家发送提醒
+
+* **备份与恢复**：默认启用，归档 PVC 及其路径由安装器生成。
+  * 手动备份：将服务器的完整数据卷（`/data`，含世界、配置、插件与模组）归档至集群内的归档存储，可回滚至任一备份点
+  * 每日恢复点：当天有玩家进入过的服务器在停止后自动生成恢复点，默认保留 7 个，保存期限 90 天；恢复点单独轮换，不影响手动备份
+  * 下载与导出：支持下载单个备份（附 sha256 校验）、删除单个备份及导出完整世界
+  * 异地副本（可选）：备份在主机上加密后同步至 S3 兼容存储（AWS S3、Cloudflare R2、Backblaze B2、MinIO 等）
+  * 控制面数据库：存放账号、服务器归属、配额与存档索引的数据库每日自动备份，每次升级迁移前额外创建快照，故障时可通过 `felis db restore` 整库原子回滚；面板「维护与备份」页显示最近一次备份的时效（参见 [故障排查 §16](docs/troubleshooting.md)）
+
+* **运维诊断**
+  * `sudo felis status`：汇总显示节点、控制面、游戏代理、各服务器、备份及未解决的告警
+  * `sudo felis doctor`：执行全部健康检查，按模块列出问题及排查方向，执行过程中不发送邮件
+  * `sudo felis support-bundle`：生成已脱敏的诊断包，供提交问题时附带（参见 [故障排查 §0](docs/troubleshooting.md)）
+  * 看门狗：每 2 分钟执行一次巡检，异常持续时向平台所有者发送邮件告警，支持外部心跳监测
+
+* **世界回收（可选）**：超过 15 天无人游玩的世界在备份后删除，以释放磁盘空间。安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认为 `/var/lib/rancher/k3s/storage`）即启用每日回收；未设置时不删除任何世界。过期备份的每日清理与此设置无关，始终执行。
+
+* **多核心支持**：兼容 Paper、Fabric、Forge 与 NeoForge，统一经由 Velocity 代理接入。
+
+* **模组包投稿**：玩家可上传模组包，经服主审批后自动构建并通过 Trivy 安全扫描；构建产物加入镜像白名单后，可直接选作服务器镜像。
+
+* **安全**
+  * Passkey 登录：支持指纹、面容识别及硬件密钥等无密码认证方式
+  * 零信任访问：面板流量经 Cloudflare Access 保护，集群内部 API 不对公网开放
+
+* **多机部署（实验性，默认关闭）**：由一台主控节点统一下发指令，其余节点仅运行游戏服务器，已停止的服务器可迁移至其他节点。该功能目前仅位于 main 分支，尚未完成三机验收（参见 [多机部署](docs/distributed.md)）。
 
 ## 使用方式
 
-在准备好的 Linux 主机上执行：
+开发测试需在已准备好的 Linux 测试主机上显式选择 `dev` 通道（跟随 `main`，从源码构建）：
 
 ```bash
-curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
+curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo FELIS_VERSION_BOOTSTRAP=dev bash
 ```
 
-脚本将自动安装 K3s、部署控制平面并启动设置向导。完成后浏览器访问已配置的域名进入控制面板即可使用。
+脚本将安装 K3s，在 K3s 中部署 PostgreSQL 与控制平面，随后启动设置向导。设置完成后，通过浏览器访问所配置的域名即可进入控制面板。
 
-> **本仓库当前为私有**，上面这条会返回 404。请改用带凭据的形式；安装器自身也需要同一个 token
-> 去解析并下载 release，所以用 `sudo -E` 把它带进去：
->
-> ```bash
-> export FELIS_GITHUB_TOKEN=<对本仓库有读权限的 token>
-> printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \
->   | curl -fsSL --config - -H "Accept: application/vnd.github.raw" \
->       https://api.github.com/repos/FelisMC/Felis/contents/deploy/bootstrap.sh \
->   | sudo -E bash
-> ```
->
-> token 经 stdin 交给 `curl --config -`，不放在命令行上：argv 在 `/proc` 下对本机任意用户可读，
-> 而这正是安装器内部 `github_api` 采用同一写法的原因。
+* **设置向导**：先完成面板访问方式与存储配置，再由主机管理员创建首位 Owner，终端会给出一次性网页设置链接（30 分钟内有效）。用浏览器打开链接、登记邮箱并创建通行密钥，即可进入面板，无需启动 Minecraft。角色关联可稍后在“账户”页选择认证源、输入角色名或 UUID 并确认；普通玩家继续通过游戏绑定码验证身份。未完成网页登录或链接过期时，再次执行 `sudo felis setup` 会提供新链接；已设置登录凭证的账号不会被重置。登录服或大厅故障不会阻止面板初始化。“账户”页会解释世界树（Yggdrasil）认证、显示进服地址与登录服/大厅所需的 Java 版客户端版本；安装器会把实际构建版本写入 `[velocity].game_version`，自定义镜像需自行填写，未配置时不会猜测版本。标准世界树接口可直接查询角色；非标准 `hasJoined` 地址可通过 `[[auth_source]].api_url` 指定认证站 API 根地址，游戏绑定码仍可作为替代方式。安装器仅在交互式终端中自动启动向导；输出重定向至日志或经由 cloud-init 安装时，请在安装结束后执行 `sudo felis setup`。设置 `FELIS_NO_SETUP=1` 时，安装器在输出摘要后直接结束。
 
-重跑这条命令也是把 felis-api 升到新版本的方式（`felis setup` 做不到，它用的是本机已有的二进制）。
-重跑会沿用已安装的根域名，但**不会**沿用通道：若本机跟随 main，需一并 `export FELIS_VERSION_BOOTSTRAP=dev`。
+* **认证源管理**：Owner 可在左侧“平台 → 认证源”添加、编辑、排序、停用第三方 Yggdrasil 认证站，并测试认证接口。保存后立即用于下一次登录和角色查询，无需重启；面板配置优先于安装配置。已保存的永久标识不能改名或删除，以保留玩家 UUID 与账号绑定；不再使用的源可停用。Mojang 始终优先验证。Nano 继续使用 TOML 配置。
+
+* **支持的系统**：CentOS Stream 9（aarch64）已在实机上验证；Ubuntu 24.04（x86_64）在每次推送时由 CI 验证开发构建的全新安装与重复安装（参见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
+
+* **安装前检查**：安装器在修改主机之前检查内存、磁盘、端口、网段冲突、已有的 Kubernetes 及外网连通性。发现问题时一次性列出全部问题并退出，主机保持原状（检查项参见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
+
+* **升级**：重新执行安装命令即可将 felis-api 升级至新版本；`felis setup` 仅使用本机已安装的二进制，无法用于升级。重新执行时沿用已安装的根域名，发布通道需重新指定：跟随 main 分支的主机须同时设置 `export FELIS_VERSION_BOOTSTRAP=dev`。早期版本安装在宿主机上的 PostgreSQL 会在重新执行时整库迁入 K3s，宿主机上的原实例停用并保留，以便回退（参见 [运维手册 §4](docs/operations.md#4-upgrading-the-pieces-around-felis)）。
+
+<details>
+<summary>安装来源与受限网络环境下的安装</summary>
+<br>
+
+以下说明发布机制；当前暂无公开 Release 附件，开发测试使用源码构建。
+
+安装发布版时，二进制文件、全部镜像及 Velocity 插件均取自该版本由 CI 预构建的 release 附件，逐一校验 `SHA256SUMS` 后导入。主机无需安装 Docker、Gradle 或 Go，也无需访问 Docker Hub。若某个附件缺失或校验失败，仅该镜像回退为本机构建，并输出提示（参见 [故障排查 §15c](docs/troubleshooting.md)）。
+
+也可将附件预先复制到主机，再通过 `FELIS_ARTIFACT_DIR=<绝对路径>` 安装，此时 Felis 自身的二进制、镜像与插件均从该目录读取。k3s 及其镜像、JRE、cloudflared、Velocity 与 Via 插件仍从 GitHub 和 PaperMC 下载；RHEL、Fedora、openSUSE Leap 等启用 SELinux 的主机还需从 rpm.rancher.io 安装 k3s-selinux；系统软件包来自发行版软件源。
+
+因此，出站网络受限的主机须放行上述地址的 HTTPS 访问，或设置 `https_proxy`。preflight 会在修改主机之前逐一探测这些地址。目前暂不支持完全离线安装（地址清单参见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
+
+</details>
 
 ## 从源码构建
 
-本项目基于 Go 和 Node.js 开发：
+本项目基于 Go 与 Node.js 开发：
 
 ```bash
 # 后端（Go 1.26+）
@@ -70,22 +116,22 @@ docker build -t felis:custom .
 
 ## 开源协议
 
-本项目遵循 [AGPL-3.0-only](LICENSE) 开源协议。
+本项目采用 [AGPL-3.0-only](LICENSE) 许可证。
 
 ### 协议注意事项
 
-1. **衍生作品同样是 AGPL**：分发本项目的副本或基于本项目衍生的软件时，必须以 AGPL-3.0 开源，并保留原作者的版权声明和许可声明。
-2. **通过网络提供服务同样要开源**（AGPL 第 13 条）：如果你把修改过的 Felis 架起来给别人用，即使从不分发任何二进制，也必须向这些用户提供你那份修改后的完整源码。这是 AGPL 相对 GPL 的唯一实质区别，而 Felis 正是一个跑在网络上的托管平台，所以这一条基本总会触发。
-3. **免责声明**：本项目按"原样"提供，作者不承担任何因使用本项目而产生的法律责任。
+1. **衍生作品须采用 AGPL**：分发本项目副本或基于本项目的衍生软件时，须以 AGPL-3.0 开源，并保留原作者的版权声明与许可声明。
+2. **网络服务同样须提供源码**（AGPL 第 13 条）：通过网络向他人提供经修改的 Felis 服务时，即使未分发任何二进制文件，也须向这些用户提供修改后的完整源码。这是 AGPL 与 GPL 唯一的实质区别；Felis 作为通过网络访问的托管平台，几乎所有部署场景都适用此条款。
+3. **免责声明**：本项目按"原样"提供，作者不承担因使用本项目而产生的任何法律责任。
 
 ## 致谢
 
-- [Kubernetes](https://kubernetes.io/)：底层容器编排引擎
-- [K3s](https://k3s.io/)：轻量级 Kubernetes 发行版
-- [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/)：零信任安全基础设施
-- [PostgreSQL](https://www.postgresql.org/)：数据持久化
-- [React](https://react.dev/)：前端用户界面框架
-- [Vite](https://vitejs.dev/)：前端构建工具
-- [TailwindCSS](https://tailwindcss.com/)：CSS 框架
-- [Bubble Tea](https://github.com/charmbracelet/bubbletea)：TUI 框架
-- [Minecraft](https://www.minecraft.net/)：让这一切值得做
+* [Kubernetes](https://kubernetes.io/)：容器编排引擎
+* [K3s](https://k3s.io/)：轻量级 Kubernetes 发行版
+* [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/)：零信任安全基础设施
+* [PostgreSQL](https://www.postgresql.org/)：数据持久化
+* [React](https://react.dev/)：前端用户界面框架
+* [Vite](https://vitejs.dev/)：前端构建工具
+* [TailwindCSS](https://tailwindcss.com/)：CSS 框架
+* [Bubble Tea](https://github.com/charmbracelet/bubbletea)：TUI 框架
+* [Minecraft](https://www.minecraft.net/)：本项目服务的游戏

README_EN.md

@@ -1,11 +1,24 @@
-# Felis
+<div align="center">
+  <h1 align="center">
+    <img src="docs/assets/felis-logo.png" alt="Felis logo" width="270"><br>
+    Felis
+  </h1>
+  <p align="center">
+    A Kubernetes-driven Minecraft server hosting platform<br>
+    One command to deploy, with automatic lifecycle, backup, and security
+    <br><br>
+    <a href="README.md">简体中文</a> | <a href="README_EN.md">English</a>
+    <br>
+    <a href="https://felismc.com/">Website</a> | <a href="https://docs.felismc.com/en/">Documentation</a>
+  </p>
+</div>
 
-A Kubernetes-driven Minecraft server hosting platform — one command to deploy, automatic lifecycle, backup, and security.  
-一款 Kubernetes 驱动的 Minecraft 服务器托管平台，一行命令部署，自动管理生命周期与安全。
+> [!CAUTION]
+> **Only development snapshots are available; public releases have been withdrawn. These builds are for developers testing in isolated environments and are not recommended for general deployment.**
+> **THIS PROJECT IS STILL WIP, YOU SHOULD DO NOT USE THIS PROJECT IN ANY PRODUCTION USAGE. WE ARE NOT RESPOND FOR ANY LEGAL OR HUMANLY PROBLEM.**
 
-[简体中文](README.md) | [English](README_EN.md)
-
-Table of Contents
+<details>
+<summary>Table of Contents</summary>
 
 - [Features](#features)
 - [Getting Started](#getting-started)
@@ -13,47 +26,73 @@ Table of Contents
 - [License](#license)
 - [Acknowledgements](#acknowledgements)
 
+</details>
+
 ## Features
 
-- **Wake on Join**: Servers start automatically when a player connects, and stop when idle — like hibernate for your server.
-- **Web Dashboard**: Monitor server status, online players, and resource usage from your browser, with backup and restore management.
-- **Backup & Restore**: One-click snapshots of a server's whole data volume (worlds, config, plugins/mods — the entire /data volume) into the cluster's archive store, with rollback from any backup point — enabled by default (the installer renders the archive PVC and its path).
-- **World Reaper** (opt in): Worlds idle for more than 15 days are automatically backed up and removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is ever deleted.
-- **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, federated behind a Velocity proxy.
-- **Modpack Submission**: Players submit custom modpacks; admin approval triggers an automatic build, and the result is whitelisted as a server image you can select to deploy.
-- **Passkey Login**: Passwordless authentication via fingerprint, face recognition, or hardware security keys.
-- **Zero Trust Security**: Panel traffic protected by Cloudflare Access; the internal API is never exposed to the internet.
+* **On-demand Start and Stop**: A server starts when a player connects to the proxy. The player waits in a queue during start-up and is transferred once the server is ready. Idle servers stop automatically to free memory.
+
+* **Web Dashboard**: Monitor server status, online players, and resource usage from your browser.
+  * Console (RCON), whitelist, bans, OPs and LuckPerms permissions
+  * File manager: create, delete, rename, chunked upload, download, and unzip while the server is stopped; also used to import worlds
+  * Schedules: run commands, restart, stop, start or back up by weekday and time zone, with an in-game warning to players beforehand
+
+* **Backup & Restore**: Enabled by default; the installer renders the archive PVC and its path.
+  * Manual backups: archive a server's entire data volume (`/data`, including worlds, configuration, plugins and mods) to the cluster's archive store, with rollback to any backup point
+  * Daily restore points: a server played that day gets a restore point once it stops; by default 7 are kept for up to 90 days, rotated separately from manual backups
+  * Download and export: download a single backup (with sha256 verification), delete a single backup, or export a whole world
+  * Off-site copy (optional): backups are encrypted on the host and synced to S3-compatible storage (AWS S3, Cloudflare R2, Backblaze B2, MinIO and others)
+  * Control-plane database: the database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migration; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows the age of the latest backup (see [troubleshooting §16](docs/troubleshooting.md))
+
+* **Diagnostics**
+  * `sudo felis status`: a summary of the node, control plane, game proxy, each server, backups and open alerts
+  * `sudo felis doctor`: runs all health checks and lists problems by area with troubleshooting pointers; sends no email
+  * `sudo felis support-bundle`: generates a redacted diagnostics archive to attach to support requests (see [troubleshooting §0](docs/troubleshooting.md))
+  * Watchdog: runs a check every 2 minutes and emails the platform owners when a problem persists; supports an external heartbeat monitor
+
+* **World Reaper** (optional): Worlds idle for more than 15 days are backed up and then removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is deleted. Expired backups are cleaned up daily regardless of this setting.
+
+* **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, accessed through a single Velocity proxy.
+
+* **Modpack Submission**: Players can upload modpacks. After admin approval, each modpack is built automatically and scanned with Trivy; the result is added to the image whitelist and can be selected as a server image.
+
+* **Security**
+  * Passkey login: passwordless authentication via fingerprint, face recognition, or hardware security keys
+  * Zero-trust access: panel traffic is protected by Cloudflare Access, and the internal API is not exposed to the internet
+
+* **Multi-node Deployment** (experimental, off by default): a single controller node issues all commands, the other nodes run game servers only, and a stopped server can be migrated to another node. Currently available only on the main branch; three-node acceptance testing is not yet complete (see [distributed mode](docs/distributed.md), in Chinese).
 
 ## Getting Started
 
-On a prepared Linux host, run:
+For development testing on a prepared Linux test host, explicitly select the `dev` channel, which follows `main` and builds from source:
 
 ```bash
-curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
+curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo FELIS_VERSION_BOOTSTRAP=dev bash
 ```
 
-The script installs K3s, deploys the control plane, and launches a setup wizard. Once done, open your browser at the configured domain.
+The script installs K3s, deploys PostgreSQL and the control plane inside it, and launches a setup wizard. When setup completes, open the configured domain in a browser to reach the control panel.
 
-> **This repository is currently private**, so the command above returns 404. Use the
-> credentialed form instead; the installer itself needs the same token to resolve and
-> download the release, so pass it through with `sudo -E`:
->
-> ```bash
-> export FELIS_GITHUB_TOKEN=<a token with read access to this repository>
-> printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \
->   | curl -fsSL --config - -H "Accept: application/vnd.github.raw" \
->       https://api.github.com/repos/FelisMC/Felis/contents/deploy/bootstrap.sh \
->   | sudo -E bash
-> ```
->
-> The token reaches `curl --config -` over stdin instead of the command line: argv is
-> readable by any local user via `/proc`, and that is exactly why the installer's
-> internal `github_api` uses the same form.
+* **Setup wizard**: Configure panel access and storage first. The host administrator then initializes the first Owner and receives a one-time browser setup link (valid for 30 minutes). Open it, record an email, and create a passkey to enter the panel; Minecraft is not required. Later, link a game role from Account by selecting an authentication source, entering a role name or UUID, and confirming it. Players retain the in-game bind-code flow. Rerun `sudo felis setup` if login setup is unfinished or the link expires; accounts with an established login factor are never reset. Login/lobby failures do not block panel initialization. Account explains Yggdrasil authentication and shows the join address and Java client version for the login/lobby servers. Bootstrap records the built protocol in `[velocity].game_version`; set it yourself for custom images, otherwise the panel reports it as unknown. Standard Yggdrasil endpoints support role lookup directly; sources with a nonstandard `hasJoined` path can set `[[auth_source]].api_url` to their API root, with game-code linking still available as a fallback. The installer launches the wizard automatically only on an interactive terminal; when output is redirected to a log or the install runs under cloud-init, run `sudo felis setup` after it finishes. Setting `FELIS_NO_SETUP=1` makes the installer end at its summary.
 
-Rerunning this command is also how you upgrade felis-api to a newer version (`felis setup`
-cannot — it uses the binary already installed on the host). The rerun keeps the installed
-root domain but **not** the channel: if this host follows main, also
-`export FELIS_VERSION_BOOTSTRAP=dev`.
+* **Supported hosts**: CentOS Stream 9 (aarch64) is verified on physical hardware; CI tests fresh installation and repeat installation of development builds on Ubuntu 24.04 (x86_64) on every push (see [operations §1](docs/operations.md#1-supported-hosts)).
+
+* **Preflight checks**: Before modifying the host, the installer checks memory, disk, ports, network range conflicts, existing Kubernetes installations and outbound connectivity. If any check fails, it lists all problems and exits, leaving the host unchanged (see [operations §1](docs/operations.md#1-supported-hosts) for the checks).
+
+* **Upgrading**: Rerun the install command to upgrade felis-api to a newer version; `felis setup` only uses the binary already installed on the host and cannot upgrade it. A rerun keeps the installed root domain, and the release channel must be specified again: hosts that follow the main branch must also set `export FELIS_VERSION_BOOTSTRAP=dev`. A PostgreSQL instance installed on the host by an earlier release is migrated into K3s during the rerun; the original instance on the host is stopped and retained for rollback (see [operations §4](docs/operations.md#4-upgrading-the-pieces-around-felis)).
+
+<details>
+<summary>Installation sources and restricted networks</summary>
+<br>
+
+The following describes the release mechanism. No public release assets are currently available; development testing uses source builds.
+
+A release installation takes the binary, all images and the Velocity plugin from the release assets prebuilt in CI, verifying each against `SHA256SUMS` before import. The host requires no Docker, Gradle or Go, and no access to Docker Hub. If an asset is missing or fails verification, only that image falls back to a local build, and the installer prints a notice (see [troubleshooting §15c](docs/troubleshooting.md)).
+
+The assets can also be copied to the host in advance and installed with `FELIS_ARTIFACT_DIR=<absolute path>`; the Felis binary, images and plugin are then read from that directory. k3s and its images, the JRE, cloudflared, Velocity and the Via plugins are still downloaded from GitHub and PaperMC; hosts with SELinux enabled, such as RHEL, Fedora and openSUSE Leap, additionally install k3s-selinux from rpm.rancher.io; system packages come from the distribution's repositories.
+
+A host with restricted outbound access must therefore allow HTTPS to these addresses or set `https_proxy`. Preflight probes each address before changing the host. Fully offline installation is not yet supported (see [operations §1](docs/operations.md#1-supported-hosts) for the address list).
+
+</details>
 
 ## Build from Source
 
@@ -74,22 +113,22 @@ docker build -t felis:custom .
 
 ## License
 
-The source code is released under [AGPL-3.0-only](LICENSE).
+This project is licensed under [AGPL-3.0-only](LICENSE).
 
 ### License Notes
 
-1. **Derivative works are AGPL too**: Any distribution of this project or of software derived from it must be released under AGPL-3.0 and must include the original copyright notice and license statement.
-2. **Running it as a network service also triggers the source obligation** (AGPL section 13): if you host a modified Felis for other people to use, you must offer those users the complete source of your modified version — even if you never distribute a binary. This is the one substantive difference between AGPL and GPL, and since Felis is a hosting platform reached over a network, it will essentially always apply.
+1. **Derivative works must use AGPL**: Any distribution of this project or of software derived from it must be released under AGPL-3.0 and must include the original copyright notice and license statement.
+2. **Network services must also provide source** (AGPL section 13): anyone who offers a modified Felis to others over a network must provide those users with the complete source of the modified version, even without distributing any binary. This is the only substantive difference between AGPL and GPL; as Felis is a hosting platform accessed over a network, this clause applies to virtually every deployment.
 3. **Disclaimer**: This project is provided "as is", without warranty of any kind.
 
 ## Acknowledgements
 
-- [Kubernetes](https://kubernetes.io/): Container orchestration engine
-- [K3s](https://k3s.io/): Lightweight Kubernetes distribution
-- [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/): Zero trust security infrastructure
-- [PostgreSQL](https://www.postgresql.org/): Data persistence
-- [React](https://react.dev/): User interface framework
-- [Vite](https://vitejs.dev/): Frontend build tool
-- [TailwindCSS](https://tailwindcss.com/): CSS framework
-- [Bubble Tea](https://github.com/charmbracelet/bubbletea): TUI framework
-- [Minecraft](https://www.minecraft.net/): What makes this all worthwhile
+* [Kubernetes](https://kubernetes.io/): Container orchestration engine
+* [K3s](https://k3s.io/): Lightweight Kubernetes distribution
+* [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/): Zero trust security infrastructure
+* [PostgreSQL](https://www.postgresql.org/): Data persistence
+* [React](https://react.dev/): User interface framework
+* [Vite](https://vitejs.dev/): Frontend build tool
+* [TailwindCSS](https://tailwindcss.com/): CSS framework
+* [Bubble Tea](https://github.com/charmbracelet/bubbletea): TUI framework
+* [Minecraft](https://www.minecraft.net/): The game this project serves

bootstrap_asset.go

@@ -22,15 +22,18 @@ var bootstrapAssets embed.FS
 // developer's working tree carries gradle output (plugins/*/build, plugins/*/bin,
 // and for the modded loaders a decompiled Minecraft under build/) which would
 // otherwise be baked into every felis binary. Keep them explicit — add a source
-// directory here, never a parent.
+// directory here, never a parent. Each module's gradle/verification-metadata.xml rides
+// along, since Gradle builds unverified without it; the wrapper stays out, because the
+// image builds run the pinned build image's own gradle.
 //
+//go:embed deploy/game-stack.lock
 //go:embed deploy/limbo/Dockerfile deploy/limbo/entrypoint.sh
 //go:embed deploy/lobby/Dockerfile deploy/lobby/entrypoint.sh
 //go:embed deploy/paper/Dockerfile deploy/paper/entrypoint.sh
-//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src
-//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src
-//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src
-//go:embed plugins/shared/src
+//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src plugins/limbo/gradle/verification-metadata.xml
+//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src plugins/paper/gradle/verification-metadata.xml
+//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src plugins/velocity/gradle/verification-metadata.xml
+//go:embed plugins/shared/src plugins/shared/build-progress.gradle
 var gameStackAssets embed.FS
 
 // GameStackTar streams the embedded game-stack sources as a tar, rooted so that

bootstrap_asset_test.go

@@ -1,7 +1,9 @@
 package felis
 
 import (
+	"encoding/xml"
 	"io/fs"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -145,6 +147,9 @@ func TestBootstrapPinsViaBlockConnectionsOff(t *testing.T) {
 // inputs from the script and from each Dockerfile's own COPY lines instead of restating
 // them here; a fourth image inherits the check for free.
 func TestGameStackTarCarriesEveryBuildInput(t *testing.T) {
+	// The plugin builds invoke this shared init script after COPYing the directory;
+	// a directory entry alone would pass the COPY check even if the script was omitted.
+	requireEmbedded(t, "plugins/shared/build-progress.gradle")
 	// Matches the path only when GAME_STACK_DIR is followed by one, which skips the
 	// build-context arguments (`"$GAME_STACK_DIR"`, `"${GAME_STACK_DIR}:/src:z"`) and
 	// the glob for gradle's output, none of which are inputs this tar has to carry.
@@ -205,6 +210,299 @@ func requireEmbedded(t *testing.T, path string) {
 	}
 }
 
+// The lock file is the install's only source of upstream builds, and bootstrap.sh reads it
+// with a strict KEY=value parser that dies on anything unexpected, so a malformed lock is a
+// failed install on every host. Check the shipped copy the same way here.
+func TestGameStackLockIsComplete(t *testing.T) {
+	lock := gameStackLock(t)
+	m := regexp.MustCompile(`GAME_STACK_LOCK_KEYS="([^"]*)"`).FindStringSubmatch(BootstrapScript())
+	if m == nil {
+		t.Fatal("bootstrap.sh no longer declares GAME_STACK_LOCK_KEYS")
+	}
+	keys := strings.Fields(m[1])
+	sha := regexp.MustCompile(`^[0-9a-f]{64}$`)
+	for _, k := range keys {
+		v, ok := lock[k]
+		if !ok || v == "" {
+			t.Errorf("game-stack.lock does not set %s", k)
+			continue
+		}
+		if strings.HasSuffix(k, "_SHA256") && !sha.MatchString(v) {
+			t.Errorf("%s=%q is not a lowercase sha256", k, v)
+		}
+		// A moving URL pins nothing: the digest check would start failing the day
+		// upstream publishes the next build.
+		if strings.HasSuffix(k, "_URL") && strings.Contains(v, "lastSuccessfulBuild") {
+			t.Errorf("%s names a moving build: %s", k, v)
+		}
+	}
+	for k := range lock {
+		if !strings.Contains(" "+m[1]+" ", " "+k+" ") {
+			t.Errorf("game-stack.lock sets %s, which bootstrap.sh refuses as an unknown key", k)
+		}
+	}
+	// Fill's URLs are content-addressed; a lock whose digest disagrees with its own URL
+	// was edited by hand and half-way.
+	for _, name := range []string{"PAPER", "VELOCITY"} {
+		if !strings.Contains(lock[name+"_JAR_URL"], "/objects/"+lock[name+"_JAR_SHA256"]+"/") {
+			t.Errorf("%s_JAR_SHA256 is not the digest in %s_JAR_URL", name, name)
+		}
+	}
+	if !strings.Contains(lock["LIMBO_JAR_URL"], "-"+lock["MC_VERSION"]+".jar") {
+		t.Errorf("LIMBO_JAR_URL %s is not a Minecraft %s build", lock["LIMBO_JAR_URL"], lock["MC_VERSION"])
+	}
+	if !strings.Contains(lock["PAPER_JAR_URL"], "/paper-"+lock["MC_VERSION"]+"-") {
+		t.Errorf("PAPER_JAR_URL %s is not a Minecraft %s build; the lobby would not speak the login gate's protocol", lock["PAPER_JAR_URL"], lock["MC_VERSION"])
+	}
+}
+
+// Each downloaded jar's digest is a build-arg bootstrap.sh passes and the Dockerfile must
+// both require and spend on the file it downloaded; docker only warns about an unknown
+// --build-arg, so a renamed arg would ship an unchecked jar.
+func TestGameStackDigestsReachTheImageBuilds(t *testing.T) {
+	script := BootstrapScript()
+	for _, c := range []struct{ dockerfile, arg, path string }{
+		{"deploy/limbo/Dockerfile", "LIMBO_JAR_SHA256", "/limbo/Limbo.jar"},
+		{"deploy/limbo/Dockerfile", "LIMBO_SCHEM_SHA256", "/limbo/spawn.schem"},
+		{"deploy/lobby/Dockerfile", "LUCKPERMS_JAR_SHA256", "/paper/plugins/LuckPerms.jar"},
+	} {
+		if !strings.Contains(script, "--build-arg "+c.arg+"=\"$"+c.arg+"\"") {
+			t.Errorf("bootstrap.sh never passes --build-arg %s", c.arg)
+		}
+		dockerfile := readGameStackFile(t, c.dockerfile)
+		if !strings.Contains(dockerfile, "ARG "+c.arg) {
+			t.Errorf("%s declares no ARG %s", c.dockerfile, c.arg)
+		}
+		if !strings.Contains(dockerfile, `echo "$`+c.arg+`  `+c.path+`" | sha256sum -c`) {
+			t.Errorf("%s never verifies %s against %s", c.dockerfile, c.path, c.arg)
+		}
+	}
+}
+
+// A base image named by tag alone is whatever the tag points at on build day.
+func TestDockerfileBaseImagesArePinnedByDigest(t *testing.T) {
+	root, err := os.ReadFile("Dockerfile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	files := map[string]string{"Dockerfile": string(root)}
+	for _, name := range []string{"deploy/limbo/Dockerfile", "deploy/lobby/Dockerfile", "deploy/paper/Dockerfile"} {
+		files[name] = readGameStackFile(t, name)
+	}
+	pinned := regexp.MustCompile(`^FROM (--platform=\S+ )?\S+:\S+@sha256:[0-9a-f]{64}( AS \S+)?$`)
+	for name, body := range files {
+		n := 0
+		for line := range strings.SplitSeq(body, "\n") {
+			if !strings.HasPrefix(line, "FROM ") {
+				continue
+			}
+			n++
+			if !pinned.MatchString(line) {
+				t.Errorf("%s: %q is not pinned by digest", name, line)
+			}
+		}
+		if n == 0 {
+			t.Errorf("%s has no FROM line", name)
+		}
+	}
+}
+
+// The plugin jars are built in four places the installer controls — the lobby and limbo
+// image builds, bootstrap's Velocity build and the release build of the same jar — and
+// through each module's wrapper by a developer or CI. A tag alone is whatever it points
+// at on build day, and two Gradle versions are two chances for a build to pass in one
+// place and break in the other, so all of them run one image, pinned by digest, whose
+// Gradle is the wrappers' Gradle.
+func TestPluginBuildsRunOnePinnedGradle(t *testing.T) {
+	sources := map[string]string{
+		"deploy/lobby/Dockerfile": readGameStackFile(t, "deploy/lobby/Dockerfile"),
+		"deploy/limbo/Dockerfile": readGameStackFile(t, "deploy/limbo/Dockerfile"),
+		"deploy/bootstrap.sh":     BootstrapScript(),
+		// The release build compiles felis-velocity.jar for hosts that install prebuilt.
+		"deploy/build-release-artifacts.sh": readRepoFile(t, "deploy/build-release-artifacts.sh"),
+	}
+	anyRef := regexp.MustCompile(`gradle:[\w.-]+(@sha256:\w+)?`)
+	pinned := regexp.MustCompile(`^gradle:(\d+\.\d+(?:\.\d+)?)-jdk\d+@sha256:[0-9a-f]{64}$`)
+	images := map[string]bool{}
+	gradle := ""
+	for name, body := range sources {
+		refs := anyRef.FindAllString(body, -1)
+		if len(refs) == 0 {
+			t.Errorf("%s names no gradle image", name)
+		}
+		for _, ref := range refs {
+			m := pinned.FindStringSubmatch(ref)
+			if m == nil {
+				t.Errorf("%s: %s is not a gradle image pinned by digest", name, ref)
+				continue
+			}
+			images[ref] = true
+			gradle = m[1]
+		}
+	}
+	if len(images) != 1 {
+		t.Fatalf("the plugin builds use %d different gradle images, want one: %v", len(images), images)
+	}
+	// bootstrap names the image once and has to spend it where it builds the jar.
+	if !strings.Contains(BootstrapScript(), `"$PLUGIN_BUILD_IMAGE" gradle --no-daemon clean build`) {
+		t.Error("build_velocity_plugin does not build in $PLUGIN_BUILD_IMAGE")
+	}
+
+	for _, module := range []string{"velocity", "paper", "limbo"} {
+		props := wrapperProperties(t, module)
+		if want := "gradle-" + gradle + "-bin.zip"; !strings.HasSuffix(props["distributionUrl"], "/"+want) {
+			t.Errorf("plugins/%s wrapper runs %s; the image builds run Gradle %s", module, props["distributionUrl"], gradle)
+		}
+	}
+	// The mods are no part of the install, but a wrapper without a checksum runs whatever
+	// the download handed it.
+	for _, module := range []string{"velocity", "paper", "limbo", "fabric", "forge", "neoforge"} {
+		if sum := wrapperProperties(t, module)["distributionSha256Sum"]; !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(sum) {
+			t.Errorf("plugins/%s wrapper pins no distribution sha256 (got %q)", module, sum)
+		}
+	}
+}
+
+// Each plugin compiles against the API of the exact build the install runs, and Gradle
+// checks those bytes against the module's verification file. Nothing but this test ties
+// the three to deploy/game-stack.lock: a lock refresh that leaves them behind builds the
+// lobby against yesterday's API, or fails every image build on a checksum the file does
+// not have.
+func TestPluginApisAreTheLockedBuilds(t *testing.T) {
+	lock := gameStackLock(t)
+
+	// Paper: paper-<mc>-<build>.jar runs; paper-api <mc>.build.<build>-<channel> compiles.
+	jar := regexp.MustCompile(`/paper-([^/]+)-(\d+)\.jar$`).FindStringSubmatch(lock["PAPER_JAR_URL"])
+	if jar == nil {
+		t.Fatalf("PAPER_JAR_URL %s does not name paper-<mc>-<build>.jar", lock["PAPER_JAR_URL"])
+	}
+	dep := regexp.MustCompile(`compileOnly 'io\.papermc\.paper:paper-api:([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/paper/build.gradle"))
+	if dep == nil {
+		t.Fatal("plugins/paper/build.gradle declares no paper-api dependency")
+	}
+	if !regexp.MustCompile(`^` + regexp.QuoteMeta(jar[1]+".build."+jar[2]) + `(-[a-z]+)?$`).MatchString(dep[1]) {
+		t.Errorf("paper-api %s is not the API of the locked server paper-%s-%s.jar", dep[1], jar[1], jar[2])
+	}
+	requireVerified(t, "paper", "io.papermc.paper", "paper-api", dep[1])
+
+	// Limbo: the lock's release, passed to the image build, which refuses to guess one.
+	limbo := lock["LIMBO_VERSION"]
+	if !strings.Contains(BootstrapScript(), `--build-arg LIMBO_VERSION="$LIMBO_VERSION"`) {
+		t.Error("bootstrap.sh does not pass the locked LIMBO_VERSION to the limbo image build")
+	}
+	dockerfile := readGameStackFile(t, "deploy/limbo/Dockerfile")
+	if !regexp.MustCompile(`(?m)^ARG LIMBO_VERSION$`).MatchString(dockerfile) ||
+		!strings.Contains(dockerfile, `if [ -z "${LIMBO_VERSION:-}" ]`) {
+		t.Error("deploy/limbo/Dockerfile does not require LIMBO_VERSION; a build without it would " +
+			"compile against a version nobody chose")
+	}
+	// LOOHP publishes the jar the login gate runs as the Limbo API artifact itself, so the
+	// checksum Gradle holds for it is the lock's: compiled-against and running are one file.
+	if got := requireVerified(t, "limbo", "com.loohp", "Limbo", limbo)["Limbo-"+limbo+".jar"]; got != lock["LIMBO_JAR_SHA256"] {
+		t.Errorf("verification-metadata.xml holds %q for Limbo-%s.jar; the login gate runs %s", got, limbo, lock["LIMBO_JAR_SHA256"])
+	}
+
+	// Velocity: the API default is the proxy the install runs.
+	api := regexp.MustCompile(`findProperty\('velocityApi'\) \?: '([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/velocity/build.gradle"))
+	if api == nil {
+		t.Fatal("plugins/velocity/build.gradle has no velocityApi default")
+	}
+	if api[1] != lock["VELOCITY_VERSION"] {
+		t.Errorf("velocity-api defaults to %s; the install runs Velocity %s", api[1], lock["VELOCITY_VERSION"])
+	}
+	requireVerified(t, "velocity", "com.velocitypowered", "velocity-api", api[1])
+}
+
+// requireVerified asserts the module's shipped verification file checks metadata and
+// pins group:name:version, and returns that component's artifact sha256s by file name.
+func requireVerified(t *testing.T, module, group, name, version string) map[string]string {
+	t.Helper()
+	path := "plugins/" + module + "/gradle/verification-metadata.xml"
+	var doc struct {
+		VerifyMetadata bool `xml:"configuration>verify-metadata"`
+		Components     []struct {
+			Group     string `xml:"group,attr"`
+			Name      string `xml:"name,attr"`
+			Version   string `xml:"version,attr"`
+			Artifacts []struct {
+				Name   string `xml:"name,attr"`
+				SHA256 []struct {
+					Value string `xml:"value,attr"`
+				} `xml:"sha256"`
+			} `xml:"artifact"`
+		} `xml:"components>component"`
+	}
+	if err := xml.Unmarshal([]byte(readGameStackFile(t, path)), &doc); err != nil {
+		t.Fatalf("%s: %v", path, err)
+	}
+	if !doc.VerifyMetadata {
+		t.Errorf("%s does not verify metadata; a swapped pom could redirect the graph", path)
+	}
+	for _, c := range doc.Components {
+		if c.Group != group || c.Name != name || c.Version != version {
+			continue
+		}
+		sums := map[string]string{}
+		for _, a := range c.Artifacts {
+			if len(a.SHA256) > 0 {
+				sums[a.Name] = a.SHA256[0].Value
+			}
+		}
+		if sums[name+"-"+version+".jar"] == "" {
+			t.Errorf("%s pins %s:%s:%s but no sha256 for its jar", path, group, name, version)
+		}
+		return sums
+	}
+	t.Errorf("%s has no checksum for %s:%s:%s; the build would refuse it", path, group, name, version)
+	return nil
+}
+
+// wrapperProperties reads a module's gradle-wrapper.properties off disk: the wrappers are
+// for developers and CI, and nothing embeds them.
+func wrapperProperties(t *testing.T, module string) map[string]string {
+	t.Helper()
+	b, err := os.ReadFile("plugins/" + module + "/gradle/wrapper/gradle-wrapper.properties")
+	if err != nil {
+		t.Fatal(err)
+	}
+	props := map[string]string{}
+	for line := range strings.SplitSeq(string(b), "\n") {
+		if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok && !strings.HasPrefix(k, "#") {
+			props[k] = strings.ReplaceAll(v, `\:`, ":")
+		}
+	}
+	return props
+}
+
+// gameStackLock parses the shipped deploy/game-stack.lock the way bootstrap.sh does.
+func gameStackLock(t *testing.T) map[string]string {
+	t.Helper()
+	lock := map[string]string{}
+	for line := range strings.SplitSeq(readGameStackFile(t, "deploy/game-stack.lock"), "\n") {
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			t.Fatalf("not a KEY=value line: %q", line)
+		}
+		lock[k] = v
+	}
+	return lock
+}
+
+// readRepoFile reads a file of the checkout that the binary does not embed.
+func readRepoFile(t *testing.T, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
 func readGameStackFile(t *testing.T, name string) string {
 	t.Helper()
 	b, err := gameStackAssets.ReadFile(name)

cmd/felis/api.go

@@ -5,10 +5,14 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"felis.lolicon.best/internal/api"
@@ -16,20 +20,30 @@ import (
 	"felis.lolicon.best/internal/backupjob"
 	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
 	"felis.lolicon.best/internal/fileedit"
+	"felis.lolicon.best/internal/imagepin"
 	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/metrics"
 	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/nodecontrol"
 	"felis.lolicon.best/internal/panel"
 	"felis.lolicon.best/internal/passkey"
+	"felis.lolicon.best/internal/placement"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/reaper"
+	"felis.lolicon.best/internal/registryprune"
 	"felis.lolicon.best/internal/restore"
-	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/retention"
 	"felis.lolicon.best/internal/submit"
+	"felis.lolicon.best/internal/worldexport"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -39,6 +53,22 @@ import (
 // the code marks Identity (UUIDs trusted verbatim); config can never add another.
 const mojangSessionServer = "https://sessionserver.mojang.com/session/minecraft/hasJoined"
 
+// passkeyRelyingParty is the one WebAuthn relying party both web faces share: its id
+// is the player console host, derived from server.root_domain the way the panel
+// handler derives it when auth.panel_hostname is unset, and its origins are that host
+// plus the operator host. An empty id means the install names no panel host at all.
+func passkeyRelyingParty(cfg *config.Config) (string, []string) {
+	rpID := defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname)
+	if rpID == "" {
+		return "", nil
+	}
+	origins := []string{"https://" + rpID, fmt.Sprintf("https://%s:%d", rpID, setupPanelNodePort())}
+	if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != rpID {
+		origins = append(origins, "https://"+admin, fmt.Sprintf("https://%s:%d", admin, setupPanelNodePort()))
+	}
+	return rpID, origins
+}
+
 // authSourcesFromConfig builds the multiplexer's priority list from the configured
 // [[auth_source]] entries: Mojang leads as the code-owned identity anchor (正版优先, the ONLY
 // Identity source — config can only append namespace-rewritten third-party sources, never a
@@ -48,16 +78,14 @@ func authSourcesFromConfig(configured []config.AuthSourceConfig) []api.AuthSourc
 	sources := make([]api.AuthSource, 0, len(configured)+1)
 	sources = append(sources, api.AuthSource{Tag: "mojang", URL: mojangSessionServer, Identity: true})
 	for _, s := range configured {
-		sources = append(sources, api.AuthSource{Tag: s.Tag, Prefix: s.Prefix, URL: s.URL})
+		sources = append(sources, api.AuthSource{Tag: s.Tag, Prefix: s.Prefix, URL: s.URL, APIURL: s.APIURL})
 	}
 	return sources
 }
 
 // cmdAPI runs felis-api: two listeners, two middleware chains (spec §7). The
-// internal face (service token) is fully wired. The external face is wired but
-// fails closed until an Access JWKS key function is configured — the verifier's
-// audience logic is unit-tested (internal/api), the JWKS source is a deployment
-// integration point.
+// internal face authenticates per-caller service tokens; the external face
+// authenticates the local session cookie.
 func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("api", flag.ContinueOnError)
 	fs.SetOutput(stderr)
@@ -80,9 +108,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	// Load already refused a malformed [audit] retention.
+	auditRetention, _ := cfg.Audit.RetentionPeriod()
+	if auditRetention == 0 {
+		fmt.Fprintln(stdout, "felis api: audit rows are kept forever ([audit] retention = \"forever\")")
+	} else {
+		fmt.Fprintf(stdout, "felis api: audit rows older than %d days are deleted ([audit] retention; export them first with felis db audit-export)\n", int(auditRetention/(24*time.Hour)))
+	}
+
 	ctx := ctrl.SetupSignalHandler()
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Before anything serves: an api on a schema it was not built for answers with
+	// errors, or writes rows the other version cannot read.
+	drv, err := openPodStore(ctx, cfg.Database.URL, "api", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis api: open database: %v\n", err)
 		return 1
@@ -109,15 +147,24 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	token := os.Getenv("FELIS_SERVICE_TOKEN")
-	if token == "" {
-		fmt.Fprintln(stderr, "felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers")
+	metrics.SetBuildInfo("api", resolvedVersion())
+
+	internalAuth, err := internalCallerTokens(os.Getenv)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: internal face tokens: %v\n", err)
+		return 1
+	}
+	for _, ct := range naming.CallerTokens {
+		if internalAuth[api.Caller(ct.Caller)] == "" {
+			fmt.Fprintf(stderr, "felis api: warning: %s unset — the internal face turns the %s caller away\n", ct.APIEnv, ct.Caller)
+		}
 	}
 
 	// Email one-time codes go through the [smtp] relay when one is configured; the
 	// password is read from the env var password_ref names (default SMTPPasswordEnv,
 	// injected from the felis-smtp Secret). No [smtp] host ⇒ mailer stays nil and
-	// deliverOTP logs each code server-side (the pre-SMTP bootstrap posture).
+	// every door that mails a code answers 503 mail_unavailable: a code that is
+	// not mailed is never written anywhere else either.
 	var mailer api.OTPMailer
 	if cfg.SMTP.Host != "" {
 		passRef := cfg.SMTP.PasswordRef
@@ -128,15 +175,12 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		if cfg.SMTP.Username != "" && password == "" {
 			fmt.Fprintf(stderr, "felis api: warning: [smtp] username is set but credentials env %s is empty — OTP sends will fail AUTH\n", passRef)
 		}
-		mailer = &mail.SMTP{
-			Host:     cfg.SMTP.Host,
-			Port:     cfg.SMTP.Port,
-			From:     cfg.SMTP.From,
-			Username: cfg.SMTP.Username,
-			Password: password,
+		mailer = smtpRelay(cfg.SMTP, password)
+		if !cfg.SMTP.TLSRequired() {
+			fmt.Fprintf(stderr, "felis api: warning: [smtp] %s may be sent codes without TLS (require_tls off or a relay on this host)\n", cfg.SMTP.Host)
 		}
 	} else {
-		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email one-time codes are logged, not mailed")
+		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email sign-in and verification are off (503 mail_unavailable); sign in with a passkey, or run felis setup to add a relay")
 	}
 
 	// Build subsystem (spec §16): the weak-SA build Job runs in the configured
@@ -147,11 +191,14 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// The fetch initContainer runs THIS image's fetch-context entrypoint, so the
 	// build config carries the api's own image (the platform sets FELIS_IMAGE).
 	buildCfg.FelisImage = os.Getenv("FELIS_IMAGE")
+	buildJobs := build.NewK8sJobs(cl, buildCfg)
 	builder := &build.Builder{
-		Store:  build.NewPGStore(drv.DB()),
-		Jobs:   build.NewK8sJobs(cl, buildCfg),
-		Config: buildCfg,
+		Store:    build.NewPGStore(drv.DB()),
+		Jobs:     buildJobs,
+		Config:   buildCfg,
+		Outcomes: build.NewK8sOutcomes(clientset, buildCfg),
 	}
+	go probeBuildUserNamespaces(ctx, buildJobs, buildCfg, stderr)
 
 	// User-modpack approval lane (user-directed extension over §16; see
 	// internal/submit). An ordinary user may only SUBMIT a
@@ -203,6 +250,21 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		ContextBaseURL: internalAPIBaseURL(),
 		Blobs:          blobs,
 	}
+	if blobs != nil {
+		submissions.Parts = &submit.PartStore{Dir: uploadPartsDir(contextBase)}
+	}
+	if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
+		if n, err := parseByteSize(v); err != nil || n <= 0 {
+			fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
+		} else {
+			submissions.MaxStoredBytesTotal = n
+		}
+	}
+	if n, err := contextMaxBytes(cfg); err != nil {
+		fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 512Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
+	} else {
+		submissions.MaxContextBytes = n
+	}
 
 	// Restore subsystem (spec §7): the weak-SA restore Job mounts the target
 	// world PVC + the backup PVC and runs `felis restore`. It needs deployment-
@@ -214,9 +276,23 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// store at load, so by this point cfg.Archive.Store is guaranteed tarLocal.)
 	var restorer api.Restorer
 	felisImage, backupPVC := os.Getenv("FELIS_IMAGE"), os.Getenv("FELIS_BACKUP_PVC")
+	worldResolver := placement.Resolve(cl, cfg.K8s.Namespace)
+	distribution, err := distributionManager(cl, cfg, felisImage)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if distribution != nil {
+		worldResolver = distribution.Resolve
+	}
 	if felisImage != "" && backupPVC != "" {
 		rcfg := restoreConfig(cfg, felisImage, backupPVC)
-		restorer = &restore.Restorer{Jobs: restore.NewK8sJobs(cl), Config: rcfg}
+		rcfg.ResolveWorld = worldResolver
+		var jobs restore.Jobs = restore.NewK8sJobs(cl)
+		if distribution != nil {
+			jobs = distribution
+		}
+		restorer = &restore.Restorer{Jobs: jobs, Config: rcfg}
 	} else {
 		fmt.Fprintln(stderr, "felis api: restore executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — restore endpoint returns 503")
 	}
@@ -229,11 +305,30 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// endpoint honestly returns 503.
 	var backuper api.Backuper
 	if felisImage != "" && backupPVC != "" {
-		backuper = &backupjob.Backuper{Jobs: backupjob.NewK8sJobs(cl), Config: backupConfig(cfg, felisImage, backupPVC)}
+		bcfg := backupConfig(cfg, felisImage, backupPVC)
+		bcfg.ResolveWorld = worldResolver
+		var jobs backupjob.Jobs = backupjob.NewK8sJobs(cl)
+		if distribution != nil {
+			jobs = distribution
+		}
+		backuper = &backupjob.Backuper{Jobs: jobs, Config: bcfg}
 	} else {
 		fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503")
 	}
 
+	// World export: a weak-SA Job mounts the world PVC, or the backup PVC, read-only
+	// and PUTs the archive to the internal face, which streams it on to the owner's
+	// browser (internal/worldexport). A backup export mounts the backup PVC, so it
+	// is wired under the restore gate; otherwise the export routes return 503.
+	var exporter api.Exporter
+	if felisImage != "" && backupPVC != "" {
+		ecfg := exportConfig(cfg, felisImage, backupPVC)
+		ecfg.ResolveWorld = worldResolver
+		exporter = worldexport.New(clientset, ecfg)
+	} else {
+		fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503")
+	}
+
 	// Server file editor: a weak-SA Job mounts ONLY the target world PVC and runs
 	// `felis files`, printing its result for felis-api to read back through
 	// pods/log (see internal/fileedit). It needs FELIS_IMAGE but — unlike restore
@@ -242,11 +337,27 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// endpoints honestly return 503. It takes the typed clientset rather than the
 	// controller-runtime client because the log subresource lives only on the typed
 	// CoreV1 client, and one client covers its Job create, Pod list, and log read.
+	//
+	// Uploads additionally stage their bytes on this pod's disk until the Job
+	// fetches them from the internal face; whatever a previous process staged is
+	// orphaned (the index is in memory), so the stage starts empty.
 	var files api.FileEditor
+	var fileStage *fileedit.Stage
+	var fileBrowser *fileedit.Browser
 	if felisImage != "" {
+		fcfg := fileEditConfig(cfg, felisImage)
+		fcfg.ResolveWorld = worldResolver
+		fileBrowser = &fileedit.Browser{BaseURL: internalAPIBaseURL()}
+		runner := fileedit.NewK8sRunner(clientset)
+		runner.Browser = fileBrowser
 		files = &fileedit.Editor{
-			Runner: fileedit.NewK8sRunner(clientset),
-			Config: fileEditConfig(cfg, felisImage),
+			Runner: runner,
+			Config: fcfg,
+		}
+		fileStage = &fileedit.Stage{Dir: fileStagingDir()}
+		if err := fileStage.Sweep(); err != nil {
+			fmt.Fprintf(stderr, "felis api: %v — file uploads return 503\n", err)
+			fileStage = nil
 		}
 	} else {
 		fmt.Fprintln(stderr, "felis api: file editor disabled (needs FELIS_IMAGE) — file endpoints return 503")
@@ -257,33 +368,73 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// the same store the auth handlers write to, so a login and the next request
 	// agree on what local auth knows.
 	repo := api.NewPGRepo(drv.DB())
+	if err := api.RegisterStorePool(drv.DB()); err != nil {
+		fmt.Fprintf(stderr, "felis api: store pool metrics unavailable: %v\n", err)
+	}
 
+	// The owner's on-demand backup levers come from [archive], the same keys the
+	// backup Job and the reaper read. A malformed key leaves the defaults in
+	// place here; the reaper Job fails on it and names it.
+	rcfg, err := reaperConfig(cfg)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: %v; using the default backup limits\n", err)
+		rcfg = reaper.DefaultConfig()
+	}
+
+	serverCache, serversSynced, err := startServerCache(ctx, restCfg, scheme, cfg.K8s.Namespace, stderr)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err)
+		return 1
+	}
+	cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced).WithDistributed(distribution != nil, os.Getenv("FELIS_CONTROLLER_NODE"))
+	if distribution != nil {
+		distribution.Record = func(ctx context.Context, b distributed.Backup) error {
+			keep, retention, ok := backupPolicy(b.Reason, rcfg)
+			if !ok {
+				return fmt.Errorf("unknown backup reason %s", b.Reason)
+			}
+			st := reaper.NewPGStore(drv.DB())
+			if err := st.InsertBackup(ctx, reaper.BackupRecord{ID: "bk-" + b.ID, ServerName: b.Server, FormerOwner: b.Owner, BackupRef: b.Receipt.Ref, SizeBytes: b.Receipt.Size, SHA256: b.Receipt.SHA256, Reason: b.Reason, ExpiresAt: time.Now().Add(retention)}); err != nil {
+				return err
+			}
+			pruneBackups(ctx, st, distribution.Archive, b.Server, b.Owner, b.Reason, keep, b.Protect, stdout, stderr)
+			return nil
+		}
+	}
+	jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace)
 	a := &api.API{
 		Repo:    repo,
-		Cluster: api.NewK8sCluster(cl, cfg.K8s.Namespace),
+		Cluster: cluster,
 		Console: api.NewK8sConsole(cl, cfg.K8s.Namespace),
 		Logs:    api.NewK8sLogStreamer(clientset, cfg.K8s.Namespace),
 		// Build-log stream (spec §16) is scoped to the BUILD namespace — the same
 		// value the Builder renders Jobs into — so it follows where build Pods run.
-		BuildLogs:   api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
-		Internal:    api.BearerTokenAuth{Token: token},
-		Builder:     builder,
-		Restorer:    restorer,
-		Backuper:    backuper,
-		JobStatus:   api.NewK8sJobStatus(cl, cfg.K8s.Namespace),
-		Files:       files,
-		Submissions: submissions,
-		Mailer:      mailer,
-		// The external face is fronted by SessionAuth: it prefers a local session
-		// cookie (minted by the passwordless doors) and otherwise delegates to the
-		// Cloudflare-Access JWT verifier, so both auth models coexist on one face. The
-		// delegate's Keyfunc is intentionally nil — the JWT path fails closed until a
-		// JWKS-backed key function is wired (deployment integration point) — while the
-		// local session path is live the moment `felis breakGlass` flips
-		// local_auth_enabled on.
+		BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
+		Internal:  internalAuth,
+		Builder:   builder,
+		Images:    imagePinner(cfg.Registry.URL),
+		Restorer:  restorer,
+		Backuper:  backuper,
+		JobStatus: jobStatus,
+		// A restore starts with a safety snapshot; settleRestoreChains starts the
+		// restore behind each one.
+		RestoreChains: jobStatus,
+		Files:         files,
+		FileStage:     fileStage,
+		FileBrowser:   fileBrowser,
+		// The file Job fetches an upload from here; it runs in the minecraft
+		// namespace, where the internal face is reachable like it is for the login
+		// gate.
+		InternalBaseURL: internalAPIBaseURL(),
+		Exporter:        exporter,
+		Submissions:     submissions,
+		Mailer:          mailer,
+		Schedules:       repo,
+		// The external face authenticates the local session cookie the sign-in doors
+		// mint, live once `felis breakGlass` flips local_auth_enabled on. Cloudflare
+		// Access, when the install sits behind it, is enforced at the edge only.
 		External: api.SessionAuth{
 			Repo:          repo,
-			Delegate:      api.AccessVerifier{Audience: cfg.Auth.AccessJWTAud},
 			RootDomain:    cfg.Server.RootDomain,
 			AdminHostname: cfg.Auth.AdminHostname,
 		},
@@ -291,6 +442,10 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		AdminHostname: cfg.Auth.AdminHostname,
 		PanelHostname: cfg.Auth.PanelHostname,
 		WakeCooldown:  30 * time.Second,
+		// An owner may start one backup per server per manual_cooldown, and none
+		// while the store is at max_local_bytes (data-durability-9).
+		BackupCooldown: rcfg.ManualCooldown,
+		BackupStoreCap: rcfg.MaxLocalBytes,
 		// The user-modpack lane's per-user throttles: a create spaces out
 		// review-queue rows, an upload spaces out (up to 1 GiB) context streams.
 		// Separate keys, so the normal create→upload sequence stays immediate.
@@ -300,8 +455,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		// for legitimate multi-tab / multi-server watching, while capping how many
 		// upstream follow connections a single caller can tie up if their streams stall.
 		MaxStreamsPerPrincipal: 16,
+		// Public sign-in doors, per client address: a person signing in makes a
+		// handful of calls, so 20 at once refilled at 20 a minute never bites a
+		// real user and still turns a spray into a trickle. The client address
+		// is the edge's header when the install names one (config.AuthConfig).
+		AuthDoorLimit:  api.RateLimit{Burst: 20, PerMinute: 20},
+		ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(),
+		MailLimit:      mailLimit(cfg.SMTP.MaxPerHour),
+	}
+	if a.ClientIPHeader != "" {
+		fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader)
+	} else {
+		fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)")
 	}
-	fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)")
 
 	// Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned
 	// identity anchor (正版优先); config can only append namespace-rewritten third-party
@@ -310,7 +476,8 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// [[auth_source]] is configured, so an empty list has to mean a Mojang-only relay, the
 	// same as under `felis nano`. A nil list would 204 every login, premium ones included.
 	a.AuthSources = authSourcesFromConfig(cfg.AuthSources)
-	fmt.Fprintf(stderr, "felis api: hasJoined multiplexer active — Mojang + %d third-party source(s)\n", len(cfg.AuthSources))
+	a.AuthSourceSettings = &api.AuthSourceSettings{Repo: repo, Defaults: a.AuthSources}
+	fmt.Fprintf(stderr, "felis api: hasJoined multiplexer active — Mojang + %d default third-party source(s); panel settings take precedence\n", len(cfg.AuthSources))
 
 	// Passkey (WebAuthn) verifier (spec §14, Phase 6). One relying party spans BOTH
 	// web faces: the RP id is the panel hostname (console.<root>), and because that is
@@ -318,22 +485,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// enrolled once asserts on either face — one binding, usable on the player console
 	// AND the operator console. Both hosts are therefore listed as permitted origins,
 	// while the RP id stays the panel host so the credential's scope is ONE relying
-	// party, not two. Wired only when auth.panel_hostname is configured; otherwise
-	// a.Passkey stays nil and the passkey routes honestly return 503 (the authenticated
-	// enrollment boundary is still enforced by the handlers).
-	if cfg.Auth.PanelHostname != "" {
-		origins := []string{"https://" + cfg.Auth.PanelHostname}
-		if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != cfg.Auth.PanelHostname {
-			origins = append(origins, "https://"+admin)
-		}
-		pv, err := passkey.New(cfg.Auth.PanelHostname, "Felis", origins)
-		if err != nil {
-			fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
-		} else {
-			a.Passkey = pv
-		}
+	// party, not two. Without a panel host (neither auth.panel_hostname nor
+	// server.root_domain) a.Passkey stays nil and the passkey routes honestly return
+	// 503 (the authenticated enrollment boundary is still enforced by the handlers).
+	if rpID, origins := passkeyRelyingParty(cfg); rpID == "" {
+		fmt.Fprintln(stderr, "felis api: passkey verifier disabled (no panel host: set server.root_domain or auth.panel_hostname) — passkey endpoints return 503")
+	} else if pv, err := passkey.New(rpID, "Felis", origins); err != nil {
+		fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
 	} else {
-		fmt.Fprintln(stderr, "felis api: passkey verifier disabled (auth.panel_hostname unset) — passkey endpoints return 503")
+		a.Passkey = pv
 	}
 
 	// Derive the console hostnames when felis.toml leaves them unset, exactly as the
@@ -342,7 +502,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain,
 		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname),
 		defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname),
-		resolvedVersion())
+		cfg.Velocity.GamePort, cfg.Velocity.GameVersion, resolvedVersion(), distribution != nil)
 	internalSrv := newAPIServer(*internalAddr, a.InternalHandler())
 	externalSrv := newAPIServer(cfg.Server.Listen, externalHandler)
 
@@ -364,16 +524,47 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// and advance any whose Job has reached a terminal phase. GET on a build also
 	// reconciles it, but this loop converges builds nobody is polling.
 	go reconcileBuilds(ctx, builder, stderr)
+	go settleRestoreChains(ctx, a, stderr)
+	go runSchedules(ctx, a, stderr)
+	// A daily restore point of every world played since its last one, taken
+	// once the server stops ([archive] scheduled_every; 0s turns it off).
+	if backuper != nil && rcfg.ScheduledEvery > 0 {
+		go scheduleBackups(ctx, &api.BackupScheduler{API: a, Store: repo, Jobs: jobStatus, Every: rcfg.ScheduledEvery}, stderr)
+	} else {
+		fmt.Fprintln(stderr, "felis api: scheduled backups off (needs the backup executor and [archive] scheduled_every above 0s)")
+	}
+
+	if pruner := registryPruner(cfg, builder.Store, cluster, stderr); pruner != nil {
+		go pruner.Loop(ctx, registryPruneInterval)
+	}
+	go reapRejectedContexts(ctx, submissions, stderr)
+	if fileStage != nil {
+		go expireFileSessions(ctx, fileStage, fileSessionSweep, stderr)
+	}
+	if exporter != nil {
+		go expireExports(ctx, a, exportSweep)
+	}
+	go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
+
+	if distribution != nil {
+		a.Distribution = distribution
+		go reconcileDistribution(ctx, distribution, stderr)
+	}
+	if socket := os.Getenv("FELIS_NODE_CONTROL_SOCKET"); socket != "" {
+		a.NodeControl = nodecontrol.NewClient(socket)
+		cluster.NodeMaintenanceGuard = api.NodeMaintenanceGuard(a.NodeControl)
+	}
+	servers := []*http.Server{internalSrv, externalSrv}
+	if httpsSrv != nil {
+		servers = append(servers, httpsSrv)
+	}
+	for _, srv := range servers {
+		srv.RegisterOnShutdown(a.CloseStreams)
+	}
 
 	select {
 	case <-ctx.Done():
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		_ = internalSrv.Shutdown(shutdownCtx)
-		_ = externalSrv.Shutdown(shutdownCtx)
-		if httpsSrv != nil {
-			_ = httpsSrv.Shutdown(shutdownCtx)
-		}
+		shutdownServers(servers, apiShutdownGrace, stderr)
 		return 0
 	case err := <-errc:
 		if err != nil && err != http.ErrServerClosed {
@@ -393,8 +584,30 @@ const (
 	// apiIdleTimeout caps how long a kept-alive connection may sit idle between
 	// requests before the server closes it, bounding idle-connection exhaustion.
 	apiIdleTimeout = 120 * time.Second
+	// apiShutdownGrace is how long the listeners drain after SIGTERM. The pod gets
+	// the Kubernetes default of 30s before SIGKILL; this leaves the rest for the
+	// process to exit.
+	apiShutdownGrace = 20 * time.Second
 )
 
+// shutdownServers drains every listener at once under one deadline: in turn, a
+// slow first listener would spend the time the others needed. Log streams end
+// through RegisterOnShutdown (API.CloseStreams); what is still running when the
+// deadline passes is cut off with the process.
+func shutdownServers(servers []*http.Server, grace time.Duration, stderr io.Writer) {
+	ctx, cancel := context.WithTimeout(context.Background(), grace)
+	defer cancel()
+	var wg sync.WaitGroup
+	for _, srv := range servers {
+		wg.Go(func() {
+			if err := srv.Shutdown(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: shutdown %s: %v\n", srv.Addr, err)
+			}
+		})
+	}
+	wg.Wait()
+}
+
 // newAPIServer builds an http.Server with hardened header/idle timeouts (gosec
 // G112) shared by all three felis-api listeners (internal, external, https).
 // WriteTimeout and ReadTimeout are deliberately LEFT UNSET: the external and https
@@ -418,18 +631,49 @@ func buildConfig(cfg *config.Config) build.Config {
 	return build.Config{
 		Namespace:   cfg.Registry.BuildNamespace,
 		RegistryURL: cfg.Registry.URL,
-		// Empty overrides fall back to the build package's defaults, so an
-		// install that has not imported kaniko/trivy keeps the compiled-in refs
-		// (and fails loudly on pull rather than silently building with the wrong
-		// image).
+		// Empty overrides fall back to the registry's copies of the tools
+		// (build.Tools), which felis mirror-build-tools keeps current.
 		KanikoImage: cfg.Registry.KanikoImage,
 		TrivyImage:  cfg.Registry.TrivyImage,
 		CPULimit:    cfg.Registry.BuildCPULimit,
 		MemLimit:    cfg.Registry.BuildMemLimit,
-		// Empty keeps Trivy's own default; an install with builds points this at
-		// the internal DB mirror (see config.RegistryConfig.TrivyDBRepository).
+		DiskLimit:   cfg.Registry.BuildDiskLimit,
+		// "auto" follows the startup probe (see probeBuildUserNamespaces).
+		UserNamespaces:        cfg.Registry.BuildUserNamespaces,
+		UserNamespacesProbe:   new(atomic.Bool),
+		RuntimeClass:          cfg.Registry.BuildRuntimeClass,
+		MaxConcurrent:         cfg.Registry.MaxConcurrentBuilds,
 		TrivyDBRepository:     cfg.Registry.TrivyDBRepository,
 		TrivyJavaDBRepository: cfg.Registry.TrivyJavaDBRepository,
+		ScanFailOn:            cfg.Registry.ScanFailOn,
+		ScanFailUnfixed:       cfg.Registry.ScanFailUnfixed,
+		ScanAccept:            cfg.Registry.ScanAccept,
+		// The submit lane's derived context URLs live here; the fetch step's
+		// service token goes nowhere else.
+		ContextOrigin: internalAPIBaseURL(),
+	}
+}
+
+// probeBuildUserNamespaces settles build_user_namespaces = "auto": one probe
+// pod with hostUsers: false tells whether this node's kernel and runtime can run
+// build pods in a user namespace. Builds submitted before it answers run without.
+func probeBuildUserNamespaces(ctx context.Context, jobs *build.K8sJobs, cfg build.Config, stderr io.Writer) {
+	if mode := cfg.UserNamespaces; mode != "" && mode != build.UserNamespacesAuto {
+		return
+	}
+	if cfg.FelisImage == "" {
+		fmt.Fprintln(stderr, "felis api: FELIS_IMAGE unset — build pods run without a user namespace")
+		return
+	}
+	ok, err := jobs.ProbeUserNamespaces(ctx, cfg.FelisImage)
+	cfg.UserNamespacesProbe.Store(ok)
+	switch {
+	case ok:
+		fmt.Fprintln(stderr, "felis api: build pods run in a user namespace (hostUsers: false)")
+	case err != nil:
+		fmt.Fprintf(stderr, "felis api: build pods run without a user namespace: the probe failed: %v\n", err)
+	default:
+		fmt.Fprintln(stderr, "felis api: build pods run without a user namespace: this node cannot start a pod with hostUsers: false")
 	}
 }
 
@@ -517,6 +761,17 @@ func backupConfig(cfg *config.Config, image, backupPVC string) backupjob.Config
 	}
 }
 
+// exportConfig builds the world export executor's config. BackupRoot mirrors
+// restoreConfig: the stored refs are absolute paths under [archive] local_path.
+func exportConfig(cfg *config.Config, image, backupPVC string) worldexport.Config {
+	return worldexport.Config{
+		Namespace:  cfg.K8s.Namespace,
+		Image:      image,
+		BackupPVC:  backupPVC,
+		BackupRoot: cfg.Archive.LocalPath,
+	}
+}
+
 // fileEditConfig builds the file editor's config from felis.toml plus the
 // deployment-supplied image. It is the shortest of the three: the editor mounts
 // only the world PVC, so it needs no archive coordinates at all, and everything
@@ -546,3 +801,339 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
 		}
 	}
 }
+
+// settleRestoreChains starts the restore behind each safety snapshot that has
+// finished (and gives up the one behind a snapshot that failed). The world stays
+// locked in between, so the interval is how long a finished snapshot keeps the
+// server down before its restore begins.
+func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
+	t := time.NewTicker(5 * time.Second)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := a.SettleRestoreChains(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: restore chains: %v\n", err)
+			}
+		}
+	}
+}
+
+// runSchedules runs the servers' scheduled tasks (api.API.RunSchedules). The
+// interval is how late a task may start, and how often a restart or backup in
+// progress checks whether it can take its next step.
+func runSchedules(ctx context.Context, a *api.API, stderr io.Writer) {
+	t := time.NewTicker(15 * time.Second)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := a.RunSchedules(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: scheduled tasks: %v\n", err)
+			}
+		}
+	}
+}
+
+// scheduleBackups starts the scheduled backups (api.BackupScheduler). Each tick
+// starts at most one, so the interval also spaces the worlds that stopped at
+// the same time: a world that stops waits at most this long for its point to
+// start once the Jobs ahead of it are done.
+func scheduleBackups(ctx context.Context, s *api.BackupScheduler, stderr io.Writer) {
+	t := time.NewTicker(2 * time.Minute)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := s.Tick(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: scheduled backups: %v\n", err)
+			}
+		}
+	}
+}
+
+// fileSessionSweep is how often expireFileSessions looks for idle upload
+// sessions: small beside fileedit.SessionIdle, so an abandoned one gives its
+// room back within minutes of going stale.
+const fileSessionSweep = 10 * time.Minute
+
+// expireFileSessions drops the file manager's upload sessions left untouched
+// for fileedit.SessionIdle. Each reserved room on the staging disk for its whole
+// file when it began, so one abandoned would otherwise hold that room until
+// felis-api restarts.
+func expireFileSessions(ctx context.Context, s *fileedit.Stage, every time.Duration, stderr io.Writer) {
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if n := s.Expire(); n > 0 {
+				fmt.Fprintf(stderr, "felis api: dropped %d upload session(s) left idle for %s\n", n, fileedit.SessionIdle)
+			}
+		}
+	}
+}
+
+// exportSweep is how often expireExports runs: an export whose Job never
+// connected is stopped within a minute of going stale.
+const exportSweep = time.Minute
+
+// expireExports runs the export sweep (api.API.ExpireExports) on a ticker. The
+// export routes sweep as they are called, and an owner who closed the tab calls
+// none; a Job whose Pod never got going would then keep the server from
+// starting until the Job's deadline.
+func expireExports(ctx context.Context, a interface{ ExpireExports() }, every time.Duration) {
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			a.ExpireExports()
+		}
+	}
+}
+
+// reapRejectedContexts deletes, once an hour, the uploaded contexts of
+// submissions rejected more than submit.RejectedContextRetention ago, and the
+// chunked uploads left untouched for submit.StalePartRetention. Without it a
+// rejected modpack or an abandoned upload keeps its bytes on the uploads store
+// (and against its submitter's budget) until an admin deletes the row.
+func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
+	t := time.NewTicker(time.Hour)
+	defer t.Stop()
+	for {
+		n, err := m.ReapRejected(ctx, submit.RejectedContextRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap rejected uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
+		}
+		n, err = m.ReapStaleParts(submit.StalePartRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap abandoned uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted %d abandoned chunked upload(s)\n", n)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// retentionInterval spaces the runs that delete spent sign-in rows and audit rows
+// past [audit] retention. The rows are spent for weeks before they go, so a few
+// runs a day keep the tables flat.
+const retentionInterval = 6 * time.Hour
+
+// registryPruneInterval spaces the registry pruner's runs. The registry-gc
+// sidecar sweeps once a day, so pruning more often only changes which sweep frees
+// a layer.
+const registryPruneInterval = 6 * time.Hour
+
+// registryPruner deletes the registry manifests nothing references
+// (internal/registryprune); the registry-gc sidecar frees their layers on its next
+// sweep. It acts as the gate's prune principal, whose token the api Deployment
+// injects from felis-registry-auth. Without the token the registry only grows,
+// which is said once here.
+func registryPruner(cfg *config.Config, store imageRefStore, servers serverLister, stderr io.Writer) *registryprune.Pruner {
+	if cfg.Registry.URL == "" {
+		return nil
+	}
+	token := os.Getenv(platform.RegistryPruneTokenEnv)
+	if token == "" {
+		fmt.Fprintf(stderr, "felis api: registry pruner disabled (%s unset) — images nothing uses are never deleted from the registry\n", platform.RegistryPruneTokenEnv)
+		return nil
+	}
+	static := append([]string{os.Getenv("FELIS_IMAGE")}, buildConfig(cfg).ToolRefs()...)
+	return &registryprune.Pruner{
+		Registry: &registryprune.Client{Endpoint: "http://" + cfg.Registry.URL, Token: token},
+		Host:     cfg.Registry.URL,
+		Refs: func(ctx context.Context) ([]string, error) {
+			return inUseImageRefs(ctx, store, servers, static)
+		},
+		Log: slog.New(slog.NewTextHandler(stderr, nil)),
+	}
+}
+
+type imageRefStore interface {
+	ListImages(ctx context.Context) ([]build.Image, error)
+	ListUnfinishedBuilds(ctx context.Context) ([]build.Build, error)
+}
+
+type serverLister interface {
+	ListServers(ctx context.Context) ([]api.ServerInfo, error)
+	PodImages(ctx context.Context) ([]string, error)
+}
+
+// inUseImageRefs lists every image reference the platform still depends on: the
+// whitelist (disabled rows too, an admin may enable them again), every server's
+// spec, the images the game pods run, builds still running, and the images the
+// control plane and the build Jobs run. Any source failing fails the whole list,
+// so the pruner never decides on a partial view.
+//
+// The pods matter for the felis image: a running server keeps the one it started
+// with across platform upgrades (operator.PodTemplateAnnotation), which after a
+// few releases is no longer among the newest tags the pruner keeps anyway, and
+// the pod needs it again whenever it is recreated.
+func inUseImageRefs(ctx context.Context, store imageRefStore, servers serverLister, static []string) ([]string, error) {
+	refs := append([]string(nil), static...)
+	images, err := store.ListImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("image whitelist: %w", err)
+	}
+	for _, img := range images {
+		refs = append(refs, img.ImageRef)
+	}
+	srvs, err := servers.ListServers(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("servers: %w", err)
+	}
+	for _, s := range srvs {
+		refs = append(refs, s.Image)
+	}
+	podImages, err := servers.PodImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("game pods: %w", err)
+	}
+	refs = append(refs, podImages...)
+	builds, err := store.ListUnfinishedBuilds(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("running builds: %w", err)
+	}
+	for _, b := range builds {
+		refs = append(refs, b.ImageRef)
+	}
+	return refs, nil
+}
+
+// mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket:
+// the hourly cap as the refill rate, with a quarter of it (at least 5) allowed
+// at once so a burst of real sign-ins is not queued behind the average.
+func mailLimit(perHour int) api.RateLimit {
+	if perHour <= 0 {
+		perHour = config.DefaultMailPerHour
+	}
+	return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60}
+}
+
+// imagePinner resolves a new server's image against the platform registry
+// through its in-cluster Service, the address its refs already spell. An install
+// without a registry has no platform-built images to pin.
+func imagePinner(registry string) api.ImagePinner {
+	if registry == "" {
+		return nil
+	}
+	return imagepin.Resolver{Registry: registry}
+}
+
+// internalCallerTokens reads each internal caller's token from the env var the
+// Deployment feeds it from (naming.CallerTokens). Two callers sharing a value
+// would make the caller ambiguous, so that refuses to start.
+func internalCallerTokens(getenv func(string) string) (api.CallerTokens, error) {
+	tokens := map[api.Caller]string{}
+	for _, ct := range naming.CallerTokens {
+		tokens[api.Caller(ct.Caller)] = strings.TrimSpace(getenv(ct.APIEnv))
+	}
+	return api.NewCallerTokens(tokens)
+}
+
+// smtpRelay is the relay [smtp] names, with the resolved password and the TLS
+// posture config.SMTPConfig.TLSRequired picks. felis api, the reaper and the
+// watchdog all send through it, so none can drift to a weaker posture.
+func smtpRelay(c config.SMTPConfig, password string) *mail.SMTP {
+	return &mail.SMTP{
+		Host:       c.Host,
+		Port:       c.Port,
+		From:       c.From,
+		Username:   c.Username,
+		Password:   password,
+		RequireTLS: c.TLSRequired(),
+	}
+}
+
+// startServerCache starts the informer that serves the api's fleet-wide
+// MinecraftServer reads (api.K8sCluster.WithServerCache): one watch on the
+// namespace instead of a full List per velocity pull, fleet page and wake. It
+// caches MinecraftServers only — ReaderFailOnMissingInformer turns any other read
+// through it into an error rather than a new informer the api's Role cannot back —
+// indexes spec.subdomain for GetBySubdomain, and drops managedFields to keep the
+// copy small. It returns without waiting: the reads block until the first list
+// lands and /readyz reports not-ready until then.
+func startServerCache(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, stderr io.Writer) (cache.Cache, func() bool, error) {
+	c, err := cache.New(cfg, cache.Options{
+		Scheme:                      scheme,
+		DefaultNamespaces:           map[string]cache.Config{namespace: {}},
+		DefaultTransform:            cache.TransformStripManagedFields(),
+		ReaderFailOnMissingInformer: true,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := c.IndexField(ctx, &v1alpha1.MinecraftServer{}, api.SubdomainIndex, api.SubdomainOf); err != nil {
+		return nil, nil, fmt.Errorf("index %s: %w", api.SubdomainIndex, err)
+	}
+	inf, err := c.GetInformer(ctx, &v1alpha1.MinecraftServer{})
+	if err != nil {
+		return nil, nil, err
+	}
+	go func() {
+		if err := c.Start(ctx); err != nil {
+			fmt.Fprintf(stderr, "felis api: MinecraftServer cache stopped: %v\n", err)
+		}
+	}()
+	return c, inf.HasSynced, nil
+}
+
+// uploadPartsDir is where chunked uploads are staged: beside a local store's
+// contexts, so the room check and the budget see one disk and a staged upload
+// survives an API restart; for an s3:// store, on the uploads volume the
+// platform mounts either way, or the pod's /tmp when run by hand without it.
+func uploadPartsDir(contextBase string) string {
+	if isLocalUploadsPath(contextBase) {
+		return filepath.Join(strings.TrimPrefix(contextBase, "file://"), ".parts")
+	}
+	if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
+		return filepath.Join(platform.UploadsLocalPath, ".parts")
+	}
+	return filepath.Join(os.TempDir(), "felis-upload-parts")
+}
+
+// fileStagingDir is where file uploads wait for their Job: on the uploads
+// volume, whose capacity is its own, or the pod's /tmp when run by hand without
+// it — /tmp is the node's disk, which a burst of uploads should not fill.
+func fileStagingDir() string {
+	if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
+		return filepath.Join(platform.UploadsLocalPath, ".file-staging")
+	}
+	return filepath.Join(os.TempDir(), "felis-file-staging")
+}
+
+// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
+// package's own default (1 GiB). The Cloudflare edge refuses a single request
+// body over 100 MB, which the panel's chunked upload stays under, so the edge
+// does not lower the cap.
+func contextMaxBytes(cfg *config.Config) (int64, error) {
+	v := cfg.Registry.ContextMaxBytes
+	if v == "" {
+		return 0, nil
+	}
+	n, err := parseByteSize(v)
+	if err != nil || n <= 0 {
+		return 0, fmt.Errorf("not a positive size: %q", v)
+	}
+	return n, nil
+}

cmd/felis/api_test.go

@@ -1,12 +1,71 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
 	"net/http"
+	"slices"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/fileedit"
 )
 
+// The passkey relying party follows the panel host the SPA is served on: an install
+// that names only its root domain still gets passkeys, on console.<root>, with the
+// operator host as the second origin; only an install with no panel host goes without.
+func TestPasskeyRelyingParty(t *testing.T) {
+	t.Setenv("FELIS_PANEL_NODEPORT", "")
+	for _, tc := range []struct {
+		name               string
+		root, panel, admin string
+		wantRP             string
+		wantOrigins        []string
+	}{
+		{"root domain only", "example.net", "", "", "console.example.net",
+			[]string{"https://console.example.net", "https://op.console.example.net"}},
+		{"configured hosts", "example.net", " play.example.net ", "ops.example.net", "play.example.net",
+			[]string{"https://play.example.net", "https://ops.example.net"}},
+		{"operator host equal to the panel host", "example.net", "console.example.net", "console.example.net",
+			"console.example.net", []string{"https://console.example.net"}},
+		{"panel host without a root domain", "", "console.example.org", "", "console.example.org",
+			[]string{"https://console.example.org"}},
+		{"no host at all", "", "", "", "", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := &config.Config{}
+			cfg.Server.RootDomain, cfg.Auth.PanelHostname, cfg.Auth.AdminHostname = tc.root, tc.panel, tc.admin
+			var wantOrigins []string
+			for _, origin := range tc.wantOrigins {
+				wantOrigins = append(wantOrigins, origin, fmt.Sprintf("%s:%d", origin, defaultPanelNodePort))
+			}
+			rp, origins := passkeyRelyingParty(cfg)
+			if rp != tc.wantRP || !slices.Equal(origins, wantOrigins) {
+				t.Fatalf("relying party = %q %q, want %q %q", rp, origins, tc.wantRP, wantOrigins)
+			}
+		})
+	}
+}
+
+func TestPasskeyRelyingPartyIncludesConfiguredNodePort(t *testing.T) {
+	t.Setenv("FELIS_PANEL_NODEPORT", "30445")
+	cfg := &config.Config{}
+	cfg.Server.RootDomain = "example.com"
+	_, origins := passkeyRelyingParty(cfg)
+	if !slices.Contains(origins, "https://op.console.example.com:30445") {
+		t.Fatalf("configured NodePort origin missing: %v", origins)
+	}
+}
+
 // TestAuthSourcesFromConfig pins the one place the hasJoined identity anchor is decided:
 // Mojang is prepended in code, first, and is the only source whose UUIDs are trusted as-is.
 // The empty case matters on its own — both `felis api` and `felis nano` call this with a
@@ -20,7 +79,7 @@ func TestAuthSourcesFromConfig(t *testing.T) {
 		{"no configured sources", nil},
 		{"configured sources", []config.AuthSourceConfig{
 			{Tag: "littleskin", Prefix: "LS", URL: "https://littleskin.example/hasJoined"},
-			{Tag: "guild", Prefix: "GD", URL: "https://guild.example/hasJoined"},
+			{Tag: "guild", Prefix: "GD", URL: "https://guild.example/hasJoined", APIURL: "https://guild.example/api"},
 		}},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
@@ -36,7 +95,7 @@ func TestAuthSourcesFromConfig(t *testing.T) {
 				if s.Identity {
 					t.Errorf("configured source %q is marked Identity; only Mojang may be", c.Tag)
 				}
-				if s.Tag != c.Tag || s.Prefix != c.Prefix || s.URL != c.URL {
+				if s.Tag != c.Tag || s.Prefix != c.Prefix || s.URL != c.URL || s.APIURL != c.APIURL {
 					t.Errorf("source %d = %+v, want %+v in config order", i+1, s, c)
 				}
 			}
@@ -91,3 +150,179 @@ func TestNewAPIServerSetsHardenedTimeouts(t *testing.T) {
 		t.Errorf("ReadTimeout = %v, want 0 (unset) so a slow SSE attach is not capped", srv.ReadTimeout)
 	}
 }
+
+// Every listener drains at once, and the shutdown hook (API.CloseStreams in
+// cmdAPI) runs on each: two listeners each holding a request that ends when
+// the hook fires take one hook's worth of time, well inside the deadline.
+func TestShutdownServersDrainsListenersTogether(t *testing.T) {
+	release := make(chan struct{})
+	var hooks int32
+	started := make(chan struct{}, 2)
+	var servers []*http.Server
+	for range 2 {
+		srv := newAPIServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			started <- struct{}{}
+			<-release
+		}))
+		srv.RegisterOnShutdown(func() {
+			if atomic.AddInt32(&hooks, 1) == 1 {
+				time.AfterFunc(100*time.Millisecond, func() { close(release) })
+			}
+		})
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			t.Fatal(err)
+		}
+		srv.Addr = l.Addr().String()
+		go func() { _ = srv.Serve(l) }()
+		go func() {
+			if resp, err := http.Get("http://" + srv.Addr); err == nil {
+				resp.Body.Close()
+			}
+		}()
+		servers = append(servers, srv)
+	}
+	<-started
+	<-started
+
+	begun := time.Now()
+	shutdownServers(servers, 5*time.Second, io.Discard)
+	if took := time.Since(begun); took > 2*time.Second {
+		t.Fatalf("shutdown took %v", took)
+	}
+	if got := atomic.LoadInt32(&hooks); got != 2 {
+		t.Fatalf("shutdown hook ran %d times, want once per listener", got)
+	}
+}
+
+type fakeRefStore struct {
+	images []build.Image
+	builds []build.Build
+	err    error
+}
+
+func (f fakeRefStore) ListImages(context.Context) ([]build.Image, error) { return f.images, f.err }
+func (f fakeRefStore) ListUnfinishedBuilds(context.Context) ([]build.Build, error) {
+	return f.builds, nil
+}
+
+type fakeServers struct {
+	list    []api.ServerInfo
+	pods    []string
+	podsErr error
+}
+
+func (f fakeServers) ListServers(context.Context) ([]api.ServerInfo, error) { return f.list, nil }
+func (f fakeServers) PodImages(context.Context) ([]string, error)           { return f.pods, f.podsErr }
+
+// The registry pruner deletes whatever this list does not name, so every source of
+// a reference has to be in it, and a failing source must fail the list.
+func TestInUseImageRefsCoversEverySource(t *testing.T) {
+	const reg = "registry.felis.svc:5000/"
+	store := fakeRefStore{
+		images: []build.Image{{ImageRef: reg + "modpacks/pack:*"}, {ImageRef: reg + "felis/paper:demo"}},
+		builds: []build.Build{{ImageRef: reg + "user-uploads/sub-9:latest"}},
+	}
+	servers := fakeServers{
+		list: []api.ServerInfo{{Name: "s1", Image: reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1)}},
+		pods: []string{reg + "felis/felis:v1.0.0"},
+	}
+	got, err := inUseImageRefs(context.Background(), store, servers, []string{reg + "felis/felis:b60"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		reg + "felis/felis:b60",
+		reg + "modpacks/pack:*", reg + "felis/paper:demo",
+		reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1),
+		reg + "felis/felis:v1.0.0",
+		reg + "user-uploads/sub-9:latest",
+	}
+	if fmt.Sprint(got) != fmt.Sprint(want) {
+		t.Fatalf("refs = %v\nwant %v", got, want)
+	}
+
+	servers.podsErr = errors.New("apiserver down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing pod list produced a reference list")
+	}
+	servers.podsErr = nil
+
+	store.err = errors.New("db down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing whitelist read produced a reference list")
+	}
+}
+
+// TestExpireFileSessions runs the loop against a stage whose clock the test
+// holds: the session idle past fileedit.SessionIdle goes, the one touched since
+// stays, and the drop is said once.
+func TestExpireFileSessions(t *testing.T) {
+	var mu sync.Mutex
+	now := time.Date(2026, 9, 28, 10, 0, 0, 0, time.UTC)
+	advance := func(d time.Duration) { mu.Lock(); now = now.Add(d); mu.Unlock() }
+	st := &fileedit.Stage{Dir: t.TempDir(), MinFree: 1e-9, Now: func() time.Time {
+		mu.Lock()
+		defer mu.Unlock()
+		return now
+	}}
+	idle, err := st.Begin("u1", "survival", "a.jar", 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	advance(fileedit.SessionIdle - time.Minute)
+	fresh, err := st.Begin("u1", "survival", "b.jar", 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	advance(2 * time.Minute)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var out bytes.Buffer
+	done := make(chan struct{})
+	go func() { expireFileSessions(ctx, st, time.Millisecond, &out); close(done) }()
+	for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(time.Millisecond) {
+		if _, err := st.Status("u1", "survival", idle.ID); errors.Is(err, fileedit.ErrNotStaged) {
+			break
+		}
+		if time.Now().After(deadline) {
+			cancel()
+			t.Fatal("the idle session was never dropped")
+		}
+	}
+	// A few more ticks with nothing idle, which must stay quiet.
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+	<-done
+	if _, err := st.Status("u1", "survival", fresh.ID); err != nil {
+		t.Fatalf("the session touched since went too: %v", err)
+	}
+	if got := out.String(); got != "felis api: dropped 1 upload session(s) left idle for 6h0m0s\n" {
+		t.Fatalf("said %q", got)
+	}
+}
+
+type sweepCount struct{ n atomic.Int32 }
+
+func (s *sweepCount) ExpireExports() { s.n.Add(1) }
+
+// TestExpireExports: the loop sweeps on each tick, and returns once felis-api
+// shuts down.
+func TestExpireExports(t *testing.T) {
+	var s sweepCount
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { expireExports(ctx, &s, time.Millisecond); close(done) }()
+	for deadline := time.Now().Add(5 * time.Second); s.n.Load() < 3; time.Sleep(time.Millisecond) {
+		if time.Now().After(deadline) {
+			cancel()
+			t.Fatalf("swept %d times in 5s at a 1ms tick", s.n.Load())
+		}
+	}
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the loop outlived its context")
+	}
+}

cmd/felis/api_tokens_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/api"
+)
+
+// felis-api maps each env var onto the caller the Deployment feeds it for, so the
+// build Job's token (FELIS_BUILD_TOKEN) authenticates as build and only as build.
+func TestInternalCallerTokensReadEachCallersEnv(t *testing.T) {
+	env := map[string]string{
+		"FELIS_SERVICE_TOKEN": "v-tok",
+		"FELIS_LIMBO_TOKEN":   "l-tok",
+		"FELIS_BUILD_TOKEN":   " b-tok\n",
+		"FELIS_OPS_TOKEN":     "o-tok",
+	}
+	auth, err := internalCallerTokens(func(k string) string { return env[k] })
+	if err != nil {
+		t.Fatal(err)
+	}
+	for tok, want := range map[string]api.Caller{"v-tok": api.CallerVelocity, "l-tok": api.CallerLimbo, "b-tok": api.CallerBuild, "o-tok": api.CallerOps} {
+		r := httptest.NewRequest("GET", "/", nil)
+		r.Header.Set("Authorization", "Bearer "+tok)
+		if got, err := auth.Authenticate(r); err != nil || got != want {
+			t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want)
+		}
+	}
+
+	// An install where the build namespace still holds a copy of the proxy's token
+	// would let that copy act as the proxy; the api refuses to start on it.
+	env["FELIS_BUILD_TOKEN"] = "v-tok"
+	if _, err := internalCallerTokens(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "same value") {
+		t.Fatalf("shared token: err = %v, want a same-value refusal", err)
+	}
+}

cmd/felis/apply.go

@@ -8,7 +8,9 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/naming"
@@ -16,10 +18,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -111,20 +109,15 @@ func cmdApply(args []string, stdout, stderr io.Writer) int {
 	}
 
 	// ------- K8s client (one context, one client) -------
-	// SetupSignalHandler must be called exactly once per process —
-	// controller-runtime panics on a second call. We create ctx and the
-	// K8s client here and thread both through every downstream call so no
-	// callee ever needs to call SetupSignalHandler again.
-	ctx := ctrl.SetupSignalHandler()
-
-	scheme := runtime.NewScheme()
-	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-	utilruntime.Must(v1alpha1.AddToScheme(scheme))
-
-	cfg := ctrl.GetConfigOrDie()
-	cl, err := client.New(cfg, client.Options{Scheme: scheme})
+	// The operator runs this on the node, where the kubeconfig is k3s's own file and
+	// neither $KUBECONFIG nor ~/.kube is set. buildSystemServerClient falls back to that
+	// file and names what it tried; ctrl.GetConfigOrDie exited 1 there without a word,
+	// because controller-runtime's logger is never set up in a CLI command.
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	cl, err := buildSystemServerClient()
 	if err != nil {
-		fmt.Fprintf(stderr, "felis apply: build client: %v\n", err)
+		fmt.Fprintf(stderr, "felis apply: %v\n", err)
 		return 1
 	}
 
@@ -167,6 +160,10 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 	if err := naming.ValidateServerName(req.Subdomain); err != nil {
 		return nil, fmt.Errorf("invalid subdomain: %w", err)
 	}
+	displayName, err := naming.CleanDisplayName(req.DisplayName)
+	if err != nil {
+		return nil, fmt.Errorf("invalid displayName: %w", err)
+	}
 	if strings.TrimSpace(req.Image) == "" {
 		return nil, fmt.Errorf("image is required")
 	}
@@ -236,13 +233,26 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 		},
 		Spec: v1alpha1.MinecraftServerSpec{
 			Subdomain:       req.Subdomain,
-			DisplayName:     req.DisplayName,
+			DisplayName:     displayName,
 			Image:           req.Image,
 			JavaMemory:      deriveApplyJavaHeap(memLim),
 			DesiredState:    v1alpha1.DesiredStopped,
 			AutostartPolicy: policy,
 			Storage:         v1alpha1.StorageSpec{Size: storageQ.String()},
 			Resources:       corev1.ResourceRequirements{Limits: limits, Requests: requests},
+			// The rest matches what felis-api's create writes (K8sCluster.CreateServer):
+			// fall back to the login gate while stopped, RCON on (readiness, the
+			// player count and the console all ride it; the operator mints the
+			// password), and the default idle stop.
+			FallbackServer: naming.SystemLoginServer,
+			Rcon: v1alpha1.RconSpec{
+				Enabled: true,
+				SecretRef: v1alpha1.SecretKeyRef{
+					Name: naming.RconSecretName(req.Name),
+					Key:  naming.RconSecretKey,
+				},
+			},
+			Idle: v1alpha1.DefaultIdle(),
 		},
 	}, nil
 }
@@ -251,8 +261,7 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 // request if any CRD already carries the given spec.subdomain. metadata.name
 // uniqueness is enforced by K8s on Create, but spec.subdomain must be checked
 // here because two CRDs with different names could otherwise share a subdomain.
-// It reuses the caller's context and K8s client — it never calls
-// SetupSignalHandler or builds its own client.
+// It reuses the caller's context and K8s client.
 func checkSubdomainUnique(ctx context.Context, cl client.Client, namespace, subdomain string) error {
 	var list v1alpha1.MinecraftServerList
 	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {

cmd/felis/apply_test.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
@@ -136,11 +140,12 @@ func TestDeriveApplyJavaHeap(t *testing.T) {
 
 func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	req := applyRequest{
-		Name:      "test-server",
-		Subdomain: "test-server",
-		Image:     "registry.felis.svc/paper:1.21",
-		Memory:    "4Gi",
-		Storage:   "20Gi",
+		Name:        "test-server",
+		Subdomain:   "test-server",
+		DisplayName: "  Test Server  ",
+		Image:       "registry.felis.svc/paper:1.21",
+		Memory:      "4Gi",
+		Storage:     "20Gi",
 	}
 	ms, err := buildMinecraftServerFromApplyRequest(req, "minecraft")
 	if err != nil {
@@ -155,6 +160,9 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	if ms.Spec.Subdomain != "test-server" {
 		t.Errorf("Subdomain = %q", ms.Spec.Subdomain)
 	}
+	if ms.Spec.DisplayName != "Test Server" {
+		t.Errorf("DisplayName = %q, want it trimmed to Test Server", ms.Spec.DisplayName)
+	}
 	if ms.Spec.Image != "registry.felis.svc/paper:1.21" {
 		t.Errorf("Image = %q", ms.Spec.Image)
 	}
@@ -167,6 +175,18 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	if ms.Spec.Storage.Size != "20Gi" {
 		t.Errorf("Storage.Size = %q, want 20Gi", ms.Spec.Storage.Size)
 	}
+	// Same operational defaults as the API create path: without RCON the server
+	// never reports players and the console answers 503; without spec.idle it
+	// never stops on its own.
+	if !ms.Spec.Rcon.Enabled || ms.Spec.Rcon.SecretRef.Name != naming.RconSecretName("test-server") {
+		t.Errorf("Rcon = %+v, want enabled with the operator-minted secret", ms.Spec.Rcon)
+	}
+	if ms.Spec.Idle != v1alpha1.DefaultIdle() {
+		t.Errorf("Idle = %+v, want the default %+v", ms.Spec.Idle, v1alpha1.DefaultIdle())
+	}
+	if ms.Spec.FallbackServer != naming.SystemLoginServer {
+		t.Errorf("FallbackServer = %q, want the login gate", ms.Spec.FallbackServer)
+	}
 	mem, ok := ms.Spec.Resources.Limits[corev1.ResourceMemory]
 	if !ok {
 		t.Fatal("memory limit missing")
@@ -267,6 +287,11 @@ func TestBuildMinecraftServerFromApplyRequest_Errors(t *testing.T) {
 			applyRequest{Name: ok, Subdomain: "", Image: "x", Memory: "1Gi", Storage: "1Gi"},
 			"invalid subdomain",
 		},
+		{
+			"display name with a tab",
+			applyRequest{Name: ok, Subdomain: ok, DisplayName: "a" + string(rune(0x09)) + "b", Image: "x", Memory: "1Gi", Storage: "1Gi"},
+			"invalid displayName",
+		},
 		{
 			"empty image",
 			applyRequest{Name: ok, Subdomain: ok, Image: "", Memory: "1Gi", Storage: "1Gi"},
@@ -357,3 +382,27 @@ func resList(specs ...string) corev1.ResourceList {
 	}
 	return rl
 }
+
+// TestApplyReportsAMissingKubeconfig pins the node-side failure: with no kubeconfig to
+// find, apply says which ones it tried and exits 1. It used to call
+// ctrl.GetConfigOrDie, which ended the process with exit 1 and nothing printed.
+func TestApplyReportsAMissingKubeconfig(t *testing.T) {
+	if _, err := os.Stat(hostBootstrapKubeconfigPath); err == nil {
+		t.Skipf("%s exists on this machine", hostBootstrapKubeconfigPath)
+	}
+	dir := t.TempDir()
+	t.Setenv("KUBECONFIG", filepath.Join(dir, "missing"))
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	form := filepath.Join(dir, "server.json")
+	if err := os.WriteFile(form, []byte(`{"name":"alpha","subdomain":"alpha","image":"registry.felis.svc:5000/felis/paper:demo","memory":"1Gi","storage":"1Gi"}`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var out, errw bytes.Buffer
+	if code := cmdApply([]string{"-f", form}, &out, &errw); code != 1 {
+		t.Fatalf("exit = %d, want 1; stderr %q", code, errw.String())
+	}
+	want := "felis apply: no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and " + hostBootstrapKubeconfigPath + "): stat " + hostBootstrapKubeconfigPath + ": no such file or directory\n"
+	if errw.String() != want || out.Len() != 0 {
+		t.Fatalf("stdout %q, stderr %q, want stderr %q", out.String(), errw.String(), want)
+	}
+}

cmd/felis/archive_serve.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	"felis.lolicon.best/internal/archivetransfer"
+)
+
+func cmdArchiveServe(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("archive-serve", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	root := fs.String("root", "/backups", "archive PVC mount (must match archive.local_path)")
+	addr := fs.String("listen", ":8090", "private archive listener")
+	limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum compressed archive bytes")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	key := os.Getenv(archivetransfer.KeyEnv)
+	if len(key) < 32 || *limit <= 0 {
+		fmt.Fprintln(stderr, "archive-serve: signing key and positive size limit required")
+		return 2
+	}
+	if err := os.MkdirAll(*root, 0750); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	limitHeapToCgroup()
+	transport := &archivetransfer.Server{Root: *root, Key: key, Limit: *limit}
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		ticker := time.NewTicker(time.Hour)
+		defer ticker.Stop()
+		for {
+			if err := transport.Sweep(time.Now()); err != nil {
+				fmt.Fprintln(stderr, "archive journal cleanup:", err)
+			}
+			select {
+			case <-done:
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+	srv := &http.Server{Addr: *addr, Handler: transport, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Hour, WriteTimeout: 2 * time.Hour, MaxHeaderBytes: 16 << 10}
+	fmt.Fprintln(stdout, "archive transport listening", *addr)
+	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	return 0
+}

cmd/felis/backup.go

@@ -1,18 +1,20 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	"felis.lolicon.best/internal/backup"
+	"felis.lolicon.best/internal/backupjob"
 	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/reaper"
-	"felis.lolicon.best/internal/store"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
@@ -35,6 +37,8 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	server := fs.String("server", "", "server name whose world is being backed up")
 	formerOwner := fs.String("former-owner", "", "owner recorded on the backup row (empty for an unowned server)")
 	worldsRoot := fs.String("worlds-root", "/world", "mount path of the world PVC being archived")
+	reason := fs.String("reason", reasonManual, "world_backups reason: manual, pre_restore for the safety snapshot in front of a restore, or scheduled for felis-api's daily restore point")
+	protect := fs.String("protect", "", "backup id the prune must keep (the one a chained restore extracts)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -42,6 +46,10 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintln(stderr, "felis backup: --server is required")
 		return 2
 	}
+	if _, _, ok := backupPolicy(*reason, reaper.DefaultConfig()); !ok {
+		fmt.Fprintf(stderr, "felis backup: unknown --reason %q (manual, %s or %s)\n", *reason, backupjob.ReasonPreRestore, backupjob.ReasonScheduled)
+		return 2
+	}
 
 	cfg, err := config.Load(*cfgPath)
 	if err != nil {
@@ -52,13 +60,14 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis backup: archive store %q is not implemented in this build (only tarLocal)\n", cfg.Archive.Store)
 		return 1
 	}
-	// Reuse the reaper's retention derivation so an on-demand backup expires on the
-	// same clock as an inactivity backup — one retention policy, not two.
+	// The [archive] parse the reaper uses: it holds each reason's keep and
+	// retention.
 	rcfg, err := reaperConfig(cfg)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: %v\n", err)
 		return 1
 	}
+	keep, retention, _ := backupPolicy(*reason, rcfg)
 
 	// The world PVC is mounted directly at worldsRoot; the resolver returns it for
 	// any target, exactly as in cmdRestore. This is the same TarLocal the reaper
@@ -72,13 +81,25 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 
 	ctx := ctrl.SetupSignalHandler()
 
-	ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
+	// The archive store shares the node's disk with every world and the
+	// database: an owner's backup must not be what tips it into eviction.
+	if err := backup.CheckRoom(cfg.Archive.LocalPath, *worldsRoot, backup.MinFreeAfter); err != nil {
+		fmt.Fprintf(stderr, "felis backup: %v\n", err)
+		return 1
+	}
+
+	a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
 		return 1
 	}
+	ref, size := a.Ref, a.Size
+	if len(a.Skipped) > 0 {
+		fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",
+			len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))
+	}
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	drv, err := openPodStore(ctx, cfg.Database.URL, "backup", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: open database: %v\n", err)
 		return 1
@@ -91,10 +112,14 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		FormerOwner: *formerOwner,
 		BackupRef:   string(ref),
 		SizeBytes:   size,
-		Reason:      "manual",
-		ExpiresAt:   time.Now().Add(rcfg.Retention),
+		Reason:      *reason,
+		ExpiresAt:   time.Now().Add(retention),
+
+		SHA256:         a.SHA256,
+		SkippedEntries: len(a.Skipped),
 	}
-	if err := reaper.NewPGStore(drv.DB()).InsertBackup(ctx, rec); err != nil {
+	st := reaper.NewPGStore(drv.DB())
+	if err := st.InsertBackup(ctx, rec); err != nil {
 		// The archive is written but unrecorded — an orphan the retention pass would
 		// never expire. Delete it so a failed backup leaves no leaked bytes, mirroring
 		// the reaper's archive-then-record atomicity.
@@ -107,9 +132,63 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	}
 
 	fmt.Fprintf(stdout, "felis backup: server=%s archived %d bytes to %s (backup %s)\n", *server, size, ref, rec.ID)
+	pruneBackups(ctx, st, archiver, *server, *formerOwner, *reason, keep, *protect, stdout, stderr)
 	return 0
 }
 
+const (
+	reasonManual = "manual"
+	// preRestoreKeep is how many safety snapshots a server keeps: enough to walk
+	// back a couple of restores in a row, without every restore adding a world's
+	// worth of bytes for the full manual retention.
+	preRestoreKeep = 3
+)
+
+// backupPolicy is how many backups of one reason a server keeps and how long
+// each lives: an owner's own backups and the safety snapshots in front of a
+// restore by [archive] manual_keep / manual_retention (the snapshots capped at
+// preRestoreKeep), felis-api's daily restore points by scheduled_keep /
+// scheduled_retention, so neither kind crowds out the other. ok is false for a
+// reason this command does not record.
+func backupPolicy(reason string, rcfg reaper.Config) (keep int, retention time.Duration, ok bool) {
+	switch reason {
+	case reasonManual:
+		return rcfg.ManualKeep, rcfg.ManualRetention, true
+	case backupjob.ReasonPreRestore:
+		return preRestoreKeep, rcfg.ManualRetention, true
+	case backupjob.ReasonScheduled:
+		return rcfg.ScheduledKeep, rcfg.ScheduledRetention, true
+	}
+	return 0, 0, false
+}
+
+// pruneBackups keeps the newest keep backups of this reason that owner holds of
+// server and removes the rest, oldest first, so repeated backups of one world
+// cannot fill the shared archive store and a new owner's backups never remove a
+// previous owner's. protect is never removed: it is the backup a chained restore
+// is about to extract. The new backup is already recorded; a removal that fails
+// is reported and retried after the next backup.
+func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver interface {
+	Delete(context.Context, backup.ArchiveRef) error
+}, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) {
+	excess, err := st.ExcessBackups(ctx, server, owner, reason, keep, protect)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)
+		return
+	}
+	for _, b := range excess {
+		if err := archiver.Delete(ctx, backup.ArchiveRef(b.BackupRef)); err != nil {
+			fmt.Fprintf(stderr, "felis backup: remove older backup %s: %v\n", b.ID, err)
+			continue
+		}
+		if err := st.MarkBackupDeleted(ctx, b.ID, time.Now()); err != nil {
+			fmt.Fprintf(stderr, "felis backup: record the removal of %s: %v\n", b.ID, err)
+			continue
+		}
+		fmt.Fprintf(stdout, "felis backup: removed older %s backup %s of %s (keeping the newest %d)\n", reason, b.ID, server, keep)
+	}
+}
+
 // newBackupID mints a world_backups primary key, matching the reaper's "bk-"+hex
 // scheme so a manual and an inactivity backup are indistinguishable downstream.
 func newBackupID() string {

cmd/felis/backup_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/reaper"
+)
+
+// The reason decides which backups the new one's prune may remove, so an
+// unknown one is refused before anything is archived.
+func TestBackupSubcommandRejectsUnknownReason(t *testing.T) {
+	var stderr bytes.Buffer
+	if code := cmdBackup([]string{"--server", "survival", "--reason", "inactive_15d"}, &bytes.Buffer{}, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2 (%s)", code, stderr.String())
+	}
+	if !strings.Contains(stderr.String(), "unknown --reason") {
+		t.Fatalf("stderr = %q", stderr.String())
+	}
+}
+
+// Each reason is pruned and expired by its own [archive] keys: a daily
+// restore point must never count against, or take the lifetime of, the
+// backups an owner asked for.
+func TestBackupPolicyPerReason(t *testing.T) {
+	rcfg := reaper.DefaultConfig()
+	rcfg.ManualKeep, rcfg.ManualRetention = 5, 30*reaper.Day
+	rcfg.ScheduledKeep, rcfg.ScheduledRetention = 7, 90*reaper.Day
+	for _, tc := range []struct {
+		reason    string
+		keep      int
+		retention time.Duration
+	}{
+		{"manual", 5, 30 * reaper.Day},
+		{"pre_restore", preRestoreKeep, 30 * reaper.Day},
+		{"scheduled", 7, 90 * reaper.Day},
+	} {
+		keep, retention, ok := backupPolicy(tc.reason, rcfg)
+		if !ok || keep != tc.keep || retention != tc.retention {
+			t.Errorf("backupPolicy(%q) = (%d, %v, %v); want (%d, %v, true)", tc.reason, keep, retention, ok, tc.keep, tc.retention)
+		}
+	}
+	if _, _, ok := backupPolicy("inactive_15d", rcfg); ok {
+		t.Error("backupPolicy accepted inactive_15d; the reaper records those itself")
+	}
+}

cmd/felis/backupnow.go

@@ -4,15 +4,28 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"os/signal"
+	"sort"
+	"strings"
+	"syscall"
 	"time"
 
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/operator"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/store"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -20,9 +33,11 @@ import (
 // backupnow is the break-glass "back up a world now" op (§B4 "Sync"). Unlike halt —
 // which writes the CRD directly — a backup needs felis-api's deployment coordinates
 // (FELIS_IMAGE / FELIS_BACKUP_PVC) to render the one-shot backup Job, so the console
-// cannot do it in-process. It POSTs the felis-api INTERNAL face (service-token auth)
+// cannot do it in-process. It POSTs the felis-api INTERNAL face (ops-token auth)
 // while the API is alive, and the API renders the Job and audits the action. This file
-// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue.
+// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue. The
+// `felis backup-now` command (cmdBackupNow, below) takes the same route for every
+// user server in turn.
 
 // backupNowOutcome is the durable result of a backup request, re-printed after the TUI
 // alt-screen tears down.
@@ -33,7 +48,7 @@ type backupNowOutcome struct {
 
 // resolveInternalAPI reads the two things the on-node console needs to reach the
 // felis-api internal face: the felis-api-internal Service ClusterIP (the host's
-// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the service
+// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the ops
 // token. Both live in the control namespace.
 func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace string) (baseURL, token string, err error) {
 	var svc corev1.Service
@@ -45,13 +60,14 @@ func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace
 		return "", "", fmt.Errorf("%s Service has no ClusterIP yet", platform.APIInternalServiceName)
 	}
 
+	// The console's own token, which the api serves on the backup route alone.
 	var sec corev1.Secret
-	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.ServiceTokenSecretName}, &sec); err != nil {
-		return "", "", fmt.Errorf("get %s Secret: %w", naming.ServiceTokenSecretName, err)
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.OpsTokenSecretName}, &sec); err != nil {
+		return "", "", fmt.Errorf("get %s Secret (re-run the installer to create it): %w", naming.OpsTokenSecretName, err)
 	}
 	token = string(sec.Data[naming.ServiceTokenSecretKey])
 	if token == "" {
-		return "", "", fmt.Errorf("secret %s has no %s key", naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey)
+		return "", "", fmt.Errorf("secret %s has no %s key", naming.OpsTokenSecretName, naming.ServiceTokenSecretKey)
 	}
 
 	return fmt.Sprintf("http://%s:%d", ip, platform.APIInternalPort), token, nil
@@ -74,7 +90,7 @@ func requestBackup(ctx context.Context, hc *http.Client, baseURL, token, name, o
 
 	resp, err := hc.Do(req)
 	if err != nil {
-		return backupNowOutcome{}, fmt.Errorf("felis-api unreachable (a backup needs it alive): %w", err)
+		return backupNowOutcome{}, fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
 	}
 	defer resp.Body.Close()
 
@@ -144,3 +160,439 @@ func backupPickable(servers []haltableServer) []haltableServer {
 	}
 	return out
 }
+
+// cmdBackupNow is `felis backup-now`: the world of every user server (or of the
+// named ones) archived now, one at a time, through the internal backup route the
+// console's Sync uses. A world lives only in its volume and the off-site copy holds
+// only its archives, so this is the lever in front of a planned move to another
+// host, a disk swap or anything else that could lose a volume.
+func cmdBackupNow(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("backup-now", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	yes := fs.Bool("yes", false, "back up; without it the plan is printed and nothing changes")
+	stop := fs.Bool("stop", false, "stop the running servers first: their players are disconnected and the servers stay stopped")
+	fs.Usage = func() {
+		fmt.Fprintln(stderr, "Usage: felis backup-now [-yes] [-stop] [server ...]")
+		fmt.Fprintln(stderr)
+		fmt.Fprintln(stderr, "Archives the world of every user server, or of the named ones, one at a time, and waits for each archive.")
+		fmt.Fprintln(stderr, "A running server is skipped unless -stop is given. Without -yes it prints what it would do.")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis backup-now: refused — it reads the cluster's ops token, so it must run as root (try: sudo felis backup-now)")
+		return 1
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
+		return 1
+	}
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer cancel()
+
+	ns := cfg.K8s.Namespace
+	osUser := accountableOSUser()
+	jobs := api.NewK8sJobStatus(cl, ns)
+	var baseURL, token string
+	var repo ownerStore
+	var drv *store.PostgresDriver
+	defer func() {
+		if drv != nil {
+			_ = drv.Close()
+		}
+	}()
+	hc := &http.Client{Timeout: 10 * time.Second}
+	r := backupNowRun{
+		out:  stdout,
+		errw: stderr,
+		ns:   ns,
+		list: func(ctx context.Context) ([]backupNowWorld, error) { return listBackupNowWorlds(ctx, cl, ns) },
+		stopped: func(ctx context.Context, name string) (bool, error) {
+			var ms v1alpha1.MinecraftServer
+			if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &ms); err != nil {
+				return false, err
+			}
+			return backupNowStopped(ctx, cl, &ms)
+		},
+		halt: func(ctx context.Context, name string) error {
+			// The database is opened only once a server is to be stopped: the halt
+			// is audited like the console's, and a run with nothing running needs
+			// no more than the API.
+			if repo == nil {
+				d, err := store.Open(ctx, cfg.Database.URL)
+				if err != nil {
+					return fmt.Errorf("open the database for the audit log: %w", err)
+				}
+				drv, repo = d, api.NewPGRepo(d.DB())
+			}
+			out, err := performHalt(ctx, cl, repo, ns, name, osUser)
+			if err != nil {
+				return err
+			}
+			if out.auditErr != nil {
+				fmt.Fprintf(stderr, "felis backup-now: the audit row for stopping %s was not written: %v\n", name, out.auditErr)
+			}
+			return nil
+		},
+		request: func(ctx context.Context, name string) error {
+			if baseURL == "" {
+				var err error
+				if baseURL, token, err = resolveInternalAPI(ctx, cl, platform.DefaultControlNamespace); err != nil {
+					return fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
+				}
+			}
+			_, err := requestBackup(ctx, hc, baseURL, token, name, osUser)
+			return err
+		},
+		jobs:  jobs.LatestJobs,
+		now:   time.Now,
+		sleep: func(ctx context.Context, d time.Duration) { sleepCtx(ctx, d) },
+	}
+	return r.run(ctx, fs.Args(), *yes, *stop)
+}
+
+// sleepCtx waits d or until ctx ends.
+func sleepCtx(ctx context.Context, d time.Duration) {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+	case <-t.C:
+	}
+}
+
+// backupNowWorld is one user server as backup-now sees it.
+type backupNowWorld struct {
+	name     string
+	phase    string // the observed phase, or the desired state before the operator reconciled it
+	stopped  bool   // the backup route's stopped gate admits it
+	hasWorld bool   // its world volume exists
+}
+
+// listBackupNowWorlds lists the user servers of namespace in the API server's
+// order (by name), each with what the backup route checks. System servers are
+// left out: they have no row in the servers table, so the route refuses them
+// (backupPickable).
+func listBackupNowWorlds(ctx context.Context, cl client.Client, namespace string) ([]backupNowWorld, error) {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return nil, err
+	}
+	var out []backupNowWorld
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if isSystemServer(ms.Name) {
+			continue
+		}
+		stopped, err := backupNowStopped(ctx, cl, ms)
+		if err != nil {
+			return nil, err
+		}
+		var pvc corev1.PersistentVolumeClaim
+		err = cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: naming.WorldPVCName(ms.Name)}, &pvc)
+		if err != nil && !apierrors.IsNotFound(err) {
+			return nil, fmt.Errorf("look up the world volume of %s: %w", ms.Name, err)
+		}
+		phase := string(ms.Status.Phase)
+		if phase == "" {
+			phase = string(ms.Spec.DesiredState)
+		}
+		out = append(out, backupNowWorld{name: ms.Name, phase: phase, stopped: stopped, hasWorld: err == nil})
+	}
+	return out, nil
+}
+
+// backupNowStopped is the backup route's stopped gate (api.enqueueBackup and
+// K8sCluster.AcquireMaintenance together): desired Stopped, not ready, phase
+// Stopped and no game pod left.
+func backupNowStopped(ctx context.Context, cl client.Client, ms *v1alpha1.MinecraftServer) (bool, error) {
+	if ms.Spec.DesiredState != v1alpha1.DesiredStopped || ms.Status.Ready || ms.Status.Phase != v1alpha1.PhaseStopped {
+		return false, nil
+	}
+	var pods corev1.PodList
+	if err := cl.List(ctx, &pods, client.InNamespace(ms.Namespace), client.MatchingLabels{
+		v1alpha1.LabelServer: ms.Name, v1alpha1.LabelComponent: operator.ComponentValue,
+	}); err != nil {
+		return false, fmt.Errorf("look up the pod of %s: %w", ms.Name, err)
+	}
+	return len(pods.Items) == 0, nil
+}
+
+// errBackupAPIUnreachable is a backup request that never reached felis-api. It
+// ends a backup-now run: every later world would fail the same way, and stopping
+// servers for backups that cannot be taken only takes them away from players.
+var errBackupAPIUnreachable = errors.New("felis-api unreachable (a backup needs it alive)")
+
+// Polling of backup-now. A graceful stop saves the world first; the backup Job's
+// own deadline (backupjob, 30 minutes) ends a Job that hangs, so its wait needs no
+// cap of its own.
+const (
+	backupNowPoll      = 2 * time.Second
+	backupNowStopWait  = 10 * time.Minute
+	backupNowJobAppear = time.Minute
+)
+
+// backupNowRun is backup-now over seams, so the plan and the run are tested
+// without a cluster or felis-api.
+type backupNowRun struct {
+	out, errw io.Writer
+	ns        string // where the servers and their Jobs live, for the kubectl hints
+	list      func(ctx context.Context) ([]backupNowWorld, error)
+	stopped   func(ctx context.Context, name string) (bool, error)
+	halt      func(ctx context.Context, name string) error
+	request   func(ctx context.Context, name string) error
+	jobs      func(ctx context.Context, name string) ([]api.AsyncJob, error)
+	now       func() time.Time
+	sleep     func(ctx context.Context, d time.Duration)
+}
+
+// pickBackupNowWorlds narrows worlds to names, in the order given, or keeps them
+// all when names is empty.
+func pickBackupNowWorlds(worlds []backupNowWorld, names []string) ([]backupNowWorld, error) {
+	if len(names) == 0 {
+		return worlds, nil
+	}
+	byName := make(map[string]backupNowWorld, len(worlds))
+	for _, w := range worlds {
+		byName[w.name] = w
+	}
+	var out []backupNowWorld
+	seen := map[string]bool{}
+	for _, n := range names {
+		if seen[n] {
+			continue
+		}
+		seen[n] = true
+		w, ok := byName[n]
+		switch {
+		case ok:
+			out = append(out, w)
+		case isSystemServer(n):
+			return nil, fmt.Errorf("%s is a system server: its world is rebuilt by felis setup and has no backups", n)
+		default:
+			return nil, fmt.Errorf("no server named %q", n)
+		}
+	}
+	return out, nil
+}
+
+// backupNowAction is what the plan does with a world.
+func backupNowAction(w backupNowWorld, stop bool) string {
+	switch {
+	case w.stopped && !w.hasWorld:
+		return "skip: no world volume (never started, nothing to save)"
+	case w.stopped:
+		return "back up"
+	case stop:
+		return "stop, then back up"
+	default:
+		return "skip: running (stop it first, or pass -stop)"
+	}
+}
+
+func (r *backupNowRun) run(ctx context.Context, names []string, yes, stop bool) int {
+	all, err := r.list(ctx)
+	if err != nil {
+		fmt.Fprintf(r.errw, "felis backup-now: list the servers: %v\n", err)
+		return 1
+	}
+	worlds, err := pickBackupNowWorlds(all, names)
+	if err != nil {
+		fmt.Fprintf(r.errw, "felis backup-now: %v\n", err)
+		return 2
+	}
+	if len(worlds) == 0 {
+		fmt.Fprintln(r.out, "felis backup-now: there are no user servers")
+		return 0
+	}
+
+	// The stopped worlds go first, so a felis-api that cannot take a backup is
+	// found before any server is stopped for one.
+	sort.SliceStable(worlds, func(i, j int) bool { return worlds[i].stopped && !worlds[j].stopped })
+	width := 0
+	for _, w := range worlds {
+		width = max(width, len(w.name))
+	}
+	fmt.Fprintf(r.out, "felis backup-now: %d server(s), backed up one at a time:\n", len(worlds))
+	stopping, work := false, 0
+	for _, w := range worlds {
+		fmt.Fprintf(r.out, "  %-*s  %-8s  %s\n", width, w.name, w.phase, backupNowAction(w, stop))
+		stopping = stopping || (!w.stopped && stop)
+		if (w.stopped && w.hasWorld) || (!w.stopped && stop) {
+			work++
+		}
+	}
+	if work > 0 {
+		fmt.Fprintln(r.out, "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.")
+	}
+	if stopping {
+		fmt.Fprintln(r.out, "Stopping disconnects the players on those servers, and they stay stopped afterwards.")
+	}
+	if !yes {
+		if work == 0 {
+			fmt.Fprintln(r.out, "Nothing to back up.")
+		} else {
+			fmt.Fprintln(r.out, "Nothing changed. Run again with -yes to back them up.")
+		}
+		return 0
+	}
+
+	var done, failed, running, empty int
+	var leftStopped []string
+	for _, w := range worlds {
+		if err := ctx.Err(); err != nil {
+			break
+		}
+		switch {
+		case w.stopped && !w.hasWorld:
+			empty++
+			continue
+		case !w.stopped && !stop:
+			running++
+			continue
+		}
+		if !w.stopped {
+			halted, err := r.stopWorld(ctx, w.name)
+			if halted {
+				leftStopped = append(leftStopped, w.name)
+			}
+			if err != nil {
+				fmt.Fprintf(r.out, "  %s: failed: %v\n", w.name, err)
+				failed++
+				continue
+			}
+		}
+		err := r.backUp(ctx, w.name)
+		if errors.Is(err, errBackupAPIUnreachable) {
+			fmt.Fprintf(r.out, "  %s: failed: %v\n", w.name, err)
+			fmt.Fprintln(r.out, "Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).")
+			failed++
+			break
+		}
+		if err != nil {
+			fmt.Fprintf(r.out, "  %s: failed: %v\n", w.name, err)
+			failed++
+			continue
+		}
+		done++
+	}
+
+	interrupted := ctx.Err() != nil
+	if interrupted {
+		fmt.Fprintln(r.out, "Interrupted: a backup Job already started runs to its end.")
+	}
+	parts := []string{fmt.Sprintf("%d backed up", done)}
+	if failed > 0 {
+		parts = append(parts, fmt.Sprintf("%d failed", failed))
+	}
+	if running > 0 {
+		parts = append(parts, fmt.Sprintf("%d skipped (running)", running))
+	}
+	if empty > 0 {
+		parts = append(parts, fmt.Sprintf("%d without a world", empty))
+	}
+	fmt.Fprintf(r.out, "%s.\n", strings.Join(parts, ", "))
+	if len(leftStopped) > 0 {
+		fmt.Fprintf(r.out, "Left stopped: %s. Start them from the panel when you are done.\n", strings.Join(leftStopped, ", "))
+	}
+	if done > 0 {
+		fmt.Fprintln(r.out, "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.")
+	}
+	if failed > 0 || running > 0 || interrupted {
+		return 1
+	}
+	return 0
+}
+
+// stopWorld stops one server and waits until the backup route would admit it.
+// halted is whether the stop was asked for: the server then stays stopped, even
+// when it takes longer than the wait.
+func (r *backupNowRun) stopWorld(ctx context.Context, name string) (halted bool, err error) {
+	fmt.Fprintf(r.out, "  %s: stopping\n", name)
+	if err := r.halt(ctx, name); err != nil {
+		return false, err
+	}
+	start := r.now()
+	for {
+		ok, err := r.stopped(ctx, name)
+		if err != nil {
+			return true, err
+		}
+		if ok {
+			fmt.Fprintf(r.out, "  %s: stopped after %s\n", name, r.now().Sub(start).Round(time.Second))
+			return true, nil
+		}
+		if r.now().Sub(start) >= backupNowStopWait {
+			return true, fmt.Errorf("did not stop within %s (kubectl -n %s describe minecraftserver %s)", backupNowStopWait, r.ns, name)
+		}
+		if err := ctx.Err(); err != nil {
+			return true, err
+		}
+		r.sleep(ctx, backupNowPoll)
+	}
+}
+
+// backUp requests one world's backup and waits for its Job to finish. The Job is
+// the one of this server that was not there before the request.
+func (r *backupNowRun) backUp(ctx context.Context, name string) error {
+	before, err := r.jobs(ctx, name)
+	if err != nil {
+		return fmt.Errorf("list its backup Jobs: %w", err)
+	}
+	known := make(map[string]bool, len(before))
+	for _, j := range before {
+		known[j.Name] = true
+	}
+	if err := r.request(ctx, name); err != nil {
+		return err
+	}
+	fmt.Fprintf(r.out, "  %s: backing up\n", name)
+	start := r.now()
+	seen := ""
+	for {
+		jobs, err := r.jobs(ctx, name)
+		if err != nil {
+			return fmt.Errorf("list its backup Jobs: %w", err)
+		}
+		var job *api.AsyncJob
+		for i := range jobs {
+			if jobs[i].Kind == "backup" && (jobs[i].Name == seen || seen == "" && !known[jobs[i].Name]) {
+				job = &jobs[i]
+				break
+			}
+		}
+		switch {
+		case job == nil && seen != "":
+			return fmt.Errorf("its backup Job %s was deleted before it finished", seen)
+		case job == nil && r.now().Sub(start) >= backupNowJobAppear:
+			return fmt.Errorf("felis-api accepted the backup, but no backup Job appeared within %s (kubectl -n %s get jobs)", backupNowJobAppear, r.ns)
+		case job != nil && job.State == "succeeded":
+			fmt.Fprintf(r.out, "  %s: archived in %s\n", name, r.now().Sub(start).Round(time.Second))
+			return nil
+		case job != nil && job.State == "failed":
+			msg := job.Message
+			if msg == "" {
+				msg = "the backup Job failed"
+			}
+			return fmt.Errorf("%s (kubectl -n %s logs job/%s)", msg, r.ns, job.Name)
+		case job != nil:
+			seen = job.Name
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		r.sleep(ctx, backupNowPoll)
+	}
+}

cmd/felis/backupnow_cmd_test.go

@@ -0,0 +1,563 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// bnRig drives backupNowRun against scripted servers and Jobs on a fake clock that
+// moves only when the run sleeps.
+type bnRig struct {
+	out, errw bytes.Buffer
+	clock     time.Time
+	events    []string
+	worlds    []backupNowWorld
+	// stopAfter is how many polls a halted server takes to stop; -1 never does.
+	stopAfter map[string]int
+	polls     map[string]int
+	reqErr    map[string]error
+	// states is what the server's new backup Job reports on each poll after the
+	// request, the last one repeating; "" is no Job.
+	states   map[string][]string
+	messages map[string]string
+	jobPolls map[string]int
+	// stopErr fails a server's stop polls; jobsFail fails the nth (1-based) Job
+	// list of a server.
+	stopErr  map[string]error
+	jobsFail map[string]int
+	jobCalls map[string]int
+	// ctx is the run's context, and onAct runs after each halt and request.
+	ctx   context.Context
+	onAct func(event string)
+}
+
+func newBNRig(worlds ...backupNowWorld) *bnRig {
+	return &bnRig{
+		clock:     time.Unix(1_800_000_000, 0),
+		worlds:    worlds,
+		stopAfter: map[string]int{},
+		polls:     map[string]int{},
+		reqErr:    map[string]error{},
+		states:    map[string][]string{},
+		messages:  map[string]string{},
+		jobPolls:  map[string]int{},
+		stopErr:   map[string]error{},
+		jobsFail:  map[string]int{},
+		jobCalls:  map[string]int{},
+		ctx:       context.Background(),
+		onAct:     func(string) {},
+	}
+}
+
+func (g *bnRig) run(names []string, yes, stop bool) int {
+	r := backupNowRun{
+		out: &g.out, errw: &g.errw, ns: "minecraft",
+		list: func(context.Context) ([]backupNowWorld, error) {
+			return append([]backupNowWorld(nil), g.worlds...), nil
+		},
+		stopped: func(_ context.Context, name string) (bool, error) {
+			g.polls[name]++
+			if err := g.stopErr[name]; err != nil {
+				return false, err
+			}
+			n := g.stopAfter[name]
+			return n >= 0 && g.polls[name] > n, nil
+		},
+		halt: func(_ context.Context, name string) error {
+			g.events = append(g.events, "halt "+name)
+			g.onAct("halt " + name)
+			return nil
+		},
+		request: func(_ context.Context, name string) error {
+			g.events = append(g.events, "request "+name)
+			g.onAct("request " + name)
+			if err := g.reqErr[name]; err != nil {
+				return err
+			}
+			g.jobPolls[name] = 0
+			return nil
+		},
+		jobs: func(_ context.Context, name string) ([]api.AsyncJob, error) {
+			// Every server has an older finished backup, and a restore Job that
+			// shows up with the new backup: neither is the Job to wait for.
+			g.jobCalls[name]++
+			if g.jobCalls[name] == g.jobsFail[name] {
+				return nil, errors.New("the apiserver is gone")
+			}
+			out := []api.AsyncJob{{Name: "backup-" + name + "-old", Kind: "backup", State: "succeeded"}}
+			n, requested := g.jobPolls[name]
+			if !requested {
+				return out, nil
+			}
+			g.jobPolls[name] = n + 1
+			states := g.states[name]
+			if len(states) == 0 {
+				states = []string{"succeeded"}
+			}
+			state := states[min(n, len(states)-1)]
+			if state == "" {
+				return out, nil
+			}
+			return append([]api.AsyncJob{
+				{Name: "restore-" + name + "-x", Kind: "restore", State: "succeeded"},
+				{Name: "backup-" + name + "-new", Kind: "backup", State: state, Message: g.messages[name]},
+			}, out...), nil
+		},
+		now:   func() time.Time { return g.clock },
+		sleep: func(_ context.Context, d time.Duration) { g.clock = g.clock.Add(d) },
+	}
+	return r.run(g.ctx, names, yes, stop)
+}
+
+func stoppedWorld(name string) backupNowWorld {
+	return backupNowWorld{name: name, phase: "Stopped", stopped: true, hasWorld: true}
+}
+
+func runningWorld(name string) backupNowWorld {
+	return backupNowWorld{name: name, phase: "Running", hasWorld: true}
+}
+
+func emptyWorld(name string) backupNowWorld {
+	return backupNowWorld{name: name, phase: "Stopped", stopped: true}
+}
+
+const (
+	bnManualKeep = "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.\n"
+	bnStopping   = "Stopping disconnects the players on those servers, and they stay stopped afterwards.\n"
+	bnOffsite    = "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.\n"
+)
+
+func TestBackupNowPlanChangesNothing(t *testing.T) {
+	for _, tc := range []struct {
+		stop bool
+		want string
+	}{
+		{false, "felis backup-now: 4 server(s), backed up one at a time:\n" +
+			"  alpha    Stopped   back up\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   skip: running (stop it first, or pass -stop)\n" +
+			"  delta    Starting  skip: running (stop it first, or pass -stop)\n" +
+			bnManualKeep +
+			"Nothing changed. Run again with -yes to back them up.\n"},
+		{true, "felis backup-now: 4 server(s), backed up one at a time:\n" +
+			"  alpha    Stopped   back up\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   stop, then back up\n" +
+			"  delta    Starting  stop, then back up\n" +
+			bnManualKeep + bnStopping +
+			"Nothing changed. Run again with -yes to back them up.\n"},
+	} {
+		t.Run(fmt.Sprintf("stop=%v", tc.stop), func(t *testing.T) {
+			delta := runningWorld("delta")
+			delta.phase = "Starting"
+			g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), delta)
+			if code := g.run(nil, false, tc.stop); code != 0 {
+				t.Fatalf("exit = %d, want 0; stderr %q", code, g.errw.String())
+			}
+			if g.out.String() != tc.want {
+				t.Fatalf("plan =\n%s\nwant\n%s", g.out.String(), tc.want)
+			}
+			if len(g.events) != 0 || len(g.polls) != 0 {
+				t.Fatalf("the plan acted: events %v, polls %v", g.events, g.polls)
+			}
+		})
+	}
+}
+
+// A plan that saves nothing says so, without the manual_keep warning; a running
+// server counts as something to save once -stop is given.
+func TestBackupNowPlanWithNothingToSave(t *testing.T) {
+	for _, tc := range []struct {
+		stop bool
+		want string
+	}{
+		{false, "felis backup-now: 2 server(s), backed up one at a time:\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   skip: running (stop it first, or pass -stop)\n" +
+			"Nothing to back up.\n"},
+		{true, "felis backup-now: 2 server(s), backed up one at a time:\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   stop, then back up\n" +
+			bnManualKeep + bnStopping +
+			"Nothing changed. Run again with -yes to back them up.\n"},
+	} {
+		g := newBNRig(runningWorld("bravo"), emptyWorld("charlie"))
+		if code := g.run(nil, false, tc.stop); code != 0 {
+			t.Fatalf("stop=%v: exit = %d, want 0", tc.stop, code)
+		}
+		if g.out.String() != tc.want {
+			t.Fatalf("stop=%v: plan =\n%s\nwant\n%s", tc.stop, g.out.String(), tc.want)
+		}
+	}
+}
+
+func TestBackupNowBacksUpEachWorldInTurn(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), stoppedWorld("delta"))
+	g.states["alpha"] = []string{"", "running", "succeeded"}
+	g.states["delta"] = []string{"running", "failed"}
+	g.messages["delta"] = "felis backup: not enough free disk for the archive"
+	g.stopAfter["bravo"] = 2
+	g.states["bravo"] = []string{"running", "running", "running", "succeeded"}
+
+	code := g.run(nil, true, true)
+	if code != 1 {
+		t.Fatalf("exit = %d, want 1 (delta failed)", code)
+	}
+	if want := []string{"request alpha", "request delta", "halt bravo", "request bravo"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping+"")
+	want := "  alpha: backing up\n" +
+		"  alpha: archived in 4s\n" +
+		"  delta: backing up\n" +
+		"  delta: failed: felis backup: not enough free disk for the archive (kubectl -n minecraft logs job/backup-delta-new)\n" +
+		"  bravo: stopping\n" +
+		"  bravo: stopped after 4s\n" +
+		"  bravo: backing up\n" +
+		"  bravo: archived in 6s\n" +
+		"2 backed up, 1 failed, 1 without a world.\n" +
+		"Left stopped: bravo. Start them from the panel when you are done.\n" +
+		bnOffsite
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+}
+
+func TestBackupNowSkipsRunningServersWithoutStop(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"))
+	if code := g.run(nil, true, false); code != 1 {
+		t.Fatalf("exit = %d, want 1 (bravo was not backed up)", code)
+	}
+	if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	if len(g.polls) != 0 {
+		t.Fatalf("polled a server it did not stop: %v", g.polls)
+	}
+	_, run, _ := strings.Cut(g.out.String(), "Nothing changed")
+	if run != "" {
+		t.Fatalf("-yes printed the plan's closing line")
+	}
+	_, run, _ = strings.Cut(g.out.String(), bnManualKeep)
+	want := "  alpha: backing up\n  alpha: archived in 0s\n1 backed up, 1 skipped (running).\n" + bnOffsite
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+}
+
+func TestBackupNowExitsCleanWhenEverythingIsSaved(t *testing.T) {
+	// -stop with nothing running stops nothing and says nothing about stopping.
+	g := newBNRig(stoppedWorld("alpha"), emptyWorld("charlie"))
+	if code := g.run(nil, true, true); code != 0 {
+		t.Fatalf("exit = %d, want 0; output\n%s", code, g.out.String())
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
+	if want := "  alpha: backing up\n  alpha: archived in 0s\n1 backed up, 1 without a world.\n" + bnOffsite; run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+
+	g = newBNRig(emptyWorld("charlie"))
+	if code := g.run(nil, true, false); code != 0 {
+		t.Fatalf("exit = %d, want 0 for a server with nothing to save", code)
+	}
+	// Nothing to archive: no manual_keep warning, and no off-site hint.
+	if want := "felis backup-now: 1 server(s), backed up one at a time:\n" +
+		"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+		"0 backed up, 1 without a world.\n"; g.out.String() != want {
+		t.Fatalf("output =\n%s\nwant\n%s", g.out.String(), want)
+	}
+	if len(g.events) != 0 {
+		t.Fatalf("events = %v, want none", g.events)
+	}
+}
+
+func TestBackupNowStopsAtAnUnreachableAPI(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), runningWorld("charlie"))
+	g.reqErr["alpha"] = fmt.Errorf("%w: dial tcp 10.43.0.9:8081: connect: connection refused", errBackupAPIUnreachable)
+	if code := g.run(nil, true, true); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v: nothing after the API proved unreachable, and no server stopped", g.events, want)
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping)
+	want := "  alpha: failed: felis-api unreachable (a backup needs it alive): dial tcp 10.43.0.9:8081: connect: connection refused\n" +
+		"Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).\n" +
+		"0 backed up, 1 failed.\n"
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+
+	// Any other refusal is that world's alone: the run goes on.
+	g = newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
+	g.reqErr["alpha"] = errors.New("felis-api: the world is being restored")
+	if code := g.run(nil, true, false); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	if want := []string{"request alpha", "request bravo"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+}
+
+func TestBackupNowGivesUpOnAServerThatDoesNotStop(t *testing.T) {
+	g := newBNRig(runningWorld("bravo"), runningWorld("echo"))
+	g.stopAfter["bravo"] = -1
+	if code := g.run(nil, true, true); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	if want := []string{"halt bravo", "halt echo", "request echo"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	// One poll at the start and one per 2s sleep up to the 10-minute mark.
+	if g.polls["bravo"] != 301 {
+		t.Fatalf("bravo polled %d times, want 301", g.polls["bravo"])
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping)
+	want := "  bravo: stopping\n" +
+		"  bravo: failed: did not stop within 10m0s (kubectl -n minecraft describe minecraftserver bravo)\n" +
+		"  echo: stopping\n" +
+		"  echo: stopped after 0s\n" +
+		"  echo: backing up\n" +
+		"  echo: archived in 0s\n" +
+		"1 backed up, 1 failed.\n" +
+		"Left stopped: bravo, echo. Start them from the panel when you are done.\n" +
+		bnOffsite
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+}
+
+func TestBackupNowReportsAJobThatNeverRuns(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		states []string
+		polls  int
+		want   string
+	}{
+		{"never appears", []string{""}, 31,
+			"felis-api accepted the backup, but no backup Job appeared within 1m0s (kubectl -n minecraft get jobs)"},
+		{"deleted while running", []string{"running", "running", ""}, 3,
+			"its backup Job backup-alpha-new was deleted before it finished"},
+		{"fails without a message", []string{"failed"}, 1,
+			"the backup Job failed (kubectl -n minecraft logs job/backup-alpha-new)"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			g := newBNRig(stoppedWorld("alpha"))
+			g.states["alpha"] = tc.states
+			if code := g.run(nil, true, false); code != 1 {
+				t.Fatalf("exit = %d, want 1", code)
+			}
+			if !strings.Contains(g.out.String(), "  alpha: failed: "+tc.want+"\n") {
+				t.Fatalf("output =\n%s\nwant the line %q", g.out.String(), tc.want)
+			}
+			if g.jobPolls["alpha"] != tc.polls {
+				t.Fatalf("polled the Jobs %d times after the request, want %d", g.jobPolls["alpha"], tc.polls)
+			}
+		})
+	}
+}
+
+func TestBackupNowNamedServers(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), stoppedWorld("delta"))
+	if code := g.run([]string{"delta", "alpha", "delta"}, true, false); code != 0 {
+		t.Fatalf("exit = %d, want 0", code)
+	}
+	if want := []string{"request delta", "request alpha"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	if !strings.HasPrefix(g.out.String(), "felis backup-now: 2 server(s), backed up one at a time:\n  delta  Stopped   back up\n  alpha  Stopped   back up\n") {
+		t.Fatalf("plan =\n%s", g.out.String())
+	}
+
+	for _, tc := range []struct{ name, want string }{
+		{"login", "felis backup-now: login is a system server: its world is rebuilt by felis setup and has no backups\n"},
+		{"lobby", "felis backup-now: lobby is a system server: its world is rebuilt by felis setup and has no backups\n"},
+		{"nope", "felis backup-now: no server named \"nope\"\n"},
+	} {
+		g := newBNRig(stoppedWorld("alpha"))
+		if code := g.run([]string{"alpha", tc.name}, true, false); code != 2 {
+			t.Fatalf("%s: exit = %d, want 2", tc.name, code)
+		}
+		if g.errw.String() != tc.want {
+			t.Fatalf("%s: stderr = %q, want %q", tc.name, g.errw.String(), tc.want)
+		}
+		if len(g.events) != 0 || g.out.Len() != 0 {
+			t.Fatalf("%s: acted on a bad name: events %v, output %q", tc.name, g.events, g.out.String())
+		}
+	}
+
+	g = newBNRig()
+	if code := g.run(nil, true, false); code != 0 || g.out.String() != "felis backup-now: there are no user servers\n" {
+		t.Fatalf("empty fleet: exit %d, output %q", code, g.out.String())
+	}
+}
+
+// The world list mirrors the backup route's own gate, so the plan says exactly what
+// the route would refuse.
+func TestListBackupNowWorlds(t *testing.T) {
+	withStatus := func(ms *v1alpha1.MinecraftServer, ready bool) *v1alpha1.MinecraftServer {
+		ms.Status.Ready = ready
+		return ms
+	}
+	pvc := func(server, ns string) *corev1.PersistentVolumeClaim {
+		return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: naming.WorldPVCName(server), Namespace: ns}}
+	}
+	pod := func(name, ns string, labels map[string]string) *corev1.Pod {
+		return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}}
+	}
+	gameLabels := func(server string) map[string]string {
+		return map[string]string{v1alpha1.LabelServer: server, v1alpha1.LabelComponent: "server"}
+	}
+	other := mcServer("zulu", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped)
+	other.Namespace = "elsewhere"
+	hotel := mcServer("hotel", v1alpha1.DesiredStopped, "")
+	objs := []client.Object{
+		mcServer("alpha", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("alpha", haltNS),
+		// A backup Job's pod carries the server label with its own component, and
+		// a game pod of the same name in another namespace is somebody else's.
+		pod("backup-alpha-1-x", haltNS, map[string]string{v1alpha1.LabelServer: "alpha", "app.kubernetes.io/component": "world-backup"}),
+		pod("alpha-0", "elsewhere", gameLabels("alpha")),
+		withStatus(mcServer("bravo", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning), true), pvc("bravo", haltNS), pod("bravo-0", haltNS, gameLabels("bravo")),
+		mcServer("charlie", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped),
+		mcServer("delta", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("delta", haltNS), pod("delta-0", haltNS, gameLabels("delta")),
+		mcServer("echo", v1alpha1.DesiredStopped, v1alpha1.PhaseStopping), pvc("echo", haltNS),
+		mcServer("foxtrot", v1alpha1.DesiredRunning, v1alpha1.PhaseStopped), pvc("foxtrot", haltNS),
+		withStatus(mcServer("golf", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), true), pvc("golf", haltNS),
+		hotel, pvc("hotel", haltNS),
+		mcServer("login", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("login", haltNS),
+		mcServer("lobby", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("lobby", haltNS),
+		other, pvc("zulu", "elsewhere"),
+	}
+	got, err := listBackupNowWorlds(context.Background(), haltClient(t, objs...), haltNS)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []backupNowWorld{
+		{name: "alpha", phase: "Stopped", stopped: true, hasWorld: true},
+		{name: "bravo", phase: "Running", hasWorld: true},
+		{name: "charlie", phase: "Stopped", stopped: true},
+		{name: "delta", phase: "Stopped", hasWorld: true},   // its pod is still going
+		{name: "echo", phase: "Stopping", hasWorld: true},   // still stopping
+		{name: "foxtrot", phase: "Stopped", hasWorld: true}, // asked to start
+		{name: "golf", phase: "Stopped", hasWorld: true},    // still reports ready
+		{name: "hotel", phase: "Stopped", hasWorld: true},   // not reconciled yet: the desired state shows
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("worlds =\n%+v\nwant\n%+v", got, want)
+	}
+}
+
+func TestBackupNowStopsWhenInterrupted(t *testing.T) {
+	t.Run("between worlds", func(t *testing.T) {
+		g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		g.ctx = ctx
+		g.onAct = func(e string) {
+			if e == "request alpha" {
+				cancel()
+			}
+		}
+		if code := g.run(nil, true, false); code != 1 {
+			t.Fatalf("exit = %d, want 1", code)
+		}
+		if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
+			t.Fatalf("events = %v, want %v", g.events, want)
+		}
+		_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
+		want := "  alpha: backing up\n  alpha: archived in 0s\n" +
+			"Interrupted: a backup Job already started runs to its end.\n" +
+			"1 backed up.\n" + bnOffsite
+		if run != want {
+			t.Fatalf("run =\n%s\nwant\n%s", run, want)
+		}
+	})
+	t.Run("while a Job runs", func(t *testing.T) {
+		g := newBNRig(stoppedWorld("alpha"))
+		g.states["alpha"] = []string{"running"}
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		g.ctx = ctx
+		g.onAct = func(string) { cancel() }
+		if code := g.run(nil, true, false); code != 1 {
+			t.Fatalf("exit = %d, want 1", code)
+		}
+		if g.jobPolls["alpha"] != 1 {
+			t.Fatalf("polled the Job %d times after the interrupt, want 1", g.jobPolls["alpha"])
+		}
+		if !strings.Contains(g.out.String(), "  alpha: failed: context canceled\nInterrupted: a backup Job already started runs to its end.\n0 backed up, 1 failed.\n") {
+			t.Fatalf("output =\n%s", g.out.String())
+		}
+	})
+	t.Run("while a server stops", func(t *testing.T) {
+		g := newBNRig(runningWorld("bravo"))
+		g.stopAfter["bravo"] = -1
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		g.ctx = ctx
+		g.onAct = func(string) { cancel() }
+		if code := g.run(nil, true, true); code != 1 {
+			t.Fatalf("exit = %d, want 1", code)
+		}
+		if g.polls["bravo"] != 1 {
+			t.Fatalf("polled bravo %d times after the interrupt, want 1", g.polls["bravo"])
+		}
+		_, run, _ := strings.Cut(g.out.String(), bnStopping)
+		want := "  bravo: stopping\n  bravo: failed: context canceled\n" +
+			"Interrupted: a backup Job already started runs to its end.\n" +
+			"0 backed up, 1 failed.\n" +
+			"Left stopped: bravo. Start them from the panel when you are done.\n"
+		if run != want {
+			t.Fatalf("run =\n%s\nwant\n%s", run, want)
+		}
+	})
+}
+
+func TestBackupNowReportsClusterErrors(t *testing.T) {
+	g := newBNRig(runningWorld("bravo"))
+	g.stopErr["bravo"] = errors.New("the apiserver is gone")
+	if code := g.run(nil, true, true); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping)
+	// The stop was asked for, so the server stays stopped whatever the poll said.
+	want := "  bravo: stopping\n  bravo: failed: the apiserver is gone\n0 backed up, 1 failed.\n" +
+		"Left stopped: bravo. Start them from the panel when you are done.\n"
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+
+	for _, tc := range []struct {
+		call   int
+		events []string
+	}{
+		{1, nil},                       // before the request: nothing is asked for
+		{2, []string{"request alpha"}}, // the first poll after it
+	} {
+		g := newBNRig(stoppedWorld("alpha"))
+		g.jobsFail["alpha"] = tc.call
+		if code := g.run(nil, true, false); code != 1 {
+			t.Fatalf("call %d: exit = %d, want 1", tc.call, code)
+		}
+		if !reflect.DeepEqual(g.events, tc.events) {
+			t.Fatalf("call %d: events = %v, want %v", tc.call, g.events, tc.events)
+		}
+		if !strings.Contains(g.out.String(), "  alpha: failed: list its backup Jobs: the apiserver is gone\n") {
+			t.Fatalf("call %d: output =\n%s", tc.call, g.out.String())
+		}
+	}
+}

cmd/felis/backupnow_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,7 +12,6 @@ import (
 	"testing"
 	"time"
 
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 
 	corev1 "k8s.io/api/core/v1"
@@ -28,9 +28,15 @@ func internalAPIObjs(clusterIP, token string) []client.Object {
 			ObjectMeta: metav1.ObjectMeta{Name: platform.APIInternalServiceName, Namespace: bgControlNS},
 			Spec:       corev1.ServiceSpec{ClusterIP: clusterIP},
 		},
+		// The console presents the ops token; the proxy's felis-service-token sits
+		// beside it and must not be the one picked up.
 		&corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: bgControlNS},
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-ops-token", Namespace: bgControlNS},
+			Data:       map[string][]byte{"token": []byte(token)},
+		},
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-service-token", Namespace: bgControlNS},
+			Data:       map[string][]byte{"token": []byte("proxy-" + token)},
 		},
 	}
 }
@@ -147,6 +153,10 @@ func TestRequestBackup(t *testing.T) {
 		if err == nil || !strings.Contains(err.Error(), "unreachable") {
 			t.Fatalf("err = %v, want an 'unreachable' transport error", err)
 		}
+		// backup-now ends its run on this error alone.
+		if !errors.Is(err, errBackupAPIUnreachable) {
+			t.Fatalf("err = %v, want it to wrap errBackupAPIUnreachable", err)
+		}
 	})
 }
 

cmd/felis/bootstrap_assets.go

@@ -9,10 +9,17 @@ import (
 
 func cmdBootstrapAssets(args []string, stdout, stderr io.Writer) int {
 	if len(args) != 1 {
-		fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack")
+		fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack|config-keys")
 		return 2
 	}
 	switch args[0] {
+	case "config-keys":
+		// Optional keys the installer may emit; older binaries reject this verb.
+		_, err := fmt.Fprintln(stdout, "velocity.game_version")
+		if err != nil {
+			fmt.Fprintf(stderr, "felis bootstrap-assets: write: %v\n", err)
+			return 1
+		}
 	case "crd":
 		crd, err := felis.MinecraftServerCRD()
 		if err != nil {
@@ -32,7 +39,7 @@ func cmdBootstrapAssets(args []string, stdout, stderr io.Writer) int {
 			return 1
 		}
 	default:
-		fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack")
+		fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack|config-keys")
 		return 2
 	}
 	return 0

cmd/felis/breakglass.go

@@ -11,12 +11,14 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"strings"
 	"time"
 
 	"felis.lolicon.best/internal/api"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/store"
 
 	tea "github.com/charmbracelet/bubbletea"
@@ -33,13 +35,15 @@ import (
 //
 // Root is necessary but NOT sufficient for accountability: root is machine
 // authority, not a human identity, so the console additionally captures WHO is
-// breaking the glass. When a staff account already exists it asks the operator to
-// authenticate as an existing admin (the verified identity is the accountable
-// actor); when none exists yet it bootstraps the first Owner from the typed
-// credential and attributes the act to the OS user. The audit row records the
-// difference. This attribution is best-effort, not tamper-proof — whoever runs
-// this is root and can edit Postgres directly — but it produces an honest trail
-// for an honest operator, which is the point.
+// breaking the glass. When a staff account already exists the operator names one
+// and types the one-time code the console mails to its verified address
+// (breakglass_otp.go); that account is then the accountable actor. When no code can
+// be sent or proven, the typed OVERRIDE proceeds as the OS user and the audit row
+// says why. When no staff account exists yet it bootstraps the first Owner and
+// attributes the act to the OS user. The audit row records which of these
+// happened. This attribution is best-effort, not tamper-proof — whoever runs this
+// is root and can edit Postgres directly — but it produces an honest trail for an
+// honest operator, which is the point.
 //
 // When a staff account already exists the console opens on a thin top-level menu
 // (menuModel) so that operations are peers, not tails of one wizard. Two account
@@ -62,7 +66,7 @@ import (
 // suspension for the interactive `cloudflared tunnel login` browser consent.
 
 // breakGlassOverrideToken is the literal an operator must type to proceed when no
-// admin credential could be verified. Requiring an explicit, deliberate word (not a
+// admin could be verified by a mailed code. Requiring an explicit, deliberate word (not a
 // bare Enter) keeps the unverified root override from happening by reflex.
 const breakGlassOverrideToken = "OVERRIDE"
 
@@ -90,11 +94,9 @@ type ownerStore interface {
 	// role=owner identity (migration 0011 adds that role); the two are the only
 	// staff roles.
 	InsertOperator(ctx context.Context, id, username, email string) error
-	// CompleteOwnerSetup atomically consumes the in-game link code, creates or
-	// promotes the bound Owner, enables local auth, and stores the one-time setup
-	// token. A failure rolls all four writes back so setup is always retryable.
-	CompleteOwnerSetup(ctx context.Context, newUserID, code string, now time.Time,
-		tokenHash string, tokenExpiresAt time.Time) (userID, mcUUID, authSource string, err error)
+	// CompleteOwnerSetup creates or resumes the first panel login atomically.
+	CompleteOwnerSetup(ctx context.Context, newUserID string, now time.Time,
+		tokenHash string, tokenExpiresAt time.Time) (userID, username string, err error)
 	SetSetting(ctx context.Context, key string, value []byte) error
 	// Audit records the break-glass accountability row.
 	Audit(ctx context.Context, e api.AuditEntry) error
@@ -142,7 +144,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 	repo := api.NewPGRepo(drv.DB())
 
 	// Decide bootstrap (no admin yet → typed credential mints the first Owner) vs
-	// recovery (an admin exists → the operator must authenticate as one) BEFORE the
+	// recovery (an admin exists → the operator proves one with a mailed code) BEFORE the
 	// alt-screen TUI takes over, so a database fault surfaces as a plain error.
 	adminExists, err := repo.AdminExists(ctx)
 	if err != nil {
@@ -150,7 +152,12 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	res, err := runBreakGlassTUI(ctx, repo, cfg.Database.URL, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists)
+	// Recovery mails its code through [smtp]; the relay is opened only if a code is
+	// asked for.
+	host, _ := os.Hostname()
+	recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, hostSMTPPasswordPath, platform.DefaultControlNamespace), host: host}
+
+	res, err := runBreakGlassTUI(ctx, repo, cfg.Database, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists, recovery)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis breakGlass: %v\n", err)
 		return 1
@@ -174,6 +181,9 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 			fmt.Fprintf(stdout, "\nfelis breakGlass: Owner account %q provisioned; local session sign-in is ENABLED.\n", res.username)
 		}
 		fmt.Fprintf(stdout, "Recorded as %q (mode: %s, os user: %s).\n", res.accountable, res.mode, res.osUser)
+		if res.mode == "root_override" {
+			fmt.Fprintln(stdout, "No admin was proven by an email code; the audit row records this run as an unverified root override and why.")
+		}
 		if res.setupTokenURL != "" {
 			fmt.Fprintf(stdout, "One-time setup URL (opens a lockdown session to verify email / enroll passkey):\n\n    %s\n\n", res.setupTokenURL)
 		}
@@ -258,29 +268,6 @@ func newOwnerID() string {
 	return "usr-" + hex.EncodeToString(b[:])
 }
 
-// authenticateAdmin resolves a typed admin username for recovery-mode attribution.
-// Password verification is gone (passwordless design); Phase 3 replaces this with
-// email-OTP recovery. For now it confirms the named admin exists.
-func authenticateAdmin(ctx context.Context, s ownerStore, username string) (matched string, ok bool, err error) {
-	username = strings.TrimSpace(username)
-	if username == "" {
-		return "", false, nil
-	}
-	u, err := s.UserByUsername(ctx, username)
-	if errors.Is(err, api.ErrNotFound) {
-		return "", false, nil
-	}
-	if err != nil {
-		return "", false, err
-	}
-	// Staff means admin OR owner: recovery attribution must accept the Owner (the
-	// primary break-glass identity), not just plain admins.
-	if u.Role != "admin" && u.Role != "owner" {
-		return "", false, nil
-	}
-	return u.Username, true, nil
-}
-
 // provisionOwner mints or resets the single Owner account direct-to-Postgres,
 // passwordless. The account is role=owner with no password — the Owner completes
 // passwordless login setup via the web setup-token flow after `felis setup`.
@@ -371,12 +358,16 @@ type breakGlassOp struct {
 	ownerUsername  string
 	ownerEmail     string
 	attemptedAdmin string // recovery / override: the admin username the operator typed
+	verifiedBy     string // recovery: how the admin was proven (verifiedByEmailOTP)
+	codeSentTo     string // recovery: the address the proving code went to
+	otpSkipped     string // root_override: why no code proved an admin (otpSkip*)
+	otpSkipDetail  string // root_override: what failed, when something did
 }
 
 // breakGlassOutcome is what performBreakGlass reports back to the TUI.
 type breakGlassOutcome struct {
 	setupTokenURL string // non-empty when setup minted a one-time first-login URL
-	ownerIdentity string // verified Minecraft UUID for the setup Owner-bind path
+	ownerUsername string // panel Owner created or resumed by setup
 	auditErr      error  // non-nil if the accountability row could not be written
 }
 
@@ -416,25 +407,12 @@ func newSetupToken() (raw, hash string, err error) {
 	return raw, hex.EncodeToString(sum[:]), nil
 }
 
-// performSetupMCBind is the `felis setup` Owner-establishment path: the operator
-// binds their Minecraft account via a one-time link code the login gate handed
-// them in-game, the bound user is promoted to role='owner' (passwordless Owner),
-// local auth is enabled, and a one-time setup URL is minted for the first web
-// login where the Owner verifies email / enrolls a passkey. adminHostname is the
-// operator-console host the URL points at (op.console.<root>): the Owner is staff,
-// so first-run onboarding belongs on the operator face, not the player panel. The
-// passkey verifier's RP id is the panel host, but its permitted origins now include
-// op.console (cmd/felis/api.go), so enrollment on op.console is a valid ceremony —
-// one binding that works on both faces. osUser is recorded as the accountable actor.
-//
-// Local auth is as load-bearing here as it is in break-glass, and for a sharper
-// reason: an MC-bound Owner has no password AND no email, so the setup token is
-// their ONLY door. CompleteOwnerSetup therefore commits the identity bind, auth
-// toggle, and token together; any failed write leaves the link code retryable.
-func performSetupMCBind(ctx context.Context, s ownerStore, code, adminHostname, osUser string) (breakGlassOutcome, error) {
-	code = strings.TrimSpace(strings.ToUpper(code))
-	if code == "" {
-		return breakGlassOutcome{}, errors.New("link code is required")
+// performSetupOwner establishes panel access under the caller's host-root
+// authority. Minecraft identity can be linked later from the authenticated panel.
+func performSetupOwner(ctx context.Context, s ownerStore, panelURL, osUser string) (breakGlassOutcome, error) {
+	base, err := url.Parse(strings.TrimRight(panelURL, "/"))
+	if err != nil || base.Scheme != "https" || base.Host == "" {
+		return breakGlassOutcome{}, errors.New("a configured HTTPS operator console is required")
 	}
 	newID := newOwnerID()
 	if newID == "" {
@@ -445,48 +423,21 @@ func performSetupMCBind(ctx context.Context, s ownerStore, code, adminHostname,
 		return breakGlassOutcome{}, err
 	}
 	now := time.Now()
-	_, mcUUID, authSource, err := s.CompleteOwnerSetup(
-		ctx, newID, code, now, hash, now.Add(setupTokenTTL))
+	userID, username, err := s.CompleteOwnerSetup(ctx, newID, now, hash, now.Add(setupTokenTTL))
+	out := breakGlassOutcome{ownerUsername: username}
 	if err != nil {
-		return breakGlassOutcome{}, fmt.Errorf("complete owner setup: %w", err)
+		return out, err
 	}
-	// The load-bearing writes committed together above. Accountability remains
-	// best-effort: an unhappy audit sink never costs the operator their install.
-	out := breakGlassOutcome{
-		ownerIdentity: mcUUID,
-		auditErr:      auditSetupMCBind(ctx, s, osUser, mcUUID, authSource),
-	}
-	host := strings.TrimSpace(adminHostname)
-	if host == "" {
-		host = "op.console.localhost"
-	}
-	out.setupTokenURL = "https://" + host + "/setup?token=" + raw
+	base.Path = "/setup"
+	base.RawQuery = url.Values{"token": {raw}}.Encode()
+	out.setupTokenURL = base.String()
+	blob, _ := json.Marshal(map[string]string{"os_user": osUser, "user_id": userID, "username": username})
+	out.auditErr = s.Audit(ctx, api.AuditEntry{
+		Actor: osUser, Source: "setup", Action: "setup.owner_login", Payload: blob,
+	})
 	return out, nil
 }
 
-// auditSetupMCBind records who claimed the Owner seat at setup. It carries the
-// Minecraft identity rather than a username because that IS the evidence: the
-// login gate only issues a link code to a player it authenticated, so mc_uuid +
-// auth_source say which account was verified and by whom. Actor is the OS user who
-// ran `felis setup` — honest attribution, not proof (root can edit the row).
-func auditSetupMCBind(ctx context.Context, s ownerStore, osUser, mcUUID, authSource string) error {
-	blob, err := json.Marshal(map[string]any{
-		"mode":        "setup",
-		"os_user":     osUser,
-		"mc_uuid":     mcUUID,
-		"auth_source": authSource,
-	})
-	if err != nil {
-		return err
-	}
-	return s.Audit(ctx, api.AuditEntry{
-		Actor:   osUser,
-		Source:  "setup",
-		Action:  "setup.owner_bind",
-		Payload: blob,
-	})
-}
-
 // auditBreakGlass writes the break-glass accountability row. The actor is the
 // resolved human identity (a verified admin in recovery, the OS user otherwise);
 // the payload carries the full who/what/how so an after-the-fact reader can tell a
@@ -494,16 +445,7 @@ func auditSetupMCBind(ctx context.Context, s ownerStore, osUser, mcUUID, authSou
 // does not fail the recovery if this write fails — and intentionally honest: it
 // records attribution, it does not prove it (a malicious root can edit the row).
 func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
-	payload := map[string]any{
-		"mode":     op.mode,
-		"owner":    op.ownerUsername,
-		"os_user":  op.osUser,
-		"verified": op.mode == "recovery",
-	}
-	if op.attemptedAdmin != "" {
-		payload["admin_account"] = op.attemptedAdmin
-	}
-	blob, err := json.Marshal(payload)
+	blob, err := json.Marshal(breakGlassPayload(op, "owner"))
 	if err != nil {
 		return err
 	}
@@ -515,6 +457,33 @@ func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
 	})
 }
 
+// breakGlassPayload is the who/how both account audits carry, with the account the
+// run wrote under subjectKey. verified is true only for a run a mailed code proved;
+// such a run names the address the code went to, and an override names why no code
+// proved anyone.
+func breakGlassPayload(op breakGlassOp, subjectKey string) map[string]any {
+	payload := map[string]any{
+		"mode":     op.mode,
+		subjectKey: op.ownerUsername,
+		"os_user":  op.osUser,
+		"verified": op.verifiedBy != "",
+	}
+	if op.attemptedAdmin != "" {
+		payload["admin_account"] = op.attemptedAdmin
+	}
+	if op.verifiedBy != "" {
+		payload["verified_by"] = op.verifiedBy
+		payload["code_sent_to"] = op.codeSentTo
+	}
+	if op.otpSkipped != "" {
+		payload["otp_skipped"] = op.otpSkipped
+		if op.otpSkipDetail != "" {
+			payload["otp_skip_detail"] = op.otpSkipDetail
+		}
+	}
+	return payload
+}
+
 // performAddOperator mints a NEW Operator account and records a best-effort
 // accountability row. It mirrors performBreakGlass — passwordless — with two
 // deliberate differences. (1) It provisions insert-only (provisionOperator), so it
@@ -538,16 +507,7 @@ func performAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) (bre
 // break_glass.operator_create action, naming the new account under an "operator" key
 // rather than "owner".
 func auditAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) error {
-	payload := map[string]any{
-		"mode":     op.mode,
-		"operator": op.ownerUsername,
-		"os_user":  op.osUser,
-		"verified": op.mode == "recovery",
-	}
-	if op.attemptedAdmin != "" {
-		payload["admin_account"] = op.attemptedAdmin
-	}
-	blob, err := json.Marshal(payload)
+	blob, err := json.Marshal(breakGlassPayload(op, "operator"))
 	if err != nil {
 		return err
 	}
@@ -626,16 +586,31 @@ const (
 	cloudflareAPITokenDocsURL        = "https://developers.cloudflare.com/fundamentals/api/how-to/account-owned-token-template/"
 )
 
-func runBreakGlassTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
-	return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass)
+func runBreakGlassTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, recovery recoveryConfig) (breakGlassResult, error) {
+	return runConsoleTUI(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass, recovery)
 }
 
-func runSetupTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
-	return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup)
+// runSetupTUI configures deployment before issuing the first panel login link.
+func runSetupTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
+	return runConsoleRoot(newConsoleRoot(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup, recoveryConfig{}))
 }
 
-func runConsoleTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) (breakGlassResult, error) {
-	rm := newRootModel(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
+// newConsoleRoot is the console's root model as the host runs it: the summary
+// reads this host's alert route.
+func newConsoleRoot(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) *rootModel {
+	rm := newRootModel(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
+	rm.recovery = recovery
+	rm.alertRoute = func(ctx context.Context) alertRoute {
+		return hostAlertRoute(ctx, hostSetupConfigPath, db.URL, defaultHeartbeatFile)
+	}
+	return rm
+}
+
+func runConsoleTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) (breakGlassResult, error) {
+	return runConsoleRoot(newConsoleRoot(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode, recovery))
+}
+
+func runConsoleRoot(rm *rootModel) (breakGlassResult, error) {
 	final, err := tea.NewProgram(rm, tea.WithAltScreen()).Run()
 	if err != nil {
 		return breakGlassResult{}, err

cmd/felis/breakglass_otp.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/subtle"
+	"errors"
+	"fmt"
+	"math/big"
+	"os"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
+)
+
+// Recovery mode proves who is breaking the glass (#13). Naming a staff account is
+// where it starts: the console then mails a one-time code to that account's verified
+// address, and only that code, typed within recoveryCodeTTL, makes the run a
+// recovery attributed to the account. Every other ending — no such account, no
+// verified address, no relay, a send that fails, a wrong or late code, or the
+// operator giving up on the mail — leads to the typed OVERRIDE, which the audit row
+// records as an unverified root_override together with the reason (otp_skipped).
+// The code goes through the same [smtp] relay as the panel's login codes, so with
+// that relay down recovery still works, as an override that says why.
+
+const (
+	recoveryCodeTTL      = 10 * time.Minute
+	recoveryCodeAttempts = 5
+)
+
+// The reasons a run fell back to the override, recorded as otp_skipped.
+const (
+	otpSkipUnknownAdmin    = "unknown_admin"
+	otpSkipNoVerifiedEmail = "no_verified_email"
+	otpSkipNoRelay         = "no_relay"
+	otpSkipSendFailed      = "send_failed"
+	otpSkipCodeExpired     = "code_expired"
+	otpSkipCodeRejected    = "code_rejected"
+	otpSkipByOperator      = "operator_skipped"
+)
+
+// verifiedByEmailOTP is the audit's verified_by for a recovery the mailed code proved.
+const verifiedByEmailOTP = "email_otp"
+
+// recoveryMailer is the one relay call a recovery code needs; *mail.SMTP has it.
+type recoveryMailer interface {
+	SendNotice(ctx context.Context, email, subject, body string) error
+}
+
+// recoveryConfig is what the console needs to mail a recovery code. open resolves
+// the relay only when a code is about to go out, so a console used to halt a server
+// never touches [smtp] or the cluster; its error says why no relay is available.
+// host names this machine in the mail. The zero value has no relay.
+type recoveryConfig struct {
+	open func(ctx context.Context) (recoveryMailer, error)
+	host string
+	now  func() time.Time
+}
+
+func (r recoveryConfig) clock() time.Time {
+	if r.now != nil {
+		return r.now()
+	}
+	return time.Now()
+}
+
+// recoveryCode is one mailed code: its value, when it stops working, and how many
+// wrong codes were typed against it.
+type recoveryCode struct {
+	value    string
+	expires  time.Time
+	failures int
+}
+
+func newRecoveryCode(now time.Time) (*recoveryCode, error) {
+	n, err := rand.Int(rand.Reader, big.NewInt(1_000_000))
+	if err != nil {
+		return nil, fmt.Errorf("generate recovery code: %w", err)
+	}
+	return &recoveryCode{value: fmt.Sprintf("%06d", n.Int64()), expires: now.Add(recoveryCodeTTL)}, nil
+}
+
+type codeVerdict int
+
+const (
+	codeAccepted codeVerdict = iota
+	codeWrong
+	codeExpired
+	codeExhausted
+)
+
+// check compares a typed code in constant time. Each wrong code counts; the one
+// that reaches recoveryCodeAttempts exhausts the code, which then accepts nothing,
+// and neither does an expired one.
+func (c *recoveryCode) check(typed string, now time.Time) codeVerdict {
+	if c.failures >= recoveryCodeAttempts {
+		return codeExhausted
+	}
+	if !now.Before(c.expires) {
+		return codeExpired
+	}
+	if subtle.ConstantTimeCompare([]byte(strings.TrimSpace(typed)), []byte(c.value)) == 1 {
+		return codeAccepted
+	}
+	c.failures++
+	if c.failures >= recoveryCodeAttempts {
+		return codeExhausted
+	}
+	return codeWrong
+}
+
+func (c *recoveryCode) attemptsLeft() int { return recoveryCodeAttempts - c.failures }
+
+// recoveryStart is where naming an admin led: a code on its way to that admin, or
+// the reason the run has to fall back to the override.
+type recoveryStart struct {
+	admin  *api.StaffUser // the named staff account; nil when none matched
+	code   *recoveryCode  // set when the code went out
+	skip   string         // otpSkip* when it did not
+	detail string         // what failed, for the override screen and the audit row
+}
+
+// resolveAdmin loads the staff account (admin or owner) a typed username names, or
+// nil when there is none.
+func resolveAdmin(ctx context.Context, s ownerStore, username string) (*api.StaffUser, error) {
+	username = strings.TrimSpace(username)
+	if username == "" {
+		return nil, nil
+	}
+	u, err := s.UserByUsername(ctx, username)
+	if errors.Is(err, api.ErrNotFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	// Staff means admin or owner: the Owner is the primary break-glass identity.
+	if u.Role != "admin" && u.Role != "owner" {
+		return nil, nil
+	}
+	return u, nil
+}
+
+// beginRecovery resolves the named admin and mails it a recovery code. Only a
+// datastore or entropy fault is an error; every other way the code cannot go out is
+// a recoveryStart with skip set.
+func beginRecovery(ctx context.Context, s ownerStore, rc recoveryConfig, username, osUser string, op bgOperation) (recoveryStart, error) {
+	admin, err := resolveAdmin(ctx, s, username)
+	if err != nil {
+		return recoveryStart{}, err
+	}
+	if admin == nil {
+		return recoveryStart{skip: otpSkipUnknownAdmin}, nil
+	}
+	st := recoveryStart{admin: admin}
+	// An address nobody ever proved vouches for nobody.
+	email := strings.TrimSpace(admin.Email)
+	if email == "" || !admin.EmailVerified {
+		st.skip = otpSkipNoVerifiedEmail
+		return st, nil
+	}
+	if rc.open == nil {
+		st.skip, st.detail = otpSkipNoRelay, "this console has no mail relay"
+		return st, nil
+	}
+	relay, err := rc.open(ctx)
+	if err != nil {
+		st.skip, st.detail = otpSkipNoRelay, err.Error()
+		return st, nil
+	}
+	code, err := newRecoveryCode(rc.clock())
+	if err != nil {
+		return recoveryStart{}, err
+	}
+	sendCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	subject, body := recoveryMail(code.value, rc.host, osUser, admin.Username, op)
+	if err := relay.SendNotice(sendCtx, email, subject, body); err != nil {
+		st.skip, st.detail = otpSkipSendFailed, err.Error()
+		return st, nil
+	}
+	st.code = code
+	return st, nil
+}
+
+// recoveryMail words the code mail. It says where, by whom and for what the console
+// was opened, so an admin who did not ask for it learns that root on that machine is
+// in someone else's hands.
+func recoveryMail(code, host, osUser, admin string, op bgOperation) (subject, body string) {
+	what, whatZH := "reset the Owner account", "重置 Owner 账号"
+	if op == bgAddOperator {
+		what, whatZH = "add an Operator account", "添加 Operator 账号"
+	}
+	if host == "" {
+		host = "the Felis host"
+	}
+	minutes := int(recoveryCodeTTL / time.Minute)
+	subject = "Felis break-glass recovery code / 紧急恢复验证码"
+	body = fmt.Sprintf(`Someone with root on %[1]s (OS user %[2]s) opened felis breakGlass and named your staff account %[3]q to %[4]s.
+
+Recovery code: %[6]s
+It works for %[7]d minutes.
+
+If this was not you, root on that machine is in someone else's hands: change its credentials and read the audit log for break_glass entries.
+
+有人在 %[1]s 上以 root 身份（系统用户 %[2]s）打开了 felis breakGlass，指名你的管理员账号 %[3]q 来%[5]s。
+
+恢复验证码：%[6]s
+%[7]d 分钟内有效。
+
+如果不是你本人，这台机器的 root 已落入他人之手：请更换它的凭据，并查看审计日志中的 break_glass 记录。
+`, host, osUser, admin, what, whatZH, code, minutes)
+	return subject, body
+}
+
+// maskEmail keeps the first character of the local part and the domain, enough for
+// the operator to recognise the address without putting it on screen whole.
+func maskEmail(email string) string {
+	at := strings.LastIndex(email, "@")
+	if at <= 0 {
+		return "***"
+	}
+	return email[:1] + strings.Repeat("*", max(at-1, 3)) + email[at:]
+}
+
+// hostRecoveryMailer opens the [smtp] relay from the host the way the watchdog does:
+// the password is the env var password_ref names when that is set, else the host
+// copy at passwordPath, else the felis-smtp Secret, whose absence means a relay
+// without AUTH. The cluster is reached only when the host copy is missing, so a
+// break-glass on a host whose k3s is down still gets its code.
+func hostRecoveryMailer(c config.SMTPConfig, passwordPath, controlNS string) func(context.Context) (recoveryMailer, error) {
+	return func(ctx context.Context) (recoveryMailer, error) {
+		if strings.TrimSpace(c.Host) == "" {
+			return nil, errors.New("[smtp] is not configured in felis.toml")
+		}
+		if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" {
+			return smtpRelay(c, os.Getenv(ref)), nil
+		}
+		if password, ok, err := readHostCredential(passwordPath); err != nil {
+			return nil, fmt.Errorf("read the relay password: %w", err)
+		} else if ok {
+			return smtpRelay(c, password), nil
+		}
+		cl, err := buildSystemServerClient()
+		if err != nil {
+			return nil, fmt.Errorf("reach the cluster for the relay password: %w", err)
+		}
+		ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+		defer cancel()
+		password, err := smtpSecretPassword(ctx, cl, controlNS)
+		if err != nil {
+			return nil, fmt.Errorf("read the relay password from %s/%s: %w", controlNS, platform.SMTPSecretName, err)
+		}
+		return smtpRelay(c, password), nil
+	}
+}

cmd/felis/breakglass_otp_test.go

@@ -0,0 +1,498 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/platform"
+
+	tea "github.com/charmbracelet/bubbletea"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// These tests cover the recovery proof (#13): the mailed code's rules, where
+// naming an admin leads, what the mail says, how the console walks from a name to
+// a proven (or overridden) run, and what the audit row then records. No mail
+// leaves the process: the relay is a fake that keeps what it was handed.
+
+type sentMail struct{ to, subject, body string }
+
+type fakeRelay struct {
+	sent []sentMail
+	err  error
+}
+
+func (r *fakeRelay) SendNotice(_ context.Context, to, subject, body string) error {
+	if r.err != nil {
+		return r.err
+	}
+	r.sent = append(r.sent, sentMail{to, subject, body})
+	return nil
+}
+
+// sentCode pulls the code out of the one mail the relay carried.
+func (r *fakeRelay) sentCode(t *testing.T) string {
+	t.Helper()
+	if len(r.sent) != 1 {
+		t.Fatalf("relay carried %d mails, want 1", len(r.sent))
+	}
+	_, after, ok := strings.Cut(r.sent[0].body, "Recovery code: ")
+	if !ok || len(after) < 6 {
+		t.Fatalf("mail carries no recovery code:\n%s", r.sent[0].body)
+	}
+	return after[:6]
+}
+
+func relayConfig(r *fakeRelay, now func() time.Time) recoveryConfig {
+	return recoveryConfig{
+		open: func(context.Context) (recoveryMailer, error) { return r, nil },
+		host: "felis-host-1",
+		now:  now,
+	}
+}
+
+func verifiedAdmin(username, email string) *api.StaffUser {
+	return &api.StaffUser{ID: "usr-" + username, Username: username, Role: "owner", Email: email, EmailVerified: true}
+}
+
+func TestRecoveryCodeRules(t *testing.T) {
+	t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
+
+	t.Run("a fresh code is six digits and lives for the TTL", func(t *testing.T) {
+		seen := map[string]bool{}
+		for i := 0; i < 20; i++ {
+			c, err := newRecoveryCode(t0)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !isRecoveryCodeShape(c.value) {
+				t.Fatalf("code %q is not six digits", c.value)
+			}
+			if !c.expires.Equal(t0.Add(recoveryCodeTTL)) {
+				t.Fatalf("expires = %v, want %v", c.expires, t0.Add(recoveryCodeTTL))
+			}
+			seen[c.value] = true
+		}
+		if len(seen) < 2 {
+			t.Error("twenty codes were all the same")
+		}
+	})
+
+	t.Run("the right code is accepted, surrounding space ignored", func(t *testing.T) {
+		c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		if v := c.check(" 042917 ", t0); v != codeAccepted {
+			t.Errorf("check = %v, want accepted", v)
+		}
+	})
+
+	t.Run("wrong codes count down and the last one exhausts it", func(t *testing.T) {
+		c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		for i := 1; i < recoveryCodeAttempts; i++ {
+			if v := c.check("000000", t0); v != codeWrong {
+				t.Fatalf("wrong code %d: check = %v, want wrong", i, v)
+			}
+			if left := c.attemptsLeft(); left != recoveryCodeAttempts-i {
+				t.Fatalf("after %d wrong codes attemptsLeft = %d, want %d", i, left, recoveryCodeAttempts-i)
+			}
+		}
+		if v := c.check("000000", t0); v != codeExhausted {
+			t.Fatalf("wrong code %d: check = %v, want exhausted", recoveryCodeAttempts, v)
+		}
+		if v := c.check("042917", t0); v != codeExhausted {
+			t.Errorf("the right code after exhaustion: check = %v, want exhausted", v)
+		}
+	})
+
+	t.Run("an expired code accepts nothing", func(t *testing.T) {
+		c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		if v := c.check("042917", t0.Add(recoveryCodeTTL-time.Second)); v != codeAccepted {
+			t.Fatalf("a second before expiry: check = %v, want accepted", v)
+		}
+		c = &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		if v := c.check("042917", t0.Add(recoveryCodeTTL)); v != codeExpired {
+			t.Errorf("at expiry: check = %v, want expired", v)
+		}
+	})
+}
+
+func TestBeginRecovery(t *testing.T) {
+	ctx := context.Background()
+	t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
+	clock := func() time.Time { return t0 }
+
+	t.Run("mails a code to the named admin's verified address", func(t *testing.T) {
+		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
+		r := &fakeRelay{}
+		st, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgAddOperator)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if st.code == nil || st.skip != "" {
+			t.Fatalf("start = %+v, want a code and no skip", st)
+		}
+		if st.admin == nil || st.admin.Username != "root" {
+			t.Fatalf("start.admin = %+v, want root", st.admin)
+		}
+		if got := r.sentCode(t); got != st.code.value {
+			t.Errorf("mailed code %q, want the code the console checks (%q)", got, st.code.value)
+		}
+		m := r.sent[0]
+		if m.to != "[email protected]" {
+			t.Errorf("mail went to %q, want [email protected]", m.to)
+		}
+		for _, want := range []string{"felis-host-1", "OS user alice", `"root"`, "add an Operator account", "添加 Operator 账号", "10 minutes"} {
+			if !strings.Contains(m.body, want) {
+				t.Errorf("mail body lacks %q:\n%s", want, m.body)
+			}
+		}
+		if !st.code.expires.Equal(t0.Add(recoveryCodeTTL)) {
+			t.Errorf("code expires %v, want %v", st.code.expires, t0.Add(recoveryCodeTTL))
+		}
+	})
+
+	t.Run("the Owner reset is named as such", func(t *testing.T) {
+		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
+		r := &fakeRelay{}
+		if _, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgProvisionOwner); err != nil {
+			t.Fatal(err)
+		}
+		if body := r.sent[0].body; !strings.Contains(body, "reset the Owner account") || !strings.Contains(body, "重置 Owner 账号") {
+			t.Errorf("mail body does not name the Owner reset:\n%s", body)
+		}
+	})
+
+	// Each way the code cannot go out ends in a skip reason and no mail.
+	unverified := verifiedAdmin("root", "[email protected]")
+	unverified.EmailVerified = false
+	noEmail := verifiedAdmin("root", "")
+	cases := []struct {
+		name       string
+		user       *api.StaffUser
+		rc         func(r *fakeRelay) recoveryConfig
+		skip       string
+		detail     string
+		wantAdmin  bool
+		relayError error
+	}{
+		{name: "unknown admin", user: nil, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipUnknownAdmin},
+		{name: "unverified address", user: unverified, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
+		{name: "no address", user: noEmail, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
+		{name: "no relay wired", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig { return recoveryConfig{} }, skip: otpSkipNoRelay, detail: "this console has no mail relay", wantAdmin: true},
+		{name: "relay cannot open", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig {
+			return recoveryConfig{open: func(context.Context) (recoveryMailer, error) {
+				return nil, errors.New("[smtp] is not configured in felis.toml")
+			}}
+		}, skip: otpSkipNoRelay, detail: "[smtp] is not configured in felis.toml", wantAdmin: true},
+		{name: "send fails", user: verifiedAdmin("root", "[email protected]"), rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) },
+			skip: otpSkipSendFailed, detail: "554 relay refused", wantAdmin: true, relayError: errors.New("554 relay refused")},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := &fakeOwnerStore{users: map[string]*api.StaffUser{}}
+			if tc.user != nil {
+				f.users["root"] = tc.user
+			}
+			r := &fakeRelay{err: tc.relayError}
+			st, err := beginRecovery(ctx, f, tc.rc(r), "root", "alice", bgProvisionOwner)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if st.code != nil || st.skip != tc.skip || st.detail != tc.detail {
+				t.Errorf("start = {code:%v skip:%q detail:%q}, want no code, skip %q, detail %q", st.code, st.skip, st.detail, tc.skip, tc.detail)
+			}
+			if (st.admin != nil) != tc.wantAdmin {
+				t.Errorf("start.admin = %+v, want present=%v", st.admin, tc.wantAdmin)
+			}
+			if len(r.sent) != 0 {
+				t.Errorf("relay carried %d mails, want none", len(r.sent))
+			}
+		})
+	}
+
+	t.Run("a datastore fault is an error", func(t *testing.T) {
+		f := &fakeOwnerStore{userErr: errors.New("db down")}
+		if _, err := beginRecovery(ctx, f, relayConfig(&fakeRelay{}, clock), "root", "alice", bgProvisionOwner); err == nil {
+			t.Fatal("want the store fault")
+		}
+	})
+}
+
+func TestMaskEmail(t *testing.T) {
+	for in, want := range map[string]string{
+		"[email protected]": "a****@example.com",
+		"[email protected]":    "a***@example.com",
+		"@example.com":      "***",
+		"nonsense":          "***",
+	} {
+		if got := maskEmail(in); got != want {
+			t.Errorf("maskEmail(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestHostRecoveryMailer(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("no [smtp] host is no relay", func(t *testing.T) {
+		_, err := hostRecoveryMailer(config.SMTPConfig{}, hostSMTPPasswordPath, "felis")(ctx)
+		if err == nil || !strings.Contains(err.Error(), "[smtp]") {
+			t.Fatalf("err = %v, want it to name [smtp]", err)
+		}
+	})
+
+	t.Run("the password_ref env var supplies the password", func(t *testing.T) {
+		t.Setenv("FELIS_TEST_RELAY_PW", "from-env")
+		off := false
+		c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_RELAY_PW", RequireTLS: &off}
+		got, err := hostRecoveryMailer(c, hostSMTPPasswordPath, "felis")(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		relay, ok := got.(*mail.SMTP)
+		if !ok {
+			t.Fatalf("relay is %T, want *mail.SMTP", got)
+		}
+		if relay.Password != "from-env" || relay.Host != "mail.example.com" || relay.Port != 2525 || relay.From != "[email protected]" || relay.Username != "felis" || relay.RequireTLS {
+			t.Errorf("relay = %+v, want the [smtp] fields with the env password and TLS as configured", *relay)
+		}
+	})
+}
+
+func TestSMTPSecretPassword(t *testing.T) {
+	ctx := context.Background()
+	scheme := haltScheme(t)
+
+	t.Run("reads the password key", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
+			Data:       map[string][]byte{platform.SMTPSecretPasswordKey: []byte("s3cret")},
+		}).Build()
+		if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "s3cret" {
+			t.Errorf("smtpSecretPassword = (%q, %v), want (s3cret, nil)", pw, err)
+		}
+	})
+
+	t.Run("a missing Secret is a relay without AUTH", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
+		if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "" {
+			t.Errorf("smtpSecretPassword = (%q, %v), want (\"\", nil)", pw, err)
+		}
+	})
+
+	t.Run("any other read failure is an error", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{
+			Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
+				return apierrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, platform.SMTPSecretName, errors.New("rbac"))
+			},
+		}).Build()
+		if _, err := smtpSecretPassword(ctx, cl, "felis"); err == nil {
+			t.Fatal("want the read failure")
+		}
+	})
+}
+
+// recoveryModel is an Owner-reset console for admin "root" (verified address
+// [email protected]), run by OS user alice, with the fake relay behind it.
+func recoveryModel(t *testing.T, f *fakeOwnerStore, r *fakeRelay, now func() time.Time) *ownerModel {
+	t.Helper()
+	if f.users == nil {
+		f.users = map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}
+	}
+	return newOwnerModel(context.Background(), f, "alice", true).withRecovery(relayConfig(r, now))
+}
+
+// nameAdmin submits the admin-name form and feeds the result of the send back in.
+func nameAdmin(t *testing.T, m *ownerModel, name string) *ownerModel {
+	t.Helper()
+	m.authUser = name
+	_, cmd := m.onFormComplete()
+	if m.step != owWorking {
+		t.Fatalf("after naming the admin step = %v, want owWorking", m.step)
+	}
+	msg := findMsg[owAuthMsg](t, cmd)
+	next, _ := m.Update(msg)
+	return next.(*ownerModel)
+}
+
+// findMsg runs a (possibly batched) command and returns the first T it produces.
+func findMsg[T any](t *testing.T, cmd tea.Cmd) T {
+	t.Helper()
+	var zero T
+	if cmd == nil {
+		t.Fatalf("no command, want one producing %T", zero)
+	}
+	switch msg := cmd().(type) {
+	case T:
+		return msg
+	case tea.BatchMsg:
+		for _, c := range msg {
+			if c == nil {
+				continue
+			}
+			if got, ok := c().(T); ok {
+				return got
+			}
+		}
+	}
+	t.Fatalf("command produced no %T", zero)
+	return zero
+}
+
+// typeCode submits the code form with typed.
+func typeCode(t *testing.T, m *ownerModel, typed string) {
+	t.Helper()
+	if m.step != owCode {
+		t.Fatalf("step = %v, want owCode", m.step)
+	}
+	m.codeInput = typed
+	m.onFormComplete()
+}
+
+// provisionAudit finishes the run as an Owner reset and returns its audit payload.
+func provisionAudit(t *testing.T, m *ownerModel, f *fakeOwnerStore) (api.AuditEntry, map[string]any) {
+	t.Helper()
+	if m.step != owProvision {
+		t.Fatalf("step = %v, want owProvision", m.step)
+	}
+	m.username = "owner"
+	msg := m.provisionCmd()().(owProvisionMsg)
+	if msg.err != nil {
+		t.Fatalf("provision: %v", msg.err)
+	}
+	return auditOf(t, f)
+}
+
+func TestRecoveryConsoleFlow(t *testing.T) {
+	t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
+	fixed := func() time.Time { return t0 }
+
+	t.Run("the mailed code proves the admin and the audit says so", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		typeCode(t, m, r.sentCode(t))
+		if m.mode != "recovery" || m.accountable != "root" {
+			t.Fatalf("mode=%q accountable=%q, want recovery attributed to root", m.mode, m.accountable)
+		}
+		e, payload := provisionAudit(t, m, f)
+		if e.Actor != "root" || e.Action != "break_glass.recovery" {
+			t.Errorf("audit = %+v, want actor=root action=break_glass.recovery", e)
+		}
+		if payload["verified"] != true || payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" || payload["os_user"] != "alice" {
+			t.Errorf("payload = %v, want verified by email_otp to [email protected], os_user alice", payload)
+		}
+	})
+
+	t.Run("a wrong code asks again, and the last wrong one leads to the override", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		wrong := "000000"
+		if r.sentCode(t) == wrong {
+			wrong = "111111"
+		}
+		for i := 1; i < recoveryCodeAttempts; i++ {
+			typeCode(t, m, wrong)
+			if m.step != owCode || !strings.Contains(m.codeNote, "wrong") {
+				t.Fatalf("wrong code %d: step=%v note=%q, want the code form again with a note", i, m.step, m.codeNote)
+			}
+		}
+		typeCode(t, m, wrong)
+		if m.step != owOverride || m.skip != otpSkipCodeRejected {
+			t.Fatalf("after %d wrong codes step=%v skip=%q, want the override for code_rejected", recoveryCodeAttempts, m.step, m.skip)
+		}
+		m.onFormComplete() // OVERRIDE typed
+		e, payload := provisionAudit(t, m, f)
+		if e.Actor != "alice" || e.Action != "break_glass.root_override" {
+			t.Errorf("audit = %+v, want actor=alice action=break_glass.root_override", e)
+		}
+		if payload["verified"] != false || payload["otp_skipped"] != otpSkipCodeRejected || payload["admin_account"] != "root" {
+			t.Errorf("payload = %v, want unverified, otp_skipped=code_rejected, admin_account=root", payload)
+		}
+	})
+
+	t.Run("a late code leads to the override", func(t *testing.T) {
+		now := t0
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, func() time.Time { return now }), "root")
+		now = t0.Add(recoveryCodeTTL)
+		typeCode(t, m, r.sentCode(t))
+		if m.step != owOverride || m.skip != otpSkipCodeExpired {
+			t.Fatalf("step=%v skip=%q, want the override for code_expired", m.step, m.skip)
+		}
+	})
+
+	t.Run("OVERRIDE at the code prompt goes on unverified, saying the operator skipped", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		typeCode(t, m, breakGlassOverrideToken)
+		if m.mode != "root_override" || m.accountable != "alice" {
+			t.Fatalf("mode=%q accountable=%q, want root_override as alice", m.mode, m.accountable)
+		}
+		_, payload := provisionAudit(t, m, f)
+		if payload["verified"] != false || payload["otp_skipped"] != otpSkipByOperator {
+			t.Errorf("payload = %v, want unverified, otp_skipped=operator_skipped", payload)
+		}
+	})
+
+	t.Run("a relay failure leads to the override naming it", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{err: errors.New("dial tcp 10.0.0.9:587: connect: connection refused")}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		if m.step != owOverride || m.skip != otpSkipSendFailed {
+			t.Fatalf("step=%v skip=%q, want the override for send_failed", m.step, m.skip)
+		}
+		if reason := m.overrideReason(); !strings.Contains(reason, "connection refused") {
+			t.Errorf("override reason %q does not name the failure", reason)
+		}
+		m.onFormComplete()
+		_, payload := provisionAudit(t, m, f)
+		if payload["otp_skipped"] != otpSkipSendFailed || !strings.Contains(payload["otp_skip_detail"].(string), "connection refused") {
+			t.Errorf("payload = %v, want otp_skipped=send_failed with the relay's error", payload)
+		}
+	})
+
+	t.Run("esc at the code prompt starts over and forgets the code", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		code := r.sentCode(t)
+		next, _ := m.Update(key(tea.KeyEsc))
+		m = next.(*ownerModel)
+		if m.step != owAuth || m.code != nil || m.admin != nil {
+			t.Fatalf("after esc step=%v code=%v admin=%v, want owAuth with the attempt forgotten", m.step, m.code, m.admin)
+		}
+		// The old code cannot be replayed: the next name mails a new one.
+		m = nameAdmin(t, m, "root")
+		if len(r.sent) != 2 {
+			t.Fatalf("relay carried %d mails, want a second one for the new attempt", len(r.sent))
+		}
+		_, fresh, _ := strings.Cut(r.sent[1].body, "Recovery code: ")
+		if m.code.value != fresh[:6] {
+			t.Errorf("the console checks %q, want the newly mailed %q (old one was %q)", m.code.value, fresh[:6], code)
+		}
+	})
+}
+
+func TestRootHandsRecoveryToAccountOperations(t *testing.T) {
+	for _, op := range []bgOperation{bgProvisionOwner, bgAddOperator} {
+		m := newTestRoot(true, consoleModeBreakGlass, "")
+		m.recovery = recoveryConfig{host: "felis-host-1"}
+		m = drive(t, m, menuChoiceMsg{op: op})
+		om, ok := m.screen.(*ownerModel)
+		if !ok {
+			t.Fatalf("op %v: screen = %T, want *ownerModel", op, m.screen)
+		}
+		if om.recovery.host != "felis-host-1" {
+			t.Errorf("op %v: the account screen has no relay config; its codes could never go out", op)
+		}
+	}
+}

cmd/felis/breakglass_test.go

@@ -24,16 +24,14 @@ type fakeOwnerStore struct {
 	settings  map[string][]byte
 	audits    []api.AuditEntry
 	tokens    []setupTokenCall
-	redeems   []redeemCall
+	setupIDs  []string
 	users     map[string]*api.StaffUser // keyed by username
 	admins    bool                      // AdminExists answer
 	ownerSeat string                    // OwnerUsername answer: the occupied seat, "" when none
 
-	// CompleteOwnerSetup's success result. redeemUserID defaults to the fresh id
-	// the caller passes (the unlinked-UUID case) when left empty.
-	redeemUserID     string
-	redeemMCUUID     string
-	redeemAuthSource string
+	setupUserID   string
+	setupUsername string
+	onboarded     bool
 
 	upsertErr      error
 	insertErr      error
@@ -59,12 +57,6 @@ type setupTokenCall struct {
 	expiresAt time.Time
 }
 
-// redeemCall records the inputs CompleteOwnerSetup was called with.
-type redeemCall struct {
-	newUserID string
-	code      string
-}
-
 func (f *fakeOwnerStore) AdminExists(_ context.Context) (bool, error) {
 	if f.adminErr != nil {
 		return false, f.adminErr
@@ -118,30 +110,31 @@ func (f *fakeOwnerStore) InsertOperator(_ context.Context, id, username, email s
 	return nil
 }
 
-// CompleteOwnerSetup models the real all-or-nothing transaction: injected failures
-// record none of the redeem, auth-toggle, or setup-token writes.
-func (f *fakeOwnerStore) CompleteOwnerSetup(_ context.Context, newUserID, code string, _ time.Time,
-	tokenHash string, expiresAt time.Time) (string, string, string, error) {
-	if f.redeemErr != nil {
-		return "", "", "", f.redeemErr
+// CompleteOwnerSetup models the transaction without any game link code.
+func (f *fakeOwnerStore) CompleteOwnerSetup(_ context.Context, newUserID string, _ time.Time,
+	tokenHash string, expiresAt time.Time) (string, string, error) {
+	for _, err := range []error{f.redeemErr, f.setErr, f.createTokenErr} {
+		if err != nil {
+			return "", "", err
+		}
 	}
-	if f.setErr != nil {
-		return "", "", "", f.setErr
-	}
-	if f.createTokenErr != nil {
-		return "", "", "", f.createTokenErr
-	}
-	f.redeems = append(f.redeems, redeemCall{newUserID, code})
-	userID := f.redeemUserID
+	userID, username := f.setupUserID, f.setupUsername
 	if userID == "" {
-		userID = newUserID // unlinked UUID → the fresh id becomes the Owner
+		userID = newUserID
 	}
+	if username == "" {
+		username = "owner"
+	}
+	if f.onboarded {
+		return userID, username, api.ErrConflict
+	}
+	f.setupIDs = append(f.setupIDs, newUserID)
 	if f.settings == nil {
 		f.settings = map[string][]byte{}
 	}
 	f.settings[api.LocalAuthEnabledKey] = []byte("true")
 	f.tokens = append(f.tokens, setupTokenCall{tokenHash, userID, expiresAt})
-	return userID, f.redeemMCUUID, f.redeemAuthSource, nil
+	return userID, username, nil
 }
 
 func (f *fakeOwnerStore) SetSetting(_ context.Context, key string, value []byte) error {
@@ -269,71 +262,57 @@ func TestEnableLocalAuth(t *testing.T) {
 	}
 }
 
-func TestAuthenticateAdmin(t *testing.T) {
+func TestResolveAdmin(t *testing.T) {
 	ctx := context.Background()
 
-	// Password verification is gone (passwordless design): authenticateAdmin now only
-	// resolves the named admin so recovery can attribute the audit to a real identity.
-	// The security boundary is the break-glass root gate, not a typed secret.
+	// resolveAdmin only finds the staff account a typed name points at; proving the
+	// operator holds it is the mailed code's job (beginRecovery).
 
-	t.Run("resolves an existing admin for attribution", func(t *testing.T) {
+	t.Run("resolves an existing admin", func(t *testing.T) {
 		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": mkAdmin("root")}}
-		matched, ok, err := authenticateAdmin(ctx, f, "root")
+		u, err := resolveAdmin(ctx, f, "  root ")
 		if err != nil {
-			t.Fatalf("authenticateAdmin: %v", err)
+			t.Fatalf("resolveAdmin: %v", err)
 		}
-		if !ok {
-			t.Fatal("ok = false, want true for an existing admin")
-		}
-		if matched != "root" {
-			t.Errorf("matched = %q, want root", matched)
+		if u == nil || u.Username != "root" {
+			t.Fatalf("resolveAdmin = %+v, want the root admin", u)
 		}
 	})
 
-	t.Run("a non-admin role can never attribute a break-glass", func(t *testing.T) {
+	t.Run("a non-admin role is no staff account", func(t *testing.T) {
 		player := mkAdmin("alice")
 		player.Role = "user" // a player row is not staff
 		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"alice": player}}
-		_, ok, err := authenticateAdmin(ctx, f, "alice")
-		if err != nil {
-			t.Fatalf("unexpected error: %v", err)
-		}
-		if ok {
-			t.Error("ok = true, want false for a non-admin role")
+		if u, err := resolveAdmin(ctx, f, "alice"); u != nil || err != nil {
+			t.Errorf("resolveAdmin(player) = (%+v, %v), want (nil, nil)", u, err)
 		}
 	})
 
-	t.Run("the owner role attributes like an admin", func(t *testing.T) {
+	t.Run("the owner role counts as staff", func(t *testing.T) {
 		owner := mkAdmin("root")
 		owner.Role = "owner" // the platform owner is staff too (migration 0011)
 		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": owner}}
-		matched, ok, err := authenticateAdmin(ctx, f, "root")
-		if err != nil || !ok || matched != "root" {
-			t.Fatalf("authenticateAdmin(owner) = (%q, %v, %v), want (root, true, nil)", matched, ok, err)
+		if u, err := resolveAdmin(ctx, f, "root"); err != nil || u == nil {
+			t.Fatalf("resolveAdmin(owner) = (%+v, %v), want the owner", u, err)
 		}
 	})
 
-	t.Run("an unknown user is a non-match, not an error", func(t *testing.T) {
-		f := &fakeOwnerStore{}
-		_, ok, err := authenticateAdmin(ctx, f, "nobody")
-		if err != nil {
-			t.Fatalf("unexpected error: %v", err)
-		}
-		if ok {
-			t.Error("ok = true, want false for an unknown user")
+	t.Run("an unknown user is nil, not an error", func(t *testing.T) {
+		if u, err := resolveAdmin(ctx, &fakeOwnerStore{}, "nobody"); u != nil || err != nil {
+			t.Errorf("resolveAdmin(unknown) = (%+v, %v), want (nil, nil)", u, err)
 		}
 	})
 
-	t.Run("an empty username is a non-match with no store call", func(t *testing.T) {
+	t.Run("an empty username makes no store call", func(t *testing.T) {
 		f := &fakeOwnerStore{userErr: errors.New("must not be called")}
-		if _, ok, err := authenticateAdmin(ctx, f, ""); ok || err != nil {
-			t.Errorf("empty username: ok=%v err=%v, want false,nil", ok, err)
+		if u, err := resolveAdmin(ctx, f, " "); u != nil || err != nil {
+			t.Errorf("empty username: (%+v, %v), want (nil, nil)", u, err)
 		}
 	})
 
 	t.Run("a datastore fault is surfaced", func(t *testing.T) {
 		f := &fakeOwnerStore{userErr: errors.New("db down")}
-		if _, _, err := authenticateAdmin(ctx, f, "root"); err == nil {
+		if _, err := resolveAdmin(ctx, f, "root"); err == nil {
 			t.Fatal("want error when the store fails")
 		}
 	})
@@ -401,6 +380,8 @@ func TestPerformBreakGlass(t *testing.T) {
 			osUser:         "alice",
 			ownerUsername:  "owner",
 			attemptedAdmin: "root",
+			verifiedBy:     verifiedByEmailOTP,
+			codeSentTo:     "[email protected]",
 		}
 		out, err := performBreakGlass(ctx, f, op)
 		if err != nil {
@@ -425,6 +406,23 @@ func TestPerformBreakGlass(t *testing.T) {
 		if payload["admin_account"] != "root" {
 			t.Errorf("payload.admin_account = %v, want root", payload["admin_account"])
 		}
+		if payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" {
+			t.Errorf("payload = %v, want verified_by=email_otp [email protected]", payload)
+		}
+		if _, present := payload["otp_skipped"]; present {
+			t.Error("a proven recovery carries no otp_skipped")
+		}
+	})
+
+	t.Run("recovery without a proof is recorded unverified", func(t *testing.T) {
+		f := &fakeOwnerStore{}
+		op := breakGlassOp{mode: "recovery", accountable: "root", osUser: "alice", ownerUsername: "owner", attemptedAdmin: "root"}
+		if _, err := performBreakGlass(ctx, f, op); err != nil {
+			t.Fatalf("performBreakGlass: %v", err)
+		}
+		if _, payload := auditOf(t, f); payload["verified"] != false {
+			t.Errorf("payload.verified = %v, want false: only a mailed code verifies", payload["verified"])
+		}
 	})
 
 	t.Run("root override records an unverified row attributed to the OS user", func(t *testing.T) {
@@ -435,6 +433,8 @@ func TestPerformBreakGlass(t *testing.T) {
 			osUser:         "alice",
 			ownerUsername:  "owner",
 			attemptedAdmin: "typo-admin",
+			otpSkipped:     otpSkipSendFailed,
+			otpSkipDetail:  "dial tcp: connection refused",
 		}
 		if _, err := performBreakGlass(ctx, f, op); err != nil {
 			t.Fatalf("performBreakGlass: %v", err)
@@ -450,6 +450,13 @@ func TestPerformBreakGlass(t *testing.T) {
 		if payload["admin_account"] != "typo-admin" {
 			t.Errorf("payload.admin_account = %v, want typo-admin", payload["admin_account"])
 		}
+		// Why no code proved anyone is part of the record.
+		if payload["otp_skipped"] != otpSkipSendFailed || payload["otp_skip_detail"] != "dial tcp: connection refused" {
+			t.Errorf("payload = %v, want otp_skipped=send_failed with its detail", payload)
+		}
+		if _, present := payload["verified_by"]; present {
+			t.Error("an override carries no verified_by")
+		}
 	})
 
 	t.Run("an audit failure does not fail the recovery", func(t *testing.T) {
@@ -616,6 +623,8 @@ func TestPerformAddOperator(t *testing.T) {
 			ownerUsername:  "ops-jordan",
 			ownerEmail:     "[email protected]",
 			attemptedAdmin: "root",
+			verifiedBy:     verifiedByEmailOTP,
+			codeSentTo:     "[email protected]",
 		}
 		out, err := performAddOperator(ctx, f, op)
 		if err != nil {
@@ -642,14 +651,14 @@ func TestPerformAddOperator(t *testing.T) {
 		if _, present := payload["owner"]; present {
 			t.Error("payload.owner present, want the new account under the operator key")
 		}
-		if payload["verified"] != true || payload["admin_account"] != "root" {
-			t.Errorf("payload = %v, want verified=true admin_account=root", payload)
+		if payload["verified"] != true || payload["admin_account"] != "root" || payload["verified_by"] != verifiedByEmailOTP {
+			t.Errorf("payload = %v, want verified=true admin_account=root verified_by=email_otp", payload)
 		}
 	})
 
 	t.Run("root override records an unverified operator row", func(t *testing.T) {
 		f := &fakeOwnerStore{}
-		op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin"}
+		op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin", otpSkipped: otpSkipUnknownAdmin}
 		if _, err := performAddOperator(ctx, f, op); err != nil {
 			t.Fatalf("performAddOperator: %v", err)
 		}
@@ -657,8 +666,8 @@ func TestPerformAddOperator(t *testing.T) {
 			t.Fatalf("want 1 insert, got %d", len(f.inserts))
 		}
 		_, payload := auditOf(t, f)
-		if payload["verified"] != false {
-			t.Errorf("payload.verified = %v, want false for root_override", payload["verified"])
+		if payload["verified"] != false || payload["otp_skipped"] != otpSkipUnknownAdmin {
+			t.Errorf("payload = %v, want verified=false otp_skipped=unknown_admin", payload)
 		}
 	})
 
@@ -694,149 +703,70 @@ func TestPerformAddOperator(t *testing.T) {
 	})
 }
 
-func TestPerformSetupMCBind(t *testing.T) {
+func TestPerformSetupOwner(t *testing.T) {
 	ctx := context.Background()
+	f := &fakeOwnerStore{setupUserID: "usr-owner-1"}
+	out, err := performSetupOwner(ctx, f, "https://op.console.example.com:30443", "deploybot")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if out.ownerUsername != "owner" {
+		t.Fatalf("username = %q", out.ownerUsername)
+	}
+	const prefix = "https://op.console.example.com:30443/setup?token="
+	if !strings.HasPrefix(out.setupTokenURL, prefix) {
+		t.Fatalf("URL = %q", out.setupTokenURL)
+	}
+	if len(f.tokens) != 1 || f.tokens[0].userID != "usr-owner-1" {
+		t.Fatalf("tokens = %+v", f.tokens)
+	}
+	raw := strings.TrimPrefix(out.setupTokenURL, prefix)
+	sum := sha256.Sum256([]byte(raw))
+	if f.tokens[0].tokenHash != hex.EncodeToString(sum[:]) {
+		t.Fatal("stored token does not match URL")
+	}
+	if !f.tokens[0].expiresAt.After(time.Now()) {
+		t.Fatal("token already expired")
+	}
+	if string(f.settings[api.LocalAuthEnabledKey]) != "true" {
+		t.Fatal("local auth stayed disabled")
+	}
+	e, payload := auditOf(t, f)
+	if e.Actor != "deploybot" || e.Action != "setup.owner_login" || payload["user_id"] != "usr-owner-1" {
+		t.Fatalf("audit = %+v, %v", e, payload)
+	}
 
-	t.Run("binds the owner and mints a setup URL whose token hash is what is stored", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1", redeemMCUUID: "mc-uuid-1", redeemAuthSource: "mojang"}
-		out, err := performSetupMCBind(ctx, f, "  abc-123  ", "console.example.com", "deploybot")
-		if err != nil {
-			t.Fatalf("performSetupMCBind: %v", err)
-		}
-		// The Owner this mints has no password and no email, so the setup token is the
-		// only door — and handleSetupRedeem is gated on local_auth_enabled. A bind that
-		// leaves the toggle off hands back a URL that answers 403.
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; !ok {
-			t.Error("local auth was not enabled — the setup URL would 403 local_auth_disabled")
-		}
-		// The bind is attributed by Minecraft identity, because that is what the login
-		// gate verified; a username would be the one thing nobody checked.
-		e, payload := auditOf(t, f)
-		if e.Actor != "deploybot" || e.Source != "setup" || e.Action != "setup.owner_bind" {
-			t.Errorf("audit = %+v, want actor=deploybot source=setup action=setup.owner_bind", e)
-		}
-		if payload["mc_uuid"] != "mc-uuid-1" || payload["auth_source"] != "mojang" {
-			t.Errorf("audit payload = %v, want the redeemed mc_uuid + auth_source", payload)
-		}
-		if out.ownerIdentity != "mc-uuid-1" {
-			t.Errorf("owner identity = %q, want the verified Minecraft UUID", out.ownerIdentity)
-		}
-		const prefix = "https://console.example.com/setup?token="
-		if !strings.HasPrefix(out.setupTokenURL, prefix) {
-			t.Fatalf("setup URL = %q, want prefix %q", out.setupTokenURL, prefix)
-		}
-		// The link code is trimmed and upper-cased before redemption.
-		if len(f.redeems) != 1 {
-			t.Fatalf("want 1 redeem, got %d", len(f.redeems))
-		}
-		if f.redeems[0].code != "ABC-123" {
-			t.Errorf("redeemed code = %q, want ABC-123 (trimmed + upper-cased)", f.redeems[0].code)
-		}
-		if !strings.HasPrefix(f.redeems[0].newUserID, "usr-") {
-			t.Errorf("redeem newUserID = %q, want usr- prefix", f.redeems[0].newUserID)
-		}
-		// Exactly one token minted, for the redeemed user, and only its hash stored —
-		// the stored hash must be sha-256 of the raw token carried in the URL.
-		if len(f.tokens) != 1 {
-			t.Fatalf("want 1 setup token, got %d", len(f.tokens))
-		}
-		tok := f.tokens[0]
-		if tok.userID != "usr-owner-1" {
-			t.Errorf("token userID = %q, want usr-owner-1 (the redeemed owner)", tok.userID)
-		}
-		raw := strings.TrimPrefix(out.setupTokenURL, prefix)
-		sum := sha256.Sum256([]byte(raw))
-		if tok.tokenHash != hex.EncodeToString(sum[:]) {
-			t.Error("stored token hash is not sha-256 of the raw token in the URL")
-		}
-		if tok.tokenHash == raw || tok.tokenHash == "" {
-			t.Error("the raw token (or nothing) was stored instead of its hash")
-		}
-		// The token is short-lived and in the future.
-		if !tok.expiresAt.After(time.Now()) {
-			t.Errorf("token expiresAt = %v, want a future time", tok.expiresAt)
+	t.Run("failed writes leave no partially initialized login", func(t *testing.T) {
+		for _, store := range []*fakeOwnerStore{
+			{redeemErr: errors.New("database unavailable")},
+			{setErr: errors.New("settings write failed")},
+			{createTokenErr: errors.New("token write failed")},
+		} {
+			if _, err := performSetupOwner(ctx, store, "https://op.console.example.com", "root"); err == nil {
+				t.Fatal("want failure")
+			}
+			if len(store.tokens) != 0 || len(store.setupIDs) != 0 || len(store.settings) != 0 {
+				t.Fatal("partial setup")
+			}
 		}
 	})
-
-	t.Run("an empty link code mints nothing", func(t *testing.T) {
-		f := &fakeOwnerStore{}
-		if _, err := performSetupMCBind(ctx, f, "   ", "console.example.com", "root"); err == nil {
-			t.Fatal("want error for an empty link code")
-		}
-		if len(f.redeems) != 0 || len(f.tokens) != 0 {
-			t.Errorf("want no redeem/token on an empty code, got redeems=%d tokens=%d", len(f.redeems), len(f.tokens))
-		}
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; ok {
-			t.Error("local auth was enabled without an owner — the gate must not open on a failed bind")
+	t.Run("settled account is not reset", func(t *testing.T) {
+		store := &fakeOwnerStore{setupUserID: "existing", setupUsername: "alice", onboarded: true}
+		out, err := performSetupOwner(ctx, store, "https://op.console.example.com", "root")
+		if !errors.Is(err, api.ErrConflict) || out.ownerUsername != "alice" || len(store.tokens) != 0 {
+			t.Fatalf("out = %+v err = %v", out, err)
 		}
 	})
-
-	t.Run("a link-code redemption failure mints no token", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemErr: errors.New("code expired")}
-		if _, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root"); err == nil {
-			t.Fatal("want error when the link code cannot be redeemed")
-		}
-		if len(f.tokens) != 0 {
-			t.Errorf("want no token minted on a redeem failure, got %d", len(f.tokens))
-		}
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; ok {
-			t.Error("local auth was enabled without an owner — the gate must not open on a failed redeem")
+	t.Run("audit failure still returns the login link", func(t *testing.T) {
+		out, err := performSetupOwner(ctx, &fakeOwnerStore{auditErr: errors.New("audit down")}, "https://op.console.example.com", "root")
+		if err != nil || out.auditErr == nil || out.setupTokenURL == "" {
+			t.Fatalf("out = %+v err = %v", out, err)
 		}
 	})
-
-	t.Run("a local-auth failure fails the bind rather than minting an unredeemable URL", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1", setErr: errors.New("db down")}
-		if _, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root"); err == nil {
-			t.Fatal("want error when local auth cannot be enabled")
-		}
-		if len(f.redeems) != 0 || len(f.tokens) != 0 {
-			t.Errorf("atomic setup was partially recorded: redeems=%d tokens=%d", len(f.redeems), len(f.tokens))
-		}
-	})
-
-	t.Run("a token-store failure rolls the bind back", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1", createTokenErr: errors.New("db down")}
-		if _, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root"); err == nil {
-			t.Fatal("want error when the setup token cannot be stored")
-		}
-		if len(f.redeems) != 0 {
-			t.Errorf("link code was consumed despite token failure, got %d redeems", len(f.redeems))
-		}
-		if len(f.tokens) != 0 {
-			t.Errorf("want no recorded token when the store fails, got %d", len(f.tokens))
-		}
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; ok {
-			t.Error("local auth stayed enabled despite transaction rollback")
-		}
-	})
-
-	t.Run("an audit failure does not cost the operator their install", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1", auditErr: errors.New("audit sink down")}
-		out, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root")
-		if err != nil {
-			t.Fatalf("an audit failure must not fail the bind: %v", err)
-		}
-		if out.auditErr == nil {
-			t.Error("the audit failure was swallowed instead of surfaced on the outcome")
-		}
-		if out.setupTokenURL == "" {
-			t.Error("no setup URL minted despite a recoverable audit failure")
-		}
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; !ok {
-			t.Error("local auth was not enabled despite a recoverable audit failure")
-		}
-	})
-
-	t.Run("defaults to the op.console host when adminHostname is empty", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1"}
-		out, err := performSetupMCBind(ctx, f, "abc-123", "  ", "root")
-		if err != nil {
-			t.Fatalf("performSetupMCBind: %v", err)
-		}
-		// The Owner is staff, so onboarding lands on the operator console, not the
-		// player panel — the empty-host fallback must reflect that.
-		if !strings.HasPrefix(out.setupTokenURL, "https://op.console.localhost/setup?token=") {
-			t.Errorf("setup URL = %q, want the op.console.localhost default host", out.setupTokenURL)
+	t.Run("missing HTTPS origin cannot create an account", func(t *testing.T) {
+		store := &fakeOwnerStore{}
+		if _, err := performSetupOwner(ctx, store, "", "root"); err == nil || len(store.tokens) != 0 {
+			t.Fatal("invalid origin accepted")
 		}
 	})
 }

cmd/felis/conflict_test.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// racingClient lands one concurrent write (race) on the stored object just before
+// setup's first write of it goes out, the way the operator's status update or
+// felis-api's idle patch can, and counts setup's writes.
+func racingClient(t *testing.T, race func(ctx context.Context, c client.WithWatch), objs ...client.Object) (client.Client, *int) {
+	t.Helper()
+	writes := 0
+	before := func(ctx context.Context, c client.WithWatch) {
+		writes++
+		if writes == 1 {
+			race(ctx, c)
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
+		WithStatusSubresource(&v1alpha1.MinecraftServer{}).
+		WithInterceptorFuncs(interceptor.Funcs{
+			Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
+				before(ctx, c)
+				return c.Update(ctx, obj, opts...)
+			},
+			Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+				before(ctx, c)
+				return c.Patch(ctx, obj, patch, opts...)
+			},
+		}).Build()
+	return cl, &writes
+}
+
+func staleLoginGate(t *testing.T) *v1alpha1.MinecraftServer {
+	t.Helper()
+	ms, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://old.internal:8081", "203.0.113.10.nip.io", "console.203.0.113.10.nip.io")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ms
+}
+
+func getLogin(t *testing.T, cl client.Client) *v1alpha1.MinecraftServer {
+	t.Helper()
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	return &ms
+}
+
+func envMap(ms *v1alpha1.MinecraftServer) map[string]string {
+	m := map[string]string{}
+	for _, e := range ms.Spec.Env {
+		m[e.Name] = e.Value
+	}
+	return m
+}
+
+// A hand edit that lands while setup refreshes the console hostnames costs setup a
+// re-read and a second write; both the edit and the refresh survive.
+func TestRefreshDerivedEnvRetriesAConcurrentEdit(t *testing.T) {
+	ctx := context.Background()
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		ms := getLogin(t, c)
+		ms.Spec.Env = append(ms.Spec.Env, v1alpha1.EnvVar{Name: "HAND_TUNED", Value: "1"})
+		if err := c.Update(ctx, ms); err != nil {
+			t.Fatal(err)
+		}
+	}, staleLoginGate(t))
+
+	desired, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatal(err)
+	}
+	refreshed, err := refreshDerivedEnv(ctx, cl, getLogin(t, cl), desired)
+	if err != nil || !refreshed {
+		t.Fatalf("refreshed=%v err=%v, want a refresh after the retry", refreshed, err)
+	}
+	env := envMap(getLogin(t, cl))
+	if env[envPanelHostname] != "console.mc.example.net" || env[envRootDomain] != "mc.example.net" {
+		t.Errorf("env = %v, want the new hostnames", env)
+	}
+	if env["HAND_TUNED"] != "1" {
+		t.Errorf("env = %v: the concurrent hand edit was dropped", env)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2 (one conflict, one retry)", *writes)
+	}
+}
+
+// converge reports each fill once even when a status write forced a retry.
+func TestConvergeRetriesAConcurrentStatusWrite(t *testing.T) {
+	ctx := context.Background()
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatal(err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Status.Phase = v1alpha1.PhaseRunning
+		if err := c.Status().Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, lobby)
+
+	var got systemServerOutcome
+	for _, o := range convergeSystemServers(ctx, cl, "minecraft", "", "reg/lobby:1",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
+		if o.name == naming.SystemLobbyServer {
+			got = o
+		}
+	}
+	if got.err != nil || !got.updated || !slices.Equal(got.changes, []string{"spec.rcon"}) {
+		t.Fatalf("lobby outcome = %+v, want spec.rcon filled once", got)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if !ms.Spec.Rcon.Enabled || ms.Status.Phase != v1alpha1.PhaseRunning {
+		t.Errorf("rcon.enabled=%v phase=%q, want the fill and the status write both kept", ms.Spec.Rcon.Enabled, ms.Status.Phase)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+// A replica refresh racing another writer keeps that writer's key.
+func TestSecretReplicaRefreshRetriesAConcurrentWrite(t *testing.T) {
+	secret := func(ns, body string) *corev1.Secret {
+		return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
+			Data: map[string][]byte{"felis.toml": []byte(body)}}
+	}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var s corev1.Secret
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+			t.Fatal(err)
+		}
+		s.Data["extra"] = []byte("x")
+		if err := c.Update(ctx, &s); err != nil {
+			t.Fatal(err)
+		}
+	}, secret("felis", "current"), secret("minecraft", "stale"))
+
+	out := ensureSecretReplica(context.Background(), cl, "felis", "minecraft",
+		"felis-config", "felis.toml", "config", "minecraft ns", true)
+	if out.err != nil || !out.updated {
+		t.Fatalf("outcome = %+v, want refreshed", out)
+	}
+	var s corev1.Secret
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+		t.Fatal(err)
+	}
+	if string(s.Data["felis.toml"]) != "current" || string(s.Data["extra"]) != "x" {
+		t.Errorf("replica data = %q, want felis.toml=current and extra=x", s.Data)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+const (
+	sysOldDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
+	sysNewDigest = "sha256:4444444444444444444444444444444444444444444444444444444444444444"
+)
+
+func systemPinRegistry(t *testing.T) imagepin.Resolver {
+	t.Helper()
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v2/felis/limbo/manifests/demo", "/v2/felis/lobby/manifests/demo":
+			w.Header().Set("Docker-Content-Digest", sysNewDigest)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(reg.Close)
+	return imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+}
+
+func systemServer(name, image string) *v1alpha1.MinecraftServer {
+	ms := &v1alpha1.MinecraftServer{}
+	ms.Name, ms.Namespace = name, "minecraft"
+	ms.Labels = map[string]string{v1alpha1.LabelSystemRole: name}
+	ms.Spec.Image = image
+	return ms
+}
+
+// The installer's --system pin moves a system server onto the build its tag names
+// now, from the bare tag or from an earlier digest, and is a no-op the second time.
+func TestPinSystemServerImage(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	limbo := defaultRegistryURL + "/felis/limbo:demo"
+	lobby := defaultRegistryURL + "/felis/lobby:demo"
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		systemServer(naming.SystemLoginServer, limbo),
+		systemServer(naming.SystemLobbyServer, lobby+"@"+sysOldDigest),
+	).Build()
+	image := func(name string) string {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		return ms.Spec.Image
+	}
+
+	for name, want := range map[string]string{
+		naming.SystemLoginServer: limbo + "@" + sysNewDigest,
+		naming.SystemLobbyServer: lobby + "@" + sysNewDigest,
+	} {
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || o.err != nil || !o.updated {
+			t.Fatalf("%s: outcome=%+v err=%v, want pinned", name, o, err)
+		}
+		if got := image(name); got != want {
+			t.Errorf("%s image = %q, want %q", name, got, want)
+		}
+		again, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || again.updated || again.skipped != "already runs "+want {
+			t.Errorf("%s second pass = %+v, %v; want already runs %s", name, again, err, want)
+		}
+	}
+}
+
+func TestPinSystemServerImageLeavesOthersAlone(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	pin := func(t *testing.T, ms *v1alpha1.MinecraftServer) (systemServerOutcome, string) {
+		t.Helper()
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(ms).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(ms), &got); err != nil {
+			t.Fatal(err)
+		}
+		return o, got.Spec.Image
+	}
+
+	t.Run("external image", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, "docker.io/example/limbo:1.2"))
+		if o.err != nil || o.updated || img != "docker.io/example/limbo:1.2" ||
+			o.skipped != "runs docker.io/example/limbo:1.2, which names no platform registry tag to follow; left alone" {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("digest without a tag", func(t *testing.T) {
+		ref := defaultRegistryURL + "/felis/limbo@" + sysOldDigest
+		o, img := pin(t, systemServer(naming.SystemLoginServer, ref))
+		if o.err != nil || o.updated || img != ref {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("not a system server", func(t *testing.T) {
+		ms := systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo")
+		ms.Labels = nil
+		o, img := pin(t, ms)
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:demo" {
+			t.Fatalf("outcome=%+v image=%q, want refused", o, img)
+		}
+	})
+	t.Run("tag the registry lost", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:gone"))
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:gone" {
+			t.Fatalf("outcome=%+v image=%q, want an error the installer falls back on", o, img)
+		}
+	})
+	t.Run("absent", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.skipped != "not present yet; sudo felis setup creates it" {
+			t.Fatalf("outcome=%+v err=%v, want a skip", o, err)
+		}
+	})
+	t.Run("retargeted while pinning", func(t *testing.T) {
+		cl, _ := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+			ms := getLogin(t, c)
+			ms.Spec.Image = "docker.io/example/limbo:1.2"
+			if err := c.Update(ctx, ms); err != nil {
+				t.Fatal(err)
+			}
+		}, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo"))
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.updated {
+			t.Fatalf("outcome=%+v err=%v, want nothing written", o, err)
+		}
+		if img := getLogin(t, cl).Spec.Image; img != "docker.io/example/limbo:1.2" {
+			t.Errorf("image = %q, want the admin's retarget kept", img)
+		}
+	})
+}
+
+// --system names one of the two system servers; anything else is a usage error
+// before any cluster is touched.
+func TestPinImagesSystemFlagTakesOnlySystemServers(t *testing.T) {
+	var stdout, stderr strings.Builder
+	if code := cmdPinImages([]string{"--system", "survival"}, &stdout, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2", code)
+	}
+	if got := stderr.String(); got != "felis pin-images: --system takes login or lobby, not \"survival\"\n" {
+		t.Errorf("stderr = %q", got)
+	}
+}
+
+// An idle setting the panel saves while converge fills the default is kept.
+func TestConvergeIdleKeepsAConcurrentPanelEdit(t *testing.T) {
+	ctx := context.Background()
+	srv := &v1alpha1.MinecraftServer{}
+	srv.Name, srv.Namespace = "survival", "minecraft"
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Spec.Idle = v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}
+		if err := c.Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, srv)
+
+	if out := convergeUserServerIdle(ctx, cl, "minecraft"); len(out) != 0 {
+		t.Fatalf("outcomes = %+v, want none: the server has a setting by the time converge writes", out)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if want := (v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}); ms.Spec.Idle != want {
+		t.Errorf("idle = %+v, want the panel's %+v", ms.Spec.Idle, want)
+	}
+	if *writes != 1 {
+		t.Errorf("writes = %d, want 1 (the conflicted attempt; the retry sends nothing)", *writes)
+	}
+}

cmd/felis/context_max_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+)
+
+func TestContextMaxBytes(t *testing.T) {
+	cases := []struct {
+		name    string
+		reg     config.RegistryConfig
+		auth    config.AuthConfig
+		want    int64
+		wantErr bool
+	}{
+		{name: "direct install keeps the package default", want: 0},
+		// The panel uploads in parts under the edge's 100 MB body limit, so the
+		// Cloudflare edge keeps the full default.
+		{name: "the Cloudflare edge keeps the package default", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 0},
+		{name: "CF-Connecting-IP keeps the package default", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 0},
+		{name: "explicit value behind the edge", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
+		{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
+		{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
+		{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := contextMaxBytes(&config.Config{Registry: tc.reg, Auth: tc.auth})
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
+			}
+			if got != tc.want {
+				t.Fatalf("contextMaxBytes = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestUploadPartsDir(t *testing.T) {
+	if got := uploadPartsDir("/var/lib/felis/uploads"); got != "/var/lib/felis/uploads/.parts" {
+		t.Errorf("local store: parts dir = %q, want beside the contexts", got)
+	}
+	if got := uploadPartsDir("file:///srv/uploads"); got != "/srv/uploads/.parts" {
+		t.Errorf("file:// store: parts dir = %q, want /srv/uploads/.parts", got)
+	}
+	// This machine has no /var/lib/felis/uploads mount, so an s3:// store falls
+	// back to the temp dir.
+	if _, err := os.Stat("/var/lib/felis/uploads"); err == nil {
+		t.Skip("/var/lib/felis/uploads exists here")
+	}
+	if got := uploadPartsDir("s3://bucket/uploads"); got != filepath.Join(os.TempDir(), "felis-upload-parts") {
+		t.Errorf("s3 store without the uploads mount: parts dir = %q", got)
+	}
+}

cmd/felis/converge.go

@@ -9,12 +9,16 @@ import (
 	"os"
 	"strings"
 
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // cmdConverge is the explicit convergence pass over already-installed system
-// servers (#1). Provisioning is create-if-absent, so a field the desired spec
+// servers (#1), plus the idle-stop default for user servers that predate it, and
+// with -user-rcon their RCON block (#3). Provisioning is create-if-absent, so a field the desired spec
 // gained after an install (spec.rcon, spec.startup.healthHTTPPort, a derived env
 // key) never reaches the existing CR — and nothing says so. This command fills
 // exactly those zero-value fields; see convergeSystemServers for the full contract
@@ -27,6 +31,7 @@ func cmdConverge(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("converge", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	userRcon := fs.Bool("user-rcon", false, "also turn RCON on for user servers created before it was the default")
 	if err := fs.Parse(args); err != nil {
 		if errors.Is(err, flag.ErrHelp) {
 			return 0
@@ -56,7 +61,10 @@ func cmdConverge(args []string, stdout, stderr io.Writer) int {
 		platform.InternalAPIBaseURL(controlNS), cfg.Server.RootDomain,
 		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
 
-	fmt.Fprintln(stdout, "felis converge: filling fields an installed system server predates (operator-set values are never overwritten):")
+	outcomes = append(outcomes, convergeUserServerIdle(context.Background(), cl, cfg.K8s.Namespace)...)
+	outcomes = append(outcomes, convergeUserServerRcon(context.Background(), cl, cfg.K8s.Namespace, *userRcon)...)
+
+	fmt.Fprintln(stdout, "felis converge: filling fields an installed server predates (operator-set values are never overwritten):")
 	exit := 0
 	for _, o := range outcomes {
 		switch {
@@ -71,3 +79,89 @@ func cmdConverge(args []string, stdout, stderr io.Writer) int {
 	}
 	return exit
 }
+
+// convergeUserServerIdle gives every user server that predates the idle default
+// (spec.idle entirely unset) the default idle stop. A server whose idle stop was
+// turned off keeps a duration on its spec, so it is not "unset" and is left
+// alone; system servers never idle out and are skipped. Servers that already
+// carry a value produce no line, so a converged fleet prints nothing here.
+func convergeUserServerIdle(ctx context.Context, cl client.Client, namespace string) []systemServerOutcome {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+			continue
+		}
+		// Re-checked on the copy each attempt reads: an idle setting the panel saved
+		// meanwhile is the user's, and the default must not land over it.
+		changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
+			if ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+				return false
+			}
+			ms.Spec.Idle = v1alpha1.DefaultIdle()
+			return true
+		})
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
+			continue
+		}
+		if !changed {
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{fmt.Sprintf("spec.idle (stop after %ds empty)", v1alpha1.DefaultEmptySecondsBeforeStop)}})
+	}
+	return out
+}
+
+// convergeUserServerRcon handles user servers created before RCON was part of every
+// new server (694e3cb): spec.rcon entirely unset. Such a server has a dead console,
+// reports nobody online, and never idles out, because all three ride RCON.
+//
+// Only with fill does it turn RCON on, with the same block CreateServer writes
+// today; without it each such server gets a line saying so. The fill is opt-in
+// because the operator gates readiness on the RCON probe: a server whose image does
+// not open the listener RCON_PASSWORD asks for would sit in Starting until it is
+// marked Failed. Felis's own paper and lobby images open it; an image a user brought
+// may not, and only the operator running this can tell. A server that already
+// carries any RCON setting (on or off) is left alone and produces no line.
+func convergeUserServerRcon(ctx context.Context, cl client.Client, namespace string, fill bool) []systemServerOutcome {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
+			continue
+		}
+		if !fill {
+			out = append(out, systemServerOutcome{name: ms.Name, available: true,
+				skipped: "no RCON (console, online count and idle stop are off); once its image serves RCON, sudo felis converge -user-rcon turns it on"})
+			continue
+		}
+		changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
+			if ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
+				return false
+			}
+			ms.Spec.Rcon = v1alpha1.RconSpec{
+				Enabled:   true,
+				SecretRef: v1alpha1.SecretKeyRef{Name: naming.RconSecretName(ms.Name), Key: naming.RconSecretKey},
+			}
+			return true
+		})
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
+			continue
+		}
+		if changed {
+			out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true, changes: []string{"spec.rcon"}})
+		}
+	}
+	return out
+}

cmd/felis/converge_test.go

@@ -183,3 +183,113 @@ func TestConvergeSystemServersGuards(t *testing.T) {
 		}
 	})
 }
+
+// TestConvergeUserServerIdle fills the idle default only where spec.idle was
+// never set: a server whose idle stop was turned off (duration kept), one with
+// its own duration, and a system server all stay as they are.
+func TestConvergeUserServerIdle(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	mk := func(name string, idle v1alpha1.IdleSpec, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Idle = idle
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		mk("legacy", v1alpha1.IdleSpec{}, ""),
+		mk("off", v1alpha1.IdleSpec{EmptySecondsBeforeStop: 600}, ""),
+		mk("custom", v1alpha1.IdleSpec{AutoStopEnabled: true, EmptySecondsBeforeStop: 1800}, ""),
+		mk(naming.SystemLobbyServer, v1alpha1.IdleSpec{}, naming.SystemLobbyServer),
+	).Build()
+
+	outcomes := convergeUserServerIdle(ctx, cl, "minecraft")
+	if len(outcomes) != 1 || outcomes[0].name != "legacy" || outcomes[0].err != nil {
+		t.Fatalf("outcomes = %+v, want exactly one fill for legacy", outcomes)
+	}
+	want := map[string]v1alpha1.IdleSpec{
+		"legacy":                 v1alpha1.DefaultIdle(),
+		"off":                    {EmptySecondsBeforeStop: 600},
+		"custom":                 {AutoStopEnabled: true, EmptySecondsBeforeStop: 1800},
+		naming.SystemLobbyServer: {},
+	}
+	for name, idle := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Idle != idle {
+			t.Errorf("%s idle = %+v, want %+v", name, ms.Spec.Idle, idle)
+		}
+	}
+	if again := convergeUserServerIdle(ctx, cl, "minecraft"); len(again) != 0 {
+		t.Fatalf("second pass = %+v, want nothing to do", again)
+	}
+}
+
+// TestConvergeUserServerRcon reports a user server with no RCON block at all and
+// fills it only when asked, with the block CreateServer writes. RCON turned off on
+// purpose, a server with its own secret, and a system server stay as they are and
+// produce no line.
+func TestConvergeUserServerRcon(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	mk := func(name string, rcon v1alpha1.RconSpec, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Rcon = rcon
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	own := v1alpha1.RconSpec{Enabled: true, Port: 25580, SecretRef: v1alpha1.SecretKeyRef{Name: "own", Key: "pw"}}
+	off := v1alpha1.RconSpec{SecretRef: v1alpha1.SecretKeyRef{Name: "rcon-off", Key: naming.RconSecretKey}}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		mk("demo", v1alpha1.RconSpec{}, ""),
+		mk("off", off, ""),
+		mk("own", own, ""),
+		mk(naming.SystemLobbyServer, v1alpha1.RconSpec{}, naming.SystemLobbyServer),
+	).Build()
+	get := func(name string) v1alpha1.RconSpec {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		return ms.Spec.Rcon
+	}
+
+	report := convergeUserServerRcon(ctx, cl, "minecraft", false)
+	if len(report) != 1 || report[0].name != "demo" || report[0].updated || report[0].err != nil ||
+		!strings.Contains(report[0].skipped, "-user-rcon") {
+		t.Fatalf("report = %+v, want one skipped line for demo naming -user-rcon", report)
+	}
+	if got := get("demo"); got != (v1alpha1.RconSpec{}) {
+		t.Fatalf("the report-only pass wrote demo's rcon: %+v", got)
+	}
+
+	filled := convergeUserServerRcon(ctx, cl, "minecraft", true)
+	if len(filled) != 1 || filled[0].name != "demo" || !filled[0].updated || filled[0].err != nil {
+		t.Fatalf("fill = %+v, want exactly one update for demo", filled)
+	}
+	want := map[string]v1alpha1.RconSpec{
+		"demo": {Enabled: true, SecretRef: v1alpha1.SecretKeyRef{
+			Name: naming.RconSecretName("demo"), Key: naming.RconSecretKey}},
+		"off":                    off,
+		"own":                    own,
+		naming.SystemLobbyServer: {},
+	}
+	for name, rcon := range want {
+		if got := get(name); got != rcon {
+			t.Errorf("%s rcon = %+v, want %+v", name, got, rcon)
+		}
+	}
+	for _, fill := range []bool{false, true} {
+		if again := convergeUserServerRcon(ctx, cl, "minecraft", fill); len(again) != 0 {
+			t.Fatalf("second pass (fill=%v) = %+v, want nothing to do", fill, again)
+		}
+	}
+}

cmd/felis/db.go

@@ -0,0 +1,501 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	neturl "net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/retention"
+)
+
+const dbUsage = `usage:
+  felis db backup  [-config path] [-dir dir] [-label daily|manual|...] [-keep n] [-state-dir dir]
+                   [-no-servers] [-metrics-file path]
+  felis db restore [-config path] [-dir dir] [-yes] [-force] [-no-safety-backup] <bundle>
+  felis db verify  [-dir dir] <bundle>
+  felis db list    [-dir dir]
+  felis db check   [-dir dir] [-max-age 26h]
+  felis db audit-export [-config path] [-since date] [-until date] [-out file]
+`
+
+// defaultKeep is how many bundles of a label a backup leaves behind. Manual
+// bundles are the operator's own and are never pruned.
+var defaultKeep = map[string]int{
+	dbbackup.LabelDaily:      14,
+	dbbackup.LabelPreMigrate: 10,
+	dbbackup.LabelPreRestore: 5,
+	dbbackup.LabelOffsite:    1,
+}
+
+// cmdDB implements `felis db`: logical backups of the control-plane database
+// together with the host state a rebuild needs (internal/dbbackup). The verb
+// comes first for the same reason as `felis migrate up`.
+func cmdDB(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("db "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, dbUsage) }
+	dir := fs.String("dir", dbbackup.DefaultDir, "bundle directory")
+	switch verb {
+	case "backup":
+		return dbBackup(fs, dir, rest, stdout, stderr)
+	case "restore":
+		return dbRestore(fs, dir, rest, stdout, stderr)
+	case "verify":
+		return dbVerify(fs, dir, rest, stdout, stderr)
+	case "list":
+		return dbList(fs, dir, rest, stdout, stderr)
+	case "check":
+		return dbCheck(fs, dir, rest, stdout, stderr)
+	case "audit-export":
+		return dbAuditExport(fs, rest, stdout, stderr)
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, dbUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis db: unknown verb %q\n%s", verb, dbUsage)
+	return 2
+}
+
+// parseWithArg parses flags that may sit on either side of one positional
+// argument (`restore -yes x.tar` and `restore x.tar -yes` both work) and
+// returns that argument.
+func parseWithArg(fs *flag.FlagSet, args []string) (string, bool) {
+	if err := fs.Parse(args); err != nil {
+		return "", false
+	}
+	if fs.NArg() == 0 {
+		return "", true
+	}
+	arg := fs.Arg(0)
+	if err := fs.Parse(fs.Args()[1:]); err != nil {
+		return "", false
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprintf(fs.Output(), "felis db: unexpected argument %q\n", fs.Arg(0))
+		return "", false
+	}
+	return arg, true
+}
+
+func dbDatabaseURL(path string) (string, error) {
+	db, err := dbDatabase(path)
+	return db.URL, err
+}
+
+func dbDatabase(path string) (config.DatabaseConfig, error) {
+	cfg, err := config.Load(path)
+	if err != nil {
+		return config.DatabaseConfig{}, err
+	}
+	return cfg.Database, nil
+}
+
+// dbTools places pg_dump, pg_restore and psql. The installer's database runs in
+// k3s and the host has no PostgreSQL client, so when the host config names the
+// [database] deployment the tools run in its postgres container over the
+// container's socket, as the URL's role on the URL's database. Otherwise they
+// come from PATH and connect with the URL.
+func dbTools(db config.DatabaseConfig) (dbbackup.Tools, error) {
+	if db.Deployment == "" {
+		return dbbackup.Tools{}, nil
+	}
+	ns, name, _ := strings.Cut(db.Deployment, "/")
+	u, err := neturl.Parse(db.URL)
+	if err != nil || u.User == nil || u.User.Username() == "" || strings.TrimPrefix(u.Path, "/") == "" {
+		return dbbackup.Tools{}, errors.New("[database] url must name the role and the database to run the tools in the database's pod")
+	}
+	return dbbackup.Tools{
+		Exec: []string{"k3s", "kubectl", "exec", "-i", "-n", ns, "deploy/" + name, "-c", platform.PostgresContainer, "--"},
+		Conn: fmt.Sprintf("host=%s port=%d dbname=%s user=%s connect_timeout=15",
+			platform.PostgresSocketDir, platform.PostgresPort,
+			libpqQuote(strings.TrimPrefix(u.Path, "/")), libpqQuote(u.User.Username())),
+	}, nil
+}
+
+// libpqQuote renders v as a single-quoted libpq connection-string value.
+func libpqQuote(v string) string {
+	return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(v) + "'"
+}
+
+func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never")
+	keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, offsite 1, manual all)")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`)
+	noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle")
+	metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	db, err := dbDatabase(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	tools, err := dbTools(db)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	if *keep < 0 {
+		*keep = defaultKeep[*label]
+	}
+	o := dbbackup.BackupOptions{
+		DatabaseURL: db.URL, Tools: tools, Dir: *dir, Label: *label, Keep: *keep,
+		StateDir: *stateDir, Version: resolvedVersion(), Log: stderr,
+		MetricsFile: *metrics, Record: true,
+	}
+	if !*noServers {
+		o.ExportServers = exportMinecraftServers
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	path, err := dbbackup.Backup(ctx, o)
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// The bundle is on disk and holds the database; the exit status fails
+		// the timer's run so the gap shows in systemctl and the journal, and
+		// the panel and the watchdog read it from the record and the manifest.
+		fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+		fmt.Fprintf(stderr, "felis db backup: %v\n  a restore from %s brings back the database but no servers; check `k3s kubectl get minecraftservers -A`, then run `felis db backup` again\n", err, filepath.Base(path))
+		return 1
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+	return 0
+}
+
+// resolveBundle accepts a path, or a bare bundle name looked up in dir.
+func resolveBundle(dir, arg string) string {
+	if strings.ContainsRune(arg, os.PathSeparator) {
+		return arg
+	}
+	if _, err := os.Stat(arg); err == nil {
+		return arg
+	}
+	return filepath.Join(dir, arg)
+}
+
+func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	yes := fs.Bool("yes", false, "replace the database's contents (required)")
+	force := fs.Bool("force", false, "restore even while other clients are connected")
+	noSafety := fs.Bool("no-safety-backup", false, "skip the bundle of the current database taken first")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, "host state directory for the safety bundle")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	if !*yes {
+		fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d, holding %s).\n",
+			filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion, m.Counts.String())
+		fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.")
+		return 2
+	}
+	db, err := dbDatabase(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	tools, err := dbTools(db)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
+	defer cancel()
+	_, safety, err := dbbackup.Restore(ctx, dbbackup.RestoreOptions{
+		DatabaseURL: db.URL, Tools: tools, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
+		Safety: dbbackup.BackupOptions{Keep: defaultKeep[dbbackup.LabelPreRestore], StateDir: *stateDir,
+			Version: resolvedVersion(), ExportServers: exportMinecraftServers},
+		Log: stderr,
+	})
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		if errors.Is(err, dbbackup.ErrClientsConnected) {
+			fmt.Fprintln(stderr, "  kubectl -n felis scale deployment felis-api felis-operator --replicas=0")
+		}
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db restore: restored %s (schema %d)\n", filepath.Base(bundle), m.SchemaVersion)
+	if safety != "" {
+		fmt.Fprintf(stdout, "  the database as it was before is in %s\n", safety)
+	}
+	// Nothing migrates at startup, so a control plane newer than the bundle needs
+	// its migrations re-applied; rolling back to the release that wrote the bundle
+	// must skip that, or the rollback is undone.
+	fmt.Fprintf(stdout, "  next: felis migrate up -config %s (skip it when rolling back to felis %s, which wrote this bundle)\n", *cfgPath, orUnknown(m.FelisVersion))
+	fmt.Fprintln(stdout, "        kubectl -n felis scale deployment felis-api felis-operator --replicas=1")
+	return 0
+}
+
+func dbVerify(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db verify: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "%s: ok\n  taken   %s (%s)\n  felis   %s\n  schema  %d\n  holds   %s\n  %s\n",
+		filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion,
+		m.Counts.String(), orUnknown(m.PGDumpVersion))
+	for _, f := range m.Files {
+		if f.Link != "" {
+			fmt.Fprintf(stdout, "  %-40s -> %s\n", f.Name, f.Link)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %-40s %d bytes\n", f.Name, f.Size)
+	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  (no MinecraftServer objects: %s)\n", m.ServersError)
+	}
+	return 0
+}
+
+func orUnknown(s string) string {
+	if s == "" {
+		return "unknown"
+	}
+	return s
+}
+
+func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	all, err := dbbackup.List(*dir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db list: %v\n", err)
+		return 1
+	}
+	if len(all) == 0 {
+		fmt.Fprintf(stdout, "no database backups in %s\n", *dir)
+		return 0
+	}
+	now := time.Now()
+	for _, b := range all {
+		fmt.Fprintf(stdout, "%-50s %-12s %10s  %s ago\n", b.Name, b.Label, humanBytes(b.Size), dbbackup.Age(now.Sub(b.Created)))
+	}
+	return 0
+}
+
+// dbAuditExport writes audit rows to a file (or stdout) as JSON lines, so an
+// install can keep them past [audit] retention, after which felis-api deletes them.
+func dbAuditExport(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	sinceFlag := fs.String("since", "", "first day (or RFC 3339 instant) to export, inclusive; empty starts at the oldest row")
+	untilFlag := fs.String("until", "", "day (or RFC 3339 instant) to stop before, exclusive; empty runs to the newest row")
+	out := fs.String("out", "", "file to write (created 0600, never overwritten); empty writes to stdout")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	since, err := parseExportBound(*sinceFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -since: %v\n", err)
+		return 2
+	}
+	until, err := parseExportBound(*untilFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -until: %v\n", err)
+		return 2
+	}
+	if !since.IsZero() && !until.IsZero() && !until.After(since) {
+		fmt.Fprintf(stderr, "felis db audit-export: -until %s is not after -since %s\n", *untilFlag, *sinceFlag)
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+		return 1
+	}
+	w := stdout
+	var f *os.File
+	if *out != "" {
+		if f, err = os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600); err != nil {
+			fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+			return 1
+		}
+		w = f
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	n, err := exportAudit(ctx, url, since, until, w)
+	if f != nil {
+		if cerr := f.Close(); err == nil {
+			err = cerr
+		}
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v (%d rows written)\n", err, n)
+		return 1
+	}
+	fmt.Fprintf(stderr, "felis db audit-export: %d audit rows written\n", n)
+	return 0
+}
+
+func exportAudit(ctx context.Context, url string, since, until time.Time, w io.Writer) (int, error) {
+	drv, err := openStore(ctx, url, false)
+	if err != nil {
+		return 0, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	return retention.ExportAudit(ctx, drv.DB(), since, until, w)
+}
+
+// parseExportBound reads a -since/-until value: a day (midnight UTC) or an
+// RFC 3339 instant; empty is an open bound.
+func parseExportBound(v string) (time.Time, error) {
+	if v == "" {
+		return time.Time{}, nil
+	}
+	if t, err := time.Parse(time.DateOnly, v); err == nil {
+		return t, nil
+	}
+	if t, err := time.Parse(time.RFC3339, v); err == nil {
+		return t.UTC(), nil
+	}
+	return time.Time{}, fmt.Errorf("%q is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)", v)
+}
+
+func humanBytes(n int64) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%d B", n)
+	}
+	div, exp := int64(unit), 0
+	for m := n / unit; m >= unit; m /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
+}
+
+// dbCheck is the freshness probe: exit 1 when the newest daily bundle is
+// missing or older than -max-age, for a monitor or the break-glass console to
+// act on.
+func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest daily bundle")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	b, err := dbbackup.Check(*dir, *maxAge, time.Now())
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db check: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db check: ok, newest daily backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
+	return 0
+}
+
+// serverExportTries and serverExportRetry are how long a backup waits out a
+// cluster that is briefly away (an apiserver restart) before its bundle goes
+// without the MinecraftServer objects.
+const serverExportTries = 3
+
+var serverExportRetry = 10 * time.Second
+
+// exportMinecraftServers is dbbackup's ExportServers on the host: the
+// MinecraftServer objects through k3s kubectl, tried serverExportTries times.
+func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+	for try := 1; ; try++ {
+		out, err := getMinecraftServers(ctx)
+		if err == nil {
+			return out, nil
+		}
+		if try == serverExportTries {
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		}
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		case <-time.After(serverExportRetry):
+		}
+	}
+}
+
+// getMinecraftServers reads every MinecraftServer through the host's k3s
+// kubectl and strips what the API server owns, so the result can be fed back
+// with `kubectl apply -f` on a rebuilt cluster.
+func getMinecraftServers(ctx context.Context) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning
+	// on stderr must not end up inside the JSON.
+	cmd := exec.CommandContext(ctx, "k3s", "kubectl", "get", "minecraftservers.felis.lolicon.best", "-A", "-o", "json")
+	cmd.Env = append(os.Environ(), "KUBECONFIG="+hostBootstrapKubeconfigPath)
+	var errBuf strings.Builder
+	cmd.Stderr = &errBuf
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("k3s kubectl get minecraftservers: %w: %s", err, strings.TrimSpace(errBuf.String()))
+	}
+	return cleanServerList(out)
+}
+
+// cleanServerList drops status and the server-assigned metadata from a
+// `kubectl get -o json` List.
+func cleanServerList(raw []byte) ([]byte, error) {
+	var list struct {
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(raw, &list); err != nil {
+		return nil, fmt.Errorf("parse MinecraftServer list: %w", err)
+	}
+	for _, it := range list.Items {
+		delete(it, "status")
+		if md, ok := it["metadata"].(map[string]any); ok {
+			for _, k := range []string{"resourceVersion", "uid", "creationTimestamp", "generation", "managedFields", "selfLink"} {
+				delete(md, k)
+			}
+		}
+	}
+	if list.Items == nil {
+		list.Items = []map[string]any{}
+	}
+	return json.MarshalIndent(map[string]any{"apiVersion": "v1", "kind": "List", "items": list.Items}, "", "  ")
+}

cmd/felis/db_test.go

@@ -0,0 +1,567 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/store"
+)
+
+func TestDBUsage(t *testing.T) {
+	for _, args := range [][]string{{"db"}, {"db", "frobnicate"}, {"db", "restore"}, {"db", "verify"}, {"db", "backup", "extra"}} {
+		var out, errBuf bytes.Buffer
+		if code := run(args, &out, &errBuf); code != 2 {
+			t.Errorf("%v: exit %d, want 2", args, code)
+		}
+		if !strings.Contains(errBuf.String(), "felis db restore") {
+			t.Errorf("%v: no usage on stderr: %q", args, errBuf.String())
+		}
+	}
+}
+
+func TestDBVerifySaysWhatTheBundleHolds(t *testing.T) {
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	bundles := filepath.Join(dir, "bundles")
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
+		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	out.Reset()
+	if code := run([]string{"db", "verify", "-dir", bundles, filepath.Base(bundle)}, &out, &errBuf); code != 0 {
+		t.Fatalf("verify: exit %d: %s", code, errBuf.String())
+	}
+	for _, want := range []string{filepath.Base(bundle) + ": ok", "schema  3", "holds   4 accounts, 2 servers", "pg_dump (PostgreSQL) 18.6"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("verify output lacks %q:\n%s", want, out.String())
+		}
+	}
+}
+
+// TestDBRestoreNeedsYes: without -yes a restore describes the bundle and stops
+// before anything reaches the database, even with -force and
+// -no-safety-backup, which would otherwise let the replay run at once.
+func TestDBRestoreNeedsYes(t *testing.T) {
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	bundles := filepath.Join(dir, "bundles")
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
+		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	podRuns(t, dir)
+
+	out.Reset()
+	errBuf.Reset()
+	code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-force", "-no-safety-backup", filepath.Base(bundle)}, &out, &errBuf)
+	if code != 2 {
+		t.Fatalf("exit %d, want 2; stderr %q", code, errBuf.String())
+	}
+	if want := filepath.Base(bundle) + " (manual, taken "; !strings.Contains(errBuf.String(), want) || !strings.Contains(errBuf.String(), "schema 3, holding 4 accounts, 2 servers).") {
+		t.Errorf("stderr %q does not describe the bundle", errBuf.String())
+	}
+	if !strings.Contains(errBuf.String(), "re-run with -yes") {
+		t.Errorf("stderr %q does not say how to go on", errBuf.String())
+	}
+	if argv, err := os.ReadFile(filepath.Join(dir, "k3s.args")); err == nil {
+		t.Errorf("a restore without -yes ran in the database pod:\n%s", argv)
+	}
+
+	// A bundle that does not verify is refused before -yes is weighed.
+	errBuf.Reset()
+	if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "missing.tar"}, &out, &errBuf); code != 1 {
+		t.Errorf("missing bundle: exit %d, want 1; stderr %q", code, errBuf.String())
+	}
+	if _, err := os.Stat(filepath.Join(dir, "k3s.args")); err == nil {
+		t.Error("a missing bundle reached the database pod")
+	}
+}
+
+func TestParseWithArg(t *testing.T) {
+	for _, args := range [][]string{{"-yes", "b.tar"}, {"b.tar", "-yes"}} {
+		fs := flag.NewFlagSet("t", flag.ContinueOnError)
+		fs.SetOutput(io.Discard)
+		yes := fs.Bool("yes", false, "")
+		arg, ok := parseWithArg(fs, args)
+		if !ok || arg != "b.tar" || !*yes {
+			t.Errorf("%v -> %q ok=%v yes=%v", args, arg, ok, *yes)
+		}
+	}
+	fs := flag.NewFlagSet("t", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	if _, ok := parseWithArg(fs, []string{"a.tar", "b.tar"}); ok {
+		t.Error("two positional arguments accepted")
+	}
+}
+
+func TestResolveBundle(t *testing.T) {
+	if got := resolveBundle("/var/lib/felis/db-backups", "felis-db-x.tar"); got != "/var/lib/felis/db-backups/felis-db-x.tar" {
+		t.Errorf("bare name -> %s", got)
+	}
+	if got := resolveBundle("/var/lib/felis/db-backups", "/root/copy.tar"); got != "/root/copy.tar" {
+		t.Errorf("path -> %s", got)
+	}
+}
+
+func TestCleanServerList(t *testing.T) {
+	raw := `{"apiVersion":"v1","kind":"List","metadata":{"resourceVersion":""},"items":[{
+		"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer",
+		"metadata":{"name":"survival","namespace":"minecraft","uid":"u","resourceVersion":"42","generation":3,
+			"creationTimestamp":"2026-09-01T00:00:00Z","managedFields":[{}],"labels":{"a":"b"}},
+		"spec":{"desiredState":"Running"},"status":{"phase":"Running"}}]}`
+	out, err := cleanServerList([]byte(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Kind  string           `json:"kind"`
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatal(err)
+	}
+	if got.Kind != "List" || len(got.Items) != 1 {
+		t.Fatalf("got %s", out)
+	}
+	it := got.Items[0]
+	if _, ok := it["status"]; ok {
+		t.Error("status kept")
+	}
+	md := it["metadata"].(map[string]any)
+	for _, k := range []string{"uid", "resourceVersion", "generation", "creationTimestamp", "managedFields"} {
+		if _, ok := md[k]; ok {
+			t.Errorf("metadata.%s kept", k)
+		}
+	}
+	if md["name"] != "survival" || md["namespace"] != "minecraft" || md["labels"] == nil {
+		t.Errorf("identity lost: %v", md)
+	}
+	if it["spec"].(map[string]any)["desiredState"] != "Running" {
+		t.Error("spec lost")
+	}
+
+	empty, err := cleanServerList([]byte(`{"items":null}`))
+	if err != nil || !strings.Contains(string(empty), `"items": []`) {
+		t.Errorf("empty list -> %s, %v", empty, err)
+	}
+	if _, err := cleanServerList([]byte("Warning: x\n{")); err == nil {
+		t.Error("garbage parsed")
+	}
+}
+
+func TestHasPending(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}, {Version: 3}}
+	if hasPending(map[int]struct{}{1: {}, 2: {}, 3: {}}, ms) {
+		t.Error("fully applied reported pending")
+	}
+	if !hasPending(map[int]struct{}{1: {}, 2: {}}, ms) {
+		t.Error("missing 3 not reported")
+	}
+}
+
+type appliedDriver struct {
+	store.Driver
+	done map[int]struct{}
+}
+
+func (d appliedDriver) EnsureVersionTable(context.Context) error { return nil }
+func (d appliedDriver) AppliedVersions(context.Context) (map[int]struct{}, error) {
+	return d.done, nil
+}
+
+func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	// An unusable URL makes an attempted backup observable as an error without
+	// any PostgreSQL tooling.
+	const badURL = "not-a-url"
+	for _, tc := range []struct {
+		name    string
+		done    map[int]struct{}
+		attempt bool
+	}{
+		{"fresh database", map[int]struct{}{}, false},
+		{"up to date", map[int]struct{}{1: {}, 2: {}}, false},
+		{"pending on a populated database", map[int]struct{}{1: {}}, true},
+	} {
+		path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, config.DatabaseConfig{URL: badURL}, t.TempDir(), io.Discard)
+		if attempted := err != nil; attempted != tc.attempt {
+			t.Errorf("%s: attempted = %v (err %v), want %v", tc.name, attempted, err, tc.attempt)
+		}
+		if path != "" {
+			t.Errorf("%s: path = %q", tc.name, path)
+		}
+	}
+}
+
+// audit-export takes a day or an RFC 3339 instant for each bound, and refuses a
+// malformed or inverted window before it opens the config or the database.
+func TestAuditExportBounds(t *testing.T) {
+	for _, tc := range []struct{ in, want string }{
+		{"", "0001-01-01T00:00:00Z"},
+		{"2026-01-31", "2026-01-31T00:00:00Z"},
+		{"2026-01-31T12:30:00+08:00", "2026-01-31T04:30:00Z"},
+	} {
+		got, err := parseExportBound(tc.in)
+		if err != nil || got.Format(time.RFC3339) != tc.want {
+			t.Errorf("parseExportBound(%q) = %v, %v; want %s", tc.in, got, err, tc.want)
+		}
+	}
+	if _, err := parseExportBound("31/01/2026"); err == nil || err.Error() != `"31/01/2026" is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)` {
+		t.Errorf("parseExportBound(31/01/2026) err = %v", err)
+	}
+	for _, tc := range []struct {
+		args    []string
+		wantErr string
+	}{
+		{[]string{"db", "audit-export", "-since", "yesterday"}, `felis db audit-export: -since: "yesterday" is neither`},
+		{[]string{"db", "audit-export", "-until", "2026-13-01"}, `felis db audit-export: -until: "2026-13-01" is neither`},
+		{[]string{"db", "audit-export", "-since", "2026-02-01", "-until", "2026-02-01"}, "felis db audit-export: -until 2026-02-01 is not after -since 2026-02-01"},
+		{[]string{"db", "audit-export", "extra"}, "felis db audit-export [-config path]"},
+	} {
+		var out, errBuf bytes.Buffer
+		code := run(append(tc.args, "-config", "/nonexistent/felis.toml"), &out, &errBuf)
+		if code != 2 || !strings.Contains(errBuf.String(), tc.wantErr) {
+			t.Errorf("%v: exit %d, stderr %q; want 2 and %q", tc.args, code, errBuf.String(), tc.wantErr)
+		}
+	}
+}
+
+// podK3s stands in for `k3s kubectl exec ... --`: it logs its argv and runs the
+// command after -- from the "container" directory, which is the only place the
+// PostgreSQL tools exist, as on an installed host. `kubectl get` lists one
+// MinecraftServer, logged to k3s.get, and refuses its first N calls while
+// servers_fail holds N.
+const podK3s = `#!/bin/sh
+if [ "$1" = kubectl ] && [ "$2" = get ]; then
+  printf '%s\n' "$*" >> "$FAKE_DIR/k3s.get"
+  n=$(/usr/bin/wc -l < "$FAKE_DIR/k3s.get")
+  if [ -f "$FAKE_DIR/servers_fail" ] && [ "$n" -le "$(/bin/cat "$FAKE_DIR/servers_fail")" ]; then
+    echo "The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?" >&2
+    exit 1
+  fi
+  echo '{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer","metadata":{"name":"lobby","namespace":"felis-servers","uid":"u-1"},"spec":{"type":"PAPER"},"status":{"phase":"Running"}}]}'
+  exit 0
+fi
+printf '%s\n' "$*" >> "$FAKE_DIR/k3s.args"
+while [ $# -gt 0 ] && [ "$1" != "--" ]; do shift; done
+shift
+tool=$1; shift
+exec /usr/bin/env -i FAKE_DIR="$FAKE_DIR" PATH=/usr/bin:/bin "$FAKE_DIR/container/$tool" "$@"
+`
+
+var podTools = map[string]string{
+	"pg_dump": `#!/bin/sh
+case "$1" in --version) echo "pg_dump (PostgreSQL) 18.6"; exit 0 ;; esac
+printf 'PGDMP-fake-archive'
+`,
+	"pg_restore": `#!/bin/sh
+cat > /dev/null
+`,
+	"psql": `#!/bin/sh
+for a in "$@"; do case "$a" in *"FROM users"*) echo "4|2"; exit 0 ;; esac; done
+for a in "$@"; do [ "$a" = "-c" ] && { echo 3; exit 0; }; done
+cat > /dev/null
+`,
+}
+
+const podPassword = "pw-must-stay-on-the-host"
+
+// podDB is the host config's [database] on an installed host.
+var podDB = config.DatabaseConfig{
+	URL:        "postgres://felis:" + podPassword + "@127.0.0.1:15432/felis?sslmode=disable",
+	Deployment: "felis/felis-postgres",
+}
+
+const podExecPrefix = "kubectl exec -i -n felis deploy/felis-postgres -c postgres -- "
+
+// newPodRig puts the fake k3s on PATH, alone, and returns the directory its
+// k3s.args log lands in.
+func newPodRig(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	bin := filepath.Join(dir, "bin")
+	container := filepath.Join(dir, "container")
+	for _, d := range []string{bin, container} {
+		if err := os.Mkdir(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeTestFile(t, filepath.Join(bin, "k3s"), podK3s, 0o755)
+	for name, body := range podTools {
+		writeTestFile(t, filepath.Join(container, name), body, 0o755)
+	}
+	t.Setenv("PATH", bin)
+	t.Setenv("FAKE_DIR", dir)
+	return dir
+}
+
+// podRuns returns what the fake k3s ran since the last call, failing on any
+// run outside the database container or with the password on its command
+// line (visible to every local user in ps).
+func podRuns(t *testing.T, dir string) []string {
+	t.Helper()
+	log := filepath.Join(dir, "k3s.args")
+	argv, err := os.ReadFile(log)
+	if err != nil {
+		t.Fatalf("nothing ran through k3s: %v", err)
+	}
+	os.Remove(log)
+	var runs []string
+	for _, line := range strings.Split(strings.TrimSpace(string(argv)), "\n") {
+		if !strings.HasPrefix(line, podExecPrefix) {
+			t.Errorf("k3s ran %q, want everything under %q", line, podExecPrefix)
+		}
+		if strings.Contains(line, podPassword) {
+			t.Errorf("the password crossed into the pod on a command line: %q", line)
+		}
+		runs = append(runs, strings.TrimPrefix(line, podExecPrefix))
+	}
+	return runs
+}
+
+// podConfig writes an installed host's felis.toml, [database] pointing at the
+// pod, into dir.
+func podConfig(t *testing.T, dir string) string {
+	t.Helper()
+	toml := strings.Replace(installerTOML("example.com", "127.0.0.1"),
+		`url = "postgres://felis:[email protected]:5432/felis?sslmode=disable"`,
+		`url = "`+podDB.URL+`"
+deployment = "`+podDB.Deployment+`"`, 1)
+	cfg := filepath.Join(dir, "felis.toml")
+	writeTestFile(t, cfg, toml, 0o600)
+	return cfg
+}
+
+func ranIn(runs []string, prefix string) bool {
+	for _, r := range runs {
+		if strings.HasPrefix(r, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+// TestDBBackupAndRestoreRunTheToolsInTheDatabasePod: on an installed host the
+// database is a k3s Deployment and no PostgreSQL client exists outside it, so
+// `felis db backup` and `restore` must reach the tools through kubectl exec,
+// over the pod's socket, and without putting the role's password on a command
+// line.
+func TestDBBackupAndRestoreRunTheToolsInTheDatabasePod(t *testing.T) {
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+
+	var out, errBuf bytes.Buffer
+	bundles := filepath.Join(dir, "bundles")
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
+		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	if _, err := dbbackupVerify(bundle); err != nil {
+		t.Fatalf("the bundle does not verify: %v", err)
+	}
+	runs := podRuns(t, dir)
+	if !ranIn(runs, "pg_dump --format=custom --no-password --dbname=host=/var/run/postgresql port=5432 dbname='felis' user='felis'") {
+		t.Errorf("pg_dump did not dump over the pod's socket as felis on felis: %q", runs)
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "-force", "-no-safety-backup", bundle}, &out, &errBuf); code != 0 {
+		t.Fatalf("restore: exit %d: %s", code, errBuf.String())
+	}
+	runs = podRuns(t, dir)
+	if !ranIn(runs, "pg_restore --no-owner --no-privileges --file=-") || !ranIn(runs, "psql -X -q -w -v ON_ERROR_STOP=1 -d host=/var/run/postgresql") {
+		t.Errorf("the replay did not run in the pod: %q", runs)
+	}
+}
+
+// TestPreMigrateBackupRunsInTheDatabasePod: the snapshot in front of an upgrade
+// is the one taken most often, by bootstrap on every rerun.
+func TestPreMigrateBackupRunsInTheDatabasePod(t *testing.T) {
+	dir := newPodRig(t)
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	// The snapshot also bundles /etc/felis, which a test machine may lack; the
+	// dump runs first either way.
+	_, err := preMigrateBackup(context.Background(), appliedDriver{done: map[int]struct{}{1: {}}}, ms, podDB, filepath.Join(dir, "bundles"), io.Discard)
+	if err != nil && !strings.Contains(err.Error(), "read host state") {
+		t.Fatalf("snapshot: %v", err)
+	}
+	if runs := podRuns(t, dir); !ranIn(runs, "pg_dump --format=custom") {
+		t.Errorf("pg_dump did not run in the pod: %q", runs)
+	}
+}
+
+func TestDBToolsNeedTheRoleAndDatabase(t *testing.T) {
+	if tools, err := dbTools(config.DatabaseConfig{URL: "postgres://felis:pw@db:5432/felis"}); err != nil || len(tools.Exec) != 0 {
+		t.Errorf("no deployment: tools %+v err %v, want the PATH tools", tools, err)
+	}
+	for _, u := range []string{"postgres://db:5432/felis", "postgres://felis:pw@db:5432/"} {
+		if _, err := dbTools(config.DatabaseConfig{URL: u, Deployment: "felis/felis-postgres"}); err == nil {
+			t.Errorf("%s: no error, want a refusal (the pod connection needs the role and the database)", u)
+		}
+	}
+	tools, err := dbTools(config.DatabaseConfig{URL: `postgres://o%27brien@db/my%20db`, Deployment: "felis/felis-postgres"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(tools.Conn, `dbname='my db' user='o\'brien'`) {
+		t.Errorf("Conn = %q, want the values quoted for libpq", tools.Conn)
+	}
+}
+
+var dbbackupVerify = dbbackup.Verify
+
+func noServerExportWait(t *testing.T) {
+	t.Helper()
+	old := serverExportRetry
+	serverExportRetry = 0
+	t.Cleanup(func() { serverExportRetry = old })
+}
+
+// serverGets counts the `kubectl get` calls the fake k3s answered or refused.
+func serverGets(dir string) int {
+	b, _ := os.ReadFile(filepath.Join(dir, "k3s.get"))
+	return strings.Count(string(b), "\n")
+}
+
+// bundleServers returns the bundle's k8s/minecraftservers.json, or nil.
+func bundleServers(t *testing.T, bundle string) []byte {
+	t.Helper()
+	f, err := os.Open(bundle)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	tr := tar.NewReader(f)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		if h.Name == "k8s/minecraftservers.json" {
+			data, err := io.ReadAll(tr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return data
+		}
+	}
+}
+
+// TestDBBackupWithoutServersFails: when the cluster stays away the daily
+// bundle is still written, and `felis db backup` exits 1, so the timer's run
+// shows failed, saying a restore from the bundle brings back no servers.
+func TestDBBackupWithoutServersFails(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	var out, errBuf bytes.Buffer
+	code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", "", "-label", "daily"}, &out, &errBuf)
+	if code != 1 {
+		t.Fatalf("exit %d, want 1: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	m, err := dbbackupVerify(bundle)
+	if err != nil {
+		t.Fatalf("the database must still be bundled: %v", err)
+	}
+	if !strings.Contains(m.ServersError, "6443 was refused") || !strings.Contains(m.ServersError, "(tried 3 times)") || bundleServers(t, bundle) != nil {
+		t.Errorf("manifest servers error = %q", m.ServersError)
+	}
+	if n := serverGets(dir); n != serverExportTries {
+		t.Errorf("export tried %d times, want %d", n, serverExportTries)
+	}
+	if msg := errBuf.String(); !strings.Contains(msg, "a restore from "+filepath.Base(bundle)+" brings back the database but no servers") {
+		t.Errorf("stderr = %q", msg)
+	}
+}
+
+// TestDBBackupRetriesTheServerExport: a cluster back on the last try costs the
+// bundle nothing, and what it holds is ready for kubectl apply.
+func TestDBBackupRetriesTheServerExport(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "2", 0o600)
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", ""}, &out, &errBuf); code != 0 {
+		t.Fatalf("exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	servers := string(bundleServers(t, bundle))
+	if !strings.Contains(servers, `"name": "lobby"`) || strings.Contains(servers, "status") || strings.Contains(servers, "u-1") {
+		t.Errorf("k8s/minecraftservers.json = %s", servers)
+	}
+	if n := serverGets(dir); n != 3 {
+		t.Errorf("export tried %d times, want 3", n)
+	}
+}
+
+// TestPreMigrateBackupExportsServers: the snapshot every upgrade takes, often
+// the newest bundle, carries the MinecraftServer objects too; a cluster that
+// is away does not hold back the migration, whose rollback needs the database
+// alone.
+func TestPreMigrateBackupExportsServers(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	old := preMigrateStateDir
+	preMigrateStateDir = ""
+	t.Cleanup(func() { preMigrateStateDir = old })
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	bundles := filepath.Join(dir, "bundles")
+	pending := appliedDriver{done: map[int]struct{}{1: {}}}
+
+	path, err := preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil {
+		t.Fatalf("snapshot: %v", err)
+	}
+	if !strings.Contains(string(bundleServers(t, path)), `"name": "lobby"`) {
+		t.Errorf("%s holds no MinecraftServer objects", path)
+	}
+
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	path, err = preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil || path == "" {
+		t.Fatalf("snapshot with the cluster away = %q, %v; want the bundle and no error", path, err)
+	}
+	if m, err := dbbackupVerify(path); err != nil || m.ServersError == "" || bundleServers(t, path) != nil {
+		t.Errorf("snapshot with the cluster away: %+v, %v", m, err)
+	}
+}
+
+// TestServerExportStopsWaitingWithTheContext: a backup whose time is up stops
+// waiting for the cluster between tries.
+func TestServerExportStopsWaitingWithTheContext(t *testing.T) {
+	dir := newPodRig(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	// Long enough for the first try to run to its refusal: starting the fake
+	// k3s on a busy machine can take a few hundred ms. Still far below the
+	// 10s retry wait, so waiting it out would fail the check below.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	start := time.Now()
+	_, err := exportMinecraftServers(ctx)
+	if err == nil || !strings.Contains(err.Error(), "(tried 1 times)") || serverGets(dir) != 1 {
+		t.Fatalf("err = %v after %d tries, want the first failure alone", err, serverGets(dir))
+	}
+	if took := time.Since(start); took > 5*time.Second {
+		t.Errorf("took %s, want the context's deadline, not the %s retry wait", took, serverExportRetry)
+	}
+}

cmd/felis/distributed.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"felis.lolicon.best/internal/archivetransfer"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/placement"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func distributionManager(cl client.Client, cfg *config.Config, image string) (*distributed.Manager, error) {
+	if os.Getenv("FELIS_DISTRIBUTED") != "true" {
+		return nil, nil
+	}
+	controller, url, key := os.Getenv("FELIS_CONTROLLER_NODE"), os.Getenv("FELIS_ARCHIVE_URL"), os.Getenv(archivetransfer.KeyEnv)
+	if controller == "" || url == "" || len(key) < 32 || image == "" || cfg.Archive.Store != "tarLocal" {
+		return nil, fmt.Errorf("distributed mode requires controller identity, archive service/key, Felis image and tarLocal")
+	}
+	return &distributed.Manager{Client: cl, Namespace: cfg.K8s.Namespace, Image: image, Controller: controller, Archive: archivetransfer.Client{URL: url, Root: cfg.Archive.LocalPath, Key: key}, Resolve: placement.Resolve(cl, cfg.K8s.Namespace, controller)}, nil
+}
+
+func reconcileDistribution(ctx context.Context, m *distributed.Manager, stderr io.Writer) {
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+	for {
+		if err := m.SettleBackups(ctx); err != nil {
+			fmt.Fprintln(stderr, "distributed backup:", err)
+		}
+		if err := m.ReconcileMigrations(ctx); err != nil {
+			fmt.Fprintln(stderr, "migration:", err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}

cmd/felis/doctor.go

@@ -0,0 +1,387 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/watchdog"
+)
+
+// systemdUnitDir is where the installer writes its units.
+const systemdUnitDir = "/etc/systemd/system"
+
+// hostCommand runs a host tool (systemctl, journalctl, k3s) and returns its
+// stdout. A tool that exits non-zero still returns what it printed:
+// `systemctl is-active` prints "inactive" and exits 3.
+func hostCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
+	return exec.CommandContext(ctx, name, args...).Output()
+}
+
+// hostServices are the long-running units a full install depends on, checked
+// when their unit file is present: k3s runs the cluster, felis-velocity is the
+// game proxy, felis-nano the single-binary host that runs without k3s.
+var hostServices = []string{"k3s.service", "felis-velocity.service", "felis-nano.service"}
+
+// cmdDoctor runs every check felis watchdog runs, with the settings
+// felis-watchdog.service gives it, plus what only the host shows (systemd
+// units that failed or stopped, timers that no longer fire, alerts that reach
+// no one), and prints them grouped by area with where to look next. It mails
+// nothing, pings no heartbeat and leaves the watchdog's state alone: it is
+// safe to run at any time, as often as wanted.
+func cmdDoctor(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("doctor", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis doctor: run as root (sudo felis doctor): the checks read root-only state under /etc/felis and /var/lib/felis")
+		return 1
+	}
+	host, _ := os.Hostname()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	return runDoctor(ctx, doctorEnv{unitDir: *unitDir, run: hostCommand, now: time.Now(), host: host}, stdout)
+}
+
+// doctorEnv is what one doctor run reads the host through.
+type doctorEnv struct {
+	unitDir string
+	run     func(ctx context.Context, name string, args ...string) ([]byte, error)
+	now     time.Time
+	host    string
+}
+
+// doctorAreas are the report's headings in order, by the area findingArea
+// puts a finding under.
+var doctorAreas = []struct{ key, title string }{
+	{key: "config", title: "configuration"},
+	{key: "cluster", title: "Kubernetes cluster"},
+	{key: "postgres", title: "PostgreSQL"},
+	{key: "proxy", title: "game proxy"},
+	{key: "db-backup", title: "database backups"},
+	{key: "offsite", title: "off-site copy"},
+	{key: "scan-db", title: "build scan database"},
+	{key: "disk", title: "disk space"},
+	{key: "memory", title: "memory"},
+	{key: "k3s-certs", title: "k3s certificates"},
+	{key: "host-address", title: "node address"},
+	{key: "clock", title: "clock"},
+	{key: "systemd", title: "systemd units and timers"},
+	{key: "alerts", title: "alerting"},
+}
+
+func runDoctor(ctx context.Context, env doctorEnv, stdout io.Writer) int {
+	unit := filepath.Join(env.unitDir, "felis-watchdog.service")
+	w, found, unitErr := watchdogUnitFlags(unit)
+	var report watchdog.Report
+	var notes []string
+	fmt.Fprintf(stdout, "felis doctor on %s at %s\n", env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
+	switch {
+	case unitErr != nil:
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "watchdog/unit", Severity: watchdog.Critical,
+			SummaryEN: fmt.Sprintf("cannot read the watchdog's settings: %v", unitErr),
+			Hint:      "rerun the installer (deploy/bootstrap.sh) to rewrite felis-watchdog.service",
+		})
+		fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
+	case !found:
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "watchdog/unit", Severity: watchdog.Critical,
+			SummaryEN: fmt.Sprintf("%s is not installed: nothing checks this host or mails anyone when it breaks", unit),
+			Hint:      "rerun the installer (deploy/bootstrap.sh), which installs felis-watchdog.timer",
+		})
+		fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
+	default:
+		fmt.Fprintf(stdout, "checks run as %s runs them (config %s)\n", unit, w.cfgPath)
+	}
+	fmt.Fprintln(stdout)
+
+	skip := map[string]string{}
+	cfg, cfgErr := config.Load(w.cfgPath)
+	if cfgErr != nil {
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "config", Severity: watchdog.Critical,
+			SummaryEN: cfgErr.Error(),
+			Hint:      "the installer writes it (deploy/bootstrap.sh); felis watchdog fails on every run until it loads",
+		})
+		// The host's units are read without it; whether alerts reach anyone
+		// is not known without its relay.
+		for _, a := range doctorAreas {
+			if a.key != "config" && a.key != "systemd" {
+				skip[a.key] = "the configuration did not load"
+			}
+		}
+	} else {
+		_, owners, ownersErr := watchdogProbes(ctx, w, cfg, env.now, &report)
+		report.Findings = append(report.Findings, alertReachFindings(cfg, owners, ownersErr)...)
+		if w.proxyAddr == "" {
+			skip["proxy"] = "no -proxy-addr"
+		}
+		if w.backupDir == "" {
+			skip["db-backup"] = "no -backup-dir"
+		}
+		if !cfg.Offsite.Enabled() {
+			skip["offsite"] = "not configured"
+		}
+		if !usesMirroredScanDB(cfg) {
+			skip["scan-db"] = "builds do not scan against the registry's copy"
+		}
+		if len(splitList(w.certDirs)) == 0 {
+			skip["k3s-certs"] = "no -k3s-cert-dirs"
+		}
+		if w.nodeIP == "" {
+			skip["host-address"] = "no -node-ip"
+		}
+	}
+	report.Findings = append(report.Findings, unitFindings(ctx, env)...)
+
+	switch url, err := readHeartbeatURL(w.heartbeatFile); {
+	case err != nil:
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "watchdog/heartbeat", Severity: watchdog.Warning,
+			SummaryEN: fmt.Sprintf("the heartbeat URL is unusable, so no run pings it: %v", err),
+			Hint:      "rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL set (docs/troubleshooting.md §14)",
+		})
+	case url == "":
+		notes = append(notes, "no heartbeat URL is set: a host that goes down entirely, or a watchdog that stops running, alerts no one. "+
+			"Rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL (docs/troubleshooting.md §14)")
+	}
+	if until := watchdog.QuietUntil(w.quietPath); env.now.Before(until) {
+		notes = append(notes, fmt.Sprintf("the watchdog mails nothing until %s (%s): the installer holds it while it restarts things on purpose, "+
+			"and a marker an installer killed mid-run left behind holds it until then",
+			until.UTC().Format("2006-01-02 15:04 UTC"), w.quietPath))
+	}
+	return printDoctorReport(stdout, report.Findings, skip, notes)
+}
+
+// printDoctorReport prints each area's findings under its heading, an area
+// with none as fine or, when skip says why, as not checked, then the notes
+// and the count. It returns the exit status: 1 when anything was found.
+func printDoctorReport(stdout io.Writer, findings []watchdog.Finding, skip map[string]string, notes []string) int {
+	byArea := map[string][]watchdog.Finding{}
+	for _, f := range findings {
+		a := findingArea(f.Key)
+		byArea[a] = append(byArea[a], f)
+	}
+	var critical, warning int
+	for _, a := range doctorAreas {
+		fs := byArea[a.key]
+		switch {
+		case len(fs) > 0:
+		case skip[a.key] != "":
+			fmt.Fprintf(stdout, "-  %s: not checked, %s\n", a.title, skip[a.key])
+			continue
+		default:
+			fmt.Fprintf(stdout, "✓  %s\n", a.title)
+			continue
+		}
+		mark := "!"
+		if slices.ContainsFunc(fs, func(f watchdog.Finding) bool { return f.Severity == watchdog.Critical }) {
+			mark = "✗"
+		}
+		fmt.Fprintf(stdout, "%s  %s\n", mark, a.title)
+		for _, f := range fs {
+			if f.Severity == watchdog.Critical {
+				critical++
+			} else {
+				warning++
+			}
+			fmt.Fprintf(stdout, "     %-8s %s: %s\n", f.Severity, f.Key, f.SummaryEN)
+			if f.Hint != "" {
+				fmt.Fprintf(stdout, "              → %s\n", f.Hint)
+			}
+		}
+	}
+	for _, n := range notes {
+		fmt.Fprintf(stdout, "\nnote: %s\n", n)
+	}
+	fmt.Fprintln(stdout)
+	if critical+warning == 0 {
+		fmt.Fprintln(stdout, "no problems found")
+		return 0
+	}
+	fmt.Fprintf(stdout, "%d problem(s): %d critical, %d warning(s)\n", critical+warning, critical, warning)
+	return 1
+}
+
+// findingArea is the report heading a finding key goes under.
+func findingArea(key string) string {
+	head, _, _ := strings.Cut(key, "/")
+	switch head {
+	case "kube-api", "deployment", "system-server", "server-failed", "job-failed", "reaper-stale", "node":
+		return "cluster"
+	case "db-backup", "db-backup-servers":
+		return "db-backup"
+	case "unit", "timer":
+		return "systemd"
+	case "watchdog":
+		return "alerts"
+	}
+	return head
+}
+
+// alertReachFindings is why the watchdog's alerts would reach no one, which
+// its own runs only log: no relay, or no owner with a verified address.
+func alertReachFindings(cfg *config.Config, owners []string, ownersErr error) []watchdog.Finding {
+	var out []watchdog.Finding
+	if cfg.SMTP.Host == "" {
+		out = append(out, watchdog.Finding{
+			Key: "alerts/relay", Severity: watchdog.Warning,
+			SummaryEN: "no [smtp] relay is configured: the watchdog logs its alerts to the journal and mails no one",
+			Hint:      "sudo felis setup, step SMTP",
+		})
+	}
+	if ownersErr == nil && len(owners) == 0 {
+		out = append(out, watchdog.Finding{
+			Key: "alerts/recipients", Severity: watchdog.Warning,
+			SummaryEN: "no owner account has a verified email: the watchdog's alerts reach no one",
+			Hint:      "an owner verifies an address in the panel's account settings",
+		})
+	}
+	return out
+}
+
+// unitFindings reports the installer's systemd units that failed, the
+// long-running ones that are not running, and timers that no longer fire.
+func unitFindings(ctx context.Context, env doctorEnv) []watchdog.Finding {
+	var out []watchdog.Finding
+	seen := map[string]bool{}
+	failed, err := env.run(ctx, "systemctl", "list-units", "--all", "--plain", "--no-legend", "--no-pager", "--state=failed", "felis-*", "k3s.service")
+	if err != nil && len(failed) == 0 {
+		return []watchdog.Finding{{
+			Key: "unit/systemctl", Severity: watchdog.Warning,
+			SummaryEN: fmt.Sprintf("systemctl list-units failed, so no unit was checked: %v", err),
+		}}
+	}
+	for _, line := range strings.Split(string(failed), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
+		}
+		name := fields[0]
+		seen[name] = true
+		out = append(out, watchdog.Finding{
+			Key: "unit/" + name, Severity: watchdog.Critical,
+			SummaryEN: name + " failed",
+			Hint:      fmt.Sprintf("journalctl -u %s -n 100 --no-pager; once fixed, sudo systemctl reset-failed %s (a timer's job clears on its next good run)", name, name),
+		})
+	}
+	for _, name := range hostServices {
+		if seen[name] {
+			continue
+		}
+		if _, err := os.Stat(filepath.Join(env.unitDir, name)); err != nil {
+			continue
+		}
+		if state := unitActiveState(ctx, env, name); state != "active" {
+			out = append(out, watchdog.Finding{
+				Key: "unit/" + name, Severity: watchdog.Critical,
+				SummaryEN: fmt.Sprintf("%s is %s", name, state),
+				Hint:      fmt.Sprintf("sudo systemctl start %s; journalctl -u %s -n 100 --no-pager", name, name),
+			})
+		}
+	}
+	timers, _ := filepath.Glob(filepath.Join(env.unitDir, "felis-*.timer"))
+	for _, path := range timers {
+		name := filepath.Base(path)
+		if state := unitActiveState(ctx, env, name); state != "active" {
+			out = append(out, watchdog.Finding{
+				Key: "timer/" + name, Severity: watchdog.Warning,
+				SummaryEN: fmt.Sprintf("%s is %s: the job it starts no longer runs", name, state),
+				Hint:      fmt.Sprintf("sudo systemctl enable --now %s", name),
+			})
+		}
+	}
+	return out
+}
+
+// unitActiveState is what `systemctl is-active` says of unit.
+func unitActiveState(ctx context.Context, env doctorEnv, unit string) string {
+	out, err := env.run(ctx, "systemctl", "is-active", unit)
+	if state := strings.TrimSpace(string(out)); state != "" {
+		return state
+	}
+	return fmt.Sprintf("in an unknown state (systemctl is-active: %v)", err)
+}
+
+// watchdogUnitFlags reads the flags felis-watchdog.service runs felis
+// watchdog with. found is false when there is no such unit; w is then the
+// watchdog's defaults.
+func watchdogUnitFlags(path string) (w watchdogFlags, found bool, err error) {
+	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	w.register(fs)
+	raw, err := os.ReadFile(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return w, false, nil
+	}
+	if err != nil {
+		return w, false, err
+	}
+	for _, line := range strings.Split(string(raw), "\n") {
+		cmd, ok := strings.CutPrefix(strings.TrimSpace(line), "ExecStart=")
+		if !ok {
+			continue
+		}
+		fields := execArgs(cmd)
+		i := slices.Index(fields, "watchdog")
+		if i < 0 {
+			continue
+		}
+		if err := fs.Parse(fields[i+1:]); err != nil {
+			return w, true, fmt.Errorf("%s: %w", path, err)
+		}
+		return w, true, nil
+	}
+	return w, true, fmt.Errorf("%s runs no `felis watchdog`", path)
+}
+
+// execArgs splits an ExecStart= command line into its words. A word may be
+// quoted with " or ', as systemd allows, which is how an empty value is
+// written.
+func execArgs(s string) []string {
+	var out []string
+	var cur strings.Builder
+	inWord := false
+	var quote rune
+	for _, r := range s {
+		switch {
+		case quote != 0:
+			if r == quote {
+				quote = 0
+			} else {
+				cur.WriteRune(r)
+			}
+		case r == '"' || r == '\'':
+			quote, inWord = r, true
+		case r == ' ' || r == '\t':
+			if inWord {
+				out = append(out, cur.String())
+				cur.Reset()
+				inWord = false
+			}
+		default:
+			cur.WriteRune(r)
+			inWord = true
+		}
+	}
+	if inWord {
+		out = append(out, cur.String())
+	}
+	return out
+}

cmd/felis/doctor_test.go

@@ -0,0 +1,423 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/watchdog"
+)
+
+// bootstrapWatchdogExecStart is the ExecStart= line deploy/bootstrap.sh writes
+// into felis-watchdog.service, with its variables filled in as an install
+// fills them.
+func bootstrapWatchdogExecStart(t *testing.T, vars map[string]string) string {
+	t.Helper()
+	raw, err := os.ReadFile("../../deploy/bootstrap.sh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var line string
+	for _, l := range strings.Split(string(raw), "\n") {
+		if strings.HasPrefix(l, "ExecStart=${HOST_BIN} watchdog -config") {
+			line = l
+		}
+	}
+	if line == "" {
+		t.Fatal("deploy/bootstrap.sh writes no `ExecStart=${HOST_BIN} watchdog -config` line")
+	}
+	line = strings.ReplaceAll(line, "${NODE_IP:+ -node-ip ${NODE_IP}}", " -node-ip "+vars["NODE_IP"])
+	line = regexp.MustCompile(`\$\{([A-Za-z_]+)\}`).ReplaceAllStringFunc(line, func(m string) string {
+		v, ok := vars[m[2:len(m)-1]]
+		if !ok {
+			t.Fatalf("bootstrap's watchdog ExecStart= uses %s, which this test does not fill in", m)
+		}
+		return v
+	})
+	return line
+}
+
+// felis doctor reads the watchdog's settings from the unit the installer
+// writes, so it checks the paths the timer's runs check.
+func TestWatchdogUnitFlagsReadsTheInstallersUnit(t *testing.T) {
+	dir := t.TempDir()
+	exec := bootstrapWatchdogExecStart(t, map[string]string{
+		"HOST_BIN": "/usr/local/bin/felis", "STATE_DIR": "/srv/felis-etc", "WATCHDOG_STATE": "/srv/watchdog/state.json",
+		"WATCHDOG_QUIET_FILE": "/srv/quiet-until", "FELIS_DB_BACKUP_DIR": "/srv/db-backups", "FELIS_GAME_PORT": "25577",
+		"disks": "/,/srv/data", "NODE_IP": "10.0.0.5",
+	})
+	unit := filepath.Join(dir, "felis-watchdog.service")
+	writeTestFile(t, unit, "[Unit]\nDescription=Felis watchdog\n\n[Service]\nType=oneshot\n"+exec+"\nTimeoutStartSec=3min\n", 0o644)
+
+	w, found, err := watchdogUnitFlags(unit)
+	if err != nil || !found {
+		t.Fatalf("found %v, err %v", found, err)
+	}
+	got := []string{w.cfgPath, w.statePath, w.quietPath, w.backupDir, w.proxyAddr, w.diskPaths, w.nodeIP, w.heartbeatFile, w.controlNS}
+	want := []string{"/srv/felis-etc/felis.host.toml", "/srv/watchdog/state.json", "/srv/quiet-until", "/srv/db-backups", "127.0.0.1:25577", "/,/srv/data", "10.0.0.5", defaultHeartbeatFile, "felis"}
+	if strings.Join(got, "|") != strings.Join(want, "|") {
+		t.Errorf("read\n %q\nwant\n %q", got, want)
+	}
+
+	w, found, err = watchdogUnitFlags(filepath.Join(dir, "missing.service"))
+	if err != nil || found || w.cfgPath != "/etc/felis/felis.toml" || w.backupDir != "/var/lib/felis/db-backups" {
+		t.Errorf("no unit: found %v, err %v, config %q, backups %q; want the watchdog's defaults", found, err, w.cfgPath, w.backupDir)
+	}
+
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis version\n", 0o644)
+	if _, found, err := watchdogUnitFlags(unit); !found || err == nil || !strings.Contains(err.Error(), "runs no `felis watchdog`") {
+		t.Errorf("a unit that runs something else: found %v, err %v", found, err)
+	}
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
+	if _, _, err := watchdogUnitFlags(unit); err == nil {
+		t.Error("a flag this binary does not know was accepted")
+	}
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -backup-dir \"\"\t-proxy-addr '127.0.0.1:1' -disk-paths \"/a b\"\n", 0o644)
+	if w, _, err := watchdogUnitFlags(unit); err != nil || w.backupDir != "" || w.proxyAddr != "127.0.0.1:1" || w.diskPaths != "/a b" {
+		t.Errorf("quoted words: backups %q, proxy %q, disks %q, err %v", w.backupDir, w.proxyAddr, w.diskPaths, err)
+	}
+}
+
+func TestFindingArea(t *testing.T) {
+	for key, want := range map[string]string{
+		"kube-api":                           "cluster",
+		"deployment/felis-api":               "cluster",
+		"system-server/lobby":                "cluster",
+		"server-failed/survival":             "cluster",
+		"job-failed/reaper-123":              "cluster",
+		"reaper-stale":                       "cluster",
+		"node/felis-1/NotReady":              "cluster",
+		"postgres":                           "postgres",
+		"proxy":                              "proxy",
+		"db-backup":                          "db-backup",
+		"db-backup-servers":                  "db-backup",
+		"offsite":                            "offsite",
+		"scan-db":                            "scan-db",
+		"disk//var/lib/felis":                "disk",
+		"memory":                             "memory",
+		"k3s-certs":                          "k3s-certs",
+		"host-address":                       "host-address",
+		"clock":                              "clock",
+		"config":                             "config",
+		"unit/felis-offsite.service":         "systemd",
+		"timer/felis-db-backup.timer":        "systemd",
+		"watchdog/unit":                      "alerts",
+		"watchdog/heartbeat":                 "alerts",
+		"alerts/relay":                       "alerts",
+		"alerts/recipients":                  "alerts",
+		"something-a-later-release-reported": "something-a-later-release-reported",
+	} {
+		if got := findingArea(key); got != want {
+			t.Errorf("findingArea(%q) = %q, want %q", key, got, want)
+		}
+	}
+}
+
+func TestAlertReachFindings(t *testing.T) {
+	relay := &config.Config{SMTP: config.SMTPConfig{Host: "smtp.example.com"}}
+	keys := func(fs []watchdog.Finding) string {
+		var k []string
+		for _, f := range fs {
+			k = append(k, f.Key)
+		}
+		return strings.Join(k, ",")
+	}
+	for _, tc := range []struct {
+		what      string
+		cfg       *config.Config
+		owners    []string
+		ownersErr error
+		want      string
+	}{
+		{"a relay and an owner", relay, []string{"[email protected]"}, nil, ""},
+		{"no relay", &config.Config{}, []string{"[email protected]"}, nil, "alerts/relay"},
+		{"no owner with an address", relay, nil, nil, "alerts/recipients"},
+		{"PostgreSQL down: its own finding says so", relay, nil, errors.New("refused"), ""},
+		{"neither", &config.Config{}, nil, nil, "alerts/relay,alerts/recipients"},
+	} {
+		if got := keys(alertReachFindings(tc.cfg, tc.owners, tc.ownersErr)); got != tc.want {
+			t.Errorf("%s: %q, want %q", tc.what, got, tc.want)
+		}
+	}
+}
+
+// fakeSystemctl answers list-units with failed and is-active from states;
+// a unit missing from states is "inactive", as systemctl says, and one whose
+// state is "" gets no answer.
+func fakeSystemctl(failed string, states map[string]string) func(ctx context.Context, name string, args ...string) ([]byte, error) {
+	return func(_ context.Context, name string, args ...string) ([]byte, error) {
+		if name != "systemctl" || len(args) == 0 {
+			return nil, errors.New("unexpected command " + name)
+		}
+		switch args[0] {
+		case "list-units":
+			return []byte(failed), nil
+		case "is-active":
+			if s, ok := states[args[1]]; ok && s == "" {
+				return nil, errors.New("signal: killed")
+			} else if ok {
+				return []byte(s + "\n"), nil
+			}
+			return []byte("inactive\n"), errors.New("exit status 3")
+		}
+		return nil, errors.New("unexpected systemctl " + args[0])
+	}
+}
+
+func TestUnitFindings(t *testing.T) {
+	dir := t.TempDir()
+	for _, f := range []string{"k3s.service", "felis-velocity.service", "felis-db-backup.timer", "felis-offsite.timer", "felis-offsite.service"} {
+		writeTestFile(t, filepath.Join(dir, f), "[Unit]\n", 0o644)
+	}
+	env := doctorEnv{unitDir: dir, run: fakeSystemctl(
+		"felis-offsite.service loaded failed failed Felis off-site copy\nfelis-velocity.service loaded failed failed Velocity\n",
+		map[string]string{"k3s.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "inactive"},
+	)}
+	var got []string
+	for _, f := range unitFindings(context.Background(), env) {
+		got = append(got, string(f.Severity)+" "+f.Key+": "+f.SummaryEN)
+	}
+	want := []string{
+		"critical unit/felis-offsite.service: felis-offsite.service failed",
+		"critical unit/felis-velocity.service: felis-velocity.service failed",
+		"warning timer/felis-offsite.timer: felis-offsite.timer is inactive: the job it starts no longer runs",
+	}
+	if strings.Join(got, "\n") != strings.Join(want, "\n") {
+		t.Errorf("findings:\n%s\nwant (felis-nano.service has no unit file here, and a failed unit is reported once):\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
+	}
+
+	env.run = fakeSystemctl("", map[string]string{"k3s.service": "activating", "felis-velocity.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "active"})
+	got = nil
+	for _, f := range unitFindings(context.Background(), env) {
+		got = append(got, f.Key+": "+f.SummaryEN)
+	}
+	if strings.Join(got, "\n") != "unit/k3s.service: k3s.service is activating" {
+		t.Errorf("findings %q, want only k3s.service, which is not active", got)
+	}
+
+	env.run = fakeSystemctl("", map[string]string{"k3s.service": "active", "felis-velocity.service": "active", "felis-db-backup.timer": "", "felis-offsite.timer": "active"})
+	got = nil
+	for _, f := range unitFindings(context.Background(), env) {
+		got = append(got, f.Key+": "+f.SummaryEN)
+	}
+	if want := "timer/felis-db-backup.timer: felis-db-backup.timer is in an unknown state (systemctl is-active: signal: killed): the job it starts no longer runs"; strings.Join(got, "\n") != want {
+		t.Errorf("findings %q, want %q", got, want)
+	}
+
+	env.run = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("no systemctl") }
+	if fs := unitFindings(context.Background(), env); len(fs) != 1 || fs[0].Key != "unit/systemctl" || fs[0].Severity != watchdog.Warning {
+		t.Errorf("without systemctl: %+v, want the one unit/systemctl warning", fs)
+	}
+}
+
+// quoteArgs writes args as an ExecStart= line does, each word in quotes so an
+// empty one survives.
+func quoteArgs(args []string) string {
+	q := make([]string, len(args))
+	for i, a := range args {
+		q[i] = `"` + a + `"`
+	}
+	return strings.Join(q, " ")
+}
+
+// doctorHost is a host with the installer's watchdog unit, whose API server
+// and PostgreSQL are down.
+func doctorHost(t *testing.T, cfg string, extraFlags string) (env doctorEnv, h *watchdogHost) {
+	t.Helper()
+	h = newWatchdogHost(t, cfg, nil)
+	unitDir := t.TempDir()
+	writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"),
+		"[Service]\nType=oneshot\nExecStart=/usr/local/bin/felis watchdog "+quoteArgs(h.args)+
+			// Off this machine's disk, whose free space is not the test's.
+			` -disk-paths "/nonexistent-felis-doctor-test"`+extraFlags+"\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-offsite.timer"), "[Unit]\n", 0o644)
+	return doctorEnv{
+		unitDir: unitDir, now: time.Now(), host: "felis-test",
+		run: fakeSystemctl("felis-db-backup.service loaded failed failed Felis database backup\n",
+			map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"}),
+	}, h
+}
+
+func TestDoctorReportsByArea(t *testing.T) {
+	env, _ := doctorHost(t, testWatchdogConfig, "")
+	var out bytes.Buffer
+	code := runDoctor(context.Background(), env, &out)
+	got := out.String()
+	if code != 1 {
+		t.Errorf("exit %d, want 1 with problems found", code)
+	}
+	for _, want := range []string{
+		"felis doctor on felis-test at ",
+		"checks run as " + filepath.Join(env.unitDir, "felis-watchdog.service") + " runs them (config ",
+		"✓  configuration\n",
+		"✗  Kubernetes cluster\n     critical kube-api: ",
+		"✗  PostgreSQL\n     critical postgres: ",
+		"-  game proxy: not checked, no -proxy-addr\n",
+		"-  database backups: not checked, no -backup-dir\n",
+		"-  off-site copy: not checked, not configured\n",
+		"-  build scan database: not checked, builds do not scan against the registry's copy\n",
+		"-  k3s certificates: not checked, no -k3s-cert-dirs\n",
+		"-  node address: not checked, no -node-ip\n",
+		"✗  systemd units and timers\n     critical unit/felis-db-backup.service: felis-db-backup.service failed\n" +
+			"              → journalctl -u felis-db-backup.service -n 100 --no-pager; once fixed, sudo systemctl reset-failed felis-db-backup.service",
+		"!  alerting\n     warning  alerts/relay: no [smtp] relay is configured",
+		"\n4 problem(s): 3 critical, 1 warning(s)\n",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("report lacks %q:\n%s", want, got)
+		}
+	}
+	if strings.Contains(got, "alerts/recipients") {
+		t.Errorf("reported no recipients although PostgreSQL, which names them, is down:\n%s", got)
+	}
+	if strings.Contains(got, "note: no heartbeat URL") {
+		t.Errorf("the host has a heartbeat URL:\n%s", got)
+	}
+}
+
+// A doctor run is only a look: whatever is due to be mailed stays due, no
+// heartbeat is pinged, and the watchdog's state is left as it was.
+func TestDoctorMailsPingsAndSavesNothing(t *testing.T) {
+	cfg := testWatchdogConfig + "[smtp]\nhost = \"smtp.example.com\"\nport = 587\nfrom = \"[email protected]\"\n"
+	env, h := doctorHost(t, cfg, "")
+	if err := watchdog.SaveState(h.statePath, duePostgres("cached-pw")); err != nil {
+		t.Fatal(err)
+	}
+	before, err := os.ReadFile(h.statePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	watchdogSender = h.rec.sender
+	defer func() { watchdogSender = smtpSender }()
+	var out bytes.Buffer
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "critical postgres: ") {
+		t.Fatalf("the due PostgreSQL alert was not seen:\n%s", out.String())
+	}
+	if len(h.rec.sent) != 0 || len(h.rec.relays) != 0 {
+		t.Errorf("mailed %v", h.rec.sent)
+	}
+	if n := len(h.pings.pings); n != 0 {
+		t.Errorf("pinged the heartbeat %d times", n)
+	}
+	after, err := os.ReadFile(h.statePath)
+	if err != nil || !bytes.Equal(before, after) {
+		t.Errorf("the watchdog state changed (err %v)", err)
+	}
+	if _, err := os.Stat(h.fallbackPath); !errors.Is(err, os.ErrNotExist) {
+		t.Errorf("a fallback state was written: %v", err)
+	}
+}
+
+func TestDoctorNotes(t *testing.T) {
+	env, h := doctorHost(t, testWatchdogConfig, "")
+	if err := os.Remove(filepath.Join(h.dir, "watchdog-heartbeat-url")); err != nil {
+		t.Fatal(err)
+	}
+	until := env.now.Add(10 * time.Minute).Unix()
+	writeTestFile(t, filepath.Join(h.dir, "quiet"), strconv.FormatInt(until, 10)+"\n", 0o644)
+	var out bytes.Buffer
+	runDoctor(context.Background(), env, &out)
+	for _, want := range []string{
+		"\nnote: no heartbeat URL is set: ",
+		"\nnote: the watchdog mails nothing until " + time.Unix(until, 0).UTC().Format("2006-01-02 15:04 UTC") + " (" + filepath.Join(h.dir, "quiet") + ")",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("report lacks %q:\n%s", want, out.String())
+		}
+	}
+
+	writeTestFile(t, filepath.Join(h.dir, "watchdog-heartbeat-url"), "not a url\n", 0o600)
+	out.Reset()
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "warning  watchdog/heartbeat: the heartbeat URL is unusable") {
+		t.Errorf("an unusable heartbeat URL is not reported:\n%s", out.String())
+	}
+}
+
+// Without the watchdog's unit the doctor still checks, with the watchdog's
+// defaults, and says the host has no watchdog; a configuration that does not
+// load leaves the checks that need it unchecked and the host's own checks on.
+func TestDoctorWithoutUnitOrConfig(t *testing.T) {
+	env, _ := doctorHost(t, testWatchdogConfig, "")
+	if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
+		t.Fatal(err)
+	}
+	writeTestFile(t, filepath.Join(env.unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+filepath.Join(t.TempDir(), "absent.toml")+"\n", 0o644)
+	env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"})
+	var out bytes.Buffer
+	if code := runDoctor(context.Background(), env, &out); code != 1 {
+		t.Errorf("exit %d, want 1", code)
+	}
+	for _, want := range []string{
+		"✗  configuration\n     critical config: ",
+		"-  Kubernetes cluster: not checked, the configuration did not load\n",
+		"-  PostgreSQL: not checked, the configuration did not load\n",
+		"-  disk space: not checked, the configuration did not load\n",
+		"✓  systemd units and timers\n",
+		"-  alerting: not checked, the configuration did not load\n",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("report lacks %q:\n%s", want, out.String())
+		}
+	}
+
+	if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
+		t.Fatal(err)
+	}
+	out.Reset()
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "critical watchdog/unit: "+filepath.Join(env.unitDir, "felis-watchdog.service")+" is not installed") ||
+		!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
+		t.Errorf("a host without the watchdog's unit:\n%s", out.String())
+	}
+
+	unit := filepath.Join(env.unitDir, "felis-watchdog.service")
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
+	out.Reset()
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "critical watchdog/unit: cannot read the watchdog's settings: "+unit+": flag provided but not defined: -no-such-flag\n") ||
+		!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
+		t.Errorf("a watchdog unit this binary cannot read:\n%s", out.String())
+	}
+}
+
+func TestPrintDoctorReport(t *testing.T) {
+	var out bytes.Buffer
+	if code := printDoctorReport(&out, nil, map[string]string{"proxy": "no -proxy-addr"}, nil); code != 0 {
+		t.Errorf("exit %d with nothing found, want 0", code)
+	}
+	want := "✓  configuration\n✓  Kubernetes cluster\n✓  PostgreSQL\n-  game proxy: not checked, no -proxy-addr\n✓  database backups\n✓  off-site copy\n" +
+		"✓  build scan database\n✓  disk space\n✓  memory\n✓  k3s certificates\n✓  node address\n✓  clock\n✓  systemd units and timers\n✓  alerting\n" +
+		"\nno problems found\n"
+	if out.String() != want {
+		t.Errorf("report:\n%s\nwant:\n%s", out.String(), want)
+	}
+
+	out.Reset()
+	code := printDoctorReport(&out, []watchdog.Finding{
+		{Key: "unit/systemctl", Severity: watchdog.Warning, SummaryEN: "systemctl list-units failed"},
+		{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "PostgreSQL is down", Hint: "kubectl -n felis get pods"},
+	}, map[string]string{"postgres": "a skip loses to what was found"}, []string{"a note"})
+	if code != 1 {
+		t.Errorf("exit %d with problems, want 1", code)
+	}
+	for _, want := range []string{
+		"✗  PostgreSQL\n     critical postgres: PostgreSQL is down\n              → kubectl -n felis get pods\n✓  game proxy\n",
+		"!  systemd units and timers\n     warning  unit/systemctl: systemctl list-units failed\n✓  alerting\n",
+		"✓  alerting\n\nnote: a note\n\n2 problem(s): 1 critical, 1 warning(s)\n",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("report lacks %q:\n%s", want, out.String())
+		}
+	}
+}

cmd/felis/domain_test.go

@@ -0,0 +1,892 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// installerTOML is felis.toml as deploy/bootstrap.sh write_felis_toml renders it,
+// comments included: the domain move has to leave all of it but three values alone.
+func installerTOML(root, dbHost string) string {
+	return `# Generated by deploy/bootstrap.sh; rerun the installer to regenerate. Hand edits are
+# overwritten, except [smtp], [[auth_source]], [offsite], and the operator-owned
+# [registry] / [archive] overrides, which carry forward.
+[server]
+listen = "0.0.0.0:8080"
+root_domain = "` + root + `"
+
+[database]
+url = "postgres://felis:pw@` + dbHost + `:5432/felis?sslmode=disable"
+
+[k8s]
+namespace = "minecraft"
+egress_mode = "nodeport"
+
+[velocity]
+# The two always-on system servers that felis setup provisions.
+login_image = "felis/limbo:1"
+lobby_image = "felis/lobby:1"
+game_port = 25565
+
+[registry]
+url = "registry.felis.svc:5000"
+build_namespace = "felis-build"
+
+[archive]
+store = "tarLocal"
+local_path = "/var/lib/felis/archives"
+
+[auth]
+admin_hostname = "op.console.` + root + `"
+panel_hostname = "console.` + root + `"
+access_jwt_aud = "aud123"
+
+# Third-party Yggdrasil sources federated by the hasJoined multiplexer.
+[[auth_source]]
+tag = "littleskin"
+prefix = "LS"
+url = "https://littleskin.cn/api/yggdrasil/sessionserver/session/minecraft/hasJoined"
+`
+}
+
+const linkPropsBody = `# Generated by deploy/bootstrap.sh — do not edit by hand; rerun the installer.
+api-base-url=http://10.43.0.9:8081
+service-token=TOKEN-NOT-TO-TOUCH
+root-domain=old.example
+panel-hostname=console.old.example
+admin-hostname=op.console.old.example
+login-server=login
+lobby-server=lobby
+`
+
+var oldNames = domainNames{root: "old.example", panel: "console.old.example", admin: "op.console.old.example"}
+var newNames = domainNames{root: "new.example", panel: "console.new.example", admin: "op.console.new.example"}
+
+// domainRig models the host: the files, the cluster, and a felis-api, proxy and
+// operator that pick up config the way the real ones do — the api serves what it
+// read at its last restart, the proxy runs since its last restart, and the login
+// pod carries the env its MinecraftServer had when it was last rolled.
+type domainRig struct {
+	h      domainHost
+	cl     client.Client
+	out    *bytes.Buffer
+	dir    string
+	events []string
+
+	served      domainNames
+	servedCert  *x509.Certificate
+	proxySince  time.Time
+	proxyLoaded bool
+	unresolved  map[string]bool
+	// The fake operator: the CR env the login pod was last rolled to, and how
+	// many looks at the pod since the CR moved on.
+	rolledTo string
+	pending  int
+}
+
+func (rig *domainRig) path(name string) string { return filepath.Join(rig.dir, name) }
+
+func newDomainRig(t *testing.T) *domainRig {
+	t.Helper()
+	rig := &domainRig{out: &bytes.Buffer{}, dir: t.TempDir(), proxyLoaded: true, unresolved: map[string]bool{}}
+	writeTestFile(t, rig.path("felis.host.toml"), installerTOML("old.example", "127.0.0.1"), 0o600)
+	writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
+	if err := os.Symlink(rig.path("felis.host.toml"), rig.path("felis.toml")); err != nil {
+		t.Fatal(err)
+	}
+	certPEM, keyPEM, err := issuePanelCert(oldNames, []net.IP{net.ParseIP("10.211.55.6")}, time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
+	writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
+	// The proxy started before its config was last written, which is how it
+	// stands after an install.
+	rig.proxySince = time.Now().Add(-time.Hour)
+
+	pod := []byte(installerTOML("old.example", "10.211.55.6"))
+	login, err := loginSystemServer("felis/limbo:1", "minecraft", platform.InternalAPIBaseURL("felis"), oldNames.root, oldNames.panel)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lobby, err := lobbySystemServer("felis/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatal(err)
+	}
+	loginPod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"},
+		Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "minecraft", Env: podEnv(login.Spec.Env)}}},
+		Status:     corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}},
+	}
+	rig.rolledTo = envKey(login.Spec.Env)
+	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
+		Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+			if key.Name == naming.SystemLoginServer+"-0" {
+				rig.operatorTick(t, c)
+			}
+			return c.Get(ctx, key, obj, opts...)
+		},
+	}).WithObjects(
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.ConfigSecretName},
+			Data: map[string][]byte{platform.ConfigSecretKey: pod}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: platform.ConfigSecretName},
+			Data: map[string][]byte{platform.ConfigSecretKey: pod}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.APITLSSecretName},
+			Type: corev1.SecretTypeTLS, Data: map[string][]byte{corev1.TLSCertKey: certPEM, corev1.TLSPrivateKeyKey: keyPEM}},
+		login, lobby, loginPod,
+	).Build()
+	rig.served = oldNames
+	rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
+
+	rig.h = domainHost{
+		paths: domainPaths{
+			hostTOML: rig.path("felis.host.toml"), podTOML: rig.path("felis.pod.toml"), defaultTOML: rig.path("felis.toml"),
+			cert: rig.path("panel-tls.crt"), key: rig.path("panel-tls.key"),
+			linkProps: rig.path("felis-link.properties"), tunnelConfig: rig.path("cloudflared.yml"),
+		},
+		cl:        rig.cl,
+		controlNS: "felis",
+		rollAPI: func(ctx context.Context) error {
+			rig.events = append(rig.events, "roll-api")
+			rig.restartAPI(t)
+			return nil
+		},
+		restartUnit: func(_ context.Context, unit string) error {
+			rig.events = append(rig.events, "restart "+unit)
+			rig.proxySince = time.Now().Add(time.Second)
+			return nil
+		},
+		unitState: func(context.Context, string) (unitStatus, error) {
+			return unitStatus{loaded: rig.proxyLoaded, active: rig.proxyLoaded, since: rig.proxySince}, nil
+		},
+		liveAPI: func(context.Context, string) (liveAPIView, error) {
+			return liveAPIView{names: rig.served, cert: rig.servedCert}, nil
+		},
+		lookupHost: func(_ context.Context, host string) ([]string, error) {
+			if rig.unresolved[host] {
+				return nil, errors.New("no such host")
+			}
+			return []string{"10.211.55.6"}, nil
+		},
+		passkeys:  func(context.Context) (int, int, error) { return 3, 2, nil },
+		now:       time.Now,
+		out:       rig.out,
+		loginWait: 50 * time.Millisecond,
+		pollEvery: time.Millisecond,
+	}
+	return rig
+}
+
+func podEnv(env []v1alpha1.EnvVar) []corev1.EnvVar {
+	out := make([]corev1.EnvVar, len(env))
+	for i, e := range env {
+		out[i] = corev1.EnvVar{Name: e.Name, Value: e.Value}
+	}
+	return out
+}
+
+// restartAPI makes the fake felis-api load the config and certificate its
+// Secrets hold now.
+func (rig *domainRig) restartAPI(t *testing.T) {
+	t.Helper()
+	var cfg, tlsSec corev1.Secret
+	ctx := context.Background()
+	if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.ConfigSecretName}, &cfg); err != nil {
+		t.Fatal(err)
+	}
+	if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.APITLSSecretName}, &tlsSec); err != nil {
+		t.Fatal(err)
+	}
+	names, err := tomlDomainNames(cfg.Data[platform.ConfigSecretKey])
+	if err != nil {
+		t.Fatal(err)
+	}
+	rig.served = names
+	rig.servedCert, err = x509.ParseCertificate(mustCertDER(tlsSec.Data[corev1.TLSCertKey]))
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// rollLoginPod is the operator restarting the login pod onto its CR's env.
+// operatorTick is the operator as the login pod is watched: once the CR's env
+// changes it takes operatorLag looks at the pod before the restarted pod
+// carries the new env, the way a real rollout lags the CR.
+func (rig *domainRig) operatorTick(t *testing.T, c client.Client) {
+	t.Helper()
+	ctx := context.Background()
+	var ms v1alpha1.MinecraftServer
+	if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if envKey(ms.Spec.Env) == rig.rolledTo {
+		return
+	}
+	if rig.pending++; rig.pending < operatorLag {
+		return
+	}
+	var pod corev1.Pod
+	if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"}, &pod); err != nil {
+		t.Fatal(err)
+	}
+	pod.Spec.Containers[0].Env = podEnv(ms.Spec.Env)
+	if err := c.Update(ctx, &pod); err != nil {
+		t.Fatal(err)
+	}
+	rig.rolledTo, rig.pending = envKey(ms.Spec.Env), 0
+}
+
+const operatorLag = 3
+
+func envKey(env []v1alpha1.EnvVar) string {
+	var b strings.Builder
+	for _, e := range env {
+		b.WriteString(e.Name + "=" + e.Value + "\n")
+	}
+	return b.String()
+}
+
+func (rig *domainRig) read(t *testing.T, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(rig.path(name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
+func (rig *domainRig) secret(t *testing.T, ns, name string) map[string][]byte {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		t.Fatal(err)
+	}
+	return s.Data
+}
+
+func (rig *domainRig) crEnv(t *testing.T, name string) map[string]string {
+	t.Helper()
+	var ms v1alpha1.MinecraftServer
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	env := map[string]string{}
+	for _, e := range ms.Spec.Env {
+		env[e.Name] = e.Value
+	}
+	return env
+}
+
+// snapshot is every byte `set` may touch, for proving a refused or dry run
+// touched none of it.
+func (rig *domainRig) snapshot(t *testing.T) string {
+	t.Helper()
+	var b strings.Builder
+	entries, _ := os.ReadDir(rig.dir)
+	for _, e := range entries {
+		b.WriteString(e.Name() + "\n" + rig.read(t, e.Name()) + "\n")
+	}
+	var lines []string
+	for _, s := range []struct{ ns, name string }{{"felis", platform.ConfigSecretName}, {"minecraft", platform.ConfigSecretName}, {"felis", platform.APITLSSecretName}} {
+		for k, v := range rig.secret(t, s.ns, s.name) {
+			lines = append(lines, s.ns+"/"+s.name+"/"+k+"\n"+string(v))
+		}
+	}
+	for k, v := range rig.crEnv(t, naming.SystemLoginServer) {
+		lines = append(lines, "env "+k+"="+v)
+	}
+	sort.Strings(lines)
+	b.WriteString(strings.Join(lines, "\n"))
+	return b.String()
+}
+
+func TestNormalizeRootDomain(t *testing.T) {
+	for in, want := range map[string]string{
+		"Example.COM.":          "example.com",
+		"  mc.example.org ":     "mc.example.org",
+		"10.211.55.6.nip.io":    "10.211.55.6.nip.io",
+		"xn--bcher-kva.example": "xn--bcher-kva.example",
+	} {
+		got, err := normalizeRootDomain(in)
+		if err != nil || got != want {
+			t.Errorf("normalizeRootDomain(%q) = %q, %v; want %q", in, got, err, want)
+		}
+	}
+	for _, in := range []string{"", "https://example.com", "example.com:443", "example.com/x", "10.0.0.1", "::1",
+		"localhost", "a_b.example", "-a.example", "a-.example", strings.Repeat("a", 64) + ".example",
+		strings.Repeat("abcdefghi.", 25) + "example"} {
+		if got, err := normalizeRootDomain(in); err == nil {
+			t.Errorf("normalizeRootDomain(%q) = %q, want an error", in, got)
+		}
+	}
+}
+
+func TestPlanDomainChangeMovesDefaultsAndKeepsHandSetNames(t *testing.T) {
+	p := planDomainChange(oldNames, "new.example")
+	if p.to != newNames || p.customPanel || p.customAdmin {
+		t.Fatalf("defaults: %+v", p)
+	}
+	p = planDomainChange(domainNames{root: "old.example", panel: "play.corp.net", admin: "op.console.old.example"}, "new.example")
+	if p.to.panel != "play.corp.net" || !p.customPanel || p.to.admin != "op.console.new.example" || p.customAdmin {
+		t.Fatalf("hand-set panel: %+v", p)
+	}
+	p = planDomainChange(domainNames{root: "old.example", panel: "console.old.example", admin: "admin.corp.net"}, "new.example")
+	if p.to.admin != "admin.corp.net" || !p.customAdmin || p.to.panel != "console.new.example" {
+		t.Fatalf("hand-set admin: %+v", p)
+	}
+}
+
+func TestEditTOMLStringsChangesOnlyTheDomainLines(t *testing.T) {
+	orig := installerTOML("old.example", "127.0.0.1")
+	out, err := editTOMLStrings([]byte(orig), domainTOMLEdits(newNames))
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, b := strings.Split(orig, "\n"), strings.Split(string(out), "\n")
+	if len(a) != len(b) {
+		t.Fatalf("line count %d → %d:\n%s", len(a), len(b), out)
+	}
+	changed := map[string]string{}
+	for i := range a {
+		if a[i] != b[i] {
+			changed[a[i]] = b[i]
+		}
+	}
+	want := map[string]string{
+		`root_domain = "old.example"`:               `root_domain = "new.example"`,
+		`admin_hostname = "op.console.old.example"`: `admin_hostname = "op.console.new.example"`,
+		`panel_hostname = "console.old.example"`:    `panel_hostname = "console.new.example"`,
+	}
+	if len(changed) != len(want) {
+		t.Fatalf("changed lines %v, want %v", changed, want)
+	}
+	for k, v := range want {
+		if changed[k] != v {
+			t.Errorf("%q → %q, want %q", k, changed[k], v)
+		}
+	}
+}
+
+func TestEditTOMLStringsAddsMissingKeysInTheirTable(t *testing.T) {
+	in := "[server]\nroot_domain = \"old.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\n\n[smtp]\nhost = \"relay\"\n"
+	out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "[server]\nroot_domain = \"new.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\npanel_hostname = \"console.new.example\"\nadmin_hostname = \"op.console.new.example\"\n\n[smtp]\nhost = \"relay\"\n"
+	if string(out) != want {
+		t.Fatalf("got:\n%s\nwant:\n%s", out, want)
+	}
+
+	out, err = editTOMLStrings([]byte("[server]\nroot_domain = \"old.example\"\n\n[[auth_source]]\ntag = \"ls\"\n"), domainTOMLEdits(newNames))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := tomlDomainNames(out)
+	if err != nil || got != newNames || !strings.Contains(string(out), "[[auth_source]]\ntag = \"ls\"\n") {
+		t.Fatalf("no [auth] table: %v %+v\n%s", err, got, out)
+	}
+}
+
+func TestEditTOMLStringsRefusesWhatItCannotEditExactly(t *testing.T) {
+	for name, in := range map[string]string{
+		"multi-line value": "[server]\nroot_domain = \"\"\"\nold.example\"\"\"\n[auth]\n",
+		// The key's line sits inside another value; the real key is absent.
+		"key inside a string": "[server]\nmotd = \"\"\"\nroot_domain = \"old.example\"\n\"\"\"\n[auth]\n",
+		"quoted header":       "[server]\nroot_domain = \"old.example\"\n[\"auth\"]\npanel_hostname = \"console.old.example\"\n",
+		"dotted key":          "server.root_domain = \"old.example\"\n",
+		"inline table":        "server = { root_domain = \"old.example\" }\n",
+	} {
+		if out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames)); err == nil {
+			t.Errorf("%s: edited instead of refusing:\n%s", name, out)
+		}
+	}
+}
+
+// caSignedCert is an operator's certificate from their own CA; it names
+// localhost too, so only the issuer tells it apart from the installer's.
+func caSignedCert(t *testing.T, hosts ...string) (certPEM, keyPEM []byte) {
+	t.Helper()
+	caKey, _ := rsa.GenerateKey(rand.Reader, 2048)
+	ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Corp CA"}, IsCA: true,
+		BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
+	caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	caCert, _ := x509.ParseCertificate(caDER)
+	key, _ := rsa.GenerateKey(rand.Reader, 2048)
+	leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: hosts[0]},
+		DNSNames: append(hosts, "localhost"), IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
+		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
+	der, err := x509.CreateCertificate(rand.Reader, leaf, caCert, &key.PublicKey, caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pk, _ := x509.MarshalPKCS8PrivateKey(key)
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pk})
+}
+
+func TestFelisIssuedCert(t *testing.T) {
+	mine, _, err := issuePanelCert(oldNames, nil, time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	c, _ := x509.ParseCertificate(mustCertDER(mine))
+	if !felisIssuedCert(c) {
+		t.Error("the installer's kind of certificate is not recognised as Felis-issued")
+	}
+	theirs, _ := caSignedCert(t, "console.old.example")
+	c, _ = x509.ParseCertificate(mustCertDER(theirs))
+	if felisIssuedCert(c) {
+		t.Error("a CA-signed certificate is taken for Felis-issued")
+	}
+
+	// Self-signed but without one of the installer's localhost names: someone
+	// else's.
+	key, _ := rsa.GenerateKey(rand.Reader, 2048)
+	for name, self := range map[string]*x509.Certificate{
+		"no localhost": {DNSNames: []string{"console.old.example"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}},
+		"no 127.0.0.1": {DNSNames: []string{"console.old.example", "localhost"}},
+	} {
+		self.SerialNumber, self.Subject = big.NewInt(3), pkix.Name{CommonName: "x"}
+		self.NotBefore, self.NotAfter = time.Now().Add(-time.Hour), time.Now().Add(time.Hour)
+		der, _ := x509.CreateCertificate(rand.Reader, self, self, &key.PublicKey, key)
+		c, _ = x509.ParseCertificate(der)
+		if felisIssuedCert(c) {
+			t.Errorf("%s: a self-signed certificate is taken for Felis-issued", name)
+		}
+	}
+}
+
+func TestIssuePanelCertIsTheInstallersShape(t *testing.T) {
+	now := time.Now()
+	certPEM, keyPEM, err := issuePanelCert(newNames, []net.IP{net.ParseIP("10.211.55.6"), net.IPv4(127, 0, 0, 1)}, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil {
+		t.Fatalf("key does not match the certificate: %v", err)
+	}
+	if b, _ := pem.Decode(keyPEM); b == nil || b.Type != "PRIVATE KEY" {
+		t.Fatalf("key is not PKCS#8 PEM like openssl writes")
+	}
+	c, _ := x509.ParseCertificate(mustCertDER(certPEM))
+	if !certCovers(c, newNames.panel, newNames.admin, "localhost") || !felisIssuedCert(c) {
+		t.Fatalf("names %v", c.DNSNames)
+	}
+	if len(c.IPAddresses) != 2 || !c.IPAddresses[1].Equal(net.ParseIP("10.211.55.6")) {
+		t.Fatalf("addresses %v, want 127.0.0.1 and the node address once each", c.IPAddresses)
+	}
+	if c.Subject.CommonName != newNames.admin || c.NotAfter.Sub(now) < 824*24*time.Hour || c.NotAfter.Sub(now) > 826*24*time.Hour {
+		t.Fatalf("CN %q, valid until %s", c.Subject.CommonName, c.NotAfter)
+	}
+	if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth || c.IsCA {
+		t.Fatalf("usage %v, CA %v", c.ExtKeyUsage, c.IsCA)
+	}
+}
+
+func TestDomainSetMovesEverySurface(t *testing.T) {
+	rig := newDomainRig(t)
+	hostBefore := rig.read(t, "felis.host.toml")
+	code, err := rig.h.set(context.Background(), "New.Example", true)
+	if err != nil || code != 0 {
+		t.Fatalf("set = %d, %v\n%s", code, err, rig.out)
+	}
+
+	// The configs: the three values moved, everything else — comments, the
+	// database host of each copy, the Access audience — is as it was.
+	host := rig.read(t, "felis.host.toml")
+	if want := strings.NewReplacer("old.example", "new.example").Replace(hostBefore); host != want {
+		t.Fatalf("host toml:\n%s", host)
+	}
+	pod := rig.read(t, "felis.pod.toml")
+	if got, _ := tomlDomainNames([]byte(pod)); got != newNames || !strings.Contains(pod, "@10.211.55.6:5432") {
+		t.Fatalf("pod toml:\n%s", pod)
+	}
+	if link, err := os.Readlink(rig.path("felis.toml")); err != nil || link != rig.path("felis.host.toml") {
+		t.Fatalf("felis.toml is no longer the link to the host copy: %q %v", link, err)
+	}
+	if st, _ := os.Stat(rig.path("felis.host.toml")); st.Mode().Perm() != 0o600 {
+		t.Fatalf("host toml mode %v", st.Mode().Perm())
+	}
+
+	// The certificate: reissued for the new names, the node address kept, the old
+	// pair beside it.
+	c, err := readCertFile(rig.path("panel-tls.crt"))
+	if err != nil || !certCovers(c, newNames.panel, newNames.admin) || !felisIssuedCert(c) {
+		t.Fatalf("certificate: %v %v", err, c.DNSNames)
+	}
+	if !c.IPAddresses[len(c.IPAddresses)-1].Equal(net.ParseIP("10.211.55.6")) {
+		t.Fatalf("addresses %v", c.IPAddresses)
+	}
+	if _, err := tls.LoadX509KeyPair(rig.path("panel-tls.crt"), rig.path("panel-tls.key")); err != nil {
+		t.Fatalf("new pair: %v", err)
+	}
+	if st, _ := os.Stat(rig.path("panel-tls.key")); st.Mode().Perm() != 0o600 {
+		t.Fatalf("key mode %v", st.Mode().Perm())
+	}
+	backups, _ := filepath.Glob(rig.path("panel-tls.*.pre-domain-*"))
+	if len(backups) != 2 {
+		t.Fatalf("old pair kept as %v", backups)
+	}
+	for _, b := range backups {
+		if st, _ := os.Stat(b); strings.Contains(b, ".key.") && st.Mode().Perm() != 0o600 {
+			t.Fatalf("kept key %s has mode %v", b, st.Mode().Perm())
+		}
+		old, _ := os.ReadFile(b)
+		if strings.Contains(b, ".crt.") {
+			oc, _ := x509.ParseCertificate(mustCertDER(old))
+			if oc == nil || !certCovers(oc, oldNames.panel) {
+				t.Fatalf("kept certificate is not the old one")
+			}
+		}
+	}
+
+	// The Secrets carry the files.
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secret(t, ns, platform.ConfigSecretName)[platform.ConfigSecretKey]; string(got) != pod {
+			t.Fatalf("%s/felis-config is not felis.pod.toml", ns)
+		}
+	}
+	tlsData := rig.secret(t, "felis", platform.APITLSSecretName)
+	if string(tlsData[corev1.TLSCertKey]) != rig.read(t, "panel-tls.crt") || string(tlsData[corev1.TLSPrivateKeyKey]) != rig.read(t, "panel-tls.key") {
+		t.Fatal("felis-api-tls does not hold the new pair")
+	}
+
+	// The login gate's env moved and nothing else did.
+	env := rig.crEnv(t, naming.SystemLoginServer)
+	if env[envRootDomain] != newNames.root || env[envPanelHostname] != newNames.panel || env[envAPIBaseURL] != platform.InternalAPIBaseURL("felis") {
+		t.Fatalf("login env %v", env)
+	}
+
+	// The proxy's file: the three keys moved, its token and mode did not.
+	props := rig.read(t, "felis-link.properties")
+	if want := strings.NewReplacer("old.example", "new.example").Replace(linkPropsBody); props != want {
+		t.Fatalf("felis-link.properties:\n%s", props)
+	}
+	if st, _ := os.Stat(rig.path("felis-link.properties")); st.Mode().Perm() != 0o640 {
+		t.Fatalf("felis-link.properties mode %v", st.Mode().Perm())
+	}
+
+	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+		t.Fatalf("events %v", rig.events)
+	}
+	if !strings.Contains(rig.out.String(), "Every surface is on new.example.") {
+		t.Fatalf("the closing check did not pass:\n%s", rig.out)
+	}
+}
+
+func TestDomainSetWithoutYesChangesNothing(t *testing.T) {
+	rig := newDomainRig(t)
+	before := rig.snapshot(t)
+	code, err := rig.h.set(context.Background(), "new.example", false)
+	if err != nil || code != 0 {
+		t.Fatalf("set = %d, %v", code, err)
+	}
+	if rig.snapshot(t) != before || len(rig.events) != 0 {
+		t.Fatalf("a dry run changed something (events %v)", rig.events)
+	}
+	out := rig.out.String()
+	for _, want := range []string{
+		"console.old.example → console.new.example",
+		"3 passkey(s) of 2 user(s) are bound to console.old.example",
+		"does not cover op.console.new.example",
+		"No [smtp] relay is configured",
+		"sudo felis domain set -yes new.example",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("plan lacks %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestDomainSetKeepsAHandSetPanelHostname(t *testing.T) {
+	rig := newDomainRig(t)
+	for _, f := range []string{"felis.host.toml", "felis.pod.toml"} {
+		writeTestFile(t, rig.path(f), strings.Replace(rig.read(t, f), `panel_hostname = "console.old.example"`, `panel_hostname = "play.corp.net"`, 1), 0o600)
+	}
+	code, err := rig.h.set(context.Background(), "new.example", true)
+	if err != nil {
+		t.Fatalf("set: %v\n%s", err, rig.out)
+	}
+	got, _ := tomlDomainNames([]byte(rig.read(t, "felis.host.toml")))
+	if got != (domainNames{root: "new.example", panel: "play.corp.net", admin: "op.console.new.example"}) {
+		t.Fatalf("names %+v", got)
+	}
+	c, _ := readCertFile(rig.path("panel-tls.crt"))
+	if !certCovers(c, "play.corp.net", "op.console.new.example") {
+		t.Fatalf("certificate names %v", c.DNSNames)
+	}
+	if env := rig.crEnv(t, naming.SystemLoginServer); env[envPanelHostname] != "play.corp.net" {
+		t.Fatalf("login env %v", env)
+	}
+	if code != 0 || !strings.Contains(rig.out.String(), "play.corp.net (set by hand, kept") {
+		t.Fatalf("code %d:\n%s", code, rig.out)
+	}
+}
+
+func TestDomainSetRefusesAnOperatorCertificateForOtherNames(t *testing.T) {
+	rig := newDomainRig(t)
+	certPEM, keyPEM := caSignedCert(t, "console.old.example", "op.console.old.example")
+	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
+	before := rig.snapshot(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "not issued by Felis") {
+		t.Fatalf("err = %v", err)
+	}
+	if rig.snapshot(t) != before || len(rig.events) != 0 {
+		t.Fatal("a refused move changed something")
+	}
+
+	// The operator's certificate for the new names is kept as it is.
+	certPEM, keyPEM = caSignedCert(t, "console.new.example", "op.console.new.example")
+	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatalf("set: %v", err)
+	}
+	if rig.read(t, "panel-tls.crt") != string(certPEM) {
+		t.Fatal("the operator's certificate was replaced")
+	}
+}
+
+func TestDomainSetRefusesAConfigItCannotEdit(t *testing.T) {
+	rig := newDomainRig(t)
+	writeTestFile(t, rig.path("felis.pod.toml"), strings.Replace(rig.read(t, "felis.pod.toml"), "[auth]", "[\"auth\"]", 1), 0o600)
+	before := rig.snapshot(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "by hand") {
+		t.Fatalf("err = %v", err)
+	}
+	if rig.snapshot(t) != before || len(rig.events) != 0 {
+		t.Fatal("a refused move changed something")
+	}
+}
+
+func TestDomainSetAgainOnlyConvergesWhatIsBehind(t *testing.T) {
+	rig := newDomainRig(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	rig.events, rig.out = nil, &bytes.Buffer{}
+	rig.h.out = rig.out
+	before := rig.snapshot(t)
+	code, err := rig.h.set(context.Background(), "new.example", true)
+	if err != nil || code != 0 {
+		t.Fatalf("second set = %d, %v\n%s", code, err, rig.out)
+	}
+	if len(rig.events) != 0 || rig.snapshot(t) != before {
+		t.Fatalf("a converged install was touched again: %v\n%s", rig.events, rig.out)
+	}
+
+	// A proxy that was not restarted after the move is restarted by a re-run, and
+	// an api still on the old config is rolled.
+	rig.proxySince = time.Now().Add(-time.Hour)
+	rig.served = oldNames
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+		t.Fatalf("events %v", rig.events)
+	}
+
+	// An api on the new names that still presents the old certificate is rolled.
+	rig.events = nil
+	oldCert, _, _ := issuePanelCert(oldNames, nil, time.Now())
+	rig.servedCert, _ = x509.ParseCertificate(mustCertDER(oldCert))
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll-api" {
+		t.Fatalf("events %v", rig.events)
+	}
+}
+
+func TestDomainSetRefusesALoginServerItDoesNotOwn(t *testing.T) {
+	rig := newDomainRig(t)
+	var ms v1alpha1.MinecraftServer
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	delete(ms.Labels, v1alpha1.LabelSystemRole)
+	if err := rig.cl.Update(context.Background(), &ms); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "system role") {
+		t.Fatalf("err = %v", err)
+	}
+	if env := rig.crEnv(t, naming.SystemLoginServer); env[envRootDomain] != oldNames.root {
+		t.Fatalf("a server not marked as the login gate was changed: %v", env)
+	}
+}
+
+func TestDomainCheckNamesTheSurfaceThatIsBehind(t *testing.T) {
+	cases := []struct {
+		surface string
+		breakIt func(t *testing.T, rig *domainRig)
+	}{
+		{"felis.pod.toml", func(t *testing.T, rig *domainRig) {
+			writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
+		}},
+		{"Secret minecraft/felis-config", func(t *testing.T, rig *domainRig) {
+			rig.putSecret(t, "minecraft", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
+		}},
+		{"Secret felis/felis-config", func(t *testing.T, rig *domainRig) {
+			rig.putSecret(t, "felis", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
+		}},
+		{"panel certificate", func(t *testing.T, rig *domainRig) {
+			// The Secret follows the file, so only the certificate's names are wrong.
+			certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
+			writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+			rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
+		}},
+		{"Secret felis/felis-api-tls", func(t *testing.T, rig *domainRig) {
+			certPEM, _, _ := issuePanelCert(newNames, nil, time.Now())
+			rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
+		}},
+		{"felis-api", func(t *testing.T, rig *domainRig) { rig.served = oldNames }},
+		{"felis-api", func(t *testing.T, rig *domainRig) {
+			certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
+			rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
+		}},
+		{"proxy", func(t *testing.T, rig *domainRig) {
+			writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
+			rig.proxySince = time.Now().Add(time.Hour)
+		}},
+		{"proxy", func(t *testing.T, rig *domainRig) { rig.proxySince = time.Now().Add(-time.Hour) }},
+		{"login gate", func(t *testing.T, rig *domainRig) {
+			var ms v1alpha1.MinecraftServer
+			_ = rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms)
+			for i := range ms.Spec.Env {
+				if ms.Spec.Env[i].Name == envRootDomain {
+					ms.Spec.Env[i].Value = "old.example"
+				}
+			}
+			if err := rig.cl.Update(context.Background(), &ms); err != nil {
+				t.Fatal(err)
+			}
+		}},
+		{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envRootDomain, "old.example") }},
+		{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envPanelHostname, "console.old.example") }},
+		{"Cloudflare tunnel", func(t *testing.T, rig *domainRig) {
+			writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n  service: https://127.0.0.1:30443\n- hostname: op.console.old.example\n  service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
+		}},
+	}
+	for _, tc := range cases {
+		rig := newDomainRig(t)
+		if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+			t.Fatal(err)
+		}
+		rig.out.Reset()
+		tc.breakIt(t, rig)
+		if code := rig.h.check(context.Background()); code != 1 {
+			t.Errorf("%s behind: check = %d\n%s", tc.surface, code, rig.out)
+			continue
+		}
+		var failed []string
+		for _, ln := range strings.Split(rig.out.String(), "\n") {
+			if strings.HasPrefix(ln, "  FAIL ") {
+				failed = append(failed, ln)
+			}
+		}
+		if len(failed) != 1 || !strings.Contains(failed[0], tc.surface) {
+			t.Errorf("%s behind: FAIL lines %q", tc.surface, failed)
+		}
+	}
+}
+
+func TestDomainCheckPassesAConvergedInstallAndWarnsOnDNS(t *testing.T) {
+	rig := newDomainRig(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n  service: https://127.0.0.1:30443\n- hostname: op.console.new.example\n  service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
+	rig.unresolved["op.console.new.example"] = true
+	rig.unresolved[dnsProbeLabel+".new.example"] = true
+	rig.out.Reset()
+	if code := rig.h.check(context.Background()); code != 0 {
+		t.Fatalf("check = %d\n%s", code, rig.out)
+	}
+	out := rig.out.String()
+	for _, want := range []string{"  ok   Cloudflare tunnel: routes both names", "  warn DNS: op.console.new.example, *.new.example do not resolve from this host\n"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("check lacks %q:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "TOKEN-NOT-TO-TOUCH") {
+		t.Fatal("check printed the proxy's service token")
+	}
+
+	// A zone with only the wildcard: the admin name alone is missing, and why is said.
+	delete(rig.unresolved, dnsProbeLabel+".new.example")
+	rig.out.Reset()
+	rig.h.check(context.Background())
+	if want := "  warn DNS: op.console.new.example does not resolve from this host: the *.new.example wildcard does not cover op.console.new.example, which needs its own record\n"; !strings.Contains(rig.out.String(), want) {
+		t.Errorf("check lacks %q:\n%s", want, rig.out)
+	}
+}
+
+func (rig *domainRig) setPodEnv(t *testing.T, name, value string) {
+	t.Helper()
+	var pod corev1.Pod
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+		t.Fatal(err)
+	}
+	for i, e := range pod.Spec.Containers[0].Env {
+		if e.Name == name {
+			pod.Spec.Containers[0].Env[i].Value = value
+		}
+	}
+	if err := rig.cl.Update(context.Background(), &pod); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (rig *domainRig) putSecret(t *testing.T, ns, name, key, val string) {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		t.Fatal(err)
+	}
+	s.Data[key] = []byte(val)
+	if err := rig.cl.Update(context.Background(), &s); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestParseUnitShow(t *testing.T) {
+	st := parseUnitShow("LoadState=loaded\nActiveState=active\nActiveEnterTimestamp=@1790000000\n")
+	if !st.loaded || !st.active || !st.since.Equal(time.Unix(1790000000, 0)) {
+		t.Fatalf("%+v", st)
+	}
+	st = parseUnitShow("LoadState=not-found\nActiveState=inactive\nActiveEnterTimestamp=\n")
+	if st.loaded || st.active || !st.since.IsZero() {
+		t.Fatalf("%+v", st)
+	}
+}

cmd/felis/egressgate.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"time"
+)
+
+// Vars so tests can shrink them. A dial that neither connects nor is refused
+// within egressDialTimeout counts as blocked: a policy that drops packets looks
+// exactly like that.
+var (
+	egressDialTimeout  = 500 * time.Millisecond
+	egressPollInterval = 200 * time.Millisecond
+)
+
+// cmdEgressGate is the first initContainer of every build pod and the last of
+// every game server pod. A pod's NetworkPolicy is programmed asynchronously after
+// the pod starts (live on k3s: a build-labelled pod reached the internet and the
+// Kubernetes API for its first ~0.7 s, a server-labelled one felis-api's internal
+// face on its first request), so the gate dials a destination the policy denies
+// until it stops answering, and only then lets the pod's next container, the
+// untrusted Dockerfile or server image, start.
+//
+// The default probe is the Kubernetes API Service, which the kubelet names in
+// every pod's environment and neither policy admits. A probe that still answers
+// after --wait means the policy is not enforced at all (a CNI without
+// NetworkPolicy support, or k3s run with --disable-network-policy), and the
+// build fails closed. A server passes --fail-open: an operator's
+// --server-egress-allow-cidr may cover the node the API Service leads to, so a
+// probe that keeps answering does not prove the fence is missing, and by then
+// the policy has had --wait to land.
+func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	probe := fs.String("probe", "", "host:port the pod's NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
+	positive := fs.String("positive-probe", "", "allowed host:port that must remain reachable during denial checks")
+	wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the gate gives up")
+	failOpen := fs.Bool("fail-open", false, "when --wait runs out, warn and let the pod go on instead of refusing it")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *probe == "" {
+		host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
+		if host == "" || port == "" {
+			fmt.Fprintln(stderr, "felis egress-gate: no --probe and no KUBERNETES_SERVICE_HOST/PORT to default to")
+			return 2
+		}
+		*probe = net.JoinHostPort(host, port)
+	}
+
+	start := time.Now()
+	for {
+		if *positive != "" {
+			allowed, err := net.DialTimeout("tcp", *positive, egressDialTimeout)
+			if err != nil {
+				if time.Since(start) >= *wait {
+					fmt.Fprintln(stderr, "felis egress-gate: positive probe unavailable; refusing to start", err)
+					return 1
+				}
+				time.Sleep(egressPollInterval)
+				continue
+			}
+			allowed.Close()
+		}
+		conn, err := net.DialTimeout("tcp", *probe, egressDialTimeout)
+		if err != nil {
+			fmt.Fprintf(stdout, "felis egress-gate: %s is unreachable after %s (%v); the egress lock is in effect\n",
+				*probe, time.Since(start).Round(time.Millisecond), err)
+			return 0
+		}
+		_ = conn.Close()
+		if time.Since(start) >= *wait {
+			if *failOpen {
+				fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s; starting anyway. Either this namespace's "+
+					"NetworkPolicy is not enforced (a CNI without NetworkPolicy support, or k3s started with "+
+					"--disable-network-policy), or an allowed CIDR admits the address behind it\n", *probe, *wait)
+				return 0
+			}
+			fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s: the build namespace's NetworkPolicy is not enforced "+
+				"(a CNI without NetworkPolicy support, or k3s started with --disable-network-policy); refusing to run the build\n",
+				*probe, *wait)
+			return 1
+		}
+		time.Sleep(egressPollInterval)
+	}
+}

cmd/felis/egressgate_test.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"bytes"
+	"net"
+	"strings"
+	"testing"
+	"time"
+)
+
+func shrinkEgressGate(t *testing.T) {
+	t.Helper()
+	dial, poll := egressDialTimeout, egressPollInterval
+	egressDialTimeout, egressPollInterval = 200*time.Millisecond, 10*time.Millisecond
+	t.Cleanup(func() { egressDialTimeout, egressPollInterval = dial, poll })
+}
+
+func TestEgressGateRequiresPositiveReachability(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	closed := ln.Addr().String()
+	ln.Close()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--positive-probe", closed, "--probe", closed, "--wait", "20ms"}, &out, &errb); code != 1 {
+		t.Fatal("unavailable probes allowed startup", code)
+	}
+}
+
+// The gate holds while the probe answers and lets the pod go on once the policy
+// lands, which the test plays by closing the listener.
+func TestEgressGateWaitsForTheLock(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	accepted := make(chan struct{}, 100)
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+			accepted <- struct{}{}
+		}
+	}()
+	go func() {
+		for i := 0; i < 3; i++ {
+			<-accepted
+		}
+		_ = ln.Close()
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "10s"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if !strings.Contains(out.String(), "egress lock is in effect") {
+		t.Errorf("stdout = %q", out.String())
+	}
+}
+
+// A probe that keeps answering means no policy is enforced: the build must not run.
+func TestEgressGateRefusesAnOpenNetwork(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+		}
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "100ms"}, &out, &errb); code != 1 {
+		t.Fatalf("exit %d, want 1", code)
+	}
+	if !strings.Contains(errb.String(), "not enforced") {
+		t.Errorf("stderr = %q", errb.String())
+	}
+}
+
+// A server's gate waits out --wait all the same, then lets the pod start with a
+// warning in its log.
+func TestEgressGateFailOpenWaitsThenWarns(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+		}
+	}()
+	var out, errb bytes.Buffer
+	start := time.Now()
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "150ms", "--fail-open"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d, want 0: %s", code, errb.String())
+	}
+	if waited := time.Since(start); waited < 150*time.Millisecond {
+		t.Errorf("gave up after %s, before --wait ran out", waited)
+	}
+	if !strings.Contains(errb.String(), ln.Addr().String()+" still answers after 150ms; starting anyway") {
+		t.Errorf("stderr = %q", errb.String())
+	}
+	if out.Len() != 0 {
+		t.Errorf("stdout = %q, want nothing: the lock was never seen", out.String())
+	}
+}
+
+func TestEgressGateDefaultsToTheKubernetesService(t *testing.T) {
+	shrinkEgressGate(t)
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	t.Setenv("KUBERNETES_SERVICE_PORT", "")
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate(nil, &out, &errb); code != 2 {
+		t.Fatalf("exit %d without a probe, want 2", code)
+	}
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	host, port, _ := net.SplitHostPort(ln.Addr().String())
+	_ = ln.Close() // closed: the lock reads as in effect at once
+	t.Setenv("KUBERNETES_SERVICE_HOST", host)
+	t.Setenv("KUBERNETES_SERVICE_PORT", port)
+	out.Reset()
+	if code := cmdEgressGate(nil, &out, &errb); code != 0 || !strings.Contains(out.String(), ln.Addr().String()) {
+		t.Fatalf("exit %d, stdout %q", code, out.String())
+	}
+}

cmd/felis/export.go

@@ -0,0 +1,307 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"hash"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/backup"
+	"felis.lolicon.best/internal/fileedit"
+	"felis.lolicon.best/internal/worldexport"
+)
+
+// cmdExport is the in-Pod entrypoint the export Job runs. internal/worldexport
+// renders a Pod whose command is `/usr/local/bin/felis export`. It archives the
+// mounted world, re-streams one archive from the mounted backup store, or sends
+// one file or folder of the world, PUTs it to felis-api's internal face, and
+// exits once felis-api says the owner's browser got all of it. It is NOT a
+// user-facing command and is never invoked by hand.
+//
+// Like cmdRestore it holds no database credentials and never calls config.Load:
+// felis-api made every decision (who may download what, that the server is
+// stopped, which archive) before the Job existed. Its input is the flags below
+// plus the one-time upload token in the environment, which opens this one
+// export and nothing else.
+//
+// Whatever leaves goes through the same guards as the file editor
+// (fileedit.Guard): the proxy forwarding secret, which every server on the
+// install shares, never leaves, and server.properties leaves with its RCON
+// password redacted. A backup is stored with both, since a restore must bring
+// the world back whole, so it is filtered on the way out rather than handed
+// over as stored.
+//
+// Exit status: 0 once felis-api answers 204 (the download completed), 1 when
+// the export could not be read or handed over, a backup failed its digest
+// check, or felis-api refused it (the browser never came or left early), 2 on
+// bad flags. The last stderr line reaches the export's status and the jobs list.
+func cmdExport(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("export", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	mode := fs.String("mode", "", "what to export: world, backup or files")
+	server := fs.String("server", "", "server name being exported (for logging)")
+	target := fs.String("target-url", "", "felis-api URL to PUT the export to")
+	ref := fs.String("ref", "", "backup only: absolute path to the archive on the backup mount")
+	backupRoot := fs.String("backup-root", "/backups", "backup only: mount path of the backup PVC (the ref must resolve under it)")
+	sum := fs.String("sha256", "", "backup only: the sha256 recorded when the archive was written; a mismatch fails the export before its end is sent")
+	worldsRoot := fs.String("worlds-root", "/world", "world and files: mount path of the world PVC")
+	path := fs.String("path", "", "files only: the file or folder to send, relative to the world root")
+	rawArchive := fs.Bool("archive-raw", false, "internal archive transfer: preserve the complete world")
+	dir := fs.Bool("dir", false, "files only: the path is a folder, sent as a zip")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	token := os.Getenv(worldexport.TokenEnv)
+	if *target == "" || token == "" {
+		fmt.Fprintf(stderr, "felis export: --target-url and %s are required\n", worldexport.TokenEnv)
+		return 2
+	}
+	limitHeapToCgroup()
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	var err error
+	switch *mode {
+	case worldexport.ModeBackup:
+		if *ref == "" {
+			fmt.Fprintln(stderr, "felis export: --ref is required for a backup")
+			return 2
+		}
+		err = exportBackup(ctx, *target, token, *ref, *backupRoot, *sum, stdout)
+	case worldexport.ModeWorld:
+		if *rawArchive {
+			err = streamExport(ctx, *target, token, archiveType, -1, func(w io.Writer) error { _, _, err := backup.WriteTarGz(ctx, w, *worldsRoot, nil); return err })
+		} else {
+			err = exportWorld(ctx, *target, token, *worldsRoot, stdout)
+		}
+	case worldexport.ModeFiles:
+		if *path == "" {
+			fmt.Fprintln(stderr, "felis export: --path is required for files")
+			return 2
+		}
+		err = exportFiles(ctx, *target, token, *worldsRoot, *path, *dir, stdout)
+	default:
+		fmt.Fprintf(stderr, "felis export: --mode must be %s, %s or %s\n", worldexport.ModeWorld, worldexport.ModeBackup, worldexport.ModeFiles)
+		return 2
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis export: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis export: server=%s mode=%s downloaded\n", *server, *mode)
+	return 0
+}
+
+// archiveType is the content type of a world or backup export.
+const archiveType = "application/gzip"
+
+// errBackupDigest fails a backup export whose stored archive no longer hashes
+// to what was recorded when it was written.
+var errBackupDigest = errors.New("the backup archive does not match the sha256 recorded when it was written")
+
+// exportBackup re-streams one stored archive through the export guards
+// (backup.FilterTarGz with archiveFilter). Its length changes on the way, so it
+// goes chunked. With want set, the stored bytes are hashed as they are read,
+// and FilterTarGz reads them to their end before it closes its own archive: a
+// mismatch aborts the upload while what felis-api has passed on still lacks
+// its end, so the browser never keeps a complete-looking corrupt file.
+func exportBackup(ctx context.Context, target, token, ref, root, want string, stdout io.Writer) error {
+	// Defense in depth, as in cmdRestore: the ref comes from felis-api, but this
+	// process opens it, so it confirms the ref stays on the backup mount.
+	if !refWithinRoot(ref, root) {
+		return fmt.Errorf("ref %q is not under backup root %q", ref, root)
+	}
+	f, err := os.Open(ref)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	var src io.Reader = f
+	if want != "" {
+		src = &digestReader{r: f, sum: sha256.New(), want: want}
+	}
+	var withheld []string
+	err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
+		var err error
+		withheld, err = backup.FilterTarGz(ctx, w, src, archiveFilter)
+		return err
+	})
+	if errors.Is(err, errBackupDigest) {
+		return errBackupDigest // the jobs list shows it as it is, not wrapped as a read error
+	}
+	reportWithheld(stdout, len(withheld))
+	return err
+}
+
+// exportWorld archives the world straight into the request body: nothing is
+// staged, so a world bigger than the Pod's memory or any scratch disk exports
+// the same.
+func exportWorld(ctx context.Context, target, token, root string, stdout io.Writer) error {
+	r, err := os.OpenRoot(root)
+	if err != nil {
+		return err
+	}
+	guard := fileedit.NewGuard(r)
+	r.Close()
+	var skipped, withheld []string
+	err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
+		var err error
+		skipped, withheld, err = backup.WriteTarGz(ctx, w, root, worldFilter(guard))
+		return err
+	})
+	if len(skipped) > 0 {
+		fmt.Fprintf(stdout, "felis export: left out %d entries a tar cannot hold (symbolic links, devices, sockets)\n", len(skipped))
+	}
+	reportWithheld(stdout, len(withheld))
+	return err
+}
+
+// exportFiles sends one file or folder of the world (fileedit.OpenDownload): a
+// file with its exact length, a folder as a zip made as it streams. dir is
+// what the owner saw at path when they asked.
+func exportFiles(ctx context.Context, target, token, root, path string, dir bool, stdout io.Writer) error {
+	d, err := fileedit.OpenDownload(root, path, dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	err = streamExport(ctx, target, token, d.ContentType, d.Size, func(w io.Writer) error { return d.WriteTo(ctx, w) })
+	if d.Skipped > 0 {
+		fmt.Fprintf(stdout, "felis export: left out %d entries a zip does not carry (symbolic links, devices, sockets)\n", d.Skipped)
+	}
+	reportWithheld(stdout, d.Withheld)
+	return err
+}
+
+func reportWithheld(stdout io.Writer, n int) {
+	if n > 0 {
+		fmt.Fprintf(stdout, "felis export: left out %d files that hold platform secrets\n", n)
+	}
+}
+
+// worldFilter guards a live world by file identity, so a link to a guarded
+// file under another name is caught as well.
+func worldFilter(g fileedit.Guard) backup.Filter {
+	return func(_ string, info fs.FileInfo) (bool, func([]byte) []byte) {
+		return guardAction(g.Rule(info))
+	}
+}
+
+// archiveFilter guards a stored archive, which has only names.
+func archiveFilter(name string, _ fs.FileInfo) (bool, func([]byte) []byte) {
+	return guardAction(fileedit.ArchiveRule(name))
+}
+
+func guardAction(withhold, redact bool) (bool, func([]byte) []byte) {
+	if redact {
+		return withhold, fileedit.RedactProps
+	}
+	return withhold, nil
+}
+
+// digestReader passes r through, hashing it, and turns r's EOF into
+// errBackupDigest when the bytes do not hash to want.
+type digestReader struct {
+	r    io.Reader
+	sum  hash.Hash
+	want string
+}
+
+func (d *digestReader) Read(p []byte) (int, error) {
+	n, err := d.r.Read(p)
+	d.sum.Write(p[:n])
+	if err == io.EOF && !strings.EqualFold(hex.EncodeToString(d.sum.Sum(nil)), d.want) {
+		return n, errBackupDigest
+	}
+	return n, err
+}
+
+// streamExport runs write straight into the body of the PUT, hashing it as it
+// goes. Once write has finished, the SHA-256 of all it wrote rides the
+// request's trailer (worldexport.DigestTrailer), and felis-api holds back the
+// last bytes from the browser until what it received hashes the same. An error
+// from write aborts the chunked body before the trailer, and felis-api then
+// cuts the browser's download off rather than end it; that error is the one
+// reported, since the PUT's own error only wraps it. When the PUT ends first,
+// write is stopped.
+func streamExport(ctx context.Context, target, token, contentType string, size int64, write func(io.Writer) error) error {
+	pr, pw := io.Pipe()
+	trailer := http.Header{worldexport.DigestTrailer: nil}
+	werr := make(chan error, 1)
+	go func() {
+		sum := sha256.New()
+		err := write(io.MultiWriter(pw, sum))
+		if err == nil {
+			// Set before the body ends: the transport reads the trailer once it
+			// has read the body to its end.
+			trailer.Set(worldexport.DigestTrailer, "sha-256=:"+base64.StdEncoding.EncodeToString(sum.Sum(nil))+":")
+		}
+		pw.CloseWithError(err)
+		werr <- err
+	}()
+	err := putExport(ctx, target, token, contentType, pr, size, trailer)
+	pr.CloseWithError(io.ErrClosedPipe)
+	if w := <-werr; w != nil && !errors.Is(w, io.ErrClosedPipe) {
+		return w
+	}
+	return err
+}
+
+// putExport PUTs the export to felis-api. There is no retry: the token opens
+// the export once, so a second attempt could only be refused. Redirects are
+// refused because the request carries the token and the internal face never
+// redirects. felis-api answers only after the whole download, which the Job's
+// activeDeadlineSeconds bounds, so the header timeout is a backstop for a
+// wedged endpoint and not the real limit.
+//
+// The body always goes chunked, which is what lets it end with a trailer; a
+// size the Job knows (-1 when it does not) goes as worldexport.LengthHeader in
+// place of Content-Length.
+func putExport(ctx context.Context, target, token, contentType string, body io.Reader, size int64, trailer http.Header) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body)
+	if err != nil {
+		return err
+	}
+	req.ContentLength = -1
+	req.Trailer = trailer
+	if size >= 0 {
+		req.Header.Set(worldexport.LengthHeader, strconv.FormatInt(size, 10))
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("Content-Type", contentType)
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: 10 * time.Minute},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNoContent {
+		return nil
+	}
+	var e struct {
+		Error struct {
+			Message string `json:"message"`
+		} `json:"error"`
+	}
+	if json.NewDecoder(io.LimitReader(resp.Body, 4<<10)).Decode(&e) == nil && e.Error.Message != "" {
+		return fmt.Errorf("felis-api answered %s: %s", resp.Status, e.Error.Message)
+	}
+	return fmt.Errorf("felis-api answered %s", resp.Status)
+}

cmd/felis/export_test.go

@@ -0,0 +1,532 @@
+package main
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"slices"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/worldexport"
+)
+
+// exportReceiver stands in for felis-api's internal upload route: it records
+// the PUT it gets (or the error reading it ended on) and answers with reply.
+type exportReceiver struct {
+	srv     *httptest.Server
+	hits    atomic.Int32
+	req     *http.Request
+	body    []byte
+	readErr error
+	served  chan struct{} // one send per request, once it is answered
+}
+
+func receiveExport(t *testing.T, reply func(w http.ResponseWriter)) *exportReceiver {
+	t.Helper()
+	rcv := &exportReceiver{served: make(chan struct{}, 1)}
+	rcv.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		rcv.hits.Add(1)
+		rcv.req = r
+		rcv.body, rcv.readErr = io.ReadAll(r.Body)
+		reply(w)
+		rcv.served <- struct{}{}
+	}))
+	t.Cleanup(rcv.srv.Close)
+	return rcv
+}
+
+func noContent(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) }
+
+// sentWhole fails unless the upload rcv got ended with the Content-Digest
+// trailer of its own bytes, and declared length as its size (-1: none).
+func sentWhole(t *testing.T, rcv *exportReceiver, length int64) {
+	t.Helper()
+	sum := sha256.Sum256(rcv.body)
+	want := "sha-256=:" + base64.StdEncoding.EncodeToString(sum[:]) + ":"
+	wantLength := ""
+	if length >= 0 {
+		wantLength = strconv.FormatInt(length, 10)
+	}
+	r := rcv.req
+	if rcv.readErr != nil || r.Trailer.Get(worldexport.DigestTrailer) != want || r.Header.Get(worldexport.LengthHeader) != wantLength ||
+		r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
+		t.Fatalf("upload read %v, trailer %v, %s %q, length %d, encoding %v; want trailer %q and %s %q, chunked",
+			rcv.readErr, r.Trailer, worldexport.LengthHeader, r.Header.Get(worldexport.LengthHeader), r.ContentLength, r.TransferEncoding,
+			want, worldexport.LengthHeader, wantLength)
+	}
+}
+
+func tarEntries(t *testing.T, archive []byte) map[string]string {
+	t.Helper()
+	gz, err := gzip.NewReader(bytes.NewReader(archive))
+	if err != nil {
+		t.Fatalf("not gzip: %v", err)
+	}
+	out := map[string]string{}
+	tr := tar.NewReader(gz)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return out
+		}
+		if err != nil {
+			t.Fatalf("tar: %v", err)
+		}
+		b, _ := io.ReadAll(tr)
+		out[h.Name] = string(b)
+	}
+}
+
+// writeTree writes name → body under root, making the folders on the way.
+func writeTree(t *testing.T, root string, files map[string]string) {
+	t.Helper()
+	for name, body := range files {
+		p := filepath.Join(root, name)
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// The two secrets a world holds, and what server.properties reads as once
+// redacted.
+const (
+	secretProps   = "motd=hi\nrcon.password=hunter2\n"
+	redactedProps = "motd=hi\nrcon.password=<redacted by felis>\n"
+	forwardingKey = "secret: aVeryRealForwardingKey\n"
+)
+
+// secretWorld is a world holding both secrets, with a hard link to the
+// forwarding secret under a name nothing would guard by.
+func secretWorld(t *testing.T) string {
+	t.Helper()
+	root := t.TempDir()
+	writeTree(t, root, map[string]string{
+		"server.properties":       secretProps,
+		"config/paper-global.yml": forwardingKey,
+		"world/region/r.0.0.mca":  "chunks",
+	})
+	if err := os.MkdirAll(filepath.Join(root, "plugins"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(root, "plugins/copy.yml")); err != nil {
+		t.Fatal(err)
+	}
+	return root
+}
+
+func TestCmdExportWorld(t *testing.T) {
+	root := secretWorld(t)
+	if err := os.Symlink("server.properties", filepath.Join(root, "props-link")); err != nil {
+		t.Fatal(err)
+	}
+	rcv := receiveExport(t, noContent)
+	t.Setenv(worldexport.TokenEnv, "tok")
+	var stdout, stderr bytes.Buffer
+	code := cmdExport([]string{"--mode", "world", "--server", "survival", "--target-url", rcv.srv.URL + "/api/v1/internal/exports/ab",
+		"--worlds-root", root}, &stdout, &stderr)
+	if code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, stderr.String())
+	}
+	r := rcv.req
+	if r.Method != http.MethodPut || r.URL.Path != "/api/v1/internal/exports/ab" || r.Header.Get("Authorization") != "Bearer tok" ||
+		r.Header.Get("Content-Type") != "application/gzip" || r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
+		t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
+	}
+	want := map[string]string{
+		"server.properties": redactedProps, "config/": "", "plugins/": "",
+		"world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks",
+	}
+	if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
+		t.Fatalf("archive holds %v\nwant %v", got, want)
+	}
+	sentWhole(t, rcv, -1)
+	want2 := "felis export: left out 1 entries a tar cannot hold (symbolic links, devices, sockets)\n" +
+		"felis export: left out 2 files that hold platform secrets\n" +
+		"felis export: server=survival mode=world downloaded\n"
+	if stdout.String() != want2 {
+		t.Errorf("stdout = %q, want %q", stdout.String(), want2)
+	}
+}
+
+// A world root that cannot be opened fails before anything reaches felis-api.
+func TestCmdExportWorldUnreadable(t *testing.T) {
+	rcv := receiveExport(t, noContent)
+	t.Setenv(worldexport.TokenEnv, "tok")
+	var stdout, stderr bytes.Buffer
+	code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
+	if code != 1 || rcv.hits.Load() != 0 {
+		t.Fatalf("exit %d with %d requests, want 1 and none", code, rcv.hits.Load())
+	}
+}
+
+// An export that fails part-way must never reach felis-api as a complete body:
+// the chunked upload is cut off, so felis-api aborts the browser's download,
+// and the failure itself is what the Job reports.
+func TestStreamExportWriteErrorAbortsTheUpload(t *testing.T) {
+	broken := errors.New("disk read failed")
+	rcv := receiveExport(t, noContent)
+	err := streamExport(context.Background(), rcv.srv.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
+		if _, err := w.Write(bytes.Repeat([]byte("x"), 100_000)); err != nil {
+			return err
+		}
+		return broken
+	})
+	if err != broken {
+		t.Fatalf("err = %v, want the write's own error, unwrapped", err)
+	}
+	select {
+	case <-rcv.served:
+		if rcv.readErr == nil {
+			t.Fatalf("felis-api read a complete %d-byte body from a failed export", len(rcv.body))
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("the request never reached felis-api")
+	}
+}
+
+// When felis-api refuses first, its reason is reported, not the closed pipe
+// that then stops the writer.
+func TestStreamExportRefusalStopsTheWriter(t *testing.T) {
+	refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer refusing.Close()
+	stopped := make(chan error, 1)
+	err := streamExport(context.Background(), refusing.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
+		for {
+			if _, err := w.Write(make([]byte, 32<<10)); err != nil {
+				stopped <- err
+				return err
+			}
+		}
+	})
+	if err == nil || err.Error() != "felis-api answered 404 Not Found" {
+		t.Fatalf("err = %v, want felis-api's answer", err)
+	}
+	if werr := <-stopped; !errors.Is(werr, io.ErrClosedPipe) {
+		t.Fatalf("the writer stopped on %v, want the closed pipe", werr)
+	}
+
+	// A PUT that never starts leaves no transport to close the body: the writer
+	// is still stopped, and the export fails rather than hangs.
+	done := make(chan error, 1)
+	go func() {
+		done <- streamExport(context.Background(), "http://[::1", "tok", "application/gzip", -1, func(w io.Writer) error {
+			_, err := w.Write([]byte("x"))
+			return err
+		})
+	}()
+	select {
+	case err := <-done:
+		if err == nil || !strings.Contains(err.Error(), "missing ']'") {
+			t.Fatalf("err = %v, want the bad URL", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("an export whose PUT never started hung")
+	}
+}
+
+// storedBackup writes, at path, a gzip+tar like one the backup store holds:
+// the world whole, both secrets included, and a region file that does not
+// compress. It returns the archive's sha256.
+func storedBackup(t *testing.T, path string) string {
+	t.Helper()
+	region := make([]byte, 64<<10)
+	if _, err := rand.Read(region); err != nil {
+		t.Fatal(err)
+	}
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zw)
+	for _, e := range []struct{ name, body string }{
+		{"server.properties", secretProps},
+		{"config/paper-global.yml", forwardingKey},
+		{"world/level.dat", "level"},
+		{"world/region/r.0.0.mca", string(region)},
+	} {
+		if err := tw.WriteHeader(&tar.Header{Name: e.name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(e.body))}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := io.WriteString(tw, e.body); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256(buf.Bytes())
+	return hex.EncodeToString(sum[:])
+}
+
+func TestCmdExportBackup(t *testing.T) {
+	root := t.TempDir()
+	ref := filepath.Join(root, "survival-1.tar.gz")
+	sum := storedBackup(t, ref)
+	stored, err := os.ReadFile(ref)
+	if err != nil {
+		t.Fatal(err)
+	}
+	region := tarEntries(t, stored)["world/region/r.0.0.mca"]
+	args := func(url, ref string) []string {
+		return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
+	}
+
+	for name, extra := range map[string][]string{
+		"no digest recorded":      nil,
+		"recorded digest matches": {"--sha256", sum},
+	} {
+		t.Run(name+": re-streamed through the guards", func(t *testing.T) {
+			rcv := receiveExport(t, noContent)
+			t.Setenv(worldexport.TokenEnv, "tok")
+			var stdout, stderr bytes.Buffer
+			if code := cmdExport(append(args(rcv.srv.URL, ref), extra...), &stdout, &stderr); code != 0 {
+				t.Fatalf("exit %d, stderr %q", code, stderr.String())
+			}
+			r := rcv.req
+			if r.ContentLength != -1 || r.Header.Get("Content-Type") != "application/gzip" || r.Header.Get("Authorization") != "Bearer tok" {
+				t.Fatalf("length %d, headers %v", r.ContentLength, r.Header)
+			}
+			want := map[string]string{"server.properties": redactedProps, "world/level.dat": "level", "world/region/r.0.0.mca": region}
+			if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
+				t.Fatalf("archive holds %d entries, want exactly the redacted properties, level.dat and the region file", len(got))
+			}
+			sentWhole(t, rcv, -1)
+			if want := "felis export: left out 1 files that hold platform secrets\nfelis export: server=survival mode=backup downloaded\n"; stdout.String() != want {
+				t.Errorf("stdout = %q, want %q", stdout.String(), want)
+			}
+		})
+	}
+
+	// The stored bytes are checked as they stream, and the archive the Job sends
+	// is only closed once they are all read: a mismatch cuts the upload off
+	// short of its end, so felis-api never passes on a complete-looking copy.
+	t.Run("a digest mismatch cuts the upload off before its end", func(t *testing.T) {
+		rcv := receiveExport(t, noContent)
+		t.Setenv(worldexport.TokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdExport(append(args(rcv.srv.URL, ref), "--sha256", strings.Repeat("ab", 32)), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if want := "felis export: the backup archive does not match the sha256 recorded when it was written\n"; stderr.String() != want {
+			t.Fatalf("stderr = %q, want %q", stderr.String(), want)
+		}
+		select {
+		case <-rcv.served:
+		case <-time.After(5 * time.Second):
+			t.Fatal("the upload never reached felis-api")
+		}
+		if rcv.readErr == nil {
+			t.Fatalf("felis-api read a complete %d-byte body", len(rcv.body))
+		}
+		zr, err := gzip.NewReader(bytes.NewReader(rcv.body))
+		if err == nil {
+			_, err = io.ReadAll(zr)
+		}
+		if err == nil {
+			t.Fatal("what felis-api got is a complete archive")
+		}
+	})
+
+	t.Run("a ref outside the backup root is refused before any request", func(t *testing.T) {
+		outside := filepath.Join(t.TempDir(), "secret.tar.gz")
+		if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		rcv := receiveExport(t, noContent)
+		t.Setenv(worldexport.TokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdExport(args(rcv.srv.URL, outside), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if n := rcv.hits.Load(); n != 0 {
+			t.Fatalf("felis-api got %d requests", n)
+		}
+	})
+
+	t.Run("a refusal exits 1 with felis-api's reason", func(t *testing.T) {
+		rcv := receiveExport(t, func(w http.ResponseWriter) {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusGone)
+			io.WriteString(w, `{"error":{"code":"export_expired","message":"nobody opened the download"}}`)
+		})
+		t.Setenv(worldexport.TokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if got := stderr.String(); got != "felis export: felis-api answered 410 Gone: nobody opened the download\n" {
+			t.Fatalf("stderr = %q", got)
+		}
+	})
+
+	// The request carries the token and the internal face never redirects, so a
+	// redirect is refused rather than followed with the token attached. A 302 or
+	// 303 is the one net/http would follow on its own (as a GET, and to the same
+	// host with the Authorization header still on it).
+	for _, status := range []int{http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
+		t.Run("a redirect is not followed: "+strconv.Itoa(status), func(t *testing.T) {
+			elsewhere := receiveExport(t, noContent)
+			redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.srv.URL, status))
+			defer redirecting.Close()
+			t.Setenv(worldexport.TokenEnv, "tok")
+			var stdout, stderr bytes.Buffer
+			if code := cmdExport(args(redirecting.URL, ref), &stdout, &stderr); code != 1 {
+				t.Fatalf("exit %d, want 1", code)
+			}
+			if n := elsewhere.hits.Load(); n != 0 {
+				t.Fatalf("the redirect target got %d requests", n)
+			}
+			if got, want := stderr.String(), "felis export: felis-api answered "+strconv.Itoa(status)+" "+http.StatusText(status)+"\n"; got != want {
+				t.Fatalf("stderr = %q, want %q", got, want)
+			}
+		})
+	}
+}
+
+func TestCmdExportFiles(t *testing.T) {
+	root := secretWorld(t)
+	writeTree(t, root, map[string]string{"plugins/Essentials/config.yml": "x: 1"})
+	if err := os.Symlink("config.yml", filepath.Join(root, "plugins/Essentials/link.yml")); err != nil {
+		t.Fatal(err)
+	}
+	export := func(t *testing.T, path string, dir bool) (*exportReceiver, int, string, string) {
+		t.Helper()
+		rcv := receiveExport(t, noContent)
+		t.Setenv(worldexport.TokenEnv, "tok")
+		args := []string{"--mode", "files", "--server", "survival", "--target-url", rcv.srv.URL, "--worlds-root", root, "--path", path}
+		if dir {
+			args = append(args, "--dir")
+		}
+		var stdout, stderr bytes.Buffer
+		code := cmdExport(args, &stdout, &stderr)
+		return rcv, code, stdout.String(), stderr.String()
+	}
+
+	for path, want := range map[string]string{
+		"world/region/r.0.0.mca": "chunks",
+		"server.properties":      redactedProps,
+	} {
+		t.Run("a file goes with its exact length: "+path, func(t *testing.T) {
+			rcv, code, stdout, stderr := export(t, path, false)
+			if code != 0 || stdout != "felis export: server=survival mode=files downloaded\n" {
+				t.Fatalf("exit %d, stdout %q, stderr %q", code, stdout, stderr)
+			}
+			if string(rcv.body) != want || rcv.req.Header.Get("Content-Type") != "application/octet-stream" {
+				t.Fatalf("body %q, type %q; want %q", rcv.body, rcv.req.Header.Get("Content-Type"), want)
+			}
+			sentWhole(t, rcv, int64(len(want)))
+		})
+	}
+
+	t.Run("a folder goes as a zip, guarded", func(t *testing.T) {
+		rcv, code, stdout, stderr := export(t, "plugins", true)
+		if code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr)
+		}
+		if rcv.req.ContentLength != -1 || rcv.req.Header.Get("Content-Type") != "application/zip" {
+			t.Fatalf("length %d, type %q", rcv.req.ContentLength, rcv.req.Header.Get("Content-Type"))
+		}
+		zr, err := zip.NewReader(bytes.NewReader(rcv.body), int64(len(rcv.body)))
+		if err != nil {
+			t.Fatal(err)
+		}
+		var names []string
+		for _, f := range zr.File {
+			names = append(names, f.Name)
+		}
+		if want := []string{"plugins/", "plugins/Essentials/", "plugins/Essentials/config.yml"}; !slices.Equal(names, want) {
+			t.Fatalf("zip holds %v, want %v", names, want)
+		}
+		want := "felis export: left out 1 entries a zip does not carry (symbolic links, devices, sockets)\n" +
+			"felis export: left out 1 files that hold platform secrets\n" +
+			"felis export: server=survival mode=files downloaded\n"
+		if stdout != want {
+			t.Errorf("stdout = %q, want %q", stdout, want)
+		}
+	})
+
+	for _, c := range []struct {
+		path string
+		dir  bool
+		want string
+	}{
+		{"config/paper-global.yml", false, "forwarding secret"},
+		{"plugins/copy.yml", false, "forwarding secret"},
+		{"plugins", false, "is a folder now"},
+		{"server.properties", true, "is not a folder now"},
+	} {
+		t.Run("refused before any request: "+c.path, func(t *testing.T) {
+			rcv, code, _, stderr := export(t, c.path, c.dir)
+			if code != 1 || rcv.hits.Load() != 0 || !strings.Contains(stderr, c.want) {
+				t.Fatalf("exit %d, %d requests, stderr %q; want 1, none, and %q", code, rcv.hits.Load(), stderr, c.want)
+			}
+		})
+	}
+}
+
+func TestCmdExportUsage(t *testing.T) {
+	for name, tc := range map[string]struct {
+		token string
+		args  []string
+	}{
+		"no token":           {"", []string{"--mode", "world", "--target-url", "http://api/x"}},
+		"no target":          {"tok", []string{"--mode", "world"}},
+		"unknown mode":       {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
+		"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
+		"files without path": {"tok", []string{"--mode", "files", "--target-url", "http://api/x"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Setenv(worldexport.TokenEnv, tc.token)
+			var stdout, stderr bytes.Buffer
+			if code := cmdExport(tc.args, &stdout, &stderr); code != 2 {
+				t.Fatalf("exit %d, want 2; stderr %q", code, stderr.String())
+			}
+		})
+	}
+}
+
+// TestCmdExportWiring: the Job's `felis export` reaches cmdExport, and
+// felis-api's executor mounts the backup store at the path the archives were
+// written under, since a backup's ref is an absolute path there.
+func TestCmdExportWiring(t *testing.T) {
+	t.Setenv(worldexport.TokenEnv, "")
+	var stdout, stderr bytes.Buffer
+	if code := run([]string{"export"}, &stdout, &stderr); code != 2 ||
+		stderr.String() != "felis export: --target-url and "+worldexport.TokenEnv+" are required\n" {
+		t.Fatalf("felis export = %d, stderr %q", code, stderr.String())
+	}
+	cfg := &config.Config{}
+	cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups"
+	want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"}
+	if got := exportConfig(cfg, "felis:1", "felis-backups"); !reflect.DeepEqual(got, want) {
+		t.Fatalf("exportConfig = %+v, want %+v", got, want)
+	}
+}

cmd/felis/fetchcontext.go

@@ -4,6 +4,8 @@ import (
 	"archive/tar"
 	"compress/gzip"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"flag"
 	"fmt"
@@ -15,6 +17,8 @@ import (
 	"strings"
 	"syscall"
 	"time"
+
+	"felis.lolicon.best/internal/build"
 )
 
 // cmdFetchContext is the in-Pod entrypoint the build Job's context-fetch
@@ -41,9 +45,14 @@ func cmdFetchContext(args []string, _, stderr io.Writer) int {
 	fs.SetOutput(stderr)
 	url := fs.String("url", "", "internal-face URL of the submission's build-context tarball")
 	out := fs.String("out", "/context", "directory to extract the build context into")
+	want := fs.String("sha256", "", "refuse the context unless the tarball's sha256 is this lowercase hex digest")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
+	if *want != "" && !build.IsSHA256Hex(*want) {
+		fmt.Fprintf(stderr, "felis fetch-context: --sha256 %q is not a lowercase hex sha256\n", *want)
+		return 2
+	}
 	if *url == "" {
 		fmt.Fprintln(stderr, "felis fetch-context: --url is required")
 		return 2
@@ -66,7 +75,12 @@ func cmdFetchContext(args []string, _, stderr io.Writer) int {
 	// No overall client timeout: a legitimate modpack context can be large and the
 	// Job's activeDeadlineSeconds is the real bound. The header timeout catches a
 	// wedged endpoint without capping a healthy download.
-	client := &http.Client{Transport: &http.Transport{ResponseHeaderTimeout: time.Minute}}
+	// Redirects are refused: the request carries the service token, and the
+	// internal face never redirects, so a 3xx is someone steering the token.
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: time.Minute},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
 	resp, err := fetchContextWithRetry(ctx, client, *url, token, stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
@@ -74,10 +88,28 @@ func cmdFetchContext(args []string, _, stderr io.Writer) int {
 	}
 	defer resp.Body.Close()
 
-	if err := extractTarGz(resp.Body, *out); err != nil {
+	h := sha256.New()
+	body := io.TeeReader(resp.Body, h)
+	if err := extractTarGz(body, *out); err != nil {
 		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
 		return 1
 	}
+	if *want == "" {
+		return 0
+	}
+	// The tar end marker comes before the gzip trailer and whatever follows it,
+	// so read to EOF: the digest must cover every byte the blob holds. The blob
+	// itself is size-capped at upload, which bounds this read.
+	if _, err := io.Copy(io.Discard, io.LimitReader(body, maxContextBytes)); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	if got := hex.EncodeToString(h.Sum(nil)); got != *want {
+		// The init container failing is what keeps Kaniko from ever starting on
+		// the extracted tree.
+		fmt.Fprintf(stderr, "felis fetch-context: the context's sha256 is %s, the approved digest is %s: it changed after approval; refusing to build\n", got, *want)
+		return 1
+	}
 	return 0
 }
 
@@ -136,13 +168,27 @@ func fetchContextWithRetry(ctx context.Context, client *http.Client, url, token
 	}
 }
 
+// maxContextBytes / maxContextEntries bound what one context may expand to. The
+// compressed upload is capped at 1 GiB, but gzip turns that into hundreds of GiB
+// or millions of empty files, and the emptyDir's 4 GiB sizeLimit is only
+// enforced by the kubelet's periodic sweep, after the disk has filled. The byte
+// cap matches that sizeLimit; the entry cap is far above any real modpack (a
+// large one is a few thousand files) and far below an inode exhaustion.
+//
+// Vars, not consts, so tests can shrink them.
+var (
+	maxContextBytes   int64 = 4 << 30
+	maxContextEntries       = 200_000
+)
+
 // extractTarGz streams a gzip'd tarball into root, creating directories as
 // needed. Every entry is vetted BEFORE anything is written: a path that is
 // absolute or escapes root (via ".."), a link (symlink or hardlink), or any
 // special file kind aborts the whole extraction. Refusing rather than skipping is
 // deliberate — a context that needs one of those constructs is not a context this
 // transport carries, and silently dropping entries would build from a corpus the
-// submitter did not upload.
+// submitter did not upload. The whole extraction is also bounded by
+// maxContextBytes and maxContextEntries.
 func extractTarGz(r io.Reader, root string) error {
 	if err := os.MkdirAll(root, 0o755); err != nil {
 		return fmt.Errorf("create context dir: %w", err)
@@ -153,6 +199,8 @@ func extractTarGz(r io.Reader, root string) error {
 	}
 	defer zr.Close()
 	tr := tar.NewReader(zr)
+	var written int64
+	entries := 0
 	for {
 		hdr, err := tr.Next()
 		if errors.Is(err, io.EOF) {
@@ -161,6 +209,9 @@ func extractTarGz(r io.Reader, root string) error {
 		if err != nil {
 			return fmt.Errorf("read context tarball: %w", err)
 		}
+		if entries++; entries > maxContextEntries {
+			return fmt.Errorf("the build context has more than %d entries", maxContextEntries)
+		}
 		name := filepath.Clean(hdr.Name)
 		if name == "." {
 			continue
@@ -176,7 +227,7 @@ func extractTarGz(r io.Reader, root string) error {
 			if err := os.MkdirAll(target, 0o755); err != nil {
 				return fmt.Errorf("create %q: %w", name, err)
 			}
-		case tar.TypeReg, tar.TypeRegA:
+		case tar.TypeReg:
 			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
 				return fmt.Errorf("create parent of %q: %w", name, err)
 			}
@@ -188,10 +239,16 @@ func extractTarGz(r io.Reader, root string) error {
 			if err != nil {
 				return fmt.Errorf("create %q: %w", name, err)
 			}
-			if _, err := io.Copy(f, tr); err != nil {
+			n, err := io.Copy(f, io.LimitReader(tr, maxContextBytes-written+1))
+			written += n
+			if err != nil {
 				_ = f.Close()
 				return fmt.Errorf("write %q: %w", name, err)
 			}
+			if written > maxContextBytes {
+				_ = f.Close()
+				return fmt.Errorf("the build context expands past %d bytes", maxContextBytes)
+			}
 			if err := f.Close(); err != nil {
 				return fmt.Errorf("close %q: %w", name, err)
 			}

cmd/felis/fetchcontext_test.go

@@ -4,6 +4,8 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
 	"io"
 	"net"
 	"net/http"
@@ -121,6 +123,30 @@ func TestExtractTarGzRefusesEscapes(t *testing.T) {
 	}
 }
 
+// A context that expands past the byte or entry cap is refused, however small
+// it was compressed: gzip bombs and inode floods stop at the cap.
+func TestExtractTarGzCapsExpansion(t *testing.T) {
+	bytesCap, entriesCap := maxContextBytes, maxContextEntries
+	t.Cleanup(func() { maxContextBytes, maxContextEntries = bytesCap, entriesCap })
+	maxContextBytes, maxContextEntries = 1000, 5
+
+	fits := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 400)})
+	if err := extractTarGz(bytes.NewReader(fits), t.TempDir()); err != nil {
+		t.Fatalf("a context exactly at the byte cap: %v", err)
+	}
+	big := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 401)})
+	if err := extractTarGz(bytes.NewReader(big), t.TempDir()); err == nil || !strings.Contains(err.Error(), "expands past") {
+		t.Fatalf("one byte over the cap: err = %v", err)
+	}
+	var many []tarEntry
+	for i := 0; i < 6; i++ {
+		many = append(many, tarEntry{name: "d" + string(rune('0'+i)) + "/", typ: tar.TypeDir})
+	}
+	if err := extractTarGz(bytes.NewReader(tgzBody(t, many...)), t.TempDir()); err == nil || !strings.Contains(err.Error(), "entries") {
+		t.Fatalf("six entries over a cap of five: err = %v", err)
+	}
+}
+
 // The command end to end: it dials the URL with the bearer token from the
 // environment, and refuses to run without it (the internal face would 401
 // anyway; failing at parse time is the honest earlier error).
@@ -163,6 +189,69 @@ func TestCmdFetchContextFetchAndExtract(t *testing.T) {
 	}
 }
 
+// With --sha256 the fetch refuses any bytes but the approved ones, including
+// a tarball that extracts cleanly: that is exactly the context an uploader
+// swapped in after the review.
+func TestCmdFetchContextChecksDigest(t *testing.T) {
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	sum := sha256.Sum256(body)
+	good := hex.EncodeToString(sum[:])
+	args := func(digest string) []string {
+		return []string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir(), "--sha256=" + digest}
+	}
+
+	if code := cmdFetchContext(args(good), io.Discard, io.Discard); code != 0 {
+		t.Fatalf("matching digest exit = %d, want 0", code)
+	}
+	var stderr bytes.Buffer
+	other := strings.Repeat("0", 64)
+	if code := cmdFetchContext(args(other), io.Discard, &stderr); code != 1 || !strings.Contains(stderr.String(), "changed after approval") {
+		t.Fatalf("mismatched digest exit = %d, stderr %q; want 1 naming the change", code, stderr.String())
+	}
+	stderr.Reset()
+	if code := cmdFetchContext(args("ABC"), io.Discard, &stderr); code != 2 {
+		t.Fatalf("malformed digest exit = %d, want 2 (stderr %q)", code, stderr.String())
+	}
+
+	// Bytes after the tar end marker still count: appending to an approved blob
+	// must change what the fetch accepts.
+	padded := append(append([]byte{}, body...), "trailing"...)
+	srvPadded := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(padded)
+	}))
+	defer srvPadded.Close()
+	if code := cmdFetchContext([]string{"--url=" + srvPadded.URL + "/c", "--out=" + t.TempDir(), "--sha256=" + good}, io.Discard, io.Discard); code != 1 {
+		t.Fatalf("padded blob exit = %d, want 1", code)
+	}
+}
+
+// The request carries the service token, so a redirect is a failure: the token
+// never follows it to another host (build-supply-chain-13).
+func TestCmdFetchContextRefusesRedirects(t *testing.T) {
+	var leaked bool
+	elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		leaked = true
+	}))
+	defer elsewhere.Close()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, elsewhere.URL+"/steal", http.StatusFound)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/c", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("redirect exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if leaked {
+		t.Fatal("the fetch followed the redirect")
+	}
+}
+
 // A body that is not a gzip tarball must fail the extraction rather than produce
 // an empty (or partial) context Kaniko would then try to build.
 func TestExtractTarGzRejectsNonGzip(t *testing.T) {

cmd/felis/files.go

@@ -1,11 +1,15 @@
 package main
 
 import (
-	"encoding/base64"
+	"context"
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
 	"felis.lolicon.best/internal/fileedit"
 )
@@ -19,32 +23,55 @@ import (
 // Like cmdRestore it deliberately holds NO database credentials and never calls
 // config.Load: felis-api made the authorization decision (the caller owns this
 // server, and the server is stopped so the RWO world volume is free); this process
-// is the unprivileged hands that touch bytes. Its entire input is the three flags
-// below plus, for a write, one environment variable. Every isolation guarantee
-// lives in the Pod spec (internal/fileedit/jobspec.go), and the path-containment
-// guarantee lives in fileedit.Execute, which resolves the path through os.Root and
-// therefore cannot be walked out of the world mount.
+// is the unprivileged hands that touch bytes. Its entire input is the flags
+// below plus, for a write, the content variables and, for an upload, one token.
+// Every isolation guarantee lives in the Pod spec (internal/fileedit/jobspec.go),
+// and the path-containment guarantee lives in fileedit.Execute, which resolves
+// every path through os.Root and therefore cannot be walked out of the world
+// mount.
 //
 // Exit status carries a specific meaning that felis-api depends on: a CALLER-fault
 // outcome — a path that escapes the root, a file that is missing or too large — is
 // a SUCCESSFUL run that prints a Result carrying an error code, so the API can map
 // it to a precise 4xx. A non-zero exit means the operation could not be attempted
-// at all (the world mount is unreadable, the result unprintable), which the API
-// reports as a 500.
+// at all (the world mount is unreadable, an upload's bytes could not be fetched
+// intact, the result unprintable), which the API reports as a 500.
 func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("files", flag.ContinueOnError)
 	fs.SetOutput(stderr)
-	op := fs.String("op", "", "operation: list, read, or write")
+	op := fs.String("op", "", "operation: list, read, write, mkdir, delete, rename, upload or unzip")
+	browseURL := fs.String("browse-url", "", "internal command channel for a read-only file browser")
 	path := fs.String("path", "", "path to operate on, relative to the world root (empty = the root itself)")
 	worldsRoot := fs.String("worlds-root", "/data", "mount path of the world PVC; every path resolves under it")
+	expect := fs.String("expect-sha256", "", "write only: refuse unless the file's current SHA-256 (hex) is this")
+	createOnly := fs.Bool("create-only", false, "write only: refuse a path that already exists")
+	to := fs.String("to", "", "rename only: the destination path")
+	sourceURL := fs.String("source-url", "", "upload only: felis-api URL to fetch the bytes from")
+	size := fs.Int64("size", -1, "upload only: the byte count the fetched file must have")
+	sum := fs.String("sha256", "", "write and upload: the SHA-256 (hex) the content or the fetched file must have")
+	overwrite := fs.Bool("overwrite", false, "upload and unzip: replace files already there")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
 
+	if *browseURL != "" {
+		limitHeapToCgroup()
+		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer stop()
+		if err := fileedit.Browse(ctx, *worldsRoot, *browseURL, os.Getenv(fileedit.BrowserTokenEnv)); err != nil {
+			fmt.Fprintf(stderr, "felis files: %v\n", err)
+			return 1
+		}
+		return 0
+	}
 	if *op == "" {
-		fmt.Fprintln(stderr, "felis files: --op is required (list, read, or write)")
+		fmt.Fprintln(stderr, "felis files: --op is required")
 		return 2
 	}
+	limitHeapToCgroup()
+	req := fileedit.Request{
+		Op: *op, Path: *path, To: *to, Expect: *expect, CreateOnly: *createOnly, Overwrite: *overwrite,
+	}
 
 	// New content arrives base64-encoded in the environment rather than in argv:
 	// a process's arguments are world-readable on the node (/proc/<pid>/cmdline),
@@ -52,22 +79,46 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	// secrets — an RCON password in server.properties is the obvious case. The
 	// encoding is what lets arbitrary bytes (CRLF endings, a BOM, a NUL) survive a
 	// channel that must be a valid string.
-	var content []byte
-	if *op == fileedit.OpWrite {
-		raw, ok := os.LookupEnv(fileedit.ContentEnv)
-		if !ok {
-			fmt.Fprintf(stderr, "felis files: a write needs %s in the environment\n", fileedit.ContentEnv)
+	switch *op {
+	case fileedit.OpWrite:
+		// The content's SHA-256 comes with it, so bytes that changed on the way
+		// to this Job are refused rather than written (Request.ContentSHA256).
+		if *sum == "" {
+			fmt.Fprintln(stderr, "felis files: a write needs --sha256")
 			return 2
 		}
-		decoded, err := base64.StdEncoding.DecodeString(raw)
+		content, err := fileedit.ContentFromEnv(os.LookupEnv)
 		if err != nil {
-			fmt.Fprintf(stderr, "felis files: %s is not valid base64: %v\n", fileedit.ContentEnv, err)
+			fmt.Fprintf(stderr, "felis files: %v\n", err)
 			return 2
 		}
-		content = decoded
+		req.Content, req.ContentSHA256 = content, *sum
+	case fileedit.OpUpload:
+		token := os.Getenv(fileedit.UploadTokenEnv)
+		if *sourceURL == "" || token == "" {
+			fmt.Fprintf(stderr, "felis files: an upload needs --source-url and %s\n", fileedit.UploadTokenEnv)
+			return 2
+		}
+		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer stop()
+		req.Upload = &fileedit.Upload{
+			Size: *size, SHA256: *sum,
+			Open: func() (io.ReadCloser, error) { return fetchUpload(ctx, *sourceURL, token) },
+			Landed: func() {
+				if err := reportLanded(ctx, *sourceURL, token); err != nil {
+					// The file is in place; felis-api drops its copy when it
+					// has sat idle long enough, and the panel cancels it too.
+					fmt.Fprintf(stderr, "felis files: tell felis-api the upload landed: %v\n", err)
+				}
+			},
+		}
 	}
+	// An upload or an unzip (the only ops that report progress) can run long
+	// enough that felis-api does not wait on its Job, and the panel shows how far
+	// it has got from the latest of these lines (fileedit.K8sRunner.Ops).
+	req.Progress = fileedit.ThrottledProgress(stdout, time.Second, time.Now)
 
-	res, err := fileedit.Execute(*worldsRoot, *op, *path, content)
+	res, err := fileedit.Execute(*worldsRoot, req)
 	if err != nil {
 		// The operation could not be attempted — infrastructure, not caller fault.
 		fmt.Fprintf(stderr, "felis files: %v\n", err)
@@ -82,3 +133,57 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// fetchUpload opens the staged upload on felis-api's internal face. There is no
+// retry: the token opens the upload once (fileedit.Stage), so a second attempt
+// could only be refused, and the caller retries the failed Job whole (a file
+// sent in parts stays staged until its Job reports it landed, so that retry
+// does not send it again). Redirects are refused because the request carries the
+// token and the internal face never redirects; the header timeout catches a
+// wedged endpoint, and the Job's activeDeadlineSeconds bounds the body.
+func fetchUpload(ctx context.Context, url, token string) (io.ReadCloser, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: 30 * time.Second},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, fmt.Errorf("GET returned %s", resp.Status)
+	}
+	return resp.Body, nil
+}
+
+// reportLanded tells felis-api the upload's file is in place (DELETE on the URL
+// it was fetched from, with the same token), so it deletes the copy it staged.
+// One try: the file has landed whatever the answer, and a copy nobody deletes
+// is dropped once it has sat idle for fileedit.SessionIdle.
+func reportLanded(ctx context.Context, url, token string) error {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	client := &http.Client{
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		return fmt.Errorf("DELETE returned %s", resp.Status)
+	}
+	return nil
+}

cmd/felis/files_test.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"cmp"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"felis.lolicon.best/internal/fileedit"
+)
+
+// filesResult is the Result a `felis files` run printed on its marked line,
+// the last it prints.
+func filesResult(t *testing.T, stdout string) fileedit.Result {
+	t.Helper()
+	lines := strings.Split(strings.TrimSpace(stdout), "\n")
+	line, ok := strings.CutPrefix(lines[len(lines)-1], fileedit.ResultPrefix)
+	if !ok {
+		t.Fatalf("stdout has no result line: %q", stdout)
+	}
+	var res fileedit.Result
+	if err := json.Unmarshal([]byte(line), &res); err != nil {
+		t.Fatalf("result line %q: %v", line, err)
+	}
+	return res
+}
+
+// stagedSource is felis-api's internal face for one staged upload. It serves
+// body to a GET carrying Bearer token and 404 to any other, and answers the
+// DELETE that reports the file landed with landedCode (204 when unset),
+// redirecting to landedTo when that is a redirect. reports counts those
+// DELETEs, each with the token and at the path the bytes came from.
+type stagedSource struct {
+	*httptest.Server
+	reports, strays atomic.Int32
+	landedCode      int
+	landedTo        string
+}
+
+func stagedUpload(t *testing.T, token string, body []byte) *stagedSource {
+	t.Helper()
+	s := &stagedSource{}
+	s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer "+token || r.URL.Path != "/u" {
+			s.strays.Add(1)
+			http.Error(w, "no such upload", http.StatusNotFound)
+			return
+		}
+		switch r.Method {
+		case http.MethodGet:
+			w.Write(body)
+		case http.MethodDelete:
+			s.reports.Add(1)
+			if s.landedTo != "" {
+				w.Header().Set("Location", s.landedTo)
+			}
+			w.WriteHeader(cmp.Or(s.landedCode, http.StatusNoContent))
+		default:
+			s.strays.Add(1)
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		}
+	}))
+	t.Cleanup(s.Close)
+	return s
+}
+
+func uploadArgs(root, sourceURL string, body []byte) []string {
+	sum := sha256.Sum256(body)
+	return []string{
+		"--op", "upload", "--path", "plugins/a.jar", "--worlds-root", root,
+		"--source-url", sourceURL, "--size", "4", "--sha256", hex.EncodeToString(sum[:]),
+	}
+}
+
+// uploadRoot is a world with the plugins folder an upload lands in.
+func uploadRoot(t *testing.T) string {
+	t.Helper()
+	root := t.TempDir()
+	if err := os.Mkdir(filepath.Join(root, "plugins"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	return root
+}
+
+func TestCmdFilesUpload(t *testing.T) {
+	body := []byte("PK\x03\x04")
+
+	t.Run("fetches the staged bytes with its token and lands them", func(t *testing.T) {
+		root := uploadRoot(t)
+		srv := stagedUpload(t, "tok", body)
+		t.Setenv(fileedit.UploadTokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix+`{"done":4,"total":4}`+"\n") {
+			t.Fatalf("stdout %q does not start with the progress to the last byte", stdout.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != "" {
+			t.Fatalf("result = %+v", res)
+		}
+		got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar"))
+		if err != nil || !bytes.Equal(got, body) {
+			t.Fatalf("landed %q, %v", got, err)
+		}
+		if n, strays := srv.reports.Load(), srv.strays.Load(); n != 1 || strays != 0 || stderr.Len() != 0 {
+			t.Fatalf("reported landed %d times, %d stray requests, stderr %q; want once", n, strays, stderr.String())
+		}
+	})
+
+	t.Run("a file already there is a result, and nothing is reported landed", func(t *testing.T) {
+		root := uploadRoot(t)
+		if err := os.WriteFile(filepath.Join(root, "plugins", "a.jar"), []byte("old!"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		srv := stagedUpload(t, "tok", body)
+		t.Setenv(fileedit.UploadTokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeExists || srv.reports.Load() != 0 {
+			t.Fatalf("result = %+v, reported landed %d times", res, srv.reports.Load())
+		}
+	})
+
+	// The file is in place whatever felis-api answers, so the Job still succeeds
+	// and says why the staged copy may linger. A redirect is not followed, since
+	// the request carries the token.
+	for name, tc := range map[string]struct {
+		code   int
+		stderr string
+	}{
+		"refused":    {http.StatusNotFound, "felis files: tell felis-api the upload landed: DELETE returned 404 Not Found\n"},
+		"redirected": {http.StatusFound, "felis files: tell felis-api the upload landed: DELETE returned 302 Found\n"},
+	} {
+		t.Run("a landed report "+name+" still lands the file", func(t *testing.T) {
+			var elsewhere atomic.Int32
+			away := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { elsewhere.Add(1) }))
+			defer away.Close()
+			root := uploadRoot(t)
+			srv := stagedUpload(t, "tok", body)
+			srv.landedCode, srv.landedTo = tc.code, away.URL+"/u"
+			t.Setenv(fileedit.UploadTokenEnv, "tok")
+			var stdout, stderr bytes.Buffer
+			if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
+				t.Fatalf("exit %d, stderr %q", code, stderr.String())
+			}
+			if res := filesResult(t, stdout.String()); res.Code != "" || stderr.String() != tc.stderr || elsewhere.Load() != 0 {
+				t.Fatalf("result = %+v, stderr %q, redirect followed %d times", res, stderr.String(), elsewhere.Load())
+			}
+			if got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar")); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("landed %q, %v", got, err)
+			}
+		})
+	}
+
+	// A refused fetch is the Job failing, never a Result: the API answers it with a
+	// 500 the caller retries whole.
+	t.Run("a refused fetch exits 1 and lands nothing", func(t *testing.T) {
+		root := uploadRoot(t)
+		srv := stagedUpload(t, "tok", body)
+		t.Setenv(fileedit.UploadTokenEnv, "wrong")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1; stdout %q", code, stdout.String())
+		}
+		if !strings.Contains(stderr.String(), "404") {
+			t.Fatalf("stderr %q does not name the status", stderr.String())
+		}
+		if srv.reports.Load() != 0 {
+			t.Fatal("a refused fetch was reported landed")
+		}
+		if _, err := os.Lstat(filepath.Join(root, "plugins", "a.jar")); !os.IsNotExist(err) {
+			t.Fatalf("a refused fetch left a file: %v", err)
+		}
+	})
+
+	// The request carries the token, and the internal face never redirects, so a
+	// redirect is refused rather than followed with the token attached.
+	t.Run("a redirect is not followed", func(t *testing.T) {
+		root := uploadRoot(t)
+		var hits atomic.Int32
+		elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			w.Write(body)
+		}))
+		defer elsewhere.Close()
+		redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.URL+"/u", http.StatusFound))
+		defer redirecting.Close()
+		t.Setenv(fileedit.UploadTokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, redirecting.URL+"/u", body), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if n := hits.Load(); n != 0 {
+			t.Fatalf("the redirect target was fetched %d times", n)
+		}
+	})
+
+	for name, tc := range map[string]struct {
+		token string
+		drop  string
+	}{
+		"no token":      {"", ""},
+		"no source URL": {"tok", "--source-url"},
+	} {
+		t.Run(name+" exits 2", func(t *testing.T) {
+			srv := stagedUpload(t, "tok", body)
+			t.Setenv(fileedit.UploadTokenEnv, tc.token)
+			args := uploadArgs(uploadRoot(t), srv.URL+"/u", body)
+			if tc.drop != "" {
+				for i, a := range args {
+					if a == tc.drop {
+						args = append(args[:i:i], args[i+2:]...)
+						break
+					}
+				}
+			}
+			var stdout, stderr bytes.Buffer
+			if code := cmdFiles(args, &stdout, &stderr); code != 2 {
+				t.Fatalf("exit %d, want 2", code)
+			}
+		})
+	}
+}
+
+func TestCmdFilesWrite(t *testing.T) {
+	root := t.TempDir()
+	content := []byte("[]\r\n")
+	sum := sha256.Sum256(content)
+	args := []string{"--op", "write", "--path", "ops.json", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}
+
+	t.Run("reassembles the content parts", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "1")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(args, &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != "" {
+			t.Fatalf("result = %+v", res)
+		}
+		if got, err := os.ReadFile(filepath.Join(root, "ops.json")); err != nil || !bytes.Equal(got, content) {
+			t.Fatalf("wrote %q, %v", got, err)
+		}
+	})
+
+	// Writing what did arrive of an incomplete spec would truncate the file.
+	t.Run("an incomplete content spec exits 2 and writes nothing", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "2")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("x")))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 2 {
+			t.Fatalf("exit %d, want 2", code)
+		}
+		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
+			t.Fatalf("an incomplete spec wrote a file: %v", err)
+		}
+	})
+
+	// Without the content's SHA-256 the Job could not tell bytes changed on the
+	// way from the bytes felis-api sent.
+	t.Run("a write without its SHA-256 exits 2 and writes nothing", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "1")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root}, &stdout, &stderr); code != 2 {
+			t.Fatalf("exit %d, want 2", code)
+		}
+		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
+			t.Fatalf("a write without its SHA-256 wrote a file: %v", err)
+		}
+	})
+
+	t.Run("content that changed on the way is a result and writes nothing", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "1")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("[]\n")))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeDigestMismatch {
+			t.Fatalf("result = %+v, want %s", res, fileedit.CodeDigestMismatch)
+		}
+		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
+			t.Fatalf("changed content wrote a file: %v", err)
+		}
+	})
+}
+
+// A caller-fault outcome is a successful run carrying a code, so felis-api can
+// answer the precise 4xx instead of a 500.
+func TestCmdFilesCallerFaultIsAResult(t *testing.T) {
+	root := t.TempDir()
+	var stdout, stderr bytes.Buffer
+	if code := cmdFiles([]string{"--op", "mkdir", "--path", "../out", "--worlds-root", root}, &stdout, &stderr); code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, stderr.String())
+	}
+	if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeBadPath {
+		t.Fatalf("result = %+v, want code %s", res, fileedit.CodeBadPath)
+	}
+	stdout.Reset()
+	if code := cmdFiles([]string{"--worlds-root", root}, &stdout, &stderr); code != 2 {
+		t.Fatalf("no --op: exit %d, want 2", code)
+	}
+}
+
+// TestCmdFilesUnzip checks an unzip extracts next to the archive and reports its
+// progress before its result, the same way an upload does.
+func TestCmdFilesUnzip(t *testing.T) {
+	root := t.TempDir()
+	if err := os.Mkdir(filepath.Join(root, "maps"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	var zb bytes.Buffer
+	zw := zip.NewWriter(&zb)
+	for name, body := range map[string]string{"world/level.dat": "level", "world/region/r.0.0.mca": "region!"} {
+		w, err := zw.Create(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		io.WriteString(w, body)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "maps", "a.zip"), zb.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var stdout, stderr bytes.Buffer
+	if code := cmdFiles([]string{"--op", "unzip", "--path", "maps/a.zip", "--worlds-root", root}, &stdout, &stderr); code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, stderr.String())
+	}
+	if res := filesResult(t, stdout.String()); res.Code != "" || res.Files != 2 || res.Bytes != 12 {
+		t.Fatalf("result = %+v", res)
+	}
+	if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix) ||
+		!strings.Contains(stdout.String(), fileedit.ProgressPrefix+`{"done":12,"total":12}`+"\n") {
+		t.Fatalf("stdout %q does not report the progress to the last byte", stdout.String())
+	}
+	got, err := os.ReadFile(filepath.Join(root, "maps", "world", "region", "r.0.0.mca"))
+	if err != nil || string(got) != "region!" {
+		t.Fatalf("extracted %q, %v", got, err)
+	}
+}

cmd/felis/hostcreds.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// Host copies of the credentials `felis setup` takes at the keyboard: the [smtp]
+// relay password and the uploads bucket's keys. The cluster reads them from the
+// felis-smtp and felis-uploads-s3 Secrets, and a Secret lives in k3s's datastore,
+// which a reinstall (uninstall.sh keeps /etc/felis) or a host rebuilt from a
+// database bundle's state/ starts empty. Each file holds the bare value, mode
+// 0600, directly in /etc/felis beside secrets.env: every installer run applies
+// the Secrets from these files, and every database bundle, so the off-site copy
+// too, carries them.
+const (
+	hostSMTPPasswordPath       = "/etc/felis/smtp-password"
+	hostUploadsS3AccessKeyPath = "/etc/felis/uploads-s3-access-key"
+	hostUploadsS3SecretKeyPath = "/etc/felis/uploads-s3-secret-key"
+)
+
+// writeHostCredential replaces the file at path with value, mode 0600, through a
+// temporary file in the same directory, so a crash leaves the old value or the
+// new one and never a partial one.
+func writeHostCredential(path, value string) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	defer os.Remove(tmpPath)
+	// CreateTemp already makes the file 0600; the Chmod states it rather than
+	// leaning on that.
+	if err := tmp.Chmod(0o600); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if _, err := tmp.WriteString(value); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpPath, path)
+}
+
+// readHostCredential returns the value in path; ok is false when there is no
+// such file. An empty file is a value: the relay password of a relay without AUTH.
+func readHostCredential(path string) (value string, ok bool, err error) {
+	b, err := os.ReadFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return string(b), true, nil
+}
+
+// relayPassword is the [smtp] relay password as the host holds it: the copy
+// `felis setup` keeps at path, else, on an install from before that copy, the
+// felis-smtp Secret in ns, whose absence means a relay without AUTH. cl is only
+// used when the file is missing; a nil cl then reports errClusterUnreachable.
+func relayPassword(ctx context.Context, path string, cl client.Client, ns string) (string, error) {
+	if pw, ok, err := readHostCredential(path); err != nil || ok {
+		return pw, err
+	}
+	if cl == nil {
+		return "", errClusterUnreachable
+	}
+	return smtpSecretPassword(ctx, cl, ns)
+}
+
+var errClusterUnreachable = errors.New("the cluster did not answer")

cmd/felis/hostcreds_test.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/watchdog"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func TestHostCredentialFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "smtp-password")
+
+	if _, ok, err := readHostCredential(path); ok || err != nil {
+		t.Fatalf("a missing file read as ok=%v err=%v; want not there", ok, err)
+	}
+	// A copy an operator put there by hand, readable by everyone, is tightened.
+	if err := os.WriteFile(path, []byte("by hand"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range []string{"first secret", "a \"quoted\" $second\nsecret", ""} {
+		if err := writeHostCredential(path, v); err != nil {
+			t.Fatal(err)
+		}
+		got, ok, err := readHostCredential(path)
+		if err != nil || !ok || got != v {
+			t.Fatalf("read back (%q, %v, %v); want (%q, true, nil)", got, ok, err, v)
+		}
+		fi, err := os.Stat(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if mode := fi.Mode().Perm(); mode != 0o600 {
+			t.Fatalf("mode %v; want 0600", mode)
+		}
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 1 {
+		t.Fatalf("the directory holds %d entries; want the credential alone, no leftover temporary file", len(entries))
+	}
+
+	if _, _, err := readHostCredential(dir); err == nil {
+		t.Fatal("an unreadable credential read as fine")
+	}
+}
+
+func smtpSecretClient(t *testing.T, password string) client.Client {
+	t.Helper()
+	return fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
+		Data:       map[string][]byte{platform.SMTPSecretPasswordKey: []byte(password)},
+	}).Build()
+}
+
+func TestRelayPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	cl := smtpSecretClient(t, "from-secret")
+
+	if pw, err := relayPassword(ctx, host, cl, "felis"); err != nil || pw != "from-secret" {
+		t.Fatalf("without a host copy = (%q, %v); want the Secret's", pw, err)
+	}
+	if _, err := relayPassword(ctx, host, nil, "felis"); !errors.Is(err, errClusterUnreachable) {
+		t.Fatalf("without a host copy or a cluster err = %v; want errClusterUnreachable", err)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []client.Client{cl, nil} {
+		if pw, err := relayPassword(ctx, host, c, "felis"); err != nil || pw != "from-host" {
+			t.Fatalf("with a host copy (cluster %v) = (%q, %v); want the host copy", c != nil, pw, err)
+		}
+	}
+	if _, err := relayPassword(ctx, dir, cl, "felis"); err == nil {
+		t.Fatal("an unreadable host copy fell through to the Secret")
+	}
+}
+
+// The watchdog mails the most while the cluster is down: the host copy must
+// reach it then, and without one the password the last good run cached stays.
+func TestRefreshSMTPPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	var stderr bytes.Buffer
+
+	state := &watchdog.State{SMTPPassword: "cached"}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "cached" || stderr.Len() != 0 {
+		t.Fatalf("cluster down, no host copy: password %q, stderr %q; want the cached one kept quietly", state.SMTPPassword, stderr.String())
+	}
+	refreshSMTPPassword(ctx, host, smtpSecretClient(t, "from-secret"), "felis", state, &stderr)
+	if state.SMTPPassword != "from-secret" {
+		t.Fatalf("cluster up, no host copy: password %q; want the Secret's", state.SMTPPassword)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cluster down, host copy: password %q; want the host copy", state.SMTPPassword)
+	}
+	refreshSMTPPassword(ctx, dir, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" || !strings.Contains(stderr.String(), "keeping the cached one") {
+		t.Fatalf("unreadable host copy: password %q, stderr %q; want the cached one kept and the failure said", state.SMTPPassword, stderr.String())
+	}
+}
+
+func TestHostRecoveryMailerHostCopy(t *testing.T) {
+	ctx := context.Background()
+	// No kubeconfig anywhere: reaching for the cluster fails, so a pass proves
+	// the host copy was enough.
+	t.Setenv("KUBECONFIG", filepath.Join(t.TempDir(), "no-kubeconfig"))
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	off := false
+	c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_UNSET_RELAY_PW", RequireTLS: &off}
+
+	got, err := hostRecoveryMailer(c, host, "felis")(ctx)
+	if err != nil {
+		t.Fatalf("with the host copy: %v", err)
+	}
+	if relay, ok := got.(*mail.SMTP); !ok || relay.Password != "from-host" {
+		t.Fatalf("relay = %#v; want the host copy's password", got)
+	}
+	if _, err := hostRecoveryMailer(c, dir, "felis")(ctx); err == nil || !strings.Contains(err.Error(), "read the relay password") {
+		t.Fatalf("unreadable host copy: err = %v; want it named", err)
+	}
+	if _, err := hostRecoveryMailer(c, filepath.Join(dir, "none"), "felis")(ctx); err == nil || !strings.Contains(err.Error(), "reach the cluster") {
+		t.Fatalf("no host copy and no cluster: err = %v; want the cluster named", err)
+	}
+}
+
+// A whole watchdog run with the API server and PostgreSQL both down still
+// takes the relay password from the host copy, so the outage mail can
+// authenticate even when no earlier run cached it.
+func TestWatchdogReadsHostCopyWhileClusterDown(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
+	cfgPath := filepath.Join(dir, "felis.toml")
+	if err := os.WriteFile(cfgPath, []byte(`[database]
+url = "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
+[server]
+root_domain = "example.com"
+[archive]
+store = "tarLocal"
+[k8s]
+egress_mode = "nodeport"
+[smtp]
+host = "127.0.0.1"
+port = 1
+from = "[email protected]"
+username = "felis"
+password_ref = "FELIS_TEST_UNSET_RELAY_PW"
+`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	pwPath := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(pwPath, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	statePath := filepath.Join(dir, "state.json")
+	var stdout, stderr bytes.Buffer
+	cmdWatchdog([]string{
+		"-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"),
+		"-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", pwPath, "-heartbeat-file", filepath.Join(dir, "no-heartbeat"),
+	}, &stdout, &stderr)
+	if !strings.Contains(stdout.String(), "kube-api") {
+		t.Fatalf("the run found the API server up; the test needs it down (stdout %s)", stdout.String())
+	}
+	state, err := watchdog.LoadState(statePath)
+	if err != nil {
+		t.Fatalf("load state: %v (stderr %s)", err, stderr.String())
+	}
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cached relay password %q; want the host copy (stdout %s, stderr %s)", state.SMTPPassword, stdout.String(), stderr.String())
+	}
+}

cmd/felis/imagebundle.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"felis.lolicon.best/internal/imagepush"
+)
+
+// cmdImageBundle writes a release's image bundle: one OCI layout tar holding every
+// image an install runs, for one platform, plus its listing (one "role name
+// manifest-digest config-digest" line per image). deploy/build-release-artifacts.sh
+// runs it in CI; deploy/bootstrap.sh imports the tar into k3s's containerd and
+// pushes it into the platform registry with push-image --image.
+//
+//	--layout role=name=path   an image buildx wrote with --output type=oci
+//	--pull role=ref           a digest-pinned public image, named repository@digest
+//
+// The tar and the listing are written beside their final paths and renamed into
+// place, so a failed run leaves neither behind.
+func cmdImageBundle(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("image-bundle", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	platform := fs.String("platform", "", "os/arch the bundle is for, e.g. linux/arm64")
+	out := fs.String("out", "", "path of the bundle tar to write")
+	list := fs.String("list", "", "path of the listing to write")
+	var images []imagepush.BundleImage
+	fs.Func("layout", "role=name=path of an OCI layout tar (repeatable)", func(v string) error {
+		role, rest, ok := strings.Cut(v, "=")
+		name, path, ok2 := strings.Cut(rest, "=")
+		if !ok || !ok2 || role == "" || name == "" || path == "" {
+			return fmt.Errorf("want role=name=path, got %q", v)
+		}
+		images = append(images, imagepush.BundleImage{Role: role, Name: name, Layout: path})
+		return nil
+	})
+	fs.Func("pull", "role=ref of a digest-pinned public image (repeatable)", func(v string) error {
+		role, ref, ok := strings.Cut(v, "=")
+		if !ok || role == "" || !strings.Contains(ref, "@sha256:") {
+			return fmt.Errorf("want role=ref with ref pinned by digest, got %q", v)
+		}
+		images = append(images, imagepush.BundleImage{Role: role, Name: imagepush.PinnedName(ref), Source: ref})
+		return nil
+	})
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if *platform == "" || *out == "" || *list == "" || len(images) == 0 {
+		fmt.Fprintln(stderr, "felis image-bundle: --platform, --out, --list and at least one --layout or --pull are required")
+		return 2
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	if err := writeImageBundle(ctx, &imagepush.Source{Platform: *platform}, images, *out, *list); err != nil {
+		fmt.Fprintf(stderr, "felis image-bundle: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+func writeImageBundle(ctx context.Context, s *imagepush.Source, images []imagepush.BundleImage, out, list string) (err error) {
+	tmpOut, tmpList := out+".tmp", list+".tmp"
+	defer func() {
+		if err != nil {
+			os.Remove(tmpOut)
+			os.Remove(tmpList)
+		}
+	}()
+	f, err := os.Create(tmpOut)
+	if err != nil {
+		return err
+	}
+	w := bufio.NewWriterSize(f, 1<<20)
+	entries, err := imagepush.WriteBundle(ctx, s, images, w)
+	if err == nil {
+		err = w.Flush()
+	}
+	if err == nil {
+		err = f.Sync()
+	}
+	if cerr := f.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return err
+	}
+	var b strings.Builder
+	for _, e := range entries {
+		fmt.Fprintf(&b, "%s %s %s %s\n", e.Role, e.Name, e.Digest, e.Config)
+	}
+	if err := os.WriteFile(tmpList, []byte(b.String()), 0o644); err != nil {
+		return err
+	}
+	if err := os.Rename(tmpOut, out); err != nil {
+		return err
+	}
+	return os.Rename(tmpList, list)
+}

cmd/felis/imagebundle_test.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// writeTestLayout writes an OCI layout tar holding one linux/arch image with one
+// layer, the way buildx --output type=oci leaves a single-platform build, and
+// returns its path with the manifest and config digests.
+func writeTestLayout(t *testing.T, dir, arch, layer string) (path, manifestDigest, configDigest string) {
+	t.Helper()
+	blobs := map[string][]byte{}
+	add := func(b []byte) (string, int) {
+		sum := sha256.Sum256(b)
+		d := "sha256:" + hex.EncodeToString(sum[:])
+		blobs[d] = b
+		return d, len(b)
+	}
+	cfgDigest, cfgSize := add([]byte(`{"architecture":"` + arch + `","os":"linux","rootfs":{"type":"layers"}}`))
+	layerDigest, layerSize := add([]byte(layer))
+	manifest := fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",`+
+		`"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":%q,"size":%d},`+
+		`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":%q,"size":%d}]}`,
+		cfgDigest, cfgSize, layerDigest, layerSize)
+	mDigest, mSize := add([]byte(manifest))
+	index, _ := json.Marshal(map[string]any{
+		"schemaVersion": 2,
+		"manifests": []map[string]any{{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": mDigest, "size": mSize,
+		}},
+	})
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	put := func(name string, b []byte) {
+		tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(b)), Typeflag: tar.TypeReg})
+		tw.Write(b)
+	}
+	put("oci-layout", []byte(`{"imageLayoutVersion":"1.0.0"}`))
+	put("index.json", index)
+	for d, b := range blobs {
+		put("blobs/sha256/"+strings.TrimPrefix(d, "sha256:"), b)
+	}
+	tw.Close()
+	path = filepath.Join(dir, arch+"-"+layer+".tar")
+	if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return path, mDigest, cfgDigest
+}
+
+func TestImageBundleWritesTheListingTheInstallerReads(t *testing.T) {
+	dir := t.TempDir()
+	limbo, limboDigest, limboConfig := writeTestLayout(t, dir, "arm64", "limbo")
+	lobby, lobbyDigest, lobbyConfig := writeTestLayout(t, dir, "arm64", "lobby")
+	out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
+	var stderr bytes.Buffer
+	code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
+		"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo,
+		"--layout", "lobby=registry.felis.svc:5000/felis/lobby:demo=" + lobby,
+	}, nil, &stderr)
+	if code != 0 {
+		t.Fatalf("image-bundle = %d: %s", code, stderr.String())
+	}
+	// deploy/bootstrap.sh reads this with `read -r role name digest config`.
+	got, _ := os.ReadFile(list)
+	want := "limbo registry.felis.svc:5000/felis/limbo:demo " + limboDigest + " " + limboConfig + "\n" +
+		"lobby registry.felis.svc:5000/felis/lobby:demo " + lobbyDigest + " " + lobbyConfig + "\n"
+	if string(got) != want {
+		t.Errorf("listing:\n%s\nwant:\n%s", got, want)
+	}
+	if fi, err := os.Stat(out); err != nil || fi.Size() == 0 {
+		t.Errorf("bundle: %v", err)
+	}
+	if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
+		t.Errorf("left behind %v", leftovers)
+	}
+}
+
+func TestImageBundleLeavesNothingWhenAnImageIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	amd, _, _ := writeTestLayout(t, dir, "amd64", "limbo")
+	out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
+	// The pair an earlier run wrote stays as it was: a listing beside a bundle it
+	// does not describe would have the installer look for images that are not there.
+	os.WriteFile(out, []byte("old bundle"), 0o644)
+	os.WriteFile(list, []byte("old listing\n"), 0o644)
+	var stderr bytes.Buffer
+	code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
+		"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + amd}, nil, &stderr)
+	if code != 1 || !strings.Contains(stderr.String(), "is a linux/amd64 image") {
+		t.Fatalf("image-bundle = %d: %s", code, stderr.String())
+	}
+	if got, _ := os.ReadFile(out); string(got) != "old bundle" {
+		t.Errorf("bundle was replaced with %d bytes", len(got))
+	}
+	if got, _ := os.ReadFile(list); string(got) != "old listing\n" {
+		t.Errorf("listing was replaced with %q", got)
+	}
+	if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
+		t.Errorf("left behind %v", leftovers)
+	}
+}
+
+func TestPushImageReadsABundleByImageName(t *testing.T) {
+	dir := t.TempDir()
+	limbo, _, _ := writeTestLayout(t, dir, "arm64", "limbo")
+	out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
+	if code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
+		"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo}, nil, &bytes.Buffer{}); code != 0 {
+		t.Fatal("image-bundle failed")
+	}
+	t.Setenv("FELIS_REGISTRY_USERNAME", "platform")
+	t.Setenv("FELIS_REGISTRY_PASSWORD", "x")
+	// The name is looked up in the bundle's index before the registry is contacted,
+	// so a name the bundle lacks fails here with what it does hold.
+	var stderr bytes.Buffer
+	code := cmdPushImage([]string{"--tar", out, "--image", "registry.felis.svc:5000/felis/lobby:demo",
+		"--ref", "127.0.0.1:1/felis/lobby:demo"}, &bytes.Buffer{}, &stderr)
+	if code != 1 || !strings.Contains(stderr.String(), "holds no image named registry.felis.svc:5000/felis/lobby:demo (it holds: registry.felis.svc:5000/felis/limbo:demo") {
+		t.Fatalf("push-image = %d: %s", code, stderr.String())
+	}
+}

cmd/felis/initforwarding.go

@@ -20,18 +20,14 @@ const forwardingSecretEnv = "FELIS_FORWARDING_SECRET"
 // config/ and server.properties live under it.
 const defaultForwardingDataDir = "/data"
 
-// fwd*Mode make the written config readable AND rewritable by the main server
-// container, whose UID we do not control (an arbitrary user image). The
-// initContainer runs as root (see buildStatefulSet) so it can write into a data
-// volume of unknown ownership; 0666/0777 then let a non-root Paper rewrite the
-// same files on boot.
-//
-// This relies on the initContainer running as root to write into a volume of
-// unknown ownership; that is how the operator schedules it. If that ever changes,
-// give the server pod an fsGroup so the shared volume is group-writable instead.
+// fwd*Mode are the modes the written config lands with. The initContainer runs as
+// the same uid as the server container (naming.GameUID, pinned by the operator in
+// the pod securityContext) after the prepare-data initContainer has handed the
+// whole volume to that uid, so owner read/write is all the server needs to rewrite
+// these files on boot and nothing else on the node gets write access to them.
 const (
-	fwdFileMode os.FileMode = 0o666
-	fwdDirMode  os.FileMode = 0o777
+	fwdFileMode os.FileMode = 0o644
+	fwdDirMode  os.FileMode = 0o755
 )
 
 // cmdInitForwarding is the felis-image initContainer entrypoint that makes an
@@ -96,9 +92,8 @@ func writePaperGlobal(dataDir, secret string) error {
 	if err := os.MkdirAll(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("create %s: %w", dir, err)
 	}
-	// MkdirAll honours the process umask (root's is typically 022 → 0755); chmod
-	// does not, and a non-root main container must be able to place/replace the
-	// file in this directory on boot.
+	// MkdirAll honours the process umask; chmod does not, so a directory an older
+	// release left at 0777 is brought back to fwdDirMode here.
 	if err := os.Chmod(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("chmod %s: %w", dir, err)
 	}
@@ -188,8 +183,8 @@ func upsertProperty(content []byte, key, value string) []byte {
 }
 
 // writeFileMode writes data then forces the mode, since WriteFile honours the
-// umask (root's is typically 022 → 0644) but a non-root main container must be
-// able to rewrite these files on boot.
+// umask and leaves an existing file's mode alone: a file an older release wrote
+// world-writable (0666) is tightened back to fwdFileMode on the next boot.
 func writeFileMode(path string, data []byte) error {
 	if err := os.WriteFile(path, data, fwdFileMode); err != nil {
 		return fmt.Errorf("write %s: %w", path, err)

cmd/felis/initforwarding_test.go

@@ -164,8 +164,9 @@ func TestUpsertPropertyAppends(t *testing.T) {
 	}
 }
 
-// The written files must be group/world writable so a non-root main container can
-// rewrite them. chmod semantics are POSIX-only, so this asserts on non-Windows.
+// The written files land at fwdFileMode: owner-writable for the game uid the init
+// shares with the server container, and no longer world-writable. chmod semantics
+// are POSIX-only, so this asserts on non-Windows.
 func TestWriteForwardingFileModes(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("POSIX file modes not represented on Windows")

cmd/felis/initvolume.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"syscall"
+
+	"felis.lolicon.best/internal/naming"
+)
+
+// cmdInitVolume is the felis-image `prepare-data` initContainer entrypoint: it
+// hands every entry of a server's world volume to the game uid/gid before the
+// server container starts. The operator runs the server itself as naming.GameUID,
+// so a world written by an earlier release (whose server ran as root), a restore
+// Job (which extracts as root), or a storage provisioner that creates the volume
+// root-owned would otherwise leave files the server cannot write — a world that
+// boots and then fails every save.
+//
+// fsGroup covers only part of this: kubelet applies it to volume types that
+// support ownership management, and a k3s local-path PV is a hostPath underneath,
+// which it skips. A walk from inside the pod works for every volume type.
+//
+// Only mismatched entries are touched, so a volume already owned by the game uid
+// costs one lstat per entry and no writes. The walk runs inside an os.Root at the
+// data dir and uses lchown, so a symlink a plugin planted is re-owned as a link
+// and never followed out of the volume.
+//
+// A single entry that cannot be chowned is reported and skipped: failing the pod
+// over one odd file would keep the whole server down, while the server itself
+// reports the one file it cannot write. Only an unreadable data dir fails.
+func cmdInitVolume(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("init-volume", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	dataDir := fs.String("data", defaultForwardingDataDir, "world volume mount to hand to the game uid")
+	uid := fs.Int64("uid", naming.GameUID, "owner uid for every entry")
+	gid := fs.Int64("gid", naming.GameGID, "owner gid for every entry")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	root, err := os.OpenRoot(*dataDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: open %s: %v\n", *dataDir, err)
+		return 1
+	}
+	defer root.Close()
+	res, err := chownTree(root, int(*uid), int(*gid), root.Lchown)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: %v\n", err)
+		return 1
+	}
+	for _, f := range res.failures {
+		fmt.Fprintf(stderr, "felis init-volume: %s\n", f)
+	}
+	fmt.Fprintf(stdout, "felis init-volume: %d entries checked, %d handed to %d:%d, %d failed\n",
+		res.checked, res.changed, *uid, *gid, len(res.failures))
+	return 0
+}
+
+// chownResult tallies one walk; failures is capped so a volume of thousands of
+// unownable files cannot flood the pod log.
+type chownResult struct {
+	checked  int
+	changed  int
+	failures []string
+}
+
+const maxReportedChownFailures = 20
+
+// chownTree walks root and calls chown on every entry (the root dir included)
+// whose owner is not uid:gid. It returns an error only when the root itself
+// cannot be read; per-entry failures are collected in the result.
+func chownTree(root *os.Root, uid, gid int, chown func(name string, uid, gid int) error) (chownResult, error) {
+	var res chownResult
+	fail := func(name string, err error) {
+		if len(res.failures) < maxReportedChownFailures {
+			res.failures = append(res.failures, fmt.Sprintf("%s: %v", name, err))
+		} else if len(res.failures) == maxReportedChownFailures {
+			res.failures = append(res.failures, "further failures not listed")
+		}
+	}
+	err := fs.WalkDir(root.FS(), ".", func(name string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			if name == "." {
+				return walkErr
+			}
+			fail(name, walkErr)
+			// A directory that cannot be listed is skipped as a whole; a file
+			// error has nothing below it to skip.
+			if d != nil && d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		res.checked++
+		info, err := d.Info()
+		if err != nil {
+			fail(name, err)
+			return nil
+		}
+		if st, ok := info.Sys().(*syscall.Stat_t); ok && int(st.Uid) == uid && int(st.Gid) == gid {
+			return nil
+		}
+		if err := chown(name, uid, gid); err != nil {
+			if !errors.Is(err, fs.ErrNotExist) { // gone mid-walk: nothing left to own
+				fail(name, err)
+			}
+			return nil
+		}
+		res.changed++
+		return nil
+	})
+	return res, err
+}

cmd/felis/initvolume_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strconv"
+	"testing"
+)
+
+// openTree builds a small world under a temp dir: nested dirs, a file, and a
+// symlink pointing out of the volume that the walk must not follow.
+func openTree(t *testing.T) *os.Root {
+	t.Helper()
+	dir := t.TempDir()
+	for _, d := range []string{"world/region", "plugins"} {
+		if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, f := range []string{"level.dat", "world/region/r.0.0.mca"} {
+		if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	outside := t.TempDir()
+	if err := os.Symlink(outside, filepath.Join(dir, "plugins", "escape")); err != nil {
+		t.Fatal(err)
+	}
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { root.Close() })
+	return root
+}
+
+// Every entry owned by someone else is handed over, the root dir included, and a
+// symlink is re-owned as a link rather than walked into.
+func TestChownTreeHandsOverMismatchedEntries(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	var got []string
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		got = append(got, name)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	want := []string{".", "level.dat", "plugins", "plugins/escape", "world", "world/region", "world/region/r.0.0.mca"}
+	slices.Sort(got)
+	if !slices.Equal(got, want) {
+		t.Errorf("chowned %v, want %v", got, want)
+	}
+	if res.changed != len(want) || res.checked != len(want) || len(res.failures) != 0 {
+		t.Errorf("result = %+v, want %d checked and changed, no failures", res, len(want))
+	}
+}
+
+// A volume already owned by the game uid costs no chown at all: this is the steady
+// state every restart after the first one hits.
+func TestChownTreeSkipsMatchingOwner(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	calls := 0
+	res, err := chownTree(root, os.Getuid(), os.Getgid(), func(string, int, int) error {
+		calls++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if calls != 0 || res.changed != 0 {
+		t.Errorf("chown called %d times on an already-owned tree (result %+v)", calls, res)
+	}
+}
+
+// One entry that refuses the chown is reported and the walk carries on: a single
+// odd file must not keep the whole server from starting.
+func TestChownTreeContinuesPastFailures(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		if name == "level.dat" {
+			return os.ErrPermission
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if len(res.failures) != 1 || res.changed != 6 {
+		t.Errorf("result = %+v, want 1 failure and 6 changed", res)
+	}
+}
+
+// Against a real directory the owner already matches, so the command succeeds
+// without needing CAP_CHOWN — the path every test runner (non-root) can take.
+func TestCmdInitVolumeOwnedTree(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "server.properties"), []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	var out, errb bytes.Buffer
+	code := cmdInitVolume([]string{"--data", dir, "--uid", strconv.Itoa(os.Getuid()), "--gid", strconv.Itoa(os.Getgid())}, &out, &errb)
+	if code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, errb.String())
+	}
+	if code := cmdInitVolume([]string{"--data", filepath.Join(dir, "missing")}, &out, &errb); code != 1 {
+		t.Errorf("missing data dir exit = %d, want 1", code)
+	}
+}

cmd/felis/main.go

@@ -6,8 +6,32 @@ package main
 
 import (
 	"os"
+	"path/filepath"
 )
 
 func main() {
+	ensureHostBinDirOnPath()
 	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
 }
+
+// hostBinDir is where deploy/bootstrap.sh installs felis, k3s and cloudflared.
+const hostBinDir = "/usr/local/bin"
+
+// ensureHostBinDirOnPath appends hostBinDir to PATH when it is missing, so the
+// k3s and cloudflared this binary execs are found beside it. sudo's secure_path
+// on EL leaves /usr/local/bin out: `sudo /usr/local/bin/felis db backup` would
+// otherwise run with no k3s to reach the database's pod through. Appended, so a
+// PATH that names another k3s first keeps it.
+func ensureHostBinDirOnPath() {
+	path := os.Getenv("PATH")
+	for _, dir := range filepath.SplitList(path) {
+		if dir == hostBinDir {
+			return
+		}
+	}
+	if path == "" {
+		os.Setenv("PATH", hostBinDir)
+		return
+	}
+	os.Setenv("PATH", path+string(os.PathListSeparator)+hostBinDir)
+}

cmd/felis/main_test.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+	"regexp"
+	"testing"
+)
+
+func TestEnsureHostBinDirOnPath(t *testing.T) {
+	for _, c := range []struct{ in, want string }{
+		{"/usr/sbin:/usr/bin", "/usr/sbin:/usr/bin:/usr/local/bin"},
+		{"/usr/local/bin:/usr/bin", "/usr/local/bin:/usr/bin"},
+		{"/opt/k3s:/usr/bin:/usr/local/bin", "/opt/k3s:/usr/bin:/usr/local/bin"},
+		{"", "/usr/local/bin"},
+	} {
+		t.Setenv("PATH", c.in)
+		ensureHostBinDirOnPath()
+		if got := os.Getenv("PATH"); got != c.want {
+			t.Errorf("PATH %q became %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+// run is what the tests drive, so the PATH fix must sit in main, before it.
+func TestMainFixesPathBeforeRunning(t *testing.T) {
+	b, err := os.ReadFile("main.go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !regexp.MustCompile(`func main\(\) \{\n\tensureHostBinDirOnPath\(\)\n\tos\.Exit\(run\(`).Match(b) {
+		t.Fatal("main does not call ensureHostBinDirOnPath before run")
+	}
+}

cmd/felis/manifests.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"path/filepath"
 	"strings"
 
 	"felis.lolicon.best/internal/platform"
@@ -35,9 +36,21 @@ func (m *multiFlag) Set(v string) error {
 // --velocity-cidr records the proxy host addresses allowed by the game NetworkPolicy.
 // Kubernetes permits resident-node traffic regardless, but remote proxy deployments
 // need an explicit CIDR, so the renderer refuses to guess.
+//
+// --only postgres renders just the control-plane database (platform.PostgresObjects),
+// which the installer brings up before migrations, before it has anything else
+// to render the full bundle with; it needs neither --felis-image nor
+// --velocity-cidr.
 func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("manifests", flag.ContinueOnError)
 	fs.SetOutput(stderr)
+	distributed := fs.Bool("distributed", false, "enable approved workers and archive transport")
+	controller := fs.String("controller-node", "", "protected controller identity for A")
+	probe := fs.String("egress-probe", "", "reachable controller host:port denied to game Pods")
+	var registryNodes multiFlag
+	fs.Var(&registryNodes, "registry-node-cidr", "exact node pull source for the registry (repeatable)")
+	socket := fs.String("node-control-socket", "", "host node-control Unix socket (optional, API only)")
+	nodeControlNode := fs.String("node-control-node", "", "controller hostname hosting the socket")
 	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace the control plane (api/operator/reaper) runs in")
 	minecraftNS := fs.String("minecraft-namespace", platform.DefaultMinecraftNamespace, "namespace MinecraftServer workloads run in")
 	buildNS := fs.String("build-namespace", platform.DefaultBuildNamespace, "namespace image-build Jobs run in")
@@ -45,18 +58,40 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	registryPort := fs.Int("registry-port", 5000, "port the in-cluster registry listens on")
 	panelNodePort := fs.Int("panel-node-port", int(platform.DefaultPanelNodePort), "NodePort that exposes the built-in HTTPS panel/API origin")
 	felisImage := fs.String("felis-image", "", "container image the felis-api/operator Deployments run, also passed through as FELIS_IMAGE (REQUIRED)")
-	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry:2)")
+	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry 2.8.3, pinned by digest)")
+	postgresImage := fs.String("postgres-image", "", "control-plane database image (default: PostgreSQL 18.6, pinned by digest)")
+	only := fs.String("only", "", `render one part of the bundle instead of all of it; "postgres" is the control-plane database`)
 	backupPVC := fs.String("backup-pvc", "felis-backups", "name of the world-archive PVC this bundle renders in the Minecraft namespace and advertises to the backup/restore executors via FELIS_BACKUP_PVC (default: felis-backups; pass an empty value to render none, leaving backup/restore answering 503)")
 	worldsHostPath := fs.String("worlds-host-path", "", "node directory the reaper reads worlds from: each world PVC resolves as <path>/<pvc>, or as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (k3s storage root: /var/lib/rancher/k3s/storage); enables the reaper CronJob (requires --archive-local-path and a non-empty --backup-pvc)")
-	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
+	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path. With the backup PVC alone it renders the retention-only CronJob, which deletes backups past their expiry and never touches a world")
+	registryStorage := fs.String("registry-storage", "", "capacity the registry PVC requests (default 10Gi; k3s local-path does not enforce it)")
+	uploadsStorage := fs.String("uploads-storage", "", "capacity the uploads PVC requests (default 5Gi; k3s local-path does not enforce it)")
+	backupStorage := fs.String("backup-storage", "", "capacity the world-archive PVC requests (default 10Gi; k3s local-path does not enforce it)")
 	reaperNode := fs.String("reaper-node", "", "node that holds --worlds-host-path: pins the reaper CronJob's pod there via nodeSelector kubernetes.io/hostname (multi-node clusters need this, or the reaper may schedule where the hostPath is empty)")
 	var velocityCIDRs multiFlag
 	fs.Var(&velocityCIDRs, "velocity-cidr", "CIDR of a Velocity proxy host allowed to reach game port 25565 (repeatable, REQUIRED)")
 	var packageCIDRs multiFlag
 	fs.Var(&packageCIDRs, "package-cidr", "CIDR of a package mirror build Pods may reach (repeatable; default none = no internet egress)")
+	var serverDenyCIDRs multiFlag
+	fs.Var(&serverDenyCIDRs, "server-egress-deny-cidr", "extra CIDR game server pods may never reach, e.g. the node's public address (repeatable)")
+	var serverAllowCIDRs multiFlag
+	fs.Var(&serverAllowCIDRs, "server-egress-allow-cidr", "private CIDR game server pods may reach despite the private-range block, e.g. a LAN database (repeatable)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
+	switch *only {
+	case "":
+	case "postgres":
+		return renderManifests(stdout, stderr, platform.PostgresObjects(platform.Params{
+			ControllerNode:     *controller,
+			ControlNamespace:   *controlNS,
+			MinecraftNamespace: *minecraftNS,
+			PostgresImage:      *postgresImage,
+		}))
+	default:
+		fmt.Fprintf(stderr, "felis manifests: --only %q: the one part that renders alone is \"postgres\"\n", *only)
+		return 2
+	}
 
 	// Keep proxy placement explicit. This matters for remote proxies and documents
 	// the expected source even when Velocity runs on the resident node.
@@ -74,7 +109,9 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"(the felis-api/operator Deployments run it and it is passed through as FELIS_IMAGE, e.g. --felis-image registry.felis.svc:5000/felis:v1)")
 		return 2
 	}
-	for _, cidr := range append(append([]string{}, velocityCIDRs...), packageCIDRs...) {
+	allCIDRs := append(append([]string{}, velocityCIDRs...), packageCIDRs...)
+	allCIDRs = append(append(allCIDRs, serverDenyCIDRs...), serverAllowCIDRs...)
+	for _, cidr := range allCIDRs {
 		if _, _, err := net.ParseCIDR(cidr); err != nil {
 			fmt.Fprintf(stderr, "felis manifests: invalid CIDR %q: %v\n", cidr, err)
 			return 2
@@ -87,7 +124,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	// The node pin exists only for the reaper's hostPath: naming a node without the
 	// worlds root would be silently dropped (no CronJob renders), so fail loud like
 	// the storage-trio check below.
-	if *reaperNode != "" && *worldsHostPath == "" {
+	if *reaperNode != "" && *worldsHostPath == "" && !*distributed {
 		fmt.Fprintln(stderr, "felis manifests: --reaper-node requires --worlds-host-path "+
 			"(it pins the reaper CronJob, which renders only with the retention storage trio)")
 		return 2
@@ -118,8 +155,8 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"multi-node cluster you MUST pass --reaper-node <name> (or add a nodeSelector) for the node holding the " +
 			"worlds, or the reaper may schedule where the hostPath is empty"
 		if *reaperNode != "" {
-			pin = fmt.Sprintf("the CronJob is pinned to node %q via kubernetes.io/hostname — keep this pointed at the "+
-				"node that actually holds the world volumes", *reaperNode)
+			pin = fmt.Sprintf("the CronJob and its worlds-root PV are pinned to node %q via kubernetes.io/hostname — "+
+				"keep this pointed at the node that actually holds the world volumes", *reaperNode)
 		}
 		fmt.Fprintf(stderr, "felis manifests: note: rendering the retention reaper CronJob (worlds hostPath %q). "+
 			"These points are NOT verified here:\n"+
@@ -128,12 +165,32 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"writes under /var/lib/rancher/k3s/storage). Any other provisioner needs its volumes exposed as "+
 			"<path>/<pvc>, or each candidate's archive fails and the world is preserved;\n"+
 			"  - %s.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, pin)
-	} else {
-		fmt.Fprintln(stderr, "felis manifests: note: retention reaper CronJob not rendered "+
-			"(pass --worlds-host-path and --archive-local-path — the archive PVC defaults to felis-backups — to enable it)")
+	} else if !*distributed {
+		switch {
+		case *archiveLocalPath != "" && *backupPVC == "":
+			fmt.Fprintln(stderr, "felis manifests: --archive-local-path names where the backup PVC is mounted, "+
+				"but --backup-pvc is empty (no archive store renders); drop one or the other")
+			return 2
+		case *archiveLocalPath != "":
+			fmt.Fprintln(stderr, "felis manifests: note: rendering the reaper CronJob retention-only: backups past "+
+				"their expiry are deleted daily, idle worlds are never archived or deleted "+
+				"(pass --worlds-host-path to reap them too)")
+		case *backupPVC != "":
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered: backups are never expired, so "+
+				"the archive store only grows until the disk fills (pass --archive-local-path, equal to felis.toml "+
+				"[archive] local_path, for the retention-only CronJob, and --worlds-host-path as well to reap idle worlds)")
+		default:
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered (backups are disabled)")
+		}
 	}
 
-	out, err := platform.RenderYAML(platform.Params{
+	if *socket != "" && (!filepath.IsAbs(*socket) || *nodeControlNode == "") {
+		fmt.Fprintln(stderr, "node-control requires an absolute socket and its controller hostname")
+		return 2
+	}
+	params := platform.Params{
+		NodeControlSocket: *socket, NodeControlNode: *nodeControlNode,
+		Distributed: *distributed, ControllerNode: *controller, EgressProbe: *probe, RegistryNodeCIDRs: registryNodes,
 		ControlNamespace:   *controlNS,
 		MinecraftNamespace: *minecraftNS,
 		BuildNamespace:     *buildNS,
@@ -142,13 +199,30 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 		PanelNodePort:      int32(*panelNodePort),
 		FelisImage:         *felisImage,
 		RegistryImage:      *registryImage,
+		PostgresImage:      *postgresImage,
 		BackupPVC:          *backupPVC,
 		WorldsHostPath:     *worldsHostPath,
 		ReaperNode:         *reaperNode,
 		ArchiveLocalPath:   *archiveLocalPath,
 		VelocityCIDRs:      []string(velocityCIDRs),
 		PackageSourceCIDRs: []string(packageCIDRs),
-	})
+
+		ServerEgressDenyCIDRs:  []string(serverDenyCIDRs),
+		ServerEgressAllowCIDRs: []string(serverAllowCIDRs),
+
+		RegistryStorage: *registryStorage,
+		UploadsStorage:  *uploadsStorage,
+		BackupStorage:   *backupStorage,
+	}
+	if err := params.Validate(); err != nil {
+		fmt.Fprintf(stderr, "felis manifests: %v\n", err)
+		return 2
+	}
+	return renderManifests(stdout, stderr, platform.Objects(params))
+}
+
+func renderManifests(stdout, stderr io.Writer, objs []platform.Object) int {
+	out, err := platform.RenderObjects(objs)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis manifests: render: %v\n", err)
 		return 1

cmd/felis/manifests_test.go

@@ -90,12 +90,13 @@ func TestManifestsRendersBundle(t *testing.T) {
 		t.Error("rendered bundle must not contain ClusterRole/ClusterRoleBinding")
 	}
 	// Without the retention flags, the reaper CronJob is not rendered and the
-	// generator says so on stderr.
+	// generator says so on stderr, naming what that leaves: backups that never
+	// expire.
 	if strings.Contains(text, "kind: CronJob") {
-		t.Error("no reaper CronJob must render without --worlds-host-path")
+		t.Error("no reaper CronJob must render without --archive-local-path")
 	}
-	if !strings.Contains(errBuf.String(), "not rendered") {
-		t.Errorf("expected a 'reaper not rendered' notice on stderr, got %q", errBuf.String())
+	if !strings.Contains(errBuf.String(), "not rendered") || !strings.Contains(errBuf.String(), "never expired") {
+		t.Errorf("expected a 'reaper not rendered, backups never expired' notice on stderr, got %q", errBuf.String())
 	}
 }
 
@@ -144,6 +145,40 @@ func TestManifestsBackupPVCOptOut(t *testing.T) {
 	}
 }
 
+// TestManifestsRendersRetentionOnly: the archive store without a worlds root
+// still gets the daily CronJob, retention-only, so backups past their expiry
+// leave the store on an install that never reaps a world; the operator is told
+// which of the two it got. An archive path with the store switched off is a
+// mistake and fails loud.
+func TestManifestsRendersRetentionOnly(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	text := out.String()
+	for _, want := range []string{"kind: CronJob", "name: felis-reaper", "--retention-only", "claimName: felis-backups"} {
+		if !strings.Contains(text, want) {
+			t.Errorf("retention-only bundle missing %q", want)
+		}
+	}
+	if strings.Contains(text, "kind: PersistentVolume\n") || strings.Contains(text, "--worlds-root") {
+		t.Error("a retention-only bundle must not reach for a worlds root")
+	}
+	if !strings.Contains(errBuf.String(), "retention-only") {
+		t.Errorf("stderr must say the CronJob is retention-only, got %q", errBuf.String())
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives", "--backup-pvc="}, &out, &errBuf)
+	if code != 2 || out.Len() != 0 || !strings.Contains(errBuf.String(), "--backup-pvc is empty") {
+		t.Errorf("archive path without an archive store: exit=%d out=%d bytes stderr=%q, want a fail-loud 2", code, out.Len(), errBuf.String())
+	}
+}
+
 // TestManifestsRendersReaper proves the happy path with the full retention trio:
 // a batch/v1 CronJob is emitted, named felis-reaper, mounting the backup PVC at the
 // supplied archive path.
@@ -219,3 +254,61 @@ func TestManifestsReaperNodePin(t *testing.T) {
 		t.Errorf("--reaper-node without --worlds-host-path: exit = %d, want 2", code)
 	}
 }
+
+// TestManifestsStorageSizes proves the PVC size flags reach the rendered claims
+// and a size the API server would reject fails before anything is applied.
+func TestManifestsStorageSizes(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "40Gi", "--uploads-storage", "8Gi"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, stderr = %s", code, errBuf.String())
+	}
+	for _, want := range []string{"storage: 40Gi", "storage: 8Gi"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("bundle lacks %q", want)
+		}
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "lots"}, &out, &errBuf)
+	if code == 0 || !strings.Contains(errBuf.String(), "registry storage") {
+		t.Errorf("--registry-storage lots: exit %d, stderr %q; want a refusal naming the flag", code, errBuf.String())
+	}
+}
+
+// TestManifestsOnlyPostgres: the installer renders the database before it has
+// an image or a proxy address for the rest of the bundle, so --only postgres
+// must render without them, and render the database and nothing else (a stray
+// Deployment in that apply would start without its identities).
+func TestManifestsOnlyPostgres(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--only", "postgres", "--control-namespace", "ctl", "--postgres-image", "example/pg:18@sha256:abc"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	var kinds []string
+	for _, doc := range strings.Split(out.String(), "\n---\n") {
+		for _, line := range strings.Split(doc, "\n") {
+			if strings.HasPrefix(line, "kind: ") {
+				kinds = append(kinds, strings.TrimPrefix(line, "kind: "))
+			}
+		}
+	}
+	if got, want := strings.Join(kinds, ","), "Namespace,ConfigMap,NetworkPolicy,Deployment,Service"; got != want {
+		t.Errorf("rendered kinds %s, want %s", got, want)
+	}
+	for _, want := range []string{"name: felis-postgres", "namespace: ctl", "image: example/pg:18@sha256:abc"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("rendered database missing %q", want)
+		}
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	if code := run([]string{"manifests", "--only", "registry"}, &out, &errBuf); code != 2 || out.Len() != 0 {
+		t.Errorf("--only registry: exit %d with %d bytes of YAML, want 2 and none", code, out.Len())
+	}
+}

cmd/felis/memlimit.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"os"
+	"runtime/debug"
+	"strconv"
+	"strings"
+)
+
+// cgroupMemoryFiles are where a container reads the memory it is allowed:
+// cgroup v2 first, then v1.
+var cgroupMemoryFiles = []string{"/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"}
+
+// limitHeapToCgroup sets the Go heap's soft limit from the container's memory
+// limit, so the collector works harder as a Job nears it and the kernel does not
+// kill the Job first. An extraction or a folder zipped for download keeps a few
+// hundred bytes per entry for as long as it runs; without the limit the heap
+// grows to twice that before a collection, and a 256 MiB Job was killed at
+// 400,000 entries whose live heap was 115 MB. GOMEMLIMIT set by hand wins.
+func limitHeapToCgroup() {
+	if os.Getenv("GOMEMLIMIT") != "" {
+		return
+	}
+	for _, f := range cgroupMemoryFiles {
+		b, err := os.ReadFile(f)
+		if err != nil {
+			continue
+		}
+		if n, ok := softMemoryLimit(string(b)); ok {
+			debug.SetMemoryLimit(n)
+		}
+		return
+	}
+}
+
+// softMemoryLimit answers three fifths of the limit a cgroup memory file holds,
+// or false for "max" (no limit) and anything unreadable. The rest is left for
+// what the kernel charges the container beyond the Go heap: the page cache of
+// the files it reads and writes, and the inodes it creates. Under a 256 MiB
+// limit, 400,000 extracted entries peaked at 184 MB resident with the heap held
+// to 150 MiB.
+func softMemoryLimit(content string) (int64, bool) {
+	n, err := strconv.ParseInt(strings.TrimSpace(content), 10, 64)
+	if err != nil || n <= 0 {
+		return 0, false
+	}
+	return n / 5 * 3, true
+}

cmd/felis/memlimit_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"math"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"testing"
+)
+
+func TestSoftMemoryLimit(t *testing.T) {
+	for _, c := range []struct {
+		in   string
+		want int64
+		ok   bool
+	}{
+		{"268435456\n", 161061273, true}, // 256 MiB, as memory.max holds it
+		{"max\n", 0, false},
+		{"0\n", 0, false},
+		{"-1", 0, false},
+		{"", 0, false},
+	} {
+		got, ok := softMemoryLimit(c.in)
+		if got != c.want || ok != c.ok {
+			t.Errorf("softMemoryLimit(%q) = %d %v, want %d %v", c.in, got, ok, c.want, c.ok)
+		}
+	}
+}
+
+// TestLimitHeapToCgroup checks the limit comes from the first cgroup file there
+// is, and that GOMEMLIMIT set by hand leaves the heap alone.
+func TestLimitHeapToCgroup(t *testing.T) {
+	prevFiles, prevLimit := cgroupMemoryFiles, debug.SetMemoryLimit(-1)
+	t.Cleanup(func() { cgroupMemoryFiles = prevFiles; debug.SetMemoryLimit(prevLimit) })
+	dir := t.TempDir()
+	v1, v1b := filepath.Join(dir, "v1"), filepath.Join(dir, "v1b")
+	if err := os.WriteFile(v1, []byte("268435456\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(v1b, []byte("536870912\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	cgroupMemoryFiles = []string{filepath.Join(dir, "missing"), v1, v1b}
+
+	t.Setenv("GOMEMLIMIT", "")
+	debug.SetMemoryLimit(math.MaxInt64)
+	limitHeapToCgroup()
+	if got := debug.SetMemoryLimit(-1); got != 161061273 {
+		t.Fatalf("limit = %d, want three fifths of 256 MiB", got)
+	}
+
+	t.Setenv("GOMEMLIMIT", "1GiB")
+	debug.SetMemoryLimit(math.MaxInt64)
+	limitHeapToCgroup()
+	if got := debug.SetMemoryLimit(-1); got != math.MaxInt64 {
+		t.Fatalf("limit = %d with GOMEMLIMIT set, want it left alone", got)
+	}
+}

cmd/felis/migrate.go

@@ -2,20 +2,32 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"time"
 
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
 // cmdMigrate implements `felis migrate up`: load config, open the database, and
 // apply every pending embedded migration under the advisory lock (spec §6).
+//
+// Migrations only roll forward, and some drop data (0017_drop_password), so a
+// database that already holds a schema and has migrations pending is bundled
+// first (internal/dbbackup, label pre-migrate). A failed snapshot stops the
+// upgrade; -no-backup is the explicit way past it, e.g. for an external
+// database (no [database] deployment, so the host's own pg_dump runs) whose
+// server is newer than that pg_dump.
 func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("migrate", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	backupDir := fs.String("backup-dir", dbbackup.DefaultDir, "where the pre-migration snapshot goes")
+	noBackup := fs.Bool("no-backup", false, "apply pending migrations without snapshotting the database first")
 	// The "up" verb precedes any flags (felis migrate up -config path). Go's
 	// flag.Parse stops at the first non-flag token and would never see a flag
 	// placed after "up", silently falling back to the default -config. Pull the
@@ -48,6 +60,18 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	if !*noBackup {
+		path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database, *backupDir, stderr)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis migrate: pre-migration backup failed, nothing applied: %v\n", err)
+			fmt.Fprintln(stderr, "  fix the backup, or re-run with -no-backup to migrate without one")
+			return 1
+		}
+		if path != "" {
+			fmt.Fprintf(stdout, "felis migrate: database snapshot %s\n", path)
+		}
+	}
+
 	applied, err := store.Up(ctx, drv, migrations)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis migrate: %v\n", err)
@@ -60,3 +84,98 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// preMigrateStateDir is the host state a pre-migrate bundle carries; tests
+// point it at a directory of their own.
+var preMigrateStateDir = dbbackup.DefaultStateDir
+
+// preMigrateBackup bundles the database when it already carries a schema and
+// some of migrations are not applied yet, and returns the bundle's path ("" when
+// there was nothing to protect: a fresh database, or nothing pending).
+func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, db config.DatabaseConfig, dir string, log io.Writer) (string, error) {
+	if err := drv.EnsureVersionTable(ctx); err != nil {
+		return "", fmt.Errorf("ensure version table: %w", err)
+	}
+	done, err := drv.AppliedVersions(ctx)
+	if err != nil {
+		return "", fmt.Errorf("read applied versions: %w", err)
+	}
+	if len(done) == 0 || !hasPending(done, migrations) {
+		return "", nil
+	}
+	tools, err := dbTools(db)
+	if err != nil {
+		return "", err
+	}
+	path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
+		DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelPreMigrate,
+		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: preMigrateStateDir,
+		Version: resolvedVersion(), ExportServers: exportMinecraftServers, Log: log, Record: true,
+	})
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// Rolling the migration back needs the database alone. Backup logged
+		// the gap, and the panel and the watchdog show it while this is the
+		// newest bundle.
+		return path, nil
+	}
+	return path, err
+}
+
+func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
+	for _, m := range migrations {
+		if _, ok := done[m.Version]; !ok {
+			return true
+		}
+	}
+	return false
+}
+
+// openStore opens the business database for a command that reads and writes its
+// tables, and refuses one whose schema this build was not written against: a newer
+// Felis migrated it (a rolled-back binary), or, unless allowPending, migrations this
+// build embeds have not run yet (a binary swapped in ahead of `felis migrate up`).
+func openStore(ctx context.Context, url string, allowPending bool) (*store.PostgresDriver, error) {
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	return checkSchema(ctx, drv, allowPending)
+}
+
+// podDBWindow and podDBInterval bound how long a pod that has just started retries its
+// first database dial while the network policy has yet to admit it
+// (store.OpenRetrying). A minute is far past the sync lag and far inside every Job's
+// deadline. Vars so a test can shrink them.
+var (
+	podDBWindow   = time.Minute
+	podDBInterval = time.Second
+)
+
+// openPodStore is openStore for felis-api and the reaper and backup Jobs, whose first
+// dial comes milliseconds after their pod starts.
+func openPodStore(ctx context.Context, url, prog string, stderr io.Writer) (*store.PostgresDriver, error) {
+	drv, err := store.OpenRetrying(ctx, url, podDBWindow, podDBInterval, func(err error) {
+		fmt.Fprintf(stderr, "felis %s: %v; retrying (a pod that has just started waits for the network policy to admit it)\n", prog, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return checkSchema(ctx, drv, false)
+}
+
+// checkSchema closes drv and fails when its schema is not the one this build was
+// written against (see openStore).
+func checkSchema(ctx context.Context, drv *store.PostgresDriver, allowPending bool) (*store.PostgresDriver, error) {
+	s, err := store.ReadSchema(ctx, drv)
+	if err == nil {
+		err = s.Err()
+		if allowPending {
+			err = s.Newer()
+		}
+	}
+	if err != nil {
+		drv.Close()
+		return nil, err
+	}
+	return drv, nil
+}

cmd/felis/migrate_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// openPodStore retries a refused first dial for podDBWindow, saying so on stderr
+// under the calling command's name each time.
+func TestOpenPodStoreRetriesARefusedDial(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := ln.Addr().String()
+	ln.Close()
+
+	window, interval := podDBWindow, podDBInterval
+	podDBWindow, podDBInterval = 200*time.Millisecond, 20*time.Millisecond
+	t.Cleanup(func() { podDBWindow, podDBInterval = window, interval })
+
+	var stderr bytes.Buffer
+	start := time.Now()
+	_, err = openPodStore(context.Background(), fmt.Sprintf("postgres://felis@%s/felis?sslmode=disable", addr), "reaper", &stderr)
+	if !errors.Is(err, syscall.ECONNREFUSED) {
+		t.Fatalf("err = %v, want a refused dial", err)
+	}
+	if elapsed := time.Since(start); elapsed < podDBWindow {
+		t.Fatalf("gave up after %s, inside the %s window", elapsed, podDBWindow)
+	}
+	lines := strings.Split(strings.TrimSuffix(stderr.String(), "\n"), "\n")
+	if len(lines) < 2 || len(lines) > 11 {
+		t.Fatalf("%d retry lines in a 200ms window at 20ms:\n%s", len(lines), stderr.String())
+	}
+	for _, l := range lines {
+		if !strings.HasPrefix(l, "felis reaper: failed to connect to `user=felis database=felis`: ") ||
+			!strings.HasSuffix(l, "; retrying (a pod that has just started waits for the network policy to admit it)") {
+			t.Fatalf("retry line %q", l)
+		}
+	}
+}

cmd/felis/mirrortools.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/build"
+	"felis.lolicon.best/internal/imagepush"
+)
+
+// defaultBuildToolsStatus is where mirror-build-tools records its last run; the
+// watchdog reads it to tell a vulnerability DB that stopped refreshing.
+const defaultBuildToolsStatus = "/var/lib/felis/build-tools/status.json"
+
+// cmdMirrorBuildTools copies the build lane's tools (build.Tools: the kaniko and
+// trivy images, Trivy's vulnerability and Java DBs) from upstream into the
+// platform registry, where build Jobs pull them. deploy/bootstrap.sh runs it at
+// install and from felis-build-tools.timer twice a day, which is what keeps the
+// DBs fresh; a root shell can run it the same way to refresh now.
+//
+// It writes as the platform principal through the node's loopback hostPort, the
+// same way the installer pushes, reading the token from the environment or from
+// /etc/felis/secrets.env.
+func cmdMirrorBuildTools(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("mirror-build-tools", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	endpoint := fs.String("endpoint", "127.0.0.1:5000", "host[:port] of the registry to write to (plain HTTP)")
+	only := fs.String("only", "", "comma-separated tool names to copy (default: all of "+toolNames()+")")
+	status := fs.String("status", defaultBuildToolsStatus, `file to record the run in ("" records nothing)`)
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	platformFlag := fs.String("platform", "", "os/arch of the images to copy (default: this machine's)")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	tools, err := selectTools(*only)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: "http", Username: user, Password: pass, Log: stdout}
+	src := &imagepush.Source{Platform: *platformFlag}
+	started := time.Now()
+	var failed []string
+	for _, t := range tools {
+		dst := strings.TrimSuffix(*endpoint, "/") + "/" + t.Mirror
+		if _, err := p.Mirror(ctx, src, t.Source, dst); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: %s: %v\n", t.Name, err)
+			failed = append(failed, t.Name+": "+err.Error())
+		}
+	}
+	if *status != "" {
+		st, err := imagepush.ReadMirrorStatus(*status)
+		if err != nil || st == nil {
+			st = &imagepush.MirrorStatus{}
+		}
+		st.LastAttempt = started
+		st.LastError = strings.Join(failed, "; ")
+		if len(failed) == 0 {
+			st.LastSuccess = started
+		}
+		if err := imagepush.WriteMirrorStatus(*status, *st); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: record %s: %v\n", *status, err)
+		}
+	}
+	if len(failed) > 0 {
+		return 1
+	}
+	return 0
+}
+
+func toolNames() string {
+	var names []string
+	for _, t := range build.Tools {
+		names = append(names, t.Name)
+	}
+	return strings.Join(names, ",")
+}
+
+func selectTools(only string) ([]build.Tool, error) {
+	if only == "" {
+		return build.Tools, nil
+	}
+	var out []build.Tool
+	for _, name := range strings.Split(only, ",") {
+		name = strings.TrimSpace(name)
+		found := false
+		for _, t := range build.Tools {
+			if t.Name == name {
+				out = append(out, t)
+				found = true
+			}
+		}
+		if !found {
+			return nil, fmt.Errorf("unknown tool %q (known: %s)", name, toolNames())
+		}
+	}
+	return out, nil
+}

cmd/felis/node_approve.go

@@ -0,0 +1,466 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/archivetransfer"
+	"felis.lolicon.best/internal/operator"
+	"felis.lolicon.best/internal/placement"
+	"felis.lolicon.best/internal/platform"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func approveNode(ctx context.Context, cl client.Client, cs kubernetes.Interface, ns, controlNS, name, image, remote string, stdout io.Writer) error {
+	var n corev1.Node
+	if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil {
+		return err
+	}
+	_, controlPlane := n.Labels["node-role.kubernetes.io/control-plane"]
+	if !placement.Online(&n) || controlPlane || n.Labels[placement.LabelRole] == placement.RoleController {
+		return fmt.Errorf("candidate must be an online agent")
+	}
+	var nodes corev1.NodeList
+	if err := cl.List(ctx, &nodes); err != nil {
+		return err
+	}
+	var controller *corev1.Node
+	var peers []string
+	var denied []string
+	for i := range nodes.Items {
+		node := &nodes.Items[i]
+		if node.Labels[placement.LabelRole] == placement.RoleController {
+			if controller != nil {
+				return fmt.Errorf("multiple controllers found")
+			}
+			controller = node
+		}
+		for _, addr := range node.Status.Addresses {
+			if addr.Type == corev1.NodeInternalIP || addr.Type == corev1.NodeExternalIP {
+				cidr, err := exactCIDR(addr.Address)
+				if err != nil {
+					return err
+				}
+				peers = append(peers, cidr)
+				for _, p := range []string{"6443", "10250", "5000", "30443"} {
+					denied = append(denied, net.JoinHostPort(addr.Address, p))
+				}
+			}
+		}
+	}
+	if controller == nil || controller.Name == n.Name || controller.Status.NodeInfo.Architecture != n.Status.NodeInfo.Architecture {
+		return fmt.Errorf("candidate architecture must match the sole controller")
+	}
+	if n.Status.NodeInfo.KubeletVersion != controller.Status.NodeInfo.KubeletVersion {
+		return fmt.Errorf("candidate k3s version must match A")
+	}
+	var apiSvc, registrySvc, archiveSvc, kubernetesSvc corev1.Service
+	for _, svc := range []struct {
+		obj      *corev1.Service
+		ns, name string
+	}{{&kubernetesSvc, "default", "kubernetes"}, {&apiSvc, controlNS, platform.SAAPI}, {&registrySvc, controlNS, "registry"}, {&archiveSvc, ns, platform.ArchiveName}} {
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: svc.ns, Name: svc.name}, svc.obj); err != nil {
+			return err
+		}
+	}
+	if len(apiSvc.Spec.Ports) == 0 || apiSvc.Spec.Ports[0].NodePort == 0 {
+		return fmt.Errorf("controller API NodePort is absent")
+	}
+	// A's exact interface addresses let the probe observe DNAT/SNAT before the production policy is adjusted.
+	var aCIDRs []string
+	if addrs, err := net.InterfaceAddrs(); err == nil {
+		for _, a := range addrs {
+			ip, _, err := net.ParseCIDR(a.String())
+			if err == nil && !ip.IsLoopback() {
+				cidr, _ := exactCIDR(ip.String())
+				aCIDRs = append(aCIDRs, cidr)
+			}
+		}
+	}
+	if len(aCIDRs) == 0 {
+		return fmt.Errorf("cannot determine A's exact interface addresses")
+	}
+	onController := false
+	for _, addr := range controller.Status.Addresses {
+		cidr, err := exactCIDR(addr.Address)
+		if err != nil {
+			continue
+		}
+		for _, local := range aCIDRs {
+			if cidr == local {
+				onController = true
+			}
+		}
+	}
+	if !onController || !placement.Online(controller) {
+		return fmt.Errorf("run admission on the online controller A")
+	}
+	// Quarantine first. Approval is the final write, after all checks have succeeded.
+	before := n.DeepCopy()
+	if n.Labels == nil {
+		n.Labels = map[string]string{}
+	}
+	delete(n.Labels, placement.LabelApproved)
+	found := false
+	for _, t := range n.Spec.Taints {
+		if t.Key == "felis.lolicon.best/unapproved" {
+			found = true
+		}
+	}
+	if !found {
+		n.Spec.Taints = append(n.Spec.Taints, corev1.Taint{Key: "felis.lolicon.best/unapproved", Value: "true", Effect: corev1.TaintEffectNoSchedule})
+	}
+	if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil {
+		return err
+	}
+	if err := denyNodeAddresses(ctx, cl, ns, nodes.Items); err != nil {
+		return err
+	}
+	// The host's Service dial may be masqueraded to its bridge gateway. Permit exact host addresses only.
+	pullSources := append([]string{}, peers...)
+	if ip, network, err := net.ParseCIDR(n.Spec.PodCIDR); err == nil && ip.To4() != nil {
+		v := append(net.IP(nil), network.IP.To4()...)
+		pullSources = append(pullSources, v.String()+"/32")
+		v[3]++
+		pullSources = append(pullSources, v.String()+"/32")
+	}
+	var registryNP networkingv1.NetworkPolicy
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNS, Name: "felis-registry-ingress"}, &registryNP); err != nil {
+		return err
+	}
+	if len(registryNP.Spec.Ingress) == 0 {
+		return fmt.Errorf("registry ingress policy is not configured")
+	}
+	prev := registryNP.DeepCopy()
+	for _, cidr := range pullSources {
+		registryNP.Spec.Ingress[0].From = append(registryNP.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
+	}
+	if err := cl.Patch(ctx, &registryNP, client.MergeFrom(prev)); err != nil {
+		return err
+	}
+	ctrlIP := ""
+	for _, a := range controller.Status.Addresses {
+		if a.Type == corev1.NodeInternalIP {
+			ctrlIP = a.Address
+			break
+		}
+	}
+	if ctrlIP == "" {
+		return fmt.Errorf("controller has no address")
+	}
+	script := "set -euo pipefail\numask 077\n" +
+		"systemctl is-active --quiet k3s-agent\n! systemctl is-active --quiet k3s\ntest ! -f /etc/rancher/k3s/k3s.yaml\n" +
+		"ip -d link show flannel-wg | grep -q wireguard\n" +
+		"/usr/local/bin/felis node firewall --peers " + shellQuote(strings.Join(peers, ",")) + " --controller-ip " + shellQuote(ctrlIP) + " --api-service-ip " + shellQuote(kubernetesSvc.Spec.ClusterIP) + " --node-port " + strconv.Itoa(int(apiSvc.Spec.Ports[0].NodePort)) + " --namespace " + shellQuote(ns) + " --control-namespace " + shellQuote(controlNS) + "\n" +
+		"kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig\n" +
+		"/usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" get node " + shellQuote(name) + " -o name >/dev/null\n" +
+		"proof=$(mktemp); trap 'rm -f \"$proof\"' EXIT\n" +
+		"if /usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" label node " + shellQuote(name) + " felis.node-restriction.kubernetes.io/probe=controller --overwrite 2>\"$proof\"; then echo 'NodeRestriction failed' >&2;exit 1;fi\ngrep -qi forbidden \"$proof\"\n" +
+		"image=" + shellQuote(image) + "\nif [ -n \"$(/usr/local/bin/k3s crictl images -q \"$image\")\" ]; then /usr/local/bin/k3s crictl rmi \"$image\" >/dev/null; fi\ntest -z \"$(/usr/local/bin/k3s crictl images -q \"$image\")\"\n/usr/local/bin/k3s crictl pull \"$image\" >/dev/null\n"
+	cmd := exec.CommandContext(ctx, "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=10", "--", remote, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi")
+	cmd.Stdin = strings.NewReader(script)
+	cmd.Stdout = stdout
+	cmd.Stderr = stdout
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("worker host, NodeRestriction or uncached registry pull check failed: %w", err)
+	}
+	id := "felis-probe-" + archivetransfer.ID()[:12]
+	var objects []client.Object
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		for i := len(objects) - 1; i >= 0; i-- {
+			cl.Delete(cleanup, objects[i])
+		}
+	}()
+	create := func(o client.Object) error {
+		if err := cl.Create(ctx, o); err != nil {
+			return err
+		}
+		objects = append(objects, o)
+		return nil
+	}
+	var endpoints []string
+	var echoPods []*corev1.Pod
+	for i := range nodes.Items {
+		node := &nodes.Items[i]
+		if !placement.Online(node) || (node.Name != name && node.Name != controller.Name && node.Labels[placement.LabelApproved] != "true") {
+			continue
+		}
+		echoName := fmt.Sprintf("%s-%d", id, i)
+		p := probePod(echoName, ns, node.Name, image, []string{"--listen", ":25565"}, true)
+		p.Labels["felis.lolicon.best/probe-echo"] = id
+		if err := create(p); err != nil {
+			return err
+		}
+		echoPods = append(echoPods, p)
+		svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: echoName, Namespace: ns}, Spec: corev1.ServiceSpec{Selector: map[string]string{"felis.lolicon.best/probe-name": echoName}, Ports: []corev1.ServicePort{{Port: 25565, TargetPort: intstr.FromInt32(25565)}}}}
+		if err := create(svc); err != nil {
+			return err
+		}
+		endpoints = append(endpoints, net.JoinHostPort(svc.Spec.ClusterIP, "25565"))
+	}
+	tcp := corev1.ProtocolTCP
+	port := intstr.FromInt32(25565)
+	policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id, Namespace: ns}, Spec: networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-echo": id}}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}}}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &port}}}}}}
+	for _, cidr := range aCIDRs {
+		policy.Spec.Ingress[0].From = append(policy.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
+	}
+	if err := create(policy); err != nil {
+		return err
+	}
+	for _, p := range echoPods {
+		if err := waitProbePod(ctx, cl, p); err != nil {
+			return err
+		}
+	}
+	open := append([]string{}, endpoints...)
+	open = append(open, net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), net.JoinHostPort(registrySvc.Spec.ClusterIP, "5000"), net.JoinHostPort(archiveSvc.Spec.ClusterIP, "8090"))
+	// Positive probes need temporary, narrowly scoped ingress grants where the
+	// production fence admits only the controller or archive-transfer jobs.
+	for _, svc := range []*corev1.Service{&apiSvc, &registrySvc, &archiveSvc} {
+		if len(svc.Spec.Selector) == 0 || len(svc.Spec.Ports) == 0 {
+			return fmt.Errorf("probe Service %s has no backend", svc.Name)
+		}
+		targetPort := svc.Spec.Ports[0].TargetPort
+		if targetPort.IntVal == 0 && targetPort.StrVal == "" {
+			targetPort = intstr.FromInt32(svc.Spec.Ports[0].Port)
+		}
+		allow := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id + "-" + svc.Name, Namespace: svc.Namespace}, Spec: networkingv1.NetworkPolicySpec{
+			PodSelector: metav1.LabelSelector{MatchLabels: svc.Spec.Selector}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
+			Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{
+				NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ns}},
+				PodSelector:       &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}},
+			}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &targetPort}}}},
+		}}
+		if err := create(allow); err != nil {
+			return err
+		}
+	}
+	positive := probeJob(id+"-positive", ns, name, image, probeArgs("--open", open), false)
+	positive.Spec.Template.Labels["felis.lolicon.best/probe-source"] = id
+	if err := create(positive); err != nil {
+		return err
+	}
+	if err := waitProbeJob(ctx, cl, positive); err != nil {
+		return err
+	}
+	denied = append(denied, open...)
+	denied = append(denied, "169.254.169.254:80", "169.254.170.2:80")
+	negative := probeJob(id+"-negative", ns, name, image, probeArgs("--closed", denied), true)
+	negative.Spec.Template.Spec.InitContainers = []corev1.Container{{Name: "egress-gate", Image: image, Command: []string{"/usr/local/bin/felis", "egress-gate", "--probe", net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), "--wait", "2m"}, SecurityContext: negative.Spec.Template.Spec.Containers[0].SecurityContext}}
+	if err := create(negative); err != nil {
+		return err
+	}
+	if err := waitProbeJob(ctx, cl, negative); err != nil {
+		return err
+	}
+	// Dial from Velocity's host namespace and record the address actually observed inside each backend.
+	for _, endpoint := range endpoints {
+		c, err := net.DialTimeout("tcp", endpoint, 5*time.Second)
+		if err != nil {
+			return fmt.Errorf("velocity host cannot dial backend Service %s: %w", endpoint, err)
+		}
+		c.Close()
+	}
+	var observed []string
+	time.Sleep(500 * time.Millisecond)
+	for _, p := range echoPods {
+		foundA := false
+		stream, err := cs.CoreV1().Pods(ns).GetLogs(p.Name, &corev1.PodLogOptions{}).Stream(ctx)
+		if err != nil {
+			return err
+		}
+		scan := bufio.NewScanner(stream)
+		for scan.Scan() {
+			line := scan.Text()
+			if strings.HasPrefix(line, "felis-probe-source ") {
+				host, _, err := net.SplitHostPort(strings.TrimPrefix(line, "felis-probe-source "))
+				if err == nil {
+					cidr, _ := exactCIDR(host)
+					for _, a := range aCIDRs {
+						if cidr == a {
+							observed = append(observed, cidr)
+							foundA = true
+						}
+					}
+				}
+			}
+		}
+		if !foundA {
+			stream.Close()
+			return fmt.Errorf("backend %s did not observe A as an exact source", p.Name)
+		}
+		err = scan.Err()
+		stream.Close()
+		if err != nil {
+			return err
+		}
+	}
+	if len(observed) < len(echoPods) {
+		return fmt.Errorf("backend observed a source outside A's exact interfaces")
+	}
+	var game networkingv1.NetworkPolicy
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-allow-game-from-velocity"}, &game); err != nil {
+		return err
+	}
+	if len(game.Spec.Ingress) == 0 {
+		return fmt.Errorf("velocity ingress policy is not configured")
+	}
+	prevGame := game.DeepCopy()
+	for _, cidr := range observed {
+		exists := false
+		for _, peer := range game.Spec.Ingress[0].From {
+			if peer.IPBlock != nil && peer.IPBlock.CIDR == cidr {
+				exists = true
+				break
+			}
+		}
+		if exists {
+			continue
+		}
+		game.Spec.Ingress[0].From = append(game.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
+	}
+	if err := cl.Patch(ctx, &game, client.MergeFrom(prevGame)); err != nil {
+		return err
+	}
+	// Read fresh resourceVersion so concurrent node health writes cannot be overwritten.
+	if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil {
+		return err
+	}
+	if !placement.Online(&n) {
+		return fmt.Errorf("worker became offline during validation")
+	}
+	before = n.DeepCopy()
+	n.Labels[placement.LabelIdentity] = name
+	n.Labels[placement.LabelRole] = placement.RoleWorker
+	n.Labels[placement.LabelApproved] = "true"
+	taints := n.Spec.Taints[:0]
+	for _, t := range n.Spec.Taints {
+		if t.Key != "felis.lolicon.best/unapproved" {
+			taints = append(taints, t)
+		}
+	}
+	n.Spec.Taints = taints
+	if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil {
+		return err
+	}
+	fmt.Fprintln(stdout, "approved worker", name, "Velocity sources", strings.Join(observed, ","))
+	return nil
+}
+
+func probeArgs(flag string, addresses []string) []string {
+	var args []string
+	for _, a := range addresses {
+		args = append(args, flag, a)
+	}
+	return args
+}
+func probePod(name, ns, node, image string, args []string, game bool) *corev1.Pod {
+	no, yes := false, true
+	uid := int64(1000)
+	labels := map[string]string{"felis.lolicon.best/probe-name": name}
+	if game {
+		labels[v1alpha1.LabelManagedBy] = operator.ManagedByValue
+		labels[v1alpha1.LabelComponent] = operator.ComponentValue
+		labels[v1alpha1.LabelServer] = name
+	}
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, Spec: corev1.PodSpec{NodeName: node, RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &no, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &yes, RunAsUser: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "probe", Image: image, ImagePullPolicy: corev1.PullAlways, Command: []string{"/usr/local/bin/felis", "node-probe"}, Args: args, Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi"), corev1.ResourceCPU: resource.MustParse("200m")}}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &no, ReadOnlyRootFilesystem: &yes, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}}}}}
+}
+func probeJob(name, ns, node, image string, args []string, game bool) *batchv1.Job {
+	p := probePod(name, ns, node, image, args, game)
+	zero := int32(0)
+	deadline := int64(600)
+	return &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, Spec: batchv1.JobSpec{BackoffLimit: &zero, ActiveDeadlineSeconds: &deadline, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: p.Labels}, Spec: p.Spec}}}
+}
+func waitProbePod(ctx context.Context, cl client.Client, p *corev1.Pod) error {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+	for {
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(p), p); err != nil {
+			return err
+		}
+		for _, c := range p.Status.Conditions {
+			if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue {
+				return nil
+			}
+		}
+		if p.Status.Phase == corev1.PodFailed {
+			return fmt.Errorf("probe Pod %s failed", p.Name)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-t.C:
+		}
+	}
+}
+func waitProbeJob(ctx context.Context, cl client.Client, j *batchv1.Job) error {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+	for {
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(j), j); err != nil {
+			return err
+		}
+		for _, c := range j.Status.Conditions {
+			if c.Status != corev1.ConditionTrue {
+				continue
+			}
+			if c.Type == batchv1.JobComplete {
+				return nil
+			}
+			if c.Type == batchv1.JobFailed {
+				return fmt.Errorf("admission probe Job %s failed; inspect its Pod log", j.Name)
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-t.C:
+		}
+	}
+}
+
+func denyNodeAddresses(ctx context.Context, cl client.Client, ns string, nodes []corev1.Node) error {
+	var np networkingv1.NetworkPolicy
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-server-egress"}, &np); err != nil {
+		return err
+	}
+	before := np.DeepCopy()
+	for _, n := range nodes {
+		for _, a := range n.Status.Addresses {
+			if a.Type != corev1.NodeInternalIP && a.Type != corev1.NodeExternalIP {
+				continue
+			}
+			cidr, err := exactCIDR(a.Address)
+			if err != nil {
+				return err
+			}
+			for i := range np.Spec.Egress {
+				for k := range np.Spec.Egress[i].To {
+					block := np.Spec.Egress[i].To[k].IPBlock
+					if block != nil && ((strings.Contains(cidr, ":") && block.CIDR == "::/0") || (!strings.Contains(cidr, ":") && block.CIDR == "0.0.0.0/0")) {
+						block.Except = append(block.Except, cidr)
+					}
+				}
+			}
+		}
+	}
+	return cl.Patch(ctx, &np, client.MergeFrom(before))
+}

cmd/felis/node_control.go

@@ -0,0 +1,581 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strings"
+	"syscall"
+	"time"
+
+	felis "felis.lolicon.best"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/nodecontrol"
+	"felis.lolicon.best/internal/placement"
+	"felis.lolicon.best/internal/platform"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+func cmdNodeControl(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("node-control", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	socket := fs.String("socket", nodecontrol.Socket, "local API-only Unix socket")
+	dir := fs.String("state", "/var/lib/felis/node-control", "root-owned persistent task state")
+	kubeconfig := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "controller kubeconfig")
+	config := fs.String("config", "/etc/felis/felis.host.toml", "host configuration")
+	ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
+	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "control namespace")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "node-control requires root")
+		return 1
+	}
+	cfg, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	cs, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	exe, err := os.Executable()
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer cancel()
+	execute := nodeExecutor{cs: cs, binary: exe, kubeconfig: *kubeconfig, config: *config, namespace: *ns, controlNamespace: *controlNS, stateDir: *dir}
+	socketDir := filepath.Dir(*socket)
+	if err = os.MkdirAll(socketDir, 0750); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err = os.Chown(socketDir, 0, 65532); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err = os.Chmod(socketDir, 0750); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+
+	if existing, err := os.Lstat(*socket); err == nil {
+		if existing.Mode()&os.ModeSocket == 0 {
+			fmt.Fprintln(stderr, "refusing to replace non-socket path")
+			return 1
+		}
+		conn, err := net.DialTimeout("unix", *socket, time.Second)
+		if err == nil {
+			conn.Close()
+			fmt.Fprintln(stderr, "node-control is already running")
+			return 1
+		}
+		if err = os.Remove(*socket); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	listener, err := net.Listen("unix", *socket)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	defer listener.Close()
+	if err = os.Chown(*socket, 0, 65532); err == nil {
+		err = os.Chmod(*socket, 0660)
+	}
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	// /run is recreated at boot; label both the directory and the new socket inode.
+	if os.Getenv("FELIS_NODE_CONTROL_SELINUX") == "1" {
+		if err := exec.Command("chcon", "-R", "-t", "felis_node_control_socket_t", socketDir).Run(); err != nil {
+			fmt.Fprintln(stderr, "node-control socket labeling failed:", err)
+			return 1
+		}
+	}
+	manager, err := nodecontrol.Open(ctx, *dir, execute.run)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	server := &http.Server{Handler: manager.Handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second}
+	go func() { <-ctx.Done(); server.Close() }()
+	fmt.Fprintln(stdout, "node-control listening on", *socket)
+	err = server.Serve(listener)
+	cancel()
+	manager.Wait()
+	if err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	return 0
+}
+
+type nodeExecutor struct {
+	cs                                                                kubernetes.Interface
+	binary, kubeconfig, config, namespace, controlNamespace, stateDir string
+}
+
+func (e nodeExecutor) run(ctx context.Context, r nodecontrol.Request, stage func(string) error, out io.Writer) (resultErr error) {
+	// Host-wide changes and cache-removing admission probes are serialized by the manager.
+	if r.Action == "approve" {
+		node, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		if node.Labels[placement.LabelRole] != placement.RoleWorker {
+			return errors.New("only worker nodes can be approved")
+		}
+		patch := []byte(`{"spec":{"unschedulable":true}}`)
+		if !node.Spec.Unschedulable {
+			patch = []byte(`{"spec":{"unschedulable":true},"metadata":{"annotations":{"felis.lolicon.best/node-control-cordon":"true"}}}`)
+		}
+		if _, err = e.cs.CoreV1().Nodes().Patch(ctx, r.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
+			return err
+		}
+	}
+	if err := e.requireStopped(ctx, r); err != nil {
+		return err
+	}
+	nodes, err := e.cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	controller, peers, err := nodeController(nodes.Items, r.ExternalIP, r.Peers)
+	if err != nil {
+		return err
+	}
+	deployment, err := e.cs.AppsV1().Deployments(e.controlNamespace).Get(ctx, platform.SAAPI, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if len(deployment.Spec.Template.Spec.Containers) == 0 {
+		return errors.New("the Felis API image is unavailable")
+	}
+	image := deployment.Spec.Template.Spec.Containers[0].Image
+	if r.Action == "enable" {
+		if err := stage("pause_operator"); err != nil {
+			return err
+		}
+		scale, err := e.cs.AppsV1().Deployments(e.controlNamespace).GetScale(ctx, platform.SAOperator, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		replicas := scale.Spec.Replicas
+		if replicas < 1 {
+			replicas = 1
+		}
+		scale.Spec.Replicas = 0
+		if _, err = e.cs.AppsV1().Deployments(e.controlNamespace).UpdateScale(ctx, platform.SAOperator, scale, metav1.UpdateOptions{}); err != nil {
+			return err
+		}
+		defer func() {
+			cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+			defer cancel()
+			latest, err := e.cs.AppsV1().Deployments(e.controlNamespace).GetScale(cleanup, platform.SAOperator, metav1.GetOptions{})
+			if err == nil {
+				latest.Spec.Replicas = replicas
+				_, err = e.cs.AppsV1().Deployments(e.controlNamespace).UpdateScale(cleanup, platform.SAOperator, latest, metav1.UpdateOptions{})
+			}
+			if err != nil {
+				fmt.Fprintln(out, "Operator restoration failed:", err)
+				resultErr = fmt.Errorf("operator restoration failed: %w", err)
+			}
+		}()
+		// Drain the old reconciler before the second stopped-state check.
+		for {
+			pods, err := e.cs.CoreV1().Pods(e.controlNamespace).List(ctx, metav1.ListOptions{LabelSelector: platform.LabelComponent + "=" + platform.ComponentOperator})
+			if err != nil {
+				return err
+			}
+			if len(pods.Items) == 0 {
+				break
+			}
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(time.Second):
+			}
+		}
+		if err = e.requireStopped(ctx, r); err != nil {
+			return err
+		}
+		return e.enable(ctx, r, controller, peers, stage, out)
+	}
+	if controller.Labels[placement.LabelRole] != placement.RoleController {
+		return errors.New("distributed mode is not configured on the controller")
+	}
+	if r.Name == controller.Name {
+		return errors.New("worker name must differ from the controller")
+	}
+	// SSH trust and keys are configured on A; no password, key or bootstrap token crosses the API.
+	if err := stage("ssh_check"); err != nil {
+		return err
+	}
+	arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[runtime.GOARCH]
+	if arch == "" {
+		return errors.New("unsupported host architecture")
+	}
+	check := "set -eu\n[ \"$(uname -s)\" = Linux ] || { echo 'worker requires Linux' >&2; exit 1; }\n[ \"$(uname -m)\" = " + shellQuote(arch) + " ] || { echo 'worker architecture differs from controller' >&2; exit 1; }\n"
+	if r.Action == "join" {
+		check += "[ ! -e /etc/rancher/k3s/k3s.yaml ] && [ ! -e /var/lib/rancher/k3s/agent ] || { echo 'k3s is already installed; enrollment will not overwrite an existing node' >&2; exit 1; }\n"
+		check += "ip -o address show | awk '{print $4}' | cut -d/ -f1 | grep -Fx -- " + shellQuote(r.ExternalIP) + " >/dev/null || { echo 'fixed node IP is not assigned to the worker' >&2; exit 1; }\n"
+	}
+
+	if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi", strings.NewReader(check), out); err != nil {
+		return fmt.Errorf("worker preflight or SSH connection failed: %w", err)
+	}
+	if r.Action == "join" {
+		for _, n := range nodes.Items {
+			if n.Name == r.Name {
+				return errors.New("node already exists; use approval instead of reinstalling it")
+			}
+		}
+		if err = e.join(ctx, r, controller, peers, stage, out); err != nil {
+			return err
+		}
+	}
+	if err = stage("approval"); err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, e.binary, "node", "approve", "--name", r.Name, "--ssh-target", r.SSHTarget, "--image", image, "--kubeconfig", e.kubeconfig, "--namespace", e.namespace, "--control-namespace", e.controlNamespace)
+	cmd.Stdout = out
+	cmd.Stderr = out
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("node approval failed; node remains quarantined: %w", err)
+	}
+	// Admission succeeded; release only the task's scheduling quarantine.
+	admitted, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if admitted.Annotations["felis.lolicon.best/node-control-cordon"] == "true" {
+		if _, err = e.cs.CoreV1().Nodes().Patch(ctx, r.Name, types.MergePatchType, []byte(`{"spec":{"unschedulable":false},"metadata":{"annotations":{"felis.lolicon.best/node-control-cordon":null}}}`), metav1.PatchOptions{}); err != nil {
+			return fmt.Errorf("node approved but scheduling remains disabled: %w", err)
+		}
+	}
+	return stage("complete")
+}
+func (e nodeExecutor) requireStopped(ctx context.Context, r nodecontrol.Request) error {
+	// Even pre-existing worker names may contain worlds. Never run installer/admission on an occupied worker.
+	ss, err := e.cs.AppsV1().StatefulSets(e.namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, s := range ss.Items {
+		if r.Action != "enable" && s.Spec.Template.Spec.NodeSelector[placement.LabelIdentity] != r.Name {
+			continue
+		}
+		if s.Spec.Replicas != nil && *s.Spec.Replicas > 0 {
+			return fmt.Errorf("StatefulSet %s still requests %d replicas; stop the server before changing nodes", s.Name, *s.Spec.Replicas)
+		}
+	}
+	pods, err := e.cs.CoreV1().Pods(e.namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, p := range pods.Items {
+		if r.Action != "enable" && p.Spec.NodeName != r.Name && p.Spec.NodeSelector[placement.LabelIdentity] != r.Name {
+			continue
+		}
+		if p.Status.Phase != corev1.PodSucceeded && p.Status.Phase != corev1.PodFailed {
+			return fmt.Errorf("pod %s has not exited (phase %s); wait for game and maintenance workloads to stop", p.Name, p.Status.Phase)
+		}
+	}
+	// Prevent a pending wake intent racing admission or an installation.
+	raw, err := e.cs.CoreV1().RESTClient().Get().AbsPath("/apis/" + v1alpha1.GroupVersion.Group + "/" + v1alpha1.GroupVersion.Version + "/namespaces/" + e.namespace + "/minecraftservers").DoRaw(ctx)
+	if err != nil {
+		return err
+	}
+	var servers v1alpha1.MinecraftServerList
+	if err = json.Unmarshal(raw, &servers); err != nil {
+		return err
+	}
+	for _, s := range servers.Items {
+		if r.Action != "enable" && s.Spec.NodeName != r.Name && s.Status.NodeName != r.Name {
+			continue
+		}
+		if s.Spec.DesiredState != "" && string(s.Spec.DesiredState) != "Stopped" {
+			return fmt.Errorf("server %s must have stopped intent", s.Name)
+		}
+	}
+	return nil
+}
+
+func nodeController(nodes []corev1.Node, ip string, extra []string) (*corev1.Node, []string, error) {
+	var controller *corev1.Node
+	peers := append([]string{}, extra...)
+	for i := range nodes {
+		n := &nodes[i]
+		_, controlPlane := n.Labels["node-role.kubernetes.io/control-plane"]
+		if controlPlane || n.Labels[placement.LabelRole] == placement.RoleController {
+			if controller != nil {
+				return nil, nil, errors.New("exactly one controller is required")
+			}
+			controller = n
+		}
+		for _, a := range n.Status.Addresses {
+			if a.Type == corev1.NodeExternalIP || a.Type == corev1.NodeInternalIP {
+				cidr, err := exactCIDR(a.Address)
+				if err != nil {
+					return nil, nil, err
+				}
+				peers = append(peers, cidr)
+			}
+		}
+	}
+	if controller == nil || !placement.Online(controller) {
+		return nil, nil, errors.New("the sole controller must be online")
+	}
+	if ip != "" {
+		cidr, err := exactCIDR(ip)
+		if err != nil {
+			return nil, nil, err
+		}
+		peers = append(peers, cidr)
+	}
+	slices.Sort(peers)
+	peers = slices.Compact(peers)
+	return controller, peers, nil
+}
+func controllerIP(n *corev1.Node) string {
+	for _, kind := range []corev1.NodeAddressType{corev1.NodeExternalIP, corev1.NodeInternalIP} {
+		for _, a := range n.Status.Addresses {
+			if a.Type == kind {
+				return a.Address
+			}
+		}
+	}
+	return ""
+}
+func (e nodeExecutor) ssh(ctx context.Context, target, command string, in io.Reader, out io.Writer) error {
+	cmd := exec.CommandContext(ctx, "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=10", "--", target, command)
+	cmd.Stdin = in
+	cmd.Stdout = out
+	cmd.Stderr = out
+	return cmd.Run()
+}
+func (e nodeExecutor) join(ctx context.Context, r nodecontrol.Request, controller *corev1.Node, peers []string, stage func(string) error, out io.Writer) error {
+	registry, err := e.cs.CoreV1().Services(e.controlNamespace).Get(ctx, "registry", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if err = stage("firewall"); err != nil {
+		return err
+	}
+	// All peers must be updated before the new agent is admitted. Existing worker SSH targets must be configured by stable node name.
+	list, err := e.cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, n := range list.Items {
+		if n.Name == controller.Name || n.Name == r.Name {
+			continue
+		}
+		if n.Labels[placement.LabelRole] != placement.RoleWorker {
+			return fmt.Errorf("node %s has an unknown role", n.Name)
+		}
+		script := shellQuote(e.binary) + " node firewall --peers " + shellQuote(strings.Join(peers, ",")) + " --controller-ip " + shellQuote(controllerIP(controller))
+		script += " --namespace " + shellQuote(e.namespace) + " --control-namespace " + shellQuote(e.controlNamespace)
+		if err = e.ssh(ctx, n.Name, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n "+script+"; fi", nil, out); err != nil {
+			return fmt.Errorf("update peer firewall on %s: %w", n.Name, err)
+		}
+	}
+	cmd := exec.CommandContext(ctx, e.binary, "node", "firewall", "--controller", "--controller-ip", controllerIP(controller), "--peers", strings.Join(peers, ","), "--namespace", e.namespace, "--control-namespace", e.controlNamespace)
+	cmd.Stdout = out
+	cmd.Stderr = out
+	if err = cmd.Run(); err != nil {
+		return err
+	}
+	if err = stage("bootstrap_token"); err != nil {
+		return err
+	}
+	temp, err := os.MkdirTemp("", "felis-node-join-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(temp)
+	tokenPath := filepath.Join(temp, "bootstrap")
+	tokenCmd := exec.CommandContext(ctx, e.binary, "node", "token", "--name", r.Name, "--ttl", "1h", "--out", tokenPath)
+	tokenCmd.Stdout = out
+	tokenCmd.Stderr = out
+	if err = tokenCmd.Run(); err != nil {
+		return err
+	}
+	// Revoke even on failure. The worker exchanges bootstrap credentials for its node identity.
+	token, err := os.ReadFile(tokenPath)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		exec.CommandContext(cleanup, "k3s", "token", "delete", bootstrapTokenID(string(token))).Run()
+	}()
+	remoteDir := "/var/tmp/felis-node-" + filepath.Base(temp)
+	if err = stage("transfer"); err != nil {
+		return err
+	}
+	script := "set -eu; umask 077; mkdir " + shellQuote(remoteDir) + "; cat > " + shellQuote(remoteDir+"/bootstrap")
+	rootCommand := "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi"
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		e.ssh(cleanup, r.SSHTarget, rootCommand, strings.NewReader("rm -rf -- "+shellQuote(remoteDir)), io.Discard)
+	}()
+	// stdin carries the token; it is never in command arguments, task records or logs.
+	if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n sh -c "+shellQuote(script)+"; fi", strings.NewReader(string(token)), out); err != nil {
+		return err
+	}
+
+	binary, err := os.Open(e.binary)
+	if err != nil {
+		return err
+	}
+	defer binary.Close()
+	script = "set -eu; cat > " + shellQuote(remoteDir+"/felis") + "; chmod 0700 " + shellQuote(remoteDir+"/felis")
+	if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n sh -c "+shellQuote(script)+"; fi", binary, out); err != nil {
+		return err
+	}
+	if err = stage("install_worker"); err != nil {
+		return err
+	}
+	script = "set -eu\nexport FELIS_K3S_VERSION=" + shellQuote(controller.Status.NodeInfo.KubeletVersion) + "\n" + shellQuote(remoteDir+"/felis") + " node join --name " + shellQuote(r.Name) + " --server " + shellQuote("https://"+net.JoinHostPort(controllerIP(controller), "6443")) + " --external-ip " + shellQuote(r.ExternalIP) + " --token-file " + shellQuote(remoteDir+"/bootstrap") + " --registry-ip " + shellQuote(registry.Spec.ClusterIP) + " --peers " + shellQuote(strings.Join(peers, ",")) + "\n"
+	if err = e.ssh(ctx, r.SSHTarget, rootCommand, strings.NewReader(script), out); err != nil {
+		return fmt.Errorf("worker installation failed: %w", err)
+	}
+	if err = stage("wait_ready"); err != nil {
+		return err
+	}
+	deadline := time.NewTimer(5 * time.Minute)
+	defer deadline.Stop()
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+	for {
+		n, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
+		if err == nil && placement.Online(n) {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-deadline.C:
+			return errors.New("worker did not become Ready within five minutes")
+		case <-ticker.C:
+		}
+	}
+}
+func (e nodeExecutor) enable(ctx context.Context, r nodecontrol.Request, controller *corev1.Node, peers []string, stage func(string) error, out io.Writer) error {
+	if controllerIP(controller) != r.ExternalIP {
+		return errors.New("controller IP must match the registered fixed node address")
+	}
+	if len(peers) == 0 {
+		return errors.New("peer addresses are required")
+	}
+	if err := stage("database_backup"); err != nil {
+		return err
+	}
+	backup := exec.CommandContext(ctx, e.binary, "db", "backup", "-config", e.config, "-dir", "/var/lib/felis/db-backups", "-label", "pre-migrate")
+	backup.Stdout = out
+	backup.Stderr = out
+	if err := backup.Run(); err != nil {
+		return err
+	}
+	if err := stage("cluster_backup"); err != nil {
+		return err
+	}
+	if err := e.backupCluster(ctx, out); err != nil {
+		return err
+	}
+	if err := stage("configure_controller"); err != nil {
+		return err
+	}
+	patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s","%s":"controller"}}}`, placement.LabelIdentity, controller.Name, placement.LabelRole)
+	if _, err := e.cs.CoreV1().Nodes().Patch(ctx, controller.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil {
+		return err
+	}
+	for _, name := range []string{platform.SAAPI, platform.SAOperator, platform.PostgresName, "registry"} {
+		body := fmt.Sprintf(`{"spec":{"template":{"spec":{"nodeSelector":{"%s":"%s"}}}}}`, placement.LabelIdentity, controller.Name)
+		if _, err := e.cs.AppsV1().Deployments(e.controlNamespace).Patch(ctx, name, types.MergePatchType, []byte(body), metav1.PatchOptions{}); err != nil {
+			return err
+		}
+	}
+	if err := stage("install_controller"); err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, "bash", "-s")
+	cmd.Stdin = strings.NewReader(felis.BootstrapScript())
+	cmd.Stdout = out
+	cmd.Stderr = out
+	cmd.Env = append(os.Environ(), "FELIS_DISTRIBUTED=1", "FELIS_NODE_EXTERNAL_IP="+r.ExternalIP, "FELIS_PEER_CIDRS="+strings.Join(peers, ","), "FELIS_BOOTSTRAP_FROM_TUI=1", "FELIS_BOOTSTRAP_BINARY="+e.binary, "FELIS_NO_SETUP=1", "FELIS_NODE_CONTROL_TASK=1", "FELIS_INSTALL_MODE=full")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("controller installation failed; retain the database backup: %w", err)
+	}
+	return stage("complete")
+}
+
+func bootstrapTokenID(token string) string {
+	token = strings.TrimSpace(token)
+	if _, short, ok := strings.Cut(token, "::"); ok {
+		token = short
+	}
+	id, _, _ := strings.Cut(token, ".")
+	return id
+}
+
+func (e nodeExecutor) backupCluster(ctx context.Context, out io.Writer) (err error) {
+	dir := filepath.Join(filepath.Dir(e.stateDir), "cluster-backups")
+	if err = os.MkdirAll(dir, 0700); err != nil {
+		return err
+	}
+	path := filepath.Join(dir, "pre-distributed-"+time.Now().UTC().Format("20060102T150405.000000000Z")+".tar")
+	stop := exec.CommandContext(ctx, "systemctl", "stop", "k3s")
+	stop.Stdout = out
+	stop.Stderr = out
+	if err = stop.Run(); err != nil {
+		return err
+	}
+	defer func() {
+		restart, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		defer cancel()
+		cmd := exec.CommandContext(restart, "systemctl", "start", "k3s")
+		cmd.Stdout = out
+		cmd.Stderr = out
+		if restartErr := cmd.Run(); restartErr != nil {
+			err = fmt.Errorf("k3s restart failed after snapshot: %w", restartErr)
+		}
+	}()
+	cmd := exec.CommandContext(ctx, "tar", "-cf", path, "-C", "/var/lib/rancher/k3s/server", "db", "token", "tls")
+	cmd.Stdout = out
+	cmd.Stderr = out
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("cluster snapshot failed: %w", err)
+	}
+	if err = os.Chmod(path, 0600); err != nil {
+		return err
+	}
+	fmt.Fprintln(out, "Cluster snapshot:", path)
+	return nil
+}

cmd/felis/node_control_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/nodecontrol"
+	"felis.lolicon.best/internal/platform"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func TestNodeControlTopologyAndBootstrapID(t *testing.T) {
+	node := corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "controller", Labels: map[string]string{"node-role.kubernetes.io/control-plane": "true"}}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.0.2.1"}}}}
+	controller, peers, err := nodeController([]corev1.Node{node}, "192.0.2.2", []string{"192.0.2.1/32"})
+	if err != nil || controller.Name != "controller" || strings.Join(peers, ",") != "192.0.2.1/32,192.0.2.2/32" {
+		t.Fatal(controller, peers, err)
+	}
+	duplicate := node
+	duplicate.Name = "second"
+	if _, _, err = nodeController([]corev1.Node{node, duplicate}, "", nil); err == nil {
+		t.Fatal("multiple controllers accepted")
+	}
+	for _, token := range []string{"abcdef.0123456789abcdef", "K10hash::abcdef.0123456789abcdef\n"} {
+		if bootstrapTokenID(token) != "abcdef" {
+			t.Fatal("token secret passed to revocation")
+		}
+	}
+}
+func TestNodeControlRejectsActiveWorkloadsBeforeHostCommands(t *testing.T) {
+	replicas := int32(1)
+	set := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"}, Spec: appsv1.StatefulSetSpec{Replicas: &replicas}}
+	executor := nodeExecutor{cs: fake.NewSimpleClientset(set), namespace: platform.DefaultMinecraftNamespace}
+	if err := executor.requireStopped(context.Background(), nodecontrol.Request{Action: "enable"}); err == nil {
+		t.Fatal("host changes permitted with active worlds")
+	}
+}

cmd/felis/node_firewall.go

@@ -0,0 +1,187 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+func nodeFirewall(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("node firewall", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	peers := fs.String("peers", "", "comma-separated exact node IP CIDRs")
+	controller := fs.String("controller-ip", "", "controller address (optionally :6443)")
+	main := fs.Bool("controller", false, "A hosts the public API NodePort")
+	pod := fs.String("pod-cidr", "10.42.0.0/16", "cluster Pod CIDR")
+	dryRun := fs.Bool("dry-run", false, "print firewall scripts without installing")
+	controlNS := fs.String("control-namespace", "felis", "controller namespace")
+	minecraftNS := fs.String("namespace", "minecraft", "game namespace")
+	apiIP := fs.String("api-service-ip", "10.43.0.1", "Kubernetes API Service IP")
+	port := fs.Int("node-port", 30443, "API NodePort to block on workers before DNAT")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if host, _, err := net.SplitHostPort(*controller); err == nil {
+		*controller = host
+	}
+	if net.ParseIP(*apiIP) == nil || net.ParseIP(*controller) == nil || *port < 30000 || *port > 32767 {
+		fmt.Fprintln(stderr, "node firewall: controller IP and NodePort required")
+		return 2
+	}
+	if _, _, err := net.ParseCIDR(*pod); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+	var v4, v6 []string
+	for _, p := range strings.Split(*peers, ",") {
+		ip, n, err := net.ParseCIDR(p)
+		if err != nil {
+			fmt.Fprintln(stderr, "node firewall: invalid peer CIDR")
+			return 2
+		}
+		ones, bits := n.Mask.Size()
+		if ones != bits {
+			fmt.Fprintln(stderr, "node firewall: only exact peer addresses accepted")
+			return 2
+		}
+		if ip.To4() != nil {
+			v4 = append(v4, n.String())
+		} else {
+			v6 = append(v6, n.String())
+		}
+	}
+	script := "#!/bin/bash\nset -euo pipefail\n"
+	for _, f := range []struct {
+		bin      string
+		peers    []string
+		metadata string
+	}{{"iptables", v4, "169.254.0.0/16"}, {"ip6tables", v6, "fe80::/10"}} {
+		if f.bin == "ip6tables" && len(f.peers) == 0 {
+			continue
+		}
+		b := f.bin + " -w 10"
+		script += b + " -N FELIS-HOST 2>/dev/null || true\n" + b + " -F FELIS-HOST\n"
+		script += b + " -N FELIS-FORWARD 2>/dev/null || true\n" + b + " -F FELIS-FORWARD\n"
+		script += b + " -t raw -N FELIS-NODEPORT 2>/dev/null || true\n" + b + " -t raw -F FELIS-NODEPORT\n"
+		for _, c := range []struct{ table, parent, chain string }{{"filter", "INPUT", "FELIS-HOST"}, {"filter", "FORWARD", "FELIS-FORWARD"}, {"raw", "PREROUTING", "FELIS-NODEPORT"}} {
+			script += "while " + b + " -t " + c.table + " -D " + c.parent + " -j " + c.chain + " 2>/dev/null; do :; done\n" + b + " -t " + c.table + " -I " + c.parent + " 1 -j " + c.chain + "\n"
+		}
+		script += b + " -A FELIS-HOST -i lo -j RETURN\n"
+		if *main {
+			script += b + " -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -A FELIS-HOST -j FELIS-CONTROL\n"
+			script += b + " -t raw -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -t raw -A FELIS-NODEPORT -j FELIS-CONTROL\n"
+		}
+		script += b + " -A FELIS-HOST -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n" + b + " -A FELIS-FORWARD -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n"
+		family := 4
+		if f.bin == "ip6tables" {
+			family = 6
+		}
+		podIP, _, _ := net.ParseCIDR(*pod)
+		podFamily := 6
+		if podIP.To4() != nil {
+			podFamily = 4
+		}
+		if family == podFamily {
+			script += b + " -A FELIS-HOST -s " + *pod + " -j DROP\n"
+			// raw precedes DNAT and kube-router's filter ACCEPT rules. Block new
+			// host connections while preserving established Velocity/RCON replies.
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --syn -j DROP\n"
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p udp -j DROP\n"
+			if (net.ParseIP(*apiIP).To4() != nil) == (family == 4) {
+				script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + *apiIP + " -p tcp --dport 443 --syn -j DROP\n"
+			}
+
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n"
+			script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + f.metadata + " -j DROP\n"
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + f.metadata + " -j DROP\n"
+			for _, p := range f.peers {
+				script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + p + " -j DROP\n"
+				script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p tcp --syn -j DROP\n"
+				script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p udp -j DROP\n"
+			}
+		}
+		if !*main {
+			script += b + " -t raw -A FELIS-NODEPORT -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n"
+		}
+		for _, p := range f.peers {
+			script += b + " -A FELIS-HOST -s " + p + " -p udp --dport 51820:51821 -j RETURN\n"
+		}
+		script += b + " -A FELIS-HOST -p udp --dport 51820:51821 -j DROP\n"
+		ctrlIP := net.ParseIP(*controller)
+		ctrlFamily := 6
+		if ctrlIP.To4() != nil {
+			ctrlFamily = 4
+		}
+		if *main {
+			for _, p := range f.peers {
+				script += b + " -A FELIS-HOST -s " + p + " -p tcp --dport 6443 -j RETURN\n"
+			}
+		}
+		if ctrlFamily == family {
+			script += b + " -A FELIS-HOST -s " + ctrlIP.String() + " -p tcp --dport 10250 -j RETURN\n"
+		}
+		script += b + " -A FELIS-HOST -p tcp -m multiport --dports 6443,6444,10250,10255,2379,2380,5000,5001,15432 -j DROP\n"
+	}
+	if *dryRun {
+		fmt.Fprint(stdout, script)
+		if *main {
+			fmt.Fprint(stdout, controlFirewallScript(*controlNS, *minecraftNS))
+		}
+		return 0
+	}
+	if err := os.MkdirAll("/etc/felis", 0700); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err := os.WriteFile("/etc/felis/node-firewall.sh", []byte(script), 0700); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	unit := "[Unit]\nDescription=Felis host and NodePort isolation\nAfter=network-online.target firewalld.service ufw.service\nBefore=k3s.service k3s-agent.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/node-firewall.sh\nRemainAfterExit=yes\n[Install]\nWantedBy=multi-user.target\n"
+	if err := os.WriteFile("/etc/systemd/system/felis-node-firewall.service", []byte(unit), 0644); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if *main {
+		refresh := controlFirewallScript(*controlNS, *minecraftNS)
+		if err := os.WriteFile("/etc/felis/control-firewall.sh", []byte(refresh), 0700); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		svc := "[Unit]\nDescription=Refresh exact controller Pod access to the apiserver\nAfter=k3s.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/control-firewall.sh\n"
+		timer := "[Unit]\nDescription=Track trusted controller Pods after rescheduling\n[Timer]\nOnBootSec=5s\nOnUnitActiveSec=5s\n[Install]\nWantedBy=timers.target\n"
+		if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.service", []byte(svc), 0644); err != nil {
+			return 1
+		}
+		if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.timer", []byte(timer), 0644); err != nil {
+			return 1
+		}
+	}
+	for _, cmd := range []*exec.Cmd{exec.Command("systemctl", "daemon-reload"), exec.Command("systemctl", "enable", "felis-node-firewall.service"), exec.Command("bash", "/etc/felis/node-firewall.sh")} {
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		if err := cmd.Run(); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	if *main {
+		cmd := exec.Command("systemctl", "enable", "--now", "felis-control-firewall.timer")
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		if err := cmd.Run(); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	return 0
+}
+
+func controlFirewallScript(controlNS, minecraftNS string) string {
+	return "#!/bin/bash\nset -euo pipefail\niptables -w 10 -F FELIS-CONTROL\niptables -w 10 -t raw -F FELIS-CONTROL\n/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{\" \"}{.spec.serviceAccountName}{\" \"}{.status.podIP}{\"\\n\"}{end}' | while read -r ns sa ip; do\ncase \"$ns/$sa\" in " + shellQuote(controlNS+"/felis-api") + "|" + shellQuote(controlNS+"/felis-operator") + "|" + shellQuote(minecraftNS+"/felis-reaper") + "|kube-system/*) ;; *) continue;; esac\n[[ $ip =~ ^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$ ]] || continue\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p tcp --dport 6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p tcp -m multiport --dports 443,6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\ndone\n"
+}

cmd/felis/node_firewall_test.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"bytes"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestDistributedFirewallPathsAndSyntax(t *testing.T) {
+	for _, controller := range []bool{false, true} {
+		args := []string{"--dry-run", "--peers", "192.0.2.1/32,192.0.2.2/32", "--controller-ip", "192.0.2.1"}
+		if controller {
+			args = append(args, "--controller")
+		}
+		var out, err bytes.Buffer
+		if code := nodeFirewall(args, &out, &err); code != 0 {
+			t.Fatal(code, err.String())
+		}
+		script := out.String()
+		for _, must := range []string{"-I INPUT 1 -j FELIS-HOST", "-I FORWARD 1 -j FELIS-FORWARD", "-t raw -I PREROUTING 1 -j FELIS-NODEPORT", "-A FELIS-HOST -s 10.42.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --dport 30443 -j DROP", "-d 169.254.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --syn -j DROP", "-d 10.43.0.1 -p tcp --dport 443 --syn -j DROP"} {
+			if !strings.Contains(script, must) {
+				t.Fatal("missing protection", must)
+			}
+		}
+		if !controller && strings.Contains(script, " -p tcp --dport 6443 -j RETURN") {
+			t.Fatal("worker exposed apiserver")
+		}
+		check := exec.Command("bash", "-n")
+		check.Stdin = strings.NewReader(script)
+		if result, e := check.CombinedOutput(); e != nil {
+			t.Fatalf("invalid firewall script: %s %v", result, e)
+		}
+	}
+	var out, err bytes.Buffer
+	if code := nodeFirewall([]string{"--dry-run", "--peers", "10.0.0.0/8", "--controller-ip", "10.0.0.1"}, &out, &err); code != 2 {
+		t.Fatal("broad node range accepted")
+	}
+}

cmd/felis/node_probe.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"time"
+)
+
+// A trusted probe runs with the same server labels and security context before node admission.
+func cmdNodeProbe(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("node-probe", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	listen := fs.String("listen", "", "listen and print observed source addresses (admission only)")
+	var open, closed multiFlag
+	fs.Var(&open, "open", "required reachable host:port")
+	fs.Var(&closed, "closed", "required blocked host:port")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *listen != "" {
+		l, err := net.Listen("tcp", *listen)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		defer l.Close()
+		for {
+			c, err := l.Accept()
+			if err != nil {
+				return 1
+			}
+			fmt.Fprintln(stdout, "felis-probe-source", c.RemoteAddr().String())
+			c.Write([]byte("felis-probe\n"))
+			c.Close()
+		}
+	}
+	time.Sleep(3 * time.Second)
+	for _, check := range []struct {
+		addresses []string
+		wantOpen  bool
+	}{{open, true}, {closed, false}} {
+		for _, addr := range check.addresses {
+			c, err := net.DialTimeout("tcp", addr, 2*time.Second)
+			if c != nil {
+				c.Close()
+			}
+			if (err == nil) != check.wantOpen {
+				fmt.Fprintf(stderr, "node-probe: %s open=%t, expected %t\n", addr, err == nil, check.wantOpen)
+				return 1
+			}
+		}
+	}
+	return 0
+}

cmd/felis/nodes.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	felis "felis.lolicon.best"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/platform"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func cmdNode(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprintln(stderr, "felis node: list | token | join | approve | firewall")
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis node requires root/sudo")
+		return 1
+	}
+	if args[0] == "firewall" {
+		return nodeFirewall(args[1:], stdout, stderr)
+	}
+	fs := flag.NewFlagSet("node "+args[0], flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	name := fs.String("name", "", "stable worker node name")
+	kubeconfig := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local administrator kubeconfig")
+	ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
+	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "controller namespace")
+	image := fs.String("image", "", "Felis image to re-pull and use for admission probes")
+	remote := fs.String("ssh-target", "", "SSH target of the trusted worker (normal SSH host verification applies)")
+	out := fs.String("out", "", "token output file; never printed to stdout")
+	ttl := fs.Duration("ttl", 10*time.Minute, "bootstrap token lifetime (maximum 1h)")
+	server := fs.String("server", "", "A's https://address:6443 endpoint for join")
+	tokenFile := fs.String("token-file", "", "CA-pinned bootstrap token file for join")
+	registry := fs.String("registry-ip", "", "registry ClusterIP for join")
+	peers := fs.String("peers", "", "comma-separated exact peer CIDRs for host firewall")
+	external := fs.String("external-ip", "", "this worker's fixed external IP")
+	if err := fs.Parse(args[1:]); err != nil {
+		return 2
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer cancel()
+	switch args[0] {
+	case "token":
+		if *name == "" || *out == "" || *ttl <= 0 || *ttl > time.Hour {
+			fmt.Fprintln(stderr, "node token: name, output file and lifetime <=1h required")
+			return 2
+		}
+		cmd := exec.CommandContext(ctx, "k3s", "token", "create", "--ttl", ttl.String(), "--description", "Felis worker "+*name)
+		cmd.Stderr = stderr
+		raw, err := cmd.Output()
+		if err != nil {
+			fmt.Fprintln(stderr, "node token: creation failed:", err)
+			return 1
+		}
+		f, err := os.OpenFile(*out, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		_, err = f.Write(raw)
+		if err == nil {
+			err = f.Sync()
+		}
+		f.Close()
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		fmt.Fprintln(stdout, "limited bootstrap token written to", *out)
+		return 0
+	case "join":
+		exe, err := os.Executable()
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		cmd := exec.CommandContext(ctx, "bash", "-s")
+		cmd.Stdin = strings.NewReader(felis.BootstrapScript())
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		cmd.Env = append(os.Environ(), "FELIS_INSTALL_MODE=worker", "FELIS_BOOTSTRAP_FROM_TUI=1", "FELIS_BOOTSTRAP_BINARY="+exe, "FELIS_NODE_NAME="+*name, "FELIS_SERVER_URL="+*server, "FELIS_BOOTSTRAP_TOKEN_FILE="+*tokenFile, "FELIS_REGISTRY_CLUSTER_IP="+*registry, "FELIS_PEER_CIDRS="+*peers, "FELIS_NODE_EXTERNAL_IP="+*external)
+		if err = cmd.Run(); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		return 0
+	case "list", "approve":
+	default:
+		fmt.Fprintln(stderr, "unknown node operation")
+		return 2
+	}
+	cfg, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	scheme := runtime.NewScheme()
+	clientgoscheme.AddToScheme(scheme)
+	v1alpha1.AddToScheme(scheme)
+	cl, err := client.New(cfg, client.Options{Scheme: scheme})
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	m := &distributed.Manager{Client: cl, Namespace: *ns}
+	if args[0] == "list" {
+		nodes, err := m.Nodes(ctx)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		json.NewEncoder(stdout).Encode(nodes)
+		return 0
+	}
+	if *name == "" || *image == "" || *remote == "" || strings.HasPrefix(*remote, "-") {
+		fmt.Fprintln(stderr, "node approve requires name, image and SSH target")
+		return 2
+	}
+	cs, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err = approveNode(ctx, cl, cs, *ns, *controlNS, *name, *image, *remote, stdout); err != nil {
+		fmt.Fprintln(stderr, "node remains quarantined:", err)
+		return 1
+	}
+	return 0
+}
+
+func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" }
+func exactCIDR(ip string) (string, error) {
+	parsed := net.ParseIP(ip)
+	if parsed == nil {
+		return "", fmt.Errorf("invalid node address %q", ip)
+	}
+	if parsed.To4() != nil {
+		return parsed.String() + "/32", nil
+	}
+	return parsed.String() + "/128", nil
+}

cmd/felis/offsite.go

@@ -0,0 +1,946 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const offsiteUsage = `usage:
+  felis offsite sync         [-config path] [-archive-dir dir] [-db-dir dir] [-registry host:port|off]
+                             [-uploads-dir dir] [-status-file path]
+  felis offsite status       [-config path] [-status-file path]
+  felis offsite list         [-config path]
+  felis offsite fetch-db     [-config path | -endpoint url -bucket name [-region r] [-prefix p]]
+                             [-dir dir] latest|<bundle>
+  felis offsite fetch-worlds [-config path] [-archive-dir dir]
+  felis offsite fetch-images [-config path] [-registry host:port] [-at version]
+  felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
+  felis offsite check-key    [-config path]
+  felis offsite take-over    [-config path] [-status-file path] [-yes]
+  felis offsite keygen
+
+Every verb but keygen reads the bucket credentials and the encryption key from
+the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY,
+FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
+-env-file (default /etc/felis/offsite.env).
+
+check-key tells whether the key is the one the bucket's objects are sealed
+with, writing nothing; it exits 3 when they are sealed with another key, and
+sync then refuses to write or prune anything in the bucket.
+
+take-over names the host that writes the bucket, writing nothing; it exits 4
+when that is another host and this one never wrote it, and 5 when another
+host took the bucket over from this one. A host built from another host's
+backup (a rehearsal, or a rebuild) copies nothing into that host's bucket
+until -yes makes it the writer; the host it replaces then stops copying and
+says so.
+`
+
+// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
+// felis-offsite.service unit loads it as its EnvironmentFile.
+const defaultOffsiteEnvFile = "/etc/felis/offsite.env"
+
+// cmdOffsite implements `felis offsite`: the off-site copy of the world
+// archives, the database bundles, the registry's user images and the
+// submission uploads (internal/offsite). felis-offsite.timer runs `sync`
+// hourly on the host; the fetch verbs are the way back after the node is lost
+// (docs/troubleshooting.md §16).
+func cmdOffsite(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("offsite "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, offsiteUsage) }
+	switch verb {
+	case "sync":
+		return offsiteSync(fs, rest, stdout, stderr)
+	case "status":
+		return offsiteStatus(fs, rest, stdout, stderr)
+	case "list":
+		return offsiteList(fs, rest, stdout, stderr)
+	case "fetch-db":
+		return offsiteFetchDB(fs, rest, stdout, stderr)
+	case "fetch-worlds":
+		return offsiteFetchWorlds(fs, rest, stdout, stderr)
+	case "fetch-images":
+		return offsiteFetchImages(fs, rest, stdout, stderr)
+	case "fetch-uploads":
+		return offsiteFetchUploads(fs, rest, stdout, stderr)
+	case "check-key":
+		return offsiteCheckKey(fs, rest, stdout, stderr)
+	case "take-over":
+		return offsiteTakeOver(fs, rest, stdout, stderr)
+	case "keygen":
+		k, err := offsite.NewKey()
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite keygen: %v\n", err)
+			return 1
+		}
+		fmt.Fprintln(stdout, k)
+		return 0
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, offsiteUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis offsite: unknown verb %q\n%s", verb, offsiteUsage)
+	return 2
+}
+
+// offsiteEnv is the resolved [offsite] binding: the bucket and the key.
+type offsiteEnv struct {
+	cfg    config.OffsiteConfig
+	bucket *offsite.S3
+	key    []byte
+}
+
+// loadEnvFile sets each KEY=VALUE of path that is not already in the
+// environment, so a root shell runs a command the same way its unit does.
+// A missing file is not an error.
+func loadEnvFile(path string) error {
+	if path == "" {
+		return nil
+	}
+	f, err := os.Open(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		k = strings.TrimSpace(strings.TrimPrefix(k, "export "))
+		v = strings.TrimSpace(v)
+		if len(v) >= 2 && (v[0] == '"' || v[0] == '\'') && v[len(v)-1] == v[0] {
+			v = v[1 : len(v)-1]
+		}
+		if os.Getenv(k) == "" {
+			os.Setenv(k, v)
+		}
+	}
+	return sc.Err()
+}
+
+// resolveOffsite builds the bucket client and parses the key for c.
+func resolveOffsite(c config.OffsiteConfig) (*offsiteEnv, error) {
+	if !c.Enabled() {
+		return nil, errors.New("no [offsite] bucket is configured (docs/troubleshooting.md §16, \"Keep a copy somewhere else\")")
+	}
+	need := func(ref, what string) (string, error) {
+		v := os.Getenv(ref)
+		if v == "" {
+			return "", fmt.Errorf("%s: environment variable %s is empty (set it, or put it in %s)", what, ref, defaultOffsiteEnvFile)
+		}
+		return v, nil
+	}
+	ak, err := need(c.AccessKeyRef, "access key")
+	if err != nil {
+		return nil, err
+	}
+	sk, err := need(c.SecretKeyRef, "secret key")
+	if err != nil {
+		return nil, err
+	}
+	rawKey, err := need(c.KeyRef, "encryption key")
+	if err != nil {
+		return nil, err
+	}
+	key, err := offsite.ParseKey(rawKey)
+	if err != nil {
+		return nil, err
+	}
+	b, err := offsite.NewS3(offsite.S3Config{
+		Endpoint: c.Endpoint, Region: c.Region, Bucket: c.Bucket, Prefix: c.Prefix,
+		AccessKey: ak, SecretKey: sk,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &offsiteEnv{cfg: c, bucket: b, key: key}, nil
+}
+
+// loadOffsite loads felis.toml and the env file and resolves [offsite].
+func loadOffsite(cfgPath, envFile string) (*config.Config, *offsiteEnv, error) {
+	if err := loadEnvFile(envFile); err != nil {
+		return nil, nil, fmt.Errorf("read %s: %w", envFile, err)
+	}
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	env, err := resolveOffsite(cfg.Offsite)
+	if err != nil {
+		return cfg, nil, err
+	}
+	return cfg, env, nil
+}
+
+func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
+	dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory bundled into the database bundle taken after archives are copied ("" for none)`)
+	registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC through the cluster)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, `the submission uploads PVC, in the control-plane namespace ("" copies no uploads)`)
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "where the result of this run is recorded for the watchdog and `status`")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	st, lease := startRun(env.cfg, env.key, *statusFile, time.Now())
+	res, err := runOffsiteSync(cfg, env, offsiteSources{
+		archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir, stateDir: *stateDir,
+		registry:   offsiteRegistryEndpoint(*registry, cfg.Registry),
+		uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
+	}, &lease, stderr)
+	recordRun(&st, res, err, lease)
+	if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
+	}
+	return reportRun(res, err, stdout, stderr)
+}
+
+// reportRun prints one pass's outcome and its exit code. A run stopped before
+// it copied anything (a bucket that did not answer, another key's objects,
+// another host writing the bucket) prints no counts: its zeros would read as
+// an empty bucket.
+func reportRun(res offsite.Result, err error, stdout, stderr io.Writer) int {
+	if err == nil || len(res.Errors) > 0 {
+		fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n",
+			res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
+			res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
+			res.UploadsUploaded, res.UploadObjectsPruned,
+			res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes),
+			res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes))
+	}
+	for _, m := range res.WorldsMissing {
+		fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
+	}
+	for _, m := range res.ImagesIncomplete {
+		fmt.Fprintf(stderr, "felis offsite sync: registry image not whole: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// startRun begins a pass: its status record, in this release's format and
+// carrying the last success over, and this host's lease, read from the record
+// the last pass left.
+func startRun(cfg config.OffsiteConfig, key []byte, statusFile string, now time.Time) (offsite.Status, offsite.Lease) {
+	st := offsite.Status{
+		LastAttempt: now.UTC(), Endpoint: cfg.Endpoint, Bucket: cfg.Bucket,
+		Prefix: cfg.Prefix, KeyID: offsite.KeyID(key), Format: offsite.StatusFormat,
+	}
+	if prev, _ := offsite.ReadStatus(statusFile); prev != nil {
+		st.LastSuccess = prev.LastSuccess
+	}
+	return st, offsite.HostLease(statusFile)
+}
+
+// recordRun puts one pass's outcome into its status record. A host that
+// inherited the bucket from an older release keeps that until it has an id.
+func recordRun(st *offsite.Status, res offsite.Result, err error, lease offsite.Lease) {
+	st.Result = res
+	if err != nil {
+		st.LastError = err.Error()
+		st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch)
+		var we *offsite.WriterError
+		if errors.As(err, &we) {
+			st.Standby = errors.Is(err, offsite.ErrStandby)
+			st.Displaced = errors.Is(err, offsite.ErrDisplaced)
+			st.Writer = we.Writer
+		}
+	} else {
+		st.LastSuccess = st.LastAttempt
+	}
+	if lease.Inherited {
+		id, _ := lease.ID()
+		st.Inherited = id == ""
+	}
+}
+
+// offsiteSources is where one sync pass reads from: the world archive volume
+// (archiveDir, or the backupPVC's directory), the bundle directory (with the
+// host state the pass bundles, stateDir), the registry's loopback endpoint and
+// the uploads volume (uploadsDir, or the uploadsPVC's directory). An empty
+// source is skipped.
+type offsiteSources struct {
+	archiveDir, backupPVC  string
+	dbDir, stateDir        string
+	registry               string
+	uploadsDir, uploadsPVC string
+}
+
+// offsiteRunLimit backstops one sync pass. Each upload has its own deadline,
+// scaled to its size (internal/offsite), so a pass over a big archive may run
+// for hours; the timer starts no second pass while one runs, and the unit's
+// TimeoutStartSec sits above this.
+const offsiteRunLimit = 23 * time.Hour
+
+func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lease *offsite.Lease, log io.Writer) (offsite.Result, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit)
+	defer cancel()
+	checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
+	err := env.bucket.Check(checkCtx)
+	checkCancel()
+	if err != nil {
+		return offsite.Result{}, err
+	}
+	archiveDir, uploadsDir := src.archiveDir, src.uploadsDir
+	if archiveDir == "" && src.backupPVC != "" {
+		dir, err := resolveVolumeDir(ctx, cfg.K8s.Namespace, src.backupPVC, archiveVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		archiveDir = dir
+	}
+	// An s3:// uploads store is off the host already; only a local one, on
+	// the uploads PVC, needs the copy.
+	if uploadsDir == "" && src.uploadsPVC != "" && isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+		dir, err := resolveVolumeDir(ctx, platform.DefaultControlNamespace, src.uploadsPVC, uploadsVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		uploadsDir = dir
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		return offsite.Result{}, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	s := offsiteSyncer(cfg, env, src, archiveDir, uploadsDir, lease, log)
+	s.Catalog = offsite.PGCatalog{DB: drv.DB()}
+	if src.registry != "" {
+		s.Images = newRegistryImages(src.registry)
+		s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL)
+	}
+	return s.Run(ctx)
+}
+
+// offsiteSyncer is the pass runOffsiteSync runs over the resolved archive and
+// uploads directories, before its catalog and registry are attached. It
+// snapshots the database into the bundle directory after copying archives, and
+// sweeps world objects no backup records once they outlive every retention in
+// [archive]; a retention that does not parse sweeps none.
+func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, archiveDir, uploadsDir string, lease *offsite.Lease, log io.Writer) *offsite.Syncer {
+	s := &offsite.Syncer{
+		Bucket: env.bucket, Key: env.key,
+		ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log,
+	}
+	if src.dbDir != "" {
+		s.Snapshot = offsiteSnapshot(cfg.Database, src.dbDir, src.stateDir, log)
+	}
+	if rc, err := reaperConfig(cfg); err != nil {
+		fmt.Fprintf(log, "felis offsite: world objects no backup records are kept: %v\n", err)
+	} else {
+		s.OrphanAfter = max(rc.Retention, rc.ManualRetention, rc.ScheduledRetention)
+	}
+	return s
+}
+
+// offsiteSnapshot takes the bundle a pass sends after copying world archives
+// (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite,
+// with the newest one kept in dir. It is not recorded for the panel, whose
+// backup card watches felis-db-backup.timer: snapshots come only when archives
+// are copied, and would hide a daily timer that stopped. It requires the
+// MinecraftServer objects: it becomes the newest bundle in the bucket, which a
+// lost host restores from, and a pass that cannot take a whole one fails and
+// tries again next hour.
+func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error {
+	return func(ctx context.Context) error {
+		tools, err := dbTools(db)
+		if err != nil {
+			return err
+		}
+		ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+		defer cancel()
+		path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
+			DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite,
+			Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(),
+			ExportServers: exportMinecraftServers, RequireServers: true, Log: log,
+		})
+		if err == nil {
+			fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path))
+		}
+		return err
+	}
+}
+
+// volumeKind names a PVC the off-site copy reads or restores, for messages,
+// with the flag that bypasses finding it through the cluster.
+type volumeKind struct{ what, dirFlag, empty string }
+
+var (
+	archiveVolume = volumeKind{"archive volume", "-archive-dir", "no world has been archived"}
+	uploadsVolume = volumeKind{"uploads volume", "-uploads-dir", "no modpack has been uploaded"}
+)
+
+// resolveVolumeDir finds the host directory behind a PVC: a local-path volume
+// is a directory on this node. A PVC still waiting for its first consumer
+// holds nothing yet: without bind that is "" (nothing to copy), with bind it
+// is bound first, for a fetch to write into.
+func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind, bind bool, log io.Writer) (string, error) {
+	if ns == "" {
+		ns = platform.DefaultMinecraftNamespace
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		return "", fmt.Errorf("reach the cluster to find the %s (or pass %s): %w", kind.what, kind.dirFlag, err)
+	}
+	var pvc corev1.PersistentVolumeClaim
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+		return "", fmt.Errorf("%s %s/%s: %w", kind.what, ns, pvcName, err)
+	}
+	if pvc.Spec.VolumeName == "" {
+		if !bind {
+			fmt.Fprintf(log, "felis offsite: %s %s/%s is not bound yet; %s\n", kind.what, ns, pvcName, kind.empty)
+			return "", nil
+		}
+		if err := bindVolume(ctx, cl, ns, pvcName, kind, log); err != nil {
+			return "", err
+		}
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+			return "", err
+		}
+	}
+	var pv corev1.PersistentVolume
+	if err := cl.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil {
+		return "", fmt.Errorf("%s %s: %w", kind.what, pvc.Spec.VolumeName, err)
+	}
+	var dir string
+	switch {
+	case pv.Spec.Local != nil:
+		dir = pv.Spec.Local.Path
+	case pv.Spec.HostPath != nil:
+		dir = pv.Spec.HostPath.Path
+	default:
+		return "", fmt.Errorf("%s %s is not a directory on a node (local or hostPath); pass %s with where it is mounted on this host", kind.what, pv.Name, kind.dirFlag)
+	}
+	if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
+		return "", fmt.Errorf("%s %s is %s on its node, which is not a directory here; run this on the node that holds it, or pass %s", kind.what, pv.Name, dir, kind.dirFlag)
+	}
+	return dir, nil
+}
+
+// bindVolume runs a pod that mounts the PVC and exits, which is what makes a
+// WaitForFirstConsumer volume (k3s local-path) get provisioned. The pod uses
+// the control plane's own image, which every install already has.
+func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind volumeKind, log io.Writer) error {
+	var api appsv1.Deployment
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: platform.DefaultControlNamespace, Name: "felis-api"}, &api); err != nil {
+		return fmt.Errorf("find the felis image to bind the %s with: %w", kind.what, err)
+	}
+	if len(api.Spec.Template.Spec.Containers) == 0 {
+		return errors.New("felis-api has no container to take the image from")
+	}
+	image := api.Spec.Template.Spec.Containers[0].Image
+	pod := platform.VolumeBinderPod(ns, pvcName, image)
+	if err := cl.Create(ctx, pod); err != nil {
+		return fmt.Errorf("start a pod to bind the %s: %w", kind.what, err)
+	}
+	fmt.Fprintf(log, "felis offsite: binding the %s %s/%s (pod %s)\n", kind.what, ns, pvcName, pod.Name)
+	defer func() {
+		_ = cl.Delete(context.Background(), pod, client.PropagationPolicy(metav1.DeletePropagationBackground))
+	}()
+	deadline := time.Now().Add(3 * time.Minute)
+	for time.Now().Before(deadline) {
+		var pvc corev1.PersistentVolumeClaim
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err == nil && pvc.Spec.VolumeName != "" && pvc.Status.Phase == corev1.ClaimBound {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+	return fmt.Errorf("the %s %s/%s did not bind within 3 minutes; see kubectl -n %s describe pod %s", kind.what, ns, pvcName, ns, pod.Name)
+}
+
+func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if !cfg.Offsite.Enabled() {
+		fmt.Fprintln(stdout, "off-site copy: not configured. World archives, database bundles, user images and uploaded modpacks exist on this machine only.")
+		fmt.Fprintln(stdout, "See docs/troubleshooting.md §16, \"Keep a copy somewhere else\".")
+		return 1
+	}
+	o := cfg.Offsite
+	fmt.Fprintf(stdout, "bucket:       %s at %s", o.Bucket, o.Endpoint)
+	if o.Prefix != "" {
+		fmt.Fprintf(stdout, ", prefix %s", o.Prefix)
+	}
+	fmt.Fprintln(stdout)
+	st, err := offsite.ReadStatus(*statusFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if st == nil {
+		fmt.Fprintln(stdout, "last sync:    never (sudo systemctl start felis-offsite.service)")
+		return 1
+	}
+	now := time.Now()
+	fmt.Fprintf(stdout, "key id:       %s\n", st.KeyID)
+	fmt.Fprintf(stdout, "last attempt: %s (%s ago)\n", st.LastAttempt.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastAttempt)))
+	if st.LastSuccess.IsZero() {
+		fmt.Fprintln(stdout, "last success: never")
+	} else {
+		fmt.Fprintf(stdout, "last success: %s (%s ago)\n", st.LastSuccess.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastSuccess)))
+	}
+	if st.LastError != "" {
+		fmt.Fprintf(stdout, "last error:   %s\n", st.LastError)
+	}
+	if st.KeyMismatch {
+		fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile)
+		return 1
+	}
+	if st.Displaced && st.Writer != nil {
+		fmt.Fprintf(stdout, "\nThe last run was refused: %s took the bucket over (it last wrote it at %s), and this host copies nothing there any more. If that host is a rehearsal machine, take the bucket back: sudo felis offsite take-over -yes\n",
+			st.Writer, st.Writer.At.Local().Format(time.DateTime))
+		return 1
+	}
+	if st.Standby {
+		switch w := st.StandsBy(now); {
+		case w != nil:
+			fmt.Fprintf(stdout, "\nThis host stands by: %s writes the bucket (last at %s). This host was built from its backup, copies nothing into the bucket and, while that host keeps writing, mails no watchdog alert.", w, w.At.Local().Format(time.DateTime))
+		case st.Writer != nil:
+			fmt.Fprintf(stdout, "\nThis host copies nothing into the bucket: %s wrote it, last at %s, and this host was built from its backup.", st.Writer, st.Writer.At.Local().Format(time.DateTime))
+		default:
+			fmt.Fprint(stdout, "\nThis host copies nothing into the bucket: it holds copies this host did not write, and names no host writing it.")
+		}
+		fmt.Fprintln(stdout, " Once this host replaces that one for good: sudo felis offsite take-over -yes")
+		return 1
+	}
+	r := st.Result
+	fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
+		r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
+	if r.ImageIndex != "" {
+		fmt.Fprintf(stdout, "images:       %d in %d repositories (%s), registry index %s\n",
+			r.Images, r.ImageRepos, offsite.HumanBytes(r.RemoteImageBytes), r.ImageIndex)
+	} else {
+		fmt.Fprintln(stdout, "images:       not copied (no in-cluster registry, or no sync has reached it yet)")
+	}
+	if r.UploadIndex != "" {
+		fmt.Fprintf(stdout, "uploads:      %d submission contexts (%s), uploads index %s\n",
+			r.Uploads, offsite.HumanBytes(r.RemoteUploadBytes), r.UploadIndex)
+	} else {
+		fmt.Fprintln(stdout, "uploads:      not copied (an s3:// uploads store, or no sync has reached the volume yet)")
+	}
+	fmt.Fprintf(stdout, "waiting:      %d world archives not yet copied\n", r.WorldsPending)
+	for _, m := range r.WorldsMissing {
+		fmt.Fprintf(stdout, "missing:      %s is recorded but not on the volume\n", m)
+	}
+	for _, m := range r.ImagesIncomplete {
+		fmt.Fprintf(stdout, "not whole:    %s\n", m)
+	}
+	if st.LastSuccess.IsZero() || now.Sub(st.LastSuccess) > offsite.StaleAfter {
+		fmt.Fprintf(stdout, "\nThe last successful sync is older than %s: journalctl -u felis-offsite -n 50\n", dbbackup.Age(offsite.StaleAfter))
+		return 1
+	}
+	return 0
+}
+
+func orNone(s string) string {
+	if s == "" {
+		return "none"
+	}
+	return s
+}
+
+func offsiteList(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	return printOffsiteList(env, stdout, stderr)
+}
+
+func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	bundles, err := offsite.ListDB(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	worlds, err := env.bucket.List(ctx, "worlds/")
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	printDBBundles(ctx, env.bucket, env.key, bundles, stdout)
+	var total int64
+	for _, w := range worlds {
+		total += w.Size
+	}
+	fmt.Fprintf(stdout, "world archives: %d (%s)\n", len(worlds), offsite.HumanBytes(total))
+	versions, err := offsite.ImageIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "registry index versions (%d, newest first; restore one with fetch-images -at):\n", len(versions))
+	for i := len(versions) - 1; i >= 0; i-- {
+		x, err := offsite.LoadImageIndex(ctx, env.bucket, env.key, versions[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", versions[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d images in %d repositories\n", versions[i], x.Images(), len(x.Repositories))
+	}
+	uploads, err := offsite.UploadIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "uploads index versions (%d, newest first; restore one with fetch-uploads -at):\n", len(uploads))
+	for i := len(uploads) - 1; i >= 0; i-- {
+		x, err := offsite.LoadUploadIndex(ctx, env.bucket, env.key, uploads[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", uploads[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d submission contexts (%s)\n", uploads[i], len(x.Contexts), offsite.HumanBytes(x.Bytes()))
+	}
+	return 0
+}
+
+// printDBBundles lists the database bundles with what each one's database
+// held, read off the front of each, so a restore can pick one by its contents.
+func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles []offsite.Object, stdout io.Writer) {
+	fmt.Fprintf(stdout, "database bundles (%d, newest first; restore one with fetch-db):\n", len(bundles))
+	for _, o := range bundles {
+		m, err := offsite.PeekDB(ctx, b, key, o.Key)
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  %s  unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err)
+			continue
+		}
+		gap := ""
+		if m.ServersError != "" {
+			gap = ", no MinecraftServer objects"
+		}
+		fmt.Fprintf(stdout, "  %s  %s  %s%s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String(), gap)
+	}
+}
+
+func offsiteCheckKey(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	if err := env.bucket.Check(ctx); err != nil {
+		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
+		return 1
+	}
+	return checkKey(ctx, env.bucket, env.key, stdout, stderr)
+}
+
+// checkKey is check-key once the bucket is open: 0 when the key fits, 3 when
+// the bucket's objects are sealed with another one, 1 when it cannot tell.
+func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr io.Writer) int {
+	fit, err := offsite.CheckKey(ctx, b, key)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
+		if errors.Is(err, offsite.ErrKeyMismatch) {
+			return 3
+		}
+		return 1
+	}
+	id := offsite.KeyID(key)
+	switch fit {
+	case offsite.KeyRecorded:
+		fmt.Fprintf(stdout, "felis offsite check-key: the bucket records key id %s, this key's\n", id)
+	case offsite.KeyOpens:
+		fmt.Fprintf(stdout, "felis offsite check-key: the bucket's newest objects open with this key (key id %s); the next sync records it\n", id)
+	case offsite.KeyUnused:
+		fmt.Fprintf(stdout, "felis offsite check-key: the bucket holds no sealed object yet; the first sync records key id %s\n", id)
+	}
+	return 0
+}
+
+func offsiteTakeOver(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes; this host's id is kept next to it")
+	yes := fs.Bool("yes", false, "make this host the one that writes the bucket")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	if err := env.bucket.Check(ctx); err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	return takeOver(ctx, env.bucket, env.key, offsite.HostLease(*statusFile), *statusFile, *yes, time.Now(), stdout, stderr)
+}
+
+// takeOver is take-over once the bucket is open: without yes it says which
+// host writes the bucket, 0 for this one (or none yet), 4 for another and 5
+// for one that took the bucket over from this host; with
+// yes it records this host as the writer. A key the bucket's objects refuse
+// is 3, as in check-key: taking over a bucket this host cannot copy into
+// would only stop the host that can.
+func takeOver(ctx context.Context, b offsite.Bucket, key []byte, lease offsite.Lease, statusFile string, yes bool, now time.Time, stdout, stderr io.Writer) int {
+	fit, err := offsite.CheckKey(ctx, b, key)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		if errors.Is(err, offsite.ErrKeyMismatch) {
+			return 3
+		}
+		return 1
+	}
+	role, w, err := lease.Plan(ctx, b, fit == offsite.KeyUnused)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	switch role {
+	case offsite.RoleWrites:
+		id, _ := lease.ID()
+		fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket; nothing to take over\n", id)
+		return 0
+	case offsite.RoleClaims:
+		fmt.Fprintln(stdout, "felis offsite take-over: the bucket names no host writing it; this host's next sync records itself")
+		return 0
+	}
+	who := "another host"
+	if w != nil {
+		who = w.String()
+		fmt.Fprintf(stdout, "felis offsite take-over: %s writes the bucket, last at %s (%s ago)\n", w, w.At.Local().Format(time.DateTime), dbbackup.Age(now.Sub(w.At)))
+	} else {
+		fmt.Fprintln(stdout, "felis offsite take-over: the bucket holds copies this host did not write, and names no host writing it")
+	}
+	if !yes {
+		if role == offsite.RoleDisplaced {
+			fmt.Fprintf(stdout, "It took the bucket over from this host: this host copies nothing there any more, and its watchdog mails the owners about it. If that host is a rehearsal machine, take the bucket back:\n  sudo felis offsite take-over -yes\n")
+			return 5
+		}
+		fmt.Fprintf(stdout, "This host was built from its backup and copies nothing into the bucket.\n")
+		fmt.Fprintf(stdout, "Taking it over makes this host the one that copies into the bucket and prunes it; %s stops at its next copy and mails its owners. Do it once that host is gone for good, or is a rehearsal machine you are done with:\n  sudo felis offsite take-over -yes\n", who)
+		return 4
+	}
+	if _, err := lease.TakeOver(ctx, b, now); err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	// The refusal the last sync recorded is over: the watchdog mails again
+	// from now on, and status shows the next run's outcome.
+	if st, err := offsite.ReadStatus(statusFile); err == nil && st != nil && (st.Standby || st.Displaced) {
+		st.Standby, st.Displaced, st.Writer, st.LastError, st.Inherited = false, false, nil, "", false
+		if err := offsite.WriteStatus(statusFile, *st); err != nil {
+			fmt.Fprintf(stderr, "felis offsite take-over: record status: %v\n", err)
+		}
+	}
+	id, _ := lease.ID()
+	fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket now; %s stops at its next copy.\nStart the first copy: sudo systemctl start felis-offsite.service\n", id, who)
+	return 0
+}
+
+// keyHint explains an object the key cannot open when the bucket records
+// another key's id, "" otherwise.
+func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string {
+	if !errors.Is(err, offsite.ErrAuth) {
+		return ""
+	}
+	id, ierr := offsite.BucketKeyID(ctx, b)
+	if ierr != nil || id == "" || id == offsite.KeyID(key) {
+		return ""
+	}
+	return fmt.Sprintf("\n  the bucket records key id %s, and this key is %s: set FELIS_OFFSITE_KEY to the key the bucket was written with", id, offsite.KeyID(key))
+}
+
+func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	endpoint := fs.String("endpoint", "", "bucket endpoint, when there is no felis.toml")
+	bucket := fs.String("bucket", "", "bucket name, when there is no felis.toml")
+	region := fs.String("region", "", "bucket region, when there is no felis.toml")
+	prefix := fs.String("prefix", "", "key prefix, when there is no felis.toml")
+	dir := fs.String("dir", dbbackup.DefaultDir, "directory to write the bundle to")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	if err := loadEnvFile(*envFile); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: read %s: %v\n", *envFile, err)
+		return 1
+	}
+	var oc config.OffsiteConfig
+	if *bucket != "" {
+		oc = config.OffsiteConfig{
+			Endpoint: *endpoint, Bucket: *bucket, Region: *region, Prefix: *prefix,
+			AccessKeyRef: config.DefaultOffsiteAccessKeyEnv, SecretKeyRef: config.DefaultOffsiteSecretKeyEnv,
+			KeyRef: config.DefaultOffsiteKeyEnv,
+		}
+	} else {
+		cfg, err := config.Load(*cfgPath)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v (on a host with no install yet, pass -endpoint and -bucket)\n", err)
+			return 1
+		}
+		oc = cfg.Offsite
+	}
+	env, err := resolveOffsite(oc)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	return fetchDB(ctx, env.bucket, env.key, arg, *dir, time.Now(), stdout, stderr)
+}
+
+// fetchDB is fetch-db once the bucket is open: arg is a bundle name or latest.
+func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string, now time.Time, stdout, stderr io.Writer) int {
+	name := arg
+	if name == "latest" {
+		var err error
+		if name, _, err = offsite.ChooseDB(ctx, b, key); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
+			return 1
+		}
+	}
+	if _, _, ok := dbbackup.ParseBundleName(name); !ok {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %q is not a bundle name (felis-db-<stamp>-<label>.tar); see `felis offsite list`\n", name)
+		return 2
+	}
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	dst := filepath.Join(dir, name)
+	if err := offsite.FetchObject(ctx, b, key, offsite.DBKey(name), dst, 0o600); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
+		return 1
+	}
+	m, err := dbbackup.Verify(dst)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: fetched %s but it does not verify: %v\n", dst, err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-db: wrote %s (verified)\n", dst)
+	fmt.Fprintf(stdout, "  taken   %s (%s, %s ago)\n  felis   %s, schema %d\n  holds   %s\n",
+		m.CreatedAt.Format(time.RFC3339), m.Label, dbbackup.Age(now.Sub(m.CreatedAt)),
+		orUnknown(m.FelisVersion), m.SchemaVersion, m.Counts.String())
+	if m.Counts.Fresh() {
+		fmt.Fprintln(stdout, "  This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.")
+	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  This bundle lacks the MinecraftServer objects (%s): `felis db restore` brings back the database, and the servers come from k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap.\n", m.ServersError)
+	}
+	return 0
+}
+
+func offsiteFetchWorlds(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC, binding it if needed)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", "the world archive PVC, in the [k8s] namespace")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	dir := *archiveDir
+	if dir == "" {
+		if dir, err = resolveVolumeDir(ctx, cfg.K8s.Namespace, *backupPVC, archiveVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+			return 1
+		}
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: open database: %v\n", err)
+		return 1
+	}
+	defer drv.Close()
+	res, err := offsite.FetchWorlds(ctx, env.bucket, offsite.PGCatalog{DB: drv.DB()}, env.key, dir, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-worlds: %d recorded archives, %d fetched into %s, %d with no copy in the bucket\n",
+		res.Present, len(res.Fetched), dir, len(res.Missing))
+	for _, m := range res.Missing {
+		fmt.Fprintf(stdout, "  no off-site copy: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_images.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/registrygate"
+	"felis.lolicon.best/internal/registryprune"
+)
+
+// registryImages is the platform registry as the off-site copy sees it: read
+// anonymously through the gate (the catalog, its manifest index, manifests and
+// blobs) and, for a restore, written as the platform principal. Both go through
+// the node's loopback hostPort, the way the installer pushes.
+type registryImages struct {
+	host   string
+	index  *registryprune.Client
+	source *imagepush.Source
+	pusher *imagepush.Pusher
+}
+
+func newRegistryImages(endpoint string) *registryImages {
+	return &registryImages{
+		host:   endpoint,
+		index:  &registryprune.Client{Endpoint: "http://" + endpoint},
+		source: &imagepush.Source{Scheme: "http"},
+	}
+}
+
+func (r *registryImages) Repositories(ctx context.Context) ([]string, error) {
+	return r.index.Repositories(ctx)
+}
+
+func (r *registryImages) Revisions(ctx context.Context, repo string) ([]string, map[string]string, error) {
+	idx, err := r.index.Index(ctx, repo)
+	if err != nil {
+		return nil, nil, err
+	}
+	digests := make([]string, 0, len(idx.Revisions))
+	for _, rev := range idx.Revisions {
+		digests = append(digests, rev.Digest)
+	}
+	return digests, idx.Tags, nil
+}
+
+func (r *registryImages) Manifest(ctx context.Context, repo, digest string) ([]byte, string, error) {
+	body, mt, err := r.source.Manifest(ctx, r.host, repo, digest)
+	return body, mt, registryGone(err)
+}
+
+func (r *registryImages) Blob(ctx context.Context, repo, digest string) (io.ReadCloser, error) {
+	rc, err := r.source.Blob(ctx, r.host, repo, digest)
+	return rc, registryGone(err)
+}
+
+func (r *registryImages) PutBlob(ctx context.Context, repo, digest string, size int64, open func() (io.ReadCloser, error)) error {
+	return r.pusher.UploadBlob(ctx, r.host, repo, digest, size, open)
+}
+
+func (r *registryImages) PutManifest(ctx context.Context, repo, reference, mediaType string, body []byte) error {
+	_, err := r.pusher.PutManifest(ctx, r.host, repo, reference, mediaType, body)
+	return err
+}
+
+// imagePins lists, per repository of the registry refs spell as host, the
+// digests the MinecraftServers' specs and the image whitelist pin: what a
+// restored database and its servers will ask the registry for.
+func imagePins(db *sql.DB, host string) func(ctx context.Context) (map[string][]string, error) {
+	return func(ctx context.Context) (map[string][]string, error) {
+		cl, err := buildSystemServerClient()
+		if err != nil {
+			return nil, fmt.Errorf("reach the cluster: %w", err)
+		}
+		var servers v1alpha1.MinecraftServerList
+		if err := cl.List(ctx, &servers); err != nil {
+			return nil, fmt.Errorf("list MinecraftServers: %w", err)
+		}
+		refs := make([]string, 0, len(servers.Items))
+		for _, s := range servers.Items {
+			refs = append(refs, s.Spec.Image)
+		}
+		rows, err := db.QueryContext(ctx, `SELECT image_ref FROM image_whitelist`)
+		if err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var ref string
+			if err := rows.Scan(&ref); err != nil {
+				return nil, fmt.Errorf("read the image whitelist: %w", err)
+			}
+			refs = append(refs, ref)
+		}
+		if err := rows.Err(); err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		pins := map[string][]string{}
+		for _, ref := range refs {
+			repo, _, digest, ok := registryprune.ParseRef(ref, host)
+			if ok && digest != "" && !slices.Contains(pins[repo], digest) {
+				pins[repo] = append(pins[repo], digest)
+			}
+		}
+		return pins, nil
+	}
+}
+
+// registryGone marks a 404 as a manifest or blob the registry no longer holds.
+func registryGone(err error) error {
+	var se *imagepush.StatusError
+	if errors.As(err, &se) && se.Code == http.StatusNotFound {
+		return fmt.Errorf("%w: %v", offsite.ErrImageGone, err)
+	}
+	return err
+}
+
+// offsiteRegistryEndpoint is where the host reaches the registry whose images
+// the off-site copy covers: flag when given ("off" for none), otherwise the
+// loopback hostPort of the in-cluster registry [registry] url names. A
+// registry outside the cluster is not this install's to copy.
+func offsiteRegistryEndpoint(flag string, reg config.RegistryConfig) string {
+	switch flag {
+	case "off":
+		return ""
+	case "":
+	default:
+		return flag
+	}
+	host, _, _ := strings.Cut(reg.URL, "/")
+	name, _, _ := strings.Cut(host, ":")
+	if strings.HasSuffix(name, ".svc") || strings.HasSuffix(name, ".svc.cluster.local") {
+		return loopbackEndpoint(host)
+	}
+	return ""
+}
+
+// registryWriteCredential is the credential a host-side push uses:
+// FELIS_REGISTRY_USERNAME/PASSWORD when set, otherwise the platform principal
+// with REGISTRY_PLATFORM_TOKEN from the installer's secrets file.
+func registryWriteCredential(secrets string) (string, string, error) {
+	if err := loadEnvFile(secrets); err != nil {
+		return "", "", fmt.Errorf("read %s: %w", secrets, err)
+	}
+	user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if pass == "" {
+		user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
+	}
+	if pass == "" {
+		return "", "", errors.New("no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
+	}
+	return user, pass, nil
+}
+
+func offsiteFetchImages(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	registry := fs.String("registry", "", "host[:port] of the registry to push into (default: the loopback hostPort of the in-cluster registry)")
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	at := fs.String("at", "", "registry index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	endpoint := offsiteRegistryEndpoint(*registry, cfg.Registry)
+	if endpoint == "" {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: [registry] url %q is not the in-cluster registry; pass -registry host:port\n", cfg.Registry.URL)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 2
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseImageIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-images: restoring registry index %s (%d repositories, %d images) into %s\n",
+		stamp, len(idx.Repositories), idx.Images(), endpoint)
+	target := newRegistryImages(endpoint)
+	target.pusher = &imagepush.Pusher{Scheme: "http", Username: user, Password: pass}
+	res, err := offsite.FetchImages(ctx, env.bucket, env.key, idx, target, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-images: %d of %d repositories restored, %d images, %d tags, %d blobs pushed (%s)\n",
+		res.Repositories, len(idx.Repositories), res.Manifests, res.Tags, res.BlobsPushed, offsite.HumanBytes(res.BytesPushed))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v (a second run pushes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_test.go

@@ -0,0 +1,775 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+)
+
+func TestLoadOffsiteEnvFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "offsite.env")
+	body := `# written by bootstrap
+FELIS_OFFSITE_ACCESS_KEY=AKIA123
+export FELIS_OFFSITE_SECRET_KEY="se=cret"
+FELIS_OFFSITE_KEY='k'
+
+not a line
+`
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "from-the-shell")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "")
+	t.Setenv("FELIS_OFFSITE_KEY", "")
+	if err := loadEnvFile(path); err != nil {
+		t.Fatal(err)
+	}
+	for k, want := range map[string]string{
+		"FELIS_OFFSITE_ACCESS_KEY": "from-the-shell", // the environment wins
+		"FELIS_OFFSITE_SECRET_KEY": "se=cret",
+		"FELIS_OFFSITE_KEY":        "k",
+	} {
+		if got := os.Getenv(k); got != want {
+			t.Errorf("%s = %q, want %q", k, got, want)
+		}
+	}
+	if err := loadEnvFile(filepath.Join(t.TempDir(), "absent")); err != nil {
+		t.Errorf("a missing env file is not an error: %v", err)
+	}
+}
+
+func TestResolveOffsiteNamesTheMissingVariable(t *testing.T) {
+	c := config.OffsiteConfig{
+		Endpoint: "https://s3.example", Bucket: "b",
+		AccessKeyRef: "T_AK", SecretKeyRef: "T_SK", KeyRef: "T_KEY",
+	}
+	t.Setenv("T_AK", "ak")
+	t.Setenv("T_SK", "sk")
+	t.Setenv("T_KEY", "")
+	if _, err := resolveOffsite(c); err == nil || !strings.Contains(err.Error(), "T_KEY") {
+		t.Fatalf("err = %v, want it to name T_KEY", err)
+	}
+	t.Setenv("T_KEY", "not base64 at all")
+	if _, err := resolveOffsite(c); err == nil {
+		t.Fatal("a malformed key was accepted")
+	}
+	key, _ := offsite.NewKey()
+	t.Setenv("T_KEY", key)
+	env, err := resolveOffsite(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(env.key) != offsite.KeySize {
+		t.Fatalf("key is %d bytes", len(env.key))
+	}
+	if _, err := resolveOffsite(config.OffsiteConfig{}); err == nil {
+		t.Fatal("an unconfigured [offsite] resolved")
+	}
+}
+
+func TestOffsiteKeygen(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := cmdOffsite([]string{"keygen"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if _, err := offsite.ParseKey(strings.TrimSpace(out.String())); err != nil {
+		t.Fatalf("keygen printed %q: %v", out.String(), err)
+	}
+}
+
+func TestOffsiteFetchDBRejectsOddNames(t *testing.T) {
+	key, _ := offsite.NewKey()
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "ak")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "sk")
+	t.Setenv("FELIS_OFFSITE_KEY", key)
+	var out, errb bytes.Buffer
+	code := cmdOffsite([]string{"fetch-db", "-env-file", "", "-endpoint", "http://127.0.0.1:1", "-bucket", "b",
+		"-dir", t.TempDir(), "../../etc/shadow"}, &out, &errb)
+	if code != 2 || !strings.Contains(errb.String(), "not a bundle name") {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+}
+
+func TestOffsiteRegistryEndpoint(t *testing.T) {
+	for _, tc := range []struct{ flag, url, want string }{
+		{"", "registry.felis.svc:5000", "127.0.0.1:5000"},
+		{"", "registry.felis.svc.cluster.local:5001", "127.0.0.1:5001"},
+		{"", "ghcr.io/acme", ""},
+		{"", "", ""},
+		{"off", "registry.felis.svc:5000", ""},
+		{"10.0.0.5:5000", "ghcr.io/acme", "10.0.0.5:5000"},
+	} {
+		if got := offsiteRegistryEndpoint(tc.flag, config.RegistryConfig{URL: tc.url}); got != tc.want {
+			t.Errorf("offsiteRegistryEndpoint(%q, %q) = %q, want %q", tc.flag, tc.url, got, tc.want)
+		}
+	}
+}
+
+func TestRegistryGoneMarksNotFound(t *testing.T) {
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 404}); !errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("404 = %v, want ErrImageGone", err)
+	}
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 503}); errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("503 = %v, want it kept an ordinary failure", err)
+	}
+}
+
+// mapBucket is an in-memory offsite.Bucket.
+type mapBucket map[string][]byte
+
+func (b mapBucket) Put(_ context.Context, key string, r io.Reader, _ int64) error {
+	data, err := io.ReadAll(r)
+	b[key] = data
+	return err
+}
+
+func (b mapBucket) Get(_ context.Context, key string) (io.ReadCloser, error) {
+	data, ok := b[key]
+	if !ok {
+		return nil, offsite.ErrNotFound
+	}
+	return io.NopCloser(bytes.NewReader(data)), nil
+}
+
+func (b mapBucket) List(_ context.Context, prefix string) ([]offsite.Object, error) {
+	var out []offsite.Object
+	for k, v := range b {
+		if strings.HasPrefix(k, prefix) {
+			out = append(out, offsite.Object{Key: k, Size: int64(len(v))})
+		}
+	}
+	return out, nil
+}
+
+func (b mapBucket) Remove(_ context.Context, key string) error {
+	delete(b, key)
+	return nil
+}
+
+var fetchT0 = time.Date(2026, 9, 20, 3, 30, 0, 0, time.UTC)
+
+// putBundle seals a bundle that verifies, taken daysAgo days before fetchT0,
+// into b and returns its name.
+func putBundle(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts) string {
+	t.Helper()
+	return putBundleWith(t, b, key, daysAgo, counts, "")
+}
+
+// putBundleWith is putBundle for a bundle whose server export failed with
+// serversError, when that is not empty.
+func putBundleWith(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts, serversError string) string {
+	t.Helper()
+	created := fetchT0.AddDate(0, 0, -daysAgo)
+	dump := []byte("PGDMP " + created.String())
+	sum := sha256.Sum256(dump)
+	manifest, err := json.Marshal(dbbackup.Manifest{
+		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, ServersError: serversError,
+		Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var plain bytes.Buffer
+	tw := tar.NewWriter(&plain)
+	for _, f := range []struct {
+		name string
+		data []byte
+	}{{"MANIFEST.json", manifest}, {"db.dump", dump}} {
+		if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o600, Size: int64(len(f.data)), Typeflag: tar.TypeReg}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tw.Write(f.data); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	var sealed bytes.Buffer
+	if err := offsite.Encrypt(&sealed, &plain, key); err != nil {
+		t.Fatal(err)
+	}
+	name := dbbackup.BundleName(created, dbbackup.LabelDaily)
+	b[offsite.DBKey(name)] = sealed.Bytes()
+	return name
+}
+
+func TestOffsiteFetchDB(t *testing.T) {
+	rawKey, _ := offsite.NewKey()
+	key, _ := offsite.ParseKey(rawKey)
+	now := fetchT0.Add(2 * time.Hour)
+	fetch := func(b mapBucket, arg string) (dir string, code int, stdout, stderr string) {
+		dir = t.TempDir()
+		var out, errb bytes.Buffer
+		code = fetchDB(context.Background(), b, key, arg, dir, now, &out, &errb)
+		return dir, code, out.String(), errb.String()
+	}
+	fetched := func(t *testing.T, dir string) []string {
+		t.Helper()
+		var names []string
+		entries, _ := os.ReadDir(dir)
+		for _, e := range entries {
+			names = append(names, e.Name())
+		}
+		return names
+	}
+
+	t.Run("latest skips a rebuilt host's empty bundle", func(t *testing.T) {
+		b := mapBucket{}
+		full := putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
+		empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
+		dir, code, out, errb := fetch(b, "latest")
+		if code != 1 || !strings.Contains(errb, empty) || !strings.Contains(errb, full+" (5 accounts, 3 servers)") {
+			t.Fatalf("exit %d, stdout %q, stderr %q; want a refusal naming %s", code, out, errb, full)
+		}
+		if got := fetched(t, dir); len(got) != 0 {
+			t.Errorf("a refused fetch wrote %v", got)
+		}
+	})
+
+	t.Run("latest takes the newest bundle and says what it holds", func(t *testing.T) {
+		b := mapBucket{}
+		putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 2})
+		newest := putBundle(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3})
+		dir, code, out, errb := fetch(b, "latest")
+		if code != 0 {
+			t.Fatalf("exit %d: %s", code, errb)
+		}
+		if got := fetched(t, dir); !slices.Equal(got, []string{newest}) {
+			t.Errorf("wrote %v, want %s", got, newest)
+		}
+		for _, want := range []string{
+			"wrote " + filepath.Join(dir, newest) + " (verified)",
+			"taken   2026-09-19T03:30:00Z (daily, 26h0m ago)",
+			"felis   v1.2.3, schema 21",
+			"holds   5 accounts, 3 servers",
+		} {
+			if !strings.Contains(out, want) {
+				t.Errorf("stdout lacks %q:\n%s", want, out)
+			}
+		}
+		if strings.Contains(out, "new install") {
+			t.Errorf("a bundle with servers flagged as a new install's:\n%s", out)
+		}
+	})
+
+	t.Run("an empty bundle named outright is fetched with a warning", func(t *testing.T) {
+		b := mapBucket{}
+		putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
+		empty := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 1})
+		dir, code, out, errb := fetch(b, empty)
+		if code != 0 || !slices.Equal(fetched(t, dir), []string{empty}) {
+			t.Fatalf("exit %d, wrote %v: %s", code, fetched(t, dir), errb)
+		}
+		if !strings.Contains(out, "holds   1 account, 0 servers") || !strings.Contains(out, "like a new install's") {
+			t.Errorf("stdout = %s", out)
+		}
+	})
+
+	t.Run("a bundle without the servers says where they come from", func(t *testing.T) {
+		b := mapBucket{}
+		gapped := putBundleWith(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused (tried 3 times)")
+		_, code, out, errb := fetch(b, gapped)
+		if code != 0 || !strings.Contains(out, "This bundle lacks the MinecraftServer objects (connection refused (tried 3 times))") ||
+			!strings.Contains(out, "k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap") {
+			t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
+		}
+		whole := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+		if _, _, out, _ := fetch(b, whole); strings.Contains(out, "lacks the MinecraftServer objects") {
+			t.Errorf("a whole bundle flagged:\n%s", out)
+		}
+	})
+
+	t.Run("a bundle from before counts says so", func(t *testing.T) {
+		b := mapBucket{}
+		old := putBundle(t, b, key, 0, nil)
+		_, code, out, errb := fetch(b, "latest")
+		if code != 0 || !strings.Contains(out, old) || !strings.Contains(out, "holds   not recorded") || strings.Contains(out, "new install") {
+			t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
+		}
+	})
+}
+
+func TestOffsiteCheckKey(t *testing.T) {
+	newKey := func() []byte {
+		raw, _ := offsite.NewKey()
+		k, _ := offsite.ParseKey(raw)
+		return k
+	}
+	key, other := newKey(), newKey()
+	marked := func(k []byte) mapBucket { return mapBucket{"felis-key-id": []byte(offsite.KeyID(k) + "\n")} }
+	unmarked := func(k []byte) mapBucket {
+		b := mapBucket{}
+		putBundle(t, b, k, 0, nil)
+		return b
+	}
+	for _, tc := range []struct {
+		what   string
+		bucket mapBucket
+		code   int
+		says   []string
+	}{
+		{"the recorded key", marked(key), 0, []string{"records key id " + offsite.KeyID(key)}},
+		{"an unmarked bucket the key opens", unmarked(key), 0, []string{"open with this key", "the next sync records it"}},
+		{"an empty bucket", mapBucket{}, 0, []string{"no sealed object yet", "records key id " + offsite.KeyID(key)}},
+		{"another recorded key", marked(other), 3, []string{offsite.KeyID(other), offsite.KeyID(key), "FELIS_OFFSITE_KEY"}},
+		{"an unmarked bucket under another key", unmarked(other), 3, []string{"opens none of db/felis-db-"}},
+		{"a marker Felis did not write", mapBucket{"felis-key-id": []byte("hello")}, 1, []string{"not a key id"}},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			before := len(tc.bucket)
+			var out, errb bytes.Buffer
+			code := checkKey(context.Background(), tc.bucket, key, &out, &errb)
+			if code != tc.code {
+				t.Fatalf("exit %d, want %d; stdout %q, stderr %q", code, tc.code, out.String(), errb.String())
+			}
+			said := out.String() + errb.String()
+			for _, s := range tc.says {
+				if !strings.Contains(said, s) {
+					t.Errorf("output lacks %q: %s", s, said)
+				}
+			}
+			if len(tc.bucket) != before {
+				t.Errorf("check-key wrote to the bucket: %d objects, had %d", len(tc.bucket), before)
+			}
+		})
+	}
+
+	// fetch-db names both ids when the bucket records another key.
+	b := marked(other)
+	name := putBundle(t, b, other, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+	for _, arg := range []string{"latest", name} {
+		var out, errb bytes.Buffer
+		if code := fetchDB(context.Background(), b, key, arg, t.TempDir(), fetchT0, &out, &errb); code != 1 ||
+			!strings.Contains(errb.String(), "the bucket records key id "+offsite.KeyID(other)+", and this key is "+offsite.KeyID(key)) {
+			t.Errorf("fetch-db %s under another key: exit %d, stderr %q", arg, code, errb.String())
+		}
+	}
+	// A bundle the bucket lacks is not the key's fault.
+	var missOut, missErr bytes.Buffer
+	if code := fetchDB(context.Background(), b, key, "felis-db-20200101T000000Z-daily.tar", t.TempDir(), fetchT0, &missOut, &missErr); code != 1 || strings.Contains(missErr.String(), "records key id") {
+		t.Errorf("missing bundle: exit %d, stderr %q", code, missErr.String())
+	}
+	// A bundle damaged under the recorded key gets no such hint.
+	b = marked(key)
+	name = putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+	b[offsite.DBKey(name)][60] ^= 1
+	var out, errb bytes.Buffer
+	if code := fetchDB(context.Background(), b, key, name, t.TempDir(), fetchT0, &out, &errb); code != 1 || strings.Contains(errb.String(), "records key id") {
+		t.Errorf("damaged bundle: exit %d, stderr %q", code, errb.String())
+	}
+}
+
+func TestRecordRun(t *testing.T) {
+	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}
+	for _, tc := range []struct {
+		err                                   error
+		mismatch, standby, displaced, success bool
+	}{
+		{nil, false, false, false, true},
+		{errors.New("list worlds/ in the bucket: connection reset"), false, false, false, false},
+		{fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false, false, false},
+		{&offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, false, true, false, false},
+		{&offsite.WriterError{Kind: offsite.ErrDisplaced, Writer: w}, false, false, true, false},
+	} {
+		st := offsite.Status{LastAttempt: t0}
+		recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err, offsite.Lease{})
+		if st.KeyMismatch != tc.mismatch || st.Standby != tc.standby || st.Displaced != tc.displaced ||
+			(st.Writer != nil) != (tc.standby || tc.displaced) || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 {
+			t.Errorf("err %v: status %+v", tc.err, st)
+		}
+	}
+
+	// A host that copied before writers were recorded keeps that claim over
+	// failed runs until it has an id.
+	l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Inherited: true}
+	st := offsite.Status{}
+	recordRun(&st, offsite.Result{}, errors.New("cannot reach bucket"), l)
+	if !st.Inherited {
+		t.Error("a failed run on a host with no id dropped the older release's claim")
+	}
+	writeTestFile(t, l.IDFile, "aaaaaaaaaaaaaaaa\n", 0o600)
+	st = offsite.Status{Inherited: true}
+	recordRun(&st, offsite.Result{}, nil, l)
+	if st.Inherited {
+		t.Error("a host with an id still carries the older release's claim")
+	}
+}
+
+// A refused run has copied nothing and listed nothing: its zero counts would
+// tell the journal the bucket is empty. A pass that ran and failed a step
+// shows what it did get to.
+func TestReportRun(t *testing.T) {
+	w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)}
+	for _, tc := range []struct {
+		name   string
+		res    offsite.Result
+		err    error
+		code   int
+		counts bool
+	}{
+		{"a pass", offsite.Result{DBUploaded: 1, RemoteDB: 3}, nil, 0, true},
+		{"a pass with a failed step", offsite.Result{RemoteDB: 3, Errors: []string{"copy db/x: timeout"}}, errors.New("1 of this run's steps failed; first: copy db/x: timeout"), 1, true},
+		{"standing by", offsite.Result{}, &offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, 1, false},
+		{"another key", offsite.Result{}, fmt.Errorf("%w: the bucket records key id 1111111111111111", offsite.ErrKeyMismatch), 1, false},
+		{"no bucket", offsite.Result{}, errors.New("bucket: access denied"), 1, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var out, errOut bytes.Buffer
+			code := reportRun(tc.res, tc.err, &out, &errOut)
+			if code != tc.code {
+				t.Errorf("exit %d, want %d", code, tc.code)
+			}
+			if got := strings.Contains(out.String(), "bucket holds 0 worlds (0 B), 3 bundles"); got != tc.counts {
+				t.Errorf("counts shown = %v, want %v: %q", got, tc.counts, out.String())
+			}
+			if !tc.counts && out.Len() > 0 {
+				t.Errorf("a refused run printed %q", out.String())
+			}
+			if tc.err != nil && !strings.Contains(errOut.String(), "felis offsite sync: "+tc.err.Error()) {
+				t.Errorf("stderr %q lacks the error", errOut.String())
+			}
+		})
+	}
+}
+
+func TestOffsiteTakeOver(t *testing.T) {
+	newKey := func() []byte {
+		raw, _ := offsite.NewKey()
+		k, _ := offsite.ParseKey(raw)
+		return k
+	}
+	key, other := newKey(), newKey()
+	const mine, theirs = "aaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbb"
+	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	sealed := func(k []byte, writer string) mapBucket {
+		b := mapBucket{}
+		putBundle(t, b, k, 0, nil)
+		b["felis-key-id"] = []byte(offsite.KeyID(k) + "\n")
+		if writer != "" {
+			raw, _ := json.Marshal(offsite.Writer{HostID: writer, Host: "prod-1", At: t0.Add(-20 * time.Minute)})
+			b["felis-writer"] = raw
+		}
+		return b
+	}
+	lease := func(id string) offsite.Lease {
+		l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Host: "spare-1"}
+		if id != "" {
+			writeTestFile(t, l.IDFile, id+"\n", 0o600)
+		}
+		return l
+	}
+	run := func(b mapBucket, l offsite.Lease, statusFile string, yes bool) (int, string, string) {
+		t.Helper()
+		var out, errb bytes.Buffer
+		code := takeOver(context.Background(), b, key, l, statusFile, yes, t0, &out, &errb)
+		return code, out.String(), errb.String()
+	}
+	for _, tc := range []struct {
+		what   string
+		bucket mapBucket
+		id     string
+		code   int
+		says   []string
+	}{
+		{"this host writes the bucket", sealed(key, mine), mine, 0, []string{"this host (id " + mine + ") writes the bucket; nothing to take over"}},
+		{"an empty bucket", mapBucket{}, "", 0, []string{"names no host writing it; this host's next sync records itself"}},
+		{"a host built from the writer's backup", sealed(key, theirs), "", 4, []string{"host prod-1 (id " + theirs + ") writes the bucket, last at", "(20m ago)", "built from its backup", "sudo felis offsite take-over -yes"}},
+		{"another host's copies, no writer named", sealed(key, ""), "", 4, []string{"holds copies this host did not write, and names no host writing it", "take-over -yes"}},
+		{"a host another one took over from", sealed(key, theirs), mine, 5, []string{"took the bucket over from this host"}},
+		{"a key the bucket refuses", sealed(other, theirs), "", 3, []string{"sealed with another key"}},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			before := string(tc.bucket["felis-writer"])
+			l := lease(tc.id)
+			code, out, errb := run(tc.bucket, l, filepath.Join(t.TempDir(), "status.json"), false)
+			if code != tc.code {
+				t.Fatalf("exit %d, want %d\n%s%s", code, tc.code, out, errb)
+			}
+			for _, s := range tc.says {
+				if !strings.Contains(out+errb, s) {
+					t.Errorf("output lacks %q:\n%s%s", s, out, errb)
+				}
+			}
+			if string(tc.bucket["felis-writer"]) != before {
+				t.Error("take-over without -yes wrote the bucket's writer")
+			}
+			if tc.id == "" {
+				if _, err := os.Stat(l.IDFile); !errors.Is(err, os.ErrNotExist) {
+					t.Errorf("take-over without -yes made this host an id: %v", err)
+				}
+			}
+		})
+	}
+
+	// -yes on a standby host: the bucket names it, and the refusal the last
+	// sync recorded is cleared, so the watchdog mails again at once.
+	b := sealed(key, theirs)
+	l := lease("")
+	statusFile := filepath.Join(t.TempDir(), "status.json")
+	lastSuccess := t0.Add(-48 * time.Hour)
+	if err := offsite.WriteStatus(statusFile, offsite.Status{
+		LastAttempt: t0.Add(-time.Hour), LastSuccess: lastSuccess, LastError: "offsite: another host writes this bucket", Format: offsite.StatusFormat,
+		Standby: true, Writer: &offsite.Writer{HostID: theirs, Host: "prod-1", At: t0}, Inherited: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	code, out, errb := run(b, l, statusFile, true)
+	id, _ := l.ID()
+	if code != 0 || id == "" || !strings.Contains(out, "this host (id "+id+") writes the bucket now; host prod-1 (id "+theirs+") stops at its next copy") || !strings.Contains(out, "systemctl start felis-offsite.service") {
+		t.Fatalf("take-over -yes: exit %d, id %q\n%s%s", code, id, out, errb)
+	}
+	if w, err := offsite.BucketWriter(context.Background(), b); err != nil || w.HostID != id || w.Host != "spare-1" || !w.At.Equal(t0) {
+		t.Errorf("writer after take-over -yes = %+v, %v", w, err)
+	}
+	st, err := offsite.ReadStatus(statusFile)
+	if err != nil || st.Standby || st.Writer != nil || st.LastError != "" || st.Inherited || !st.LastSuccess.Equal(lastSuccess) {
+		t.Errorf("status after take-over -yes = %+v, %v; want the refusal cleared and the last success kept", st, err)
+	}
+
+	// -yes with a key the bucket refuses writes nothing.
+	b = sealed(other, theirs)
+	before := string(b["felis-writer"])
+	if code, _, _ := run(b, lease(""), filepath.Join(t.TempDir(), "status.json"), true); code != 3 || string(b["felis-writer"]) != before {
+		t.Errorf("take-over -yes under another key: exit %d, writer %s", code, b["felis-writer"])
+	}
+}
+
+func TestOffsiteStatusSaysWhoWritesTheBucket(t *testing.T) {
+	dir := t.TempDir()
+	cfg := filepath.Join(dir, "felis.toml")
+	writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
+	statusFile := filepath.Join(dir, "status.json")
+	now := time.Now()
+	w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-30 * time.Minute)}
+	stale := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-offsite.WriterLive - time.Hour)}
+	for _, tc := range []struct {
+		what string
+		st   offsite.Status
+		says []string
+		not  string
+	}{
+		{"standing by for a live writer", offsite.Status{Standby: true, Writer: w}, []string{"This host stands by: host prod-1 (id bbbbbbbbbbbbbbbb) writes the bucket", "mails no watchdog alert", "take-over -yes"}, "wrote it, last at"},
+		{"standing by for a writer gone quiet", offsite.Status{Standby: true, Writer: stale}, []string{"copies nothing into the bucket: host prod-1 (id bbbbbbbbbbbbbbbb) wrote it, last at", "take-over -yes"}, "mails no watchdog alert"},
+		{"standing by, no writer named", offsite.Status{Standby: true}, []string{"names no host writing it", "take-over -yes"}, "stands by:"},
+		{"displaced", offsite.Status{Displaced: true, Writer: w}, []string{"host prod-1 (id bbbbbbbbbbbbbbbb) took the bucket over", "rehearsal machine", "take-over -yes"}, "stands by"},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			tc.st.LastAttempt, tc.st.LastSuccess, tc.st.LastError = now.Add(-time.Minute), now.Add(-time.Hour), "offsite: another host writes this bucket"
+			if err := offsite.WriteStatus(statusFile, tc.st); err != nil {
+				t.Fatal(err)
+			}
+			var out, errb bytes.Buffer
+			code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
+			if code != 1 || strings.Contains(out.String(), "bucket holds:") || strings.Contains(out.String(), tc.not) {
+				t.Errorf("exit %d\n%s%s", code, out.String(), errb.String())
+			}
+			for _, s := range tc.says {
+				if !strings.Contains(out.String(), s) {
+					t.Errorf("output lacks %q:\n%s", s, out.String())
+				}
+			}
+		})
+	}
+}
+
+func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) {
+	dir := t.TempDir()
+	cfg := filepath.Join(dir, "felis.toml")
+	writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
+	statusFile := filepath.Join(dir, "status.json")
+	st := offsite.Status{LastAttempt: time.Now().Add(-time.Minute), LastSuccess: time.Now().Add(-time.Hour), KeyID: "0123456789abcdef"}
+	status := func() (int, string) {
+		t.Helper()
+		if err := offsite.WriteStatus(statusFile, st); err != nil {
+			t.Fatal(err)
+		}
+		var out, errb bytes.Buffer
+		code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
+		return code, out.String() + errb.String()
+	}
+	if code, out := status(); code != 0 || strings.Contains(out, "refused") {
+		t.Fatalf("a recent success: exit %d\n%s", code, out)
+	}
+	st.LastError, st.KeyMismatch = "offsite: the bucket's objects are sealed with another key", true
+	code, out := status()
+	if code != 1 || !strings.Contains(out, "The last run was refused") || !strings.Contains(out, "key id 0123456789abcdef") || strings.Contains(out, "bucket holds:") {
+		t.Fatalf("a refused run: exit %d\n%s", code, out)
+	}
+}
+
+func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
+	rawKey, _ := offsite.NewKey()
+	key, _ := offsite.ParseKey(rawKey)
+	otherRaw, _ := offsite.NewKey()
+	other, _ := offsite.ParseKey(otherRaw)
+	b := mapBucket{}
+	gapped := putBundleWith(t, b, key, 4, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused")
+	old := putBundle(t, b, key, 3, nil)
+	full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3})
+	sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3})
+	empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
+	bundles, err := offsite.ListDB(context.Background(), b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var out bytes.Buffer
+	printDBBundles(context.Background(), b, key, bundles, &out)
+	lines := strings.Split(strings.TrimSpace(out.String()), "\n")
+	want := []struct{ name, holds string }{
+		{empty, "0 accounts, 0 servers"},
+		{sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"},
+		{full, "5 accounts, 3 servers"},
+		{old, "not recorded"},
+		{gapped, "5 accounts, 3 servers, no MinecraftServer objects"},
+	}
+	if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (5, newest first") {
+		t.Fatalf("output:\n%s", out.String())
+	}
+	for i, w := range want {
+		if l := lines[i+1]; !strings.HasPrefix(l, "  "+w.name+"  ") || !strings.Contains(l, w.holds) || strings.Contains(l, "no MinecraftServer") != (w.name == gapped) {
+			t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds)
+		}
+	}
+}
+
+// TestRestoredHostKeepsStandingBy walks the status file across runs: a host
+// restored from the writer's backup stands by on its first run and on every
+// run after it, a host an older release left copying claims the bucket once,
+// and a failed first run after the upgrade keeps that claim.
+func TestRestoredHostKeepsStandingBy(t *testing.T) {
+	rawKey, _ := offsite.NewKey()
+	key, _ := offsite.ParseKey(rawKey)
+	cfg := config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-backups"}
+	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	standby := &offsite.WriterError{Kind: offsite.ErrStandby, Writer: &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}}
+	pass := func(statusFile string, at time.Time, err error) offsite.Lease {
+		t.Helper()
+		st, lease := startRun(cfg, key, statusFile, at)
+		recordRun(&st, offsite.Result{}, err, lease)
+		if werr := offsite.WriteStatus(statusFile, st); werr != nil {
+			t.Fatal(werr)
+		}
+		return lease
+	}
+
+	restored := filepath.Join(t.TempDir(), "status.json")
+	for i := range 3 {
+		if l := pass(restored, t0.Add(time.Duration(i)*time.Hour), standby); l.Inherited {
+			t.Fatalf("run %d of a restored host claims the bucket", i+1)
+		}
+	}
+	if st, _ := offsite.ReadStatus(restored); !st.Standby || st.Format != offsite.StatusFormat || st.KeyID != offsite.KeyID(key) || st.Bucket != "felis-backups" {
+		t.Errorf("restored host's status = %+v", st)
+	}
+
+	upgraded := filepath.Join(t.TempDir(), "status.json")
+	if err := offsite.WriteStatus(upgraded, offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-time.Hour)}); err != nil {
+		t.Fatal(err)
+	}
+	if l := pass(upgraded, t0, errors.New("cannot reach bucket")); !l.Inherited {
+		t.Fatal("the first run after the upgrade does not claim the bucket")
+	}
+	l := pass(upgraded, t0.Add(time.Hour), nil)
+	if !l.Inherited {
+		t.Fatal("a failed first run after the upgrade lost the claim")
+	}
+	if st, _ := offsite.ReadStatus(upgraded); !st.LastSuccess.Equal(t0.Add(time.Hour)) {
+		t.Errorf("upgraded host's status = %+v", st)
+	}
+}
+
+// TestOffsiteSyncerSnapshotsAndSweeps: the pass `offsite sync` runs takes its
+// snapshot the way `felis db backup` does, in the database pod, into the
+// bundle directory, keeping the newest one there; and it sweeps unrecorded
+// world objects only past the longest retention [archive] gives any backup.
+func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) {
+	dir := newPodRig(t)
+	bundles := filepath.Join(dir, "bundles")
+	env := &offsiteEnv{cfg: config.OffsiteConfig{DBKeep: 5}}
+	var log bytes.Buffer
+	cfg := &config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "120d"}}
+	s := offsiteSyncer(cfg, env, offsiteSources{dbDir: bundles}, "/archives", "/uploads", nil, &log)
+	if s.DBDir != bundles || s.DBKeep != 5 || s.ArchiveDir != "/archives" || s.UploadsDir != "/uploads" {
+		t.Fatalf("syncer = %+v", s)
+	}
+	if s.OrphanAfter != 120*24*time.Hour {
+		t.Errorf("OrphanAfter = %s, want the 120d retention", s.OrphanAfter)
+	}
+	if s.Snapshot == nil {
+		t.Fatal("the pass takes no snapshot after copying archives")
+	}
+	for i := 0; i < 2; i++ {
+		if err := s.Snapshot(context.Background()); err != nil {
+			t.Fatalf("snapshot %d: %v", i, err)
+		}
+	}
+	got, err := dbbackup.List(bundles)
+	if err != nil || len(got) != 1 || got[0].Label != dbbackup.LabelOffsite {
+		t.Fatalf("bundle directory = %+v, %v; want the newest offsite bundle alone", got, err)
+	}
+	if _, err := dbbackupVerify(got[0].Path); err != nil {
+		t.Fatalf("the snapshot does not verify: %v", err)
+	}
+	// The MinecraftServer objects are exported alongside, as in the daily bundle.
+	argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args"))
+	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") {
+		t.Errorf("k3s ran %q, want pg_dump in the pod", ran)
+	}
+	if !strings.Contains(string(bundleServers(t, got[0].Path)), `"name": "lobby"`) {
+		t.Errorf("the snapshot holds no MinecraftServer objects")
+	}
+	if !strings.Contains(log.String(), "took database bundle "+got[0].Name) {
+		t.Errorf("the snapshot is not logged:\n%s", log.String())
+	}
+	// It becomes the newest bundle in the bucket, so with the cluster away it
+	// fails, leaves no bundle, and the next pass tries again.
+	noServerExportWait(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	if err := s.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "export the MinecraftServer objects") {
+		t.Errorf("snapshot with the cluster away: %v, want a failure", err)
+	}
+	if again, _ := dbbackup.List(bundles); len(again) != 1 || again[0].Name != got[0].Name {
+		t.Errorf("bundle directory after the failed snapshot = %+v, want %s alone", again, got[0].Name)
+	}
+
+	for _, c := range []struct {
+		archive config.ArchiveConfig
+		want    time.Duration
+	}{
+		{config.ArchiveConfig{}, 90 * 24 * time.Hour},
+		{config.ArchiveConfig{ScheduledRetention: "200d"}, 200 * 24 * time.Hour},
+		{config.ArchiveConfig{ManualRetention: "150d", Retention: "30d", ScheduledRetention: "60d"}, 150 * 24 * time.Hour},
+	} {
+		s := offsiteSyncer(&config.Config{Database: podDB, Archive: c.archive}, env, offsiteSources{dbDir: bundles}, "", "", nil, io.Discard)
+		if s.OrphanAfter != c.want {
+			t.Errorf("%+v: OrphanAfter = %s, want %s", c.archive, s.OrphanAfter, c.want)
+		}
+	}
+	log.Reset()
+	s = offsiteSyncer(&config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "soon"}}, env, offsiteSources{}, "", "", nil, &log)
+	if s.OrphanAfter != 0 || !strings.Contains(log.String(), "world objects no backup records are kept") {
+		t.Errorf("a retention that does not parse: OrphanAfter %s, log %q; want no sweep, said", s.OrphanAfter, log.String())
+	}
+	if s.Snapshot != nil {
+		t.Error("a pass that copies no bundles takes a snapshot")
+	}
+}

cmd/felis/offsite_uploads.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+)
+
+func offsiteFetchUploads(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC, binding it if needed)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, "the submission uploads PVC, in the control-plane namespace")
+	at := fs.String("at", "", "uploads index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseUploadIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	dir := *uploadsDir
+	if dir == "" {
+		if !isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: [registry] user_uploads_context %q is not the uploads volume; pass -uploads-dir to restore into a directory anyway\n", cfg.Registry.UserUploadsContext)
+			return 2
+		}
+		if dir, err = resolveVolumeDir(ctx, platform.DefaultControlNamespace, *uploadsPVC, uploadsVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+			return 1
+		}
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: restoring uploads index %s (%d submission contexts, %s) into %s\n",
+		stamp, len(idx.Contexts), offsite.HumanBytes(idx.Bytes()), dir)
+	res, err := offsite.FetchUploads(ctx, env.bucket, env.key, idx, dir, platform.ControlPlaneUID, platform.ControlPlaneUID, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: %d written (%s), %d already in place, %d failed\n",
+		res.Written, offsite.HumanBytes(res.Bytes), res.Present, len(res.Failures))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v (a second run writes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/operator.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"log/slog"
+	"net/http"
 	"os"
+	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	felismetrics "felis.lolicon.best/internal/metrics"
@@ -75,16 +79,19 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	fmt.Fprintf(stderr, "felis operator: watching namespace %q\n", *namespace)
 
-	// Register the two probe endpoints. controller-runtime only mounts /healthz and
-	// /readyz once at least one check is registered, so a bare listener would 404.
-	// The checks are the canonical always-pass ping: the probes' contract is "the
-	// manager process is up and serving", and a dependency hiccup (e.g. an API blip)
-	// must not restart the operator.
-	if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
+	// /healthz fails while a reconcile pass has been stuck past its limit, so the
+	// liveness probe restarts an operator whose workers are wedged (a Pod whose
+	// process answers but no server starts or stops). /readyz waits for the
+	// informer caches: until they sync the operator acts on nothing, and one that
+	// never syncs (lost RBAC, an unreachable API) never reports Available.
+	// A dependency hiccup fails neither: the caches ride through API blips, and
+	// each pass is bounded well inside the stuck limit.
+	watch := &operator.ReconcileWatch{}
+	if err := mgr.AddHealthzCheck("reconcile", watch.Check); err != nil {
 		fmt.Fprintf(stderr, "felis operator: register healthz check: %v\n", err)
 		return 1
 	}
-	if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil {
+	if err := mgr.AddReadyzCheck("informers", cacheSynced(mgr.GetCache())); err != nil {
 		fmt.Fprintf(stderr, "felis operator: register readyz check: %v\n", err)
 		return 1
 	}
@@ -98,6 +105,7 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis operator: register metrics: %v\n", err)
 		return 1
 	}
+	felismetrics.SetBuildInfo("operator", resolvedVersion())
 
 	r := &operator.Reconciler{
 		Client: mgr.GetClient(),
@@ -106,7 +114,24 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		// The operator's own image, for the forwarding-config initContainer it
 		// injects into user servers. The Deployment passes it as FELIS_IMAGE (see
 		// platform.OperatorDeployment); absent, that injection is simply skipped.
-		FelisImage: os.Getenv("FELIS_IMAGE"),
+		FelisImage:     os.Getenv("FELIS_IMAGE"),
+		Nodes:          nil,
+		EgressProbe:    os.Getenv("FELIS_EGRESS_PROBE"),
+		ControllerNode: os.Getenv("FELIS_CONTROLLER_NODE"),
+		// Uncached: the maintenance-lock check lists Jobs only when a server is
+		// about to start, which does not justify a namespace-wide Job informer.
+		Jobs: mgr.GetAPIReader(),
+		// Uncached too: RCON Secrets are read by name, so the Role grants
+		// secrets:get without the list/watch an informer would need.
+		Secrets: mgr.GetAPIReader(),
+		// And pods: pod-0 is read by name, so the Role grants pods:get and no
+		// namespace-wide pod informer runs.
+		Pods:     mgr.GetAPIReader(),
+		Recorder: mgr.GetEventRecorderFor("felis-operator"),
+		Watch:    watch,
+	}
+	if os.Getenv("FELIS_DISTRIBUTED") == "true" {
+		r.Nodes = mgr.GetAPIReader()
 	}
 	if err := r.SetupWithManager(mgr); err != nil {
 		fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
@@ -128,3 +153,18 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// cacheSynced is a readyz check that passes once every informer the manager
+// started has synced. It waits at most a second, well inside the probe timeout.
+func cacheSynced(c interface {
+	WaitForCacheSync(ctx context.Context) bool
+}) healthz.Checker {
+	return func(req *http.Request) error {
+		ctx, cancel := context.WithTimeout(req.Context(), time.Second)
+		defer cancel()
+		if !c.WaitForCacheSync(ctx) {
+			return errors.New("informer caches not synced")
+		}
+		return nil
+	}
+}

cmd/felis/operator_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"context"
+	"net/http/httptest"
+	"testing"
+)
+
+type fakeCache bool
+
+func (f fakeCache) WaitForCacheSync(ctx context.Context) bool {
+	if !f {
+		<-ctx.Done()
+	}
+	return bool(f)
+}
+
+// TestCacheSynced: the operator reports ready only once its informers synced,
+// and a check against caches that never sync returns within its own deadline.
+func TestCacheSynced(t *testing.T) {
+	req := httptest.NewRequest("GET", "/readyz", nil)
+	if err := cacheSynced(fakeCache(true))(req); err != nil {
+		t.Errorf("synced: %v", err)
+	}
+	if err := cacheSynced(fakeCache(false))(req); err == nil {
+		t.Error("unsynced caches reported ready")
+	}
+}

cmd/felis/pinimages.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// defaultRegistryURL is the [registry] url every install uses; deploy/bootstrap.sh
+// spells the same value as REGISTRY_URL.
+const defaultRegistryURL = "registry.felis.svc:5000"
+
+// cmdPinImages pins every user server whose spec.image still names a mutable tag
+// in the platform registry to the digest that tag names now (internal/imagepin).
+// felis-api pins on create, so this covers the servers created before it did.
+//
+// deploy/bootstrap.sh runs it before it rebuilds the game images and pushes them
+// over the same tags: run after the push, it would pin those servers to the new
+// build, which is exactly the silent Minecraft upgrade pinning exists to stop.
+// It reaches the registry through the node's loopback hostPort, the same way the
+// installer pushes.
+func cmdPinImages(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("pin-images", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	namespace := fs.String("namespace", platform.DefaultMinecraftNamespace, "namespace the MinecraftServers live in")
+	registry := fs.String("registry", defaultRegistryURL, "registry host[:port] the image refs spell")
+	endpoint := fs.String("endpoint", "", "host[:port] to reach the registry at (default: 127.0.0.1 on the registry's port, its hostPort on this node)")
+	system := fs.String("system", "", "pin this system server ("+naming.SystemLoginServer+" or "+naming.SystemLobbyServer+") to the build its tag names now, instead of the user servers")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if *system != "" && *system != naming.SystemLoginServer && *system != naming.SystemLobbyServer {
+		fmt.Fprintf(stderr, "felis pin-images: --system takes %s or %s, not %q\n", naming.SystemLoginServer, naming.SystemLobbyServer, *system)
+		return 2
+	}
+	if *endpoint == "" {
+		*endpoint = loopbackEndpoint(*registry)
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	if *system != "" {
+		return reportSystemPin(ctx, cl, *namespace, *system, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint}, stdout, stderr)
+	}
+	outcomes, err := pinUserServerImages(ctx, cl, *namespace, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint})
+	if meta.IsNoMatchError(err) {
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	if len(outcomes) == 0 {
+		fmt.Fprintln(stdout, "felis pin-images: every user server already runs a pinned image")
+		return 0
+	}
+	fmt.Fprintln(stdout, "felis pin-images: pinning user servers to the build their tag names now:")
+	exit := 0
+	for _, o := range outcomes {
+		if o.err != nil {
+			fmt.Fprintf(stdout, "  - %s: ERROR %v\n", o.name, o.err)
+			exit = 1
+			continue
+		}
+		fmt.Fprintf(stdout, "  - %s: %s\n", o.name, strings.Join(o.changes, ", "))
+	}
+	return exit
+}
+
+// reportSystemPin runs pinSystemServerImage for `felis pin-images --system` and
+// prints what it did. Only a failed pin exits non-zero: the installer falls back
+// to restarting the pod on its tag then.
+func reportSystemPin(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver, stdout, stderr io.Writer) int {
+	o, err := pinSystemServerImage(ctx, cl, namespace, name, r)
+	switch {
+	case meta.IsNoMatchError(err):
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	case err == nil && o.err != nil:
+		err = o.err
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %s: %v\n", name, err)
+		return 1
+	}
+	switch {
+	case o.updated:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s; the operator rolls it onto that build\n", name, strings.Join(o.changes, ", "))
+	default:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s\n", name, o.skipped)
+	}
+	return 0
+}
+
+// pinSystemServerImage fixes a system server to the build its image tag names now,
+// replacing the digest of an earlier build. The installer runs it after pushing a
+// rebuilt login or lobby image, and the operator rolls the StatefulSet onto the new
+// ref, so the build a system server runs is written in its spec and moves only when
+// a build did. An image outside the platform registry, or one naming no tag to
+// follow, is the admin's choice and is left alone; so is a server whose image an
+// admin retargets while this runs.
+func pinSystemServerImage(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver) (systemServerOutcome, error) {
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &ms); err != nil {
+		if apierrors.IsNotFound(err) {
+			return systemServerOutcome{name: name, skipped: "not present yet; sudo felis setup creates it"}, nil
+		}
+		return systemServerOutcome{}, err
+	}
+	if ms.Labels[v1alpha1.LabelSystemRole] != name {
+		return systemServerOutcome{name: name, err: fmt.Errorf(
+			"MinecraftServer %s/%s is not marked as the Felis %q system server; left alone", namespace, name, name)}, nil
+	}
+	tagged := withoutDigest(ms.Spec.Image)
+	if !r.Covers(tagged) || !strings.Contains(tagged[strings.LastIndex(tagged, "/")+1:], ":") {
+		return systemServerOutcome{name: name, available: true, skipped: "runs " + ms.Spec.Image +
+			", which names no platform registry tag to follow; left alone"}, nil
+	}
+	pinned, err := r.Pin(ctx, tagged)
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("resolve %s: %w", tagged, err)}, nil
+	}
+	changed, err := patchOnConflictRetry(ctx, cl, &ms, func() bool {
+		if withoutDigest(ms.Spec.Image) != tagged || ms.Spec.Image == pinned {
+			return false
+		}
+		ms.Spec.Image = pinned
+		return true
+	})
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("patch %s: %w", name, err)}, nil
+	}
+	if !changed {
+		return systemServerOutcome{name: name, available: true, skipped: "already runs " + ms.Spec.Image}, nil
+	}
+	return systemServerOutcome{name: name, available: true, updated: true,
+		changes: []string{"spec.image pinned to " + pinned}}, nil
+}
+
+// withoutDigest drops the @sha256:… of a pinned ref, leaving the tag it came from.
+func withoutDigest(ref string) string {
+	if i := strings.Index(ref, "@"); i >= 0 {
+		return ref[:i]
+	}
+	return ref
+}
+
+// loopbackEndpoint is the registry's port on 127.0.0.1: the registry Deployment
+// binds it as a hostPort, and containerd's mirror and the installer's pushes use
+// the same address.
+func loopbackEndpoint(registry string) string {
+	if i := strings.LastIndex(registry, ":"); i >= 0 {
+		return "127.0.0.1" + registry[i:]
+	}
+	return "127.0.0.1"
+}
+
+// pinUserServerImages patches spec.image of every user server whose image the
+// resolver covers and is not yet pinned. System servers are the installer's to move:
+// it re-pins them with --system when it rolls them onto a new build
+// (restart_existing_system_servers).
+// A server that is already pinned, or runs an image from elsewhere, produces no
+// outcome, so a pinned fleet reports nothing. A running server restarts once as
+// the operator rolls its StatefulSet onto the pinned ref, which is the build it
+// already runs.
+func pinUserServerImages(ctx context.Context, cl client.Client, namespace string, r imagepin.Resolver) ([]systemServerOutcome, error) {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return nil, fmt.Errorf("list servers: %w", err)
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || imagepin.Pinned(ms.Spec.Image) || !r.Covers(ms.Spec.Image) {
+			continue
+		}
+		pinned, err := r.Pin(ctx, ms.Spec.Image)
+		if errors.Is(err, imagepin.ErrNotFound) {
+			err = fmt.Errorf("%s is not in the registry, so there is no build to pin it to; left unpinned: %w", ms.Spec.Image, err)
+		}
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: err})
+			continue
+		}
+		patch := client.MergeFrom(ms.DeepCopy())
+		ms.Spec.Image = pinned
+		if err := cl.Patch(ctx, ms, patch); err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("patch %s: %w", ms.Name, err)})
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{"spec.image pinned to " + pinned}})
+	}
+	return out, nil
+}

cmd/felis/pinimages_test.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+const pinTestDigest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
+
+// TestPinUserServerImages pins exactly the user servers still on a platform tag,
+// reports a tag the registry lost as an error without touching that server, and
+// has nothing left to do on a second pass.
+func TestPinUserServerImages(t *testing.T) {
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v2/felis/paper/manifests/demo" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Docker-Content-Digest", pinTestDigest)
+	}))
+	defer reg.Close()
+	res := imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+
+	paper := defaultRegistryURL + "/felis/paper:demo"
+	mk := func(name, image, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Image = image
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		mk("legacy", paper, ""),
+		mk("pinned", paper+"@sha256:"+strings.Repeat("3", 64), ""),
+		mk("external", "docker.io/itzg/minecraft-server:java21", ""),
+		mk("gone", defaultRegistryURL+"/felis/paper:old", ""),
+		mk(naming.SystemLobbyServer, defaultRegistryURL+"/felis/felis-lobby:demo", naming.SystemLobbyServer),
+	).Build()
+	ctx := context.Background()
+
+	outcomes, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil {
+		t.Fatalf("pinUserServerImages: %v", err)
+	}
+	byName := map[string]systemServerOutcome{}
+	for _, o := range outcomes {
+		byName[o.name] = o
+	}
+	if len(outcomes) != 2 || byName["legacy"].err != nil || byName["gone"].err == nil {
+		t.Fatalf("outcomes = %+v, want legacy pinned and gone reported", outcomes)
+	}
+
+	want := map[string]string{
+		"legacy":                 paper + "@" + pinTestDigest,
+		"pinned":                 paper + "@sha256:" + strings.Repeat("3", 64),
+		"external":               "docker.io/itzg/minecraft-server:java21",
+		"gone":                   defaultRegistryURL + "/felis/paper:old",
+		naming.SystemLobbyServer: defaultRegistryURL + "/felis/felis-lobby:demo",
+	}
+	for name, image := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Image != image {
+			t.Errorf("%s image = %q, want %q", name, ms.Spec.Image, image)
+		}
+	}
+
+	again, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil || len(again) != 1 || again[0].name != "gone" {
+		t.Fatalf("second pass = %+v, %v; want only the unresolvable server again", again, err)
+	}
+}
+
+// A fresh install has no CRD yet; the command must read that as nothing to pin.
+func TestPinUserServerImagesNoCRD(t *testing.T) {
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
+		List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error {
+			return &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "felis.lolicon.best", Kind: "MinecraftServer"}}
+		},
+	}).Build()
+	_, err := pinUserServerImages(context.Background(), cl, "minecraft", imagepin.Resolver{Registry: defaultRegistryURL})
+	if !meta.IsNoMatchError(err) {
+		t.Fatalf("err = %v, want a NoMatch error the command can recognise", err)
+	}
+}
+
+func TestLoopbackEndpoint(t *testing.T) {
+	for in, want := range map[string]string{
+		"registry.felis.svc:5000": "127.0.0.1:5000",
+		"registry.example":        "127.0.0.1",
+	} {
+		if got := loopbackEndpoint(in); got != want {
+			t.Errorf("loopbackEndpoint(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

cmd/felis/reaper.go

@@ -16,10 +16,9 @@ import (
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/backup"
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/distributed"
 	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/reaper"
-	"felis.lolicon.best/internal/store"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -33,11 +32,17 @@ import (
 // cadence, and RunOnce is idempotent and restart-safe, so a missed or retried
 // run simply converges. Only the tarLocal archive backend is wired in this
 // build; the snapshot backends (§19) are a later integration.
+//
+// --retention-only runs the archive-store half alone (reaper.RunRetention):
+// the CronJob an install without a worlds root gets, so backups past their
+// expiry still leave the store there. It builds no Kubernetes client and reads
+// no world.
 func cmdReaper(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("reaper", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
 	worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>, else the stock local-path <root>/<pv-name>_<ns>_<pvc-name>)")
+	retentionOnly := fs.Bool("retention-only", false, "only expire, read back and sweep the archive store: no server is evaluated and no world is read or deleted, so neither the worlds root nor the Kubernetes API is needed")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -56,13 +61,16 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 
 	ctx := ctrl.SetupSignalHandler()
 
-	scheme := runtime.NewScheme()
-	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-	utilruntime.Must(v1alpha1.AddToScheme(scheme))
-	cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
-	if err != nil {
-		fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
-		return 1
+	var cl client.Client
+	if !*retentionOnly {
+		scheme := runtime.NewScheme()
+		utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+		utilruntime.Must(v1alpha1.AddToScheme(scheme))
+		cl, err = client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
+		if err != nil {
+			fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
+			return 1
+		}
 	}
 
 	archiver, err := buildArchiver(ctx, cfg, *worldsRoot, cl)
@@ -71,7 +79,20 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	if os.Getenv("FELIS_DISTRIBUTED") == "true" && !*retentionOnly {
+		m, err := distributionManager(cl, cfg, os.Getenv("FELIS_IMAGE"))
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		local, ok := archiver.(*backup.TarLocal)
+		if !ok {
+			fmt.Fprintln(stderr, "remote reaper requires tarLocal")
+			return 1
+		}
+		archiver = &distributed.RemoteArchiver{TarLocal: local, Manager: m}
+	}
+	drv, err := openPodStore(ctx, cfg.Database.URL, "reaper", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
 		return 1
@@ -81,9 +102,13 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 	r := &reaper.Reaper{
 		Cfg:      rcfg,
 		Store:    reaper.NewPGStore(drv.DB()),
-		Cluster:  reaper.NewK8sCluster(cl, cfg.K8s.Namespace),
 		Archiver: archiver,
 	}
+	if *retentionOnly {
+		fmt.Fprintln(stderr, "felis reaper: retention only — no worlds root is configured, so idle worlds are neither archived nor released")
+		return reportReaperRun(r.RunRetention(ctx), stdout, stderr)
+	}
+	r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace).WithDistributed(os.Getenv("FELIS_DISTRIBUTED") == "true")
 
 	// Pre-reap warnings go out by email when [smtp] is configured (the same
 	// relay and password_ref convention felis-api uses); without it the channel
@@ -114,13 +139,7 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 				}
 				return email, nil
 			},
-			notifier: &mail.SMTP{
-				Host:     cfg.SMTP.Host,
-				Port:     cfg.SMTP.Port,
-				From:     cfg.SMTP.From,
-				Username: cfg.SMTP.Username,
-				Password: password,
-			},
+			notifier: smtpRelay(cfg.SMTP, password),
 		}
 	} else {
 		fmt.Fprintln(stderr, "felis reaper: [smtp] not configured — pre-reap warnings are logged and NOT marked sent")
@@ -131,9 +150,33 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis reaper: %v\n", err)
 		return 1
 	}
-	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d warned=%d skipped=%d evicted=%d expired=%d\n",
-		sum.Evaluated, sum.WorldsReaped, sum.Warned, sum.Skipped, sum.EvictedEarly, sum.BackupsExpired)
-	return 0
+	return reportReaperRun(sum, stdout, stderr)
+}
+
+// reportReaperRun prints the run's tally and turns a run that left work undone
+// into exit 1, so the Job fails and the watchdog's job-failed check (and the
+// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
+// is kept, and without this nobody would learn that it is never reaped.
+func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
+	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d released=%d deleted=%d awaiting_offsite=%d awaiting_stop=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
+		sum.Evaluated, sum.WorldsReaped, sum.Released, sum.ServersDeleted, sum.AwaitingOffsite, sum.AwaitingStop, sum.Warned, sum.Skipped, sum.StoreFull,
+		sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
+		sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
+	if !sum.Failed() {
+		return 0
+	}
+	if sum.Skipped > 0 || sum.ExpireFailed > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
+			sum.Skipped, sum.StoreFull, sum.ExpireFailed)
+	}
+	if sum.Corrupt > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)
+	}
+	if sum.VerifyFailed > 0 || sum.SweepFailed {
+		fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",
+			sum.VerifyFailed, !sum.SweepFailed)
+	}
+	return 1
 }
 
 // mailWarner delivers a pre-reap notice to the owner's verified email — the
@@ -169,8 +212,10 @@ func (w *mailWarner) Warn(ctx context.Context, ownerID, server, remaining string
 }
 
 // reaperConfig derives the reaper's retention windows from felis.toml. The 15d
-// idle deadline is fixed by §18; only the warning offsets, retention, and the
-// store soft-cap are configurable (§24).
+// idle deadline is fixed by §18; only the warning offsets, retention, the
+// store soft-cap, the on-demand backup bounds and the scheduled restore points
+// are configurable (§24). The backup Job and felis-api read the manual_* and
+// scheduled_* keys through it too.
 func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 	rc := reaper.DefaultConfig()
 	if v := cfg.Archive.Retention; v != "" {
@@ -198,19 +243,66 @@ func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 		}
 		rc.MaxLocalBytes = b
 	}
+	if v := cfg.Archive.ManualRetention; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d <= 0 {
+			return rc, fmt.Errorf("[archive] manual_retention %q: want a positive span such as 30d", v)
+		}
+		rc.ManualRetention = d
+	}
+	switch n := cfg.Archive.ManualKeep; {
+	case n < 0:
+		return rc, fmt.Errorf("[archive] manual_keep %d: want 1 or more", n)
+	case n > 0:
+		rc.ManualKeep = n
+	}
+	if v := cfg.Archive.ManualCooldown; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d < 0 {
+			return rc, fmt.Errorf("[archive] manual_cooldown %q: want a span such as 10m (0s for none)", v)
+		}
+		rc.ManualCooldown = d
+	}
+	if v := cfg.Archive.ScheduledEvery; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d < 0 {
+			return rc, fmt.Errorf("[archive] scheduled_every %q: want a span such as 1d (0s for none)", v)
+		}
+		rc.ScheduledEvery = d
+	}
+	switch n := cfg.Archive.ScheduledKeep; {
+	case n < 0:
+		return rc, fmt.Errorf("[archive] scheduled_keep %d: want 1 or more", n)
+	case n > 0:
+		rc.ScheduledKeep = n
+	}
+	if v := cfg.Archive.ScheduledRetention; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d <= 0 {
+			return rc, fmt.Errorf("[archive] scheduled_retention %q: want a positive span such as 90d", v)
+		}
+		rc.ScheduledRetention = d
+	}
+	rc.RequireOffsite = cfg.Offsite.Enabled()
 	return rc, nil
 }
 
 // buildArchiver constructs the WorldArchiver. Only tarLocal is implemented in
 // this build; the resolver maps each world PVC to its directory under worldsRoot
-// (resolveWorldDir).
+// (resolveWorldDir). A nil cl is the retention-only run, which never archives a
+// world, so its archiver resolves none.
 func buildArchiver(ctx context.Context, cfg *config.Config, worldsRoot string, cl client.Client) (backup.WorldArchiver, error) {
 	switch cfg.Archive.Store {
 	case "tarLocal":
-		return &backup.TarLocal{
-			BackupRoot: cfg.Archive.LocalPath,
-			Resolve:    resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot),
-		}, nil
+		t := &backup.TarLocal{BackupRoot: cfg.Archive.LocalPath}
+		if cl != nil {
+			t.Resolve = resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot)
+		} else {
+			t.Resolve = func(pvc string) (string, error) {
+				return "", fmt.Errorf("resolve world PVC %s: this run has no worlds root (retention only)", pvc)
+			}
+		}
+		return t, nil
 	default:
 		return nil, fmt.Errorf("[archive] store %q is not implemented in this build (only tarLocal)", cfg.Archive.Store)
 	}
@@ -253,27 +345,9 @@ func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoo
 	}
 }
 
-// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
-// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
-func parseSpanDuration(s string) (time.Duration, error) {
-	s = strings.TrimSpace(s)
-	switch {
-	case strings.HasSuffix(s, "mo"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
-		if err != nil {
-			return 0, err
-		}
-		return time.Duration(n) * 30 * 24 * time.Hour, nil
-	case strings.HasSuffix(s, "d"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
-		if err != nil {
-			return 0, err
-		}
-		return time.Duration(n) * 24 * time.Hour, nil
-	default:
-		return time.ParseDuration(s)
-	}
-}
+// parseSpanDuration parses the human spans used in felis.toml's [archive] table
+// (config.ParseSpan).
+func parseSpanDuration(s string) (time.Duration, error) { return config.ParseSpan(s) }
 
 // parseByteSize parses a Kubernetes-style quantity ("200Gi", "10G") into bytes.
 // An empty string means unlimited (0).

cmd/felis/reaper_test.go

@@ -1,22 +1,129 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/reaper"
 )
 
+// TestReportReaperRunFailsTheJob: a run that could not process a server, or
+// could not remove an expired backup, exits 1 so the Job shows as failed.
+func TestReportReaperRunFailsTheJob(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		sum  reaper.Summary
+		want int
+	}{
+		{"clean", reaper.Summary{Evaluated: 3, WorldsReaped: 1, AwaitingOffsite: 1}, 0},
+		{"retirements", reaper.Summary{Evaluated: 5, WorldsReaped: 3, Released: 2, ServersDeleted: 1}, 0},
+		{"waiting for a stop", reaper.Summary{Evaluated: 3, AwaitingStop: 1}, 0},
+		{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
+		{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
+		{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},
+		{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},
+		{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},
+		{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},
+		{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},
+	} {
+		var out, errb bytes.Buffer
+		if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {
+			t.Errorf("%s: exit %d, want %d", tc.name, got, tc.want)
+		}
+		if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") ||
+			!strings.Contains(out.String(), fmt.Sprintf(" released=%d deleted=%d ", tc.sum.Released, tc.sum.ServersDeleted)) {
+			t.Errorf("%s: summary line = %q", tc.name, out.String())
+		}
+		if (tc.want == 1) != (errb.Len() > 0) {
+			t.Errorf("%s: stderr = %q", tc.name, errb.String())
+		}
+	}
+}
+
 // TestResolveWorldDir pins the two world layouts the reaper must find, and the
 // fail-closed miss. The stock local-path arm is derived from the live PVC's
 // volumeName — a name-based guess (glob) could tar a stale deleted PV's bytes and
 // then delete the current world, which is why it is read from the API instead.
+// TestReaperConfigManualKeys: the on-demand backup keys default to 30 days,
+// five per server and a ten-minute cooldown, accept overrides, and refuse
+// values that would keep nothing or throttle backwards.
+func TestReaperConfigManualKeys(t *testing.T) {
+	rc, err := reaperConfig(&config.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 30*reaper.Day || rc.ManualKeep != 5 || rc.ManualCooldown != 10*time.Minute {
+		t.Fatalf("defaults = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
+		ManualRetention: "7d", ManualKeep: 2, ManualCooldown: "0s"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 7*reaper.Day || rc.ManualKeep != 2 || rc.ManualCooldown != 0 {
+		t.Fatalf("overrides = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	for _, bad := range []config.ArchiveConfig{
+		{ManualRetention: "0d"},
+		{ManualRetention: "soon"},
+		{ManualKeep: -1},
+		{ManualCooldown: "-5m"},
+		{ManualCooldown: "often"},
+	} {
+		if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
+			t.Errorf("%+v was accepted", bad)
+		}
+	}
+}
+
+// TestReaperConfigScheduledKeys: the scheduled restore points default to one a
+// day, seven per server and the reaper's 90 days, accept overrides ("0s" turns
+// them off), and refuse values that would keep nothing or run backwards.
+func TestReaperConfigScheduledKeys(t *testing.T) {
+	rc, err := reaperConfig(&config.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ScheduledEvery != reaper.Day || rc.ScheduledKeep != 7 || rc.ScheduledRetention != 90*reaper.Day {
+		t.Fatalf("defaults = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
+		ScheduledEvery: "12h", ScheduledKeep: 3, ScheduledRetention: "14d"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ScheduledEvery != 12*time.Hour || rc.ScheduledKeep != 3 || rc.ScheduledRetention != 14*reaper.Day {
+		t.Fatalf("overrides = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{ScheduledEvery: "0s"}})
+	if err != nil || rc.ScheduledEvery != 0 {
+		t.Fatalf("scheduled_every 0s = %v, %v; want off", rc.ScheduledEvery, err)
+	}
+	for _, bad := range []config.ArchiveConfig{
+		{ScheduledEvery: "-1h"},
+		{ScheduledEvery: "daily"},
+		{ScheduledKeep: -1},
+		{ScheduledRetention: "0d"},
+		{ScheduledRetention: "forever"},
+	} {
+		if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
+			t.Errorf("%+v was accepted", bad)
+		}
+	}
+}
+
 func TestResolveWorldDir(t *testing.T) {
 	ctx := context.Background()
 	root := t.TempDir()

cmd/felis/registrygate.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/registrygate"
+)
+
+// cmdRegistryGate is the sidecar entrypoint in the registry pod: it owns the
+// registry port (and the loopback hostPort containerd pulls through), lets reads
+// through anonymously, and forwards writes to the loopback-only registry:2 only
+// for an authenticated principal allowed to write that repository. See
+// internal/registrygate for the policy.
+//
+// Tokens are files under --auth-dir, one per principal (platform, build, prune),
+// mounted from the registry-auth Secret. A missing file disables that principal:
+// writes fail closed while every pull keeps working, which is the right way round
+// for a registry the running workloads depend on.
+//
+// --maint-listen is the GC sidecar's read-only handshake (registrygate.MaintHandler).
+// It has no authentication, so it must name a loopback address; --maint-dir keeps
+// an open window across a gate restart.
+func cmdRegistryGate(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("registry-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	listen := fs.String("listen", ":5000", "address the gate serves the registry API on")
+	upstream := fs.String("upstream", "http://127.0.0.1:5001", "the loopback registry the gate forwards to")
+	authDir := fs.String("auth-dir", "/etc/felis-registry-auth", "directory holding one token file per principal")
+	maintListen := fs.String("maint-listen", "", "loopback address for the GC sidecar's read-only handshake (empty disables it)")
+	maintDir := fs.String("maint-dir", "", "directory that keeps an open read-only window across a gate restart")
+	quiet := fs.Duration("maint-quiet", registrygate.DefaultQuiet, "how long writes must be idle before a read-only window is granted")
+	dataDir := fs.String("data-dir", "", "the registry's storage root, mounted read-only, for the manifest index (empty disables it)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *maintListen != "" && !loopbackAddr(*maintListen) {
+		fmt.Fprintf(stderr, "felis registry-gate: --maint-listen %q must be a loopback address: the handshake has no authentication\n", *maintListen)
+		return 2
+	}
+	target, err := url.Parse(*upstream)
+	if err != nil || target.Scheme == "" || target.Host == "" {
+		fmt.Fprintf(stderr, "felis registry-gate: bad --upstream %q\n", *upstream)
+		return 2
+	}
+	log := slog.New(slog.NewTextHandler(stderr, nil))
+	tokens := map[string]string{}
+	for _, p := range registrygate.Principals {
+		b, err := os.ReadFile(filepath.Join(*authDir, p))
+		tok := strings.TrimSpace(string(b))
+		if err != nil || tok == "" {
+			log.Warn("registry principal disabled: no token", "principal", p, "dir", *authDir)
+			continue
+		}
+		tokens[p] = tok
+	}
+
+	gate := registrygate.New(target, tokens, log)
+	gate.SetQuiet(*quiet)
+	gate.DataDir = *dataDir
+	if *maintDir != "" {
+		if err := gate.SetMaintenanceState(registrygate.MaintStatePath(*maintDir)); err != nil {
+			// A corrupt file must not keep the registry from serving pulls.
+			log.Warn("ignoring the saved read-only window", "err", err)
+		}
+	}
+	srv := &http.Server{
+		Addr:              *listen,
+		Handler:           gate,
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	var maint *http.Server
+	if *maintListen != "" {
+		maint = &http.Server{Addr: *maintListen, Handler: gate.MaintHandler(), ReadHeaderTimeout: 10 * time.Second}
+		go func() {
+			if err := maint.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				log.Error("maintenance listener stopped; garbage collection cannot get a read-only window", "err", err)
+			}
+		}()
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	go func() {
+		<-ctx.Done()
+		shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(shutdown)
+		if maint != nil {
+			_ = maint.Shutdown(shutdown)
+		}
+	}()
+	log.Info("registry gate listening", "addr", *listen, "upstream", target.String(), "principals", len(tokens))
+	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintf(stderr, "felis registry-gate: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// loopbackAddr reports whether a host:port listen address binds loopback only.
+func loopbackAddr(addr string) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return false
+	}
+	if host == "localhost" {
+		return true
+	}
+	ip := net.ParseIP(host)
+	return ip != nil && ip.IsLoopback()
+}
+
+// cmdPushImage is the build Job's publish step. It runs after Kaniko built the
+// image into a tarball (--no-push) and Trivy passed that tarball, and it is the
+// only container of the build pod that holds the registry credential — the one
+// executing the untrusted Dockerfile never sees it.
+func cmdPushImage(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("push-image", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path, or with --image an OCI layout tar")
+	image := fs.String("image", "", "push the image this name (io.containerd.image.name) marks in the OCI layout tar --tar, e.g. a release's image bundle")
+	ref := fs.String("ref", "", "host/repository:tag to publish it as")
+	scheme := fs.String("scheme", "http", "registry scheme: http for the in-cluster registry, https otherwise")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *tarPath == "" || *ref == "" {
+		fmt.Fprintln(stderr, "felis push-image: --tar and --ref are required")
+		return 2
+	}
+	if *scheme != "http" && *scheme != "https" {
+		fmt.Fprintf(stderr, "felis push-image: bad --scheme %q\n", *scheme)
+		return 2
+	}
+	user := os.Getenv("FELIS_REGISTRY_USERNAME")
+	pass := os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if user == "" || pass == "" {
+		fmt.Fprintln(stderr, "felis push-image: FELIS_REGISTRY_USERNAME/FELIS_REGISTRY_PASSWORD are empty — the registry refuses anonymous writes")
+		return 2
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: *scheme, Username: user, Password: pass, Log: stderr}
+	var digest string
+	var err error
+	if *image != "" {
+		digest, err = p.PushLayout(ctx, *tarPath, *image, *ref)
+	} else {
+		digest, err = p.Push(ctx, *tarPath, *ref)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis push-image: %v\n", err)
+		return 1
+	}
+	fmt.Fprintln(stdout, digest)
+	return 0
+}

cmd/felis/restore.go

@@ -1,14 +1,18 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io"
+	"os"
+	"os/signal"
 	"path/filepath"
 	"strings"
+	"syscall"
 
+	"felis.lolicon.best/internal/archivetransfer"
 	"felis.lolicon.best/internal/backup"
-	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 // cmdRestore is the in-Pod entrypoint the restore Job runs. internal/restore
@@ -26,6 +30,9 @@ import (
 func cmdRestore(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("restore", flag.ContinueOnError)
 	fs.SetOutput(stderr)
+	source := fs.String("source-url", "", "one-use archive download URL")
+	sum := fs.String("sha256", "", "required digest for remote archive")
+	limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum download size")
 	server := fs.String("server", "", "server name being restored (for logging)")
 	ref := fs.String("ref", "", "absolute path to the archive on the backup mount")
 	store := fs.String("archive-store", "tarLocal", "archive backend (only tarLocal is implemented)")
@@ -35,6 +42,22 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int {
 		return 2
 	}
 
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+	if *source != "" {
+		dir, err := os.MkdirTemp("/tmp", "felis-restore-")
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		defer os.RemoveAll(dir)
+		*backupRoot = dir
+		*ref = filepath.Join(dir, "world.tar.gz")
+		if err := archivetransfer.Fetch(ctx, *source, os.Getenv(archivetransfer.TokenEnv), *ref, *sum, *limit); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
 	if *ref == "" {
 		fmt.Fprintln(stderr, "felis restore: --ref is required")
 		return 2
@@ -62,11 +85,16 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int {
 		},
 	}
 
-	ctx := ctrl.SetupSignalHandler()
 	if err := archiver.Restore(ctx, backup.ArchiveRef(*ref), *server); err != nil {
 		fmt.Fprintf(stderr, "felis restore: %v\n", err)
 		return 1
 	}
+	if *source != "" {
+		if err := backup.VerifyRestored(ctx, *ref, *worldsRoot); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
 	fmt.Fprintf(stdout, "felis restore: server=%s restored from %s into %s\n", *server, *ref, *worldsRoot)
 	return 0
 }

cmd/felis/restore_test.go

@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup
 		BackupRoot: backupRoot,
 		Resolve:    func(string) (string, error) { return srcDir, nil },
 	}
-	got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0")
+	got, err := ar.Archive(context.Background(), "survival", "world-survival-0")
 	if err != nil {
 		t.Fatalf("Archive: %v", err)
 	}
-	return string(got), backupRoot
+	return string(got.Ref), backupRoot
 }
 
 // The restore subcommand must extract the archived world into the target world

cmd/felis/rotatetoken.go

@@ -0,0 +1,854 @@
+package main
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/pbkdf2"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	neturl "net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/maintenance"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/operator"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/registrygate"
+
+	"github.com/BurntSushi/toml"
+	"github.com/jackc/pgx/v5"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// rotate-token replaces one credential the installer generated: an internal
+// caller's token (naming.CallerTokens), the registry's write tokens, the
+// Velocity forwarding secret or the database password. The new value goes into
+// the installer's record first, so that whatever fails later a re-run of the
+// installer puts it everywhere; then into the Secrets and files that carry it;
+// then whatever read the old value at start restarts. Without -yes it prints
+// what it would change and what that interrupts, and changes nothing.
+
+const (
+	defaultSecretsEnvPath = "/etc/felis/secrets.env"
+	defaultLinkPropsPath  = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
+	defaultForwardingPath = "/opt/felis/velocity/forwarding.secret"
+	velocityUnit          = "felis-velocity"
+	apiDeployment         = "felis-api"
+	registryDeployment    = "registry"
+
+	kindRegistry   = "registry"
+	kindForwarding = "forwarding"
+	kindDB         = "db"
+
+	// proxyReloadWait is how long the host proxy gets, once felis-api has rolled,
+	// to show it re-read its token: it reads the file again on its next call to
+	// felis-api, and it calls every 15 seconds.
+	proxyReloadWait = 60 * time.Second
+
+	// apiRestartNote is in every plan: felis-api runs as one replica replaced in
+	// place, so its restart is a short outage of everything that talks to it.
+	apiRestartNote = "felis-api restarts (a single replica): the panel, sign-in and the proxy's calls are unavailable for the few seconds that takes"
+)
+
+// installerTokenKeys names each caller's token in the installer's secrets.env
+// (deploy/bootstrap.sh load_or_make_secrets). A re-run of the installer applies
+// these values to the Secrets, so a rotation that skipped the file would be
+// undone by the next upgrade.
+var installerTokenKeys = map[string]string{
+	"velocity": "SERVICE_TOKEN",
+	"limbo":    "LIMBO_TOKEN",
+	"build":    "BUILD_TOKEN",
+	"ops":      "OPS_TOKEN",
+}
+
+// installerRegistryKeys names the registry principals' tokens in secrets.env,
+// in registrygate.Principals order.
+var installerRegistryKeys = map[string]string{
+	registrygate.PrincipalPlatform: "REGISTRY_PLATFORM_TOKEN",
+	registrygate.PrincipalBuild:    "REGISTRY_BUILD_TOKEN",
+	registrygate.PrincipalPrune:    "REGISTRY_PRUNE_TOKEN",
+}
+
+// installerForwardingKey and installerDBKey name the forwarding secret and the
+// database password in secrets.env.
+const (
+	installerForwardingKey = "FORWARDING_SECRET"
+	installerDBKey         = "DB_PASSWORD"
+)
+
+type tokenRotator struct {
+	cl          client.Client
+	controlNS   string
+	minecraftNS string
+	buildNS     string
+	// secretsEnv, linkProps and forwardingFile are the installer's record and the
+	// host proxy's felis-link.properties and forwarding.secret; a missing file is
+	// reported and skipped.
+	secretsEnv     string
+	linkProps      string
+	forwardingFile string
+	// hostTOML, podTOML and defaultTOML are the config copies that carry the
+	// database URL (defaultTOML only when it is a file of its own).
+	hostTOML    string
+	podTOML     string
+	defaultTOML string
+	newToken    func() (string, error)
+	// rollout restarts a control-namespace Deployment and waits for it.
+	rollout func(ctx context.Context, deployment string) error
+	// restartUnit restarts a systemd unit on this host.
+	restartUnit func(ctx context.Context, unit string) error
+	// proxyLog is what the host proxy has logged since a moment, as far as it
+	// can be read.
+	proxyLog func(ctx context.Context, since time.Time) string
+	// alterRole stores a password verifier for a role of the database that runs
+	// as deployment ("namespace/name"); verifyDB connects with a URL.
+	alterRole func(ctx context.Context, deployment, role, verifier string) error
+	verifyDB  func(ctx context.Context, url string) error
+	now       func() time.Time
+	// reloadWait bounds the wait for the proxy to re-read its token, polled
+	// every pollEvery.
+	reloadWait time.Duration
+	pollEvery  time.Duration
+	out        io.Writer
+}
+
+func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("rotate-token", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
+	linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
+	forwarding := fs.String("forwarding-secret", defaultForwardingPath, "the host proxy's forwarding secret file (forwarding only)")
+	yes := fs.Bool("yes", false, "rotate; without it the plan is printed and nothing changes")
+	fs.Usage = func() {
+		fmt.Fprintf(stderr, "Usage: felis rotate-token [-yes] [flags] <%s>\n\n", strings.Join(rotationKinds(), "|"))
+		fmt.Fprintln(stderr, "Replaces one generated credential: the installer's record, the Secrets and files that carry it, then what reads it.")
+		fmt.Fprintln(stderr, "Without -yes it prints what would change and what that interrupts.")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if fs.NArg() != 1 {
+		fs.Usage()
+		return 2
+	}
+	kind := fs.Arg(0)
+	if !knownRotation(kind) {
+		fmt.Fprintf(stderr, "felis rotate-token: unknown credential %q (one of %s)\n", kind, strings.Join(rotationKinds(), ", "))
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+kind+")")
+		return 1
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	buildNS := cfg.Registry.BuildNamespace
+	if buildNS == "" {
+		buildNS = platform.DefaultBuildNamespace
+	}
+	controlNS := platform.DefaultControlNamespace
+	r := tokenRotator{
+		cl:             cl,
+		controlNS:      controlNS,
+		minecraftNS:    cfg.K8s.Namespace,
+		buildNS:        buildNS,
+		secretsEnv:     *secretsEnv,
+		linkProps:      *linkProps,
+		forwardingFile: *forwarding,
+		hostTOML:       hostSetupConfigPath,
+		podTOML:        podSetupConfigPath,
+		defaultTOML:    defaultSetupConfigPath,
+		newToken:       randomToken,
+		rollout: func(ctx context.Context, deployment string) error {
+			if err := kubectl(ctx, "-n", controlNS, "rollout", "restart", "deployment/"+deployment); err != nil {
+				return err
+			}
+			return kubectl(ctx, "-n", controlNS, "rollout", "status", "deployment/"+deployment, "--timeout=180s")
+		},
+		restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
+		proxyLog:    journalSince,
+		alterRole:   alterRoleInPod,
+		verifyDB: func(ctx context.Context, url string) error {
+			ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+			defer cancel()
+			conn, err := pgx.Connect(ctx, url)
+			if err != nil {
+				return err
+			}
+			return conn.Close(ctx)
+		},
+		now:        time.Now,
+		reloadWait: proxyReloadWait,
+		pollEvery:  3 * time.Second,
+		out:        stdout,
+	}
+	if err := r.rotate(context.Background(), kind, *yes); err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+func callerNames() []string {
+	names := make([]string, 0, len(naming.CallerTokens))
+	for _, ct := range naming.CallerTokens {
+		names = append(names, ct.Caller)
+	}
+	return names
+}
+
+func callerToken(name string) (naming.CallerToken, bool) {
+	for _, ct := range naming.CallerTokens {
+		if ct.Caller == name {
+			return ct, true
+		}
+	}
+	return naming.CallerToken{}, false
+}
+
+// rotationKinds is every credential rotate-token replaces: the callers' tokens
+// and the installer's other generated secrets.
+func rotationKinds() []string {
+	return append(callerNames(), kindRegistry, kindForwarding, kindDB)
+}
+
+func knownRotation(kind string) bool {
+	for _, k := range rotationKinds() {
+		if k == kind {
+			return true
+		}
+	}
+	return false
+}
+
+// randomToken is 32 random bytes in hex, the shape the installer generates.
+func randomToken() (string, error) {
+	b := make([]byte, 32)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(b), nil
+}
+
+// tokenFingerprint is how the proxy names the token it reloaded in its log
+// (plugins/shared FileToken.fingerprint): the first twelve hex digits of its
+// SHA-256, enough to tell tokens apart and useless for finding one.
+func tokenFingerprint(token string) string {
+	sum := sha256.Sum256([]byte(token))
+	return hex.EncodeToString(sum[:])[:12]
+}
+
+func (r tokenRotator) rotate(ctx context.Context, kind string, apply bool) error {
+	switch kind {
+	case kindRegistry:
+		return r.rotateRegistry(ctx, apply)
+	case kindForwarding:
+		return r.rotateForwarding(ctx, apply)
+	case kindDB:
+		return r.rotateDB(ctx, apply)
+	}
+	ct, ok := callerToken(kind)
+	if !ok {
+		return fmt.Errorf("unknown credential %q (one of %s)", kind, strings.Join(rotationKinds(), ", "))
+	}
+	return r.rotateCaller(ctx, ct, apply)
+}
+
+// confirm ends the plan: without apply it says nothing changed and how to go
+// ahead, and reports false.
+func (r tokenRotator) confirm(kind string, apply bool) bool {
+	if !apply {
+		fmt.Fprintf(r.out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes %s\n", kind)
+		return false
+	}
+	fmt.Fprintln(r.out, "\nRotating:")
+	return true
+}
+
+// record writes new values into the installer's secrets.env. It comes first in
+// every rotation: from then on, whatever fails, a re-run of the installer puts
+// the new values everywhere.
+func (r tokenRotator) record(kv ...[2]string) error {
+	keys := make([]string, len(kv))
+	for i, p := range kv {
+		keys[i] = p[0]
+	}
+	switch err := setKeyValueLines(r.secretsEnv, "=", kv); {
+	case errors.Is(err, fs.ErrNotExist):
+		fmt.Fprintf(r.out, "  - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
+	case err != nil:
+		return fmt.Errorf("record the new value in %s: %w", r.secretsEnv, err)
+	default:
+		fmt.Fprintf(r.out, "  - %s: %s updated\n", r.secretsEnv, strings.Join(keys, ", "))
+	}
+	return nil
+}
+
+func (r tokenRotator) putSecret(ctx context.Context, ns, name string, data map[string][]byte) error {
+	if _, err := putSecretKeys(ctx, r.cl, ns, name, corev1.SecretTypeOpaque, data); err != nil {
+		return fmt.Errorf("write Secret %s/%s: %w", ns, name, err)
+	}
+	fmt.Fprintf(r.out, "  - Secret %s/%s: updated\n", ns, name)
+	return nil
+}
+
+func (r tokenRotator) rollAPI(ctx context.Context) error {
+	if err := r.rollout(ctx, apiDeployment); err != nil {
+		return fmt.Errorf("roll felis-api: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - felis-api: rolled out on the new value")
+	return nil
+}
+
+// withMinecraft is the control namespace plus the minecraft namespace when that
+// is a different one: where the Secrets game pods and Jobs mount are mirrored.
+func (r tokenRotator) withMinecraft() []string {
+	if r.minecraftNS == "" || r.minecraftNS == r.controlNS {
+		return []string{r.controlNS}
+	}
+	return []string{r.controlNS, r.minecraftNS}
+}
+
+func (r tokenRotator) rotateCaller(ctx context.Context, ct naming.CallerToken, apply bool) error {
+	namespaces := []string{r.controlNS}
+	replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
+	if replica != "" && replica != r.controlNS {
+		namespaces = append(namespaces, replica)
+	}
+	key := installerTokenKeys[ct.Caller]
+
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new internal token for the %s caller\n", ct.Caller, ct.Caller)
+	secrets := make([]string, len(namespaces))
+	for i, ns := range namespaces {
+		secrets[i] = "Secret " + ns + "/" + ct.Secret
+	}
+	writes := append([]string{r.secretsEnv + " (" + key + ")"}, secrets...)
+	hostProxy := ct.Caller == "velocity" && fileExists(r.linkProps)
+	if hostProxy {
+		writes = append(writes, r.linkProps+" (service-token)")
+	}
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
+	switch ct.Caller {
+	case "velocity":
+		if hostProxy {
+			fmt.Fprintf(r.out, "  - the proxy re-reads its token from the file and keeps its players; if it has not within %s of felis-api's restart, it is restarted, which disconnects everyone online\n", r.reloadWait)
+		} else {
+			fmt.Fprintf(r.out, "  - no proxy on this host (%s): set service-token in your proxy's felis-link.properties to the value in Secret %s/%s afterwards; it re-reads the file without a restart\n",
+				r.linkProps, r.controlNS, ct.Secret)
+		}
+	case "limbo":
+		fmt.Fprintln(r.out, "  - the login gate's pod restarts: a player signing in at that moment reconnects")
+	case "build":
+		fmt.Fprintln(r.out, "  - a build fetching its context at that moment fails and can be submitted again")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now presents the new token on its next run")
+	}
+	if !r.confirm(ct.Caller, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a token: %w", err)
+	}
+	if err := r.record([2]string{key, tok}); err != nil {
+		return err
+	}
+	for _, ns := range namespaces {
+		if err := r.putSecret(ctx, ns, ct.Secret, map[string][]byte{naming.ServiceTokenSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+
+	// The proxy's file changes before felis-api rolls, so the file never falls
+	// behind the Secret; the proxy's log is read from just before the write,
+	// since it may pick the new token up before the rollout ends.
+	since := r.now()
+	if hostProxy {
+		if err := setProxyToken(r.linkProps, tok); err != nil {
+			return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: service-token updated\n", r.linkProps)
+	}
+
+	if err := r.rollAPI(ctx); err != nil {
+		return err
+	}
+
+	switch ct.Caller {
+	case "velocity":
+		if !hostProxy {
+			break
+		}
+		if r.proxyReloaded(ctx, since, tokenFingerprint(tok)) {
+			fmt.Fprintf(r.out, "  - %s: took the new token from its properties; players stayed connected\n", velocityUnit)
+			break
+		}
+		if err := r.restartUnit(ctx, velocityUnit); err != nil {
+			return fmt.Errorf("restart %s: %w", velocityUnit, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: had not taken the new token within %s, restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit, r.reloadWait)
+	case "limbo":
+		// Only the operator's server pods carry its managed-by label; a backup or
+		// restore Job's pod carries the server label too, and app.kubernetes.io ones.
+		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), client.MatchingLabels{
+			v1alpha1.LabelServer:    naming.SystemLoginServer,
+			v1alpha1.LabelManagedBy: operator.ManagedByValue,
+		}); err != nil {
+			return fmt.Errorf("restart the login gate: %w", err)
+		}
+		fmt.Fprintln(r.out, "  - login gate: pod restarted to read the new token")
+	case "build":
+		fmt.Fprintln(r.out, "  - builds: the next build Job reads the new token")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now reads the new token on its next run")
+	}
+	return nil
+}
+
+// proxyReloaded waits up to reloadWait for the host proxy to log that it
+// reloaded the token with this fingerprint (plugins/shared FileToken).
+func (r tokenRotator) proxyReloaded(ctx context.Context, since time.Time, fingerprint string) bool {
+	want := "(fingerprint " + fingerprint + ")"
+	deadline := r.now().Add(r.reloadWait)
+	for {
+		if strings.Contains(r.proxyLog(ctx, since), want) {
+			return true
+		}
+		if !r.now().Before(deadline) {
+			return false
+		}
+		select {
+		case <-ctx.Done():
+			return false
+		case <-time.After(r.pollEvery):
+		}
+	}
+}
+
+// journalSince is the proxy unit's journal from the second since falls in. A
+// journal that cannot be read reads as one without the line, which ends in the
+// restart a rotation made before the proxy could reload.
+func journalSince(ctx context.Context, since time.Time) string {
+	out, _ := exec.CommandContext(ctx, "journalctl", "-u", velocityUnit, "--since", "@"+strconv.FormatInt(since.Unix(), 10),
+		"-o", "cat", "--no-pager", "-q").Output()
+	return string(out)
+}
+
+// setProxyToken writes the proxy's token into felis-link.properties and puts
+// the file's modification time back. The proxy re-reads its token by itself,
+// while `felis domain check` and `felis domain set` read a file newer than the
+// proxy's start as config it has not loaded (the installer's
+// install_if_changed keeps the time for the same reason).
+func setProxyToken(path, token string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	if err := setKeyValueLine(path, "service-token", "=", token); err != nil {
+		return err
+	}
+	return os.Chtimes(path, time.Time{}, info.ModTime())
+}
+
+func (r tokenRotator) rotateRegistry(ctx context.Context, apply bool) error {
+	keys := make([]string, len(registrygate.Principals))
+	for i, p := range registrygate.Principals {
+		keys[i] = installerRegistryKeys[p]
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: new write tokens for the image registry's principals (%s)\n", kindRegistry, strings.Join(registrygate.Principals, ", "))
+	fmt.Fprintf(r.out, "  - writes %s (%s), Secret %s/%s, Secret %s/%s\n", r.secretsEnv, strings.Join(keys, ", "),
+		r.controlNS, naming.RegistryAuthSecretName, r.buildNS, naming.RegistryPushSecretName)
+	fmt.Fprintln(r.out, "  - the registry restarts to load them: a build pushing its image at that moment fails and can be submitted again, and an image pull in that moment retries")
+	fmt.Fprintf(r.out, "  - %s (it presents the prune token)\n", apiRestartNote)
+	if !r.confirm(kindRegistry, apply) {
+		return nil
+	}
+
+	tokens := map[string][]byte{}
+	kv := make([][2]string, 0, len(registrygate.Principals))
+	for _, p := range registrygate.Principals {
+		tok, err := r.newToken()
+		if err != nil {
+			return fmt.Errorf("generate a token: %w", err)
+		}
+		tokens[p] = []byte(tok)
+		kv = append(kv, [2]string{installerRegistryKeys[p], tok})
+	}
+	if err := r.record(kv...); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.controlNS, naming.RegistryAuthSecretName, tokens); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.buildNS, naming.RegistryPushSecretName, map[string][]byte{
+		naming.RegistryPushUsernameKey: []byte(registrygate.PrincipalBuild),
+		naming.RegistryPushPasswordKey: tokens[registrygate.PrincipalBuild],
+	}); err != nil {
+		return err
+	}
+	if err := r.rollout(ctx, registryDeployment); err != nil {
+		return fmt.Errorf("restart the registry: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - registry: restarted on the new tokens")
+	return r.rollAPI(ctx)
+}
+
+func (r tokenRotator) rotateForwarding(ctx context.Context, apply bool) error {
+	restart, held, err := r.gamePods(ctx)
+	if err != nil {
+		return err
+	}
+	hostProxy := fileExists(r.forwardingFile)
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new Velocity forwarding secret, the key a server checks each player's identity with\n", kindForwarding)
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, "Secret "+ns+"/"+naming.ForwardingSecretName)
+	}
+	writes := append([]string{r.secretsEnv + " (" + installerForwardingKey + ")"}, secrets...)
+	if hostProxy {
+		writes = append(writes, r.forwardingFile)
+	}
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - every running server restarts to read it (%d now), saving its world on the way down, and the proxy restarts: everyone online is disconnected and can rejoin once their server is back\n", len(restart))
+	if len(held) > 0 {
+		fmt.Fprintf(r.out, "  - left running, because a backup, restore or file write holds its world: %s. Players cannot join it until it restarts: stop and start it from the panel once that finishes\n", strings.Join(held, ", "))
+	}
+	if !hostProxy {
+		fmt.Fprintf(r.out, "  - no proxy on this host (%s): put the value in Secret %s/%s into your proxy's forwarding secret file and restart it\n",
+			r.forwardingFile, r.controlNS, naming.ForwardingSecretName)
+	}
+	if !r.confirm(kindForwarding, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a secret: %w", err)
+	}
+	if err := r.record([2]string{installerForwardingKey, tok}); err != nil {
+		return err
+	}
+	if hostProxy {
+		info, err := os.Stat(r.forwardingFile)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(r.forwardingFile, info, []byte(tok)); err != nil {
+			return fmt.Errorf("write %s: %w", r.forwardingFile, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: updated\n", r.forwardingFile)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, naming.ForwardingSecretName, map[string][]byte{naming.ForwardingSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+	// The servers go first: their pods read the Secret as they are recreated,
+	// and a player who rejoins through the restarted proxy meets a server on the
+	// new secret, or one still starting.
+	for i := range restart {
+		p := &restart[i]
+		if err := r.cl.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) {
+			return fmt.Errorf("restart %s: %w", p.Labels[v1alpha1.LabelServer], err)
+		}
+	}
+	fmt.Fprintf(r.out, "  - servers: %d restarting on the new secret\n", len(restart))
+	if hostProxy {
+		if err := r.restartUnit(ctx, velocityUnit); err != nil {
+			return fmt.Errorf("restart %s: %w", velocityUnit, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: restarted on the new secret\n", velocityUnit)
+	}
+	return nil
+}
+
+// gamePods lists the running game server pods: those a rotation may restart,
+// and the servers left alone because a backup, restore or file write holds
+// their world (internal/maintenance), which a restart in the middle of would
+// break.
+func (r tokenRotator) gamePods(ctx context.Context) ([]corev1.Pod, []string, error) {
+	var pods corev1.PodList
+	// The operator's managed-by label is on its server pods alone (see rotateCaller).
+	if err := r.cl.List(ctx, &pods, client.InNamespace(r.minecraftNS), client.MatchingLabels{v1alpha1.LabelManagedBy: operator.ManagedByValue}); err != nil {
+		return nil, nil, fmt.Errorf("list the servers' pods: %w", err)
+	}
+	var jobs batchv1.JobList
+	if err := r.cl.List(ctx, &jobs, client.InNamespace(r.minecraftNS)); err != nil {
+		return nil, nil, fmt.Errorf("list the maintenance Jobs: %w", err)
+	}
+	var restart []corev1.Pod
+	var held []string
+	for _, p := range pods.Items {
+		if p.DeletionTimestamp != nil {
+			continue
+		}
+		server := p.Labels[v1alpha1.LabelServer]
+		var ms v1alpha1.MinecraftServer
+		if err := r.cl.Get(ctx, client.ObjectKey{Namespace: r.minecraftNS, Name: server}, &ms); client.IgnoreNotFound(err) != nil {
+			return nil, nil, fmt.Errorf("read server %s: %w", server, err)
+		}
+		if kind, ok := maintenance.Holder(server, ms.Annotations, jobs.Items, r.now()); ok {
+			held = append(held, server+" ("+kind+")")
+			continue
+		}
+		restart = append(restart, p)
+	}
+	return restart, held, nil
+}
+
+func (r tokenRotator) rotateDB(ctx context.Context, apply bool) error {
+	cfg, err := config.Load(r.hostTOML)
+	if err != nil {
+		return err
+	}
+	if cfg.Database.Deployment == "" {
+		return fmt.Errorf("[database] deployment is unset in %s, so the database is not the installer's felis-postgres: change the role's password where it runs, then in [database] url of each config copy", r.hostTOML)
+	}
+	u, err := neturl.Parse(cfg.Database.URL)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return fmt.Errorf("[database] url in %s names no role", r.hostTOML)
+	}
+	role := u.User.Username()
+	targets, err := tomlTargetsOf(r.hostTOML, r.podTOML, r.defaultTOML)
+	if err != nil {
+		return err
+	}
+	paths := make([]string, len(targets))
+	for i, t := range targets {
+		paths[i] = t.path
+	}
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, ns+"/"+platform.ConfigSecretName)
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new password for the database role %q\n", kindDB, role)
+	fmt.Fprintf(r.out, "  - writes %s (%s), the role in %s, [database] url in %s, Secret %s\n",
+		r.secretsEnv, installerDBKey, cfg.Database.Deployment, strings.Join(paths, " and "), strings.Join(secrets, " and "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
+	fmt.Fprintln(r.out, "  - a backup, restore or file Job that connects in the seconds between the password change and felis-api's restart fails and can be run again; the host's timers read the new config on their next run")
+	if !r.confirm(kindDB, apply) {
+		return nil
+	}
+
+	password, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a password: %w", err)
+	}
+	// Every config copy is edited in memory first, so one this cannot edit stops
+	// the rotation before the role's password changes.
+	edited := make([][]byte, len(targets))
+	var hostURL string
+	var podConfig []byte
+	for i, t := range targets {
+		raw, err := os.ReadFile(t.real)
+		if err != nil {
+			return err
+		}
+		var doc struct {
+			Database struct {
+				URL string `toml:"url"`
+			} `toml:"database"`
+		}
+		if _, err := toml.Decode(string(raw), &doc); err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		next, err := withPassword(doc.Database.URL, password)
+		if err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		if edited[i], err = editTOMLStrings(raw, []tomlStringEdit{{"database", "url", next}}); err != nil {
+			return fmt.Errorf("%s: %w; set the password in its [database] url by hand", t.path, err)
+		}
+		switch t.path {
+		case r.hostTOML:
+			hostURL = next
+		case r.podTOML:
+			podConfig = edited[i]
+		}
+	}
+	if podConfig == nil {
+		return fmt.Errorf("%s resolves to the same file as %s; the pods reach the database at another address and need a copy of their own", r.podTOML, r.hostTOML)
+	}
+
+	if err := r.record([2]string{installerDBKey, password}); err != nil {
+		return err
+	}
+	salt := make([]byte, 16)
+	if _, err := rand.Read(salt); err != nil {
+		return err
+	}
+	verifier, err := scramVerifier(password, salt, scramIterations)
+	if err != nil {
+		return err
+	}
+	if err := r.alterRole(ctx, cfg.Database.Deployment, role, verifier); err != nil {
+		return fmt.Errorf("set the role's password: %w", err)
+	}
+	if err := r.verifyDB(ctx, hostURL); err != nil {
+		return fmt.Errorf("the database does not accept the new password (%v); the config copies still hold the old one: run the installer again (sudo bash deploy/bootstrap.sh), which sets the password in %s everywhere", err, r.secretsEnv)
+	}
+	fmt.Fprintf(r.out, "  - role %s: password changed, and the database accepts it\n", role)
+	for i, t := range targets {
+		info, err := os.Stat(t.real)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(t.real, info, edited[i]); err != nil {
+			return fmt.Errorf("write %s: %w", t.path, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: [database] url updated\n", t.path)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, platform.ConfigSecretName, map[string][]byte{platform.ConfigSecretKey: podConfig}); err != nil {
+			return err
+		}
+	}
+	return r.rollAPI(ctx)
+}
+
+// withPassword is a database URL with its password replaced.
+func withPassword(raw, password string) (string, error) {
+	u, err := neturl.Parse(raw)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return "", errors.New("its [database] url names no role")
+	}
+	u.User = neturl.UserPassword(u.User.Username(), password)
+	return u.String(), nil
+}
+
+// scramIterations is PostgreSQL's default scram_iterations.
+const scramIterations = 4096
+
+// scramVerifier is the SCRAM-SHA-256 verifier PostgreSQL stores for a password
+// (RFC 5802 and RFC 7677, in the form libpq's PQencryptPasswordConn makes).
+// ALTER ROLE stores a verifier as it is given, so the password itself never
+// reaches the server, where a failing statement is logged with its text.
+func scramVerifier(password string, salt []byte, iterations int) (string, error) {
+	salted, err := pbkdf2.Key(sha256.New, password, salt, iterations, sha256.Size)
+	if err != nil {
+		return "", err
+	}
+	mac := func(msg string) []byte {
+		h := hmac.New(sha256.New, salted)
+		h.Write([]byte(msg))
+		return h.Sum(nil)
+	}
+	stored := sha256.Sum256(mac("Client Key"))
+	b64 := base64.StdEncoding.EncodeToString
+	return fmt.Sprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, b64(salt), b64(stored[:]), b64(mac("Server Key"))), nil
+}
+
+// alterRoleInPod runs ALTER ROLE as the superuser inside the database's
+// container, over its socket, with the statement on stdin.
+func alterRoleInPod(ctx context.Context, deployment, role, verifier string) error {
+	ns, name, _ := strings.Cut(deployment, "/")
+	return kubectlWithInput(ctx, []byte(alterRoleSQL(role, verifier)), "-n", ns, "exec", "-i", "deploy/"+name, "-c", platform.PostgresContainer, "--",
+		"psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres")
+}
+
+// alterRoleSQL sets role's password to a SCRAM verifier, which holds no quote.
+func alterRoleSQL(role, verifier string) string {
+	return "ALTER ROLE \"" + strings.ReplaceAll(role, `"`, `""`) + "\" WITH PASSWORD '" + verifier + "';\n"
+}
+
+// setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file
+// (secrets.env, a .properties file), appending one when the key is absent. The
+// file is replaced atomically and keeps its mode and owner: felis-link.properties
+// is root:felis-velocity 0640, and the proxy must still be able to read it.
+func setKeyValueLine(path, key, sep, value string) error {
+	return setKeyValueLines(path, sep, [][2]string{{key, value}})
+}
+
+// setKeyValueLines is setKeyValueLine for several keys in one rewrite.
+func setKeyValueLines(path, sep string, kv [][2]string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
+	for _, p := range kv {
+		key, value := p[0], p[1]
+		found := false
+		for i, ln := range lines {
+			k, _, ok := strings.Cut(ln, sep)
+			if ok && strings.TrimSpace(k) == key {
+				lines[i] = key + sep + value
+				found = true
+			}
+		}
+		if !found {
+			lines = append(lines, key+sep+value)
+		}
+	}
+	return replaceFileKeepingMode(path, info, []byte(strings.Join(lines, "\n")+"\n"))
+}
+
+// replaceFileKeepingMode atomically replaces path with data, keeping the mode and
+// owner info describes: these files are read by other users (the proxy's) and
+// some hold credentials, so a rewrite must not widen or narrow who can read them.
+func replaceFileKeepingMode(path string, info os.FileInfo, data []byte) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	if err := tmp.Chmod(info.Mode().Perm()); err != nil {
+		tmp.Close()
+		return err
+	}
+	if st, ok := info.Sys().(*syscall.Stat_t); ok {
+		if err := tmp.Chown(int(st.Uid), int(st.Gid)); err != nil {
+			tmp.Close()
+			return err
+		}
+	}
+	if _, err := tmp.Write(data); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
+}

cmd/felis/rotatetoken_test.go

@@ -0,0 +1,883 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/maintenance"
+
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+type rotationRig struct {
+	r      tokenRotator
+	cl     client.Client
+	out    *bytes.Buffer
+	events []string
+	dir    string
+	clock  time.Time
+}
+
+func tokenSecret(ns, name, val string) *corev1.Secret {
+	return keySecret(ns, name, "token", val)
+}
+
+func keySecret(ns, name, key, val string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name},
+		Data:       map[string][]byte{key: []byte(val)},
+	}
+}
+
+// serverPod is a game server's pod as the operator labels it.
+func serverPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{
+			"felis.lolicon.best/server":     server,
+			"felis.lolicon.best/managed-by": "felis-operator",
+			"felis.lolicon.best/component":  "server",
+		}}}
+}
+
+// backupPod is a backup Job's pod as internal/backupjob labels it: it carries
+// the server's label too, and no rotation may take it for the server's own.
+func backupPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{
+			"felis.lolicon.best/server":    server,
+			"app.kubernetes.io/managed-by": "felis-backup",
+			"app.kubernetes.io/component":  "world-backup",
+		}}}
+}
+
+func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
+	t.Helper()
+	rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir(), clock: time.Unix(1_800_000_000, 0)}
+	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
+	rig.r = tokenRotator{
+		cl:             rig.cl,
+		controlNS:      "felis",
+		minecraftNS:    "minecraft",
+		buildNS:        "felis-build",
+		secretsEnv:     filepath.Join(rig.dir, "secrets.env"),
+		linkProps:      filepath.Join(rig.dir, "felis-link.properties"),
+		forwardingFile: filepath.Join(rig.dir, "forwarding.secret"),
+		hostTOML:       filepath.Join(rig.dir, "felis.host.toml"),
+		podTOML:        filepath.Join(rig.dir, "felis.pod.toml"),
+		defaultTOML:    filepath.Join(rig.dir, "felis.toml"),
+		newToken:       func() (string, error) { return "NEWTOKEN", nil },
+		rollout: func(_ context.Context, deployment string) error {
+			rig.events = append(rig.events, "roll "+deployment)
+			return nil
+		},
+		restartUnit: func(_ context.Context, unit string) error {
+			rig.events = append(rig.events, "restart "+unit)
+			return nil
+		},
+		proxyLog: func(context.Context, time.Time) string { return "" },
+		alterRole: func(context.Context, string, string, string) error {
+			rig.events = append(rig.events, "alter-role")
+			return nil
+		},
+		verifyDB: func(context.Context, string) error {
+			rig.events = append(rig.events, "verify-db")
+			return nil
+		},
+		// Every look at the clock moves it a second on, so a wait measured with it
+		// ends after a known number of looks.
+		now: func() time.Time {
+			rig.clock = rig.clock.Add(time.Second)
+			return rig.clock
+		},
+		reloadWait: 5 * time.Second,
+		out:        rig.out,
+	}
+	return rig
+}
+
+func (rig *rotationRig) secretKey(t *testing.T, ns, name, key string) string {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		return "<missing>"
+	}
+	return string(s.Data[key])
+}
+
+func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
+	t.Helper()
+	return rig.secretKey(t, ns, name, "token")
+}
+
+func (rig *rotationRig) pods(t *testing.T) string {
+	t.Helper()
+	var pods corev1.PodList
+	if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
+		t.Fatal(err)
+	}
+	names := make([]string, len(pods.Items))
+	for i, p := range pods.Items {
+		names[i] = p.Name
+	}
+	return strings.Join(names, ",")
+}
+
+func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(body), mode); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(path, mode); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func readTestFile(t *testing.T, path string) string {
+	t.Helper()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(raw)
+}
+
+func TestRotateLimboToken(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-service-token", "proxy"),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+		backupPod("minecraft", "login-backup-x", "login"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600)
+
+	// felis-api must roll only after both copies hold the new value, and the login
+	// pod must still be there then: restarting it earlier would have it present
+	// the new token to an api that does not know it yet.
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the control Secret held %q", got)
+		}
+		if got := rig.secret(t, "minecraft", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the minecraft replica held %q", got)
+		}
+		var pod corev1.Pod
+		if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+			t.Errorf("the login pod was restarted before the api rolled")
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), "limbo", true); err != nil {
+		t.Fatal(err)
+	}
+
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 {
+		t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm())
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" {
+		t.Errorf("the proxy's token changed to %q", got)
+	}
+	// The login pod restarted; a user server and the backup Job's pod, which
+	// carries the login server's label too, are untouched.
+	if got := rig.pods(t); got != "login-backup-x,survival-0" {
+		t.Errorf("pods left = %s, want login-backup-x,survival-0", got)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new token was printed: %s", rig.out.String())
+	}
+}
+
+const testLinkProps = "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n"
+
+func reloadLine(token string) string {
+	return "[12:00:00 INFO] [felis-link]: Felis: service-token reloaded from /x/felis-link.properties (fingerprint " + tokenFingerprint(token) + ")\n"
+}
+
+// The host proxy re-reads its token: the rotation waits for it to say so and
+// leaves it running.
+func TestRotateVelocityTokenReloadsTheHostProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	if err := os.Chtimes(rig.r.linkProps, written, written); err != nil {
+		t.Fatal(err)
+	}
+
+	// The proxy logs its reload on its next call to felis-api, which may be
+	// while the api rolls: the log must be read from before that.
+	var reloadedAt time.Time
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := readTestFile(t, rig.r.linkProps); !strings.Contains(got, "service-token=NEWTOKEN\n") {
+			t.Errorf("api rolled before the proxy's file held the new token: %q", got)
+		}
+		reloadedAt = rig.clock
+		return nil
+	}
+	looks := 0
+	rig.r.proxyLog = func(_ context.Context, since time.Time) string {
+		looks++
+		log := reloadLine("old") // an earlier rotation's
+		if looks >= 3 && !since.After(reloadedAt) {
+			log += reloadLine("NEWTOKEN")
+		}
+		return log
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.linkProps); got != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
+		t.Errorf("felis-link.properties = %q", got)
+	}
+	info, _ := os.Stat(rig.r.linkProps)
+	if info.Mode().Perm() != 0o640 {
+		t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm())
+	}
+	if !info.ModTime().Equal(written) {
+		t.Errorf("properties mtime = %v, want it kept at %v (felis domain check would read the proxy as stale)", info.ModTime(), written)
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	// The proxy's token has no replica: it must not appear in a workload namespace.
+	if got := rig.secret(t, "minecraft", "felis-service-token"); got != "<missing>" {
+		t.Errorf("rotation copied the proxy token into minecraft (%q)", got)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want the api roll and no proxy restart", rig.events)
+	}
+	if looks != 3 {
+		t.Errorf("the log was read %d times, want 3 (until the line appeared)", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: took the new token from its properties; players stayed connected") {
+		t.Errorf("output does not say the players stayed: %s", out)
+	}
+}
+
+// A proxy that never logs the new fingerprint (an older plugin, a stuck proxy)
+// is restarted once the wait is over.
+func TestRotateVelocityTokenRestartsAProxyThatDoesNotReload(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	looks := 0
+	rig.r.proxyLog = func(context.Context, time.Time) string {
+		looks++
+		return reloadLine("old")
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api,restart felis-velocity" {
+		t.Errorf("events = %v, want the api roll then the proxy restart", rig.events)
+	}
+	// reloadWait is 5s and each look moves the clock a second.
+	if looks != 5 {
+		t.Errorf("the log was read %d times, want 5", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: had not taken the new token within 5s, restarted") {
+		t.Errorf("output does not explain the restart: %s", out)
+	}
+}
+
+// The proxy and the Java plugin name a token by the same fingerprint
+// (plugins/shared LinkConfigLoaderTest fileTokenFollowsTheFile).
+func TestTokenFingerprintMatchesThePlugin(t *testing.T) {
+	if got := tokenFingerprint("new-token"); got != "348e9df2a42b" {
+		t.Errorf("tokenFingerprint(new-token) = %q, want 348e9df2a42b", got)
+	}
+}
+
+// An external proxy has no felis-link.properties here: the Secret still rotates,
+// nothing is restarted on this host, and the output says where the value is
+// without printing it.
+func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want no proxy restart", rig.events)
+	}
+	out := rig.out.String()
+	if !strings.Contains(out, "felis/felis-service-token") || strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("output should point at the Secret without the value: %s", out)
+	}
+}
+
+func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old"))
+	// An install from before per-caller tokens has no BUILD_TOKEN line yet.
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600)
+	if err := rig.r.rotate(context.Background(), "build", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("build replica = %q, want NEWTOKEN (created when absent)", got)
+	}
+	if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+}
+
+// A failed api rollout stops the rotation before the caller restarts: the login
+// gate keeps running on the old value the old api pods still accept.
+func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		serverPod("minecraft", "login-0", "login"),
+	)
+	rig.r.rollout = func(context.Context, string) error { return errors.New("rollout timed out") }
+	err := rig.r.rotate(context.Background(), "limbo", true)
+	if err == nil || !strings.Contains(err.Error(), "rollout timed out") {
+		t.Fatalf("err = %v, want the rollout failure", err)
+	}
+	if got := rig.pods(t); got != "login-0" {
+		t.Errorf("pods left = %s: the login pod was restarted although the api never rolled", got)
+	}
+}
+
+func TestRotateRefusesAnUnknownCredential(t *testing.T) {
+	rig := newRotationRig(t)
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600)
+	if err := rig.r.rotate(context.Background(), "admin", true); err == nil {
+		t.Fatal("rotated an unknown credential")
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\n" {
+		t.Errorf("secrets.env = %q, want it untouched", got)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want nothing touched", rig.events)
+	}
+}
+
+const (
+	testHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	testPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+// installedRig is a host as the installer leaves it, with every credential in
+// place, for the plan test.
+func installedRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-service-token", "old"),
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-build-token", "old"),
+		tokenSecret("felis-build", "felis-build-token", "old"),
+		tokenSecret("felis", "felis-ops-token", "old"),
+		keySecret("felis", "felis-registry-auth", "platform", "old"),
+		keySecret("felis-build", "felis-registry-push", "password", "old"),
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	return rig
+}
+
+// Without -yes every rotation prints its plan and changes nothing.
+func TestRotatePlanChangesNothing(t *testing.T) {
+	for _, kind := range rotationKinds() {
+		t.Run(kind, func(t *testing.T) {
+			rig := installedRig(t)
+			files := map[string]string{}
+			for _, p := range []string{rig.r.secretsEnv, rig.r.linkProps, rig.r.forwardingFile, rig.r.hostTOML, rig.r.podTOML} {
+				files[p] = readTestFile(t, p)
+			}
+			rig.r.newToken = func() (string, error) {
+				t.Error("a plan generated a token")
+				return "NEWTOKEN", nil
+			}
+			if err := rig.r.rotate(context.Background(), kind, false); err != nil {
+				t.Fatal(err)
+			}
+			for p, before := range files {
+				if got := readTestFile(t, p); got != before {
+					t.Errorf("%s changed to %q", filepath.Base(p), got)
+				}
+			}
+			for _, s := range [][3]string{
+				{"felis", "felis-service-token", "token"}, {"felis", "felis-limbo-token", "token"},
+				{"felis-build", "felis-build-token", "token"}, {"felis", "felis-ops-token", "token"},
+				{"felis", "felis-registry-auth", "platform"}, {"felis-build", "felis-registry-push", "password"},
+				{"minecraft", "felis-forwarding-secret", "secret"},
+			} {
+				if got := rig.secretKey(t, s[0], s[1], s[2]); got != "old" {
+					t.Errorf("Secret %s/%s changed to %q", s[0], s[1], got)
+				}
+			}
+			if got := rig.secretKey(t, "minecraft", "felis-config", "felis.toml"); got != testPodTOML {
+				t.Errorf("felis-config changed to %q", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want none", rig.events)
+			}
+			if got := rig.pods(t); got != "login-0,survival-0" {
+				t.Errorf("pods left = %s, want all", got)
+			}
+			out := rig.out.String()
+			if !strings.HasSuffix(out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes "+kind+"\n") {
+				t.Errorf("the plan does not end with how to go ahead: %s", out)
+			}
+			// Each plan names the interruption it causes.
+			want := apiRestartNote
+			if kind == kindForwarding {
+				want = "everyone online is disconnected"
+			}
+			if !strings.Contains(out, want) {
+				t.Errorf("the plan does not say %q: %s", want, out)
+			}
+		})
+	}
+}
+
+func TestRotateRegistryTokens(t *testing.T) {
+	rig := newRotationRig(t,
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: "felis-registry-auth"},
+			Data: map[string][]byte{"platform": []byte("a"), "build": []byte("b"), "prune": []byte("c")}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis-build", Name: "felis-registry-push"},
+			Data: map[string][]byte{"username": []byte("build"), "password": []byte("b")}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=a\nREGISTRY_BUILD_TOKEN=b\nREGISTRY_PRUNE_TOKEN=c\n", 0o600)
+	n := 0
+	rig.r.newToken = func() (string, error) {
+		n++
+		return fmt.Sprintf("NEWTOKEN%d", n), nil
+	}
+	// The registry reads its tokens as it starts: it restarts after the Secret
+	// holds them, and felis-api (the prune token) after that.
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", "prune"); got != "NEWTOKEN3" {
+			t.Errorf("%s rolled while the prune token was %q", deployment, got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindRegistry, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=NEWTOKEN1\nREGISTRY_BUILD_TOKEN=NEWTOKEN2\nREGISTRY_PRUNE_TOKEN=NEWTOKEN3\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	for key, want := range map[string]string{"platform": "NEWTOKEN1", "build": "NEWTOKEN2", "prune": "NEWTOKEN3"} {
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", key); got != want {
+			t.Errorf("felis-registry-auth %s = %q, want %s", key, got, want)
+		}
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "password"); got != "NEWTOKEN2" {
+		t.Errorf("the build Jobs' push password = %q, want the build token", got)
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "username"); got != "build" {
+		t.Errorf("the build Jobs' push username = %q, want build", got)
+	}
+	if strings.Join(rig.events, ",") != "roll registry,roll felis-api" {
+		t.Errorf("events = %v, want the registry then felis-api", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("a new token was printed: %s", rig.out.String())
+	}
+}
+
+func TestRotateForwardingSecret(t *testing.T) {
+	lock := time.Unix(1_800_000_000, 0)
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		serverPod("minecraft", "survival-0", "survival"),
+		serverPod("minecraft", "lobby-0", "lobby"),
+		// creative is being backed up: a running Job holds its world.
+		serverPod("minecraft", "creative-0", "creative"),
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "creative-backup",
+			Labels: map[string]string{maintenance.LabelServer: "creative", maintenance.LabelManagedBy: "felis-backup"}}},
+		// survival's last backup is over; its finished pod stays until the Job's TTL.
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival-backup",
+			Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: "felis-backup"}},
+			Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}},
+		backupPod("minecraft", "survival-backup-x", "survival"),
+		// A pod already on its way out is neither counted nor deleted again.
+		func() *corev1.Pod {
+			p := serverPod("minecraft", "lobby-old", "lobby")
+			p.DeletionTimestamp = &metav1.Time{Time: lock}
+			p.Finalizers = []string{"felis.lolicon.best/test"}
+			return p
+		}(),
+		// skyblock's restore has just been admitted, its Job not created yet.
+		serverPod("minecraft", "skyblock-0", "skyblock"),
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "skyblock",
+			Annotations: map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lock)}}},
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival"}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nFORWARDING_SECRET=old\n", 0o600)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	// The proxy restarts after the servers were told to: a player who rejoins
+	// must not meet a server still on the old secret.
+	rig.r.restartUnit = func(_ context.Context, unit string) error {
+		rig.events = append(rig.events, "restart "+unit)
+		if got := rig.pods(t); strings.Contains(got, "survival-0") {
+			t.Errorf("the proxy restarted before the servers (pods %s)", got)
+		}
+		if got := readTestFile(t, rig.r.forwardingFile); got != "NEWTOKEN" {
+			t.Errorf("the proxy restarted on forwarding.secret %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nFORWARDING_SECRET=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if info, _ := os.Stat(rig.r.forwardingFile); info.Mode().Perm() != 0o640 {
+		t.Errorf("forwarding.secret mode = %v, want 0640", info.Mode().Perm())
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+			t.Errorf("Secret %s/felis-forwarding-secret = %q, want NEWTOKEN", ns, got)
+		}
+	}
+	if got := rig.pods(t); got != "creative-0,lobby-old,skyblock-0,survival-backup-x" {
+		t.Errorf("pods left = %s, want the held servers, the terminating pod and the backup's pod", got)
+	}
+	if strings.Join(rig.events, ",") != "restart felis-velocity" {
+		t.Errorf("events = %v, want only the proxy restart (felis-api does not hold this secret)", rig.events)
+	}
+	out := rig.out.String()
+	for _, want := range []string{
+		"every running server restarts to read it (2 now)",
+		"left running, because a backup, restore or file write holds its world: creative (backup), skyblock (restore).",
+		"  - servers: 2 restarting on the new secret\n",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output lacks %q: %s", want, out)
+		}
+	}
+	if strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("the new secret was printed: %s", out)
+	}
+}
+
+func TestRotateForwardingSecretForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, keySecret("felis", "felis-forwarding-secret", "secret", "old"))
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secretKey(t, "felis", "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want no proxy restart on this host", rig.events)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "put the value in Secret felis/felis-forwarding-secret into your proxy's forwarding secret file") {
+		t.Errorf("output does not say where the value is: %s", out)
+	}
+}
+
+// dbRig is an installed host for the database rotation: felis.toml links to the
+// host copy, as the installer makes it.
+func dbRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n", 0o600)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	if err := os.Symlink("felis.host.toml", rig.r.defaultTOML); err != nil {
+		t.Fatal(err)
+	}
+	return rig
+}
+
+const (
+	wantHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	wantPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+func TestRotateDatabasePassword(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.alterRole = func(_ context.Context, deployment, role, verifier string) error {
+		rig.events = append(rig.events, "alter-role")
+		if deployment != "felis/felis-postgres" || role != "felis" {
+			t.Errorf("alterRole(%q, %q), want felis/felis-postgres and felis", deployment, role)
+		}
+		if !strings.Contains(readTestFile(t, rig.r.secretsEnv), "DB_PASSWORD=NEWTOKEN\n") {
+			t.Error("the role changed before secrets.env recorded the new password")
+		}
+		// The verifier is the new password's, under the salt it carries.
+		m := regexp.MustCompile(`^SCRAM-SHA-256\$4096:([^$]+)\$`).FindStringSubmatch(verifier)
+		if m == nil {
+			t.Fatalf("verifier %q is not a SCRAM-SHA-256 verifier", verifier)
+		}
+		salt, err := base64.StdEncoding.DecodeString(m[1])
+		if err != nil || len(salt) != 16 {
+			t.Fatalf("verifier salt %q: %v", m[1], err)
+		}
+		if want, _ := scramVerifier("NEWTOKEN", salt, 4096); verifier != want {
+			t.Errorf("verifier = %q, want %q", verifier, want)
+		}
+		return nil
+	}
+	// The configs change only once the database accepts the new password.
+	rig.r.verifyDB = func(_ context.Context, url string) error {
+		rig.events = append(rig.events, "verify-db")
+		if url != "postgres://felis:[email protected]:30432/felis?sslmode=disable" {
+			t.Errorf("verified with %q, want the host URL with the new password", url)
+		}
+		if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+			t.Errorf("the host config changed before the database accepted the password: %q", got)
+		}
+		return nil
+	}
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("felis-api rolled on felis-config %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindDB, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=NEWTOKEN\nSERVICE_TOKEN=s\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != wantHostTOML {
+		t.Errorf("host config = %q", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != wantPodTOML {
+		t.Errorf("pod config = %q", got)
+	}
+	if info, err := os.Lstat(rig.r.defaultTOML); err != nil || info.Mode()&os.ModeSymlink == 0 {
+		t.Errorf("felis.toml is no longer the link to the host copy (%v)", err)
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("Secret %s/felis-config = %q", ns, got)
+		}
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db,roll felis-api" {
+		t.Errorf("events = %v", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new password was printed: %s", rig.out.String())
+	}
+}
+
+// A password the database does not accept leaves the configs on the old one
+// and felis-api running: the installer's record, already new, is the way back.
+func TestRotateDatabasePasswordStopsWhenTheDatabaseRefuses(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.verifyDB = func(context.Context, string) error {
+		rig.events = append(rig.events, "verify-db")
+		return errors.New("password authentication failed")
+	}
+	err := rig.r.rotate(context.Background(), kindDB, true)
+	if err == nil || !strings.Contains(err.Error(), "password authentication failed") || !strings.Contains(err.Error(), "run the installer again") {
+		t.Fatalf("err = %v, want the refusal and the way back", err)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+		t.Errorf("host config = %q, want it unchanged", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != testPodTOML {
+		t.Errorf("pod config = %q, want it unchanged", got)
+	}
+	if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != testPodTOML {
+		t.Errorf("felis-config = %q, want it unchanged", got)
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db" {
+		t.Errorf("events = %v, want no api roll", rig.events)
+	}
+}
+
+// Everything that can refuse does so before the installer's record or the
+// role change; what the host config alone shows is refused by the plan too.
+func TestRotateDatabasePasswordRefusesEarly(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		apply bool
+		setup func(t *testing.T, rig *rotationRig)
+		want  string
+	}{
+		{"a database the installer does not run", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "deployment = \"felis/felis-postgres\"\n", "", 1), 0o600)
+		}, "[database] deployment is unset"},
+		{"a database url without a role", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "felis:oldpw@", "", 1), 0o600)
+		}, "names no role"},
+		{"a config copy the line editor cannot edit", true, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.podTOML, "[database]\nurl = \"\"\"\npostgres://felis:[email protected]:5432/felis\"\"\"\n", 0o600)
+		}, "set the password in its [database] url by hand"},
+		{"a pod copy that is the host copy", true, func(t *testing.T, rig *rotationRig) {
+			if err := os.Remove(rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.Symlink("felis.host.toml", rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+		}, "resolves to the same file as"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			rig := dbRig(t)
+			tc.setup(t, rig)
+			err := rig.r.rotate(context.Background(), kindDB, tc.apply)
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err = %v, want %q", err, tc.want)
+			}
+			if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n" {
+				t.Errorf("secrets.env = %q, want it untouched", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want nothing done", rig.events)
+			}
+		})
+	}
+}
+
+// The RFC 7677 example (user "user", password "pencil"), with the stored and
+// server keys computed by openssl's PBKDF2 and HMAC rather than this code.
+func TestScramVerifier(t *testing.T) {
+	salt, _ := base64.StdEncoding.DecodeString("W22ZaJ0SNY7soEsUEjb6gQ==")
+	got, err := scramVerifier("pencil", salt, 4096)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := "SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==$WG5d8oPm3OtcPnkdi4Uo7BkeZkBFzpcXkuLmtbsT4qY=:wfPLwcE6nTWhTAmQ7tl2KeoiWGPlZqQxSrmfPwDl2dU="; got != want {
+		t.Errorf("scramVerifier = %q\nwant            %q", got, want)
+	}
+}
+
+func TestAlterRoleSQL(t *testing.T) {
+	if got := alterRoleSQL(`fe"lis`, "SCRAM-SHA-256$4096:c2FsdA==$a:b"); got != "ALTER ROLE \"fe\"\"lis\" WITH PASSWORD 'SCRAM-SHA-256$4096:c2FsdA==$a:b';\n" {
+		t.Errorf("alterRoleSQL = %q", got)
+	}
+}
+
+// installerSecretKeys is every secrets.env key a rotation writes.
+func installerSecretKeys() map[string]bool {
+	keys := map[string]bool{installerForwardingKey: true, installerDBKey: true}
+	for _, k := range installerTokenKeys {
+		keys[k] = true
+	}
+	for _, k := range installerRegistryKeys {
+		keys[k] = true
+	}
+	return keys
+}
+
+// The installer and rotate-token must agree on where each caller's token lives:
+// rotate-token writes installerTokenKeys into secrets.env, and the installer
+// applies those same keys to the Secrets on its next run. A key the installer
+// does not read would be silently reverted by the next upgrade.
+func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	for caller, key := range installerTokenKeys {
+		ct, ok := callerToken(caller)
+		if !ok {
+			t.Fatalf("installerTokenKeys names unknown caller %q", caller)
+		}
+		apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`)
+		if !apply.MatchString(script) {
+			t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key)
+		}
+	}
+	// The replicas the installer applies straight into the workload namespaces, so an
+	// upgrade has them before the new operator and build Jobs reference them.
+	for _, line := range []string{
+		`apply_literal_secret "$MINECRAFT_NS" felis-limbo-token token "$LIMBO_TOKEN"`,
+		`apply_literal_secret "$BUILD_NS" felis-build-token token "$BUILD_TOKEN"`,
+		`kube -n "$MINECRAFT_NS" delete secret felis-service-token --ignore-not-found`,
+		`kube -n "$BUILD_NS" delete secret felis-service-token --ignore-not-found`,
+	} {
+		if !strings.Contains(script, line) {
+			t.Errorf("bootstrap.sh lacks %s", line)
+		}
+	}
+	for _, ns := range []string{"MINECRAFT_NS", "BUILD_NS"} {
+		if strings.Contains(script, `apply_literal_secret "$`+ns+`" felis-service-token`) {
+			t.Errorf("bootstrap.sh still copies the proxy's token into %s", ns)
+		}
+	}
+	if len(installerTokenKeys) != len(callerNames()) {
+		t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames()))
+	}
+}
+
+// Every value the installer keeps in secrets.env has a rotation that rewrites
+// that very key, and every key a rotation writes is one the installer
+// generates, keeps and so re-applies on its next run.
+func TestInstallerSecretsAreAllRotatable(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	m := regexp.MustCompile(`(?s)write_file_atomic "\$SECRETS_ENV" 0600 <<EOF\n(.*?)\nEOF\n`).FindStringSubmatch(script)
+	if m == nil {
+		t.Fatal("bootstrap.sh: no secrets.env heredoc found")
+	}
+	persisted := map[string]bool{}
+	for _, line := range strings.Split(m[1], "\n") {
+		key, value, _ := strings.Cut(line, "=")
+		if value != "${"+key+"}" {
+			t.Errorf("secrets.env line %q is not KEY=${KEY}", line)
+		}
+		persisted[key] = true
+	}
+	rotated := installerSecretKeys()
+	for key := range persisted {
+		if !rotated[key] {
+			t.Errorf("secrets.env keeps %s, which no rotation replaces", key)
+		}
+	}
+	for key := range rotated {
+		if !persisted[key] {
+			t.Errorf("a rotation writes %s, which the installer does not keep in secrets.env", key)
+		}
+		if !regexp.MustCompile(`\n  ` + key + `="\$\{` + key + `:-\$\(openssl rand -hex [0-9]+\)\}"\n`).MatchString(script) {
+			t.Errorf("bootstrap.sh does not generate %s when secrets.env lacks it", key)
+		}
+	}
+}

cmd/felis/run.go

@@ -11,21 +11,41 @@ Usage:
   felis <command> [flags]
 
 Commands:
-  migrate up        Apply embedded database migrations under an advisory lock
+  migrate up        Apply embedded database migrations under an advisory lock (snapshots the database first)
+  db                Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check)
+  offsite           Copy world archives, database bundles, user images and uploads to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|fetch-images|fetch-uploads|keygen)
   operator          Run the MinecraftServer controller-manager
   api               Run the felis-api HTTP server
   nano              Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)
   reaper            Run the world reaper / backup batch
   restore           Extract a world archive into a world volume (internal Job entrypoint)
   backup            Archive a world into the backup store and record it (internal Job entrypoint)
-  files             List/read/write one file in a stopped server's world (internal Job entrypoint)
+  backup-now        Archive every user server's world now, one at a time (or the named ones; -stop stops running ones first; prints the plan, -yes applies; requires root/sudo)
+  files             List, read, write, mkdir, delete, rename, upload or unzip one path in a stopped server's world (internal Job entrypoint)
+  export            Archive a stopped server's world, or read one of its backups, and hand it to felis-api for download (internal Job entrypoint)
+  egress-gate       Hold a build or game server pod until its egress NetworkPolicy is enforced (internal init container entrypoint)
   fetch-context     Fetch and extract a submission's build context (internal Job entrypoint)
+  scan-gate         Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
+  push-image        Push a scanned image tarball to the registry (internal Job entrypoint)
+  mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
+  server-migrate    Move a stopped world between approved nodes (start|status|retry; requires root)
+  node              Join and approve trusted daemon nodes (list|token|join|approve|firewall; requires root)
+  node-control      Run the host node-task service on an API-only Unix socket (requires root)
+  node-probe        Verify reachability and observed sources (internal admission probe)
+  archive-serve     Serve scoped one-use archive transfers on the controller (internal entrypoint)
+  registry-gate     Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
   manifests         Render the control-plane RBAC + NetworkPolicy install bundle as YAML
   apply             Create a MinecraftServer CRD (direct K8s write; use -f server.json)
   setup             Run host bootstrap + first-run setup console (TUI; requires root/sudo)
   converge          Fill in fields a newer desired spec added to already-installed system servers
+  rotate-token      Replace a generated credential and restart what reads it (velocity|limbo|build|ops|registry|forwarding|db; prints the plan, -yes applies; requires root/sudo)
+  domain            Move the install to a new root domain on every surface that carries it, or check each one (set|check; requires root/sudo)
+  status            Print the platform at a glance: node, control plane, proxy, servers, backups, host, open alerts (requires root/sudo)
+  doctor            Run every health check once, grouped by area, with where to look next; mails nothing (requires root/sudo)
+  support-bundle    Collect status, doctor, logs and cluster state into one redacted tar.gz to share when asking for help (requires root/sudo)
+  watchdog          Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
   version           Print the build stamp of this binary
-  update            Report which platform components have updates available
+  update            Check platform versions; --apply installs a reviewed target
   breakGlass        Open the local break-glass emergency console (TUI; requires root/sudo)
 
 Run "felis <command> -h" for command-specific flags.
@@ -41,24 +61,47 @@ Run "felis <command> -h" for command-specific flags.
 // The help aliases are deliberately NOT entries: they print usage rather than run a
 // subcommand, and listing them would make the table disagree with the command list.
 var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
-	"migrate":          cmdMigrate,
-	"operator":         cmdOperator,
-	"api":              cmdAPI,
-	"nano":             cmdNano,
-	"reaper":           cmdReaper,
-	"restore":          cmdRestore,
-	"backup":           cmdBackup,
-	"files":            cmdFiles,
-	"fetch-context":    cmdFetchContext,
-	"manifests":        cmdManifests,
-	"apply":            cmdApply,
-	"setup":            cmdSetup,
-	"converge":         cmdConverge,
-	"breakGlass":       cmdBreakGlass,
-	"bootstrap-assets": cmdBootstrapAssets,
-	"init-forwarding":  cmdInitForwarding,
-	"version":          cmdVersion,
-	"update":           cmdUpdate,
+	"migrate":            cmdMigrate,
+	"db":                 cmdDB,
+	"offsite":            cmdOffsite,
+	"operator":           cmdOperator,
+	"api":                cmdAPI,
+	"nano":               cmdNano,
+	"reaper":             cmdReaper,
+	"restore":            cmdRestore,
+	"backup":             cmdBackup,
+	"backup-now":         cmdBackupNow,
+	"files":              cmdFiles,
+	"export":             cmdExport,
+	"egress-gate":        cmdEgressGate,
+	"fetch-context":      cmdFetchContext,
+	"scan-gate":          cmdScanGate,
+	"push-image":         cmdPushImage,
+	"mirror-build-tools": cmdMirrorBuildTools,
+	"registry-gate":      cmdRegistryGate,
+	"archive-serve":      cmdArchiveServe,
+	"node":               cmdNode,
+	"node-control":       cmdNodeControl,
+	"server-migrate":     cmdServerMigrate,
+	"node-probe":         cmdNodeProbe,
+	"manifests":          cmdManifests,
+	"apply":              cmdApply,
+	"setup":              cmdSetup,
+	"converge":           cmdConverge,
+	"rotate-token":       cmdRotateToken,
+	"domain":             cmdDomain,
+	"breakGlass":         cmdBreakGlass,
+	"bootstrap-assets":   cmdBootstrapAssets,
+	"init-forwarding":    cmdInitForwarding,
+	"init-volume":        cmdInitVolume,
+	"pin-images":         cmdPinImages,
+	"image-bundle":       cmdImageBundle,
+	"version":            cmdVersion,
+	"update":             cmdUpdate,
+	"watchdog":           cmdWatchdog,
+	"status":             cmdStatus,
+	"doctor":             cmdDoctor,
+	"support-bundle":     cmdSupportBundle,
 }
 
 // run dispatches a subcommand. It is separate from main so the router is

cmd/felis/run_test.go

@@ -5,8 +5,27 @@ import (
 	"os"
 	"strings"
 	"testing"
+
+	"github.com/BurntSushi/toml"
+
+	"felis.lolicon.best/internal/config"
 )
 
+func TestBootstrapConfigKeys(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"bootstrap-assets", "config-keys"}, &out, &errBuf); code != 0 {
+		t.Fatalf("exit code = %d: %s", code, errBuf.String())
+	}
+	var cfg config.Config
+	meta, err := toml.Decode(strings.TrimSpace(out.String())+` = "26.3"`, &cfg)
+	if err != nil || len(meta.Undecoded()) != 0 {
+		t.Fatalf("advertised config key is unsupported: %v, undecoded: %v", err, meta.Undecoded())
+	}
+	if cfg.Velocity.GameVersion != "26.3" {
+		t.Fatalf("advertised key did not set the game version: %q", cfg.Velocity.GameVersion)
+	}
+}
+
 func TestRunNoArgsPrintsUsage(t *testing.T) {
 	var out, errBuf bytes.Buffer
 	if code := run(nil, &out, &errBuf); code != 2 {
@@ -38,9 +57,13 @@ func TestRunUnknownCommand(t *testing.T) {
 }
 
 // undocumentedCommands are routable on purpose but kept out of the usage text: they
-// are called by deploy/bootstrap.sh, not by a human at a prompt. Listing them here is
-// what makes their absence from usage a deliberate decision rather than an oversight.
-var undocumentedCommands = map[string]bool{"bootstrap-assets": true, "init-forwarding": true}
+// are called by deploy/bootstrap.sh or the operator's initContainers, not by a human
+// at a prompt. Listing them here is what makes their absence from usage a deliberate
+// decision rather than an oversight.
+var undocumentedCommands = map[string]bool{
+	"bootstrap-assets": true, "init-forwarding": true, "init-volume": true,
+	"pin-images": true, "image-bundle": true,
+}
 
 // The usage text and the dispatch table must describe the same set of commands.
 //

cmd/felis/scangate.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"strings"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// maxScanDocument bounds each document scan-gate reads: a modpack report lists a
+// few thousand packages, far below this. Tests shrink it.
+var maxScanDocument int64 = 64 << 20
+
+// cmdScanGate is the build pod's verdict step, after trivy wrote its full JSON
+// report and trivy convert wrote the CycloneDX SBOM. It applies the scan policy
+// to the report, prints the verdict and every blocking finding, then appends the
+// verdict, the report and the SBOM to its log as the envelope felis-api keeps on
+// the build (build.WriteScanEnvelope). It exits 1 when a finding blocks, which
+// fails the pod before the push step runs, and 2 when the report cannot be read,
+// so a scan that produced nothing usable never admits an image.
+func cmdScanGate(args []string, stdout, stderr io.Writer) int {
+	fset := flag.NewFlagSet("scan-gate", flag.ContinueOnError)
+	fset.SetOutput(stderr)
+	reportPath := fset.String("report", "", "trivy JSON report (required)")
+	sbomPath := fset.String("sbom", "", "CycloneDX SBOM to keep with the report")
+	failOn := fset.String("fail-on", strings.Join(build.DefaultScanFailOn, ","), "comma-separated severities that block the image")
+	failUnfixed := fset.Bool("fail-unfixed", false, "block on vulnerabilities that have no fixed release too")
+	accept := fset.String("accept", "", "comma-separated vulnerability ids and secret rule ids that never block")
+	termLog := fset.String("termination-log", "/dev/termination-log", "where the one-line verdict goes for the pod status")
+	if err := fset.Parse(args); err != nil {
+		return 2
+	}
+	// A stray argument is a policy the gate would otherwise drop without a word
+	// (a comma split out of --fail-on, say).
+	if fset.NArg() > 0 {
+		fmt.Fprintf(stderr, "felis scan-gate: unexpected argument %q\n", fset.Arg(0))
+		return 2
+	}
+	sevs, err := build.ParseSeverities(*failOn)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --fail-on: %v\n", err)
+		return 2
+	}
+	accepted, err := build.ParseScanAccept(*accept)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --accept: %v\n", err)
+		return 2
+	}
+	if *reportPath == "" {
+		fmt.Fprintln(stderr, "felis scan-gate: --report is required")
+		return 2
+	}
+	fail := func(msg string) int {
+		fmt.Fprintln(stderr, "felis scan-gate: "+msg)
+		writeTerminationLog(*termLog, msg)
+		return 2
+	}
+	report, err := readScanDocument(*reportPath)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	policy := build.ScanPolicy{FailOn: sevs, FailUnfixed: *failUnfixed, Accept: accepted}
+	summary, err := build.Summarize(report, policy)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	env := build.ScanEnvelope{Summary: summary, Report: report}
+	if *sbomPath != "" {
+		switch sbom, err := readScanDocument(*sbomPath); {
+		case err == nil:
+			env.SBOM = sbom
+		case errors.Is(err, fs.ErrNotExist):
+			fmt.Fprintf(stdout, "felis scan-gate: no SBOM at %s; keeping the report alone\n", *sbomPath)
+		default:
+			return fail("the SBOM is unreadable: " + err.Error())
+		}
+	}
+
+	printScanVerdict(stdout, summary)
+	written, err := build.WriteScanEnvelope(stdout, env)
+	if err != nil {
+		return fail("could not write the scan envelope: " + err.Error())
+	}
+	for _, doc := range written.Summary.Omitted {
+		fmt.Fprintf(stderr, "felis scan-gate: the %s is too large to keep with the build and was left out\n", doc)
+	}
+	if summary.Blocked {
+		writeTerminationLog(*termLog, summary.Reason())
+		return 1
+	}
+	writeTerminationLog(*termLog, "the scan passed")
+	return 0
+}
+
+// printScanVerdict writes the human half of scan-gate's log.
+func printScanVerdict(w io.Writer, s build.ScanSummary) {
+	var counts []string
+	for _, sev := range build.Severities {
+		counts = append(counts, fmt.Sprintf("%s %d", sev, s.Counts[sev]))
+	}
+	fmt.Fprintf(w, "felis scan-gate: %d packages; findings: %s\n", s.Packages, strings.Join(counts, ", "))
+	unfixed := "vulnerabilities with no fixed release do not block"
+	if s.Policy.FailUnfixed {
+		unfixed = "vulnerabilities with no fixed release block too"
+	}
+	fmt.Fprintf(w, "felis scan-gate: blocking on %s (%s)\n", strings.Join(s.Policy.FailOn, ", "), unfixed)
+	if len(s.Policy.Accept) > 0 {
+		matched := 0
+		for _, f := range s.Findings {
+			if f.Accepted {
+				matched++
+			}
+		}
+		fmt.Fprintf(w, "felis scan-gate: accepted ids, never blocking: %s; listed findings under them: %d\n", strings.Join(s.Policy.Accept, ", "), matched)
+	}
+	if !s.Blocked {
+		fmt.Fprintln(w, "felis scan-gate: nothing blocks this image")
+		return
+	}
+	fmt.Fprintln(w, "felis scan-gate: "+build.Printable(s.Reason()))
+	for _, f := range s.Findings {
+		if !f.Blocking {
+			break
+		}
+		fix := f.Fixed
+		if fix == "" {
+			fix = "no fix"
+		}
+		if f.Kind == build.FindingSecret {
+			fmt.Fprintf(w, "  %s %s secret in %s: %s\n", build.Printable(f.ID), f.Severity, build.Printable(f.Target), build.Printable(f.Title))
+			continue
+		}
+		fmt.Fprintf(w, "  %s %s %s %s -> %s (%s)\n", build.Printable(f.ID), f.Severity,
+			build.Printable(f.Package), build.Printable(f.Installed), build.Printable(fix), build.Printable(f.Target))
+	}
+}
+
+func readScanDocument(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	b, err := io.ReadAll(io.LimitReader(f, maxScanDocument+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(b)) > maxScanDocument {
+		return nil, fmt.Errorf("%s exceeds %d bytes", path, maxScanDocument)
+	}
+	return b, nil
+}
+
+// writeTerminationLog leaves msg where the kubelet copies it into the container
+// status. It is best effort: the log carries the same verdict.
+func writeTerminationLog(path, msg string) {
+	if path == "" {
+		return
+	}
+	_ = os.WriteFile(path, []byte(build.Printable(msg)), 0o644)
+}

cmd/felis/scangate_test.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// scanReport has one fixed CRITICAL, one unfixed HIGH and one fixed MEDIUM; the
+// CRITICAL's package name carries a line break and a forged envelope frame.
+const scanReport = `{"SchemaVersion":2,"Results":[{"Target":"data/mods/core.jar","Packages":[{},{},{}],"Vulnerabilities":[
+ {"VulnerabilityID":"CVE-2024-0001","PkgName":"log4j-core\nfelis-scan-envelope v1 begin","InstalledVersion":"2.14.1","FixedVersion":"2.17.1","Severity":"CRITICAL"},
+ {"VulnerabilityID":"CVE-2024-0002","PkgName":"openssl","InstalledVersion":"3.0.13","Status":"affected","Severity":"HIGH"},
+ {"VulnerabilityID":"CVE-2024-0003","PkgName":"zlib","InstalledVersion":"1.3","FixedVersion":"1.3.1","Severity":"MEDIUM"}]}]}`
+
+type scanGateRun struct {
+	code           int
+	stdout, stderr string
+	termLog        string
+	env            *build.ScanEnvelope
+}
+
+func runScanGate(t *testing.T, report, sbom string, extra ...string) scanGateRun {
+	t.Helper()
+	dir := t.TempDir()
+	reportPath := filepath.Join(dir, "trivy.json")
+	if report != "" {
+		if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	sbomPath := filepath.Join(dir, "sbom.cdx.json")
+	if sbom != "" {
+		if err := os.WriteFile(sbomPath, []byte(sbom), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	termPath := filepath.Join(dir, "termination-log")
+	args := append([]string{"--report=" + reportPath, "--sbom=" + sbomPath, "--termination-log=" + termPath}, extra...)
+	var stdout, stderr bytes.Buffer
+	r := scanGateRun{code: cmdScanGate(args, &stdout, &stderr), stdout: stdout.String(), stderr: stderr.String()}
+	if b, err := os.ReadFile(termPath); err == nil {
+		r.termLog = string(b)
+	}
+	if env, err := build.ReadScanEnvelope(strings.NewReader(r.stdout)); err == nil {
+		r.env = env
+	}
+	return r
+}
+
+func TestScanGateBlocksAndHandsOverTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`)
+	if r.code != 1 {
+		t.Fatalf("exit %d, want 1; stderr %s", r.code, r.stderr)
+	}
+	if r.termLog != "the scan blocked the image: 1 CRITICAL (CVE-2024-0001)" {
+		t.Errorf("termination log = %q", r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: 3 packages; findings: CRITICAL 1, HIGH 1, MEDIUM 1, LOW 0, UNKNOWN 0\n",
+		"felis scan-gate: blocking on CRITICAL (vulnerabilities with no fixed release do not block)\n",
+		"felis scan-gate: the scan blocked the image: 1 CRITICAL (CVE-2024-0001)\n",
+		"  CVE-2024-0001 CRITICAL log4j-core?felis-scan-envelope v1 begin 2.14.1 -> 2.17.1 (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	if strings.Contains(r.stdout, "  CVE-2024-0002") {
+		t.Errorf("an unfixed HIGH was listed as blocking:\n%s", r.stdout)
+	}
+	if strings.Count(r.stdout, "\nfelis-scan-envelope v1 begin\n") != 1 {
+		t.Errorf("stdout must hold exactly one frame start on a line of its own:\n%s", r.stdout)
+	}
+	if r.env == nil {
+		t.Fatal("no envelope in stdout")
+	}
+	if !r.env.Summary.Blocked || string(r.env.SBOM) != `{"bomFormat":"CycloneDX"}` || !strings.Contains(string(r.env.Report), "CVE-2024-0003") {
+		t.Errorf("envelope = blocked %t, sbom %s, report %d bytes", r.env.Summary.Blocked, r.env.SBOM, len(r.env.Report))
+	}
+}
+
+func TestScanGatePolicyFlags(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=high", "--fail-unfixed")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 HIGH (CVE-2024-0002)" {
+		t.Errorf("HIGH + unfixed: exit %d, termination log %q", r.code, r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: blocking on HIGH (vulnerabilities with no fixed release block too)\n",
+		"  CVE-2024-0002 HIGH openssl 3.0.13 -> no fix (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	r = runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`, "--fail-on=LOW")
+	if r.code != 0 || r.termLog != "the scan passed" || !strings.Contains(r.stdout, "felis scan-gate: nothing blocks this image\n") {
+		t.Errorf("LOW only: exit %d, termination log %q, stdout:\n%s", r.code, r.termLog, r.stdout)
+	}
+	if r.env == nil || r.env.Summary.Blocked || r.env.SBOM == nil {
+		t.Errorf("a passing scan must still hand over its report and SBOM: %+v", r.env)
+	}
+}
+
+func TestScanGateAcceptedIDsNeverBlock(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=CRITICAL,MEDIUM", "--accept=CVE-2024-0001, CVE-2024-0002,CVE-2099-0001")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 MEDIUM (CVE-2024-0003)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "felis scan-gate: accepted ids, never blocking: CVE-2024-0001, CVE-2024-0002, CVE-2099-0001; listed findings under them: 2\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+	if r.env == nil || strings.Join(r.env.Summary.Policy.Accept, ",") != "CVE-2024-0001,CVE-2024-0002,CVE-2099-0001" {
+		t.Fatalf("envelope = %+v", r.env)
+	}
+	for _, f := range r.env.Summary.Findings {
+		if f.ID == "CVE-2024-0001" && (f.Blocking || !f.Accepted) {
+			t.Errorf("accepted finding = %+v", f)
+		}
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001")
+	if r.code != 0 || r.termLog != "the scan passed" {
+		t.Errorf("only an accepted CRITICAL: exit %d, termination log %q", r.code, r.termLog)
+	}
+}
+
+func TestScanGateWithoutAnSBOMKeepsTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=LOW")
+	if r.code != 0 || !strings.Contains(r.stdout, "felis scan-gate: no SBOM at ") {
+		t.Errorf("exit %d, stdout:\n%s", r.code, r.stdout)
+	}
+	if r.env == nil || r.env.SBOM != nil || r.env.Report == nil {
+		t.Errorf("envelope = %+v", r.env)
+	}
+}
+
+func TestScanGateListsABlockingSecret(t *testing.T) {
+	r := runScanGate(t, `{"SchemaVersion":2,"Results":[{"Target":"config/keys.txt","Secrets":[
+ {"RuleID":"aws-access-key-id","Severity":"CRITICAL","Title":"AWS Access\nKey ID"}]}]}`, "")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 CRITICAL (aws-access-key-id)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "  aws-access-key-id CRITICAL secret in config/keys.txt: AWS Access?Key ID\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+}
+
+func TestScanGateFailsClosed(t *testing.T) {
+	r := runScanGate(t, "", "")
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the scan report is unreadable: open ") || r.env != nil {
+		t.Errorf("missing report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	r = runScanGate(t, `{"SchemaVersion":1}`, "")
+	if r.code != 2 || r.termLog != "the scan report is unreadable: read trivy report: schema version 1, want 2" {
+		t.Errorf("old schema: exit %d, termination log %q", r.code, r.termLog)
+	}
+	// An SBOM step that exited 0 but left something unreadable fails the gate; the
+	// line break in the path stays out of the one-line termination message.
+	sbomDir := filepath.Join(t.TempDir(), "sb\nom")
+	if err := os.Mkdir(sbomDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	r = runScanGate(t, scanReport, "", "--sbom="+sbomDir)
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the SBOM is unreadable: read ") ||
+		!strings.HasSuffix(r.termLog, "/sb?om: is a directory") || r.env != nil {
+		t.Errorf("unreadable SBOM: exit %d, termination log %q", r.code, r.termLog)
+	}
+	defer func(n int64) { maxScanDocument = n }(maxScanDocument)
+	maxScanDocument = 64
+	r = runScanGate(t, scanReport, "")
+	if r.code != 2 || !strings.HasSuffix(r.termLog, "/trivy.json exceeds 64 bytes") || r.env != nil {
+		t.Errorf("oversized report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	maxScanDocument = 64 << 20
+	r = runScanGate(t, scanReport, "", "--fail-on=SEVERE")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --fail-on: unknown severity "SEVERE"`) {
+		t.Errorf("bad severity: exit %d, stderr %q", r.code, r.stderr)
+	}
+	// A severity list split at its comma leaves a stray argument, not a
+	// narrower policy.
+	r = runScanGate(t, scanReport, "", "--fail-on=LOW", "CRITICAL")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: unexpected argument "CRITICAL"`) || r.env != nil {
+		t.Errorf("stray argument: exit %d, stderr %q", r.code, r.stderr)
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001 CVE-2024-0003")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --accept: "CVE-2024-0001 CVE-2024-0003" is not a vulnerability id or secret rule id`) {
+		t.Errorf("bad accept list: exit %d, stderr %q", r.code, r.stderr)
+	}
+	var stdout, stderr bytes.Buffer
+	if code := cmdScanGate(nil, &stdout, &stderr); code != 2 || !strings.Contains(stderr.String(), "--report is required") {
+		t.Errorf("no report flag: exit %d, stderr %q", code, stderr.String())
+	}
+}

cmd/felis/server_migrate.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/placement"
+	"felis.lolicon.best/internal/platform"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func cmdServerMigrate(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprintln(stderr, "server-migrate: start | status | retry (separate from database migrate)")
+		return 2
+	}
+	fs := flag.NewFlagSet("server-migrate "+args[0], flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	name := fs.String("name", "", "stopped user server")
+	target := fs.String("target-node", "", "approved target worker")
+	id := fs.String("id", "", "operation id (required for retry)")
+	kube := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local kubeconfig")
+	ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
+	cfgPath := fs.String("config", "/var/lib/felis/felis.host.toml", "host config used to record the current owner on the safety backup")
+	if err := fs.Parse(args[1:]); err != nil {
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "server-migrate requires root/sudo")
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	cfg, err := clientcmd.BuildConfigFromFlags("", *kube)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	scheme := runtime.NewScheme()
+	clientgoscheme.AddToScheme(scheme)
+	v1alpha1.AddToScheme(scheme)
+	cl, err := client.New(cfg, client.Options{Scheme: scheme})
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	m := &distributed.Manager{Client: cl, Namespace: *ns, Resolve: placement.Resolve(cl, *ns)}
+	var nodes corev1.NodeList
+	if err = cl.List(ctx, &nodes); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	for _, n := range nodes.Items {
+		if n.Labels[placement.LabelRole] == placement.RoleController {
+			m.Controller = n.Name
+		}
+	}
+	if m.Controller == "" {
+		fmt.Fprintln(stderr, "distributed controller identity is not configured")
+		return 1
+	}
+	m.Resolve = placement.Resolve(cl, *ns, m.Controller)
+	var op distributed.Operation
+	switch args[0] {
+	case "start":
+		host, cfgErr := config.Load(*cfgPath)
+		if cfgErr != nil {
+			fmt.Fprintln(stderr, cfgErr)
+			return 1
+		}
+		drv, dbErr := openPodStore(ctx, host.Database.URL, "migration", stderr)
+		if dbErr != nil {
+			fmt.Fprintln(stderr, dbErr)
+			return 1
+		}
+		defer drv.Close()
+		var owner sql.NullString
+		if err := drv.DB().QueryRowContext(ctx, "SELECT owner_id FROM servers WHERE name=$1 AND deleted_at IS NULL AND retire_requested_at IS NULL", *name).Scan(&owner); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		op, err = m.BeginMigration(ctx, *name, *target, owner.String)
+	case "status":
+		op, err = m.Migration(ctx, *name, *id)
+	case "retry":
+		if *id == "" {
+			fmt.Fprintln(stderr, "retry requires --id")
+			return 2
+		}
+		op, err = m.RetryMigration(ctx, *name, *id)
+	default:
+		fmt.Fprintln(stderr, "unknown migration operation")
+		return 2
+	}
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	json.NewEncoder(stdout).Encode(op)
+	return 0
+}

cmd/felis/setup.go

@@ -11,9 +11,7 @@ import (
 	"time"
 
 	"felis.lolicon.best/internal/api"
-	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/store"
 )
@@ -101,19 +99,7 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
 	}
 	defer setup.drv.Close()
 
-	// The wizard's first screen asks the operator to join the server and run /link:
-	// the Owner IS the Minecraft account, so the login gate must be UP before we ask
-	// for a link code. This used to run after the wizard, which is why setup asked
-	// for a code from a server that had never been started. On a re-run the Owner
-	// already exists, so provisioning stays best-effort and never blocks the
-	// operator from reaching the status screen.
-	if err := provisionSystemServers(ctx, setup.cfg, stdout, !setup.adminExists); err != nil {
-		fmt.Fprintf(stderr, "felis setup: %v\n", err)
-		fmt.Fprintln(stderr, "The Owner is bound by joining the login gate in-game, so setup cannot continue without it.")
-		return 1
-	}
-
-	res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database.URL, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
+	res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis setup: %v\n", err)
 		return 1
@@ -122,34 +108,50 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
 	if panelURL == "" {
 		panelURL = localPanelURL(setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname)
 	}
+	// Game services are provisioned independently. A failed login/lobby must not
+	// stop the host administrator from opening the panel to diagnose it.
+	if err := provisionSystemServers(ctx, setup.cfg, stdout); err != nil {
+		fmt.Fprintf(stdout, "felis setup: Minecraft services need attention: %v\n", err)
+	}
+
+	reportSetupResult(stdout, res, bootstrapped, setup.adminExists, panelURL)
+	return 0
+}
+
+// reportSetupResult preserves the browser handoff after the TUI closes.
+func reportSetupResult(stdout io.Writer, res breakGlassResult, bootstrapped, adminExisted bool, panelURL string) {
+	if !adminExisted && !res.provisioned {
+		defer fmt.Fprintln(stdout, "\nOwner login is unfinished. Run sudo felis setup again to get a panel setup link; Minecraft is not required.")
+	}
 
 	if !res.provisioned && !res.connectConfigured {
 		if bootstrapped {
-			fmt.Fprintln(stdout, "felis setup: host bootstrap completed; Owner/connection setup skipped.")
+			fmt.Fprintln(stdout, "felis setup: host bootstrap completed; panel/connection setup unfinished.")
 			if panelURL != "" {
 				fmt.Fprintf(stdout, "Panel: %s\n", panelURL)
 				fmt.Fprintln(stdout, "The local HTTPS certificate is self-signed; your browser may ask for confirmation on first visit.")
 			}
-			return 0
+			return
 		}
 		// A re-run lands on the status screen, which changes nothing by design —
 		// reporting that as "cancelled" reads as a failure the operator did not cause.
 		msg := "felis setup: cancelled — no changes made."
-		if res.alreadySetUp {
+		switch {
+		case res.alreadySetUp:
 			msg = "felis setup: already set up — nothing to change."
 		}
 		fmt.Fprintln(stdout, msg)
 		if panelURL != "" {
 			fmt.Fprintf(stdout, "Panel: %s\n", panelURL)
 		}
-		return 0
+		return
 	}
 
 	if res.provisioned {
 		fmt.Fprintf(stdout, "\nfelis setup: Owner account %q provisioned (passwordless).\n", res.username)
 		fmt.Fprintf(stdout, "Recorded as %q (mode: %s, os user: %s).\n", res.accountable, res.mode, res.osUser)
 		if res.setupTokenURL != "" {
-			fmt.Fprintf(stdout, "Open this URL to complete passwordless login setup (verify email / enroll passkey):\n\n    %s\n\n", res.setupTokenURL)
+			fmt.Fprintf(stdout, "Open this URL to record your email and create a passkey (Minecraft is optional):\n\n    %s\n\n", res.setupTokenURL)
 		}
 		if res.auditWarning != "" {
 			fmt.Fprintf(stdout, "WARNING: the accountability audit row was NOT written: %s\n", res.auditWarning)
@@ -177,28 +179,15 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
 			fmt.Fprintln(stdout, "Felis config, Kubernetes Secret and API rollout were updated.")
 		}
 	}
-
-	return 0
 }
 
 // provisionSystemServers ensures the login limbo and lobby system services exist,
 // then prints the login-first Velocity wiring. deploy/bootstrap.sh writes this
 // configuration for its host proxy; operators only need to mirror it when they
 // deliberately run Velocity elsewhere.
-//
-// required is set on a first run, where the next screen asks the operator to join
-// the server and run /link. There a gate that never comes up is not a degraded
-// install, it is an impossible one — so every soft landing below becomes a hard
-// error and we block until the gate reports Ready. On a re-run the Owner already
-// exists and nothing downstream needs the gate, so unconfigured images or an
-// unreachable cluster degrade to printed guidance exactly as before.
-func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writer, required bool) error {
-	// fail is the one place the two modes diverge: fatal on a first run, guidance
-	// on a re-run.
+
+func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writer) error {
 	fail := func(format string, args ...any) error {
-		if required {
-			return fmt.Errorf(format, args...)
-		}
 		fmt.Fprintf(out, "\nfelis setup: "+format+"\n", args...)
 		return nil
 	}
@@ -206,30 +195,26 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 		return fail("login/lobby system servers NOT provisioned — set [velocity] login_image " +
 			"and lobby_image in felis.toml (build them from deploy/limbo and deploy/lobby), then re-run `sudo felis setup`")
 	}
-	if required && cfg.Velocity.LoginImage == "" {
-		return errors.New("the Owner binds by joining the login gate, but [velocity] login_image is not set in felis.toml " +
-			"(build it from deploy/limbo), then re-run `sudo felis setup`")
-	}
 	cl, err := buildSystemServerClient()
 	if err != nil {
 		return fail("could not reach the cluster to provision the login/lobby system servers: %v\n"+
 			"Re-run `sudo felis setup` on the control-plane host once the cluster is reachable", err)
 	}
 	// The login limbo authenticates to the felis-api INTERNAL face, so it needs the
-	// internal base URL, the root domain (to link players at the console), and the
-	// service token. The first two are plain env baked into the pod here; the token
-	// is a Secret the operator injects by reference — but a secretKeyRef is
-	// namespace-local, so first replicate the token Secret from the control namespace
-	// into the minecraft namespace where the login pod runs. The control namespace is
+	// internal base URL, the root domain (to link players at the console), and its
+	// own token (felis-limbo-token). The first two are plain env baked into the pod
+	// here; the token is a Secret the operator injects by reference — but a
+	// secretKeyRef is namespace-local, so first replicate the token Secret from the
+	// control namespace into the minecraft namespace where the login pod runs. The control namespace is
 	// the platform default (there is no felis.toml override for it); a deployment that
 	// renamed it must replicate the Secret by hand.
 	controlNS := platform.DefaultControlNamespace
 	apiBaseURL := platform.InternalAPIBaseURL(controlNS)
 	// These Secrets must land in the minecraft namespace before the pods that
-	// mount them are created: the service token (login authenticates to felis-api
-	// with it), the Velocity forwarding secret (every backend verifies the proxy's
+	// mount them are created: the login gate's token (login authenticates to
+	// felis-api with it), the Velocity forwarding secret (every backend verifies the proxy's
 	// signed handshake with it — without it the login gate would derive an OFFLINE
-	// UUID and the Owner would bind the wrong Minecraft identity), and felis-config
+	// UUID and players would bind the wrong Minecraft identity), and felis-config
 	// (the on-demand BACKUP Job runs in the minecraft namespace and mounts it to
 	// self-record its world_backups row; without the replica the Job's volume
 	// mount fails and every backup request strands in the cluster).
@@ -239,31 +224,7 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 	if buildNS == "" {
 		buildNS = platform.DefaultBuildNamespace
 	}
-	secretOutcomes := []systemServerOutcome{
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false),
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
-		// refresh=true: felis-config is the rendered config, not a credential. The
-		// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
-		// must update it when the control plane's render has moved on (a stale copy
-		// e.g. keeps an old database URL after a credential rotation).
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			"felis-config", "felis.toml", "config", "minecraft ns", true),
-		// The reaper's pre-reap warning emails authenticate with the same relay
-		// password felis-api uses; the reaper pod runs in the minecraft namespace,
-		// where a secretKeyRef resolves only against a local mirror. Skipped while
-		// the relay is not configured yet — the "configure email" screen refreshes
-		// both mirrors when it applies.
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			"felis-smtp", "password", "smtp", "minecraft ns", false),
-		// The build namespace needs the same token: the build Job's fetch
-		// initContainer reads the submission context from the internal face. Best
-		// effort — a deployment that only installs the control plane simply never
-		// builds a user submission.
-		ensureSecretReplica(ctx, cl, controlNS, buildNS,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "felis-build ns", false),
-	}
+	secretOutcomes := provisionSecretReplicas(ctx, cl, controlNS, cfg.K8s.Namespace, buildNS)
 	outcomes := ensureSystemServers(ctx, cl, cfg.K8s.Namespace, cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, apiBaseURL, cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
 	outcomes = append(secretOutcomes, outcomes...)
 	fmt.Fprintln(out, "\nfelis setup: login/lobby system servers (always-on, reaper-exempt):")
@@ -279,19 +240,6 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 			fmt.Fprintf(out, "  - %s: skipped (%s)\n", o.name, o.skipped)
 		}
 	}
-	if required {
-		if err := requiredProvisioningError(outcomes); err != nil {
-			return fmt.Errorf("required Minecraft provisioning failed: %w", err)
-		}
-		fmt.Fprintln(out, "\nfelis setup: waiting for the login gate to accept players…")
-		err := awaitLoginGateReady(ctx, cl, cfg.K8s.Namespace, loginGateReadyTimeout, loginGatePollInterval, func(p v1alpha1.Phase) {
-			fmt.Fprintf(out, "  login: %s\n", phaseOrPending(p))
-		})
-		if err != nil {
-			return err
-		}
-		fmt.Fprintln(out, "  login: Ready")
-	}
 	printVelocityWiringGuidance(out, cfg.Server.RootDomain)
 	return nil
 }
@@ -345,7 +293,9 @@ func openConfiguredSetup(ctx context.Context, cfgPath string) (*configuredSetup,
 	if err != nil {
 		return nil, &setupOpenError{stage: "load config", err: err}
 	}
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Pending migrations are the preflight's to apply; a newer schema is a rolled-back
+	// binary, and nothing this console writes would match it.
+	drv, err := openStore(ctx, cfg.Database.URL, true)
 	if err != nil {
 		return nil, &setupOpenError{stage: "open database", err: err}
 	}

cmd/felis/setup_panel.go

@@ -33,9 +33,6 @@ func setupPanelNodePort() int {
 }
 
 func localPanelURL(rootDomain, adminHostname string) string {
-	if ip := rootDomainEmbeddedIP(rootDomain); ip != "" {
-		return fmt.Sprintf("https://%s:%d", ip, setupPanelNodePort())
-	}
 	host := defaultAdminHostname(rootDomain, adminHostname)
 	if host == "" {
 		return ""
@@ -57,6 +54,24 @@ func rootDomainEmbeddedIP(rootDomain string) string {
 	return ""
 }
 
+// setupGameAddress is where players join Minecraft: the
+// IP a nip.io or sslip.io root domain spells out (nothing to resolve), otherwise the
+// root domain, with the port when it is not Minecraft's default. The proxy lands
+// every fresh connection on the login server whatever name it was dialled by.
+func setupGameAddress(rootDomain string, gamePort int) string {
+	host := rootDomainEmbeddedIP(rootDomain)
+	if host == "" {
+		host = strings.TrimSpace(strings.TrimSuffix(rootDomain, "."))
+	}
+	if host == "" {
+		return ""
+	}
+	if gamePort == 0 || gamePort == 25565 {
+		return host
+	}
+	return net.JoinHostPort(host, strconv.Itoa(gamePort))
+}
+
 func localPanelOrigin() string {
 	return fmt.Sprintf("https://127.0.0.1:%d", setupPanelNodePort())
 }

cmd/felis/status.go

@@ -0,0 +1,438 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+	"text/tabwriter"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// cmdStatus prints the platform at a glance: the release and the node, the
+// control plane's workloads, the game proxy, every server with its players
+// and newest world backup, the database backups and the off-site copy, the
+// host's disks and memory, and what the watchdog has open. It changes
+// nothing, and a part that is down (the cluster, PostgreSQL) reads as such
+// while the rest still prints. felis doctor says what is wrong and where to
+// look.
+func cmdStatus(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("status", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis status: run as root (sudo felis status): it reads root-only state under /etc/felis and /var/lib/felis")
+		return 1
+	}
+	env, err := hostStatusEnv(*unitDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis status: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	printStatus(ctx, env, stdout)
+	return 0
+}
+
+// statusEnv is what one status report reads the host through.
+type statusEnv struct {
+	cfg *config.Config
+	// w is how felis-watchdog.service runs the watchdog: where the backups,
+	// the off-site record and the proxy are.
+	w       watchdogFlags
+	cl      client.Client // nil while the API server is unreachable
+	clErr   error
+	backups func(ctx context.Context) (map[string]time.Time, error)
+	run     func(ctx context.Context, name string, args ...string) ([]byte, error)
+	unitDir string
+	meminfo string
+	host    string
+	now     time.Time
+}
+
+// hostStatusEnv reads this host: the watchdog's settings, the configuration
+// they name, and the cluster.
+func hostStatusEnv(unitDir string) (statusEnv, error) {
+	w, _, err := watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
+	if err != nil {
+		return statusEnv{}, err
+	}
+	cfg, err := config.Load(w.cfgPath)
+	if err != nil {
+		return statusEnv{}, err
+	}
+	cl, clErr := buildSystemServerClient()
+	if clErr != nil {
+		cl = nil
+	}
+	host, _ := os.Hostname()
+	return statusEnv{
+		cfg: cfg, w: w, cl: cl, clErr: clErr,
+		backups: func(ctx context.Context) (map[string]time.Time, error) {
+			return newestWorldBackups(ctx, cfg.Database.URL)
+		},
+		run: hostCommand, unitDir: unitDir, meminfo: "/proc/meminfo", host: host, now: time.Now(),
+	}, nil
+}
+
+func printStatus(ctx context.Context, env statusEnv, out io.Writer) {
+	fmt.Fprintf(out, "felis %s on %s at %s\n", resolvedVersion(), env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
+	if env.cl == nil {
+		fmt.Fprintf(out, "cluster:  unreachable (%v)\n", env.clErr)
+	} else {
+		statusCluster(ctx, env, out)
+	}
+	statusProxy(ctx, env, out)
+	statusServers(ctx, env, out)
+	statusBackups(env, out)
+	statusHost(env, out)
+	statusWatchdog(ctx, env, out)
+}
+
+func statusCluster(ctx context.Context, env statusEnv, out io.Writer) {
+	var nodes corev1.NodeList
+	if err := env.cl.List(ctx, &nodes); err != nil {
+		fmt.Fprintf(out, "cluster:  unreachable (%v)\n", err)
+		return
+	}
+	for _, n := range nodes.Items {
+		ready := "NotReady"
+		for _, c := range n.Status.Conditions {
+			if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue {
+				ready = "Ready"
+			}
+		}
+		info := n.Status.NodeInfo
+		fmt.Fprintf(out, "node:     %s %s, k3s %s, %s, kernel %s\n", n.Name, ready, info.KubeletVersion, info.OSImage, info.KernelVersion)
+	}
+
+	ns := env.w.controlNS
+	var deps appsv1.DeploymentList
+	var pods corev1.PodList
+	err := env.cl.List(ctx, &deps, client.InNamespace(ns))
+	if err == nil {
+		err = env.cl.List(ctx, &pods, client.InNamespace(ns))
+	}
+	fmt.Fprintf(out, "\ncontrol plane (namespace %s):\n", ns)
+	if err != nil {
+		fmt.Fprintf(out, "  cannot list it: %v\n", err)
+		return
+	}
+	sort.Slice(deps.Items, func(i, j int) bool { return deps.Items[i].Name < deps.Items[j].Name })
+	tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	for _, d := range deps.Items {
+		want := int32(1)
+		if d.Spec.Replicas != nil {
+			want = *d.Spec.Replicas
+		}
+		image := "-"
+		if cs := d.Spec.Template.Spec.Containers; len(cs) > 0 {
+			image = shortImage(cs[0].Image)
+		}
+		fmt.Fprintf(tw, "  %s\t%d/%d ready\t%s\trestarts %d\n", d.Name, d.Status.ReadyReplicas, want, image, podRestarts(d.Spec.Selector, pods.Items))
+	}
+	tw.Flush()
+}
+
+// podRestarts adds up the container restarts of the pods selector picks (the
+// API server refuses a Deployment whose selector is empty).
+func podRestarts(selector *metav1.LabelSelector, pods []corev1.Pod) int32 {
+	sel, err := metav1.LabelSelectorAsSelector(selector)
+	if err != nil {
+		return 0
+	}
+	var n int32
+	for _, p := range pods {
+		if !sel.Matches(labels.Set(p.Labels)) {
+			continue
+		}
+		for _, cs := range p.Status.ContainerStatuses {
+			n += cs.RestartCount
+		}
+	}
+	return n
+}
+
+// shortImage is an image reference without its registry and repository path,
+// and with its digest cut to 12 characters.
+func shortImage(ref string) string {
+	if i := strings.LastIndex(ref, "/"); i >= 0 {
+		ref = ref[i+1:]
+	}
+	if name, digest, ok := strings.Cut(ref, "@sha256:"); ok && len(digest) > 12 {
+		ref = name + "@" + digest[:12]
+	}
+	return ref
+}
+
+func statusProxy(ctx context.Context, env statusEnv, out io.Writer) {
+	var parts []string
+	if _, err := os.Stat(filepath.Join(env.unitDir, "felis-velocity.service")); err == nil {
+		parts = append(parts, "felis-velocity "+unitActiveState(ctx, doctorEnv{run: env.run}, "felis-velocity.service"))
+	}
+	if env.w.proxyAddr != "" {
+		d := net.Dialer{Timeout: 3 * time.Second}
+		if conn, err := d.DialContext(ctx, "tcp", env.w.proxyAddr); err != nil {
+			parts = append(parts, fmt.Sprintf("%s refuses connections (%v)", env.w.proxyAddr, err))
+		} else {
+			conn.Close()
+			parts = append(parts, env.w.proxyAddr+" accepts connections")
+		}
+	}
+	if len(parts) == 0 {
+		parts = append(parts, "not on this host")
+	}
+	fmt.Fprintf(out, "\nproxy:    %s\n", strings.Join(parts, ", "))
+}
+
+func statusServers(ctx context.Context, env statusEnv, out io.Writer) {
+	ns := env.cfg.K8s.Namespace
+	if ns == "" {
+		ns = platform.DefaultMinecraftNamespace
+	}
+	if env.cl == nil {
+		fmt.Fprintf(out, "\nservers (namespace %s): unknown while the cluster is unreachable\n", ns)
+		return
+	}
+	var list v1alpha1.MinecraftServerList
+	if err := env.cl.List(ctx, &list, client.InNamespace(ns)); err != nil {
+		fmt.Fprintf(out, "\nservers (namespace %s): cannot list them: %v\n", ns, err)
+		return
+	}
+	newest, backupErr := env.backups(ctx)
+	running, online := 0, int32(0)
+	for _, ms := range list.Items {
+		if ms.Status.Phase == v1alpha1.PhaseRunning {
+			running++
+			online += ms.Status.Players.Online
+		}
+	}
+	fmt.Fprintf(out, "\nservers (namespace %s): %d, %d running, %d players online\n", ns, len(list.Items), running, online)
+	if len(list.Items) == 0 {
+		return
+	}
+	sort.Slice(list.Items, func(i, j int) bool { return list.Items[i].Name < list.Items[j].Name })
+	tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "  NAME\tROLE\tDESIRED\tPHASE\tPLAYERS\tNEWEST WORLD BACKUP")
+	for _, ms := range list.Items {
+		role := ms.Labels[v1alpha1.LabelSystemRole]
+		if role == "" {
+			role = "-"
+		}
+		phase := string(ms.Status.Phase)
+		if phase == "" {
+			phase = "-"
+		}
+		players := "-"
+		if ms.Status.Phase == v1alpha1.PhaseRunning {
+			players = fmt.Sprintf("%d/%d", ms.Status.Players.Online, ms.Status.Players.Max)
+		}
+		backup := "none"
+		switch at, ok := newest[ms.Name]; {
+		case backupErr != nil:
+			backup = "?"
+		case ok:
+			backup = dbbackup.Age(env.now.Sub(at)) + " ago"
+		}
+		fmt.Fprintf(tw, "  %s\t%s\t%s\t%s\t%s\t%s\n", ms.Name, role, orDash(string(ms.Spec.DesiredState)), phase, players, backup)
+	}
+	tw.Flush()
+	if backupErr != nil {
+		fmt.Fprintf(out, "  world backups unknown: %v\n", backupErr)
+	}
+}
+
+func orDash(s string) string {
+	if s == "" {
+		return "-"
+	}
+	return s
+}
+
+// newestWorldBackups is when each server's newest world backup that a restore
+// can use was taken.
+func newestWorldBackups(ctx context.Context, url string) (map[string]time.Time, error) {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	defer drv.Close()
+	rows, err := drv.DB().QueryContext(ctx, `SELECT server_name, max(created_at) FROM world_backups
+		WHERE status = 'present' AND corrupt_at IS NULL GROUP BY server_name`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]time.Time{}
+	for rows.Next() {
+		var name string
+		var at time.Time
+		if err := rows.Scan(&name, &at); err != nil {
+			return nil, err
+		}
+		out[name] = at
+	}
+	return out, rows.Err()
+}
+
+func statusBackups(env statusEnv, out io.Writer) {
+	fmt.Fprintln(out, "\nbackups:")
+	switch bundles, err := dbbackup.List(env.w.backupDir); {
+	case env.w.backupDir == "":
+		fmt.Fprintln(out, "  database: not checked (felis-watchdog.service names no -backup-dir)")
+	case err != nil:
+		fmt.Fprintf(out, "  database: cannot read %s: %v\n", env.w.backupDir, err)
+	case len(bundles) == 0:
+		fmt.Fprintf(out, "  database: none in %s\n", env.w.backupDir)
+	default:
+		fmt.Fprintf(out, "  database: newest %s, %s ago; %d bundles in %s\n",
+			bundles[0].Name, dbbackup.Age(env.now.Sub(bundles[0].Created)), len(bundles), env.w.backupDir)
+	}
+	if !env.cfg.Offsite.Enabled() {
+		fmt.Fprintln(out, "  off-site: not configured, every backup is on this machine only")
+		return
+	}
+	switch st, err := offsite.ReadStatus(env.w.offsiteStatus); {
+	case err != nil:
+		fmt.Fprintf(out, "  off-site: %v\n", err)
+	case st == nil:
+		fmt.Fprintln(out, "  off-site: never synced")
+	case st.LastSuccess.IsZero():
+		fmt.Fprintf(out, "  off-site: never succeeded; last attempt %s ago: %s\n", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
+	default:
+		line := fmt.Sprintf("  off-site: last good sync %s ago to %s", dbbackup.Age(env.now.Sub(st.LastSuccess)), st.Bucket)
+		if st.LastAttempt.After(st.LastSuccess) { // a failed run records no success
+			line += fmt.Sprintf("; the last attempt, %s ago, failed: %s", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
+		}
+		fmt.Fprintln(out, line)
+	}
+}
+
+func statusHost(env statusEnv, out io.Writer) {
+	fmt.Fprintln(out, "\nhost:")
+	seen := map[uint64]bool{}
+	for _, p := range splitList(env.w.diskPaths) {
+		var st syscall.Stat_t
+		if err := syscall.Stat(p, &st); err != nil {
+			continue
+		}
+		dev := uint64(st.Dev) // int32 on darwin
+		if seen[dev] {
+			continue
+		}
+		seen[dev] = true
+		var fs syscall.Statfs_t
+		if err := syscall.Statfs(p, &fs); err != nil || fs.Blocks == 0 {
+			continue
+		}
+		bsize := uint64(fs.Bsize) // uint32 on darwin
+		total, avail := uint64(fs.Blocks)*bsize, uint64(fs.Bavail)*bsize
+		fmt.Fprintf(out, "  disk %s: %s free of %s (%.0f%% free)\n", p, offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)), float64(fs.Bavail)/float64(fs.Blocks)*100)
+	}
+	if total, avail, ok := readMeminfo(env.meminfo); ok {
+		fmt.Fprintf(out, "  memory: %s available of %s\n", offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)))
+	}
+}
+
+// readMeminfo reads MemTotal and MemAvailable, in bytes, from a /proc/meminfo
+// style file.
+func readMeminfo(path string) (total, avail uint64, ok bool) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, 0, false
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		fields := strings.Fields(sc.Text())
+		if len(fields) < 2 {
+			continue
+		}
+		v, _ := strconv.ParseUint(fields[1], 10, 64) // the kernel writes numbers
+		switch fields[0] {
+		case "MemTotal:":
+			total = v * 1024
+		case "MemAvailable:":
+			avail = v * 1024
+		}
+	}
+	return total, avail, total > 0
+}
+
+func statusWatchdog(ctx context.Context, env statusEnv, out io.Writer) {
+	fmt.Fprintln(out, "\nwatchdog:")
+	if _, err := os.Stat(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
+		fmt.Fprintln(out, "  not installed: nothing checks this host")
+		return
+	}
+	line := "  timer " + unitActiveState(ctx, doctorEnv{run: env.run}, "felis-watchdog.timer")
+	show, _ := env.run(ctx, "systemctl", "show", "--timestamp=unix", "-p", "Result", "-p", "ExecMainExitTimestamp", "felis-watchdog.service")
+	props := map[string]string{}
+	for _, ln := range strings.Split(string(show), "\n") {
+		if k, v, ok := strings.Cut(strings.TrimSpace(ln), "="); ok {
+			props[k] = v
+		}
+	}
+	// ExecMainExitTimestamp is empty until the service has run.
+	if sec, _ := strconv.ParseInt(strings.TrimPrefix(props["ExecMainExitTimestamp"], "@"), 10, 64); sec > 0 {
+		line += fmt.Sprintf(", last run %s ago (%s)", dbbackup.Age(env.now.Sub(time.Unix(sec, 0))), orDash(props["Result"]))
+	}
+	fmt.Fprintln(out, line)
+
+	state, err := watchdog.LoadState(watchdog.NewestState(env.w.statePath, env.w.fallbackState))
+	if err != nil {
+		fmt.Fprintf(out, "  alerts: unknown (%v)\n", err)
+		return
+	}
+	var open []string
+	for key, a := range state.Alerts {
+		if !a.ClearedAt.IsZero() {
+			continue
+		}
+		if a.Notified.IsZero() {
+			open = append(open, fmt.Sprintf("%s (%s, seen %s ago, not mailed yet)", key, a.Severity, dbbackup.Age(env.now.Sub(a.FirstSeen))))
+		} else {
+			open = append(open, fmt.Sprintf("%s (%s, mailed %s ago)", key, a.Severity, dbbackup.Age(env.now.Sub(a.Notified))))
+		}
+	}
+	if len(open) == 0 {
+		fmt.Fprintln(out, "  alerts: none open")
+		return
+	}
+	sort.Strings(open)
+	fmt.Fprintf(out, "  alerts: %d open (sudo felis doctor says where to look)\n", len(open))
+	for _, o := range open {
+		fmt.Fprintf(out, "    %s\n", o)
+	}
+}

cmd/felis/status_test.go

@@ -0,0 +1,490 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// statusCluster is a one-node install: felis-api with a restarted pod, three
+// servers (one of them the lobby), and a pod of another app whose restarts
+// are not felis-api's.
+func statusClusterObjects() []client.Object {
+	replicas := int32(1)
+	apiLabels := map[string]string{"app": "felis-api"}
+	return []client.Object{
+		&corev1.Node{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-1"},
+			Status: corev1.NodeStatus{
+				Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}},
+				NodeInfo:   corev1.NodeSystemInfo{KubeletVersion: "v1.36.4+k3s1", OSImage: "CentOS Stream 9", KernelVersion: "5.14.0-630.el9.aarch64"},
+			},
+		},
+		&appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
+			Spec: appsv1.DeploymentSpec{
+				Replicas: &replicas,
+				Selector: &metav1.LabelSelector{MatchLabels: apiLabels},
+				Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{
+					Name: "api", Image: "registry.felis.svc:5000/felis/felis-api:v1.4.0@sha256:0123456789abcdef0123456789abcdef",
+				}}}},
+			},
+			Status: appsv1.DeploymentStatus{ReadyReplicas: 1},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-api-7d9", Namespace: "felis", Labels: apiLabels},
+			Status:     corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 2}}},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "other-1", Namespace: "felis", Labels: map[string]string{"app": "other"}},
+			Status:     corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "x", RestartCount: 5}}},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
+			Spec:       v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
+			Status:     v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseRunning, Players: v1alpha1.PlayersStatus{Online: 2, Max: 20}},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "creative", Namespace: "minecraft"},
+			Spec:       v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped},
+			Status:     v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "lobby", Namespace: "minecraft", Labels: map[string]string{v1alpha1.LabelSystemRole: "lobby"}},
+			Spec:       v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
+			Status:     v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStarting},
+		},
+		// Another namespace's server is not this install's.
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "elsewhere", Namespace: "other"}},
+	}
+}
+
+// statusTestEnv is a host with the cluster above, a database backup nine hours
+// old, an off-site copy last good two hours ago, and one alert open.
+func statusTestEnv(t *testing.T) statusEnv {
+	t.Helper()
+	now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	dir := t.TempDir()
+	var w watchdogFlags
+	w.controlNS = "felis"
+	w.backupDir = filepath.Join(dir, "db-backups")
+	w.offsiteStatus = filepath.Join(dir, "offsite-status.json")
+	w.statePath = filepath.Join(dir, "state.json")
+	w.fallbackState = filepath.Join(dir, "fallback.json")
+	w.diskPaths = "/nonexistent-felis-status-test"
+	unitDir := filepath.Join(dir, "systemd")
+	for _, d := range []string{w.backupDir, unitDir} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-9*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
+	writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-33*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
+	if err := offsite.WriteStatus(w.offsiteStatus, offsite.Status{
+		LastAttempt: now.Add(-2 * time.Hour), LastSuccess: now.Add(-2 * time.Hour), Bucket: "felis-dr", Endpoint: "https://s3.example.com",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	state := &watchdog.State{Alerts: map[string]*watchdog.Alert{
+		"db-backup-servers": {Finding: watchdog.Finding{Key: "db-backup-servers", Severity: watchdog.Warning}, FirstSeen: now.Add(-3 * time.Hour), Notified: now.Add(-90 * time.Minute)},
+		"proxy":             {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: now.Add(-time.Hour), Notified: now.Add(-time.Hour), ClearedAt: now.Add(-10 * time.Minute)},
+		"disk//var":         {Finding: watchdog.Finding{Key: "disk//var", Severity: watchdog.Critical}, FirstSeen: now.Add(-4 * time.Minute)},
+	}}
+	if err := watchdog.SaveState(w.statePath, state); err != nil {
+		t.Fatal(err)
+	}
+	meminfo := filepath.Join(dir, "meminfo")
+	writeTestFile(t, meminfo, "MemTotal:        8000000 kB\nMemFree:          100000 kB\nMemAvailable:    2000000 kB\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.timer"), "[Unit]\n", 0o644)
+	cfg := &config.Config{Offsite: config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-dr"}}
+	return statusEnv{
+		cfg: cfg, w: w, cl: fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).Build(),
+		backups: func(context.Context) (map[string]time.Time, error) {
+			return map[string]time.Time{"survival": now.Add(-3 * time.Hour)}, nil
+		},
+		run: func(_ context.Context, name string, args ...string) ([]byte, error) {
+			switch strings.Join(append([]string{name}, args...), " ") {
+			case "systemctl is-active felis-watchdog.timer":
+				return []byte("active\n"), nil
+			case "systemctl show --timestamp=unix -p Result -p ExecMainExitTimestamp felis-watchdog.service":
+				return []byte("Result=success\nExecMainExitTimestamp=@" + strconv.FormatInt(now.Add(-70*time.Second).Unix(), 10) + "\n"), nil
+			}
+			return nil, errors.New("unexpected")
+		},
+		unitDir: unitDir, meminfo: meminfo, host: "felis-test", now: now,
+	}
+}
+
+// lineFields is the words of the first line of out that starts with prefix
+// once trimmed.
+func lineFields(out, prefix string) []string {
+	for _, l := range strings.Split(out, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(l), prefix) {
+			return strings.Fields(l)
+		}
+	}
+	return nil
+}
+
+func TestStatusReport(t *testing.T) {
+	env := statusTestEnv(t)
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	got := out.String()
+	for _, want := range []string{
+		"on felis-test at 2026-09-27 12:00 UTC\n",
+		"node:     felis-1 Ready, k3s v1.36.4+k3s1, CentOS Stream 9, kernel 5.14.0-630.el9.aarch64\n",
+		"\ncontrol plane (namespace felis):\n",
+		"\nproxy:    not on this host\n",
+		"\nservers (namespace minecraft): 3, 1 running, 2 players online\n",
+		"  database: newest " + dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily) + ", 9h0m ago; 2 bundles in " + env.w.backupDir + "\n",
+		"  off-site: last good sync 2h0m ago to felis-dr\n",
+		"  memory: 1.9 GiB available of 7.6 GiB\n",
+		"  timer active, last run 1m ago (success)\n",
+		"  alerts: 2 open (sudo felis doctor says where to look)\n" +
+			"    db-backup-servers (warning, mailed 1h30m ago)\n" +
+			"    disk//var (critical, seen 4m ago, not mailed yet)\n",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("status lacks %q:\n%s", want, got)
+		}
+	}
+	for prefix, want := range map[string]string{
+		"felis-api": "felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
+		"NAME":      "NAME ROLE DESIRED PHASE PLAYERS NEWEST WORLD BACKUP",
+		"creative":  "creative - Stopped Stopped - none",
+		"lobby":     "lobby lobby Running Starting - none",
+		"survival":  "survival - Running Running 2/20 3h0m ago",
+	} {
+		if f := strings.Join(lineFields(got, prefix), " "); f != want {
+			t.Errorf("row %q = %q, want %q\n%s", prefix, f, want, got)
+		}
+	}
+	if strings.Contains(got, "elsewhere") || strings.Contains(got, "other-1") {
+		t.Errorf("status shows what is not this install's:\n%s", got)
+	}
+}
+
+// Whatever is down reads as down, and the rest of the report still prints.
+func TestStatusWithPartsDown(t *testing.T) {
+	env := statusTestEnv(t)
+	env.cl, env.clErr = nil, errors.New("connection refused")
+	env.cfg = &config.Config{}
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	for _, want := range []string{
+		"cluster:  unreachable (connection refused)\n",
+		"\nservers (namespace minecraft): unknown while the cluster is unreachable\n",
+		"  off-site: not configured, every backup is on this machine only\n",
+		"  timer active, last run",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("status lacks %q:\n%s", want, out.String())
+		}
+	}
+
+	env = statusTestEnv(t)
+	env.backups = func(context.Context) (map[string]time.Time, error) { return nil, errors.New("postgres is down") }
+	out.Reset()
+	printStatus(context.Background(), env, &out)
+	if f := strings.Join(lineFields(out.String(), "survival"), " "); f != "survival - Running Running 2/20 ?" {
+		t.Errorf("survival with PostgreSQL down = %q", f)
+	}
+	if !strings.Contains(out.String(), "  world backups unknown: postgres is down\n") {
+		t.Errorf("status does not say why the backups are unknown:\n%s", out.String())
+	}
+}
+
+func TestShortImage(t *testing.T) {
+	for in, want := range map[string]string{
+		"registry.felis.svc:5000/felis/felis-api:v1.4.0":                                       "felis-api:v1.4.0",
+		"docker.io/library/postgres:17@sha256:0123456789abcdef0123":                            "postgres:17@0123456789ab",
+		"felis-operator@sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba98765432": "felis-operator@fedcba987654",
+		"busybox": "busybox",
+	} {
+		if got := shortImage(in); got != want {
+			t.Errorf("shortImage(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+// A node that is not ready, a Deployment without replicas or containers, and
+// lists the API server refuses each read as such.
+func TestStatusClusterEdges(t *testing.T) {
+	env := statusTestEnv(t)
+	env.cfg = &config.Config{K8s: config.K8sConfig{Namespace: "games"}}
+	env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}, Status: corev1.NodeStatus{
+			Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionFalse}, {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionTrue}}}},
+		&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-bare", Namespace: "felis"}},
+		&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "felis"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 4}}}},
+	).Build()
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	got := out.String()
+	for _, want := range []string{
+		"node:     felis-1 NotReady, k3s , , kernel \n",
+		"\nservers (namespace games): 0, 0 running, 0 players online\n\nbackups:\n",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("status lacks %q:\n%s", want, got)
+		}
+	}
+	if f := strings.Join(lineFields(got, "felis-bare"), " "); f != "felis-bare 0/1 ready - restarts 0" {
+		t.Errorf("row felis-bare = %q, want its one replica wanted, no image, and no pod of its own:\n%s", f, got)
+	}
+
+	failing := func(what client.ObjectList) {
+		env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).
+			WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
+				if fmt.Sprintf("%T", list) == fmt.Sprintf("%T", what) {
+					return errors.New("forbidden")
+				}
+				return c.List(ctx, list, opts...)
+			}}).Build()
+		out.Reset()
+		printStatus(context.Background(), env, &out)
+	}
+	for _, tc := range []struct {
+		list client.ObjectList
+		want string
+	}{
+		{&corev1.NodeList{}, "cluster:  unreachable (forbidden)\n\nproxy:"},
+		{&appsv1.DeploymentList{}, "\ncontrol plane (namespace felis):\n  cannot list it: forbidden\n\nproxy:"},
+		{&corev1.PodList{}, "\ncontrol plane (namespace felis):\n  cannot list it: forbidden\n\nproxy:"},
+		{&v1alpha1.MinecraftServerList{}, "\nservers (namespace games): cannot list them: forbidden\n\nbackups:"},
+	} {
+		failing(tc.list)
+		if !strings.Contains(out.String(), tc.want) {
+			t.Errorf("listing %T refused: status lacks %q:\n%s", tc.list, tc.want, out.String())
+		}
+	}
+}
+
+func TestStatusProxy(t *testing.T) {
+	env := statusTestEnv(t)
+	writeTestFile(t, filepath.Join(env.unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
+	env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active"})
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	env.w.proxyAddr = ln.Addr().String()
+	var out bytes.Buffer
+	statusProxy(context.Background(), env, &out)
+	if want := "\nproxy:    felis-velocity active, " + env.w.proxyAddr + " accepts connections\n"; out.String() != want {
+		t.Errorf("proxy listening: %q, want %q", out.String(), want)
+	}
+	ln.Close()
+	out.Reset()
+	statusProxy(context.Background(), env, &out)
+	if want := "\nproxy:    felis-velocity active, " + env.w.proxyAddr + " refuses connections (dial tcp " + env.w.proxyAddr + ": "; !strings.HasPrefix(out.String(), want) {
+		t.Errorf("proxy gone: %q, want it to start %q", out.String(), want)
+	}
+}
+
+func TestStatusBackups(t *testing.T) {
+	env := statusTestEnv(t)
+	status := func() string {
+		var out bytes.Buffer
+		statusBackups(env, &out)
+		return out.String()
+	}
+	for _, tc := range []struct {
+		what, dir, want string
+	}{
+		{"no -backup-dir", "", "  database: not checked (felis-watchdog.service names no -backup-dir)\n"},
+		{"an empty directory", t.TempDir(), "  database: none in %s\n"},
+		{"a file where the directory should be", filepath.Join(env.w.backupDir, dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily)), "  database: cannot read %s: "},
+	} {
+		env.w.backupDir = tc.dir
+		want := tc.want
+		if strings.Contains(want, "%s") {
+			want = fmt.Sprintf(want, tc.dir)
+		}
+		if got := status(); !strings.Contains(got, want) {
+			t.Errorf("%s: %q, want %q", tc.what, got, want)
+		}
+	}
+
+	for _, tc := range []struct {
+		what   string
+		status *offsite.Status
+		raw    string
+		want   string
+	}{
+		{"never synced", nil, "", "  off-site: never synced\n"},
+		{"never succeeded", &offsite.Status{LastAttempt: env.now.Add(-time.Hour), LastError: "403 Forbidden"}, "", "  off-site: never succeeded; last attempt 1h0m ago: 403 Forbidden\n"},
+		{"the last attempt failed", &offsite.Status{LastAttempt: env.now.Add(-30 * time.Minute), LastSuccess: env.now.Add(-26 * time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
+			"  off-site: last good sync 26h0m ago to felis-dr; the last attempt, 30m ago, failed: timeout\n"},
+		{"an error an earlier attempt left", &offsite.Status{LastAttempt: env.now.Add(-2 * time.Hour), LastSuccess: env.now.Add(-time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
+			"  off-site: last good sync 1h0m ago to felis-dr\n"},
+		{"an unreadable record", nil, "{", "  off-site: unexpected end of JSON input\n"},
+	} {
+		os.Remove(env.w.offsiteStatus)
+		switch {
+		case tc.status != nil:
+			if err := offsite.WriteStatus(env.w.offsiteStatus, *tc.status); err != nil {
+				t.Fatal(err)
+			}
+		case tc.raw != "":
+			writeTestFile(t, env.w.offsiteStatus, tc.raw, 0o600)
+		}
+		if got := status(); !strings.HasSuffix(got, tc.want) {
+			t.Errorf("%s: %q, want it to end %q", tc.what, got, tc.want)
+		}
+	}
+}
+
+func TestStatusHost(t *testing.T) {
+	env := statusTestEnv(t)
+	dir := t.TempDir()
+	env.w.diskPaths = "/nonexistent-felis-status-test," + dir + "," + dir
+	env.meminfo = filepath.Join(dir, "meminfo")
+	var out bytes.Buffer
+	statusHost(env, &out)
+	lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
+	if len(lines) != 3 || lines[0] != "" || lines[1] != "host:" || !strings.HasPrefix(lines[2], "  disk "+dir+": ") || !strings.HasSuffix(lines[2], "% free)") {
+		t.Errorf("host: %q, want one line for %s (a path on a disk already shown, and one that is not there, print none) and no memory line", lines, dir)
+	}
+
+	writeTestFile(t, env.meminfo, "MemTotal: 4096 kB\ngarbage\nMemAvailable: 1024 kB\n", 0o644)
+	if total, avail, ok := readMeminfo(env.meminfo); !ok || total != 4096*1024 || avail != 1024*1024 {
+		t.Errorf("readMeminfo = %d, %d, %v", total, avail, ok)
+	}
+	writeTestFile(t, env.meminfo, "MemAvailable: 1024 kB\n", 0o644)
+	if _, _, ok := readMeminfo(env.meminfo); ok {
+		t.Error("readMeminfo without MemTotal: ok")
+	}
+}
+
+func TestStatusWatchdog(t *testing.T) {
+	env := statusTestEnv(t)
+	status := func() string {
+		var out bytes.Buffer
+		statusWatchdog(context.Background(), env, &out)
+		return out.String()
+	}
+	run := env.run
+	env.run = func(ctx context.Context, name string, args ...string) ([]byte, error) {
+		if len(args) > 0 && args[0] == "show" {
+			return []byte("Result=success\nExecMainExitTimestamp=\n"), nil
+		}
+		return run(ctx, name, args...)
+	}
+	if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: map[string]*watchdog.Alert{
+		"proxy": {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: env.now.Add(-time.Hour), ClearedAt: env.now.Add(-time.Minute)},
+	}}); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := status(), "\nwatchdog:\n  timer active\n  alerts: none open\n"; got != want {
+		t.Errorf("a watchdog that has not run and has nothing open: %q, want %q", got, want)
+	}
+
+	writeTestFile(t, env.w.statePath, "{", 0o600)
+	if got := status(); !strings.Contains(got, "\n  alerts: unknown (") {
+		t.Errorf("an unreadable state: %q", got)
+	}
+
+	if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := status(), "\nwatchdog:\n  not installed: nothing checks this host\n"; got != want {
+		t.Errorf("no watchdog timer: %q, want %q", got, want)
+	}
+}
+
+// Rows print by name in whatever order the API server lists them, a server
+// the operator has not reconciled yet reads as dashes, and open alerts print
+// by key in whatever order the state's map yields them.
+func TestStatusOrder(t *testing.T) {
+	env := statusTestEnv(t)
+	objs := append(statusClusterObjects(),
+		&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-operator", Namespace: "felis"}},
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "fresh", Namespace: "minecraft"}})
+	env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
+		WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
+			// The fake lists by name; the other way round, then.
+			if err := c.List(ctx, list, opts...); err != nil {
+				return err
+			}
+			items, err := meta.ExtractList(list)
+			if err != nil {
+				return err
+			}
+			slices.Reverse(items)
+			return meta.SetList(list, items)
+		}}).Build()
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	var rows []string
+	for _, l := range strings.Split(out.String(), "\n") {
+		if f := strings.Fields(l); len(f) > 0 && (strings.HasPrefix(f[0], "felis-") || slices.Contains([]string{"creative", "fresh", "lobby", "survival"}, f[0])) {
+			rows = append(rows, strings.Join(f, " "))
+		}
+	}
+	want := []string{
+		"felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
+		"felis-operator 0/1 ready - restarts 0",
+		"creative - Stopped Stopped - none",
+		"fresh - - - - none",
+		"lobby lobby Running Starting - none",
+		"survival - Running Running 2/20 3h0m ago",
+	}
+	if !slices.Equal(rows, want) {
+		t.Errorf("rows %q, want %q:\n%s", rows, want, out.String())
+	}
+
+	alerts := map[string]*watchdog.Alert{}
+	for i, key := range []string{"a", "b", "c", "d"} {
+		alerts[key] = &watchdog.Alert{Finding: watchdog.Finding{Key: key, Severity: watchdog.Warning}, FirstSeen: env.now.Add(-time.Duration(i+1) * time.Minute)}
+	}
+	if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: alerts}); err != nil {
+		t.Fatal(err)
+	}
+	wantAlerts := "  alerts: 4 open (sudo felis doctor says where to look)\n" +
+		"    a (warning, seen 1m ago, not mailed yet)\n    b (warning, seen 2m ago, not mailed yet)\n" +
+		"    c (warning, seen 3m ago, not mailed yet)\n    d (warning, seen 4m ago, not mailed yet)\n"
+	// A map starts its walk at random: fifty walks all in order by chance
+	// is out of the question.
+	for range 50 {
+		out.Reset()
+		statusWatchdog(context.Background(), env, &out)
+		if !strings.HasSuffix(out.String(), wantAlerts) {
+			t.Fatalf("alerts %q, want them to end %q", out.String(), wantAlerts)
+		}
+	}
+}
+
+// A selector the API server would have refused picks no pods.
+func TestPodRestartsBadSelector(t *testing.T) {
+	pods := []corev1.Pod{{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 3}}}}}
+	if n := podRestarts(&metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "app", Operator: "Near"}}}, pods); n != 0 {
+		t.Errorf("podRestarts with a bad selector = %d, want 0", n)
+	}
+	if n := podRestarts(&metav1.LabelSelector{}, pods); n != 3 {
+		t.Errorf("podRestarts with a selector that picks all = %d, want 3", n)
+	}
+}

cmd/felis/supportbundle.go

@@ -0,0 +1,862 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+)
+
+// supportBundleDir is where felis support-bundle writes unless told otherwise.
+const supportBundleDir = "/var/lib/felis/support"
+
+// redacted stands in for every value the bundle leaves out.
+const redacted = "<redacted>"
+
+// minScrubLen is the shortest known secret value the bundle searches for: a
+// shorter one would blank ordinary words, and every secret the installer
+// generates is far longer.
+const minScrubLen = 8
+
+// hostSecretSources are the files on a Felis host that hold secrets; the
+// bundle reads them only to take each value out of what it collects.
+var hostSecretSources = secretSources{
+	envFiles:   []string{"/etc/felis/secrets.env", defaultOffsiteEnvFile},
+	valueFiles: []string{hostSMTPPasswordPath, hostUploadsS3AccessKeyPath, hostUploadsS3SecretKeyPath, defaultHeartbeatFile, "/opt/felis/velocity/forwarding.secret"},
+	propsFiles: []string{"/opt/felis/velocity/plugins/felis-link/felis-link.properties"},
+	tokenFiles: []string{"/var/lib/rancher/k3s/server/token", "/var/lib/rancher/k3s/server/agent-token"},
+}
+
+// cmdSupportBundle collects what someone helping with this host needs into
+// one tar.gz: felis status and felis doctor, the logs of the control plane,
+// the builds and the systemd units, the cluster's workloads and events, and a
+// summary of the configuration. It never collects a Secret, a ConfigMap, the
+// contents of a configuration file, the database or a world, and takes every
+// value of the host's secret files out of what it does collect. Game server
+// logs, which carry player names, addresses and chat, only with -server-logs.
+func cmdSupportBundle(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("support-bundle", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	outDir := fs.String("o", supportBundleDir, "directory to write the bundle to (created mode 0700 when missing)")
+	logLines := fs.Int64("log-lines", 2000, "lines kept from the end of each pod log and each unit's journal")
+	since := fs.Duration("since", 48*time.Hour, "how far back each unit's journal is read")
+	serverLogs := fs.Bool("server-logs", false, "also collect the game servers' own logs, which carry player names, IP addresses and chat")
+	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis support-bundle: run as root (sudo felis support-bundle): it reads root-only logs and state")
+		return 1
+	}
+	b := hostSupportBundle(*unitDir, *logLines, *since, *serverLogs)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	path, err := b.write(ctx, *outDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis support-bundle: %v\n", err)
+		return 1
+	}
+	size := int64(0)
+	if st, err := os.Stat(path); err == nil {
+		size = st.Size()
+	}
+	fmt.Fprintf(stdout, "wrote %s (%s, mode 0600)\n", path, humanSize(size))
+	fmt.Fprintln(stdout, "MANIFEST.txt inside says what it holds and what was taken out. Read it through before you send it anywhere:")
+	fmt.Fprintln(stdout, "redaction finds this host's known secrets and the common ways a secret is logged, and a secret logged another way stays in.")
+	if !*serverLogs {
+		fmt.Fprintln(stdout, "Game server logs are left out; -server-logs adds them (player names, IP addresses, chat).")
+	}
+	return 0
+}
+
+func humanSize(n int64) string {
+	switch {
+	case n >= 1<<20:
+		return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
+	case n >= 1<<10:
+		return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10))
+	}
+	return fmt.Sprintf("%d B", n)
+}
+
+// shortDuration is d as Duration.String writes it, less the zero minutes and
+// seconds after whole hours or minutes: 48h, and 90m as 1h30m.
+func shortDuration(d time.Duration) string {
+	s := d.String()
+	if strings.HasSuffix(s, "m0s") {
+		s = strings.TrimSuffix(s, "0s")
+	}
+	if strings.HasSuffix(s, "h0m") {
+		s = strings.TrimSuffix(s, "0m")
+	}
+	return s
+}
+
+// supportBundle is one collection and what it reads the host through.
+type supportBundle struct {
+	host    string
+	now     time.Time
+	unitDir string
+	// w is how felis-watchdog.service runs the watchdog, wErr why it could
+	// not be read (w then holds the defaults).
+	w          watchdogFlags
+	wErr       error
+	cfg        *config.Config // nil when cfgErr
+	cfgErr     error
+	cl         client.Client // nil when clErr
+	clErr      error
+	logs       func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error)
+	run        func(ctx context.Context, name string, args ...string) ([]byte, error)
+	backups    func(ctx context.Context) (map[string]time.Time, error)
+	doctor     func(ctx context.Context, out io.Writer)
+	secrets    secretSources
+	logLines   int64
+	since      time.Duration
+	serverLogs bool
+	// Host files read whole, and the directory whose listing is kept.
+	meminfo, osRelease, procVersion, stateDir string
+}
+
+func hostSupportBundle(unitDir string, logLines int64, since time.Duration, serverLogs bool) *supportBundle {
+	host, _ := os.Hostname()
+	b := &supportBundle{
+		host: host, now: time.Now(), unitDir: unitDir, run: hostCommand, secrets: hostSecretSources,
+		logLines: logLines, since: since, serverLogs: serverLogs,
+		meminfo: "/proc/meminfo", osRelease: "/etc/os-release", procVersion: "/proc/version", stateDir: "/etc/felis",
+	}
+	var found bool
+	b.w, found, b.wErr = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
+	if b.wErr == nil && !found {
+		b.wErr = fmt.Errorf("%s is not installed; read the watchdog's defaults", filepath.Join(unitDir, "felis-watchdog.service"))
+	}
+	if b.cfg, b.cfgErr = config.Load(b.w.cfgPath); b.cfgErr == nil {
+		cfg := b.cfg
+		b.backups = func(ctx context.Context) (map[string]time.Time, error) {
+			return newestWorldBackups(ctx, cfg.Database.URL)
+		}
+	} else {
+		b.cfg = nil
+	}
+	if b.cl, b.clErr = buildSystemServerClient(); b.clErr != nil {
+		b.cl = nil
+	} else if rc, err := hostRESTConfig(); err != nil {
+		b.clErr = err
+		b.cl = nil
+	} else if cs, err := kubernetes.NewForConfig(rc); err != nil {
+		b.clErr = err
+		b.cl = nil
+	} else {
+		limit := int64(8 << 20)
+		b.logs = func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error) {
+			return cs.CoreV1().Pods(ns).GetLogs(pod, &corev1.PodLogOptions{
+				Container: container, Previous: previous, Timestamps: true, TailLines: &logLines, LimitBytes: &limit,
+			}).DoRaw(ctx)
+		}
+	}
+	b.doctor = func(ctx context.Context, out io.Writer) {
+		runDoctor(ctx, doctorEnv{unitDir: unitDir, run: hostCommand, now: b.now, host: host}, out)
+	}
+	return b
+}
+
+// bundleWriter streams scrubbed files into the archive and keeps what went
+// wrong while collecting, for MANIFEST.txt.
+type bundleWriter struct {
+	tw     *tar.Writer
+	prefix string
+	now    time.Time
+	scrub  *scrubber
+	errs   []string
+}
+
+// logText adds a log, or a report that quotes errors: known secrets and
+// anything logged as one come out.
+func (bw *bundleWriter) logText(name string, data []byte) error {
+	return bw.add(name, bw.scrub.text(data))
+}
+
+// plain adds a file whose secrets were already taken out by structure (the
+// cluster's objects, the configuration summary) or that names none (the
+// release, disk use, addresses): known secret values still come out, and
+// nothing else is rewritten.
+func (bw *bundleWriter) plain(name string, data []byte) error {
+	return bw.add(name, bw.scrub.values(data))
+}
+
+func (bw *bundleWriter) add(name string, data []byte) error {
+	hdr := &tar.Header{Name: path.Join(bw.prefix, name), Mode: 0o600, Size: int64(len(data)), ModTime: bw.now, Typeflag: tar.TypeReg}
+	if err := bw.tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+	_, err := bw.tw.Write(data)
+	return err
+}
+
+func (bw *bundleWriter) failed(what string, err error) {
+	bw.errs = append(bw.errs, string(bw.scrub.text([]byte(fmt.Sprintf("%s: %v", what, err)))))
+}
+
+// write collects the bundle into dir and returns its path.
+func (b *supportBundle) write(ctx context.Context, dir string) (string, error) {
+	if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {
+		if err := os.MkdirAll(dir, 0o700); err != nil {
+			return "", err
+		}
+	}
+	stamp := b.now.UTC().Format("20060102T150405Z")
+	base := fmt.Sprintf("felis-support-%s-%s", safeName(b.host), stamp)
+	final := filepath.Join(dir, base+".tar.gz")
+	f, err := os.CreateTemp(dir, "."+base+".*.partial") // mode 0600
+	if err != nil {
+		return "", err
+	}
+	keep := false
+	defer func() {
+		if !keep {
+			f.Close()
+			os.Remove(f.Name())
+		}
+	}()
+	gz := gzip.NewWriter(f)
+	bw := &bundleWriter{tw: tar.NewWriter(gz), prefix: base, now: b.now, scrub: b.scrubber()}
+	if err := b.collect(ctx, bw); err != nil {
+		return "", err
+	}
+	if err := bw.tw.Close(); err != nil {
+		return "", err
+	}
+	if err := gz.Close(); err != nil {
+		return "", err
+	}
+	if err := f.Sync(); err != nil {
+		return "", err
+	}
+	if err := f.Close(); err != nil {
+		return "", err
+	}
+	if err := os.Rename(f.Name(), final); err != nil {
+		return "", err
+	}
+	keep = true
+	return final, nil
+}
+
+// safeName keeps a host name usable in a file name.
+func safeName(s string) string {
+	s = strings.Map(func(r rune) rune {
+		if r == '-' || r == '.' || r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
+			return r
+		}
+		return '_'
+	}, s)
+	if s == "" {
+		return "host"
+	}
+	return s
+}
+
+func (b *supportBundle) collect(ctx context.Context, bw *bundleWriter) error {
+	var buf bytes.Buffer
+	cmdVersion(nil, &buf, io.Discard)
+	for _, p := range []string{b.osRelease, b.procVersion} {
+		if raw, err := os.ReadFile(p); err == nil {
+			fmt.Fprintf(&buf, "\n# %s\n%s", p, raw)
+		}
+	}
+	if err := bw.plain("version.txt", buf.Bytes()); err != nil {
+		return err
+	}
+
+	buf.Reset()
+	switch {
+	case b.cfgErr != nil:
+		fmt.Fprintf(&buf, "felis status: the configuration did not load: %v\n", b.cfgErr)
+	default:
+		printStatus(ctx, statusEnv{
+			cfg: b.cfg, w: b.w, cl: b.cl, clErr: b.clErr, backups: b.backups, run: b.run,
+			unitDir: b.unitDir, meminfo: b.meminfo, host: b.host, now: b.now,
+		}, &buf)
+	}
+	if err := bw.logText("status.txt", buf.Bytes()); err != nil {
+		return err
+	}
+	buf.Reset()
+	b.doctor(ctx, &buf)
+	if err := bw.logText("doctor.txt", buf.Bytes()); err != nil {
+		return err
+	}
+	if b.cfg != nil {
+		if err := bw.plain("config.txt", configSummary(b.cfg, b.w.cfgPath)); err != nil {
+			return err
+		}
+	}
+	if err := b.collectHost(ctx, bw); err != nil {
+		return err
+	}
+	if err := b.collectJournal(ctx, bw); err != nil {
+		return err
+	}
+	if b.cl == nil {
+		bw.failed("cluster", b.clErr)
+	} else if err := b.collectCluster(ctx, bw); err != nil {
+		return err
+	}
+	return bw.add("MANIFEST.txt", b.manifest(bw))
+}
+
+func (b *supportBundle) collectHost(ctx context.Context, bw *bundleWriter) error {
+	commands := []struct {
+		name string
+		argv []string
+	}{
+		{"host/systemd-units.txt", []string{"systemctl", "list-units", "--all", "--no-pager", "--plain", "felis-*", "k3s.service"}},
+		{"host/systemd-timers.txt", []string{"systemctl", "list-timers", "--all", "--no-pager", "felis-*"}},
+		{"host/df.txt", []string{"df", "-h"}},
+		{"host/addresses.txt", []string{"ip", "-brief", "address"}},
+	}
+	for _, c := range commands {
+		out, err := b.run(ctx, c.argv[0], c.argv[1:]...)
+		if err != nil && len(out) == 0 {
+			bw.failed(strings.Join(c.argv, " "), err)
+			continue
+		}
+		if err := bw.plain(c.name, out); err != nil {
+			return err
+		}
+	}
+	if raw, err := os.ReadFile(b.meminfo); err == nil {
+		if err := bw.plain("host/meminfo.txt", raw); err != nil {
+			return err
+		}
+	}
+	listing, err := dirListing(b.stateDir)
+	if err != nil {
+		bw.failed("list "+b.stateDir, err)
+		return nil
+	}
+	return bw.plain("host/etc-felis.txt", listing)
+}
+
+// dirListing names every file under dir with its mode, size and time, and
+// holds nothing of what is in them.
+func dirListing(dir string) ([]byte, error) {
+	var buf bytes.Buffer
+	tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
+	fmt.Fprintf(tw, "# %s: names, modes, sizes and times only; no contents\n", dir)
+	err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(tw, "%s\t%d\t%s\t%s\n", info.Mode(), info.Size(), info.ModTime().UTC().Format(time.RFC3339), p)
+		return nil
+	})
+	tw.Flush()
+	return buf.Bytes(), err
+}
+
+func (b *supportBundle) collectJournal(ctx context.Context, bw *bundleWriter) error {
+	units, _ := filepath.Glob(filepath.Join(b.unitDir, "felis-*.service"))
+	if _, err := os.Stat(filepath.Join(b.unitDir, "k3s.service")); err == nil {
+		units = append(units, filepath.Join(b.unitDir, "k3s.service"))
+	}
+	since := "@" + strconv.FormatInt(b.now.Add(-b.since).Unix(), 10)
+	for _, u := range units {
+		unit := filepath.Base(u)
+		out, err := b.run(ctx, "journalctl", "-u", unit, "--since", since, "-n", strconv.FormatInt(b.logLines, 10), "--no-pager", "-o", "short-iso")
+		if err != nil && len(out) == 0 {
+			bw.failed("journalctl -u "+unit, err)
+			continue
+		}
+		if err := bw.logText("journal/"+strings.TrimSuffix(unit, ".service")+".log", out); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// bundleNamespaces are the namespaces whose objects and logs the bundle
+// collects: the control plane, the builds and the game servers.
+func (b *supportBundle) bundleNamespaces() (control, build, minecraft string) {
+	control, build, minecraft = b.w.controlNS, platform.DefaultBuildNamespace, platform.DefaultMinecraftNamespace
+	if b.cfg != nil {
+		if b.cfg.Registry.BuildNamespace != "" {
+			build = b.cfg.Registry.BuildNamespace
+		}
+		if b.cfg.K8s.Namespace != "" {
+			minecraft = b.cfg.K8s.Namespace
+		}
+	}
+	return control, build, minecraft
+}
+
+func (b *supportBundle) collectCluster(ctx context.Context, bw *bundleWriter) error {
+	control, build, minecraft := b.bundleNamespaces()
+	dump := func(name string, list client.ObjectList, opts ...client.ListOption) error {
+		if err := b.cl.List(ctx, list, opts...); err != nil {
+			bw.failed("list "+name, err)
+			return nil
+		}
+		redactList(list)
+		out, err := yaml.Marshal(list)
+		if err != nil {
+			bw.failed("encode "+name, err)
+			return nil
+		}
+		return bw.plain(name, out)
+	}
+	if err := dump("cluster/nodes.yaml", &corev1.NodeList{}); err != nil {
+		return err
+	}
+	if err := dump("cluster/persistentvolumes.yaml", &corev1.PersistentVolumeList{}); err != nil {
+		return err
+	}
+	if err := dump("cluster/minecraftservers.yaml", &v1alpha1.MinecraftServerList{}, client.InNamespace(minecraft)); err != nil {
+		return err
+	}
+	for _, ns := range []string{control, build, minecraft} {
+		for _, k := range []struct {
+			name string
+			list client.ObjectList
+		}{
+			{"pods", &corev1.PodList{}},
+			{"deployments", &appsv1.DeploymentList{}},
+			{"statefulsets", &appsv1.StatefulSetList{}},
+			{"jobs", &batchv1.JobList{}},
+			{"cronjobs", &batchv1.CronJobList{}},
+			{"services", &corev1.ServiceList{}},
+			{"persistentvolumeclaims", &corev1.PersistentVolumeClaimList{}},
+			{"networkpolicies", &networkingv1.NetworkPolicyList{}},
+			{"events", &corev1.EventList{}},
+		} {
+			if err := dump("cluster/"+ns+"/"+k.name+".yaml", k.list, client.InNamespace(ns)); err != nil {
+				return err
+			}
+		}
+	}
+
+	var all corev1.PodList
+	if err := b.cl.List(ctx, &all); err != nil {
+		bw.failed("list every pod", err)
+	} else if err := bw.plain("cluster/pods-all-namespaces.txt", podTable(all.Items, b.now)); err != nil {
+		return err
+	}
+
+	for _, ns := range []string{control, build, minecraft} {
+		var pods corev1.PodList
+		if err := b.cl.List(ctx, &pods, client.InNamespace(ns)); err != nil {
+			continue // already recorded by the dump above
+		}
+		for _, p := range pods.Items {
+			if err := b.collectPodLogs(ctx, bw, p, ns == minecraft && !b.serverLogs); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// collectPodLogs keeps the tail of each container's log, and of its previous
+// run when it restarted. initOnly keeps only the init containers: a game
+// server's own log carries player names, addresses and chat.
+func (b *supportBundle) collectPodLogs(ctx context.Context, bw *bundleWriter, p corev1.Pod, initOnly bool) error {
+	type ctr struct {
+		name     string
+		restarts int32
+	}
+	var ctrs []ctr
+	restarts := map[string]int32{}
+	for _, cs := range append(append([]corev1.ContainerStatus(nil), p.Status.InitContainerStatuses...), p.Status.ContainerStatuses...) {
+		restarts[cs.Name] = cs.RestartCount
+	}
+	for _, c := range p.Spec.InitContainers {
+		ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
+	}
+	if !initOnly {
+		for _, c := range p.Spec.Containers {
+			ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
+		}
+	}
+	for _, c := range ctrs {
+		for _, previous := range []bool{false, true} {
+			if previous && c.restarts == 0 {
+				continue
+			}
+			name := fmt.Sprintf("logs/%s/%s/%s.log", p.Namespace, p.Name, c.name)
+			if previous {
+				name = fmt.Sprintf("logs/%s/%s/%s.previous.log", p.Namespace, p.Name, c.name)
+			}
+			out, err := b.logs(ctx, p.Namespace, p.Name, c.name, previous)
+			if err != nil {
+				bw.failed(name, err)
+				continue
+			}
+			if err := bw.logText(name, out); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// podTable is every pod on the node, one line each.
+func podTable(pods []corev1.Pod, now time.Time) []byte {
+	sort.Slice(pods, func(i, j int) bool {
+		if pods[i].Namespace != pods[j].Namespace {
+			return pods[i].Namespace < pods[j].Namespace
+		}
+		return pods[i].Name < pods[j].Name
+	})
+	var buf bytes.Buffer
+	tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "NAMESPACE\tNAME\tPHASE\tREADY\tRESTARTS\tAGE")
+	for _, p := range pods {
+		var ready, restarts int32
+		for _, cs := range p.Status.ContainerStatuses {
+			if cs.Ready {
+				ready++
+			}
+			restarts += cs.RestartCount
+		}
+		age := "-"
+		if !p.CreationTimestamp.IsZero() {
+			age = now.Sub(p.CreationTimestamp.Time).Round(time.Minute).String()
+		}
+		fmt.Fprintf(tw, "%s\t%s\t%s\t%d/%d\t%d\t%s\n", p.Namespace, p.Name, p.Status.Phase, ready, len(p.Spec.Containers), restarts, age)
+	}
+	tw.Flush()
+	return buf.Bytes()
+}
+
+// redactList takes out of every object what the bundle must not carry: the
+// literal env values of pod specs and of MinecraftServers (valueFrom
+// references stay, naming the Secret without its contents), the
+// last-applied-configuration annotation that repeats them, and managedFields.
+func redactList(list client.ObjectList) {
+	items, err := meta.ExtractList(list)
+	if err != nil {
+		return
+	}
+	for _, it := range items {
+		if acc, err := meta.Accessor(it); err == nil {
+			acc.SetManagedFields(nil)
+			if ann := acc.GetAnnotations(); ann != nil {
+				delete(ann, corev1.LastAppliedConfigAnnotation)
+				acc.SetAnnotations(ann)
+			}
+		}
+		switch o := it.(type) {
+		case *corev1.Pod:
+			redactPodSpec(&o.Spec)
+		case *appsv1.Deployment:
+			redactPodSpec(&o.Spec.Template.Spec)
+		case *appsv1.StatefulSet:
+			redactPodSpec(&o.Spec.Template.Spec)
+		case *batchv1.Job:
+			redactPodSpec(&o.Spec.Template.Spec)
+		case *batchv1.CronJob:
+			redactPodSpec(&o.Spec.JobTemplate.Spec.Template.Spec)
+		case *v1alpha1.MinecraftServer:
+			for i := range o.Spec.Env {
+				if o.Spec.Env[i].Value != "" {
+					o.Spec.Env[i].Value = redacted
+				}
+			}
+		}
+	}
+}
+
+func redactPodSpec(s *corev1.PodSpec) {
+	blank := func(cs []corev1.Container) {
+		for i := range cs {
+			for j := range cs[i].Env {
+				if cs[i].Env[j].Value != "" {
+					cs[i].Env[j].Value = redacted
+				}
+			}
+		}
+	}
+	blank(s.InitContainers)
+	blank(s.Containers)
+	for i := range s.EphemeralContainers {
+		for j := range s.EphemeralContainers[i].Env {
+			if s.EphemeralContainers[i].Env[j].Value != "" {
+				s.EphemeralContainers[i].Env[j].Value = redacted
+			}
+		}
+	}
+}
+
+// configSummary is felis.toml without a single credential: the hostnames,
+// namespaces and which features are on. Fields are picked one by one, so a
+// field added later stays out until someone decides it is safe.
+func configSummary(c *config.Config, path string) []byte {
+	var buf bytes.Buffer
+	line := func(k string, v any) { fmt.Fprintf(&buf, "%-34s %v\n", k, v) }
+	fmt.Fprintf(&buf, "# a summary of %s; no password, key or token is in it\n", path)
+	line("server.root_domain", c.Server.RootDomain)
+	line("server.listen", c.Server.Listen)
+	db := "(unparsable)"
+	if u, err := url.Parse(c.Database.URL); err == nil {
+		db = u.Scheme + "://" + u.User.Username() + "@" + u.Host + u.Path
+	}
+	line("database.url (no password)", db)
+	line("database.deployment", c.Database.Deployment)
+	line("velocity.public_ip", c.Velocity.PublicIP)
+	line("velocity.game_port", c.Velocity.GamePort)
+	line("velocity.login_image", c.Velocity.LoginImage)
+	line("velocity.lobby_image", c.Velocity.LobbyImage)
+	line("auth.panel_hostname", c.Auth.PanelHostname)
+	line("auth.admin_hostname", c.Auth.AdminHostname)
+	line("auth.client_ip_header", c.Auth.ClientIPHeader)
+	line("k8s.namespace", c.K8s.Namespace)
+	line("k8s.egress_mode", c.K8s.EgressMode)
+	line("k8s.metallb_pool", c.K8s.MetalLBPool)
+	line("registry.url", c.Registry.URL)
+	line("registry.build_namespace", c.Registry.BuildNamespace)
+	line("registry.trivy_db_repository", c.Registry.TrivyDBRepository)
+	line("registry.build_user_namespaces", c.Registry.BuildUserNamespaces)
+	line("registry.build_runtime_class", c.Registry.BuildRuntimeClass)
+	line("registry.max_concurrent_builds", c.Registry.MaxConcurrentBuilds)
+	line("registry.user_uploads_context", c.Registry.UserUploadsContext)
+	line("archive.store", c.Archive.Store)
+	line("archive.local_path", c.Archive.LocalPath)
+	line("archive.retention", c.Archive.Retention)
+	line("archive.scheduled_every", c.Archive.ScheduledEvery)
+	line("archive.scheduled_keep", c.Archive.ScheduledKeep)
+	line("offsite (configured)", c.Offsite.Enabled())
+	if c.Offsite.Enabled() {
+		line("offsite.endpoint", c.Offsite.Endpoint)
+		line("offsite.bucket", c.Offsite.Bucket)
+		line("offsite.prefix", c.Offsite.Prefix)
+	}
+	line("smtp.host", c.SMTP.Host)
+	if c.SMTP.Host != "" {
+		line("smtp.port", c.SMTP.Port)
+		line("smtp.require_tls", c.SMTP.TLSRequired())
+		line("smtp.max_per_hour", c.SMTP.MaxPerHour)
+	}
+	for i, s := range c.AuthSources {
+		line(fmt.Sprintf("auth_source[%d]", i), s.Tag+" "+s.Prefix+" "+s.URL)
+	}
+	return buf.Bytes()
+}
+
+func (b *supportBundle) manifest(bw *bundleWriter) []byte {
+	control, build, minecraft := b.bundleNamespaces()
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "Felis support bundle\nhost %s, collected %s, felis %s\n\n", b.host, b.now.UTC().Format(time.RFC3339), resolvedVersion())
+	fmt.Fprintf(&buf, `What it holds:
+  status.txt, doctor.txt   felis status and felis doctor at collection time
+  version.txt              the release, the OS and the kernel
+  config.txt               a summary of felis.toml: hostnames, namespaces, which features are on
+  host/                    systemd units and timers, disk use, memory, addresses, and the names,
+                           modes, sizes and times of the files under %s (not what is in them)
+  journal/                 up to %d lines per Felis unit and k3s, from the last %s
+  logs/                    up to %d lines of each container of the pods in %s and %s, and of the
+                           init containers of the game server pods in %s; the previous run too
+                           where a container restarted
+  cluster/                 nodes, volumes, the MinecraftServers, and the pods, workloads, services,
+                           volume claims, network policies and events of %s, %s and %s
+`, b.stateDir, b.logLines, shortDuration(b.since), b.logLines, control, build, minecraft, control, build, minecraft)
+	if b.serverLogs {
+		fmt.Fprintln(&buf, "\nThe game servers' own logs are in logs/ (-server-logs): they carry player names, IP addresses and chat.")
+	} else {
+		fmt.Fprintln(&buf, "\nThe game servers' own logs are left out (they carry player names, IP addresses and chat; -server-logs adds them).")
+	}
+	fmt.Fprint(&buf, `
+Never collected: Kubernetes Secrets and ConfigMaps, what is in /etc/felis or any configuration
+file, the database, worlds, uploads.
+
+Taken out:
+`)
+	if len(bw.scrub.sources) > 0 {
+		fmt.Fprintf(&buf, "  - %d secret values, wherever they appear, read from:\n", len(bw.scrub.vals))
+		for _, s := range bw.scrub.sources {
+			fmt.Fprintf(&buf, "      %s\n", s)
+		}
+	} else {
+		fmt.Fprintln(&buf, "  - no secret file was found on this host to take values from")
+	}
+	fmt.Fprint(&buf, `  - passwords in URLs, private keys, Bearer and Basic credentials
+  - in logs and command output, whatever follows password=, secret=, token=, api_key=,
+    access_key=, private_key= or credentials= (and the same with a colon)
+  - every literal env value in pod specs and MinecraftServers (valueFrom references stay)
+
+Read it through before you send it anywhere: redaction finds this host's known secrets and the
+common ways a secret is logged, and a secret logged another way stays in.
+`)
+	if b.wErr != nil {
+		fmt.Fprintf(&buf, "\nThe watchdog's settings: %v\n", b.wErr)
+	}
+	if len(bw.errs) > 0 {
+		fmt.Fprintln(&buf, "\nNot collected:")
+		for _, e := range bw.errs {
+			fmt.Fprintf(&buf, "  - %s\n", e)
+		}
+	}
+	// Its own words name what is redacted, and would be redacted themselves;
+	// what it quotes was scrubbed as it was recorded.
+	return buf.Bytes()
+}
+
+// secretSources are files that hold secrets, by how each is laid out.
+type secretSources struct {
+	envFiles   []string // KEY=VALUE lines, every value a secret (a commented-out one too)
+	valueFiles []string // one secret, the whole file
+	propsFiles []string // key=value lines; the keys naming a token, secret, password or key hold one
+	tokenFiles []string // k3s join tokens: the whole token and its secret part after the last ':'
+}
+
+// scrubber takes secrets out of what the bundle collects.
+type scrubber struct {
+	vals    []string // longest first, so a secret that contains another goes whole
+	sources []string // the files vals came from
+}
+
+var (
+	pemPrivateKey = regexp.MustCompile(`(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?-----END [A-Z0-9 ]*PRIVATE KEY-----`)
+	urlUserinfo   = regexp.MustCompile(`([A-Za-z][A-Za-z0-9+.-]*://[^/\s:@]*:)[^/\s@]+@`)
+	authScheme    = regexp.MustCompile(`(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}`)
+	secretAssign  = regexp.MustCompile(`(?i)((?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|credentials?)[A-Za-z0-9_.-]*"?[ \t]*[=:][ \t]*"?)([^\s"',;&]{4,})`)
+	secretPropKey = regexp.MustCompile(`(?i)token|secret|password|key`)
+)
+
+func (b *supportBundle) scrubber() *scrubber {
+	s := &scrubber{}
+	s.readSources(b.secrets)
+	if b.cfg != nil {
+		if u, err := url.Parse(b.cfg.Database.URL); err == nil {
+			if pw, ok := u.User.Password(); ok {
+				s.add(pw)
+			}
+		}
+		for _, ref := range []string{
+			b.cfg.Velocity.ServiceTokenRef, b.cfg.SMTP.PasswordRef,
+			b.cfg.Offsite.AccessKeyRef, b.cfg.Offsite.SecretKeyRef, b.cfg.Offsite.KeyRef,
+			b.cfg.Registry.S3.AccessKeyRef, b.cfg.Registry.S3.SecretKeyRef,
+			b.cfg.Archive.S3.AccessKeyRef, b.cfg.Archive.S3.SecretKeyRef,
+		} {
+			if ref != "" {
+				s.add(os.Getenv(ref))
+			}
+		}
+	}
+	if st, err := watchdog.LoadState(watchdog.NewestState(b.w.statePath, b.w.fallbackState)); err == nil {
+		s.add(st.SMTPPassword)
+	}
+	return s
+}
+
+func (s *scrubber) add(v string) bool {
+	v = strings.TrimSpace(v)
+	if len(v) < minScrubLen {
+		return false
+	}
+	for _, have := range s.vals {
+		if have == v {
+			return true
+		}
+	}
+	s.vals = append(s.vals, v)
+	sort.SliceStable(s.vals, func(i, j int) bool { return len(s.vals[i]) > len(s.vals[j]) })
+	return true
+}
+
+func (s *scrubber) readSources(src secretSources) {
+	read := func(p string, take func(content string) bool) {
+		raw, err := os.ReadFile(p)
+		if err != nil {
+			return
+		}
+		if take(string(raw)) {
+			s.sources = append(s.sources, p)
+		}
+	}
+	keyValues := func(content string, keep func(key string) bool) bool {
+		found := false
+		for _, line := range strings.Split(content, "\n") {
+			k, v, ok := strings.Cut(line, "=")
+			if !ok || !keep(strings.TrimSpace(k)) {
+				continue
+			}
+			v = strings.TrimSpace(v)
+			if len(v) >= 2 && (v[0] == '\'' || v[0] == '"') && v[len(v)-1] == v[0] {
+				v = v[1 : len(v)-1]
+			}
+			found = s.add(v) || found
+		}
+		return found
+	}
+	for _, p := range src.envFiles {
+		read(p, func(c string) bool { return keyValues(c, func(string) bool { return true }) })
+	}
+	for _, p := range src.propsFiles {
+		read(p, func(c string) bool { return keyValues(c, secretPropKey.MatchString) })
+	}
+	for _, p := range src.valueFiles {
+		read(p, func(c string) bool { return s.add(c) })
+	}
+	for _, p := range src.tokenFiles {
+		read(p, func(c string) bool {
+			c = strings.TrimSpace(c)
+			whole := s.add(c)
+			part := false
+			if i := strings.LastIndex(c, ":"); i >= 0 {
+				part = s.add(c[i+1:])
+			}
+			return whole || part
+		})
+	}
+}
+
+// values takes every known secret value out of data.
+func (s *scrubber) values(data []byte) []byte {
+	t := pemPrivateKey.ReplaceAllString(string(data), "<redacted private key>")
+	for _, v := range s.vals {
+		t = strings.ReplaceAll(t, v, redacted)
+	}
+	t = urlUserinfo.ReplaceAllString(t, "${1}"+redacted+"@")
+	t = authScheme.ReplaceAllString(t, "${1} "+redacted)
+	return []byte(t)
+}
+
+// text is values, and whatever a log names as a secret as well.
+func (s *scrubber) text(data []byte) []byte {
+	return secretAssign.ReplaceAll(s.values(data), []byte("${1}"+redacted))
+}

cmd/felis/supportbundle_test.go

@@ -0,0 +1,433 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+// The secrets a bundle host holds, each of which must be nowhere in the bundle.
+var plantedSecrets = map[string]string{
+	"secrets.env SERVICE_TOKEN":     "svc-token-planted-0a1b2c3d",
+	"secrets.env DB_PASSWORD":       "db-pass-planted-4e5f6a7b",
+	"offsite.env FELIS_OFFSITE_KEY": "offsite-key-planted+8c9d/0e1f=",
+	"smtp-password":                 "smtp-pass-planted-2a3b",
+	"felis-link.properties token":   "props-token-planted-4c5d",
+	"k3s token secret part":         "k3s-secret-part-planted-6e7f",
+	"watchdog state relay password": "cached-relay-pw-planted-8a9b",
+	"pod env literal":               "env-literal-planted-0c1d",
+	"deployment env literal":        "deploy-env-planted-2e3f",
+	"MinecraftServer env literal":   "cr-env-planted-4a5b",
+	"last-applied annotation":       "annotation-planted-6c7d",
+	"logged password":               "hunter2-planted-8e9f",
+	"logged bearer":                 "bearerplanted0a1b2c3d",
+	"URL password":                  "urlpass-planted-4e5f",
+	"token in an error":             "errtoken-planted-0f1e",
+	"felis.host.toml DB password":   "cfg-db-pass-planted-1c2d",
+	"service token by its env ref":  "env-ref-token-planted-3e4f",
+	"init container env literal":    "init-env-planted-5a6b",
+	"ephemeral container env":       "ephemeral-env-planted-7c8d",
+	"statefulset env literal":       "sts-env-planted-9e0f",
+	"job env literal":               "job-env-planted-1a2b",
+	"cronjob env literal":           "cronjob-env-planted-3c4d",
+	"commented-out offsite key":     "old-offsite-key-planted-5e6f",
+	"doctor quoting a password":     "doctor-pw-planted-7a8b",
+	"status quoting a password":     "status-pw-planted-9c0d",
+	"journal quoting a password":    "journal-pw-planted-1e2f",
+}
+
+// bundleHost is a Felis host for felis support-bundle: its secret files, its
+// watchdog unit and state, a cluster with a control-plane pod that restarted
+// and a game server, and logs and a journal that name secrets.
+func bundleHost(t *testing.T) *supportBundle {
+	t.Helper()
+	p := plantedSecrets
+	dir := t.TempDir()
+	etc := filepath.Join(dir, "etc-felis")
+	for _, d := range []string{etc, filepath.Join(dir, "systemd"), filepath.Join(dir, "k3s")} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeTestFile(t, filepath.Join(etc, "secrets.env"), "SERVICE_TOKEN="+p["secrets.env SERVICE_TOKEN"]+"\nDB_PASSWORD="+p["secrets.env DB_PASSWORD"]+"\nSHORT=abc\n", 0o600)
+	writeTestFile(t, filepath.Join(etc, "offsite.env"), "# before the rotation\n# FELIS_OFFSITE_KEY="+p["commented-out offsite key"]+"\nFELIS_OFFSITE_KEY='"+p["offsite.env FELIS_OFFSITE_KEY"]+"'\n", 0o600)
+	writeTestFile(t, filepath.Join(etc, "smtp-password"), p["smtp-password"]+"\n", 0o600)
+	writeTestFile(t, filepath.Join(etc, "felis-link.properties"), "api-base-url=http://10.43.0.10:8081\nservice-token="+p["felis-link.properties token"]+"\nroot-domain=games.example.org\n", 0o640)
+	writeTestFile(t, filepath.Join(dir, "k3s", "token"), "K10deadbeefcafe::server:"+p["k3s token secret part"]+"\n", 0o600)
+	cfgPath := filepath.Join(etc, "felis.host.toml")
+	writeTestFile(t, cfgPath, "[database]\nurl = \"postgres://felis:"+p["felis.host.toml DB password"]+"@127.0.0.1:1/felis?sslmode=disable&connect_timeout=1\"\n"+
+		"[server]\nroot_domain = \"games.example.org\"\n[archive]\nstore = \"tarLocal\"\n[k8s]\negress_mode = \"nodeport\"\n"+
+		"[velocity]\nservice_token_ref = \"FELIS_BUNDLE_TEST_SERVICE_TOKEN\"\n", 0o600)
+	t.Setenv("FELIS_BUNDLE_TEST_SERVICE_TOKEN", p["service token by its env ref"])
+	statePath := filepath.Join(dir, "state.json")
+	if err := watchdog.SaveState(statePath, &watchdog.State{SMTPPassword: p["watchdog state relay password"]}); err != nil {
+		t.Fatal(err)
+	}
+	unitDir := filepath.Join(dir, "systemd")
+	writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+cfgPath+" -state "+statePath+"\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-offsite.service"), "[Unit]\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "k3s.service"), "[Unit]\n", 0o644)
+	meminfo := filepath.Join(dir, "meminfo")
+	writeTestFile(t, meminfo, "MemTotal: 8000000 kB\nMemAvailable: 2000000 kB\n", 0o644)
+
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var w watchdogFlags
+	w, _, err = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	w.diskPaths = "/nonexistent-felis-bundle-test"
+
+	secretEnv := corev1.EnvVar{Name: "FELIS_SMTP_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
+		LocalObjectReference: corev1.LocalObjectReference{Name: "felis-smtp"}, Key: "password"}}}
+	replicas := int32(1)
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "felis-api-7d9", Namespace: "felis",
+				Annotations:   map[string]string{corev1.LastAppliedConfigAnnotation: `{"env":"` + p["last-applied annotation"] + `"}`, "felis.lolicon.best/kept": "yes"},
+				ManagedFields: []metav1.ManagedFieldsEntry{{Manager: "kubectl-client-side-apply"}},
+			},
+			Spec: corev1.PodSpec{
+				InitContainers: []corev1.Container{{Name: "wait-db", Image: "busybox", Env: []corev1.EnvVar{{Name: "FELIS_INIT_SETTING", Value: p["init container env literal"]}}}},
+				Containers: []corev1.Container{{Name: "api", Image: "felis-api:v1", Env: []corev1.EnvVar{
+					{Name: "FELIS_PLAIN_SETTING", Value: p["pod env literal"]}, secretEnv,
+				}}},
+				EphemeralContainers: []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{
+					Name: "debug", Env: []corev1.EnvVar{{Name: "FELIS_DEBUG_SETTING", Value: p["ephemeral container env"]}}}}},
+			},
+			Status: corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 1, Ready: true}}},
+		},
+		&appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
+			Spec: appsv1.DeploymentSpec{Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "felis-api"}},
+				Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "api", Env: []corev1.EnvVar{
+					{Name: "FELIS_DEPLOY_SETTING", Value: p["deployment env literal"]},
+				}}}}}},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "survival-0", Namespace: "minecraft"},
+			Spec: corev1.PodSpec{
+				InitContainers: []corev1.Container{{Name: "prepare-data"}, {Name: "egress-gate"}},
+				Containers:     []corev1.Container{{Name: "server"}},
+			},
+			Status: corev1.PodStatus{Phase: corev1.PodRunning},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
+			Spec:       v1alpha1.MinecraftServerSpec{Env: []v1alpha1.EnvVar{{Name: "DISCORD_WEBHOOK", Value: p["MinecraftServer env literal"]}}},
+		},
+		&appsv1.StatefulSet{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-postgres", Namespace: "felis"},
+			Spec:       appsv1.StatefulSetSpec{Template: envTemplate("FELIS_STS_SETTING", p["statefulset env literal"])},
+		},
+		&batchv1.Job{
+			ObjectMeta: metav1.ObjectMeta{Name: "build-1", Namespace: "felis-build"},
+			Spec:       batchv1.JobSpec{Template: envTemplate("FELIS_JOB_SETTING", p["job env literal"])},
+		},
+		&batchv1.CronJob{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-reaper", Namespace: "felis"},
+			Spec:       batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: envTemplate("FELIS_CRON_SETTING", p["cronjob env literal"])}}},
+		},
+		&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}},
+	).Build()
+
+	secretLog := fmt.Sprintf("started with SERVICE_TOKEN=%s\nlogin password=%s ok\nAuthorization: Bearer %s\ndial postgres://felis:%s@db:5432/felis\n",
+		p["secrets.env SERVICE_TOKEN"], p["logged password"], p["logged bearer"], p["URL password"])
+	return &supportBundle{
+		host: "felis-test", now: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC), unitDir: unitDir,
+		w: w, cfg: cfg, cl: cl,
+		logs: func(_ context.Context, ns, pod, container string, previous bool) ([]byte, error) {
+			if container == "wait-db" {
+				return nil, errors.New("container \"wait-db\" is waiting to start; token=" + p["token in an error"])
+			}
+			return []byte(fmt.Sprintf("log of %s/%s/%s previous=%v\n%s", ns, pod, container, previous, secretLog)), nil
+		},
+		run: func(_ context.Context, name string, args ...string) ([]byte, error) {
+			switch name {
+			case "journalctl":
+				return []byte("journal of " + args[1] + "\nFELIS_OFFSITE_KEY=" + p["offsite.env FELIS_OFFSITE_KEY"] + "\nrelay " + p["smtp-password"] + " refused\n" +
+					"relay login with the cached " + p["watchdog state relay password"] + "\ndatabase auth failed for " + p["felis.host.toml DB password"] +
+					"\nservice token " + p["service token by its env ref"] + " rejected\nnode joined with " + p["k3s token secret part"] + "\n" +
+					"old copies sealed with " + p["commented-out offsite key"] + "\nretrying with password=" + p["journal quoting a password"] + "\n"), nil
+			case "systemctl", "df", "ip":
+				return []byte(name + " output\n"), nil
+			}
+			return nil, errors.New("unexpected " + name)
+		},
+		backups: func(context.Context) (map[string]time.Time, error) {
+			return nil, errors.New("connect: password=" + p["status quoting a password"])
+		},
+		doctor: func(_ context.Context, out io.Writer) {
+			fmt.Fprintf(out, "doctor report; the k3s token K10deadbeefcafe::server:%s leaked here\nprobe said password=%s\n", p["k3s token secret part"], p["doctor quoting a password"])
+		},
+		secrets: secretSources{
+			envFiles:   []string{filepath.Join(etc, "secrets.env"), filepath.Join(etc, "offsite.env"), filepath.Join(etc, "absent.env")},
+			valueFiles: []string{filepath.Join(etc, "smtp-password"), filepath.Join(etc, "uploads-s3-secret-key")},
+			propsFiles: []string{filepath.Join(etc, "felis-link.properties")},
+			tokenFiles: []string{filepath.Join(dir, "k3s", "token")},
+		},
+		logLines: 500, since: 48 * time.Hour,
+		meminfo: meminfo, osRelease: filepath.Join(dir, "os-release"), procVersion: filepath.Join(dir, "version"), stateDir: etc,
+	}
+}
+
+// envTemplate is a pod template whose one container sets name to a literal.
+func envTemplate(name, value string) corev1.PodTemplateSpec {
+	return corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Env: []corev1.EnvVar{{Name: name, Value: value}}}}}}
+}
+
+// readBundle is every file in the bundle at path, by its name inside the
+// bundle's directory, and each one's mode.
+func readBundle(t *testing.T, path string) (map[string]string, map[string]int64) {
+	t.Helper()
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr := tar.NewReader(gz)
+	files, modes := map[string]string{}, map[string]int64{}
+	prefix := strings.TrimSuffix(filepath.Base(path), ".tar.gz") + "/"
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		name, ok := strings.CutPrefix(hdr.Name, prefix)
+		if !ok {
+			t.Errorf("%s is outside the bundle's directory %s", hdr.Name, prefix)
+		}
+		raw, err := io.ReadAll(tr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		files[name], modes[name] = string(raw), hdr.Mode
+	}
+	return files, modes
+}
+
+func TestSupportBundle(t *testing.T) {
+	b := bundleHost(t)
+	out := filepath.Join(t.TempDir(), "support")
+	path, err := b.write(context.Background(), out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := filepath.Join(out, "felis-support-felis-test-20260927T120000Z.tar.gz"); path != want {
+		t.Errorf("path %s, want %s", path, want)
+	}
+	for p, want := range map[string]os.FileMode{out: 0o700 | os.ModeDir, path: 0o600} {
+		if st, err := os.Stat(p); err != nil || st.Mode() != want {
+			t.Errorf("%s: mode %v (err %v), want %v", p, st.Mode(), err, want)
+		}
+	}
+	if left, _ := filepath.Glob(filepath.Join(out, ".*partial")); len(left) != 0 {
+		t.Errorf("left behind %v", left)
+	}
+
+	files, modes := readBundle(t, path)
+	var names []string
+	for n := range files {
+		names = append(names, n)
+		if modes[n] != 0o600 {
+			t.Errorf("%s: mode %o in the archive, want 600", n, modes[n])
+		}
+	}
+	sort.Strings(names)
+	want := []string{
+		"MANIFEST.txt",
+		"cluster/felis-build/cronjobs.yaml", "cluster/felis-build/deployments.yaml", "cluster/felis-build/events.yaml", "cluster/felis-build/jobs.yaml",
+		"cluster/felis-build/networkpolicies.yaml", "cluster/felis-build/persistentvolumeclaims.yaml", "cluster/felis-build/pods.yaml",
+		"cluster/felis-build/services.yaml", "cluster/felis-build/statefulsets.yaml",
+		"cluster/felis/cronjobs.yaml", "cluster/felis/deployments.yaml", "cluster/felis/events.yaml", "cluster/felis/jobs.yaml",
+		"cluster/felis/networkpolicies.yaml", "cluster/felis/persistentvolumeclaims.yaml", "cluster/felis/pods.yaml",
+		"cluster/felis/services.yaml", "cluster/felis/statefulsets.yaml",
+		"cluster/minecraft/cronjobs.yaml", "cluster/minecraft/deployments.yaml", "cluster/minecraft/events.yaml", "cluster/minecraft/jobs.yaml",
+		"cluster/minecraft/networkpolicies.yaml", "cluster/minecraft/persistentvolumeclaims.yaml", "cluster/minecraft/pods.yaml",
+		"cluster/minecraft/services.yaml", "cluster/minecraft/statefulsets.yaml",
+		"cluster/minecraftservers.yaml", "cluster/nodes.yaml", "cluster/persistentvolumes.yaml", "cluster/pods-all-namespaces.txt",
+		"config.txt", "doctor.txt",
+		"host/addresses.txt", "host/df.txt", "host/etc-felis.txt", "host/meminfo.txt", "host/systemd-timers.txt", "host/systemd-units.txt",
+		"journal/felis-offsite.log", "journal/felis-watchdog.log", "journal/k3s.log",
+		"logs/felis/felis-api-7d9/api.log", "logs/felis/felis-api-7d9/api.previous.log",
+		"logs/minecraft/survival-0/egress-gate.log", "logs/minecraft/survival-0/prepare-data.log",
+		"status.txt", "version.txt",
+	}
+	if strings.Join(names, "\n") != strings.Join(want, "\n") {
+		t.Errorf("bundle holds\n  %s\nwant\n  %s", strings.Join(names, "\n  "), strings.Join(want, "\n  "))
+	}
+
+	for what, secret := range plantedSecrets {
+		for n, body := range files {
+			if strings.Contains(body, secret) {
+				t.Errorf("%s (%s) is in %s:\n%s", what, secret, n, body)
+			}
+		}
+	}
+
+	// What was taken out leaves what a reader needs around it.
+	for name, wants := range map[string][]string{
+		"logs/felis/felis-api-7d9/api.previous.log": {"log of felis/felis-api-7d9/api previous=true\n", "SERVICE_TOKEN=<redacted>\n", "login password=<redacted> ok\n", "Authorization: Bearer <redacted>\n", "postgres://felis:<redacted>@db:5432/felis\n"},
+		"journal/k3s.log": {"journal of k3s.service\n", "FELIS_OFFSITE_KEY=<redacted>\n", "relay <redacted> refused\n",
+			"relay login with the cached <redacted>\n", "database auth failed for <redacted>\n", "service token <redacted> rejected\n", "node joined with <redacted>\n"},
+		"cluster/felis/statefulsets.yaml": {"name: FELIS_STS_SETTING\n"},
+		"cluster/felis/cronjobs.yaml":     {"name: FELIS_CRON_SETTING\n"},
+		"cluster/felis-build/jobs.yaml":   {"name: FELIS_JOB_SETTING\n"},
+		"cluster/felis/pods.yaml":         {"name: FELIS_PLAIN_SETTING\n        value: <redacted>\n", "name: FELIS_SMTP_PASSWORD\n        valueFrom:\n          secretKeyRef:\n            key: password\n            name: felis-smtp\n", "felis.lolicon.best/kept: \"yes\"", "name: FELIS_INIT_SETTING\n", "name: FELIS_DEBUG_SETTING\n"},
+		"cluster/felis/deployments.yaml":  {"name: FELIS_DEPLOY_SETTING\n            value: <redacted>\n"},
+		"cluster/minecraftservers.yaml":   {"name: DISCORD_WEBHOOK\n      value: <redacted>\n"},
+		"config.txt":                      {fmt.Sprintf("%-34s %s\n", "server.root_domain", "games.example.org"), fmt.Sprintf("%-34s %s\n", "database.url (no password)", "postgres://[email protected]:1/felis")},
+		"doctor.txt":                      {"the k3s token <redacted> leaked here\nprobe said password=<redacted>\n"},
+		"status.txt":                      {"  world backups unknown: connect: password=<redacted>\n"},
+		"host/etc-felis.txt":              {"secrets.env\n", "smtp-password\n", "felis-link.properties\n"},
+		"MANIFEST.txt": {
+			"The game servers' own logs are left out",
+			"  journal/                 up to 500 lines per Felis unit and k3s, from the last 48h\n",
+			"  - 11 secret values, wherever they appear, read from:\n",
+			"secrets.env\n", "offsite.env\n", "smtp-password\n", "felis-link.properties\n", "token\n",
+			"logs/felis/felis-api-7d9/wait-db.log: container \"wait-db\" is waiting to start; token=<redacted>\n",
+			// Its own words about what is redacted come through whole.
+			"  - passwords in URLs, private keys, Bearer and Basic credentials\n",
+			"access_key=, private_key= or credentials= (and the same with a colon)\n",
+			"Read it through before you send it anywhere",
+		},
+	} {
+		for _, w := range wants {
+			if !strings.Contains(files[name], w) {
+				t.Errorf("%s lacks %q:\n%s", name, w, files[name])
+			}
+		}
+	}
+	for _, gone := range []string{"managedFields", "last-applied-configuration"} {
+		if strings.Contains(files["cluster/felis/pods.yaml"], gone) {
+			t.Errorf("pods.yaml keeps %s:\n%s", gone, files["cluster/felis/pods.yaml"])
+		}
+	}
+	if strings.Contains(files["MANIFEST.txt"], "absent.env") || strings.Contains(files["MANIFEST.txt"], "uploads-s3-secret-key") {
+		t.Errorf("MANIFEST names files this host does not have:\n%s", files["MANIFEST.txt"])
+	}
+}
+
+// -server-logs adds the game servers' own logs, and says so.
+func TestSupportBundleServerLogs(t *testing.T) {
+	b := bundleHost(t)
+	b.serverLogs = true
+	path, err := b.write(context.Background(), t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	files, _ := readBundle(t, path)
+	if _, ok := files["logs/minecraft/survival-0/server.log"]; !ok {
+		t.Error("no server.log with -server-logs")
+	}
+	if !strings.Contains(files["MANIFEST.txt"], "The game servers' own logs are in logs/ (-server-logs)") {
+		t.Errorf("MANIFEST does not say server logs are in:\n%s", files["MANIFEST.txt"])
+	}
+}
+
+// With the cluster and the configuration gone the bundle still holds what the
+// host shows, and MANIFEST says what it could not collect.
+func TestSupportBundleWithTheClusterDown(t *testing.T) {
+	b := bundleHost(t)
+	b.cl, b.clErr = nil, errors.New("connection refused")
+	b.cfg, b.cfgErr = nil, errors.New("felis.host.toml: no such file")
+	b.wErr = errors.New("felis-watchdog.service is not installed; read the watchdog's defaults")
+	path, err := b.write(context.Background(), t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	files, _ := readBundle(t, path)
+	for _, n := range []string{"status.txt", "doctor.txt", "host/etc-felis.txt", "journal/k3s.log"} {
+		if _, ok := files[n]; !ok {
+			t.Errorf("no %s", n)
+		}
+	}
+	for n := range files {
+		if strings.HasPrefix(n, "cluster/") || strings.HasPrefix(n, "logs/") || n == "config.txt" {
+			t.Errorf("%s with no cluster and no configuration", n)
+		}
+	}
+	if !strings.Contains(files["status.txt"], "felis status: the configuration did not load: felis.host.toml: no such file") {
+		t.Errorf("status.txt:\n%s", files["status.txt"])
+	}
+	if !strings.Contains(files["MANIFEST.txt"], "\nThe watchdog's settings: felis-watchdog.service is not installed; read the watchdog's defaults\n") ||
+		!strings.Contains(files["MANIFEST.txt"], "  - cluster: connection refused\n") {
+		t.Errorf("MANIFEST.txt:\n%s", files["MANIFEST.txt"])
+	}
+}
+
+func TestShortDuration(t *testing.T) {
+	for d, want := range map[time.Duration]string{
+		48 * time.Hour:             "48h",
+		90 * time.Minute:           "1h30m",
+		45 * time.Minute:           "45m",
+		30 * time.Second:           "30s",
+		time.Hour + 30*time.Second: "1h0m30s",
+	} {
+		if got := shortDuration(d); got != want {
+			t.Errorf("shortDuration(%v) = %q, want %q", d, got, want)
+		}
+	}
+}
+
+func TestScrubber(t *testing.T) {
+	s := &scrubber{}
+	for _, v := range []string{"known-secret-value", "known-secret-value-longer", "short", "known-secret-value"} {
+		s.add(v)
+	}
+	if got := strings.Join(s.vals, ","); got != "known-secret-value-longer,known-secret-value" {
+		t.Errorf("kept %q; want each value once, longest first, none under %d characters", s.vals, minScrubLen)
+	}
+	for in, want := range map[string]string{
+		"a known-secret-value-longer b":                                     "a <redacted> b",
+		"a known-secret-value b":                                            "a <redacted> b",
+		"password=abcd1234 next":                                            "password=<redacted> next",
+		`{"token": "abcd1234"}`:                                             `{"token": "<redacted>"}`,
+		"SMTP_PASSWORD: s3cr3t!x":                                           "SMTP_PASSWORD: <redacted>",
+		"x-api-key=zzzz9999&q=1":                                            "x-api-key=<redacted>&q=1",
+		"Authorization: Basic dXNlcjpwYXNzd29yZA==":                         "Authorization: Basic <redacted>",
+		"s3://AKIA:secretpart123@bucket/key":                                "s3://AKIA:<redacted>@bucket/key",
+		"https://user@host/path":                                            "https://user@host/path",
+		"-----BEGIN EC PRIVATE KEY-----\nMHc\n-----END EC PRIVATE KEY-----": "<redacted private key>",
+		"tokens: 3": "tokens: 3",
+		"read the relay password (keeping the cached one)": "read the relay password (keeping the cached one)",
+		`secrets "felis-smtp" not found`:                   `secrets "felis-smtp" not found`,
+	} {
+		if got := string(s.text([]byte(in))); got != want {
+			t.Errorf("text(%q) = %q, want %q", in, got, want)
+		}
+	}
+	// Structured files keep what only looks like a secret.
+	if got := string(s.values([]byte("secretName: felis-forwarding-secret and known-secret-value"))); got != "secretName: felis-forwarding-secret and <redacted>" {
+		t.Errorf("values() = %q", got)
+	}
+}

cmd/felis/systemservers.go

@@ -4,18 +4,18 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/naming"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -242,6 +242,15 @@ func buildSystemServerClient() (client.Client, error) {
 	if err := v1alpha1.AddToScheme(scheme); err != nil {
 		return nil, err
 	}
+	cfg, err := hostRESTConfig()
+	if err != nil {
+		return nil, err
+	}
+	return client.New(cfg, client.Options{Scheme: scheme})
+}
+
+// hostRESTConfig is the cluster connection buildSystemServerClient describes.
+func hostRESTConfig() (*rest.Config, error) {
 	cfg, err := ctrl.GetConfig()
 	if err != nil {
 		cfg, err = clientcmd.BuildConfigFromFlags("", hostBootstrapKubeconfigPath)
@@ -249,7 +258,7 @@ func buildSystemServerClient() (client.Client, error) {
 			return nil, fmt.Errorf("no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and %s): %w", hostBootstrapKubeconfigPath, err)
 		}
 	}
-	return client.New(cfg, client.Options{Scheme: scheme})
+	return cfg, nil
 }
 
 // systemServerOutcome records what ensureSystemServers did with one service so
@@ -392,21 +401,48 @@ var derivedSystemEnv = map[string]bool{
 // operator every run.
 func refreshDerivedEnv(ctx context.Context, cl client.Client, existing, desired *v1alpha1.MinecraftServer) (bool, error) {
 	want := derivedEnvWanted(desired)
-
-	changed := false
-	for i, e := range existing.Spec.Env {
-		if v, ok := want[e.Name]; ok && v != e.Value {
-			existing.Spec.Env[i].Value = v
-			changed = true
+	changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
+		changed := false
+		for i, e := range existing.Spec.Env {
+			if v, ok := want[e.Name]; ok && v != e.Value {
+				existing.Spec.Env[i].Value = v
+				changed = true
+			}
 		}
-	}
-	if !changed {
-		return false, nil
-	}
-	if err := cl.Update(ctx, existing); err != nil {
+		return changed
+	})
+	if err != nil {
 		return false, fmt.Errorf("refresh %s env: %w", existing.Name, err)
 	}
-	return true, nil
+	return changed, nil
+}
+
+// patchOnConflictRetry applies mutate to obj and sends only the difference, as a
+// merge patch that carries the resourceVersion obj was read at. The operator writes
+// status and felis-api patches spec.idle on these same objects, so a write can land
+// between setup's read and its patch: the apiserver then answers 409, and this
+// re-reads obj and runs mutate again on the fresh copy, up to retry.DefaultRetry's
+// five attempts. The pinned resourceVersion is what keeps a list field such as
+// spec.env safe — a merge patch replaces a list whole, and without the lock an
+// entry added concurrently would be dropped. mutate reports whether it changed
+// anything; nothing is sent when it did not. obj holds the stored object after.
+func patchOnConflictRetry(ctx context.Context, cl client.Client, obj client.Object, mutate func() bool) (bool, error) {
+	key := client.ObjectKeyFromObject(obj)
+	changed, reread := false, false
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		if reread {
+			if err := cl.Get(ctx, key, obj); err != nil {
+				return err
+			}
+		}
+		reread = true
+		base := obj.DeepCopyObject().(client.Object)
+		if changed = mutate(); !changed {
+			return nil
+		}
+		return cl.Patch(ctx, obj, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}))
+	})
+	return changed, err
 }
 
 // derivedEnvWanted maps the derived env keys of desired onto their values.
@@ -469,22 +505,25 @@ func convergeSystemServers(ctx context.Context, cl client.Client, namespace, log
 		}
 
 		var changes []string
-		if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
-			existing.Spec.Rcon = desired.Spec.Rcon
-			changes = append(changes, "spec.rcon")
-		}
-		if existing.Spec.Startup.HealthHTTPPort == 0 && desired.Spec.Startup.HealthHTTPPort != 0 {
-			existing.Spec.Startup.HealthHTTPPort = desired.Spec.Startup.HealthHTTPPort
-			changes = append(changes, "spec.startup.healthHTTPPort")
-		}
-		changes = append(changes, convergeDerivedEnv(&existing, desired)...)
-
-		if len(changes) == 0 {
-			outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
+		changed, err := patchOnConflictRetry(ctx, cl, &existing, func() bool {
+			changes = nil
+			if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
+				existing.Spec.Rcon = desired.Spec.Rcon
+				changes = append(changes, "spec.rcon")
+			}
+			if existing.Spec.Startup.HealthHTTPPort == 0 && desired.Spec.Startup.HealthHTTPPort != 0 {
+				existing.Spec.Startup.HealthHTTPPort = desired.Spec.Startup.HealthHTTPPort
+				changes = append(changes, "spec.startup.healthHTTPPort")
+			}
+			changes = append(changes, convergeDerivedEnv(&existing, desired)...)
+			return len(changes) > 0
+		})
+		if err != nil {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
 			continue
 		}
-		if err := cl.Update(ctx, &existing); err != nil {
-			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
+		if !changed {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
 			continue
 		}
 		outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, updated: true, changes: changes})
@@ -519,84 +558,51 @@ func convergeDerivedEnv(existing, desired *v1alpha1.MinecraftServer) []string {
 	return changes
 }
 
-// The login gate is a hard prerequisite of the Owner bind, so setup waits for it
-// rather than racing it. The ceiling covers a cold image pull on a fresh node;
-// the poll is fast enough that a warm start feels immediate.
-const (
-	loginGateReadyTimeout = 5 * time.Minute
-	loginGatePollInterval = 3 * time.Second
-)
-
-// awaitLoginGateReady blocks until the login system server reports status.ready.
-//
-// The Owner claims their seat by JOINING the game and running /link, so the gate
-// being up is not a nicety — it is the precondition for the very next thing setup
-// asks of the operator. progress is called on each phase change so the caller can
-// show movement during a cold image pull; it may be nil.
-func awaitLoginGateReady(ctx context.Context, cl client.Client, namespace string, timeout, poll time.Duration, progress func(v1alpha1.Phase)) error {
-	key := client.ObjectKey{Namespace: namespace, Name: naming.SystemLoginServer}
-	deadline := time.Now().Add(timeout)
-	last := v1alpha1.Phase("")
-	for {
-		var ms v1alpha1.MinecraftServer
-		switch err := cl.Get(ctx, key, &ms); {
-		case err == nil:
-			if ms.Status.Ready {
-				return nil
-			}
-			if ms.Status.Phase != last {
-				last = ms.Status.Phase
-				if progress != nil {
-					progress(last)
-				}
-			}
-			// The operator only marks Failed once its OWN startup deadline has already
-			// elapsed, so Failed is a settled verdict rather than a transient — sitting
-			// out the rest of our timeout on top of it would only hide the reason.
-			if ms.Status.Phase == v1alpha1.PhaseFailed {
-				return fmt.Errorf("the login gate failed to start: %s", readyConditionMessage(&ms))
-			}
-		case !apierrors.IsNotFound(err):
-			return err
-		}
-		if !time.Now().Before(deadline) {
-			return fmt.Errorf("timed out after %s waiting for the login gate to become ready (last phase: %s)", timeout, phaseOrPending(last))
-		}
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-time.After(poll):
-		}
+// provisionSecretReplicas copies the Secrets workload pods mount from the control
+// namespace into the namespaces those pods run in. The proxy's felis-service-token
+// is not among them: it lives in the control namespace and on the host, and a copy
+// anywhere else would open every game route to whoever reads that namespace.
+func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, minecraftNS, buildNS string) []systemServerOutcome {
+	return []systemServerOutcome{
+		// refresh=true for both caller tokens: the control namespace holds the
+		// current value and `felis rotate-token` replaces it there, so a replica
+		// that differs is stale and the login gate would be turned away with it.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.LimboTokenSecretName, naming.ServiceTokenSecretKey, "limbo-token", "minecraft ns", true),
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
+		// refresh=true: felis-config is the rendered config, not a credential. The
+		// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
+		// must update it when the control plane's render has moved on (a stale copy
+		// e.g. keeps an old database URL after a credential rotation).
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-config", "felis.toml", "config", "minecraft ns", true),
+		// The reaper's pre-reap warning emails authenticate with the same relay
+		// password felis-api uses; the reaper pod runs in the minecraft namespace,
+		// where a secretKeyRef resolves only against a local mirror. Skipped while
+		// the relay is not configured yet — the "configure email" screen refreshes
+		// both mirrors when it applies.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-smtp", "password", "smtp", "minecraft ns", false),
+		// The build namespace needs the build token: the build Job's fetch
+		// initContainer reads the submission context from the internal face, and
+		// that is all this token opens. Best effort — a deployment that only
+		// installs the control plane simply never builds a user submission.
+		ensureSecretReplica(ctx, cl, controlNS, buildNS,
+			naming.BuildTokenSecretName, naming.ServiceTokenSecretKey, "build-token", "felis-build ns", true),
 	}
 }
 
-// readyConditionMessage is the operator's own account of why the gate is not
-// ready — far more useful to an operator than "phase: Failed".
-func readyConditionMessage(ms *v1alpha1.MinecraftServer) string {
-	if c := meta.FindStatusCondition(ms.Status.Conditions, v1alpha1.ConditionReady); c != nil && c.Message != "" {
-		return c.Message
-	}
-	return "no Ready condition was reported"
-}
-
-// phaseOrPending names the empty phase, which means the operator has not
-// reconciled the server yet (commonly: the operator itself is not running).
-func phaseOrPending(p v1alpha1.Phase) string {
-	if p == "" {
-		return "not yet reconciled — is the felis operator running?"
-	}
-	return string(p)
-}
-
 // ensureSecretReplica copies one Secret from the control namespace into a workload
 // namespace (minecraft — or the build namespace, whose fetch initContainer reads the
-// context from the felis-api internal face with the same token) so a pod can mount it
+// context from the felis-api internal face with the build token) so a pod can mount it
 // via secretKeyRef. A secretKeyRef is namespace-local, but those workloads do not run
 // beside the control plane — so without this replica the secretKeyRef would dangle and
 // wedge the pod in CreateContainerConfigError.
 //
-// Three Secrets need it, for different reasons: the service token (the login limbo and
-// the build Pod's context fetch — both authenticate to the felis-api internal face),
+// Several Secrets need it, for different reasons: the caller tokens of the login
+// limbo and the build Pod's context fetch (both authenticate to the felis-api
+// internal face, each with its own token),
 // the Velocity modern-forwarding secret (every backend — it is how a backend knows
 // a login really came from the proxy, and so that the player's UUID is Mojang-verified
 // rather than offline-derived), and the SMTP relay password (the reaper's pre-reap
@@ -609,12 +615,14 @@ func phaseOrPending(p v1alpha1.Phase) string {
 // copies only Type and Data — never labels/annotations/ownerRefs — so the replica
 // carries no accidental GC owner or managed-by lineage.
 //
-// refreshExisting switches the felis-config mirror to refresh-in-place: that Secret is
-// a rendered config, never a hand-rotated credential, and the workload Jobs that mount
-// it (backup/restore/fileedit) plus the reaper silently misbehave on a stale copy —
-// e.g. after a database credential rotation the control plane moves on while every
-// backup Job keeps failing auth. Credential Secrets keep the never-overwrite rule so a
-// rotated value survives; to rotate those, delete the replica and re-run setup.
+// refreshExisting switches a replica to refresh-in-place from the control namespace.
+// The felis-config mirror uses it because that Secret is a rendered config and the
+// workload Jobs that mount it (backup/restore/fileedit) plus the reaper silently
+// misbehave on a stale copy — e.g. after a database credential rotation the control
+// plane moves on while every backup Job keeps failing auth. The caller tokens use it
+// because `felis rotate-token` replaces them in the control namespace, which makes a
+// differing replica stale by definition. The forwarding and SMTP Secrets keep the
+// never-overwrite rule.
 func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label, where string, refreshExisting bool) systemServerOutcome {
 	name := label + " (" + where + ")"
 	validate := func(secret *corev1.Secret, location, skipped string) systemServerOutcome {
@@ -639,16 +647,22 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 		if out := validate(&src, controlNamespace, ""); !out.available {
 			return out
 		}
-		if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
-			return validate(existing, minecraftNamespace, "already current")
-		}
-		if existing.Data == nil {
-			existing.Data = map[string][]byte{}
-		}
-		existing.Data[secretKey] = src.Data[secretKey]
-		if err := cl.Update(ctx, existing); err != nil {
+		changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
+			if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
+				return false
+			}
+			if existing.Data == nil {
+				existing.Data = map[string][]byte{}
+			}
+			existing.Data[secretKey] = src.Data[secretKey]
+			return true
+		})
+		if err != nil {
 			return systemServerOutcome{name: name, err: err}
 		}
+		if !changed {
+			return validate(existing, minecraftNamespace, "already current")
+		}
 		return systemServerOutcome{name: name, updated: true, available: true}
 	}
 	if controlNamespace == minecraftNamespace {
@@ -709,24 +723,3 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 	}
 	return systemServerOutcome{name: name, created: true, available: true}
 }
-
-func requiredProvisioningError(outcomes []systemServerOutcome) error {
-	required := map[string]struct{}{
-		"service-token (minecraft ns)":     {},
-		"forwarding-secret (minecraft ns)": {},
-		naming.SystemLoginServer:           {},
-	}
-	for _, o := range outcomes {
-		if o.err != nil {
-			return fmt.Errorf("%s: %w", o.name, o.err)
-		}
-		if _, ok := required[o.name]; ok && !o.available {
-			reason := o.skipped
-			if reason == "" {
-				reason = "object was not created"
-			}
-			return fmt.Errorf("%s unavailable: %s", o.name, reason)
-		}
-	}
-	return nil
-}

cmd/felis/systemservers_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"strings"
 	"testing"
-	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/naming"
@@ -142,21 +141,21 @@ func TestLoginSystemServerEnv(t *testing.T) {
 // namespace (create-if-absent), so the operator's secretKeyRef on the backend pod
 // resolves. It must not overwrite an existing replica, and must degrade gracefully
 // when the source is missing or the namespaces coincide. Exercised here with the
-// service token; setup runs it a second time for the Velocity forwarding secret.
+// Velocity forwarding secret, which setup replicates in this never-overwrite mode.
 func TestEnsureSecretReplica(t *testing.T) {
 	scheme := newSystemServerScheme(t)
 	ctx := context.Background()
 
 	srcSecret := func() *corev1.Secret {
 		return &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "felis"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "felis"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("s3cr3t")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("s3cr3t")},
 		}
 	}
 	replicate := func(cl client.Client, controlNS, mcNS string) systemServerOutcome {
 		return ensureSecretReplica(ctx, cl, controlNS, mcNS,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false)
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false)
 	}
 
 	t.Run("replicates when absent", func(t *testing.T) {
@@ -166,19 +165,19 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want created", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "s3cr3t" {
-			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ServiceTokenSecretKey])
+		if string(replica.Data[naming.ForwardingSecretKey]) != "s3cr3t" {
+			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ForwardingSecretKey])
 		}
 	})
 
 	t.Run("does not overwrite existing replica", func(t *testing.T) {
 		existing := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("rotated")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("rotated")},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), existing).Build()
 		out := replicate(cl, "felis", "minecraft")
@@ -186,11 +185,11 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want skipped (not clobbered)", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "rotated" {
-			t.Error("existing replica was overwritten — a rotated token must survive")
+		if string(replica.Data[naming.ForwardingSecretKey]) != "rotated" {
+			t.Error("existing replica was overwritten — a hand-set value must survive")
 		}
 	})
 
@@ -204,22 +203,22 @@ func TestEnsureSecretReplica(t *testing.T) {
 
 	t.Run("rejects a source with an empty required key", func(t *testing.T) {
 		bad := srcSecret()
-		bad.Data[naming.ServiceTokenSecretKey] = nil
+		bad.Data[naming.ForwardingSecretKey] = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
 
 	t.Run("rejects an existing replica with an empty required key", func(t *testing.T) {
 		bad := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Data:       map[string][]byte{},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable existing replica", out)
 		}
 	})
@@ -245,7 +244,7 @@ func TestEnsureSecretReplica(t *testing.T) {
 		bad.Data = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "felis")
-		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
@@ -332,90 +331,6 @@ func TestEnsureSecretReplicaRefresh(t *testing.T) {
 	})
 }
 
-func TestRequiredProvisioningError(t *testing.T) {
-	ready := []systemServerOutcome{
-		{name: "service-token (minecraft ns)", available: true},
-		{name: "forwarding-secret (minecraft ns)", available: true},
-		{name: naming.SystemLoginServer, available: true},
-		{name: naming.SystemLobbyServer, skipped: "image not configured"},
-	}
-	if err := requiredProvisioningError(ready); err != nil {
-		t.Fatalf("ready outcomes: %v", err)
-	}
-
-	missing := append([]systemServerOutcome(nil), ready...)
-	missing[1] = systemServerOutcome{name: "forwarding-secret (minecraft ns)", skipped: "source missing"}
-	if err := requiredProvisioningError(missing); err == nil || !strings.Contains(err.Error(), "forwarding-secret") {
-		t.Fatalf("missing forwarding secret = %v, want named error", err)
-	}
-
-	failed := append([]systemServerOutcome(nil), ready...)
-	failed[3] = systemServerOutcome{name: naming.SystemLobbyServer, err: context.DeadlineExceeded}
-	if err := requiredProvisioningError(failed); err == nil || !strings.Contains(err.Error(), naming.SystemLobbyServer) {
-		t.Fatalf("lobby create failure = %v, want immediate named error", err)
-	}
-}
-
-// The Owner binds by joining the game, so setup blocks on the login gate rather
-// than racing it. What matters is that each ending is distinguishable: Ready
-// proceeds, Failed reports the operator's own reason instead of waiting out the
-// clock, and a gate that never appears (no operator reconciling it) times out
-// saying so rather than dropping the operator on a bind screen that cannot work.
-func TestAwaitLoginGateReady(t *testing.T) {
-	scheme := newSystemServerScheme(t)
-	ctx := context.Background()
-
-	gate := func(mut func(*v1alpha1.MinecraftServer)) *v1alpha1.MinecraftServer {
-		ms := &v1alpha1.MinecraftServer{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.SystemLoginServer, Namespace: "minecraft"},
-		}
-		mut(ms)
-		return ms
-	}
-
-	t.Run("returns once the gate is ready", func(t *testing.T) {
-		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gate(func(ms *v1alpha1.MinecraftServer) {
-			ms.Status.Phase = v1alpha1.PhaseRunning
-			ms.Status.Ready = true
-		})).Build()
-		if err := awaitLoginGateReady(ctx, cl, "minecraft", time.Second, 10*time.Millisecond, nil); err != nil {
-			t.Fatalf("await: %v", err)
-		}
-	})
-
-	t.Run("fails fast on Failed, carrying the operator's reason", func(t *testing.T) {
-		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gate(func(ms *v1alpha1.MinecraftServer) {
-			ms.Status.Phase = v1alpha1.PhaseFailed
-			ms.Status.Conditions = []metav1.Condition{{
-				Type:               v1alpha1.ConditionReady,
-				Status:             metav1.ConditionFalse,
-				Reason:             "StartupTimeout",
-				Message:            "pod never became ready: ImagePullBackOff",
-				LastTransitionTime: metav1.Now(),
-			}}
-		})).Build()
-		start := time.Now()
-		err := awaitLoginGateReady(ctx, cl, "minecraft", time.Minute, 10*time.Millisecond, nil)
-		if err == nil {
-			t.Fatal("await: nil error, want failure")
-		}
-		if !strings.Contains(err.Error(), "ImagePullBackOff") {
-			t.Errorf("error = %q, want the operator's Ready-condition message", err)
-		}
-		if time.Since(start) > 5*time.Second {
-			t.Error("await sat out the full timeout on a settled Failed verdict")
-		}
-	})
-
-	t.Run("times out when nothing ever reconciles the gate", func(t *testing.T) {
-		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
-		err := awaitLoginGateReady(ctx, cl, "minecraft", 30*time.Millisecond, 10*time.Millisecond, nil)
-		if err == nil || !strings.Contains(err.Error(), "timed out") {
-			t.Fatalf("await = %v, want a timeout", err)
-		}
-	})
-}
-
 func newSystemServerScheme(t *testing.T) *runtime.Scheme {
 	t.Helper()
 	scheme := runtime.NewScheme()
@@ -662,3 +577,62 @@ func TestEnsureSystemServersRefreshesDerivedEnv(t *testing.T) {
 		}
 	})
 }
+
+// Setup's replicas carry each workload its own caller token and nothing more: the
+// login gate gets felis-limbo-token in the minecraft namespace, the build Jobs get
+// felis-build-token in the build namespace, a replica left stale by a rotation is
+// brought up to date, and the proxy's felis-service-token is copied nowhere.
+func TestProvisionSecretReplicasCarryCallerTokens(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	secret := func(ns, name, key, val string) *corev1.Secret {
+		return &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{key: []byte(val)},
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		secret("felis", "felis-service-token", "token", "proxy-tok"),
+		secret("felis", "felis-limbo-token", "token", "limbo-new"),
+		secret("felis", "felis-build-token", "token", "build-tok"),
+		secret("felis", "felis-ops-token", "token", "ops-tok"),
+		secret("felis", naming.ForwardingSecretName, naming.ForwardingSecretKey, "fwd"),
+		// What a rotation leaves behind before setup runs again.
+		secret("minecraft", "felis-limbo-token", "token", "limbo-old"),
+	).Build()
+
+	outcomes := provisionSecretReplicas(ctx, cl, "felis", "minecraft", "felis-build")
+	for _, o := range outcomes {
+		if o.err != nil {
+			t.Fatalf("%s: %v", o.name, o.err)
+		}
+	}
+
+	read := func(ns, name string) (string, bool) {
+		var s corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+			return "", false
+		}
+		return string(s.Data["token"]), true
+	}
+	if got, _ := read("minecraft", "felis-limbo-token"); got != "limbo-new" {
+		t.Errorf("minecraft/felis-limbo-token = %q, want the rotated limbo-new", got)
+	}
+	if got, _ := read("felis-build", "felis-build-token"); got != "build-tok" {
+		t.Errorf("felis-build/felis-build-token = %q, want build-tok", got)
+	}
+	for _, ns := range []string{"minecraft", "felis-build"} {
+		for _, name := range []string{"felis-service-token", "felis-ops-token"} {
+			if _, ok := read(ns, name); ok {
+				t.Errorf("%s/%s was replicated; only the control namespace holds it", ns, name)
+			}
+		}
+	}
+	if _, ok := read("minecraft", "felis-build-token"); ok {
+		t.Error("the build token was copied into the minecraft namespace")
+	}
+	if _, ok := read("felis-build", "felis-limbo-token"); ok {
+		t.Error("the limbo token was copied into the build namespace")
+	}
+}

cmd/felis/tui_alerts_test.go

@@ -0,0 +1,203 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/charmbracelet/lipgloss"
+
+	"felis.lolicon.best/internal/config"
+)
+
+// TestAlertRouteLines: the summary's alert rows say where the watchdog's
+// alerts go, and flag every route that reaches no one.
+func TestAlertRouteLines(t *testing.T) {
+	cases := []struct {
+		name     string
+		r        alertRoute
+		alerts   string
+		alertsOK bool
+		beat     string
+		beatOK   bool
+	}{
+		{
+			name:   "fresh install",
+			alerts: "only logged: no email relay (press e)",
+			beat:   "none: no outside check (troubleshooting.md §14)",
+		},
+		{
+			name:   "relay, no verified owner",
+			r:      alertRoute{relay: "smtp.example.com:587", heartbeat: "https://hc-ping.com/..."},
+			alerts: "only logged: no verified Owner email (panel → Account)",
+			beat:   "https://hc-ping.com/... every 2 minutes", beatOK: true,
+		},
+		{
+			name:   "relay, owners unreadable",
+			r:      alertRoute{relay: "smtp.example.com:587", lookupErr: errors.New("connection refused")},
+			alerts: "via smtp.example.com:587; could not read the Owner addresses",
+			beat:   "none: no outside check (troubleshooting.md §14)",
+		},
+		{
+			name:   "owners but no relay",
+			r:      alertRoute{recipients: []string{"[email protected]"}},
+			alerts: "only logged: no email relay (press e)",
+			beat:   "none: no outside check (troubleshooting.md §14)",
+		},
+		{
+			name:   "mailed",
+			r:      alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]", "[email protected]"}, heartbeatErr: errors.New("/etc/felis/watchdog-heartbeat-url: the heartbeat URL is not an http:// or https:// URL")},
+			alerts: "mailed to [email protected], [email protected] via smtp.example.com:587", alertsOK: true,
+			beat: "unreadable: /etc/felis/watchdog-heartbeat-url: the heartbeat URL is not an http:// or https:// URL",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if line, ok := c.r.alertsLine(); line != c.alerts || ok != c.alertsOK {
+				t.Errorf("alerts = %q, %v; want %q, %v", line, ok, c.alerts, c.alertsOK)
+			}
+			if line, ok := c.r.heartbeatLine(); line != c.beat || ok != c.beatOK {
+				t.Errorf("heartbeat = %q, %v; want %q, %v", line, ok, c.beat, c.beatOK)
+			}
+		})
+	}
+}
+
+// TestSummaryAlertRows: the rows show on the card, a route that needs action
+// marked so it reads without colour, and stay off a summary with no route.
+func TestSummaryAlertRows(t *testing.T) {
+	m := &summaryModel{ownerUsername: "owner", alerts: &alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]"}}}
+	v := m.View()
+	for _, want := range []string{"alerts    mailed to [email protected] via smtp.example.com:587", "heartbeat ⚠ none: no outside check"} {
+		if !strings.Contains(v, want) {
+			t.Errorf("summary lacks %q:\n%s", want, v)
+		}
+	}
+	if strings.Contains(v, "⚠ mailed") {
+		t.Errorf("a route that reaches the owners is marked:\n%s", v)
+	}
+	m.alerts = &alertRoute{heartbeat: "https://hc-ping.com/..."}
+	v = m.View()
+	for _, want := range []string{"alerts    ⚠ only logged: no email relay (press e)", "heartbeat https://hc-ping.com/... every 2 minutes"} {
+		if !strings.Contains(v, want) {
+			t.Errorf("summary lacks %q:\n%s", want, v)
+		}
+	}
+	m.alerts = nil
+	if v = m.View(); strings.Contains(v, "alerts") || strings.Contains(v, "heartbeat") {
+		t.Errorf("a summary with no route shows alert rows:\n%s", v)
+	}
+}
+
+// TestRootSummaryReadsAlertRoute: the summary and the re-run status screen
+// read the route each time they open, so a relay configured with e shows at
+// once.
+func TestRootSummaryReadsAlertRoute(t *testing.T) {
+	m := newTestRoot(false, consoleModeSetup, "")
+	reads := 0
+	route := alertRoute{}
+	m.alertRoute = func(context.Context) alertRoute {
+		reads++
+		return route
+	}
+	m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk"})
+	m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
+	sum, ok := m.screen.(*summaryModel)
+	if !ok || sum.alerts == nil || sum.alerts.relay != "" {
+		t.Fatalf("summary after storage: %T %+v", m.screen, sum)
+	}
+	route = alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]"}}
+	m = drive(t, m, smtpResultMsg{configured: true})
+	sum, ok = m.screen.(*summaryModel)
+	if !ok || sum.alerts == nil || sum.alerts.relay != "smtp.example.com:587" {
+		t.Fatalf("summary after configuring email: %T %+v", m.screen, sum)
+	}
+	if reads != 2 {
+		t.Errorf("route read %d times, want 2", reads)
+	}
+
+	status := newTestRoot(true, consoleModeSetup, "")
+	status.alertRoute = m.alertRoute
+	status.showStatus()
+	if sum, ok := status.screen.(*summaryModel); !ok || sum.alerts == nil || sum.alerts.relay != "smtp.example.com:587" {
+		t.Fatalf("status screen: %T %+v", status.screen, status.screen)
+	}
+}
+
+// TestHostAlertRoute reads the relay from the host config and the heartbeat
+// file, shows the heartbeat by its host only, and reports a database it cannot
+// reach.
+func TestHostAlertRoute(t *testing.T) {
+	dir := t.TempDir()
+	cfg := filepath.Join(dir, "felis.host.toml")
+	writeTestFile(t, cfg, testWatchdogConfig+testWatchdogSMTP, 0o600)
+	beat := filepath.Join(dir, "watchdog-heartbeat-url")
+	writeTestFile(t, beat, "https://hc-ping.com/check-key\n", 0o600)
+	dbURL := "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
+
+	r := hostAlertRoute(context.Background(), cfg, dbURL, beat)
+	if r.relay != "smtp.config.example:2525" {
+		t.Errorf("relay = %q", r.relay)
+	}
+	if r.heartbeat != "https://hc-ping.com/..." || r.heartbeatErr != nil {
+		t.Errorf("heartbeat = %q, %v", r.heartbeat, r.heartbeatErr)
+	}
+	if r.lookupErr == nil {
+		t.Errorf("an unreachable database reads as %v", r.recipients)
+	}
+
+	noRelay := filepath.Join(dir, "no-relay.toml")
+	writeTestFile(t, noRelay, testWatchdogConfig, 0o600)
+	writeTestFile(t, beat, "hc-ping.com/check-key\n", 0o600)
+	r = hostAlertRoute(context.Background(), noRelay, dbURL, beat)
+	if r.relay != "" {
+		t.Errorf("no [smtp]: relay = %q", r.relay)
+	}
+	if r.heartbeatErr == nil || strings.Contains(r.heartbeatErr.Error(), "check-key") {
+		t.Errorf("a bad heartbeat file: %v", r.heartbeatErr)
+	}
+	r = hostAlertRoute(context.Background(), noRelay, dbURL, filepath.Join(dir, "none"))
+	if r.heartbeat != "" || r.heartbeatErr != nil {
+		t.Errorf("no heartbeat file: %q, %v", r.heartbeat, r.heartbeatErr)
+	}
+}
+
+// TestSummaryWithAlertsFitsTerminal: the two rows keep the summary inside the
+// terminal, with the longest route lines.
+func TestSummaryWithAlertsFitsTerminal(t *testing.T) {
+	for _, w := range []int{60, 80, 90} {
+		for _, h := range []int{24, 30, 45} {
+			m := newTestRoot(false, consoleModeSetup, "")
+			m.alertRoute = func(context.Context) alertRoute {
+				return alertRoute{relay: "smtp.example.com:587", lookupErr: errors.New("dial tcp 127.0.0.1:5432: connect: connection refused")}
+			}
+			m = drive(t, m, tea.WindowSizeMsg{Width: w, Height: h})
+			m = drive(t, m, preflightDoneMsg{})
+			m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
+			m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.example.com"})
+			m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"})
+			if _, ok := m.screen.(*summaryModel); !ok {
+				t.Fatalf("screen = %T, want the summary", m.screen)
+			}
+			if got := lipgloss.Height(m.View()); got > h {
+				t.Errorf("terminal %dx%d: summary with alert rows = %d rows (exceeds height)", w, h, got)
+			}
+		}
+	}
+}
+
+// TestConsoleRootReadsHostAlertRoute: the console the host runs gives its
+// summary this host's route, read against its database.
+func TestConsoleRootReadsHostAlertRoute(t *testing.T) {
+	db := config.DatabaseConfig{URL: "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"}
+	rm := newConsoleRoot(context.Background(), &fakeOwnerStore{}, db, "felis.example.com", "admin.felis.example.com", "panel.felis.example.com", "", "minecraft", "root", false, consoleModeSetup, recoveryConfig{})
+	if rm.alertRoute == nil {
+		t.Fatal("the console's summary reads no alert route")
+	}
+	if r := rm.alertRoute(context.Background()); r.lookupErr == nil {
+		t.Errorf("the route did not query the console's database: %+v", r)
+	}
+}