style: unify panel cards and form controls

Manage Yggdrasil providers from the panel using durable platform settings, protected identity namespaces and atomic revisions. Apply changes to subsequent logins and profile lookups without restarting. Return operator-host logouts to the login method selection page.

.dockerignore

@@ -31,3 +31,14 @@ Dockerfile
 *.key
 felis
 felis.exe
+
+# macOS materializes extended attributes as ._<name> sidecars (BSD tar uploads,
+# Finder copies, network volumes) and leaves .DS_Store behind. Neither is
+# source, and one is actively harmful: a ._*.sql beside the migrations is
+# //go:embed-ed into the binary and makes every `felis migrate` fail
+# ("non-numeric version") — observed live on a Mac-staged tree. Same exposure
+# for any tree the other //go:embed patterns walk (deploy/, plugins/).
+._*
+**/._*
+.DS_Store
+**/.DS_Store

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# The workflows pin every action to a commit SHA, and every Dockerfile pins its base images
+# by digest. This keeps those pins moving: Dependabot reads the "# vX.Y.Z" comment next to
+# each action SHA, and the tag in front of each image digest, and opens a PR that bumps both
+# together.
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directories:
+      - /
+      - /deploy/limbo
+      - /deploy/lobby
+      - /deploy/paper
+    schedule:
+      interval: weekly
+    # A new major is a runtime change (Paper 26.x needs Java 25, Limbo's jar is Java 21
+    # bytecode), so only digests and minors are proposed; majors move by hand.
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/workflows/ci.yml

@@ -9,15 +9,23 @@
 # The push trigger is limited to main rather than every branch, for the reason release.yml is
 # not repeated here: a branch with an open PR would otherwise run the whole suite twice per
 # push, once for refs/heads/<branch> and once for refs/pull/N/merge. Those are different
-# concurrency groups, so neither cancels the other, and this repository is private and billed
-# for both. A branch with no PR open yet is the one case that loses coverage, and opening the
+# concurrency groups, so neither cancels the other, and both would occupy runners for the
+# same commit. A branch with no PR open yet is the one case that loses coverage, and opening the
 # PR is what asks for the answer.
 name: ci
 
+#
+# release.yml calls this workflow (workflow_call) before it builds anything, so a tag passes
+# exactly these gates and there is one list of them.
+#
+# workflow_dispatch reruns the suite on a commit whose push produced no run, such as one
+# pushed while Actions was unavailable.
 on:
   push:
     branches: [main]
   pull_request:
+  workflow_call:
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -30,9 +38,9 @@ jobs:
   go:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
           go-version-file: go.mod
 
@@ -43,12 +51,70 @@ jobs:
             echo "gofmt needed on:"; echo "$unformatted"; exit 1
           fi
       - run: go vet ./...
-      - run: go test ./...
+      # -race: felis-api and the operator are mostly goroutines (watchers, the
+      # registry pruner, the backup scheduler, the rate limiters).
+      - run: go test -race ./...
+      # The version is pinned here and bumped by hand; Dependabot does not read `go run`.
+      - name: staticcheck
+        run: go run honnef.co/go/tools/cmd/[email protected] ./...
+
+  # Separate from the go job so a newly published advisory reads as what it is. govulncheck
+  # exits non-zero only for vulnerable code this module can actually reach, standard
+  # library included: setup-go installs the newest patch of go.mod's Go line, so a finding
+  # there means the Dockerfile's golang digest (which ships the release) needs a bump too.
+  vuln:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go run golang.org/x/vuln/cmd/[email protected] ./...
+
+  # The business stores' SQL against a real PostgreSQL (internal/pgint): the unit suites run
+  # on fakes, and PGRepo drifted from them three times while those stayed green. 13 is the
+  # oldest server a supported distribution installs (EL9), 18 the newest (Arch).
+  # `felis db backup` and `restore` run there too, with the tools inside the service
+  # container, as production runs them inside felis-postgres: the runner's own client is one
+  # major version, and pg_dump refuses a newer server.
+  pgint:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        postgres: ['13', '18']
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres }}
+        env:
+          POSTGRES_USER: felis
+          POSTGRES_PASSWORD: pgint
+          POSTGRES_DB: felis_pgint
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U felis -d felis_pgint"
+          --health-interval 2s
+          --health-timeout 5s
+          --health-retries 30
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go test -race -tags pgint -count=1 ./internal/pgint/
+        env:
+          FELIS_TEST_PG_URL: postgres://felis:pgint@localhost:5432/felis_pgint?sslmode=disable
+          FELIS_TEST_PG_EXEC: docker exec -i ${{ job.services.postgres.id }}
 
   shell:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # bootstrap.sh is the only thing that ever runs on a fresh host, and nothing here can
       # run it — it wants root, a package manager and k3s. Syntax plus the extracted-block
@@ -66,12 +132,50 @@ jobs:
             esac
           done
 
+      # A pinned release rather than the runner image's copy, so a runner update cannot
+      # change what fails. Warnings and errors fail the job; style notes (info) do not.
+      - name: shellcheck
+        run: |
+          curl -fsSL -o shellcheck.tar.xz \
+            https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz
+          echo "8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198  shellcheck.tar.xz" | sha256sum -c
+          tar -xJf shellcheck.tar.xz
+          ./shellcheck-v0.11.0/shellcheck -S warning $(git ls-files '*.sh')
+
+      # An exit status is 8 bits, so `exit "$fails"` reads 256 failures as a pass. The suites
+      # exit 1 on any failure; this keeps the next one from carrying its count out.
+      - name: No script exits with its failure count
+        run: |
+          if git grep -nE 'exit +"?\$\{?[a-z_]*fail[a-z_]*\}?"?[[:space:]]*$' -- '*.sh'; then exit 1; fi
+
       - run: sh deploy/bootstrap_test.sh
+      - run: sh deploy/uninstall_test.sh
+      - run: bash deploy/e2e_release_test.sh
+      - run: bash deploy/e2e_upstream_test.sh
+
+  # The shipped alert rules (deploy/alerts): promtool parses them and runs their unit tests,
+  # which pin when each alert fires and that it stays quiet before. internal/metrics'
+  # alerts_test.go pins what promtool cannot see from there: the PrometheusRule twin and the
+  # metric names the rules read.
+  alerts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # A pinned release, like shellcheck's, so a new Prometheus cannot change what fails.
+      - name: promtool
+        run: |
+          curl -fsSL -o prometheus.tar.gz \
+            https://github.com/prometheus/prometheus/releases/download/v3.15.0/prometheus-3.15.0.linux-amd64.tar.gz
+          echo "2a542df32eac02ee17b9d844fb2aa1de00dafa5476579ba8a3ba862e9d572ea0  prometheus.tar.gz" | sha256sum -c
+          tar -xzf prometheus.tar.gz --strip-components=1 prometheus-3.15.0.linux-amd64/promtool
+          ./promtool check rules deploy/alerts/felis-alerts.yaml
+          ./promtool test rules deploy/alerts/felis-alerts_test.yml
 
   panel:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # The Dockerfile's `FROM node:<major>` is the only place the panel's Node version is
       # declared — there is no .nvmrc and no engines field. Reading it here rather than
@@ -84,7 +188,7 @@ jobs:
           [ -n "$version" ] || { echo "Dockerfile has no 'FROM ... node:<major>' line"; exit 1; }
           echo "version=${version}" >> "$GITHUB_OUTPUT"
 
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: ${{ steps.node.outputs.version }}
           cache: npm
@@ -96,5 +200,71 @@ jobs:
       - run: npm test
         working-directory: panel
 
+      # `tsc -b`: tsconfig.json is a solution file (files: [] plus references), so a plain
+      # `tsc --noEmit` checked nothing and passed with type errors in the tree.
       - run: npm run typecheck
         working-directory: panel
+
+      # Rules of hooks and effect dependency lists, with --max-warnings 0.
+      - run: npm run lint
+        working-directory: panel
+
+      # openapi.gen.ts is generated from docs/openapi.yaml and checked in, so the
+      # compile-time parity in src/lib/types.parity.ts needs no generator in the build;
+      # a schema edit that was not regenerated fails here.
+      - name: openapi.gen.ts matches docs/openapi.yaml
+        working-directory: panel
+        run: |
+          npm run gen:api
+          git diff --exit-code -- src/lib/openapi.gen.ts
+
+      # Browser smoke over the mock-mode dev server, in the runner's installed Chrome
+      # (playwright.config.ts sets channel: chrome, so nothing is downloaded).
+      - run: npm run test:e2e
+        working-directory: panel
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: panel-playwright-report
+          path: |
+            panel/playwright-report
+            panel/test-results
+          retention-days: 7
+
+  plugins:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The other jobs never touch the Java layer: the plugin jars were only ever
+      # compiled by bootstrap on a live host, and the test mains under plugins/*/test
+      # were run by hand. JDK 25 is what the plugin build image runs (paper-api 26.x
+      # needs it); each module's wrapper brings the Gradle the image pins.
+      - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
+        with:
+          distribution: temurin
+          java-version: '25'
+
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
+
+      - run: bash plugins/test.sh
+
+  mods:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The three loader mods (Minecraft 1.20.1 / 1.20.4, Java-17 lines) compile
+      # through their vendored Gradle wrappers, which fetch their own Gradle. Until
+      # this job nothing ever built them: no install path touches them, and their
+      # gradlew scripts were committed without the exec bit, so the README's
+      # one-liners failed on a fresh clone.
+      - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
+
+      - run: bash plugins/test-mods.sh

.github/workflows/e2e.yml

@@ -0,0 +1,343 @@
+# deploy/bootstrap.sh end to end on a fresh Ubuntu 24.04 x86_64 runner: the paths every
+# host goes through, run for real instead of by hand on a VM.
+#
+#   artifacts  this commit's release assets, built by deploy/build-release-artifacts.sh the
+#              way release.yml builds a tag's (amd64 only, the runners' architecture)
+#   install    a full install from those assets, the way a release installs, on a host with
+#              ufw enabled, then the same assets again (a rerun must converge without
+#              restarting what did not change, importing or uploading an image again, or
+#              touching Docker)
+#   readme     the README's one-line install as a new host runs it today: this commit's
+#              installer on its default channel, which installs the newest published
+#              release's binary, images and plugin from that release's assets; skipped until
+#              a release exists
+#   upgrade    the newest published release through its own installer and its own assets,
+#              rows of every kind seeded into its database, then this commit's assets on
+#              top of it; skipped until a release exists
+#   source     a full install built on the host from this checkout (FELIS_SKIP_FETCH):
+#              the fallback a release without assets, or an unsupported one, takes. It builds
+#              everything with Docker, so it runs by hand and weekly only
+#
+# This runs on pushes that touch what gets installed, by hand, and weekly (a moving
+# upstream: apt mirrors, k3s's install script and release assets, Adoptium).
+# deploy/e2e_check.sh holds the assertions, deploy/e2e_seed.sh the upgrade's seed and its check.
+#
+# An installer that stops on an upstream download refused or dropped (a GitHub 403, a 5xx, a
+# timeout) ends its job green, with a warning on the run: each install step hands a failed
+# run's log to deploy/e2e_upstream.sh, which sets E2E_UPSTREAM_SKIP for the job's later
+# steps. Every other installer failure, a 404 included, fails the job.
+name: e2e
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'deploy/**'
+      - 'cmd/**'
+      - 'internal/**'
+      - 'panel/**'
+      - 'plugins/**'
+      - 'Dockerfile'
+      - 'go.mod'
+      - 'go.sum'
+      - '.github/workflows/e2e.yml'
+  workflow_dispatch:
+  schedule:
+    - cron: '23 4 * * 1'
+
+permissions:
+  contents: read
+
+# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` carries the
+# installer's status to deploy/e2e_upstream.sh; the default shell reports tee's status.
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  artifacts:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0 # the newest tag is the version's base, as on the dev channel
+
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      # Stamped the way bootstrap stamps a build of main: "<newest tag>+g<sha>".
+      - name: Build the release assets
+        run: |
+          base="$(git describe --tags --abbrev=0 --match 'v*' 2>/dev/null || echo v0.0.0)"
+          FELIS_RELEASE_ARCHES=amd64 deploy/build-release-artifacts.sh "${base}+g$(git rev-parse --short HEAD)" dist
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: e2e-assets
+          path: dist/
+          if-no-files-found: error
+          retention-days: 1
+
+  install:
+    needs: artifacts
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The images, k3s and the JRE need more room than a stock runner leaves free.
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: e2e-assets
+          path: dist
+
+      # The runner has Docker preinstalled, so its absence proves nothing: the log shows
+      # whether bootstrap reached for it. From FELIS_ARTIFACT_DIR every image and the plugin
+      # come out of the assets, so it must not have. (`! grep` is spelled `if grep ... exit 1`
+      # below because bash -e ignores a failing `!` command.)
+      # ufw, enabled on many Ubuntu hosts, drops every inbound packet no rule admits, the
+      # pods' traffic to the API server among them; the runner ships it disabled. The
+      # runner's own traffic is outbound, which ufw lets through.
+      - name: Enable ufw
+        run: sudo ufw --force enable
+
+      - name: Install
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log || bash deploy/e2e_upstream.sh install.log $?
+
+      - name: Check the install
+        if: env.E2E_UPSTREAM_SKIP != '1'
+        run: |
+          sudo bash deploy/e2e_check.sh install
+          for tag in felis-k3s-pods felis-k3s-services felis-panel felis-proxy; do
+            sudo ufw status | grep -qE "# ${tag} *$"
+          done
+          if grep -E 'docker already installed|installing docker|docker running' install.log; then exit 1; fi
+          for role in felis limbo lobby paper; do
+            grep -q "felis/${role}:[^ ]* is the release's" install.log
+          done
+          grep -q "docker.io/library/registry@sha256:[0-9a-f]* is the release's" install.log
+          grep -q "docker.io/library/postgres@sha256:[0-9a-f]* is the release's" install.log
+          grep -q "felis-velocity.jar is the release's" install.log
+
+      - name: Rerun the same assets
+        if: env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log || bash deploy/e2e_upstream.sh rerun.log $?
+
+      # containerd and the registry already hold every image under the digest the listing
+      # names, so nothing is imported or uploaded twice.
+      - name: Check the rerun
+        if: env.E2E_UPSTREAM_SKIP != '1'
+        run: |
+          sudo bash deploy/e2e_check.sh rerun
+          grep -q 'felis-velocity unchanged; left running' rerun.log
+          if grep -E 'docker already installed|installing docker|docker running' rerun.log; then exit 1; fi
+          if grep -E 'importing felis-image-|mirroring .* into the internal registry' rerun.log; then exit 1; fi
+          grep -q 'is already in the internal registry' rerun.log
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          k() { sudo -E /usr/local/bin/k3s kubectl "$@"; }
+          k get pods -A -o wide || true
+          k get events -A --sort-by=.lastTimestamp | tail -n 60 || true
+          for p in $(k get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" {print $1 "/" $2}'); do
+            k -n "${p%%/*}" describe pod "${p#*/}" | tail -n 40 || true
+            k -n "${p%%/*}" logs "${p#*/}" --all-containers --tail=60 || true
+          done
+          k -n felis logs deploy/felis-postgres --tail=60 || true
+          sudo journalctl -u k3s -u felis-velocity --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-install-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  # The README's command on a fresh host: this commit's installer, as main serves it, on its
+  # default channel with nothing pinned. It resolves the newest release and installs that
+  # release's binary, images and plugin from its assets, each checked against its
+  # SHA256SUMS. The workflow's token is the only thing added: it lifts GitHub's limit of 60 API
+  # calls an hour for an address without one. FELIS_INSTALL_MODE picks the mode the setup
+  # console would ask for.
+  readme:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - name: Find the newest release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: bash deploy/e2e_release.sh find
+
+      - name: Install as the README does
+        if: steps.release.outputs.tag != ''
+        env:
+          TOKEN: ${{ github.token }}
+        run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log || bash deploy/e2e_upstream.sh readme.log $?
+
+      # The binary is the release's, so the phase is `release`: its database backup and
+      # timers are the release's to have or lack.
+      - name: Check the install
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          BINARY: ${{ steps.release.outputs.binary }}
+          SUMS: ${{ steps.release.outputs.sums }}
+        run: |
+          sudo bash deploy/e2e_check.sh release
+          sudo /usr/local/bin/felis version | grep -qx "felis ${TAG}"
+          bash deploy/e2e_release.sh check-readme readme.log
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
+          sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-readme-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  upgrade:
+    needs: artifacts
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: e2e-assets
+          path: dist
+
+      - name: Find the newest release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: bash deploy/e2e_release.sh find
+
+      # The release's own installer on its default channel, fetching the release's own
+      # binary: what a host that installed that release is running today. FELIS_REF would
+      # build the tag from source instead, a path no host takes by default. FELIS_RELEASE pins
+      # the tag found above; an installer older than FELIS_RELEASE ignores it and resolves
+      # the newest release, the same tag.
+      - name: Install the newest release
+        if: steps.release.outputs.tag != ''
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          TOKEN: ${{ github.token }}
+        run: |
+          git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh
+          sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log || bash deploy/e2e_upstream.sh release.log $?
+
+      - name: Check the release install
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          BINARY: ${{ steps.release.outputs.binary }}
+        run: |
+          sudo bash deploy/e2e_check.sh release
+          bash deploy/e2e_release.sh check-own release.log
+
+      # A fresh install's database holds only what its migrations wrote: the seed gives the
+      # upgrade's move and its pending migrations existing rows to carry.
+      - name: Seed the release's database
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo bash deploy/e2e_seed.sh seed
+
+      - name: Upgrade to this commit
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log || bash deploy/e2e_upstream.sh upgrade.log $?
+
+      # Both checks run and report: the seeded rows go last, after the restore drill has
+      # also put them back from a bundle.
+      - name: Check the upgrade
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        run: |
+          rc=0
+          sudo bash deploy/e2e_check.sh upgrade || rc=1
+          sudo bash deploy/e2e_seed.sh check || rc=1
+          exit "$rc"
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
+          # The release ran the database on the host; the upgrade moves it into k3s.
+          sudo journalctl -u k3s -u felis-velocity -u postgresql --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-upgrade-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  source:
+    if: github.event_name != 'push'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      # FELIS_SKIP_FETCH builds whatever sits in /opt/felis/src; the .git directory is what
+      # stamps the build. Root owns it so git, run as root by the installer, does not refuse
+      # it as dubious.
+      - name: Stage this commit as the installer's source
+        run: |
+          sudo mkdir -p /opt/felis
+          sudo cp -a "$GITHUB_WORKSPACE" /opt/felis/src
+          sudo chown -R root:root /opt/felis/src
+
+      - name: Install
+        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log || bash deploy/e2e_upstream.sh source.log $?
+
+      - name: Check the install
+        if: env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo bash deploy/e2e_check.sh install
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-source-logs
+          path: '*.log'
+          if-no-files-found: ignore

.github/workflows/release.yml

@@ -2,14 +2,15 @@
 #
 # Two things depend on this job. /repos/{repo}/releases/latest must ANSWER — that endpoint is
 # what `felis update` polls (internal/updater/github.go) and what deploy/bootstrap.sh's default
-# "release" channel resolves its ref from. And the binaries below are what that channel then
-# INSTALLS: bootstrap downloads felis-linux-<arch> instead of compiling on the target host, so
-# these are the shipped artifact, not a convenience.
+# "release" channel resolves its ref from. And the assets below are what that channel then
+# INSTALLS: bootstrap downloads the felis binary, the image tars, their listing and the Velocity
+# plugin instead of building anything on the target host, so these are the shipped artifact,
+# not a convenience. A host installing a release needs neither Docker, Gradle, Go nor Docker Hub.
 #
-# The asset NAME is a contract with deploy/bootstrap.sh (download_release_binary builds
-# "felis-linux-${arch}"). It is deliberately a plain literal on both sides: a GitHub Actions
-# YAML and a go:embed'ed shell script have no honest way to share a constant, and the failure
-# mode is benign — bootstrap warns and falls back to a source build of the same tag.
+# deploy/build-release-artifacts.sh builds every asset in one run and documents each name; the
+# names are a contract with deploy/bootstrap.sh. They are plain literals on both sides: a YAML
+# workflow and a go:embed'ed shell script have no honest way to share a constant, and the
+# failure mode is benign — bootstrap warns and falls back to building on the host.
 #
 # The binary is built through the repo Dockerfile rather than a plain `go build`.
 # internal/panel/static holds a tracked PLACEHOLDER index.html so the //go:embed
@@ -17,6 +18,15 @@
 # quietly ships a release whose panel is that placeholder. The Dockerfile runs the npm
 # build first, and is the same recipe bootstrap uses, so there is one way to build felis
 # rather than two that can drift.
+#
+# SHA256SUMS is a contract with bootstrap too: it refuses any asset whose hash is not listed
+# there, BEFORE it runs or imports it.
+#
+# The write token never meets the test suite: `gates` (ci.yml) and `build` run the tests,
+# Gradle and the Docker builds (each of which executes third-party code) with a read-only
+# token, and `build` hands the assets over as a workflow artifact; `publish` holds contents:write and runs only
+# pinned actions and gh. Every action is pinned to a commit SHA (the tag in the trailing
+# comment is for humans); .github/dependabot.yml proposes the bumps.
 name: release
 
 on:
@@ -24,59 +34,99 @@ on:
     tags: ['v*']
 
 permissions:
-  contents: write # gh release create/upload
+  contents: read
 
 jobs:
-  release:
+  # A tag that ships red is worse than a tag that fails to ship. These are ci.yml's gates,
+  # called rather than copied: Go (race, vet, staticcheck), govulncheck, the PostgreSQL
+  # contract suite, shellcheck and the bootstrap tests, promtool on the alert rules, the
+  # panel, and the Java layer the binary EMBEDS (bootstrap_asset.go ships the plugin
+  # sources, so a tag whose plugins do not compile turns every install of that release into
+  # a failed bootstrap).
+  gates:
+    uses: ./.github/workflows/ci.yml
+
+  build:
+    needs: gates
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-
-      # A tag that ships red is worse than a tag that fails to ship.
-      - run: go vet ./...
-      - run: go test ./...
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # Both architectures, because bootstrap's default release channel DOWNLOADS these
-      # rather than compiling on the target host — an arm64 host with no asset silently
-      # falls back to a slow source build. Neither stage is emulated: the Dockerfile pins
-      # both build stages to $BUILDPLATFORM and the Go stage cross-compiles via TARGETARCH,
-      # so the second architecture costs about a minute.
-      - uses: docker/setup-buildx-action@v3
+      # rather than building on the target host — an arm64 host with no asset falls back to
+      # a slow on-host build. The felis binary and the plugin jars cross-compile on the
+      # runner (their build stages are pinned to $BUILDPLATFORM); the game images' runtime
+      # stages run apt-get for the target platform, which for arm64 takes QEMU.
+      - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
 
-      - name: Build the stamped binaries
-        run: |
-          docker buildx build --platform linux/amd64,linux/arm64 \
-            --build-arg FELIS_VERSION="${GITHUB_REF_NAME}" \
-            --output type=local,dest=out .
-          mv out/linux_amd64/usr/local/bin/felis ./felis-linux-amd64
-          mv out/linux_arm64/usr/local/bin/felis ./felis-linux-arm64
-          chmod +x ./felis-linux-amd64 ./felis-linux-arm64
+      # Two architectures of five images plus BuildKit's cache outgrow the runner's free disk
+      # with its preinstalled SDKs in place; none of them is used here.
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
 
-      # The stamp is the whole point and it fails silently: an unstamped binary reports
-      # "dev", which `felis update` refuses to compare, disabling update reporting for
-      # every install built from it. Assert it end to end instead of trusting the ARG
-      # reached the linker.
-      - name: Verify the version stamp
-        run: |
-          got="$(./felis-linux-amd64 version | head -1)"
-          echo "reported: ${got}"
-          [ "$got" = "felis ${GITHUB_REF_NAME}" ] \
-            || { echo "expected 'felis ${GITHUB_REF_NAME}' — the -X main.version stamp did not reach the binary"; exit 1; }
-          # The arm64 binary is checked by ELF machine type, NOT by running it. Runners have
-          # binfmt/QEMU registered, so `./felis-linux-arm64 version` would happily succeed on
-          # an amd64 binary misnamed arm64 — which is exactly the failure the Dockerfile's
-          # ${TARGETARCH:-$(go env GOARCH)} fallback produces if buildx did not take. Both
-          # binaries come out of one RUN with one -ldflags string, so the stamp is checked once.
-          file ./felis-linux-arm64
-          file ./felis-linux-arm64 | grep -q 'ARM aarch64' \
-            || { echo "felis-linux-arm64 is not an arm64 ELF — TARGETARCH did not reach the go build"; exit 1; }
+      # The script checks what the old inline steps did: the host binary reports exactly
+      # "felis <tag>" (unstamped, `felis update` refuses to compare), and the other one is an
+      # ELF of its architecture, read with file(1) — binfmt would run a misnamed binary happily.
+      - name: Build the release assets
+        run: deploy/build-release-artifacts.sh "${GITHUB_REF_NAME}" dist
 
-      # --verify-tag refuses to invent a release for a tag that is not pushed. The upload
-      # fallback makes a re-run converge rather than failing on an existing release.
+      # A CycloneDX SBOM per binary: the Go modules (and versions) linked into it, read
+      # from the build info the linker embeds. Written after SHA256SUMS, so beside it in the
+      # release but outside it: that lists what bootstrap installs. Straight into dist/,
+      # because the action does not create a missing output directory.
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: dist/felis-linux-amd64
+          format: cyclonedx-json
+          output-file: dist/felis-linux-amd64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: dist/felis-linux-arm64
+          format: cyclonedx-json
+          output-file: dist/felis-linux-arm64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: release-assets
+          path: dist/
+          if-no-files-found: error
+          retention-days: 7
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # gh release create/upload
+      id-token: write # the Sigstore certificate behind the provenance attestation
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: release-assets
+
+      # The artifact store sits between the two jobs, so check the handover too.
+      - run: sha256sum -c SHA256SUMS
+
+      # Signed SLSA provenance: which workflow run, commit and repository produced each
+      # asset. Check one with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`.
+      # GitHub only stores attestations for private repositories on Enterprise Cloud, and a
+      # failure here would block the release, so a private repository skips the step and
+      # relies on SHA256SUMS alone.
+      - name: Attest build provenance
+        if: ${{ !github.event.repository.private }}
+        uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0
+        with:
+          subject-path: |
+            felis-linux-*
+            felis-image-*.tar
+            felis-velocity.jar
+
+      # --verify-tag refuses to invent a release for a tag that is not pushed.
       #
       # The prerelease flag has to be passed explicitly: the trigger glob is v*, so v1.2.3-rc1
       # lands here too, and gh does not read semver out of the tag name. Published as a full
@@ -84,13 +134,35 @@ jobs:
       # channel installs from and `felis update` polls — so every fresh install would get the
       # RC binary and every deployed felis-api would error on the felis component until a
       # stable tag was cut. Flagged, GitHub keeps latest pointing at the last stable release.
+      #
+      # A re-run (the release already exists) uploads only what is missing and never
+      # replaces a published asset: hosts may already have installed it, and their
+      # SHA256SUMS check would start failing against a swapped file. An asset that is
+      # there with different bytes stops the job; cut a new tag instead.
       - name: Publish the release
         env:
           GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
         run: |
+          # Every file the build handed over: SHA256SUMS names the installable ones, and the
+          # SBOMs ride along.
+          assets="$(ls)"
           flags=""
           case "$GITHUB_REF_NAME" in *-*) flags="--prerelease" ;; esac
-          gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags \
-              ./felis-linux-amd64 ./felis-linux-arm64 \
-            || gh release upload "$GITHUB_REF_NAME" \
-                 ./felis-linux-amd64 ./felis-linux-arm64 --clobber
+          if ! gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
+            # shellcheck disable=SC2086 # word-splitting the list is the point
+            gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags $assets
+            exit 0
+          fi
+          # The REST payload's per-asset "digest" is GitHub's own sha256 of the stored file.
+          published="$(gh api "repos/${GH_REPO}/releases/tags/${GITHUB_REF_NAME}" --jq '.assets[] | "\(.name) \(.digest)"')"
+          for a in $assets; do
+            have="$(printf '%s\n' "$published" | awk -v n="$a" '$1 == n { print $2 }')"
+            want="sha256:$(sha256sum < "$a" | cut -d' ' -f1)"
+            if [ -z "$have" ]; then
+              gh release upload "$GITHUB_REF_NAME" "$a"
+            elif [ "$have" != "$want" ]; then
+              echo "::error::$a is already published with $have; this run built $want. Published assets are never replaced."
+              exit 1
+            fi
+          done

.gitignore

@@ -51,3 +51,7 @@ AGENTS.md
 docs/Felis-Spec-V4.1.md
 panel/DESIGN.md
 panel/DESIGN-WEB-3SIDES.md
+
+# Audit ledgers and readiness reviews stay on the maintainer's disk.
+/AUDIT-*.md
+/READINESS-*.md

AUDIT-2026-09-22.md

@@ -1,61 +0,0 @@
-# Felis 生产就绪审计 — 2026-09-22（真机 E2E + 混沌）
-
-分支：`audit-fixes-20260922`（已推送）。环境：CentOS Stream 9 / aarch64 / k3s v1.36.4，
-IPv6-only 接入（`ssh -6 -i ~/.ssh/id_ed25519 root@fdb2:2c26:f4e4:0:21c:42ff:fede:69ec`），
-面板经 `socat TCP6:443 → 127.0.0.1:30443` 中继（手动启动，重启 VM 后需重开）。
-
-## 已修复并验证（分支内）
-
-| # | 缺陷 | 证据 | 修复 |
-|---|------|------|------|
-| 1 | **失败恢复后重试被静默吞掉**：restore Job 固定名 + `ErrAlreadyExists` 一律当"幂等成功"；失败 Job 占名 10 分钟（TTL），期间重试返回 202 `restoring` 但什么都不跑 | 真机：坏 ref 制造失败 → 立刻合法重试 → Job 原地不动、无新 Pod | `k8sjobs.go`：撞名时检查已完成（成功/失败）→ 删除+**等 finalizer 释放**（有界 10s）+ 重建；进行中仍幂等吸收。单测 3 个。**真机复验：重试 4s 完成恢复** |
-| 2 | **备份 Job 全部 FailedMount**：Job 在 minecraft 命名空间挂 `felis-config`，而 bootstrap 只在 felis 命名空间创建该 Secret | 事件：`MountVolume.SetUp failed: secret "felis-config" not found` | `felis setup` 用既有 `ensureSecretReplica` 把 felis-config(`felis.toml`) 复制到 minecraft；VM 上手工复制后备份成功（167MB 归档） |
-| 3 | RBAC 缺 `jobs:get/delete`（修复 #1 需要） | Role 检查 | `APIMinecraftRole` jobs → create/get/delete，测试同步更新 |
-| 4 | gofmt 9 文件漂移 + CI 无 gofmt 门禁；staticcheck 9 处 | 基线扫描 | 全部修复；CI 加 gofmt job |
-| 5 | 依赖漏洞：pgx v5.7.1（GO-2026-5004，可达 pgrepo.go）、x/net、x/text | govulncheck | 升 pgx v5.9.2 / x/net v0.55.0 / x/text v0.39.0 |
-| 16 | **`ConsumeLoginEmailOTP` PG 实现与接口契约漂移**：契约/`fakeRepo` 说「同 VerifyEmailOTP 生命周期（扣尝试/锁定/ErrOTPInvalid/ErrOTPLocked）」，PG 却是单条 UPDATE+`ErrNotFound` → 错码/重放/过期在 login 门、op-login finish、migration confirm 三个入口全部 500；且尝试次数永不累计、`otpMaxAttempts` 锁定失效 | 真机：login 门重放正确码 → **HTTP 500**；修复前错码不扣次。修复后复测：5 次错码 400 且 attempts=5（正确码因锁定也 400、码未消费）、新码可用、重放 400 | `pgrepo.go` 改置为 VerifyEmailOTP 同构事务（FOR UPDATE、先锁后比、mismatch 扣次、match 消费），无 users 写副作用。提交 `52549f7` |
-| 17 | **`ListPendingOpLogins` PG 少列**：接口注释承诺「joined to its staff username」，handler 输出 `username`/`created_at`，fake 正确填充；PG SQL 未 JOIN 也未取 `created_at` → 真机 pending 列表 username 为空、created_at 为 `0001-01-01` | 真机 internal `/op-login/pending` 响应 | `pgrepo.go` SQL 改为 JOIN users + 取 created_at。提交见分支 |
-| 18 | **绑定码并发兑换 500**：`RedeemPlayerBindCode` 无 `FOR UPDATE`（同文件 `VerifyLinkCode` 有），且裸 INSERT。6 路并发同码兑换 → 3×HTTP 500（`users_username_key` 唯一冲突）+ 1×400 + 2×200；跨码并发同 UUID 同样会撞 | 真机四组并发测试 + API 日志 `unmapped error ... duplicate key` | 同码：码行 `FOR UPDATE`（输家干净地 400 invalid_code）；跨码：`INSERT users ... ON CONFLICT (username) DO NOTHING`+重读、`account_links ON CONFLICT (mc_uuid) DO NOTHING`（两路汇聚同一 user）。复测：同码×6=1×200+5×400、双码×2=2×200 同 ID、DB 干净、日志零 unmapped |
-| 19 | **reaper CronJob 渲染位置错误，永远无法调度**：`felis manifests` 把 CronJob 渲染到控制 ns，却引用 minecraft ns 的备份 PVC（Pod 不能跨 ns 挂 PVC：真机 `FailedScheduling: persistentvolumeclaim "felis-backups" not found`）；修正 ns 后又发现 ServerAccount 也不能跨 ns 使用（`serviceaccount "felis-reaper" not found`）。而 reaper 的 Role/RoleBinding 本就在 minecraft ns | 真机三层取证（PVC/SA/调度）| CronJob 与 SA、RoleBinding subject 全部移到 `MinecraftNamespace`（提交 `e4f2cff`+`c839454`）。**修复后完整演练通过**（见下） |
-
-## 待决策台账（未修）
-
-| # | 主题 | 说明 |
-|---|------|------|
-| 6 | 默认安装无备份能力 | backup/restore 端点默认 503（需 `FELIS_BACKUP_PVC`+PVC），reaper CronJob 需 `--backup-pvc/--archive-local-path/--worlds-host-path` 三旗标渲染，bootstrap 一个都不传；文档无说明；README 与"自动备份"口径不符。另：归档 3 个月过期依赖 reaper 清理，未启用则磁盘只增不减 |
-| 7 | 异步失败不可感知 | backup/restore 失败后无状态出口：restore 行不变、backup 无行；只有集群侧 Job/日志可查。建议状态字段或 `?failed` 查询 |
-| 8 | 磁盘打满灾难链 | DiskPressure → kubelet 驱逐控制面（无 PriorityClass 保护）→ 镜像被 GC（无外网、registry 空）→ 全部 ImagePullBackOff；释放后约 8 分钟才恢复调度。恢复靠 `docker save felis:* | k3s ctr images import -`（docker 守护进程存储是唯一副本，需固化回源路径）。建议：PriorityClass、镜像入内置 registry、磁盘告警 |
-| 9 | 升级策略 Recreate | 单副本 + Recreate：任何控制面升级=停机；坏升级（实测错 tag）服务中断约 95s 且需人工 `rollout undo`（无自动回滚）。建议 runbook/文档化 |
-| 10 | 备份语义 | 归档包含整个 /data（jar、libraries、cache），167MB；是否符合"world backup"定位待评估 |
-| 11 | PG 断连表现 | 会话查询失败报 401 而非 503（fail-closed 但误导；用户以为没登录） |
-| 12 | ready 门滞后 | 容器 Ready 后 6~10s 内 API 仍 409 not_running |
-| 13 | setup token 截断 | 43 字符 token + 长域名，80 列终端下 TUI 截断显示（复现：tmux 80 列） |
-| 14 | 日志噪音 | controller-runtime 未 SetLogger，首用打印整段堆栈；TLS handshake EOF 噪音（kubelet 探针） |
-| 15 | reaper 启用未演练 | **已演练完毕（2026-09-22）**：绑定挂载演练 world → 一次 pass 归档+删 PVC+保留 servers 行/CRD；过期归档驱逐（行转 deleted、文件删除、合法归档未动）。启用仍受 #6 三旗标约束；另注意 `--worlds-host-path` 的 `<path>/<pvc>` 布局在 stock local-path 下不成立（编排责任），且 VM 上演练用的 CronJob 已 **suspend** 防误删（真实世界目录不在 /srv/worlds-root）|
-
-## 已验证事实（正向清单）
-
-- 安装→hook 发码→Owner 绑定→passkey（虚拟认证器）→面板管理员全链路 ✅
-- 建服（POST /servers）→ 唤醒（operator 拉 StatefulSet pod）→ RCON `list` → SSE 控制台 → 停止 ✅
-- 文件编辑：列目录/读/写（wire 为 base64）/256KiB 413/路径穿越 5 变体全拦截/运行中 409 ✅
-- **备份→篡改→恢复数据演练**：v1→备份→v2→恢复→读回 v1 ✅（G2 数据可恢复）
-- **毒档案 fail-closed**：穿越/绝对路径/符号链接条目 → `archive entry escapes target` 退出码 1，零写入 ✅
-- 混沌：PG 掉线（healthz 仍 200、恢复后连接池自愈）、API pod 击杀（~2s 中断）、整机重启（32s 回归、会话/CRD/停止态全保留）✅
-- `felis update` 报告（k3s/velocity 有更新、私有仓库 404 优雅处理）✅
-- 账户全套真机 E2E：绑定码新玩家注册（幂等/并发见 #18）、邮箱验证（onboarding 门）、邮箱 OTP 登录（错码扣次/5 次锁定后正确码也 400、重放 400、staff 账号 403 拒绝）、Passkey 注册+discoverable 登录+邮箱优先登录（虚拟认证器；Chrome 要求 `Page.bringToFront` 才能过 focus 检查）、登出吊销会话（旧 cookie 401）
-- op-login 全状态机（start→status→approve→finish；早 finish 不烧码、错码扣次且请求保留、重放/重复批准/非管理员批准/未知 handle 全部按契约返回）✅
-- 并发：OTP 风暴 8 路 = 1×202 + 7×429 且仅铸 1 码；绑定码并发（同码/双码）见 #18 ✅
-- **reaper 全链路真机演练**（修复 #19 后）：绑定挂载假世界 → `felis reaper` 一 pass：归档 tar 落盘（内容含标记文件）、`world_backups` 行 `inactive_15d`+90d 过期、**PVC 删除且宿主目录回收**、servers 行保留且 activity 时钟重置（红线②）、CRD 保留；第二 pass：伪造过期归档被驱逐（`expired=1`，文件删、行转 `deleted`），合法归档未动；`evaluated=2 reaped=1` 只动到期的世界 ✅
-
-## 系统性观察
-
-- **PGRepo 与接口契约/fake 漂移**（#16/#17/#18）：`fakeRepo` 与接口注释是对的、PG 实现在细节上落后，单测全绿也发现不了。建议后续引入 PG 级契约测试（testcontainers 或针对关键写路径的集成测试），重点覆盖「接口注释承诺了字段/错误码/生命周期」的方法。
-- 部署知识：交叉编译必须先把 `panel/dist` 拷进 `internal/panel/static` 再 `go build`，否则镜像内 SPA 缺失（页面显示 “assets were not built”）。本批镜像为 `felis:auditfix7`。
-
-## 复现入口速查
-
-- 面板会话 cookie：`/tmp/felis-cookies.json`；API 助手：`/tmp/fcurl.sh`
-- 测试服：`test-one`（minecraft ns，stopped）；合法备份 `bk-47ee2e7e96e5a4ca9d0e51b805518bac`
-- CDP 调试口：Mac `127.0.0.1:9333`（独立 Chrome，profile `/tmp/felis-chrome2`）；WebAuthn 虚拟认证器需在**同一 CDP 会话**内完成仪式，且先 `Page.bringToFront`（否则 NotAllowedError: page does not have focus）
-- 分支已部署到 VM：`felis-api` 镜像 = `felis:auditfix7`（含全部修复）
-- VM 内部面：`k3s kubectl -n felis port-forward svc/felis-api-internal 18081:8081`（Pod 重建后转发会悬死，需重启）；reaper CronJob（minecraft ns）已应用但 `suspend=true`

CONTRIBUTING.md

@@ -23,6 +23,28 @@ The codebase is intentionally split by responsibility. Prefer changing the
 smallest owning module instead of adding broad abstractions or rebuilding nearby
 code.
 
+## Interface Copy
+
+Interface copy must use a formal documentation style: objective, concise, and
+precise. State the current condition, its cause or impact, and the available
+action. Describe confirmed facts only; avoid conversational phrasing,
+personification, and unsupported duration estimates. Chinese instructions should
+use explicit verbs such as “执行”, “选择”, “查看”, and “配置”, with operation names
+matching the actual UI labels. English and Chinese copy must retain the same
+meaning and degree of formality. Update existing translation entries rather than
+adding duplicate messages or components.
+
+## Panel Visual Style
+
+Reuse `PageHeader`, the `Card` family, and the shared form and button components.
+Card titles, borders, corner radii, spacing, and save footers follow their shared
+definitions; avoid redefining these styles in individual pages. Use white card
+backgrounds in the light theme, restrained semantic colors, and 16px functional
+icons. Use `CardFooter` for save actions and `Button` for interactive controls.
+Tables, consoles, compact statistics, and status messages may retain spacing and
+colors suited to their content. Preserve the shared page margins and bottom
+spacing.
+
 ## Local Development
 
 You can do most day-to-day development on macOS or Linux without a full cluster.
@@ -67,6 +89,27 @@ go test ./internal/api
 go test ./cmd/felis
 ```
 
+The hermetic suites run against in-memory fakes; the business stores' SQL is
+verified separately against a real Postgres, on a throwaway database whose name
+must contain `pgint` (the harness drops and recreates its schema and replays the
+embedded migrations):
+
+```bash
+FELIS_TEST_PG_URL='postgres://felis:***@127.0.0.1:5432/felis_pgint?sslmode=disable' \
+  go test -tags pgint ./internal/pgint/ -v
+```
+
+Run it after touching anything under `internal/api/pgrepo.go`, `internal/submit`,
+`internal/build` or `internal/dbbackup` that speaks SQL: the fakes encode the
+contract, and this suite exists to catch the drift between the fakes and the real
+queries. The `felis db backup` and `restore` tests also run `pg_dump`, `pg_restore`
+and `psql`, which must be the server's major version. For a server in a container,
+run them in it, as production does in felis-postgres:
+
+```bash
+FELIS_TEST_PG_EXEC='docker exec -i <container>' FELIS_TEST_PG_URL=... go test -tags pgint ./internal/pgint/
+```
+
 Build the CLI:
 
 ```bash
@@ -191,13 +234,13 @@ export FELIS_IMAGE=felis:dev
 export FELIS_ROOT_DOMAIN=<node-ip>.nip.io
 ```
 
-By default the installer builds the newest **published GitHub release**. While this
-repository is private that lookup — and the clone itself — needs a token, and building
-the development tip needs an opt-in:
+By default the installer builds the newest **published GitHub release**. Building the
+development tip needs an opt-in, and installing from a private fork additionally needs a
+token for the release lookup and the clone:
 
 ```bash
-export FELIS_GITHUB_TOKEN=<token with read access to the repo>
 export FELIS_VERSION_BOOTSTRAP=dev      # build main instead of the newest release
+export FELIS_GITHUB_TOKEN=<token>       # private forks only: read access to the fork
 ```
 
 `dev` is also the escape hatch before the first `vX.Y.Z` tag exists: with no published

Dockerfile

@@ -21,14 +21,18 @@
 # minutes. The FINAL stage is deliberately NOT pinned — it must stay on the target platform
 # or the published arm64 image would carry amd64 layers. It contains only COPY, which
 # BuildKit performs itself, so it needs no QEMU either; adding a RUN there would.
-FROM --platform=$BUILDPLATFORM node:22-bookworm AS panel
+#
+# Every base image here and in deploy/{limbo,lobby,paper} is pinned by digest, so a rebuild
+# of one release uses the same bytes; .github/dependabot.yml proposes the bumps (tag and
+# digest together).
+FROM --platform=$BUILDPLATFORM node:22-bookworm@sha256:363e1587494626837fa7f9a23bdb453d13b0ff3c67c705c2805cfc69c2d2fad7 AS panel
 WORKDIR /panel
 COPY panel/package*.json ./
 RUN npm ci
 COPY panel/ ./
 RUN npm run build
 
-FROM --platform=$BUILDPLATFORM golang:1.26 AS build
+FROM --platform=$BUILDPLATFORM golang:1.26@sha256:6c2a5538f964f1c82f97ad14988bf05de100d922d159d0e398b54c7b0ca0c6c9 AS build
 WORKDIR /src
 ARG TARGETOS=linux
 ARG TARGETARCH
@@ -55,7 +59,7 @@ ARG FELIS_VERSION=dev
 RUN CGO_ENABLED=0 GOOS="$TARGETOS" GOARCH="${TARGETARCH:-$(go env GOARCH)}" \
     go build -trimpath -ldflags="-s -w -X main.version=${FELIS_VERSION}" -o /out/felis ./cmd/felis
 
-FROM gcr.io/distroless/static-debian12:nonroot
+FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
 ENV PATH=/usr/local/bin:/usr/bin:/bin
 COPY --chmod=0755 --from=build /out/felis /usr/local/bin/felis
 # distroless "nonroot" is uid 65532; the rendered PodSecurityContext pins

README.md

@@ -1,11 +1,25 @@
-# Felis
+<div align="center">
+  <h1 align="center">
+    <img src="docs/assets/felis-logo.png" alt="Felis logo" width="270"><br>
+    Felis
+  </h1>
+  <p align="center">
+    基于 Kubernetes 的 Minecraft 服务器托管平台<br>
+    单条命令完成部署，自动管理服务器生命周期、备份与安全
+    <br><br>
+    <a href="README.md">简体中文</a> | <a href="README_EN.md">English</a>
+    <br>
+    <a href="https://felismc.com/">官网</a> | <a href="https://docs.felismc.com/">文档</a>
+  </p>
+</div>
 
-一款 Kubernetes 驱动的 Minecraft 服务器托管平台，一行命令部署，自动管理生命周期与安全。  
-A Kubernetes-driven Minecraft server hosting platform — one command to deploy, automatic lifecycle, backup, and security.
+> [!CAUTION]
+> **当前仅提供开发快照，已撤下公开 Release。仅供开发者在隔离的测试环境中验证，不建议普通用户部署。**
+> **此项目仍处于早期开发阶段，您不该在任何生产环境使用该项目。若产生任何问题，贵用户的使用行为与 FelisMC 团队无任何民事刑事法律关系。**<br>
+> **THIS PROJECT IS STILL WIP, YOU SHOULD DO NOT USE THIS PROJECT IN ANY PRODUCTION USAGE. WE ARE NOT RESPOND FOR ANY LEGAL OR HUMANLY PROBLEM.**
 
-[简体中文](README.md) | [English](README_EN.md)
-
-目录
+<details>
+<summary>目录</summary>
 
 - [特性](#特性)
 - [使用方式](#使用方式)
@@ -13,47 +27,79 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy,
 - [开源协议](#开源协议)
 - [致谢](#致谢)
 
+</details>
+
 ## 特性
 
-- **即开即玩**：玩家尝试连接时自动唤醒服务器，空闲后自动休眠，像游戏主机一样省资源。
-- **Web 控制面板**：浏览器中查看服务器状态、在线玩家与资源用量，管理备份与恢复。
-- **自动备份与恢复**：定时将世界打包存档，支持从任意备份点一键回滚。
-- **智慧回收**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间。
-- **多核心支持**：兼容 Paper、Fabric、Forge、NeoForge，经由 Velocity 代理统一入口。
-- **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建并部署。
-- **Passkey 登录**：支持指纹、面容、硬件密钥等无密码认证方式。
-- **零信任安全**：面板流量由 Cloudflare Access 保护，集群内 API 不暴露到公网。
+* **按需启停**：玩家连接代理时自动启动目标服务器，启动期间玩家进入等待队列，服务器就绪后自动传送；服务器空闲后自动停止，释放内存。
+
+* **Web 控制面板**：在浏览器中查看服务器状态、在线玩家与资源用量。
+  * 控制台（RCON）、白名单、封禁、OP 与 LuckPerms 权限管理
+  * 文件管理：新建、删除、重命名、分片上传、下载，以及停服状态下解压 zip，可用于导入世界
+  * 计划任务：按星期与时区定时执行命令、重启、停止、启动或备份，执行前在游戏内向玩家发送提醒
+
+* **备份与恢复**：默认启用，归档 PVC 及其路径由安装器生成。
+  * 手动备份：将服务器的完整数据卷（`/data`，含世界、配置、插件与模组）归档至集群内的归档存储，可回滚至任一备份点
+  * 每日恢复点：当天有玩家进入过的服务器在停止后自动生成恢复点，默认保留 7 个，保存期限 90 天；恢复点单独轮换，不影响手动备份
+  * 下载与导出：支持下载单个备份（附 sha256 校验）、删除单个备份及导出完整世界
+  * 异地副本（可选）：备份在主机上加密后同步至 S3 兼容存储（AWS S3、Cloudflare R2、Backblaze B2、MinIO 等）
+  * 控制面数据库：存放账号、服务器归属、配额与存档索引的数据库每日自动备份，每次升级迁移前额外创建快照，故障时可通过 `felis db restore` 整库原子回滚；面板「维护与备份」页显示最近一次备份的时效（参见 [故障排查 §16](docs/troubleshooting.md)）
+
+* **运维诊断**
+  * `sudo felis status`：汇总显示节点、控制面、游戏代理、各服务器、备份及未解决的告警
+  * `sudo felis doctor`：执行全部健康检查，按模块列出问题及排查方向，执行过程中不发送邮件
+  * `sudo felis support-bundle`：生成已脱敏的诊断包，供提交问题时附带（参见 [故障排查 §0](docs/troubleshooting.md)）
+  * 看门狗：每 2 分钟执行一次巡检，异常持续时向平台所有者发送邮件告警，支持外部心跳监测
+
+* **世界回收（可选）**：超过 15 天无人游玩的世界在备份后删除，以释放磁盘空间。安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认为 `/var/lib/rancher/k3s/storage`）即启用每日回收；未设置时不删除任何世界。过期备份的每日清理与此设置无关，始终执行。
+
+* **多核心支持**：兼容 Paper、Fabric、Forge 与 NeoForge，统一经由 Velocity 代理接入。
+
+* **模组包投稿**：玩家可上传模组包，经服主审批后自动构建并通过 Trivy 安全扫描；构建产物加入镜像白名单后，可直接选作服务器镜像。
+
+* **安全**
+  * Passkey 登录：支持指纹、面容识别及硬件密钥等无密码认证方式
+  * 零信任访问：面板流量经 Cloudflare Access 保护，集群内部 API 不对公网开放
+
+* **多机部署（实验性，默认关闭）**：由一台主控节点统一下发指令，其余节点仅运行游戏服务器，已停止的服务器可迁移至其他节点。该功能目前仅位于 main 分支，尚未完成三机验收（参见 [多机部署](docs/distributed.md)）。
 
 ## 使用方式
 
-在准备好的 Linux 主机上执行：
+开发测试需在已准备好的 Linux 测试主机上显式选择 `dev` 通道（跟随 `main`，从源码构建）：
 
 ```bash
-curl -fsSL https://raw.githubusercontent.com/MliroLirrorsIngenuity/Felis/main/deploy/bootstrap.sh | sudo bash
+curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo FELIS_VERSION_BOOTSTRAP=dev bash
 ```
 
-脚本将自动安装 K3s、部署控制平面并启动设置向导。完成后浏览器访问已配置的域名进入控制面板即可使用。
+脚本将安装 K3s，在 K3s 中部署 PostgreSQL 与控制平面，随后启动设置向导。设置完成后，通过浏览器访问所配置的域名即可进入控制面板。
 
-> **本仓库当前为私有**，上面这条会返回 404。请改用带凭据的形式；安装器自身也需要同一个 token
-> 去解析并下载 release，所以用 `sudo -E` 把它带进去：
->
-> ```bash
-> export FELIS_GITHUB_TOKEN=<对本仓库有读权限的 token>
-> printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \
->   | curl -fsSL --config - -H "Accept: application/vnd.github.raw" \
->       https://api.github.com/repos/MliroLirrorsIngenuity/Felis/contents/deploy/bootstrap.sh \
->   | sudo -E bash
-> ```
->
-> token 经 stdin 交给 `curl --config -`，不放在命令行上：argv 在 `/proc` 下对本机任意用户可读，
-> 而这正是安装器内部 `github_api` 采用同一写法的原因。
+* **设置向导**：先完成面板访问方式与存储配置，再由主机管理员创建首位 Owner，终端会给出一次性网页设置链接（30 分钟内有效）。用浏览器打开链接、登记邮箱并创建通行密钥，即可进入面板，无需启动 Minecraft。角色关联可稍后在“账户”页选择认证源、输入角色名或 UUID 并确认；普通玩家继续通过游戏绑定码验证身份。未完成网页登录或链接过期时，再次执行 `sudo felis setup` 会提供新链接；已设置登录凭证的账号不会被重置。登录服或大厅故障不会阻止面板初始化。“账户”页会解释世界树（Yggdrasil）认证、显示进服地址与登录服/大厅所需的 Java 版客户端版本；安装器会把实际构建版本写入 `[velocity].game_version`，自定义镜像需自行填写，未配置时不会猜测版本。标准世界树接口可直接查询角色；非标准 `hasJoined` 地址可通过 `[[auth_source]].api_url` 指定认证站 API 根地址，游戏绑定码仍可作为替代方式。安装器仅在交互式终端中自动启动向导；输出重定向至日志或经由 cloud-init 安装时，请在安装结束后执行 `sudo felis setup`。设置 `FELIS_NO_SETUP=1` 时，安装器在输出摘要后直接结束。
 
-重跑这条命令也是把 felis-api 升到新版本的方式（`felis setup` 做不到，它用的是本机已有的二进制）。
-重跑会沿用已安装的根域名，但**不会**沿用通道：若本机跟随 main，需一并 `export FELIS_VERSION_BOOTSTRAP=dev`。
+* **认证源管理**：Owner 可在左侧“平台 → 认证源”添加、编辑、排序、停用第三方 Yggdrasil 认证站，并测试认证接口。保存后立即用于下一次登录和角色查询，无需重启；面板配置优先于安装配置。已保存的永久标识不能改名或删除，以保留玩家 UUID 与账号绑定；不再使用的源可停用。Mojang 始终优先验证。Nano 继续使用 TOML 配置。
+
+* **支持的系统**：CentOS Stream 9（aarch64）已在实机上验证；Ubuntu 24.04（x86_64）在每次推送时由 CI 验证开发构建的全新安装与重复安装（参见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
+
+* **安装前检查**：安装器在修改主机之前检查内存、磁盘、端口、网段冲突、已有的 Kubernetes 及外网连通性。发现问题时一次性列出全部问题并退出，主机保持原状（检查项参见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
+
+* **升级**：重新执行安装命令即可将 felis-api 升级至新版本；`felis setup` 仅使用本机已安装的二进制，无法用于升级。重新执行时沿用已安装的根域名，发布通道需重新指定：跟随 main 分支的主机须同时设置 `export FELIS_VERSION_BOOTSTRAP=dev`。早期版本安装在宿主机上的 PostgreSQL 会在重新执行时整库迁入 K3s，宿主机上的原实例停用并保留，以便回退（参见 [运维手册 §4](docs/operations.md#4-upgrading-the-pieces-around-felis)）。
+
+<details>
+<summary>安装来源与受限网络环境下的安装</summary>
+<br>
+
+以下说明发布机制；当前暂无公开 Release 附件，开发测试使用源码构建。
+
+安装发布版时，二进制文件、全部镜像及 Velocity 插件均取自该版本由 CI 预构建的 release 附件，逐一校验 `SHA256SUMS` 后导入。主机无需安装 Docker、Gradle 或 Go，也无需访问 Docker Hub。若某个附件缺失或校验失败，仅该镜像回退为本机构建，并输出提示（参见 [故障排查 §15c](docs/troubleshooting.md)）。
+
+也可将附件预先复制到主机，再通过 `FELIS_ARTIFACT_DIR=<绝对路径>` 安装，此时 Felis 自身的二进制、镜像与插件均从该目录读取。k3s 及其镜像、JRE、cloudflared、Velocity 与 Via 插件仍从 GitHub 和 PaperMC 下载；RHEL、Fedora、openSUSE Leap 等启用 SELinux 的主机还需从 rpm.rancher.io 安装 k3s-selinux；系统软件包来自发行版软件源。
+
+因此，出站网络受限的主机须放行上述地址的 HTTPS 访问，或设置 `https_proxy`。preflight 会在修改主机之前逐一探测这些地址。目前暂不支持完全离线安装（地址清单参见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
+
+</details>
 
 ## 从源码构建
 
-本项目基于 Go 和 Node.js 开发：
+本项目基于 Go 与 Node.js 开发：
 
 ```bash
 # 后端（Go 1.26+）
@@ -70,22 +116,22 @@ docker build -t felis:custom .
 
 ## 开源协议
 
-本项目遵循 [AGPL-3.0-only](LICENSE) 开源协议。
+本项目采用 [AGPL-3.0-only](LICENSE) 许可证。
 
 ### 协议注意事项
 
-1. **衍生作品同样是 AGPL**：分发本项目的副本或基于本项目衍生的软件时，必须以 AGPL-3.0 开源，并保留原作者的版权声明和许可声明。
-2. **通过网络提供服务同样要开源**（AGPL 第 13 条）：如果你把修改过的 Felis 架起来给别人用，即使从不分发任何二进制，也必须向这些用户提供你那份修改后的完整源码。这是 AGPL 相对 GPL 的唯一实质区别，而 Felis 正是一个跑在网络上的托管平台，所以这一条基本总会触发。
-3. **免责声明**：本项目按"原样"提供，作者不承担任何因使用本项目而产生的法律责任。
+1. **衍生作品须采用 AGPL**：分发本项目副本或基于本项目的衍生软件时，须以 AGPL-3.0 开源，并保留原作者的版权声明与许可声明。
+2. **网络服务同样须提供源码**（AGPL 第 13 条）：通过网络向他人提供经修改的 Felis 服务时，即使未分发任何二进制文件，也须向这些用户提供修改后的完整源码。这是 AGPL 与 GPL 唯一的实质区别；Felis 作为通过网络访问的托管平台，几乎所有部署场景都适用此条款。
+3. **免责声明**：本项目按"原样"提供，作者不承担因使用本项目而产生的任何法律责任。
 
 ## 致谢
 
-- [Kubernetes](https://kubernetes.io/)：底层容器编排引擎
-- [K3s](https://k3s.io/)：轻量级 Kubernetes 发行版
-- [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/)：零信任安全基础设施
-- [PostgreSQL](https://www.postgresql.org/)：数据持久化
-- [React](https://react.dev/)：前端用户界面框架
-- [Vite](https://vitejs.dev/)：前端构建工具
-- [TailwindCSS](https://tailwindcss.com/)：CSS 框架
-- [Bubble Tea](https://github.com/charmbracelet/bubbletea)：TUI 框架
-- [Minecraft](https://www.minecraft.net/)：让这一切值得做
+* [Kubernetes](https://kubernetes.io/)：容器编排引擎
+* [K3s](https://k3s.io/)：轻量级 Kubernetes 发行版
+* [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/)：零信任安全基础设施
+* [PostgreSQL](https://www.postgresql.org/)：数据持久化
+* [React](https://react.dev/)：前端用户界面框架
+* [Vite](https://vitejs.dev/)：前端构建工具
+* [TailwindCSS](https://tailwindcss.com/)：CSS 框架
+* [Bubble Tea](https://github.com/charmbracelet/bubbletea)：TUI 框架
+* [Minecraft](https://www.minecraft.net/)：本项目服务的游戏

README_EN.md

@@ -1,11 +1,24 @@
-# Felis
+<div align="center">
+  <h1 align="center">
+    <img src="docs/assets/felis-logo.png" alt="Felis logo" width="270"><br>
+    Felis
+  </h1>
+  <p align="center">
+    A Kubernetes-driven Minecraft server hosting platform<br>
+    One command to deploy, with automatic lifecycle, backup, and security
+    <br><br>
+    <a href="README.md">简体中文</a> | <a href="README_EN.md">English</a>
+    <br>
+    <a href="https://felismc.com/">Website</a> | <a href="https://docs.felismc.com/en/">Documentation</a>
+  </p>
+</div>
 
-A Kubernetes-driven Minecraft server hosting platform — one command to deploy, automatic lifecycle, backup, and security.  
-一款 Kubernetes 驱动的 Minecraft 服务器托管平台，一行命令部署，自动管理生命周期与安全。
+> [!CAUTION]
+> **Only development snapshots are available; public releases have been withdrawn. These builds are for developers testing in isolated environments and are not recommended for general deployment.**
+> **THIS PROJECT IS STILL WIP, YOU SHOULD DO NOT USE THIS PROJECT IN ANY PRODUCTION USAGE. WE ARE NOT RESPOND FOR ANY LEGAL OR HUMANLY PROBLEM.**
 
-[简体中文](README.md) | [English](README_EN.md)
-
-Table of Contents
+<details>
+<summary>Table of Contents</summary>
 
 - [Features](#features)
 - [Getting Started](#getting-started)
@@ -13,26 +26,73 @@ Table of Contents
 - [License](#license)
 - [Acknowledgements](#acknowledgements)
 
+</details>
+
 ## Features
 
-- **Wake on Join**: Servers start automatically when a player connects, and stop when idle — like hibernate for your server.
-- **Web Dashboard**: Monitor server status, online players, and resource usage from your browser, with backup and restore management.
-- **Auto Backup & Restore**: Scheduled world backups with one-click rollback from any backup point.
-- **World Reaper**: Worlds idle for more than 15 days are automatically backed up and removed to free disk space.
-- **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, federated behind a Velocity proxy.
-- **Modpack Submission**: Players submit custom modpacks; admin approval triggers automatic build and deployment.
-- **Passkey Login**: Passwordless authentication via fingerprint, face recognition, or hardware security keys.
-- **Zero Trust Security**: Panel traffic protected by Cloudflare Access; the internal API is never exposed to the internet.
+* **On-demand Start and Stop**: A server starts when a player connects to the proxy. The player waits in a queue during start-up and is transferred once the server is ready. Idle servers stop automatically to free memory.
+
+* **Web Dashboard**: Monitor server status, online players, and resource usage from your browser.
+  * Console (RCON), whitelist, bans, OPs and LuckPerms permissions
+  * File manager: create, delete, rename, chunked upload, download, and unzip while the server is stopped; also used to import worlds
+  * Schedules: run commands, restart, stop, start or back up by weekday and time zone, with an in-game warning to players beforehand
+
+* **Backup & Restore**: Enabled by default; the installer renders the archive PVC and its path.
+  * Manual backups: archive a server's entire data volume (`/data`, including worlds, configuration, plugins and mods) to the cluster's archive store, with rollback to any backup point
+  * Daily restore points: a server played that day gets a restore point once it stops; by default 7 are kept for up to 90 days, rotated separately from manual backups
+  * Download and export: download a single backup (with sha256 verification), delete a single backup, or export a whole world
+  * Off-site copy (optional): backups are encrypted on the host and synced to S3-compatible storage (AWS S3, Cloudflare R2, Backblaze B2, MinIO and others)
+  * Control-plane database: the database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migration; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows the age of the latest backup (see [troubleshooting §16](docs/troubleshooting.md))
+
+* **Diagnostics**
+  * `sudo felis status`: a summary of the node, control plane, game proxy, each server, backups and open alerts
+  * `sudo felis doctor`: runs all health checks and lists problems by area with troubleshooting pointers; sends no email
+  * `sudo felis support-bundle`: generates a redacted diagnostics archive to attach to support requests (see [troubleshooting §0](docs/troubleshooting.md))
+  * Watchdog: runs a check every 2 minutes and emails the platform owners when a problem persists; supports an external heartbeat monitor
+
+* **World Reaper** (optional): Worlds idle for more than 15 days are backed up and then removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is deleted. Expired backups are cleaned up daily regardless of this setting.
+
+* **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, accessed through a single Velocity proxy.
+
+* **Modpack Submission**: Players can upload modpacks. After admin approval, each modpack is built automatically and scanned with Trivy; the result is added to the image whitelist and can be selected as a server image.
+
+* **Security**
+  * Passkey login: passwordless authentication via fingerprint, face recognition, or hardware security keys
+  * Zero-trust access: panel traffic is protected by Cloudflare Access, and the internal API is not exposed to the internet
+
+* **Multi-node Deployment** (experimental, off by default): a single controller node issues all commands, the other nodes run game servers only, and a stopped server can be migrated to another node. Currently available only on the main branch; three-node acceptance testing is not yet complete (see [distributed mode](docs/distributed.md), in Chinese).
 
 ## Getting Started
 
-On a prepared Linux host, run:
+For development testing on a prepared Linux test host, explicitly select the `dev` channel, which follows `main` and builds from source:
 
 ```bash
-curl -fsSL https://raw.githubusercontent.com/MliroLirrorsIngenuity/Felis/main/deploy/bootstrap.sh | sudo bash
+curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo FELIS_VERSION_BOOTSTRAP=dev bash
 ```
 
-The script installs K3s, deploys the control plane, and launches a setup wizard. Once done, open your browser at the configured domain.
+The script installs K3s, deploys PostgreSQL and the control plane inside it, and launches a setup wizard. When setup completes, open the configured domain in a browser to reach the control panel.
+
+* **Setup wizard**: Configure panel access and storage first. The host administrator then initializes the first Owner and receives a one-time browser setup link (valid for 30 minutes). Open it, record an email, and create a passkey to enter the panel; Minecraft is not required. Later, link a game role from Account by selecting an authentication source, entering a role name or UUID, and confirming it. Players retain the in-game bind-code flow. Rerun `sudo felis setup` if login setup is unfinished or the link expires; accounts with an established login factor are never reset. Login/lobby failures do not block panel initialization. Account explains Yggdrasil authentication and shows the join address and Java client version for the login/lobby servers. Bootstrap records the built protocol in `[velocity].game_version`; set it yourself for custom images, otherwise the panel reports it as unknown. Standard Yggdrasil endpoints support role lookup directly; sources with a nonstandard `hasJoined` path can set `[[auth_source]].api_url` to their API root, with game-code linking still available as a fallback. The installer launches the wizard automatically only on an interactive terminal; when output is redirected to a log or the install runs under cloud-init, run `sudo felis setup` after it finishes. Setting `FELIS_NO_SETUP=1` makes the installer end at its summary.
+
+* **Supported hosts**: CentOS Stream 9 (aarch64) is verified on physical hardware; CI tests fresh installation and repeat installation of development builds on Ubuntu 24.04 (x86_64) on every push (see [operations §1](docs/operations.md#1-supported-hosts)).
+
+* **Preflight checks**: Before modifying the host, the installer checks memory, disk, ports, network range conflicts, existing Kubernetes installations and outbound connectivity. If any check fails, it lists all problems and exits, leaving the host unchanged (see [operations §1](docs/operations.md#1-supported-hosts) for the checks).
+
+* **Upgrading**: Rerun the install command to upgrade felis-api to a newer version; `felis setup` only uses the binary already installed on the host and cannot upgrade it. A rerun keeps the installed root domain, and the release channel must be specified again: hosts that follow the main branch must also set `export FELIS_VERSION_BOOTSTRAP=dev`. A PostgreSQL instance installed on the host by an earlier release is migrated into K3s during the rerun; the original instance on the host is stopped and retained for rollback (see [operations §4](docs/operations.md#4-upgrading-the-pieces-around-felis)).
+
+<details>
+<summary>Installation sources and restricted networks</summary>
+<br>
+
+The following describes the release mechanism. No public release assets are currently available; development testing uses source builds.
+
+A release installation takes the binary, all images and the Velocity plugin from the release assets prebuilt in CI, verifying each against `SHA256SUMS` before import. The host requires no Docker, Gradle or Go, and no access to Docker Hub. If an asset is missing or fails verification, only that image falls back to a local build, and the installer prints a notice (see [troubleshooting §15c](docs/troubleshooting.md)).
+
+The assets can also be copied to the host in advance and installed with `FELIS_ARTIFACT_DIR=<absolute path>`; the Felis binary, images and plugin are then read from that directory. k3s and its images, the JRE, cloudflared, Velocity and the Via plugins are still downloaded from GitHub and PaperMC; hosts with SELinux enabled, such as RHEL, Fedora and openSUSE Leap, additionally install k3s-selinux from rpm.rancher.io; system packages come from the distribution's repositories.
+
+A host with restricted outbound access must therefore allow HTTPS to these addresses or set `https_proxy`. Preflight probes each address before changing the host. Fully offline installation is not yet supported (see [operations §1](docs/operations.md#1-supported-hosts) for the address list).
+
+</details>
 
 ## Build from Source
 
@@ -53,22 +113,22 @@ docker build -t felis:custom .
 
 ## License
 
-The source code is released under [AGPL-3.0-only](LICENSE).
+This project is licensed under [AGPL-3.0-only](LICENSE).
 
 ### License Notes
 
-1. **Derivative works are AGPL too**: Any distribution of this project or of software derived from it must be released under AGPL-3.0 and must include the original copyright notice and license statement.
-2. **Running it as a network service also triggers the source obligation** (AGPL section 13): if you host a modified Felis for other people to use, you must offer those users the complete source of your modified version — even if you never distribute a binary. This is the one substantive difference between AGPL and GPL, and since Felis is a hosting platform reached over a network, it will essentially always apply.
+1. **Derivative works must use AGPL**: Any distribution of this project or of software derived from it must be released under AGPL-3.0 and must include the original copyright notice and license statement.
+2. **Network services must also provide source** (AGPL section 13): anyone who offers a modified Felis to others over a network must provide those users with the complete source of the modified version, even without distributing any binary. This is the only substantive difference between AGPL and GPL; as Felis is a hosting platform accessed over a network, this clause applies to virtually every deployment.
 3. **Disclaimer**: This project is provided "as is", without warranty of any kind.
 
 ## Acknowledgements
 
-- [Kubernetes](https://kubernetes.io/): Container orchestration engine
-- [K3s](https://k3s.io/): Lightweight Kubernetes distribution
-- [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/): Zero trust security infrastructure
-- [PostgreSQL](https://www.postgresql.org/): Data persistence
-- [React](https://react.dev/): User interface framework
-- [Vite](https://vitejs.dev/): Frontend build tool
-- [TailwindCSS](https://tailwindcss.com/): CSS framework
-- [Bubble Tea](https://github.com/charmbracelet/bubbletea): TUI framework
-- [Minecraft](https://www.minecraft.net/): What makes this all worthwhile
+* [Kubernetes](https://kubernetes.io/): Container orchestration engine
+* [K3s](https://k3s.io/): Lightweight Kubernetes distribution
+* [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/): Zero trust security infrastructure
+* [PostgreSQL](https://www.postgresql.org/): Data persistence
+* [React](https://react.dev/): User interface framework
+* [Vite](https://vitejs.dev/): Frontend build tool
+* [TailwindCSS](https://tailwindcss.com/): CSS framework
+* [Bubble Tea](https://github.com/charmbracelet/bubbletea): TUI framework
+* [Minecraft](https://www.minecraft.net/): The game this project serves

bootstrap_asset.go

@@ -22,15 +22,18 @@ var bootstrapAssets embed.FS
 // developer's working tree carries gradle output (plugins/*/build, plugins/*/bin,
 // and for the modded loaders a decompiled Minecraft under build/) which would
 // otherwise be baked into every felis binary. Keep them explicit — add a source
-// directory here, never a parent.
+// directory here, never a parent. Each module's gradle/verification-metadata.xml rides
+// along, since Gradle builds unverified without it; the wrapper stays out, because the
+// image builds run the pinned build image's own gradle.
 //
+//go:embed deploy/game-stack.lock
 //go:embed deploy/limbo/Dockerfile deploy/limbo/entrypoint.sh
 //go:embed deploy/lobby/Dockerfile deploy/lobby/entrypoint.sh
 //go:embed deploy/paper/Dockerfile deploy/paper/entrypoint.sh
-//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src
-//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src
-//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src
-//go:embed plugins/shared/src
+//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src plugins/limbo/gradle/verification-metadata.xml
+//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src plugins/paper/gradle/verification-metadata.xml
+//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src plugins/velocity/gradle/verification-metadata.xml
+//go:embed plugins/shared/src plugins/shared/build-progress.gradle
 var gameStackAssets embed.FS
 
 // GameStackTar streams the embedded game-stack sources as a tar, rooted so that

bootstrap_asset_test.go

@@ -1,7 +1,9 @@
 package felis
 
 import (
+	"encoding/xml"
 	"io/fs"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -145,6 +147,9 @@ func TestBootstrapPinsViaBlockConnectionsOff(t *testing.T) {
 // inputs from the script and from each Dockerfile's own COPY lines instead of restating
 // them here; a fourth image inherits the check for free.
 func TestGameStackTarCarriesEveryBuildInput(t *testing.T) {
+	// The plugin builds invoke this shared init script after COPYing the directory;
+	// a directory entry alone would pass the COPY check even if the script was omitted.
+	requireEmbedded(t, "plugins/shared/build-progress.gradle")
 	// Matches the path only when GAME_STACK_DIR is followed by one, which skips the
 	// build-context arguments (`"$GAME_STACK_DIR"`, `"${GAME_STACK_DIR}:/src:z"`) and
 	// the glob for gradle's output, none of which are inputs this tar has to carry.
@@ -205,6 +210,299 @@ func requireEmbedded(t *testing.T, path string) {
 	}
 }
 
+// The lock file is the install's only source of upstream builds, and bootstrap.sh reads it
+// with a strict KEY=value parser that dies on anything unexpected, so a malformed lock is a
+// failed install on every host. Check the shipped copy the same way here.
+func TestGameStackLockIsComplete(t *testing.T) {
+	lock := gameStackLock(t)
+	m := regexp.MustCompile(`GAME_STACK_LOCK_KEYS="([^"]*)"`).FindStringSubmatch(BootstrapScript())
+	if m == nil {
+		t.Fatal("bootstrap.sh no longer declares GAME_STACK_LOCK_KEYS")
+	}
+	keys := strings.Fields(m[1])
+	sha := regexp.MustCompile(`^[0-9a-f]{64}$`)
+	for _, k := range keys {
+		v, ok := lock[k]
+		if !ok || v == "" {
+			t.Errorf("game-stack.lock does not set %s", k)
+			continue
+		}
+		if strings.HasSuffix(k, "_SHA256") && !sha.MatchString(v) {
+			t.Errorf("%s=%q is not a lowercase sha256", k, v)
+		}
+		// A moving URL pins nothing: the digest check would start failing the day
+		// upstream publishes the next build.
+		if strings.HasSuffix(k, "_URL") && strings.Contains(v, "lastSuccessfulBuild") {
+			t.Errorf("%s names a moving build: %s", k, v)
+		}
+	}
+	for k := range lock {
+		if !strings.Contains(" "+m[1]+" ", " "+k+" ") {
+			t.Errorf("game-stack.lock sets %s, which bootstrap.sh refuses as an unknown key", k)
+		}
+	}
+	// Fill's URLs are content-addressed; a lock whose digest disagrees with its own URL
+	// was edited by hand and half-way.
+	for _, name := range []string{"PAPER", "VELOCITY"} {
+		if !strings.Contains(lock[name+"_JAR_URL"], "/objects/"+lock[name+"_JAR_SHA256"]+"/") {
+			t.Errorf("%s_JAR_SHA256 is not the digest in %s_JAR_URL", name, name)
+		}
+	}
+	if !strings.Contains(lock["LIMBO_JAR_URL"], "-"+lock["MC_VERSION"]+".jar") {
+		t.Errorf("LIMBO_JAR_URL %s is not a Minecraft %s build", lock["LIMBO_JAR_URL"], lock["MC_VERSION"])
+	}
+	if !strings.Contains(lock["PAPER_JAR_URL"], "/paper-"+lock["MC_VERSION"]+"-") {
+		t.Errorf("PAPER_JAR_URL %s is not a Minecraft %s build; the lobby would not speak the login gate's protocol", lock["PAPER_JAR_URL"], lock["MC_VERSION"])
+	}
+}
+
+// Each downloaded jar's digest is a build-arg bootstrap.sh passes and the Dockerfile must
+// both require and spend on the file it downloaded; docker only warns about an unknown
+// --build-arg, so a renamed arg would ship an unchecked jar.
+func TestGameStackDigestsReachTheImageBuilds(t *testing.T) {
+	script := BootstrapScript()
+	for _, c := range []struct{ dockerfile, arg, path string }{
+		{"deploy/limbo/Dockerfile", "LIMBO_JAR_SHA256", "/limbo/Limbo.jar"},
+		{"deploy/limbo/Dockerfile", "LIMBO_SCHEM_SHA256", "/limbo/spawn.schem"},
+		{"deploy/lobby/Dockerfile", "LUCKPERMS_JAR_SHA256", "/paper/plugins/LuckPerms.jar"},
+	} {
+		if !strings.Contains(script, "--build-arg "+c.arg+"=\"$"+c.arg+"\"") {
+			t.Errorf("bootstrap.sh never passes --build-arg %s", c.arg)
+		}
+		dockerfile := readGameStackFile(t, c.dockerfile)
+		if !strings.Contains(dockerfile, "ARG "+c.arg) {
+			t.Errorf("%s declares no ARG %s", c.dockerfile, c.arg)
+		}
+		if !strings.Contains(dockerfile, `echo "$`+c.arg+`  `+c.path+`" | sha256sum -c`) {
+			t.Errorf("%s never verifies %s against %s", c.dockerfile, c.path, c.arg)
+		}
+	}
+}
+
+// A base image named by tag alone is whatever the tag points at on build day.
+func TestDockerfileBaseImagesArePinnedByDigest(t *testing.T) {
+	root, err := os.ReadFile("Dockerfile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	files := map[string]string{"Dockerfile": string(root)}
+	for _, name := range []string{"deploy/limbo/Dockerfile", "deploy/lobby/Dockerfile", "deploy/paper/Dockerfile"} {
+		files[name] = readGameStackFile(t, name)
+	}
+	pinned := regexp.MustCompile(`^FROM (--platform=\S+ )?\S+:\S+@sha256:[0-9a-f]{64}( AS \S+)?$`)
+	for name, body := range files {
+		n := 0
+		for line := range strings.SplitSeq(body, "\n") {
+			if !strings.HasPrefix(line, "FROM ") {
+				continue
+			}
+			n++
+			if !pinned.MatchString(line) {
+				t.Errorf("%s: %q is not pinned by digest", name, line)
+			}
+		}
+		if n == 0 {
+			t.Errorf("%s has no FROM line", name)
+		}
+	}
+}
+
+// The plugin jars are built in four places the installer controls — the lobby and limbo
+// image builds, bootstrap's Velocity build and the release build of the same jar — and
+// through each module's wrapper by a developer or CI. A tag alone is whatever it points
+// at on build day, and two Gradle versions are two chances for a build to pass in one
+// place and break in the other, so all of them run one image, pinned by digest, whose
+// Gradle is the wrappers' Gradle.
+func TestPluginBuildsRunOnePinnedGradle(t *testing.T) {
+	sources := map[string]string{
+		"deploy/lobby/Dockerfile": readGameStackFile(t, "deploy/lobby/Dockerfile"),
+		"deploy/limbo/Dockerfile": readGameStackFile(t, "deploy/limbo/Dockerfile"),
+		"deploy/bootstrap.sh":     BootstrapScript(),
+		// The release build compiles felis-velocity.jar for hosts that install prebuilt.
+		"deploy/build-release-artifacts.sh": readRepoFile(t, "deploy/build-release-artifacts.sh"),
+	}
+	anyRef := regexp.MustCompile(`gradle:[\w.-]+(@sha256:\w+)?`)
+	pinned := regexp.MustCompile(`^gradle:(\d+\.\d+(?:\.\d+)?)-jdk\d+@sha256:[0-9a-f]{64}$`)
+	images := map[string]bool{}
+	gradle := ""
+	for name, body := range sources {
+		refs := anyRef.FindAllString(body, -1)
+		if len(refs) == 0 {
+			t.Errorf("%s names no gradle image", name)
+		}
+		for _, ref := range refs {
+			m := pinned.FindStringSubmatch(ref)
+			if m == nil {
+				t.Errorf("%s: %s is not a gradle image pinned by digest", name, ref)
+				continue
+			}
+			images[ref] = true
+			gradle = m[1]
+		}
+	}
+	if len(images) != 1 {
+		t.Fatalf("the plugin builds use %d different gradle images, want one: %v", len(images), images)
+	}
+	// bootstrap names the image once and has to spend it where it builds the jar.
+	if !strings.Contains(BootstrapScript(), `"$PLUGIN_BUILD_IMAGE" gradle --no-daemon clean build`) {
+		t.Error("build_velocity_plugin does not build in $PLUGIN_BUILD_IMAGE")
+	}
+
+	for _, module := range []string{"velocity", "paper", "limbo"} {
+		props := wrapperProperties(t, module)
+		if want := "gradle-" + gradle + "-bin.zip"; !strings.HasSuffix(props["distributionUrl"], "/"+want) {
+			t.Errorf("plugins/%s wrapper runs %s; the image builds run Gradle %s", module, props["distributionUrl"], gradle)
+		}
+	}
+	// The mods are no part of the install, but a wrapper without a checksum runs whatever
+	// the download handed it.
+	for _, module := range []string{"velocity", "paper", "limbo", "fabric", "forge", "neoforge"} {
+		if sum := wrapperProperties(t, module)["distributionSha256Sum"]; !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(sum) {
+			t.Errorf("plugins/%s wrapper pins no distribution sha256 (got %q)", module, sum)
+		}
+	}
+}
+
+// Each plugin compiles against the API of the exact build the install runs, and Gradle
+// checks those bytes against the module's verification file. Nothing but this test ties
+// the three to deploy/game-stack.lock: a lock refresh that leaves them behind builds the
+// lobby against yesterday's API, or fails every image build on a checksum the file does
+// not have.
+func TestPluginApisAreTheLockedBuilds(t *testing.T) {
+	lock := gameStackLock(t)
+
+	// Paper: paper-<mc>-<build>.jar runs; paper-api <mc>.build.<build>-<channel> compiles.
+	jar := regexp.MustCompile(`/paper-([^/]+)-(\d+)\.jar$`).FindStringSubmatch(lock["PAPER_JAR_URL"])
+	if jar == nil {
+		t.Fatalf("PAPER_JAR_URL %s does not name paper-<mc>-<build>.jar", lock["PAPER_JAR_URL"])
+	}
+	dep := regexp.MustCompile(`compileOnly 'io\.papermc\.paper:paper-api:([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/paper/build.gradle"))
+	if dep == nil {
+		t.Fatal("plugins/paper/build.gradle declares no paper-api dependency")
+	}
+	if !regexp.MustCompile(`^` + regexp.QuoteMeta(jar[1]+".build."+jar[2]) + `(-[a-z]+)?$`).MatchString(dep[1]) {
+		t.Errorf("paper-api %s is not the API of the locked server paper-%s-%s.jar", dep[1], jar[1], jar[2])
+	}
+	requireVerified(t, "paper", "io.papermc.paper", "paper-api", dep[1])
+
+	// Limbo: the lock's release, passed to the image build, which refuses to guess one.
+	limbo := lock["LIMBO_VERSION"]
+	if !strings.Contains(BootstrapScript(), `--build-arg LIMBO_VERSION="$LIMBO_VERSION"`) {
+		t.Error("bootstrap.sh does not pass the locked LIMBO_VERSION to the limbo image build")
+	}
+	dockerfile := readGameStackFile(t, "deploy/limbo/Dockerfile")
+	if !regexp.MustCompile(`(?m)^ARG LIMBO_VERSION$`).MatchString(dockerfile) ||
+		!strings.Contains(dockerfile, `if [ -z "${LIMBO_VERSION:-}" ]`) {
+		t.Error("deploy/limbo/Dockerfile does not require LIMBO_VERSION; a build without it would " +
+			"compile against a version nobody chose")
+	}
+	// LOOHP publishes the jar the login gate runs as the Limbo API artifact itself, so the
+	// checksum Gradle holds for it is the lock's: compiled-against and running are one file.
+	if got := requireVerified(t, "limbo", "com.loohp", "Limbo", limbo)["Limbo-"+limbo+".jar"]; got != lock["LIMBO_JAR_SHA256"] {
+		t.Errorf("verification-metadata.xml holds %q for Limbo-%s.jar; the login gate runs %s", got, limbo, lock["LIMBO_JAR_SHA256"])
+	}
+
+	// Velocity: the API default is the proxy the install runs.
+	api := regexp.MustCompile(`findProperty\('velocityApi'\) \?: '([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/velocity/build.gradle"))
+	if api == nil {
+		t.Fatal("plugins/velocity/build.gradle has no velocityApi default")
+	}
+	if api[1] != lock["VELOCITY_VERSION"] {
+		t.Errorf("velocity-api defaults to %s; the install runs Velocity %s", api[1], lock["VELOCITY_VERSION"])
+	}
+	requireVerified(t, "velocity", "com.velocitypowered", "velocity-api", api[1])
+}
+
+// requireVerified asserts the module's shipped verification file checks metadata and
+// pins group:name:version, and returns that component's artifact sha256s by file name.
+func requireVerified(t *testing.T, module, group, name, version string) map[string]string {
+	t.Helper()
+	path := "plugins/" + module + "/gradle/verification-metadata.xml"
+	var doc struct {
+		VerifyMetadata bool `xml:"configuration>verify-metadata"`
+		Components     []struct {
+			Group     string `xml:"group,attr"`
+			Name      string `xml:"name,attr"`
+			Version   string `xml:"version,attr"`
+			Artifacts []struct {
+				Name   string `xml:"name,attr"`
+				SHA256 []struct {
+					Value string `xml:"value,attr"`
+				} `xml:"sha256"`
+			} `xml:"artifact"`
+		} `xml:"components>component"`
+	}
+	if err := xml.Unmarshal([]byte(readGameStackFile(t, path)), &doc); err != nil {
+		t.Fatalf("%s: %v", path, err)
+	}
+	if !doc.VerifyMetadata {
+		t.Errorf("%s does not verify metadata; a swapped pom could redirect the graph", path)
+	}
+	for _, c := range doc.Components {
+		if c.Group != group || c.Name != name || c.Version != version {
+			continue
+		}
+		sums := map[string]string{}
+		for _, a := range c.Artifacts {
+			if len(a.SHA256) > 0 {
+				sums[a.Name] = a.SHA256[0].Value
+			}
+		}
+		if sums[name+"-"+version+".jar"] == "" {
+			t.Errorf("%s pins %s:%s:%s but no sha256 for its jar", path, group, name, version)
+		}
+		return sums
+	}
+	t.Errorf("%s has no checksum for %s:%s:%s; the build would refuse it", path, group, name, version)
+	return nil
+}
+
+// wrapperProperties reads a module's gradle-wrapper.properties off disk: the wrappers are
+// for developers and CI, and nothing embeds them.
+func wrapperProperties(t *testing.T, module string) map[string]string {
+	t.Helper()
+	b, err := os.ReadFile("plugins/" + module + "/gradle/wrapper/gradle-wrapper.properties")
+	if err != nil {
+		t.Fatal(err)
+	}
+	props := map[string]string{}
+	for line := range strings.SplitSeq(string(b), "\n") {
+		if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok && !strings.HasPrefix(k, "#") {
+			props[k] = strings.ReplaceAll(v, `\:`, ":")
+		}
+	}
+	return props
+}
+
+// gameStackLock parses the shipped deploy/game-stack.lock the way bootstrap.sh does.
+func gameStackLock(t *testing.T) map[string]string {
+	t.Helper()
+	lock := map[string]string{}
+	for line := range strings.SplitSeq(readGameStackFile(t, "deploy/game-stack.lock"), "\n") {
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			t.Fatalf("not a KEY=value line: %q", line)
+		}
+		lock[k] = v
+	}
+	return lock
+}
+
+// readRepoFile reads a file of the checkout that the binary does not embed.
+func readRepoFile(t *testing.T, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
 func readGameStackFile(t *testing.T, name string) string {
 	t.Helper()
 	b, err := gameStackAssets.ReadFile(name)

cmd/felis/api.go

@@ -5,10 +5,14 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"felis.lolicon.best/internal/api"
@@ -16,19 +20,30 @@ import (
 	"felis.lolicon.best/internal/backupjob"
 	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
 	"felis.lolicon.best/internal/fileedit"
+	"felis.lolicon.best/internal/imagepin"
 	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/metrics"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/nodecontrol"
 	"felis.lolicon.best/internal/panel"
 	"felis.lolicon.best/internal/passkey"
+	"felis.lolicon.best/internal/placement"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/reaper"
+	"felis.lolicon.best/internal/registryprune"
 	"felis.lolicon.best/internal/restore"
-	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/retention"
 	"felis.lolicon.best/internal/submit"
+	"felis.lolicon.best/internal/worldexport"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -38,6 +53,22 @@ import (
 // the code marks Identity (UUIDs trusted verbatim); config can never add another.
 const mojangSessionServer = "https://sessionserver.mojang.com/session/minecraft/hasJoined"
 
+// passkeyRelyingParty is the one WebAuthn relying party both web faces share: its id
+// is the player console host, derived from server.root_domain the way the panel
+// handler derives it when auth.panel_hostname is unset, and its origins are that host
+// plus the operator host. An empty id means the install names no panel host at all.
+func passkeyRelyingParty(cfg *config.Config) (string, []string) {
+	rpID := defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname)
+	if rpID == "" {
+		return "", nil
+	}
+	origins := []string{"https://" + rpID, fmt.Sprintf("https://%s:%d", rpID, setupPanelNodePort())}
+	if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != rpID {
+		origins = append(origins, "https://"+admin, fmt.Sprintf("https://%s:%d", admin, setupPanelNodePort()))
+	}
+	return rpID, origins
+}
+
 // authSourcesFromConfig builds the multiplexer's priority list from the configured
 // [[auth_source]] entries: Mojang leads as the code-owned identity anchor (正版优先, the ONLY
 // Identity source — config can only append namespace-rewritten third-party sources, never a
@@ -47,16 +78,14 @@ func authSourcesFromConfig(configured []config.AuthSourceConfig) []api.AuthSourc
 	sources := make([]api.AuthSource, 0, len(configured)+1)
 	sources = append(sources, api.AuthSource{Tag: "mojang", URL: mojangSessionServer, Identity: true})
 	for _, s := range configured {
-		sources = append(sources, api.AuthSource{Tag: s.Tag, Prefix: s.Prefix, URL: s.URL})
+		sources = append(sources, api.AuthSource{Tag: s.Tag, Prefix: s.Prefix, URL: s.URL, APIURL: s.APIURL})
 	}
 	return sources
 }
 
 // cmdAPI runs felis-api: two listeners, two middleware chains (spec §7). The
-// internal face (service token) is fully wired. The external face is wired but
-// fails closed until an Access JWKS key function is configured — the verifier's
-// audience logic is unit-tested (internal/api), the JWKS source is a deployment
-// integration point.
+// internal face authenticates per-caller service tokens; the external face
+// authenticates the local session cookie.
 func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("api", flag.ContinueOnError)
 	fs.SetOutput(stderr)
@@ -79,9 +108,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	// Load already refused a malformed [audit] retention.
+	auditRetention, _ := cfg.Audit.RetentionPeriod()
+	if auditRetention == 0 {
+		fmt.Fprintln(stdout, "felis api: audit rows are kept forever ([audit] retention = \"forever\")")
+	} else {
+		fmt.Fprintf(stdout, "felis api: audit rows older than %d days are deleted ([audit] retention; export them first with felis db audit-export)\n", int(auditRetention/(24*time.Hour)))
+	}
+
 	ctx := ctrl.SetupSignalHandler()
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Before anything serves: an api on a schema it was not built for answers with
+	// errors, or writes rows the other version cannot read.
+	drv, err := openPodStore(ctx, cfg.Database.URL, "api", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis api: open database: %v\n", err)
 		return 1
@@ -108,15 +147,24 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	token := os.Getenv("FELIS_SERVICE_TOKEN")
-	if token == "" {
-		fmt.Fprintln(stderr, "felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers")
+	metrics.SetBuildInfo("api", resolvedVersion())
+
+	internalAuth, err := internalCallerTokens(os.Getenv)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: internal face tokens: %v\n", err)
+		return 1
+	}
+	for _, ct := range naming.CallerTokens {
+		if internalAuth[api.Caller(ct.Caller)] == "" {
+			fmt.Fprintf(stderr, "felis api: warning: %s unset — the internal face turns the %s caller away\n", ct.APIEnv, ct.Caller)
+		}
 	}
 
 	// Email one-time codes go through the [smtp] relay when one is configured; the
 	// password is read from the env var password_ref names (default SMTPPasswordEnv,
 	// injected from the felis-smtp Secret). No [smtp] host ⇒ mailer stays nil and
-	// deliverOTP logs each code server-side (the pre-SMTP bootstrap posture).
+	// every door that mails a code answers 503 mail_unavailable: a code that is
+	// not mailed is never written anywhere else either.
 	var mailer api.OTPMailer
 	if cfg.SMTP.Host != "" {
 		passRef := cfg.SMTP.PasswordRef
@@ -127,26 +175,30 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		if cfg.SMTP.Username != "" && password == "" {
 			fmt.Fprintf(stderr, "felis api: warning: [smtp] username is set but credentials env %s is empty — OTP sends will fail AUTH\n", passRef)
 		}
-		mailer = &mail.SMTP{
-			Host:     cfg.SMTP.Host,
-			Port:     cfg.SMTP.Port,
-			From:     cfg.SMTP.From,
-			Username: cfg.SMTP.Username,
-			Password: password,
+		mailer = smtpRelay(cfg.SMTP, password)
+		if !cfg.SMTP.TLSRequired() {
+			fmt.Fprintf(stderr, "felis api: warning: [smtp] %s may be sent codes without TLS (require_tls off or a relay on this host)\n", cfg.SMTP.Host)
 		}
 	} else {
-		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email one-time codes are logged, not mailed")
+		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email sign-in and verification are off (503 mail_unavailable); sign in with a passkey, or run felis setup to add a relay")
 	}
 
 	// Build subsystem (spec §16): the weak-SA build Job runs in the configured
 	// build namespace and pushes to the internal registry. The build Pod never
 	// holds DB credentials — felis-api owns the PG store and admits scanned
 	// images, so the Builder is constructed here with both bindings.
+	buildCfg := buildConfig(cfg)
+	// The fetch initContainer runs THIS image's fetch-context entrypoint, so the
+	// build config carries the api's own image (the platform sets FELIS_IMAGE).
+	buildCfg.FelisImage = os.Getenv("FELIS_IMAGE")
+	buildJobs := build.NewK8sJobs(cl, buildCfg)
 	builder := &build.Builder{
-		Store:  build.NewPGStore(drv.DB()),
-		Jobs:   build.NewK8sJobs(cl, buildConfig(cfg)),
-		Config: buildConfig(cfg),
+		Store:    build.NewPGStore(drv.DB()),
+		Jobs:     buildJobs,
+		Config:   buildCfg,
+		Outcomes: build.NewK8sOutcomes(clientset, buildCfg),
 	}
+	go probeBuildUserNamespaces(ctx, buildJobs, buildCfg, stderr)
 
 	// User-modpack approval lane (user-directed extension over §16; see
 	// internal/submit). An ordinary user may only SUBMIT a
@@ -159,14 +211,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// The blob upload transport is selected by the shape of user_uploads_context —
 	// the two backends the setup wizard chooses between. A local path wires
 	// LocalContextStore (the mounted uploads PVC); an s3:// base wires
-	// S3ContextStore when its credentials resolve. Either way the store's target is
-	// derived from the SAME config field the context ref uses, so the blob lands
-	// exactly where Kaniko's --context points. Anything else — or an s3:// base with
-	// no credentials configured — leaves Blobs nil so POST
+	// S3ContextStore when its credentials resolve. Anything else — or an s3:// base
+	// with no credentials configured — leaves Blobs nil so POST
 	// /me/submissions/{id}/context returns 503, honest like the restore executor
-	// when its PVC is not supplied. (Letting the sandboxed Kaniko build Pod READ the
-	// context — PVC mount for local, creds+egress for S3 — is a separate deployment
-	// integration.)
+	// when its PVC is not supplied.
+	//
+	// Reading the blob back is the API's job, not Kaniko's: the build Pod runs in
+	// another namespace and can neither mount the uploads PVC (a PVC does not cross
+	// namespaces) nor hold object-store credentials, so ContextBaseURL makes the
+	// derived context ref an internal-face URL that the build Job's fetch
+	// initContainer streams (cmd/felis fetch-context). The platform renders this
+	// address into the api Deployment (felis API base URL env); the fallback keeps
+	// a hand-rolled deployment working under the platform's default control
+	// namespace.
 	contextBase := cfg.Registry.UserUploadsContext
 	var blobs submit.Blobs
 	switch {
@@ -186,11 +243,27 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis api: user-uploads context %q is neither a local path nor an s3:// base — modpack upload transport disabled (POST /api/v1/me/submissions/{id}/context returns 503)\n", contextBase)
 	}
 	submissions := &submit.Manager{
-		Store:        submit.NewPGStore(drv.DB()),
-		Builds:       builder,
-		Registry:     cfg.Registry.URL,
-		ContextStore: contextBase,
-		Blobs:        blobs,
+		Store:          submit.NewPGStore(drv.DB()),
+		Builds:         builder,
+		Registry:       cfg.Registry.URL,
+		ContextStore:   contextBase,
+		ContextBaseURL: internalAPIBaseURL(),
+		Blobs:          blobs,
+	}
+	if blobs != nil {
+		submissions.Parts = &submit.PartStore{Dir: uploadPartsDir(contextBase)}
+	}
+	if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
+		if n, err := parseByteSize(v); err != nil || n <= 0 {
+			fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
+		} else {
+			submissions.MaxStoredBytesTotal = n
+		}
+	}
+	if n, err := contextMaxBytes(cfg); err != nil {
+		fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 512Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
+	} else {
+		submissions.MaxContextBytes = n
 	}
 
 	// Restore subsystem (spec §7): the weak-SA restore Job mounts the target
@@ -203,9 +276,23 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// store at load, so by this point cfg.Archive.Store is guaranteed tarLocal.)
 	var restorer api.Restorer
 	felisImage, backupPVC := os.Getenv("FELIS_IMAGE"), os.Getenv("FELIS_BACKUP_PVC")
+	worldResolver := placement.Resolve(cl, cfg.K8s.Namespace)
+	distribution, err := distributionManager(cl, cfg, felisImage)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if distribution != nil {
+		worldResolver = distribution.Resolve
+	}
 	if felisImage != "" && backupPVC != "" {
 		rcfg := restoreConfig(cfg, felisImage, backupPVC)
-		restorer = &restore.Restorer{Jobs: restore.NewK8sJobs(cl), Config: rcfg}
+		rcfg.ResolveWorld = worldResolver
+		var jobs restore.Jobs = restore.NewK8sJobs(cl)
+		if distribution != nil {
+			jobs = distribution
+		}
+		restorer = &restore.Restorer{Jobs: jobs, Config: rcfg}
 	} else {
 		fmt.Fprintln(stderr, "felis api: restore executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — restore endpoint returns 503")
 	}
@@ -218,11 +305,30 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// endpoint honestly returns 503.
 	var backuper api.Backuper
 	if felisImage != "" && backupPVC != "" {
-		backuper = &backupjob.Backuper{Jobs: backupjob.NewK8sJobs(cl), Config: backupConfig(cfg, felisImage, backupPVC)}
+		bcfg := backupConfig(cfg, felisImage, backupPVC)
+		bcfg.ResolveWorld = worldResolver
+		var jobs backupjob.Jobs = backupjob.NewK8sJobs(cl)
+		if distribution != nil {
+			jobs = distribution
+		}
+		backuper = &backupjob.Backuper{Jobs: jobs, Config: bcfg}
 	} else {
 		fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503")
 	}
 
+	// World export: a weak-SA Job mounts the world PVC, or the backup PVC, read-only
+	// and PUTs the archive to the internal face, which streams it on to the owner's
+	// browser (internal/worldexport). A backup export mounts the backup PVC, so it
+	// is wired under the restore gate; otherwise the export routes return 503.
+	var exporter api.Exporter
+	if felisImage != "" && backupPVC != "" {
+		ecfg := exportConfig(cfg, felisImage, backupPVC)
+		ecfg.ResolveWorld = worldResolver
+		exporter = worldexport.New(clientset, ecfg)
+	} else {
+		fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503")
+	}
+
 	// Server file editor: a weak-SA Job mounts ONLY the target world PVC and runs
 	// `felis files`, printing its result for felis-api to read back through
 	// pods/log (see internal/fileedit). It needs FELIS_IMAGE but — unlike restore
@@ -231,11 +337,27 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// endpoints honestly return 503. It takes the typed clientset rather than the
 	// controller-runtime client because the log subresource lives only on the typed
 	// CoreV1 client, and one client covers its Job create, Pod list, and log read.
+	//
+	// Uploads additionally stage their bytes on this pod's disk until the Job
+	// fetches them from the internal face; whatever a previous process staged is
+	// orphaned (the index is in memory), so the stage starts empty.
 	var files api.FileEditor
+	var fileStage *fileedit.Stage
+	var fileBrowser *fileedit.Browser
 	if felisImage != "" {
+		fcfg := fileEditConfig(cfg, felisImage)
+		fcfg.ResolveWorld = worldResolver
+		fileBrowser = &fileedit.Browser{BaseURL: internalAPIBaseURL()}
+		runner := fileedit.NewK8sRunner(clientset)
+		runner.Browser = fileBrowser
 		files = &fileedit.Editor{
-			Runner: fileedit.NewK8sRunner(clientset),
-			Config: fileEditConfig(cfg, felisImage),
+			Runner: runner,
+			Config: fcfg,
+		}
+		fileStage = &fileedit.Stage{Dir: fileStagingDir()}
+		if err := fileStage.Sweep(); err != nil {
+			fmt.Fprintf(stderr, "felis api: %v — file uploads return 503\n", err)
+			fileStage = nil
 		}
 	} else {
 		fmt.Fprintln(stderr, "felis api: file editor disabled (needs FELIS_IMAGE) — file endpoints return 503")
@@ -246,32 +368,73 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// the same store the auth handlers write to, so a login and the next request
 	// agree on what local auth knows.
 	repo := api.NewPGRepo(drv.DB())
+	if err := api.RegisterStorePool(drv.DB()); err != nil {
+		fmt.Fprintf(stderr, "felis api: store pool metrics unavailable: %v\n", err)
+	}
 
+	// The owner's on-demand backup levers come from [archive], the same keys the
+	// backup Job and the reaper read. A malformed key leaves the defaults in
+	// place here; the reaper Job fails on it and names it.
+	rcfg, err := reaperConfig(cfg)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: %v; using the default backup limits\n", err)
+		rcfg = reaper.DefaultConfig()
+	}
+
+	serverCache, serversSynced, err := startServerCache(ctx, restCfg, scheme, cfg.K8s.Namespace, stderr)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err)
+		return 1
+	}
+	cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced).WithDistributed(distribution != nil, os.Getenv("FELIS_CONTROLLER_NODE"))
+	if distribution != nil {
+		distribution.Record = func(ctx context.Context, b distributed.Backup) error {
+			keep, retention, ok := backupPolicy(b.Reason, rcfg)
+			if !ok {
+				return fmt.Errorf("unknown backup reason %s", b.Reason)
+			}
+			st := reaper.NewPGStore(drv.DB())
+			if err := st.InsertBackup(ctx, reaper.BackupRecord{ID: "bk-" + b.ID, ServerName: b.Server, FormerOwner: b.Owner, BackupRef: b.Receipt.Ref, SizeBytes: b.Receipt.Size, SHA256: b.Receipt.SHA256, Reason: b.Reason, ExpiresAt: time.Now().Add(retention)}); err != nil {
+				return err
+			}
+			pruneBackups(ctx, st, distribution.Archive, b.Server, b.Owner, b.Reason, keep, b.Protect, stdout, stderr)
+			return nil
+		}
+	}
+	jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace)
 	a := &api.API{
 		Repo:    repo,
-		Cluster: api.NewK8sCluster(cl, cfg.K8s.Namespace),
+		Cluster: cluster,
 		Console: api.NewK8sConsole(cl, cfg.K8s.Namespace),
 		Logs:    api.NewK8sLogStreamer(clientset, cfg.K8s.Namespace),
 		// Build-log stream (spec §16) is scoped to the BUILD namespace — the same
 		// value the Builder renders Jobs into — so it follows where build Pods run.
-		BuildLogs:   api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
-		Internal:    api.BearerTokenAuth{Token: token},
-		Builder:     builder,
-		Restorer:    restorer,
-		Backuper:    backuper,
-		Files:       files,
-		Submissions: submissions,
-		Mailer:      mailer,
-		// The external face is fronted by SessionAuth: it prefers a local session
-		// cookie (minted by the passwordless doors) and otherwise delegates to the
-		// Cloudflare-Access JWT verifier, so both auth models coexist on one face. The
-		// delegate's Keyfunc is intentionally nil — the JWT path fails closed until a
-		// JWKS-backed key function is wired (deployment integration point) — while the
-		// local session path is live the moment `felis breakGlass` flips
-		// local_auth_enabled on.
+		BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
+		Internal:  internalAuth,
+		Builder:   builder,
+		Images:    imagePinner(cfg.Registry.URL),
+		Restorer:  restorer,
+		Backuper:  backuper,
+		JobStatus: jobStatus,
+		// A restore starts with a safety snapshot; settleRestoreChains starts the
+		// restore behind each one.
+		RestoreChains: jobStatus,
+		Files:         files,
+		FileStage:     fileStage,
+		FileBrowser:   fileBrowser,
+		// The file Job fetches an upload from here; it runs in the minecraft
+		// namespace, where the internal face is reachable like it is for the login
+		// gate.
+		InternalBaseURL: internalAPIBaseURL(),
+		Exporter:        exporter,
+		Submissions:     submissions,
+		Mailer:          mailer,
+		Schedules:       repo,
+		// The external face authenticates the local session cookie the sign-in doors
+		// mint, live once `felis breakGlass` flips local_auth_enabled on. Cloudflare
+		// Access, when the install sits behind it, is enforced at the edge only.
 		External: api.SessionAuth{
 			Repo:          repo,
-			Delegate:      api.AccessVerifier{Audience: cfg.Auth.AccessJWTAud},
 			RootDomain:    cfg.Server.RootDomain,
 			AdminHostname: cfg.Auth.AdminHostname,
 		},
@@ -279,12 +442,32 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		AdminHostname: cfg.Auth.AdminHostname,
 		PanelHostname: cfg.Auth.PanelHostname,
 		WakeCooldown:  30 * time.Second,
+		// An owner may start one backup per server per manual_cooldown, and none
+		// while the store is at max_local_bytes (data-durability-9).
+		BackupCooldown: rcfg.ManualCooldown,
+		BackupStoreCap: rcfg.MaxLocalBytes,
+		// The user-modpack lane's per-user throttles: a create spaces out
+		// review-queue rows, an upload spaces out (up to 1 GiB) context streams.
+		// Separate keys, so the normal create→upload sequence stays immediate.
+		SubmitCreateCooldown: 30 * time.Second,
+		SubmitUploadCooldown: 15 * time.Second,
 		// Bound concurrent console/build-log SSE streams per principal. Generous enough
 		// for legitimate multi-tab / multi-server watching, while capping how many
 		// upstream follow connections a single caller can tie up if their streams stall.
 		MaxStreamsPerPrincipal: 16,
+		// Public sign-in doors, per client address: a person signing in makes a
+		// handful of calls, so 20 at once refilled at 20 a minute never bites a
+		// real user and still turns a spray into a trickle. The client address
+		// is the edge's header when the install names one (config.AuthConfig).
+		AuthDoorLimit:  api.RateLimit{Burst: 20, PerMinute: 20},
+		ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(),
+		MailLimit:      mailLimit(cfg.SMTP.MaxPerHour),
+	}
+	if a.ClientIPHeader != "" {
+		fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader)
+	} else {
+		fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)")
 	}
-	fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)")
 
 	// Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned
 	// identity anchor (正版优先); config can only append namespace-rewritten third-party
@@ -293,7 +476,8 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// [[auth_source]] is configured, so an empty list has to mean a Mojang-only relay, the
 	// same as under `felis nano`. A nil list would 204 every login, premium ones included.
 	a.AuthSources = authSourcesFromConfig(cfg.AuthSources)
-	fmt.Fprintf(stderr, "felis api: hasJoined multiplexer active — Mojang + %d third-party source(s)\n", len(cfg.AuthSources))
+	a.AuthSourceSettings = &api.AuthSourceSettings{Repo: repo, Defaults: a.AuthSources}
+	fmt.Fprintf(stderr, "felis api: hasJoined multiplexer active — Mojang + %d default third-party source(s); panel settings take precedence\n", len(cfg.AuthSources))
 
 	// Passkey (WebAuthn) verifier (spec §14, Phase 6). One relying party spans BOTH
 	// web faces: the RP id is the panel hostname (console.<root>), and because that is
@@ -301,22 +485,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// enrolled once asserts on either face — one binding, usable on the player console
 	// AND the operator console. Both hosts are therefore listed as permitted origins,
 	// while the RP id stays the panel host so the credential's scope is ONE relying
-	// party, not two. Wired only when auth.panel_hostname is configured; otherwise
-	// a.Passkey stays nil and the passkey routes honestly return 503 (the authenticated
-	// enrollment boundary is still enforced by the handlers).
-	if cfg.Auth.PanelHostname != "" {
-		origins := []string{"https://" + cfg.Auth.PanelHostname}
-		if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != cfg.Auth.PanelHostname {
-			origins = append(origins, "https://"+admin)
-		}
-		pv, err := passkey.New(cfg.Auth.PanelHostname, "Felis", origins)
-		if err != nil {
-			fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
-		} else {
-			a.Passkey = pv
-		}
+	// party, not two. Without a panel host (neither auth.panel_hostname nor
+	// server.root_domain) a.Passkey stays nil and the passkey routes honestly return
+	// 503 (the authenticated enrollment boundary is still enforced by the handlers).
+	if rpID, origins := passkeyRelyingParty(cfg); rpID == "" {
+		fmt.Fprintln(stderr, "felis api: passkey verifier disabled (no panel host: set server.root_domain or auth.panel_hostname) — passkey endpoints return 503")
+	} else if pv, err := passkey.New(rpID, "Felis", origins); err != nil {
+		fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
 	} else {
-		fmt.Fprintln(stderr, "felis api: passkey verifier disabled (auth.panel_hostname unset) — passkey endpoints return 503")
+		a.Passkey = pv
 	}
 
 	// Derive the console hostnames when felis.toml leaves them unset, exactly as the
@@ -325,7 +502,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain,
 		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname),
 		defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname),
-		resolvedVersion())
+		cfg.Velocity.GamePort, cfg.Velocity.GameVersion, resolvedVersion(), distribution != nil)
 	internalSrv := newAPIServer(*internalAddr, a.InternalHandler())
 	externalSrv := newAPIServer(cfg.Server.Listen, externalHandler)
 
@@ -347,16 +524,47 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// and advance any whose Job has reached a terminal phase. GET on a build also
 	// reconciles it, but this loop converges builds nobody is polling.
 	go reconcileBuilds(ctx, builder, stderr)
+	go settleRestoreChains(ctx, a, stderr)
+	go runSchedules(ctx, a, stderr)
+	// A daily restore point of every world played since its last one, taken
+	// once the server stops ([archive] scheduled_every; 0s turns it off).
+	if backuper != nil && rcfg.ScheduledEvery > 0 {
+		go scheduleBackups(ctx, &api.BackupScheduler{API: a, Store: repo, Jobs: jobStatus, Every: rcfg.ScheduledEvery}, stderr)
+	} else {
+		fmt.Fprintln(stderr, "felis api: scheduled backups off (needs the backup executor and [archive] scheduled_every above 0s)")
+	}
+
+	if pruner := registryPruner(cfg, builder.Store, cluster, stderr); pruner != nil {
+		go pruner.Loop(ctx, registryPruneInterval)
+	}
+	go reapRejectedContexts(ctx, submissions, stderr)
+	if fileStage != nil {
+		go expireFileSessions(ctx, fileStage, fileSessionSweep, stderr)
+	}
+	if exporter != nil {
+		go expireExports(ctx, a, exportSweep)
+	}
+	go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
+
+	if distribution != nil {
+		a.Distribution = distribution
+		go reconcileDistribution(ctx, distribution, stderr)
+	}
+	if socket := os.Getenv("FELIS_NODE_CONTROL_SOCKET"); socket != "" {
+		a.NodeControl = nodecontrol.NewClient(socket)
+		cluster.NodeMaintenanceGuard = api.NodeMaintenanceGuard(a.NodeControl)
+	}
+	servers := []*http.Server{internalSrv, externalSrv}
+	if httpsSrv != nil {
+		servers = append(servers, httpsSrv)
+	}
+	for _, srv := range servers {
+		srv.RegisterOnShutdown(a.CloseStreams)
+	}
 
 	select {
 	case <-ctx.Done():
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		_ = internalSrv.Shutdown(shutdownCtx)
-		_ = externalSrv.Shutdown(shutdownCtx)
-		if httpsSrv != nil {
-			_ = httpsSrv.Shutdown(shutdownCtx)
-		}
+		shutdownServers(servers, apiShutdownGrace, stderr)
 		return 0
 	case err := <-errc:
 		if err != nil && err != http.ErrServerClosed {
@@ -376,8 +584,30 @@ const (
 	// apiIdleTimeout caps how long a kept-alive connection may sit idle between
 	// requests before the server closes it, bounding idle-connection exhaustion.
 	apiIdleTimeout = 120 * time.Second
+	// apiShutdownGrace is how long the listeners drain after SIGTERM. The pod gets
+	// the Kubernetes default of 30s before SIGKILL; this leaves the rest for the
+	// process to exit.
+	apiShutdownGrace = 20 * time.Second
 )
 
+// shutdownServers drains every listener at once under one deadline: in turn, a
+// slow first listener would spend the time the others needed. Log streams end
+// through RegisterOnShutdown (API.CloseStreams); what is still running when the
+// deadline passes is cut off with the process.
+func shutdownServers(servers []*http.Server, grace time.Duration, stderr io.Writer) {
+	ctx, cancel := context.WithTimeout(context.Background(), grace)
+	defer cancel()
+	var wg sync.WaitGroup
+	for _, srv := range servers {
+		wg.Go(func() {
+			if err := srv.Shutdown(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: shutdown %s: %v\n", srv.Addr, err)
+			}
+		})
+	}
+	wg.Wait()
+}
+
 // newAPIServer builds an http.Server with hardened header/idle timeouts (gosec
 // G112) shared by all three felis-api listeners (internal, external, https).
 // WriteTimeout and ReadTimeout are deliberately LEFT UNSET: the external and https
@@ -401,9 +631,63 @@ func buildConfig(cfg *config.Config) build.Config {
 	return build.Config{
 		Namespace:   cfg.Registry.BuildNamespace,
 		RegistryURL: cfg.Registry.URL,
+		// Empty overrides fall back to the registry's copies of the tools
+		// (build.Tools), which felis mirror-build-tools keeps current.
+		KanikoImage: cfg.Registry.KanikoImage,
+		TrivyImage:  cfg.Registry.TrivyImage,
+		CPULimit:    cfg.Registry.BuildCPULimit,
+		MemLimit:    cfg.Registry.BuildMemLimit,
+		DiskLimit:   cfg.Registry.BuildDiskLimit,
+		// "auto" follows the startup probe (see probeBuildUserNamespaces).
+		UserNamespaces:        cfg.Registry.BuildUserNamespaces,
+		UserNamespacesProbe:   new(atomic.Bool),
+		RuntimeClass:          cfg.Registry.BuildRuntimeClass,
+		MaxConcurrent:         cfg.Registry.MaxConcurrentBuilds,
+		TrivyDBRepository:     cfg.Registry.TrivyDBRepository,
+		TrivyJavaDBRepository: cfg.Registry.TrivyJavaDBRepository,
+		ScanFailOn:            cfg.Registry.ScanFailOn,
+		ScanFailUnfixed:       cfg.Registry.ScanFailUnfixed,
+		ScanAccept:            cfg.Registry.ScanAccept,
+		// The submit lane's derived context URLs live here; the fetch step's
+		// service token goes nowhere else.
+		ContextOrigin: internalAPIBaseURL(),
 	}
 }
 
+// probeBuildUserNamespaces settles build_user_namespaces = "auto": one probe
+// pod with hostUsers: false tells whether this node's kernel and runtime can run
+// build pods in a user namespace. Builds submitted before it answers run without.
+func probeBuildUserNamespaces(ctx context.Context, jobs *build.K8sJobs, cfg build.Config, stderr io.Writer) {
+	if mode := cfg.UserNamespaces; mode != "" && mode != build.UserNamespacesAuto {
+		return
+	}
+	if cfg.FelisImage == "" {
+		fmt.Fprintln(stderr, "felis api: FELIS_IMAGE unset — build pods run without a user namespace")
+		return
+	}
+	ok, err := jobs.ProbeUserNamespaces(ctx, cfg.FelisImage)
+	cfg.UserNamespacesProbe.Store(ok)
+	switch {
+	case ok:
+		fmt.Fprintln(stderr, "felis api: build pods run in a user namespace (hostUsers: false)")
+	case err != nil:
+		fmt.Fprintf(stderr, "felis api: build pods run without a user namespace: the probe failed: %v\n", err)
+	default:
+		fmt.Fprintln(stderr, "felis api: build pods run without a user namespace: this node cannot start a pod with hostUsers: false")
+	}
+}
+
+// internalAPIBaseURL resolves the platform's internal-face base URL: the address
+// the platform rendered into this pod (felis API base URL env), or — for a
+// hand-rolled deployment that set none — the platform default control namespace,
+// the same fallback setup.go uses to hand the login gate its address.
+func internalAPIBaseURL() string {
+	if base := os.Getenv(naming.EnvAPIBaseURL); base != "" {
+		return base
+	}
+	return platform.InternalAPIBaseURL(platform.DefaultControlNamespace)
+}
+
 // uploadsSchemeRE matches a leading URL scheme like "s3://" or "gs://".
 var uploadsSchemeRE = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://`)
 
@@ -477,6 +761,17 @@ func backupConfig(cfg *config.Config, image, backupPVC string) backupjob.Config
 	}
 }
 
+// exportConfig builds the world export executor's config. BackupRoot mirrors
+// restoreConfig: the stored refs are absolute paths under [archive] local_path.
+func exportConfig(cfg *config.Config, image, backupPVC string) worldexport.Config {
+	return worldexport.Config{
+		Namespace:  cfg.K8s.Namespace,
+		Image:      image,
+		BackupPVC:  backupPVC,
+		BackupRoot: cfg.Archive.LocalPath,
+	}
+}
+
 // fileEditConfig builds the file editor's config from felis.toml plus the
 // deployment-supplied image. It is the shortest of the three: the editor mounts
 // only the world PVC, so it needs no archive coordinates at all, and everything
@@ -506,3 +801,339 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
 		}
 	}
 }
+
+// settleRestoreChains starts the restore behind each safety snapshot that has
+// finished (and gives up the one behind a snapshot that failed). The world stays
+// locked in between, so the interval is how long a finished snapshot keeps the
+// server down before its restore begins.
+func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
+	t := time.NewTicker(5 * time.Second)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := a.SettleRestoreChains(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: restore chains: %v\n", err)
+			}
+		}
+	}
+}
+
+// runSchedules runs the servers' scheduled tasks (api.API.RunSchedules). The
+// interval is how late a task may start, and how often a restart or backup in
+// progress checks whether it can take its next step.
+func runSchedules(ctx context.Context, a *api.API, stderr io.Writer) {
+	t := time.NewTicker(15 * time.Second)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := a.RunSchedules(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: scheduled tasks: %v\n", err)
+			}
+		}
+	}
+}
+
+// scheduleBackups starts the scheduled backups (api.BackupScheduler). Each tick
+// starts at most one, so the interval also spaces the worlds that stopped at
+// the same time: a world that stops waits at most this long for its point to
+// start once the Jobs ahead of it are done.
+func scheduleBackups(ctx context.Context, s *api.BackupScheduler, stderr io.Writer) {
+	t := time.NewTicker(2 * time.Minute)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := s.Tick(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: scheduled backups: %v\n", err)
+			}
+		}
+	}
+}
+
+// fileSessionSweep is how often expireFileSessions looks for idle upload
+// sessions: small beside fileedit.SessionIdle, so an abandoned one gives its
+// room back within minutes of going stale.
+const fileSessionSweep = 10 * time.Minute
+
+// expireFileSessions drops the file manager's upload sessions left untouched
+// for fileedit.SessionIdle. Each reserved room on the staging disk for its whole
+// file when it began, so one abandoned would otherwise hold that room until
+// felis-api restarts.
+func expireFileSessions(ctx context.Context, s *fileedit.Stage, every time.Duration, stderr io.Writer) {
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if n := s.Expire(); n > 0 {
+				fmt.Fprintf(stderr, "felis api: dropped %d upload session(s) left idle for %s\n", n, fileedit.SessionIdle)
+			}
+		}
+	}
+}
+
+// exportSweep is how often expireExports runs: an export whose Job never
+// connected is stopped within a minute of going stale.
+const exportSweep = time.Minute
+
+// expireExports runs the export sweep (api.API.ExpireExports) on a ticker. The
+// export routes sweep as they are called, and an owner who closed the tab calls
+// none; a Job whose Pod never got going would then keep the server from
+// starting until the Job's deadline.
+func expireExports(ctx context.Context, a interface{ ExpireExports() }, every time.Duration) {
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			a.ExpireExports()
+		}
+	}
+}
+
+// reapRejectedContexts deletes, once an hour, the uploaded contexts of
+// submissions rejected more than submit.RejectedContextRetention ago, and the
+// chunked uploads left untouched for submit.StalePartRetention. Without it a
+// rejected modpack or an abandoned upload keeps its bytes on the uploads store
+// (and against its submitter's budget) until an admin deletes the row.
+func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
+	t := time.NewTicker(time.Hour)
+	defer t.Stop()
+	for {
+		n, err := m.ReapRejected(ctx, submit.RejectedContextRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap rejected uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
+		}
+		n, err = m.ReapStaleParts(submit.StalePartRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap abandoned uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted %d abandoned chunked upload(s)\n", n)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// retentionInterval spaces the runs that delete spent sign-in rows and audit rows
+// past [audit] retention. The rows are spent for weeks before they go, so a few
+// runs a day keep the tables flat.
+const retentionInterval = 6 * time.Hour
+
+// registryPruneInterval spaces the registry pruner's runs. The registry-gc
+// sidecar sweeps once a day, so pruning more often only changes which sweep frees
+// a layer.
+const registryPruneInterval = 6 * time.Hour
+
+// registryPruner deletes the registry manifests nothing references
+// (internal/registryprune); the registry-gc sidecar frees their layers on its next
+// sweep. It acts as the gate's prune principal, whose token the api Deployment
+// injects from felis-registry-auth. Without the token the registry only grows,
+// which is said once here.
+func registryPruner(cfg *config.Config, store imageRefStore, servers serverLister, stderr io.Writer) *registryprune.Pruner {
+	if cfg.Registry.URL == "" {
+		return nil
+	}
+	token := os.Getenv(platform.RegistryPruneTokenEnv)
+	if token == "" {
+		fmt.Fprintf(stderr, "felis api: registry pruner disabled (%s unset) — images nothing uses are never deleted from the registry\n", platform.RegistryPruneTokenEnv)
+		return nil
+	}
+	static := append([]string{os.Getenv("FELIS_IMAGE")}, buildConfig(cfg).ToolRefs()...)
+	return &registryprune.Pruner{
+		Registry: &registryprune.Client{Endpoint: "http://" + cfg.Registry.URL, Token: token},
+		Host:     cfg.Registry.URL,
+		Refs: func(ctx context.Context) ([]string, error) {
+			return inUseImageRefs(ctx, store, servers, static)
+		},
+		Log: slog.New(slog.NewTextHandler(stderr, nil)),
+	}
+}
+
+type imageRefStore interface {
+	ListImages(ctx context.Context) ([]build.Image, error)
+	ListUnfinishedBuilds(ctx context.Context) ([]build.Build, error)
+}
+
+type serverLister interface {
+	ListServers(ctx context.Context) ([]api.ServerInfo, error)
+	PodImages(ctx context.Context) ([]string, error)
+}
+
+// inUseImageRefs lists every image reference the platform still depends on: the
+// whitelist (disabled rows too, an admin may enable them again), every server's
+// spec, the images the game pods run, builds still running, and the images the
+// control plane and the build Jobs run. Any source failing fails the whole list,
+// so the pruner never decides on a partial view.
+//
+// The pods matter for the felis image: a running server keeps the one it started
+// with across platform upgrades (operator.PodTemplateAnnotation), which after a
+// few releases is no longer among the newest tags the pruner keeps anyway, and
+// the pod needs it again whenever it is recreated.
+func inUseImageRefs(ctx context.Context, store imageRefStore, servers serverLister, static []string) ([]string, error) {
+	refs := append([]string(nil), static...)
+	images, err := store.ListImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("image whitelist: %w", err)
+	}
+	for _, img := range images {
+		refs = append(refs, img.ImageRef)
+	}
+	srvs, err := servers.ListServers(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("servers: %w", err)
+	}
+	for _, s := range srvs {
+		refs = append(refs, s.Image)
+	}
+	podImages, err := servers.PodImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("game pods: %w", err)
+	}
+	refs = append(refs, podImages...)
+	builds, err := store.ListUnfinishedBuilds(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("running builds: %w", err)
+	}
+	for _, b := range builds {
+		refs = append(refs, b.ImageRef)
+	}
+	return refs, nil
+}
+
+// mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket:
+// the hourly cap as the refill rate, with a quarter of it (at least 5) allowed
+// at once so a burst of real sign-ins is not queued behind the average.
+func mailLimit(perHour int) api.RateLimit {
+	if perHour <= 0 {
+		perHour = config.DefaultMailPerHour
+	}
+	return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60}
+}
+
+// imagePinner resolves a new server's image against the platform registry
+// through its in-cluster Service, the address its refs already spell. An install
+// without a registry has no platform-built images to pin.
+func imagePinner(registry string) api.ImagePinner {
+	if registry == "" {
+		return nil
+	}
+	return imagepin.Resolver{Registry: registry}
+}
+
+// internalCallerTokens reads each internal caller's token from the env var the
+// Deployment feeds it from (naming.CallerTokens). Two callers sharing a value
+// would make the caller ambiguous, so that refuses to start.
+func internalCallerTokens(getenv func(string) string) (api.CallerTokens, error) {
+	tokens := map[api.Caller]string{}
+	for _, ct := range naming.CallerTokens {
+		tokens[api.Caller(ct.Caller)] = strings.TrimSpace(getenv(ct.APIEnv))
+	}
+	return api.NewCallerTokens(tokens)
+}
+
+// smtpRelay is the relay [smtp] names, with the resolved password and the TLS
+// posture config.SMTPConfig.TLSRequired picks. felis api, the reaper and the
+// watchdog all send through it, so none can drift to a weaker posture.
+func smtpRelay(c config.SMTPConfig, password string) *mail.SMTP {
+	return &mail.SMTP{
+		Host:       c.Host,
+		Port:       c.Port,
+		From:       c.From,
+		Username:   c.Username,
+		Password:   password,
+		RequireTLS: c.TLSRequired(),
+	}
+}
+
+// startServerCache starts the informer that serves the api's fleet-wide
+// MinecraftServer reads (api.K8sCluster.WithServerCache): one watch on the
+// namespace instead of a full List per velocity pull, fleet page and wake. It
+// caches MinecraftServers only — ReaderFailOnMissingInformer turns any other read
+// through it into an error rather than a new informer the api's Role cannot back —
+// indexes spec.subdomain for GetBySubdomain, and drops managedFields to keep the
+// copy small. It returns without waiting: the reads block until the first list
+// lands and /readyz reports not-ready until then.
+func startServerCache(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, stderr io.Writer) (cache.Cache, func() bool, error) {
+	c, err := cache.New(cfg, cache.Options{
+		Scheme:                      scheme,
+		DefaultNamespaces:           map[string]cache.Config{namespace: {}},
+		DefaultTransform:            cache.TransformStripManagedFields(),
+		ReaderFailOnMissingInformer: true,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := c.IndexField(ctx, &v1alpha1.MinecraftServer{}, api.SubdomainIndex, api.SubdomainOf); err != nil {
+		return nil, nil, fmt.Errorf("index %s: %w", api.SubdomainIndex, err)
+	}
+	inf, err := c.GetInformer(ctx, &v1alpha1.MinecraftServer{})
+	if err != nil {
+		return nil, nil, err
+	}
+	go func() {
+		if err := c.Start(ctx); err != nil {
+			fmt.Fprintf(stderr, "felis api: MinecraftServer cache stopped: %v\n", err)
+		}
+	}()
+	return c, inf.HasSynced, nil
+}
+
+// uploadPartsDir is where chunked uploads are staged: beside a local store's
+// contexts, so the room check and the budget see one disk and a staged upload
+// survives an API restart; for an s3:// store, on the uploads volume the
+// platform mounts either way, or the pod's /tmp when run by hand without it.
+func uploadPartsDir(contextBase string) string {
+	if isLocalUploadsPath(contextBase) {
+		return filepath.Join(strings.TrimPrefix(contextBase, "file://"), ".parts")
+	}
+	if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
+		return filepath.Join(platform.UploadsLocalPath, ".parts")
+	}
+	return filepath.Join(os.TempDir(), "felis-upload-parts")
+}
+
+// fileStagingDir is where file uploads wait for their Job: on the uploads
+// volume, whose capacity is its own, or the pod's /tmp when run by hand without
+// it — /tmp is the node's disk, which a burst of uploads should not fill.
+func fileStagingDir() string {
+	if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
+		return filepath.Join(platform.UploadsLocalPath, ".file-staging")
+	}
+	return filepath.Join(os.TempDir(), "felis-file-staging")
+}
+
+// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
+// package's own default (1 GiB). The Cloudflare edge refuses a single request
+// body over 100 MB, which the panel's chunked upload stays under, so the edge
+// does not lower the cap.
+func contextMaxBytes(cfg *config.Config) (int64, error) {
+	v := cfg.Registry.ContextMaxBytes
+	if v == "" {
+		return 0, nil
+	}
+	n, err := parseByteSize(v)
+	if err != nil || n <= 0 {
+		return 0, fmt.Errorf("not a positive size: %q", v)
+	}
+	return n, nil
+}

cmd/felis/api_test.go

@@ -1,12 +1,71 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
 	"net/http"
+	"slices"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/fileedit"
 )
 
+// The passkey relying party follows the panel host the SPA is served on: an install
+// that names only its root domain still gets passkeys, on console.<root>, with the
+// operator host as the second origin; only an install with no panel host goes without.
+func TestPasskeyRelyingParty(t *testing.T) {
+	t.Setenv("FELIS_PANEL_NODEPORT", "")
+	for _, tc := range []struct {
+		name               string
+		root, panel, admin string
+		wantRP             string
+		wantOrigins        []string
+	}{
+		{"root domain only", "example.net", "", "", "console.example.net",
+			[]string{"https://console.example.net", "https://op.console.example.net"}},
+		{"configured hosts", "example.net", " play.example.net ", "ops.example.net", "play.example.net",
+			[]string{"https://play.example.net", "https://ops.example.net"}},
+		{"operator host equal to the panel host", "example.net", "console.example.net", "console.example.net",
+			"console.example.net", []string{"https://console.example.net"}},
+		{"panel host without a root domain", "", "console.example.org", "", "console.example.org",
+			[]string{"https://console.example.org"}},
+		{"no host at all", "", "", "", "", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := &config.Config{}
+			cfg.Server.RootDomain, cfg.Auth.PanelHostname, cfg.Auth.AdminHostname = tc.root, tc.panel, tc.admin
+			var wantOrigins []string
+			for _, origin := range tc.wantOrigins {
+				wantOrigins = append(wantOrigins, origin, fmt.Sprintf("%s:%d", origin, defaultPanelNodePort))
+			}
+			rp, origins := passkeyRelyingParty(cfg)
+			if rp != tc.wantRP || !slices.Equal(origins, wantOrigins) {
+				t.Fatalf("relying party = %q %q, want %q %q", rp, origins, tc.wantRP, wantOrigins)
+			}
+		})
+	}
+}
+
+func TestPasskeyRelyingPartyIncludesConfiguredNodePort(t *testing.T) {
+	t.Setenv("FELIS_PANEL_NODEPORT", "30445")
+	cfg := &config.Config{}
+	cfg.Server.RootDomain = "example.com"
+	_, origins := passkeyRelyingParty(cfg)
+	if !slices.Contains(origins, "https://op.console.example.com:30445") {
+		t.Fatalf("configured NodePort origin missing: %v", origins)
+	}
+}
+
 // TestAuthSourcesFromConfig pins the one place the hasJoined identity anchor is decided:
 // Mojang is prepended in code, first, and is the only source whose UUIDs are trusted as-is.
 // The empty case matters on its own — both `felis api` and `felis nano` call this with a
@@ -20,7 +79,7 @@ func TestAuthSourcesFromConfig(t *testing.T) {
 		{"no configured sources", nil},
 		{"configured sources", []config.AuthSourceConfig{
 			{Tag: "littleskin", Prefix: "LS", URL: "https://littleskin.example/hasJoined"},
-			{Tag: "guild", Prefix: "GD", URL: "https://guild.example/hasJoined"},
+			{Tag: "guild", Prefix: "GD", URL: "https://guild.example/hasJoined", APIURL: "https://guild.example/api"},
 		}},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
@@ -36,7 +95,7 @@ func TestAuthSourcesFromConfig(t *testing.T) {
 				if s.Identity {
 					t.Errorf("configured source %q is marked Identity; only Mojang may be", c.Tag)
 				}
-				if s.Tag != c.Tag || s.Prefix != c.Prefix || s.URL != c.URL {
+				if s.Tag != c.Tag || s.Prefix != c.Prefix || s.URL != c.URL || s.APIURL != c.APIURL {
 					t.Errorf("source %d = %+v, want %+v in config order", i+1, s, c)
 				}
 			}
@@ -44,6 +103,32 @@ func TestAuthSourcesFromConfig(t *testing.T) {
 	}
 }
 
+// TestBuildConfig_ProjectsOverrides pins the [registry] overrides reaching the
+// build subsystem: unset fields must stay EMPTY (the build package's compiled-in
+// defaults apply there, not here), and set fields must pass through verbatim —
+// an air-gapped install points these at its imported mirrors.
+func TestBuildConfig_ProjectsOverrides(t *testing.T) {
+	empty := buildConfig(&config.Config{})
+	if empty.KanikoImage != "" || empty.TrivyImage != "" || empty.CPULimit != "" || empty.MemLimit != "" {
+		t.Errorf("empty registry config must project empty overrides (defaults live in internal/build), got %+v", empty)
+	}
+	full := buildConfig(&config.Config{Registry: config.RegistryConfig{
+		URL:            "registry.felis.svc:5000",
+		BuildNamespace: "felis-build",
+		KanikoImage:    "reg/kaniko:v1",
+		TrivyImage:     "reg/trivy:v1",
+		BuildCPULimit:  "1",
+		BuildMemLimit:  "2Gi",
+	}})
+	if full.KanikoImage != "reg/kaniko:v1" || full.TrivyImage != "reg/trivy:v1" ||
+		full.CPULimit != "1" || full.MemLimit != "2Gi" {
+		t.Errorf("registry overrides did not reach build.Config: %+v", full)
+	}
+	if full.Namespace != "felis-build" || full.RegistryURL != "registry.felis.svc:5000" {
+		t.Errorf("namespace/registry url must keep projecting: %+v", full)
+	}
+}
+
 // TestNewAPIServerSetsHardenedTimeouts pins the gosec-G112 hardening on every
 // felis-api listener: the shared factory must bound the header and idle phases
 // (Slowloris + idle-connection exhaustion) while leaving WriteTimeout UNSET, because
@@ -65,3 +150,179 @@ func TestNewAPIServerSetsHardenedTimeouts(t *testing.T) {
 		t.Errorf("ReadTimeout = %v, want 0 (unset) so a slow SSE attach is not capped", srv.ReadTimeout)
 	}
 }
+
+// Every listener drains at once, and the shutdown hook (API.CloseStreams in
+// cmdAPI) runs on each: two listeners each holding a request that ends when
+// the hook fires take one hook's worth of time, well inside the deadline.
+func TestShutdownServersDrainsListenersTogether(t *testing.T) {
+	release := make(chan struct{})
+	var hooks int32
+	started := make(chan struct{}, 2)
+	var servers []*http.Server
+	for range 2 {
+		srv := newAPIServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			started <- struct{}{}
+			<-release
+		}))
+		srv.RegisterOnShutdown(func() {
+			if atomic.AddInt32(&hooks, 1) == 1 {
+				time.AfterFunc(100*time.Millisecond, func() { close(release) })
+			}
+		})
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			t.Fatal(err)
+		}
+		srv.Addr = l.Addr().String()
+		go func() { _ = srv.Serve(l) }()
+		go func() {
+			if resp, err := http.Get("http://" + srv.Addr); err == nil {
+				resp.Body.Close()
+			}
+		}()
+		servers = append(servers, srv)
+	}
+	<-started
+	<-started
+
+	begun := time.Now()
+	shutdownServers(servers, 5*time.Second, io.Discard)
+	if took := time.Since(begun); took > 2*time.Second {
+		t.Fatalf("shutdown took %v", took)
+	}
+	if got := atomic.LoadInt32(&hooks); got != 2 {
+		t.Fatalf("shutdown hook ran %d times, want once per listener", got)
+	}
+}
+
+type fakeRefStore struct {
+	images []build.Image
+	builds []build.Build
+	err    error
+}
+
+func (f fakeRefStore) ListImages(context.Context) ([]build.Image, error) { return f.images, f.err }
+func (f fakeRefStore) ListUnfinishedBuilds(context.Context) ([]build.Build, error) {
+	return f.builds, nil
+}
+
+type fakeServers struct {
+	list    []api.ServerInfo
+	pods    []string
+	podsErr error
+}
+
+func (f fakeServers) ListServers(context.Context) ([]api.ServerInfo, error) { return f.list, nil }
+func (f fakeServers) PodImages(context.Context) ([]string, error)           { return f.pods, f.podsErr }
+
+// The registry pruner deletes whatever this list does not name, so every source of
+// a reference has to be in it, and a failing source must fail the list.
+func TestInUseImageRefsCoversEverySource(t *testing.T) {
+	const reg = "registry.felis.svc:5000/"
+	store := fakeRefStore{
+		images: []build.Image{{ImageRef: reg + "modpacks/pack:*"}, {ImageRef: reg + "felis/paper:demo"}},
+		builds: []build.Build{{ImageRef: reg + "user-uploads/sub-9:latest"}},
+	}
+	servers := fakeServers{
+		list: []api.ServerInfo{{Name: "s1", Image: reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1)}},
+		pods: []string{reg + "felis/felis:v1.0.0"},
+	}
+	got, err := inUseImageRefs(context.Background(), store, servers, []string{reg + "felis/felis:b60"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		reg + "felis/felis:b60",
+		reg + "modpacks/pack:*", reg + "felis/paper:demo",
+		reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1),
+		reg + "felis/felis:v1.0.0",
+		reg + "user-uploads/sub-9:latest",
+	}
+	if fmt.Sprint(got) != fmt.Sprint(want) {
+		t.Fatalf("refs = %v\nwant %v", got, want)
+	}
+
+	servers.podsErr = errors.New("apiserver down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing pod list produced a reference list")
+	}
+	servers.podsErr = nil
+
+	store.err = errors.New("db down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing whitelist read produced a reference list")
+	}
+}
+
+// TestExpireFileSessions runs the loop against a stage whose clock the test
+// holds: the session idle past fileedit.SessionIdle goes, the one touched since
+// stays, and the drop is said once.
+func TestExpireFileSessions(t *testing.T) {
+	var mu sync.Mutex
+	now := time.Date(2026, 9, 28, 10, 0, 0, 0, time.UTC)
+	advance := func(d time.Duration) { mu.Lock(); now = now.Add(d); mu.Unlock() }
+	st := &fileedit.Stage{Dir: t.TempDir(), MinFree: 1e-9, Now: func() time.Time {
+		mu.Lock()
+		defer mu.Unlock()
+		return now
+	}}
+	idle, err := st.Begin("u1", "survival", "a.jar", 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	advance(fileedit.SessionIdle - time.Minute)
+	fresh, err := st.Begin("u1", "survival", "b.jar", 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	advance(2 * time.Minute)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var out bytes.Buffer
+	done := make(chan struct{})
+	go func() { expireFileSessions(ctx, st, time.Millisecond, &out); close(done) }()
+	for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(time.Millisecond) {
+		if _, err := st.Status("u1", "survival", idle.ID); errors.Is(err, fileedit.ErrNotStaged) {
+			break
+		}
+		if time.Now().After(deadline) {
+			cancel()
+			t.Fatal("the idle session was never dropped")
+		}
+	}
+	// A few more ticks with nothing idle, which must stay quiet.
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+	<-done
+	if _, err := st.Status("u1", "survival", fresh.ID); err != nil {
+		t.Fatalf("the session touched since went too: %v", err)
+	}
+	if got := out.String(); got != "felis api: dropped 1 upload session(s) left idle for 6h0m0s\n" {
+		t.Fatalf("said %q", got)
+	}
+}
+
+type sweepCount struct{ n atomic.Int32 }
+
+func (s *sweepCount) ExpireExports() { s.n.Add(1) }
+
+// TestExpireExports: the loop sweeps on each tick, and returns once felis-api
+// shuts down.
+func TestExpireExports(t *testing.T) {
+	var s sweepCount
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { expireExports(ctx, &s, time.Millisecond); close(done) }()
+	for deadline := time.Now().Add(5 * time.Second); s.n.Load() < 3; time.Sleep(time.Millisecond) {
+		if time.Now().After(deadline) {
+			cancel()
+			t.Fatalf("swept %d times in 5s at a 1ms tick", s.n.Load())
+		}
+	}
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the loop outlived its context")
+	}
+}

cmd/felis/api_tokens_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/api"
+)
+
+// felis-api maps each env var onto the caller the Deployment feeds it for, so the
+// build Job's token (FELIS_BUILD_TOKEN) authenticates as build and only as build.
+func TestInternalCallerTokensReadEachCallersEnv(t *testing.T) {
+	env := map[string]string{
+		"FELIS_SERVICE_TOKEN": "v-tok",
+		"FELIS_LIMBO_TOKEN":   "l-tok",
+		"FELIS_BUILD_TOKEN":   " b-tok\n",
+		"FELIS_OPS_TOKEN":     "o-tok",
+	}
+	auth, err := internalCallerTokens(func(k string) string { return env[k] })
+	if err != nil {
+		t.Fatal(err)
+	}
+	for tok, want := range map[string]api.Caller{"v-tok": api.CallerVelocity, "l-tok": api.CallerLimbo, "b-tok": api.CallerBuild, "o-tok": api.CallerOps} {
+		r := httptest.NewRequest("GET", "/", nil)
+		r.Header.Set("Authorization", "Bearer "+tok)
+		if got, err := auth.Authenticate(r); err != nil || got != want {
+			t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want)
+		}
+	}
+
+	// An install where the build namespace still holds a copy of the proxy's token
+	// would let that copy act as the proxy; the api refuses to start on it.
+	env["FELIS_BUILD_TOKEN"] = "v-tok"
+	if _, err := internalCallerTokens(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "same value") {
+		t.Fatalf("shared token: err = %v, want a same-value refusal", err)
+	}
+}

cmd/felis/apply.go

@@ -8,7 +8,9 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/naming"
@@ -16,10 +18,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -111,20 +109,15 @@ func cmdApply(args []string, stdout, stderr io.Writer) int {
 	}
 
 	// ------- K8s client (one context, one client) -------
-	// SetupSignalHandler must be called exactly once per process —
-	// controller-runtime panics on a second call. We create ctx and the
-	// K8s client here and thread both through every downstream call so no
-	// callee ever needs to call SetupSignalHandler again.
-	ctx := ctrl.SetupSignalHandler()
-
-	scheme := runtime.NewScheme()
-	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-	utilruntime.Must(v1alpha1.AddToScheme(scheme))
-
-	cfg := ctrl.GetConfigOrDie()
-	cl, err := client.New(cfg, client.Options{Scheme: scheme})
+	// The operator runs this on the node, where the kubeconfig is k3s's own file and
+	// neither $KUBECONFIG nor ~/.kube is set. buildSystemServerClient falls back to that
+	// file and names what it tried; ctrl.GetConfigOrDie exited 1 there without a word,
+	// because controller-runtime's logger is never set up in a CLI command.
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	cl, err := buildSystemServerClient()
 	if err != nil {
-		fmt.Fprintf(stderr, "felis apply: build client: %v\n", err)
+		fmt.Fprintf(stderr, "felis apply: %v\n", err)
 		return 1
 	}
 
@@ -167,6 +160,10 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 	if err := naming.ValidateServerName(req.Subdomain); err != nil {
 		return nil, fmt.Errorf("invalid subdomain: %w", err)
 	}
+	displayName, err := naming.CleanDisplayName(req.DisplayName)
+	if err != nil {
+		return nil, fmt.Errorf("invalid displayName: %w", err)
+	}
 	if strings.TrimSpace(req.Image) == "" {
 		return nil, fmt.Errorf("image is required")
 	}
@@ -236,13 +233,26 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 		},
 		Spec: v1alpha1.MinecraftServerSpec{
 			Subdomain:       req.Subdomain,
-			DisplayName:     req.DisplayName,
+			DisplayName:     displayName,
 			Image:           req.Image,
 			JavaMemory:      deriveApplyJavaHeap(memLim),
 			DesiredState:    v1alpha1.DesiredStopped,
 			AutostartPolicy: policy,
 			Storage:         v1alpha1.StorageSpec{Size: storageQ.String()},
 			Resources:       corev1.ResourceRequirements{Limits: limits, Requests: requests},
+			// The rest matches what felis-api's create writes (K8sCluster.CreateServer):
+			// fall back to the login gate while stopped, RCON on (readiness, the
+			// player count and the console all ride it; the operator mints the
+			// password), and the default idle stop.
+			FallbackServer: naming.SystemLoginServer,
+			Rcon: v1alpha1.RconSpec{
+				Enabled: true,
+				SecretRef: v1alpha1.SecretKeyRef{
+					Name: naming.RconSecretName(req.Name),
+					Key:  naming.RconSecretKey,
+				},
+			},
+			Idle: v1alpha1.DefaultIdle(),
 		},
 	}, nil
 }
@@ -251,8 +261,7 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 // request if any CRD already carries the given spec.subdomain. metadata.name
 // uniqueness is enforced by K8s on Create, but spec.subdomain must be checked
 // here because two CRDs with different names could otherwise share a subdomain.
-// It reuses the caller's context and K8s client — it never calls
-// SetupSignalHandler or builds its own client.
+// It reuses the caller's context and K8s client.
 func checkSubdomainUnique(ctx context.Context, cl client.Client, namespace, subdomain string) error {
 	var list v1alpha1.MinecraftServerList
 	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {

cmd/felis/apply_test.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
@@ -136,11 +140,12 @@ func TestDeriveApplyJavaHeap(t *testing.T) {
 
 func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	req := applyRequest{
-		Name:      "test-server",
-		Subdomain: "test-server",
-		Image:     "registry.felis.svc/paper:1.21",
-		Memory:    "4Gi",
-		Storage:   "20Gi",
+		Name:        "test-server",
+		Subdomain:   "test-server",
+		DisplayName: "  Test Server  ",
+		Image:       "registry.felis.svc/paper:1.21",
+		Memory:      "4Gi",
+		Storage:     "20Gi",
 	}
 	ms, err := buildMinecraftServerFromApplyRequest(req, "minecraft")
 	if err != nil {
@@ -155,6 +160,9 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	if ms.Spec.Subdomain != "test-server" {
 		t.Errorf("Subdomain = %q", ms.Spec.Subdomain)
 	}
+	if ms.Spec.DisplayName != "Test Server" {
+		t.Errorf("DisplayName = %q, want it trimmed to Test Server", ms.Spec.DisplayName)
+	}
 	if ms.Spec.Image != "registry.felis.svc/paper:1.21" {
 		t.Errorf("Image = %q", ms.Spec.Image)
 	}
@@ -167,6 +175,18 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	if ms.Spec.Storage.Size != "20Gi" {
 		t.Errorf("Storage.Size = %q, want 20Gi", ms.Spec.Storage.Size)
 	}
+	// Same operational defaults as the API create path: without RCON the server
+	// never reports players and the console answers 503; without spec.idle it
+	// never stops on its own.
+	if !ms.Spec.Rcon.Enabled || ms.Spec.Rcon.SecretRef.Name != naming.RconSecretName("test-server") {
+		t.Errorf("Rcon = %+v, want enabled with the operator-minted secret", ms.Spec.Rcon)
+	}
+	if ms.Spec.Idle != v1alpha1.DefaultIdle() {
+		t.Errorf("Idle = %+v, want the default %+v", ms.Spec.Idle, v1alpha1.DefaultIdle())
+	}
+	if ms.Spec.FallbackServer != naming.SystemLoginServer {
+		t.Errorf("FallbackServer = %q, want the login gate", ms.Spec.FallbackServer)
+	}
 	mem, ok := ms.Spec.Resources.Limits[corev1.ResourceMemory]
 	if !ok {
 		t.Fatal("memory limit missing")
@@ -267,6 +287,11 @@ func TestBuildMinecraftServerFromApplyRequest_Errors(t *testing.T) {
 			applyRequest{Name: ok, Subdomain: "", Image: "x", Memory: "1Gi", Storage: "1Gi"},
 			"invalid subdomain",
 		},
+		{
+			"display name with a tab",
+			applyRequest{Name: ok, Subdomain: ok, DisplayName: "a" + string(rune(0x09)) + "b", Image: "x", Memory: "1Gi", Storage: "1Gi"},
+			"invalid displayName",
+		},
 		{
 			"empty image",
 			applyRequest{Name: ok, Subdomain: ok, Image: "", Memory: "1Gi", Storage: "1Gi"},
@@ -357,3 +382,27 @@ func resList(specs ...string) corev1.ResourceList {
 	}
 	return rl
 }
+
+// TestApplyReportsAMissingKubeconfig pins the node-side failure: with no kubeconfig to
+// find, apply says which ones it tried and exits 1. It used to call
+// ctrl.GetConfigOrDie, which ended the process with exit 1 and nothing printed.
+func TestApplyReportsAMissingKubeconfig(t *testing.T) {
+	if _, err := os.Stat(hostBootstrapKubeconfigPath); err == nil {
+		t.Skipf("%s exists on this machine", hostBootstrapKubeconfigPath)
+	}
+	dir := t.TempDir()
+	t.Setenv("KUBECONFIG", filepath.Join(dir, "missing"))
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	form := filepath.Join(dir, "server.json")
+	if err := os.WriteFile(form, []byte(`{"name":"alpha","subdomain":"alpha","image":"registry.felis.svc:5000/felis/paper:demo","memory":"1Gi","storage":"1Gi"}`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var out, errw bytes.Buffer
+	if code := cmdApply([]string{"-f", form}, &out, &errw); code != 1 {
+		t.Fatalf("exit = %d, want 1; stderr %q", code, errw.String())
+	}
+	want := "felis apply: no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and " + hostBootstrapKubeconfigPath + "): stat " + hostBootstrapKubeconfigPath + ": no such file or directory\n"
+	if errw.String() != want || out.Len() != 0 {
+		t.Fatalf("stdout %q, stderr %q, want stderr %q", out.String(), errw.String(), want)
+	}
+}

cmd/felis/archive_serve.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	"felis.lolicon.best/internal/archivetransfer"
+)
+
+func cmdArchiveServe(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("archive-serve", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	root := fs.String("root", "/backups", "archive PVC mount (must match archive.local_path)")
+	addr := fs.String("listen", ":8090", "private archive listener")
+	limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum compressed archive bytes")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	key := os.Getenv(archivetransfer.KeyEnv)
+	if len(key) < 32 || *limit <= 0 {
+		fmt.Fprintln(stderr, "archive-serve: signing key and positive size limit required")
+		return 2
+	}
+	if err := os.MkdirAll(*root, 0750); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	limitHeapToCgroup()
+	transport := &archivetransfer.Server{Root: *root, Key: key, Limit: *limit}
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		ticker := time.NewTicker(time.Hour)
+		defer ticker.Stop()
+		for {
+			if err := transport.Sweep(time.Now()); err != nil {
+				fmt.Fprintln(stderr, "archive journal cleanup:", err)
+			}
+			select {
+			case <-done:
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+	srv := &http.Server{Addr: *addr, Handler: transport, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Hour, WriteTimeout: 2 * time.Hour, MaxHeaderBytes: 16 << 10}
+	fmt.Fprintln(stdout, "archive transport listening", *addr)
+	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	return 0
+}

cmd/felis/backup.go

@@ -1,18 +1,20 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	"felis.lolicon.best/internal/backup"
+	"felis.lolicon.best/internal/backupjob"
 	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/reaper"
-	"felis.lolicon.best/internal/store"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
@@ -35,6 +37,8 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	server := fs.String("server", "", "server name whose world is being backed up")
 	formerOwner := fs.String("former-owner", "", "owner recorded on the backup row (empty for an unowned server)")
 	worldsRoot := fs.String("worlds-root", "/world", "mount path of the world PVC being archived")
+	reason := fs.String("reason", reasonManual, "world_backups reason: manual, pre_restore for the safety snapshot in front of a restore, or scheduled for felis-api's daily restore point")
+	protect := fs.String("protect", "", "backup id the prune must keep (the one a chained restore extracts)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -42,6 +46,10 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintln(stderr, "felis backup: --server is required")
 		return 2
 	}
+	if _, _, ok := backupPolicy(*reason, reaper.DefaultConfig()); !ok {
+		fmt.Fprintf(stderr, "felis backup: unknown --reason %q (manual, %s or %s)\n", *reason, backupjob.ReasonPreRestore, backupjob.ReasonScheduled)
+		return 2
+	}
 
 	cfg, err := config.Load(*cfgPath)
 	if err != nil {
@@ -52,13 +60,14 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis backup: archive store %q is not implemented in this build (only tarLocal)\n", cfg.Archive.Store)
 		return 1
 	}
-	// Reuse the reaper's retention derivation so an on-demand backup expires on the
-	// same clock as an inactivity backup — one retention policy, not two.
+	// The [archive] parse the reaper uses: it holds each reason's keep and
+	// retention.
 	rcfg, err := reaperConfig(cfg)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: %v\n", err)
 		return 1
 	}
+	keep, retention, _ := backupPolicy(*reason, rcfg)
 
 	// The world PVC is mounted directly at worldsRoot; the resolver returns it for
 	// any target, exactly as in cmdRestore. This is the same TarLocal the reaper
@@ -72,13 +81,25 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 
 	ctx := ctrl.SetupSignalHandler()
 
-	ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
+	// The archive store shares the node's disk with every world and the
+	// database: an owner's backup must not be what tips it into eviction.
+	if err := backup.CheckRoom(cfg.Archive.LocalPath, *worldsRoot, backup.MinFreeAfter); err != nil {
+		fmt.Fprintf(stderr, "felis backup: %v\n", err)
+		return 1
+	}
+
+	a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
 		return 1
 	}
+	ref, size := a.Ref, a.Size
+	if len(a.Skipped) > 0 {
+		fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",
+			len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))
+	}
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	drv, err := openPodStore(ctx, cfg.Database.URL, "backup", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: open database: %v\n", err)
 		return 1
@@ -91,10 +112,14 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		FormerOwner: *formerOwner,
 		BackupRef:   string(ref),
 		SizeBytes:   size,
-		Reason:      "manual",
-		ExpiresAt:   time.Now().Add(rcfg.Retention),
+		Reason:      *reason,
+		ExpiresAt:   time.Now().Add(retention),
+
+		SHA256:         a.SHA256,
+		SkippedEntries: len(a.Skipped),
 	}
-	if err := reaper.NewPGStore(drv.DB()).InsertBackup(ctx, rec); err != nil {
+	st := reaper.NewPGStore(drv.DB())
+	if err := st.InsertBackup(ctx, rec); err != nil {
 		// The archive is written but unrecorded — an orphan the retention pass would
 		// never expire. Delete it so a failed backup leaves no leaked bytes, mirroring
 		// the reaper's archive-then-record atomicity.
@@ -107,9 +132,63 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	}
 
 	fmt.Fprintf(stdout, "felis backup: server=%s archived %d bytes to %s (backup %s)\n", *server, size, ref, rec.ID)
+	pruneBackups(ctx, st, archiver, *server, *formerOwner, *reason, keep, *protect, stdout, stderr)
 	return 0
 }
 
+const (
+	reasonManual = "manual"
+	// preRestoreKeep is how many safety snapshots a server keeps: enough to walk
+	// back a couple of restores in a row, without every restore adding a world's
+	// worth of bytes for the full manual retention.
+	preRestoreKeep = 3
+)
+
+// backupPolicy is how many backups of one reason a server keeps and how long
+// each lives: an owner's own backups and the safety snapshots in front of a
+// restore by [archive] manual_keep / manual_retention (the snapshots capped at
+// preRestoreKeep), felis-api's daily restore points by scheduled_keep /
+// scheduled_retention, so neither kind crowds out the other. ok is false for a
+// reason this command does not record.
+func backupPolicy(reason string, rcfg reaper.Config) (keep int, retention time.Duration, ok bool) {
+	switch reason {
+	case reasonManual:
+		return rcfg.ManualKeep, rcfg.ManualRetention, true
+	case backupjob.ReasonPreRestore:
+		return preRestoreKeep, rcfg.ManualRetention, true
+	case backupjob.ReasonScheduled:
+		return rcfg.ScheduledKeep, rcfg.ScheduledRetention, true
+	}
+	return 0, 0, false
+}
+
+// pruneBackups keeps the newest keep backups of this reason that owner holds of
+// server and removes the rest, oldest first, so repeated backups of one world
+// cannot fill the shared archive store and a new owner's backups never remove a
+// previous owner's. protect is never removed: it is the backup a chained restore
+// is about to extract. The new backup is already recorded; a removal that fails
+// is reported and retried after the next backup.
+func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver interface {
+	Delete(context.Context, backup.ArchiveRef) error
+}, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) {
+	excess, err := st.ExcessBackups(ctx, server, owner, reason, keep, protect)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)
+		return
+	}
+	for _, b := range excess {
+		if err := archiver.Delete(ctx, backup.ArchiveRef(b.BackupRef)); err != nil {
+			fmt.Fprintf(stderr, "felis backup: remove older backup %s: %v\n", b.ID, err)
+			continue
+		}
+		if err := st.MarkBackupDeleted(ctx, b.ID, time.Now()); err != nil {
+			fmt.Fprintf(stderr, "felis backup: record the removal of %s: %v\n", b.ID, err)
+			continue
+		}
+		fmt.Fprintf(stdout, "felis backup: removed older %s backup %s of %s (keeping the newest %d)\n", reason, b.ID, server, keep)
+	}
+}
+
 // newBackupID mints a world_backups primary key, matching the reaper's "bk-"+hex
 // scheme so a manual and an inactivity backup are indistinguishable downstream.
 func newBackupID() string {

cmd/felis/backup_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/reaper"
+)
+
+// The reason decides which backups the new one's prune may remove, so an
+// unknown one is refused before anything is archived.
+func TestBackupSubcommandRejectsUnknownReason(t *testing.T) {
+	var stderr bytes.Buffer
+	if code := cmdBackup([]string{"--server", "survival", "--reason", "inactive_15d"}, &bytes.Buffer{}, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2 (%s)", code, stderr.String())
+	}
+	if !strings.Contains(stderr.String(), "unknown --reason") {
+		t.Fatalf("stderr = %q", stderr.String())
+	}
+}
+
+// Each reason is pruned and expired by its own [archive] keys: a daily
+// restore point must never count against, or take the lifetime of, the
+// backups an owner asked for.
+func TestBackupPolicyPerReason(t *testing.T) {
+	rcfg := reaper.DefaultConfig()
+	rcfg.ManualKeep, rcfg.ManualRetention = 5, 30*reaper.Day
+	rcfg.ScheduledKeep, rcfg.ScheduledRetention = 7, 90*reaper.Day
+	for _, tc := range []struct {
+		reason    string
+		keep      int
+		retention time.Duration
+	}{
+		{"manual", 5, 30 * reaper.Day},
+		{"pre_restore", preRestoreKeep, 30 * reaper.Day},
+		{"scheduled", 7, 90 * reaper.Day},
+	} {
+		keep, retention, ok := backupPolicy(tc.reason, rcfg)
+		if !ok || keep != tc.keep || retention != tc.retention {
+			t.Errorf("backupPolicy(%q) = (%d, %v, %v); want (%d, %v, true)", tc.reason, keep, retention, ok, tc.keep, tc.retention)
+		}
+	}
+	if _, _, ok := backupPolicy("inactive_15d", rcfg); ok {
+		t.Error("backupPolicy accepted inactive_15d; the reaper records those itself")
+	}
+}

cmd/felis/backupnow.go

@@ -4,15 +4,28 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"os/signal"
+	"sort"
+	"strings"
+	"syscall"
 	"time"
 
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/operator"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/store"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -20,9 +33,11 @@ import (
 // backupnow is the break-glass "back up a world now" op (§B4 "Sync"). Unlike halt —
 // which writes the CRD directly — a backup needs felis-api's deployment coordinates
 // (FELIS_IMAGE / FELIS_BACKUP_PVC) to render the one-shot backup Job, so the console
-// cannot do it in-process. It POSTs the felis-api INTERNAL face (service-token auth)
+// cannot do it in-process. It POSTs the felis-api INTERNAL face (ops-token auth)
 // while the API is alive, and the API renders the Job and audits the action. This file
-// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue.
+// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue. The
+// `felis backup-now` command (cmdBackupNow, below) takes the same route for every
+// user server in turn.
 
 // backupNowOutcome is the durable result of a backup request, re-printed after the TUI
 // alt-screen tears down.
@@ -33,7 +48,7 @@ type backupNowOutcome struct {
 
 // resolveInternalAPI reads the two things the on-node console needs to reach the
 // felis-api internal face: the felis-api-internal Service ClusterIP (the host's
-// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the service
+// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the ops
 // token. Both live in the control namespace.
 func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace string) (baseURL, token string, err error) {
 	var svc corev1.Service
@@ -45,13 +60,14 @@ func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace
 		return "", "", fmt.Errorf("%s Service has no ClusterIP yet", platform.APIInternalServiceName)
 	}
 
+	// The console's own token, which the api serves on the backup route alone.
 	var sec corev1.Secret
-	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.ServiceTokenSecretName}, &sec); err != nil {
-		return "", "", fmt.Errorf("get %s Secret: %w", naming.ServiceTokenSecretName, err)
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.OpsTokenSecretName}, &sec); err != nil {
+		return "", "", fmt.Errorf("get %s Secret (re-run the installer to create it): %w", naming.OpsTokenSecretName, err)
 	}
 	token = string(sec.Data[naming.ServiceTokenSecretKey])
 	if token == "" {
-		return "", "", fmt.Errorf("secret %s has no %s key", naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey)
+		return "", "", fmt.Errorf("secret %s has no %s key", naming.OpsTokenSecretName, naming.ServiceTokenSecretKey)
 	}
 
 	return fmt.Sprintf("http://%s:%d", ip, platform.APIInternalPort), token, nil
@@ -74,7 +90,7 @@ func requestBackup(ctx context.Context, hc *http.Client, baseURL, token, name, o
 
 	resp, err := hc.Do(req)
 	if err != nil {
-		return backupNowOutcome{}, fmt.Errorf("felis-api unreachable (a backup needs it alive): %w", err)
+		return backupNowOutcome{}, fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
 	}
 	defer resp.Body.Close()
 
@@ -86,23 +102,31 @@ func requestBackup(ctx context.Context, hc *http.Client, baseURL, token, name, o
 
 // backupErrorFromResponse turns a non-202 into a human message. The well-known codes get
 // an operator-facing explanation; anything else falls back to the API's
-// {"error":{message}} body, then the bare status code.
+// {"error":{code,message}} body, then the bare status code.
 func backupErrorFromResponse(resp *http.Response) error {
-	switch resp.StatusCode {
-	case http.StatusConflict: // not_stopped
-		return fmt.Errorf("the server must be stopped before its world can be backed up — halt it first")
-	case http.StatusServiceUnavailable: // backup_unavailable
-		return fmt.Errorf("the backup subsystem is not configured on felis-api (FELIS_IMAGE / FELIS_BACKUP_PVC unset)")
-	case http.StatusNotFound:
-		return fmt.Errorf("no such server")
-	}
 	var e struct {
 		Error struct {
+			Code    string `json:"code"`
 			Message string `json:"message"`
 		} `json:"error"`
 	}
 	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
 	_ = json.Unmarshal(raw, &e)
+
+	switch resp.StatusCode {
+	case http.StatusConflict:
+		// Two refusals share 409: the stopped gate and the missing-world-volume
+		// gate. The body's code distinguishes them; a code-less body reads as the
+		// stopped gate (the only 409 before the volume gate existed), and any other
+		// coded 409 falls through to the API's own operator text.
+		if e.Error.Code == "" || e.Error.Code == "not_stopped" {
+			return fmt.Errorf("the server must be stopped before its world can be backed up — halt it first")
+		}
+	case http.StatusServiceUnavailable: // backup_unavailable
+		return fmt.Errorf("the backup subsystem is not configured on felis-api (FELIS_IMAGE / FELIS_BACKUP_PVC unset)")
+	case http.StatusNotFound:
+		return fmt.Errorf("no such server")
+	}
 	if e.Error.Message != "" {
 		return fmt.Errorf("felis-api: %s", e.Error.Message)
 	}
@@ -120,3 +144,455 @@ func performBackupNow(ctx context.Context, cl client.Client, controlNamespace, n
 	hc := &http.Client{Timeout: 10 * time.Second}
 	return requestBackup(ctx, hc, baseURL, token, name, osUser)
 }
+
+// backupPickable narrows the backup picker to servers the backup API can accept.
+// System servers (login/lobby) are excluded: they have no row in the servers
+// table and carry reserved names, so every attempt dies in name validation —
+// offering them would be a dead pick. The halt picker keeps them on purpose
+// (break-glass retains full power over system servers); only the API-backed
+// backup op cannot reach them.
+func backupPickable(servers []haltableServer) []haltableServer {
+	out := make([]haltableServer, 0, len(servers))
+	for _, s := range servers {
+		if !s.system {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+// cmdBackupNow is `felis backup-now`: the world of every user server (or of the
+// named ones) archived now, one at a time, through the internal backup route the
+// console's Sync uses. A world lives only in its volume and the off-site copy holds
+// only its archives, so this is the lever in front of a planned move to another
+// host, a disk swap or anything else that could lose a volume.
+func cmdBackupNow(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("backup-now", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	yes := fs.Bool("yes", false, "back up; without it the plan is printed and nothing changes")
+	stop := fs.Bool("stop", false, "stop the running servers first: their players are disconnected and the servers stay stopped")
+	fs.Usage = func() {
+		fmt.Fprintln(stderr, "Usage: felis backup-now [-yes] [-stop] [server ...]")
+		fmt.Fprintln(stderr)
+		fmt.Fprintln(stderr, "Archives the world of every user server, or of the named ones, one at a time, and waits for each archive.")
+		fmt.Fprintln(stderr, "A running server is skipped unless -stop is given. Without -yes it prints what it would do.")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis backup-now: refused — it reads the cluster's ops token, so it must run as root (try: sudo felis backup-now)")
+		return 1
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
+		return 1
+	}
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer cancel()
+
+	ns := cfg.K8s.Namespace
+	osUser := accountableOSUser()
+	jobs := api.NewK8sJobStatus(cl, ns)
+	var baseURL, token string
+	var repo ownerStore
+	var drv *store.PostgresDriver
+	defer func() {
+		if drv != nil {
+			_ = drv.Close()
+		}
+	}()
+	hc := &http.Client{Timeout: 10 * time.Second}
+	r := backupNowRun{
+		out:  stdout,
+		errw: stderr,
+		ns:   ns,
+		list: func(ctx context.Context) ([]backupNowWorld, error) { return listBackupNowWorlds(ctx, cl, ns) },
+		stopped: func(ctx context.Context, name string) (bool, error) {
+			var ms v1alpha1.MinecraftServer
+			if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &ms); err != nil {
+				return false, err
+			}
+			return backupNowStopped(ctx, cl, &ms)
+		},
+		halt: func(ctx context.Context, name string) error {
+			// The database is opened only once a server is to be stopped: the halt
+			// is audited like the console's, and a run with nothing running needs
+			// no more than the API.
+			if repo == nil {
+				d, err := store.Open(ctx, cfg.Database.URL)
+				if err != nil {
+					return fmt.Errorf("open the database for the audit log: %w", err)
+				}
+				drv, repo = d, api.NewPGRepo(d.DB())
+			}
+			out, err := performHalt(ctx, cl, repo, ns, name, osUser)
+			if err != nil {
+				return err
+			}
+			if out.auditErr != nil {
+				fmt.Fprintf(stderr, "felis backup-now: the audit row for stopping %s was not written: %v\n", name, out.auditErr)
+			}
+			return nil
+		},
+		request: func(ctx context.Context, name string) error {
+			if baseURL == "" {
+				var err error
+				if baseURL, token, err = resolveInternalAPI(ctx, cl, platform.DefaultControlNamespace); err != nil {
+					return fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
+				}
+			}
+			_, err := requestBackup(ctx, hc, baseURL, token, name, osUser)
+			return err
+		},
+		jobs:  jobs.LatestJobs,
+		now:   time.Now,
+		sleep: func(ctx context.Context, d time.Duration) { sleepCtx(ctx, d) },
+	}
+	return r.run(ctx, fs.Args(), *yes, *stop)
+}
+
+// sleepCtx waits d or until ctx ends.
+func sleepCtx(ctx context.Context, d time.Duration) {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+	case <-t.C:
+	}
+}
+
+// backupNowWorld is one user server as backup-now sees it.
+type backupNowWorld struct {
+	name     string
+	phase    string // the observed phase, or the desired state before the operator reconciled it
+	stopped  bool   // the backup route's stopped gate admits it
+	hasWorld bool   // its world volume exists
+}
+
+// listBackupNowWorlds lists the user servers of namespace in the API server's
+// order (by name), each with what the backup route checks. System servers are
+// left out: they have no row in the servers table, so the route refuses them
+// (backupPickable).
+func listBackupNowWorlds(ctx context.Context, cl client.Client, namespace string) ([]backupNowWorld, error) {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return nil, err
+	}
+	var out []backupNowWorld
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if isSystemServer(ms.Name) {
+			continue
+		}
+		stopped, err := backupNowStopped(ctx, cl, ms)
+		if err != nil {
+			return nil, err
+		}
+		var pvc corev1.PersistentVolumeClaim
+		err = cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: naming.WorldPVCName(ms.Name)}, &pvc)
+		if err != nil && !apierrors.IsNotFound(err) {
+			return nil, fmt.Errorf("look up the world volume of %s: %w", ms.Name, err)
+		}
+		phase := string(ms.Status.Phase)
+		if phase == "" {
+			phase = string(ms.Spec.DesiredState)
+		}
+		out = append(out, backupNowWorld{name: ms.Name, phase: phase, stopped: stopped, hasWorld: err == nil})
+	}
+	return out, nil
+}
+
+// backupNowStopped is the backup route's stopped gate (api.enqueueBackup and
+// K8sCluster.AcquireMaintenance together): desired Stopped, not ready, phase
+// Stopped and no game pod left.
+func backupNowStopped(ctx context.Context, cl client.Client, ms *v1alpha1.MinecraftServer) (bool, error) {
+	if ms.Spec.DesiredState != v1alpha1.DesiredStopped || ms.Status.Ready || ms.Status.Phase != v1alpha1.PhaseStopped {
+		return false, nil
+	}
+	var pods corev1.PodList
+	if err := cl.List(ctx, &pods, client.InNamespace(ms.Namespace), client.MatchingLabels{
+		v1alpha1.LabelServer: ms.Name, v1alpha1.LabelComponent: operator.ComponentValue,
+	}); err != nil {
+		return false, fmt.Errorf("look up the pod of %s: %w", ms.Name, err)
+	}
+	return len(pods.Items) == 0, nil
+}
+
+// errBackupAPIUnreachable is a backup request that never reached felis-api. It
+// ends a backup-now run: every later world would fail the same way, and stopping
+// servers for backups that cannot be taken only takes them away from players.
+var errBackupAPIUnreachable = errors.New("felis-api unreachable (a backup needs it alive)")
+
+// Polling of backup-now. A graceful stop saves the world first; the backup Job's
+// own deadline (backupjob, 30 minutes) ends a Job that hangs, so its wait needs no
+// cap of its own.
+const (
+	backupNowPoll      = 2 * time.Second
+	backupNowStopWait  = 10 * time.Minute
+	backupNowJobAppear = time.Minute
+)
+
+// backupNowRun is backup-now over seams, so the plan and the run are tested
+// without a cluster or felis-api.
+type backupNowRun struct {
+	out, errw io.Writer
+	ns        string // where the servers and their Jobs live, for the kubectl hints
+	list      func(ctx context.Context) ([]backupNowWorld, error)
+	stopped   func(ctx context.Context, name string) (bool, error)
+	halt      func(ctx context.Context, name string) error
+	request   func(ctx context.Context, name string) error
+	jobs      func(ctx context.Context, name string) ([]api.AsyncJob, error)
+	now       func() time.Time
+	sleep     func(ctx context.Context, d time.Duration)
+}
+
+// pickBackupNowWorlds narrows worlds to names, in the order given, or keeps them
+// all when names is empty.
+func pickBackupNowWorlds(worlds []backupNowWorld, names []string) ([]backupNowWorld, error) {
+	if len(names) == 0 {
+		return worlds, nil
+	}
+	byName := make(map[string]backupNowWorld, len(worlds))
+	for _, w := range worlds {
+		byName[w.name] = w
+	}
+	var out []backupNowWorld
+	seen := map[string]bool{}
+	for _, n := range names {
+		if seen[n] {
+			continue
+		}
+		seen[n] = true
+		w, ok := byName[n]
+		switch {
+		case ok:
+			out = append(out, w)
+		case isSystemServer(n):
+			return nil, fmt.Errorf("%s is a system server: its world is rebuilt by felis setup and has no backups", n)
+		default:
+			return nil, fmt.Errorf("no server named %q", n)
+		}
+	}
+	return out, nil
+}
+
+// backupNowAction is what the plan does with a world.
+func backupNowAction(w backupNowWorld, stop bool) string {
+	switch {
+	case w.stopped && !w.hasWorld:
+		return "skip: no world volume (never started, nothing to save)"
+	case w.stopped:
+		return "back up"
+	case stop:
+		return "stop, then back up"
+	default:
+		return "skip: running (stop it first, or pass -stop)"
+	}
+}
+
+func (r *backupNowRun) run(ctx context.Context, names []string, yes, stop bool) int {
+	all, err := r.list(ctx)
+	if err != nil {
+		fmt.Fprintf(r.errw, "felis backup-now: list the servers: %v\n", err)
+		return 1
+	}
+	worlds, err := pickBackupNowWorlds(all, names)
+	if err != nil {
+		fmt.Fprintf(r.errw, "felis backup-now: %v\n", err)
+		return 2
+	}
+	if len(worlds) == 0 {
+		fmt.Fprintln(r.out, "felis backup-now: there are no user servers")
+		return 0
+	}
+
+	// The stopped worlds go first, so a felis-api that cannot take a backup is
+	// found before any server is stopped for one.
+	sort.SliceStable(worlds, func(i, j int) bool { return worlds[i].stopped && !worlds[j].stopped })
+	width := 0
+	for _, w := range worlds {
+		width = max(width, len(w.name))
+	}
+	fmt.Fprintf(r.out, "felis backup-now: %d server(s), backed up one at a time:\n", len(worlds))
+	stopping, work := false, 0
+	for _, w := range worlds {
+		fmt.Fprintf(r.out, "  %-*s  %-8s  %s\n", width, w.name, w.phase, backupNowAction(w, stop))
+		stopping = stopping || (!w.stopped && stop)
+		if (w.stopped && w.hasWorld) || (!w.stopped && stop) {
+			work++
+		}
+	}
+	if work > 0 {
+		fmt.Fprintln(r.out, "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.")
+	}
+	if stopping {
+		fmt.Fprintln(r.out, "Stopping disconnects the players on those servers, and they stay stopped afterwards.")
+	}
+	if !yes {
+		if work == 0 {
+			fmt.Fprintln(r.out, "Nothing to back up.")
+		} else {
+			fmt.Fprintln(r.out, "Nothing changed. Run again with -yes to back them up.")
+		}
+		return 0
+	}
+
+	var done, failed, running, empty int
+	var leftStopped []string
+	for _, w := range worlds {
+		if err := ctx.Err(); err != nil {
+			break
+		}
+		switch {
+		case w.stopped && !w.hasWorld:
+			empty++
+			continue
+		case !w.stopped && !stop:
+			running++
+			continue
+		}
+		if !w.stopped {
+			halted, err := r.stopWorld(ctx, w.name)
+			if halted {
+				leftStopped = append(leftStopped, w.name)
+			}
+			if err != nil {
+				fmt.Fprintf(r.out, "  %s: failed: %v\n", w.name, err)
+				failed++
+				continue
+			}
+		}
+		err := r.backUp(ctx, w.name)
+		if errors.Is(err, errBackupAPIUnreachable) {
+			fmt.Fprintf(r.out, "  %s: failed: %v\n", w.name, err)
+			fmt.Fprintln(r.out, "Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).")
+			failed++
+			break
+		}
+		if err != nil {
+			fmt.Fprintf(r.out, "  %s: failed: %v\n", w.name, err)
+			failed++
+			continue
+		}
+		done++
+	}
+
+	interrupted := ctx.Err() != nil
+	if interrupted {
+		fmt.Fprintln(r.out, "Interrupted: a backup Job already started runs to its end.")
+	}
+	parts := []string{fmt.Sprintf("%d backed up", done)}
+	if failed > 0 {
+		parts = append(parts, fmt.Sprintf("%d failed", failed))
+	}
+	if running > 0 {
+		parts = append(parts, fmt.Sprintf("%d skipped (running)", running))
+	}
+	if empty > 0 {
+		parts = append(parts, fmt.Sprintf("%d without a world", empty))
+	}
+	fmt.Fprintf(r.out, "%s.\n", strings.Join(parts, ", "))
+	if len(leftStopped) > 0 {
+		fmt.Fprintf(r.out, "Left stopped: %s. Start them from the panel when you are done.\n", strings.Join(leftStopped, ", "))
+	}
+	if done > 0 {
+		fmt.Fprintln(r.out, "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.")
+	}
+	if failed > 0 || running > 0 || interrupted {
+		return 1
+	}
+	return 0
+}
+
+// stopWorld stops one server and waits until the backup route would admit it.
+// halted is whether the stop was asked for: the server then stays stopped, even
+// when it takes longer than the wait.
+func (r *backupNowRun) stopWorld(ctx context.Context, name string) (halted bool, err error) {
+	fmt.Fprintf(r.out, "  %s: stopping\n", name)
+	if err := r.halt(ctx, name); err != nil {
+		return false, err
+	}
+	start := r.now()
+	for {
+		ok, err := r.stopped(ctx, name)
+		if err != nil {
+			return true, err
+		}
+		if ok {
+			fmt.Fprintf(r.out, "  %s: stopped after %s\n", name, r.now().Sub(start).Round(time.Second))
+			return true, nil
+		}
+		if r.now().Sub(start) >= backupNowStopWait {
+			return true, fmt.Errorf("did not stop within %s (kubectl -n %s describe minecraftserver %s)", backupNowStopWait, r.ns, name)
+		}
+		if err := ctx.Err(); err != nil {
+			return true, err
+		}
+		r.sleep(ctx, backupNowPoll)
+	}
+}
+
+// backUp requests one world's backup and waits for its Job to finish. The Job is
+// the one of this server that was not there before the request.
+func (r *backupNowRun) backUp(ctx context.Context, name string) error {
+	before, err := r.jobs(ctx, name)
+	if err != nil {
+		return fmt.Errorf("list its backup Jobs: %w", err)
+	}
+	known := make(map[string]bool, len(before))
+	for _, j := range before {
+		known[j.Name] = true
+	}
+	if err := r.request(ctx, name); err != nil {
+		return err
+	}
+	fmt.Fprintf(r.out, "  %s: backing up\n", name)
+	start := r.now()
+	seen := ""
+	for {
+		jobs, err := r.jobs(ctx, name)
+		if err != nil {
+			return fmt.Errorf("list its backup Jobs: %w", err)
+		}
+		var job *api.AsyncJob
+		for i := range jobs {
+			if jobs[i].Kind == "backup" && (jobs[i].Name == seen || seen == "" && !known[jobs[i].Name]) {
+				job = &jobs[i]
+				break
+			}
+		}
+		switch {
+		case job == nil && seen != "":
+			return fmt.Errorf("its backup Job %s was deleted before it finished", seen)
+		case job == nil && r.now().Sub(start) >= backupNowJobAppear:
+			return fmt.Errorf("felis-api accepted the backup, but no backup Job appeared within %s (kubectl -n %s get jobs)", backupNowJobAppear, r.ns)
+		case job != nil && job.State == "succeeded":
+			fmt.Fprintf(r.out, "  %s: archived in %s\n", name, r.now().Sub(start).Round(time.Second))
+			return nil
+		case job != nil && job.State == "failed":
+			msg := job.Message
+			if msg == "" {
+				msg = "the backup Job failed"
+			}
+			return fmt.Errorf("%s (kubectl -n %s logs job/%s)", msg, r.ns, job.Name)
+		case job != nil:
+			seen = job.Name
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		r.sleep(ctx, backupNowPoll)
+	}
+}

cmd/felis/backupnow_cmd_test.go

@@ -0,0 +1,563 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// bnRig drives backupNowRun against scripted servers and Jobs on a fake clock that
+// moves only when the run sleeps.
+type bnRig struct {
+	out, errw bytes.Buffer
+	clock     time.Time
+	events    []string
+	worlds    []backupNowWorld
+	// stopAfter is how many polls a halted server takes to stop; -1 never does.
+	stopAfter map[string]int
+	polls     map[string]int
+	reqErr    map[string]error
+	// states is what the server's new backup Job reports on each poll after the
+	// request, the last one repeating; "" is no Job.
+	states   map[string][]string
+	messages map[string]string
+	jobPolls map[string]int
+	// stopErr fails a server's stop polls; jobsFail fails the nth (1-based) Job
+	// list of a server.
+	stopErr  map[string]error
+	jobsFail map[string]int
+	jobCalls map[string]int
+	// ctx is the run's context, and onAct runs after each halt and request.
+	ctx   context.Context
+	onAct func(event string)
+}
+
+func newBNRig(worlds ...backupNowWorld) *bnRig {
+	return &bnRig{
+		clock:     time.Unix(1_800_000_000, 0),
+		worlds:    worlds,
+		stopAfter: map[string]int{},
+		polls:     map[string]int{},
+		reqErr:    map[string]error{},
+		states:    map[string][]string{},
+		messages:  map[string]string{},
+		jobPolls:  map[string]int{},
+		stopErr:   map[string]error{},
+		jobsFail:  map[string]int{},
+		jobCalls:  map[string]int{},
+		ctx:       context.Background(),
+		onAct:     func(string) {},
+	}
+}
+
+func (g *bnRig) run(names []string, yes, stop bool) int {
+	r := backupNowRun{
+		out: &g.out, errw: &g.errw, ns: "minecraft",
+		list: func(context.Context) ([]backupNowWorld, error) {
+			return append([]backupNowWorld(nil), g.worlds...), nil
+		},
+		stopped: func(_ context.Context, name string) (bool, error) {
+			g.polls[name]++
+			if err := g.stopErr[name]; err != nil {
+				return false, err
+			}
+			n := g.stopAfter[name]
+			return n >= 0 && g.polls[name] > n, nil
+		},
+		halt: func(_ context.Context, name string) error {
+			g.events = append(g.events, "halt "+name)
+			g.onAct("halt " + name)
+			return nil
+		},
+		request: func(_ context.Context, name string) error {
+			g.events = append(g.events, "request "+name)
+			g.onAct("request " + name)
+			if err := g.reqErr[name]; err != nil {
+				return err
+			}
+			g.jobPolls[name] = 0
+			return nil
+		},
+		jobs: func(_ context.Context, name string) ([]api.AsyncJob, error) {
+			// Every server has an older finished backup, and a restore Job that
+			// shows up with the new backup: neither is the Job to wait for.
+			g.jobCalls[name]++
+			if g.jobCalls[name] == g.jobsFail[name] {
+				return nil, errors.New("the apiserver is gone")
+			}
+			out := []api.AsyncJob{{Name: "backup-" + name + "-old", Kind: "backup", State: "succeeded"}}
+			n, requested := g.jobPolls[name]
+			if !requested {
+				return out, nil
+			}
+			g.jobPolls[name] = n + 1
+			states := g.states[name]
+			if len(states) == 0 {
+				states = []string{"succeeded"}
+			}
+			state := states[min(n, len(states)-1)]
+			if state == "" {
+				return out, nil
+			}
+			return append([]api.AsyncJob{
+				{Name: "restore-" + name + "-x", Kind: "restore", State: "succeeded"},
+				{Name: "backup-" + name + "-new", Kind: "backup", State: state, Message: g.messages[name]},
+			}, out...), nil
+		},
+		now:   func() time.Time { return g.clock },
+		sleep: func(_ context.Context, d time.Duration) { g.clock = g.clock.Add(d) },
+	}
+	return r.run(g.ctx, names, yes, stop)
+}
+
+func stoppedWorld(name string) backupNowWorld {
+	return backupNowWorld{name: name, phase: "Stopped", stopped: true, hasWorld: true}
+}
+
+func runningWorld(name string) backupNowWorld {
+	return backupNowWorld{name: name, phase: "Running", hasWorld: true}
+}
+
+func emptyWorld(name string) backupNowWorld {
+	return backupNowWorld{name: name, phase: "Stopped", stopped: true}
+}
+
+const (
+	bnManualKeep = "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.\n"
+	bnStopping   = "Stopping disconnects the players on those servers, and they stay stopped afterwards.\n"
+	bnOffsite    = "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.\n"
+)
+
+func TestBackupNowPlanChangesNothing(t *testing.T) {
+	for _, tc := range []struct {
+		stop bool
+		want string
+	}{
+		{false, "felis backup-now: 4 server(s), backed up one at a time:\n" +
+			"  alpha    Stopped   back up\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   skip: running (stop it first, or pass -stop)\n" +
+			"  delta    Starting  skip: running (stop it first, or pass -stop)\n" +
+			bnManualKeep +
+			"Nothing changed. Run again with -yes to back them up.\n"},
+		{true, "felis backup-now: 4 server(s), backed up one at a time:\n" +
+			"  alpha    Stopped   back up\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   stop, then back up\n" +
+			"  delta    Starting  stop, then back up\n" +
+			bnManualKeep + bnStopping +
+			"Nothing changed. Run again with -yes to back them up.\n"},
+	} {
+		t.Run(fmt.Sprintf("stop=%v", tc.stop), func(t *testing.T) {
+			delta := runningWorld("delta")
+			delta.phase = "Starting"
+			g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), delta)
+			if code := g.run(nil, false, tc.stop); code != 0 {
+				t.Fatalf("exit = %d, want 0; stderr %q", code, g.errw.String())
+			}
+			if g.out.String() != tc.want {
+				t.Fatalf("plan =\n%s\nwant\n%s", g.out.String(), tc.want)
+			}
+			if len(g.events) != 0 || len(g.polls) != 0 {
+				t.Fatalf("the plan acted: events %v, polls %v", g.events, g.polls)
+			}
+		})
+	}
+}
+
+// A plan that saves nothing says so, without the manual_keep warning; a running
+// server counts as something to save once -stop is given.
+func TestBackupNowPlanWithNothingToSave(t *testing.T) {
+	for _, tc := range []struct {
+		stop bool
+		want string
+	}{
+		{false, "felis backup-now: 2 server(s), backed up one at a time:\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   skip: running (stop it first, or pass -stop)\n" +
+			"Nothing to back up.\n"},
+		{true, "felis backup-now: 2 server(s), backed up one at a time:\n" +
+			"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+			"  bravo    Running   stop, then back up\n" +
+			bnManualKeep + bnStopping +
+			"Nothing changed. Run again with -yes to back them up.\n"},
+	} {
+		g := newBNRig(runningWorld("bravo"), emptyWorld("charlie"))
+		if code := g.run(nil, false, tc.stop); code != 0 {
+			t.Fatalf("stop=%v: exit = %d, want 0", tc.stop, code)
+		}
+		if g.out.String() != tc.want {
+			t.Fatalf("stop=%v: plan =\n%s\nwant\n%s", tc.stop, g.out.String(), tc.want)
+		}
+	}
+}
+
+func TestBackupNowBacksUpEachWorldInTurn(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), stoppedWorld("delta"))
+	g.states["alpha"] = []string{"", "running", "succeeded"}
+	g.states["delta"] = []string{"running", "failed"}
+	g.messages["delta"] = "felis backup: not enough free disk for the archive"
+	g.stopAfter["bravo"] = 2
+	g.states["bravo"] = []string{"running", "running", "running", "succeeded"}
+
+	code := g.run(nil, true, true)
+	if code != 1 {
+		t.Fatalf("exit = %d, want 1 (delta failed)", code)
+	}
+	if want := []string{"request alpha", "request delta", "halt bravo", "request bravo"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping+"")
+	want := "  alpha: backing up\n" +
+		"  alpha: archived in 4s\n" +
+		"  delta: backing up\n" +
+		"  delta: failed: felis backup: not enough free disk for the archive (kubectl -n minecraft logs job/backup-delta-new)\n" +
+		"  bravo: stopping\n" +
+		"  bravo: stopped after 4s\n" +
+		"  bravo: backing up\n" +
+		"  bravo: archived in 6s\n" +
+		"2 backed up, 1 failed, 1 without a world.\n" +
+		"Left stopped: bravo. Start them from the panel when you are done.\n" +
+		bnOffsite
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+}
+
+func TestBackupNowSkipsRunningServersWithoutStop(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"))
+	if code := g.run(nil, true, false); code != 1 {
+		t.Fatalf("exit = %d, want 1 (bravo was not backed up)", code)
+	}
+	if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	if len(g.polls) != 0 {
+		t.Fatalf("polled a server it did not stop: %v", g.polls)
+	}
+	_, run, _ := strings.Cut(g.out.String(), "Nothing changed")
+	if run != "" {
+		t.Fatalf("-yes printed the plan's closing line")
+	}
+	_, run, _ = strings.Cut(g.out.String(), bnManualKeep)
+	want := "  alpha: backing up\n  alpha: archived in 0s\n1 backed up, 1 skipped (running).\n" + bnOffsite
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+}
+
+func TestBackupNowExitsCleanWhenEverythingIsSaved(t *testing.T) {
+	// -stop with nothing running stops nothing and says nothing about stopping.
+	g := newBNRig(stoppedWorld("alpha"), emptyWorld("charlie"))
+	if code := g.run(nil, true, true); code != 0 {
+		t.Fatalf("exit = %d, want 0; output\n%s", code, g.out.String())
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
+	if want := "  alpha: backing up\n  alpha: archived in 0s\n1 backed up, 1 without a world.\n" + bnOffsite; run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+
+	g = newBNRig(emptyWorld("charlie"))
+	if code := g.run(nil, true, false); code != 0 {
+		t.Fatalf("exit = %d, want 0 for a server with nothing to save", code)
+	}
+	// Nothing to archive: no manual_keep warning, and no off-site hint.
+	if want := "felis backup-now: 1 server(s), backed up one at a time:\n" +
+		"  charlie  Stopped   skip: no world volume (never started, nothing to save)\n" +
+		"0 backed up, 1 without a world.\n"; g.out.String() != want {
+		t.Fatalf("output =\n%s\nwant\n%s", g.out.String(), want)
+	}
+	if len(g.events) != 0 {
+		t.Fatalf("events = %v, want none", g.events)
+	}
+}
+
+func TestBackupNowStopsAtAnUnreachableAPI(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), runningWorld("charlie"))
+	g.reqErr["alpha"] = fmt.Errorf("%w: dial tcp 10.43.0.9:8081: connect: connection refused", errBackupAPIUnreachable)
+	if code := g.run(nil, true, true); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v: nothing after the API proved unreachable, and no server stopped", g.events, want)
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping)
+	want := "  alpha: failed: felis-api unreachable (a backup needs it alive): dial tcp 10.43.0.9:8081: connect: connection refused\n" +
+		"Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).\n" +
+		"0 backed up, 1 failed.\n"
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+
+	// Any other refusal is that world's alone: the run goes on.
+	g = newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
+	g.reqErr["alpha"] = errors.New("felis-api: the world is being restored")
+	if code := g.run(nil, true, false); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	if want := []string{"request alpha", "request bravo"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+}
+
+func TestBackupNowGivesUpOnAServerThatDoesNotStop(t *testing.T) {
+	g := newBNRig(runningWorld("bravo"), runningWorld("echo"))
+	g.stopAfter["bravo"] = -1
+	if code := g.run(nil, true, true); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	if want := []string{"halt bravo", "halt echo", "request echo"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	// One poll at the start and one per 2s sleep up to the 10-minute mark.
+	if g.polls["bravo"] != 301 {
+		t.Fatalf("bravo polled %d times, want 301", g.polls["bravo"])
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping)
+	want := "  bravo: stopping\n" +
+		"  bravo: failed: did not stop within 10m0s (kubectl -n minecraft describe minecraftserver bravo)\n" +
+		"  echo: stopping\n" +
+		"  echo: stopped after 0s\n" +
+		"  echo: backing up\n" +
+		"  echo: archived in 0s\n" +
+		"1 backed up, 1 failed.\n" +
+		"Left stopped: bravo, echo. Start them from the panel when you are done.\n" +
+		bnOffsite
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+}
+
+func TestBackupNowReportsAJobThatNeverRuns(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		states []string
+		polls  int
+		want   string
+	}{
+		{"never appears", []string{""}, 31,
+			"felis-api accepted the backup, but no backup Job appeared within 1m0s (kubectl -n minecraft get jobs)"},
+		{"deleted while running", []string{"running", "running", ""}, 3,
+			"its backup Job backup-alpha-new was deleted before it finished"},
+		{"fails without a message", []string{"failed"}, 1,
+			"the backup Job failed (kubectl -n minecraft logs job/backup-alpha-new)"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			g := newBNRig(stoppedWorld("alpha"))
+			g.states["alpha"] = tc.states
+			if code := g.run(nil, true, false); code != 1 {
+				t.Fatalf("exit = %d, want 1", code)
+			}
+			if !strings.Contains(g.out.String(), "  alpha: failed: "+tc.want+"\n") {
+				t.Fatalf("output =\n%s\nwant the line %q", g.out.String(), tc.want)
+			}
+			if g.jobPolls["alpha"] != tc.polls {
+				t.Fatalf("polled the Jobs %d times after the request, want %d", g.jobPolls["alpha"], tc.polls)
+			}
+		})
+	}
+}
+
+func TestBackupNowNamedServers(t *testing.T) {
+	g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), stoppedWorld("delta"))
+	if code := g.run([]string{"delta", "alpha", "delta"}, true, false); code != 0 {
+		t.Fatalf("exit = %d, want 0", code)
+	}
+	if want := []string{"request delta", "request alpha"}; !reflect.DeepEqual(g.events, want) {
+		t.Fatalf("events = %v, want %v", g.events, want)
+	}
+	if !strings.HasPrefix(g.out.String(), "felis backup-now: 2 server(s), backed up one at a time:\n  delta  Stopped   back up\n  alpha  Stopped   back up\n") {
+		t.Fatalf("plan =\n%s", g.out.String())
+	}
+
+	for _, tc := range []struct{ name, want string }{
+		{"login", "felis backup-now: login is a system server: its world is rebuilt by felis setup and has no backups\n"},
+		{"lobby", "felis backup-now: lobby is a system server: its world is rebuilt by felis setup and has no backups\n"},
+		{"nope", "felis backup-now: no server named \"nope\"\n"},
+	} {
+		g := newBNRig(stoppedWorld("alpha"))
+		if code := g.run([]string{"alpha", tc.name}, true, false); code != 2 {
+			t.Fatalf("%s: exit = %d, want 2", tc.name, code)
+		}
+		if g.errw.String() != tc.want {
+			t.Fatalf("%s: stderr = %q, want %q", tc.name, g.errw.String(), tc.want)
+		}
+		if len(g.events) != 0 || g.out.Len() != 0 {
+			t.Fatalf("%s: acted on a bad name: events %v, output %q", tc.name, g.events, g.out.String())
+		}
+	}
+
+	g = newBNRig()
+	if code := g.run(nil, true, false); code != 0 || g.out.String() != "felis backup-now: there are no user servers\n" {
+		t.Fatalf("empty fleet: exit %d, output %q", code, g.out.String())
+	}
+}
+
+// The world list mirrors the backup route's own gate, so the plan says exactly what
+// the route would refuse.
+func TestListBackupNowWorlds(t *testing.T) {
+	withStatus := func(ms *v1alpha1.MinecraftServer, ready bool) *v1alpha1.MinecraftServer {
+		ms.Status.Ready = ready
+		return ms
+	}
+	pvc := func(server, ns string) *corev1.PersistentVolumeClaim {
+		return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: naming.WorldPVCName(server), Namespace: ns}}
+	}
+	pod := func(name, ns string, labels map[string]string) *corev1.Pod {
+		return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}}
+	}
+	gameLabels := func(server string) map[string]string {
+		return map[string]string{v1alpha1.LabelServer: server, v1alpha1.LabelComponent: "server"}
+	}
+	other := mcServer("zulu", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped)
+	other.Namespace = "elsewhere"
+	hotel := mcServer("hotel", v1alpha1.DesiredStopped, "")
+	objs := []client.Object{
+		mcServer("alpha", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("alpha", haltNS),
+		// A backup Job's pod carries the server label with its own component, and
+		// a game pod of the same name in another namespace is somebody else's.
+		pod("backup-alpha-1-x", haltNS, map[string]string{v1alpha1.LabelServer: "alpha", "app.kubernetes.io/component": "world-backup"}),
+		pod("alpha-0", "elsewhere", gameLabels("alpha")),
+		withStatus(mcServer("bravo", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning), true), pvc("bravo", haltNS), pod("bravo-0", haltNS, gameLabels("bravo")),
+		mcServer("charlie", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped),
+		mcServer("delta", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("delta", haltNS), pod("delta-0", haltNS, gameLabels("delta")),
+		mcServer("echo", v1alpha1.DesiredStopped, v1alpha1.PhaseStopping), pvc("echo", haltNS),
+		mcServer("foxtrot", v1alpha1.DesiredRunning, v1alpha1.PhaseStopped), pvc("foxtrot", haltNS),
+		withStatus(mcServer("golf", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), true), pvc("golf", haltNS),
+		hotel, pvc("hotel", haltNS),
+		mcServer("login", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("login", haltNS),
+		mcServer("lobby", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("lobby", haltNS),
+		other, pvc("zulu", "elsewhere"),
+	}
+	got, err := listBackupNowWorlds(context.Background(), haltClient(t, objs...), haltNS)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []backupNowWorld{
+		{name: "alpha", phase: "Stopped", stopped: true, hasWorld: true},
+		{name: "bravo", phase: "Running", hasWorld: true},
+		{name: "charlie", phase: "Stopped", stopped: true},
+		{name: "delta", phase: "Stopped", hasWorld: true},   // its pod is still going
+		{name: "echo", phase: "Stopping", hasWorld: true},   // still stopping
+		{name: "foxtrot", phase: "Stopped", hasWorld: true}, // asked to start
+		{name: "golf", phase: "Stopped", hasWorld: true},    // still reports ready
+		{name: "hotel", phase: "Stopped", hasWorld: true},   // not reconciled yet: the desired state shows
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("worlds =\n%+v\nwant\n%+v", got, want)
+	}
+}
+
+func TestBackupNowStopsWhenInterrupted(t *testing.T) {
+	t.Run("between worlds", func(t *testing.T) {
+		g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		g.ctx = ctx
+		g.onAct = func(e string) {
+			if e == "request alpha" {
+				cancel()
+			}
+		}
+		if code := g.run(nil, true, false); code != 1 {
+			t.Fatalf("exit = %d, want 1", code)
+		}
+		if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
+			t.Fatalf("events = %v, want %v", g.events, want)
+		}
+		_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
+		want := "  alpha: backing up\n  alpha: archived in 0s\n" +
+			"Interrupted: a backup Job already started runs to its end.\n" +
+			"1 backed up.\n" + bnOffsite
+		if run != want {
+			t.Fatalf("run =\n%s\nwant\n%s", run, want)
+		}
+	})
+	t.Run("while a Job runs", func(t *testing.T) {
+		g := newBNRig(stoppedWorld("alpha"))
+		g.states["alpha"] = []string{"running"}
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		g.ctx = ctx
+		g.onAct = func(string) { cancel() }
+		if code := g.run(nil, true, false); code != 1 {
+			t.Fatalf("exit = %d, want 1", code)
+		}
+		if g.jobPolls["alpha"] != 1 {
+			t.Fatalf("polled the Job %d times after the interrupt, want 1", g.jobPolls["alpha"])
+		}
+		if !strings.Contains(g.out.String(), "  alpha: failed: context canceled\nInterrupted: a backup Job already started runs to its end.\n0 backed up, 1 failed.\n") {
+			t.Fatalf("output =\n%s", g.out.String())
+		}
+	})
+	t.Run("while a server stops", func(t *testing.T) {
+		g := newBNRig(runningWorld("bravo"))
+		g.stopAfter["bravo"] = -1
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		g.ctx = ctx
+		g.onAct = func(string) { cancel() }
+		if code := g.run(nil, true, true); code != 1 {
+			t.Fatalf("exit = %d, want 1", code)
+		}
+		if g.polls["bravo"] != 1 {
+			t.Fatalf("polled bravo %d times after the interrupt, want 1", g.polls["bravo"])
+		}
+		_, run, _ := strings.Cut(g.out.String(), bnStopping)
+		want := "  bravo: stopping\n  bravo: failed: context canceled\n" +
+			"Interrupted: a backup Job already started runs to its end.\n" +
+			"0 backed up, 1 failed.\n" +
+			"Left stopped: bravo. Start them from the panel when you are done.\n"
+		if run != want {
+			t.Fatalf("run =\n%s\nwant\n%s", run, want)
+		}
+	})
+}
+
+func TestBackupNowReportsClusterErrors(t *testing.T) {
+	g := newBNRig(runningWorld("bravo"))
+	g.stopErr["bravo"] = errors.New("the apiserver is gone")
+	if code := g.run(nil, true, true); code != 1 {
+		t.Fatalf("exit = %d, want 1", code)
+	}
+	_, run, _ := strings.Cut(g.out.String(), bnStopping)
+	// The stop was asked for, so the server stays stopped whatever the poll said.
+	want := "  bravo: stopping\n  bravo: failed: the apiserver is gone\n0 backed up, 1 failed.\n" +
+		"Left stopped: bravo. Start them from the panel when you are done.\n"
+	if run != want {
+		t.Fatalf("run =\n%s\nwant\n%s", run, want)
+	}
+
+	for _, tc := range []struct {
+		call   int
+		events []string
+	}{
+		{1, nil},                       // before the request: nothing is asked for
+		{2, []string{"request alpha"}}, // the first poll after it
+	} {
+		g := newBNRig(stoppedWorld("alpha"))
+		g.jobsFail["alpha"] = tc.call
+		if code := g.run(nil, true, false); code != 1 {
+			t.Fatalf("call %d: exit = %d, want 1", tc.call, code)
+		}
+		if !reflect.DeepEqual(g.events, tc.events) {
+			t.Fatalf("call %d: events = %v, want %v", tc.call, g.events, tc.events)
+		}
+		if !strings.Contains(g.out.String(), "  alpha: failed: list its backup Jobs: the apiserver is gone\n") {
+			t.Fatalf("call %d: output =\n%s", tc.call, g.out.String())
+		}
+	}
+}

cmd/felis/backupnow_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,7 +12,6 @@ import (
 	"testing"
 	"time"
 
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 
 	corev1 "k8s.io/api/core/v1"
@@ -28,9 +28,15 @@ func internalAPIObjs(clusterIP, token string) []client.Object {
 			ObjectMeta: metav1.ObjectMeta{Name: platform.APIInternalServiceName, Namespace: bgControlNS},
 			Spec:       corev1.ServiceSpec{ClusterIP: clusterIP},
 		},
+		// The console presents the ops token; the proxy's felis-service-token sits
+		// beside it and must not be the one picked up.
 		&corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: bgControlNS},
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-ops-token", Namespace: bgControlNS},
+			Data:       map[string][]byte{"token": []byte(token)},
+		},
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-service-token", Namespace: bgControlNS},
+			Data:       map[string][]byte{"token": []byte("proxy-" + token)},
 		},
 	}
 }
@@ -114,16 +120,23 @@ func TestRequestBackup(t *testing.T) {
 	cases := []struct {
 		name   string
 		code   int
+		body   string // optional JSON error body
 		expect string
 	}{
-		{"409 not_stopped", http.StatusConflict, "must be stopped"},
-		{"503 backup_unavailable", http.StatusServiceUnavailable, "not configured"},
-		{"404 not found", http.StatusNotFound, "no such server"},
+		{"409 not_stopped", http.StatusConflict, "", "must be stopped"},
+		{"409 no_world_volume surfaces the API's own text", http.StatusConflict,
+			`{"error":{"code":"no_world_volume","message":"this server has no world volume yet — start it once to create it, then retry"}}`,
+			"no world volume yet"},
+		{"503 backup_unavailable", http.StatusServiceUnavailable, "", "not configured"},
+		{"404 not found", http.StatusNotFound, "", "no such server"},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(tc.code)
+				if tc.body != "" {
+					_, _ = io.WriteString(w, tc.body)
+				}
 			}))
 			defer srv.Close()
 			_, err := requestBackup(context.Background(), hc, srv.URL, "tok", "survival", "alice")
@@ -140,5 +153,31 @@ func TestRequestBackup(t *testing.T) {
 		if err == nil || !strings.Contains(err.Error(), "unreachable") {
 			t.Fatalf("err = %v, want an 'unreachable' transport error", err)
 		}
+		// backup-now ends its run on this error alone.
+		if !errors.Is(err, errBackupAPIUnreachable) {
+			t.Fatalf("err = %v, want it to wrap errBackupAPIUnreachable", err)
+		}
 	})
 }
+
+// The Sync picker must not offer system servers: the backup API validates names
+// and resolves a servers-table row, so a lobby/login pick can only die in
+// validation — a dead choice in an emergency console.
+func TestBackupPickable(t *testing.T) {
+	got := backupPickable([]haltableServer{
+		{name: "lobby", phase: "Running", system: true},
+		{name: "login", phase: "Running", system: true},
+		{name: "test-one", phase: "Stopped"},
+	})
+	if len(got) != 1 || got[0].name != "test-one" || got[0].system {
+		t.Fatalf("backupPickable = %+v, want only the user server", got)
+	}
+
+	// Survivors keep their input order (the picker's cursor math depends on it).
+	got = backupPickable([]haltableServer{
+		{name: "alpha"}, {name: "login", system: true}, {name: "beta"},
+	})
+	if len(got) != 2 || got[0].name != "alpha" || got[1].name != "beta" {
+		t.Fatalf("backupPickable order = %+v, want [alpha beta]", got)
+	}
+}

cmd/felis/bootstrap_assets.go

@@ -9,10 +9,17 @@ import (
 
 func cmdBootstrapAssets(args []string, stdout, stderr io.Writer) int {
 	if len(args) != 1 {
-		fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack")
+		fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack|config-keys")
 		return 2
 	}
 	switch args[0] {
+	case "config-keys":
+		// Optional keys the installer may emit; older binaries reject this verb.
+		_, err := fmt.Fprintln(stdout, "velocity.game_version")
+		if err != nil {
+			fmt.Fprintf(stderr, "felis bootstrap-assets: write: %v\n", err)
+			return 1
+		}
 	case "crd":
 		crd, err := felis.MinecraftServerCRD()
 		if err != nil {
@@ -32,7 +39,7 @@ func cmdBootstrapAssets(args []string, stdout, stderr io.Writer) int {
 			return 1
 		}
 	default:
-		fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack")
+		fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack|config-keys")
 		return 2
 	}
 	return 0

cmd/felis/breakglass.go

@@ -11,12 +11,14 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"strings"
 	"time"
 
 	"felis.lolicon.best/internal/api"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/store"
 
 	tea "github.com/charmbracelet/bubbletea"
@@ -33,13 +35,15 @@ import (
 //
 // Root is necessary but NOT sufficient for accountability: root is machine
 // authority, not a human identity, so the console additionally captures WHO is
-// breaking the glass. When a staff account already exists it asks the operator to
-// authenticate as an existing admin (the verified identity is the accountable
-// actor); when none exists yet it bootstraps the first Owner from the typed
-// credential and attributes the act to the OS user. The audit row records the
-// difference. This attribution is best-effort, not tamper-proof — whoever runs
-// this is root and can edit Postgres directly — but it produces an honest trail
-// for an honest operator, which is the point.
+// breaking the glass. When a staff account already exists the operator names one
+// and types the one-time code the console mails to its verified address
+// (breakglass_otp.go); that account is then the accountable actor. When no code can
+// be sent or proven, the typed OVERRIDE proceeds as the OS user and the audit row
+// says why. When no staff account exists yet it bootstraps the first Owner and
+// attributes the act to the OS user. The audit row records which of these
+// happened. This attribution is best-effort, not tamper-proof — whoever runs this
+// is root and can edit Postgres directly — but it produces an honest trail for an
+// honest operator, which is the point.
 //
 // When a staff account already exists the console opens on a thin top-level menu
 // (menuModel) so that operations are peers, not tails of one wizard. Two account
@@ -62,7 +66,7 @@ import (
 // suspension for the interactive `cloudflared tunnel login` browser consent.
 
 // breakGlassOverrideToken is the literal an operator must type to proceed when no
-// admin credential could be verified. Requiring an explicit, deliberate word (not a
+// admin could be verified by a mailed code. Requiring an explicit, deliberate word (not a
 // bare Enter) keeps the unverified root override from happening by reflex.
 const breakGlassOverrideToken = "OVERRIDE"
 
@@ -76,18 +80,23 @@ type ownerStore interface {
 	AdminExists(ctx context.Context) (bool, error)
 	// UserByUsername loads a staff login projection.
 	UserByUsername(ctx context.Context, username string) (*api.StaffUser, error)
+	// OwnerUsername names the single active Owner seat, or "" when none exists.
+	// provisionOwner refuses to re-target anything but this username: with the
+	// seat occupied, a fresh name would mint a second owner row (UpsertOwner's
+	// insert arm) while the existing — possibly compromised — seat stays live,
+	// and no supported path can delete an owner row afterwards.
+	OwnerUsername(ctx context.Context) (string, error)
 	UpsertOwner(ctx context.Context, id, username, email string) error
 	// InsertOperator mints a NEW Operator staff account. Unlike UpsertOwner it is
 	// insert-only: a username already taken is a conflict (api.ErrConflict), never a
 	// silent reset, so adding an Operator can never clobber the Owner or an existing
-	// Operator. The row is role=admin, identical in shape to the Owner — Felis has no
-	// separate operator DB role (migration 0003: staff = role=admin).
+	// Operator. The row is role=admin — an Operator is staff BELOW the single
+	// role=owner identity (migration 0011 adds that role); the two are the only
+	// staff roles.
 	InsertOperator(ctx context.Context, id, username, email string) error
-	// CompleteOwnerSetup atomically consumes the in-game link code, creates or
-	// promotes the bound Owner, enables local auth, and stores the one-time setup
-	// token. A failure rolls all four writes back so setup is always retryable.
-	CompleteOwnerSetup(ctx context.Context, newUserID, code string, now time.Time,
-		tokenHash string, tokenExpiresAt time.Time) (userID, mcUUID, authSource string, err error)
+	// CompleteOwnerSetup creates or resumes the first panel login atomically.
+	CompleteOwnerSetup(ctx context.Context, newUserID string, now time.Time,
+		tokenHash string, tokenExpiresAt time.Time) (userID, username string, err error)
 	SetSetting(ctx context.Context, key string, value []byte) error
 	// Audit records the break-glass accountability row.
 	Audit(ctx context.Context, e api.AuditEntry) error
@@ -135,7 +144,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 	repo := api.NewPGRepo(drv.DB())
 
 	// Decide bootstrap (no admin yet → typed credential mints the first Owner) vs
-	// recovery (an admin exists → the operator must authenticate as one) BEFORE the
+	// recovery (an admin exists → the operator proves one with a mailed code) BEFORE the
 	// alt-screen TUI takes over, so a database fault surfaces as a plain error.
 	adminExists, err := repo.AdminExists(ctx)
 	if err != nil {
@@ -143,7 +152,12 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	res, err := runBreakGlassTUI(ctx, repo, cfg.Database.URL, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists)
+	// Recovery mails its code through [smtp]; the relay is opened only if a code is
+	// asked for.
+	host, _ := os.Hostname()
+	recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, hostSMTPPasswordPath, platform.DefaultControlNamespace), host: host}
+
+	res, err := runBreakGlassTUI(ctx, repo, cfg.Database, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists, recovery)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis breakGlass: %v\n", err)
 		return 1
@@ -167,6 +181,9 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 			fmt.Fprintf(stdout, "\nfelis breakGlass: Owner account %q provisioned; local session sign-in is ENABLED.\n", res.username)
 		}
 		fmt.Fprintf(stdout, "Recorded as %q (mode: %s, os user: %s).\n", res.accountable, res.mode, res.osUser)
+		if res.mode == "root_override" {
+			fmt.Fprintln(stdout, "No admin was proven by an email code; the audit row records this run as an unverified root override and why.")
+		}
 		if res.setupTokenURL != "" {
 			fmt.Fprintf(stdout, "One-time setup URL (opens a lockdown session to verify email / enroll passkey):\n\n    %s\n\n", res.setupTokenURL)
 		}
@@ -251,35 +268,23 @@ func newOwnerID() string {
 	return "usr-" + hex.EncodeToString(b[:])
 }
 
-// authenticateAdmin resolves a typed admin username for recovery-mode attribution.
-// Password verification is gone (passwordless design); Phase 3 replaces this with
-// email-OTP recovery. For now it confirms the named admin exists.
-func authenticateAdmin(ctx context.Context, s ownerStore, username string) (matched string, ok bool, err error) {
-	username = strings.TrimSpace(username)
-	if username == "" {
-		return "", false, nil
-	}
-	u, err := s.UserByUsername(ctx, username)
-	if errors.Is(err, api.ErrNotFound) {
-		return "", false, nil
-	}
-	if err != nil {
-		return "", false, err
-	}
-	if u.Role != "admin" {
-		return "", false, nil
-	}
-	return u.Username, true, nil
-}
-
 // provisionOwner mints or resets the single Owner account direct-to-Postgres,
-// passwordless. The account is role=admin with no password — the Owner completes
+// passwordless. The account is role=owner with no password — the Owner completes
 // passwordless login setup via the web setup-token flow after `felis setup`.
+// With a seat already occupied the reset must name that seat (ownerSeatTakenError
+// otherwise): the upsert's insert arm would silently mint a SECOND owner, and
+// every owner row is undeletable through the panel, so the tier could never
+// converge back to one.
 func provisionOwner(ctx context.Context, s ownerStore, username, email string) error {
 	username = strings.TrimSpace(username)
 	if username == "" {
 		return errors.New("owner username is required")
 	}
+	if seat, err := s.OwnerUsername(ctx); err != nil {
+		return fmt.Errorf("check the owner seat: %w", err)
+	} else if seat != "" && seat != username {
+		return &ownerSeatTakenError{seat: seat}
+	}
 	id := newOwnerID()
 	if id == "" {
 		return errors.New("generate owner id: entropy source failed")
@@ -290,13 +295,26 @@ func provisionOwner(ctx context.Context, s ownerStore, username, email string) e
 	return nil
 }
 
-// provisionOperator mints a NEW Operator staff account direct-to-Postgres. Like the
-// Owner it is role=admin and passwordless — Felis has no separate operator DB role,
-// so an Operator is simply an additional staff admin (migration 0003). UNLIKE
-// provisionOwner, which upserts the single Owner and resets it on a username
-// conflict, this is insert-only: a username already taken returns api.ErrConflict
-// rather than overwriting a live account, so adding an Operator can never silently
-// clobber the Owner's or another Operator's account.
+// ownerSeatTakenError refuses an Owner reset that names anything but the
+// occupied seat, naming it so the operator can retype. Is reports
+// api.ErrConflict so the TUI's recoverable-error branch (shared with the
+// operator path's taken-name clash) routes back to the form instead of ending
+// the console.
+type ownerSeatTakenError struct{ seat string }
+
+func (e *ownerSeatTakenError) Error() string {
+	return fmt.Sprintf("an Owner already exists as %q — enter that username to reset the Owner", e.seat)
+}
+
+func (e *ownerSeatTakenError) Is(target error) bool { return target == api.ErrConflict }
+
+// provisionOperator mints a NEW Operator staff account direct-to-Postgres. It is
+// role=admin and passwordless — an additional staff admin below the single
+// role=owner identity (migrations 0003 + 0011). UNLIKE provisionOwner, which
+// upserts the single Owner and resets it on a username conflict, this is
+// insert-only: a username already taken returns api.ErrConflict rather than
+// overwriting a live account, so adding an Operator can never silently clobber
+// the Owner's or another Operator's account.
 func provisionOperator(ctx context.Context, s ownerStore, username, email string) error {
 	username = strings.TrimSpace(username)
 	if username == "" {
@@ -340,12 +358,16 @@ type breakGlassOp struct {
 	ownerUsername  string
 	ownerEmail     string
 	attemptedAdmin string // recovery / override: the admin username the operator typed
+	verifiedBy     string // recovery: how the admin was proven (verifiedByEmailOTP)
+	codeSentTo     string // recovery: the address the proving code went to
+	otpSkipped     string // root_override: why no code proved an admin (otpSkip*)
+	otpSkipDetail  string // root_override: what failed, when something did
 }
 
 // breakGlassOutcome is what performBreakGlass reports back to the TUI.
 type breakGlassOutcome struct {
 	setupTokenURL string // non-empty when setup minted a one-time first-login URL
-	ownerIdentity string // verified Minecraft UUID for the setup Owner-bind path
+	ownerUsername string // panel Owner created or resumed by setup
 	auditErr      error  // non-nil if the accountability row could not be written
 }
 
@@ -385,25 +407,12 @@ func newSetupToken() (raw, hash string, err error) {
 	return raw, hex.EncodeToString(sum[:]), nil
 }
 
-// performSetupMCBind is the `felis setup` Owner-establishment path: the operator
-// binds their Minecraft account via a one-time link code the login gate handed
-// them in-game, the bound user is promoted to role='admin' (passwordless Owner),
-// local auth is enabled, and a one-time setup URL is minted for the first web
-// login where the Owner verifies email / enrolls a passkey. adminHostname is the
-// operator-console host the URL points at (op.console.<root>): the Owner is staff,
-// so first-run onboarding belongs on the operator face, not the player panel. The
-// passkey verifier's RP id is the panel host, but its permitted origins now include
-// op.console (cmd/felis/api.go), so enrollment on op.console is a valid ceremony —
-// one binding that works on both faces. osUser is recorded as the accountable actor.
-//
-// Local auth is as load-bearing here as it is in break-glass, and for a sharper
-// reason: an MC-bound Owner has no password AND no email, so the setup token is
-// their ONLY door. CompleteOwnerSetup therefore commits the identity bind, auth
-// toggle, and token together; any failed write leaves the link code retryable.
-func performSetupMCBind(ctx context.Context, s ownerStore, code, adminHostname, osUser string) (breakGlassOutcome, error) {
-	code = strings.TrimSpace(strings.ToUpper(code))
-	if code == "" {
-		return breakGlassOutcome{}, errors.New("link code is required")
+// performSetupOwner establishes panel access under the caller's host-root
+// authority. Minecraft identity can be linked later from the authenticated panel.
+func performSetupOwner(ctx context.Context, s ownerStore, panelURL, osUser string) (breakGlassOutcome, error) {
+	base, err := url.Parse(strings.TrimRight(panelURL, "/"))
+	if err != nil || base.Scheme != "https" || base.Host == "" {
+		return breakGlassOutcome{}, errors.New("a configured HTTPS operator console is required")
 	}
 	newID := newOwnerID()
 	if newID == "" {
@@ -414,48 +423,21 @@ func performSetupMCBind(ctx context.Context, s ownerStore, code, adminHostname,
 		return breakGlassOutcome{}, err
 	}
 	now := time.Now()
-	_, mcUUID, authSource, err := s.CompleteOwnerSetup(
-		ctx, newID, code, now, hash, now.Add(setupTokenTTL))
+	userID, username, err := s.CompleteOwnerSetup(ctx, newID, now, hash, now.Add(setupTokenTTL))
+	out := breakGlassOutcome{ownerUsername: username}
 	if err != nil {
-		return breakGlassOutcome{}, fmt.Errorf("complete owner setup: %w", err)
+		return out, err
 	}
-	// The load-bearing writes committed together above. Accountability remains
-	// best-effort: an unhappy audit sink never costs the operator their install.
-	out := breakGlassOutcome{
-		ownerIdentity: mcUUID,
-		auditErr:      auditSetupMCBind(ctx, s, osUser, mcUUID, authSource),
-	}
-	host := strings.TrimSpace(adminHostname)
-	if host == "" {
-		host = "op.console.localhost"
-	}
-	out.setupTokenURL = "https://" + host + "/setup?token=" + raw
+	base.Path = "/setup"
+	base.RawQuery = url.Values{"token": {raw}}.Encode()
+	out.setupTokenURL = base.String()
+	blob, _ := json.Marshal(map[string]string{"os_user": osUser, "user_id": userID, "username": username})
+	out.auditErr = s.Audit(ctx, api.AuditEntry{
+		Actor: osUser, Source: "setup", Action: "setup.owner_login", Payload: blob,
+	})
 	return out, nil
 }
 
-// auditSetupMCBind records who claimed the Owner seat at setup. It carries the
-// Minecraft identity rather than a username because that IS the evidence: the
-// login gate only issues a link code to a player it authenticated, so mc_uuid +
-// auth_source say which account was verified and by whom. Actor is the OS user who
-// ran `felis setup` — honest attribution, not proof (root can edit the row).
-func auditSetupMCBind(ctx context.Context, s ownerStore, osUser, mcUUID, authSource string) error {
-	blob, err := json.Marshal(map[string]any{
-		"mode":        "setup",
-		"os_user":     osUser,
-		"mc_uuid":     mcUUID,
-		"auth_source": authSource,
-	})
-	if err != nil {
-		return err
-	}
-	return s.Audit(ctx, api.AuditEntry{
-		Actor:   osUser,
-		Source:  "setup",
-		Action:  "setup.owner_bind",
-		Payload: blob,
-	})
-}
-
 // auditBreakGlass writes the break-glass accountability row. The actor is the
 // resolved human identity (a verified admin in recovery, the OS user otherwise);
 // the payload carries the full who/what/how so an after-the-fact reader can tell a
@@ -463,16 +445,7 @@ func auditSetupMCBind(ctx context.Context, s ownerStore, osUser, mcUUID, authSou
 // does not fail the recovery if this write fails — and intentionally honest: it
 // records attribution, it does not prove it (a malicious root can edit the row).
 func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
-	payload := map[string]any{
-		"mode":     op.mode,
-		"owner":    op.ownerUsername,
-		"os_user":  op.osUser,
-		"verified": op.mode == "recovery",
-	}
-	if op.attemptedAdmin != "" {
-		payload["admin_account"] = op.attemptedAdmin
-	}
-	blob, err := json.Marshal(payload)
+	blob, err := json.Marshal(breakGlassPayload(op, "owner"))
 	if err != nil {
 		return err
 	}
@@ -484,6 +457,33 @@ func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
 	})
 }
 
+// breakGlassPayload is the who/how both account audits carry, with the account the
+// run wrote under subjectKey. verified is true only for a run a mailed code proved;
+// such a run names the address the code went to, and an override names why no code
+// proved anyone.
+func breakGlassPayload(op breakGlassOp, subjectKey string) map[string]any {
+	payload := map[string]any{
+		"mode":     op.mode,
+		subjectKey: op.ownerUsername,
+		"os_user":  op.osUser,
+		"verified": op.verifiedBy != "",
+	}
+	if op.attemptedAdmin != "" {
+		payload["admin_account"] = op.attemptedAdmin
+	}
+	if op.verifiedBy != "" {
+		payload["verified_by"] = op.verifiedBy
+		payload["code_sent_to"] = op.codeSentTo
+	}
+	if op.otpSkipped != "" {
+		payload["otp_skipped"] = op.otpSkipped
+		if op.otpSkipDetail != "" {
+			payload["otp_skip_detail"] = op.otpSkipDetail
+		}
+	}
+	return payload
+}
+
 // performAddOperator mints a NEW Operator account and records a best-effort
 // accountability row. It mirrors performBreakGlass — passwordless — with two
 // deliberate differences. (1) It provisions insert-only (provisionOperator), so it
@@ -507,16 +507,7 @@ func performAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) (bre
 // break_glass.operator_create action, naming the new account under an "operator" key
 // rather than "owner".
 func auditAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) error {
-	payload := map[string]any{
-		"mode":     op.mode,
-		"operator": op.ownerUsername,
-		"os_user":  op.osUser,
-		"verified": op.mode == "recovery",
-	}
-	if op.attemptedAdmin != "" {
-		payload["admin_account"] = op.attemptedAdmin
-	}
-	blob, err := json.Marshal(payload)
+	blob, err := json.Marshal(breakGlassPayload(op, "operator"))
 	if err != nil {
 		return err
 	}
@@ -595,16 +586,31 @@ const (
 	cloudflareAPITokenDocsURL        = "https://developers.cloudflare.com/fundamentals/api/how-to/account-owned-token-template/"
 )
 
-func runBreakGlassTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
-	return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass)
+func runBreakGlassTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, recovery recoveryConfig) (breakGlassResult, error) {
+	return runConsoleTUI(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass, recovery)
 }
 
-func runSetupTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
-	return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup)
+// runSetupTUI configures deployment before issuing the first panel login link.
+func runSetupTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
+	return runConsoleRoot(newConsoleRoot(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup, recoveryConfig{}))
 }
 
-func runConsoleTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) (breakGlassResult, error) {
-	rm := newRootModel(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
+// newConsoleRoot is the console's root model as the host runs it: the summary
+// reads this host's alert route.
+func newConsoleRoot(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) *rootModel {
+	rm := newRootModel(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
+	rm.recovery = recovery
+	rm.alertRoute = func(ctx context.Context) alertRoute {
+		return hostAlertRoute(ctx, hostSetupConfigPath, db.URL, defaultHeartbeatFile)
+	}
+	return rm
+}
+
+func runConsoleTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) (breakGlassResult, error) {
+	return runConsoleRoot(newConsoleRoot(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode, recovery))
+}
+
+func runConsoleRoot(rm *rootModel) (breakGlassResult, error) {
 	final, err := tea.NewProgram(rm, tea.WithAltScreen()).Run()
 	if err != nil {
 		return breakGlassResult{}, err

cmd/felis/breakglass_otp.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/subtle"
+	"errors"
+	"fmt"
+	"math/big"
+	"os"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
+)
+
+// Recovery mode proves who is breaking the glass (#13). Naming a staff account is
+// where it starts: the console then mails a one-time code to that account's verified
+// address, and only that code, typed within recoveryCodeTTL, makes the run a
+// recovery attributed to the account. Every other ending — no such account, no
+// verified address, no relay, a send that fails, a wrong or late code, or the
+// operator giving up on the mail — leads to the typed OVERRIDE, which the audit row
+// records as an unverified root_override together with the reason (otp_skipped).
+// The code goes through the same [smtp] relay as the panel's login codes, so with
+// that relay down recovery still works, as an override that says why.
+
+const (
+	recoveryCodeTTL      = 10 * time.Minute
+	recoveryCodeAttempts = 5
+)
+
+// The reasons a run fell back to the override, recorded as otp_skipped.
+const (
+	otpSkipUnknownAdmin    = "unknown_admin"
+	otpSkipNoVerifiedEmail = "no_verified_email"
+	otpSkipNoRelay         = "no_relay"
+	otpSkipSendFailed      = "send_failed"
+	otpSkipCodeExpired     = "code_expired"
+	otpSkipCodeRejected    = "code_rejected"
+	otpSkipByOperator      = "operator_skipped"
+)
+
+// verifiedByEmailOTP is the audit's verified_by for a recovery the mailed code proved.
+const verifiedByEmailOTP = "email_otp"
+
+// recoveryMailer is the one relay call a recovery code needs; *mail.SMTP has it.
+type recoveryMailer interface {
+	SendNotice(ctx context.Context, email, subject, body string) error
+}
+
+// recoveryConfig is what the console needs to mail a recovery code. open resolves
+// the relay only when a code is about to go out, so a console used to halt a server
+// never touches [smtp] or the cluster; its error says why no relay is available.
+// host names this machine in the mail. The zero value has no relay.
+type recoveryConfig struct {
+	open func(ctx context.Context) (recoveryMailer, error)
+	host string
+	now  func() time.Time
+}
+
+func (r recoveryConfig) clock() time.Time {
+	if r.now != nil {
+		return r.now()
+	}
+	return time.Now()
+}
+
+// recoveryCode is one mailed code: its value, when it stops working, and how many
+// wrong codes were typed against it.
+type recoveryCode struct {
+	value    string
+	expires  time.Time
+	failures int
+}
+
+func newRecoveryCode(now time.Time) (*recoveryCode, error) {
+	n, err := rand.Int(rand.Reader, big.NewInt(1_000_000))
+	if err != nil {
+		return nil, fmt.Errorf("generate recovery code: %w", err)
+	}
+	return &recoveryCode{value: fmt.Sprintf("%06d", n.Int64()), expires: now.Add(recoveryCodeTTL)}, nil
+}
+
+type codeVerdict int
+
+const (
+	codeAccepted codeVerdict = iota
+	codeWrong
+	codeExpired
+	codeExhausted
+)
+
+// check compares a typed code in constant time. Each wrong code counts; the one
+// that reaches recoveryCodeAttempts exhausts the code, which then accepts nothing,
+// and neither does an expired one.
+func (c *recoveryCode) check(typed string, now time.Time) codeVerdict {
+	if c.failures >= recoveryCodeAttempts {
+		return codeExhausted
+	}
+	if !now.Before(c.expires) {
+		return codeExpired
+	}
+	if subtle.ConstantTimeCompare([]byte(strings.TrimSpace(typed)), []byte(c.value)) == 1 {
+		return codeAccepted
+	}
+	c.failures++
+	if c.failures >= recoveryCodeAttempts {
+		return codeExhausted
+	}
+	return codeWrong
+}
+
+func (c *recoveryCode) attemptsLeft() int { return recoveryCodeAttempts - c.failures }
+
+// recoveryStart is where naming an admin led: a code on its way to that admin, or
+// the reason the run has to fall back to the override.
+type recoveryStart struct {
+	admin  *api.StaffUser // the named staff account; nil when none matched
+	code   *recoveryCode  // set when the code went out
+	skip   string         // otpSkip* when it did not
+	detail string         // what failed, for the override screen and the audit row
+}
+
+// resolveAdmin loads the staff account (admin or owner) a typed username names, or
+// nil when there is none.
+func resolveAdmin(ctx context.Context, s ownerStore, username string) (*api.StaffUser, error) {
+	username = strings.TrimSpace(username)
+	if username == "" {
+		return nil, nil
+	}
+	u, err := s.UserByUsername(ctx, username)
+	if errors.Is(err, api.ErrNotFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	// Staff means admin or owner: the Owner is the primary break-glass identity.
+	if u.Role != "admin" && u.Role != "owner" {
+		return nil, nil
+	}
+	return u, nil
+}
+
+// beginRecovery resolves the named admin and mails it a recovery code. Only a
+// datastore or entropy fault is an error; every other way the code cannot go out is
+// a recoveryStart with skip set.
+func beginRecovery(ctx context.Context, s ownerStore, rc recoveryConfig, username, osUser string, op bgOperation) (recoveryStart, error) {
+	admin, err := resolveAdmin(ctx, s, username)
+	if err != nil {
+		return recoveryStart{}, err
+	}
+	if admin == nil {
+		return recoveryStart{skip: otpSkipUnknownAdmin}, nil
+	}
+	st := recoveryStart{admin: admin}
+	// An address nobody ever proved vouches for nobody.
+	email := strings.TrimSpace(admin.Email)
+	if email == "" || !admin.EmailVerified {
+		st.skip = otpSkipNoVerifiedEmail
+		return st, nil
+	}
+	if rc.open == nil {
+		st.skip, st.detail = otpSkipNoRelay, "this console has no mail relay"
+		return st, nil
+	}
+	relay, err := rc.open(ctx)
+	if err != nil {
+		st.skip, st.detail = otpSkipNoRelay, err.Error()
+		return st, nil
+	}
+	code, err := newRecoveryCode(rc.clock())
+	if err != nil {
+		return recoveryStart{}, err
+	}
+	sendCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	subject, body := recoveryMail(code.value, rc.host, osUser, admin.Username, op)
+	if err := relay.SendNotice(sendCtx, email, subject, body); err != nil {
+		st.skip, st.detail = otpSkipSendFailed, err.Error()
+		return st, nil
+	}
+	st.code = code
+	return st, nil
+}
+
+// recoveryMail words the code mail. It says where, by whom and for what the console
+// was opened, so an admin who did not ask for it learns that root on that machine is
+// in someone else's hands.
+func recoveryMail(code, host, osUser, admin string, op bgOperation) (subject, body string) {
+	what, whatZH := "reset the Owner account", "重置 Owner 账号"
+	if op == bgAddOperator {
+		what, whatZH = "add an Operator account", "添加 Operator 账号"
+	}
+	if host == "" {
+		host = "the Felis host"
+	}
+	minutes := int(recoveryCodeTTL / time.Minute)
+	subject = "Felis break-glass recovery code / 紧急恢复验证码"
+	body = fmt.Sprintf(`Someone with root on %[1]s (OS user %[2]s) opened felis breakGlass and named your staff account %[3]q to %[4]s.
+
+Recovery code: %[6]s
+It works for %[7]d minutes.
+
+If this was not you, root on that machine is in someone else's hands: change its credentials and read the audit log for break_glass entries.
+
+有人在 %[1]s 上以 root 身份（系统用户 %[2]s）打开了 felis breakGlass，指名你的管理员账号 %[3]q 来%[5]s。
+
+恢复验证码：%[6]s
+%[7]d 分钟内有效。
+
+如果不是你本人，这台机器的 root 已落入他人之手：请更换它的凭据，并查看审计日志中的 break_glass 记录。
+`, host, osUser, admin, what, whatZH, code, minutes)
+	return subject, body
+}
+
+// maskEmail keeps the first character of the local part and the domain, enough for
+// the operator to recognise the address without putting it on screen whole.
+func maskEmail(email string) string {
+	at := strings.LastIndex(email, "@")
+	if at <= 0 {
+		return "***"
+	}
+	return email[:1] + strings.Repeat("*", max(at-1, 3)) + email[at:]
+}
+
+// hostRecoveryMailer opens the [smtp] relay from the host the way the watchdog does:
+// the password is the env var password_ref names when that is set, else the host
+// copy at passwordPath, else the felis-smtp Secret, whose absence means a relay
+// without AUTH. The cluster is reached only when the host copy is missing, so a
+// break-glass on a host whose k3s is down still gets its code.
+func hostRecoveryMailer(c config.SMTPConfig, passwordPath, controlNS string) func(context.Context) (recoveryMailer, error) {
+	return func(ctx context.Context) (recoveryMailer, error) {
+		if strings.TrimSpace(c.Host) == "" {
+			return nil, errors.New("[smtp] is not configured in felis.toml")
+		}
+		if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" {
+			return smtpRelay(c, os.Getenv(ref)), nil
+		}
+		if password, ok, err := readHostCredential(passwordPath); err != nil {
+			return nil, fmt.Errorf("read the relay password: %w", err)
+		} else if ok {
+			return smtpRelay(c, password), nil
+		}
+		cl, err := buildSystemServerClient()
+		if err != nil {
+			return nil, fmt.Errorf("reach the cluster for the relay password: %w", err)
+		}
+		ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+		defer cancel()
+		password, err := smtpSecretPassword(ctx, cl, controlNS)
+		if err != nil {
+			return nil, fmt.Errorf("read the relay password from %s/%s: %w", controlNS, platform.SMTPSecretName, err)
+		}
+		return smtpRelay(c, password), nil
+	}
+}

cmd/felis/breakglass_otp_test.go

@@ -0,0 +1,498 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/platform"
+
+	tea "github.com/charmbracelet/bubbletea"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// These tests cover the recovery proof (#13): the mailed code's rules, where
+// naming an admin leads, what the mail says, how the console walks from a name to
+// a proven (or overridden) run, and what the audit row then records. No mail
+// leaves the process: the relay is a fake that keeps what it was handed.
+
+type sentMail struct{ to, subject, body string }
+
+type fakeRelay struct {
+	sent []sentMail
+	err  error
+}
+
+func (r *fakeRelay) SendNotice(_ context.Context, to, subject, body string) error {
+	if r.err != nil {
+		return r.err
+	}
+	r.sent = append(r.sent, sentMail{to, subject, body})
+	return nil
+}
+
+// sentCode pulls the code out of the one mail the relay carried.
+func (r *fakeRelay) sentCode(t *testing.T) string {
+	t.Helper()
+	if len(r.sent) != 1 {
+		t.Fatalf("relay carried %d mails, want 1", len(r.sent))
+	}
+	_, after, ok := strings.Cut(r.sent[0].body, "Recovery code: ")
+	if !ok || len(after) < 6 {
+		t.Fatalf("mail carries no recovery code:\n%s", r.sent[0].body)
+	}
+	return after[:6]
+}
+
+func relayConfig(r *fakeRelay, now func() time.Time) recoveryConfig {
+	return recoveryConfig{
+		open: func(context.Context) (recoveryMailer, error) { return r, nil },
+		host: "felis-host-1",
+		now:  now,
+	}
+}
+
+func verifiedAdmin(username, email string) *api.StaffUser {
+	return &api.StaffUser{ID: "usr-" + username, Username: username, Role: "owner", Email: email, EmailVerified: true}
+}
+
+func TestRecoveryCodeRules(t *testing.T) {
+	t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
+
+	t.Run("a fresh code is six digits and lives for the TTL", func(t *testing.T) {
+		seen := map[string]bool{}
+		for i := 0; i < 20; i++ {
+			c, err := newRecoveryCode(t0)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !isRecoveryCodeShape(c.value) {
+				t.Fatalf("code %q is not six digits", c.value)
+			}
+			if !c.expires.Equal(t0.Add(recoveryCodeTTL)) {
+				t.Fatalf("expires = %v, want %v", c.expires, t0.Add(recoveryCodeTTL))
+			}
+			seen[c.value] = true
+		}
+		if len(seen) < 2 {
+			t.Error("twenty codes were all the same")
+		}
+	})
+
+	t.Run("the right code is accepted, surrounding space ignored", func(t *testing.T) {
+		c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		if v := c.check(" 042917 ", t0); v != codeAccepted {
+			t.Errorf("check = %v, want accepted", v)
+		}
+	})
+
+	t.Run("wrong codes count down and the last one exhausts it", func(t *testing.T) {
+		c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		for i := 1; i < recoveryCodeAttempts; i++ {
+			if v := c.check("000000", t0); v != codeWrong {
+				t.Fatalf("wrong code %d: check = %v, want wrong", i, v)
+			}
+			if left := c.attemptsLeft(); left != recoveryCodeAttempts-i {
+				t.Fatalf("after %d wrong codes attemptsLeft = %d, want %d", i, left, recoveryCodeAttempts-i)
+			}
+		}
+		if v := c.check("000000", t0); v != codeExhausted {
+			t.Fatalf("wrong code %d: check = %v, want exhausted", recoveryCodeAttempts, v)
+		}
+		if v := c.check("042917", t0); v != codeExhausted {
+			t.Errorf("the right code after exhaustion: check = %v, want exhausted", v)
+		}
+	})
+
+	t.Run("an expired code accepts nothing", func(t *testing.T) {
+		c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		if v := c.check("042917", t0.Add(recoveryCodeTTL-time.Second)); v != codeAccepted {
+			t.Fatalf("a second before expiry: check = %v, want accepted", v)
+		}
+		c = &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
+		if v := c.check("042917", t0.Add(recoveryCodeTTL)); v != codeExpired {
+			t.Errorf("at expiry: check = %v, want expired", v)
+		}
+	})
+}
+
+func TestBeginRecovery(t *testing.T) {
+	ctx := context.Background()
+	t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
+	clock := func() time.Time { return t0 }
+
+	t.Run("mails a code to the named admin's verified address", func(t *testing.T) {
+		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
+		r := &fakeRelay{}
+		st, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgAddOperator)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if st.code == nil || st.skip != "" {
+			t.Fatalf("start = %+v, want a code and no skip", st)
+		}
+		if st.admin == nil || st.admin.Username != "root" {
+			t.Fatalf("start.admin = %+v, want root", st.admin)
+		}
+		if got := r.sentCode(t); got != st.code.value {
+			t.Errorf("mailed code %q, want the code the console checks (%q)", got, st.code.value)
+		}
+		m := r.sent[0]
+		if m.to != "[email protected]" {
+			t.Errorf("mail went to %q, want [email protected]", m.to)
+		}
+		for _, want := range []string{"felis-host-1", "OS user alice", `"root"`, "add an Operator account", "添加 Operator 账号", "10 minutes"} {
+			if !strings.Contains(m.body, want) {
+				t.Errorf("mail body lacks %q:\n%s", want, m.body)
+			}
+		}
+		if !st.code.expires.Equal(t0.Add(recoveryCodeTTL)) {
+			t.Errorf("code expires %v, want %v", st.code.expires, t0.Add(recoveryCodeTTL))
+		}
+	})
+
+	t.Run("the Owner reset is named as such", func(t *testing.T) {
+		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
+		r := &fakeRelay{}
+		if _, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgProvisionOwner); err != nil {
+			t.Fatal(err)
+		}
+		if body := r.sent[0].body; !strings.Contains(body, "reset the Owner account") || !strings.Contains(body, "重置 Owner 账号") {
+			t.Errorf("mail body does not name the Owner reset:\n%s", body)
+		}
+	})
+
+	// Each way the code cannot go out ends in a skip reason and no mail.
+	unverified := verifiedAdmin("root", "[email protected]")
+	unverified.EmailVerified = false
+	noEmail := verifiedAdmin("root", "")
+	cases := []struct {
+		name       string
+		user       *api.StaffUser
+		rc         func(r *fakeRelay) recoveryConfig
+		skip       string
+		detail     string
+		wantAdmin  bool
+		relayError error
+	}{
+		{name: "unknown admin", user: nil, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipUnknownAdmin},
+		{name: "unverified address", user: unverified, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
+		{name: "no address", user: noEmail, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
+		{name: "no relay wired", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig { return recoveryConfig{} }, skip: otpSkipNoRelay, detail: "this console has no mail relay", wantAdmin: true},
+		{name: "relay cannot open", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig {
+			return recoveryConfig{open: func(context.Context) (recoveryMailer, error) {
+				return nil, errors.New("[smtp] is not configured in felis.toml")
+			}}
+		}, skip: otpSkipNoRelay, detail: "[smtp] is not configured in felis.toml", wantAdmin: true},
+		{name: "send fails", user: verifiedAdmin("root", "[email protected]"), rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) },
+			skip: otpSkipSendFailed, detail: "554 relay refused", wantAdmin: true, relayError: errors.New("554 relay refused")},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := &fakeOwnerStore{users: map[string]*api.StaffUser{}}
+			if tc.user != nil {
+				f.users["root"] = tc.user
+			}
+			r := &fakeRelay{err: tc.relayError}
+			st, err := beginRecovery(ctx, f, tc.rc(r), "root", "alice", bgProvisionOwner)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if st.code != nil || st.skip != tc.skip || st.detail != tc.detail {
+				t.Errorf("start = {code:%v skip:%q detail:%q}, want no code, skip %q, detail %q", st.code, st.skip, st.detail, tc.skip, tc.detail)
+			}
+			if (st.admin != nil) != tc.wantAdmin {
+				t.Errorf("start.admin = %+v, want present=%v", st.admin, tc.wantAdmin)
+			}
+			if len(r.sent) != 0 {
+				t.Errorf("relay carried %d mails, want none", len(r.sent))
+			}
+		})
+	}
+
+	t.Run("a datastore fault is an error", func(t *testing.T) {
+		f := &fakeOwnerStore{userErr: errors.New("db down")}
+		if _, err := beginRecovery(ctx, f, relayConfig(&fakeRelay{}, clock), "root", "alice", bgProvisionOwner); err == nil {
+			t.Fatal("want the store fault")
+		}
+	})
+}
+
+func TestMaskEmail(t *testing.T) {
+	for in, want := range map[string]string{
+		"[email protected]": "a****@example.com",
+		"[email protected]":    "a***@example.com",
+		"@example.com":      "***",
+		"nonsense":          "***",
+	} {
+		if got := maskEmail(in); got != want {
+			t.Errorf("maskEmail(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestHostRecoveryMailer(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("no [smtp] host is no relay", func(t *testing.T) {
+		_, err := hostRecoveryMailer(config.SMTPConfig{}, hostSMTPPasswordPath, "felis")(ctx)
+		if err == nil || !strings.Contains(err.Error(), "[smtp]") {
+			t.Fatalf("err = %v, want it to name [smtp]", err)
+		}
+	})
+
+	t.Run("the password_ref env var supplies the password", func(t *testing.T) {
+		t.Setenv("FELIS_TEST_RELAY_PW", "from-env")
+		off := false
+		c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_RELAY_PW", RequireTLS: &off}
+		got, err := hostRecoveryMailer(c, hostSMTPPasswordPath, "felis")(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		relay, ok := got.(*mail.SMTP)
+		if !ok {
+			t.Fatalf("relay is %T, want *mail.SMTP", got)
+		}
+		if relay.Password != "from-env" || relay.Host != "mail.example.com" || relay.Port != 2525 || relay.From != "[email protected]" || relay.Username != "felis" || relay.RequireTLS {
+			t.Errorf("relay = %+v, want the [smtp] fields with the env password and TLS as configured", *relay)
+		}
+	})
+}
+
+func TestSMTPSecretPassword(t *testing.T) {
+	ctx := context.Background()
+	scheme := haltScheme(t)
+
+	t.Run("reads the password key", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
+			Data:       map[string][]byte{platform.SMTPSecretPasswordKey: []byte("s3cret")},
+		}).Build()
+		if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "s3cret" {
+			t.Errorf("smtpSecretPassword = (%q, %v), want (s3cret, nil)", pw, err)
+		}
+	})
+
+	t.Run("a missing Secret is a relay without AUTH", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
+		if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "" {
+			t.Errorf("smtpSecretPassword = (%q, %v), want (\"\", nil)", pw, err)
+		}
+	})
+
+	t.Run("any other read failure is an error", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{
+			Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
+				return apierrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, platform.SMTPSecretName, errors.New("rbac"))
+			},
+		}).Build()
+		if _, err := smtpSecretPassword(ctx, cl, "felis"); err == nil {
+			t.Fatal("want the read failure")
+		}
+	})
+}
+
+// recoveryModel is an Owner-reset console for admin "root" (verified address
+// [email protected]), run by OS user alice, with the fake relay behind it.
+func recoveryModel(t *testing.T, f *fakeOwnerStore, r *fakeRelay, now func() time.Time) *ownerModel {
+	t.Helper()
+	if f.users == nil {
+		f.users = map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}
+	}
+	return newOwnerModel(context.Background(), f, "alice", true).withRecovery(relayConfig(r, now))
+}
+
+// nameAdmin submits the admin-name form and feeds the result of the send back in.
+func nameAdmin(t *testing.T, m *ownerModel, name string) *ownerModel {
+	t.Helper()
+	m.authUser = name
+	_, cmd := m.onFormComplete()
+	if m.step != owWorking {
+		t.Fatalf("after naming the admin step = %v, want owWorking", m.step)
+	}
+	msg := findMsg[owAuthMsg](t, cmd)
+	next, _ := m.Update(msg)
+	return next.(*ownerModel)
+}
+
+// findMsg runs a (possibly batched) command and returns the first T it produces.
+func findMsg[T any](t *testing.T, cmd tea.Cmd) T {
+	t.Helper()
+	var zero T
+	if cmd == nil {
+		t.Fatalf("no command, want one producing %T", zero)
+	}
+	switch msg := cmd().(type) {
+	case T:
+		return msg
+	case tea.BatchMsg:
+		for _, c := range msg {
+			if c == nil {
+				continue
+			}
+			if got, ok := c().(T); ok {
+				return got
+			}
+		}
+	}
+	t.Fatalf("command produced no %T", zero)
+	return zero
+}
+
+// typeCode submits the code form with typed.
+func typeCode(t *testing.T, m *ownerModel, typed string) {
+	t.Helper()
+	if m.step != owCode {
+		t.Fatalf("step = %v, want owCode", m.step)
+	}
+	m.codeInput = typed
+	m.onFormComplete()
+}
+
+// provisionAudit finishes the run as an Owner reset and returns its audit payload.
+func provisionAudit(t *testing.T, m *ownerModel, f *fakeOwnerStore) (api.AuditEntry, map[string]any) {
+	t.Helper()
+	if m.step != owProvision {
+		t.Fatalf("step = %v, want owProvision", m.step)
+	}
+	m.username = "owner"
+	msg := m.provisionCmd()().(owProvisionMsg)
+	if msg.err != nil {
+		t.Fatalf("provision: %v", msg.err)
+	}
+	return auditOf(t, f)
+}
+
+func TestRecoveryConsoleFlow(t *testing.T) {
+	t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
+	fixed := func() time.Time { return t0 }
+
+	t.Run("the mailed code proves the admin and the audit says so", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		typeCode(t, m, r.sentCode(t))
+		if m.mode != "recovery" || m.accountable != "root" {
+			t.Fatalf("mode=%q accountable=%q, want recovery attributed to root", m.mode, m.accountable)
+		}
+		e, payload := provisionAudit(t, m, f)
+		if e.Actor != "root" || e.Action != "break_glass.recovery" {
+			t.Errorf("audit = %+v, want actor=root action=break_glass.recovery", e)
+		}
+		if payload["verified"] != true || payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" || payload["os_user"] != "alice" {
+			t.Errorf("payload = %v, want verified by email_otp to [email protected], os_user alice", payload)
+		}
+	})
+
+	t.Run("a wrong code asks again, and the last wrong one leads to the override", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		wrong := "000000"
+		if r.sentCode(t) == wrong {
+			wrong = "111111"
+		}
+		for i := 1; i < recoveryCodeAttempts; i++ {
+			typeCode(t, m, wrong)
+			if m.step != owCode || !strings.Contains(m.codeNote, "wrong") {
+				t.Fatalf("wrong code %d: step=%v note=%q, want the code form again with a note", i, m.step, m.codeNote)
+			}
+		}
+		typeCode(t, m, wrong)
+		if m.step != owOverride || m.skip != otpSkipCodeRejected {
+			t.Fatalf("after %d wrong codes step=%v skip=%q, want the override for code_rejected", recoveryCodeAttempts, m.step, m.skip)
+		}
+		m.onFormComplete() // OVERRIDE typed
+		e, payload := provisionAudit(t, m, f)
+		if e.Actor != "alice" || e.Action != "break_glass.root_override" {
+			t.Errorf("audit = %+v, want actor=alice action=break_glass.root_override", e)
+		}
+		if payload["verified"] != false || payload["otp_skipped"] != otpSkipCodeRejected || payload["admin_account"] != "root" {
+			t.Errorf("payload = %v, want unverified, otp_skipped=code_rejected, admin_account=root", payload)
+		}
+	})
+
+	t.Run("a late code leads to the override", func(t *testing.T) {
+		now := t0
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, func() time.Time { return now }), "root")
+		now = t0.Add(recoveryCodeTTL)
+		typeCode(t, m, r.sentCode(t))
+		if m.step != owOverride || m.skip != otpSkipCodeExpired {
+			t.Fatalf("step=%v skip=%q, want the override for code_expired", m.step, m.skip)
+		}
+	})
+
+	t.Run("OVERRIDE at the code prompt goes on unverified, saying the operator skipped", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		typeCode(t, m, breakGlassOverrideToken)
+		if m.mode != "root_override" || m.accountable != "alice" {
+			t.Fatalf("mode=%q accountable=%q, want root_override as alice", m.mode, m.accountable)
+		}
+		_, payload := provisionAudit(t, m, f)
+		if payload["verified"] != false || payload["otp_skipped"] != otpSkipByOperator {
+			t.Errorf("payload = %v, want unverified, otp_skipped=operator_skipped", payload)
+		}
+	})
+
+	t.Run("a relay failure leads to the override naming it", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{err: errors.New("dial tcp 10.0.0.9:587: connect: connection refused")}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		if m.step != owOverride || m.skip != otpSkipSendFailed {
+			t.Fatalf("step=%v skip=%q, want the override for send_failed", m.step, m.skip)
+		}
+		if reason := m.overrideReason(); !strings.Contains(reason, "connection refused") {
+			t.Errorf("override reason %q does not name the failure", reason)
+		}
+		m.onFormComplete()
+		_, payload := provisionAudit(t, m, f)
+		if payload["otp_skipped"] != otpSkipSendFailed || !strings.Contains(payload["otp_skip_detail"].(string), "connection refused") {
+			t.Errorf("payload = %v, want otp_skipped=send_failed with the relay's error", payload)
+		}
+	})
+
+	t.Run("esc at the code prompt starts over and forgets the code", func(t *testing.T) {
+		f, r := &fakeOwnerStore{}, &fakeRelay{}
+		m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
+		code := r.sentCode(t)
+		next, _ := m.Update(key(tea.KeyEsc))
+		m = next.(*ownerModel)
+		if m.step != owAuth || m.code != nil || m.admin != nil {
+			t.Fatalf("after esc step=%v code=%v admin=%v, want owAuth with the attempt forgotten", m.step, m.code, m.admin)
+		}
+		// The old code cannot be replayed: the next name mails a new one.
+		m = nameAdmin(t, m, "root")
+		if len(r.sent) != 2 {
+			t.Fatalf("relay carried %d mails, want a second one for the new attempt", len(r.sent))
+		}
+		_, fresh, _ := strings.Cut(r.sent[1].body, "Recovery code: ")
+		if m.code.value != fresh[:6] {
+			t.Errorf("the console checks %q, want the newly mailed %q (old one was %q)", m.code.value, fresh[:6], code)
+		}
+	})
+}
+
+func TestRootHandsRecoveryToAccountOperations(t *testing.T) {
+	for _, op := range []bgOperation{bgProvisionOwner, bgAddOperator} {
+		m := newTestRoot(true, consoleModeBreakGlass, "")
+		m.recovery = recoveryConfig{host: "felis-host-1"}
+		m = drive(t, m, menuChoiceMsg{op: op})
+		om, ok := m.screen.(*ownerModel)
+		if !ok {
+			t.Fatalf("op %v: screen = %T, want *ownerModel", op, m.screen)
+		}
+		if om.recovery.host != "felis-host-1" {
+			t.Errorf("op %v: the account screen has no relay config; its codes could never go out", op)
+		}
+	}
+}

cmd/felis/breakglass_test.go

@@ -19,20 +19,19 @@ import (
 // terminal. The design is passwordless: accounts carry no credential, and the
 // Owner completes first-login through the setup-token web flow.
 type fakeOwnerStore struct {
-	upserts  []upsertCall
-	inserts  []upsertCall
-	settings map[string][]byte
-	audits   []api.AuditEntry
-	tokens   []setupTokenCall
-	redeems  []redeemCall
-	users    map[string]*api.StaffUser // keyed by username
-	admins   bool                      // AdminExists answer
+	upserts   []upsertCall
+	inserts   []upsertCall
+	settings  map[string][]byte
+	audits    []api.AuditEntry
+	tokens    []setupTokenCall
+	setupIDs  []string
+	users     map[string]*api.StaffUser // keyed by username
+	admins    bool                      // AdminExists answer
+	ownerSeat string                    // OwnerUsername answer: the occupied seat, "" when none
 
-	// CompleteOwnerSetup's success result. redeemUserID defaults to the fresh id
-	// the caller passes (the unlinked-UUID case) when left empty.
-	redeemUserID     string
-	redeemMCUUID     string
-	redeemAuthSource string
+	setupUserID   string
+	setupUsername string
+	onboarded     bool
 
 	upsertErr      error
 	insertErr      error
@@ -40,6 +39,7 @@ type fakeOwnerStore struct {
 	auditErr       error
 	userErr        error // non-not-found error from UserByUsername
 	adminErr       error
+	seatErr        error
 	redeemErr      error
 	createTokenErr error
 }
@@ -57,12 +57,6 @@ type setupTokenCall struct {
 	expiresAt time.Time
 }
 
-// redeemCall records the inputs CompleteOwnerSetup was called with.
-type redeemCall struct {
-	newUserID string
-	code      string
-}
-
 func (f *fakeOwnerStore) AdminExists(_ context.Context) (bool, error) {
 	if f.adminErr != nil {
 		return false, f.adminErr
@@ -80,6 +74,15 @@ func (f *fakeOwnerStore) UserByUsername(_ context.Context, username string) (*ap
 	return nil, api.ErrNotFound
 }
 
+// OwnerUsername reports the single active Owner seat. Tests set ownerSeat; the
+// zero value models a fresh install where bootstrap is free to mint.
+func (f *fakeOwnerStore) OwnerUsername(_ context.Context) (string, error) {
+	if f.seatErr != nil {
+		return "", f.seatErr
+	}
+	return f.ownerSeat, nil
+}
+
 func (f *fakeOwnerStore) UpsertOwner(_ context.Context, id, username, email string) error {
 	if f.upsertErr != nil {
 		return f.upsertErr
@@ -107,30 +110,31 @@ func (f *fakeOwnerStore) InsertOperator(_ context.Context, id, username, email s
 	return nil
 }
 
-// CompleteOwnerSetup models the real all-or-nothing transaction: injected failures
-// record none of the redeem, auth-toggle, or setup-token writes.
-func (f *fakeOwnerStore) CompleteOwnerSetup(_ context.Context, newUserID, code string, _ time.Time,
-	tokenHash string, expiresAt time.Time) (string, string, string, error) {
-	if f.redeemErr != nil {
-		return "", "", "", f.redeemErr
+// CompleteOwnerSetup models the transaction without any game link code.
+func (f *fakeOwnerStore) CompleteOwnerSetup(_ context.Context, newUserID string, _ time.Time,
+	tokenHash string, expiresAt time.Time) (string, string, error) {
+	for _, err := range []error{f.redeemErr, f.setErr, f.createTokenErr} {
+		if err != nil {
+			return "", "", err
+		}
 	}
-	if f.setErr != nil {
-		return "", "", "", f.setErr
-	}
-	if f.createTokenErr != nil {
-		return "", "", "", f.createTokenErr
-	}
-	f.redeems = append(f.redeems, redeemCall{newUserID, code})
-	userID := f.redeemUserID
+	userID, username := f.setupUserID, f.setupUsername
 	if userID == "" {
-		userID = newUserID // unlinked UUID → the fresh id becomes the Owner
+		userID = newUserID
 	}
+	if username == "" {
+		username = "owner"
+	}
+	if f.onboarded {
+		return userID, username, api.ErrConflict
+	}
+	f.setupIDs = append(f.setupIDs, newUserID)
 	if f.settings == nil {
 		f.settings = map[string][]byte{}
 	}
 	f.settings[api.LocalAuthEnabledKey] = []byte("true")
 	f.tokens = append(f.tokens, setupTokenCall{tokenHash, userID, expiresAt})
-	return userID, f.redeemMCUUID, f.redeemAuthSource, nil
+	return userID, username, nil
 }
 
 func (f *fakeOwnerStore) SetSetting(_ context.Context, key string, value []byte) error {
@@ -201,6 +205,34 @@ func TestProvisionOwner(t *testing.T) {
 		}
 	})
 
+	t.Run("an occupied seat refuses any other username", func(t *testing.T) {
+		// The seat is the single owner row: upserting a fresh name would take the
+		// insert arm and mint a SECOND owner, while the existing seat — possibly the
+		// compromised account this reset was meant to replace — stays live, and no
+		// supported path can delete an owner row.
+		f := &fakeOwnerStore{ownerSeat: "seat-holder"}
+		err := provisionOwner(ctx, f, "someone-else", "")
+		if !errors.Is(err, api.ErrConflict) {
+			t.Fatalf("error = %v, want it to wrap api.ErrConflict so the TUI routes back to the form", err)
+		}
+		if !strings.Contains(err.Error(), `"seat-holder"`) {
+			t.Errorf("error = %q, want it to name the occupied seat", err)
+		}
+		if len(f.upserts) != 0 {
+			t.Errorf("want no write against an occupied seat, got %d", len(f.upserts))
+		}
+	})
+
+	t.Run("the occupied seat's own username still resets", func(t *testing.T) {
+		f := &fakeOwnerStore{ownerSeat: "seat-holder"}
+		if err := provisionOwner(ctx, f, "seat-holder", "[email protected]"); err != nil {
+			t.Fatalf("provisionOwner(reset): %v", err)
+		}
+		if len(f.upserts) != 1 || f.upserts[0].username != "seat-holder" || f.upserts[0].email != "[email protected]" {
+			t.Fatalf("want 1 reset upsert for the seat, got %+v", f.upserts)
+		}
+	})
+
 	t.Run("propagates a store error", func(t *testing.T) {
 		f := &fakeOwnerStore{upsertErr: errors.New("boom")}
 		if err := provisionOwner(ctx, f, "owner", ""); err == nil {
@@ -230,61 +262,57 @@ func TestEnableLocalAuth(t *testing.T) {
 	}
 }
 
-func TestAuthenticateAdmin(t *testing.T) {
+func TestResolveAdmin(t *testing.T) {
 	ctx := context.Background()
 
-	// Password verification is gone (passwordless design): authenticateAdmin now only
-	// resolves the named admin so recovery can attribute the audit to a real identity.
-	// The security boundary is the break-glass root gate, not a typed secret.
+	// resolveAdmin only finds the staff account a typed name points at; proving the
+	// operator holds it is the mailed code's job (beginRecovery).
 
-	t.Run("resolves an existing admin for attribution", func(t *testing.T) {
+	t.Run("resolves an existing admin", func(t *testing.T) {
 		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": mkAdmin("root")}}
-		matched, ok, err := authenticateAdmin(ctx, f, "root")
+		u, err := resolveAdmin(ctx, f, "  root ")
 		if err != nil {
-			t.Fatalf("authenticateAdmin: %v", err)
+			t.Fatalf("resolveAdmin: %v", err)
 		}
-		if !ok {
-			t.Fatal("ok = false, want true for an existing admin")
-		}
-		if matched != "root" {
-			t.Errorf("matched = %q, want root", matched)
+		if u == nil || u.Username != "root" {
+			t.Fatalf("resolveAdmin = %+v, want the root admin", u)
 		}
 	})
 
-	t.Run("a non-admin role can never attribute a break-glass", func(t *testing.T) {
+	t.Run("a non-admin role is no staff account", func(t *testing.T) {
 		player := mkAdmin("alice")
 		player.Role = "user" // a player row is not staff
 		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"alice": player}}
-		_, ok, err := authenticateAdmin(ctx, f, "alice")
-		if err != nil {
-			t.Fatalf("unexpected error: %v", err)
-		}
-		if ok {
-			t.Error("ok = true, want false for a non-admin role")
+		if u, err := resolveAdmin(ctx, f, "alice"); u != nil || err != nil {
+			t.Errorf("resolveAdmin(player) = (%+v, %v), want (nil, nil)", u, err)
 		}
 	})
 
-	t.Run("an unknown user is a non-match, not an error", func(t *testing.T) {
-		f := &fakeOwnerStore{}
-		_, ok, err := authenticateAdmin(ctx, f, "nobody")
-		if err != nil {
-			t.Fatalf("unexpected error: %v", err)
-		}
-		if ok {
-			t.Error("ok = true, want false for an unknown user")
+	t.Run("the owner role counts as staff", func(t *testing.T) {
+		owner := mkAdmin("root")
+		owner.Role = "owner" // the platform owner is staff too (migration 0011)
+		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": owner}}
+		if u, err := resolveAdmin(ctx, f, "root"); err != nil || u == nil {
+			t.Fatalf("resolveAdmin(owner) = (%+v, %v), want the owner", u, err)
 		}
 	})
 
-	t.Run("an empty username is a non-match with no store call", func(t *testing.T) {
+	t.Run("an unknown user is nil, not an error", func(t *testing.T) {
+		if u, err := resolveAdmin(ctx, &fakeOwnerStore{}, "nobody"); u != nil || err != nil {
+			t.Errorf("resolveAdmin(unknown) = (%+v, %v), want (nil, nil)", u, err)
+		}
+	})
+
+	t.Run("an empty username makes no store call", func(t *testing.T) {
 		f := &fakeOwnerStore{userErr: errors.New("must not be called")}
-		if _, ok, err := authenticateAdmin(ctx, f, ""); ok || err != nil {
-			t.Errorf("empty username: ok=%v err=%v, want false,nil", ok, err)
+		if u, err := resolveAdmin(ctx, f, " "); u != nil || err != nil {
+			t.Errorf("empty username: (%+v, %v), want (nil, nil)", u, err)
 		}
 	})
 
 	t.Run("a datastore fault is surfaced", func(t *testing.T) {
 		f := &fakeOwnerStore{userErr: errors.New("db down")}
-		if _, _, err := authenticateAdmin(ctx, f, "root"); err == nil {
+		if _, err := resolveAdmin(ctx, f, "root"); err == nil {
 			t.Fatal("want error when the store fails")
 		}
 	})
@@ -352,6 +380,8 @@ func TestPerformBreakGlass(t *testing.T) {
 			osUser:         "alice",
 			ownerUsername:  "owner",
 			attemptedAdmin: "root",
+			verifiedBy:     verifiedByEmailOTP,
+			codeSentTo:     "[email protected]",
 		}
 		out, err := performBreakGlass(ctx, f, op)
 		if err != nil {
@@ -376,6 +406,23 @@ func TestPerformBreakGlass(t *testing.T) {
 		if payload["admin_account"] != "root" {
 			t.Errorf("payload.admin_account = %v, want root", payload["admin_account"])
 		}
+		if payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" {
+			t.Errorf("payload = %v, want verified_by=email_otp [email protected]", payload)
+		}
+		if _, present := payload["otp_skipped"]; present {
+			t.Error("a proven recovery carries no otp_skipped")
+		}
+	})
+
+	t.Run("recovery without a proof is recorded unverified", func(t *testing.T) {
+		f := &fakeOwnerStore{}
+		op := breakGlassOp{mode: "recovery", accountable: "root", osUser: "alice", ownerUsername: "owner", attemptedAdmin: "root"}
+		if _, err := performBreakGlass(ctx, f, op); err != nil {
+			t.Fatalf("performBreakGlass: %v", err)
+		}
+		if _, payload := auditOf(t, f); payload["verified"] != false {
+			t.Errorf("payload.verified = %v, want false: only a mailed code verifies", payload["verified"])
+		}
 	})
 
 	t.Run("root override records an unverified row attributed to the OS user", func(t *testing.T) {
@@ -386,6 +433,8 @@ func TestPerformBreakGlass(t *testing.T) {
 			osUser:         "alice",
 			ownerUsername:  "owner",
 			attemptedAdmin: "typo-admin",
+			otpSkipped:     otpSkipSendFailed,
+			otpSkipDetail:  "dial tcp: connection refused",
 		}
 		if _, err := performBreakGlass(ctx, f, op); err != nil {
 			t.Fatalf("performBreakGlass: %v", err)
@@ -401,6 +450,13 @@ func TestPerformBreakGlass(t *testing.T) {
 		if payload["admin_account"] != "typo-admin" {
 			t.Errorf("payload.admin_account = %v, want typo-admin", payload["admin_account"])
 		}
+		// Why no code proved anyone is part of the record.
+		if payload["otp_skipped"] != otpSkipSendFailed || payload["otp_skip_detail"] != "dial tcp: connection refused" {
+			t.Errorf("payload = %v, want otp_skipped=send_failed with its detail", payload)
+		}
+		if _, present := payload["verified_by"]; present {
+			t.Error("an override carries no verified_by")
+		}
 	})
 
 	t.Run("an audit failure does not fail the recovery", func(t *testing.T) {
@@ -567,6 +623,8 @@ func TestPerformAddOperator(t *testing.T) {
 			ownerUsername:  "ops-jordan",
 			ownerEmail:     "[email protected]",
 			attemptedAdmin: "root",
+			verifiedBy:     verifiedByEmailOTP,
+			codeSentTo:     "[email protected]",
 		}
 		out, err := performAddOperator(ctx, f, op)
 		if err != nil {
@@ -593,14 +651,14 @@ func TestPerformAddOperator(t *testing.T) {
 		if _, present := payload["owner"]; present {
 			t.Error("payload.owner present, want the new account under the operator key")
 		}
-		if payload["verified"] != true || payload["admin_account"] != "root" {
-			t.Errorf("payload = %v, want verified=true admin_account=root", payload)
+		if payload["verified"] != true || payload["admin_account"] != "root" || payload["verified_by"] != verifiedByEmailOTP {
+			t.Errorf("payload = %v, want verified=true admin_account=root verified_by=email_otp", payload)
 		}
 	})
 
 	t.Run("root override records an unverified operator row", func(t *testing.T) {
 		f := &fakeOwnerStore{}
-		op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin"}
+		op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin", otpSkipped: otpSkipUnknownAdmin}
 		if _, err := performAddOperator(ctx, f, op); err != nil {
 			t.Fatalf("performAddOperator: %v", err)
 		}
@@ -608,8 +666,8 @@ func TestPerformAddOperator(t *testing.T) {
 			t.Fatalf("want 1 insert, got %d", len(f.inserts))
 		}
 		_, payload := auditOf(t, f)
-		if payload["verified"] != false {
-			t.Errorf("payload.verified = %v, want false for root_override", payload["verified"])
+		if payload["verified"] != false || payload["otp_skipped"] != otpSkipUnknownAdmin {
+			t.Errorf("payload = %v, want verified=false otp_skipped=unknown_admin", payload)
 		}
 	})
 
@@ -645,149 +703,70 @@ func TestPerformAddOperator(t *testing.T) {
 	})
 }
 
-func TestPerformSetupMCBind(t *testing.T) {
+func TestPerformSetupOwner(t *testing.T) {
 	ctx := context.Background()
+	f := &fakeOwnerStore{setupUserID: "usr-owner-1"}
+	out, err := performSetupOwner(ctx, f, "https://op.console.example.com:30443", "deploybot")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if out.ownerUsername != "owner" {
+		t.Fatalf("username = %q", out.ownerUsername)
+	}
+	const prefix = "https://op.console.example.com:30443/setup?token="
+	if !strings.HasPrefix(out.setupTokenURL, prefix) {
+		t.Fatalf("URL = %q", out.setupTokenURL)
+	}
+	if len(f.tokens) != 1 || f.tokens[0].userID != "usr-owner-1" {
+		t.Fatalf("tokens = %+v", f.tokens)
+	}
+	raw := strings.TrimPrefix(out.setupTokenURL, prefix)
+	sum := sha256.Sum256([]byte(raw))
+	if f.tokens[0].tokenHash != hex.EncodeToString(sum[:]) {
+		t.Fatal("stored token does not match URL")
+	}
+	if !f.tokens[0].expiresAt.After(time.Now()) {
+		t.Fatal("token already expired")
+	}
+	if string(f.settings[api.LocalAuthEnabledKey]) != "true" {
+		t.Fatal("local auth stayed disabled")
+	}
+	e, payload := auditOf(t, f)
+	if e.Actor != "deploybot" || e.Action != "setup.owner_login" || payload["user_id"] != "usr-owner-1" {
+		t.Fatalf("audit = %+v, %v", e, payload)
+	}
 
-	t.Run("binds the owner and mints a setup URL whose token hash is what is stored", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1", redeemMCUUID: "mc-uuid-1", redeemAuthSource: "mojang"}
-		out, err := performSetupMCBind(ctx, f, "  abc-123  ", "console.example.com", "deploybot")
-		if err != nil {
-			t.Fatalf("performSetupMCBind: %v", err)
-		}
-		// The Owner this mints has no password and no email, so the setup token is the
-		// only door — and handleSetupRedeem is gated on local_auth_enabled. A bind that
-		// leaves the toggle off hands back a URL that answers 403.
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; !ok {
-			t.Error("local auth was not enabled — the setup URL would 403 local_auth_disabled")
-		}
-		// The bind is attributed by Minecraft identity, because that is what the login
-		// gate verified; a username would be the one thing nobody checked.
-		e, payload := auditOf(t, f)
-		if e.Actor != "deploybot" || e.Source != "setup" || e.Action != "setup.owner_bind" {
-			t.Errorf("audit = %+v, want actor=deploybot source=setup action=setup.owner_bind", e)
-		}
-		if payload["mc_uuid"] != "mc-uuid-1" || payload["auth_source"] != "mojang" {
-			t.Errorf("audit payload = %v, want the redeemed mc_uuid + auth_source", payload)
-		}
-		if out.ownerIdentity != "mc-uuid-1" {
-			t.Errorf("owner identity = %q, want the verified Minecraft UUID", out.ownerIdentity)
-		}
-		const prefix = "https://console.example.com/setup?token="
-		if !strings.HasPrefix(out.setupTokenURL, prefix) {
-			t.Fatalf("setup URL = %q, want prefix %q", out.setupTokenURL, prefix)
-		}
-		// The link code is trimmed and upper-cased before redemption.
-		if len(f.redeems) != 1 {
-			t.Fatalf("want 1 redeem, got %d", len(f.redeems))
-		}
-		if f.redeems[0].code != "ABC-123" {
-			t.Errorf("redeemed code = %q, want ABC-123 (trimmed + upper-cased)", f.redeems[0].code)
-		}
-		if !strings.HasPrefix(f.redeems[0].newUserID, "usr-") {
-			t.Errorf("redeem newUserID = %q, want usr- prefix", f.redeems[0].newUserID)
-		}
-		// Exactly one token minted, for the redeemed user, and only its hash stored —
-		// the stored hash must be sha-256 of the raw token carried in the URL.
-		if len(f.tokens) != 1 {
-			t.Fatalf("want 1 setup token, got %d", len(f.tokens))
-		}
-		tok := f.tokens[0]
-		if tok.userID != "usr-owner-1" {
-			t.Errorf("token userID = %q, want usr-owner-1 (the redeemed owner)", tok.userID)
-		}
-		raw := strings.TrimPrefix(out.setupTokenURL, prefix)
-		sum := sha256.Sum256([]byte(raw))
-		if tok.tokenHash != hex.EncodeToString(sum[:]) {
-			t.Error("stored token hash is not sha-256 of the raw token in the URL")
-		}
-		if tok.tokenHash == raw || tok.tokenHash == "" {
-			t.Error("the raw token (or nothing) was stored instead of its hash")
-		}
-		// The token is short-lived and in the future.
-		if !tok.expiresAt.After(time.Now()) {
-			t.Errorf("token expiresAt = %v, want a future time", tok.expiresAt)
+	t.Run("failed writes leave no partially initialized login", func(t *testing.T) {
+		for _, store := range []*fakeOwnerStore{
+			{redeemErr: errors.New("database unavailable")},
+			{setErr: errors.New("settings write failed")},
+			{createTokenErr: errors.New("token write failed")},
+		} {
+			if _, err := performSetupOwner(ctx, store, "https://op.console.example.com", "root"); err == nil {
+				t.Fatal("want failure")
+			}
+			if len(store.tokens) != 0 || len(store.setupIDs) != 0 || len(store.settings) != 0 {
+				t.Fatal("partial setup")
+			}
 		}
 	})
-
-	t.Run("an empty link code mints nothing", func(t *testing.T) {
-		f := &fakeOwnerStore{}
-		if _, err := performSetupMCBind(ctx, f, "   ", "console.example.com", "root"); err == nil {
-			t.Fatal("want error for an empty link code")
-		}
-		if len(f.redeems) != 0 || len(f.tokens) != 0 {
-			t.Errorf("want no redeem/token on an empty code, got redeems=%d tokens=%d", len(f.redeems), len(f.tokens))
-		}
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; ok {
-			t.Error("local auth was enabled without an owner — the gate must not open on a failed bind")
+	t.Run("settled account is not reset", func(t *testing.T) {
+		store := &fakeOwnerStore{setupUserID: "existing", setupUsername: "alice", onboarded: true}
+		out, err := performSetupOwner(ctx, store, "https://op.console.example.com", "root")
+		if !errors.Is(err, api.ErrConflict) || out.ownerUsername != "alice" || len(store.tokens) != 0 {
+			t.Fatalf("out = %+v err = %v", out, err)
 		}
 	})
-
-	t.Run("a link-code redemption failure mints no token", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemErr: errors.New("code expired")}
-		if _, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root"); err == nil {
-			t.Fatal("want error when the link code cannot be redeemed")
-		}
-		if len(f.tokens) != 0 {
-			t.Errorf("want no token minted on a redeem failure, got %d", len(f.tokens))
-		}
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; ok {
-			t.Error("local auth was enabled without an owner — the gate must not open on a failed redeem")
+	t.Run("audit failure still returns the login link", func(t *testing.T) {
+		out, err := performSetupOwner(ctx, &fakeOwnerStore{auditErr: errors.New("audit down")}, "https://op.console.example.com", "root")
+		if err != nil || out.auditErr == nil || out.setupTokenURL == "" {
+			t.Fatalf("out = %+v err = %v", out, err)
 		}
 	})
-
-	t.Run("a local-auth failure fails the bind rather than minting an unredeemable URL", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1", setErr: errors.New("db down")}
-		if _, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root"); err == nil {
-			t.Fatal("want error when local auth cannot be enabled")
-		}
-		if len(f.redeems) != 0 || len(f.tokens) != 0 {
-			t.Errorf("atomic setup was partially recorded: redeems=%d tokens=%d", len(f.redeems), len(f.tokens))
-		}
-	})
-
-	t.Run("a token-store failure rolls the bind back", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1", createTokenErr: errors.New("db down")}
-		if _, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root"); err == nil {
-			t.Fatal("want error when the setup token cannot be stored")
-		}
-		if len(f.redeems) != 0 {
-			t.Errorf("link code was consumed despite token failure, got %d redeems", len(f.redeems))
-		}
-		if len(f.tokens) != 0 {
-			t.Errorf("want no recorded token when the store fails, got %d", len(f.tokens))
-		}
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; ok {
-			t.Error("local auth stayed enabled despite transaction rollback")
-		}
-	})
-
-	t.Run("an audit failure does not cost the operator their install", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1", auditErr: errors.New("audit sink down")}
-		out, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root")
-		if err != nil {
-			t.Fatalf("an audit failure must not fail the bind: %v", err)
-		}
-		if out.auditErr == nil {
-			t.Error("the audit failure was swallowed instead of surfaced on the outcome")
-		}
-		if out.setupTokenURL == "" {
-			t.Error("no setup URL minted despite a recoverable audit failure")
-		}
-		if _, ok := f.settings[api.LocalAuthEnabledKey]; !ok {
-			t.Error("local auth was not enabled despite a recoverable audit failure")
-		}
-	})
-
-	t.Run("defaults to the op.console host when adminHostname is empty", func(t *testing.T) {
-		f := &fakeOwnerStore{redeemUserID: "usr-owner-1"}
-		out, err := performSetupMCBind(ctx, f, "abc-123", "  ", "root")
-		if err != nil {
-			t.Fatalf("performSetupMCBind: %v", err)
-		}
-		// The Owner is staff, so onboarding lands on the operator console, not the
-		// player panel — the empty-host fallback must reflect that.
-		if !strings.HasPrefix(out.setupTokenURL, "https://op.console.localhost/setup?token=") {
-			t.Errorf("setup URL = %q, want the op.console.localhost default host", out.setupTokenURL)
+	t.Run("missing HTTPS origin cannot create an account", func(t *testing.T) {
+		store := &fakeOwnerStore{}
+		if _, err := performSetupOwner(ctx, store, "", "root"); err == nil || len(store.tokens) != 0 {
+			t.Fatal("invalid origin accepted")
 		}
 	})
 }

cmd/felis/conflict_test.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// racingClient lands one concurrent write (race) on the stored object just before
+// setup's first write of it goes out, the way the operator's status update or
+// felis-api's idle patch can, and counts setup's writes.
+func racingClient(t *testing.T, race func(ctx context.Context, c client.WithWatch), objs ...client.Object) (client.Client, *int) {
+	t.Helper()
+	writes := 0
+	before := func(ctx context.Context, c client.WithWatch) {
+		writes++
+		if writes == 1 {
+			race(ctx, c)
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
+		WithStatusSubresource(&v1alpha1.MinecraftServer{}).
+		WithInterceptorFuncs(interceptor.Funcs{
+			Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
+				before(ctx, c)
+				return c.Update(ctx, obj, opts...)
+			},
+			Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+				before(ctx, c)
+				return c.Patch(ctx, obj, patch, opts...)
+			},
+		}).Build()
+	return cl, &writes
+}
+
+func staleLoginGate(t *testing.T) *v1alpha1.MinecraftServer {
+	t.Helper()
+	ms, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://old.internal:8081", "203.0.113.10.nip.io", "console.203.0.113.10.nip.io")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ms
+}
+
+func getLogin(t *testing.T, cl client.Client) *v1alpha1.MinecraftServer {
+	t.Helper()
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	return &ms
+}
+
+func envMap(ms *v1alpha1.MinecraftServer) map[string]string {
+	m := map[string]string{}
+	for _, e := range ms.Spec.Env {
+		m[e.Name] = e.Value
+	}
+	return m
+}
+
+// A hand edit that lands while setup refreshes the console hostnames costs setup a
+// re-read and a second write; both the edit and the refresh survive.
+func TestRefreshDerivedEnvRetriesAConcurrentEdit(t *testing.T) {
+	ctx := context.Background()
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		ms := getLogin(t, c)
+		ms.Spec.Env = append(ms.Spec.Env, v1alpha1.EnvVar{Name: "HAND_TUNED", Value: "1"})
+		if err := c.Update(ctx, ms); err != nil {
+			t.Fatal(err)
+		}
+	}, staleLoginGate(t))
+
+	desired, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatal(err)
+	}
+	refreshed, err := refreshDerivedEnv(ctx, cl, getLogin(t, cl), desired)
+	if err != nil || !refreshed {
+		t.Fatalf("refreshed=%v err=%v, want a refresh after the retry", refreshed, err)
+	}
+	env := envMap(getLogin(t, cl))
+	if env[envPanelHostname] != "console.mc.example.net" || env[envRootDomain] != "mc.example.net" {
+		t.Errorf("env = %v, want the new hostnames", env)
+	}
+	if env["HAND_TUNED"] != "1" {
+		t.Errorf("env = %v: the concurrent hand edit was dropped", env)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2 (one conflict, one retry)", *writes)
+	}
+}
+
+// converge reports each fill once even when a status write forced a retry.
+func TestConvergeRetriesAConcurrentStatusWrite(t *testing.T) {
+	ctx := context.Background()
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatal(err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Status.Phase = v1alpha1.PhaseRunning
+		if err := c.Status().Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, lobby)
+
+	var got systemServerOutcome
+	for _, o := range convergeSystemServers(ctx, cl, "minecraft", "", "reg/lobby:1",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
+		if o.name == naming.SystemLobbyServer {
+			got = o
+		}
+	}
+	if got.err != nil || !got.updated || !slices.Equal(got.changes, []string{"spec.rcon"}) {
+		t.Fatalf("lobby outcome = %+v, want spec.rcon filled once", got)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if !ms.Spec.Rcon.Enabled || ms.Status.Phase != v1alpha1.PhaseRunning {
+		t.Errorf("rcon.enabled=%v phase=%q, want the fill and the status write both kept", ms.Spec.Rcon.Enabled, ms.Status.Phase)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+// A replica refresh racing another writer keeps that writer's key.
+func TestSecretReplicaRefreshRetriesAConcurrentWrite(t *testing.T) {
+	secret := func(ns, body string) *corev1.Secret {
+		return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
+			Data: map[string][]byte{"felis.toml": []byte(body)}}
+	}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var s corev1.Secret
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+			t.Fatal(err)
+		}
+		s.Data["extra"] = []byte("x")
+		if err := c.Update(ctx, &s); err != nil {
+			t.Fatal(err)
+		}
+	}, secret("felis", "current"), secret("minecraft", "stale"))
+
+	out := ensureSecretReplica(context.Background(), cl, "felis", "minecraft",
+		"felis-config", "felis.toml", "config", "minecraft ns", true)
+	if out.err != nil || !out.updated {
+		t.Fatalf("outcome = %+v, want refreshed", out)
+	}
+	var s corev1.Secret
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+		t.Fatal(err)
+	}
+	if string(s.Data["felis.toml"]) != "current" || string(s.Data["extra"]) != "x" {
+		t.Errorf("replica data = %q, want felis.toml=current and extra=x", s.Data)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+const (
+	sysOldDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
+	sysNewDigest = "sha256:4444444444444444444444444444444444444444444444444444444444444444"
+)
+
+func systemPinRegistry(t *testing.T) imagepin.Resolver {
+	t.Helper()
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v2/felis/limbo/manifests/demo", "/v2/felis/lobby/manifests/demo":
+			w.Header().Set("Docker-Content-Digest", sysNewDigest)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(reg.Close)
+	return imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+}
+
+func systemServer(name, image string) *v1alpha1.MinecraftServer {
+	ms := &v1alpha1.MinecraftServer{}
+	ms.Name, ms.Namespace = name, "minecraft"
+	ms.Labels = map[string]string{v1alpha1.LabelSystemRole: name}
+	ms.Spec.Image = image
+	return ms
+}
+
+// The installer's --system pin moves a system server onto the build its tag names
+// now, from the bare tag or from an earlier digest, and is a no-op the second time.
+func TestPinSystemServerImage(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	limbo := defaultRegistryURL + "/felis/limbo:demo"
+	lobby := defaultRegistryURL + "/felis/lobby:demo"
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		systemServer(naming.SystemLoginServer, limbo),
+		systemServer(naming.SystemLobbyServer, lobby+"@"+sysOldDigest),
+	).Build()
+	image := func(name string) string {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		return ms.Spec.Image
+	}
+
+	for name, want := range map[string]string{
+		naming.SystemLoginServer: limbo + "@" + sysNewDigest,
+		naming.SystemLobbyServer: lobby + "@" + sysNewDigest,
+	} {
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || o.err != nil || !o.updated {
+			t.Fatalf("%s: outcome=%+v err=%v, want pinned", name, o, err)
+		}
+		if got := image(name); got != want {
+			t.Errorf("%s image = %q, want %q", name, got, want)
+		}
+		again, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || again.updated || again.skipped != "already runs "+want {
+			t.Errorf("%s second pass = %+v, %v; want already runs %s", name, again, err, want)
+		}
+	}
+}
+
+func TestPinSystemServerImageLeavesOthersAlone(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	pin := func(t *testing.T, ms *v1alpha1.MinecraftServer) (systemServerOutcome, string) {
+		t.Helper()
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(ms).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(ms), &got); err != nil {
+			t.Fatal(err)
+		}
+		return o, got.Spec.Image
+	}
+
+	t.Run("external image", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, "docker.io/example/limbo:1.2"))
+		if o.err != nil || o.updated || img != "docker.io/example/limbo:1.2" ||
+			o.skipped != "runs docker.io/example/limbo:1.2, which names no platform registry tag to follow; left alone" {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("digest without a tag", func(t *testing.T) {
+		ref := defaultRegistryURL + "/felis/limbo@" + sysOldDigest
+		o, img := pin(t, systemServer(naming.SystemLoginServer, ref))
+		if o.err != nil || o.updated || img != ref {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("not a system server", func(t *testing.T) {
+		ms := systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo")
+		ms.Labels = nil
+		o, img := pin(t, ms)
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:demo" {
+			t.Fatalf("outcome=%+v image=%q, want refused", o, img)
+		}
+	})
+	t.Run("tag the registry lost", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:gone"))
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:gone" {
+			t.Fatalf("outcome=%+v image=%q, want an error the installer falls back on", o, img)
+		}
+	})
+	t.Run("absent", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.skipped != "not present yet; sudo felis setup creates it" {
+			t.Fatalf("outcome=%+v err=%v, want a skip", o, err)
+		}
+	})
+	t.Run("retargeted while pinning", func(t *testing.T) {
+		cl, _ := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+			ms := getLogin(t, c)
+			ms.Spec.Image = "docker.io/example/limbo:1.2"
+			if err := c.Update(ctx, ms); err != nil {
+				t.Fatal(err)
+			}
+		}, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo"))
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.updated {
+			t.Fatalf("outcome=%+v err=%v, want nothing written", o, err)
+		}
+		if img := getLogin(t, cl).Spec.Image; img != "docker.io/example/limbo:1.2" {
+			t.Errorf("image = %q, want the admin's retarget kept", img)
+		}
+	})
+}
+
+// --system names one of the two system servers; anything else is a usage error
+// before any cluster is touched.
+func TestPinImagesSystemFlagTakesOnlySystemServers(t *testing.T) {
+	var stdout, stderr strings.Builder
+	if code := cmdPinImages([]string{"--system", "survival"}, &stdout, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2", code)
+	}
+	if got := stderr.String(); got != "felis pin-images: --system takes login or lobby, not \"survival\"\n" {
+		t.Errorf("stderr = %q", got)
+	}
+}
+
+// An idle setting the panel saves while converge fills the default is kept.
+func TestConvergeIdleKeepsAConcurrentPanelEdit(t *testing.T) {
+	ctx := context.Background()
+	srv := &v1alpha1.MinecraftServer{}
+	srv.Name, srv.Namespace = "survival", "minecraft"
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Spec.Idle = v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}
+		if err := c.Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, srv)
+
+	if out := convergeUserServerIdle(ctx, cl, "minecraft"); len(out) != 0 {
+		t.Fatalf("outcomes = %+v, want none: the server has a setting by the time converge writes", out)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if want := (v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}); ms.Spec.Idle != want {
+		t.Errorf("idle = %+v, want the panel's %+v", ms.Spec.Idle, want)
+	}
+	if *writes != 1 {
+		t.Errorf("writes = %d, want 1 (the conflicted attempt; the retry sends nothing)", *writes)
+	}
+}

cmd/felis/context_max_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+)
+
+func TestContextMaxBytes(t *testing.T) {
+	cases := []struct {
+		name    string
+		reg     config.RegistryConfig
+		auth    config.AuthConfig
+		want    int64
+		wantErr bool
+	}{
+		{name: "direct install keeps the package default", want: 0},
+		// The panel uploads in parts under the edge's 100 MB body limit, so the
+		// Cloudflare edge keeps the full default.
+		{name: "the Cloudflare edge keeps the package default", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 0},
+		{name: "CF-Connecting-IP keeps the package default", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 0},
+		{name: "explicit value behind the edge", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
+		{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
+		{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
+		{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := contextMaxBytes(&config.Config{Registry: tc.reg, Auth: tc.auth})
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
+			}
+			if got != tc.want {
+				t.Fatalf("contextMaxBytes = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestUploadPartsDir(t *testing.T) {
+	if got := uploadPartsDir("/var/lib/felis/uploads"); got != "/var/lib/felis/uploads/.parts" {
+		t.Errorf("local store: parts dir = %q, want beside the contexts", got)
+	}
+	if got := uploadPartsDir("file:///srv/uploads"); got != "/srv/uploads/.parts" {
+		t.Errorf("file:// store: parts dir = %q, want /srv/uploads/.parts", got)
+	}
+	// This machine has no /var/lib/felis/uploads mount, so an s3:// store falls
+	// back to the temp dir.
+	if _, err := os.Stat("/var/lib/felis/uploads"); err == nil {
+		t.Skip("/var/lib/felis/uploads exists here")
+	}
+	if got := uploadPartsDir("s3://bucket/uploads"); got != filepath.Join(os.TempDir(), "felis-upload-parts") {
+		t.Errorf("s3 store without the uploads mount: parts dir = %q", got)
+	}
+}

cmd/felis/converge.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// cmdConverge is the explicit convergence pass over already-installed system
+// servers (#1), plus the idle-stop default for user servers that predate it, and
+// with -user-rcon their RCON block (#3). Provisioning is create-if-absent, so a field the desired spec
+// gained after an install (spec.rcon, spec.startup.healthHTTPPort, a derived env
+// key) never reaches the existing CR — and nothing says so. This command fills
+// exactly those zero-value fields; see convergeSystemServers for the full contract
+// and why it is a separate, operator-timed step rather than part of setup.
+//
+// It reads the same host config as setup (the control plane's felis.toml) and
+// talks to the cluster with the local kubeconfig, so it must run as root on the
+// control-plane host.
+func cmdConverge(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("converge", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	userRcon := fs.Bool("user-rcon", false, "also turn RCON on for user servers created before it was the default")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis converge: refused — converging needs the cluster credentials, so it must run as root (try: sudo felis converge)")
+		return 1
+	}
+
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis converge: %v\n", err)
+		fmt.Fprintln(stderr, "If this host was never installed, run `sudo felis setup` first.")
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis converge: %v\n", err)
+		return 1
+	}
+
+	controlNS := platform.DefaultControlNamespace
+	outcomes := convergeSystemServers(context.Background(), cl, cfg.K8s.Namespace,
+		cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage,
+		platform.InternalAPIBaseURL(controlNS), cfg.Server.RootDomain,
+		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
+
+	outcomes = append(outcomes, convergeUserServerIdle(context.Background(), cl, cfg.K8s.Namespace)...)
+	outcomes = append(outcomes, convergeUserServerRcon(context.Background(), cl, cfg.K8s.Namespace, *userRcon)...)
+
+	fmt.Fprintln(stdout, "felis converge: filling fields an installed server predates (operator-set values are never overwritten):")
+	exit := 0
+	for _, o := range outcomes {
+		switch {
+		case o.err != nil:
+			fmt.Fprintf(stdout, "  - %s: ERROR %v\n", o.name, o.err)
+			exit = 1
+		case len(o.changes) > 0:
+			fmt.Fprintf(stdout, "  - %s: updated (%s)\n", o.name, strings.Join(o.changes, ", "))
+		default:
+			fmt.Fprintf(stdout, "  - %s: %s\n", o.name, o.skipped)
+		}
+	}
+	return exit
+}
+
+// convergeUserServerIdle gives every user server that predates the idle default
+// (spec.idle entirely unset) the default idle stop. A server whose idle stop was
+// turned off keeps a duration on its spec, so it is not "unset" and is left
+// alone; system servers never idle out and are skipped. Servers that already
+// carry a value produce no line, so a converged fleet prints nothing here.
+func convergeUserServerIdle(ctx context.Context, cl client.Client, namespace string) []systemServerOutcome {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+			continue
+		}
+		// Re-checked on the copy each attempt reads: an idle setting the panel saved
+		// meanwhile is the user's, and the default must not land over it.
+		changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
+			if ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+				return false
+			}
+			ms.Spec.Idle = v1alpha1.DefaultIdle()
+			return true
+		})
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
+			continue
+		}
+		if !changed {
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{fmt.Sprintf("spec.idle (stop after %ds empty)", v1alpha1.DefaultEmptySecondsBeforeStop)}})
+	}
+	return out
+}
+
+// convergeUserServerRcon handles user servers created before RCON was part of every
+// new server (694e3cb): spec.rcon entirely unset. Such a server has a dead console,
+// reports nobody online, and never idles out, because all three ride RCON.
+//
+// Only with fill does it turn RCON on, with the same block CreateServer writes
+// today; without it each such server gets a line saying so. The fill is opt-in
+// because the operator gates readiness on the RCON probe: a server whose image does
+// not open the listener RCON_PASSWORD asks for would sit in Starting until it is
+// marked Failed. Felis's own paper and lobby images open it; an image a user brought
+// may not, and only the operator running this can tell. A server that already
+// carries any RCON setting (on or off) is left alone and produces no line.
+func convergeUserServerRcon(ctx context.Context, cl client.Client, namespace string, fill bool) []systemServerOutcome {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
+			continue
+		}
+		if !fill {
+			out = append(out, systemServerOutcome{name: ms.Name, available: true,
+				skipped: "no RCON (console, online count and idle stop are off); once its image serves RCON, sudo felis converge -user-rcon turns it on"})
+			continue
+		}
+		changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
+			if ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
+				return false
+			}
+			ms.Spec.Rcon = v1alpha1.RconSpec{
+				Enabled:   true,
+				SecretRef: v1alpha1.SecretKeyRef{Name: naming.RconSecretName(ms.Name), Key: naming.RconSecretKey},
+			}
+			return true
+		})
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
+			continue
+		}
+		if changed {
+			out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true, changes: []string{"spec.rcon"}})
+		}
+	}
+	return out
+}

cmd/felis/converge_test.go

@@ -0,0 +1,295 @@
+package main
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+// converge is the explicit pass over an installed system server whose CR predates
+// a field the desired spec has since gained (#1). It must fill exactly the
+// zero-valued whitelist fields and the derived env, and must not touch anything a
+// non-zero value already occupies — that is the operator's.
+func TestConvergeSystemServersFillsPredatedFields(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+
+	// An old install: the lobby CR was created before the desired spec began
+	// rendering spec.rcon, and the login CR before the HTTP readiness gate existed.
+	// One derived env key is absent entirely (as if it were added later), and one
+	// hand-added env var plus a non-whitelisted spec field must survive.
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatalf("build lobby: %v", err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{}
+	lobby.Spec.JavaMemory = "999Mi"
+
+	login, err := loginSystemServer("reg/limbo:1", "minecraft",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatalf("build login: %v", err)
+	}
+	login.Spec.Startup.HealthHTTPPort = 0
+	kept := login.Spec.Env
+	login.Spec.Env = nil
+	for _, e := range kept {
+		if e.Name != envPanelHostname {
+			login.Spec.Env = append(login.Spec.Env, e)
+		}
+	}
+	login.Spec.Env = append(login.Spec.Env, v1alpha1.EnvVar{Name: "OPERATOR_TUNING", Value: "keep-me"})
+
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(lobby, login).Build()
+	outcomes := convergeSystemServers(ctx, cl, "minecraft", "reg/limbo:1", "reg/lobby:1",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+
+	byName := map[string]systemServerOutcome{}
+	for _, o := range outcomes {
+		if o.err != nil {
+			t.Fatalf("%s: unexpected error: %v", o.name, o.err)
+		}
+		byName[o.name] = o
+	}
+	lobbyOut := byName[naming.SystemLobbyServer]
+	if len(lobbyOut.changes) != 1 || lobbyOut.changes[0] != "spec.rcon" {
+		t.Errorf("lobby changes = %v, want [spec.rcon] (only the zero-valued field)", lobbyOut.changes)
+	}
+	loginOut := byName[naming.SystemLoginServer]
+	if !slices.Contains(loginOut.changes, "spec.startup.healthHTTPPort") || !slices.Contains(loginOut.changes, "env "+envPanelHostname) {
+		t.Errorf("login changes = %v, want the health port plus the missing derived env key", loginOut.changes)
+	}
+
+	var gotLobby v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &gotLobby); err != nil {
+		t.Fatalf("get lobby: %v", err)
+	}
+	if !gotLobby.Spec.Rcon.Enabled ||
+		gotLobby.Spec.Rcon.SecretRef.Name != naming.RconSecretName(naming.SystemLobbyServer) ||
+		gotLobby.Spec.Rcon.SecretRef.Key != naming.RconSecretKey {
+		t.Errorf("lobby rcon = %+v, want the desired block with the %s secret",
+			gotLobby.Spec.Rcon, naming.RconSecretName(naming.SystemLobbyServer))
+	}
+	if gotLobby.Spec.JavaMemory != "999Mi" {
+		t.Errorf("lobby javaMemory = %q, want 999Mi — converge fills new fields, it does not rewrite the spec", gotLobby.Spec.JavaMemory)
+	}
+
+	var gotLogin v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &gotLogin); err != nil {
+		t.Fatalf("get login: %v", err)
+	}
+	if gotLogin.Spec.Startup.HealthHTTPPort != felisLimboHealthPort {
+		t.Errorf("login healthHTTPPort = %d, want %d", gotLogin.Spec.Startup.HealthHTTPPort, felisLimboHealthPort)
+	}
+	env := map[string]string{}
+	for _, e := range gotLogin.Spec.Env {
+		env[e.Name] = e.Value
+	}
+	if env[envPanelHostname] != "console.mc.example.net" {
+		t.Errorf("%s was not added back: %q", envPanelHostname, env[envPanelHostname])
+	}
+	if env["OPERATOR_TUNING"] != "keep-me" {
+		t.Error("a hand-added env var was dropped; converge only touches config-derived names")
+	}
+}
+
+// A field already holding a non-zero value belongs to the operator: converge must
+// report "already converged" and write nothing.
+func TestConvergeSystemServersLeavesNonZeroFieldsAlone(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatalf("build lobby: %v", err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{
+		Enabled:   true,
+		SecretRef: v1alpha1.SecretKeyRef{Name: "operator-rotated", Key: "password"},
+	}
+	login, err := loginSystemServer("reg/limbo:1", "minecraft",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatalf("build login: %v", err)
+	}
+
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(lobby, login).Build()
+	for _, o := range convergeSystemServers(ctx, cl, "minecraft", "reg/limbo:1", "reg/lobby:1",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
+		if o.err != nil {
+			t.Fatalf("%s: unexpected error: %v", o.name, o.err)
+		}
+		if len(o.changes) != 0 || o.skipped != "already converged" {
+			t.Errorf("%s outcome = %+v, want already converged with no writes", o.name, o)
+		}
+	}
+	var got v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &got); err != nil {
+		t.Fatalf("get lobby: %v", err)
+	}
+	if got.Spec.Rcon.SecretRef.Name != "operator-rotated" {
+		t.Errorf("lobby rcon secretRef = %q — converge overwrote a field the operator had already set",
+			got.Spec.Rcon.SecretRef.Name)
+	}
+}
+
+// Guards: an absent CR is reported (creation is setup's job), a foreign CR is
+// refused rather than adopted, and an unset image skips like the provisioner does.
+func TestConvergeSystemServersGuards(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	run := func(cl client.Client, loginImage, lobbyImage string) []systemServerOutcome {
+		return convergeSystemServers(ctx, cl, "minecraft", loginImage, lobbyImage,
+			"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	}
+
+	t.Run("absent CRs are reported, not created", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
+		for _, o := range run(cl, "reg/limbo:1", "reg/lobby:1") {
+			if o.err != nil {
+				t.Fatalf("%s: %v", o.name, o.err)
+			}
+			if o.created || !strings.Contains(o.skipped, "not present") {
+				t.Errorf("%s outcome = %+v, want a not-present skip", o.name, o)
+			}
+		}
+	})
+
+	t.Run("foreign CR is refused", func(t *testing.T) {
+		foreign := &v1alpha1.MinecraftServer{}
+		foreign.Name = naming.SystemLoginServer
+		foreign.Namespace = "minecraft"
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(foreign).Build()
+		out := run(cl, "reg/limbo:1", "")
+		if len(out) != 2 {
+			t.Fatalf("outcomes = %d, want 2", len(out))
+		}
+		if out[0].err == nil || !strings.Contains(out[0].err.Error(), "not marked") {
+			t.Fatalf("login error = %v, want an unmarked-name refusal", out[0].err)
+		}
+	})
+
+	t.Run("unset image skips", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
+		out := run(cl, "", "reg/lobby:1")
+		if out[0].skipped != "image not configured" {
+			t.Errorf("login skipped = %q, want %q", out[0].skipped, "image not configured")
+		}
+	})
+}
+
+// TestConvergeUserServerIdle fills the idle default only where spec.idle was
+// never set: a server whose idle stop was turned off (duration kept), one with
+// its own duration, and a system server all stay as they are.
+func TestConvergeUserServerIdle(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	mk := func(name string, idle v1alpha1.IdleSpec, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Idle = idle
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		mk("legacy", v1alpha1.IdleSpec{}, ""),
+		mk("off", v1alpha1.IdleSpec{EmptySecondsBeforeStop: 600}, ""),
+		mk("custom", v1alpha1.IdleSpec{AutoStopEnabled: true, EmptySecondsBeforeStop: 1800}, ""),
+		mk(naming.SystemLobbyServer, v1alpha1.IdleSpec{}, naming.SystemLobbyServer),
+	).Build()
+
+	outcomes := convergeUserServerIdle(ctx, cl, "minecraft")
+	if len(outcomes) != 1 || outcomes[0].name != "legacy" || outcomes[0].err != nil {
+		t.Fatalf("outcomes = %+v, want exactly one fill for legacy", outcomes)
+	}
+	want := map[string]v1alpha1.IdleSpec{
+		"legacy":                 v1alpha1.DefaultIdle(),
+		"off":                    {EmptySecondsBeforeStop: 600},
+		"custom":                 {AutoStopEnabled: true, EmptySecondsBeforeStop: 1800},
+		naming.SystemLobbyServer: {},
+	}
+	for name, idle := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Idle != idle {
+			t.Errorf("%s idle = %+v, want %+v", name, ms.Spec.Idle, idle)
+		}
+	}
+	if again := convergeUserServerIdle(ctx, cl, "minecraft"); len(again) != 0 {
+		t.Fatalf("second pass = %+v, want nothing to do", again)
+	}
+}
+
+// TestConvergeUserServerRcon reports a user server with no RCON block at all and
+// fills it only when asked, with the block CreateServer writes. RCON turned off on
+// purpose, a server with its own secret, and a system server stay as they are and
+// produce no line.
+func TestConvergeUserServerRcon(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	mk := func(name string, rcon v1alpha1.RconSpec, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Rcon = rcon
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	own := v1alpha1.RconSpec{Enabled: true, Port: 25580, SecretRef: v1alpha1.SecretKeyRef{Name: "own", Key: "pw"}}
+	off := v1alpha1.RconSpec{SecretRef: v1alpha1.SecretKeyRef{Name: "rcon-off", Key: naming.RconSecretKey}}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		mk("demo", v1alpha1.RconSpec{}, ""),
+		mk("off", off, ""),
+		mk("own", own, ""),
+		mk(naming.SystemLobbyServer, v1alpha1.RconSpec{}, naming.SystemLobbyServer),
+	).Build()
+	get := func(name string) v1alpha1.RconSpec {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		return ms.Spec.Rcon
+	}
+
+	report := convergeUserServerRcon(ctx, cl, "minecraft", false)
+	if len(report) != 1 || report[0].name != "demo" || report[0].updated || report[0].err != nil ||
+		!strings.Contains(report[0].skipped, "-user-rcon") {
+		t.Fatalf("report = %+v, want one skipped line for demo naming -user-rcon", report)
+	}
+	if got := get("demo"); got != (v1alpha1.RconSpec{}) {
+		t.Fatalf("the report-only pass wrote demo's rcon: %+v", got)
+	}
+
+	filled := convergeUserServerRcon(ctx, cl, "minecraft", true)
+	if len(filled) != 1 || filled[0].name != "demo" || !filled[0].updated || filled[0].err != nil {
+		t.Fatalf("fill = %+v, want exactly one update for demo", filled)
+	}
+	want := map[string]v1alpha1.RconSpec{
+		"demo": {Enabled: true, SecretRef: v1alpha1.SecretKeyRef{
+			Name: naming.RconSecretName("demo"), Key: naming.RconSecretKey}},
+		"off":                    off,
+		"own":                    own,
+		naming.SystemLobbyServer: {},
+	}
+	for name, rcon := range want {
+		if got := get(name); got != rcon {
+			t.Errorf("%s rcon = %+v, want %+v", name, got, rcon)
+		}
+	}
+	for _, fill := range []bool{false, true} {
+		if again := convergeUserServerRcon(ctx, cl, "minecraft", fill); len(again) != 0 {
+			t.Fatalf("second pass (fill=%v) = %+v, want nothing to do", fill, again)
+		}
+	}
+}

cmd/felis/db.go

@@ -0,0 +1,501 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	neturl "net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/retention"
+)
+
+const dbUsage = `usage:
+  felis db backup  [-config path] [-dir dir] [-label daily|manual|...] [-keep n] [-state-dir dir]
+                   [-no-servers] [-metrics-file path]
+  felis db restore [-config path] [-dir dir] [-yes] [-force] [-no-safety-backup] <bundle>
+  felis db verify  [-dir dir] <bundle>
+  felis db list    [-dir dir]
+  felis db check   [-dir dir] [-max-age 26h]
+  felis db audit-export [-config path] [-since date] [-until date] [-out file]
+`
+
+// defaultKeep is how many bundles of a label a backup leaves behind. Manual
+// bundles are the operator's own and are never pruned.
+var defaultKeep = map[string]int{
+	dbbackup.LabelDaily:      14,
+	dbbackup.LabelPreMigrate: 10,
+	dbbackup.LabelPreRestore: 5,
+	dbbackup.LabelOffsite:    1,
+}
+
+// cmdDB implements `felis db`: logical backups of the control-plane database
+// together with the host state a rebuild needs (internal/dbbackup). The verb
+// comes first for the same reason as `felis migrate up`.
+func cmdDB(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("db "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, dbUsage) }
+	dir := fs.String("dir", dbbackup.DefaultDir, "bundle directory")
+	switch verb {
+	case "backup":
+		return dbBackup(fs, dir, rest, stdout, stderr)
+	case "restore":
+		return dbRestore(fs, dir, rest, stdout, stderr)
+	case "verify":
+		return dbVerify(fs, dir, rest, stdout, stderr)
+	case "list":
+		return dbList(fs, dir, rest, stdout, stderr)
+	case "check":
+		return dbCheck(fs, dir, rest, stdout, stderr)
+	case "audit-export":
+		return dbAuditExport(fs, rest, stdout, stderr)
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, dbUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis db: unknown verb %q\n%s", verb, dbUsage)
+	return 2
+}
+
+// parseWithArg parses flags that may sit on either side of one positional
+// argument (`restore -yes x.tar` and `restore x.tar -yes` both work) and
+// returns that argument.
+func parseWithArg(fs *flag.FlagSet, args []string) (string, bool) {
+	if err := fs.Parse(args); err != nil {
+		return "", false
+	}
+	if fs.NArg() == 0 {
+		return "", true
+	}
+	arg := fs.Arg(0)
+	if err := fs.Parse(fs.Args()[1:]); err != nil {
+		return "", false
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprintf(fs.Output(), "felis db: unexpected argument %q\n", fs.Arg(0))
+		return "", false
+	}
+	return arg, true
+}
+
+func dbDatabaseURL(path string) (string, error) {
+	db, err := dbDatabase(path)
+	return db.URL, err
+}
+
+func dbDatabase(path string) (config.DatabaseConfig, error) {
+	cfg, err := config.Load(path)
+	if err != nil {
+		return config.DatabaseConfig{}, err
+	}
+	return cfg.Database, nil
+}
+
+// dbTools places pg_dump, pg_restore and psql. The installer's database runs in
+// k3s and the host has no PostgreSQL client, so when the host config names the
+// [database] deployment the tools run in its postgres container over the
+// container's socket, as the URL's role on the URL's database. Otherwise they
+// come from PATH and connect with the URL.
+func dbTools(db config.DatabaseConfig) (dbbackup.Tools, error) {
+	if db.Deployment == "" {
+		return dbbackup.Tools{}, nil
+	}
+	ns, name, _ := strings.Cut(db.Deployment, "/")
+	u, err := neturl.Parse(db.URL)
+	if err != nil || u.User == nil || u.User.Username() == "" || strings.TrimPrefix(u.Path, "/") == "" {
+		return dbbackup.Tools{}, errors.New("[database] url must name the role and the database to run the tools in the database's pod")
+	}
+	return dbbackup.Tools{
+		Exec: []string{"k3s", "kubectl", "exec", "-i", "-n", ns, "deploy/" + name, "-c", platform.PostgresContainer, "--"},
+		Conn: fmt.Sprintf("host=%s port=%d dbname=%s user=%s connect_timeout=15",
+			platform.PostgresSocketDir, platform.PostgresPort,
+			libpqQuote(strings.TrimPrefix(u.Path, "/")), libpqQuote(u.User.Username())),
+	}, nil
+}
+
+// libpqQuote renders v as a single-quoted libpq connection-string value.
+func libpqQuote(v string) string {
+	return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(v) + "'"
+}
+
+func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never")
+	keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, offsite 1, manual all)")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`)
+	noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle")
+	metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	db, err := dbDatabase(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	tools, err := dbTools(db)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	if *keep < 0 {
+		*keep = defaultKeep[*label]
+	}
+	o := dbbackup.BackupOptions{
+		DatabaseURL: db.URL, Tools: tools, Dir: *dir, Label: *label, Keep: *keep,
+		StateDir: *stateDir, Version: resolvedVersion(), Log: stderr,
+		MetricsFile: *metrics, Record: true,
+	}
+	if !*noServers {
+		o.ExportServers = exportMinecraftServers
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	path, err := dbbackup.Backup(ctx, o)
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// The bundle is on disk and holds the database; the exit status fails
+		// the timer's run so the gap shows in systemctl and the journal, and
+		// the panel and the watchdog read it from the record and the manifest.
+		fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+		fmt.Fprintf(stderr, "felis db backup: %v\n  a restore from %s brings back the database but no servers; check `k3s kubectl get minecraftservers -A`, then run `felis db backup` again\n", err, filepath.Base(path))
+		return 1
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+	return 0
+}
+
+// resolveBundle accepts a path, or a bare bundle name looked up in dir.
+func resolveBundle(dir, arg string) string {
+	if strings.ContainsRune(arg, os.PathSeparator) {
+		return arg
+	}
+	if _, err := os.Stat(arg); err == nil {
+		return arg
+	}
+	return filepath.Join(dir, arg)
+}
+
+func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	yes := fs.Bool("yes", false, "replace the database's contents (required)")
+	force := fs.Bool("force", false, "restore even while other clients are connected")
+	noSafety := fs.Bool("no-safety-backup", false, "skip the bundle of the current database taken first")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, "host state directory for the safety bundle")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	if !*yes {
+		fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d, holding %s).\n",
+			filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion, m.Counts.String())
+		fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.")
+		return 2
+	}
+	db, err := dbDatabase(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	tools, err := dbTools(db)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
+	defer cancel()
+	_, safety, err := dbbackup.Restore(ctx, dbbackup.RestoreOptions{
+		DatabaseURL: db.URL, Tools: tools, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
+		Safety: dbbackup.BackupOptions{Keep: defaultKeep[dbbackup.LabelPreRestore], StateDir: *stateDir,
+			Version: resolvedVersion(), ExportServers: exportMinecraftServers},
+		Log: stderr,
+	})
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		if errors.Is(err, dbbackup.ErrClientsConnected) {
+			fmt.Fprintln(stderr, "  kubectl -n felis scale deployment felis-api felis-operator --replicas=0")
+		}
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db restore: restored %s (schema %d)\n", filepath.Base(bundle), m.SchemaVersion)
+	if safety != "" {
+		fmt.Fprintf(stdout, "  the database as it was before is in %s\n", safety)
+	}
+	// Nothing migrates at startup, so a control plane newer than the bundle needs
+	// its migrations re-applied; rolling back to the release that wrote the bundle
+	// must skip that, or the rollback is undone.
+	fmt.Fprintf(stdout, "  next: felis migrate up -config %s (skip it when rolling back to felis %s, which wrote this bundle)\n", *cfgPath, orUnknown(m.FelisVersion))
+	fmt.Fprintln(stdout, "        kubectl -n felis scale deployment felis-api felis-operator --replicas=1")
+	return 0
+}
+
+func dbVerify(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db verify: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "%s: ok\n  taken   %s (%s)\n  felis   %s\n  schema  %d\n  holds   %s\n  %s\n",
+		filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion,
+		m.Counts.String(), orUnknown(m.PGDumpVersion))
+	for _, f := range m.Files {
+		if f.Link != "" {
+			fmt.Fprintf(stdout, "  %-40s -> %s\n", f.Name, f.Link)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %-40s %d bytes\n", f.Name, f.Size)
+	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  (no MinecraftServer objects: %s)\n", m.ServersError)
+	}
+	return 0
+}
+
+func orUnknown(s string) string {
+	if s == "" {
+		return "unknown"
+	}
+	return s
+}
+
+func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	all, err := dbbackup.List(*dir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db list: %v\n", err)
+		return 1
+	}
+	if len(all) == 0 {
+		fmt.Fprintf(stdout, "no database backups in %s\n", *dir)
+		return 0
+	}
+	now := time.Now()
+	for _, b := range all {
+		fmt.Fprintf(stdout, "%-50s %-12s %10s  %s ago\n", b.Name, b.Label, humanBytes(b.Size), dbbackup.Age(now.Sub(b.Created)))
+	}
+	return 0
+}
+
+// dbAuditExport writes audit rows to a file (or stdout) as JSON lines, so an
+// install can keep them past [audit] retention, after which felis-api deletes them.
+func dbAuditExport(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	sinceFlag := fs.String("since", "", "first day (or RFC 3339 instant) to export, inclusive; empty starts at the oldest row")
+	untilFlag := fs.String("until", "", "day (or RFC 3339 instant) to stop before, exclusive; empty runs to the newest row")
+	out := fs.String("out", "", "file to write (created 0600, never overwritten); empty writes to stdout")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	since, err := parseExportBound(*sinceFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -since: %v\n", err)
+		return 2
+	}
+	until, err := parseExportBound(*untilFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -until: %v\n", err)
+		return 2
+	}
+	if !since.IsZero() && !until.IsZero() && !until.After(since) {
+		fmt.Fprintf(stderr, "felis db audit-export: -until %s is not after -since %s\n", *untilFlag, *sinceFlag)
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+		return 1
+	}
+	w := stdout
+	var f *os.File
+	if *out != "" {
+		if f, err = os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600); err != nil {
+			fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+			return 1
+		}
+		w = f
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	n, err := exportAudit(ctx, url, since, until, w)
+	if f != nil {
+		if cerr := f.Close(); err == nil {
+			err = cerr
+		}
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v (%d rows written)\n", err, n)
+		return 1
+	}
+	fmt.Fprintf(stderr, "felis db audit-export: %d audit rows written\n", n)
+	return 0
+}
+
+func exportAudit(ctx context.Context, url string, since, until time.Time, w io.Writer) (int, error) {
+	drv, err := openStore(ctx, url, false)
+	if err != nil {
+		return 0, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	return retention.ExportAudit(ctx, drv.DB(), since, until, w)
+}
+
+// parseExportBound reads a -since/-until value: a day (midnight UTC) or an
+// RFC 3339 instant; empty is an open bound.
+func parseExportBound(v string) (time.Time, error) {
+	if v == "" {
+		return time.Time{}, nil
+	}
+	if t, err := time.Parse(time.DateOnly, v); err == nil {
+		return t, nil
+	}
+	if t, err := time.Parse(time.RFC3339, v); err == nil {
+		return t.UTC(), nil
+	}
+	return time.Time{}, fmt.Errorf("%q is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)", v)
+}
+
+func humanBytes(n int64) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%d B", n)
+	}
+	div, exp := int64(unit), 0
+	for m := n / unit; m >= unit; m /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
+}
+
+// dbCheck is the freshness probe: exit 1 when the newest daily bundle is
+// missing or older than -max-age, for a monitor or the break-glass console to
+// act on.
+func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest daily bundle")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	b, err := dbbackup.Check(*dir, *maxAge, time.Now())
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db check: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db check: ok, newest daily backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
+	return 0
+}
+
+// serverExportTries and serverExportRetry are how long a backup waits out a
+// cluster that is briefly away (an apiserver restart) before its bundle goes
+// without the MinecraftServer objects.
+const serverExportTries = 3
+
+var serverExportRetry = 10 * time.Second
+
+// exportMinecraftServers is dbbackup's ExportServers on the host: the
+// MinecraftServer objects through k3s kubectl, tried serverExportTries times.
+func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+	for try := 1; ; try++ {
+		out, err := getMinecraftServers(ctx)
+		if err == nil {
+			return out, nil
+		}
+		if try == serverExportTries {
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		}
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("%w (tried %d times)", err, try)
+		case <-time.After(serverExportRetry):
+		}
+	}
+}
+
+// getMinecraftServers reads every MinecraftServer through the host's k3s
+// kubectl and strips what the API server owns, so the result can be fed back
+// with `kubectl apply -f` on a rebuilt cluster.
+func getMinecraftServers(ctx context.Context) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning
+	// on stderr must not end up inside the JSON.
+	cmd := exec.CommandContext(ctx, "k3s", "kubectl", "get", "minecraftservers.felis.lolicon.best", "-A", "-o", "json")
+	cmd.Env = append(os.Environ(), "KUBECONFIG="+hostBootstrapKubeconfigPath)
+	var errBuf strings.Builder
+	cmd.Stderr = &errBuf
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("k3s kubectl get minecraftservers: %w: %s", err, strings.TrimSpace(errBuf.String()))
+	}
+	return cleanServerList(out)
+}
+
+// cleanServerList drops status and the server-assigned metadata from a
+// `kubectl get -o json` List.
+func cleanServerList(raw []byte) ([]byte, error) {
+	var list struct {
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(raw, &list); err != nil {
+		return nil, fmt.Errorf("parse MinecraftServer list: %w", err)
+	}
+	for _, it := range list.Items {
+		delete(it, "status")
+		if md, ok := it["metadata"].(map[string]any); ok {
+			for _, k := range []string{"resourceVersion", "uid", "creationTimestamp", "generation", "managedFields", "selfLink"} {
+				delete(md, k)
+			}
+		}
+	}
+	if list.Items == nil {
+		list.Items = []map[string]any{}
+	}
+	return json.MarshalIndent(map[string]any{"apiVersion": "v1", "kind": "List", "items": list.Items}, "", "  ")
+}

cmd/felis/db_test.go

@@ -0,0 +1,567 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/store"
+)
+
+func TestDBUsage(t *testing.T) {
+	for _, args := range [][]string{{"db"}, {"db", "frobnicate"}, {"db", "restore"}, {"db", "verify"}, {"db", "backup", "extra"}} {
+		var out, errBuf bytes.Buffer
+		if code := run(args, &out, &errBuf); code != 2 {
+			t.Errorf("%v: exit %d, want 2", args, code)
+		}
+		if !strings.Contains(errBuf.String(), "felis db restore") {
+			t.Errorf("%v: no usage on stderr: %q", args, errBuf.String())
+		}
+	}
+}
+
+func TestDBVerifySaysWhatTheBundleHolds(t *testing.T) {
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	bundles := filepath.Join(dir, "bundles")
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
+		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	out.Reset()
+	if code := run([]string{"db", "verify", "-dir", bundles, filepath.Base(bundle)}, &out, &errBuf); code != 0 {
+		t.Fatalf("verify: exit %d: %s", code, errBuf.String())
+	}
+	for _, want := range []string{filepath.Base(bundle) + ": ok", "schema  3", "holds   4 accounts, 2 servers", "pg_dump (PostgreSQL) 18.6"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("verify output lacks %q:\n%s", want, out.String())
+		}
+	}
+}
+
+// TestDBRestoreNeedsYes: without -yes a restore describes the bundle and stops
+// before anything reaches the database, even with -force and
+// -no-safety-backup, which would otherwise let the replay run at once.
+func TestDBRestoreNeedsYes(t *testing.T) {
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	bundles := filepath.Join(dir, "bundles")
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
+		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	podRuns(t, dir)
+
+	out.Reset()
+	errBuf.Reset()
+	code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-force", "-no-safety-backup", filepath.Base(bundle)}, &out, &errBuf)
+	if code != 2 {
+		t.Fatalf("exit %d, want 2; stderr %q", code, errBuf.String())
+	}
+	if want := filepath.Base(bundle) + " (manual, taken "; !strings.Contains(errBuf.String(), want) || !strings.Contains(errBuf.String(), "schema 3, holding 4 accounts, 2 servers).") {
+		t.Errorf("stderr %q does not describe the bundle", errBuf.String())
+	}
+	if !strings.Contains(errBuf.String(), "re-run with -yes") {
+		t.Errorf("stderr %q does not say how to go on", errBuf.String())
+	}
+	if argv, err := os.ReadFile(filepath.Join(dir, "k3s.args")); err == nil {
+		t.Errorf("a restore without -yes ran in the database pod:\n%s", argv)
+	}
+
+	// A bundle that does not verify is refused before -yes is weighed.
+	errBuf.Reset()
+	if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "missing.tar"}, &out, &errBuf); code != 1 {
+		t.Errorf("missing bundle: exit %d, want 1; stderr %q", code, errBuf.String())
+	}
+	if _, err := os.Stat(filepath.Join(dir, "k3s.args")); err == nil {
+		t.Error("a missing bundle reached the database pod")
+	}
+}
+
+func TestParseWithArg(t *testing.T) {
+	for _, args := range [][]string{{"-yes", "b.tar"}, {"b.tar", "-yes"}} {
+		fs := flag.NewFlagSet("t", flag.ContinueOnError)
+		fs.SetOutput(io.Discard)
+		yes := fs.Bool("yes", false, "")
+		arg, ok := parseWithArg(fs, args)
+		if !ok || arg != "b.tar" || !*yes {
+			t.Errorf("%v -> %q ok=%v yes=%v", args, arg, ok, *yes)
+		}
+	}
+	fs := flag.NewFlagSet("t", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	if _, ok := parseWithArg(fs, []string{"a.tar", "b.tar"}); ok {
+		t.Error("two positional arguments accepted")
+	}
+}
+
+func TestResolveBundle(t *testing.T) {
+	if got := resolveBundle("/var/lib/felis/db-backups", "felis-db-x.tar"); got != "/var/lib/felis/db-backups/felis-db-x.tar" {
+		t.Errorf("bare name -> %s", got)
+	}
+	if got := resolveBundle("/var/lib/felis/db-backups", "/root/copy.tar"); got != "/root/copy.tar" {
+		t.Errorf("path -> %s", got)
+	}
+}
+
+func TestCleanServerList(t *testing.T) {
+	raw := `{"apiVersion":"v1","kind":"List","metadata":{"resourceVersion":""},"items":[{
+		"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer",
+		"metadata":{"name":"survival","namespace":"minecraft","uid":"u","resourceVersion":"42","generation":3,
+			"creationTimestamp":"2026-09-01T00:00:00Z","managedFields":[{}],"labels":{"a":"b"}},
+		"spec":{"desiredState":"Running"},"status":{"phase":"Running"}}]}`
+	out, err := cleanServerList([]byte(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Kind  string           `json:"kind"`
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatal(err)
+	}
+	if got.Kind != "List" || len(got.Items) != 1 {
+		t.Fatalf("got %s", out)
+	}
+	it := got.Items[0]
+	if _, ok := it["status"]; ok {
+		t.Error("status kept")
+	}
+	md := it["metadata"].(map[string]any)
+	for _, k := range []string{"uid", "resourceVersion", "generation", "creationTimestamp", "managedFields"} {
+		if _, ok := md[k]; ok {
+			t.Errorf("metadata.%s kept", k)
+		}
+	}
+	if md["name"] != "survival" || md["namespace"] != "minecraft" || md["labels"] == nil {
+		t.Errorf("identity lost: %v", md)
+	}
+	if it["spec"].(map[string]any)["desiredState"] != "Running" {
+		t.Error("spec lost")
+	}
+
+	empty, err := cleanServerList([]byte(`{"items":null}`))
+	if err != nil || !strings.Contains(string(empty), `"items": []`) {
+		t.Errorf("empty list -> %s, %v", empty, err)
+	}
+	if _, err := cleanServerList([]byte("Warning: x\n{")); err == nil {
+		t.Error("garbage parsed")
+	}
+}
+
+func TestHasPending(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}, {Version: 3}}
+	if hasPending(map[int]struct{}{1: {}, 2: {}, 3: {}}, ms) {
+		t.Error("fully applied reported pending")
+	}
+	if !hasPending(map[int]struct{}{1: {}, 2: {}}, ms) {
+		t.Error("missing 3 not reported")
+	}
+}
+
+type appliedDriver struct {
+	store.Driver
+	done map[int]struct{}
+}
+
+func (d appliedDriver) EnsureVersionTable(context.Context) error { return nil }
+func (d appliedDriver) AppliedVersions(context.Context) (map[int]struct{}, error) {
+	return d.done, nil
+}
+
+func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	// An unusable URL makes an attempted backup observable as an error without
+	// any PostgreSQL tooling.
+	const badURL = "not-a-url"
+	for _, tc := range []struct {
+		name    string
+		done    map[int]struct{}
+		attempt bool
+	}{
+		{"fresh database", map[int]struct{}{}, false},
+		{"up to date", map[int]struct{}{1: {}, 2: {}}, false},
+		{"pending on a populated database", map[int]struct{}{1: {}}, true},
+	} {
+		path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, config.DatabaseConfig{URL: badURL}, t.TempDir(), io.Discard)
+		if attempted := err != nil; attempted != tc.attempt {
+			t.Errorf("%s: attempted = %v (err %v), want %v", tc.name, attempted, err, tc.attempt)
+		}
+		if path != "" {
+			t.Errorf("%s: path = %q", tc.name, path)
+		}
+	}
+}
+
+// audit-export takes a day or an RFC 3339 instant for each bound, and refuses a
+// malformed or inverted window before it opens the config or the database.
+func TestAuditExportBounds(t *testing.T) {
+	for _, tc := range []struct{ in, want string }{
+		{"", "0001-01-01T00:00:00Z"},
+		{"2026-01-31", "2026-01-31T00:00:00Z"},
+		{"2026-01-31T12:30:00+08:00", "2026-01-31T04:30:00Z"},
+	} {
+		got, err := parseExportBound(tc.in)
+		if err != nil || got.Format(time.RFC3339) != tc.want {
+			t.Errorf("parseExportBound(%q) = %v, %v; want %s", tc.in, got, err, tc.want)
+		}
+	}
+	if _, err := parseExportBound("31/01/2026"); err == nil || err.Error() != `"31/01/2026" is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)` {
+		t.Errorf("parseExportBound(31/01/2026) err = %v", err)
+	}
+	for _, tc := range []struct {
+		args    []string
+		wantErr string
+	}{
+		{[]string{"db", "audit-export", "-since", "yesterday"}, `felis db audit-export: -since: "yesterday" is neither`},
+		{[]string{"db", "audit-export", "-until", "2026-13-01"}, `felis db audit-export: -until: "2026-13-01" is neither`},
+		{[]string{"db", "audit-export", "-since", "2026-02-01", "-until", "2026-02-01"}, "felis db audit-export: -until 2026-02-01 is not after -since 2026-02-01"},
+		{[]string{"db", "audit-export", "extra"}, "felis db audit-export [-config path]"},
+	} {
+		var out, errBuf bytes.Buffer
+		code := run(append(tc.args, "-config", "/nonexistent/felis.toml"), &out, &errBuf)
+		if code != 2 || !strings.Contains(errBuf.String(), tc.wantErr) {
+			t.Errorf("%v: exit %d, stderr %q; want 2 and %q", tc.args, code, errBuf.String(), tc.wantErr)
+		}
+	}
+}
+
+// podK3s stands in for `k3s kubectl exec ... --`: it logs its argv and runs the
+// command after -- from the "container" directory, which is the only place the
+// PostgreSQL tools exist, as on an installed host. `kubectl get` lists one
+// MinecraftServer, logged to k3s.get, and refuses its first N calls while
+// servers_fail holds N.
+const podK3s = `#!/bin/sh
+if [ "$1" = kubectl ] && [ "$2" = get ]; then
+  printf '%s\n' "$*" >> "$FAKE_DIR/k3s.get"
+  n=$(/usr/bin/wc -l < "$FAKE_DIR/k3s.get")
+  if [ -f "$FAKE_DIR/servers_fail" ] && [ "$n" -le "$(/bin/cat "$FAKE_DIR/servers_fail")" ]; then
+    echo "The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?" >&2
+    exit 1
+  fi
+  echo '{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer","metadata":{"name":"lobby","namespace":"felis-servers","uid":"u-1"},"spec":{"type":"PAPER"},"status":{"phase":"Running"}}]}'
+  exit 0
+fi
+printf '%s\n' "$*" >> "$FAKE_DIR/k3s.args"
+while [ $# -gt 0 ] && [ "$1" != "--" ]; do shift; done
+shift
+tool=$1; shift
+exec /usr/bin/env -i FAKE_DIR="$FAKE_DIR" PATH=/usr/bin:/bin "$FAKE_DIR/container/$tool" "$@"
+`
+
+var podTools = map[string]string{
+	"pg_dump": `#!/bin/sh
+case "$1" in --version) echo "pg_dump (PostgreSQL) 18.6"; exit 0 ;; esac
+printf 'PGDMP-fake-archive'
+`,
+	"pg_restore": `#!/bin/sh
+cat > /dev/null
+`,
+	"psql": `#!/bin/sh
+for a in "$@"; do case "$a" in *"FROM users"*) echo "4|2"; exit 0 ;; esac; done
+for a in "$@"; do [ "$a" = "-c" ] && { echo 3; exit 0; }; done
+cat > /dev/null
+`,
+}
+
+const podPassword = "pw-must-stay-on-the-host"
+
+// podDB is the host config's [database] on an installed host.
+var podDB = config.DatabaseConfig{
+	URL:        "postgres://felis:" + podPassword + "@127.0.0.1:15432/felis?sslmode=disable",
+	Deployment: "felis/felis-postgres",
+}
+
+const podExecPrefix = "kubectl exec -i -n felis deploy/felis-postgres -c postgres -- "
+
+// newPodRig puts the fake k3s on PATH, alone, and returns the directory its
+// k3s.args log lands in.
+func newPodRig(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	bin := filepath.Join(dir, "bin")
+	container := filepath.Join(dir, "container")
+	for _, d := range []string{bin, container} {
+		if err := os.Mkdir(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeTestFile(t, filepath.Join(bin, "k3s"), podK3s, 0o755)
+	for name, body := range podTools {
+		writeTestFile(t, filepath.Join(container, name), body, 0o755)
+	}
+	t.Setenv("PATH", bin)
+	t.Setenv("FAKE_DIR", dir)
+	return dir
+}
+
+// podRuns returns what the fake k3s ran since the last call, failing on any
+// run outside the database container or with the password on its command
+// line (visible to every local user in ps).
+func podRuns(t *testing.T, dir string) []string {
+	t.Helper()
+	log := filepath.Join(dir, "k3s.args")
+	argv, err := os.ReadFile(log)
+	if err != nil {
+		t.Fatalf("nothing ran through k3s: %v", err)
+	}
+	os.Remove(log)
+	var runs []string
+	for _, line := range strings.Split(strings.TrimSpace(string(argv)), "\n") {
+		if !strings.HasPrefix(line, podExecPrefix) {
+			t.Errorf("k3s ran %q, want everything under %q", line, podExecPrefix)
+		}
+		if strings.Contains(line, podPassword) {
+			t.Errorf("the password crossed into the pod on a command line: %q", line)
+		}
+		runs = append(runs, strings.TrimPrefix(line, podExecPrefix))
+	}
+	return runs
+}
+
+// podConfig writes an installed host's felis.toml, [database] pointing at the
+// pod, into dir.
+func podConfig(t *testing.T, dir string) string {
+	t.Helper()
+	toml := strings.Replace(installerTOML("example.com", "127.0.0.1"),
+		`url = "postgres://felis:[email protected]:5432/felis?sslmode=disable"`,
+		`url = "`+podDB.URL+`"
+deployment = "`+podDB.Deployment+`"`, 1)
+	cfg := filepath.Join(dir, "felis.toml")
+	writeTestFile(t, cfg, toml, 0o600)
+	return cfg
+}
+
+func ranIn(runs []string, prefix string) bool {
+	for _, r := range runs {
+		if strings.HasPrefix(r, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+// TestDBBackupAndRestoreRunTheToolsInTheDatabasePod: on an installed host the
+// database is a k3s Deployment and no PostgreSQL client exists outside it, so
+// `felis db backup` and `restore` must reach the tools through kubectl exec,
+// over the pod's socket, and without putting the role's password on a command
+// line.
+func TestDBBackupAndRestoreRunTheToolsInTheDatabasePod(t *testing.T) {
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+
+	var out, errBuf bytes.Buffer
+	bundles := filepath.Join(dir, "bundles")
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
+		t.Fatalf("backup: exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	if _, err := dbbackupVerify(bundle); err != nil {
+		t.Fatalf("the bundle does not verify: %v", err)
+	}
+	runs := podRuns(t, dir)
+	if !ranIn(runs, "pg_dump --format=custom --no-password --dbname=host=/var/run/postgresql port=5432 dbname='felis' user='felis'") {
+		t.Errorf("pg_dump did not dump over the pod's socket as felis on felis: %q", runs)
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "-force", "-no-safety-backup", bundle}, &out, &errBuf); code != 0 {
+		t.Fatalf("restore: exit %d: %s", code, errBuf.String())
+	}
+	runs = podRuns(t, dir)
+	if !ranIn(runs, "pg_restore --no-owner --no-privileges --file=-") || !ranIn(runs, "psql -X -q -w -v ON_ERROR_STOP=1 -d host=/var/run/postgresql") {
+		t.Errorf("the replay did not run in the pod: %q", runs)
+	}
+}
+
+// TestPreMigrateBackupRunsInTheDatabasePod: the snapshot in front of an upgrade
+// is the one taken most often, by bootstrap on every rerun.
+func TestPreMigrateBackupRunsInTheDatabasePod(t *testing.T) {
+	dir := newPodRig(t)
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	// The snapshot also bundles /etc/felis, which a test machine may lack; the
+	// dump runs first either way.
+	_, err := preMigrateBackup(context.Background(), appliedDriver{done: map[int]struct{}{1: {}}}, ms, podDB, filepath.Join(dir, "bundles"), io.Discard)
+	if err != nil && !strings.Contains(err.Error(), "read host state") {
+		t.Fatalf("snapshot: %v", err)
+	}
+	if runs := podRuns(t, dir); !ranIn(runs, "pg_dump --format=custom") {
+		t.Errorf("pg_dump did not run in the pod: %q", runs)
+	}
+}
+
+func TestDBToolsNeedTheRoleAndDatabase(t *testing.T) {
+	if tools, err := dbTools(config.DatabaseConfig{URL: "postgres://felis:pw@db:5432/felis"}); err != nil || len(tools.Exec) != 0 {
+		t.Errorf("no deployment: tools %+v err %v, want the PATH tools", tools, err)
+	}
+	for _, u := range []string{"postgres://db:5432/felis", "postgres://felis:pw@db:5432/"} {
+		if _, err := dbTools(config.DatabaseConfig{URL: u, Deployment: "felis/felis-postgres"}); err == nil {
+			t.Errorf("%s: no error, want a refusal (the pod connection needs the role and the database)", u)
+		}
+	}
+	tools, err := dbTools(config.DatabaseConfig{URL: `postgres://o%27brien@db/my%20db`, Deployment: "felis/felis-postgres"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(tools.Conn, `dbname='my db' user='o\'brien'`) {
+		t.Errorf("Conn = %q, want the values quoted for libpq", tools.Conn)
+	}
+}
+
+var dbbackupVerify = dbbackup.Verify
+
+func noServerExportWait(t *testing.T) {
+	t.Helper()
+	old := serverExportRetry
+	serverExportRetry = 0
+	t.Cleanup(func() { serverExportRetry = old })
+}
+
+// serverGets counts the `kubectl get` calls the fake k3s answered or refused.
+func serverGets(dir string) int {
+	b, _ := os.ReadFile(filepath.Join(dir, "k3s.get"))
+	return strings.Count(string(b), "\n")
+}
+
+// bundleServers returns the bundle's k8s/minecraftservers.json, or nil.
+func bundleServers(t *testing.T, bundle string) []byte {
+	t.Helper()
+	f, err := os.Open(bundle)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	tr := tar.NewReader(f)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		if h.Name == "k8s/minecraftservers.json" {
+			data, err := io.ReadAll(tr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return data
+		}
+	}
+}
+
+// TestDBBackupWithoutServersFails: when the cluster stays away the daily
+// bundle is still written, and `felis db backup` exits 1, so the timer's run
+// shows failed, saying a restore from the bundle brings back no servers.
+func TestDBBackupWithoutServersFails(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	var out, errBuf bytes.Buffer
+	code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", "", "-label", "daily"}, &out, &errBuf)
+	if code != 1 {
+		t.Fatalf("exit %d, want 1: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	m, err := dbbackupVerify(bundle)
+	if err != nil {
+		t.Fatalf("the database must still be bundled: %v", err)
+	}
+	if !strings.Contains(m.ServersError, "6443 was refused") || !strings.Contains(m.ServersError, "(tried 3 times)") || bundleServers(t, bundle) != nil {
+		t.Errorf("manifest servers error = %q", m.ServersError)
+	}
+	if n := serverGets(dir); n != serverExportTries {
+		t.Errorf("export tried %d times, want %d", n, serverExportTries)
+	}
+	if msg := errBuf.String(); !strings.Contains(msg, "a restore from "+filepath.Base(bundle)+" brings back the database but no servers") {
+		t.Errorf("stderr = %q", msg)
+	}
+}
+
+// TestDBBackupRetriesTheServerExport: a cluster back on the last try costs the
+// bundle nothing, and what it holds is ready for kubectl apply.
+func TestDBBackupRetriesTheServerExport(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	cfg := podConfig(t, dir)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "2", 0o600)
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", ""}, &out, &errBuf); code != 0 {
+		t.Fatalf("exit %d: %s", code, errBuf.String())
+	}
+	bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
+	servers := string(bundleServers(t, bundle))
+	if !strings.Contains(servers, `"name": "lobby"`) || strings.Contains(servers, "status") || strings.Contains(servers, "u-1") {
+		t.Errorf("k8s/minecraftservers.json = %s", servers)
+	}
+	if n := serverGets(dir); n != 3 {
+		t.Errorf("export tried %d times, want 3", n)
+	}
+}
+
+// TestPreMigrateBackupExportsServers: the snapshot every upgrade takes, often
+// the newest bundle, carries the MinecraftServer objects too; a cluster that
+// is away does not hold back the migration, whose rollback needs the database
+// alone.
+func TestPreMigrateBackupExportsServers(t *testing.T) {
+	noServerExportWait(t)
+	dir := newPodRig(t)
+	old := preMigrateStateDir
+	preMigrateStateDir = ""
+	t.Cleanup(func() { preMigrateStateDir = old })
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	bundles := filepath.Join(dir, "bundles")
+	pending := appliedDriver{done: map[int]struct{}{1: {}}}
+
+	path, err := preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil {
+		t.Fatalf("snapshot: %v", err)
+	}
+	if !strings.Contains(string(bundleServers(t, path)), `"name": "lobby"`) {
+		t.Errorf("%s holds no MinecraftServer objects", path)
+	}
+
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	path, err = preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
+	if err != nil || path == "" {
+		t.Fatalf("snapshot with the cluster away = %q, %v; want the bundle and no error", path, err)
+	}
+	if m, err := dbbackupVerify(path); err != nil || m.ServersError == "" || bundleServers(t, path) != nil {
+		t.Errorf("snapshot with the cluster away: %+v, %v", m, err)
+	}
+}
+
+// TestServerExportStopsWaitingWithTheContext: a backup whose time is up stops
+// waiting for the cluster between tries.
+func TestServerExportStopsWaitingWithTheContext(t *testing.T) {
+	dir := newPodRig(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	// Long enough for the first try to run to its refusal: starting the fake
+	// k3s on a busy machine can take a few hundred ms. Still far below the
+	// 10s retry wait, so waiting it out would fail the check below.
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	start := time.Now()
+	_, err := exportMinecraftServers(ctx)
+	if err == nil || !strings.Contains(err.Error(), "(tried 1 times)") || serverGets(dir) != 1 {
+		t.Fatalf("err = %v after %d tries, want the first failure alone", err, serverGets(dir))
+	}
+	if took := time.Since(start); took > 5*time.Second {
+		t.Errorf("took %s, want the context's deadline, not the %s retry wait", took, serverExportRetry)
+	}
+}

cmd/felis/distributed.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"felis.lolicon.best/internal/archivetransfer"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/placement"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func distributionManager(cl client.Client, cfg *config.Config, image string) (*distributed.Manager, error) {
+	if os.Getenv("FELIS_DISTRIBUTED") != "true" {
+		return nil, nil
+	}
+	controller, url, key := os.Getenv("FELIS_CONTROLLER_NODE"), os.Getenv("FELIS_ARCHIVE_URL"), os.Getenv(archivetransfer.KeyEnv)
+	if controller == "" || url == "" || len(key) < 32 || image == "" || cfg.Archive.Store != "tarLocal" {
+		return nil, fmt.Errorf("distributed mode requires controller identity, archive service/key, Felis image and tarLocal")
+	}
+	return &distributed.Manager{Client: cl, Namespace: cfg.K8s.Namespace, Image: image, Controller: controller, Archive: archivetransfer.Client{URL: url, Root: cfg.Archive.LocalPath, Key: key}, Resolve: placement.Resolve(cl, cfg.K8s.Namespace, controller)}, nil
+}
+
+func reconcileDistribution(ctx context.Context, m *distributed.Manager, stderr io.Writer) {
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+	for {
+		if err := m.SettleBackups(ctx); err != nil {
+			fmt.Fprintln(stderr, "distributed backup:", err)
+		}
+		if err := m.ReconcileMigrations(ctx); err != nil {
+			fmt.Fprintln(stderr, "migration:", err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}

cmd/felis/doctor.go

@@ -0,0 +1,387 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/watchdog"
+)
+
+// systemdUnitDir is where the installer writes its units.
+const systemdUnitDir = "/etc/systemd/system"
+
+// hostCommand runs a host tool (systemctl, journalctl, k3s) and returns its
+// stdout. A tool that exits non-zero still returns what it printed:
+// `systemctl is-active` prints "inactive" and exits 3.
+func hostCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
+	return exec.CommandContext(ctx, name, args...).Output()
+}
+
+// hostServices are the long-running units a full install depends on, checked
+// when their unit file is present: k3s runs the cluster, felis-velocity is the
+// game proxy, felis-nano the single-binary host that runs without k3s.
+var hostServices = []string{"k3s.service", "felis-velocity.service", "felis-nano.service"}
+
+// cmdDoctor runs every check felis watchdog runs, with the settings
+// felis-watchdog.service gives it, plus what only the host shows (systemd
+// units that failed or stopped, timers that no longer fire, alerts that reach
+// no one), and prints them grouped by area with where to look next. It mails
+// nothing, pings no heartbeat and leaves the watchdog's state alone: it is
+// safe to run at any time, as often as wanted.
+func cmdDoctor(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("doctor", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis doctor: run as root (sudo felis doctor): the checks read root-only state under /etc/felis and /var/lib/felis")
+		return 1
+	}
+	host, _ := os.Hostname()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	return runDoctor(ctx, doctorEnv{unitDir: *unitDir, run: hostCommand, now: time.Now(), host: host}, stdout)
+}
+
+// doctorEnv is what one doctor run reads the host through.
+type doctorEnv struct {
+	unitDir string
+	run     func(ctx context.Context, name string, args ...string) ([]byte, error)
+	now     time.Time
+	host    string
+}
+
+// doctorAreas are the report's headings in order, by the area findingArea
+// puts a finding under.
+var doctorAreas = []struct{ key, title string }{
+	{key: "config", title: "configuration"},
+	{key: "cluster", title: "Kubernetes cluster"},
+	{key: "postgres", title: "PostgreSQL"},
+	{key: "proxy", title: "game proxy"},
+	{key: "db-backup", title: "database backups"},
+	{key: "offsite", title: "off-site copy"},
+	{key: "scan-db", title: "build scan database"},
+	{key: "disk", title: "disk space"},
+	{key: "memory", title: "memory"},
+	{key: "k3s-certs", title: "k3s certificates"},
+	{key: "host-address", title: "node address"},
+	{key: "clock", title: "clock"},
+	{key: "systemd", title: "systemd units and timers"},
+	{key: "alerts", title: "alerting"},
+}
+
+func runDoctor(ctx context.Context, env doctorEnv, stdout io.Writer) int {
+	unit := filepath.Join(env.unitDir, "felis-watchdog.service")
+	w, found, unitErr := watchdogUnitFlags(unit)
+	var report watchdog.Report
+	var notes []string
+	fmt.Fprintf(stdout, "felis doctor on %s at %s\n", env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
+	switch {
+	case unitErr != nil:
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "watchdog/unit", Severity: watchdog.Critical,
+			SummaryEN: fmt.Sprintf("cannot read the watchdog's settings: %v", unitErr),
+			Hint:      "rerun the installer (deploy/bootstrap.sh) to rewrite felis-watchdog.service",
+		})
+		fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
+	case !found:
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "watchdog/unit", Severity: watchdog.Critical,
+			SummaryEN: fmt.Sprintf("%s is not installed: nothing checks this host or mails anyone when it breaks", unit),
+			Hint:      "rerun the installer (deploy/bootstrap.sh), which installs felis-watchdog.timer",
+		})
+		fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
+	default:
+		fmt.Fprintf(stdout, "checks run as %s runs them (config %s)\n", unit, w.cfgPath)
+	}
+	fmt.Fprintln(stdout)
+
+	skip := map[string]string{}
+	cfg, cfgErr := config.Load(w.cfgPath)
+	if cfgErr != nil {
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "config", Severity: watchdog.Critical,
+			SummaryEN: cfgErr.Error(),
+			Hint:      "the installer writes it (deploy/bootstrap.sh); felis watchdog fails on every run until it loads",
+		})
+		// The host's units are read without it; whether alerts reach anyone
+		// is not known without its relay.
+		for _, a := range doctorAreas {
+			if a.key != "config" && a.key != "systemd" {
+				skip[a.key] = "the configuration did not load"
+			}
+		}
+	} else {
+		_, owners, ownersErr := watchdogProbes(ctx, w, cfg, env.now, &report)
+		report.Findings = append(report.Findings, alertReachFindings(cfg, owners, ownersErr)...)
+		if w.proxyAddr == "" {
+			skip["proxy"] = "no -proxy-addr"
+		}
+		if w.backupDir == "" {
+			skip["db-backup"] = "no -backup-dir"
+		}
+		if !cfg.Offsite.Enabled() {
+			skip["offsite"] = "not configured"
+		}
+		if !usesMirroredScanDB(cfg) {
+			skip["scan-db"] = "builds do not scan against the registry's copy"
+		}
+		if len(splitList(w.certDirs)) == 0 {
+			skip["k3s-certs"] = "no -k3s-cert-dirs"
+		}
+		if w.nodeIP == "" {
+			skip["host-address"] = "no -node-ip"
+		}
+	}
+	report.Findings = append(report.Findings, unitFindings(ctx, env)...)
+
+	switch url, err := readHeartbeatURL(w.heartbeatFile); {
+	case err != nil:
+		report.Findings = append(report.Findings, watchdog.Finding{
+			Key: "watchdog/heartbeat", Severity: watchdog.Warning,
+			SummaryEN: fmt.Sprintf("the heartbeat URL is unusable, so no run pings it: %v", err),
+			Hint:      "rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL set (docs/troubleshooting.md §14)",
+		})
+	case url == "":
+		notes = append(notes, "no heartbeat URL is set: a host that goes down entirely, or a watchdog that stops running, alerts no one. "+
+			"Rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL (docs/troubleshooting.md §14)")
+	}
+	if until := watchdog.QuietUntil(w.quietPath); env.now.Before(until) {
+		notes = append(notes, fmt.Sprintf("the watchdog mails nothing until %s (%s): the installer holds it while it restarts things on purpose, "+
+			"and a marker an installer killed mid-run left behind holds it until then",
+			until.UTC().Format("2006-01-02 15:04 UTC"), w.quietPath))
+	}
+	return printDoctorReport(stdout, report.Findings, skip, notes)
+}
+
+// printDoctorReport prints each area's findings under its heading, an area
+// with none as fine or, when skip says why, as not checked, then the notes
+// and the count. It returns the exit status: 1 when anything was found.
+func printDoctorReport(stdout io.Writer, findings []watchdog.Finding, skip map[string]string, notes []string) int {
+	byArea := map[string][]watchdog.Finding{}
+	for _, f := range findings {
+		a := findingArea(f.Key)
+		byArea[a] = append(byArea[a], f)
+	}
+	var critical, warning int
+	for _, a := range doctorAreas {
+		fs := byArea[a.key]
+		switch {
+		case len(fs) > 0:
+		case skip[a.key] != "":
+			fmt.Fprintf(stdout, "-  %s: not checked, %s\n", a.title, skip[a.key])
+			continue
+		default:
+			fmt.Fprintf(stdout, "✓  %s\n", a.title)
+			continue
+		}
+		mark := "!"
+		if slices.ContainsFunc(fs, func(f watchdog.Finding) bool { return f.Severity == watchdog.Critical }) {
+			mark = "✗"
+		}
+		fmt.Fprintf(stdout, "%s  %s\n", mark, a.title)
+		for _, f := range fs {
+			if f.Severity == watchdog.Critical {
+				critical++
+			} else {
+				warning++
+			}
+			fmt.Fprintf(stdout, "     %-8s %s: %s\n", f.Severity, f.Key, f.SummaryEN)
+			if f.Hint != "" {
+				fmt.Fprintf(stdout, "              → %s\n", f.Hint)
+			}
+		}
+	}
+	for _, n := range notes {
+		fmt.Fprintf(stdout, "\nnote: %s\n", n)
+	}
+	fmt.Fprintln(stdout)
+	if critical+warning == 0 {
+		fmt.Fprintln(stdout, "no problems found")
+		return 0
+	}
+	fmt.Fprintf(stdout, "%d problem(s): %d critical, %d warning(s)\n", critical+warning, critical, warning)
+	return 1
+}
+
+// findingArea is the report heading a finding key goes under.
+func findingArea(key string) string {
+	head, _, _ := strings.Cut(key, "/")
+	switch head {
+	case "kube-api", "deployment", "system-server", "server-failed", "job-failed", "reaper-stale", "node":
+		return "cluster"
+	case "db-backup", "db-backup-servers":
+		return "db-backup"
+	case "unit", "timer":
+		return "systemd"
+	case "watchdog":
+		return "alerts"
+	}
+	return head
+}
+
+// alertReachFindings is why the watchdog's alerts would reach no one, which
+// its own runs only log: no relay, or no owner with a verified address.
+func alertReachFindings(cfg *config.Config, owners []string, ownersErr error) []watchdog.Finding {
+	var out []watchdog.Finding
+	if cfg.SMTP.Host == "" {
+		out = append(out, watchdog.Finding{
+			Key: "alerts/relay", Severity: watchdog.Warning,
+			SummaryEN: "no [smtp] relay is configured: the watchdog logs its alerts to the journal and mails no one",
+			Hint:      "sudo felis setup, step SMTP",
+		})
+	}
+	if ownersErr == nil && len(owners) == 0 {
+		out = append(out, watchdog.Finding{
+			Key: "alerts/recipients", Severity: watchdog.Warning,
+			SummaryEN: "no owner account has a verified email: the watchdog's alerts reach no one",
+			Hint:      "an owner verifies an address in the panel's account settings",
+		})
+	}
+	return out
+}
+
+// unitFindings reports the installer's systemd units that failed, the
+// long-running ones that are not running, and timers that no longer fire.
+func unitFindings(ctx context.Context, env doctorEnv) []watchdog.Finding {
+	var out []watchdog.Finding
+	seen := map[string]bool{}
+	failed, err := env.run(ctx, "systemctl", "list-units", "--all", "--plain", "--no-legend", "--no-pager", "--state=failed", "felis-*", "k3s.service")
+	if err != nil && len(failed) == 0 {
+		return []watchdog.Finding{{
+			Key: "unit/systemctl", Severity: watchdog.Warning,
+			SummaryEN: fmt.Sprintf("systemctl list-units failed, so no unit was checked: %v", err),
+		}}
+	}
+	for _, line := range strings.Split(string(failed), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			continue
+		}
+		name := fields[0]
+		seen[name] = true
+		out = append(out, watchdog.Finding{
+			Key: "unit/" + name, Severity: watchdog.Critical,
+			SummaryEN: name + " failed",
+			Hint:      fmt.Sprintf("journalctl -u %s -n 100 --no-pager; once fixed, sudo systemctl reset-failed %s (a timer's job clears on its next good run)", name, name),
+		})
+	}
+	for _, name := range hostServices {
+		if seen[name] {
+			continue
+		}
+		if _, err := os.Stat(filepath.Join(env.unitDir, name)); err != nil {
+			continue
+		}
+		if state := unitActiveState(ctx, env, name); state != "active" {
+			out = append(out, watchdog.Finding{
+				Key: "unit/" + name, Severity: watchdog.Critical,
+				SummaryEN: fmt.Sprintf("%s is %s", name, state),
+				Hint:      fmt.Sprintf("sudo systemctl start %s; journalctl -u %s -n 100 --no-pager", name, name),
+			})
+		}
+	}
+	timers, _ := filepath.Glob(filepath.Join(env.unitDir, "felis-*.timer"))
+	for _, path := range timers {
+		name := filepath.Base(path)
+		if state := unitActiveState(ctx, env, name); state != "active" {
+			out = append(out, watchdog.Finding{
+				Key: "timer/" + name, Severity: watchdog.Warning,
+				SummaryEN: fmt.Sprintf("%s is %s: the job it starts no longer runs", name, state),
+				Hint:      fmt.Sprintf("sudo systemctl enable --now %s", name),
+			})
+		}
+	}
+	return out
+}
+
+// unitActiveState is what `systemctl is-active` says of unit.
+func unitActiveState(ctx context.Context, env doctorEnv, unit string) string {
+	out, err := env.run(ctx, "systemctl", "is-active", unit)
+	if state := strings.TrimSpace(string(out)); state != "" {
+		return state
+	}
+	return fmt.Sprintf("in an unknown state (systemctl is-active: %v)", err)
+}
+
+// watchdogUnitFlags reads the flags felis-watchdog.service runs felis
+// watchdog with. found is false when there is no such unit; w is then the
+// watchdog's defaults.
+func watchdogUnitFlags(path string) (w watchdogFlags, found bool, err error) {
+	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	w.register(fs)
+	raw, err := os.ReadFile(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return w, false, nil
+	}
+	if err != nil {
+		return w, false, err
+	}
+	for _, line := range strings.Split(string(raw), "\n") {
+		cmd, ok := strings.CutPrefix(strings.TrimSpace(line), "ExecStart=")
+		if !ok {
+			continue
+		}
+		fields := execArgs(cmd)
+		i := slices.Index(fields, "watchdog")
+		if i < 0 {
+			continue
+		}
+		if err := fs.Parse(fields[i+1:]); err != nil {
+			return w, true, fmt.Errorf("%s: %w", path, err)
+		}
+		return w, true, nil
+	}
+	return w, true, fmt.Errorf("%s runs no `felis watchdog`", path)
+}
+
+// execArgs splits an ExecStart= command line into its words. A word may be
+// quoted with " or ', as systemd allows, which is how an empty value is
+// written.
+func execArgs(s string) []string {
+	var out []string
+	var cur strings.Builder
+	inWord := false
+	var quote rune
+	for _, r := range s {
+		switch {
+		case quote != 0:
+			if r == quote {
+				quote = 0
+			} else {
+				cur.WriteRune(r)
+			}
+		case r == '"' || r == '\'':
+			quote, inWord = r, true
+		case r == ' ' || r == '\t':
+			if inWord {
+				out = append(out, cur.String())
+				cur.Reset()
+				inWord = false
+			}
+		default:
+			cur.WriteRune(r)
+			inWord = true
+		}
+	}
+	if inWord {
+		out = append(out, cur.String())
+	}
+	return out
+}

cmd/felis/doctor_test.go

@@ -0,0 +1,423 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/watchdog"
+)
+
+// bootstrapWatchdogExecStart is the ExecStart= line deploy/bootstrap.sh writes
+// into felis-watchdog.service, with its variables filled in as an install
+// fills them.
+func bootstrapWatchdogExecStart(t *testing.T, vars map[string]string) string {
+	t.Helper()
+	raw, err := os.ReadFile("../../deploy/bootstrap.sh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var line string
+	for _, l := range strings.Split(string(raw), "\n") {
+		if strings.HasPrefix(l, "ExecStart=${HOST_BIN} watchdog -config") {
+			line = l
+		}
+	}
+	if line == "" {
+		t.Fatal("deploy/bootstrap.sh writes no `ExecStart=${HOST_BIN} watchdog -config` line")
+	}
+	line = strings.ReplaceAll(line, "${NODE_IP:+ -node-ip ${NODE_IP}}", " -node-ip "+vars["NODE_IP"])
+	line = regexp.MustCompile(`\$\{([A-Za-z_]+)\}`).ReplaceAllStringFunc(line, func(m string) string {
+		v, ok := vars[m[2:len(m)-1]]
+		if !ok {
+			t.Fatalf("bootstrap's watchdog ExecStart= uses %s, which this test does not fill in", m)
+		}
+		return v
+	})
+	return line
+}
+
+// felis doctor reads the watchdog's settings from the unit the installer
+// writes, so it checks the paths the timer's runs check.
+func TestWatchdogUnitFlagsReadsTheInstallersUnit(t *testing.T) {
+	dir := t.TempDir()
+	exec := bootstrapWatchdogExecStart(t, map[string]string{
+		"HOST_BIN": "/usr/local/bin/felis", "STATE_DIR": "/srv/felis-etc", "WATCHDOG_STATE": "/srv/watchdog/state.json",
+		"WATCHDOG_QUIET_FILE": "/srv/quiet-until", "FELIS_DB_BACKUP_DIR": "/srv/db-backups", "FELIS_GAME_PORT": "25577",
+		"disks": "/,/srv/data", "NODE_IP": "10.0.0.5",
+	})
+	unit := filepath.Join(dir, "felis-watchdog.service")
+	writeTestFile(t, unit, "[Unit]\nDescription=Felis watchdog\n\n[Service]\nType=oneshot\n"+exec+"\nTimeoutStartSec=3min\n", 0o644)
+
+	w, found, err := watchdogUnitFlags(unit)
+	if err != nil || !found {
+		t.Fatalf("found %v, err %v", found, err)
+	}
+	got := []string{w.cfgPath, w.statePath, w.quietPath, w.backupDir, w.proxyAddr, w.diskPaths, w.nodeIP, w.heartbeatFile, w.controlNS}
+	want := []string{"/srv/felis-etc/felis.host.toml", "/srv/watchdog/state.json", "/srv/quiet-until", "/srv/db-backups", "127.0.0.1:25577", "/,/srv/data", "10.0.0.5", defaultHeartbeatFile, "felis"}
+	if strings.Join(got, "|") != strings.Join(want, "|") {
+		t.Errorf("read\n %q\nwant\n %q", got, want)
+	}
+
+	w, found, err = watchdogUnitFlags(filepath.Join(dir, "missing.service"))
+	if err != nil || found || w.cfgPath != "/etc/felis/felis.toml" || w.backupDir != "/var/lib/felis/db-backups" {
+		t.Errorf("no unit: found %v, err %v, config %q, backups %q; want the watchdog's defaults", found, err, w.cfgPath, w.backupDir)
+	}
+
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis version\n", 0o644)
+	if _, found, err := watchdogUnitFlags(unit); !found || err == nil || !strings.Contains(err.Error(), "runs no `felis watchdog`") {
+		t.Errorf("a unit that runs something else: found %v, err %v", found, err)
+	}
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
+	if _, _, err := watchdogUnitFlags(unit); err == nil {
+		t.Error("a flag this binary does not know was accepted")
+	}
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -backup-dir \"\"\t-proxy-addr '127.0.0.1:1' -disk-paths \"/a b\"\n", 0o644)
+	if w, _, err := watchdogUnitFlags(unit); err != nil || w.backupDir != "" || w.proxyAddr != "127.0.0.1:1" || w.diskPaths != "/a b" {
+		t.Errorf("quoted words: backups %q, proxy %q, disks %q, err %v", w.backupDir, w.proxyAddr, w.diskPaths, err)
+	}
+}
+
+func TestFindingArea(t *testing.T) {
+	for key, want := range map[string]string{
+		"kube-api":                           "cluster",
+		"deployment/felis-api":               "cluster",
+		"system-server/lobby":                "cluster",
+		"server-failed/survival":             "cluster",
+		"job-failed/reaper-123":              "cluster",
+		"reaper-stale":                       "cluster",
+		"node/felis-1/NotReady":              "cluster",
+		"postgres":                           "postgres",
+		"proxy":                              "proxy",
+		"db-backup":                          "db-backup",
+		"db-backup-servers":                  "db-backup",
+		"offsite":                            "offsite",
+		"scan-db":                            "scan-db",
+		"disk//var/lib/felis":                "disk",
+		"memory":                             "memory",
+		"k3s-certs":                          "k3s-certs",
+		"host-address":                       "host-address",
+		"clock":                              "clock",
+		"config":                             "config",
+		"unit/felis-offsite.service":         "systemd",
+		"timer/felis-db-backup.timer":        "systemd",
+		"watchdog/unit":                      "alerts",
+		"watchdog/heartbeat":                 "alerts",
+		"alerts/relay":                       "alerts",
+		"alerts/recipients":                  "alerts",
+		"something-a-later-release-reported": "something-a-later-release-reported",
+	} {
+		if got := findingArea(key); got != want {
+			t.Errorf("findingArea(%q) = %q, want %q", key, got, want)
+		}
+	}
+}
+
+func TestAlertReachFindings(t *testing.T) {
+	relay := &config.Config{SMTP: config.SMTPConfig{Host: "smtp.example.com"}}
+	keys := func(fs []watchdog.Finding) string {
+		var k []string
+		for _, f := range fs {
+			k = append(k, f.Key)
+		}
+		return strings.Join(k, ",")
+	}
+	for _, tc := range []struct {
+		what      string
+		cfg       *config.Config
+		owners    []string
+		ownersErr error
+		want      string
+	}{
+		{"a relay and an owner", relay, []string{"[email protected]"}, nil, ""},
+		{"no relay", &config.Config{}, []string{"[email protected]"}, nil, "alerts/relay"},
+		{"no owner with an address", relay, nil, nil, "alerts/recipients"},
+		{"PostgreSQL down: its own finding says so", relay, nil, errors.New("refused"), ""},
+		{"neither", &config.Config{}, nil, nil, "alerts/relay,alerts/recipients"},
+	} {
+		if got := keys(alertReachFindings(tc.cfg, tc.owners, tc.ownersErr)); got != tc.want {
+			t.Errorf("%s: %q, want %q", tc.what, got, tc.want)
+		}
+	}
+}
+
+// fakeSystemctl answers list-units with failed and is-active from states;
+// a unit missing from states is "inactive", as systemctl says, and one whose
+// state is "" gets no answer.
+func fakeSystemctl(failed string, states map[string]string) func(ctx context.Context, name string, args ...string) ([]byte, error) {
+	return func(_ context.Context, name string, args ...string) ([]byte, error) {
+		if name != "systemctl" || len(args) == 0 {
+			return nil, errors.New("unexpected command " + name)
+		}
+		switch args[0] {
+		case "list-units":
+			return []byte(failed), nil
+		case "is-active":
+			if s, ok := states[args[1]]; ok && s == "" {
+				return nil, errors.New("signal: killed")
+			} else if ok {
+				return []byte(s + "\n"), nil
+			}
+			return []byte("inactive\n"), errors.New("exit status 3")
+		}
+		return nil, errors.New("unexpected systemctl " + args[0])
+	}
+}
+
+func TestUnitFindings(t *testing.T) {
+	dir := t.TempDir()
+	for _, f := range []string{"k3s.service", "felis-velocity.service", "felis-db-backup.timer", "felis-offsite.timer", "felis-offsite.service"} {
+		writeTestFile(t, filepath.Join(dir, f), "[Unit]\n", 0o644)
+	}
+	env := doctorEnv{unitDir: dir, run: fakeSystemctl(
+		"felis-offsite.service loaded failed failed Felis off-site copy\nfelis-velocity.service loaded failed failed Velocity\n",
+		map[string]string{"k3s.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "inactive"},
+	)}
+	var got []string
+	for _, f := range unitFindings(context.Background(), env) {
+		got = append(got, string(f.Severity)+" "+f.Key+": "+f.SummaryEN)
+	}
+	want := []string{
+		"critical unit/felis-offsite.service: felis-offsite.service failed",
+		"critical unit/felis-velocity.service: felis-velocity.service failed",
+		"warning timer/felis-offsite.timer: felis-offsite.timer is inactive: the job it starts no longer runs",
+	}
+	if strings.Join(got, "\n") != strings.Join(want, "\n") {
+		t.Errorf("findings:\n%s\nwant (felis-nano.service has no unit file here, and a failed unit is reported once):\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
+	}
+
+	env.run = fakeSystemctl("", map[string]string{"k3s.service": "activating", "felis-velocity.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "active"})
+	got = nil
+	for _, f := range unitFindings(context.Background(), env) {
+		got = append(got, f.Key+": "+f.SummaryEN)
+	}
+	if strings.Join(got, "\n") != "unit/k3s.service: k3s.service is activating" {
+		t.Errorf("findings %q, want only k3s.service, which is not active", got)
+	}
+
+	env.run = fakeSystemctl("", map[string]string{"k3s.service": "active", "felis-velocity.service": "active", "felis-db-backup.timer": "", "felis-offsite.timer": "active"})
+	got = nil
+	for _, f := range unitFindings(context.Background(), env) {
+		got = append(got, f.Key+": "+f.SummaryEN)
+	}
+	if want := "timer/felis-db-backup.timer: felis-db-backup.timer is in an unknown state (systemctl is-active: signal: killed): the job it starts no longer runs"; strings.Join(got, "\n") != want {
+		t.Errorf("findings %q, want %q", got, want)
+	}
+
+	env.run = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("no systemctl") }
+	if fs := unitFindings(context.Background(), env); len(fs) != 1 || fs[0].Key != "unit/systemctl" || fs[0].Severity != watchdog.Warning {
+		t.Errorf("without systemctl: %+v, want the one unit/systemctl warning", fs)
+	}
+}
+
+// quoteArgs writes args as an ExecStart= line does, each word in quotes so an
+// empty one survives.
+func quoteArgs(args []string) string {
+	q := make([]string, len(args))
+	for i, a := range args {
+		q[i] = `"` + a + `"`
+	}
+	return strings.Join(q, " ")
+}
+
+// doctorHost is a host with the installer's watchdog unit, whose API server
+// and PostgreSQL are down.
+func doctorHost(t *testing.T, cfg string, extraFlags string) (env doctorEnv, h *watchdogHost) {
+	t.Helper()
+	h = newWatchdogHost(t, cfg, nil)
+	unitDir := t.TempDir()
+	writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"),
+		"[Service]\nType=oneshot\nExecStart=/usr/local/bin/felis watchdog "+quoteArgs(h.args)+
+			// Off this machine's disk, whose free space is not the test's.
+			` -disk-paths "/nonexistent-felis-doctor-test"`+extraFlags+"\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-offsite.timer"), "[Unit]\n", 0o644)
+	return doctorEnv{
+		unitDir: unitDir, now: time.Now(), host: "felis-test",
+		run: fakeSystemctl("felis-db-backup.service loaded failed failed Felis database backup\n",
+			map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"}),
+	}, h
+}
+
+func TestDoctorReportsByArea(t *testing.T) {
+	env, _ := doctorHost(t, testWatchdogConfig, "")
+	var out bytes.Buffer
+	code := runDoctor(context.Background(), env, &out)
+	got := out.String()
+	if code != 1 {
+		t.Errorf("exit %d, want 1 with problems found", code)
+	}
+	for _, want := range []string{
+		"felis doctor on felis-test at ",
+		"checks run as " + filepath.Join(env.unitDir, "felis-watchdog.service") + " runs them (config ",
+		"✓  configuration\n",
+		"✗  Kubernetes cluster\n     critical kube-api: ",
+		"✗  PostgreSQL\n     critical postgres: ",
+		"-  game proxy: not checked, no -proxy-addr\n",
+		"-  database backups: not checked, no -backup-dir\n",
+		"-  off-site copy: not checked, not configured\n",
+		"-  build scan database: not checked, builds do not scan against the registry's copy\n",
+		"-  k3s certificates: not checked, no -k3s-cert-dirs\n",
+		"-  node address: not checked, no -node-ip\n",
+		"✗  systemd units and timers\n     critical unit/felis-db-backup.service: felis-db-backup.service failed\n" +
+			"              → journalctl -u felis-db-backup.service -n 100 --no-pager; once fixed, sudo systemctl reset-failed felis-db-backup.service",
+		"!  alerting\n     warning  alerts/relay: no [smtp] relay is configured",
+		"\n4 problem(s): 3 critical, 1 warning(s)\n",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("report lacks %q:\n%s", want, got)
+		}
+	}
+	if strings.Contains(got, "alerts/recipients") {
+		t.Errorf("reported no recipients although PostgreSQL, which names them, is down:\n%s", got)
+	}
+	if strings.Contains(got, "note: no heartbeat URL") {
+		t.Errorf("the host has a heartbeat URL:\n%s", got)
+	}
+}
+
+// A doctor run is only a look: whatever is due to be mailed stays due, no
+// heartbeat is pinged, and the watchdog's state is left as it was.
+func TestDoctorMailsPingsAndSavesNothing(t *testing.T) {
+	cfg := testWatchdogConfig + "[smtp]\nhost = \"smtp.example.com\"\nport = 587\nfrom = \"[email protected]\"\n"
+	env, h := doctorHost(t, cfg, "")
+	if err := watchdog.SaveState(h.statePath, duePostgres("cached-pw")); err != nil {
+		t.Fatal(err)
+	}
+	before, err := os.ReadFile(h.statePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	watchdogSender = h.rec.sender
+	defer func() { watchdogSender = smtpSender }()
+	var out bytes.Buffer
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "critical postgres: ") {
+		t.Fatalf("the due PostgreSQL alert was not seen:\n%s", out.String())
+	}
+	if len(h.rec.sent) != 0 || len(h.rec.relays) != 0 {
+		t.Errorf("mailed %v", h.rec.sent)
+	}
+	if n := len(h.pings.pings); n != 0 {
+		t.Errorf("pinged the heartbeat %d times", n)
+	}
+	after, err := os.ReadFile(h.statePath)
+	if err != nil || !bytes.Equal(before, after) {
+		t.Errorf("the watchdog state changed (err %v)", err)
+	}
+	if _, err := os.Stat(h.fallbackPath); !errors.Is(err, os.ErrNotExist) {
+		t.Errorf("a fallback state was written: %v", err)
+	}
+}
+
+func TestDoctorNotes(t *testing.T) {
+	env, h := doctorHost(t, testWatchdogConfig, "")
+	if err := os.Remove(filepath.Join(h.dir, "watchdog-heartbeat-url")); err != nil {
+		t.Fatal(err)
+	}
+	until := env.now.Add(10 * time.Minute).Unix()
+	writeTestFile(t, filepath.Join(h.dir, "quiet"), strconv.FormatInt(until, 10)+"\n", 0o644)
+	var out bytes.Buffer
+	runDoctor(context.Background(), env, &out)
+	for _, want := range []string{
+		"\nnote: no heartbeat URL is set: ",
+		"\nnote: the watchdog mails nothing until " + time.Unix(until, 0).UTC().Format("2006-01-02 15:04 UTC") + " (" + filepath.Join(h.dir, "quiet") + ")",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("report lacks %q:\n%s", want, out.String())
+		}
+	}
+
+	writeTestFile(t, filepath.Join(h.dir, "watchdog-heartbeat-url"), "not a url\n", 0o600)
+	out.Reset()
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "warning  watchdog/heartbeat: the heartbeat URL is unusable") {
+		t.Errorf("an unusable heartbeat URL is not reported:\n%s", out.String())
+	}
+}
+
+// Without the watchdog's unit the doctor still checks, with the watchdog's
+// defaults, and says the host has no watchdog; a configuration that does not
+// load leaves the checks that need it unchecked and the host's own checks on.
+func TestDoctorWithoutUnitOrConfig(t *testing.T) {
+	env, _ := doctorHost(t, testWatchdogConfig, "")
+	if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
+		t.Fatal(err)
+	}
+	writeTestFile(t, filepath.Join(env.unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+filepath.Join(t.TempDir(), "absent.toml")+"\n", 0o644)
+	env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"})
+	var out bytes.Buffer
+	if code := runDoctor(context.Background(), env, &out); code != 1 {
+		t.Errorf("exit %d, want 1", code)
+	}
+	for _, want := range []string{
+		"✗  configuration\n     critical config: ",
+		"-  Kubernetes cluster: not checked, the configuration did not load\n",
+		"-  PostgreSQL: not checked, the configuration did not load\n",
+		"-  disk space: not checked, the configuration did not load\n",
+		"✓  systemd units and timers\n",
+		"-  alerting: not checked, the configuration did not load\n",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("report lacks %q:\n%s", want, out.String())
+		}
+	}
+
+	if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
+		t.Fatal(err)
+	}
+	out.Reset()
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "critical watchdog/unit: "+filepath.Join(env.unitDir, "felis-watchdog.service")+" is not installed") ||
+		!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
+		t.Errorf("a host without the watchdog's unit:\n%s", out.String())
+	}
+
+	unit := filepath.Join(env.unitDir, "felis-watchdog.service")
+	writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
+	out.Reset()
+	runDoctor(context.Background(), env, &out)
+	if !strings.Contains(out.String(), "critical watchdog/unit: cannot read the watchdog's settings: "+unit+": flag provided but not defined: -no-such-flag\n") ||
+		!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
+		t.Errorf("a watchdog unit this binary cannot read:\n%s", out.String())
+	}
+}
+
+func TestPrintDoctorReport(t *testing.T) {
+	var out bytes.Buffer
+	if code := printDoctorReport(&out, nil, map[string]string{"proxy": "no -proxy-addr"}, nil); code != 0 {
+		t.Errorf("exit %d with nothing found, want 0", code)
+	}
+	want := "✓  configuration\n✓  Kubernetes cluster\n✓  PostgreSQL\n-  game proxy: not checked, no -proxy-addr\n✓  database backups\n✓  off-site copy\n" +
+		"✓  build scan database\n✓  disk space\n✓  memory\n✓  k3s certificates\n✓  node address\n✓  clock\n✓  systemd units and timers\n✓  alerting\n" +
+		"\nno problems found\n"
+	if out.String() != want {
+		t.Errorf("report:\n%s\nwant:\n%s", out.String(), want)
+	}
+
+	out.Reset()
+	code := printDoctorReport(&out, []watchdog.Finding{
+		{Key: "unit/systemctl", Severity: watchdog.Warning, SummaryEN: "systemctl list-units failed"},
+		{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "PostgreSQL is down", Hint: "kubectl -n felis get pods"},
+	}, map[string]string{"postgres": "a skip loses to what was found"}, []string{"a note"})
+	if code != 1 {
+		t.Errorf("exit %d with problems, want 1", code)
+	}
+	for _, want := range []string{
+		"✗  PostgreSQL\n     critical postgres: PostgreSQL is down\n              → kubectl -n felis get pods\n✓  game proxy\n",
+		"!  systemd units and timers\n     warning  unit/systemctl: systemctl list-units failed\n✓  alerting\n",
+		"✓  alerting\n\nnote: a note\n\n2 problem(s): 1 critical, 1 warning(s)\n",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("report lacks %q:\n%s", want, out.String())
+		}
+	}
+}

cmd/felis/domain_test.go

@@ -0,0 +1,892 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// installerTOML is felis.toml as deploy/bootstrap.sh write_felis_toml renders it,
+// comments included: the domain move has to leave all of it but three values alone.
+func installerTOML(root, dbHost string) string {
+	return `# Generated by deploy/bootstrap.sh; rerun the installer to regenerate. Hand edits are
+# overwritten, except [smtp], [[auth_source]], [offsite], and the operator-owned
+# [registry] / [archive] overrides, which carry forward.
+[server]
+listen = "0.0.0.0:8080"
+root_domain = "` + root + `"
+
+[database]
+url = "postgres://felis:pw@` + dbHost + `:5432/felis?sslmode=disable"
+
+[k8s]
+namespace = "minecraft"
+egress_mode = "nodeport"
+
+[velocity]
+# The two always-on system servers that felis setup provisions.
+login_image = "felis/limbo:1"
+lobby_image = "felis/lobby:1"
+game_port = 25565
+
+[registry]
+url = "registry.felis.svc:5000"
+build_namespace = "felis-build"
+
+[archive]
+store = "tarLocal"
+local_path = "/var/lib/felis/archives"
+
+[auth]
+admin_hostname = "op.console.` + root + `"
+panel_hostname = "console.` + root + `"
+access_jwt_aud = "aud123"
+
+# Third-party Yggdrasil sources federated by the hasJoined multiplexer.
+[[auth_source]]
+tag = "littleskin"
+prefix = "LS"
+url = "https://littleskin.cn/api/yggdrasil/sessionserver/session/minecraft/hasJoined"
+`
+}
+
+const linkPropsBody = `# Generated by deploy/bootstrap.sh — do not edit by hand; rerun the installer.
+api-base-url=http://10.43.0.9:8081
+service-token=TOKEN-NOT-TO-TOUCH
+root-domain=old.example
+panel-hostname=console.old.example
+admin-hostname=op.console.old.example
+login-server=login
+lobby-server=lobby
+`
+
+var oldNames = domainNames{root: "old.example", panel: "console.old.example", admin: "op.console.old.example"}
+var newNames = domainNames{root: "new.example", panel: "console.new.example", admin: "op.console.new.example"}
+
+// domainRig models the host: the files, the cluster, and a felis-api, proxy and
+// operator that pick up config the way the real ones do — the api serves what it
+// read at its last restart, the proxy runs since its last restart, and the login
+// pod carries the env its MinecraftServer had when it was last rolled.
+type domainRig struct {
+	h      domainHost
+	cl     client.Client
+	out    *bytes.Buffer
+	dir    string
+	events []string
+
+	served      domainNames
+	servedCert  *x509.Certificate
+	proxySince  time.Time
+	proxyLoaded bool
+	unresolved  map[string]bool
+	// The fake operator: the CR env the login pod was last rolled to, and how
+	// many looks at the pod since the CR moved on.
+	rolledTo string
+	pending  int
+}
+
+func (rig *domainRig) path(name string) string { return filepath.Join(rig.dir, name) }
+
+func newDomainRig(t *testing.T) *domainRig {
+	t.Helper()
+	rig := &domainRig{out: &bytes.Buffer{}, dir: t.TempDir(), proxyLoaded: true, unresolved: map[string]bool{}}
+	writeTestFile(t, rig.path("felis.host.toml"), installerTOML("old.example", "127.0.0.1"), 0o600)
+	writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
+	if err := os.Symlink(rig.path("felis.host.toml"), rig.path("felis.toml")); err != nil {
+		t.Fatal(err)
+	}
+	certPEM, keyPEM, err := issuePanelCert(oldNames, []net.IP{net.ParseIP("10.211.55.6")}, time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
+	writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
+	// The proxy started before its config was last written, which is how it
+	// stands after an install.
+	rig.proxySince = time.Now().Add(-time.Hour)
+
+	pod := []byte(installerTOML("old.example", "10.211.55.6"))
+	login, err := loginSystemServer("felis/limbo:1", "minecraft", platform.InternalAPIBaseURL("felis"), oldNames.root, oldNames.panel)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lobby, err := lobbySystemServer("felis/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatal(err)
+	}
+	loginPod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"},
+		Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "minecraft", Env: podEnv(login.Spec.Env)}}},
+		Status:     corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}},
+	}
+	rig.rolledTo = envKey(login.Spec.Env)
+	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
+		Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+			if key.Name == naming.SystemLoginServer+"-0" {
+				rig.operatorTick(t, c)
+			}
+			return c.Get(ctx, key, obj, opts...)
+		},
+	}).WithObjects(
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.ConfigSecretName},
+			Data: map[string][]byte{platform.ConfigSecretKey: pod}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: platform.ConfigSecretName},
+			Data: map[string][]byte{platform.ConfigSecretKey: pod}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.APITLSSecretName},
+			Type: corev1.SecretTypeTLS, Data: map[string][]byte{corev1.TLSCertKey: certPEM, corev1.TLSPrivateKeyKey: keyPEM}},
+		login, lobby, loginPod,
+	).Build()
+	rig.served = oldNames
+	rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
+
+	rig.h = domainHost{
+		paths: domainPaths{
+			hostTOML: rig.path("felis.host.toml"), podTOML: rig.path("felis.pod.toml"), defaultTOML: rig.path("felis.toml"),
+			cert: rig.path("panel-tls.crt"), key: rig.path("panel-tls.key"),
+			linkProps: rig.path("felis-link.properties"), tunnelConfig: rig.path("cloudflared.yml"),
+		},
+		cl:        rig.cl,
+		controlNS: "felis",
+		rollAPI: func(ctx context.Context) error {
+			rig.events = append(rig.events, "roll-api")
+			rig.restartAPI(t)
+			return nil
+		},
+		restartUnit: func(_ context.Context, unit string) error {
+			rig.events = append(rig.events, "restart "+unit)
+			rig.proxySince = time.Now().Add(time.Second)
+			return nil
+		},
+		unitState: func(context.Context, string) (unitStatus, error) {
+			return unitStatus{loaded: rig.proxyLoaded, active: rig.proxyLoaded, since: rig.proxySince}, nil
+		},
+		liveAPI: func(context.Context, string) (liveAPIView, error) {
+			return liveAPIView{names: rig.served, cert: rig.servedCert}, nil
+		},
+		lookupHost: func(_ context.Context, host string) ([]string, error) {
+			if rig.unresolved[host] {
+				return nil, errors.New("no such host")
+			}
+			return []string{"10.211.55.6"}, nil
+		},
+		passkeys:  func(context.Context) (int, int, error) { return 3, 2, nil },
+		now:       time.Now,
+		out:       rig.out,
+		loginWait: 50 * time.Millisecond,
+		pollEvery: time.Millisecond,
+	}
+	return rig
+}
+
+func podEnv(env []v1alpha1.EnvVar) []corev1.EnvVar {
+	out := make([]corev1.EnvVar, len(env))
+	for i, e := range env {
+		out[i] = corev1.EnvVar{Name: e.Name, Value: e.Value}
+	}
+	return out
+}
+
+// restartAPI makes the fake felis-api load the config and certificate its
+// Secrets hold now.
+func (rig *domainRig) restartAPI(t *testing.T) {
+	t.Helper()
+	var cfg, tlsSec corev1.Secret
+	ctx := context.Background()
+	if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.ConfigSecretName}, &cfg); err != nil {
+		t.Fatal(err)
+	}
+	if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.APITLSSecretName}, &tlsSec); err != nil {
+		t.Fatal(err)
+	}
+	names, err := tomlDomainNames(cfg.Data[platform.ConfigSecretKey])
+	if err != nil {
+		t.Fatal(err)
+	}
+	rig.served = names
+	rig.servedCert, err = x509.ParseCertificate(mustCertDER(tlsSec.Data[corev1.TLSCertKey]))
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// rollLoginPod is the operator restarting the login pod onto its CR's env.
+// operatorTick is the operator as the login pod is watched: once the CR's env
+// changes it takes operatorLag looks at the pod before the restarted pod
+// carries the new env, the way a real rollout lags the CR.
+func (rig *domainRig) operatorTick(t *testing.T, c client.Client) {
+	t.Helper()
+	ctx := context.Background()
+	var ms v1alpha1.MinecraftServer
+	if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if envKey(ms.Spec.Env) == rig.rolledTo {
+		return
+	}
+	if rig.pending++; rig.pending < operatorLag {
+		return
+	}
+	var pod corev1.Pod
+	if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"}, &pod); err != nil {
+		t.Fatal(err)
+	}
+	pod.Spec.Containers[0].Env = podEnv(ms.Spec.Env)
+	if err := c.Update(ctx, &pod); err != nil {
+		t.Fatal(err)
+	}
+	rig.rolledTo, rig.pending = envKey(ms.Spec.Env), 0
+}
+
+const operatorLag = 3
+
+func envKey(env []v1alpha1.EnvVar) string {
+	var b strings.Builder
+	for _, e := range env {
+		b.WriteString(e.Name + "=" + e.Value + "\n")
+	}
+	return b.String()
+}
+
+func (rig *domainRig) read(t *testing.T, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(rig.path(name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
+func (rig *domainRig) secret(t *testing.T, ns, name string) map[string][]byte {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		t.Fatal(err)
+	}
+	return s.Data
+}
+
+func (rig *domainRig) crEnv(t *testing.T, name string) map[string]string {
+	t.Helper()
+	var ms v1alpha1.MinecraftServer
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	env := map[string]string{}
+	for _, e := range ms.Spec.Env {
+		env[e.Name] = e.Value
+	}
+	return env
+}
+
+// snapshot is every byte `set` may touch, for proving a refused or dry run
+// touched none of it.
+func (rig *domainRig) snapshot(t *testing.T) string {
+	t.Helper()
+	var b strings.Builder
+	entries, _ := os.ReadDir(rig.dir)
+	for _, e := range entries {
+		b.WriteString(e.Name() + "\n" + rig.read(t, e.Name()) + "\n")
+	}
+	var lines []string
+	for _, s := range []struct{ ns, name string }{{"felis", platform.ConfigSecretName}, {"minecraft", platform.ConfigSecretName}, {"felis", platform.APITLSSecretName}} {
+		for k, v := range rig.secret(t, s.ns, s.name) {
+			lines = append(lines, s.ns+"/"+s.name+"/"+k+"\n"+string(v))
+		}
+	}
+	for k, v := range rig.crEnv(t, naming.SystemLoginServer) {
+		lines = append(lines, "env "+k+"="+v)
+	}
+	sort.Strings(lines)
+	b.WriteString(strings.Join(lines, "\n"))
+	return b.String()
+}
+
+func TestNormalizeRootDomain(t *testing.T) {
+	for in, want := range map[string]string{
+		"Example.COM.":          "example.com",
+		"  mc.example.org ":     "mc.example.org",
+		"10.211.55.6.nip.io":    "10.211.55.6.nip.io",
+		"xn--bcher-kva.example": "xn--bcher-kva.example",
+	} {
+		got, err := normalizeRootDomain(in)
+		if err != nil || got != want {
+			t.Errorf("normalizeRootDomain(%q) = %q, %v; want %q", in, got, err, want)
+		}
+	}
+	for _, in := range []string{"", "https://example.com", "example.com:443", "example.com/x", "10.0.0.1", "::1",
+		"localhost", "a_b.example", "-a.example", "a-.example", strings.Repeat("a", 64) + ".example",
+		strings.Repeat("abcdefghi.", 25) + "example"} {
+		if got, err := normalizeRootDomain(in); err == nil {
+			t.Errorf("normalizeRootDomain(%q) = %q, want an error", in, got)
+		}
+	}
+}
+
+func TestPlanDomainChangeMovesDefaultsAndKeepsHandSetNames(t *testing.T) {
+	p := planDomainChange(oldNames, "new.example")
+	if p.to != newNames || p.customPanel || p.customAdmin {
+		t.Fatalf("defaults: %+v", p)
+	}
+	p = planDomainChange(domainNames{root: "old.example", panel: "play.corp.net", admin: "op.console.old.example"}, "new.example")
+	if p.to.panel != "play.corp.net" || !p.customPanel || p.to.admin != "op.console.new.example" || p.customAdmin {
+		t.Fatalf("hand-set panel: %+v", p)
+	}
+	p = planDomainChange(domainNames{root: "old.example", panel: "console.old.example", admin: "admin.corp.net"}, "new.example")
+	if p.to.admin != "admin.corp.net" || !p.customAdmin || p.to.panel != "console.new.example" {
+		t.Fatalf("hand-set admin: %+v", p)
+	}
+}
+
+func TestEditTOMLStringsChangesOnlyTheDomainLines(t *testing.T) {
+	orig := installerTOML("old.example", "127.0.0.1")
+	out, err := editTOMLStrings([]byte(orig), domainTOMLEdits(newNames))
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, b := strings.Split(orig, "\n"), strings.Split(string(out), "\n")
+	if len(a) != len(b) {
+		t.Fatalf("line count %d → %d:\n%s", len(a), len(b), out)
+	}
+	changed := map[string]string{}
+	for i := range a {
+		if a[i] != b[i] {
+			changed[a[i]] = b[i]
+		}
+	}
+	want := map[string]string{
+		`root_domain = "old.example"`:               `root_domain = "new.example"`,
+		`admin_hostname = "op.console.old.example"`: `admin_hostname = "op.console.new.example"`,
+		`panel_hostname = "console.old.example"`:    `panel_hostname = "console.new.example"`,
+	}
+	if len(changed) != len(want) {
+		t.Fatalf("changed lines %v, want %v", changed, want)
+	}
+	for k, v := range want {
+		if changed[k] != v {
+			t.Errorf("%q → %q, want %q", k, changed[k], v)
+		}
+	}
+}
+
+func TestEditTOMLStringsAddsMissingKeysInTheirTable(t *testing.T) {
+	in := "[server]\nroot_domain = \"old.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\n\n[smtp]\nhost = \"relay\"\n"
+	out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "[server]\nroot_domain = \"new.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\npanel_hostname = \"console.new.example\"\nadmin_hostname = \"op.console.new.example\"\n\n[smtp]\nhost = \"relay\"\n"
+	if string(out) != want {
+		t.Fatalf("got:\n%s\nwant:\n%s", out, want)
+	}
+
+	out, err = editTOMLStrings([]byte("[server]\nroot_domain = \"old.example\"\n\n[[auth_source]]\ntag = \"ls\"\n"), domainTOMLEdits(newNames))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := tomlDomainNames(out)
+	if err != nil || got != newNames || !strings.Contains(string(out), "[[auth_source]]\ntag = \"ls\"\n") {
+		t.Fatalf("no [auth] table: %v %+v\n%s", err, got, out)
+	}
+}
+
+func TestEditTOMLStringsRefusesWhatItCannotEditExactly(t *testing.T) {
+	for name, in := range map[string]string{
+		"multi-line value": "[server]\nroot_domain = \"\"\"\nold.example\"\"\"\n[auth]\n",
+		// The key's line sits inside another value; the real key is absent.
+		"key inside a string": "[server]\nmotd = \"\"\"\nroot_domain = \"old.example\"\n\"\"\"\n[auth]\n",
+		"quoted header":       "[server]\nroot_domain = \"old.example\"\n[\"auth\"]\npanel_hostname = \"console.old.example\"\n",
+		"dotted key":          "server.root_domain = \"old.example\"\n",
+		"inline table":        "server = { root_domain = \"old.example\" }\n",
+	} {
+		if out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames)); err == nil {
+			t.Errorf("%s: edited instead of refusing:\n%s", name, out)
+		}
+	}
+}
+
+// caSignedCert is an operator's certificate from their own CA; it names
+// localhost too, so only the issuer tells it apart from the installer's.
+func caSignedCert(t *testing.T, hosts ...string) (certPEM, keyPEM []byte) {
+	t.Helper()
+	caKey, _ := rsa.GenerateKey(rand.Reader, 2048)
+	ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Corp CA"}, IsCA: true,
+		BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
+	caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	caCert, _ := x509.ParseCertificate(caDER)
+	key, _ := rsa.GenerateKey(rand.Reader, 2048)
+	leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: hosts[0]},
+		DNSNames: append(hosts, "localhost"), IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
+		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
+	der, err := x509.CreateCertificate(rand.Reader, leaf, caCert, &key.PublicKey, caKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pk, _ := x509.MarshalPKCS8PrivateKey(key)
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pk})
+}
+
+func TestFelisIssuedCert(t *testing.T) {
+	mine, _, err := issuePanelCert(oldNames, nil, time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	c, _ := x509.ParseCertificate(mustCertDER(mine))
+	if !felisIssuedCert(c) {
+		t.Error("the installer's kind of certificate is not recognised as Felis-issued")
+	}
+	theirs, _ := caSignedCert(t, "console.old.example")
+	c, _ = x509.ParseCertificate(mustCertDER(theirs))
+	if felisIssuedCert(c) {
+		t.Error("a CA-signed certificate is taken for Felis-issued")
+	}
+
+	// Self-signed but without one of the installer's localhost names: someone
+	// else's.
+	key, _ := rsa.GenerateKey(rand.Reader, 2048)
+	for name, self := range map[string]*x509.Certificate{
+		"no localhost": {DNSNames: []string{"console.old.example"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}},
+		"no 127.0.0.1": {DNSNames: []string{"console.old.example", "localhost"}},
+	} {
+		self.SerialNumber, self.Subject = big.NewInt(3), pkix.Name{CommonName: "x"}
+		self.NotBefore, self.NotAfter = time.Now().Add(-time.Hour), time.Now().Add(time.Hour)
+		der, _ := x509.CreateCertificate(rand.Reader, self, self, &key.PublicKey, key)
+		c, _ = x509.ParseCertificate(der)
+		if felisIssuedCert(c) {
+			t.Errorf("%s: a self-signed certificate is taken for Felis-issued", name)
+		}
+	}
+}
+
+func TestIssuePanelCertIsTheInstallersShape(t *testing.T) {
+	now := time.Now()
+	certPEM, keyPEM, err := issuePanelCert(newNames, []net.IP{net.ParseIP("10.211.55.6"), net.IPv4(127, 0, 0, 1)}, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil {
+		t.Fatalf("key does not match the certificate: %v", err)
+	}
+	if b, _ := pem.Decode(keyPEM); b == nil || b.Type != "PRIVATE KEY" {
+		t.Fatalf("key is not PKCS#8 PEM like openssl writes")
+	}
+	c, _ := x509.ParseCertificate(mustCertDER(certPEM))
+	if !certCovers(c, newNames.panel, newNames.admin, "localhost") || !felisIssuedCert(c) {
+		t.Fatalf("names %v", c.DNSNames)
+	}
+	if len(c.IPAddresses) != 2 || !c.IPAddresses[1].Equal(net.ParseIP("10.211.55.6")) {
+		t.Fatalf("addresses %v, want 127.0.0.1 and the node address once each", c.IPAddresses)
+	}
+	if c.Subject.CommonName != newNames.admin || c.NotAfter.Sub(now) < 824*24*time.Hour || c.NotAfter.Sub(now) > 826*24*time.Hour {
+		t.Fatalf("CN %q, valid until %s", c.Subject.CommonName, c.NotAfter)
+	}
+	if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth || c.IsCA {
+		t.Fatalf("usage %v, CA %v", c.ExtKeyUsage, c.IsCA)
+	}
+}
+
+func TestDomainSetMovesEverySurface(t *testing.T) {
+	rig := newDomainRig(t)
+	hostBefore := rig.read(t, "felis.host.toml")
+	code, err := rig.h.set(context.Background(), "New.Example", true)
+	if err != nil || code != 0 {
+		t.Fatalf("set = %d, %v\n%s", code, err, rig.out)
+	}
+
+	// The configs: the three values moved, everything else — comments, the
+	// database host of each copy, the Access audience — is as it was.
+	host := rig.read(t, "felis.host.toml")
+	if want := strings.NewReplacer("old.example", "new.example").Replace(hostBefore); host != want {
+		t.Fatalf("host toml:\n%s", host)
+	}
+	pod := rig.read(t, "felis.pod.toml")
+	if got, _ := tomlDomainNames([]byte(pod)); got != newNames || !strings.Contains(pod, "@10.211.55.6:5432") {
+		t.Fatalf("pod toml:\n%s", pod)
+	}
+	if link, err := os.Readlink(rig.path("felis.toml")); err != nil || link != rig.path("felis.host.toml") {
+		t.Fatalf("felis.toml is no longer the link to the host copy: %q %v", link, err)
+	}
+	if st, _ := os.Stat(rig.path("felis.host.toml")); st.Mode().Perm() != 0o600 {
+		t.Fatalf("host toml mode %v", st.Mode().Perm())
+	}
+
+	// The certificate: reissued for the new names, the node address kept, the old
+	// pair beside it.
+	c, err := readCertFile(rig.path("panel-tls.crt"))
+	if err != nil || !certCovers(c, newNames.panel, newNames.admin) || !felisIssuedCert(c) {
+		t.Fatalf("certificate: %v %v", err, c.DNSNames)
+	}
+	if !c.IPAddresses[len(c.IPAddresses)-1].Equal(net.ParseIP("10.211.55.6")) {
+		t.Fatalf("addresses %v", c.IPAddresses)
+	}
+	if _, err := tls.LoadX509KeyPair(rig.path("panel-tls.crt"), rig.path("panel-tls.key")); err != nil {
+		t.Fatalf("new pair: %v", err)
+	}
+	if st, _ := os.Stat(rig.path("panel-tls.key")); st.Mode().Perm() != 0o600 {
+		t.Fatalf("key mode %v", st.Mode().Perm())
+	}
+	backups, _ := filepath.Glob(rig.path("panel-tls.*.pre-domain-*"))
+	if len(backups) != 2 {
+		t.Fatalf("old pair kept as %v", backups)
+	}
+	for _, b := range backups {
+		if st, _ := os.Stat(b); strings.Contains(b, ".key.") && st.Mode().Perm() != 0o600 {
+			t.Fatalf("kept key %s has mode %v", b, st.Mode().Perm())
+		}
+		old, _ := os.ReadFile(b)
+		if strings.Contains(b, ".crt.") {
+			oc, _ := x509.ParseCertificate(mustCertDER(old))
+			if oc == nil || !certCovers(oc, oldNames.panel) {
+				t.Fatalf("kept certificate is not the old one")
+			}
+		}
+	}
+
+	// The Secrets carry the files.
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secret(t, ns, platform.ConfigSecretName)[platform.ConfigSecretKey]; string(got) != pod {
+			t.Fatalf("%s/felis-config is not felis.pod.toml", ns)
+		}
+	}
+	tlsData := rig.secret(t, "felis", platform.APITLSSecretName)
+	if string(tlsData[corev1.TLSCertKey]) != rig.read(t, "panel-tls.crt") || string(tlsData[corev1.TLSPrivateKeyKey]) != rig.read(t, "panel-tls.key") {
+		t.Fatal("felis-api-tls does not hold the new pair")
+	}
+
+	// The login gate's env moved and nothing else did.
+	env := rig.crEnv(t, naming.SystemLoginServer)
+	if env[envRootDomain] != newNames.root || env[envPanelHostname] != newNames.panel || env[envAPIBaseURL] != platform.InternalAPIBaseURL("felis") {
+		t.Fatalf("login env %v", env)
+	}
+
+	// The proxy's file: the three keys moved, its token and mode did not.
+	props := rig.read(t, "felis-link.properties")
+	if want := strings.NewReplacer("old.example", "new.example").Replace(linkPropsBody); props != want {
+		t.Fatalf("felis-link.properties:\n%s", props)
+	}
+	if st, _ := os.Stat(rig.path("felis-link.properties")); st.Mode().Perm() != 0o640 {
+		t.Fatalf("felis-link.properties mode %v", st.Mode().Perm())
+	}
+
+	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+		t.Fatalf("events %v", rig.events)
+	}
+	if !strings.Contains(rig.out.String(), "Every surface is on new.example.") {
+		t.Fatalf("the closing check did not pass:\n%s", rig.out)
+	}
+}
+
+func TestDomainSetWithoutYesChangesNothing(t *testing.T) {
+	rig := newDomainRig(t)
+	before := rig.snapshot(t)
+	code, err := rig.h.set(context.Background(), "new.example", false)
+	if err != nil || code != 0 {
+		t.Fatalf("set = %d, %v", code, err)
+	}
+	if rig.snapshot(t) != before || len(rig.events) != 0 {
+		t.Fatalf("a dry run changed something (events %v)", rig.events)
+	}
+	out := rig.out.String()
+	for _, want := range []string{
+		"console.old.example → console.new.example",
+		"3 passkey(s) of 2 user(s) are bound to console.old.example",
+		"does not cover op.console.new.example",
+		"No [smtp] relay is configured",
+		"sudo felis domain set -yes new.example",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("plan lacks %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestDomainSetKeepsAHandSetPanelHostname(t *testing.T) {
+	rig := newDomainRig(t)
+	for _, f := range []string{"felis.host.toml", "felis.pod.toml"} {
+		writeTestFile(t, rig.path(f), strings.Replace(rig.read(t, f), `panel_hostname = "console.old.example"`, `panel_hostname = "play.corp.net"`, 1), 0o600)
+	}
+	code, err := rig.h.set(context.Background(), "new.example", true)
+	if err != nil {
+		t.Fatalf("set: %v\n%s", err, rig.out)
+	}
+	got, _ := tomlDomainNames([]byte(rig.read(t, "felis.host.toml")))
+	if got != (domainNames{root: "new.example", panel: "play.corp.net", admin: "op.console.new.example"}) {
+		t.Fatalf("names %+v", got)
+	}
+	c, _ := readCertFile(rig.path("panel-tls.crt"))
+	if !certCovers(c, "play.corp.net", "op.console.new.example") {
+		t.Fatalf("certificate names %v", c.DNSNames)
+	}
+	if env := rig.crEnv(t, naming.SystemLoginServer); env[envPanelHostname] != "play.corp.net" {
+		t.Fatalf("login env %v", env)
+	}
+	if code != 0 || !strings.Contains(rig.out.String(), "play.corp.net (set by hand, kept") {
+		t.Fatalf("code %d:\n%s", code, rig.out)
+	}
+}
+
+func TestDomainSetRefusesAnOperatorCertificateForOtherNames(t *testing.T) {
+	rig := newDomainRig(t)
+	certPEM, keyPEM := caSignedCert(t, "console.old.example", "op.console.old.example")
+	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
+	before := rig.snapshot(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "not issued by Felis") {
+		t.Fatalf("err = %v", err)
+	}
+	if rig.snapshot(t) != before || len(rig.events) != 0 {
+		t.Fatal("a refused move changed something")
+	}
+
+	// The operator's certificate for the new names is kept as it is.
+	certPEM, keyPEM = caSignedCert(t, "console.new.example", "op.console.new.example")
+	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatalf("set: %v", err)
+	}
+	if rig.read(t, "panel-tls.crt") != string(certPEM) {
+		t.Fatal("the operator's certificate was replaced")
+	}
+}
+
+func TestDomainSetRefusesAConfigItCannotEdit(t *testing.T) {
+	rig := newDomainRig(t)
+	writeTestFile(t, rig.path("felis.pod.toml"), strings.Replace(rig.read(t, "felis.pod.toml"), "[auth]", "[\"auth\"]", 1), 0o600)
+	before := rig.snapshot(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "by hand") {
+		t.Fatalf("err = %v", err)
+	}
+	if rig.snapshot(t) != before || len(rig.events) != 0 {
+		t.Fatal("a refused move changed something")
+	}
+}
+
+func TestDomainSetAgainOnlyConvergesWhatIsBehind(t *testing.T) {
+	rig := newDomainRig(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	rig.events, rig.out = nil, &bytes.Buffer{}
+	rig.h.out = rig.out
+	before := rig.snapshot(t)
+	code, err := rig.h.set(context.Background(), "new.example", true)
+	if err != nil || code != 0 {
+		t.Fatalf("second set = %d, %v\n%s", code, err, rig.out)
+	}
+	if len(rig.events) != 0 || rig.snapshot(t) != before {
+		t.Fatalf("a converged install was touched again: %v\n%s", rig.events, rig.out)
+	}
+
+	// A proxy that was not restarted after the move is restarted by a re-run, and
+	// an api still on the old config is rolled.
+	rig.proxySince = time.Now().Add(-time.Hour)
+	rig.served = oldNames
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+		t.Fatalf("events %v", rig.events)
+	}
+
+	// An api on the new names that still presents the old certificate is rolled.
+	rig.events = nil
+	oldCert, _, _ := issuePanelCert(oldNames, nil, time.Now())
+	rig.servedCert, _ = x509.ParseCertificate(mustCertDER(oldCert))
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll-api" {
+		t.Fatalf("events %v", rig.events)
+	}
+}
+
+func TestDomainSetRefusesALoginServerItDoesNotOwn(t *testing.T) {
+	rig := newDomainRig(t)
+	var ms v1alpha1.MinecraftServer
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	delete(ms.Labels, v1alpha1.LabelSystemRole)
+	if err := rig.cl.Update(context.Background(), &ms); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "system role") {
+		t.Fatalf("err = %v", err)
+	}
+	if env := rig.crEnv(t, naming.SystemLoginServer); env[envRootDomain] != oldNames.root {
+		t.Fatalf("a server not marked as the login gate was changed: %v", env)
+	}
+}
+
+func TestDomainCheckNamesTheSurfaceThatIsBehind(t *testing.T) {
+	cases := []struct {
+		surface string
+		breakIt func(t *testing.T, rig *domainRig)
+	}{
+		{"felis.pod.toml", func(t *testing.T, rig *domainRig) {
+			writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
+		}},
+		{"Secret minecraft/felis-config", func(t *testing.T, rig *domainRig) {
+			rig.putSecret(t, "minecraft", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
+		}},
+		{"Secret felis/felis-config", func(t *testing.T, rig *domainRig) {
+			rig.putSecret(t, "felis", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
+		}},
+		{"panel certificate", func(t *testing.T, rig *domainRig) {
+			// The Secret follows the file, so only the certificate's names are wrong.
+			certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
+			writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
+			rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
+		}},
+		{"Secret felis/felis-api-tls", func(t *testing.T, rig *domainRig) {
+			certPEM, _, _ := issuePanelCert(newNames, nil, time.Now())
+			rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
+		}},
+		{"felis-api", func(t *testing.T, rig *domainRig) { rig.served = oldNames }},
+		{"felis-api", func(t *testing.T, rig *domainRig) {
+			certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
+			rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
+		}},
+		{"proxy", func(t *testing.T, rig *domainRig) {
+			writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
+			rig.proxySince = time.Now().Add(time.Hour)
+		}},
+		{"proxy", func(t *testing.T, rig *domainRig) { rig.proxySince = time.Now().Add(-time.Hour) }},
+		{"login gate", func(t *testing.T, rig *domainRig) {
+			var ms v1alpha1.MinecraftServer
+			_ = rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms)
+			for i := range ms.Spec.Env {
+				if ms.Spec.Env[i].Name == envRootDomain {
+					ms.Spec.Env[i].Value = "old.example"
+				}
+			}
+			if err := rig.cl.Update(context.Background(), &ms); err != nil {
+				t.Fatal(err)
+			}
+		}},
+		{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envRootDomain, "old.example") }},
+		{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envPanelHostname, "console.old.example") }},
+		{"Cloudflare tunnel", func(t *testing.T, rig *domainRig) {
+			writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n  service: https://127.0.0.1:30443\n- hostname: op.console.old.example\n  service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
+		}},
+	}
+	for _, tc := range cases {
+		rig := newDomainRig(t)
+		if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+			t.Fatal(err)
+		}
+		rig.out.Reset()
+		tc.breakIt(t, rig)
+		if code := rig.h.check(context.Background()); code != 1 {
+			t.Errorf("%s behind: check = %d\n%s", tc.surface, code, rig.out)
+			continue
+		}
+		var failed []string
+		for _, ln := range strings.Split(rig.out.String(), "\n") {
+			if strings.HasPrefix(ln, "  FAIL ") {
+				failed = append(failed, ln)
+			}
+		}
+		if len(failed) != 1 || !strings.Contains(failed[0], tc.surface) {
+			t.Errorf("%s behind: FAIL lines %q", tc.surface, failed)
+		}
+	}
+}
+
+func TestDomainCheckPassesAConvergedInstallAndWarnsOnDNS(t *testing.T) {
+	rig := newDomainRig(t)
+	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
+		t.Fatal(err)
+	}
+	writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n  service: https://127.0.0.1:30443\n- hostname: op.console.new.example\n  service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
+	rig.unresolved["op.console.new.example"] = true
+	rig.unresolved[dnsProbeLabel+".new.example"] = true
+	rig.out.Reset()
+	if code := rig.h.check(context.Background()); code != 0 {
+		t.Fatalf("check = %d\n%s", code, rig.out)
+	}
+	out := rig.out.String()
+	for _, want := range []string{"  ok   Cloudflare tunnel: routes both names", "  warn DNS: op.console.new.example, *.new.example do not resolve from this host\n"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("check lacks %q:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "TOKEN-NOT-TO-TOUCH") {
+		t.Fatal("check printed the proxy's service token")
+	}
+
+	// A zone with only the wildcard: the admin name alone is missing, and why is said.
+	delete(rig.unresolved, dnsProbeLabel+".new.example")
+	rig.out.Reset()
+	rig.h.check(context.Background())
+	if want := "  warn DNS: op.console.new.example does not resolve from this host: the *.new.example wildcard does not cover op.console.new.example, which needs its own record\n"; !strings.Contains(rig.out.String(), want) {
+		t.Errorf("check lacks %q:\n%s", want, rig.out)
+	}
+}
+
+func (rig *domainRig) setPodEnv(t *testing.T, name, value string) {
+	t.Helper()
+	var pod corev1.Pod
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+		t.Fatal(err)
+	}
+	for i, e := range pod.Spec.Containers[0].Env {
+		if e.Name == name {
+			pod.Spec.Containers[0].Env[i].Value = value
+		}
+	}
+	if err := rig.cl.Update(context.Background(), &pod); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (rig *domainRig) putSecret(t *testing.T, ns, name, key, val string) {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		t.Fatal(err)
+	}
+	s.Data[key] = []byte(val)
+	if err := rig.cl.Update(context.Background(), &s); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestParseUnitShow(t *testing.T) {
+	st := parseUnitShow("LoadState=loaded\nActiveState=active\nActiveEnterTimestamp=@1790000000\n")
+	if !st.loaded || !st.active || !st.since.Equal(time.Unix(1790000000, 0)) {
+		t.Fatalf("%+v", st)
+	}
+	st = parseUnitShow("LoadState=not-found\nActiveState=inactive\nActiveEnterTimestamp=\n")
+	if st.loaded || st.active || !st.since.IsZero() {
+		t.Fatalf("%+v", st)
+	}
+}

cmd/felis/egressgate.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"time"
+)
+
+// Vars so tests can shrink them. A dial that neither connects nor is refused
+// within egressDialTimeout counts as blocked: a policy that drops packets looks
+// exactly like that.
+var (
+	egressDialTimeout  = 500 * time.Millisecond
+	egressPollInterval = 200 * time.Millisecond
+)
+
+// cmdEgressGate is the first initContainer of every build pod and the last of
+// every game server pod. A pod's NetworkPolicy is programmed asynchronously after
+// the pod starts (live on k3s: a build-labelled pod reached the internet and the
+// Kubernetes API for its first ~0.7 s, a server-labelled one felis-api's internal
+// face on its first request), so the gate dials a destination the policy denies
+// until it stops answering, and only then lets the pod's next container, the
+// untrusted Dockerfile or server image, start.
+//
+// The default probe is the Kubernetes API Service, which the kubelet names in
+// every pod's environment and neither policy admits. A probe that still answers
+// after --wait means the policy is not enforced at all (a CNI without
+// NetworkPolicy support, or k3s run with --disable-network-policy), and the
+// build fails closed. A server passes --fail-open: an operator's
+// --server-egress-allow-cidr may cover the node the API Service leads to, so a
+// probe that keeps answering does not prove the fence is missing, and by then
+// the policy has had --wait to land.
+func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	probe := fs.String("probe", "", "host:port the pod's NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
+	positive := fs.String("positive-probe", "", "allowed host:port that must remain reachable during denial checks")
+	wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the gate gives up")
+	failOpen := fs.Bool("fail-open", false, "when --wait runs out, warn and let the pod go on instead of refusing it")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *probe == "" {
+		host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
+		if host == "" || port == "" {
+			fmt.Fprintln(stderr, "felis egress-gate: no --probe and no KUBERNETES_SERVICE_HOST/PORT to default to")
+			return 2
+		}
+		*probe = net.JoinHostPort(host, port)
+	}
+
+	start := time.Now()
+	for {
+		if *positive != "" {
+			allowed, err := net.DialTimeout("tcp", *positive, egressDialTimeout)
+			if err != nil {
+				if time.Since(start) >= *wait {
+					fmt.Fprintln(stderr, "felis egress-gate: positive probe unavailable; refusing to start", err)
+					return 1
+				}
+				time.Sleep(egressPollInterval)
+				continue
+			}
+			allowed.Close()
+		}
+		conn, err := net.DialTimeout("tcp", *probe, egressDialTimeout)
+		if err != nil {
+			fmt.Fprintf(stdout, "felis egress-gate: %s is unreachable after %s (%v); the egress lock is in effect\n",
+				*probe, time.Since(start).Round(time.Millisecond), err)
+			return 0
+		}
+		_ = conn.Close()
+		if time.Since(start) >= *wait {
+			if *failOpen {
+				fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s; starting anyway. Either this namespace's "+
+					"NetworkPolicy is not enforced (a CNI without NetworkPolicy support, or k3s started with "+
+					"--disable-network-policy), or an allowed CIDR admits the address behind it\n", *probe, *wait)
+				return 0
+			}
+			fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s: the build namespace's NetworkPolicy is not enforced "+
+				"(a CNI without NetworkPolicy support, or k3s started with --disable-network-policy); refusing to run the build\n",
+				*probe, *wait)
+			return 1
+		}
+		time.Sleep(egressPollInterval)
+	}
+}

cmd/felis/egressgate_test.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"bytes"
+	"net"
+	"strings"
+	"testing"
+	"time"
+)
+
+func shrinkEgressGate(t *testing.T) {
+	t.Helper()
+	dial, poll := egressDialTimeout, egressPollInterval
+	egressDialTimeout, egressPollInterval = 200*time.Millisecond, 10*time.Millisecond
+	t.Cleanup(func() { egressDialTimeout, egressPollInterval = dial, poll })
+}
+
+func TestEgressGateRequiresPositiveReachability(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	closed := ln.Addr().String()
+	ln.Close()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--positive-probe", closed, "--probe", closed, "--wait", "20ms"}, &out, &errb); code != 1 {
+		t.Fatal("unavailable probes allowed startup", code)
+	}
+}
+
+// The gate holds while the probe answers and lets the pod go on once the policy
+// lands, which the test plays by closing the listener.
+func TestEgressGateWaitsForTheLock(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	accepted := make(chan struct{}, 100)
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+			accepted <- struct{}{}
+		}
+	}()
+	go func() {
+		for i := 0; i < 3; i++ {
+			<-accepted
+		}
+		_ = ln.Close()
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "10s"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if !strings.Contains(out.String(), "egress lock is in effect") {
+		t.Errorf("stdout = %q", out.String())
+	}
+}
+
+// A probe that keeps answering means no policy is enforced: the build must not run.
+func TestEgressGateRefusesAnOpenNetwork(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+		}
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "100ms"}, &out, &errb); code != 1 {
+		t.Fatalf("exit %d, want 1", code)
+	}
+	if !strings.Contains(errb.String(), "not enforced") {
+		t.Errorf("stderr = %q", errb.String())
+	}
+}
+
+// A server's gate waits out --wait all the same, then lets the pod start with a
+// warning in its log.
+func TestEgressGateFailOpenWaitsThenWarns(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+		}
+	}()
+	var out, errb bytes.Buffer
+	start := time.Now()
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "150ms", "--fail-open"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d, want 0: %s", code, errb.String())
+	}
+	if waited := time.Since(start); waited < 150*time.Millisecond {
+		t.Errorf("gave up after %s, before --wait ran out", waited)
+	}
+	if !strings.Contains(errb.String(), ln.Addr().String()+" still answers after 150ms; starting anyway") {
+		t.Errorf("stderr = %q", errb.String())
+	}
+	if out.Len() != 0 {
+		t.Errorf("stdout = %q, want nothing: the lock was never seen", out.String())
+	}
+}
+
+func TestEgressGateDefaultsToTheKubernetesService(t *testing.T) {
+	shrinkEgressGate(t)
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	t.Setenv("KUBERNETES_SERVICE_PORT", "")
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate(nil, &out, &errb); code != 2 {
+		t.Fatalf("exit %d without a probe, want 2", code)
+	}
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	host, port, _ := net.SplitHostPort(ln.Addr().String())
+	_ = ln.Close() // closed: the lock reads as in effect at once
+	t.Setenv("KUBERNETES_SERVICE_HOST", host)
+	t.Setenv("KUBERNETES_SERVICE_PORT", port)
+	out.Reset()
+	if code := cmdEgressGate(nil, &out, &errb); code != 0 || !strings.Contains(out.String(), ln.Addr().String()) {
+		t.Fatalf("exit %d, stdout %q", code, out.String())
+	}
+}

cmd/felis/export.go

@@ -0,0 +1,307 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"hash"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/backup"
+	"felis.lolicon.best/internal/fileedit"
+	"felis.lolicon.best/internal/worldexport"
+)
+
+// cmdExport is the in-Pod entrypoint the export Job runs. internal/worldexport
+// renders a Pod whose command is `/usr/local/bin/felis export`. It archives the
+// mounted world, re-streams one archive from the mounted backup store, or sends
+// one file or folder of the world, PUTs it to felis-api's internal face, and
+// exits once felis-api says the owner's browser got all of it. It is NOT a
+// user-facing command and is never invoked by hand.
+//
+// Like cmdRestore it holds no database credentials and never calls config.Load:
+// felis-api made every decision (who may download what, that the server is
+// stopped, which archive) before the Job existed. Its input is the flags below
+// plus the one-time upload token in the environment, which opens this one
+// export and nothing else.
+//
+// Whatever leaves goes through the same guards as the file editor
+// (fileedit.Guard): the proxy forwarding secret, which every server on the
+// install shares, never leaves, and server.properties leaves with its RCON
+// password redacted. A backup is stored with both, since a restore must bring
+// the world back whole, so it is filtered on the way out rather than handed
+// over as stored.
+//
+// Exit status: 0 once felis-api answers 204 (the download completed), 1 when
+// the export could not be read or handed over, a backup failed its digest
+// check, or felis-api refused it (the browser never came or left early), 2 on
+// bad flags. The last stderr line reaches the export's status and the jobs list.
+func cmdExport(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("export", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	mode := fs.String("mode", "", "what to export: world, backup or files")
+	server := fs.String("server", "", "server name being exported (for logging)")
+	target := fs.String("target-url", "", "felis-api URL to PUT the export to")
+	ref := fs.String("ref", "", "backup only: absolute path to the archive on the backup mount")
+	backupRoot := fs.String("backup-root", "/backups", "backup only: mount path of the backup PVC (the ref must resolve under it)")
+	sum := fs.String("sha256", "", "backup only: the sha256 recorded when the archive was written; a mismatch fails the export before its end is sent")
+	worldsRoot := fs.String("worlds-root", "/world", "world and files: mount path of the world PVC")
+	path := fs.String("path", "", "files only: the file or folder to send, relative to the world root")
+	rawArchive := fs.Bool("archive-raw", false, "internal archive transfer: preserve the complete world")
+	dir := fs.Bool("dir", false, "files only: the path is a folder, sent as a zip")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	token := os.Getenv(worldexport.TokenEnv)
+	if *target == "" || token == "" {
+		fmt.Fprintf(stderr, "felis export: --target-url and %s are required\n", worldexport.TokenEnv)
+		return 2
+	}
+	limitHeapToCgroup()
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	var err error
+	switch *mode {
+	case worldexport.ModeBackup:
+		if *ref == "" {
+			fmt.Fprintln(stderr, "felis export: --ref is required for a backup")
+			return 2
+		}
+		err = exportBackup(ctx, *target, token, *ref, *backupRoot, *sum, stdout)
+	case worldexport.ModeWorld:
+		if *rawArchive {
+			err = streamExport(ctx, *target, token, archiveType, -1, func(w io.Writer) error { _, _, err := backup.WriteTarGz(ctx, w, *worldsRoot, nil); return err })
+		} else {
+			err = exportWorld(ctx, *target, token, *worldsRoot, stdout)
+		}
+	case worldexport.ModeFiles:
+		if *path == "" {
+			fmt.Fprintln(stderr, "felis export: --path is required for files")
+			return 2
+		}
+		err = exportFiles(ctx, *target, token, *worldsRoot, *path, *dir, stdout)
+	default:
+		fmt.Fprintf(stderr, "felis export: --mode must be %s, %s or %s\n", worldexport.ModeWorld, worldexport.ModeBackup, worldexport.ModeFiles)
+		return 2
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis export: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis export: server=%s mode=%s downloaded\n", *server, *mode)
+	return 0
+}
+
+// archiveType is the content type of a world or backup export.
+const archiveType = "application/gzip"
+
+// errBackupDigest fails a backup export whose stored archive no longer hashes
+// to what was recorded when it was written.
+var errBackupDigest = errors.New("the backup archive does not match the sha256 recorded when it was written")
+
+// exportBackup re-streams one stored archive through the export guards
+// (backup.FilterTarGz with archiveFilter). Its length changes on the way, so it
+// goes chunked. With want set, the stored bytes are hashed as they are read,
+// and FilterTarGz reads them to their end before it closes its own archive: a
+// mismatch aborts the upload while what felis-api has passed on still lacks
+// its end, so the browser never keeps a complete-looking corrupt file.
+func exportBackup(ctx context.Context, target, token, ref, root, want string, stdout io.Writer) error {
+	// Defense in depth, as in cmdRestore: the ref comes from felis-api, but this
+	// process opens it, so it confirms the ref stays on the backup mount.
+	if !refWithinRoot(ref, root) {
+		return fmt.Errorf("ref %q is not under backup root %q", ref, root)
+	}
+	f, err := os.Open(ref)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	var src io.Reader = f
+	if want != "" {
+		src = &digestReader{r: f, sum: sha256.New(), want: want}
+	}
+	var withheld []string
+	err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
+		var err error
+		withheld, err = backup.FilterTarGz(ctx, w, src, archiveFilter)
+		return err
+	})
+	if errors.Is(err, errBackupDigest) {
+		return errBackupDigest // the jobs list shows it as it is, not wrapped as a read error
+	}
+	reportWithheld(stdout, len(withheld))
+	return err
+}
+
+// exportWorld archives the world straight into the request body: nothing is
+// staged, so a world bigger than the Pod's memory or any scratch disk exports
+// the same.
+func exportWorld(ctx context.Context, target, token, root string, stdout io.Writer) error {
+	r, err := os.OpenRoot(root)
+	if err != nil {
+		return err
+	}
+	guard := fileedit.NewGuard(r)
+	r.Close()
+	var skipped, withheld []string
+	err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
+		var err error
+		skipped, withheld, err = backup.WriteTarGz(ctx, w, root, worldFilter(guard))
+		return err
+	})
+	if len(skipped) > 0 {
+		fmt.Fprintf(stdout, "felis export: left out %d entries a tar cannot hold (symbolic links, devices, sockets)\n", len(skipped))
+	}
+	reportWithheld(stdout, len(withheld))
+	return err
+}
+
+// exportFiles sends one file or folder of the world (fileedit.OpenDownload): a
+// file with its exact length, a folder as a zip made as it streams. dir is
+// what the owner saw at path when they asked.
+func exportFiles(ctx context.Context, target, token, root, path string, dir bool, stdout io.Writer) error {
+	d, err := fileedit.OpenDownload(root, path, dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	err = streamExport(ctx, target, token, d.ContentType, d.Size, func(w io.Writer) error { return d.WriteTo(ctx, w) })
+	if d.Skipped > 0 {
+		fmt.Fprintf(stdout, "felis export: left out %d entries a zip does not carry (symbolic links, devices, sockets)\n", d.Skipped)
+	}
+	reportWithheld(stdout, d.Withheld)
+	return err
+}
+
+func reportWithheld(stdout io.Writer, n int) {
+	if n > 0 {
+		fmt.Fprintf(stdout, "felis export: left out %d files that hold platform secrets\n", n)
+	}
+}
+
+// worldFilter guards a live world by file identity, so a link to a guarded
+// file under another name is caught as well.
+func worldFilter(g fileedit.Guard) backup.Filter {
+	return func(_ string, info fs.FileInfo) (bool, func([]byte) []byte) {
+		return guardAction(g.Rule(info))
+	}
+}
+
+// archiveFilter guards a stored archive, which has only names.
+func archiveFilter(name string, _ fs.FileInfo) (bool, func([]byte) []byte) {
+	return guardAction(fileedit.ArchiveRule(name))
+}
+
+func guardAction(withhold, redact bool) (bool, func([]byte) []byte) {
+	if redact {
+		return withhold, fileedit.RedactProps
+	}
+	return withhold, nil
+}
+
+// digestReader passes r through, hashing it, and turns r's EOF into
+// errBackupDigest when the bytes do not hash to want.
+type digestReader struct {
+	r    io.Reader
+	sum  hash.Hash
+	want string
+}
+
+func (d *digestReader) Read(p []byte) (int, error) {
+	n, err := d.r.Read(p)
+	d.sum.Write(p[:n])
+	if err == io.EOF && !strings.EqualFold(hex.EncodeToString(d.sum.Sum(nil)), d.want) {
+		return n, errBackupDigest
+	}
+	return n, err
+}
+
+// streamExport runs write straight into the body of the PUT, hashing it as it
+// goes. Once write has finished, the SHA-256 of all it wrote rides the
+// request's trailer (worldexport.DigestTrailer), and felis-api holds back the
+// last bytes from the browser until what it received hashes the same. An error
+// from write aborts the chunked body before the trailer, and felis-api then
+// cuts the browser's download off rather than end it; that error is the one
+// reported, since the PUT's own error only wraps it. When the PUT ends first,
+// write is stopped.
+func streamExport(ctx context.Context, target, token, contentType string, size int64, write func(io.Writer) error) error {
+	pr, pw := io.Pipe()
+	trailer := http.Header{worldexport.DigestTrailer: nil}
+	werr := make(chan error, 1)
+	go func() {
+		sum := sha256.New()
+		err := write(io.MultiWriter(pw, sum))
+		if err == nil {
+			// Set before the body ends: the transport reads the trailer once it
+			// has read the body to its end.
+			trailer.Set(worldexport.DigestTrailer, "sha-256=:"+base64.StdEncoding.EncodeToString(sum.Sum(nil))+":")
+		}
+		pw.CloseWithError(err)
+		werr <- err
+	}()
+	err := putExport(ctx, target, token, contentType, pr, size, trailer)
+	pr.CloseWithError(io.ErrClosedPipe)
+	if w := <-werr; w != nil && !errors.Is(w, io.ErrClosedPipe) {
+		return w
+	}
+	return err
+}
+
+// putExport PUTs the export to felis-api. There is no retry: the token opens
+// the export once, so a second attempt could only be refused. Redirects are
+// refused because the request carries the token and the internal face never
+// redirects. felis-api answers only after the whole download, which the Job's
+// activeDeadlineSeconds bounds, so the header timeout is a backstop for a
+// wedged endpoint and not the real limit.
+//
+// The body always goes chunked, which is what lets it end with a trailer; a
+// size the Job knows (-1 when it does not) goes as worldexport.LengthHeader in
+// place of Content-Length.
+func putExport(ctx context.Context, target, token, contentType string, body io.Reader, size int64, trailer http.Header) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body)
+	if err != nil {
+		return err
+	}
+	req.ContentLength = -1
+	req.Trailer = trailer
+	if size >= 0 {
+		req.Header.Set(worldexport.LengthHeader, strconv.FormatInt(size, 10))
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("Content-Type", contentType)
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: 10 * time.Minute},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNoContent {
+		return nil
+	}
+	var e struct {
+		Error struct {
+			Message string `json:"message"`
+		} `json:"error"`
+	}
+	if json.NewDecoder(io.LimitReader(resp.Body, 4<<10)).Decode(&e) == nil && e.Error.Message != "" {
+		return fmt.Errorf("felis-api answered %s: %s", resp.Status, e.Error.Message)
+	}
+	return fmt.Errorf("felis-api answered %s", resp.Status)
+}

cmd/felis/export_test.go

@@ -0,0 +1,532 @@
+package main
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"slices"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/worldexport"
+)
+
+// exportReceiver stands in for felis-api's internal upload route: it records
+// the PUT it gets (or the error reading it ended on) and answers with reply.
+type exportReceiver struct {
+	srv     *httptest.Server
+	hits    atomic.Int32
+	req     *http.Request
+	body    []byte
+	readErr error
+	served  chan struct{} // one send per request, once it is answered
+}
+
+func receiveExport(t *testing.T, reply func(w http.ResponseWriter)) *exportReceiver {
+	t.Helper()
+	rcv := &exportReceiver{served: make(chan struct{}, 1)}
+	rcv.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		rcv.hits.Add(1)
+		rcv.req = r
+		rcv.body, rcv.readErr = io.ReadAll(r.Body)
+		reply(w)
+		rcv.served <- struct{}{}
+	}))
+	t.Cleanup(rcv.srv.Close)
+	return rcv
+}
+
+func noContent(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) }
+
+// sentWhole fails unless the upload rcv got ended with the Content-Digest
+// trailer of its own bytes, and declared length as its size (-1: none).
+func sentWhole(t *testing.T, rcv *exportReceiver, length int64) {
+	t.Helper()
+	sum := sha256.Sum256(rcv.body)
+	want := "sha-256=:" + base64.StdEncoding.EncodeToString(sum[:]) + ":"
+	wantLength := ""
+	if length >= 0 {
+		wantLength = strconv.FormatInt(length, 10)
+	}
+	r := rcv.req
+	if rcv.readErr != nil || r.Trailer.Get(worldexport.DigestTrailer) != want || r.Header.Get(worldexport.LengthHeader) != wantLength ||
+		r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
+		t.Fatalf("upload read %v, trailer %v, %s %q, length %d, encoding %v; want trailer %q and %s %q, chunked",
+			rcv.readErr, r.Trailer, worldexport.LengthHeader, r.Header.Get(worldexport.LengthHeader), r.ContentLength, r.TransferEncoding,
+			want, worldexport.LengthHeader, wantLength)
+	}
+}
+
+func tarEntries(t *testing.T, archive []byte) map[string]string {
+	t.Helper()
+	gz, err := gzip.NewReader(bytes.NewReader(archive))
+	if err != nil {
+		t.Fatalf("not gzip: %v", err)
+	}
+	out := map[string]string{}
+	tr := tar.NewReader(gz)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return out
+		}
+		if err != nil {
+			t.Fatalf("tar: %v", err)
+		}
+		b, _ := io.ReadAll(tr)
+		out[h.Name] = string(b)
+	}
+}
+
+// writeTree writes name → body under root, making the folders on the way.
+func writeTree(t *testing.T, root string, files map[string]string) {
+	t.Helper()
+	for name, body := range files {
+		p := filepath.Join(root, name)
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// The two secrets a world holds, and what server.properties reads as once
+// redacted.
+const (
+	secretProps   = "motd=hi\nrcon.password=hunter2\n"
+	redactedProps = "motd=hi\nrcon.password=<redacted by felis>\n"
+	forwardingKey = "secret: aVeryRealForwardingKey\n"
+)
+
+// secretWorld is a world holding both secrets, with a hard link to the
+// forwarding secret under a name nothing would guard by.
+func secretWorld(t *testing.T) string {
+	t.Helper()
+	root := t.TempDir()
+	writeTree(t, root, map[string]string{
+		"server.properties":       secretProps,
+		"config/paper-global.yml": forwardingKey,
+		"world/region/r.0.0.mca":  "chunks",
+	})
+	if err := os.MkdirAll(filepath.Join(root, "plugins"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(root, "plugins/copy.yml")); err != nil {
+		t.Fatal(err)
+	}
+	return root
+}
+
+func TestCmdExportWorld(t *testing.T) {
+	root := secretWorld(t)
+	if err := os.Symlink("server.properties", filepath.Join(root, "props-link")); err != nil {
+		t.Fatal(err)
+	}
+	rcv := receiveExport(t, noContent)
+	t.Setenv(worldexport.TokenEnv, "tok")
+	var stdout, stderr bytes.Buffer
+	code := cmdExport([]string{"--mode", "world", "--server", "survival", "--target-url", rcv.srv.URL + "/api/v1/internal/exports/ab",
+		"--worlds-root", root}, &stdout, &stderr)
+	if code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, stderr.String())
+	}
+	r := rcv.req
+	if r.Method != http.MethodPut || r.URL.Path != "/api/v1/internal/exports/ab" || r.Header.Get("Authorization") != "Bearer tok" ||
+		r.Header.Get("Content-Type") != "application/gzip" || r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
+		t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
+	}
+	want := map[string]string{
+		"server.properties": redactedProps, "config/": "", "plugins/": "",
+		"world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks",
+	}
+	if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
+		t.Fatalf("archive holds %v\nwant %v", got, want)
+	}
+	sentWhole(t, rcv, -1)
+	want2 := "felis export: left out 1 entries a tar cannot hold (symbolic links, devices, sockets)\n" +
+		"felis export: left out 2 files that hold platform secrets\n" +
+		"felis export: server=survival mode=world downloaded\n"
+	if stdout.String() != want2 {
+		t.Errorf("stdout = %q, want %q", stdout.String(), want2)
+	}
+}
+
+// A world root that cannot be opened fails before anything reaches felis-api.
+func TestCmdExportWorldUnreadable(t *testing.T) {
+	rcv := receiveExport(t, noContent)
+	t.Setenv(worldexport.TokenEnv, "tok")
+	var stdout, stderr bytes.Buffer
+	code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
+	if code != 1 || rcv.hits.Load() != 0 {
+		t.Fatalf("exit %d with %d requests, want 1 and none", code, rcv.hits.Load())
+	}
+}
+
+// An export that fails part-way must never reach felis-api as a complete body:
+// the chunked upload is cut off, so felis-api aborts the browser's download,
+// and the failure itself is what the Job reports.
+func TestStreamExportWriteErrorAbortsTheUpload(t *testing.T) {
+	broken := errors.New("disk read failed")
+	rcv := receiveExport(t, noContent)
+	err := streamExport(context.Background(), rcv.srv.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
+		if _, err := w.Write(bytes.Repeat([]byte("x"), 100_000)); err != nil {
+			return err
+		}
+		return broken
+	})
+	if err != broken {
+		t.Fatalf("err = %v, want the write's own error, unwrapped", err)
+	}
+	select {
+	case <-rcv.served:
+		if rcv.readErr == nil {
+			t.Fatalf("felis-api read a complete %d-byte body from a failed export", len(rcv.body))
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("the request never reached felis-api")
+	}
+}
+
+// When felis-api refuses first, its reason is reported, not the closed pipe
+// that then stops the writer.
+func TestStreamExportRefusalStopsTheWriter(t *testing.T) {
+	refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer refusing.Close()
+	stopped := make(chan error, 1)
+	err := streamExport(context.Background(), refusing.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
+		for {
+			if _, err := w.Write(make([]byte, 32<<10)); err != nil {
+				stopped <- err
+				return err
+			}
+		}
+	})
+	if err == nil || err.Error() != "felis-api answered 404 Not Found" {
+		t.Fatalf("err = %v, want felis-api's answer", err)
+	}
+	if werr := <-stopped; !errors.Is(werr, io.ErrClosedPipe) {
+		t.Fatalf("the writer stopped on %v, want the closed pipe", werr)
+	}
+
+	// A PUT that never starts leaves no transport to close the body: the writer
+	// is still stopped, and the export fails rather than hangs.
+	done := make(chan error, 1)
+	go func() {
+		done <- streamExport(context.Background(), "http://[::1", "tok", "application/gzip", -1, func(w io.Writer) error {
+			_, err := w.Write([]byte("x"))
+			return err
+		})
+	}()
+	select {
+	case err := <-done:
+		if err == nil || !strings.Contains(err.Error(), "missing ']'") {
+			t.Fatalf("err = %v, want the bad URL", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("an export whose PUT never started hung")
+	}
+}
+
+// storedBackup writes, at path, a gzip+tar like one the backup store holds:
+// the world whole, both secrets included, and a region file that does not
+// compress. It returns the archive's sha256.
+func storedBackup(t *testing.T, path string) string {
+	t.Helper()
+	region := make([]byte, 64<<10)
+	if _, err := rand.Read(region); err != nil {
+		t.Fatal(err)
+	}
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zw)
+	for _, e := range []struct{ name, body string }{
+		{"server.properties", secretProps},
+		{"config/paper-global.yml", forwardingKey},
+		{"world/level.dat", "level"},
+		{"world/region/r.0.0.mca", string(region)},
+	} {
+		if err := tw.WriteHeader(&tar.Header{Name: e.name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(e.body))}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := io.WriteString(tw, e.body); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256(buf.Bytes())
+	return hex.EncodeToString(sum[:])
+}
+
+func TestCmdExportBackup(t *testing.T) {
+	root := t.TempDir()
+	ref := filepath.Join(root, "survival-1.tar.gz")
+	sum := storedBackup(t, ref)
+	stored, err := os.ReadFile(ref)
+	if err != nil {
+		t.Fatal(err)
+	}
+	region := tarEntries(t, stored)["world/region/r.0.0.mca"]
+	args := func(url, ref string) []string {
+		return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
+	}
+
+	for name, extra := range map[string][]string{
+		"no digest recorded":      nil,
+		"recorded digest matches": {"--sha256", sum},
+	} {
+		t.Run(name+": re-streamed through the guards", func(t *testing.T) {
+			rcv := receiveExport(t, noContent)
+			t.Setenv(worldexport.TokenEnv, "tok")
+			var stdout, stderr bytes.Buffer
+			if code := cmdExport(append(args(rcv.srv.URL, ref), extra...), &stdout, &stderr); code != 0 {
+				t.Fatalf("exit %d, stderr %q", code, stderr.String())
+			}
+			r := rcv.req
+			if r.ContentLength != -1 || r.Header.Get("Content-Type") != "application/gzip" || r.Header.Get("Authorization") != "Bearer tok" {
+				t.Fatalf("length %d, headers %v", r.ContentLength, r.Header)
+			}
+			want := map[string]string{"server.properties": redactedProps, "world/level.dat": "level", "world/region/r.0.0.mca": region}
+			if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
+				t.Fatalf("archive holds %d entries, want exactly the redacted properties, level.dat and the region file", len(got))
+			}
+			sentWhole(t, rcv, -1)
+			if want := "felis export: left out 1 files that hold platform secrets\nfelis export: server=survival mode=backup downloaded\n"; stdout.String() != want {
+				t.Errorf("stdout = %q, want %q", stdout.String(), want)
+			}
+		})
+	}
+
+	// The stored bytes are checked as they stream, and the archive the Job sends
+	// is only closed once they are all read: a mismatch cuts the upload off
+	// short of its end, so felis-api never passes on a complete-looking copy.
+	t.Run("a digest mismatch cuts the upload off before its end", func(t *testing.T) {
+		rcv := receiveExport(t, noContent)
+		t.Setenv(worldexport.TokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdExport(append(args(rcv.srv.URL, ref), "--sha256", strings.Repeat("ab", 32)), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if want := "felis export: the backup archive does not match the sha256 recorded when it was written\n"; stderr.String() != want {
+			t.Fatalf("stderr = %q, want %q", stderr.String(), want)
+		}
+		select {
+		case <-rcv.served:
+		case <-time.After(5 * time.Second):
+			t.Fatal("the upload never reached felis-api")
+		}
+		if rcv.readErr == nil {
+			t.Fatalf("felis-api read a complete %d-byte body", len(rcv.body))
+		}
+		zr, err := gzip.NewReader(bytes.NewReader(rcv.body))
+		if err == nil {
+			_, err = io.ReadAll(zr)
+		}
+		if err == nil {
+			t.Fatal("what felis-api got is a complete archive")
+		}
+	})
+
+	t.Run("a ref outside the backup root is refused before any request", func(t *testing.T) {
+		outside := filepath.Join(t.TempDir(), "secret.tar.gz")
+		if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		rcv := receiveExport(t, noContent)
+		t.Setenv(worldexport.TokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdExport(args(rcv.srv.URL, outside), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if n := rcv.hits.Load(); n != 0 {
+			t.Fatalf("felis-api got %d requests", n)
+		}
+	})
+
+	t.Run("a refusal exits 1 with felis-api's reason", func(t *testing.T) {
+		rcv := receiveExport(t, func(w http.ResponseWriter) {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusGone)
+			io.WriteString(w, `{"error":{"code":"export_expired","message":"nobody opened the download"}}`)
+		})
+		t.Setenv(worldexport.TokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if got := stderr.String(); got != "felis export: felis-api answered 410 Gone: nobody opened the download\n" {
+			t.Fatalf("stderr = %q", got)
+		}
+	})
+
+	// The request carries the token and the internal face never redirects, so a
+	// redirect is refused rather than followed with the token attached. A 302 or
+	// 303 is the one net/http would follow on its own (as a GET, and to the same
+	// host with the Authorization header still on it).
+	for _, status := range []int{http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
+		t.Run("a redirect is not followed: "+strconv.Itoa(status), func(t *testing.T) {
+			elsewhere := receiveExport(t, noContent)
+			redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.srv.URL, status))
+			defer redirecting.Close()
+			t.Setenv(worldexport.TokenEnv, "tok")
+			var stdout, stderr bytes.Buffer
+			if code := cmdExport(args(redirecting.URL, ref), &stdout, &stderr); code != 1 {
+				t.Fatalf("exit %d, want 1", code)
+			}
+			if n := elsewhere.hits.Load(); n != 0 {
+				t.Fatalf("the redirect target got %d requests", n)
+			}
+			if got, want := stderr.String(), "felis export: felis-api answered "+strconv.Itoa(status)+" "+http.StatusText(status)+"\n"; got != want {
+				t.Fatalf("stderr = %q, want %q", got, want)
+			}
+		})
+	}
+}
+
+func TestCmdExportFiles(t *testing.T) {
+	root := secretWorld(t)
+	writeTree(t, root, map[string]string{"plugins/Essentials/config.yml": "x: 1"})
+	if err := os.Symlink("config.yml", filepath.Join(root, "plugins/Essentials/link.yml")); err != nil {
+		t.Fatal(err)
+	}
+	export := func(t *testing.T, path string, dir bool) (*exportReceiver, int, string, string) {
+		t.Helper()
+		rcv := receiveExport(t, noContent)
+		t.Setenv(worldexport.TokenEnv, "tok")
+		args := []string{"--mode", "files", "--server", "survival", "--target-url", rcv.srv.URL, "--worlds-root", root, "--path", path}
+		if dir {
+			args = append(args, "--dir")
+		}
+		var stdout, stderr bytes.Buffer
+		code := cmdExport(args, &stdout, &stderr)
+		return rcv, code, stdout.String(), stderr.String()
+	}
+
+	for path, want := range map[string]string{
+		"world/region/r.0.0.mca": "chunks",
+		"server.properties":      redactedProps,
+	} {
+		t.Run("a file goes with its exact length: "+path, func(t *testing.T) {
+			rcv, code, stdout, stderr := export(t, path, false)
+			if code != 0 || stdout != "felis export: server=survival mode=files downloaded\n" {
+				t.Fatalf("exit %d, stdout %q, stderr %q", code, stdout, stderr)
+			}
+			if string(rcv.body) != want || rcv.req.Header.Get("Content-Type") != "application/octet-stream" {
+				t.Fatalf("body %q, type %q; want %q", rcv.body, rcv.req.Header.Get("Content-Type"), want)
+			}
+			sentWhole(t, rcv, int64(len(want)))
+		})
+	}
+
+	t.Run("a folder goes as a zip, guarded", func(t *testing.T) {
+		rcv, code, stdout, stderr := export(t, "plugins", true)
+		if code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr)
+		}
+		if rcv.req.ContentLength != -1 || rcv.req.Header.Get("Content-Type") != "application/zip" {
+			t.Fatalf("length %d, type %q", rcv.req.ContentLength, rcv.req.Header.Get("Content-Type"))
+		}
+		zr, err := zip.NewReader(bytes.NewReader(rcv.body), int64(len(rcv.body)))
+		if err != nil {
+			t.Fatal(err)
+		}
+		var names []string
+		for _, f := range zr.File {
+			names = append(names, f.Name)
+		}
+		if want := []string{"plugins/", "plugins/Essentials/", "plugins/Essentials/config.yml"}; !slices.Equal(names, want) {
+			t.Fatalf("zip holds %v, want %v", names, want)
+		}
+		want := "felis export: left out 1 entries a zip does not carry (symbolic links, devices, sockets)\n" +
+			"felis export: left out 1 files that hold platform secrets\n" +
+			"felis export: server=survival mode=files downloaded\n"
+		if stdout != want {
+			t.Errorf("stdout = %q, want %q", stdout, want)
+		}
+	})
+
+	for _, c := range []struct {
+		path string
+		dir  bool
+		want string
+	}{
+		{"config/paper-global.yml", false, "forwarding secret"},
+		{"plugins/copy.yml", false, "forwarding secret"},
+		{"plugins", false, "is a folder now"},
+		{"server.properties", true, "is not a folder now"},
+	} {
+		t.Run("refused before any request: "+c.path, func(t *testing.T) {
+			rcv, code, _, stderr := export(t, c.path, c.dir)
+			if code != 1 || rcv.hits.Load() != 0 || !strings.Contains(stderr, c.want) {
+				t.Fatalf("exit %d, %d requests, stderr %q; want 1, none, and %q", code, rcv.hits.Load(), stderr, c.want)
+			}
+		})
+	}
+}
+
+func TestCmdExportUsage(t *testing.T) {
+	for name, tc := range map[string]struct {
+		token string
+		args  []string
+	}{
+		"no token":           {"", []string{"--mode", "world", "--target-url", "http://api/x"}},
+		"no target":          {"tok", []string{"--mode", "world"}},
+		"unknown mode":       {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
+		"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
+		"files without path": {"tok", []string{"--mode", "files", "--target-url", "http://api/x"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Setenv(worldexport.TokenEnv, tc.token)
+			var stdout, stderr bytes.Buffer
+			if code := cmdExport(tc.args, &stdout, &stderr); code != 2 {
+				t.Fatalf("exit %d, want 2; stderr %q", code, stderr.String())
+			}
+		})
+	}
+}
+
+// TestCmdExportWiring: the Job's `felis export` reaches cmdExport, and
+// felis-api's executor mounts the backup store at the path the archives were
+// written under, since a backup's ref is an absolute path there.
+func TestCmdExportWiring(t *testing.T) {
+	t.Setenv(worldexport.TokenEnv, "")
+	var stdout, stderr bytes.Buffer
+	if code := run([]string{"export"}, &stdout, &stderr); code != 2 ||
+		stderr.String() != "felis export: --target-url and "+worldexport.TokenEnv+" are required\n" {
+		t.Fatalf("felis export = %d, stderr %q", code, stderr.String())
+	}
+	cfg := &config.Config{}
+	cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups"
+	want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"}
+	if got := exportConfig(cfg, "felis:1", "felis-backups"); !reflect.DeepEqual(got, want) {
+		t.Fatalf("exportConfig = %+v, want %+v", got, want)
+	}
+}

cmd/felis/fetchcontext.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// cmdFetchContext is the in-Pod entrypoint the build Job's context-fetch
+// initContainer runs. It reads the blob the platform stored for a submission
+// from the felis-api INTERNAL face (with a bounded retry — see
+// fetchContextWithRetry) and extracts it into the shared emptyDir the Kaniko
+// container then builds from.
+//
+// Why this exists: the build Pod runs in the build namespace, where it can neither
+// mount the control-plane uploads PVC (a PVC does not cross namespaces) nor hold
+// object-store credentials, so the API that WROTE the blob is the transport. The
+// route is service-token-gated; the token arrives through a namespace-local Secret
+// mounted only into this initContainer, never into Kaniko's — so the untrusted
+// Dockerfile's build steps have no credential to read (their containers share no
+// environment, no PID namespace, and Kaniko itself mounts the context read-only).
+//
+// The extraction is deliberately paranoid: the tarball is attacker-controlled
+// input, so absolute paths, ".." escapes, links, and special files are refused
+// rather than sanitized. Kaniko treats the extracted tree as hostile regardless
+// (spec §16), but the pod's own filesystem still must not be written outside the
+// context directory it was given.
+func cmdFetchContext(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("fetch-context", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	url := fs.String("url", "", "internal-face URL of the submission's build-context tarball")
+	out := fs.String("out", "/context", "directory to extract the build context into")
+	want := fs.String("sha256", "", "refuse the context unless the tarball's sha256 is this lowercase hex digest")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *want != "" && !build.IsSHA256Hex(*want) {
+		fmt.Fprintf(stderr, "felis fetch-context: --sha256 %q is not a lowercase hex sha256\n", *want)
+		return 2
+	}
+	if *url == "" {
+		fmt.Fprintln(stderr, "felis fetch-context: --url is required")
+		return 2
+	}
+	token := os.Getenv("FELIS_SERVICE_TOKEN")
+	if token == "" {
+		fmt.Fprintln(stderr, "felis fetch-context: FELIS_SERVICE_TOKEN is empty — the internal face rejects anonymous reads")
+		return 2
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	// Validate the URL once up front: a bad one is a usage error (2), not
+	// something to sit in the retry loop.
+	if _, err := http.NewRequest(http.MethodGet, *url, nil); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: bad --url: %v\n", err)
+		return 2
+	}
+	// No overall client timeout: a legitimate modpack context can be large and the
+	// Job's activeDeadlineSeconds is the real bound. The header timeout catches a
+	// wedged endpoint without capping a healthy download.
+	// Redirects are refused: the request carries the service token, and the
+	// internal face never redirects, so a 3xx is someone steering the token.
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: time.Minute},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := fetchContextWithRetry(ctx, client, *url, token, stderr)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	defer resp.Body.Close()
+
+	h := sha256.New()
+	body := io.TeeReader(resp.Body, h)
+	if err := extractTarGz(body, *out); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	if *want == "" {
+		return 0
+	}
+	// The tar end marker comes before the gzip trailer and whatever follows it,
+	// so read to EOF: the digest must cover every byte the blob holds. The blob
+	// itself is size-capped at upload, which bounds this read.
+	if _, err := io.Copy(io.Discard, io.LimitReader(body, maxContextBytes)); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	if got := hex.EncodeToString(h.Sum(nil)); got != *want {
+		// The init container failing is what keeps Kaniko from ever starting on
+		// the extracted tree.
+		fmt.Fprintf(stderr, "felis fetch-context: the context's sha256 is %s, the approved digest is %s: it changed after approval; refusing to build\n", got, *want)
+		return 1
+	}
+	return 0
+}
+
+// fetchRetryInterval/fetchRetryWindow bound how long the fetch waits out a
+// control-plane blip before giving up. The api pod being replaced is a normal
+// event (rollout, eviction, a chaos drill), and without a retry one refused
+// dial turns it into a failed build: BackoffLimit=0 gives the Job no second
+// Pod, so the terminal verdict costs a manual re-approval — the live drill hit
+// exactly this (context-fetch exit 1 on `connect: connection refused` while
+// the api pod rolled; the new pod was serving 11 seconds later and the same
+// 198-byte blob). The window is tiny next to the Job's 30-minute
+// activeDeadline; a 4xx (missing blob, rejected token) still fails fast.
+//
+// Vars, not consts, so tests can shrink the window.
+var (
+	fetchRetryInterval = 3 * time.Second
+	fetchRetryWindow   = 45 * time.Second
+)
+
+// fetchContextWithRetry GETs the context tarball, retrying transport failures
+// and 5xx responses until fetchRetryWindow runs out. A 4xx is an answer, not a
+// blip — retrying it only delays the honest error.
+func fetchContextWithRetry(ctx context.Context, client *http.Client, url, token string, stderr io.Writer) (*http.Response, error) {
+	deadline := time.Now().Add(fetchRetryWindow)
+	for {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+		if err != nil {
+			return nil, fmt.Errorf("bad --url: %w", err)
+		}
+		req.Header.Set("Authorization", "Bearer "+token)
+
+		resp, err := client.Do(req)
+		if err == nil && resp.StatusCode == http.StatusOK {
+			return resp, nil
+		}
+		if err == nil {
+			status := resp.Status
+			_ = resp.Body.Close()
+			err = fmt.Errorf("GET returned %s", status)
+			if resp.StatusCode < 500 {
+				return nil, err
+			}
+		}
+		if ctx.Err() != nil {
+			return nil, fmt.Errorf("GET failed: %w", err)
+		}
+		if time.Now().After(deadline) {
+			return nil, fmt.Errorf("GET failed (retried for %s): %w", fetchRetryWindow, err)
+		}
+		fmt.Fprintf(stderr, "felis fetch-context: %v; retrying (the internal face may be restarting)\n", err)
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("GET failed: %w", err)
+		case <-time.After(fetchRetryInterval):
+		}
+	}
+}
+
+// maxContextBytes / maxContextEntries bound what one context may expand to. The
+// compressed upload is capped at 1 GiB, but gzip turns that into hundreds of GiB
+// or millions of empty files, and the emptyDir's 4 GiB sizeLimit is only
+// enforced by the kubelet's periodic sweep, after the disk has filled. The byte
+// cap matches that sizeLimit; the entry cap is far above any real modpack (a
+// large one is a few thousand files) and far below an inode exhaustion.
+//
+// Vars, not consts, so tests can shrink them.
+var (
+	maxContextBytes   int64 = 4 << 30
+	maxContextEntries       = 200_000
+)
+
+// extractTarGz streams a gzip'd tarball into root, creating directories as
+// needed. Every entry is vetted BEFORE anything is written: a path that is
+// absolute or escapes root (via ".."), a link (symlink or hardlink), or any
+// special file kind aborts the whole extraction. Refusing rather than skipping is
+// deliberate — a context that needs one of those constructs is not a context this
+// transport carries, and silently dropping entries would build from a corpus the
+// submitter did not upload. The whole extraction is also bounded by
+// maxContextBytes and maxContextEntries.
+func extractTarGz(r io.Reader, root string) error {
+	if err := os.MkdirAll(root, 0o755); err != nil {
+		return fmt.Errorf("create context dir: %w", err)
+	}
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return fmt.Errorf("context is not a valid gzip tarball: %w", err)
+	}
+	defer zr.Close()
+	tr := tar.NewReader(zr)
+	var written int64
+	entries := 0
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("read context tarball: %w", err)
+		}
+		if entries++; entries > maxContextEntries {
+			return fmt.Errorf("the build context has more than %d entries", maxContextEntries)
+		}
+		name := filepath.Clean(hdr.Name)
+		if name == "." {
+			continue
+		}
+		// The zip-slip guard: reject, never rewrite. filepath.Clean collapses any
+		// "a/../../b", so these two checks are sufficient once Clean has run.
+		if filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) {
+			return fmt.Errorf("context entry %q escapes the context directory", hdr.Name)
+		}
+		target := filepath.Join(root, name)
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0o755); err != nil {
+				return fmt.Errorf("create %q: %w", name, err)
+			}
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+				return fmt.Errorf("create parent of %q: %w", name, err)
+			}
+			mode := os.FileMode(0o644)
+			if hdr.FileInfo().Mode()&0o111 != 0 {
+				mode = 0o755 // preserve executability (entrypoint scripts), nothing else
+			}
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
+			if err != nil {
+				return fmt.Errorf("create %q: %w", name, err)
+			}
+			n, err := io.Copy(f, io.LimitReader(tr, maxContextBytes-written+1))
+			written += n
+			if err != nil {
+				_ = f.Close()
+				return fmt.Errorf("write %q: %w", name, err)
+			}
+			if written > maxContextBytes {
+				_ = f.Close()
+				return fmt.Errorf("the build context expands past %d bytes", maxContextBytes)
+			}
+			if err := f.Close(); err != nil {
+				return fmt.Errorf("close %q: %w", name, err)
+			}
+		default:
+			return fmt.Errorf("context entry %q has unsupported type %q (links and special files are refused)", hdr.Name, string(hdr.Typeflag))
+		}
+	}
+}

cmd/felis/fetchcontext_test.go

@@ -0,0 +1,404 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+type tarEntry struct {
+	name     string
+	body     string
+	mode     int64
+	typ      byte
+	linkname string
+}
+
+// tgzBody builds an in-memory .tar.gz from entries, preserving each entry's type
+// and mode so the tests can exercise the guards with exactly the bytes an
+// attacker could upload.
+func tgzBody(t *testing.T, entries ...tarEntry) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zw)
+	for _, e := range entries {
+		typ := e.typ
+		if typ == 0 {
+			typ = tar.TypeReg
+		}
+		mode := e.mode
+		if mode == 0 {
+			mode = 0o644
+		}
+		hdr := &tar.Header{Name: e.name, Typeflag: typ, Mode: mode, Size: int64(len(e.body))}
+		if typ == tar.TypeSymlink {
+			hdr.Linkname = e.linkname
+			hdr.Size = 0
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatalf("write header %q: %v", e.name, err)
+		}
+		if hdr.Size > 0 {
+			if _, err := tw.Write([]byte(e.body)); err != nil {
+				t.Fatalf("write body %q: %v", e.name, err)
+			}
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatalf("close tar: %v", err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatalf("close gzip: %v", err)
+	}
+	return buf.Bytes()
+}
+
+// A normal context extracts with its tree intact, and the executable bit that
+// modpack entrypoints rely on survives.
+func TestExtractTarGzRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	body := tgzBody(t,
+		tarEntry{name: "Dockerfile", body: "FROM scratch\n"},
+		tarEntry{name: "mods/example.jar", body: "jar-bytes"},
+		tarEntry{name: "start.sh", body: "#!/bin/sh\n", mode: 0o755},
+		tarEntry{name: "mods/", typ: tar.TypeDir, mode: 0o755},
+	)
+	if err := extractTarGz(bytes.NewReader(body), dir); err != nil {
+		t.Fatalf("extract: %v", err)
+	}
+	for name, want := range map[string]string{
+		"Dockerfile":       "FROM scratch\n",
+		"mods/example.jar": "jar-bytes",
+	} {
+		got, err := os.ReadFile(filepath.Join(dir, name))
+		if err != nil || string(got) != want {
+			t.Fatalf("%s = (%q, %v), want %q", name, got, err, want)
+		}
+	}
+	fi, err := os.Stat(filepath.Join(dir, "start.sh"))
+	if err != nil || fi.Mode()&0o111 == 0 {
+		t.Fatalf("entrypoint script lost its exec bit: %v (%v)", fi, err)
+	}
+}
+
+// The guards: "..", absolute paths, symlinks, and special files are refused whole
+// — nothing escapes, and nothing is silently skipped.
+func TestExtractTarGzRefusesEscapes(t *testing.T) {
+	cases := []struct {
+		name    string
+		entries []tarEntry
+	}{
+		{"dotdot", []tarEntry{{name: "../outside", body: "x"}}},
+		{"nested dotdot", []tarEntry{{name: "a/../../outside", body: "x"}}},
+		{"absolute", []tarEntry{{name: "/etc/outside", body: "x"}}},
+		{"symlink", []tarEntry{{name: "link", typ: tar.TypeSymlink, linkname: "/etc"}}},
+		{"hardlink", []tarEntry{{name: "hard", typ: tar.TypeLink, linkname: "somewhere"}}},
+		{"device", []tarEntry{{name: "dev", typ: tar.TypeChar}}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			if err := extractTarGz(bytes.NewReader(tgzBody(t, tc.entries...)), dir); err == nil {
+				t.Fatal("extract accepted a hostile entry, want an error")
+			}
+			// Nothing may have been written outside the target (or at all).
+			entries, _ := os.ReadDir(dir)
+			if len(entries) != 0 {
+				t.Fatalf("hostile archive left %d entries behind", len(entries))
+			}
+		})
+	}
+}
+
+// A context that expands past the byte or entry cap is refused, however small
+// it was compressed: gzip bombs and inode floods stop at the cap.
+func TestExtractTarGzCapsExpansion(t *testing.T) {
+	bytesCap, entriesCap := maxContextBytes, maxContextEntries
+	t.Cleanup(func() { maxContextBytes, maxContextEntries = bytesCap, entriesCap })
+	maxContextBytes, maxContextEntries = 1000, 5
+
+	fits := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 400)})
+	if err := extractTarGz(bytes.NewReader(fits), t.TempDir()); err != nil {
+		t.Fatalf("a context exactly at the byte cap: %v", err)
+	}
+	big := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 401)})
+	if err := extractTarGz(bytes.NewReader(big), t.TempDir()); err == nil || !strings.Contains(err.Error(), "expands past") {
+		t.Fatalf("one byte over the cap: err = %v", err)
+	}
+	var many []tarEntry
+	for i := 0; i < 6; i++ {
+		many = append(many, tarEntry{name: "d" + string(rune('0'+i)) + "/", typ: tar.TypeDir})
+	}
+	if err := extractTarGz(bytes.NewReader(tgzBody(t, many...)), t.TempDir()); err == nil || !strings.Contains(err.Error(), "entries") {
+		t.Fatalf("six entries over a cap of five: err = %v", err)
+	}
+}
+
+// The command end to end: it dials the URL with the bearer token from the
+// environment, and refuses to run without it (the internal face would 401
+// anyway; failing at parse time is the honest earlier error).
+func TestCmdFetchContextFetchAndExtract(t *testing.T) {
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer test-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		w.Header().Set("Content-Type", "application/gzip")
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + dir}, io.Discard, io.Discard); code != 0 {
+		t.Fatalf("cmdFetchContext exit = %d, want 0", code)
+	}
+	if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
+		t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
+	}
+
+	// No token: refuse before dialing.
+	t.Setenv("FELIS_SERVICE_TOKEN", "")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 2 {
+		t.Fatalf("missing token exit = %d, want 2 (stderr %q)", code, stderr.String())
+	}
+
+	// A non-200 answer (e.g. the route's 404 for a never-uploaded context) fails.
+	srv404 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv404.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	if code := cmdFetchContext([]string{"--url=" + srv404.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, io.Discard); code != 1 {
+		t.Fatalf("404 exit = %d, want 1", code)
+	}
+}
+
+// With --sha256 the fetch refuses any bytes but the approved ones, including
+// a tarball that extracts cleanly: that is exactly the context an uploader
+// swapped in after the review.
+func TestCmdFetchContextChecksDigest(t *testing.T) {
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	sum := sha256.Sum256(body)
+	good := hex.EncodeToString(sum[:])
+	args := func(digest string) []string {
+		return []string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir(), "--sha256=" + digest}
+	}
+
+	if code := cmdFetchContext(args(good), io.Discard, io.Discard); code != 0 {
+		t.Fatalf("matching digest exit = %d, want 0", code)
+	}
+	var stderr bytes.Buffer
+	other := strings.Repeat("0", 64)
+	if code := cmdFetchContext(args(other), io.Discard, &stderr); code != 1 || !strings.Contains(stderr.String(), "changed after approval") {
+		t.Fatalf("mismatched digest exit = %d, stderr %q; want 1 naming the change", code, stderr.String())
+	}
+	stderr.Reset()
+	if code := cmdFetchContext(args("ABC"), io.Discard, &stderr); code != 2 {
+		t.Fatalf("malformed digest exit = %d, want 2 (stderr %q)", code, stderr.String())
+	}
+
+	// Bytes after the tar end marker still count: appending to an approved blob
+	// must change what the fetch accepts.
+	padded := append(append([]byte{}, body...), "trailing"...)
+	srvPadded := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(padded)
+	}))
+	defer srvPadded.Close()
+	if code := cmdFetchContext([]string{"--url=" + srvPadded.URL + "/c", "--out=" + t.TempDir(), "--sha256=" + good}, io.Discard, io.Discard); code != 1 {
+		t.Fatalf("padded blob exit = %d, want 1", code)
+	}
+}
+
+// The request carries the service token, so a redirect is a failure: the token
+// never follows it to another host (build-supply-chain-13).
+func TestCmdFetchContextRefusesRedirects(t *testing.T) {
+	var leaked bool
+	elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		leaked = true
+	}))
+	defer elsewhere.Close()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, elsewhere.URL+"/steal", http.StatusFound)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/c", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("redirect exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if leaked {
+		t.Fatal("the fetch followed the redirect")
+	}
+}
+
+// A body that is not a gzip tarball must fail the extraction rather than produce
+// an empty (or partial) context Kaniko would then try to build.
+func TestExtractTarGzRejectsNonGzip(t *testing.T) {
+	dir := t.TempDir()
+	err := extractTarGz(strings.NewReader("not a tarball"), dir)
+	if err == nil || !strings.Contains(err.Error(), "gzip") {
+		t.Fatalf("err = %v, want a gzip complaint", err)
+	}
+}
+
+// shrinkFetchWindow swaps the retry knobs for a faster test and restores them
+// afterwards, so no test leaks a tiny window into another.
+func shrinkFetchWindow(t *testing.T, interval, window time.Duration) {
+	t.Helper()
+	oldInterval, oldWindow := fetchRetryInterval, fetchRetryWindow
+	fetchRetryInterval, fetchRetryWindow = interval, window
+	t.Cleanup(func() { fetchRetryInterval, fetchRetryWindow = oldInterval, oldWindow })
+}
+
+// A control-plane blip mid-fetch is survived: a 5xx on the first attempt is
+// retried and the second attempt's tarball extracts. This walks back the live
+// drill's failure, where the api pod rolled mid-fetch and the single attempt
+// died, failing the build Job.
+func TestFetchContextRetriesThroughBlip(t *testing.T) {
+	shrinkFetchWindow(t, 10*time.Millisecond, time.Second)
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			w.WriteHeader(http.StatusBadGateway) // the port is up, the API is not
+			return
+		}
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + dir}, io.Discard, &stderr); code != 0 {
+		t.Fatalf("exit = %d, want 0 (stderr %q)", code, stderr.String())
+	}
+	if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
+		t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
+	}
+	if !strings.Contains(stderr.String(), "retrying") {
+		t.Fatalf("stderr %q does not mention the retry", stderr.String())
+	}
+}
+
+// The live drill's exact shape: the dial itself is refused (the api pod is
+// gone and no endpoint answers). A refused dial is retried like any other
+// transport failure, and once the face is back the fetch completes.
+func TestFetchContextRetriesRefusedDial(t *testing.T) {
+	shrinkFetchWindow(t, 10*time.Millisecond, 5*time.Second)
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+
+	// Borrow a listen address, then close it: the first attempts dial into a
+	// refused connection, exactly like a restarting control plane.
+	probe := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+	addr := strings.TrimPrefix(probe.URL, "http://")
+	probe.Close()
+
+	dir := t.TempDir()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	// Start the fetch; while the retry loop burns refused dials, bring the same
+	// address back.
+	result := make(chan int, 1)
+	go func() {
+		result <- cmdFetchContext([]string{"--url=http://" + addr + "/sub-1/context", "--out=" + dir}, io.Discard, &stderr)
+	}()
+	time.Sleep(100 * time.Millisecond) // let a handful of dials be refused
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		t.Fatalf("rebind %s: %v", addr, err)
+	}
+	back := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer test-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		_, _ = w.Write(body)
+	})}
+	defer back.Close()
+	go func() { _ = back.Serve(ln) }()
+
+	code := <-result
+	if code != 0 {
+		t.Fatalf("exit = %d, want 0 (stderr %q)", code, stderr.String())
+	}
+	if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
+		t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
+	}
+	if !strings.Contains(stderr.String(), "retrying") {
+		t.Fatalf("stderr %q does not mention the retry", stderr.String())
+	}
+}
+
+// A 4xx is an answer, not a blip: a missing/never-uploaded context fails
+// immediately — no retry loop burns the build's deadline on a terminal error.
+func TestFetchContextDoesNotRetry4xx(t *testing.T) {
+	shrinkFetchWindow(t, 5*time.Millisecond, 200*time.Millisecond)
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if got := atomic.LoadInt32(&calls); got != 1 {
+		t.Fatalf("server saw %d attempts, want exactly 1", got)
+	}
+	if strings.Contains(stderr.String(), "retrying") {
+		t.Fatalf("stderr %q mentions a retry for a terminal 4xx", stderr.String())
+	}
+}
+
+// The retry is bounded: an internal face that stays down does not hang the
+// build pod; the window runs out and the fetch reports the exhausted retries.
+func TestFetchContextGivesUpAfterWindow(t *testing.T) {
+	shrinkFetchWindow(t, 5*time.Millisecond, 60*time.Millisecond)
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
+	defer srv.Close() // the face is up but never healthy: 503 forever
+
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	start := time.Now()
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if elapsed := time.Since(start); elapsed > 5*time.Second {
+		t.Fatalf("gave up after %v; the window is supposed to bound it", elapsed)
+	}
+	if got := atomic.LoadInt32(&calls); got < 2 {
+		t.Fatalf("server saw %d attempts, want at least one retry", got)
+	}
+	if !strings.Contains(stderr.String(), "retried for") {
+		t.Fatalf("stderr %q does not report the exhausted retry window", stderr.String())
+	}
+}

cmd/felis/files.go

@@ -1,11 +1,15 @@
 package main
 
 import (
-	"encoding/base64"
+	"context"
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
 	"felis.lolicon.best/internal/fileedit"
 )
@@ -19,32 +23,55 @@ import (
 // Like cmdRestore it deliberately holds NO database credentials and never calls
 // config.Load: felis-api made the authorization decision (the caller owns this
 // server, and the server is stopped so the RWO world volume is free); this process
-// is the unprivileged hands that touch bytes. Its entire input is the three flags
-// below plus, for a write, one environment variable. Every isolation guarantee
-// lives in the Pod spec (internal/fileedit/jobspec.go), and the path-containment
-// guarantee lives in fileedit.Execute, which resolves the path through os.Root and
-// therefore cannot be walked out of the world mount.
+// is the unprivileged hands that touch bytes. Its entire input is the flags
+// below plus, for a write, the content variables and, for an upload, one token.
+// Every isolation guarantee lives in the Pod spec (internal/fileedit/jobspec.go),
+// and the path-containment guarantee lives in fileedit.Execute, which resolves
+// every path through os.Root and therefore cannot be walked out of the world
+// mount.
 //
 // Exit status carries a specific meaning that felis-api depends on: a CALLER-fault
 // outcome — a path that escapes the root, a file that is missing or too large — is
 // a SUCCESSFUL run that prints a Result carrying an error code, so the API can map
 // it to a precise 4xx. A non-zero exit means the operation could not be attempted
-// at all (the world mount is unreadable, the result unprintable), which the API
-// reports as a 500.
+// at all (the world mount is unreadable, an upload's bytes could not be fetched
+// intact, the result unprintable), which the API reports as a 500.
 func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("files", flag.ContinueOnError)
 	fs.SetOutput(stderr)
-	op := fs.String("op", "", "operation: list, read, or write")
+	op := fs.String("op", "", "operation: list, read, write, mkdir, delete, rename, upload or unzip")
+	browseURL := fs.String("browse-url", "", "internal command channel for a read-only file browser")
 	path := fs.String("path", "", "path to operate on, relative to the world root (empty = the root itself)")
 	worldsRoot := fs.String("worlds-root", "/data", "mount path of the world PVC; every path resolves under it")
+	expect := fs.String("expect-sha256", "", "write only: refuse unless the file's current SHA-256 (hex) is this")
+	createOnly := fs.Bool("create-only", false, "write only: refuse a path that already exists")
+	to := fs.String("to", "", "rename only: the destination path")
+	sourceURL := fs.String("source-url", "", "upload only: felis-api URL to fetch the bytes from")
+	size := fs.Int64("size", -1, "upload only: the byte count the fetched file must have")
+	sum := fs.String("sha256", "", "write and upload: the SHA-256 (hex) the content or the fetched file must have")
+	overwrite := fs.Bool("overwrite", false, "upload and unzip: replace files already there")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
 
+	if *browseURL != "" {
+		limitHeapToCgroup()
+		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer stop()
+		if err := fileedit.Browse(ctx, *worldsRoot, *browseURL, os.Getenv(fileedit.BrowserTokenEnv)); err != nil {
+			fmt.Fprintf(stderr, "felis files: %v\n", err)
+			return 1
+		}
+		return 0
+	}
 	if *op == "" {
-		fmt.Fprintln(stderr, "felis files: --op is required (list, read, or write)")
+		fmt.Fprintln(stderr, "felis files: --op is required")
 		return 2
 	}
+	limitHeapToCgroup()
+	req := fileedit.Request{
+		Op: *op, Path: *path, To: *to, Expect: *expect, CreateOnly: *createOnly, Overwrite: *overwrite,
+	}
 
 	// New content arrives base64-encoded in the environment rather than in argv:
 	// a process's arguments are world-readable on the node (/proc/<pid>/cmdline),
@@ -52,22 +79,46 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	// secrets — an RCON password in server.properties is the obvious case. The
 	// encoding is what lets arbitrary bytes (CRLF endings, a BOM, a NUL) survive a
 	// channel that must be a valid string.
-	var content []byte
-	if *op == fileedit.OpWrite {
-		raw, ok := os.LookupEnv(fileedit.ContentEnv)
-		if !ok {
-			fmt.Fprintf(stderr, "felis files: a write needs %s in the environment\n", fileedit.ContentEnv)
+	switch *op {
+	case fileedit.OpWrite:
+		// The content's SHA-256 comes with it, so bytes that changed on the way
+		// to this Job are refused rather than written (Request.ContentSHA256).
+		if *sum == "" {
+			fmt.Fprintln(stderr, "felis files: a write needs --sha256")
 			return 2
 		}
-		decoded, err := base64.StdEncoding.DecodeString(raw)
+		content, err := fileedit.ContentFromEnv(os.LookupEnv)
 		if err != nil {
-			fmt.Fprintf(stderr, "felis files: %s is not valid base64: %v\n", fileedit.ContentEnv, err)
+			fmt.Fprintf(stderr, "felis files: %v\n", err)
 			return 2
 		}
-		content = decoded
+		req.Content, req.ContentSHA256 = content, *sum
+	case fileedit.OpUpload:
+		token := os.Getenv(fileedit.UploadTokenEnv)
+		if *sourceURL == "" || token == "" {
+			fmt.Fprintf(stderr, "felis files: an upload needs --source-url and %s\n", fileedit.UploadTokenEnv)
+			return 2
+		}
+		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer stop()
+		req.Upload = &fileedit.Upload{
+			Size: *size, SHA256: *sum,
+			Open: func() (io.ReadCloser, error) { return fetchUpload(ctx, *sourceURL, token) },
+			Landed: func() {
+				if err := reportLanded(ctx, *sourceURL, token); err != nil {
+					// The file is in place; felis-api drops its copy when it
+					// has sat idle long enough, and the panel cancels it too.
+					fmt.Fprintf(stderr, "felis files: tell felis-api the upload landed: %v\n", err)
+				}
+			},
+		}
 	}
+	// An upload or an unzip (the only ops that report progress) can run long
+	// enough that felis-api does not wait on its Job, and the panel shows how far
+	// it has got from the latest of these lines (fileedit.K8sRunner.Ops).
+	req.Progress = fileedit.ThrottledProgress(stdout, time.Second, time.Now)
 
-	res, err := fileedit.Execute(*worldsRoot, *op, *path, content)
+	res, err := fileedit.Execute(*worldsRoot, req)
 	if err != nil {
 		// The operation could not be attempted — infrastructure, not caller fault.
 		fmt.Fprintf(stderr, "felis files: %v\n", err)
@@ -82,3 +133,57 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// fetchUpload opens the staged upload on felis-api's internal face. There is no
+// retry: the token opens the upload once (fileedit.Stage), so a second attempt
+// could only be refused, and the caller retries the failed Job whole (a file
+// sent in parts stays staged until its Job reports it landed, so that retry
+// does not send it again). Redirects are refused because the request carries the
+// token and the internal face never redirects; the header timeout catches a
+// wedged endpoint, and the Job's activeDeadlineSeconds bounds the body.
+func fetchUpload(ctx context.Context, url, token string) (io.ReadCloser, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: 30 * time.Second},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, fmt.Errorf("GET returned %s", resp.Status)
+	}
+	return resp.Body, nil
+}
+
+// reportLanded tells felis-api the upload's file is in place (DELETE on the URL
+// it was fetched from, with the same token), so it deletes the copy it staged.
+// One try: the file has landed whatever the answer, and a copy nobody deletes
+// is dropped once it has sat idle for fileedit.SessionIdle.
+func reportLanded(ctx context.Context, url, token string) error {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	client := &http.Client{
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		return fmt.Errorf("DELETE returned %s", resp.Status)
+	}
+	return nil
+}

cmd/felis/files_test.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"cmp"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"felis.lolicon.best/internal/fileedit"
+)
+
+// filesResult is the Result a `felis files` run printed on its marked line,
+// the last it prints.
+func filesResult(t *testing.T, stdout string) fileedit.Result {
+	t.Helper()
+	lines := strings.Split(strings.TrimSpace(stdout), "\n")
+	line, ok := strings.CutPrefix(lines[len(lines)-1], fileedit.ResultPrefix)
+	if !ok {
+		t.Fatalf("stdout has no result line: %q", stdout)
+	}
+	var res fileedit.Result
+	if err := json.Unmarshal([]byte(line), &res); err != nil {
+		t.Fatalf("result line %q: %v", line, err)
+	}
+	return res
+}
+
+// stagedSource is felis-api's internal face for one staged upload. It serves
+// body to a GET carrying Bearer token and 404 to any other, and answers the
+// DELETE that reports the file landed with landedCode (204 when unset),
+// redirecting to landedTo when that is a redirect. reports counts those
+// DELETEs, each with the token and at the path the bytes came from.
+type stagedSource struct {
+	*httptest.Server
+	reports, strays atomic.Int32
+	landedCode      int
+	landedTo        string
+}
+
+func stagedUpload(t *testing.T, token string, body []byte) *stagedSource {
+	t.Helper()
+	s := &stagedSource{}
+	s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer "+token || r.URL.Path != "/u" {
+			s.strays.Add(1)
+			http.Error(w, "no such upload", http.StatusNotFound)
+			return
+		}
+		switch r.Method {
+		case http.MethodGet:
+			w.Write(body)
+		case http.MethodDelete:
+			s.reports.Add(1)
+			if s.landedTo != "" {
+				w.Header().Set("Location", s.landedTo)
+			}
+			w.WriteHeader(cmp.Or(s.landedCode, http.StatusNoContent))
+		default:
+			s.strays.Add(1)
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		}
+	}))
+	t.Cleanup(s.Close)
+	return s
+}
+
+func uploadArgs(root, sourceURL string, body []byte) []string {
+	sum := sha256.Sum256(body)
+	return []string{
+		"--op", "upload", "--path", "plugins/a.jar", "--worlds-root", root,
+		"--source-url", sourceURL, "--size", "4", "--sha256", hex.EncodeToString(sum[:]),
+	}
+}
+
+// uploadRoot is a world with the plugins folder an upload lands in.
+func uploadRoot(t *testing.T) string {
+	t.Helper()
+	root := t.TempDir()
+	if err := os.Mkdir(filepath.Join(root, "plugins"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	return root
+}
+
+func TestCmdFilesUpload(t *testing.T) {
+	body := []byte("PK\x03\x04")
+
+	t.Run("fetches the staged bytes with its token and lands them", func(t *testing.T) {
+		root := uploadRoot(t)
+		srv := stagedUpload(t, "tok", body)
+		t.Setenv(fileedit.UploadTokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix+`{"done":4,"total":4}`+"\n") {
+			t.Fatalf("stdout %q does not start with the progress to the last byte", stdout.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != "" {
+			t.Fatalf("result = %+v", res)
+		}
+		got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar"))
+		if err != nil || !bytes.Equal(got, body) {
+			t.Fatalf("landed %q, %v", got, err)
+		}
+		if n, strays := srv.reports.Load(), srv.strays.Load(); n != 1 || strays != 0 || stderr.Len() != 0 {
+			t.Fatalf("reported landed %d times, %d stray requests, stderr %q; want once", n, strays, stderr.String())
+		}
+	})
+
+	t.Run("a file already there is a result, and nothing is reported landed", func(t *testing.T) {
+		root := uploadRoot(t)
+		if err := os.WriteFile(filepath.Join(root, "plugins", "a.jar"), []byte("old!"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		srv := stagedUpload(t, "tok", body)
+		t.Setenv(fileedit.UploadTokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeExists || srv.reports.Load() != 0 {
+			t.Fatalf("result = %+v, reported landed %d times", res, srv.reports.Load())
+		}
+	})
+
+	// The file is in place whatever felis-api answers, so the Job still succeeds
+	// and says why the staged copy may linger. A redirect is not followed, since
+	// the request carries the token.
+	for name, tc := range map[string]struct {
+		code   int
+		stderr string
+	}{
+		"refused":    {http.StatusNotFound, "felis files: tell felis-api the upload landed: DELETE returned 404 Not Found\n"},
+		"redirected": {http.StatusFound, "felis files: tell felis-api the upload landed: DELETE returned 302 Found\n"},
+	} {
+		t.Run("a landed report "+name+" still lands the file", func(t *testing.T) {
+			var elsewhere atomic.Int32
+			away := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { elsewhere.Add(1) }))
+			defer away.Close()
+			root := uploadRoot(t)
+			srv := stagedUpload(t, "tok", body)
+			srv.landedCode, srv.landedTo = tc.code, away.URL+"/u"
+			t.Setenv(fileedit.UploadTokenEnv, "tok")
+			var stdout, stderr bytes.Buffer
+			if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
+				t.Fatalf("exit %d, stderr %q", code, stderr.String())
+			}
+			if res := filesResult(t, stdout.String()); res.Code != "" || stderr.String() != tc.stderr || elsewhere.Load() != 0 {
+				t.Fatalf("result = %+v, stderr %q, redirect followed %d times", res, stderr.String(), elsewhere.Load())
+			}
+			if got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar")); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("landed %q, %v", got, err)
+			}
+		})
+	}
+
+	// A refused fetch is the Job failing, never a Result: the API answers it with a
+	// 500 the caller retries whole.
+	t.Run("a refused fetch exits 1 and lands nothing", func(t *testing.T) {
+		root := uploadRoot(t)
+		srv := stagedUpload(t, "tok", body)
+		t.Setenv(fileedit.UploadTokenEnv, "wrong")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1; stdout %q", code, stdout.String())
+		}
+		if !strings.Contains(stderr.String(), "404") {
+			t.Fatalf("stderr %q does not name the status", stderr.String())
+		}
+		if srv.reports.Load() != 0 {
+			t.Fatal("a refused fetch was reported landed")
+		}
+		if _, err := os.Lstat(filepath.Join(root, "plugins", "a.jar")); !os.IsNotExist(err) {
+			t.Fatalf("a refused fetch left a file: %v", err)
+		}
+	})
+
+	// The request carries the token, and the internal face never redirects, so a
+	// redirect is refused rather than followed with the token attached.
+	t.Run("a redirect is not followed", func(t *testing.T) {
+		root := uploadRoot(t)
+		var hits atomic.Int32
+		elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			w.Write(body)
+		}))
+		defer elsewhere.Close()
+		redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.URL+"/u", http.StatusFound))
+		defer redirecting.Close()
+		t.Setenv(fileedit.UploadTokenEnv, "tok")
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(uploadArgs(root, redirecting.URL+"/u", body), &stdout, &stderr); code != 1 {
+			t.Fatalf("exit %d, want 1", code)
+		}
+		if n := hits.Load(); n != 0 {
+			t.Fatalf("the redirect target was fetched %d times", n)
+		}
+	})
+
+	for name, tc := range map[string]struct {
+		token string
+		drop  string
+	}{
+		"no token":      {"", ""},
+		"no source URL": {"tok", "--source-url"},
+	} {
+		t.Run(name+" exits 2", func(t *testing.T) {
+			srv := stagedUpload(t, "tok", body)
+			t.Setenv(fileedit.UploadTokenEnv, tc.token)
+			args := uploadArgs(uploadRoot(t), srv.URL+"/u", body)
+			if tc.drop != "" {
+				for i, a := range args {
+					if a == tc.drop {
+						args = append(args[:i:i], args[i+2:]...)
+						break
+					}
+				}
+			}
+			var stdout, stderr bytes.Buffer
+			if code := cmdFiles(args, &stdout, &stderr); code != 2 {
+				t.Fatalf("exit %d, want 2", code)
+			}
+		})
+	}
+}
+
+func TestCmdFilesWrite(t *testing.T) {
+	root := t.TempDir()
+	content := []byte("[]\r\n")
+	sum := sha256.Sum256(content)
+	args := []string{"--op", "write", "--path", "ops.json", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}
+
+	t.Run("reassembles the content parts", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "1")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles(args, &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != "" {
+			t.Fatalf("result = %+v", res)
+		}
+		if got, err := os.ReadFile(filepath.Join(root, "ops.json")); err != nil || !bytes.Equal(got, content) {
+			t.Fatalf("wrote %q, %v", got, err)
+		}
+	})
+
+	// Writing what did arrive of an incomplete spec would truncate the file.
+	t.Run("an incomplete content spec exits 2 and writes nothing", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "2")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("x")))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 2 {
+			t.Fatalf("exit %d, want 2", code)
+		}
+		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
+			t.Fatalf("an incomplete spec wrote a file: %v", err)
+		}
+	})
+
+	// Without the content's SHA-256 the Job could not tell bytes changed on the
+	// way from the bytes felis-api sent.
+	t.Run("a write without its SHA-256 exits 2 and writes nothing", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "1")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root}, &stdout, &stderr); code != 2 {
+			t.Fatalf("exit %d, want 2", code)
+		}
+		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
+			t.Fatalf("a write without its SHA-256 wrote a file: %v", err)
+		}
+	})
+
+	t.Run("content that changed on the way is a result and writes nothing", func(t *testing.T) {
+		t.Setenv(fileedit.ContentPartsEnv, "1")
+		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("[]\n")))
+		var stdout, stderr bytes.Buffer
+		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 0 {
+			t.Fatalf("exit %d, stderr %q", code, stderr.String())
+		}
+		if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeDigestMismatch {
+			t.Fatalf("result = %+v, want %s", res, fileedit.CodeDigestMismatch)
+		}
+		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
+			t.Fatalf("changed content wrote a file: %v", err)
+		}
+	})
+}
+
+// A caller-fault outcome is a successful run carrying a code, so felis-api can
+// answer the precise 4xx instead of a 500.
+func TestCmdFilesCallerFaultIsAResult(t *testing.T) {
+	root := t.TempDir()
+	var stdout, stderr bytes.Buffer
+	if code := cmdFiles([]string{"--op", "mkdir", "--path", "../out", "--worlds-root", root}, &stdout, &stderr); code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, stderr.String())
+	}
+	if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeBadPath {
+		t.Fatalf("result = %+v, want code %s", res, fileedit.CodeBadPath)
+	}
+	stdout.Reset()
+	if code := cmdFiles([]string{"--worlds-root", root}, &stdout, &stderr); code != 2 {
+		t.Fatalf("no --op: exit %d, want 2", code)
+	}
+}
+
+// TestCmdFilesUnzip checks an unzip extracts next to the archive and reports its
+// progress before its result, the same way an upload does.
+func TestCmdFilesUnzip(t *testing.T) {
+	root := t.TempDir()
+	if err := os.Mkdir(filepath.Join(root, "maps"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	var zb bytes.Buffer
+	zw := zip.NewWriter(&zb)
+	for name, body := range map[string]string{"world/level.dat": "level", "world/region/r.0.0.mca": "region!"} {
+		w, err := zw.Create(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		io.WriteString(w, body)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "maps", "a.zip"), zb.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var stdout, stderr bytes.Buffer
+	if code := cmdFiles([]string{"--op", "unzip", "--path", "maps/a.zip", "--worlds-root", root}, &stdout, &stderr); code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, stderr.String())
+	}
+	if res := filesResult(t, stdout.String()); res.Code != "" || res.Files != 2 || res.Bytes != 12 {
+		t.Fatalf("result = %+v", res)
+	}
+	if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix) ||
+		!strings.Contains(stdout.String(), fileedit.ProgressPrefix+`{"done":12,"total":12}`+"\n") {
+		t.Fatalf("stdout %q does not report the progress to the last byte", stdout.String())
+	}
+	got, err := os.ReadFile(filepath.Join(root, "maps", "world", "region", "r.0.0.mca"))
+	if err != nil || string(got) != "region!" {
+		t.Fatalf("extracted %q, %v", got, err)
+	}
+}

cmd/felis/hostcreds.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// Host copies of the credentials `felis setup` takes at the keyboard: the [smtp]
+// relay password and the uploads bucket's keys. The cluster reads them from the
+// felis-smtp and felis-uploads-s3 Secrets, and a Secret lives in k3s's datastore,
+// which a reinstall (uninstall.sh keeps /etc/felis) or a host rebuilt from a
+// database bundle's state/ starts empty. Each file holds the bare value, mode
+// 0600, directly in /etc/felis beside secrets.env: every installer run applies
+// the Secrets from these files, and every database bundle, so the off-site copy
+// too, carries them.
+const (
+	hostSMTPPasswordPath       = "/etc/felis/smtp-password"
+	hostUploadsS3AccessKeyPath = "/etc/felis/uploads-s3-access-key"
+	hostUploadsS3SecretKeyPath = "/etc/felis/uploads-s3-secret-key"
+)
+
+// writeHostCredential replaces the file at path with value, mode 0600, through a
+// temporary file in the same directory, so a crash leaves the old value or the
+// new one and never a partial one.
+func writeHostCredential(path, value string) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	defer os.Remove(tmpPath)
+	// CreateTemp already makes the file 0600; the Chmod states it rather than
+	// leaning on that.
+	if err := tmp.Chmod(0o600); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if _, err := tmp.WriteString(value); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpPath, path)
+}
+
+// readHostCredential returns the value in path; ok is false when there is no
+// such file. An empty file is a value: the relay password of a relay without AUTH.
+func readHostCredential(path string) (value string, ok bool, err error) {
+	b, err := os.ReadFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return string(b), true, nil
+}
+
+// relayPassword is the [smtp] relay password as the host holds it: the copy
+// `felis setup` keeps at path, else, on an install from before that copy, the
+// felis-smtp Secret in ns, whose absence means a relay without AUTH. cl is only
+// used when the file is missing; a nil cl then reports errClusterUnreachable.
+func relayPassword(ctx context.Context, path string, cl client.Client, ns string) (string, error) {
+	if pw, ok, err := readHostCredential(path); err != nil || ok {
+		return pw, err
+	}
+	if cl == nil {
+		return "", errClusterUnreachable
+	}
+	return smtpSecretPassword(ctx, cl, ns)
+}
+
+var errClusterUnreachable = errors.New("the cluster did not answer")

cmd/felis/hostcreds_test.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/watchdog"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func TestHostCredentialFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "smtp-password")
+
+	if _, ok, err := readHostCredential(path); ok || err != nil {
+		t.Fatalf("a missing file read as ok=%v err=%v; want not there", ok, err)
+	}
+	// A copy an operator put there by hand, readable by everyone, is tightened.
+	if err := os.WriteFile(path, []byte("by hand"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range []string{"first secret", "a \"quoted\" $second\nsecret", ""} {
+		if err := writeHostCredential(path, v); err != nil {
+			t.Fatal(err)
+		}
+		got, ok, err := readHostCredential(path)
+		if err != nil || !ok || got != v {
+			t.Fatalf("read back (%q, %v, %v); want (%q, true, nil)", got, ok, err, v)
+		}
+		fi, err := os.Stat(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if mode := fi.Mode().Perm(); mode != 0o600 {
+			t.Fatalf("mode %v; want 0600", mode)
+		}
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 1 {
+		t.Fatalf("the directory holds %d entries; want the credential alone, no leftover temporary file", len(entries))
+	}
+
+	if _, _, err := readHostCredential(dir); err == nil {
+		t.Fatal("an unreadable credential read as fine")
+	}
+}
+
+func smtpSecretClient(t *testing.T, password string) client.Client {
+	t.Helper()
+	return fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
+		Data:       map[string][]byte{platform.SMTPSecretPasswordKey: []byte(password)},
+	}).Build()
+}
+
+func TestRelayPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	cl := smtpSecretClient(t, "from-secret")
+
+	if pw, err := relayPassword(ctx, host, cl, "felis"); err != nil || pw != "from-secret" {
+		t.Fatalf("without a host copy = (%q, %v); want the Secret's", pw, err)
+	}
+	if _, err := relayPassword(ctx, host, nil, "felis"); !errors.Is(err, errClusterUnreachable) {
+		t.Fatalf("without a host copy or a cluster err = %v; want errClusterUnreachable", err)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []client.Client{cl, nil} {
+		if pw, err := relayPassword(ctx, host, c, "felis"); err != nil || pw != "from-host" {
+			t.Fatalf("with a host copy (cluster %v) = (%q, %v); want the host copy", c != nil, pw, err)
+		}
+	}
+	if _, err := relayPassword(ctx, dir, cl, "felis"); err == nil {
+		t.Fatal("an unreadable host copy fell through to the Secret")
+	}
+}
+
+// The watchdog mails the most while the cluster is down: the host copy must
+// reach it then, and without one the password the last good run cached stays.
+func TestRefreshSMTPPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	var stderr bytes.Buffer
+
+	state := &watchdog.State{SMTPPassword: "cached"}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "cached" || stderr.Len() != 0 {
+		t.Fatalf("cluster down, no host copy: password %q, stderr %q; want the cached one kept quietly", state.SMTPPassword, stderr.String())
+	}
+	refreshSMTPPassword(ctx, host, smtpSecretClient(t, "from-secret"), "felis", state, &stderr)
+	if state.SMTPPassword != "from-secret" {
+		t.Fatalf("cluster up, no host copy: password %q; want the Secret's", state.SMTPPassword)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cluster down, host copy: password %q; want the host copy", state.SMTPPassword)
+	}
+	refreshSMTPPassword(ctx, dir, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" || !strings.Contains(stderr.String(), "keeping the cached one") {
+		t.Fatalf("unreadable host copy: password %q, stderr %q; want the cached one kept and the failure said", state.SMTPPassword, stderr.String())
+	}
+}
+
+func TestHostRecoveryMailerHostCopy(t *testing.T) {
+	ctx := context.Background()
+	// No kubeconfig anywhere: reaching for the cluster fails, so a pass proves
+	// the host copy was enough.
+	t.Setenv("KUBECONFIG", filepath.Join(t.TempDir(), "no-kubeconfig"))
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	off := false
+	c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_UNSET_RELAY_PW", RequireTLS: &off}
+
+	got, err := hostRecoveryMailer(c, host, "felis")(ctx)
+	if err != nil {
+		t.Fatalf("with the host copy: %v", err)
+	}
+	if relay, ok := got.(*mail.SMTP); !ok || relay.Password != "from-host" {
+		t.Fatalf("relay = %#v; want the host copy's password", got)
+	}
+	if _, err := hostRecoveryMailer(c, dir, "felis")(ctx); err == nil || !strings.Contains(err.Error(), "read the relay password") {
+		t.Fatalf("unreadable host copy: err = %v; want it named", err)
+	}
+	if _, err := hostRecoveryMailer(c, filepath.Join(dir, "none"), "felis")(ctx); err == nil || !strings.Contains(err.Error(), "reach the cluster") {
+		t.Fatalf("no host copy and no cluster: err = %v; want the cluster named", err)
+	}
+}
+
+// A whole watchdog run with the API server and PostgreSQL both down still
+// takes the relay password from the host copy, so the outage mail can
+// authenticate even when no earlier run cached it.
+func TestWatchdogReadsHostCopyWhileClusterDown(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
+	cfgPath := filepath.Join(dir, "felis.toml")
+	if err := os.WriteFile(cfgPath, []byte(`[database]
+url = "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
+[server]
+root_domain = "example.com"
+[archive]
+store = "tarLocal"
+[k8s]
+egress_mode = "nodeport"
+[smtp]
+host = "127.0.0.1"
+port = 1
+from = "[email protected]"
+username = "felis"
+password_ref = "FELIS_TEST_UNSET_RELAY_PW"
+`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	pwPath := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(pwPath, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	statePath := filepath.Join(dir, "state.json")
+	var stdout, stderr bytes.Buffer
+	cmdWatchdog([]string{
+		"-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"),
+		"-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", pwPath, "-heartbeat-file", filepath.Join(dir, "no-heartbeat"),
+	}, &stdout, &stderr)
+	if !strings.Contains(stdout.String(), "kube-api") {
+		t.Fatalf("the run found the API server up; the test needs it down (stdout %s)", stdout.String())
+	}
+	state, err := watchdog.LoadState(statePath)
+	if err != nil {
+		t.Fatalf("load state: %v (stderr %s)", err, stderr.String())
+	}
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cached relay password %q; want the host copy (stdout %s, stderr %s)", state.SMTPPassword, stdout.String(), stderr.String())
+	}
+}

cmd/felis/imagebundle.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"felis.lolicon.best/internal/imagepush"
+)
+
+// cmdImageBundle writes a release's image bundle: one OCI layout tar holding every
+// image an install runs, for one platform, plus its listing (one "role name
+// manifest-digest config-digest" line per image). deploy/build-release-artifacts.sh
+// runs it in CI; deploy/bootstrap.sh imports the tar into k3s's containerd and
+// pushes it into the platform registry with push-image --image.
+//
+//	--layout role=name=path   an image buildx wrote with --output type=oci
+//	--pull role=ref           a digest-pinned public image, named repository@digest
+//
+// The tar and the listing are written beside their final paths and renamed into
+// place, so a failed run leaves neither behind.
+func cmdImageBundle(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("image-bundle", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	platform := fs.String("platform", "", "os/arch the bundle is for, e.g. linux/arm64")
+	out := fs.String("out", "", "path of the bundle tar to write")
+	list := fs.String("list", "", "path of the listing to write")
+	var images []imagepush.BundleImage
+	fs.Func("layout", "role=name=path of an OCI layout tar (repeatable)", func(v string) error {
+		role, rest, ok := strings.Cut(v, "=")
+		name, path, ok2 := strings.Cut(rest, "=")
+		if !ok || !ok2 || role == "" || name == "" || path == "" {
+			return fmt.Errorf("want role=name=path, got %q", v)
+		}
+		images = append(images, imagepush.BundleImage{Role: role, Name: name, Layout: path})
+		return nil
+	})
+	fs.Func("pull", "role=ref of a digest-pinned public image (repeatable)", func(v string) error {
+		role, ref, ok := strings.Cut(v, "=")
+		if !ok || role == "" || !strings.Contains(ref, "@sha256:") {
+			return fmt.Errorf("want role=ref with ref pinned by digest, got %q", v)
+		}
+		images = append(images, imagepush.BundleImage{Role: role, Name: imagepush.PinnedName(ref), Source: ref})
+		return nil
+	})
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if *platform == "" || *out == "" || *list == "" || len(images) == 0 {
+		fmt.Fprintln(stderr, "felis image-bundle: --platform, --out, --list and at least one --layout or --pull are required")
+		return 2
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	if err := writeImageBundle(ctx, &imagepush.Source{Platform: *platform}, images, *out, *list); err != nil {
+		fmt.Fprintf(stderr, "felis image-bundle: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+func writeImageBundle(ctx context.Context, s *imagepush.Source, images []imagepush.BundleImage, out, list string) (err error) {
+	tmpOut, tmpList := out+".tmp", list+".tmp"
+	defer func() {
+		if err != nil {
+			os.Remove(tmpOut)
+			os.Remove(tmpList)
+		}
+	}()
+	f, err := os.Create(tmpOut)
+	if err != nil {
+		return err
+	}
+	w := bufio.NewWriterSize(f, 1<<20)
+	entries, err := imagepush.WriteBundle(ctx, s, images, w)
+	if err == nil {
+		err = w.Flush()
+	}
+	if err == nil {
+		err = f.Sync()
+	}
+	if cerr := f.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return err
+	}
+	var b strings.Builder
+	for _, e := range entries {
+		fmt.Fprintf(&b, "%s %s %s %s\n", e.Role, e.Name, e.Digest, e.Config)
+	}
+	if err := os.WriteFile(tmpList, []byte(b.String()), 0o644); err != nil {
+		return err
+	}
+	if err := os.Rename(tmpOut, out); err != nil {
+		return err
+	}
+	return os.Rename(tmpList, list)
+}

cmd/felis/imagebundle_test.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// writeTestLayout writes an OCI layout tar holding one linux/arch image with one
+// layer, the way buildx --output type=oci leaves a single-platform build, and
+// returns its path with the manifest and config digests.
+func writeTestLayout(t *testing.T, dir, arch, layer string) (path, manifestDigest, configDigest string) {
+	t.Helper()
+	blobs := map[string][]byte{}
+	add := func(b []byte) (string, int) {
+		sum := sha256.Sum256(b)
+		d := "sha256:" + hex.EncodeToString(sum[:])
+		blobs[d] = b
+		return d, len(b)
+	}
+	cfgDigest, cfgSize := add([]byte(`{"architecture":"` + arch + `","os":"linux","rootfs":{"type":"layers"}}`))
+	layerDigest, layerSize := add([]byte(layer))
+	manifest := fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",`+
+		`"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":%q,"size":%d},`+
+		`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":%q,"size":%d}]}`,
+		cfgDigest, cfgSize, layerDigest, layerSize)
+	mDigest, mSize := add([]byte(manifest))
+	index, _ := json.Marshal(map[string]any{
+		"schemaVersion": 2,
+		"manifests": []map[string]any{{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": mDigest, "size": mSize,
+		}},
+	})
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	put := func(name string, b []byte) {
+		tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(b)), Typeflag: tar.TypeReg})
+		tw.Write(b)
+	}
+	put("oci-layout", []byte(`{"imageLayoutVersion":"1.0.0"}`))
+	put("index.json", index)
+	for d, b := range blobs {
+		put("blobs/sha256/"+strings.TrimPrefix(d, "sha256:"), b)
+	}
+	tw.Close()
+	path = filepath.Join(dir, arch+"-"+layer+".tar")
+	if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return path, mDigest, cfgDigest
+}
+
+func TestImageBundleWritesTheListingTheInstallerReads(t *testing.T) {
+	dir := t.TempDir()
+	limbo, limboDigest, limboConfig := writeTestLayout(t, dir, "arm64", "limbo")
+	lobby, lobbyDigest, lobbyConfig := writeTestLayout(t, dir, "arm64", "lobby")
+	out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
+	var stderr bytes.Buffer
+	code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
+		"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo,
+		"--layout", "lobby=registry.felis.svc:5000/felis/lobby:demo=" + lobby,
+	}, nil, &stderr)
+	if code != 0 {
+		t.Fatalf("image-bundle = %d: %s", code, stderr.String())
+	}
+	// deploy/bootstrap.sh reads this with `read -r role name digest config`.
+	got, _ := os.ReadFile(list)
+	want := "limbo registry.felis.svc:5000/felis/limbo:demo " + limboDigest + " " + limboConfig + "\n" +
+		"lobby registry.felis.svc:5000/felis/lobby:demo " + lobbyDigest + " " + lobbyConfig + "\n"
+	if string(got) != want {
+		t.Errorf("listing:\n%s\nwant:\n%s", got, want)
+	}
+	if fi, err := os.Stat(out); err != nil || fi.Size() == 0 {
+		t.Errorf("bundle: %v", err)
+	}
+	if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
+		t.Errorf("left behind %v", leftovers)
+	}
+}
+
+func TestImageBundleLeavesNothingWhenAnImageIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	amd, _, _ := writeTestLayout(t, dir, "amd64", "limbo")
+	out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
+	// The pair an earlier run wrote stays as it was: a listing beside a bundle it
+	// does not describe would have the installer look for images that are not there.
+	os.WriteFile(out, []byte("old bundle"), 0o644)
+	os.WriteFile(list, []byte("old listing\n"), 0o644)
+	var stderr bytes.Buffer
+	code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
+		"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + amd}, nil, &stderr)
+	if code != 1 || !strings.Contains(stderr.String(), "is a linux/amd64 image") {
+		t.Fatalf("image-bundle = %d: %s", code, stderr.String())
+	}
+	if got, _ := os.ReadFile(out); string(got) != "old bundle" {
+		t.Errorf("bundle was replaced with %d bytes", len(got))
+	}
+	if got, _ := os.ReadFile(list); string(got) != "old listing\n" {
+		t.Errorf("listing was replaced with %q", got)
+	}
+	if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
+		t.Errorf("left behind %v", leftovers)
+	}
+}
+
+func TestPushImageReadsABundleByImageName(t *testing.T) {
+	dir := t.TempDir()
+	limbo, _, _ := writeTestLayout(t, dir, "arm64", "limbo")
+	out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
+	if code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
+		"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo}, nil, &bytes.Buffer{}); code != 0 {
+		t.Fatal("image-bundle failed")
+	}
+	t.Setenv("FELIS_REGISTRY_USERNAME", "platform")
+	t.Setenv("FELIS_REGISTRY_PASSWORD", "x")
+	// The name is looked up in the bundle's index before the registry is contacted,
+	// so a name the bundle lacks fails here with what it does hold.
+	var stderr bytes.Buffer
+	code := cmdPushImage([]string{"--tar", out, "--image", "registry.felis.svc:5000/felis/lobby:demo",
+		"--ref", "127.0.0.1:1/felis/lobby:demo"}, &bytes.Buffer{}, &stderr)
+	if code != 1 || !strings.Contains(stderr.String(), "holds no image named registry.felis.svc:5000/felis/lobby:demo (it holds: registry.felis.svc:5000/felis/limbo:demo") {
+		t.Fatalf("push-image = %d: %s", code, stderr.String())
+	}
+}

cmd/felis/initforwarding.go

@@ -20,18 +20,14 @@ const forwardingSecretEnv = "FELIS_FORWARDING_SECRET"
 // config/ and server.properties live under it.
 const defaultForwardingDataDir = "/data"
 
-// fwd*Mode make the written config readable AND rewritable by the main server
-// container, whose UID we do not control (an arbitrary user image). The
-// initContainer runs as root (see buildStatefulSet) so it can write into a data
-// volume of unknown ownership; 0666/0777 then let a non-root Paper rewrite the
-// same files on boot.
-//
-// This relies on the initContainer running as root to write into a volume of
-// unknown ownership; that is how the operator schedules it. If that ever changes,
-// give the server pod an fsGroup so the shared volume is group-writable instead.
+// fwd*Mode are the modes the written config lands with. The initContainer runs as
+// the same uid as the server container (naming.GameUID, pinned by the operator in
+// the pod securityContext) after the prepare-data initContainer has handed the
+// whole volume to that uid, so owner read/write is all the server needs to rewrite
+// these files on boot and nothing else on the node gets write access to them.
 const (
-	fwdFileMode os.FileMode = 0o666
-	fwdDirMode  os.FileMode = 0o777
+	fwdFileMode os.FileMode = 0o644
+	fwdDirMode  os.FileMode = 0o755
 )
 
 // cmdInitForwarding is the felis-image initContainer entrypoint that makes an
@@ -96,9 +92,8 @@ func writePaperGlobal(dataDir, secret string) error {
 	if err := os.MkdirAll(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("create %s: %w", dir, err)
 	}
-	// MkdirAll honours the process umask (root's is typically 022 → 0755); chmod
-	// does not, and a non-root main container must be able to place/replace the
-	// file in this directory on boot.
+	// MkdirAll honours the process umask; chmod does not, so a directory an older
+	// release left at 0777 is brought back to fwdDirMode here.
 	if err := os.Chmod(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("chmod %s: %w", dir, err)
 	}
@@ -188,8 +183,8 @@ func upsertProperty(content []byte, key, value string) []byte {
 }
 
 // writeFileMode writes data then forces the mode, since WriteFile honours the
-// umask (root's is typically 022 → 0644) but a non-root main container must be
-// able to rewrite these files on boot.
+// umask and leaves an existing file's mode alone: a file an older release wrote
+// world-writable (0666) is tightened back to fwdFileMode on the next boot.
 func writeFileMode(path string, data []byte) error {
 	if err := os.WriteFile(path, data, fwdFileMode); err != nil {
 		return fmt.Errorf("write %s: %w", path, err)

cmd/felis/initforwarding_test.go

@@ -164,8 +164,9 @@ func TestUpsertPropertyAppends(t *testing.T) {
 	}
 }
 
-// The written files must be group/world writable so a non-root main container can
-// rewrite them. chmod semantics are POSIX-only, so this asserts on non-Windows.
+// The written files land at fwdFileMode: owner-writable for the game uid the init
+// shares with the server container, and no longer world-writable. chmod semantics
+// are POSIX-only, so this asserts on non-Windows.
 func TestWriteForwardingFileModes(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("POSIX file modes not represented on Windows")

cmd/felis/initvolume.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"syscall"
+
+	"felis.lolicon.best/internal/naming"
+)
+
+// cmdInitVolume is the felis-image `prepare-data` initContainer entrypoint: it
+// hands every entry of a server's world volume to the game uid/gid before the
+// server container starts. The operator runs the server itself as naming.GameUID,
+// so a world written by an earlier release (whose server ran as root), a restore
+// Job (which extracts as root), or a storage provisioner that creates the volume
+// root-owned would otherwise leave files the server cannot write — a world that
+// boots and then fails every save.
+//
+// fsGroup covers only part of this: kubelet applies it to volume types that
+// support ownership management, and a k3s local-path PV is a hostPath underneath,
+// which it skips. A walk from inside the pod works for every volume type.
+//
+// Only mismatched entries are touched, so a volume already owned by the game uid
+// costs one lstat per entry and no writes. The walk runs inside an os.Root at the
+// data dir and uses lchown, so a symlink a plugin planted is re-owned as a link
+// and never followed out of the volume.
+//
+// A single entry that cannot be chowned is reported and skipped: failing the pod
+// over one odd file would keep the whole server down, while the server itself
+// reports the one file it cannot write. Only an unreadable data dir fails.
+func cmdInitVolume(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("init-volume", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	dataDir := fs.String("data", defaultForwardingDataDir, "world volume mount to hand to the game uid")
+	uid := fs.Int64("uid", naming.GameUID, "owner uid for every entry")
+	gid := fs.Int64("gid", naming.GameGID, "owner gid for every entry")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	root, err := os.OpenRoot(*dataDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: open %s: %v\n", *dataDir, err)
+		return 1
+	}
+	defer root.Close()
+	res, err := chownTree(root, int(*uid), int(*gid), root.Lchown)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: %v\n", err)
+		return 1
+	}
+	for _, f := range res.failures {
+		fmt.Fprintf(stderr, "felis init-volume: %s\n", f)
+	}
+	fmt.Fprintf(stdout, "felis init-volume: %d entries checked, %d handed to %d:%d, %d failed\n",
+		res.checked, res.changed, *uid, *gid, len(res.failures))
+	return 0
+}
+
+// chownResult tallies one walk; failures is capped so a volume of thousands of
+// unownable files cannot flood the pod log.
+type chownResult struct {
+	checked  int
+	changed  int
+	failures []string
+}
+
+const maxReportedChownFailures = 20
+
+// chownTree walks root and calls chown on every entry (the root dir included)
+// whose owner is not uid:gid. It returns an error only when the root itself
+// cannot be read; per-entry failures are collected in the result.
+func chownTree(root *os.Root, uid, gid int, chown func(name string, uid, gid int) error) (chownResult, error) {
+	var res chownResult
+	fail := func(name string, err error) {
+		if len(res.failures) < maxReportedChownFailures {
+			res.failures = append(res.failures, fmt.Sprintf("%s: %v", name, err))
+		} else if len(res.failures) == maxReportedChownFailures {
+			res.failures = append(res.failures, "further failures not listed")
+		}
+	}
+	err := fs.WalkDir(root.FS(), ".", func(name string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			if name == "." {
+				return walkErr
+			}
+			fail(name, walkErr)
+			// A directory that cannot be listed is skipped as a whole; a file
+			// error has nothing below it to skip.
+			if d != nil && d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		res.checked++
+		info, err := d.Info()
+		if err != nil {
+			fail(name, err)
+			return nil
+		}
+		if st, ok := info.Sys().(*syscall.Stat_t); ok && int(st.Uid) == uid && int(st.Gid) == gid {
+			return nil
+		}
+		if err := chown(name, uid, gid); err != nil {
+			if !errors.Is(err, fs.ErrNotExist) { // gone mid-walk: nothing left to own
+				fail(name, err)
+			}
+			return nil
+		}
+		res.changed++
+		return nil
+	})
+	return res, err
+}

cmd/felis/initvolume_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strconv"
+	"testing"
+)
+
+// openTree builds a small world under a temp dir: nested dirs, a file, and a
+// symlink pointing out of the volume that the walk must not follow.
+func openTree(t *testing.T) *os.Root {
+	t.Helper()
+	dir := t.TempDir()
+	for _, d := range []string{"world/region", "plugins"} {
+		if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, f := range []string{"level.dat", "world/region/r.0.0.mca"} {
+		if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	outside := t.TempDir()
+	if err := os.Symlink(outside, filepath.Join(dir, "plugins", "escape")); err != nil {
+		t.Fatal(err)
+	}
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { root.Close() })
+	return root
+}
+
+// Every entry owned by someone else is handed over, the root dir included, and a
+// symlink is re-owned as a link rather than walked into.
+func TestChownTreeHandsOverMismatchedEntries(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	var got []string
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		got = append(got, name)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	want := []string{".", "level.dat", "plugins", "plugins/escape", "world", "world/region", "world/region/r.0.0.mca"}
+	slices.Sort(got)
+	if !slices.Equal(got, want) {
+		t.Errorf("chowned %v, want %v", got, want)
+	}
+	if res.changed != len(want) || res.checked != len(want) || len(res.failures) != 0 {
+		t.Errorf("result = %+v, want %d checked and changed, no failures", res, len(want))
+	}
+}
+
+// A volume already owned by the game uid costs no chown at all: this is the steady
+// state every restart after the first one hits.
+func TestChownTreeSkipsMatchingOwner(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	calls := 0
+	res, err := chownTree(root, os.Getuid(), os.Getgid(), func(string, int, int) error {
+		calls++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if calls != 0 || res.changed != 0 {
+		t.Errorf("chown called %d times on an already-owned tree (result %+v)", calls, res)
+	}
+}
+
+// One entry that refuses the chown is reported and the walk carries on: a single
+// odd file must not keep the whole server from starting.
+func TestChownTreeContinuesPastFailures(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		if name == "level.dat" {
+			return os.ErrPermission
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if len(res.failures) != 1 || res.changed != 6 {
+		t.Errorf("result = %+v, want 1 failure and 6 changed", res)
+	}
+}
+
+// Against a real directory the owner already matches, so the command succeeds
+// without needing CAP_CHOWN — the path every test runner (non-root) can take.
+func TestCmdInitVolumeOwnedTree(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "server.properties"), []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	var out, errb bytes.Buffer
+	code := cmdInitVolume([]string{"--data", dir, "--uid", strconv.Itoa(os.Getuid()), "--gid", strconv.Itoa(os.Getgid())}, &out, &errb)
+	if code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, errb.String())
+	}
+	if code := cmdInitVolume([]string{"--data", filepath.Join(dir, "missing")}, &out, &errb); code != 1 {
+		t.Errorf("missing data dir exit = %d, want 1", code)
+	}
+}

cmd/felis/main.go

@@ -6,8 +6,32 @@ package main
 
 import (
 	"os"
+	"path/filepath"
 )
 
 func main() {
+	ensureHostBinDirOnPath()
 	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
 }
+
+// hostBinDir is where deploy/bootstrap.sh installs felis, k3s and cloudflared.
+const hostBinDir = "/usr/local/bin"
+
+// ensureHostBinDirOnPath appends hostBinDir to PATH when it is missing, so the
+// k3s and cloudflared this binary execs are found beside it. sudo's secure_path
+// on EL leaves /usr/local/bin out: `sudo /usr/local/bin/felis db backup` would
+// otherwise run with no k3s to reach the database's pod through. Appended, so a
+// PATH that names another k3s first keeps it.
+func ensureHostBinDirOnPath() {
+	path := os.Getenv("PATH")
+	for _, dir := range filepath.SplitList(path) {
+		if dir == hostBinDir {
+			return
+		}
+	}
+	if path == "" {
+		os.Setenv("PATH", hostBinDir)
+		return
+	}
+	os.Setenv("PATH", path+string(os.PathListSeparator)+hostBinDir)
+}

cmd/felis/main_test.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+	"regexp"
+	"testing"
+)
+
+func TestEnsureHostBinDirOnPath(t *testing.T) {
+	for _, c := range []struct{ in, want string }{
+		{"/usr/sbin:/usr/bin", "/usr/sbin:/usr/bin:/usr/local/bin"},
+		{"/usr/local/bin:/usr/bin", "/usr/local/bin:/usr/bin"},
+		{"/opt/k3s:/usr/bin:/usr/local/bin", "/opt/k3s:/usr/bin:/usr/local/bin"},
+		{"", "/usr/local/bin"},
+	} {
+		t.Setenv("PATH", c.in)
+		ensureHostBinDirOnPath()
+		if got := os.Getenv("PATH"); got != c.want {
+			t.Errorf("PATH %q became %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+// run is what the tests drive, so the PATH fix must sit in main, before it.
+func TestMainFixesPathBeforeRunning(t *testing.T) {
+	b, err := os.ReadFile("main.go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !regexp.MustCompile(`func main\(\) \{\n\tensureHostBinDirOnPath\(\)\n\tos\.Exit\(run\(`).Match(b) {
+		t.Fatal("main does not call ensureHostBinDirOnPath before run")
+	}
+}

cmd/felis/manifests.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"path/filepath"
 	"strings"
 
 	"felis.lolicon.best/internal/platform"
@@ -26,16 +27,30 @@ func (m *multiFlag) Set(v string) error {
 // felis-reaper identity only when the retention reaper is enabled, gated with
 // its CronJob), the weak build/restore Job SAs, the build/minecraft
 // NetworkPolicies, and the running control-plane workloads (felis-api/operator
-// Deployments + the in-cluster registry Deployment/Service/PVC) — as a single
-// multi-document YAML stream on stdout, ready for `kubectl apply -f -`.
+// Deployments + the in-cluster registry Deployment/Service/PVC + the
+// world-archive PVC that backs backup/restore, unless --backup-pvc is emptied)
+// — as a single multi-document YAML stream on stdout, ready for
+// `kubectl apply -f -`.
 //
 // It is a pure renderer: it never contacts a cluster and holds no credentials.
 // --velocity-cidr records the proxy host addresses allowed by the game NetworkPolicy.
 // Kubernetes permits resident-node traffic regardless, but remote proxy deployments
 // need an explicit CIDR, so the renderer refuses to guess.
+//
+// --only postgres renders just the control-plane database (platform.PostgresObjects),
+// which the installer brings up before migrations, before it has anything else
+// to render the full bundle with; it needs neither --felis-image nor
+// --velocity-cidr.
 func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("manifests", flag.ContinueOnError)
 	fs.SetOutput(stderr)
+	distributed := fs.Bool("distributed", false, "enable approved workers and archive transport")
+	controller := fs.String("controller-node", "", "protected controller identity for A")
+	probe := fs.String("egress-probe", "", "reachable controller host:port denied to game Pods")
+	var registryNodes multiFlag
+	fs.Var(&registryNodes, "registry-node-cidr", "exact node pull source for the registry (repeatable)")
+	socket := fs.String("node-control-socket", "", "host node-control Unix socket (optional, API only)")
+	nodeControlNode := fs.String("node-control-node", "", "controller hostname hosting the socket")
 	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace the control plane (api/operator/reaper) runs in")
 	minecraftNS := fs.String("minecraft-namespace", platform.DefaultMinecraftNamespace, "namespace MinecraftServer workloads run in")
 	buildNS := fs.String("build-namespace", platform.DefaultBuildNamespace, "namespace image-build Jobs run in")
@@ -43,17 +58,40 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	registryPort := fs.Int("registry-port", 5000, "port the in-cluster registry listens on")
 	panelNodePort := fs.Int("panel-node-port", int(platform.DefaultPanelNodePort), "NodePort that exposes the built-in HTTPS panel/API origin")
 	felisImage := fs.String("felis-image", "", "container image the felis-api/operator Deployments run, also passed through as FELIS_IMAGE (REQUIRED)")
-	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry:2)")
-	backupPVC := fs.String("backup-pvc", "", "name of the backup PVC advertised to the restore executor via FELIS_BACKUP_PVC (default none = restore endpoint returns 503)")
-	worldsHostPath := fs.String("worlds-host-path", "", "node directory under which each world PVC is visible as <path>/<pvc>; enables the reaper CronJob (requires --backup-pvc and --archive-local-path)")
-	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
+	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry 2.8.3, pinned by digest)")
+	postgresImage := fs.String("postgres-image", "", "control-plane database image (default: PostgreSQL 18.6, pinned by digest)")
+	only := fs.String("only", "", `render one part of the bundle instead of all of it; "postgres" is the control-plane database`)
+	backupPVC := fs.String("backup-pvc", "felis-backups", "name of the world-archive PVC this bundle renders in the Minecraft namespace and advertises to the backup/restore executors via FELIS_BACKUP_PVC (default: felis-backups; pass an empty value to render none, leaving backup/restore answering 503)")
+	worldsHostPath := fs.String("worlds-host-path", "", "node directory the reaper reads worlds from: each world PVC resolves as <path>/<pvc>, or as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (k3s storage root: /var/lib/rancher/k3s/storage); enables the reaper CronJob (requires --archive-local-path and a non-empty --backup-pvc)")
+	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path. With the backup PVC alone it renders the retention-only CronJob, which deletes backups past their expiry and never touches a world")
+	registryStorage := fs.String("registry-storage", "", "capacity the registry PVC requests (default 10Gi; k3s local-path does not enforce it)")
+	uploadsStorage := fs.String("uploads-storage", "", "capacity the uploads PVC requests (default 5Gi; k3s local-path does not enforce it)")
+	backupStorage := fs.String("backup-storage", "", "capacity the world-archive PVC requests (default 10Gi; k3s local-path does not enforce it)")
+	reaperNode := fs.String("reaper-node", "", "node that holds --worlds-host-path: pins the reaper CronJob's pod there via nodeSelector kubernetes.io/hostname (multi-node clusters need this, or the reaper may schedule where the hostPath is empty)")
 	var velocityCIDRs multiFlag
 	fs.Var(&velocityCIDRs, "velocity-cidr", "CIDR of a Velocity proxy host allowed to reach game port 25565 (repeatable, REQUIRED)")
 	var packageCIDRs multiFlag
 	fs.Var(&packageCIDRs, "package-cidr", "CIDR of a package mirror build Pods may reach (repeatable; default none = no internet egress)")
+	var serverDenyCIDRs multiFlag
+	fs.Var(&serverDenyCIDRs, "server-egress-deny-cidr", "extra CIDR game server pods may never reach, e.g. the node's public address (repeatable)")
+	var serverAllowCIDRs multiFlag
+	fs.Var(&serverAllowCIDRs, "server-egress-allow-cidr", "private CIDR game server pods may reach despite the private-range block, e.g. a LAN database (repeatable)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
+	switch *only {
+	case "":
+	case "postgres":
+		return renderManifests(stdout, stderr, platform.PostgresObjects(platform.Params{
+			ControllerNode:     *controller,
+			ControlNamespace:   *controlNS,
+			MinecraftNamespace: *minecraftNS,
+			PostgresImage:      *postgresImage,
+		}))
+	default:
+		fmt.Fprintf(stderr, "felis manifests: --only %q: the one part that renders alone is \"postgres\"\n", *only)
+		return 2
+	}
 
 	// Keep proxy placement explicit. This matters for remote proxies and documents
 	// the expected source even when Velocity runs on the resident node.
@@ -71,7 +109,9 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"(the felis-api/operator Deployments run it and it is passed through as FELIS_IMAGE, e.g. --felis-image registry.felis.svc:5000/felis:v1)")
 		return 2
 	}
-	for _, cidr := range append(append([]string{}, velocityCIDRs...), packageCIDRs...) {
+	allCIDRs := append(append([]string{}, velocityCIDRs...), packageCIDRs...)
+	allCIDRs = append(append(allCIDRs, serverDenyCIDRs...), serverAllowCIDRs...)
+	for _, cidr := range allCIDRs {
 		if _, _, err := net.ParseCIDR(cidr); err != nil {
 			fmt.Fprintf(stderr, "felis manifests: invalid CIDR %q: %v\n", cidr, err)
 			return 2
@@ -81,39 +121,76 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis manifests: --panel-node-port must be in Kubernetes NodePort range 30000-32767 (got %d)\n", *panelNodePort)
 		return 2
 	}
+	// The node pin exists only for the reaper's hostPath: naming a node without the
+	// worlds root would be silently dropped (no CronJob renders), so fail loud like
+	// the storage-trio check below.
+	if *reaperNode != "" && *worldsHostPath == "" && !*distributed {
+		fmt.Fprintln(stderr, "felis manifests: --reaper-node requires --worlds-host-path "+
+			"(it pins the reaper CronJob, which renders only with the retention storage trio)")
+		return 2
+	}
 
-	// Retention/reaper rendering is opt-in and needs all three storage coordinates
-	// together: where worlds live (to read+archive them), the backup PVC (to write
-	// archives into), and the path it is mounted at (which MUST equal felis.toml
-	// [archive] local_path so tarLocal's absolute archive refs resolve). A partial
-	// configuration is almost certainly an operator mistake, so fail loud rather than
-	// silently drop retention. Asking for it without the other two is rejected; an
-	// empty trio renders the bundle WITHOUT the reaper and says so.
+	// Retention/reaper rendering is opt-in and needs a storage topology together:
+	// where worlds live (to read+archive them), a backup PVC (to write archives
+	// into — rendered from --backup-pvc), and the path it is mounted at (which MUST
+	// equal felis.toml [archive] local_path so tarLocal's absolute archive refs
+	// resolve). A partial configuration is almost certainly an operator mistake, so
+	// fail loud rather than silently drop retention or render a reaper with nowhere
+	// to write. The backup PVC itself defaults to felis-backups (it is what makes a
+	// default install's backup endpoint work at all); retention additionally needs
+	// --worlds-host-path.
 	if *worldsHostPath != "" {
 		if *backupPVC == "" || *archiveLocalPath == "" {
 			fmt.Fprintln(stderr, "felis manifests: --worlds-host-path enables the reaper CronJob and requires "+
-				"--backup-pvc and --archive-local-path too (--archive-local-path must equal felis.toml [archive] local_path)")
+				"--archive-local-path (must equal felis.toml [archive] local_path) and a non-empty --backup-pvc "+
+				"(the archive store; default felis-backups)")
 			return 2
 		}
-		// The reaper WILL render. Two deployment preconditions this generator cannot
-		// check would SILENTLY turn retention into a no-op if unmet — surface them as
+		// The reaper WILL render. Two deployment facts this generator cannot check
+		// would silently turn retention into a no-op if unmet — surface them as
 		// loudly as the fail-closed cases above, so an operator is never left with a
-		// reaper that reaps an empty directory. (Both are also in the WorldsHostPath
-		// flag/field docs, but nobody deploying from stdout reads those.)
+		// reaper that reaps nothing. (Both are also in the WorldsHostPath flag/field
+		// docs, but nobody deploying from stdout reads those.)
+		pin := "the CronJob sets NO nodeSelector: a single-node starter pins it to the worlds implicitly, but on a " +
+			"multi-node cluster you MUST pass --reaper-node <name> (or add a nodeSelector) for the node holding the " +
+			"worlds, or the reaper may schedule where the hostPath is empty"
+		if *reaperNode != "" {
+			pin = fmt.Sprintf("the CronJob and its worlds-root PV are pinned to node %q via kubernetes.io/hostname — "+
+				"keep this pointed at the node that actually holds the world volumes", *reaperNode)
+		}
 		fmt.Fprintf(stderr, "felis manifests: note: rendering the retention reaper CronJob (worlds hostPath %q). "+
-			"Two preconditions are NOT verified here:\n"+
-			"  - each world PVC must be visible at %s/<pvc> on the node: a stock local-path-provisioner lays "+
-			"volumes under PV-name paths (.../pvc-<uuid>_<ns>_<pvc>/), so unless the worlds StorageClass is "+
-			"arranged to expose <path>/<pvc>, the reaper tars an empty directory;\n"+
-			"  - the CronJob sets NO nodeSelector: a single-node starter pins it to the worlds implicitly, but "+
-			"on a multi-node cluster you MUST add a nodeSelector for the node holding the worlds, or the reaper "+
-			"may schedule where the hostPath is empty.\n", *worldsHostPath, *worldsHostPath)
-	} else {
-		fmt.Fprintln(stderr, "felis manifests: note: retention reaper CronJob not rendered "+
-			"(pass --worlds-host-path, --backup-pvc and --archive-local-path to enable it)")
+			"These points are NOT verified here:\n"+
+			"  - the node's world volumes must actually live below %s: the reaper resolves a world as "+
+			"%s/<pvc>, then as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (what k3s "+
+			"writes under /var/lib/rancher/k3s/storage). Any other provisioner needs its volumes exposed as "+
+			"<path>/<pvc>, or each candidate's archive fails and the world is preserved;\n"+
+			"  - %s.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, pin)
+	} else if !*distributed {
+		switch {
+		case *archiveLocalPath != "" && *backupPVC == "":
+			fmt.Fprintln(stderr, "felis manifests: --archive-local-path names where the backup PVC is mounted, "+
+				"but --backup-pvc is empty (no archive store renders); drop one or the other")
+			return 2
+		case *archiveLocalPath != "":
+			fmt.Fprintln(stderr, "felis manifests: note: rendering the reaper CronJob retention-only: backups past "+
+				"their expiry are deleted daily, idle worlds are never archived or deleted "+
+				"(pass --worlds-host-path to reap them too)")
+		case *backupPVC != "":
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered: backups are never expired, so "+
+				"the archive store only grows until the disk fills (pass --archive-local-path, equal to felis.toml "+
+				"[archive] local_path, for the retention-only CronJob, and --worlds-host-path as well to reap idle worlds)")
+		default:
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered (backups are disabled)")
+		}
 	}
 
-	out, err := platform.RenderYAML(platform.Params{
+	if *socket != "" && (!filepath.IsAbs(*socket) || *nodeControlNode == "") {
+		fmt.Fprintln(stderr, "node-control requires an absolute socket and its controller hostname")
+		return 2
+	}
+	params := platform.Params{
+		NodeControlSocket: *socket, NodeControlNode: *nodeControlNode,
+		Distributed: *distributed, ControllerNode: *controller, EgressProbe: *probe, RegistryNodeCIDRs: registryNodes,
 		ControlNamespace:   *controlNS,
 		MinecraftNamespace: *minecraftNS,
 		BuildNamespace:     *buildNS,
@@ -122,12 +199,30 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 		PanelNodePort:      int32(*panelNodePort),
 		FelisImage:         *felisImage,
 		RegistryImage:      *registryImage,
+		PostgresImage:      *postgresImage,
 		BackupPVC:          *backupPVC,
 		WorldsHostPath:     *worldsHostPath,
+		ReaperNode:         *reaperNode,
 		ArchiveLocalPath:   *archiveLocalPath,
 		VelocityCIDRs:      []string(velocityCIDRs),
 		PackageSourceCIDRs: []string(packageCIDRs),
-	})
+
+		ServerEgressDenyCIDRs:  []string(serverDenyCIDRs),
+		ServerEgressAllowCIDRs: []string(serverAllowCIDRs),
+
+		RegistryStorage: *registryStorage,
+		UploadsStorage:  *uploadsStorage,
+		BackupStorage:   *backupStorage,
+	}
+	if err := params.Validate(); err != nil {
+		fmt.Fprintf(stderr, "felis manifests: %v\n", err)
+		return 2
+	}
+	return renderManifests(stdout, stderr, platform.Objects(params))
+}
+
+func renderManifests(stdout, stderr io.Writer, objs []platform.Object) int {
+	out, err := platform.RenderObjects(objs)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis manifests: render: %v\n", err)
 		return 1

cmd/felis/manifests_test.go

@@ -77,6 +77,10 @@ func TestManifestsRendersBundle(t *testing.T) {
 		"10.0.0.5/32",
 		// The felis image flows through to the Deployments.
 		"registry.felis.svc:5000/felis:v1",
+		// Backup works out of the box: the archive PVC renders and the api gets
+		// the env that wires the backup/restore executors to it.
+		"name: felis-backups",
+		"name: FELIS_BACKUP_PVC",
 	} {
 		if !strings.Contains(text, want) {
 			t.Errorf("rendered bundle missing %q", want)
@@ -86,24 +90,29 @@ func TestManifestsRendersBundle(t *testing.T) {
 		t.Error("rendered bundle must not contain ClusterRole/ClusterRoleBinding")
 	}
 	// Without the retention flags, the reaper CronJob is not rendered and the
-	// generator says so on stderr.
+	// generator says so on stderr, naming what that leaves: backups that never
+	// expire.
 	if strings.Contains(text, "kind: CronJob") {
-		t.Error("no reaper CronJob must render without --worlds-host-path")
+		t.Error("no reaper CronJob must render without --archive-local-path")
 	}
-	if !strings.Contains(errBuf.String(), "not rendered") {
-		t.Errorf("expected a 'reaper not rendered' notice on stderr, got %q", errBuf.String())
+	if !strings.Contains(errBuf.String(), "not rendered") || !strings.Contains(errBuf.String(), "never expired") {
+		t.Errorf("expected a 'reaper not rendered, backups never expired' notice on stderr, got %q", errBuf.String())
 	}
 }
 
-// TestManifestsReaperRequiresTrio proves --worlds-host-path is a fail-loud opt-in:
-// asking for the reaper without the backup PVC and its mount path (which must equal
-// [archive] local_path) is rejected rather than silently dropping retention.
-func TestManifestsReaperRequiresTrio(t *testing.T) {
+// TestManifestsReaperRequiresStorage proves --worlds-host-path is a fail-loud
+// opt-in: asking for the reaper without a writable archive store (the backup PVC,
+// which defaults to felis-backups but can be emptied) and its mount path (which
+// must equal [archive] local_path) is rejected rather than silently dropping
+// retention or deleting worlds it could not archive first.
+func TestManifestsReaperRequiresStorage(t *testing.T) {
 	base := []string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32", "--worlds-host-path", "/var/lib/felis/worlds"}
 	for _, extra := range [][]string{
-		{},                                   // neither backup-pvc nor archive-local-path
-		{"--backup-pvc", "felis-backups"},    // missing archive-local-path
-		{"--archive-local-path", "/backups"}, // missing backup-pvc
+		{},                        // missing archive-local-path (backup-pvc defaults)
+		{"--backup-pvc", "other"}, // still missing archive-local-path
+		// A reaper with no archive store would have nowhere to write the archive
+		// it must verify before deleting a world; emptying the PVC is rejected.
+		{"--archive-local-path", "/backups", "--backup-pvc="},
 	} {
 		var out, errBuf bytes.Buffer
 		code := run(append(append([]string{}, base...), extra...), &out, &errBuf)
@@ -119,6 +128,57 @@ func TestManifestsReaperRequiresTrio(t *testing.T) {
 	}
 }
 
+// TestManifestsBackupPVCOptOut proves --backup-pvc= renders a bundle with no
+// archive store at all: no PVC and no FELIS_BACKUP_PVC env, so backup/restore
+// answer 503 instead of pointing Jobs at a claim nobody provisions.
+func TestManifestsBackupPVCOptOut(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test",
+		"--velocity-cidr", "10.0.0.5/32", "--backup-pvc="}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	for _, absent := range []string{"felis-backups", "FELIS_BACKUP_PVC"} {
+		if strings.Contains(out.String(), absent) {
+			t.Errorf("--backup-pvc= bundle must not contain %q", absent)
+		}
+	}
+}
+
+// TestManifestsRendersRetentionOnly: the archive store without a worlds root
+// still gets the daily CronJob, retention-only, so backups past their expiry
+// leave the store on an install that never reaps a world; the operator is told
+// which of the two it got. An archive path with the store switched off is a
+// mistake and fails loud.
+func TestManifestsRendersRetentionOnly(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	text := out.String()
+	for _, want := range []string{"kind: CronJob", "name: felis-reaper", "--retention-only", "claimName: felis-backups"} {
+		if !strings.Contains(text, want) {
+			t.Errorf("retention-only bundle missing %q", want)
+		}
+	}
+	if strings.Contains(text, "kind: PersistentVolume\n") || strings.Contains(text, "--worlds-root") {
+		t.Error("a retention-only bundle must not reach for a worlds root")
+	}
+	if !strings.Contains(errBuf.String(), "retention-only") {
+		t.Errorf("stderr must say the CronJob is retention-only, got %q", errBuf.String())
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives", "--backup-pvc="}, &out, &errBuf)
+	if code != 2 || out.Len() != 0 || !strings.Contains(errBuf.String(), "--backup-pvc is empty") {
+		t.Errorf("archive path without an archive store: exit=%d out=%d bytes stderr=%q, want a fail-loud 2", code, out.Len(), errBuf.String())
+	}
+}
+
 // TestManifestsRendersReaper proves the happy path with the full retention trio:
 // a batch/v1 CronJob is emitted, named felis-reaper, mounting the backup PVC at the
 // supplied archive path.
@@ -150,9 +210,105 @@ func TestManifestsRendersReaper(t *testing.T) {
 	// this generator cannot verify (else a misarranged hostPath silently no-ops
 	// retention): the <path>/<pvc> arrangement-dependency and the multi-node
 	// nodeSelector hazard.
-	for _, want := range []string{"local-path-provisioner", "nodeSelector"} {
+	for _, want := range []string{"local-path", "nodeSelector"} {
 		if !strings.Contains(errBuf.String(), want) {
 			t.Errorf("reaper render must warn operators about %q on stderr, got %q", want, errBuf.String())
 		}
 	}
 }
+
+// TestManifestsReaperNodePin: --reaper-node pins the rendered CronJob's pod via
+// kubernetes.io/hostname and replaces the "no nodeSelector" hazard note with the
+// pin confirmation; using it without the worlds root is a fail-loud 2.
+func TestManifestsReaperNodePin(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{
+		"manifests",
+		"--felis-image", "registry.felis.svc:5000/felis:v1",
+		"--velocity-cidr", "10.0.0.5/32",
+		"--worlds-host-path", "/var/lib/felis/worlds",
+		"--archive-local-path", "/backups",
+		"--reaper-node", "node-a",
+	}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	for _, want := range []string{
+		"kubernetes.io/hostname: node-a",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("pinned render missing %q", want)
+		}
+	}
+	if !strings.Contains(errBuf.String(), "node-a") {
+		t.Errorf("stderr must confirm the pin, got %q", errBuf.String())
+	}
+
+	var out2, err2 bytes.Buffer
+	if code := run([]string{
+		"manifests",
+		"--felis-image", "registry.felis.svc:5000/felis:v1",
+		"--velocity-cidr", "10.0.0.5/32",
+		"--reaper-node", "node-a",
+	}, &out2, &err2); code != 2 {
+		t.Errorf("--reaper-node without --worlds-host-path: exit = %d, want 2", code)
+	}
+}
+
+// TestManifestsStorageSizes proves the PVC size flags reach the rendered claims
+// and a size the API server would reject fails before anything is applied.
+func TestManifestsStorageSizes(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "40Gi", "--uploads-storage", "8Gi"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, stderr = %s", code, errBuf.String())
+	}
+	for _, want := range []string{"storage: 40Gi", "storage: 8Gi"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("bundle lacks %q", want)
+		}
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "lots"}, &out, &errBuf)
+	if code == 0 || !strings.Contains(errBuf.String(), "registry storage") {
+		t.Errorf("--registry-storage lots: exit %d, stderr %q; want a refusal naming the flag", code, errBuf.String())
+	}
+}
+
+// TestManifestsOnlyPostgres: the installer renders the database before it has
+// an image or a proxy address for the rest of the bundle, so --only postgres
+// must render without them, and render the database and nothing else (a stray
+// Deployment in that apply would start without its identities).
+func TestManifestsOnlyPostgres(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--only", "postgres", "--control-namespace", "ctl", "--postgres-image", "example/pg:18@sha256:abc"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	var kinds []string
+	for _, doc := range strings.Split(out.String(), "\n---\n") {
+		for _, line := range strings.Split(doc, "\n") {
+			if strings.HasPrefix(line, "kind: ") {
+				kinds = append(kinds, strings.TrimPrefix(line, "kind: "))
+			}
+		}
+	}
+	if got, want := strings.Join(kinds, ","), "Namespace,ConfigMap,NetworkPolicy,Deployment,Service"; got != want {
+		t.Errorf("rendered kinds %s, want %s", got, want)
+	}
+	for _, want := range []string{"name: felis-postgres", "namespace: ctl", "image: example/pg:18@sha256:abc"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("rendered database missing %q", want)
+		}
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	if code := run([]string{"manifests", "--only", "registry"}, &out, &errBuf); code != 2 || out.Len() != 0 {
+		t.Errorf("--only registry: exit %d with %d bytes of YAML, want 2 and none", code, out.Len())
+	}
+}

cmd/felis/memlimit.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"os"
+	"runtime/debug"
+	"strconv"
+	"strings"
+)
+
+// cgroupMemoryFiles are where a container reads the memory it is allowed:
+// cgroup v2 first, then v1.
+var cgroupMemoryFiles = []string{"/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"}
+
+// limitHeapToCgroup sets the Go heap's soft limit from the container's memory
+// limit, so the collector works harder as a Job nears it and the kernel does not
+// kill the Job first. An extraction or a folder zipped for download keeps a few
+// hundred bytes per entry for as long as it runs; without the limit the heap
+// grows to twice that before a collection, and a 256 MiB Job was killed at
+// 400,000 entries whose live heap was 115 MB. GOMEMLIMIT set by hand wins.
+func limitHeapToCgroup() {
+	if os.Getenv("GOMEMLIMIT") != "" {
+		return
+	}
+	for _, f := range cgroupMemoryFiles {
+		b, err := os.ReadFile(f)
+		if err != nil {
+			continue
+		}
+		if n, ok := softMemoryLimit(string(b)); ok {
+			debug.SetMemoryLimit(n)
+		}
+		return
+	}
+}
+
+// softMemoryLimit answers three fifths of the limit a cgroup memory file holds,
+// or false for "max" (no limit) and anything unreadable. The rest is left for
+// what the kernel charges the container beyond the Go heap: the page cache of
+// the files it reads and writes, and the inodes it creates. Under a 256 MiB
+// limit, 400,000 extracted entries peaked at 184 MB resident with the heap held
+// to 150 MiB.
+func softMemoryLimit(content string) (int64, bool) {
+	n, err := strconv.ParseInt(strings.TrimSpace(content), 10, 64)
+	if err != nil || n <= 0 {
+		return 0, false
+	}
+	return n / 5 * 3, true
+}

cmd/felis/memlimit_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"math"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"testing"
+)
+
+func TestSoftMemoryLimit(t *testing.T) {
+	for _, c := range []struct {
+		in   string
+		want int64
+		ok   bool
+	}{
+		{"268435456\n", 161061273, true}, // 256 MiB, as memory.max holds it
+		{"max\n", 0, false},
+		{"0\n", 0, false},
+		{"-1", 0, false},
+		{"", 0, false},
+	} {
+		got, ok := softMemoryLimit(c.in)
+		if got != c.want || ok != c.ok {
+			t.Errorf("softMemoryLimit(%q) = %d %v, want %d %v", c.in, got, ok, c.want, c.ok)
+		}
+	}
+}
+
+// TestLimitHeapToCgroup checks the limit comes from the first cgroup file there
+// is, and that GOMEMLIMIT set by hand leaves the heap alone.
+func TestLimitHeapToCgroup(t *testing.T) {
+	prevFiles, prevLimit := cgroupMemoryFiles, debug.SetMemoryLimit(-1)
+	t.Cleanup(func() { cgroupMemoryFiles = prevFiles; debug.SetMemoryLimit(prevLimit) })
+	dir := t.TempDir()
+	v1, v1b := filepath.Join(dir, "v1"), filepath.Join(dir, "v1b")
+	if err := os.WriteFile(v1, []byte("268435456\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(v1b, []byte("536870912\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	cgroupMemoryFiles = []string{filepath.Join(dir, "missing"), v1, v1b}
+
+	t.Setenv("GOMEMLIMIT", "")
+	debug.SetMemoryLimit(math.MaxInt64)
+	limitHeapToCgroup()
+	if got := debug.SetMemoryLimit(-1); got != 161061273 {
+		t.Fatalf("limit = %d, want three fifths of 256 MiB", got)
+	}
+
+	t.Setenv("GOMEMLIMIT", "1GiB")
+	debug.SetMemoryLimit(math.MaxInt64)
+	limitHeapToCgroup()
+	if got := debug.SetMemoryLimit(-1); got != math.MaxInt64 {
+		t.Fatalf("limit = %d with GOMEMLIMIT set, want it left alone", got)
+	}
+}

cmd/felis/migrate.go

@@ -2,20 +2,32 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"time"
 
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
 // cmdMigrate implements `felis migrate up`: load config, open the database, and
 // apply every pending embedded migration under the advisory lock (spec §6).
+//
+// Migrations only roll forward, and some drop data (0017_drop_password), so a
+// database that already holds a schema and has migrations pending is bundled
+// first (internal/dbbackup, label pre-migrate). A failed snapshot stops the
+// upgrade; -no-backup is the explicit way past it, e.g. for an external
+// database (no [database] deployment, so the host's own pg_dump runs) whose
+// server is newer than that pg_dump.
 func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("migrate", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	backupDir := fs.String("backup-dir", dbbackup.DefaultDir, "where the pre-migration snapshot goes")
+	noBackup := fs.Bool("no-backup", false, "apply pending migrations without snapshotting the database first")
 	// The "up" verb precedes any flags (felis migrate up -config path). Go's
 	// flag.Parse stops at the first non-flag token and would never see a flag
 	// placed after "up", silently falling back to the default -config. Pull the
@@ -48,6 +60,18 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	if !*noBackup {
+		path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database, *backupDir, stderr)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis migrate: pre-migration backup failed, nothing applied: %v\n", err)
+			fmt.Fprintln(stderr, "  fix the backup, or re-run with -no-backup to migrate without one")
+			return 1
+		}
+		if path != "" {
+			fmt.Fprintf(stdout, "felis migrate: database snapshot %s\n", path)
+		}
+	}
+
 	applied, err := store.Up(ctx, drv, migrations)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis migrate: %v\n", err)
@@ -60,3 +84,98 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// preMigrateStateDir is the host state a pre-migrate bundle carries; tests
+// point it at a directory of their own.
+var preMigrateStateDir = dbbackup.DefaultStateDir
+
+// preMigrateBackup bundles the database when it already carries a schema and
+// some of migrations are not applied yet, and returns the bundle's path ("" when
+// there was nothing to protect: a fresh database, or nothing pending).
+func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, db config.DatabaseConfig, dir string, log io.Writer) (string, error) {
+	if err := drv.EnsureVersionTable(ctx); err != nil {
+		return "", fmt.Errorf("ensure version table: %w", err)
+	}
+	done, err := drv.AppliedVersions(ctx)
+	if err != nil {
+		return "", fmt.Errorf("read applied versions: %w", err)
+	}
+	if len(done) == 0 || !hasPending(done, migrations) {
+		return "", nil
+	}
+	tools, err := dbTools(db)
+	if err != nil {
+		return "", err
+	}
+	path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
+		DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelPreMigrate,
+		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: preMigrateStateDir,
+		Version: resolvedVersion(), ExportServers: exportMinecraftServers, Log: log, Record: true,
+	})
+	if errors.Is(err, dbbackup.ErrServersMissing) {
+		// Rolling the migration back needs the database alone. Backup logged
+		// the gap, and the panel and the watchdog show it while this is the
+		// newest bundle.
+		return path, nil
+	}
+	return path, err
+}
+
+func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
+	for _, m := range migrations {
+		if _, ok := done[m.Version]; !ok {
+			return true
+		}
+	}
+	return false
+}
+
+// openStore opens the business database for a command that reads and writes its
+// tables, and refuses one whose schema this build was not written against: a newer
+// Felis migrated it (a rolled-back binary), or, unless allowPending, migrations this
+// build embeds have not run yet (a binary swapped in ahead of `felis migrate up`).
+func openStore(ctx context.Context, url string, allowPending bool) (*store.PostgresDriver, error) {
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	return checkSchema(ctx, drv, allowPending)
+}
+
+// podDBWindow and podDBInterval bound how long a pod that has just started retries its
+// first database dial while the network policy has yet to admit it
+// (store.OpenRetrying). A minute is far past the sync lag and far inside every Job's
+// deadline. Vars so a test can shrink them.
+var (
+	podDBWindow   = time.Minute
+	podDBInterval = time.Second
+)
+
+// openPodStore is openStore for felis-api and the reaper and backup Jobs, whose first
+// dial comes milliseconds after their pod starts.
+func openPodStore(ctx context.Context, url, prog string, stderr io.Writer) (*store.PostgresDriver, error) {
+	drv, err := store.OpenRetrying(ctx, url, podDBWindow, podDBInterval, func(err error) {
+		fmt.Fprintf(stderr, "felis %s: %v; retrying (a pod that has just started waits for the network policy to admit it)\n", prog, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return checkSchema(ctx, drv, false)
+}
+
+// checkSchema closes drv and fails when its schema is not the one this build was
+// written against (see openStore).
+func checkSchema(ctx context.Context, drv *store.PostgresDriver, allowPending bool) (*store.PostgresDriver, error) {
+	s, err := store.ReadSchema(ctx, drv)
+	if err == nil {
+		err = s.Err()
+		if allowPending {
+			err = s.Newer()
+		}
+	}
+	if err != nil {
+		drv.Close()
+		return nil, err
+	}
+	return drv, nil
+}

cmd/felis/migrate_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// openPodStore retries a refused first dial for podDBWindow, saying so on stderr
+// under the calling command's name each time.
+func TestOpenPodStoreRetriesARefusedDial(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := ln.Addr().String()
+	ln.Close()
+
+	window, interval := podDBWindow, podDBInterval
+	podDBWindow, podDBInterval = 200*time.Millisecond, 20*time.Millisecond
+	t.Cleanup(func() { podDBWindow, podDBInterval = window, interval })
+
+	var stderr bytes.Buffer
+	start := time.Now()
+	_, err = openPodStore(context.Background(), fmt.Sprintf("postgres://felis@%s/felis?sslmode=disable", addr), "reaper", &stderr)
+	if !errors.Is(err, syscall.ECONNREFUSED) {
+		t.Fatalf("err = %v, want a refused dial", err)
+	}
+	if elapsed := time.Since(start); elapsed < podDBWindow {
+		t.Fatalf("gave up after %s, inside the %s window", elapsed, podDBWindow)
+	}
+	lines := strings.Split(strings.TrimSuffix(stderr.String(), "\n"), "\n")
+	if len(lines) < 2 || len(lines) > 11 {
+		t.Fatalf("%d retry lines in a 200ms window at 20ms:\n%s", len(lines), stderr.String())
+	}
+	for _, l := range lines {
+		if !strings.HasPrefix(l, "felis reaper: failed to connect to `user=felis database=felis`: ") ||
+			!strings.HasSuffix(l, "; retrying (a pod that has just started waits for the network policy to admit it)") {
+			t.Fatalf("retry line %q", l)
+		}
+	}
+}

cmd/felis/mirrortools.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/build"
+	"felis.lolicon.best/internal/imagepush"
+)
+
+// defaultBuildToolsStatus is where mirror-build-tools records its last run; the
+// watchdog reads it to tell a vulnerability DB that stopped refreshing.
+const defaultBuildToolsStatus = "/var/lib/felis/build-tools/status.json"
+
+// cmdMirrorBuildTools copies the build lane's tools (build.Tools: the kaniko and
+// trivy images, Trivy's vulnerability and Java DBs) from upstream into the
+// platform registry, where build Jobs pull them. deploy/bootstrap.sh runs it at
+// install and from felis-build-tools.timer twice a day, which is what keeps the
+// DBs fresh; a root shell can run it the same way to refresh now.
+//
+// It writes as the platform principal through the node's loopback hostPort, the
+// same way the installer pushes, reading the token from the environment or from
+// /etc/felis/secrets.env.
+func cmdMirrorBuildTools(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("mirror-build-tools", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	endpoint := fs.String("endpoint", "127.0.0.1:5000", "host[:port] of the registry to write to (plain HTTP)")
+	only := fs.String("only", "", "comma-separated tool names to copy (default: all of "+toolNames()+")")
+	status := fs.String("status", defaultBuildToolsStatus, `file to record the run in ("" records nothing)`)
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	platformFlag := fs.String("platform", "", "os/arch of the images to copy (default: this machine's)")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	tools, err := selectTools(*only)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: "http", Username: user, Password: pass, Log: stdout}
+	src := &imagepush.Source{Platform: *platformFlag}
+	started := time.Now()
+	var failed []string
+	for _, t := range tools {
+		dst := strings.TrimSuffix(*endpoint, "/") + "/" + t.Mirror
+		if _, err := p.Mirror(ctx, src, t.Source, dst); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: %s: %v\n", t.Name, err)
+			failed = append(failed, t.Name+": "+err.Error())
+		}
+	}
+	if *status != "" {
+		st, err := imagepush.ReadMirrorStatus(*status)
+		if err != nil || st == nil {
+			st = &imagepush.MirrorStatus{}
+		}
+		st.LastAttempt = started
+		st.LastError = strings.Join(failed, "; ")
+		if len(failed) == 0 {
+			st.LastSuccess = started
+		}
+		if err := imagepush.WriteMirrorStatus(*status, *st); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: record %s: %v\n", *status, err)
+		}
+	}
+	if len(failed) > 0 {
+		return 1
+	}
+	return 0
+}
+
+func toolNames() string {
+	var names []string
+	for _, t := range build.Tools {
+		names = append(names, t.Name)
+	}
+	return strings.Join(names, ",")
+}
+
+func selectTools(only string) ([]build.Tool, error) {
+	if only == "" {
+		return build.Tools, nil
+	}
+	var out []build.Tool
+	for _, name := range strings.Split(only, ",") {
+		name = strings.TrimSpace(name)
+		found := false
+		for _, t := range build.Tools {
+			if t.Name == name {
+				out = append(out, t)
+				found = true
+			}
+		}
+		if !found {
+			return nil, fmt.Errorf("unknown tool %q (known: %s)", name, toolNames())
+		}
+	}
+	return out, nil
+}

cmd/felis/node_approve.go

@@ -0,0 +1,466 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/archivetransfer"
+	"felis.lolicon.best/internal/operator"
+	"felis.lolicon.best/internal/placement"
+	"felis.lolicon.best/internal/platform"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func approveNode(ctx context.Context, cl client.Client, cs kubernetes.Interface, ns, controlNS, name, image, remote string, stdout io.Writer) error {
+	var n corev1.Node
+	if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil {
+		return err
+	}
+	_, controlPlane := n.Labels["node-role.kubernetes.io/control-plane"]
+	if !placement.Online(&n) || controlPlane || n.Labels[placement.LabelRole] == placement.RoleController {
+		return fmt.Errorf("candidate must be an online agent")
+	}
+	var nodes corev1.NodeList
+	if err := cl.List(ctx, &nodes); err != nil {
+		return err
+	}
+	var controller *corev1.Node
+	var peers []string
+	var denied []string
+	for i := range nodes.Items {
+		node := &nodes.Items[i]
+		if node.Labels[placement.LabelRole] == placement.RoleController {
+			if controller != nil {
+				return fmt.Errorf("multiple controllers found")
+			}
+			controller = node
+		}
+		for _, addr := range node.Status.Addresses {
+			if addr.Type == corev1.NodeInternalIP || addr.Type == corev1.NodeExternalIP {
+				cidr, err := exactCIDR(addr.Address)
+				if err != nil {
+					return err
+				}
+				peers = append(peers, cidr)
+				for _, p := range []string{"6443", "10250", "5000", "30443"} {
+					denied = append(denied, net.JoinHostPort(addr.Address, p))
+				}
+			}
+		}
+	}
+	if controller == nil || controller.Name == n.Name || controller.Status.NodeInfo.Architecture != n.Status.NodeInfo.Architecture {
+		return fmt.Errorf("candidate architecture must match the sole controller")
+	}
+	if n.Status.NodeInfo.KubeletVersion != controller.Status.NodeInfo.KubeletVersion {
+		return fmt.Errorf("candidate k3s version must match A")
+	}
+	var apiSvc, registrySvc, archiveSvc, kubernetesSvc corev1.Service
+	for _, svc := range []struct {
+		obj      *corev1.Service
+		ns, name string
+	}{{&kubernetesSvc, "default", "kubernetes"}, {&apiSvc, controlNS, platform.SAAPI}, {&registrySvc, controlNS, "registry"}, {&archiveSvc, ns, platform.ArchiveName}} {
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: svc.ns, Name: svc.name}, svc.obj); err != nil {
+			return err
+		}
+	}
+	if len(apiSvc.Spec.Ports) == 0 || apiSvc.Spec.Ports[0].NodePort == 0 {
+		return fmt.Errorf("controller API NodePort is absent")
+	}
+	// A's exact interface addresses let the probe observe DNAT/SNAT before the production policy is adjusted.
+	var aCIDRs []string
+	if addrs, err := net.InterfaceAddrs(); err == nil {
+		for _, a := range addrs {
+			ip, _, err := net.ParseCIDR(a.String())
+			if err == nil && !ip.IsLoopback() {
+				cidr, _ := exactCIDR(ip.String())
+				aCIDRs = append(aCIDRs, cidr)
+			}
+		}
+	}
+	if len(aCIDRs) == 0 {
+		return fmt.Errorf("cannot determine A's exact interface addresses")
+	}
+	onController := false
+	for _, addr := range controller.Status.Addresses {
+		cidr, err := exactCIDR(addr.Address)
+		if err != nil {
+			continue
+		}
+		for _, local := range aCIDRs {
+			if cidr == local {
+				onController = true
+			}
+		}
+	}
+	if !onController || !placement.Online(controller) {
+		return fmt.Errorf("run admission on the online controller A")
+	}
+	// Quarantine first. Approval is the final write, after all checks have succeeded.
+	before := n.DeepCopy()
+	if n.Labels == nil {
+		n.Labels = map[string]string{}
+	}
+	delete(n.Labels, placement.LabelApproved)
+	found := false
+	for _, t := range n.Spec.Taints {
+		if t.Key == "felis.lolicon.best/unapproved" {
+			found = true
+		}
+	}
+	if !found {
+		n.Spec.Taints = append(n.Spec.Taints, corev1.Taint{Key: "felis.lolicon.best/unapproved", Value: "true", Effect: corev1.TaintEffectNoSchedule})
+	}
+	if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil {
+		return err
+	}
+	if err := denyNodeAddresses(ctx, cl, ns, nodes.Items); err != nil {
+		return err
+	}
+	// The host's Service dial may be masqueraded to its bridge gateway. Permit exact host addresses only.
+	pullSources := append([]string{}, peers...)
+	if ip, network, err := net.ParseCIDR(n.Spec.PodCIDR); err == nil && ip.To4() != nil {
+		v := append(net.IP(nil), network.IP.To4()...)
+		pullSources = append(pullSources, v.String()+"/32")
+		v[3]++
+		pullSources = append(pullSources, v.String()+"/32")
+	}
+	var registryNP networkingv1.NetworkPolicy
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNS, Name: "felis-registry-ingress"}, &registryNP); err != nil {
+		return err
+	}
+	if len(registryNP.Spec.Ingress) == 0 {
+		return fmt.Errorf("registry ingress policy is not configured")
+	}
+	prev := registryNP.DeepCopy()
+	for _, cidr := range pullSources {
+		registryNP.Spec.Ingress[0].From = append(registryNP.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
+	}
+	if err := cl.Patch(ctx, &registryNP, client.MergeFrom(prev)); err != nil {
+		return err
+	}
+	ctrlIP := ""
+	for _, a := range controller.Status.Addresses {
+		if a.Type == corev1.NodeInternalIP {
+			ctrlIP = a.Address
+			break
+		}
+	}
+	if ctrlIP == "" {
+		return fmt.Errorf("controller has no address")
+	}
+	script := "set -euo pipefail\numask 077\n" +
+		"systemctl is-active --quiet k3s-agent\n! systemctl is-active --quiet k3s\ntest ! -f /etc/rancher/k3s/k3s.yaml\n" +
+		"ip -d link show flannel-wg | grep -q wireguard\n" +
+		"/usr/local/bin/felis node firewall --peers " + shellQuote(strings.Join(peers, ",")) + " --controller-ip " + shellQuote(ctrlIP) + " --api-service-ip " + shellQuote(kubernetesSvc.Spec.ClusterIP) + " --node-port " + strconv.Itoa(int(apiSvc.Spec.Ports[0].NodePort)) + " --namespace " + shellQuote(ns) + " --control-namespace " + shellQuote(controlNS) + "\n" +
+		"kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig\n" +
+		"/usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" get node " + shellQuote(name) + " -o name >/dev/null\n" +
+		"proof=$(mktemp); trap 'rm -f \"$proof\"' EXIT\n" +
+		"if /usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" label node " + shellQuote(name) + " felis.node-restriction.kubernetes.io/probe=controller --overwrite 2>\"$proof\"; then echo 'NodeRestriction failed' >&2;exit 1;fi\ngrep -qi forbidden \"$proof\"\n" +
+		"image=" + shellQuote(image) + "\nif [ -n \"$(/usr/local/bin/k3s crictl images -q \"$image\")\" ]; then /usr/local/bin/k3s crictl rmi \"$image\" >/dev/null; fi\ntest -z \"$(/usr/local/bin/k3s crictl images -q \"$image\")\"\n/usr/local/bin/k3s crictl pull \"$image\" >/dev/null\n"
+	cmd := exec.CommandContext(ctx, "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=10", "--", remote, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi")
+	cmd.Stdin = strings.NewReader(script)
+	cmd.Stdout = stdout
+	cmd.Stderr = stdout
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("worker host, NodeRestriction or uncached registry pull check failed: %w", err)
+	}
+	id := "felis-probe-" + archivetransfer.ID()[:12]
+	var objects []client.Object
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		for i := len(objects) - 1; i >= 0; i-- {
+			cl.Delete(cleanup, objects[i])
+		}
+	}()
+	create := func(o client.Object) error {
+		if err := cl.Create(ctx, o); err != nil {
+			return err
+		}
+		objects = append(objects, o)
+		return nil
+	}
+	var endpoints []string
+	var echoPods []*corev1.Pod
+	for i := range nodes.Items {
+		node := &nodes.Items[i]
+		if !placement.Online(node) || (node.Name != name && node.Name != controller.Name && node.Labels[placement.LabelApproved] != "true") {
+			continue
+		}
+		echoName := fmt.Sprintf("%s-%d", id, i)
+		p := probePod(echoName, ns, node.Name, image, []string{"--listen", ":25565"}, true)
+		p.Labels["felis.lolicon.best/probe-echo"] = id
+		if err := create(p); err != nil {
+			return err
+		}
+		echoPods = append(echoPods, p)
+		svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: echoName, Namespace: ns}, Spec: corev1.ServiceSpec{Selector: map[string]string{"felis.lolicon.best/probe-name": echoName}, Ports: []corev1.ServicePort{{Port: 25565, TargetPort: intstr.FromInt32(25565)}}}}
+		if err := create(svc); err != nil {
+			return err
+		}
+		endpoints = append(endpoints, net.JoinHostPort(svc.Spec.ClusterIP, "25565"))
+	}
+	tcp := corev1.ProtocolTCP
+	port := intstr.FromInt32(25565)
+	policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id, Namespace: ns}, Spec: networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-echo": id}}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}}}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &port}}}}}}
+	for _, cidr := range aCIDRs {
+		policy.Spec.Ingress[0].From = append(policy.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
+	}
+	if err := create(policy); err != nil {
+		return err
+	}
+	for _, p := range echoPods {
+		if err := waitProbePod(ctx, cl, p); err != nil {
+			return err
+		}
+	}
+	open := append([]string{}, endpoints...)
+	open = append(open, net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), net.JoinHostPort(registrySvc.Spec.ClusterIP, "5000"), net.JoinHostPort(archiveSvc.Spec.ClusterIP, "8090"))
+	// Positive probes need temporary, narrowly scoped ingress grants where the
+	// production fence admits only the controller or archive-transfer jobs.
+	for _, svc := range []*corev1.Service{&apiSvc, &registrySvc, &archiveSvc} {
+		if len(svc.Spec.Selector) == 0 || len(svc.Spec.Ports) == 0 {
+			return fmt.Errorf("probe Service %s has no backend", svc.Name)
+		}
+		targetPort := svc.Spec.Ports[0].TargetPort
+		if targetPort.IntVal == 0 && targetPort.StrVal == "" {
+			targetPort = intstr.FromInt32(svc.Spec.Ports[0].Port)
+		}
+		allow := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id + "-" + svc.Name, Namespace: svc.Namespace}, Spec: networkingv1.NetworkPolicySpec{
+			PodSelector: metav1.LabelSelector{MatchLabels: svc.Spec.Selector}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
+			Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{
+				NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ns}},
+				PodSelector:       &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}},
+			}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &targetPort}}}},
+		}}
+		if err := create(allow); err != nil {
+			return err
+		}
+	}
+	positive := probeJob(id+"-positive", ns, name, image, probeArgs("--open", open), false)
+	positive.Spec.Template.Labels["felis.lolicon.best/probe-source"] = id
+	if err := create(positive); err != nil {
+		return err
+	}
+	if err := waitProbeJob(ctx, cl, positive); err != nil {
+		return err
+	}
+	denied = append(denied, open...)
+	denied = append(denied, "169.254.169.254:80", "169.254.170.2:80")
+	negative := probeJob(id+"-negative", ns, name, image, probeArgs("--closed", denied), true)
+	negative.Spec.Template.Spec.InitContainers = []corev1.Container{{Name: "egress-gate", Image: image, Command: []string{"/usr/local/bin/felis", "egress-gate", "--probe", net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), "--wait", "2m"}, SecurityContext: negative.Spec.Template.Spec.Containers[0].SecurityContext}}
+	if err := create(negative); err != nil {
+		return err
+	}
+	if err := waitProbeJob(ctx, cl, negative); err != nil {
+		return err
+	}
+	// Dial from Velocity's host namespace and record the address actually observed inside each backend.
+	for _, endpoint := range endpoints {
+		c, err := net.DialTimeout("tcp", endpoint, 5*time.Second)
+		if err != nil {
+			return fmt.Errorf("velocity host cannot dial backend Service %s: %w", endpoint, err)
+		}
+		c.Close()
+	}
+	var observed []string
+	time.Sleep(500 * time.Millisecond)
+	for _, p := range echoPods {
+		foundA := false
+		stream, err := cs.CoreV1().Pods(ns).GetLogs(p.Name, &corev1.PodLogOptions{}).Stream(ctx)
+		if err != nil {
+			return err
+		}
+		scan := bufio.NewScanner(stream)
+		for scan.Scan() {
+			line := scan.Text()
+			if strings.HasPrefix(line, "felis-probe-source ") {
+				host, _, err := net.SplitHostPort(strings.TrimPrefix(line, "felis-probe-source "))
+				if err == nil {
+					cidr, _ := exactCIDR(host)
+					for _, a := range aCIDRs {
+						if cidr == a {
+							observed = append(observed, cidr)
+							foundA = true
+						}
+					}
+				}
+			}
+		}
+		if !foundA {
+			stream.Close()
+			return fmt.Errorf("backend %s did not observe A as an exact source", p.Name)
+		}
+		err = scan.Err()
+		stream.Close()
+		if err != nil {
+			return err
+		}
+	}
+	if len(observed) < len(echoPods) {
+		return fmt.Errorf("backend observed a source outside A's exact interfaces")
+	}
+	var game networkingv1.NetworkPolicy
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-allow-game-from-velocity"}, &game); err != nil {
+		return err
+	}
+	if len(game.Spec.Ingress) == 0 {
+		return fmt.Errorf("velocity ingress policy is not configured")
+	}
+	prevGame := game.DeepCopy()
+	for _, cidr := range observed {
+		exists := false
+		for _, peer := range game.Spec.Ingress[0].From {
+			if peer.IPBlock != nil && peer.IPBlock.CIDR == cidr {
+				exists = true
+				break
+			}
+		}
+		if exists {
+			continue
+		}
+		game.Spec.Ingress[0].From = append(game.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
+	}
+	if err := cl.Patch(ctx, &game, client.MergeFrom(prevGame)); err != nil {
+		return err
+	}
+	// Read fresh resourceVersion so concurrent node health writes cannot be overwritten.
+	if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil {
+		return err
+	}
+	if !placement.Online(&n) {
+		return fmt.Errorf("worker became offline during validation")
+	}
+	before = n.DeepCopy()
+	n.Labels[placement.LabelIdentity] = name
+	n.Labels[placement.LabelRole] = placement.RoleWorker
+	n.Labels[placement.LabelApproved] = "true"
+	taints := n.Spec.Taints[:0]
+	for _, t := range n.Spec.Taints {
+		if t.Key != "felis.lolicon.best/unapproved" {
+			taints = append(taints, t)
+		}
+	}
+	n.Spec.Taints = taints
+	if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil {
+		return err
+	}
+	fmt.Fprintln(stdout, "approved worker", name, "Velocity sources", strings.Join(observed, ","))
+	return nil
+}
+
+func probeArgs(flag string, addresses []string) []string {
+	var args []string
+	for _, a := range addresses {
+		args = append(args, flag, a)
+	}
+	return args
+}
+func probePod(name, ns, node, image string, args []string, game bool) *corev1.Pod {
+	no, yes := false, true
+	uid := int64(1000)
+	labels := map[string]string{"felis.lolicon.best/probe-name": name}
+	if game {
+		labels[v1alpha1.LabelManagedBy] = operator.ManagedByValue
+		labels[v1alpha1.LabelComponent] = operator.ComponentValue
+		labels[v1alpha1.LabelServer] = name
+	}
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, Spec: corev1.PodSpec{NodeName: node, RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &no, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &yes, RunAsUser: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "probe", Image: image, ImagePullPolicy: corev1.PullAlways, Command: []string{"/usr/local/bin/felis", "node-probe"}, Args: args, Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi"), corev1.ResourceCPU: resource.MustParse("200m")}}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &no, ReadOnlyRootFilesystem: &yes, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}}}}}
+}
+func probeJob(name, ns, node, image string, args []string, game bool) *batchv1.Job {
+	p := probePod(name, ns, node, image, args, game)
+	zero := int32(0)
+	deadline := int64(600)
+	return &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, Spec: batchv1.JobSpec{BackoffLimit: &zero, ActiveDeadlineSeconds: &deadline, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: p.Labels}, Spec: p.Spec}}}
+}
+func waitProbePod(ctx context.Context, cl client.Client, p *corev1.Pod) error {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+	for {
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(p), p); err != nil {
+			return err
+		}
+		for _, c := range p.Status.Conditions {
+			if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue {
+				return nil
+			}
+		}
+		if p.Status.Phase == corev1.PodFailed {
+			return fmt.Errorf("probe Pod %s failed", p.Name)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-t.C:
+		}
+	}
+}
+func waitProbeJob(ctx context.Context, cl client.Client, j *batchv1.Job) error {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+	for {
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(j), j); err != nil {
+			return err
+		}
+		for _, c := range j.Status.Conditions {
+			if c.Status != corev1.ConditionTrue {
+				continue
+			}
+			if c.Type == batchv1.JobComplete {
+				return nil
+			}
+			if c.Type == batchv1.JobFailed {
+				return fmt.Errorf("admission probe Job %s failed; inspect its Pod log", j.Name)
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-t.C:
+		}
+	}
+}
+
+func denyNodeAddresses(ctx context.Context, cl client.Client, ns string, nodes []corev1.Node) error {
+	var np networkingv1.NetworkPolicy
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-server-egress"}, &np); err != nil {
+		return err
+	}
+	before := np.DeepCopy()
+	for _, n := range nodes {
+		for _, a := range n.Status.Addresses {
+			if a.Type != corev1.NodeInternalIP && a.Type != corev1.NodeExternalIP {
+				continue
+			}
+			cidr, err := exactCIDR(a.Address)
+			if err != nil {
+				return err
+			}
+			for i := range np.Spec.Egress {
+				for k := range np.Spec.Egress[i].To {
+					block := np.Spec.Egress[i].To[k].IPBlock
+					if block != nil && ((strings.Contains(cidr, ":") && block.CIDR == "::/0") || (!strings.Contains(cidr, ":") && block.CIDR == "0.0.0.0/0")) {
+						block.Except = append(block.Except, cidr)
+					}
+				}
+			}
+		}
+	}
+	return cl.Patch(ctx, &np, client.MergeFrom(before))
+}

cmd/felis/node_control.go

@@ -0,0 +1,581 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strings"
+	"syscall"
+	"time"
+
+	felis "felis.lolicon.best"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/nodecontrol"
+	"felis.lolicon.best/internal/placement"
+	"felis.lolicon.best/internal/platform"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+func cmdNodeControl(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("node-control", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	socket := fs.String("socket", nodecontrol.Socket, "local API-only Unix socket")
+	dir := fs.String("state", "/var/lib/felis/node-control", "root-owned persistent task state")
+	kubeconfig := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "controller kubeconfig")
+	config := fs.String("config", "/etc/felis/felis.host.toml", "host configuration")
+	ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
+	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "control namespace")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "node-control requires root")
+		return 1
+	}
+	cfg, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	cs, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	exe, err := os.Executable()
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer cancel()
+	execute := nodeExecutor{cs: cs, binary: exe, kubeconfig: *kubeconfig, config: *config, namespace: *ns, controlNamespace: *controlNS, stateDir: *dir}
+	socketDir := filepath.Dir(*socket)
+	if err = os.MkdirAll(socketDir, 0750); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err = os.Chown(socketDir, 0, 65532); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err = os.Chmod(socketDir, 0750); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+
+	if existing, err := os.Lstat(*socket); err == nil {
+		if existing.Mode()&os.ModeSocket == 0 {
+			fmt.Fprintln(stderr, "refusing to replace non-socket path")
+			return 1
+		}
+		conn, err := net.DialTimeout("unix", *socket, time.Second)
+		if err == nil {
+			conn.Close()
+			fmt.Fprintln(stderr, "node-control is already running")
+			return 1
+		}
+		if err = os.Remove(*socket); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	listener, err := net.Listen("unix", *socket)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	defer listener.Close()
+	if err = os.Chown(*socket, 0, 65532); err == nil {
+		err = os.Chmod(*socket, 0660)
+	}
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	// /run is recreated at boot; label both the directory and the new socket inode.
+	if os.Getenv("FELIS_NODE_CONTROL_SELINUX") == "1" {
+		if err := exec.Command("chcon", "-R", "-t", "felis_node_control_socket_t", socketDir).Run(); err != nil {
+			fmt.Fprintln(stderr, "node-control socket labeling failed:", err)
+			return 1
+		}
+	}
+	manager, err := nodecontrol.Open(ctx, *dir, execute.run)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	server := &http.Server{Handler: manager.Handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second}
+	go func() { <-ctx.Done(); server.Close() }()
+	fmt.Fprintln(stdout, "node-control listening on", *socket)
+	err = server.Serve(listener)
+	cancel()
+	manager.Wait()
+	if err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	return 0
+}
+
+type nodeExecutor struct {
+	cs                                                                kubernetes.Interface
+	binary, kubeconfig, config, namespace, controlNamespace, stateDir string
+}
+
+func (e nodeExecutor) run(ctx context.Context, r nodecontrol.Request, stage func(string) error, out io.Writer) (resultErr error) {
+	// Host-wide changes and cache-removing admission probes are serialized by the manager.
+	if r.Action == "approve" {
+		node, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		if node.Labels[placement.LabelRole] != placement.RoleWorker {
+			return errors.New("only worker nodes can be approved")
+		}
+		patch := []byte(`{"spec":{"unschedulable":true}}`)
+		if !node.Spec.Unschedulable {
+			patch = []byte(`{"spec":{"unschedulable":true},"metadata":{"annotations":{"felis.lolicon.best/node-control-cordon":"true"}}}`)
+		}
+		if _, err = e.cs.CoreV1().Nodes().Patch(ctx, r.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
+			return err
+		}
+	}
+	if err := e.requireStopped(ctx, r); err != nil {
+		return err
+	}
+	nodes, err := e.cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	controller, peers, err := nodeController(nodes.Items, r.ExternalIP, r.Peers)
+	if err != nil {
+		return err
+	}
+	deployment, err := e.cs.AppsV1().Deployments(e.controlNamespace).Get(ctx, platform.SAAPI, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if len(deployment.Spec.Template.Spec.Containers) == 0 {
+		return errors.New("the Felis API image is unavailable")
+	}
+	image := deployment.Spec.Template.Spec.Containers[0].Image
+	if r.Action == "enable" {
+		if err := stage("pause_operator"); err != nil {
+			return err
+		}
+		scale, err := e.cs.AppsV1().Deployments(e.controlNamespace).GetScale(ctx, platform.SAOperator, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		replicas := scale.Spec.Replicas
+		if replicas < 1 {
+			replicas = 1
+		}
+		scale.Spec.Replicas = 0
+		if _, err = e.cs.AppsV1().Deployments(e.controlNamespace).UpdateScale(ctx, platform.SAOperator, scale, metav1.UpdateOptions{}); err != nil {
+			return err
+		}
+		defer func() {
+			cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+			defer cancel()
+			latest, err := e.cs.AppsV1().Deployments(e.controlNamespace).GetScale(cleanup, platform.SAOperator, metav1.GetOptions{})
+			if err == nil {
+				latest.Spec.Replicas = replicas
+				_, err = e.cs.AppsV1().Deployments(e.controlNamespace).UpdateScale(cleanup, platform.SAOperator, latest, metav1.UpdateOptions{})
+			}
+			if err != nil {
+				fmt.Fprintln(out, "Operator restoration failed:", err)
+				resultErr = fmt.Errorf("operator restoration failed: %w", err)
+			}
+		}()
+		// Drain the old reconciler before the second stopped-state check.
+		for {
+			pods, err := e.cs.CoreV1().Pods(e.controlNamespace).List(ctx, metav1.ListOptions{LabelSelector: platform.LabelComponent + "=" + platform.ComponentOperator})
+			if err != nil {
+				return err
+			}
+			if len(pods.Items) == 0 {
+				break
+			}
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(time.Second):
+			}
+		}
+		if err = e.requireStopped(ctx, r); err != nil {
+			return err
+		}
+		return e.enable(ctx, r, controller, peers, stage, out)
+	}
+	if controller.Labels[placement.LabelRole] != placement.RoleController {
+		return errors.New("distributed mode is not configured on the controller")
+	}
+	if r.Name == controller.Name {
+		return errors.New("worker name must differ from the controller")
+	}
+	// SSH trust and keys are configured on A; no password, key or bootstrap token crosses the API.
+	if err := stage("ssh_check"); err != nil {
+		return err
+	}
+	arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[runtime.GOARCH]
+	if arch == "" {
+		return errors.New("unsupported host architecture")
+	}
+	check := "set -eu\n[ \"$(uname -s)\" = Linux ] || { echo 'worker requires Linux' >&2; exit 1; }\n[ \"$(uname -m)\" = " + shellQuote(arch) + " ] || { echo 'worker architecture differs from controller' >&2; exit 1; }\n"
+	if r.Action == "join" {
+		check += "[ ! -e /etc/rancher/k3s/k3s.yaml ] && [ ! -e /var/lib/rancher/k3s/agent ] || { echo 'k3s is already installed; enrollment will not overwrite an existing node' >&2; exit 1; }\n"
+		check += "ip -o address show | awk '{print $4}' | cut -d/ -f1 | grep -Fx -- " + shellQuote(r.ExternalIP) + " >/dev/null || { echo 'fixed node IP is not assigned to the worker' >&2; exit 1; }\n"
+	}
+
+	if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi", strings.NewReader(check), out); err != nil {
+		return fmt.Errorf("worker preflight or SSH connection failed: %w", err)
+	}
+	if r.Action == "join" {
+		for _, n := range nodes.Items {
+			if n.Name == r.Name {
+				return errors.New("node already exists; use approval instead of reinstalling it")
+			}
+		}
+		if err = e.join(ctx, r, controller, peers, stage, out); err != nil {
+			return err
+		}
+	}
+	if err = stage("approval"); err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, e.binary, "node", "approve", "--name", r.Name, "--ssh-target", r.SSHTarget, "--image", image, "--kubeconfig", e.kubeconfig, "--namespace", e.namespace, "--control-namespace", e.controlNamespace)
+	cmd.Stdout = out
+	cmd.Stderr = out
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("node approval failed; node remains quarantined: %w", err)
+	}
+	// Admission succeeded; release only the task's scheduling quarantine.
+	admitted, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if admitted.Annotations["felis.lolicon.best/node-control-cordon"] == "true" {
+		if _, err = e.cs.CoreV1().Nodes().Patch(ctx, r.Name, types.MergePatchType, []byte(`{"spec":{"unschedulable":false},"metadata":{"annotations":{"felis.lolicon.best/node-control-cordon":null}}}`), metav1.PatchOptions{}); err != nil {
+			return fmt.Errorf("node approved but scheduling remains disabled: %w", err)
+		}
+	}
+	return stage("complete")
+}
+func (e nodeExecutor) requireStopped(ctx context.Context, r nodecontrol.Request) error {
+	// Even pre-existing worker names may contain worlds. Never run installer/admission on an occupied worker.
+	ss, err := e.cs.AppsV1().StatefulSets(e.namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, s := range ss.Items {
+		if r.Action != "enable" && s.Spec.Template.Spec.NodeSelector[placement.LabelIdentity] != r.Name {
+			continue
+		}
+		if s.Spec.Replicas != nil && *s.Spec.Replicas > 0 {
+			return fmt.Errorf("StatefulSet %s still requests %d replicas; stop the server before changing nodes", s.Name, *s.Spec.Replicas)
+		}
+	}
+	pods, err := e.cs.CoreV1().Pods(e.namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, p := range pods.Items {
+		if r.Action != "enable" && p.Spec.NodeName != r.Name && p.Spec.NodeSelector[placement.LabelIdentity] != r.Name {
+			continue
+		}
+		if p.Status.Phase != corev1.PodSucceeded && p.Status.Phase != corev1.PodFailed {
+			return fmt.Errorf("pod %s has not exited (phase %s); wait for game and maintenance workloads to stop", p.Name, p.Status.Phase)
+		}
+	}
+	// Prevent a pending wake intent racing admission or an installation.
+	raw, err := e.cs.CoreV1().RESTClient().Get().AbsPath("/apis/" + v1alpha1.GroupVersion.Group + "/" + v1alpha1.GroupVersion.Version + "/namespaces/" + e.namespace + "/minecraftservers").DoRaw(ctx)
+	if err != nil {
+		return err
+	}
+	var servers v1alpha1.MinecraftServerList
+	if err = json.Unmarshal(raw, &servers); err != nil {
+		return err
+	}
+	for _, s := range servers.Items {
+		if r.Action != "enable" && s.Spec.NodeName != r.Name && s.Status.NodeName != r.Name {
+			continue
+		}
+		if s.Spec.DesiredState != "" && string(s.Spec.DesiredState) != "Stopped" {
+			return fmt.Errorf("server %s must have stopped intent", s.Name)
+		}
+	}
+	return nil
+}
+
+func nodeController(nodes []corev1.Node, ip string, extra []string) (*corev1.Node, []string, error) {
+	var controller *corev1.Node
+	peers := append([]string{}, extra...)
+	for i := range nodes {
+		n := &nodes[i]
+		_, controlPlane := n.Labels["node-role.kubernetes.io/control-plane"]
+		if controlPlane || n.Labels[placement.LabelRole] == placement.RoleController {
+			if controller != nil {
+				return nil, nil, errors.New("exactly one controller is required")
+			}
+			controller = n
+		}
+		for _, a := range n.Status.Addresses {
+			if a.Type == corev1.NodeExternalIP || a.Type == corev1.NodeInternalIP {
+				cidr, err := exactCIDR(a.Address)
+				if err != nil {
+					return nil, nil, err
+				}
+				peers = append(peers, cidr)
+			}
+		}
+	}
+	if controller == nil || !placement.Online(controller) {
+		return nil, nil, errors.New("the sole controller must be online")
+	}
+	if ip != "" {
+		cidr, err := exactCIDR(ip)
+		if err != nil {
+			return nil, nil, err
+		}
+		peers = append(peers, cidr)
+	}
+	slices.Sort(peers)
+	peers = slices.Compact(peers)
+	return controller, peers, nil
+}
+func controllerIP(n *corev1.Node) string {
+	for _, kind := range []corev1.NodeAddressType{corev1.NodeExternalIP, corev1.NodeInternalIP} {
+		for _, a := range n.Status.Addresses {
+			if a.Type == kind {
+				return a.Address
+			}
+		}
+	}
+	return ""
+}
+func (e nodeExecutor) ssh(ctx context.Context, target, command string, in io.Reader, out io.Writer) error {
+	cmd := exec.CommandContext(ctx, "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=10", "--", target, command)
+	cmd.Stdin = in
+	cmd.Stdout = out
+	cmd.Stderr = out
+	return cmd.Run()
+}
+func (e nodeExecutor) join(ctx context.Context, r nodecontrol.Request, controller *corev1.Node, peers []string, stage func(string) error, out io.Writer) error {
+	registry, err := e.cs.CoreV1().Services(e.controlNamespace).Get(ctx, "registry", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if err = stage("firewall"); err != nil {
+		return err
+	}
+	// All peers must be updated before the new agent is admitted. Existing worker SSH targets must be configured by stable node name.
+	list, err := e.cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, n := range list.Items {
+		if n.Name == controller.Name || n.Name == r.Name {
+			continue
+		}
+		if n.Labels[placement.LabelRole] != placement.RoleWorker {
+			return fmt.Errorf("node %s has an unknown role", n.Name)
+		}
+		script := shellQuote(e.binary) + " node firewall --peers " + shellQuote(strings.Join(peers, ",")) + " --controller-ip " + shellQuote(controllerIP(controller))
+		script += " --namespace " + shellQuote(e.namespace) + " --control-namespace " + shellQuote(e.controlNamespace)
+		if err = e.ssh(ctx, n.Name, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n "+script+"; fi", nil, out); err != nil {
+			return fmt.Errorf("update peer firewall on %s: %w", n.Name, err)
+		}
+	}
+	cmd := exec.CommandContext(ctx, e.binary, "node", "firewall", "--controller", "--controller-ip", controllerIP(controller), "--peers", strings.Join(peers, ","), "--namespace", e.namespace, "--control-namespace", e.controlNamespace)
+	cmd.Stdout = out
+	cmd.Stderr = out
+	if err = cmd.Run(); err != nil {
+		return err
+	}
+	if err = stage("bootstrap_token"); err != nil {
+		return err
+	}
+	temp, err := os.MkdirTemp("", "felis-node-join-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(temp)
+	tokenPath := filepath.Join(temp, "bootstrap")
+	tokenCmd := exec.CommandContext(ctx, e.binary, "node", "token", "--name", r.Name, "--ttl", "1h", "--out", tokenPath)
+	tokenCmd.Stdout = out
+	tokenCmd.Stderr = out
+	if err = tokenCmd.Run(); err != nil {
+		return err
+	}
+	// Revoke even on failure. The worker exchanges bootstrap credentials for its node identity.
+	token, err := os.ReadFile(tokenPath)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		exec.CommandContext(cleanup, "k3s", "token", "delete", bootstrapTokenID(string(token))).Run()
+	}()
+	remoteDir := "/var/tmp/felis-node-" + filepath.Base(temp)
+	if err = stage("transfer"); err != nil {
+		return err
+	}
+	script := "set -eu; umask 077; mkdir " + shellQuote(remoteDir) + "; cat > " + shellQuote(remoteDir+"/bootstrap")
+	rootCommand := "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi"
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		e.ssh(cleanup, r.SSHTarget, rootCommand, strings.NewReader("rm -rf -- "+shellQuote(remoteDir)), io.Discard)
+	}()
+	// stdin carries the token; it is never in command arguments, task records or logs.
+	if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n sh -c "+shellQuote(script)+"; fi", strings.NewReader(string(token)), out); err != nil {
+		return err
+	}
+
+	binary, err := os.Open(e.binary)
+	if err != nil {
+		return err
+	}
+	defer binary.Close()
+	script = "set -eu; cat > " + shellQuote(remoteDir+"/felis") + "; chmod 0700 " + shellQuote(remoteDir+"/felis")
+	if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n sh -c "+shellQuote(script)+"; fi", binary, out); err != nil {
+		return err
+	}
+	if err = stage("install_worker"); err != nil {
+		return err
+	}
+	script = "set -eu\nexport FELIS_K3S_VERSION=" + shellQuote(controller.Status.NodeInfo.KubeletVersion) + "\n" + shellQuote(remoteDir+"/felis") + " node join --name " + shellQuote(r.Name) + " --server " + shellQuote("https://"+net.JoinHostPort(controllerIP(controller), "6443")) + " --external-ip " + shellQuote(r.ExternalIP) + " --token-file " + shellQuote(remoteDir+"/bootstrap") + " --registry-ip " + shellQuote(registry.Spec.ClusterIP) + " --peers " + shellQuote(strings.Join(peers, ",")) + "\n"
+	if err = e.ssh(ctx, r.SSHTarget, rootCommand, strings.NewReader(script), out); err != nil {
+		return fmt.Errorf("worker installation failed: %w", err)
+	}
+	if err = stage("wait_ready"); err != nil {
+		return err
+	}
+	deadline := time.NewTimer(5 * time.Minute)
+	defer deadline.Stop()
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+	for {
+		n, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
+		if err == nil && placement.Online(n) {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-deadline.C:
+			return errors.New("worker did not become Ready within five minutes")
+		case <-ticker.C:
+		}
+	}
+}
+func (e nodeExecutor) enable(ctx context.Context, r nodecontrol.Request, controller *corev1.Node, peers []string, stage func(string) error, out io.Writer) error {
+	if controllerIP(controller) != r.ExternalIP {
+		return errors.New("controller IP must match the registered fixed node address")
+	}
+	if len(peers) == 0 {
+		return errors.New("peer addresses are required")
+	}
+	if err := stage("database_backup"); err != nil {
+		return err
+	}
+	backup := exec.CommandContext(ctx, e.binary, "db", "backup", "-config", e.config, "-dir", "/var/lib/felis/db-backups", "-label", "pre-migrate")
+	backup.Stdout = out
+	backup.Stderr = out
+	if err := backup.Run(); err != nil {
+		return err
+	}
+	if err := stage("cluster_backup"); err != nil {
+		return err
+	}
+	if err := e.backupCluster(ctx, out); err != nil {
+		return err
+	}
+	if err := stage("configure_controller"); err != nil {
+		return err
+	}
+	patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s","%s":"controller"}}}`, placement.LabelIdentity, controller.Name, placement.LabelRole)
+	if _, err := e.cs.CoreV1().Nodes().Patch(ctx, controller.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil {
+		return err
+	}
+	for _, name := range []string{platform.SAAPI, platform.SAOperator, platform.PostgresName, "registry"} {
+		body := fmt.Sprintf(`{"spec":{"template":{"spec":{"nodeSelector":{"%s":"%s"}}}}}`, placement.LabelIdentity, controller.Name)
+		if _, err := e.cs.AppsV1().Deployments(e.controlNamespace).Patch(ctx, name, types.MergePatchType, []byte(body), metav1.PatchOptions{}); err != nil {
+			return err
+		}
+	}
+	if err := stage("install_controller"); err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, "bash", "-s")
+	cmd.Stdin = strings.NewReader(felis.BootstrapScript())
+	cmd.Stdout = out
+	cmd.Stderr = out
+	cmd.Env = append(os.Environ(), "FELIS_DISTRIBUTED=1", "FELIS_NODE_EXTERNAL_IP="+r.ExternalIP, "FELIS_PEER_CIDRS="+strings.Join(peers, ","), "FELIS_BOOTSTRAP_FROM_TUI=1", "FELIS_BOOTSTRAP_BINARY="+e.binary, "FELIS_NO_SETUP=1", "FELIS_NODE_CONTROL_TASK=1", "FELIS_INSTALL_MODE=full")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("controller installation failed; retain the database backup: %w", err)
+	}
+	return stage("complete")
+}
+
+func bootstrapTokenID(token string) string {
+	token = strings.TrimSpace(token)
+	if _, short, ok := strings.Cut(token, "::"); ok {
+		token = short
+	}
+	id, _, _ := strings.Cut(token, ".")
+	return id
+}
+
+func (e nodeExecutor) backupCluster(ctx context.Context, out io.Writer) (err error) {
+	dir := filepath.Join(filepath.Dir(e.stateDir), "cluster-backups")
+	if err = os.MkdirAll(dir, 0700); err != nil {
+		return err
+	}
+	path := filepath.Join(dir, "pre-distributed-"+time.Now().UTC().Format("20060102T150405.000000000Z")+".tar")
+	stop := exec.CommandContext(ctx, "systemctl", "stop", "k3s")
+	stop.Stdout = out
+	stop.Stderr = out
+	if err = stop.Run(); err != nil {
+		return err
+	}
+	defer func() {
+		restart, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		defer cancel()
+		cmd := exec.CommandContext(restart, "systemctl", "start", "k3s")
+		cmd.Stdout = out
+		cmd.Stderr = out
+		if restartErr := cmd.Run(); restartErr != nil {
+			err = fmt.Errorf("k3s restart failed after snapshot: %w", restartErr)
+		}
+	}()
+	cmd := exec.CommandContext(ctx, "tar", "-cf", path, "-C", "/var/lib/rancher/k3s/server", "db", "token", "tls")
+	cmd.Stdout = out
+	cmd.Stderr = out
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("cluster snapshot failed: %w", err)
+	}
+	if err = os.Chmod(path, 0600); err != nil {
+		return err
+	}
+	fmt.Fprintln(out, "Cluster snapshot:", path)
+	return nil
+}

cmd/felis/node_control_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/nodecontrol"
+	"felis.lolicon.best/internal/platform"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func TestNodeControlTopologyAndBootstrapID(t *testing.T) {
+	node := corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "controller", Labels: map[string]string{"node-role.kubernetes.io/control-plane": "true"}}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.0.2.1"}}}}
+	controller, peers, err := nodeController([]corev1.Node{node}, "192.0.2.2", []string{"192.0.2.1/32"})
+	if err != nil || controller.Name != "controller" || strings.Join(peers, ",") != "192.0.2.1/32,192.0.2.2/32" {
+		t.Fatal(controller, peers, err)
+	}
+	duplicate := node
+	duplicate.Name = "second"
+	if _, _, err = nodeController([]corev1.Node{node, duplicate}, "", nil); err == nil {
+		t.Fatal("multiple controllers accepted")
+	}
+	for _, token := range []string{"abcdef.0123456789abcdef", "K10hash::abcdef.0123456789abcdef\n"} {
+		if bootstrapTokenID(token) != "abcdef" {
+			t.Fatal("token secret passed to revocation")
+		}
+	}
+}
+func TestNodeControlRejectsActiveWorkloadsBeforeHostCommands(t *testing.T) {
+	replicas := int32(1)
+	set := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"}, Spec: appsv1.StatefulSetSpec{Replicas: &replicas}}
+	executor := nodeExecutor{cs: fake.NewSimpleClientset(set), namespace: platform.DefaultMinecraftNamespace}
+	if err := executor.requireStopped(context.Background(), nodecontrol.Request{Action: "enable"}); err == nil {
+		t.Fatal("host changes permitted with active worlds")
+	}
+}

cmd/felis/node_firewall.go

@@ -0,0 +1,187 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+func nodeFirewall(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("node firewall", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	peers := fs.String("peers", "", "comma-separated exact node IP CIDRs")
+	controller := fs.String("controller-ip", "", "controller address (optionally :6443)")
+	main := fs.Bool("controller", false, "A hosts the public API NodePort")
+	pod := fs.String("pod-cidr", "10.42.0.0/16", "cluster Pod CIDR")
+	dryRun := fs.Bool("dry-run", false, "print firewall scripts without installing")
+	controlNS := fs.String("control-namespace", "felis", "controller namespace")
+	minecraftNS := fs.String("namespace", "minecraft", "game namespace")
+	apiIP := fs.String("api-service-ip", "10.43.0.1", "Kubernetes API Service IP")
+	port := fs.Int("node-port", 30443, "API NodePort to block on workers before DNAT")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if host, _, err := net.SplitHostPort(*controller); err == nil {
+		*controller = host
+	}
+	if net.ParseIP(*apiIP) == nil || net.ParseIP(*controller) == nil || *port < 30000 || *port > 32767 {
+		fmt.Fprintln(stderr, "node firewall: controller IP and NodePort required")
+		return 2
+	}
+	if _, _, err := net.ParseCIDR(*pod); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+	var v4, v6 []string
+	for _, p := range strings.Split(*peers, ",") {
+		ip, n, err := net.ParseCIDR(p)
+		if err != nil {
+			fmt.Fprintln(stderr, "node firewall: invalid peer CIDR")
+			return 2
+		}
+		ones, bits := n.Mask.Size()
+		if ones != bits {
+			fmt.Fprintln(stderr, "node firewall: only exact peer addresses accepted")
+			return 2
+		}
+		if ip.To4() != nil {
+			v4 = append(v4, n.String())
+		} else {
+			v6 = append(v6, n.String())
+		}
+	}
+	script := "#!/bin/bash\nset -euo pipefail\n"
+	for _, f := range []struct {
+		bin      string
+		peers    []string
+		metadata string
+	}{{"iptables", v4, "169.254.0.0/16"}, {"ip6tables", v6, "fe80::/10"}} {
+		if f.bin == "ip6tables" && len(f.peers) == 0 {
+			continue
+		}
+		b := f.bin + " -w 10"
+		script += b + " -N FELIS-HOST 2>/dev/null || true\n" + b + " -F FELIS-HOST\n"
+		script += b + " -N FELIS-FORWARD 2>/dev/null || true\n" + b + " -F FELIS-FORWARD\n"
+		script += b + " -t raw -N FELIS-NODEPORT 2>/dev/null || true\n" + b + " -t raw -F FELIS-NODEPORT\n"
+		for _, c := range []struct{ table, parent, chain string }{{"filter", "INPUT", "FELIS-HOST"}, {"filter", "FORWARD", "FELIS-FORWARD"}, {"raw", "PREROUTING", "FELIS-NODEPORT"}} {
+			script += "while " + b + " -t " + c.table + " -D " + c.parent + " -j " + c.chain + " 2>/dev/null; do :; done\n" + b + " -t " + c.table + " -I " + c.parent + " 1 -j " + c.chain + "\n"
+		}
+		script += b + " -A FELIS-HOST -i lo -j RETURN\n"
+		if *main {
+			script += b + " -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -A FELIS-HOST -j FELIS-CONTROL\n"
+			script += b + " -t raw -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -t raw -A FELIS-NODEPORT -j FELIS-CONTROL\n"
+		}
+		script += b + " -A FELIS-HOST -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n" + b + " -A FELIS-FORWARD -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n"
+		family := 4
+		if f.bin == "ip6tables" {
+			family = 6
+		}
+		podIP, _, _ := net.ParseCIDR(*pod)
+		podFamily := 6
+		if podIP.To4() != nil {
+			podFamily = 4
+		}
+		if family == podFamily {
+			script += b + " -A FELIS-HOST -s " + *pod + " -j DROP\n"
+			// raw precedes DNAT and kube-router's filter ACCEPT rules. Block new
+			// host connections while preserving established Velocity/RCON replies.
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --syn -j DROP\n"
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p udp -j DROP\n"
+			if (net.ParseIP(*apiIP).To4() != nil) == (family == 4) {
+				script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + *apiIP + " -p tcp --dport 443 --syn -j DROP\n"
+			}
+
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n"
+			script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + f.metadata + " -j DROP\n"
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + f.metadata + " -j DROP\n"
+			for _, p := range f.peers {
+				script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + p + " -j DROP\n"
+				script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p tcp --syn -j DROP\n"
+				script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p udp -j DROP\n"
+			}
+		}
+		if !*main {
+			script += b + " -t raw -A FELIS-NODEPORT -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n"
+		}
+		for _, p := range f.peers {
+			script += b + " -A FELIS-HOST -s " + p + " -p udp --dport 51820:51821 -j RETURN\n"
+		}
+		script += b + " -A FELIS-HOST -p udp --dport 51820:51821 -j DROP\n"
+		ctrlIP := net.ParseIP(*controller)
+		ctrlFamily := 6
+		if ctrlIP.To4() != nil {
+			ctrlFamily = 4
+		}
+		if *main {
+			for _, p := range f.peers {
+				script += b + " -A FELIS-HOST -s " + p + " -p tcp --dport 6443 -j RETURN\n"
+			}
+		}
+		if ctrlFamily == family {
+			script += b + " -A FELIS-HOST -s " + ctrlIP.String() + " -p tcp --dport 10250 -j RETURN\n"
+		}
+		script += b + " -A FELIS-HOST -p tcp -m multiport --dports 6443,6444,10250,10255,2379,2380,5000,5001,15432 -j DROP\n"
+	}
+	if *dryRun {
+		fmt.Fprint(stdout, script)
+		if *main {
+			fmt.Fprint(stdout, controlFirewallScript(*controlNS, *minecraftNS))
+		}
+		return 0
+	}
+	if err := os.MkdirAll("/etc/felis", 0700); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err := os.WriteFile("/etc/felis/node-firewall.sh", []byte(script), 0700); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	unit := "[Unit]\nDescription=Felis host and NodePort isolation\nAfter=network-online.target firewalld.service ufw.service\nBefore=k3s.service k3s-agent.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/node-firewall.sh\nRemainAfterExit=yes\n[Install]\nWantedBy=multi-user.target\n"
+	if err := os.WriteFile("/etc/systemd/system/felis-node-firewall.service", []byte(unit), 0644); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if *main {
+		refresh := controlFirewallScript(*controlNS, *minecraftNS)
+		if err := os.WriteFile("/etc/felis/control-firewall.sh", []byte(refresh), 0700); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		svc := "[Unit]\nDescription=Refresh exact controller Pod access to the apiserver\nAfter=k3s.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/control-firewall.sh\n"
+		timer := "[Unit]\nDescription=Track trusted controller Pods after rescheduling\n[Timer]\nOnBootSec=5s\nOnUnitActiveSec=5s\n[Install]\nWantedBy=timers.target\n"
+		if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.service", []byte(svc), 0644); err != nil {
+			return 1
+		}
+		if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.timer", []byte(timer), 0644); err != nil {
+			return 1
+		}
+	}
+	for _, cmd := range []*exec.Cmd{exec.Command("systemctl", "daemon-reload"), exec.Command("systemctl", "enable", "felis-node-firewall.service"), exec.Command("bash", "/etc/felis/node-firewall.sh")} {
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		if err := cmd.Run(); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	if *main {
+		cmd := exec.Command("systemctl", "enable", "--now", "felis-control-firewall.timer")
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		if err := cmd.Run(); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	return 0
+}
+
+func controlFirewallScript(controlNS, minecraftNS string) string {
+	return "#!/bin/bash\nset -euo pipefail\niptables -w 10 -F FELIS-CONTROL\niptables -w 10 -t raw -F FELIS-CONTROL\n/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{\" \"}{.spec.serviceAccountName}{\" \"}{.status.podIP}{\"\\n\"}{end}' | while read -r ns sa ip; do\ncase \"$ns/$sa\" in " + shellQuote(controlNS+"/felis-api") + "|" + shellQuote(controlNS+"/felis-operator") + "|" + shellQuote(minecraftNS+"/felis-reaper") + "|kube-system/*) ;; *) continue;; esac\n[[ $ip =~ ^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$ ]] || continue\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p tcp --dport 6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p tcp -m multiport --dports 443,6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\ndone\n"
+}

cmd/felis/node_firewall_test.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"bytes"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestDistributedFirewallPathsAndSyntax(t *testing.T) {
+	for _, controller := range []bool{false, true} {
+		args := []string{"--dry-run", "--peers", "192.0.2.1/32,192.0.2.2/32", "--controller-ip", "192.0.2.1"}
+		if controller {
+			args = append(args, "--controller")
+		}
+		var out, err bytes.Buffer
+		if code := nodeFirewall(args, &out, &err); code != 0 {
+			t.Fatal(code, err.String())
+		}
+		script := out.String()
+		for _, must := range []string{"-I INPUT 1 -j FELIS-HOST", "-I FORWARD 1 -j FELIS-FORWARD", "-t raw -I PREROUTING 1 -j FELIS-NODEPORT", "-A FELIS-HOST -s 10.42.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --dport 30443 -j DROP", "-d 169.254.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --syn -j DROP", "-d 10.43.0.1 -p tcp --dport 443 --syn -j DROP"} {
+			if !strings.Contains(script, must) {
+				t.Fatal("missing protection", must)
+			}
+		}
+		if !controller && strings.Contains(script, " -p tcp --dport 6443 -j RETURN") {
+			t.Fatal("worker exposed apiserver")
+		}
+		check := exec.Command("bash", "-n")
+		check.Stdin = strings.NewReader(script)
+		if result, e := check.CombinedOutput(); e != nil {
+			t.Fatalf("invalid firewall script: %s %v", result, e)
+		}
+	}
+	var out, err bytes.Buffer
+	if code := nodeFirewall([]string{"--dry-run", "--peers", "10.0.0.0/8", "--controller-ip", "10.0.0.1"}, &out, &err); code != 2 {
+		t.Fatal("broad node range accepted")
+	}
+}

cmd/felis/node_probe.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"time"
+)
+
+// A trusted probe runs with the same server labels and security context before node admission.
+func cmdNodeProbe(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("node-probe", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	listen := fs.String("listen", "", "listen and print observed source addresses (admission only)")
+	var open, closed multiFlag
+	fs.Var(&open, "open", "required reachable host:port")
+	fs.Var(&closed, "closed", "required blocked host:port")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *listen != "" {
+		l, err := net.Listen("tcp", *listen)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		defer l.Close()
+		for {
+			c, err := l.Accept()
+			if err != nil {
+				return 1
+			}
+			fmt.Fprintln(stdout, "felis-probe-source", c.RemoteAddr().String())
+			c.Write([]byte("felis-probe\n"))
+			c.Close()
+		}
+	}
+	time.Sleep(3 * time.Second)
+	for _, check := range []struct {
+		addresses []string
+		wantOpen  bool
+	}{{open, true}, {closed, false}} {
+		for _, addr := range check.addresses {
+			c, err := net.DialTimeout("tcp", addr, 2*time.Second)
+			if c != nil {
+				c.Close()
+			}
+			if (err == nil) != check.wantOpen {
+				fmt.Fprintf(stderr, "node-probe: %s open=%t, expected %t\n", addr, err == nil, check.wantOpen)
+				return 1
+			}
+		}
+	}
+	return 0
+}

cmd/felis/nodes.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	felis "felis.lolicon.best"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/platform"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func cmdNode(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprintln(stderr, "felis node: list | token | join | approve | firewall")
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis node requires root/sudo")
+		return 1
+	}
+	if args[0] == "firewall" {
+		return nodeFirewall(args[1:], stdout, stderr)
+	}
+	fs := flag.NewFlagSet("node "+args[0], flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	name := fs.String("name", "", "stable worker node name")
+	kubeconfig := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local administrator kubeconfig")
+	ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
+	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "controller namespace")
+	image := fs.String("image", "", "Felis image to re-pull and use for admission probes")
+	remote := fs.String("ssh-target", "", "SSH target of the trusted worker (normal SSH host verification applies)")
+	out := fs.String("out", "", "token output file; never printed to stdout")
+	ttl := fs.Duration("ttl", 10*time.Minute, "bootstrap token lifetime (maximum 1h)")
+	server := fs.String("server", "", "A's https://address:6443 endpoint for join")
+	tokenFile := fs.String("token-file", "", "CA-pinned bootstrap token file for join")
+	registry := fs.String("registry-ip", "", "registry ClusterIP for join")
+	peers := fs.String("peers", "", "comma-separated exact peer CIDRs for host firewall")
+	external := fs.String("external-ip", "", "this worker's fixed external IP")
+	if err := fs.Parse(args[1:]); err != nil {
+		return 2
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer cancel()
+	switch args[0] {
+	case "token":
+		if *name == "" || *out == "" || *ttl <= 0 || *ttl > time.Hour {
+			fmt.Fprintln(stderr, "node token: name, output file and lifetime <=1h required")
+			return 2
+		}
+		cmd := exec.CommandContext(ctx, "k3s", "token", "create", "--ttl", ttl.String(), "--description", "Felis worker "+*name)
+		cmd.Stderr = stderr
+		raw, err := cmd.Output()
+		if err != nil {
+			fmt.Fprintln(stderr, "node token: creation failed:", err)
+			return 1
+		}
+		f, err := os.OpenFile(*out, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		_, err = f.Write(raw)
+		if err == nil {
+			err = f.Sync()
+		}
+		f.Close()
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		fmt.Fprintln(stdout, "limited bootstrap token written to", *out)
+		return 0
+	case "join":
+		exe, err := os.Executable()
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		cmd := exec.CommandContext(ctx, "bash", "-s")
+		cmd.Stdin = strings.NewReader(felis.BootstrapScript())
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		cmd.Env = append(os.Environ(), "FELIS_INSTALL_MODE=worker", "FELIS_BOOTSTRAP_FROM_TUI=1", "FELIS_BOOTSTRAP_BINARY="+exe, "FELIS_NODE_NAME="+*name, "FELIS_SERVER_URL="+*server, "FELIS_BOOTSTRAP_TOKEN_FILE="+*tokenFile, "FELIS_REGISTRY_CLUSTER_IP="+*registry, "FELIS_PEER_CIDRS="+*peers, "FELIS_NODE_EXTERNAL_IP="+*external)
+		if err = cmd.Run(); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		return 0
+	case "list", "approve":
+	default:
+		fmt.Fprintln(stderr, "unknown node operation")
+		return 2
+	}
+	cfg, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	scheme := runtime.NewScheme()
+	clientgoscheme.AddToScheme(scheme)
+	v1alpha1.AddToScheme(scheme)
+	cl, err := client.New(cfg, client.Options{Scheme: scheme})
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	m := &distributed.Manager{Client: cl, Namespace: *ns}
+	if args[0] == "list" {
+		nodes, err := m.Nodes(ctx)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		json.NewEncoder(stdout).Encode(nodes)
+		return 0
+	}
+	if *name == "" || *image == "" || *remote == "" || strings.HasPrefix(*remote, "-") {
+		fmt.Fprintln(stderr, "node approve requires name, image and SSH target")
+		return 2
+	}
+	cs, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err = approveNode(ctx, cl, cs, *ns, *controlNS, *name, *image, *remote, stdout); err != nil {
+		fmt.Fprintln(stderr, "node remains quarantined:", err)
+		return 1
+	}
+	return 0
+}
+
+func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" }
+func exactCIDR(ip string) (string, error) {
+	parsed := net.ParseIP(ip)
+	if parsed == nil {
+		return "", fmt.Errorf("invalid node address %q", ip)
+	}
+	if parsed.To4() != nil {
+		return parsed.String() + "/32", nil
+	}
+	return parsed.String() + "/128", nil
+}

cmd/felis/offsite.go

@@ -0,0 +1,946 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const offsiteUsage = `usage:
+  felis offsite sync         [-config path] [-archive-dir dir] [-db-dir dir] [-registry host:port|off]
+                             [-uploads-dir dir] [-status-file path]
+  felis offsite status       [-config path] [-status-file path]
+  felis offsite list         [-config path]
+  felis offsite fetch-db     [-config path | -endpoint url -bucket name [-region r] [-prefix p]]
+                             [-dir dir] latest|<bundle>
+  felis offsite fetch-worlds [-config path] [-archive-dir dir]
+  felis offsite fetch-images [-config path] [-registry host:port] [-at version]
+  felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
+  felis offsite check-key    [-config path]
+  felis offsite take-over    [-config path] [-status-file path] [-yes]
+  felis offsite keygen
+
+Every verb but keygen reads the bucket credentials and the encryption key from
+the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY,
+FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
+-env-file (default /etc/felis/offsite.env).
+
+check-key tells whether the key is the one the bucket's objects are sealed
+with, writing nothing; it exits 3 when they are sealed with another key, and
+sync then refuses to write or prune anything in the bucket.
+
+take-over names the host that writes the bucket, writing nothing; it exits 4
+when that is another host and this one never wrote it, and 5 when another
+host took the bucket over from this one. A host built from another host's
+backup (a rehearsal, or a rebuild) copies nothing into that host's bucket
+until -yes makes it the writer; the host it replaces then stops copying and
+says so.
+`
+
+// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
+// felis-offsite.service unit loads it as its EnvironmentFile.
+const defaultOffsiteEnvFile = "/etc/felis/offsite.env"
+
+// cmdOffsite implements `felis offsite`: the off-site copy of the world
+// archives, the database bundles, the registry's user images and the
+// submission uploads (internal/offsite). felis-offsite.timer runs `sync`
+// hourly on the host; the fetch verbs are the way back after the node is lost
+// (docs/troubleshooting.md §16).
+func cmdOffsite(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("offsite "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, offsiteUsage) }
+	switch verb {
+	case "sync":
+		return offsiteSync(fs, rest, stdout, stderr)
+	case "status":
+		return offsiteStatus(fs, rest, stdout, stderr)
+	case "list":
+		return offsiteList(fs, rest, stdout, stderr)
+	case "fetch-db":
+		return offsiteFetchDB(fs, rest, stdout, stderr)
+	case "fetch-worlds":
+		return offsiteFetchWorlds(fs, rest, stdout, stderr)
+	case "fetch-images":
+		return offsiteFetchImages(fs, rest, stdout, stderr)
+	case "fetch-uploads":
+		return offsiteFetchUploads(fs, rest, stdout, stderr)
+	case "check-key":
+		return offsiteCheckKey(fs, rest, stdout, stderr)
+	case "take-over":
+		return offsiteTakeOver(fs, rest, stdout, stderr)
+	case "keygen":
+		k, err := offsite.NewKey()
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite keygen: %v\n", err)
+			return 1
+		}
+		fmt.Fprintln(stdout, k)
+		return 0
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, offsiteUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis offsite: unknown verb %q\n%s", verb, offsiteUsage)
+	return 2
+}
+
+// offsiteEnv is the resolved [offsite] binding: the bucket and the key.
+type offsiteEnv struct {
+	cfg    config.OffsiteConfig
+	bucket *offsite.S3
+	key    []byte
+}
+
+// loadEnvFile sets each KEY=VALUE of path that is not already in the
+// environment, so a root shell runs a command the same way its unit does.
+// A missing file is not an error.
+func loadEnvFile(path string) error {
+	if path == "" {
+		return nil
+	}
+	f, err := os.Open(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		k = strings.TrimSpace(strings.TrimPrefix(k, "export "))
+		v = strings.TrimSpace(v)
+		if len(v) >= 2 && (v[0] == '"' || v[0] == '\'') && v[len(v)-1] == v[0] {
+			v = v[1 : len(v)-1]
+		}
+		if os.Getenv(k) == "" {
+			os.Setenv(k, v)
+		}
+	}
+	return sc.Err()
+}
+
+// resolveOffsite builds the bucket client and parses the key for c.
+func resolveOffsite(c config.OffsiteConfig) (*offsiteEnv, error) {
+	if !c.Enabled() {
+		return nil, errors.New("no [offsite] bucket is configured (docs/troubleshooting.md §16, \"Keep a copy somewhere else\")")
+	}
+	need := func(ref, what string) (string, error) {
+		v := os.Getenv(ref)
+		if v == "" {
+			return "", fmt.Errorf("%s: environment variable %s is empty (set it, or put it in %s)", what, ref, defaultOffsiteEnvFile)
+		}
+		return v, nil
+	}
+	ak, err := need(c.AccessKeyRef, "access key")
+	if err != nil {
+		return nil, err
+	}
+	sk, err := need(c.SecretKeyRef, "secret key")
+	if err != nil {
+		return nil, err
+	}
+	rawKey, err := need(c.KeyRef, "encryption key")
+	if err != nil {
+		return nil, err
+	}
+	key, err := offsite.ParseKey(rawKey)
+	if err != nil {
+		return nil, err
+	}
+	b, err := offsite.NewS3(offsite.S3Config{
+		Endpoint: c.Endpoint, Region: c.Region, Bucket: c.Bucket, Prefix: c.Prefix,
+		AccessKey: ak, SecretKey: sk,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &offsiteEnv{cfg: c, bucket: b, key: key}, nil
+}
+
+// loadOffsite loads felis.toml and the env file and resolves [offsite].
+func loadOffsite(cfgPath, envFile string) (*config.Config, *offsiteEnv, error) {
+	if err := loadEnvFile(envFile); err != nil {
+		return nil, nil, fmt.Errorf("read %s: %w", envFile, err)
+	}
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	env, err := resolveOffsite(cfg.Offsite)
+	if err != nil {
+		return cfg, nil, err
+	}
+	return cfg, env, nil
+}
+
+func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
+	dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory bundled into the database bundle taken after archives are copied ("" for none)`)
+	registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC through the cluster)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, `the submission uploads PVC, in the control-plane namespace ("" copies no uploads)`)
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "where the result of this run is recorded for the watchdog and `status`")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	st, lease := startRun(env.cfg, env.key, *statusFile, time.Now())
+	res, err := runOffsiteSync(cfg, env, offsiteSources{
+		archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir, stateDir: *stateDir,
+		registry:   offsiteRegistryEndpoint(*registry, cfg.Registry),
+		uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
+	}, &lease, stderr)
+	recordRun(&st, res, err, lease)
+	if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
+	}
+	return reportRun(res, err, stdout, stderr)
+}
+
+// reportRun prints one pass's outcome and its exit code. A run stopped before
+// it copied anything (a bucket that did not answer, another key's objects,
+// another host writing the bucket) prints no counts: its zeros would read as
+// an empty bucket.
+func reportRun(res offsite.Result, err error, stdout, stderr io.Writer) int {
+	if err == nil || len(res.Errors) > 0 {
+		fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n",
+			res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
+			res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
+			res.UploadsUploaded, res.UploadObjectsPruned,
+			res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes),
+			res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes))
+	}
+	for _, m := range res.WorldsMissing {
+		fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
+	}
+	for _, m := range res.ImagesIncomplete {
+		fmt.Fprintf(stderr, "felis offsite sync: registry image not whole: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// startRun begins a pass: its status record, in this release's format and
+// carrying the last success over, and this host's lease, read from the record
+// the last pass left.
+func startRun(cfg config.OffsiteConfig, key []byte, statusFile string, now time.Time) (offsite.Status, offsite.Lease) {
+	st := offsite.Status{
+		LastAttempt: now.UTC(), Endpoint: cfg.Endpoint, Bucket: cfg.Bucket,
+		Prefix: cfg.Prefix, KeyID: offsite.KeyID(key), Format: offsite.StatusFormat,
+	}
+	if prev, _ := offsite.ReadStatus(statusFile); prev != nil {
+		st.LastSuccess = prev.LastSuccess
+	}
+	return st, offsite.HostLease(statusFile)
+}
+
+// recordRun puts one pass's outcome into its status record. A host that
+// inherited the bucket from an older release keeps that until it has an id.
+func recordRun(st *offsite.Status, res offsite.Result, err error, lease offsite.Lease) {
+	st.Result = res
+	if err != nil {
+		st.LastError = err.Error()
+		st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch)
+		var we *offsite.WriterError
+		if errors.As(err, &we) {
+			st.Standby = errors.Is(err, offsite.ErrStandby)
+			st.Displaced = errors.Is(err, offsite.ErrDisplaced)
+			st.Writer = we.Writer
+		}
+	} else {
+		st.LastSuccess = st.LastAttempt
+	}
+	if lease.Inherited {
+		id, _ := lease.ID()
+		st.Inherited = id == ""
+	}
+}
+
+// offsiteSources is where one sync pass reads from: the world archive volume
+// (archiveDir, or the backupPVC's directory), the bundle directory (with the
+// host state the pass bundles, stateDir), the registry's loopback endpoint and
+// the uploads volume (uploadsDir, or the uploadsPVC's directory). An empty
+// source is skipped.
+type offsiteSources struct {
+	archiveDir, backupPVC  string
+	dbDir, stateDir        string
+	registry               string
+	uploadsDir, uploadsPVC string
+}
+
+// offsiteRunLimit backstops one sync pass. Each upload has its own deadline,
+// scaled to its size (internal/offsite), so a pass over a big archive may run
+// for hours; the timer starts no second pass while one runs, and the unit's
+// TimeoutStartSec sits above this.
+const offsiteRunLimit = 23 * time.Hour
+
+func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lease *offsite.Lease, log io.Writer) (offsite.Result, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit)
+	defer cancel()
+	checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
+	err := env.bucket.Check(checkCtx)
+	checkCancel()
+	if err != nil {
+		return offsite.Result{}, err
+	}
+	archiveDir, uploadsDir := src.archiveDir, src.uploadsDir
+	if archiveDir == "" && src.backupPVC != "" {
+		dir, err := resolveVolumeDir(ctx, cfg.K8s.Namespace, src.backupPVC, archiveVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		archiveDir = dir
+	}
+	// An s3:// uploads store is off the host already; only a local one, on
+	// the uploads PVC, needs the copy.
+	if uploadsDir == "" && src.uploadsPVC != "" && isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+		dir, err := resolveVolumeDir(ctx, platform.DefaultControlNamespace, src.uploadsPVC, uploadsVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		uploadsDir = dir
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		return offsite.Result{}, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	s := offsiteSyncer(cfg, env, src, archiveDir, uploadsDir, lease, log)
+	s.Catalog = offsite.PGCatalog{DB: drv.DB()}
+	if src.registry != "" {
+		s.Images = newRegistryImages(src.registry)
+		s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL)
+	}
+	return s.Run(ctx)
+}
+
+// offsiteSyncer is the pass runOffsiteSync runs over the resolved archive and
+// uploads directories, before its catalog and registry are attached. It
+// snapshots the database into the bundle directory after copying archives, and
+// sweeps world objects no backup records once they outlive every retention in
+// [archive]; a retention that does not parse sweeps none.
+func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, archiveDir, uploadsDir string, lease *offsite.Lease, log io.Writer) *offsite.Syncer {
+	s := &offsite.Syncer{
+		Bucket: env.bucket, Key: env.key,
+		ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log,
+	}
+	if src.dbDir != "" {
+		s.Snapshot = offsiteSnapshot(cfg.Database, src.dbDir, src.stateDir, log)
+	}
+	if rc, err := reaperConfig(cfg); err != nil {
+		fmt.Fprintf(log, "felis offsite: world objects no backup records are kept: %v\n", err)
+	} else {
+		s.OrphanAfter = max(rc.Retention, rc.ManualRetention, rc.ScheduledRetention)
+	}
+	return s
+}
+
+// offsiteSnapshot takes the bundle a pass sends after copying world archives
+// (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite,
+// with the newest one kept in dir. It is not recorded for the panel, whose
+// backup card watches felis-db-backup.timer: snapshots come only when archives
+// are copied, and would hide a daily timer that stopped. It requires the
+// MinecraftServer objects: it becomes the newest bundle in the bucket, which a
+// lost host restores from, and a pass that cannot take a whole one fails and
+// tries again next hour.
+func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error {
+	return func(ctx context.Context) error {
+		tools, err := dbTools(db)
+		if err != nil {
+			return err
+		}
+		ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+		defer cancel()
+		path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
+			DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite,
+			Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(),
+			ExportServers: exportMinecraftServers, RequireServers: true, Log: log,
+		})
+		if err == nil {
+			fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path))
+		}
+		return err
+	}
+}
+
+// volumeKind names a PVC the off-site copy reads or restores, for messages,
+// with the flag that bypasses finding it through the cluster.
+type volumeKind struct{ what, dirFlag, empty string }
+
+var (
+	archiveVolume = volumeKind{"archive volume", "-archive-dir", "no world has been archived"}
+	uploadsVolume = volumeKind{"uploads volume", "-uploads-dir", "no modpack has been uploaded"}
+)
+
+// resolveVolumeDir finds the host directory behind a PVC: a local-path volume
+// is a directory on this node. A PVC still waiting for its first consumer
+// holds nothing yet: without bind that is "" (nothing to copy), with bind it
+// is bound first, for a fetch to write into.
+func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind, bind bool, log io.Writer) (string, error) {
+	if ns == "" {
+		ns = platform.DefaultMinecraftNamespace
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		return "", fmt.Errorf("reach the cluster to find the %s (or pass %s): %w", kind.what, kind.dirFlag, err)
+	}
+	var pvc corev1.PersistentVolumeClaim
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+		return "", fmt.Errorf("%s %s/%s: %w", kind.what, ns, pvcName, err)
+	}
+	if pvc.Spec.VolumeName == "" {
+		if !bind {
+			fmt.Fprintf(log, "felis offsite: %s %s/%s is not bound yet; %s\n", kind.what, ns, pvcName, kind.empty)
+			return "", nil
+		}
+		if err := bindVolume(ctx, cl, ns, pvcName, kind, log); err != nil {
+			return "", err
+		}
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+			return "", err
+		}
+	}
+	var pv corev1.PersistentVolume
+	if err := cl.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil {
+		return "", fmt.Errorf("%s %s: %w", kind.what, pvc.Spec.VolumeName, err)
+	}
+	var dir string
+	switch {
+	case pv.Spec.Local != nil:
+		dir = pv.Spec.Local.Path
+	case pv.Spec.HostPath != nil:
+		dir = pv.Spec.HostPath.Path
+	default:
+		return "", fmt.Errorf("%s %s is not a directory on a node (local or hostPath); pass %s with where it is mounted on this host", kind.what, pv.Name, kind.dirFlag)
+	}
+	if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
+		return "", fmt.Errorf("%s %s is %s on its node, which is not a directory here; run this on the node that holds it, or pass %s", kind.what, pv.Name, dir, kind.dirFlag)
+	}
+	return dir, nil
+}
+
+// bindVolume runs a pod that mounts the PVC and exits, which is what makes a
+// WaitForFirstConsumer volume (k3s local-path) get provisioned. The pod uses
+// the control plane's own image, which every install already has.
+func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind volumeKind, log io.Writer) error {
+	var api appsv1.Deployment
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: platform.DefaultControlNamespace, Name: "felis-api"}, &api); err != nil {
+		return fmt.Errorf("find the felis image to bind the %s with: %w", kind.what, err)
+	}
+	if len(api.Spec.Template.Spec.Containers) == 0 {
+		return errors.New("felis-api has no container to take the image from")
+	}
+	image := api.Spec.Template.Spec.Containers[0].Image
+	pod := platform.VolumeBinderPod(ns, pvcName, image)
+	if err := cl.Create(ctx, pod); err != nil {
+		return fmt.Errorf("start a pod to bind the %s: %w", kind.what, err)
+	}
+	fmt.Fprintf(log, "felis offsite: binding the %s %s/%s (pod %s)\n", kind.what, ns, pvcName, pod.Name)
+	defer func() {
+		_ = cl.Delete(context.Background(), pod, client.PropagationPolicy(metav1.DeletePropagationBackground))
+	}()
+	deadline := time.Now().Add(3 * time.Minute)
+	for time.Now().Before(deadline) {
+		var pvc corev1.PersistentVolumeClaim
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err == nil && pvc.Spec.VolumeName != "" && pvc.Status.Phase == corev1.ClaimBound {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+	return fmt.Errorf("the %s %s/%s did not bind within 3 minutes; see kubectl -n %s describe pod %s", kind.what, ns, pvcName, ns, pod.Name)
+}
+
+func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if !cfg.Offsite.Enabled() {
+		fmt.Fprintln(stdout, "off-site copy: not configured. World archives, database bundles, user images and uploaded modpacks exist on this machine only.")
+		fmt.Fprintln(stdout, "See docs/troubleshooting.md §16, \"Keep a copy somewhere else\".")
+		return 1
+	}
+	o := cfg.Offsite
+	fmt.Fprintf(stdout, "bucket:       %s at %s", o.Bucket, o.Endpoint)
+	if o.Prefix != "" {
+		fmt.Fprintf(stdout, ", prefix %s", o.Prefix)
+	}
+	fmt.Fprintln(stdout)
+	st, err := offsite.ReadStatus(*statusFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if st == nil {
+		fmt.Fprintln(stdout, "last sync:    never (sudo systemctl start felis-offsite.service)")
+		return 1
+	}
+	now := time.Now()
+	fmt.Fprintf(stdout, "key id:       %s\n", st.KeyID)
+	fmt.Fprintf(stdout, "last attempt: %s (%s ago)\n", st.LastAttempt.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastAttempt)))
+	if st.LastSuccess.IsZero() {
+		fmt.Fprintln(stdout, "last success: never")
+	} else {
+		fmt.Fprintf(stdout, "last success: %s (%s ago)\n", st.LastSuccess.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastSuccess)))
+	}
+	if st.LastError != "" {
+		fmt.Fprintf(stdout, "last error:   %s\n", st.LastError)
+	}
+	if st.KeyMismatch {
+		fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile)
+		return 1
+	}
+	if st.Displaced && st.Writer != nil {
+		fmt.Fprintf(stdout, "\nThe last run was refused: %s took the bucket over (it last wrote it at %s), and this host copies nothing there any more. If that host is a rehearsal machine, take the bucket back: sudo felis offsite take-over -yes\n",
+			st.Writer, st.Writer.At.Local().Format(time.DateTime))
+		return 1
+	}
+	if st.Standby {
+		switch w := st.StandsBy(now); {
+		case w != nil:
+			fmt.Fprintf(stdout, "\nThis host stands by: %s writes the bucket (last at %s). This host was built from its backup, copies nothing into the bucket and, while that host keeps writing, mails no watchdog alert.", w, w.At.Local().Format(time.DateTime))
+		case st.Writer != nil:
+			fmt.Fprintf(stdout, "\nThis host copies nothing into the bucket: %s wrote it, last at %s, and this host was built from its backup.", st.Writer, st.Writer.At.Local().Format(time.DateTime))
+		default:
+			fmt.Fprint(stdout, "\nThis host copies nothing into the bucket: it holds copies this host did not write, and names no host writing it.")
+		}
+		fmt.Fprintln(stdout, " Once this host replaces that one for good: sudo felis offsite take-over -yes")
+		return 1
+	}
+	r := st.Result
+	fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
+		r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
+	if r.ImageIndex != "" {
+		fmt.Fprintf(stdout, "images:       %d in %d repositories (%s), registry index %s\n",
+			r.Images, r.ImageRepos, offsite.HumanBytes(r.RemoteImageBytes), r.ImageIndex)
+	} else {
+		fmt.Fprintln(stdout, "images:       not copied (no in-cluster registry, or no sync has reached it yet)")
+	}
+	if r.UploadIndex != "" {
+		fmt.Fprintf(stdout, "uploads:      %d submission contexts (%s), uploads index %s\n",
+			r.Uploads, offsite.HumanBytes(r.RemoteUploadBytes), r.UploadIndex)
+	} else {
+		fmt.Fprintln(stdout, "uploads:      not copied (an s3:// uploads store, or no sync has reached the volume yet)")
+	}
+	fmt.Fprintf(stdout, "waiting:      %d world archives not yet copied\n", r.WorldsPending)
+	for _, m := range r.WorldsMissing {
+		fmt.Fprintf(stdout, "missing:      %s is recorded but not on the volume\n", m)
+	}
+	for _, m := range r.ImagesIncomplete {
+		fmt.Fprintf(stdout, "not whole:    %s\n", m)
+	}
+	if st.LastSuccess.IsZero() || now.Sub(st.LastSuccess) > offsite.StaleAfter {
+		fmt.Fprintf(stdout, "\nThe last successful sync is older than %s: journalctl -u felis-offsite -n 50\n", dbbackup.Age(offsite.StaleAfter))
+		return 1
+	}
+	return 0
+}
+
+func orNone(s string) string {
+	if s == "" {
+		return "none"
+	}
+	return s
+}
+
+func offsiteList(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	return printOffsiteList(env, stdout, stderr)
+}
+
+func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	bundles, err := offsite.ListDB(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	worlds, err := env.bucket.List(ctx, "worlds/")
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	printDBBundles(ctx, env.bucket, env.key, bundles, stdout)
+	var total int64
+	for _, w := range worlds {
+		total += w.Size
+	}
+	fmt.Fprintf(stdout, "world archives: %d (%s)\n", len(worlds), offsite.HumanBytes(total))
+	versions, err := offsite.ImageIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "registry index versions (%d, newest first; restore one with fetch-images -at):\n", len(versions))
+	for i := len(versions) - 1; i >= 0; i-- {
+		x, err := offsite.LoadImageIndex(ctx, env.bucket, env.key, versions[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", versions[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d images in %d repositories\n", versions[i], x.Images(), len(x.Repositories))
+	}
+	uploads, err := offsite.UploadIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "uploads index versions (%d, newest first; restore one with fetch-uploads -at):\n", len(uploads))
+	for i := len(uploads) - 1; i >= 0; i-- {
+		x, err := offsite.LoadUploadIndex(ctx, env.bucket, env.key, uploads[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", uploads[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d submission contexts (%s)\n", uploads[i], len(x.Contexts), offsite.HumanBytes(x.Bytes()))
+	}
+	return 0
+}
+
+// printDBBundles lists the database bundles with what each one's database
+// held, read off the front of each, so a restore can pick one by its contents.
+func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles []offsite.Object, stdout io.Writer) {
+	fmt.Fprintf(stdout, "database bundles (%d, newest first; restore one with fetch-db):\n", len(bundles))
+	for _, o := range bundles {
+		m, err := offsite.PeekDB(ctx, b, key, o.Key)
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  %s  unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err)
+			continue
+		}
+		gap := ""
+		if m.ServersError != "" {
+			gap = ", no MinecraftServer objects"
+		}
+		fmt.Fprintf(stdout, "  %s  %s  %s%s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String(), gap)
+	}
+}
+
+func offsiteCheckKey(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	if err := env.bucket.Check(ctx); err != nil {
+		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
+		return 1
+	}
+	return checkKey(ctx, env.bucket, env.key, stdout, stderr)
+}
+
+// checkKey is check-key once the bucket is open: 0 when the key fits, 3 when
+// the bucket's objects are sealed with another one, 1 when it cannot tell.
+func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr io.Writer) int {
+	fit, err := offsite.CheckKey(ctx, b, key)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
+		if errors.Is(err, offsite.ErrKeyMismatch) {
+			return 3
+		}
+		return 1
+	}
+	id := offsite.KeyID(key)
+	switch fit {
+	case offsite.KeyRecorded:
+		fmt.Fprintf(stdout, "felis offsite check-key: the bucket records key id %s, this key's\n", id)
+	case offsite.KeyOpens:
+		fmt.Fprintf(stdout, "felis offsite check-key: the bucket's newest objects open with this key (key id %s); the next sync records it\n", id)
+	case offsite.KeyUnused:
+		fmt.Fprintf(stdout, "felis offsite check-key: the bucket holds no sealed object yet; the first sync records key id %s\n", id)
+	}
+	return 0
+}
+
+func offsiteTakeOver(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes; this host's id is kept next to it")
+	yes := fs.Bool("yes", false, "make this host the one that writes the bucket")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	if err := env.bucket.Check(ctx); err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	return takeOver(ctx, env.bucket, env.key, offsite.HostLease(*statusFile), *statusFile, *yes, time.Now(), stdout, stderr)
+}
+
+// takeOver is take-over once the bucket is open: without yes it says which
+// host writes the bucket, 0 for this one (or none yet), 4 for another and 5
+// for one that took the bucket over from this host; with
+// yes it records this host as the writer. A key the bucket's objects refuse
+// is 3, as in check-key: taking over a bucket this host cannot copy into
+// would only stop the host that can.
+func takeOver(ctx context.Context, b offsite.Bucket, key []byte, lease offsite.Lease, statusFile string, yes bool, now time.Time, stdout, stderr io.Writer) int {
+	fit, err := offsite.CheckKey(ctx, b, key)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		if errors.Is(err, offsite.ErrKeyMismatch) {
+			return 3
+		}
+		return 1
+	}
+	role, w, err := lease.Plan(ctx, b, fit == offsite.KeyUnused)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	switch role {
+	case offsite.RoleWrites:
+		id, _ := lease.ID()
+		fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket; nothing to take over\n", id)
+		return 0
+	case offsite.RoleClaims:
+		fmt.Fprintln(stdout, "felis offsite take-over: the bucket names no host writing it; this host's next sync records itself")
+		return 0
+	}
+	who := "another host"
+	if w != nil {
+		who = w.String()
+		fmt.Fprintf(stdout, "felis offsite take-over: %s writes the bucket, last at %s (%s ago)\n", w, w.At.Local().Format(time.DateTime), dbbackup.Age(now.Sub(w.At)))
+	} else {
+		fmt.Fprintln(stdout, "felis offsite take-over: the bucket holds copies this host did not write, and names no host writing it")
+	}
+	if !yes {
+		if role == offsite.RoleDisplaced {
+			fmt.Fprintf(stdout, "It took the bucket over from this host: this host copies nothing there any more, and its watchdog mails the owners about it. If that host is a rehearsal machine, take the bucket back:\n  sudo felis offsite take-over -yes\n")
+			return 5
+		}
+		fmt.Fprintf(stdout, "This host was built from its backup and copies nothing into the bucket.\n")
+		fmt.Fprintf(stdout, "Taking it over makes this host the one that copies into the bucket and prunes it; %s stops at its next copy and mails its owners. Do it once that host is gone for good, or is a rehearsal machine you are done with:\n  sudo felis offsite take-over -yes\n", who)
+		return 4
+	}
+	if _, err := lease.TakeOver(ctx, b, now); err != nil {
+		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
+		return 1
+	}
+	// The refusal the last sync recorded is over: the watchdog mails again
+	// from now on, and status shows the next run's outcome.
+	if st, err := offsite.ReadStatus(statusFile); err == nil && st != nil && (st.Standby || st.Displaced) {
+		st.Standby, st.Displaced, st.Writer, st.LastError, st.Inherited = false, false, nil, "", false
+		if err := offsite.WriteStatus(statusFile, *st); err != nil {
+			fmt.Fprintf(stderr, "felis offsite take-over: record status: %v\n", err)
+		}
+	}
+	id, _ := lease.ID()
+	fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket now; %s stops at its next copy.\nStart the first copy: sudo systemctl start felis-offsite.service\n", id, who)
+	return 0
+}
+
+// keyHint explains an object the key cannot open when the bucket records
+// another key's id, "" otherwise.
+func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string {
+	if !errors.Is(err, offsite.ErrAuth) {
+		return ""
+	}
+	id, ierr := offsite.BucketKeyID(ctx, b)
+	if ierr != nil || id == "" || id == offsite.KeyID(key) {
+		return ""
+	}
+	return fmt.Sprintf("\n  the bucket records key id %s, and this key is %s: set FELIS_OFFSITE_KEY to the key the bucket was written with", id, offsite.KeyID(key))
+}
+
+func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	endpoint := fs.String("endpoint", "", "bucket endpoint, when there is no felis.toml")
+	bucket := fs.String("bucket", "", "bucket name, when there is no felis.toml")
+	region := fs.String("region", "", "bucket region, when there is no felis.toml")
+	prefix := fs.String("prefix", "", "key prefix, when there is no felis.toml")
+	dir := fs.String("dir", dbbackup.DefaultDir, "directory to write the bundle to")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	if err := loadEnvFile(*envFile); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: read %s: %v\n", *envFile, err)
+		return 1
+	}
+	var oc config.OffsiteConfig
+	if *bucket != "" {
+		oc = config.OffsiteConfig{
+			Endpoint: *endpoint, Bucket: *bucket, Region: *region, Prefix: *prefix,
+			AccessKeyRef: config.DefaultOffsiteAccessKeyEnv, SecretKeyRef: config.DefaultOffsiteSecretKeyEnv,
+			KeyRef: config.DefaultOffsiteKeyEnv,
+		}
+	} else {
+		cfg, err := config.Load(*cfgPath)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v (on a host with no install yet, pass -endpoint and -bucket)\n", err)
+			return 1
+		}
+		oc = cfg.Offsite
+	}
+	env, err := resolveOffsite(oc)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	return fetchDB(ctx, env.bucket, env.key, arg, *dir, time.Now(), stdout, stderr)
+}
+
+// fetchDB is fetch-db once the bucket is open: arg is a bundle name or latest.
+func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string, now time.Time, stdout, stderr io.Writer) int {
+	name := arg
+	if name == "latest" {
+		var err error
+		if name, _, err = offsite.ChooseDB(ctx, b, key); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
+			return 1
+		}
+	}
+	if _, _, ok := dbbackup.ParseBundleName(name); !ok {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %q is not a bundle name (felis-db-<stamp>-<label>.tar); see `felis offsite list`\n", name)
+		return 2
+	}
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	dst := filepath.Join(dir, name)
+	if err := offsite.FetchObject(ctx, b, key, offsite.DBKey(name), dst, 0o600); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
+		return 1
+	}
+	m, err := dbbackup.Verify(dst)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: fetched %s but it does not verify: %v\n", dst, err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-db: wrote %s (verified)\n", dst)
+	fmt.Fprintf(stdout, "  taken   %s (%s, %s ago)\n  felis   %s, schema %d\n  holds   %s\n",
+		m.CreatedAt.Format(time.RFC3339), m.Label, dbbackup.Age(now.Sub(m.CreatedAt)),
+		orUnknown(m.FelisVersion), m.SchemaVersion, m.Counts.String())
+	if m.Counts.Fresh() {
+		fmt.Fprintln(stdout, "  This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.")
+	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  This bundle lacks the MinecraftServer objects (%s): `felis db restore` brings back the database, and the servers come from k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap.\n", m.ServersError)
+	}
+	return 0
+}
+
+func offsiteFetchWorlds(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC, binding it if needed)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", "the world archive PVC, in the [k8s] namespace")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	dir := *archiveDir
+	if dir == "" {
+		if dir, err = resolveVolumeDir(ctx, cfg.K8s.Namespace, *backupPVC, archiveVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+			return 1
+		}
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: open database: %v\n", err)
+		return 1
+	}
+	defer drv.Close()
+	res, err := offsite.FetchWorlds(ctx, env.bucket, offsite.PGCatalog{DB: drv.DB()}, env.key, dir, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-worlds: %d recorded archives, %d fetched into %s, %d with no copy in the bucket\n",
+		res.Present, len(res.Fetched), dir, len(res.Missing))
+	for _, m := range res.Missing {
+		fmt.Fprintf(stdout, "  no off-site copy: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_images.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/registrygate"
+	"felis.lolicon.best/internal/registryprune"
+)
+
+// registryImages is the platform registry as the off-site copy sees it: read
+// anonymously through the gate (the catalog, its manifest index, manifests and
+// blobs) and, for a restore, written as the platform principal. Both go through
+// the node's loopback hostPort, the way the installer pushes.
+type registryImages struct {
+	host   string
+	index  *registryprune.Client
+	source *imagepush.Source
+	pusher *imagepush.Pusher
+}
+
+func newRegistryImages(endpoint string) *registryImages {
+	return &registryImages{
+		host:   endpoint,
+		index:  &registryprune.Client{Endpoint: "http://" + endpoint},
+		source: &imagepush.Source{Scheme: "http"},
+	}
+}
+
+func (r *registryImages) Repositories(ctx context.Context) ([]string, error) {
+	return r.index.Repositories(ctx)
+}
+
+func (r *registryImages) Revisions(ctx context.Context, repo string) ([]string, map[string]string, error) {
+	idx, err := r.index.Index(ctx, repo)
+	if err != nil {
+		return nil, nil, err
+	}
+	digests := make([]string, 0, len(idx.Revisions))
+	for _, rev := range idx.Revisions {
+		digests = append(digests, rev.Digest)
+	}
+	return digests, idx.Tags, nil
+}
+
+func (r *registryImages) Manifest(ctx context.Context, repo, digest string) ([]byte, string, error) {
+	body, mt, err := r.source.Manifest(ctx, r.host, repo, digest)
+	return body, mt, registryGone(err)
+}
+
+func (r *registryImages) Blob(ctx context.Context, repo, digest string) (io.ReadCloser, error) {
+	rc, err := r.source.Blob(ctx, r.host, repo, digest)
+	return rc, registryGone(err)
+}
+
+func (r *registryImages) PutBlob(ctx context.Context, repo, digest string, size int64, open func() (io.ReadCloser, error)) error {
+	return r.pusher.UploadBlob(ctx, r.host, repo, digest, size, open)
+}
+
+func (r *registryImages) PutManifest(ctx context.Context, repo, reference, mediaType string, body []byte) error {
+	_, err := r.pusher.PutManifest(ctx, r.host, repo, reference, mediaType, body)
+	return err
+}
+
+// imagePins lists, per repository of the registry refs spell as host, the
+// digests the MinecraftServers' specs and the image whitelist pin: what a
+// restored database and its servers will ask the registry for.
+func imagePins(db *sql.DB, host string) func(ctx context.Context) (map[string][]string, error) {
+	return func(ctx context.Context) (map[string][]string, error) {
+		cl, err := buildSystemServerClient()
+		if err != nil {
+			return nil, fmt.Errorf("reach the cluster: %w", err)
+		}
+		var servers v1alpha1.MinecraftServerList
+		if err := cl.List(ctx, &servers); err != nil {
+			return nil, fmt.Errorf("list MinecraftServers: %w", err)
+		}
+		refs := make([]string, 0, len(servers.Items))
+		for _, s := range servers.Items {
+			refs = append(refs, s.Spec.Image)
+		}
+		rows, err := db.QueryContext(ctx, `SELECT image_ref FROM image_whitelist`)
+		if err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var ref string
+			if err := rows.Scan(&ref); err != nil {
+				return nil, fmt.Errorf("read the image whitelist: %w", err)
+			}
+			refs = append(refs, ref)
+		}
+		if err := rows.Err(); err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		pins := map[string][]string{}
+		for _, ref := range refs {
+			repo, _, digest, ok := registryprune.ParseRef(ref, host)
+			if ok && digest != "" && !slices.Contains(pins[repo], digest) {
+				pins[repo] = append(pins[repo], digest)
+			}
+		}
+		return pins, nil
+	}
+}
+
+// registryGone marks a 404 as a manifest or blob the registry no longer holds.
+func registryGone(err error) error {
+	var se *imagepush.StatusError
+	if errors.As(err, &se) && se.Code == http.StatusNotFound {
+		return fmt.Errorf("%w: %v", offsite.ErrImageGone, err)
+	}
+	return err
+}
+
+// offsiteRegistryEndpoint is where the host reaches the registry whose images
+// the off-site copy covers: flag when given ("off" for none), otherwise the
+// loopback hostPort of the in-cluster registry [registry] url names. A
+// registry outside the cluster is not this install's to copy.
+func offsiteRegistryEndpoint(flag string, reg config.RegistryConfig) string {
+	switch flag {
+	case "off":
+		return ""
+	case "":
+	default:
+		return flag
+	}
+	host, _, _ := strings.Cut(reg.URL, "/")
+	name, _, _ := strings.Cut(host, ":")
+	if strings.HasSuffix(name, ".svc") || strings.HasSuffix(name, ".svc.cluster.local") {
+		return loopbackEndpoint(host)
+	}
+	return ""
+}
+
+// registryWriteCredential is the credential a host-side push uses:
+// FELIS_REGISTRY_USERNAME/PASSWORD when set, otherwise the platform principal
+// with REGISTRY_PLATFORM_TOKEN from the installer's secrets file.
+func registryWriteCredential(secrets string) (string, string, error) {
+	if err := loadEnvFile(secrets); err != nil {
+		return "", "", fmt.Errorf("read %s: %w", secrets, err)
+	}
+	user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if pass == "" {
+		user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
+	}
+	if pass == "" {
+		return "", "", errors.New("no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
+	}
+	return user, pass, nil
+}
+
+func offsiteFetchImages(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	registry := fs.String("registry", "", "host[:port] of the registry to push into (default: the loopback hostPort of the in-cluster registry)")
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	at := fs.String("at", "", "registry index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	endpoint := offsiteRegistryEndpoint(*registry, cfg.Registry)
+	if endpoint == "" {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: [registry] url %q is not the in-cluster registry; pass -registry host:port\n", cfg.Registry.URL)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 2
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseImageIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-images: restoring registry index %s (%d repositories, %d images) into %s\n",
+		stamp, len(idx.Repositories), idx.Images(), endpoint)
+	target := newRegistryImages(endpoint)
+	target.pusher = &imagepush.Pusher{Scheme: "http", Username: user, Password: pass}
+	res, err := offsite.FetchImages(ctx, env.bucket, env.key, idx, target, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-images: %d of %d repositories restored, %d images, %d tags, %d blobs pushed (%s)\n",
+		res.Repositories, len(idx.Repositories), res.Manifests, res.Tags, res.BlobsPushed, offsite.HumanBytes(res.BytesPushed))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v (a second run pushes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_test.go

@@ -0,0 +1,775 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+)
+
+func TestLoadOffsiteEnvFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "offsite.env")
+	body := `# written by bootstrap
+FELIS_OFFSITE_ACCESS_KEY=AKIA123
+export FELIS_OFFSITE_SECRET_KEY="se=cret"
+FELIS_OFFSITE_KEY='k'
+
+not a line
+`
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "from-the-shell")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "")
+	t.Setenv("FELIS_OFFSITE_KEY", "")
+	if err := loadEnvFile(path); err != nil {
+		t.Fatal(err)
+	}
+	for k, want := range map[string]string{
+		"FELIS_OFFSITE_ACCESS_KEY": "from-the-shell", // the environment wins
+		"FELIS_OFFSITE_SECRET_KEY": "se=cret",
+		"FELIS_OFFSITE_KEY":        "k",
+	} {
+		if got := os.Getenv(k); got != want {
+			t.Errorf("%s = %q, want %q", k, got, want)
+		}
+	}
+	if err := loadEnvFile(filepath.Join(t.TempDir(), "absent")); err != nil {
+		t.Errorf("a missing env file is not an error: %v", err)
+	}
+}
+
+func TestResolveOffsiteNamesTheMissingVariable(t *testing.T) {
+	c := config.OffsiteConfig{
+		Endpoint: "https://s3.example", Bucket: "b",
+		AccessKeyRef: "T_AK", SecretKeyRef: "T_SK", KeyRef: "T_KEY",
+	}
+	t.Setenv("T_AK", "ak")
+	t.Setenv("T_SK", "sk")
+	t.Setenv("T_KEY", "")
+	if _, err := resolveOffsite(c); err == nil || !strings.Contains(err.Error(), "T_KEY") {
+		t.Fatalf("err = %v, want it to name T_KEY", err)
+	}
+	t.Setenv("T_KEY", "not base64 at all")
+	if _, err := resolveOffsite(c); err == nil {
+		t.Fatal("a malformed key was accepted")
+	}
+	key, _ := offsite.NewKey()
+	t.Setenv("T_KEY", key)
+	env, err := resolveOffsite(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(env.key) != offsite.KeySize {
+		t.Fatalf("key is %d bytes", len(env.key))
+	}
+	if _, err := resolveOffsite(config.OffsiteConfig{}); err == nil {
+		t.Fatal("an unconfigured [offsite] resolved")
+	}
+}
+
+func TestOffsiteKeygen(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := cmdOffsite([]string{"keygen"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if _, err := offsite.ParseKey(strings.TrimSpace(out.String())); err != nil {
+		t.Fatalf("keygen printed %q: %v", out.String(), err)
+	}
+}
+
+func TestOffsiteFetchDBRejectsOddNames(t *testing.T) {
+	key, _ := offsite.NewKey()
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "ak")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "sk")
+	t.Setenv("FELIS_OFFSITE_KEY", key)
+	var out, errb bytes.Buffer
+	code := cmdOffsite([]string{"fetch-db", "-env-file", "", "-endpoint", "http://127.0.0.1:1", "-bucket", "b",
+		"-dir", t.TempDir(), "../../etc/shadow"}, &out, &errb)
+	if code != 2 || !strings.Contains(errb.String(), "not a bundle name") {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+}
+
+func TestOffsiteRegistryEndpoint(t *testing.T) {
+	for _, tc := range []struct{ flag, url, want string }{
+		{"", "registry.felis.svc:5000", "127.0.0.1:5000"},
+		{"", "registry.felis.svc.cluster.local:5001", "127.0.0.1:5001"},
+		{"", "ghcr.io/acme", ""},
+		{"", "", ""},
+		{"off", "registry.felis.svc:5000", ""},
+		{"10.0.0.5:5000", "ghcr.io/acme", "10.0.0.5:5000"},
+	} {
+		if got := offsiteRegistryEndpoint(tc.flag, config.RegistryConfig{URL: tc.url}); got != tc.want {
+			t.Errorf("offsiteRegistryEndpoint(%q, %q) = %q, want %q", tc.flag, tc.url, got, tc.want)
+		}
+	}
+}
+
+func TestRegistryGoneMarksNotFound(t *testing.T) {
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 404}); !errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("404 = %v, want ErrImageGone", err)
+	}
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 503}); errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("503 = %v, want it kept an ordinary failure", err)
+	}
+}
+
+// mapBucket is an in-memory offsite.Bucket.
+type mapBucket map[string][]byte
+
+func (b mapBucket) Put(_ context.Context, key string, r io.Reader, _ int64) error {
+	data, err := io.ReadAll(r)
+	b[key] = data
+	return err
+}
+
+func (b mapBucket) Get(_ context.Context, key string) (io.ReadCloser, error) {
+	data, ok := b[key]
+	if !ok {
+		return nil, offsite.ErrNotFound
+	}
+	return io.NopCloser(bytes.NewReader(data)), nil
+}
+
+func (b mapBucket) List(_ context.Context, prefix string) ([]offsite.Object, error) {
+	var out []offsite.Object
+	for k, v := range b {
+		if strings.HasPrefix(k, prefix) {
+			out = append(out, offsite.Object{Key: k, Size: int64(len(v))})
+		}
+	}
+	return out, nil
+}
+
+func (b mapBucket) Remove(_ context.Context, key string) error {
+	delete(b, key)
+	return nil
+}
+
+var fetchT0 = time.Date(2026, 9, 20, 3, 30, 0, 0, time.UTC)
+
+// putBundle seals a bundle that verifies, taken daysAgo days before fetchT0,
+// into b and returns its name.
+func putBundle(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts) string {
+	t.Helper()
+	return putBundleWith(t, b, key, daysAgo, counts, "")
+}
+
+// putBundleWith is putBundle for a bundle whose server export failed with
+// serversError, when that is not empty.
+func putBundleWith(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts, serversError string) string {
+	t.Helper()
+	created := fetchT0.AddDate(0, 0, -daysAgo)
+	dump := []byte("PGDMP " + created.String())
+	sum := sha256.Sum256(dump)
+	manifest, err := json.Marshal(dbbackup.Manifest{
+		Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, ServersError: serversError,
+		Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var plain bytes.Buffer
+	tw := tar.NewWriter(&plain)
+	for _, f := range []struct {
+		name string
+		data []byte
+	}{{"MANIFEST.json", manifest}, {"db.dump", dump}} {
+		if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o600, Size: int64(len(f.data)), Typeflag: tar.TypeReg}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tw.Write(f.data); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	var sealed bytes.Buffer
+	if err := offsite.Encrypt(&sealed, &plain, key); err != nil {
+		t.Fatal(err)
+	}
+	name := dbbackup.BundleName(created, dbbackup.LabelDaily)
+	b[offsite.DBKey(name)] = sealed.Bytes()
+	return name
+}
+
+func TestOffsiteFetchDB(t *testing.T) {
+	rawKey, _ := offsite.NewKey()
+	key, _ := offsite.ParseKey(rawKey)
+	now := fetchT0.Add(2 * time.Hour)
+	fetch := func(b mapBucket, arg string) (dir string, code int, stdout, stderr string) {
+		dir = t.TempDir()
+		var out, errb bytes.Buffer
+		code = fetchDB(context.Background(), b, key, arg, dir, now, &out, &errb)
+		return dir, code, out.String(), errb.String()
+	}
+	fetched := func(t *testing.T, dir string) []string {
+		t.Helper()
+		var names []string
+		entries, _ := os.ReadDir(dir)
+		for _, e := range entries {
+			names = append(names, e.Name())
+		}
+		return names
+	}
+
+	t.Run("latest skips a rebuilt host's empty bundle", func(t *testing.T) {
+		b := mapBucket{}
+		full := putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
+		empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
+		dir, code, out, errb := fetch(b, "latest")
+		if code != 1 || !strings.Contains(errb, empty) || !strings.Contains(errb, full+" (5 accounts, 3 servers)") {
+			t.Fatalf("exit %d, stdout %q, stderr %q; want a refusal naming %s", code, out, errb, full)
+		}
+		if got := fetched(t, dir); len(got) != 0 {
+			t.Errorf("a refused fetch wrote %v", got)
+		}
+	})
+
+	t.Run("latest takes the newest bundle and says what it holds", func(t *testing.T) {
+		b := mapBucket{}
+		putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 2})
+		newest := putBundle(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3})
+		dir, code, out, errb := fetch(b, "latest")
+		if code != 0 {
+			t.Fatalf("exit %d: %s", code, errb)
+		}
+		if got := fetched(t, dir); !slices.Equal(got, []string{newest}) {
+			t.Errorf("wrote %v, want %s", got, newest)
+		}
+		for _, want := range []string{
+			"wrote " + filepath.Join(dir, newest) + " (verified)",
+			"taken   2026-09-19T03:30:00Z (daily, 26h0m ago)",
+			"felis   v1.2.3, schema 21",
+			"holds   5 accounts, 3 servers",
+		} {
+			if !strings.Contains(out, want) {
+				t.Errorf("stdout lacks %q:\n%s", want, out)
+			}
+		}
+		if strings.Contains(out, "new install") {
+			t.Errorf("a bundle with servers flagged as a new install's:\n%s", out)
+		}
+	})
+
+	t.Run("an empty bundle named outright is fetched with a warning", func(t *testing.T) {
+		b := mapBucket{}
+		putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
+		empty := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 1})
+		dir, code, out, errb := fetch(b, empty)
+		if code != 0 || !slices.Equal(fetched(t, dir), []string{empty}) {
+			t.Fatalf("exit %d, wrote %v: %s", code, fetched(t, dir), errb)
+		}
+		if !strings.Contains(out, "holds   1 account, 0 servers") || !strings.Contains(out, "like a new install's") {
+			t.Errorf("stdout = %s", out)
+		}
+	})
+
+	t.Run("a bundle without the servers says where they come from", func(t *testing.T) {
+		b := mapBucket{}
+		gapped := putBundleWith(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused (tried 3 times)")
+		_, code, out, errb := fetch(b, gapped)
+		if code != 0 || !strings.Contains(out, "This bundle lacks the MinecraftServer objects (connection refused (tried 3 times))") ||
+			!strings.Contains(out, "k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap") {
+			t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
+		}
+		whole := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+		if _, _, out, _ := fetch(b, whole); strings.Contains(out, "lacks the MinecraftServer objects") {
+			t.Errorf("a whole bundle flagged:\n%s", out)
+		}
+	})
+
+	t.Run("a bundle from before counts says so", func(t *testing.T) {
+		b := mapBucket{}
+		old := putBundle(t, b, key, 0, nil)
+		_, code, out, errb := fetch(b, "latest")
+		if code != 0 || !strings.Contains(out, old) || !strings.Contains(out, "holds   not recorded") || strings.Contains(out, "new install") {
+			t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
+		}
+	})
+}
+
+func TestOffsiteCheckKey(t *testing.T) {
+	newKey := func() []byte {
+		raw, _ := offsite.NewKey()
+		k, _ := offsite.ParseKey(raw)
+		return k
+	}
+	key, other := newKey(), newKey()
+	marked := func(k []byte) mapBucket { return mapBucket{"felis-key-id": []byte(offsite.KeyID(k) + "\n")} }
+	unmarked := func(k []byte) mapBucket {
+		b := mapBucket{}
+		putBundle(t, b, k, 0, nil)
+		return b
+	}
+	for _, tc := range []struct {
+		what   string
+		bucket mapBucket
+		code   int
+		says   []string
+	}{
+		{"the recorded key", marked(key), 0, []string{"records key id " + offsite.KeyID(key)}},
+		{"an unmarked bucket the key opens", unmarked(key), 0, []string{"open with this key", "the next sync records it"}},
+		{"an empty bucket", mapBucket{}, 0, []string{"no sealed object yet", "records key id " + offsite.KeyID(key)}},
+		{"another recorded key", marked(other), 3, []string{offsite.KeyID(other), offsite.KeyID(key), "FELIS_OFFSITE_KEY"}},
+		{"an unmarked bucket under another key", unmarked(other), 3, []string{"opens none of db/felis-db-"}},
+		{"a marker Felis did not write", mapBucket{"felis-key-id": []byte("hello")}, 1, []string{"not a key id"}},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			before := len(tc.bucket)
+			var out, errb bytes.Buffer
+			code := checkKey(context.Background(), tc.bucket, key, &out, &errb)
+			if code != tc.code {
+				t.Fatalf("exit %d, want %d; stdout %q, stderr %q", code, tc.code, out.String(), errb.String())
+			}
+			said := out.String() + errb.String()
+			for _, s := range tc.says {
+				if !strings.Contains(said, s) {
+					t.Errorf("output lacks %q: %s", s, said)
+				}
+			}
+			if len(tc.bucket) != before {
+				t.Errorf("check-key wrote to the bucket: %d objects, had %d", len(tc.bucket), before)
+			}
+		})
+	}
+
+	// fetch-db names both ids when the bucket records another key.
+	b := marked(other)
+	name := putBundle(t, b, other, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+	for _, arg := range []string{"latest", name} {
+		var out, errb bytes.Buffer
+		if code := fetchDB(context.Background(), b, key, arg, t.TempDir(), fetchT0, &out, &errb); code != 1 ||
+			!strings.Contains(errb.String(), "the bucket records key id "+offsite.KeyID(other)+", and this key is "+offsite.KeyID(key)) {
+			t.Errorf("fetch-db %s under another key: exit %d, stderr %q", arg, code, errb.String())
+		}
+	}
+	// A bundle the bucket lacks is not the key's fault.
+	var missOut, missErr bytes.Buffer
+	if code := fetchDB(context.Background(), b, key, "felis-db-20200101T000000Z-daily.tar", t.TempDir(), fetchT0, &missOut, &missErr); code != 1 || strings.Contains(missErr.String(), "records key id") {
+		t.Errorf("missing bundle: exit %d, stderr %q", code, missErr.String())
+	}
+	// A bundle damaged under the recorded key gets no such hint.
+	b = marked(key)
+	name = putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
+	b[offsite.DBKey(name)][60] ^= 1
+	var out, errb bytes.Buffer
+	if code := fetchDB(context.Background(), b, key, name, t.TempDir(), fetchT0, &out, &errb); code != 1 || strings.Contains(errb.String(), "records key id") {
+		t.Errorf("damaged bundle: exit %d, stderr %q", code, errb.String())
+	}
+}
+
+func TestRecordRun(t *testing.T) {
+	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}
+	for _, tc := range []struct {
+		err                                   error
+		mismatch, standby, displaced, success bool
+	}{
+		{nil, false, false, false, true},
+		{errors.New("list worlds/ in the bucket: connection reset"), false, false, false, false},
+		{fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false, false, false},
+		{&offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, false, true, false, false},
+		{&offsite.WriterError{Kind: offsite.ErrDisplaced, Writer: w}, false, false, true, false},
+	} {
+		st := offsite.Status{LastAttempt: t0}
+		recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err, offsite.Lease{})
+		if st.KeyMismatch != tc.mismatch || st.Standby != tc.standby || st.Displaced != tc.displaced ||
+			(st.Writer != nil) != (tc.standby || tc.displaced) || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 {
+			t.Errorf("err %v: status %+v", tc.err, st)
+		}
+	}
+
+	// A host that copied before writers were recorded keeps that claim over
+	// failed runs until it has an id.
+	l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Inherited: true}
+	st := offsite.Status{}
+	recordRun(&st, offsite.Result{}, errors.New("cannot reach bucket"), l)
+	if !st.Inherited {
+		t.Error("a failed run on a host with no id dropped the older release's claim")
+	}
+	writeTestFile(t, l.IDFile, "aaaaaaaaaaaaaaaa\n", 0o600)
+	st = offsite.Status{Inherited: true}
+	recordRun(&st, offsite.Result{}, nil, l)
+	if st.Inherited {
+		t.Error("a host with an id still carries the older release's claim")
+	}
+}
+
+// A refused run has copied nothing and listed nothing: its zero counts would
+// tell the journal the bucket is empty. A pass that ran and failed a step
+// shows what it did get to.
+func TestReportRun(t *testing.T) {
+	w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)}
+	for _, tc := range []struct {
+		name   string
+		res    offsite.Result
+		err    error
+		code   int
+		counts bool
+	}{
+		{"a pass", offsite.Result{DBUploaded: 1, RemoteDB: 3}, nil, 0, true},
+		{"a pass with a failed step", offsite.Result{RemoteDB: 3, Errors: []string{"copy db/x: timeout"}}, errors.New("1 of this run's steps failed; first: copy db/x: timeout"), 1, true},
+		{"standing by", offsite.Result{}, &offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, 1, false},
+		{"another key", offsite.Result{}, fmt.Errorf("%w: the bucket records key id 1111111111111111", offsite.ErrKeyMismatch), 1, false},
+		{"no bucket", offsite.Result{}, errors.New("bucket: access denied"), 1, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var out, errOut bytes.Buffer
+			code := reportRun(tc.res, tc.err, &out, &errOut)
+			if code != tc.code {
+				t.Errorf("exit %d, want %d", code, tc.code)
+			}
+			if got := strings.Contains(out.String(), "bucket holds 0 worlds (0 B), 3 bundles"); got != tc.counts {
+				t.Errorf("counts shown = %v, want %v: %q", got, tc.counts, out.String())
+			}
+			if !tc.counts && out.Len() > 0 {
+				t.Errorf("a refused run printed %q", out.String())
+			}
+			if tc.err != nil && !strings.Contains(errOut.String(), "felis offsite sync: "+tc.err.Error()) {
+				t.Errorf("stderr %q lacks the error", errOut.String())
+			}
+		})
+	}
+}
+
+func TestOffsiteTakeOver(t *testing.T) {
+	newKey := func() []byte {
+		raw, _ := offsite.NewKey()
+		k, _ := offsite.ParseKey(raw)
+		return k
+	}
+	key, other := newKey(), newKey()
+	const mine, theirs = "aaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbb"
+	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	sealed := func(k []byte, writer string) mapBucket {
+		b := mapBucket{}
+		putBundle(t, b, k, 0, nil)
+		b["felis-key-id"] = []byte(offsite.KeyID(k) + "\n")
+		if writer != "" {
+			raw, _ := json.Marshal(offsite.Writer{HostID: writer, Host: "prod-1", At: t0.Add(-20 * time.Minute)})
+			b["felis-writer"] = raw
+		}
+		return b
+	}
+	lease := func(id string) offsite.Lease {
+		l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Host: "spare-1"}
+		if id != "" {
+			writeTestFile(t, l.IDFile, id+"\n", 0o600)
+		}
+		return l
+	}
+	run := func(b mapBucket, l offsite.Lease, statusFile string, yes bool) (int, string, string) {
+		t.Helper()
+		var out, errb bytes.Buffer
+		code := takeOver(context.Background(), b, key, l, statusFile, yes, t0, &out, &errb)
+		return code, out.String(), errb.String()
+	}
+	for _, tc := range []struct {
+		what   string
+		bucket mapBucket
+		id     string
+		code   int
+		says   []string
+	}{
+		{"this host writes the bucket", sealed(key, mine), mine, 0, []string{"this host (id " + mine + ") writes the bucket; nothing to take over"}},
+		{"an empty bucket", mapBucket{}, "", 0, []string{"names no host writing it; this host's next sync records itself"}},
+		{"a host built from the writer's backup", sealed(key, theirs), "", 4, []string{"host prod-1 (id " + theirs + ") writes the bucket, last at", "(20m ago)", "built from its backup", "sudo felis offsite take-over -yes"}},
+		{"another host's copies, no writer named", sealed(key, ""), "", 4, []string{"holds copies this host did not write, and names no host writing it", "take-over -yes"}},
+		{"a host another one took over from", sealed(key, theirs), mine, 5, []string{"took the bucket over from this host"}},
+		{"a key the bucket refuses", sealed(other, theirs), "", 3, []string{"sealed with another key"}},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			before := string(tc.bucket["felis-writer"])
+			l := lease(tc.id)
+			code, out, errb := run(tc.bucket, l, filepath.Join(t.TempDir(), "status.json"), false)
+			if code != tc.code {
+				t.Fatalf("exit %d, want %d\n%s%s", code, tc.code, out, errb)
+			}
+			for _, s := range tc.says {
+				if !strings.Contains(out+errb, s) {
+					t.Errorf("output lacks %q:\n%s%s", s, out, errb)
+				}
+			}
+			if string(tc.bucket["felis-writer"]) != before {
+				t.Error("take-over without -yes wrote the bucket's writer")
+			}
+			if tc.id == "" {
+				if _, err := os.Stat(l.IDFile); !errors.Is(err, os.ErrNotExist) {
+					t.Errorf("take-over without -yes made this host an id: %v", err)
+				}
+			}
+		})
+	}
+
+	// -yes on a standby host: the bucket names it, and the refusal the last
+	// sync recorded is cleared, so the watchdog mails again at once.
+	b := sealed(key, theirs)
+	l := lease("")
+	statusFile := filepath.Join(t.TempDir(), "status.json")
+	lastSuccess := t0.Add(-48 * time.Hour)
+	if err := offsite.WriteStatus(statusFile, offsite.Status{
+		LastAttempt: t0.Add(-time.Hour), LastSuccess: lastSuccess, LastError: "offsite: another host writes this bucket", Format: offsite.StatusFormat,
+		Standby: true, Writer: &offsite.Writer{HostID: theirs, Host: "prod-1", At: t0}, Inherited: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	code, out, errb := run(b, l, statusFile, true)
+	id, _ := l.ID()
+	if code != 0 || id == "" || !strings.Contains(out, "this host (id "+id+") writes the bucket now; host prod-1 (id "+theirs+") stops at its next copy") || !strings.Contains(out, "systemctl start felis-offsite.service") {
+		t.Fatalf("take-over -yes: exit %d, id %q\n%s%s", code, id, out, errb)
+	}
+	if w, err := offsite.BucketWriter(context.Background(), b); err != nil || w.HostID != id || w.Host != "spare-1" || !w.At.Equal(t0) {
+		t.Errorf("writer after take-over -yes = %+v, %v", w, err)
+	}
+	st, err := offsite.ReadStatus(statusFile)
+	if err != nil || st.Standby || st.Writer != nil || st.LastError != "" || st.Inherited || !st.LastSuccess.Equal(lastSuccess) {
+		t.Errorf("status after take-over -yes = %+v, %v; want the refusal cleared and the last success kept", st, err)
+	}
+
+	// -yes with a key the bucket refuses writes nothing.
+	b = sealed(other, theirs)
+	before := string(b["felis-writer"])
+	if code, _, _ := run(b, lease(""), filepath.Join(t.TempDir(), "status.json"), true); code != 3 || string(b["felis-writer"]) != before {
+		t.Errorf("take-over -yes under another key: exit %d, writer %s", code, b["felis-writer"])
+	}
+}
+
+func TestOffsiteStatusSaysWhoWritesTheBucket(t *testing.T) {
+	dir := t.TempDir()
+	cfg := filepath.Join(dir, "felis.toml")
+	writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
+	statusFile := filepath.Join(dir, "status.json")
+	now := time.Now()
+	w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-30 * time.Minute)}
+	stale := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-offsite.WriterLive - time.Hour)}
+	for _, tc := range []struct {
+		what string
+		st   offsite.Status
+		says []string
+		not  string
+	}{
+		{"standing by for a live writer", offsite.Status{Standby: true, Writer: w}, []string{"This host stands by: host prod-1 (id bbbbbbbbbbbbbbbb) writes the bucket", "mails no watchdog alert", "take-over -yes"}, "wrote it, last at"},
+		{"standing by for a writer gone quiet", offsite.Status{Standby: true, Writer: stale}, []string{"copies nothing into the bucket: host prod-1 (id bbbbbbbbbbbbbbbb) wrote it, last at", "take-over -yes"}, "mails no watchdog alert"},
+		{"standing by, no writer named", offsite.Status{Standby: true}, []string{"names no host writing it", "take-over -yes"}, "stands by:"},
+		{"displaced", offsite.Status{Displaced: true, Writer: w}, []string{"host prod-1 (id bbbbbbbbbbbbbbbb) took the bucket over", "rehearsal machine", "take-over -yes"}, "stands by"},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			tc.st.LastAttempt, tc.st.LastSuccess, tc.st.LastError = now.Add(-time.Minute), now.Add(-time.Hour), "offsite: another host writes this bucket"
+			if err := offsite.WriteStatus(statusFile, tc.st); err != nil {
+				t.Fatal(err)
+			}
+			var out, errb bytes.Buffer
+			code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
+			if code != 1 || strings.Contains(out.String(), "bucket holds:") || strings.Contains(out.String(), tc.not) {
+				t.Errorf("exit %d\n%s%s", code, out.String(), errb.String())
+			}
+			for _, s := range tc.says {
+				if !strings.Contains(out.String(), s) {
+					t.Errorf("output lacks %q:\n%s", s, out.String())
+				}
+			}
+		})
+	}
+}
+
+func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) {
+	dir := t.TempDir()
+	cfg := filepath.Join(dir, "felis.toml")
+	writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
+	statusFile := filepath.Join(dir, "status.json")
+	st := offsite.Status{LastAttempt: time.Now().Add(-time.Minute), LastSuccess: time.Now().Add(-time.Hour), KeyID: "0123456789abcdef"}
+	status := func() (int, string) {
+		t.Helper()
+		if err := offsite.WriteStatus(statusFile, st); err != nil {
+			t.Fatal(err)
+		}
+		var out, errb bytes.Buffer
+		code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
+		return code, out.String() + errb.String()
+	}
+	if code, out := status(); code != 0 || strings.Contains(out, "refused") {
+		t.Fatalf("a recent success: exit %d\n%s", code, out)
+	}
+	st.LastError, st.KeyMismatch = "offsite: the bucket's objects are sealed with another key", true
+	code, out := status()
+	if code != 1 || !strings.Contains(out, "The last run was refused") || !strings.Contains(out, "key id 0123456789abcdef") || strings.Contains(out, "bucket holds:") {
+		t.Fatalf("a refused run: exit %d\n%s", code, out)
+	}
+}
+
+func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
+	rawKey, _ := offsite.NewKey()
+	key, _ := offsite.ParseKey(rawKey)
+	otherRaw, _ := offsite.NewKey()
+	other, _ := offsite.ParseKey(otherRaw)
+	b := mapBucket{}
+	gapped := putBundleWith(t, b, key, 4, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused")
+	old := putBundle(t, b, key, 3, nil)
+	full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3})
+	sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3})
+	empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
+	bundles, err := offsite.ListDB(context.Background(), b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var out bytes.Buffer
+	printDBBundles(context.Background(), b, key, bundles, &out)
+	lines := strings.Split(strings.TrimSpace(out.String()), "\n")
+	want := []struct{ name, holds string }{
+		{empty, "0 accounts, 0 servers"},
+		{sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"},
+		{full, "5 accounts, 3 servers"},
+		{old, "not recorded"},
+		{gapped, "5 accounts, 3 servers, no MinecraftServer objects"},
+	}
+	if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (5, newest first") {
+		t.Fatalf("output:\n%s", out.String())
+	}
+	for i, w := range want {
+		if l := lines[i+1]; !strings.HasPrefix(l, "  "+w.name+"  ") || !strings.Contains(l, w.holds) || strings.Contains(l, "no MinecraftServer") != (w.name == gapped) {
+			t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds)
+		}
+	}
+}
+
+// TestRestoredHostKeepsStandingBy walks the status file across runs: a host
+// restored from the writer's backup stands by on its first run and on every
+// run after it, a host an older release left copying claims the bucket once,
+// and a failed first run after the upgrade keeps that claim.
+func TestRestoredHostKeepsStandingBy(t *testing.T) {
+	rawKey, _ := offsite.NewKey()
+	key, _ := offsite.ParseKey(rawKey)
+	cfg := config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-backups"}
+	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	standby := &offsite.WriterError{Kind: offsite.ErrStandby, Writer: &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}}
+	pass := func(statusFile string, at time.Time, err error) offsite.Lease {
+		t.Helper()
+		st, lease := startRun(cfg, key, statusFile, at)
+		recordRun(&st, offsite.Result{}, err, lease)
+		if werr := offsite.WriteStatus(statusFile, st); werr != nil {
+			t.Fatal(werr)
+		}
+		return lease
+	}
+
+	restored := filepath.Join(t.TempDir(), "status.json")
+	for i := range 3 {
+		if l := pass(restored, t0.Add(time.Duration(i)*time.Hour), standby); l.Inherited {
+			t.Fatalf("run %d of a restored host claims the bucket", i+1)
+		}
+	}
+	if st, _ := offsite.ReadStatus(restored); !st.Standby || st.Format != offsite.StatusFormat || st.KeyID != offsite.KeyID(key) || st.Bucket != "felis-backups" {
+		t.Errorf("restored host's status = %+v", st)
+	}
+
+	upgraded := filepath.Join(t.TempDir(), "status.json")
+	if err := offsite.WriteStatus(upgraded, offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-time.Hour)}); err != nil {
+		t.Fatal(err)
+	}
+	if l := pass(upgraded, t0, errors.New("cannot reach bucket")); !l.Inherited {
+		t.Fatal("the first run after the upgrade does not claim the bucket")
+	}
+	l := pass(upgraded, t0.Add(time.Hour), nil)
+	if !l.Inherited {
+		t.Fatal("a failed first run after the upgrade lost the claim")
+	}
+	if st, _ := offsite.ReadStatus(upgraded); !st.LastSuccess.Equal(t0.Add(time.Hour)) {
+		t.Errorf("upgraded host's status = %+v", st)
+	}
+}
+
+// TestOffsiteSyncerSnapshotsAndSweeps: the pass `offsite sync` runs takes its
+// snapshot the way `felis db backup` does, in the database pod, into the
+// bundle directory, keeping the newest one there; and it sweeps unrecorded
+// world objects only past the longest retention [archive] gives any backup.
+func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) {
+	dir := newPodRig(t)
+	bundles := filepath.Join(dir, "bundles")
+	env := &offsiteEnv{cfg: config.OffsiteConfig{DBKeep: 5}}
+	var log bytes.Buffer
+	cfg := &config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "120d"}}
+	s := offsiteSyncer(cfg, env, offsiteSources{dbDir: bundles}, "/archives", "/uploads", nil, &log)
+	if s.DBDir != bundles || s.DBKeep != 5 || s.ArchiveDir != "/archives" || s.UploadsDir != "/uploads" {
+		t.Fatalf("syncer = %+v", s)
+	}
+	if s.OrphanAfter != 120*24*time.Hour {
+		t.Errorf("OrphanAfter = %s, want the 120d retention", s.OrphanAfter)
+	}
+	if s.Snapshot == nil {
+		t.Fatal("the pass takes no snapshot after copying archives")
+	}
+	for i := 0; i < 2; i++ {
+		if err := s.Snapshot(context.Background()); err != nil {
+			t.Fatalf("snapshot %d: %v", i, err)
+		}
+	}
+	got, err := dbbackup.List(bundles)
+	if err != nil || len(got) != 1 || got[0].Label != dbbackup.LabelOffsite {
+		t.Fatalf("bundle directory = %+v, %v; want the newest offsite bundle alone", got, err)
+	}
+	if _, err := dbbackupVerify(got[0].Path); err != nil {
+		t.Fatalf("the snapshot does not verify: %v", err)
+	}
+	// The MinecraftServer objects are exported alongside, as in the daily bundle.
+	argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args"))
+	if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") {
+		t.Errorf("k3s ran %q, want pg_dump in the pod", ran)
+	}
+	if !strings.Contains(string(bundleServers(t, got[0].Path)), `"name": "lobby"`) {
+		t.Errorf("the snapshot holds no MinecraftServer objects")
+	}
+	if !strings.Contains(log.String(), "took database bundle "+got[0].Name) {
+		t.Errorf("the snapshot is not logged:\n%s", log.String())
+	}
+	// It becomes the newest bundle in the bucket, so with the cluster away it
+	// fails, leaves no bundle, and the next pass tries again.
+	noServerExportWait(t)
+	writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
+	if err := s.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "export the MinecraftServer objects") {
+		t.Errorf("snapshot with the cluster away: %v, want a failure", err)
+	}
+	if again, _ := dbbackup.List(bundles); len(again) != 1 || again[0].Name != got[0].Name {
+		t.Errorf("bundle directory after the failed snapshot = %+v, want %s alone", again, got[0].Name)
+	}
+
+	for _, c := range []struct {
+		archive config.ArchiveConfig
+		want    time.Duration
+	}{
+		{config.ArchiveConfig{}, 90 * 24 * time.Hour},
+		{config.ArchiveConfig{ScheduledRetention: "200d"}, 200 * 24 * time.Hour},
+		{config.ArchiveConfig{ManualRetention: "150d", Retention: "30d", ScheduledRetention: "60d"}, 150 * 24 * time.Hour},
+	} {
+		s := offsiteSyncer(&config.Config{Database: podDB, Archive: c.archive}, env, offsiteSources{dbDir: bundles}, "", "", nil, io.Discard)
+		if s.OrphanAfter != c.want {
+			t.Errorf("%+v: OrphanAfter = %s, want %s", c.archive, s.OrphanAfter, c.want)
+		}
+	}
+	log.Reset()
+	s = offsiteSyncer(&config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "soon"}}, env, offsiteSources{}, "", "", nil, &log)
+	if s.OrphanAfter != 0 || !strings.Contains(log.String(), "world objects no backup records are kept") {
+		t.Errorf("a retention that does not parse: OrphanAfter %s, log %q; want no sweep, said", s.OrphanAfter, log.String())
+	}
+	if s.Snapshot != nil {
+		t.Error("a pass that copies no bundles takes a snapshot")
+	}
+}

cmd/felis/offsite_uploads.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+)
+
+func offsiteFetchUploads(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC, binding it if needed)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, "the submission uploads PVC, in the control-plane namespace")
+	at := fs.String("at", "", "uploads index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseUploadIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	dir := *uploadsDir
+	if dir == "" {
+		if !isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: [registry] user_uploads_context %q is not the uploads volume; pass -uploads-dir to restore into a directory anyway\n", cfg.Registry.UserUploadsContext)
+			return 2
+		}
+		if dir, err = resolveVolumeDir(ctx, platform.DefaultControlNamespace, *uploadsPVC, uploadsVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+			return 1
+		}
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: restoring uploads index %s (%d submission contexts, %s) into %s\n",
+		stamp, len(idx.Contexts), offsite.HumanBytes(idx.Bytes()), dir)
+	res, err := offsite.FetchUploads(ctx, env.bucket, env.key, idx, dir, platform.ControlPlaneUID, platform.ControlPlaneUID, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: %d written (%s), %d already in place, %d failed\n",
+		res.Written, offsite.HumanBytes(res.Bytes), res.Present, len(res.Failures))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v (a second run writes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/operator.go

@@ -1,19 +1,26 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"log/slog"
+	"net/http"
 	"os"
+	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	felismetrics "felis.lolicon.best/internal/metrics"
 	"felis.lolicon.best/internal/operator"
+	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 )
@@ -25,6 +32,11 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	fs := flag.NewFlagSet("operator", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	metricsAddr := fs.String("metrics-bind-address", ":8080", "address the metric endpoint binds to")
+	// healthAddr serves the manager's health endpoints (/healthz, /readyz) that the
+	// Deployment's probes dial. Without it the operator pod would carry no probe at
+	// all, and a wedged manager would keep its endpoint forever. It must differ from
+	// metricsAddr: the metrics server owns :8080.
+	healthAddr := fs.String("health-probe-bind-address", ":8081", "address the health probe endpoint binds to")
 	// namespace MUST equal the [k8s] namespace felis-api is configured with, and
 	// the deployment manifests (felis manifests) render both from one value. It
 	// scopes the manager's cache (informers) to a single namespace so the operator
@@ -42,9 +54,16 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(v1alpha1.AddToScheme(scheme))
 
+	// controller-runtime logs through its own logr sink; without one, its first
+	// reconcile prints "log.SetLogger(...) was never called" ATTACHED TO A FULL
+	// GOROUTINE STACK — pure noise, not signal. Route it to slog's default handler
+	// so its messages appear as ordinary stderr lines.
+	ctrl.SetLogger(logr.FromSlogHandler(slog.Default().Handler()))
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:  scheme,
-		Metrics: metricsserver.Options{BindAddress: *metricsAddr},
+		Scheme:                 scheme,
+		Metrics:                metricsserver.Options{BindAddress: *metricsAddr},
+		HealthProbeBindAddress: *healthAddr,
 		// Scope every informer to the single watched namespace. Without this the
 		// cached client (mgr.GetClient) would LIST/WATCH cluster-wide, which a
 		// namespaced Role cannot grant — the operator would fail closed at runtime
@@ -60,6 +79,23 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	fmt.Fprintf(stderr, "felis operator: watching namespace %q\n", *namespace)
 
+	// /healthz fails while a reconcile pass has been stuck past its limit, so the
+	// liveness probe restarts an operator whose workers are wedged (a Pod whose
+	// process answers but no server starts or stops). /readyz waits for the
+	// informer caches: until they sync the operator acts on nothing, and one that
+	// never syncs (lost RBAC, an unreachable API) never reports Available.
+	// A dependency hiccup fails neither: the caches ride through API blips, and
+	// each pass is bounded well inside the stuck limit.
+	watch := &operator.ReconcileWatch{}
+	if err := mgr.AddHealthzCheck("reconcile", watch.Check); err != nil {
+		fmt.Fprintf(stderr, "felis operator: register healthz check: %v\n", err)
+		return 1
+	}
+	if err := mgr.AddReadyzCheck("informers", cacheSynced(mgr.GetCache())); err != nil {
+		fmt.Fprintf(stderr, "felis operator: register readyz check: %v\n", err)
+		return 1
+	}
+
 	// Publish the named felis_* metrics (spec §23) on the endpoint the manager
 	// already serves (metricsAddr). controller-runtime's metrics server exposes
 	// its global Registry, so registering into it is all that is needed for
@@ -69,6 +105,7 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis operator: register metrics: %v\n", err)
 		return 1
 	}
+	felismetrics.SetBuildInfo("operator", resolvedVersion())
 
 	r := &operator.Reconciler{
 		Client: mgr.GetClient(),
@@ -77,7 +114,24 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		// The operator's own image, for the forwarding-config initContainer it
 		// injects into user servers. The Deployment passes it as FELIS_IMAGE (see
 		// platform.OperatorDeployment); absent, that injection is simply skipped.
-		FelisImage: os.Getenv("FELIS_IMAGE"),
+		FelisImage:     os.Getenv("FELIS_IMAGE"),
+		Nodes:          nil,
+		EgressProbe:    os.Getenv("FELIS_EGRESS_PROBE"),
+		ControllerNode: os.Getenv("FELIS_CONTROLLER_NODE"),
+		// Uncached: the maintenance-lock check lists Jobs only when a server is
+		// about to start, which does not justify a namespace-wide Job informer.
+		Jobs: mgr.GetAPIReader(),
+		// Uncached too: RCON Secrets are read by name, so the Role grants
+		// secrets:get without the list/watch an informer would need.
+		Secrets: mgr.GetAPIReader(),
+		// And pods: pod-0 is read by name, so the Role grants pods:get and no
+		// namespace-wide pod informer runs.
+		Pods:     mgr.GetAPIReader(),
+		Recorder: mgr.GetEventRecorderFor("felis-operator"),
+		Watch:    watch,
+	}
+	if os.Getenv("FELIS_DISTRIBUTED") == "true" {
+		r.Nodes = mgr.GetAPIReader()
 	}
 	if err := r.SetupWithManager(mgr); err != nil {
 		fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
@@ -99,3 +153,18 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// cacheSynced is a readyz check that passes once every informer the manager
+// started has synced. It waits at most a second, well inside the probe timeout.
+func cacheSynced(c interface {
+	WaitForCacheSync(ctx context.Context) bool
+}) healthz.Checker {
+	return func(req *http.Request) error {
+		ctx, cancel := context.WithTimeout(req.Context(), time.Second)
+		defer cancel()
+		if !c.WaitForCacheSync(ctx) {
+			return errors.New("informer caches not synced")
+		}
+		return nil
+	}
+}

cmd/felis/operator_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"context"
+	"net/http/httptest"
+	"testing"
+)
+
+type fakeCache bool
+
+func (f fakeCache) WaitForCacheSync(ctx context.Context) bool {
+	if !f {
+		<-ctx.Done()
+	}
+	return bool(f)
+}
+
+// TestCacheSynced: the operator reports ready only once its informers synced,
+// and a check against caches that never sync returns within its own deadline.
+func TestCacheSynced(t *testing.T) {
+	req := httptest.NewRequest("GET", "/readyz", nil)
+	if err := cacheSynced(fakeCache(true))(req); err != nil {
+		t.Errorf("synced: %v", err)
+	}
+	if err := cacheSynced(fakeCache(false))(req); err == nil {
+		t.Error("unsynced caches reported ready")
+	}
+}

cmd/felis/pinimages.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// defaultRegistryURL is the [registry] url every install uses; deploy/bootstrap.sh
+// spells the same value as REGISTRY_URL.
+const defaultRegistryURL = "registry.felis.svc:5000"
+
+// cmdPinImages pins every user server whose spec.image still names a mutable tag
+// in the platform registry to the digest that tag names now (internal/imagepin).
+// felis-api pins on create, so this covers the servers created before it did.
+//
+// deploy/bootstrap.sh runs it before it rebuilds the game images and pushes them
+// over the same tags: run after the push, it would pin those servers to the new
+// build, which is exactly the silent Minecraft upgrade pinning exists to stop.
+// It reaches the registry through the node's loopback hostPort, the same way the
+// installer pushes.
+func cmdPinImages(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("pin-images", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	namespace := fs.String("namespace", platform.DefaultMinecraftNamespace, "namespace the MinecraftServers live in")
+	registry := fs.String("registry", defaultRegistryURL, "registry host[:port] the image refs spell")
+	endpoint := fs.String("endpoint", "", "host[:port] to reach the registry at (default: 127.0.0.1 on the registry's port, its hostPort on this node)")
+	system := fs.String("system", "", "pin this system server ("+naming.SystemLoginServer+" or "+naming.SystemLobbyServer+") to the build its tag names now, instead of the user servers")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if *system != "" && *system != naming.SystemLoginServer && *system != naming.SystemLobbyServer {
+		fmt.Fprintf(stderr, "felis pin-images: --system takes %s or %s, not %q\n", naming.SystemLoginServer, naming.SystemLobbyServer, *system)
+		return 2
+	}
+	if *endpoint == "" {
+		*endpoint = loopbackEndpoint(*registry)
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	if *system != "" {
+		return reportSystemPin(ctx, cl, *namespace, *system, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint}, stdout, stderr)
+	}
+	outcomes, err := pinUserServerImages(ctx, cl, *namespace, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint})
+	if meta.IsNoMatchError(err) {
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	if len(outcomes) == 0 {
+		fmt.Fprintln(stdout, "felis pin-images: every user server already runs a pinned image")
+		return 0
+	}
+	fmt.Fprintln(stdout, "felis pin-images: pinning user servers to the build their tag names now:")
+	exit := 0
+	for _, o := range outcomes {
+		if o.err != nil {
+			fmt.Fprintf(stdout, "  - %s: ERROR %v\n", o.name, o.err)
+			exit = 1
+			continue
+		}
+		fmt.Fprintf(stdout, "  - %s: %s\n", o.name, strings.Join(o.changes, ", "))
+	}
+	return exit
+}
+
+// reportSystemPin runs pinSystemServerImage for `felis pin-images --system` and
+// prints what it did. Only a failed pin exits non-zero: the installer falls back
+// to restarting the pod on its tag then.
+func reportSystemPin(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver, stdout, stderr io.Writer) int {
+	o, err := pinSystemServerImage(ctx, cl, namespace, name, r)
+	switch {
+	case meta.IsNoMatchError(err):
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	case err == nil && o.err != nil:
+		err = o.err
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %s: %v\n", name, err)
+		return 1
+	}
+	switch {
+	case o.updated:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s; the operator rolls it onto that build\n", name, strings.Join(o.changes, ", "))
+	default:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s\n", name, o.skipped)
+	}
+	return 0
+}
+
+// pinSystemServerImage fixes a system server to the build its image tag names now,
+// replacing the digest of an earlier build. The installer runs it after pushing a
+// rebuilt login or lobby image, and the operator rolls the StatefulSet onto the new
+// ref, so the build a system server runs is written in its spec and moves only when
+// a build did. An image outside the platform registry, or one naming no tag to
+// follow, is the admin's choice and is left alone; so is a server whose image an
+// admin retargets while this runs.
+func pinSystemServerImage(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver) (systemServerOutcome, error) {
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &ms); err != nil {
+		if apierrors.IsNotFound(err) {
+			return systemServerOutcome{name: name, skipped: "not present yet; sudo felis setup creates it"}, nil
+		}
+		return systemServerOutcome{}, err
+	}
+	if ms.Labels[v1alpha1.LabelSystemRole] != name {
+		return systemServerOutcome{name: name, err: fmt.Errorf(
+			"MinecraftServer %s/%s is not marked as the Felis %q system server; left alone", namespace, name, name)}, nil
+	}
+	tagged := withoutDigest(ms.Spec.Image)
+	if !r.Covers(tagged) || !strings.Contains(tagged[strings.LastIndex(tagged, "/")+1:], ":") {
+		return systemServerOutcome{name: name, available: true, skipped: "runs " + ms.Spec.Image +
+			", which names no platform registry tag to follow; left alone"}, nil
+	}
+	pinned, err := r.Pin(ctx, tagged)
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("resolve %s: %w", tagged, err)}, nil
+	}
+	changed, err := patchOnConflictRetry(ctx, cl, &ms, func() bool {
+		if withoutDigest(ms.Spec.Image) != tagged || ms.Spec.Image == pinned {
+			return false
+		}
+		ms.Spec.Image = pinned
+		return true
+	})
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("patch %s: %w", name, err)}, nil
+	}
+	if !changed {
+		return systemServerOutcome{name: name, available: true, skipped: "already runs " + ms.Spec.Image}, nil
+	}
+	return systemServerOutcome{name: name, available: true, updated: true,
+		changes: []string{"spec.image pinned to " + pinned}}, nil
+}
+
+// withoutDigest drops the @sha256:… of a pinned ref, leaving the tag it came from.
+func withoutDigest(ref string) string {
+	if i := strings.Index(ref, "@"); i >= 0 {
+		return ref[:i]
+	}
+	return ref
+}
+
+// loopbackEndpoint is the registry's port on 127.0.0.1: the registry Deployment
+// binds it as a hostPort, and containerd's mirror and the installer's pushes use
+// the same address.
+func loopbackEndpoint(registry string) string {
+	if i := strings.LastIndex(registry, ":"); i >= 0 {
+		return "127.0.0.1" + registry[i:]
+	}
+	return "127.0.0.1"
+}
+
+// pinUserServerImages patches spec.image of every user server whose image the
+// resolver covers and is not yet pinned. System servers are the installer's to move:
+// it re-pins them with --system when it rolls them onto a new build
+// (restart_existing_system_servers).
+// A server that is already pinned, or runs an image from elsewhere, produces no
+// outcome, so a pinned fleet reports nothing. A running server restarts once as
+// the operator rolls its StatefulSet onto the pinned ref, which is the build it
+// already runs.
+func pinUserServerImages(ctx context.Context, cl client.Client, namespace string, r imagepin.Resolver) ([]systemServerOutcome, error) {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return nil, fmt.Errorf("list servers: %w", err)
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || imagepin.Pinned(ms.Spec.Image) || !r.Covers(ms.Spec.Image) {
+			continue
+		}
+		pinned, err := r.Pin(ctx, ms.Spec.Image)
+		if errors.Is(err, imagepin.ErrNotFound) {
+			err = fmt.Errorf("%s is not in the registry, so there is no build to pin it to; left unpinned: %w", ms.Spec.Image, err)
+		}
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: err})
+			continue
+		}
+		patch := client.MergeFrom(ms.DeepCopy())
+		ms.Spec.Image = pinned
+		if err := cl.Patch(ctx, ms, patch); err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("patch %s: %w", ms.Name, err)})
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{"spec.image pinned to " + pinned}})
+	}
+	return out, nil
+}

cmd/felis/pinimages_test.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+const pinTestDigest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
+
+// TestPinUserServerImages pins exactly the user servers still on a platform tag,
+// reports a tag the registry lost as an error without touching that server, and
+// has nothing left to do on a second pass.
+func TestPinUserServerImages(t *testing.T) {
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v2/felis/paper/manifests/demo" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Docker-Content-Digest", pinTestDigest)
+	}))
+	defer reg.Close()
+	res := imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+
+	paper := defaultRegistryURL + "/felis/paper:demo"
+	mk := func(name, image, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Image = image
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		mk("legacy", paper, ""),
+		mk("pinned", paper+"@sha256:"+strings.Repeat("3", 64), ""),
+		mk("external", "docker.io/itzg/minecraft-server:java21", ""),
+		mk("gone", defaultRegistryURL+"/felis/paper:old", ""),
+		mk(naming.SystemLobbyServer, defaultRegistryURL+"/felis/felis-lobby:demo", naming.SystemLobbyServer),
+	).Build()
+	ctx := context.Background()
+
+	outcomes, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil {
+		t.Fatalf("pinUserServerImages: %v", err)
+	}
+	byName := map[string]systemServerOutcome{}
+	for _, o := range outcomes {
+		byName[o.name] = o
+	}
+	if len(outcomes) != 2 || byName["legacy"].err != nil || byName["gone"].err == nil {
+		t.Fatalf("outcomes = %+v, want legacy pinned and gone reported", outcomes)
+	}
+
+	want := map[string]string{
+		"legacy":                 paper + "@" + pinTestDigest,
+		"pinned":                 paper + "@sha256:" + strings.Repeat("3", 64),
+		"external":               "docker.io/itzg/minecraft-server:java21",
+		"gone":                   defaultRegistryURL + "/felis/paper:old",
+		naming.SystemLobbyServer: defaultRegistryURL + "/felis/felis-lobby:demo",
+	}
+	for name, image := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Image != image {
+			t.Errorf("%s image = %q, want %q", name, ms.Spec.Image, image)
+		}
+	}
+
+	again, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil || len(again) != 1 || again[0].name != "gone" {
+		t.Fatalf("second pass = %+v, %v; want only the unresolvable server again", again, err)
+	}
+}
+
+// A fresh install has no CRD yet; the command must read that as nothing to pin.
+func TestPinUserServerImagesNoCRD(t *testing.T) {
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
+		List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error {
+			return &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "felis.lolicon.best", Kind: "MinecraftServer"}}
+		},
+	}).Build()
+	_, err := pinUserServerImages(context.Background(), cl, "minecraft", imagepin.Resolver{Registry: defaultRegistryURL})
+	if !meta.IsNoMatchError(err) {
+		t.Fatalf("err = %v, want a NoMatch error the command can recognise", err)
+	}
+}
+
+func TestLoopbackEndpoint(t *testing.T) {
+	for in, want := range map[string]string{
+		"registry.felis.svc:5000": "127.0.0.1:5000",
+		"registry.example":        "127.0.0.1",
+	} {
+		if got := loopbackEndpoint(in); got != want {
+			t.Errorf("loopbackEndpoint(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

cmd/felis/reaper.go

@@ -1,9 +1,13 @@
 package main
 
 import (
+	"context"
+	"database/sql"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -12,8 +16,10 @@ import (
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/backup"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/reaper"
-	"felis.lolicon.best/internal/store"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -26,11 +32,17 @@ import (
 // cadence, and RunOnce is idempotent and restart-safe, so a missed or retried
 // run simply converges. Only the tarLocal archive backend is wired in this
 // build; the snapshot backends (§19) are a later integration.
+//
+// --retention-only runs the archive-store half alone (reaper.RunRetention):
+// the CronJob an install without a worlds root gets, so backups past their
+// expiry still leave the store there. It builds no Kubernetes client and reads
+// no world.
 func cmdReaper(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("reaper", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
-	worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>)")
+	worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>, else the stock local-path <root>/<pv-name>_<ns>_<pvc-name>)")
+	retentionOnly := fs.Bool("retention-only", false, "only expire, read back and sweep the archive store: no server is evaluated and no world is read or deleted, so neither the worlds root nor the Kubernetes API is needed")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -47,50 +59,163 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	archiver, err := buildArchiver(cfg, *worldsRoot)
+	ctx := ctrl.SetupSignalHandler()
+
+	var cl client.Client
+	if !*retentionOnly {
+		scheme := runtime.NewScheme()
+		utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+		utilruntime.Must(v1alpha1.AddToScheme(scheme))
+		cl, err = client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
+		if err != nil {
+			fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
+			return 1
+		}
+	}
+
+	archiver, err := buildArchiver(ctx, cfg, *worldsRoot, cl)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: %v\n", err)
 		return 1
 	}
 
-	ctx := ctrl.SetupSignalHandler()
-
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	if os.Getenv("FELIS_DISTRIBUTED") == "true" && !*retentionOnly {
+		m, err := distributionManager(cl, cfg, os.Getenv("FELIS_IMAGE"))
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		local, ok := archiver.(*backup.TarLocal)
+		if !ok {
+			fmt.Fprintln(stderr, "remote reaper requires tarLocal")
+			return 1
+		}
+		archiver = &distributed.RemoteArchiver{TarLocal: local, Manager: m}
+	}
+	drv, err := openPodStore(ctx, cfg.Database.URL, "reaper", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
 		return 1
 	}
 	defer drv.Close()
 
-	scheme := runtime.NewScheme()
-	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-	utilruntime.Must(v1alpha1.AddToScheme(scheme))
-	cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
-	if err != nil {
-		fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
-		return 1
-	}
-
 	r := &reaper.Reaper{
 		Cfg:      rcfg,
 		Store:    reaper.NewPGStore(drv.DB()),
-		Cluster:  reaper.NewK8sCluster(cl, cfg.K8s.Namespace),
 		Archiver: archiver,
 	}
+	if *retentionOnly {
+		fmt.Fprintln(stderr, "felis reaper: retention only — no worlds root is configured, so idle worlds are neither archived nor released")
+		return reportReaperRun(r.RunRetention(ctx), stdout, stderr)
+	}
+	r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace).WithDistributed(os.Getenv("FELIS_DISTRIBUTED") == "true")
+
+	// Pre-reap warnings go out by email when [smtp] is configured (the same
+	// relay and password_ref convention felis-api uses); without it the channel
+	// stays nil and the reaper logs each suppressed warning instead of stamping
+	// it, so a later SMTP setup still gets to warn. The owner must have a
+	// VERIFIED address — that flag is what proves the mailbox.
+	if cfg.SMTP.Host != "" {
+		passRef := cfg.SMTP.PasswordRef
+		if passRef == "" {
+			passRef = platform.SMTPPasswordEnv
+		}
+		password := os.Getenv(passRef)
+		if cfg.SMTP.Username != "" && password == "" {
+			fmt.Fprintf(stderr, "felis reaper: warning: [smtp] username is set but credentials env %s is empty — warning emails will fail AUTH\n", passRef)
+		}
+		db := drv.DB()
+		r.Warner = &mailWarner{
+			lookupEmail: func(ctx context.Context, ownerID string) (string, error) {
+				var email string
+				switch err := db.QueryRowContext(ctx,
+					`SELECT email FROM users
+					 WHERE id = $1 AND email_verified = true AND COALESCE(email, '') <> ''`,
+					ownerID).Scan(&email); {
+				case errors.Is(err, sql.ErrNoRows):
+					return "", fmt.Errorf("owner %s has no verified email", ownerID)
+				case err != nil:
+					return "", err
+				}
+				return email, nil
+			},
+			notifier: smtpRelay(cfg.SMTP, password),
+		}
+	} else {
+		fmt.Fprintln(stderr, "felis reaper: [smtp] not configured — pre-reap warnings are logged and NOT marked sent")
+	}
 
 	sum, err := r.RunOnce(ctx)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: %v\n", err)
 		return 1
 	}
-	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d warned=%d skipped=%d evicted=%d expired=%d\n",
-		sum.Evaluated, sum.WorldsReaped, sum.Warned, sum.Skipped, sum.EvictedEarly, sum.BackupsExpired)
-	return 0
+	return reportReaperRun(sum, stdout, stderr)
+}
+
+// reportReaperRun prints the run's tally and turns a run that left work undone
+// into exit 1, so the Job fails and the watchdog's job-failed check (and the
+// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
+// is kept, and without this nobody would learn that it is never reaped.
+func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
+	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d released=%d deleted=%d awaiting_offsite=%d awaiting_stop=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
+		sum.Evaluated, sum.WorldsReaped, sum.Released, sum.ServersDeleted, sum.AwaitingOffsite, sum.AwaitingStop, sum.Warned, sum.Skipped, sum.StoreFull,
+		sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
+		sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
+	if !sum.Failed() {
+		return 0
+	}
+	if sum.Skipped > 0 || sum.ExpireFailed > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
+			sum.Skipped, sum.StoreFull, sum.ExpireFailed)
+	}
+	if sum.Corrupt > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)
+	}
+	if sum.VerifyFailed > 0 || sum.SweepFailed {
+		fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",
+			sum.VerifyFailed, !sum.SweepFailed)
+	}
+	return 1
+}
+
+// mailWarner delivers a pre-reap notice to the owner's verified email — the
+// only channel this build can reach. Unowned owners and owners who never proved
+// a mailbox yield an error; the reaper retries such notices on its next run and
+// never lets them block the reap (red line ⑤).
+type mailWarner struct {
+	lookupEmail func(ctx context.Context, ownerID string) (string, error)
+	notifier    noticeNotifier
+}
+
+// noticeNotifier is the slice of mail.SMTP the warner needs (injected in tests).
+type noticeNotifier interface {
+	SendNotice(ctx context.Context, email, subject, body string) error
+}
+
+func (w *mailWarner) Warn(ctx context.Context, ownerID, server, remaining string) error {
+	email, err := w.lookupEmail(ctx, ownerID)
+	if err != nil {
+		return fmt.Errorf("resolve owner email: %w", err)
+	}
+	subject := fmt.Sprintf("Felis: 服务器 %s 将在 %s 后回收 · server reaped in %s", server, remaining, remaining)
+	body := fmt.Sprintf(
+		"Felis 世界回收提醒 / world-reaper notice\r\n"+
+			"\r\n"+
+			"服务器 / Server: %s\r\n"+
+			"距回收 / Time left: %s\r\n"+
+			"\r\n"+
+			"闲置的服务器会先自动备份，再释放世界；有人加入游戏即可重置倒计时。\r\n"+
+			"Idle servers are backed up and then released; any join resets the countdown.\r\n",
+		server, remaining)
+	return w.notifier.SendNotice(ctx, email, subject, body)
 }
 
 // reaperConfig derives the reaper's retention windows from felis.toml. The 15d
-// idle deadline is fixed by §18; only the warning offsets, retention, and the
-// store soft-cap are configurable (§24).
+// idle deadline is fixed by §18; only the warning offsets, retention, the
+// store soft-cap, the on-demand backup bounds and the scheduled restore points
+// are configurable (§24). The backup Job and felis-api read the manual_* and
+// scheduled_* keys through it too.
 func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 	rc := reaper.DefaultConfig()
 	if v := cfg.Archive.Retention; v != "" {
@@ -118,48 +243,112 @@ func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 		}
 		rc.MaxLocalBytes = b
 	}
+	if v := cfg.Archive.ManualRetention; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d <= 0 {
+			return rc, fmt.Errorf("[archive] manual_retention %q: want a positive span such as 30d", v)
+		}
+		rc.ManualRetention = d
+	}
+	switch n := cfg.Archive.ManualKeep; {
+	case n < 0:
+		return rc, fmt.Errorf("[archive] manual_keep %d: want 1 or more", n)
+	case n > 0:
+		rc.ManualKeep = n
+	}
+	if v := cfg.Archive.ManualCooldown; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d < 0 {
+			return rc, fmt.Errorf("[archive] manual_cooldown %q: want a span such as 10m (0s for none)", v)
+		}
+		rc.ManualCooldown = d
+	}
+	if v := cfg.Archive.ScheduledEvery; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d < 0 {
+			return rc, fmt.Errorf("[archive] scheduled_every %q: want a span such as 1d (0s for none)", v)
+		}
+		rc.ScheduledEvery = d
+	}
+	switch n := cfg.Archive.ScheduledKeep; {
+	case n < 0:
+		return rc, fmt.Errorf("[archive] scheduled_keep %d: want 1 or more", n)
+	case n > 0:
+		rc.ScheduledKeep = n
+	}
+	if v := cfg.Archive.ScheduledRetention; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d <= 0 {
+			return rc, fmt.Errorf("[archive] scheduled_retention %q: want a positive span such as 90d", v)
+		}
+		rc.ScheduledRetention = d
+	}
+	rc.RequireOffsite = cfg.Offsite.Enabled()
 	return rc, nil
 }
 
 // buildArchiver constructs the WorldArchiver. Only tarLocal is implemented in
-// this build; the resolver maps each world PVC to <worldsRoot>/<pvc>, the mount
-// convention the reaper Job is deployed with.
-func buildArchiver(cfg *config.Config, worldsRoot string) (backup.WorldArchiver, error) {
+// this build; the resolver maps each world PVC to its directory under worldsRoot
+// (resolveWorldDir). A nil cl is the retention-only run, which never archives a
+// world, so its archiver resolves none.
+func buildArchiver(ctx context.Context, cfg *config.Config, worldsRoot string, cl client.Client) (backup.WorldArchiver, error) {
 	switch cfg.Archive.Store {
 	case "tarLocal":
-		return &backup.TarLocal{
-			BackupRoot: cfg.Archive.LocalPath,
-			Resolve: func(pvc string) (string, error) {
-				return filepath.Join(worldsRoot, pvc), nil
-			},
-		}, nil
+		t := &backup.TarLocal{BackupRoot: cfg.Archive.LocalPath}
+		if cl != nil {
+			t.Resolve = resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot)
+		} else {
+			t.Resolve = func(pvc string) (string, error) {
+				return "", fmt.Errorf("resolve world PVC %s: this run has no worlds root (retention only)", pvc)
+			}
+		}
+		return t, nil
 	default:
 		return nil, fmt.Errorf("[archive] store %q is not implemented in this build (only tarLocal)", cfg.Archive.Store)
 	}
 }
 
-// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
-// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
-func parseSpanDuration(s string) (time.Duration, error) {
-	s = strings.TrimSpace(s)
-	switch {
-	case strings.HasSuffix(s, "mo"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
-		if err != nil {
-			return 0, err
+// resolveWorldDir maps a world PVC to its directory under worldsRoot, supporting
+// the two layouts a Felis host actually has:
+//
+//  1. <root>/<pvc> — the reaper's documented arrangement (worlds exposed by PVC
+//     name, e.g. via mounting each volume or a crafted storage class).
+//  2. <root>/<pv-name>_<namespace>_<pvc-name> — what a stock k3s install gets:
+//     local-path-provisioner stores every volume under its storage root as that
+//     exact directory name. Without this arm, retention on a default install could
+//     only ever fail to find a world (a no-op reaper, or worse an operator
+//     arranging paths by hand).
+//
+// The second path is derived EXACTLY from the live PVC's spec.volumeName, never
+// from a glob: a leftover directory of an old, deleted PV must never be mistaken
+// for the world the PVC currently binds, because the reaper archives the resolved
+// directory and then deletes that PVC — archiving stale bytes and deleting the
+// real world would be data loss. When neither path exists the first is returned,
+// so the archive walk fails loudly against the documented path.
+func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoot string) backup.PVCResolver {
+	return func(pvc string) (string, error) {
+		direct := filepath.Join(worldsRoot, pvc)
+		if _, err := os.Stat(direct); err == nil {
+			return direct, nil
 		}
-		return time.Duration(n) * 30 * 24 * time.Hour, nil
-	case strings.HasSuffix(s, "d"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
-		if err != nil {
-			return 0, err
+		var claim corev1.PersistentVolumeClaim
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: pvc}, &claim); err != nil {
+			return "", fmt.Errorf("resolve world PVC %s: %w", pvc, err)
 		}
-		return time.Duration(n) * 24 * time.Hour, nil
-	default:
-		return time.ParseDuration(s)
+		if pv := claim.Spec.VolumeName; pv != "" {
+			volDir := filepath.Join(worldsRoot, fmt.Sprintf("%s_%s_%s", pv, claim.Namespace, claim.Name))
+			if _, err := os.Stat(volDir); err == nil {
+				return volDir, nil
+			}
+		}
+		return direct, nil
 	}
 }
 
+// parseSpanDuration parses the human spans used in felis.toml's [archive] table
+// (config.ParseSpan).
+func parseSpanDuration(s string) (time.Duration, error) { return config.ParseSpan(s) }
+
 // parseByteSize parses a Kubernetes-style quantity ("200Gi", "10G") into bytes.
 // An empty string means unlimited (0).
 func parseByteSize(s string) (int64, error) {

cmd/felis/reaper_test.go

@@ -0,0 +1,230 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/reaper"
+)
+
+// TestReportReaperRunFailsTheJob: a run that could not process a server, or
+// could not remove an expired backup, exits 1 so the Job shows as failed.
+func TestReportReaperRunFailsTheJob(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		sum  reaper.Summary
+		want int
+	}{
+		{"clean", reaper.Summary{Evaluated: 3, WorldsReaped: 1, AwaitingOffsite: 1}, 0},
+		{"retirements", reaper.Summary{Evaluated: 5, WorldsReaped: 3, Released: 2, ServersDeleted: 1}, 0},
+		{"waiting for a stop", reaper.Summary{Evaluated: 3, AwaitingStop: 1}, 0},
+		{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
+		{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
+		{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},
+		{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},
+		{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},
+		{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},
+		{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},
+	} {
+		var out, errb bytes.Buffer
+		if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {
+			t.Errorf("%s: exit %d, want %d", tc.name, got, tc.want)
+		}
+		if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") ||
+			!strings.Contains(out.String(), fmt.Sprintf(" released=%d deleted=%d ", tc.sum.Released, tc.sum.ServersDeleted)) {
+			t.Errorf("%s: summary line = %q", tc.name, out.String())
+		}
+		if (tc.want == 1) != (errb.Len() > 0) {
+			t.Errorf("%s: stderr = %q", tc.name, errb.String())
+		}
+	}
+}
+
+// TestResolveWorldDir pins the two world layouts the reaper must find, and the
+// fail-closed miss. The stock local-path arm is derived from the live PVC's
+// volumeName — a name-based guess (glob) could tar a stale deleted PV's bytes and
+// then delete the current world, which is why it is read from the API instead.
+// TestReaperConfigManualKeys: the on-demand backup keys default to 30 days,
+// five per server and a ten-minute cooldown, accept overrides, and refuse
+// values that would keep nothing or throttle backwards.
+func TestReaperConfigManualKeys(t *testing.T) {
+	rc, err := reaperConfig(&config.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 30*reaper.Day || rc.ManualKeep != 5 || rc.ManualCooldown != 10*time.Minute {
+		t.Fatalf("defaults = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
+		ManualRetention: "7d", ManualKeep: 2, ManualCooldown: "0s"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 7*reaper.Day || rc.ManualKeep != 2 || rc.ManualCooldown != 0 {
+		t.Fatalf("overrides = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	for _, bad := range []config.ArchiveConfig{
+		{ManualRetention: "0d"},
+		{ManualRetention: "soon"},
+		{ManualKeep: -1},
+		{ManualCooldown: "-5m"},
+		{ManualCooldown: "often"},
+	} {
+		if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
+			t.Errorf("%+v was accepted", bad)
+		}
+	}
+}
+
+// TestReaperConfigScheduledKeys: the scheduled restore points default to one a
+// day, seven per server and the reaper's 90 days, accept overrides ("0s" turns
+// them off), and refuse values that would keep nothing or run backwards.
+func TestReaperConfigScheduledKeys(t *testing.T) {
+	rc, err := reaperConfig(&config.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ScheduledEvery != reaper.Day || rc.ScheduledKeep != 7 || rc.ScheduledRetention != 90*reaper.Day {
+		t.Fatalf("defaults = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
+		ScheduledEvery: "12h", ScheduledKeep: 3, ScheduledRetention: "14d"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ScheduledEvery != 12*time.Hour || rc.ScheduledKeep != 3 || rc.ScheduledRetention != 14*reaper.Day {
+		t.Fatalf("overrides = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{ScheduledEvery: "0s"}})
+	if err != nil || rc.ScheduledEvery != 0 {
+		t.Fatalf("scheduled_every 0s = %v, %v; want off", rc.ScheduledEvery, err)
+	}
+	for _, bad := range []config.ArchiveConfig{
+		{ScheduledEvery: "-1h"},
+		{ScheduledEvery: "daily"},
+		{ScheduledKeep: -1},
+		{ScheduledRetention: "0d"},
+		{ScheduledRetention: "forever"},
+	} {
+		if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
+			t.Errorf("%+v was accepted", bad)
+		}
+	}
+}
+
+func TestResolveWorldDir(t *testing.T) {
+	ctx := context.Background()
+	root := t.TempDir()
+
+	// Arrange a world under the documented <root>/<pvc> layout.
+	named := filepath.Join(root, "world-named-0")
+	if err := os.MkdirAll(named, 0o750); err != nil {
+		t.Fatal(err)
+	}
+	// Arrange a second world the way k3s local-path stores it.
+	pvDir := filepath.Join(root, "pvc-11111111-2222-3333-4444-555555555555_minecraft_world-live-0")
+	if err := os.MkdirAll(pvDir, 0o750); err != nil {
+		t.Fatal(err)
+	}
+
+	claim := &corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{Name: "world-live-0", Namespace: "minecraft"},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			VolumeName: "pvc-11111111-2222-3333-4444-555555555555",
+		},
+	}
+	cl := fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(claim).Build()
+	resolve := resolveWorldDir(ctx, cl, "minecraft", root)
+
+	t.Run("documented name layout wins", func(t *testing.T) {
+		got, err := resolve("world-named-0")
+		if err != nil || got != named {
+			t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, named)
+		}
+	})
+
+	t.Run("stock local-path layout resolves exactly", func(t *testing.T) {
+		got, err := resolve("world-live-0")
+		if err != nil || got != pvDir {
+			t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, pvDir)
+		}
+	})
+
+	t.Run("neither layout present falls back to the documented path", func(t *testing.T) {
+		// The claim exists but its directory does not: return the documented path so
+		// the archive walk fails there, and the reaper preserves the world.
+		missing := &corev1.PersistentVolumeClaim{
+			ObjectMeta: metav1.ObjectMeta{Name: "world-gone-0", Namespace: "minecraft"},
+			Spec:       corev1.PersistentVolumeClaimSpec{VolumeName: "pvc-99999999-0000-0000-0000-000000000000"},
+		}
+		cl := fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(missing).Build()
+		got, err := resolveWorldDir(ctx, cl, "minecraft", root)("world-gone-0")
+		if err != nil || got != filepath.Join(root, "world-gone-0") {
+			t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, filepath.Join(root, "world-gone-0"))
+		}
+	})
+
+	t.Run("unknown pvc is an error, not a guess", func(t *testing.T) {
+		_, err := resolve("world-unknown-0")
+		if err == nil || !strings.Contains(err.Error(), "resolve world PVC world-unknown-0") {
+			t.Fatalf("err = %v, want a resolve-world-PVC error", err)
+		}
+	})
+}
+
+// The pre-reap warner resolves the owner's VERIFIED email and hands the notice
+// to the mailer. Every failure (no verified address, relay refusal) returns an
+// error so the reaper retries on its next run instead of stamping a notice
+// nobody received.
+func TestMailWarner(t *testing.T) {
+	lookup := func(email string, err error) func(context.Context, string) (string, error) {
+		return func(context.Context, string) (string, error) { return email, err }
+	}
+
+	n := &captureNotifier{}
+	w := &mailWarner{lookupEmail: lookup("[email protected]", nil), notifier: n}
+	if err := w.Warn(context.Background(), "u1", "survival", "3d"); err != nil {
+		t.Fatalf("Warn: %v", err)
+	}
+	if n.email != "[email protected]" || !strings.Contains(n.subject, "survival") || !strings.Contains(n.subject, "3d") {
+		t.Fatalf("notice envelope = (%q, %q)", n.email, n.subject)
+	}
+	if !strings.Contains(n.body, "survival") || !strings.Contains(n.body, "3d") {
+		t.Fatalf("body missing server/remaining:\n%s", n.body)
+	}
+
+	w = &mailWarner{lookupEmail: lookup("", errors.New("owner u2 has no verified email")), notifier: n}
+	if err := w.Warn(context.Background(), "u2", "survival", "3d"); err == nil || !strings.Contains(err.Error(), "verified email") {
+		t.Fatalf("unverified owner = %v, want the lookup error surfaced", err)
+	}
+
+	w = &mailWarner{lookupEmail: lookup("[email protected]", nil), notifier: &captureNotifier{err: errors.New("relay down")}}
+	if err := w.Warn(context.Background(), "u1", "survival", "3d"); err == nil || !strings.Contains(err.Error(), "relay down") {
+		t.Fatalf("relay failure = %v, want it surfaced", err)
+	}
+}
+
+type captureNotifier struct {
+	email, subject, body string
+	err                  error
+}
+
+func (n *captureNotifier) SendNotice(_ context.Context, email, subject, body string) error {
+	if n.err != nil {
+		return n.err
+	}
+	n.email, n.subject, n.body = email, subject, body
+	return nil
+}

cmd/felis/registrygate.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/registrygate"
+)
+
+// cmdRegistryGate is the sidecar entrypoint in the registry pod: it owns the
+// registry port (and the loopback hostPort containerd pulls through), lets reads
+// through anonymously, and forwards writes to the loopback-only registry:2 only
+// for an authenticated principal allowed to write that repository. See
+// internal/registrygate for the policy.
+//
+// Tokens are files under --auth-dir, one per principal (platform, build, prune),
+// mounted from the registry-auth Secret. A missing file disables that principal:
+// writes fail closed while every pull keeps working, which is the right way round
+// for a registry the running workloads depend on.
+//
+// --maint-listen is the GC sidecar's read-only handshake (registrygate.MaintHandler).
+// It has no authentication, so it must name a loopback address; --maint-dir keeps
+// an open window across a gate restart.
+func cmdRegistryGate(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("registry-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	listen := fs.String("listen", ":5000", "address the gate serves the registry API on")
+	upstream := fs.String("upstream", "http://127.0.0.1:5001", "the loopback registry the gate forwards to")
+	authDir := fs.String("auth-dir", "/etc/felis-registry-auth", "directory holding one token file per principal")
+	maintListen := fs.String("maint-listen", "", "loopback address for the GC sidecar's read-only handshake (empty disables it)")
+	maintDir := fs.String("maint-dir", "", "directory that keeps an open read-only window across a gate restart")
+	quiet := fs.Duration("maint-quiet", registrygate.DefaultQuiet, "how long writes must be idle before a read-only window is granted")
+	dataDir := fs.String("data-dir", "", "the registry's storage root, mounted read-only, for the manifest index (empty disables it)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *maintListen != "" && !loopbackAddr(*maintListen) {
+		fmt.Fprintf(stderr, "felis registry-gate: --maint-listen %q must be a loopback address: the handshake has no authentication\n", *maintListen)
+		return 2
+	}
+	target, err := url.Parse(*upstream)
+	if err != nil || target.Scheme == "" || target.Host == "" {
+		fmt.Fprintf(stderr, "felis registry-gate: bad --upstream %q\n", *upstream)
+		return 2
+	}
+	log := slog.New(slog.NewTextHandler(stderr, nil))
+	tokens := map[string]string{}
+	for _, p := range registrygate.Principals {
+		b, err := os.ReadFile(filepath.Join(*authDir, p))
+		tok := strings.TrimSpace(string(b))
+		if err != nil || tok == "" {
+			log.Warn("registry principal disabled: no token", "principal", p, "dir", *authDir)
+			continue
+		}
+		tokens[p] = tok
+	}
+
+	gate := registrygate.New(target, tokens, log)
+	gate.SetQuiet(*quiet)
+	gate.DataDir = *dataDir
+	if *maintDir != "" {
+		if err := gate.SetMaintenanceState(registrygate.MaintStatePath(*maintDir)); err != nil {
+			// A corrupt file must not keep the registry from serving pulls.
+			log.Warn("ignoring the saved read-only window", "err", err)
+		}
+	}
+	srv := &http.Server{
+		Addr:              *listen,
+		Handler:           gate,
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	var maint *http.Server
+	if *maintListen != "" {
+		maint = &http.Server{Addr: *maintListen, Handler: gate.MaintHandler(), ReadHeaderTimeout: 10 * time.Second}
+		go func() {
+			if err := maint.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				log.Error("maintenance listener stopped; garbage collection cannot get a read-only window", "err", err)
+			}
+		}()
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	go func() {
+		<-ctx.Done()
+		shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(shutdown)
+		if maint != nil {
+			_ = maint.Shutdown(shutdown)
+		}
+	}()
+	log.Info("registry gate listening", "addr", *listen, "upstream", target.String(), "principals", len(tokens))
+	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintf(stderr, "felis registry-gate: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// loopbackAddr reports whether a host:port listen address binds loopback only.
+func loopbackAddr(addr string) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return false
+	}
+	if host == "localhost" {
+		return true
+	}
+	ip := net.ParseIP(host)
+	return ip != nil && ip.IsLoopback()
+}
+
+// cmdPushImage is the build Job's publish step. It runs after Kaniko built the
+// image into a tarball (--no-push) and Trivy passed that tarball, and it is the
+// only container of the build pod that holds the registry credential — the one
+// executing the untrusted Dockerfile never sees it.
+func cmdPushImage(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("push-image", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path, or with --image an OCI layout tar")
+	image := fs.String("image", "", "push the image this name (io.containerd.image.name) marks in the OCI layout tar --tar, e.g. a release's image bundle")
+	ref := fs.String("ref", "", "host/repository:tag to publish it as")
+	scheme := fs.String("scheme", "http", "registry scheme: http for the in-cluster registry, https otherwise")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *tarPath == "" || *ref == "" {
+		fmt.Fprintln(stderr, "felis push-image: --tar and --ref are required")
+		return 2
+	}
+	if *scheme != "http" && *scheme != "https" {
+		fmt.Fprintf(stderr, "felis push-image: bad --scheme %q\n", *scheme)
+		return 2
+	}
+	user := os.Getenv("FELIS_REGISTRY_USERNAME")
+	pass := os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if user == "" || pass == "" {
+		fmt.Fprintln(stderr, "felis push-image: FELIS_REGISTRY_USERNAME/FELIS_REGISTRY_PASSWORD are empty — the registry refuses anonymous writes")
+		return 2
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: *scheme, Username: user, Password: pass, Log: stderr}
+	var digest string
+	var err error
+	if *image != "" {
+		digest, err = p.PushLayout(ctx, *tarPath, *image, *ref)
+	} else {
+		digest, err = p.Push(ctx, *tarPath, *ref)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis push-image: %v\n", err)
+		return 1
+	}
+	fmt.Fprintln(stdout, digest)
+	return 0
+}

cmd/felis/restore.go

@@ -1,14 +1,18 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io"
+	"os"
+	"os/signal"
 	"path/filepath"
 	"strings"
+	"syscall"
 
+	"felis.lolicon.best/internal/archivetransfer"
 	"felis.lolicon.best/internal/backup"
-	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 // cmdRestore is the in-Pod entrypoint the restore Job runs. internal/restore
@@ -26,6 +30,9 @@ import (
 func cmdRestore(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("restore", flag.ContinueOnError)
 	fs.SetOutput(stderr)
+	source := fs.String("source-url", "", "one-use archive download URL")
+	sum := fs.String("sha256", "", "required digest for remote archive")
+	limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum download size")
 	server := fs.String("server", "", "server name being restored (for logging)")
 	ref := fs.String("ref", "", "absolute path to the archive on the backup mount")
 	store := fs.String("archive-store", "tarLocal", "archive backend (only tarLocal is implemented)")
@@ -35,6 +42,22 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int {
 		return 2
 	}
 
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+	if *source != "" {
+		dir, err := os.MkdirTemp("/tmp", "felis-restore-")
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		defer os.RemoveAll(dir)
+		*backupRoot = dir
+		*ref = filepath.Join(dir, "world.tar.gz")
+		if err := archivetransfer.Fetch(ctx, *source, os.Getenv(archivetransfer.TokenEnv), *ref, *sum, *limit); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
 	if *ref == "" {
 		fmt.Fprintln(stderr, "felis restore: --ref is required")
 		return 2
@@ -62,11 +85,16 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int {
 		},
 	}
 
-	ctx := ctrl.SetupSignalHandler()
 	if err := archiver.Restore(ctx, backup.ArchiveRef(*ref), *server); err != nil {
 		fmt.Fprintf(stderr, "felis restore: %v\n", err)
 		return 1
 	}
+	if *source != "" {
+		if err := backup.VerifyRestored(ctx, *ref, *worldsRoot); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
 	fmt.Fprintf(stdout, "felis restore: server=%s restored from %s into %s\n", *server, *ref, *worldsRoot)
 	return 0
 }

cmd/felis/restore_test.go

@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup
 		BackupRoot: backupRoot,
 		Resolve:    func(string) (string, error) { return srcDir, nil },
 	}
-	got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0")
+	got, err := ar.Archive(context.Background(), "survival", "world-survival-0")
 	if err != nil {
 		t.Fatalf("Archive: %v", err)
 	}
-	return string(got), backupRoot
+	return string(got.Ref), backupRoot
 }
 
 // The restore subcommand must extract the archived world into the target world

cmd/felis/rotatetoken.go

@@ -0,0 +1,854 @@
+package main
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/pbkdf2"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	neturl "net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/maintenance"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/operator"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/registrygate"
+
+	"github.com/BurntSushi/toml"
+	"github.com/jackc/pgx/v5"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// rotate-token replaces one credential the installer generated: an internal
+// caller's token (naming.CallerTokens), the registry's write tokens, the
+// Velocity forwarding secret or the database password. The new value goes into
+// the installer's record first, so that whatever fails later a re-run of the
+// installer puts it everywhere; then into the Secrets and files that carry it;
+// then whatever read the old value at start restarts. Without -yes it prints
+// what it would change and what that interrupts, and changes nothing.
+
+const (
+	defaultSecretsEnvPath = "/etc/felis/secrets.env"
+	defaultLinkPropsPath  = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
+	defaultForwardingPath = "/opt/felis/velocity/forwarding.secret"
+	velocityUnit          = "felis-velocity"
+	apiDeployment         = "felis-api"
+	registryDeployment    = "registry"
+
+	kindRegistry   = "registry"
+	kindForwarding = "forwarding"
+	kindDB         = "db"
+
+	// proxyReloadWait is how long the host proxy gets, once felis-api has rolled,
+	// to show it re-read its token: it reads the file again on its next call to
+	// felis-api, and it calls every 15 seconds.
+	proxyReloadWait = 60 * time.Second
+
+	// apiRestartNote is in every plan: felis-api runs as one replica replaced in
+	// place, so its restart is a short outage of everything that talks to it.
+	apiRestartNote = "felis-api restarts (a single replica): the panel, sign-in and the proxy's calls are unavailable for the few seconds that takes"
+)
+
+// installerTokenKeys names each caller's token in the installer's secrets.env
+// (deploy/bootstrap.sh load_or_make_secrets). A re-run of the installer applies
+// these values to the Secrets, so a rotation that skipped the file would be
+// undone by the next upgrade.
+var installerTokenKeys = map[string]string{
+	"velocity": "SERVICE_TOKEN",
+	"limbo":    "LIMBO_TOKEN",
+	"build":    "BUILD_TOKEN",
+	"ops":      "OPS_TOKEN",
+}
+
+// installerRegistryKeys names the registry principals' tokens in secrets.env,
+// in registrygate.Principals order.
+var installerRegistryKeys = map[string]string{
+	registrygate.PrincipalPlatform: "REGISTRY_PLATFORM_TOKEN",
+	registrygate.PrincipalBuild:    "REGISTRY_BUILD_TOKEN",
+	registrygate.PrincipalPrune:    "REGISTRY_PRUNE_TOKEN",
+}
+
+// installerForwardingKey and installerDBKey name the forwarding secret and the
+// database password in secrets.env.
+const (
+	installerForwardingKey = "FORWARDING_SECRET"
+	installerDBKey         = "DB_PASSWORD"
+)
+
+type tokenRotator struct {
+	cl          client.Client
+	controlNS   string
+	minecraftNS string
+	buildNS     string
+	// secretsEnv, linkProps and forwardingFile are the installer's record and the
+	// host proxy's felis-link.properties and forwarding.secret; a missing file is
+	// reported and skipped.
+	secretsEnv     string
+	linkProps      string
+	forwardingFile string
+	// hostTOML, podTOML and defaultTOML are the config copies that carry the
+	// database URL (defaultTOML only when it is a file of its own).
+	hostTOML    string
+	podTOML     string
+	defaultTOML string
+	newToken    func() (string, error)
+	// rollout restarts a control-namespace Deployment and waits for it.
+	rollout func(ctx context.Context, deployment string) error
+	// restartUnit restarts a systemd unit on this host.
+	restartUnit func(ctx context.Context, unit string) error
+	// proxyLog is what the host proxy has logged since a moment, as far as it
+	// can be read.
+	proxyLog func(ctx context.Context, since time.Time) string
+	// alterRole stores a password verifier for a role of the database that runs
+	// as deployment ("namespace/name"); verifyDB connects with a URL.
+	alterRole func(ctx context.Context, deployment, role, verifier string) error
+	verifyDB  func(ctx context.Context, url string) error
+	now       func() time.Time
+	// reloadWait bounds the wait for the proxy to re-read its token, polled
+	// every pollEvery.
+	reloadWait time.Duration
+	pollEvery  time.Duration
+	out        io.Writer
+}
+
+func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("rotate-token", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
+	linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
+	forwarding := fs.String("forwarding-secret", defaultForwardingPath, "the host proxy's forwarding secret file (forwarding only)")
+	yes := fs.Bool("yes", false, "rotate; without it the plan is printed and nothing changes")
+	fs.Usage = func() {
+		fmt.Fprintf(stderr, "Usage: felis rotate-token [-yes] [flags] <%s>\n\n", strings.Join(rotationKinds(), "|"))
+		fmt.Fprintln(stderr, "Replaces one generated credential: the installer's record, the Secrets and files that carry it, then what reads it.")
+		fmt.Fprintln(stderr, "Without -yes it prints what would change and what that interrupts.")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if fs.NArg() != 1 {
+		fs.Usage()
+		return 2
+	}
+	kind := fs.Arg(0)
+	if !knownRotation(kind) {
+		fmt.Fprintf(stderr, "felis rotate-token: unknown credential %q (one of %s)\n", kind, strings.Join(rotationKinds(), ", "))
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+kind+")")
+		return 1
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	buildNS := cfg.Registry.BuildNamespace
+	if buildNS == "" {
+		buildNS = platform.DefaultBuildNamespace
+	}
+	controlNS := platform.DefaultControlNamespace
+	r := tokenRotator{
+		cl:             cl,
+		controlNS:      controlNS,
+		minecraftNS:    cfg.K8s.Namespace,
+		buildNS:        buildNS,
+		secretsEnv:     *secretsEnv,
+		linkProps:      *linkProps,
+		forwardingFile: *forwarding,
+		hostTOML:       hostSetupConfigPath,
+		podTOML:        podSetupConfigPath,
+		defaultTOML:    defaultSetupConfigPath,
+		newToken:       randomToken,
+		rollout: func(ctx context.Context, deployment string) error {
+			if err := kubectl(ctx, "-n", controlNS, "rollout", "restart", "deployment/"+deployment); err != nil {
+				return err
+			}
+			return kubectl(ctx, "-n", controlNS, "rollout", "status", "deployment/"+deployment, "--timeout=180s")
+		},
+		restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
+		proxyLog:    journalSince,
+		alterRole:   alterRoleInPod,
+		verifyDB: func(ctx context.Context, url string) error {
+			ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+			defer cancel()
+			conn, err := pgx.Connect(ctx, url)
+			if err != nil {
+				return err
+			}
+			return conn.Close(ctx)
+		},
+		now:        time.Now,
+		reloadWait: proxyReloadWait,
+		pollEvery:  3 * time.Second,
+		out:        stdout,
+	}
+	if err := r.rotate(context.Background(), kind, *yes); err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+func callerNames() []string {
+	names := make([]string, 0, len(naming.CallerTokens))
+	for _, ct := range naming.CallerTokens {
+		names = append(names, ct.Caller)
+	}
+	return names
+}
+
+func callerToken(name string) (naming.CallerToken, bool) {
+	for _, ct := range naming.CallerTokens {
+		if ct.Caller == name {
+			return ct, true
+		}
+	}
+	return naming.CallerToken{}, false
+}
+
+// rotationKinds is every credential rotate-token replaces: the callers' tokens
+// and the installer's other generated secrets.
+func rotationKinds() []string {
+	return append(callerNames(), kindRegistry, kindForwarding, kindDB)
+}
+
+func knownRotation(kind string) bool {
+	for _, k := range rotationKinds() {
+		if k == kind {
+			return true
+		}
+	}
+	return false
+}
+
+// randomToken is 32 random bytes in hex, the shape the installer generates.
+func randomToken() (string, error) {
+	b := make([]byte, 32)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(b), nil
+}
+
+// tokenFingerprint is how the proxy names the token it reloaded in its log
+// (plugins/shared FileToken.fingerprint): the first twelve hex digits of its
+// SHA-256, enough to tell tokens apart and useless for finding one.
+func tokenFingerprint(token string) string {
+	sum := sha256.Sum256([]byte(token))
+	return hex.EncodeToString(sum[:])[:12]
+}
+
+func (r tokenRotator) rotate(ctx context.Context, kind string, apply bool) error {
+	switch kind {
+	case kindRegistry:
+		return r.rotateRegistry(ctx, apply)
+	case kindForwarding:
+		return r.rotateForwarding(ctx, apply)
+	case kindDB:
+		return r.rotateDB(ctx, apply)
+	}
+	ct, ok := callerToken(kind)
+	if !ok {
+		return fmt.Errorf("unknown credential %q (one of %s)", kind, strings.Join(rotationKinds(), ", "))
+	}
+	return r.rotateCaller(ctx, ct, apply)
+}
+
+// confirm ends the plan: without apply it says nothing changed and how to go
+// ahead, and reports false.
+func (r tokenRotator) confirm(kind string, apply bool) bool {
+	if !apply {
+		fmt.Fprintf(r.out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes %s\n", kind)
+		return false
+	}
+	fmt.Fprintln(r.out, "\nRotating:")
+	return true
+}
+
+// record writes new values into the installer's secrets.env. It comes first in
+// every rotation: from then on, whatever fails, a re-run of the installer puts
+// the new values everywhere.
+func (r tokenRotator) record(kv ...[2]string) error {
+	keys := make([]string, len(kv))
+	for i, p := range kv {
+		keys[i] = p[0]
+	}
+	switch err := setKeyValueLines(r.secretsEnv, "=", kv); {
+	case errors.Is(err, fs.ErrNotExist):
+		fmt.Fprintf(r.out, "  - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
+	case err != nil:
+		return fmt.Errorf("record the new value in %s: %w", r.secretsEnv, err)
+	default:
+		fmt.Fprintf(r.out, "  - %s: %s updated\n", r.secretsEnv, strings.Join(keys, ", "))
+	}
+	return nil
+}
+
+func (r tokenRotator) putSecret(ctx context.Context, ns, name string, data map[string][]byte) error {
+	if _, err := putSecretKeys(ctx, r.cl, ns, name, corev1.SecretTypeOpaque, data); err != nil {
+		return fmt.Errorf("write Secret %s/%s: %w", ns, name, err)
+	}
+	fmt.Fprintf(r.out, "  - Secret %s/%s: updated\n", ns, name)
+	return nil
+}
+
+func (r tokenRotator) rollAPI(ctx context.Context) error {
+	if err := r.rollout(ctx, apiDeployment); err != nil {
+		return fmt.Errorf("roll felis-api: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - felis-api: rolled out on the new value")
+	return nil
+}
+
+// withMinecraft is the control namespace plus the minecraft namespace when that
+// is a different one: where the Secrets game pods and Jobs mount are mirrored.
+func (r tokenRotator) withMinecraft() []string {
+	if r.minecraftNS == "" || r.minecraftNS == r.controlNS {
+		return []string{r.controlNS}
+	}
+	return []string{r.controlNS, r.minecraftNS}
+}
+
+func (r tokenRotator) rotateCaller(ctx context.Context, ct naming.CallerToken, apply bool) error {
+	namespaces := []string{r.controlNS}
+	replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
+	if replica != "" && replica != r.controlNS {
+		namespaces = append(namespaces, replica)
+	}
+	key := installerTokenKeys[ct.Caller]
+
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new internal token for the %s caller\n", ct.Caller, ct.Caller)
+	secrets := make([]string, len(namespaces))
+	for i, ns := range namespaces {
+		secrets[i] = "Secret " + ns + "/" + ct.Secret
+	}
+	writes := append([]string{r.secretsEnv + " (" + key + ")"}, secrets...)
+	hostProxy := ct.Caller == "velocity" && fileExists(r.linkProps)
+	if hostProxy {
+		writes = append(writes, r.linkProps+" (service-token)")
+	}
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
+	switch ct.Caller {
+	case "velocity":
+		if hostProxy {
+			fmt.Fprintf(r.out, "  - the proxy re-reads its token from the file and keeps its players; if it has not within %s of felis-api's restart, it is restarted, which disconnects everyone online\n", r.reloadWait)
+		} else {
+			fmt.Fprintf(r.out, "  - no proxy on this host (%s): set service-token in your proxy's felis-link.properties to the value in Secret %s/%s afterwards; it re-reads the file without a restart\n",
+				r.linkProps, r.controlNS, ct.Secret)
+		}
+	case "limbo":
+		fmt.Fprintln(r.out, "  - the login gate's pod restarts: a player signing in at that moment reconnects")
+	case "build":
+		fmt.Fprintln(r.out, "  - a build fetching its context at that moment fails and can be submitted again")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now presents the new token on its next run")
+	}
+	if !r.confirm(ct.Caller, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a token: %w", err)
+	}
+	if err := r.record([2]string{key, tok}); err != nil {
+		return err
+	}
+	for _, ns := range namespaces {
+		if err := r.putSecret(ctx, ns, ct.Secret, map[string][]byte{naming.ServiceTokenSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+
+	// The proxy's file changes before felis-api rolls, so the file never falls
+	// behind the Secret; the proxy's log is read from just before the write,
+	// since it may pick the new token up before the rollout ends.
+	since := r.now()
+	if hostProxy {
+		if err := setProxyToken(r.linkProps, tok); err != nil {
+			return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: service-token updated\n", r.linkProps)
+	}
+
+	if err := r.rollAPI(ctx); err != nil {
+		return err
+	}
+
+	switch ct.Caller {
+	case "velocity":
+		if !hostProxy {
+			break
+		}
+		if r.proxyReloaded(ctx, since, tokenFingerprint(tok)) {
+			fmt.Fprintf(r.out, "  - %s: took the new token from its properties; players stayed connected\n", velocityUnit)
+			break
+		}
+		if err := r.restartUnit(ctx, velocityUnit); err != nil {
+			return fmt.Errorf("restart %s: %w", velocityUnit, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: had not taken the new token within %s, restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit, r.reloadWait)
+	case "limbo":
+		// Only the operator's server pods carry its managed-by label; a backup or
+		// restore Job's pod carries the server label too, and app.kubernetes.io ones.
+		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), client.MatchingLabels{
+			v1alpha1.LabelServer:    naming.SystemLoginServer,
+			v1alpha1.LabelManagedBy: operator.ManagedByValue,
+		}); err != nil {
+			return fmt.Errorf("restart the login gate: %w", err)
+		}
+		fmt.Fprintln(r.out, "  - login gate: pod restarted to read the new token")
+	case "build":
+		fmt.Fprintln(r.out, "  - builds: the next build Job reads the new token")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now reads the new token on its next run")
+	}
+	return nil
+}
+
+// proxyReloaded waits up to reloadWait for the host proxy to log that it
+// reloaded the token with this fingerprint (plugins/shared FileToken).
+func (r tokenRotator) proxyReloaded(ctx context.Context, since time.Time, fingerprint string) bool {
+	want := "(fingerprint " + fingerprint + ")"
+	deadline := r.now().Add(r.reloadWait)
+	for {
+		if strings.Contains(r.proxyLog(ctx, since), want) {
+			return true
+		}
+		if !r.now().Before(deadline) {
+			return false
+		}
+		select {
+		case <-ctx.Done():
+			return false
+		case <-time.After(r.pollEvery):
+		}
+	}
+}
+
+// journalSince is the proxy unit's journal from the second since falls in. A
+// journal that cannot be read reads as one without the line, which ends in the
+// restart a rotation made before the proxy could reload.
+func journalSince(ctx context.Context, since time.Time) string {
+	out, _ := exec.CommandContext(ctx, "journalctl", "-u", velocityUnit, "--since", "@"+strconv.FormatInt(since.Unix(), 10),
+		"-o", "cat", "--no-pager", "-q").Output()
+	return string(out)
+}
+
+// setProxyToken writes the proxy's token into felis-link.properties and puts
+// the file's modification time back. The proxy re-reads its token by itself,
+// while `felis domain check` and `felis domain set` read a file newer than the
+// proxy's start as config it has not loaded (the installer's
+// install_if_changed keeps the time for the same reason).
+func setProxyToken(path, token string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	if err := setKeyValueLine(path, "service-token", "=", token); err != nil {
+		return err
+	}
+	return os.Chtimes(path, time.Time{}, info.ModTime())
+}
+
+func (r tokenRotator) rotateRegistry(ctx context.Context, apply bool) error {
+	keys := make([]string, len(registrygate.Principals))
+	for i, p := range registrygate.Principals {
+		keys[i] = installerRegistryKeys[p]
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: new write tokens for the image registry's principals (%s)\n", kindRegistry, strings.Join(registrygate.Principals, ", "))
+	fmt.Fprintf(r.out, "  - writes %s (%s), Secret %s/%s, Secret %s/%s\n", r.secretsEnv, strings.Join(keys, ", "),
+		r.controlNS, naming.RegistryAuthSecretName, r.buildNS, naming.RegistryPushSecretName)
+	fmt.Fprintln(r.out, "  - the registry restarts to load them: a build pushing its image at that moment fails and can be submitted again, and an image pull in that moment retries")
+	fmt.Fprintf(r.out, "  - %s (it presents the prune token)\n", apiRestartNote)
+	if !r.confirm(kindRegistry, apply) {
+		return nil
+	}
+
+	tokens := map[string][]byte{}
+	kv := make([][2]string, 0, len(registrygate.Principals))
+	for _, p := range registrygate.Principals {
+		tok, err := r.newToken()
+		if err != nil {
+			return fmt.Errorf("generate a token: %w", err)
+		}
+		tokens[p] = []byte(tok)
+		kv = append(kv, [2]string{installerRegistryKeys[p], tok})
+	}
+	if err := r.record(kv...); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.controlNS, naming.RegistryAuthSecretName, tokens); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.buildNS, naming.RegistryPushSecretName, map[string][]byte{
+		naming.RegistryPushUsernameKey: []byte(registrygate.PrincipalBuild),
+		naming.RegistryPushPasswordKey: tokens[registrygate.PrincipalBuild],
+	}); err != nil {
+		return err
+	}
+	if err := r.rollout(ctx, registryDeployment); err != nil {
+		return fmt.Errorf("restart the registry: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - registry: restarted on the new tokens")
+	return r.rollAPI(ctx)
+}
+
+func (r tokenRotator) rotateForwarding(ctx context.Context, apply bool) error {
+	restart, held, err := r.gamePods(ctx)
+	if err != nil {
+		return err
+	}
+	hostProxy := fileExists(r.forwardingFile)
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new Velocity forwarding secret, the key a server checks each player's identity with\n", kindForwarding)
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, "Secret "+ns+"/"+naming.ForwardingSecretName)
+	}
+	writes := append([]string{r.secretsEnv + " (" + installerForwardingKey + ")"}, secrets...)
+	if hostProxy {
+		writes = append(writes, r.forwardingFile)
+	}
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - every running server restarts to read it (%d now), saving its world on the way down, and the proxy restarts: everyone online is disconnected and can rejoin once their server is back\n", len(restart))
+	if len(held) > 0 {
+		fmt.Fprintf(r.out, "  - left running, because a backup, restore or file write holds its world: %s. Players cannot join it until it restarts: stop and start it from the panel once that finishes\n", strings.Join(held, ", "))
+	}
+	if !hostProxy {
+		fmt.Fprintf(r.out, "  - no proxy on this host (%s): put the value in Secret %s/%s into your proxy's forwarding secret file and restart it\n",
+			r.forwardingFile, r.controlNS, naming.ForwardingSecretName)
+	}
+	if !r.confirm(kindForwarding, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a secret: %w", err)
+	}
+	if err := r.record([2]string{installerForwardingKey, tok}); err != nil {
+		return err
+	}
+	if hostProxy {
+		info, err := os.Stat(r.forwardingFile)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(r.forwardingFile, info, []byte(tok)); err != nil {
+			return fmt.Errorf("write %s: %w", r.forwardingFile, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: updated\n", r.forwardingFile)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, naming.ForwardingSecretName, map[string][]byte{naming.ForwardingSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+	// The servers go first: their pods read the Secret as they are recreated,
+	// and a player who rejoins through the restarted proxy meets a server on the
+	// new secret, or one still starting.
+	for i := range restart {
+		p := &restart[i]
+		if err := r.cl.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) {
+			return fmt.Errorf("restart %s: %w", p.Labels[v1alpha1.LabelServer], err)
+		}
+	}
+	fmt.Fprintf(r.out, "  - servers: %d restarting on the new secret\n", len(restart))
+	if hostProxy {
+		if err := r.restartUnit(ctx, velocityUnit); err != nil {
+			return fmt.Errorf("restart %s: %w", velocityUnit, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: restarted on the new secret\n", velocityUnit)
+	}
+	return nil
+}
+
+// gamePods lists the running game server pods: those a rotation may restart,
+// and the servers left alone because a backup, restore or file write holds
+// their world (internal/maintenance), which a restart in the middle of would
+// break.
+func (r tokenRotator) gamePods(ctx context.Context) ([]corev1.Pod, []string, error) {
+	var pods corev1.PodList
+	// The operator's managed-by label is on its server pods alone (see rotateCaller).
+	if err := r.cl.List(ctx, &pods, client.InNamespace(r.minecraftNS), client.MatchingLabels{v1alpha1.LabelManagedBy: operator.ManagedByValue}); err != nil {
+		return nil, nil, fmt.Errorf("list the servers' pods: %w", err)
+	}
+	var jobs batchv1.JobList
+	if err := r.cl.List(ctx, &jobs, client.InNamespace(r.minecraftNS)); err != nil {
+		return nil, nil, fmt.Errorf("list the maintenance Jobs: %w", err)
+	}
+	var restart []corev1.Pod
+	var held []string
+	for _, p := range pods.Items {
+		if p.DeletionTimestamp != nil {
+			continue
+		}
+		server := p.Labels[v1alpha1.LabelServer]
+		var ms v1alpha1.MinecraftServer
+		if err := r.cl.Get(ctx, client.ObjectKey{Namespace: r.minecraftNS, Name: server}, &ms); client.IgnoreNotFound(err) != nil {
+			return nil, nil, fmt.Errorf("read server %s: %w", server, err)
+		}
+		if kind, ok := maintenance.Holder(server, ms.Annotations, jobs.Items, r.now()); ok {
+			held = append(held, server+" ("+kind+")")
+			continue
+		}
+		restart = append(restart, p)
+	}
+	return restart, held, nil
+}
+
+func (r tokenRotator) rotateDB(ctx context.Context, apply bool) error {
+	cfg, err := config.Load(r.hostTOML)
+	if err != nil {
+		return err
+	}
+	if cfg.Database.Deployment == "" {
+		return fmt.Errorf("[database] deployment is unset in %s, so the database is not the installer's felis-postgres: change the role's password where it runs, then in [database] url of each config copy", r.hostTOML)
+	}
+	u, err := neturl.Parse(cfg.Database.URL)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return fmt.Errorf("[database] url in %s names no role", r.hostTOML)
+	}
+	role := u.User.Username()
+	targets, err := tomlTargetsOf(r.hostTOML, r.podTOML, r.defaultTOML)
+	if err != nil {
+		return err
+	}
+	paths := make([]string, len(targets))
+	for i, t := range targets {
+		paths[i] = t.path
+	}
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, ns+"/"+platform.ConfigSecretName)
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new password for the database role %q\n", kindDB, role)
+	fmt.Fprintf(r.out, "  - writes %s (%s), the role in %s, [database] url in %s, Secret %s\n",
+		r.secretsEnv, installerDBKey, cfg.Database.Deployment, strings.Join(paths, " and "), strings.Join(secrets, " and "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
+	fmt.Fprintln(r.out, "  - a backup, restore or file Job that connects in the seconds between the password change and felis-api's restart fails and can be run again; the host's timers read the new config on their next run")
+	if !r.confirm(kindDB, apply) {
+		return nil
+	}
+
+	password, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a password: %w", err)
+	}
+	// Every config copy is edited in memory first, so one this cannot edit stops
+	// the rotation before the role's password changes.
+	edited := make([][]byte, len(targets))
+	var hostURL string
+	var podConfig []byte
+	for i, t := range targets {
+		raw, err := os.ReadFile(t.real)
+		if err != nil {
+			return err
+		}
+		var doc struct {
+			Database struct {
+				URL string `toml:"url"`
+			} `toml:"database"`
+		}
+		if _, err := toml.Decode(string(raw), &doc); err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		next, err := withPassword(doc.Database.URL, password)
+		if err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		if edited[i], err = editTOMLStrings(raw, []tomlStringEdit{{"database", "url", next}}); err != nil {
+			return fmt.Errorf("%s: %w; set the password in its [database] url by hand", t.path, err)
+		}
+		switch t.path {
+		case r.hostTOML:
+			hostURL = next
+		case r.podTOML:
+			podConfig = edited[i]
+		}
+	}
+	if podConfig == nil {
+		return fmt.Errorf("%s resolves to the same file as %s; the pods reach the database at another address and need a copy of their own", r.podTOML, r.hostTOML)
+	}
+
+	if err := r.record([2]string{installerDBKey, password}); err != nil {
+		return err
+	}
+	salt := make([]byte, 16)
+	if _, err := rand.Read(salt); err != nil {
+		return err
+	}
+	verifier, err := scramVerifier(password, salt, scramIterations)
+	if err != nil {
+		return err
+	}
+	if err := r.alterRole(ctx, cfg.Database.Deployment, role, verifier); err != nil {
+		return fmt.Errorf("set the role's password: %w", err)
+	}
+	if err := r.verifyDB(ctx, hostURL); err != nil {
+		return fmt.Errorf("the database does not accept the new password (%v); the config copies still hold the old one: run the installer again (sudo bash deploy/bootstrap.sh), which sets the password in %s everywhere", err, r.secretsEnv)
+	}
+	fmt.Fprintf(r.out, "  - role %s: password changed, and the database accepts it\n", role)
+	for i, t := range targets {
+		info, err := os.Stat(t.real)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(t.real, info, edited[i]); err != nil {
+			return fmt.Errorf("write %s: %w", t.path, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: [database] url updated\n", t.path)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, platform.ConfigSecretName, map[string][]byte{platform.ConfigSecretKey: podConfig}); err != nil {
+			return err
+		}
+	}
+	return r.rollAPI(ctx)
+}
+
+// withPassword is a database URL with its password replaced.
+func withPassword(raw, password string) (string, error) {
+	u, err := neturl.Parse(raw)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return "", errors.New("its [database] url names no role")
+	}
+	u.User = neturl.UserPassword(u.User.Username(), password)
+	return u.String(), nil
+}
+
+// scramIterations is PostgreSQL's default scram_iterations.
+const scramIterations = 4096
+
+// scramVerifier is the SCRAM-SHA-256 verifier PostgreSQL stores for a password
+// (RFC 5802 and RFC 7677, in the form libpq's PQencryptPasswordConn makes).
+// ALTER ROLE stores a verifier as it is given, so the password itself never
+// reaches the server, where a failing statement is logged with its text.
+func scramVerifier(password string, salt []byte, iterations int) (string, error) {
+	salted, err := pbkdf2.Key(sha256.New, password, salt, iterations, sha256.Size)
+	if err != nil {
+		return "", err
+	}
+	mac := func(msg string) []byte {
+		h := hmac.New(sha256.New, salted)
+		h.Write([]byte(msg))
+		return h.Sum(nil)
+	}
+	stored := sha256.Sum256(mac("Client Key"))
+	b64 := base64.StdEncoding.EncodeToString
+	return fmt.Sprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, b64(salt), b64(stored[:]), b64(mac("Server Key"))), nil
+}
+
+// alterRoleInPod runs ALTER ROLE as the superuser inside the database's
+// container, over its socket, with the statement on stdin.
+func alterRoleInPod(ctx context.Context, deployment, role, verifier string) error {
+	ns, name, _ := strings.Cut(deployment, "/")
+	return kubectlWithInput(ctx, []byte(alterRoleSQL(role, verifier)), "-n", ns, "exec", "-i", "deploy/"+name, "-c", platform.PostgresContainer, "--",
+		"psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres")
+}
+
+// alterRoleSQL sets role's password to a SCRAM verifier, which holds no quote.
+func alterRoleSQL(role, verifier string) string {
+	return "ALTER ROLE \"" + strings.ReplaceAll(role, `"`, `""`) + "\" WITH PASSWORD '" + verifier + "';\n"
+}
+
+// setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file
+// (secrets.env, a .properties file), appending one when the key is absent. The
+// file is replaced atomically and keeps its mode and owner: felis-link.properties
+// is root:felis-velocity 0640, and the proxy must still be able to read it.
+func setKeyValueLine(path, key, sep, value string) error {
+	return setKeyValueLines(path, sep, [][2]string{{key, value}})
+}
+
+// setKeyValueLines is setKeyValueLine for several keys in one rewrite.
+func setKeyValueLines(path, sep string, kv [][2]string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
+	for _, p := range kv {
+		key, value := p[0], p[1]
+		found := false
+		for i, ln := range lines {
+			k, _, ok := strings.Cut(ln, sep)
+			if ok && strings.TrimSpace(k) == key {
+				lines[i] = key + sep + value
+				found = true
+			}
+		}
+		if !found {
+			lines = append(lines, key+sep+value)
+		}
+	}
+	return replaceFileKeepingMode(path, info, []byte(strings.Join(lines, "\n")+"\n"))
+}
+
+// replaceFileKeepingMode atomically replaces path with data, keeping the mode and
+// owner info describes: these files are read by other users (the proxy's) and
+// some hold credentials, so a rewrite must not widen or narrow who can read them.
+func replaceFileKeepingMode(path string, info os.FileInfo, data []byte) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	if err := tmp.Chmod(info.Mode().Perm()); err != nil {
+		tmp.Close()
+		return err
+	}
+	if st, ok := info.Sys().(*syscall.Stat_t); ok {
+		if err := tmp.Chown(int(st.Uid), int(st.Gid)); err != nil {
+			tmp.Close()
+			return err
+		}
+	}
+	if _, err := tmp.Write(data); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
+}

cmd/felis/rotatetoken_test.go

@@ -0,0 +1,883 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/maintenance"
+
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+type rotationRig struct {
+	r      tokenRotator
+	cl     client.Client
+	out    *bytes.Buffer
+	events []string
+	dir    string
+	clock  time.Time
+}
+
+func tokenSecret(ns, name, val string) *corev1.Secret {
+	return keySecret(ns, name, "token", val)
+}
+
+func keySecret(ns, name, key, val string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name},
+		Data:       map[string][]byte{key: []byte(val)},
+	}
+}
+
+// serverPod is a game server's pod as the operator labels it.
+func serverPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{
+			"felis.lolicon.best/server":     server,
+			"felis.lolicon.best/managed-by": "felis-operator",
+			"felis.lolicon.best/component":  "server",
+		}}}
+}
+
+// backupPod is a backup Job's pod as internal/backupjob labels it: it carries
+// the server's label too, and no rotation may take it for the server's own.
+func backupPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{
+			"felis.lolicon.best/server":    server,
+			"app.kubernetes.io/managed-by": "felis-backup",
+			"app.kubernetes.io/component":  "world-backup",
+		}}}
+}
+
+func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
+	t.Helper()
+	rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir(), clock: time.Unix(1_800_000_000, 0)}
+	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
+	rig.r = tokenRotator{
+		cl:             rig.cl,
+		controlNS:      "felis",
+		minecraftNS:    "minecraft",
+		buildNS:        "felis-build",
+		secretsEnv:     filepath.Join(rig.dir, "secrets.env"),
+		linkProps:      filepath.Join(rig.dir, "felis-link.properties"),
+		forwardingFile: filepath.Join(rig.dir, "forwarding.secret"),
+		hostTOML:       filepath.Join(rig.dir, "felis.host.toml"),
+		podTOML:        filepath.Join(rig.dir, "felis.pod.toml"),
+		defaultTOML:    filepath.Join(rig.dir, "felis.toml"),
+		newToken:       func() (string, error) { return "NEWTOKEN", nil },
+		rollout: func(_ context.Context, deployment string) error {
+			rig.events = append(rig.events, "roll "+deployment)
+			return nil
+		},
+		restartUnit: func(_ context.Context, unit string) error {
+			rig.events = append(rig.events, "restart "+unit)
+			return nil
+		},
+		proxyLog: func(context.Context, time.Time) string { return "" },
+		alterRole: func(context.Context, string, string, string) error {
+			rig.events = append(rig.events, "alter-role")
+			return nil
+		},
+		verifyDB: func(context.Context, string) error {
+			rig.events = append(rig.events, "verify-db")
+			return nil
+		},
+		// Every look at the clock moves it a second on, so a wait measured with it
+		// ends after a known number of looks.
+		now: func() time.Time {
+			rig.clock = rig.clock.Add(time.Second)
+			return rig.clock
+		},
+		reloadWait: 5 * time.Second,
+		out:        rig.out,
+	}
+	return rig
+}
+
+func (rig *rotationRig) secretKey(t *testing.T, ns, name, key string) string {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		return "<missing>"
+	}
+	return string(s.Data[key])
+}
+
+func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
+	t.Helper()
+	return rig.secretKey(t, ns, name, "token")
+}
+
+func (rig *rotationRig) pods(t *testing.T) string {
+	t.Helper()
+	var pods corev1.PodList
+	if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
+		t.Fatal(err)
+	}
+	names := make([]string, len(pods.Items))
+	for i, p := range pods.Items {
+		names[i] = p.Name
+	}
+	return strings.Join(names, ",")
+}
+
+func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(body), mode); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(path, mode); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func readTestFile(t *testing.T, path string) string {
+	t.Helper()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(raw)
+}
+
+func TestRotateLimboToken(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-service-token", "proxy"),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+		backupPod("minecraft", "login-backup-x", "login"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600)
+
+	// felis-api must roll only after both copies hold the new value, and the login
+	// pod must still be there then: restarting it earlier would have it present
+	// the new token to an api that does not know it yet.
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the control Secret held %q", got)
+		}
+		if got := rig.secret(t, "minecraft", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the minecraft replica held %q", got)
+		}
+		var pod corev1.Pod
+		if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+			t.Errorf("the login pod was restarted before the api rolled")
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), "limbo", true); err != nil {
+		t.Fatal(err)
+	}
+
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 {
+		t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm())
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" {
+		t.Errorf("the proxy's token changed to %q", got)
+	}
+	// The login pod restarted; a user server and the backup Job's pod, which
+	// carries the login server's label too, are untouched.
+	if got := rig.pods(t); got != "login-backup-x,survival-0" {
+		t.Errorf("pods left = %s, want login-backup-x,survival-0", got)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new token was printed: %s", rig.out.String())
+	}
+}
+
+const testLinkProps = "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n"
+
+func reloadLine(token string) string {
+	return "[12:00:00 INFO] [felis-link]: Felis: service-token reloaded from /x/felis-link.properties (fingerprint " + tokenFingerprint(token) + ")\n"
+}
+
+// The host proxy re-reads its token: the rotation waits for it to say so and
+// leaves it running.
+func TestRotateVelocityTokenReloadsTheHostProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	if err := os.Chtimes(rig.r.linkProps, written, written); err != nil {
+		t.Fatal(err)
+	}
+
+	// The proxy logs its reload on its next call to felis-api, which may be
+	// while the api rolls: the log must be read from before that.
+	var reloadedAt time.Time
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := readTestFile(t, rig.r.linkProps); !strings.Contains(got, "service-token=NEWTOKEN\n") {
+			t.Errorf("api rolled before the proxy's file held the new token: %q", got)
+		}
+		reloadedAt = rig.clock
+		return nil
+	}
+	looks := 0
+	rig.r.proxyLog = func(_ context.Context, since time.Time) string {
+		looks++
+		log := reloadLine("old") // an earlier rotation's
+		if looks >= 3 && !since.After(reloadedAt) {
+			log += reloadLine("NEWTOKEN")
+		}
+		return log
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.linkProps); got != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
+		t.Errorf("felis-link.properties = %q", got)
+	}
+	info, _ := os.Stat(rig.r.linkProps)
+	if info.Mode().Perm() != 0o640 {
+		t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm())
+	}
+	if !info.ModTime().Equal(written) {
+		t.Errorf("properties mtime = %v, want it kept at %v (felis domain check would read the proxy as stale)", info.ModTime(), written)
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	// The proxy's token has no replica: it must not appear in a workload namespace.
+	if got := rig.secret(t, "minecraft", "felis-service-token"); got != "<missing>" {
+		t.Errorf("rotation copied the proxy token into minecraft (%q)", got)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want the api roll and no proxy restart", rig.events)
+	}
+	if looks != 3 {
+		t.Errorf("the log was read %d times, want 3 (until the line appeared)", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: took the new token from its properties; players stayed connected") {
+		t.Errorf("output does not say the players stayed: %s", out)
+	}
+}
+
+// A proxy that never logs the new fingerprint (an older plugin, a stuck proxy)
+// is restarted once the wait is over.
+func TestRotateVelocityTokenRestartsAProxyThatDoesNotReload(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	looks := 0
+	rig.r.proxyLog = func(context.Context, time.Time) string {
+		looks++
+		return reloadLine("old")
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api,restart felis-velocity" {
+		t.Errorf("events = %v, want the api roll then the proxy restart", rig.events)
+	}
+	// reloadWait is 5s and each look moves the clock a second.
+	if looks != 5 {
+		t.Errorf("the log was read %d times, want 5", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: had not taken the new token within 5s, restarted") {
+		t.Errorf("output does not explain the restart: %s", out)
+	}
+}
+
+// The proxy and the Java plugin name a token by the same fingerprint
+// (plugins/shared LinkConfigLoaderTest fileTokenFollowsTheFile).
+func TestTokenFingerprintMatchesThePlugin(t *testing.T) {
+	if got := tokenFingerprint("new-token"); got != "348e9df2a42b" {
+		t.Errorf("tokenFingerprint(new-token) = %q, want 348e9df2a42b", got)
+	}
+}
+
+// An external proxy has no felis-link.properties here: the Secret still rotates,
+// nothing is restarted on this host, and the output says where the value is
+// without printing it.
+func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want no proxy restart", rig.events)
+	}
+	out := rig.out.String()
+	if !strings.Contains(out, "felis/felis-service-token") || strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("output should point at the Secret without the value: %s", out)
+	}
+}
+
+func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old"))
+	// An install from before per-caller tokens has no BUILD_TOKEN line yet.
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600)
+	if err := rig.r.rotate(context.Background(), "build", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("build replica = %q, want NEWTOKEN (created when absent)", got)
+	}
+	if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+}
+
+// A failed api rollout stops the rotation before the caller restarts: the login
+// gate keeps running on the old value the old api pods still accept.
+func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		serverPod("minecraft", "login-0", "login"),
+	)
+	rig.r.rollout = func(context.Context, string) error { return errors.New("rollout timed out") }
+	err := rig.r.rotate(context.Background(), "limbo", true)
+	if err == nil || !strings.Contains(err.Error(), "rollout timed out") {
+		t.Fatalf("err = %v, want the rollout failure", err)
+	}
+	if got := rig.pods(t); got != "login-0" {
+		t.Errorf("pods left = %s: the login pod was restarted although the api never rolled", got)
+	}
+}
+
+func TestRotateRefusesAnUnknownCredential(t *testing.T) {
+	rig := newRotationRig(t)
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600)
+	if err := rig.r.rotate(context.Background(), "admin", true); err == nil {
+		t.Fatal("rotated an unknown credential")
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\n" {
+		t.Errorf("secrets.env = %q, want it untouched", got)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want nothing touched", rig.events)
+	}
+}
+
+const (
+	testHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	testPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+// installedRig is a host as the installer leaves it, with every credential in
+// place, for the plan test.
+func installedRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-service-token", "old"),
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-build-token", "old"),
+		tokenSecret("felis-build", "felis-build-token", "old"),
+		tokenSecret("felis", "felis-ops-token", "old"),
+		keySecret("felis", "felis-registry-auth", "platform", "old"),
+		keySecret("felis-build", "felis-registry-push", "password", "old"),
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	return rig
+}
+
+// Without -yes every rotation prints its plan and changes nothing.
+func TestRotatePlanChangesNothing(t *testing.T) {
+	for _, kind := range rotationKinds() {
+		t.Run(kind, func(t *testing.T) {
+			rig := installedRig(t)
+			files := map[string]string{}
+			for _, p := range []string{rig.r.secretsEnv, rig.r.linkProps, rig.r.forwardingFile, rig.r.hostTOML, rig.r.podTOML} {
+				files[p] = readTestFile(t, p)
+			}
+			rig.r.newToken = func() (string, error) {
+				t.Error("a plan generated a token")
+				return "NEWTOKEN", nil
+			}
+			if err := rig.r.rotate(context.Background(), kind, false); err != nil {
+				t.Fatal(err)
+			}
+			for p, before := range files {
+				if got := readTestFile(t, p); got != before {
+					t.Errorf("%s changed to %q", filepath.Base(p), got)
+				}
+			}
+			for _, s := range [][3]string{
+				{"felis", "felis-service-token", "token"}, {"felis", "felis-limbo-token", "token"},
+				{"felis-build", "felis-build-token", "token"}, {"felis", "felis-ops-token", "token"},
+				{"felis", "felis-registry-auth", "platform"}, {"felis-build", "felis-registry-push", "password"},
+				{"minecraft", "felis-forwarding-secret", "secret"},
+			} {
+				if got := rig.secretKey(t, s[0], s[1], s[2]); got != "old" {
+					t.Errorf("Secret %s/%s changed to %q", s[0], s[1], got)
+				}
+			}
+			if got := rig.secretKey(t, "minecraft", "felis-config", "felis.toml"); got != testPodTOML {
+				t.Errorf("felis-config changed to %q", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want none", rig.events)
+			}
+			if got := rig.pods(t); got != "login-0,survival-0" {
+				t.Errorf("pods left = %s, want all", got)
+			}
+			out := rig.out.String()
+			if !strings.HasSuffix(out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes "+kind+"\n") {
+				t.Errorf("the plan does not end with how to go ahead: %s", out)
+			}
+			// Each plan names the interruption it causes.
+			want := apiRestartNote
+			if kind == kindForwarding {
+				want = "everyone online is disconnected"
+			}
+			if !strings.Contains(out, want) {
+				t.Errorf("the plan does not say %q: %s", want, out)
+			}
+		})
+	}
+}
+
+func TestRotateRegistryTokens(t *testing.T) {
+	rig := newRotationRig(t,
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: "felis-registry-auth"},
+			Data: map[string][]byte{"platform": []byte("a"), "build": []byte("b"), "prune": []byte("c")}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis-build", Name: "felis-registry-push"},
+			Data: map[string][]byte{"username": []byte("build"), "password": []byte("b")}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=a\nREGISTRY_BUILD_TOKEN=b\nREGISTRY_PRUNE_TOKEN=c\n", 0o600)
+	n := 0
+	rig.r.newToken = func() (string, error) {
+		n++
+		return fmt.Sprintf("NEWTOKEN%d", n), nil
+	}
+	// The registry reads its tokens as it starts: it restarts after the Secret
+	// holds them, and felis-api (the prune token) after that.
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", "prune"); got != "NEWTOKEN3" {
+			t.Errorf("%s rolled while the prune token was %q", deployment, got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindRegistry, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=NEWTOKEN1\nREGISTRY_BUILD_TOKEN=NEWTOKEN2\nREGISTRY_PRUNE_TOKEN=NEWTOKEN3\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	for key, want := range map[string]string{"platform": "NEWTOKEN1", "build": "NEWTOKEN2", "prune": "NEWTOKEN3"} {
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", key); got != want {
+			t.Errorf("felis-registry-auth %s = %q, want %s", key, got, want)
+		}
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "password"); got != "NEWTOKEN2" {
+		t.Errorf("the build Jobs' push password = %q, want the build token", got)
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "username"); got != "build" {
+		t.Errorf("the build Jobs' push username = %q, want build", got)
+	}
+	if strings.Join(rig.events, ",") != "roll registry,roll felis-api" {
+		t.Errorf("events = %v, want the registry then felis-api", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("a new token was printed: %s", rig.out.String())
+	}
+}
+
+func TestRotateForwardingSecret(t *testing.T) {
+	lock := time.Unix(1_800_000_000, 0)
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		serverPod("minecraft", "survival-0", "survival"),
+		serverPod("minecraft", "lobby-0", "lobby"),
+		// creative is being backed up: a running Job holds its world.
+		serverPod("minecraft", "creative-0", "creative"),
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "creative-backup",
+			Labels: map[string]string{maintenance.LabelServer: "creative", maintenance.LabelManagedBy: "felis-backup"}}},
+		// survival's last backup is over; its finished pod stays until the Job's TTL.
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival-backup",
+			Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: "felis-backup"}},
+			Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}},
+		backupPod("minecraft", "survival-backup-x", "survival"),
+		// A pod already on its way out is neither counted nor deleted again.
+		func() *corev1.Pod {
+			p := serverPod("minecraft", "lobby-old", "lobby")
+			p.DeletionTimestamp = &metav1.Time{Time: lock}
+			p.Finalizers = []string{"felis.lolicon.best/test"}
+			return p
+		}(),
+		// skyblock's restore has just been admitted, its Job not created yet.
+		serverPod("minecraft", "skyblock-0", "skyblock"),
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "skyblock",
+			Annotations: map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lock)}}},
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival"}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nFORWARDING_SECRET=old\n", 0o600)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	// The proxy restarts after the servers were told to: a player who rejoins
+	// must not meet a server still on the old secret.
+	rig.r.restartUnit = func(_ context.Context, unit string) error {
+		rig.events = append(rig.events, "restart "+unit)
+		if got := rig.pods(t); strings.Contains(got, "survival-0") {
+			t.Errorf("the proxy restarted before the servers (pods %s)", got)
+		}
+		if got := readTestFile(t, rig.r.forwardingFile); got != "NEWTOKEN" {
+			t.Errorf("the proxy restarted on forwarding.secret %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nFORWARDING_SECRET=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if info, _ := os.Stat(rig.r.forwardingFile); info.Mode().Perm() != 0o640 {
+		t.Errorf("forwarding.secret mode = %v, want 0640", info.Mode().Perm())
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+			t.Errorf("Secret %s/felis-forwarding-secret = %q, want NEWTOKEN", ns, got)
+		}
+	}
+	if got := rig.pods(t); got != "creative-0,lobby-old,skyblock-0,survival-backup-x" {
+		t.Errorf("pods left = %s, want the held servers, the terminating pod and the backup's pod", got)
+	}
+	if strings.Join(rig.events, ",") != "restart felis-velocity" {
+		t.Errorf("events = %v, want only the proxy restart (felis-api does not hold this secret)", rig.events)
+	}
+	out := rig.out.String()
+	for _, want := range []string{
+		"every running server restarts to read it (2 now)",
+		"left running, because a backup, restore or file write holds its world: creative (backup), skyblock (restore).",
+		"  - servers: 2 restarting on the new secret\n",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output lacks %q: %s", want, out)
+		}
+	}
+	if strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("the new secret was printed: %s", out)
+	}
+}
+
+func TestRotateForwardingSecretForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, keySecret("felis", "felis-forwarding-secret", "secret", "old"))
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secretKey(t, "felis", "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want no proxy restart on this host", rig.events)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "put the value in Secret felis/felis-forwarding-secret into your proxy's forwarding secret file") {
+		t.Errorf("output does not say where the value is: %s", out)
+	}
+}
+
+// dbRig is an installed host for the database rotation: felis.toml links to the
+// host copy, as the installer makes it.
+func dbRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n", 0o600)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	if err := os.Symlink("felis.host.toml", rig.r.defaultTOML); err != nil {
+		t.Fatal(err)
+	}
+	return rig
+}
+
+const (
+	wantHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	wantPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+func TestRotateDatabasePassword(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.alterRole = func(_ context.Context, deployment, role, verifier string) error {
+		rig.events = append(rig.events, "alter-role")
+		if deployment != "felis/felis-postgres" || role != "felis" {
+			t.Errorf("alterRole(%q, %q), want felis/felis-postgres and felis", deployment, role)
+		}
+		if !strings.Contains(readTestFile(t, rig.r.secretsEnv), "DB_PASSWORD=NEWTOKEN\n") {
+			t.Error("the role changed before secrets.env recorded the new password")
+		}
+		// The verifier is the new password's, under the salt it carries.
+		m := regexp.MustCompile(`^SCRAM-SHA-256\$4096:([^$]+)\$`).FindStringSubmatch(verifier)
+		if m == nil {
+			t.Fatalf("verifier %q is not a SCRAM-SHA-256 verifier", verifier)
+		}
+		salt, err := base64.StdEncoding.DecodeString(m[1])
+		if err != nil || len(salt) != 16 {
+			t.Fatalf("verifier salt %q: %v", m[1], err)
+		}
+		if want, _ := scramVerifier("NEWTOKEN", salt, 4096); verifier != want {
+			t.Errorf("verifier = %q, want %q", verifier, want)
+		}
+		return nil
+	}
+	// The configs change only once the database accepts the new password.
+	rig.r.verifyDB = func(_ context.Context, url string) error {
+		rig.events = append(rig.events, "verify-db")
+		if url != "postgres://felis:[email protected]:30432/felis?sslmode=disable" {
+			t.Errorf("verified with %q, want the host URL with the new password", url)
+		}
+		if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+			t.Errorf("the host config changed before the database accepted the password: %q", got)
+		}
+		return nil
+	}
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("felis-api rolled on felis-config %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindDB, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=NEWTOKEN\nSERVICE_TOKEN=s\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != wantHostTOML {
+		t.Errorf("host config = %q", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != wantPodTOML {
+		t.Errorf("pod config = %q", got)
+	}
+	if info, err := os.Lstat(rig.r.defaultTOML); err != nil || info.Mode()&os.ModeSymlink == 0 {
+		t.Errorf("felis.toml is no longer the link to the host copy (%v)", err)
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("Secret %s/felis-config = %q", ns, got)
+		}
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db,roll felis-api" {
+		t.Errorf("events = %v", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new password was printed: %s", rig.out.String())
+	}
+}
+
+// A password the database does not accept leaves the configs on the old one
+// and felis-api running: the installer's record, already new, is the way back.
+func TestRotateDatabasePasswordStopsWhenTheDatabaseRefuses(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.verifyDB = func(context.Context, string) error {
+		rig.events = append(rig.events, "verify-db")
+		return errors.New("password authentication failed")
+	}
+	err := rig.r.rotate(context.Background(), kindDB, true)
+	if err == nil || !strings.Contains(err.Error(), "password authentication failed") || !strings.Contains(err.Error(), "run the installer again") {
+		t.Fatalf("err = %v, want the refusal and the way back", err)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+		t.Errorf("host config = %q, want it unchanged", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != testPodTOML {
+		t.Errorf("pod config = %q, want it unchanged", got)
+	}
+	if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != testPodTOML {
+		t.Errorf("felis-config = %q, want it unchanged", got)
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db" {
+		t.Errorf("events = %v, want no api roll", rig.events)
+	}
+}
+
+// Everything that can refuse does so before the installer's record or the
+// role change; what the host config alone shows is refused by the plan too.
+func TestRotateDatabasePasswordRefusesEarly(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		apply bool
+		setup func(t *testing.T, rig *rotationRig)
+		want  string
+	}{
+		{"a database the installer does not run", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "deployment = \"felis/felis-postgres\"\n", "", 1), 0o600)
+		}, "[database] deployment is unset"},
+		{"a database url without a role", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "felis:oldpw@", "", 1), 0o600)
+		}, "names no role"},
+		{"a config copy the line editor cannot edit", true, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.podTOML, "[database]\nurl = \"\"\"\npostgres://felis:[email protected]:5432/felis\"\"\"\n", 0o600)
+		}, "set the password in its [database] url by hand"},
+		{"a pod copy that is the host copy", true, func(t *testing.T, rig *rotationRig) {
+			if err := os.Remove(rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.Symlink("felis.host.toml", rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+		}, "resolves to the same file as"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			rig := dbRig(t)
+			tc.setup(t, rig)
+			err := rig.r.rotate(context.Background(), kindDB, tc.apply)
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err = %v, want %q", err, tc.want)
+			}
+			if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n" {
+				t.Errorf("secrets.env = %q, want it untouched", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want nothing done", rig.events)
+			}
+		})
+	}
+}
+
+// The RFC 7677 example (user "user", password "pencil"), with the stored and
+// server keys computed by openssl's PBKDF2 and HMAC rather than this code.
+func TestScramVerifier(t *testing.T) {
+	salt, _ := base64.StdEncoding.DecodeString("W22ZaJ0SNY7soEsUEjb6gQ==")
+	got, err := scramVerifier("pencil", salt, 4096)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := "SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==$WG5d8oPm3OtcPnkdi4Uo7BkeZkBFzpcXkuLmtbsT4qY=:wfPLwcE6nTWhTAmQ7tl2KeoiWGPlZqQxSrmfPwDl2dU="; got != want {
+		t.Errorf("scramVerifier = %q\nwant            %q", got, want)
+	}
+}
+
+func TestAlterRoleSQL(t *testing.T) {
+	if got := alterRoleSQL(`fe"lis`, "SCRAM-SHA-256$4096:c2FsdA==$a:b"); got != "ALTER ROLE \"fe\"\"lis\" WITH PASSWORD 'SCRAM-SHA-256$4096:c2FsdA==$a:b';\n" {
+		t.Errorf("alterRoleSQL = %q", got)
+	}
+}
+
+// installerSecretKeys is every secrets.env key a rotation writes.
+func installerSecretKeys() map[string]bool {
+	keys := map[string]bool{installerForwardingKey: true, installerDBKey: true}
+	for _, k := range installerTokenKeys {
+		keys[k] = true
+	}
+	for _, k := range installerRegistryKeys {
+		keys[k] = true
+	}
+	return keys
+}
+
+// The installer and rotate-token must agree on where each caller's token lives:
+// rotate-token writes installerTokenKeys into secrets.env, and the installer
+// applies those same keys to the Secrets on its next run. A key the installer
+// does not read would be silently reverted by the next upgrade.
+func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	for caller, key := range installerTokenKeys {
+		ct, ok := callerToken(caller)
+		if !ok {
+			t.Fatalf("installerTokenKeys names unknown caller %q", caller)
+		}
+		apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`)
+		if !apply.MatchString(script) {
+			t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key)
+		}
+	}
+	// The replicas the installer applies straight into the workload namespaces, so an
+	// upgrade has them before the new operator and build Jobs reference them.
+	for _, line := range []string{
+		`apply_literal_secret "$MINECRAFT_NS" felis-limbo-token token "$LIMBO_TOKEN"`,
+		`apply_literal_secret "$BUILD_NS" felis-build-token token "$BUILD_TOKEN"`,
+		`kube -n "$MINECRAFT_NS" delete secret felis-service-token --ignore-not-found`,
+		`kube -n "$BUILD_NS" delete secret felis-service-token --ignore-not-found`,
+	} {
+		if !strings.Contains(script, line) {
+			t.Errorf("bootstrap.sh lacks %s", line)
+		}
+	}
+	for _, ns := range []string{"MINECRAFT_NS", "BUILD_NS"} {
+		if strings.Contains(script, `apply_literal_secret "$`+ns+`" felis-service-token`) {
+			t.Errorf("bootstrap.sh still copies the proxy's token into %s", ns)
+		}
+	}
+	if len(installerTokenKeys) != len(callerNames()) {
+		t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames()))
+	}
+}
+
+// Every value the installer keeps in secrets.env has a rotation that rewrites
+// that very key, and every key a rotation writes is one the installer
+// generates, keeps and so re-applies on its next run.
+func TestInstallerSecretsAreAllRotatable(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	m := regexp.MustCompile(`(?s)write_file_atomic "\$SECRETS_ENV" 0600 <<EOF\n(.*?)\nEOF\n`).FindStringSubmatch(script)
+	if m == nil {
+		t.Fatal("bootstrap.sh: no secrets.env heredoc found")
+	}
+	persisted := map[string]bool{}
+	for _, line := range strings.Split(m[1], "\n") {
+		key, value, _ := strings.Cut(line, "=")
+		if value != "${"+key+"}" {
+			t.Errorf("secrets.env line %q is not KEY=${KEY}", line)
+		}
+		persisted[key] = true
+	}
+	rotated := installerSecretKeys()
+	for key := range persisted {
+		if !rotated[key] {
+			t.Errorf("secrets.env keeps %s, which no rotation replaces", key)
+		}
+	}
+	for key := range rotated {
+		if !persisted[key] {
+			t.Errorf("a rotation writes %s, which the installer does not keep in secrets.env", key)
+		}
+		if !regexp.MustCompile(`\n  ` + key + `="\$\{` + key + `:-\$\(openssl rand -hex [0-9]+\)\}"\n`).MatchString(script) {
+			t.Errorf("bootstrap.sh does not generate %s when secrets.env lacks it", key)
+		}
+	}
+}

cmd/felis/run.go

@@ -11,19 +11,41 @@ Usage:
   felis <command> [flags]
 
 Commands:
-  migrate up        Apply embedded database migrations under an advisory lock
+  migrate up        Apply embedded database migrations under an advisory lock (snapshots the database first)
+  db                Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check)
+  offsite           Copy world archives, database bundles, user images and uploads to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|fetch-images|fetch-uploads|keygen)
   operator          Run the MinecraftServer controller-manager
   api               Run the felis-api HTTP server
   nano              Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)
   reaper            Run the world reaper / backup batch
   restore           Extract a world archive into a world volume (internal Job entrypoint)
   backup            Archive a world into the backup store and record it (internal Job entrypoint)
-  files             List/read/write one file in a stopped server's world (internal Job entrypoint)
+  backup-now        Archive every user server's world now, one at a time (or the named ones; -stop stops running ones first; prints the plan, -yes applies; requires root/sudo)
+  files             List, read, write, mkdir, delete, rename, upload or unzip one path in a stopped server's world (internal Job entrypoint)
+  export            Archive a stopped server's world, or read one of its backups, and hand it to felis-api for download (internal Job entrypoint)
+  egress-gate       Hold a build or game server pod until its egress NetworkPolicy is enforced (internal init container entrypoint)
+  fetch-context     Fetch and extract a submission's build context (internal Job entrypoint)
+  scan-gate         Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
+  push-image        Push a scanned image tarball to the registry (internal Job entrypoint)
+  mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
+  server-migrate    Move a stopped world between approved nodes (start|status|retry; requires root)
+  node              Join and approve trusted daemon nodes (list|token|join|approve|firewall; requires root)
+  node-control      Run the host node-task service on an API-only Unix socket (requires root)
+  node-probe        Verify reachability and observed sources (internal admission probe)
+  archive-serve     Serve scoped one-use archive transfers on the controller (internal entrypoint)
+  registry-gate     Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
   manifests         Render the control-plane RBAC + NetworkPolicy install bundle as YAML
   apply             Create a MinecraftServer CRD (direct K8s write; use -f server.json)
   setup             Run host bootstrap + first-run setup console (TUI; requires root/sudo)
+  converge          Fill in fields a newer desired spec added to already-installed system servers
+  rotate-token      Replace a generated credential and restart what reads it (velocity|limbo|build|ops|registry|forwarding|db; prints the plan, -yes applies; requires root/sudo)
+  domain            Move the install to a new root domain on every surface that carries it, or check each one (set|check; requires root/sudo)
+  status            Print the platform at a glance: node, control plane, proxy, servers, backups, host, open alerts (requires root/sudo)
+  doctor            Run every health check once, grouped by area, with where to look next; mails nothing (requires root/sudo)
+  support-bundle    Collect status, doctor, logs and cluster state into one redacted tar.gz to share when asking for help (requires root/sudo)
+  watchdog          Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
   version           Print the build stamp of this binary
-  update            Report which platform components have updates available
+  update            Check platform versions; --apply installs a reviewed target
   breakGlass        Open the local break-glass emergency console (TUI; requires root/sudo)
 
 Run "felis <command> -h" for command-specific flags.
@@ -39,22 +61,47 @@ Run "felis <command> -h" for command-specific flags.
 // The help aliases are deliberately NOT entries: they print usage rather than run a
 // subcommand, and listing them would make the table disagree with the command list.
 var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
-	"migrate":          cmdMigrate,
-	"operator":         cmdOperator,
-	"api":              cmdAPI,
-	"nano":             cmdNano,
-	"reaper":           cmdReaper,
-	"restore":          cmdRestore,
-	"backup":           cmdBackup,
-	"files":            cmdFiles,
-	"manifests":        cmdManifests,
-	"apply":            cmdApply,
-	"setup":            cmdSetup,
-	"breakGlass":       cmdBreakGlass,
-	"bootstrap-assets": cmdBootstrapAssets,
-	"init-forwarding":  cmdInitForwarding,
-	"version":          cmdVersion,
-	"update":           cmdUpdate,
+	"migrate":            cmdMigrate,
+	"db":                 cmdDB,
+	"offsite":            cmdOffsite,
+	"operator":           cmdOperator,
+	"api":                cmdAPI,
+	"nano":               cmdNano,
+	"reaper":             cmdReaper,
+	"restore":            cmdRestore,
+	"backup":             cmdBackup,
+	"backup-now":         cmdBackupNow,
+	"files":              cmdFiles,
+	"export":             cmdExport,
+	"egress-gate":        cmdEgressGate,
+	"fetch-context":      cmdFetchContext,
+	"scan-gate":          cmdScanGate,
+	"push-image":         cmdPushImage,
+	"mirror-build-tools": cmdMirrorBuildTools,
+	"registry-gate":      cmdRegistryGate,
+	"archive-serve":      cmdArchiveServe,
+	"node":               cmdNode,
+	"node-control":       cmdNodeControl,
+	"server-migrate":     cmdServerMigrate,
+	"node-probe":         cmdNodeProbe,
+	"manifests":          cmdManifests,
+	"apply":              cmdApply,
+	"setup":              cmdSetup,
+	"converge":           cmdConverge,
+	"rotate-token":       cmdRotateToken,
+	"domain":             cmdDomain,
+	"breakGlass":         cmdBreakGlass,
+	"bootstrap-assets":   cmdBootstrapAssets,
+	"init-forwarding":    cmdInitForwarding,
+	"init-volume":        cmdInitVolume,
+	"pin-images":         cmdPinImages,
+	"image-bundle":       cmdImageBundle,
+	"version":            cmdVersion,
+	"update":             cmdUpdate,
+	"watchdog":           cmdWatchdog,
+	"status":             cmdStatus,
+	"doctor":             cmdDoctor,
+	"support-bundle":     cmdSupportBundle,
 }
 
 // run dispatches a subcommand. It is separate from main so the router is

cmd/felis/run_test.go

@@ -5,8 +5,27 @@ import (
 	"os"
 	"strings"
 	"testing"
+
+	"github.com/BurntSushi/toml"
+
+	"felis.lolicon.best/internal/config"
 )
 
+func TestBootstrapConfigKeys(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"bootstrap-assets", "config-keys"}, &out, &errBuf); code != 0 {
+		t.Fatalf("exit code = %d: %s", code, errBuf.String())
+	}
+	var cfg config.Config
+	meta, err := toml.Decode(strings.TrimSpace(out.String())+` = "26.3"`, &cfg)
+	if err != nil || len(meta.Undecoded()) != 0 {
+		t.Fatalf("advertised config key is unsupported: %v, undecoded: %v", err, meta.Undecoded())
+	}
+	if cfg.Velocity.GameVersion != "26.3" {
+		t.Fatalf("advertised key did not set the game version: %q", cfg.Velocity.GameVersion)
+	}
+}
+
 func TestRunNoArgsPrintsUsage(t *testing.T) {
 	var out, errBuf bytes.Buffer
 	if code := run(nil, &out, &errBuf); code != 2 {
@@ -38,9 +57,13 @@ func TestRunUnknownCommand(t *testing.T) {
 }
 
 // undocumentedCommands are routable on purpose but kept out of the usage text: they
-// are called by deploy/bootstrap.sh, not by a human at a prompt. Listing them here is
-// what makes their absence from usage a deliberate decision rather than an oversight.
-var undocumentedCommands = map[string]bool{"bootstrap-assets": true, "init-forwarding": true}
+// are called by deploy/bootstrap.sh or the operator's initContainers, not by a human
+// at a prompt. Listing them here is what makes their absence from usage a deliberate
+// decision rather than an oversight.
+var undocumentedCommands = map[string]bool{
+	"bootstrap-assets": true, "init-forwarding": true, "init-volume": true,
+	"pin-images": true, "image-bundle": true,
+}
 
 // The usage text and the dispatch table must describe the same set of commands.
 //

cmd/felis/scangate.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"strings"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// maxScanDocument bounds each document scan-gate reads: a modpack report lists a
+// few thousand packages, far below this. Tests shrink it.
+var maxScanDocument int64 = 64 << 20
+
+// cmdScanGate is the build pod's verdict step, after trivy wrote its full JSON
+// report and trivy convert wrote the CycloneDX SBOM. It applies the scan policy
+// to the report, prints the verdict and every blocking finding, then appends the
+// verdict, the report and the SBOM to its log as the envelope felis-api keeps on
+// the build (build.WriteScanEnvelope). It exits 1 when a finding blocks, which
+// fails the pod before the push step runs, and 2 when the report cannot be read,
+// so a scan that produced nothing usable never admits an image.
+func cmdScanGate(args []string, stdout, stderr io.Writer) int {
+	fset := flag.NewFlagSet("scan-gate", flag.ContinueOnError)
+	fset.SetOutput(stderr)
+	reportPath := fset.String("report", "", "trivy JSON report (required)")
+	sbomPath := fset.String("sbom", "", "CycloneDX SBOM to keep with the report")
+	failOn := fset.String("fail-on", strings.Join(build.DefaultScanFailOn, ","), "comma-separated severities that block the image")
+	failUnfixed := fset.Bool("fail-unfixed", false, "block on vulnerabilities that have no fixed release too")
+	accept := fset.String("accept", "", "comma-separated vulnerability ids and secret rule ids that never block")
+	termLog := fset.String("termination-log", "/dev/termination-log", "where the one-line verdict goes for the pod status")
+	if err := fset.Parse(args); err != nil {
+		return 2
+	}
+	// A stray argument is a policy the gate would otherwise drop without a word
+	// (a comma split out of --fail-on, say).
+	if fset.NArg() > 0 {
+		fmt.Fprintf(stderr, "felis scan-gate: unexpected argument %q\n", fset.Arg(0))
+		return 2
+	}
+	sevs, err := build.ParseSeverities(*failOn)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --fail-on: %v\n", err)
+		return 2
+	}
+	accepted, err := build.ParseScanAccept(*accept)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --accept: %v\n", err)
+		return 2
+	}
+	if *reportPath == "" {
+		fmt.Fprintln(stderr, "felis scan-gate: --report is required")
+		return 2
+	}
+	fail := func(msg string) int {
+		fmt.Fprintln(stderr, "felis scan-gate: "+msg)
+		writeTerminationLog(*termLog, msg)
+		return 2
+	}
+	report, err := readScanDocument(*reportPath)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	policy := build.ScanPolicy{FailOn: sevs, FailUnfixed: *failUnfixed, Accept: accepted}
+	summary, err := build.Summarize(report, policy)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	env := build.ScanEnvelope{Summary: summary, Report: report}
+	if *sbomPath != "" {
+		switch sbom, err := readScanDocument(*sbomPath); {
+		case err == nil:
+			env.SBOM = sbom
+		case errors.Is(err, fs.ErrNotExist):
+			fmt.Fprintf(stdout, "felis scan-gate: no SBOM at %s; keeping the report alone\n", *sbomPath)
+		default:
+			return fail("the SBOM is unreadable: " + err.Error())
+		}
+	}
+
+	printScanVerdict(stdout, summary)
+	written, err := build.WriteScanEnvelope(stdout, env)
+	if err != nil {
+		return fail("could not write the scan envelope: " + err.Error())
+	}
+	for _, doc := range written.Summary.Omitted {
+		fmt.Fprintf(stderr, "felis scan-gate: the %s is too large to keep with the build and was left out\n", doc)
+	}
+	if summary.Blocked {
+		writeTerminationLog(*termLog, summary.Reason())
+		return 1
+	}
+	writeTerminationLog(*termLog, "the scan passed")
+	return 0
+}
+
+// printScanVerdict writes the human half of scan-gate's log.
+func printScanVerdict(w io.Writer, s build.ScanSummary) {
+	var counts []string
+	for _, sev := range build.Severities {
+		counts = append(counts, fmt.Sprintf("%s %d", sev, s.Counts[sev]))
+	}
+	fmt.Fprintf(w, "felis scan-gate: %d packages; findings: %s\n", s.Packages, strings.Join(counts, ", "))
+	unfixed := "vulnerabilities with no fixed release do not block"
+	if s.Policy.FailUnfixed {
+		unfixed = "vulnerabilities with no fixed release block too"
+	}
+	fmt.Fprintf(w, "felis scan-gate: blocking on %s (%s)\n", strings.Join(s.Policy.FailOn, ", "), unfixed)
+	if len(s.Policy.Accept) > 0 {
+		matched := 0
+		for _, f := range s.Findings {
+			if f.Accepted {
+				matched++
+			}
+		}
+		fmt.Fprintf(w, "felis scan-gate: accepted ids, never blocking: %s; listed findings under them: %d\n", strings.Join(s.Policy.Accept, ", "), matched)
+	}
+	if !s.Blocked {
+		fmt.Fprintln(w, "felis scan-gate: nothing blocks this image")
+		return
+	}
+	fmt.Fprintln(w, "felis scan-gate: "+build.Printable(s.Reason()))
+	for _, f := range s.Findings {
+		if !f.Blocking {
+			break
+		}
+		fix := f.Fixed
+		if fix == "" {
+			fix = "no fix"
+		}
+		if f.Kind == build.FindingSecret {
+			fmt.Fprintf(w, "  %s %s secret in %s: %s\n", build.Printable(f.ID), f.Severity, build.Printable(f.Target), build.Printable(f.Title))
+			continue
+		}
+		fmt.Fprintf(w, "  %s %s %s %s -> %s (%s)\n", build.Printable(f.ID), f.Severity,
+			build.Printable(f.Package), build.Printable(f.Installed), build.Printable(fix), build.Printable(f.Target))
+	}
+}
+
+func readScanDocument(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	b, err := io.ReadAll(io.LimitReader(f, maxScanDocument+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(b)) > maxScanDocument {
+		return nil, fmt.Errorf("%s exceeds %d bytes", path, maxScanDocument)
+	}
+	return b, nil
+}
+
+// writeTerminationLog leaves msg where the kubelet copies it into the container
+// status. It is best effort: the log carries the same verdict.
+func writeTerminationLog(path, msg string) {
+	if path == "" {
+		return
+	}
+	_ = os.WriteFile(path, []byte(build.Printable(msg)), 0o644)
+}

cmd/felis/scangate_test.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// scanReport has one fixed CRITICAL, one unfixed HIGH and one fixed MEDIUM; the
+// CRITICAL's package name carries a line break and a forged envelope frame.
+const scanReport = `{"SchemaVersion":2,"Results":[{"Target":"data/mods/core.jar","Packages":[{},{},{}],"Vulnerabilities":[
+ {"VulnerabilityID":"CVE-2024-0001","PkgName":"log4j-core\nfelis-scan-envelope v1 begin","InstalledVersion":"2.14.1","FixedVersion":"2.17.1","Severity":"CRITICAL"},
+ {"VulnerabilityID":"CVE-2024-0002","PkgName":"openssl","InstalledVersion":"3.0.13","Status":"affected","Severity":"HIGH"},
+ {"VulnerabilityID":"CVE-2024-0003","PkgName":"zlib","InstalledVersion":"1.3","FixedVersion":"1.3.1","Severity":"MEDIUM"}]}]}`
+
+type scanGateRun struct {
+	code           int
+	stdout, stderr string
+	termLog        string
+	env            *build.ScanEnvelope
+}
+
+func runScanGate(t *testing.T, report, sbom string, extra ...string) scanGateRun {
+	t.Helper()
+	dir := t.TempDir()
+	reportPath := filepath.Join(dir, "trivy.json")
+	if report != "" {
+		if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	sbomPath := filepath.Join(dir, "sbom.cdx.json")
+	if sbom != "" {
+		if err := os.WriteFile(sbomPath, []byte(sbom), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	termPath := filepath.Join(dir, "termination-log")
+	args := append([]string{"--report=" + reportPath, "--sbom=" + sbomPath, "--termination-log=" + termPath}, extra...)
+	var stdout, stderr bytes.Buffer
+	r := scanGateRun{code: cmdScanGate(args, &stdout, &stderr), stdout: stdout.String(), stderr: stderr.String()}
+	if b, err := os.ReadFile(termPath); err == nil {
+		r.termLog = string(b)
+	}
+	if env, err := build.ReadScanEnvelope(strings.NewReader(r.stdout)); err == nil {
+		r.env = env
+	}
+	return r
+}
+
+func TestScanGateBlocksAndHandsOverTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`)
+	if r.code != 1 {
+		t.Fatalf("exit %d, want 1; stderr %s", r.code, r.stderr)
+	}
+	if r.termLog != "the scan blocked the image: 1 CRITICAL (CVE-2024-0001)" {
+		t.Errorf("termination log = %q", r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: 3 packages; findings: CRITICAL 1, HIGH 1, MEDIUM 1, LOW 0, UNKNOWN 0\n",
+		"felis scan-gate: blocking on CRITICAL (vulnerabilities with no fixed release do not block)\n",
+		"felis scan-gate: the scan blocked the image: 1 CRITICAL (CVE-2024-0001)\n",
+		"  CVE-2024-0001 CRITICAL log4j-core?felis-scan-envelope v1 begin 2.14.1 -> 2.17.1 (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	if strings.Contains(r.stdout, "  CVE-2024-0002") {
+		t.Errorf("an unfixed HIGH was listed as blocking:\n%s", r.stdout)
+	}
+	if strings.Count(r.stdout, "\nfelis-scan-envelope v1 begin\n") != 1 {
+		t.Errorf("stdout must hold exactly one frame start on a line of its own:\n%s", r.stdout)
+	}
+	if r.env == nil {
+		t.Fatal("no envelope in stdout")
+	}
+	if !r.env.Summary.Blocked || string(r.env.SBOM) != `{"bomFormat":"CycloneDX"}` || !strings.Contains(string(r.env.Report), "CVE-2024-0003") {
+		t.Errorf("envelope = blocked %t, sbom %s, report %d bytes", r.env.Summary.Blocked, r.env.SBOM, len(r.env.Report))
+	}
+}
+
+func TestScanGatePolicyFlags(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=high", "--fail-unfixed")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 HIGH (CVE-2024-0002)" {
+		t.Errorf("HIGH + unfixed: exit %d, termination log %q", r.code, r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: blocking on HIGH (vulnerabilities with no fixed release block too)\n",
+		"  CVE-2024-0002 HIGH openssl 3.0.13 -> no fix (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	r = runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`, "--fail-on=LOW")
+	if r.code != 0 || r.termLog != "the scan passed" || !strings.Contains(r.stdout, "felis scan-gate: nothing blocks this image\n") {
+		t.Errorf("LOW only: exit %d, termination log %q, stdout:\n%s", r.code, r.termLog, r.stdout)
+	}
+	if r.env == nil || r.env.Summary.Blocked || r.env.SBOM == nil {
+		t.Errorf("a passing scan must still hand over its report and SBOM: %+v", r.env)
+	}
+}
+
+func TestScanGateAcceptedIDsNeverBlock(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=CRITICAL,MEDIUM", "--accept=CVE-2024-0001, CVE-2024-0002,CVE-2099-0001")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 MEDIUM (CVE-2024-0003)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "felis scan-gate: accepted ids, never blocking: CVE-2024-0001, CVE-2024-0002, CVE-2099-0001; listed findings under them: 2\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+	if r.env == nil || strings.Join(r.env.Summary.Policy.Accept, ",") != "CVE-2024-0001,CVE-2024-0002,CVE-2099-0001" {
+		t.Fatalf("envelope = %+v", r.env)
+	}
+	for _, f := range r.env.Summary.Findings {
+		if f.ID == "CVE-2024-0001" && (f.Blocking || !f.Accepted) {
+			t.Errorf("accepted finding = %+v", f)
+		}
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001")
+	if r.code != 0 || r.termLog != "the scan passed" {
+		t.Errorf("only an accepted CRITICAL: exit %d, termination log %q", r.code, r.termLog)
+	}
+}
+
+func TestScanGateWithoutAnSBOMKeepsTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=LOW")
+	if r.code != 0 || !strings.Contains(r.stdout, "felis scan-gate: no SBOM at ") {
+		t.Errorf("exit %d, stdout:\n%s", r.code, r.stdout)
+	}
+	if r.env == nil || r.env.SBOM != nil || r.env.Report == nil {
+		t.Errorf("envelope = %+v", r.env)
+	}
+}
+
+func TestScanGateListsABlockingSecret(t *testing.T) {
+	r := runScanGate(t, `{"SchemaVersion":2,"Results":[{"Target":"config/keys.txt","Secrets":[
+ {"RuleID":"aws-access-key-id","Severity":"CRITICAL","Title":"AWS Access\nKey ID"}]}]}`, "")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 CRITICAL (aws-access-key-id)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "  aws-access-key-id CRITICAL secret in config/keys.txt: AWS Access?Key ID\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+}
+
+func TestScanGateFailsClosed(t *testing.T) {
+	r := runScanGate(t, "", "")
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the scan report is unreadable: open ") || r.env != nil {
+		t.Errorf("missing report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	r = runScanGate(t, `{"SchemaVersion":1}`, "")
+	if r.code != 2 || r.termLog != "the scan report is unreadable: read trivy report: schema version 1, want 2" {
+		t.Errorf("old schema: exit %d, termination log %q", r.code, r.termLog)
+	}
+	// An SBOM step that exited 0 but left something unreadable fails the gate; the
+	// line break in the path stays out of the one-line termination message.
+	sbomDir := filepath.Join(t.TempDir(), "sb\nom")
+	if err := os.Mkdir(sbomDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	r = runScanGate(t, scanReport, "", "--sbom="+sbomDir)
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the SBOM is unreadable: read ") ||
+		!strings.HasSuffix(r.termLog, "/sb?om: is a directory") || r.env != nil {
+		t.Errorf("unreadable SBOM: exit %d, termination log %q", r.code, r.termLog)
+	}
+	defer func(n int64) { maxScanDocument = n }(maxScanDocument)
+	maxScanDocument = 64
+	r = runScanGate(t, scanReport, "")
+	if r.code != 2 || !strings.HasSuffix(r.termLog, "/trivy.json exceeds 64 bytes") || r.env != nil {
+		t.Errorf("oversized report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	maxScanDocument = 64 << 20
+	r = runScanGate(t, scanReport, "", "--fail-on=SEVERE")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --fail-on: unknown severity "SEVERE"`) {
+		t.Errorf("bad severity: exit %d, stderr %q", r.code, r.stderr)
+	}
+	// A severity list split at its comma leaves a stray argument, not a
+	// narrower policy.
+	r = runScanGate(t, scanReport, "", "--fail-on=LOW", "CRITICAL")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: unexpected argument "CRITICAL"`) || r.env != nil {
+		t.Errorf("stray argument: exit %d, stderr %q", r.code, r.stderr)
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001 CVE-2024-0003")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --accept: "CVE-2024-0001 CVE-2024-0003" is not a vulnerability id or secret rule id`) {
+		t.Errorf("bad accept list: exit %d, stderr %q", r.code, r.stderr)
+	}
+	var stdout, stderr bytes.Buffer
+	if code := cmdScanGate(nil, &stdout, &stderr); code != 2 || !strings.Contains(stderr.String(), "--report is required") {
+		t.Errorf("no report flag: exit %d, stderr %q", code, stderr.String())
+	}
+}

cmd/felis/server_migrate.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/placement"
+	"felis.lolicon.best/internal/platform"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func cmdServerMigrate(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprintln(stderr, "server-migrate: start | status | retry (separate from database migrate)")
+		return 2
+	}
+	fs := flag.NewFlagSet("server-migrate "+args[0], flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	name := fs.String("name", "", "stopped user server")
+	target := fs.String("target-node", "", "approved target worker")
+	id := fs.String("id", "", "operation id (required for retry)")
+	kube := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local kubeconfig")
+	ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
+	cfgPath := fs.String("config", "/var/lib/felis/felis.host.toml", "host config used to record the current owner on the safety backup")
+	if err := fs.Parse(args[1:]); err != nil {
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "server-migrate requires root/sudo")
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	cfg, err := clientcmd.BuildConfigFromFlags("", *kube)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	scheme := runtime.NewScheme()
+	clientgoscheme.AddToScheme(scheme)
+	v1alpha1.AddToScheme(scheme)
+	cl, err := client.New(cfg, client.Options{Scheme: scheme})
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	m := &distributed.Manager{Client: cl, Namespace: *ns, Resolve: placement.Resolve(cl, *ns)}
+	var nodes corev1.NodeList
+	if err = cl.List(ctx, &nodes); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	for _, n := range nodes.Items {
+		if n.Labels[placement.LabelRole] == placement.RoleController {
+			m.Controller = n.Name
+		}
+	}
+	if m.Controller == "" {
+		fmt.Fprintln(stderr, "distributed controller identity is not configured")
+		return 1
+	}
+	m.Resolve = placement.Resolve(cl, *ns, m.Controller)
+	var op distributed.Operation
+	switch args[0] {
+	case "start":
+		host, cfgErr := config.Load(*cfgPath)
+		if cfgErr != nil {
+			fmt.Fprintln(stderr, cfgErr)
+			return 1
+		}
+		drv, dbErr := openPodStore(ctx, host.Database.URL, "migration", stderr)
+		if dbErr != nil {
+			fmt.Fprintln(stderr, dbErr)
+			return 1
+		}
+		defer drv.Close()
+		var owner sql.NullString
+		if err := drv.DB().QueryRowContext(ctx, "SELECT owner_id FROM servers WHERE name=$1 AND deleted_at IS NULL AND retire_requested_at IS NULL", *name).Scan(&owner); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		op, err = m.BeginMigration(ctx, *name, *target, owner.String)
+	case "status":
+		op, err = m.Migration(ctx, *name, *id)
+	case "retry":
+		if *id == "" {
+			fmt.Fprintln(stderr, "retry requires --id")
+			return 2
+		}
+		op, err = m.RetryMigration(ctx, *name, *id)
+	default:
+		fmt.Fprintln(stderr, "unknown migration operation")
+		return 2
+	}
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	json.NewEncoder(stdout).Encode(op)
+	return 0
+}

cmd/felis/setup.go

@@ -11,9 +11,7 @@ import (
 	"time"
 
 	"felis.lolicon.best/internal/api"
-	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/store"
 )
@@ -101,19 +99,7 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
 	}
 	defer setup.drv.Close()
 
-	// The wizard's first screen asks the operator to join the server and run /link:
-	// the Owner IS the Minecraft account, so the login gate must be UP before we ask
-	// for a link code. This used to run after the wizard, which is why setup asked
-	// for a code from a server that had never been started. On a re-run the Owner
-	// already exists, so provisioning stays best-effort and never blocks the
-	// operator from reaching the status screen.
-	if err := provisionSystemServers(ctx, setup.cfg, stdout, !setup.adminExists); err != nil {
-		fmt.Fprintf(stderr, "felis setup: %v\n", err)
-		fmt.Fprintln(stderr, "The Owner is bound by joining the login gate in-game, so setup cannot continue without it.")
-		return 1
-	}
-
-	res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database.URL, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
+	res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis setup: %v\n", err)
 		return 1
@@ -122,34 +108,50 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
 	if panelURL == "" {
 		panelURL = localPanelURL(setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname)
 	}
+	// Game services are provisioned independently. A failed login/lobby must not
+	// stop the host administrator from opening the panel to diagnose it.
+	if err := provisionSystemServers(ctx, setup.cfg, stdout); err != nil {
+		fmt.Fprintf(stdout, "felis setup: Minecraft services need attention: %v\n", err)
+	}
+
+	reportSetupResult(stdout, res, bootstrapped, setup.adminExists, panelURL)
+	return 0
+}
+
+// reportSetupResult preserves the browser handoff after the TUI closes.
+func reportSetupResult(stdout io.Writer, res breakGlassResult, bootstrapped, adminExisted bool, panelURL string) {
+	if !adminExisted && !res.provisioned {
+		defer fmt.Fprintln(stdout, "\nOwner login is unfinished. Run sudo felis setup again to get a panel setup link; Minecraft is not required.")
+	}
 
 	if !res.provisioned && !res.connectConfigured {
 		if bootstrapped {
-			fmt.Fprintln(stdout, "felis setup: host bootstrap completed; Owner/connection setup skipped.")
+			fmt.Fprintln(stdout, "felis setup: host bootstrap completed; panel/connection setup unfinished.")
 			if panelURL != "" {
 				fmt.Fprintf(stdout, "Panel: %s\n", panelURL)
 				fmt.Fprintln(stdout, "The local HTTPS certificate is self-signed; your browser may ask for confirmation on first visit.")
 			}
-			return 0
+			return
 		}
 		// A re-run lands on the status screen, which changes nothing by design —
 		// reporting that as "cancelled" reads as a failure the operator did not cause.
 		msg := "felis setup: cancelled — no changes made."
-		if res.alreadySetUp {
+		switch {
+		case res.alreadySetUp:
 			msg = "felis setup: already set up — nothing to change."
 		}
 		fmt.Fprintln(stdout, msg)
 		if panelURL != "" {
 			fmt.Fprintf(stdout, "Panel: %s\n", panelURL)
 		}
-		return 0
+		return
 	}
 
 	if res.provisioned {
 		fmt.Fprintf(stdout, "\nfelis setup: Owner account %q provisioned (passwordless).\n", res.username)
 		fmt.Fprintf(stdout, "Recorded as %q (mode: %s, os user: %s).\n", res.accountable, res.mode, res.osUser)
 		if res.setupTokenURL != "" {
-			fmt.Fprintf(stdout, "Open this URL to complete passwordless login setup (verify email / enroll passkey):\n\n    %s\n\n", res.setupTokenURL)
+			fmt.Fprintf(stdout, "Open this URL to record your email and create a passkey (Minecraft is optional):\n\n    %s\n\n", res.setupTokenURL)
 		}
 		if res.auditWarning != "" {
 			fmt.Fprintf(stdout, "WARNING: the accountability audit row was NOT written: %s\n", res.auditWarning)
@@ -177,28 +179,15 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
 			fmt.Fprintln(stdout, "Felis config, Kubernetes Secret and API rollout were updated.")
 		}
 	}
-
-	return 0
 }
 
 // provisionSystemServers ensures the login limbo and lobby system services exist,
 // then prints the login-first Velocity wiring. deploy/bootstrap.sh writes this
 // configuration for its host proxy; operators only need to mirror it when they
 // deliberately run Velocity elsewhere.
-//
-// required is set on a first run, where the next screen asks the operator to join
-// the server and run /link. There a gate that never comes up is not a degraded
-// install, it is an impossible one — so every soft landing below becomes a hard
-// error and we block until the gate reports Ready. On a re-run the Owner already
-// exists and nothing downstream needs the gate, so unconfigured images or an
-// unreachable cluster degrade to printed guidance exactly as before.
-func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writer, required bool) error {
-	// fail is the one place the two modes diverge: fatal on a first run, guidance
-	// on a re-run.
+
+func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writer) error {
 	fail := func(format string, args ...any) error {
-		if required {
-			return fmt.Errorf(format, args...)
-		}
 		fmt.Fprintf(out, "\nfelis setup: "+format+"\n", args...)
 		return nil
 	}
@@ -206,41 +195,36 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 		return fail("login/lobby system servers NOT provisioned — set [velocity] login_image " +
 			"and lobby_image in felis.toml (build them from deploy/limbo and deploy/lobby), then re-run `sudo felis setup`")
 	}
-	if required && cfg.Velocity.LoginImage == "" {
-		return errors.New("the Owner binds by joining the login gate, but [velocity] login_image is not set in felis.toml " +
-			"(build it from deploy/limbo), then re-run `sudo felis setup`")
-	}
 	cl, err := buildSystemServerClient()
 	if err != nil {
 		return fail("could not reach the cluster to provision the login/lobby system servers: %v\n"+
 			"Re-run `sudo felis setup` on the control-plane host once the cluster is reachable", err)
 	}
 	// The login limbo authenticates to the felis-api INTERNAL face, so it needs the
-	// internal base URL, the root domain (to link players at the console), and the
-	// service token. The first two are plain env baked into the pod here; the token
-	// is a Secret the operator injects by reference — but a secretKeyRef is
-	// namespace-local, so first replicate the token Secret from the control namespace
-	// into the minecraft namespace where the login pod runs. The control namespace is
+	// internal base URL, the root domain (to link players at the console), and its
+	// own token (felis-limbo-token). The first two are plain env baked into the pod
+	// here; the token is a Secret the operator injects by reference — but a
+	// secretKeyRef is namespace-local, so first replicate the token Secret from the
+	// control namespace into the minecraft namespace where the login pod runs. The control namespace is
 	// the platform default (there is no felis.toml override for it); a deployment that
 	// renamed it must replicate the Secret by hand.
 	controlNS := platform.DefaultControlNamespace
 	apiBaseURL := platform.InternalAPIBaseURL(controlNS)
 	// These Secrets must land in the minecraft namespace before the pods that
-	// mount them are created: the service token (login authenticates to felis-api
-	// with it), the Velocity forwarding secret (every backend verifies the proxy's
+	// mount them are created: the login gate's token (login authenticates to
+	// felis-api with it), the Velocity forwarding secret (every backend verifies the proxy's
 	// signed handshake with it — without it the login gate would derive an OFFLINE
-	// UUID and the Owner would bind the wrong Minecraft identity), and felis-config
+	// UUID and players would bind the wrong Minecraft identity), and felis-config
 	// (the on-demand BACKUP Job runs in the minecraft namespace and mounts it to
 	// self-record its world_backups row; without the replica the Job's volume
 	// mount fails and every backup request strands in the cluster).
-	secretOutcomes := []systemServerOutcome{
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token"),
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret"),
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			"felis-config", "felis.toml", "config"),
+	// An empty build_namespace means the build system's compiled-in default; the
+	// replica must target the namespace the Jobs actually run in.
+	buildNS := cfg.Registry.BuildNamespace
+	if buildNS == "" {
+		buildNS = platform.DefaultBuildNamespace
 	}
+	secretOutcomes := provisionSecretReplicas(ctx, cl, controlNS, cfg.K8s.Namespace, buildNS)
 	outcomes := ensureSystemServers(ctx, cl, cfg.K8s.Namespace, cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, apiBaseURL, cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
 	outcomes = append(secretOutcomes, outcomes...)
 	fmt.Fprintln(out, "\nfelis setup: login/lobby system servers (always-on, reaper-exempt):")
@@ -250,23 +234,12 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 			fmt.Fprintf(out, "  - %s: ERROR %v\n", o.name, o.err)
 		case o.created:
 			fmt.Fprintf(out, "  - %s: created (DesiredState=Running)\n", o.name)
+		case o.updated:
+			fmt.Fprintf(out, "  - %s: refreshed from the control namespace\n", o.name)
 		default:
 			fmt.Fprintf(out, "  - %s: skipped (%s)\n", o.name, o.skipped)
 		}
 	}
-	if required {
-		if err := requiredProvisioningError(outcomes); err != nil {
-			return fmt.Errorf("required Minecraft provisioning failed: %w", err)
-		}
-		fmt.Fprintln(out, "\nfelis setup: waiting for the login gate to accept players…")
-		err := awaitLoginGateReady(ctx, cl, cfg.K8s.Namespace, loginGateReadyTimeout, loginGatePollInterval, func(p v1alpha1.Phase) {
-			fmt.Fprintf(out, "  login: %s\n", phaseOrPending(p))
-		})
-		if err != nil {
-			return err
-		}
-		fmt.Fprintln(out, "  login: Ready")
-	}
 	printVelocityWiringGuidance(out, cfg.Server.RootDomain)
 	return nil
 }
@@ -320,7 +293,9 @@ func openConfiguredSetup(ctx context.Context, cfgPath string) (*configuredSetup,
 	if err != nil {
 		return nil, &setupOpenError{stage: "load config", err: err}
 	}
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Pending migrations are the preflight's to apply; a newer schema is a rolled-back
+	// binary, and nothing this console writes would match it.
+	drv, err := openStore(ctx, cfg.Database.URL, true)
 	if err != nil {
 		return nil, &setupOpenError{stage: "open database", err: err}
 	}

cmd/felis/setup_panel.go

@@ -33,9 +33,6 @@ func setupPanelNodePort() int {
 }
 
 func localPanelURL(rootDomain, adminHostname string) string {
-	if ip := rootDomainEmbeddedIP(rootDomain); ip != "" {
-		return fmt.Sprintf("https://%s:%d", ip, setupPanelNodePort())
-	}
 	host := defaultAdminHostname(rootDomain, adminHostname)
 	if host == "" {
 		return ""
@@ -57,6 +54,24 @@ func rootDomainEmbeddedIP(rootDomain string) string {
 	return ""
 }
 
+// setupGameAddress is where players join Minecraft: the
+// IP a nip.io or sslip.io root domain spells out (nothing to resolve), otherwise the
+// root domain, with the port when it is not Minecraft's default. The proxy lands
+// every fresh connection on the login server whatever name it was dialled by.
+func setupGameAddress(rootDomain string, gamePort int) string {
+	host := rootDomainEmbeddedIP(rootDomain)
+	if host == "" {
+		host = strings.TrimSpace(strings.TrimSuffix(rootDomain, "."))
+	}
+	if host == "" {
+		return ""
+	}
+	if gamePort == 0 || gamePort == 25565 {
+		return host
+	}
+	return net.JoinHostPort(host, strconv.Itoa(gamePort))
+}
+
 func localPanelOrigin() string {
 	return fmt.Sprintf("https://127.0.0.1:%d", setupPanelNodePort())
 }

cmd/felis/status.go

@@ -0,0 +1,438 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+	"text/tabwriter"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// cmdStatus prints the platform at a glance: the release and the node, the
+// control plane's workloads, the game proxy, every server with its players
+// and newest world backup, the database backups and the off-site copy, the
+// host's disks and memory, and what the watchdog has open. It changes
+// nothing, and a part that is down (the cluster, PostgreSQL) reads as such
+// while the rest still prints. felis doctor says what is wrong and where to
+// look.
+func cmdStatus(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("status", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis status: run as root (sudo felis status): it reads root-only state under /etc/felis and /var/lib/felis")
+		return 1
+	}
+	env, err := hostStatusEnv(*unitDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis status: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	printStatus(ctx, env, stdout)
+	return 0
+}
+
+// statusEnv is what one status report reads the host through.
+type statusEnv struct {
+	cfg *config.Config
+	// w is how felis-watchdog.service runs the watchdog: where the backups,
+	// the off-site record and the proxy are.
+	w       watchdogFlags
+	cl      client.Client // nil while the API server is unreachable
+	clErr   error
+	backups func(ctx context.Context) (map[string]time.Time, error)
+	run     func(ctx context.Context, name string, args ...string) ([]byte, error)
+	unitDir string
+	meminfo string
+	host    string
+	now     time.Time
+}
+
+// hostStatusEnv reads this host: the watchdog's settings, the configuration
+// they name, and the cluster.
+func hostStatusEnv(unitDir string) (statusEnv, error) {
+	w, _, err := watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
+	if err != nil {
+		return statusEnv{}, err
+	}
+	cfg, err := config.Load(w.cfgPath)
+	if err != nil {
+		return statusEnv{}, err
+	}
+	cl, clErr := buildSystemServerClient()
+	if clErr != nil {
+		cl = nil
+	}
+	host, _ := os.Hostname()
+	return statusEnv{
+		cfg: cfg, w: w, cl: cl, clErr: clErr,
+		backups: func(ctx context.Context) (map[string]time.Time, error) {
+			return newestWorldBackups(ctx, cfg.Database.URL)
+		},
+		run: hostCommand, unitDir: unitDir, meminfo: "/proc/meminfo", host: host, now: time.Now(),
+	}, nil
+}
+
+func printStatus(ctx context.Context, env statusEnv, out io.Writer) {
+	fmt.Fprintf(out, "felis %s on %s at %s\n", resolvedVersion(), env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
+	if env.cl == nil {
+		fmt.Fprintf(out, "cluster:  unreachable (%v)\n", env.clErr)
+	} else {
+		statusCluster(ctx, env, out)
+	}
+	statusProxy(ctx, env, out)
+	statusServers(ctx, env, out)
+	statusBackups(env, out)
+	statusHost(env, out)
+	statusWatchdog(ctx, env, out)
+}
+
+func statusCluster(ctx context.Context, env statusEnv, out io.Writer) {
+	var nodes corev1.NodeList
+	if err := env.cl.List(ctx, &nodes); err != nil {
+		fmt.Fprintf(out, "cluster:  unreachable (%v)\n", err)
+		return
+	}
+	for _, n := range nodes.Items {
+		ready := "NotReady"
+		for _, c := range n.Status.Conditions {
+			if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue {
+				ready = "Ready"
+			}
+		}
+		info := n.Status.NodeInfo
+		fmt.Fprintf(out, "node:     %s %s, k3s %s, %s, kernel %s\n", n.Name, ready, info.KubeletVersion, info.OSImage, info.KernelVersion)
+	}
+
+	ns := env.w.controlNS
+	var deps appsv1.DeploymentList
+	var pods corev1.PodList
+	err := env.cl.List(ctx, &deps, client.InNamespace(ns))
+	if err == nil {
+		err = env.cl.List(ctx, &pods, client.InNamespace(ns))
+	}
+	fmt.Fprintf(out, "\ncontrol plane (namespace %s):\n", ns)
+	if err != nil {
+		fmt.Fprintf(out, "  cannot list it: %v\n", err)
+		return
+	}
+	sort.Slice(deps.Items, func(i, j int) bool { return deps.Items[i].Name < deps.Items[j].Name })
+	tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	for _, d := range deps.Items {
+		want := int32(1)
+		if d.Spec.Replicas != nil {
+			want = *d.Spec.Replicas
+		}
+		image := "-"
+		if cs := d.Spec.Template.Spec.Containers; len(cs) > 0 {
+			image = shortImage(cs[0].Image)
+		}
+		fmt.Fprintf(tw, "  %s\t%d/%d ready\t%s\trestarts %d\n", d.Name, d.Status.ReadyReplicas, want, image, podRestarts(d.Spec.Selector, pods.Items))
+	}
+	tw.Flush()
+}
+
+// podRestarts adds up the container restarts of the pods selector picks (the
+// API server refuses a Deployment whose selector is empty).
+func podRestarts(selector *metav1.LabelSelector, pods []corev1.Pod) int32 {
+	sel, err := metav1.LabelSelectorAsSelector(selector)
+	if err != nil {
+		return 0
+	}
+	var n int32
+	for _, p := range pods {
+		if !sel.Matches(labels.Set(p.Labels)) {
+			continue
+		}
+		for _, cs := range p.Status.ContainerStatuses {
+			n += cs.RestartCount
+		}
+	}
+	return n
+}
+
+// shortImage is an image reference without its registry and repository path,
+// and with its digest cut to 12 characters.
+func shortImage(ref string) string {
+	if i := strings.LastIndex(ref, "/"); i >= 0 {
+		ref = ref[i+1:]
+	}
+	if name, digest, ok := strings.Cut(ref, "@sha256:"); ok && len(digest) > 12 {
+		ref = name + "@" + digest[:12]
+	}
+	return ref
+}
+
+func statusProxy(ctx context.Context, env statusEnv, out io.Writer) {
+	var parts []string
+	if _, err := os.Stat(filepath.Join(env.unitDir, "felis-velocity.service")); err == nil {
+		parts = append(parts, "felis-velocity "+unitActiveState(ctx, doctorEnv{run: env.run}, "felis-velocity.service"))
+	}
+	if env.w.proxyAddr != "" {
+		d := net.Dialer{Timeout: 3 * time.Second}
+		if conn, err := d.DialContext(ctx, "tcp", env.w.proxyAddr); err != nil {
+			parts = append(parts, fmt.Sprintf("%s refuses connections (%v)", env.w.proxyAddr, err))
+		} else {
+			conn.Close()
+			parts = append(parts, env.w.proxyAddr+" accepts connections")
+		}
+	}
+	if len(parts) == 0 {
+		parts = append(parts, "not on this host")
+	}
+	fmt.Fprintf(out, "\nproxy:    %s\n", strings.Join(parts, ", "))
+}
+
+func statusServers(ctx context.Context, env statusEnv, out io.Writer) {
+	ns := env.cfg.K8s.Namespace
+	if ns == "" {
+		ns = platform.DefaultMinecraftNamespace
+	}
+	if env.cl == nil {
+		fmt.Fprintf(out, "\nservers (namespace %s): unknown while the cluster is unreachable\n", ns)
+		return
+	}
+	var list v1alpha1.MinecraftServerList
+	if err := env.cl.List(ctx, &list, client.InNamespace(ns)); err != nil {
+		fmt.Fprintf(out, "\nservers (namespace %s): cannot list them: %v\n", ns, err)
+		return
+	}
+	newest, backupErr := env.backups(ctx)
+	running, online := 0, int32(0)
+	for _, ms := range list.Items {
+		if ms.Status.Phase == v1alpha1.PhaseRunning {
+			running++
+			online += ms.Status.Players.Online
+		}
+	}
+	fmt.Fprintf(out, "\nservers (namespace %s): %d, %d running, %d players online\n", ns, len(list.Items), running, online)
+	if len(list.Items) == 0 {
+		return
+	}
+	sort.Slice(list.Items, func(i, j int) bool { return list.Items[i].Name < list.Items[j].Name })
+	tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "  NAME\tROLE\tDESIRED\tPHASE\tPLAYERS\tNEWEST WORLD BACKUP")
+	for _, ms := range list.Items {
+		role := ms.Labels[v1alpha1.LabelSystemRole]
+		if role == "" {
+			role = "-"
+		}
+		phase := string(ms.Status.Phase)
+		if phase == "" {
+			phase = "-"
+		}
+		players := "-"
+		if ms.Status.Phase == v1alpha1.PhaseRunning {
+			players = fmt.Sprintf("%d/%d", ms.Status.Players.Online, ms.Status.Players.Max)
+		}
+		backup := "none"
+		switch at, ok := newest[ms.Name]; {
+		case backupErr != nil:
+			backup = "?"
+		case ok:
+			backup = dbbackup.Age(env.now.Sub(at)) + " ago"
+		}
+		fmt.Fprintf(tw, "  %s\t%s\t%s\t%s\t%s\t%s\n", ms.Name, role, orDash(string(ms.Spec.DesiredState)), phase, players, backup)
+	}
+	tw.Flush()
+	if backupErr != nil {
+		fmt.Fprintf(out, "  world backups unknown: %v\n", backupErr)
+	}
+}
+
+func orDash(s string) string {
+	if s == "" {
+		return "-"
+	}
+	return s
+}
+
+// newestWorldBackups is when each server's newest world backup that a restore
+// can use was taken.
+func newestWorldBackups(ctx context.Context, url string) (map[string]time.Time, error) {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	defer drv.Close()
+	rows, err := drv.DB().QueryContext(ctx, `SELECT server_name, max(created_at) FROM world_backups
+		WHERE status = 'present' AND corrupt_at IS NULL GROUP BY server_name`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]time.Time{}
+	for rows.Next() {
+		var name string
+		var at time.Time
+		if err := rows.Scan(&name, &at); err != nil {
+			return nil, err
+		}
+		out[name] = at
+	}
+	return out, rows.Err()
+}
+
+func statusBackups(env statusEnv, out io.Writer) {
+	fmt.Fprintln(out, "\nbackups:")
+	switch bundles, err := dbbackup.List(env.w.backupDir); {
+	case env.w.backupDir == "":
+		fmt.Fprintln(out, "  database: not checked (felis-watchdog.service names no -backup-dir)")
+	case err != nil:
+		fmt.Fprintf(out, "  database: cannot read %s: %v\n", env.w.backupDir, err)
+	case len(bundles) == 0:
+		fmt.Fprintf(out, "  database: none in %s\n", env.w.backupDir)
+	default:
+		fmt.Fprintf(out, "  database: newest %s, %s ago; %d bundles in %s\n",
+			bundles[0].Name, dbbackup.Age(env.now.Sub(bundles[0].Created)), len(bundles), env.w.backupDir)
+	}
+	if !env.cfg.Offsite.Enabled() {
+		fmt.Fprintln(out, "  off-site: not configured, every backup is on this machine only")
+		return
+	}
+	switch st, err := offsite.ReadStatus(env.w.offsiteStatus); {
+	case err != nil:
+		fmt.Fprintf(out, "  off-site: %v\n", err)
+	case st == nil:
+		fmt.Fprintln(out, "  off-site: never synced")
+	case st.LastSuccess.IsZero():
+		fmt.Fprintf(out, "  off-site: never succeeded; last attempt %s ago: %s\n", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
+	default:
+		line := fmt.Sprintf("  off-site: last good sync %s ago to %s", dbbackup.Age(env.now.Sub(st.LastSuccess)), st.Bucket)
+		if st.LastAttempt.After(st.LastSuccess) { // a failed run records no success
+			line += fmt.Sprintf("; the last attempt, %s ago, failed: %s", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
+		}
+		fmt.Fprintln(out, line)
+	}
+}
+
+func statusHost(env statusEnv, out io.Writer) {
+	fmt.Fprintln(out, "\nhost:")
+	seen := map[uint64]bool{}
+	for _, p := range splitList(env.w.diskPaths) {
+		var st syscall.Stat_t
+		if err := syscall.Stat(p, &st); err != nil {
+			continue
+		}
+		dev := uint64(st.Dev) // int32 on darwin
+		if seen[dev] {
+			continue
+		}
+		seen[dev] = true
+		var fs syscall.Statfs_t
+		if err := syscall.Statfs(p, &fs); err != nil || fs.Blocks == 0 {
+			continue
+		}
+		bsize := uint64(fs.Bsize) // uint32 on darwin
+		total, avail := uint64(fs.Blocks)*bsize, uint64(fs.Bavail)*bsize
+		fmt.Fprintf(out, "  disk %s: %s free of %s (%.0f%% free)\n", p, offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)), float64(fs.Bavail)/float64(fs.Blocks)*100)
+	}
+	if total, avail, ok := readMeminfo(env.meminfo); ok {
+		fmt.Fprintf(out, "  memory: %s available of %s\n", offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)))
+	}
+}
+
+// readMeminfo reads MemTotal and MemAvailable, in bytes, from a /proc/meminfo
+// style file.
+func readMeminfo(path string) (total, avail uint64, ok bool) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, 0, false
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		fields := strings.Fields(sc.Text())
+		if len(fields) < 2 {
+			continue
+		}
+		v, _ := strconv.ParseUint(fields[1], 10, 64) // the kernel writes numbers
+		switch fields[0] {
+		case "MemTotal:":
+			total = v * 1024
+		case "MemAvailable:":
+			avail = v * 1024
+		}
+	}
+	return total, avail, total > 0
+}
+
+func statusWatchdog(ctx context.Context, env statusEnv, out io.Writer) {
+	fmt.Fprintln(out, "\nwatchdog:")
+	if _, err := os.Stat(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
+		fmt.Fprintln(out, "  not installed: nothing checks this host")
+		return
+	}
+	line := "  timer " + unitActiveState(ctx, doctorEnv{run: env.run}, "felis-watchdog.timer")
+	show, _ := env.run(ctx, "systemctl", "show", "--timestamp=unix", "-p", "Result", "-p", "ExecMainExitTimestamp", "felis-watchdog.service")
+	props := map[string]string{}
+	for _, ln := range strings.Split(string(show), "\n") {
+		if k, v, ok := strings.Cut(strings.TrimSpace(ln), "="); ok {
+			props[k] = v
+		}
+	}
+	// ExecMainExitTimestamp is empty until the service has run.
+	if sec, _ := strconv.ParseInt(strings.TrimPrefix(props["ExecMainExitTimestamp"], "@"), 10, 64); sec > 0 {
+		line += fmt.Sprintf(", last run %s ago (%s)", dbbackup.Age(env.now.Sub(time.Unix(sec, 0))), orDash(props["Result"]))
+	}
+	fmt.Fprintln(out, line)
+
+	state, err := watchdog.LoadState(watchdog.NewestState(env.w.statePath, env.w.fallbackState))
+	if err != nil {
+		fmt.Fprintf(out, "  alerts: unknown (%v)\n", err)
+		return
+	}
+	var open []string
+	for key, a := range state.Alerts {
+		if !a.ClearedAt.IsZero() {
+			continue
+		}
+		if a.Notified.IsZero() {
+			open = append(open, fmt.Sprintf("%s (%s, seen %s ago, not mailed yet)", key, a.Severity, dbbackup.Age(env.now.Sub(a.FirstSeen))))
+		} else {
+			open = append(open, fmt.Sprintf("%s (%s, mailed %s ago)", key, a.Severity, dbbackup.Age(env.now.Sub(a.Notified))))
+		}
+	}
+	if len(open) == 0 {
+		fmt.Fprintln(out, "  alerts: none open")
+		return
+	}
+	sort.Strings(open)
+	fmt.Fprintf(out, "  alerts: %d open (sudo felis doctor says where to look)\n", len(open))
+	for _, o := range open {
+		fmt.Fprintf(out, "    %s\n", o)
+	}
+}

cmd/felis/status_test.go

@@ -0,0 +1,490 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// statusCluster is a one-node install: felis-api with a restarted pod, three
+// servers (one of them the lobby), and a pod of another app whose restarts
+// are not felis-api's.
+func statusClusterObjects() []client.Object {
+	replicas := int32(1)
+	apiLabels := map[string]string{"app": "felis-api"}
+	return []client.Object{
+		&corev1.Node{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-1"},
+			Status: corev1.NodeStatus{
+				Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}},
+				NodeInfo:   corev1.NodeSystemInfo{KubeletVersion: "v1.36.4+k3s1", OSImage: "CentOS Stream 9", KernelVersion: "5.14.0-630.el9.aarch64"},
+			},
+		},
+		&appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
+			Spec: appsv1.DeploymentSpec{
+				Replicas: &replicas,
+				Selector: &metav1.LabelSelector{MatchLabels: apiLabels},
+				Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{
+					Name: "api", Image: "registry.felis.svc:5000/felis/felis-api:v1.4.0@sha256:0123456789abcdef0123456789abcdef",
+				}}}},
+			},
+			Status: appsv1.DeploymentStatus{ReadyReplicas: 1},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-api-7d9", Namespace: "felis", Labels: apiLabels},
+			Status:     corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 2}}},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "other-1", Namespace: "felis", Labels: map[string]string{"app": "other"}},
+			Status:     corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "x", RestartCount: 5}}},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
+			Spec:       v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
+			Status:     v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseRunning, Players: v1alpha1.PlayersStatus{Online: 2, Max: 20}},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "creative", Namespace: "minecraft"},
+			Spec:       v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped},
+			Status:     v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "lobby", Namespace: "minecraft", Labels: map[string]string{v1alpha1.LabelSystemRole: "lobby"}},
+			Spec:       v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
+			Status:     v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStarting},
+		},
+		// Another namespace's server is not this install's.
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "elsewhere", Namespace: "other"}},
+	}
+}
+
+// statusTestEnv is a host with the cluster above, a database backup nine hours
+// old, an off-site copy last good two hours ago, and one alert open.
+func statusTestEnv(t *testing.T) statusEnv {
+	t.Helper()
+	now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	dir := t.TempDir()
+	var w watchdogFlags
+	w.controlNS = "felis"
+	w.backupDir = filepath.Join(dir, "db-backups")
+	w.offsiteStatus = filepath.Join(dir, "offsite-status.json")
+	w.statePath = filepath.Join(dir, "state.json")
+	w.fallbackState = filepath.Join(dir, "fallback.json")
+	w.diskPaths = "/nonexistent-felis-status-test"
+	unitDir := filepath.Join(dir, "systemd")
+	for _, d := range []string{w.backupDir, unitDir} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-9*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
+	writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-33*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
+	if err := offsite.WriteStatus(w.offsiteStatus, offsite.Status{
+		LastAttempt: now.Add(-2 * time.Hour), LastSuccess: now.Add(-2 * time.Hour), Bucket: "felis-dr", Endpoint: "https://s3.example.com",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	state := &watchdog.State{Alerts: map[string]*watchdog.Alert{
+		"db-backup-servers": {Finding: watchdog.Finding{Key: "db-backup-servers", Severity: watchdog.Warning}, FirstSeen: now.Add(-3 * time.Hour), Notified: now.Add(-90 * time.Minute)},
+		"proxy":             {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: now.Add(-time.Hour), Notified: now.Add(-time.Hour), ClearedAt: now.Add(-10 * time.Minute)},
+		"disk//var":         {Finding: watchdog.Finding{Key: "disk//var", Severity: watchdog.Critical}, FirstSeen: now.Add(-4 * time.Minute)},
+	}}
+	if err := watchdog.SaveState(w.statePath, state); err != nil {
+		t.Fatal(err)
+	}
+	meminfo := filepath.Join(dir, "meminfo")
+	writeTestFile(t, meminfo, "MemTotal:        8000000 kB\nMemFree:          100000 kB\nMemAvailable:    2000000 kB\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.timer"), "[Unit]\n", 0o644)
+	cfg := &config.Config{Offsite: config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-dr"}}
+	return statusEnv{
+		cfg: cfg, w: w, cl: fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).Build(),
+		backups: func(context.Context) (map[string]time.Time, error) {
+			return map[string]time.Time{"survival": now.Add(-3 * time.Hour)}, nil
+		},
+		run: func(_ context.Context, name string, args ...string) ([]byte, error) {
+			switch strings.Join(append([]string{name}, args...), " ") {
+			case "systemctl is-active felis-watchdog.timer":
+				return []byte("active\n"), nil
+			case "systemctl show --timestamp=unix -p Result -p ExecMainExitTimestamp felis-watchdog.service":
+				return []byte("Result=success\nExecMainExitTimestamp=@" + strconv.FormatInt(now.Add(-70*time.Second).Unix(), 10) + "\n"), nil
+			}
+			return nil, errors.New("unexpected")
+		},
+		unitDir: unitDir, meminfo: meminfo, host: "felis-test", now: now,
+	}
+}
+
+// lineFields is the words of the first line of out that starts with prefix
+// once trimmed.
+func lineFields(out, prefix string) []string {
+	for _, l := range strings.Split(out, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(l), prefix) {
+			return strings.Fields(l)
+		}
+	}
+	return nil
+}
+
+func TestStatusReport(t *testing.T) {
+	env := statusTestEnv(t)
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	got := out.String()
+	for _, want := range []string{
+		"on felis-test at 2026-09-27 12:00 UTC\n",
+		"node:     felis-1 Ready, k3s v1.36.4+k3s1, CentOS Stream 9, kernel 5.14.0-630.el9.aarch64\n",
+		"\ncontrol plane (namespace felis):\n",
+		"\nproxy:    not on this host\n",
+		"\nservers (namespace minecraft): 3, 1 running, 2 players online\n",
+		"  database: newest " + dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily) + ", 9h0m ago; 2 bundles in " + env.w.backupDir + "\n",
+		"  off-site: last good sync 2h0m ago to felis-dr\n",
+		"  memory: 1.9 GiB available of 7.6 GiB\n",
+		"  timer active, last run 1m ago (success)\n",
+		"  alerts: 2 open (sudo felis doctor says where to look)\n" +
+			"    db-backup-servers (warning, mailed 1h30m ago)\n" +
+			"    disk//var (critical, seen 4m ago, not mailed yet)\n",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("status lacks %q:\n%s", want, got)
+		}
+	}
+	for prefix, want := range map[string]string{
+		"felis-api": "felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
+		"NAME":      "NAME ROLE DESIRED PHASE PLAYERS NEWEST WORLD BACKUP",
+		"creative":  "creative - Stopped Stopped - none",
+		"lobby":     "lobby lobby Running Starting - none",
+		"survival":  "survival - Running Running 2/20 3h0m ago",
+	} {
+		if f := strings.Join(lineFields(got, prefix), " "); f != want {
+			t.Errorf("row %q = %q, want %q\n%s", prefix, f, want, got)
+		}
+	}
+	if strings.Contains(got, "elsewhere") || strings.Contains(got, "other-1") {
+		t.Errorf("status shows what is not this install's:\n%s", got)
+	}
+}
+
+// Whatever is down reads as down, and the rest of the report still prints.
+func TestStatusWithPartsDown(t *testing.T) {
+	env := statusTestEnv(t)
+	env.cl, env.clErr = nil, errors.New("connection refused")
+	env.cfg = &config.Config{}
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	for _, want := range []string{
+		"cluster:  unreachable (connection refused)\n",
+		"\nservers (namespace minecraft): unknown while the cluster is unreachable\n",
+		"  off-site: not configured, every backup is on this machine only\n",
+		"  timer active, last run",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("status lacks %q:\n%s", want, out.String())
+		}
+	}
+
+	env = statusTestEnv(t)
+	env.backups = func(context.Context) (map[string]time.Time, error) { return nil, errors.New("postgres is down") }
+	out.Reset()
+	printStatus(context.Background(), env, &out)
+	if f := strings.Join(lineFields(out.String(), "survival"), " "); f != "survival - Running Running 2/20 ?" {
+		t.Errorf("survival with PostgreSQL down = %q", f)
+	}
+	if !strings.Contains(out.String(), "  world backups unknown: postgres is down\n") {
+		t.Errorf("status does not say why the backups are unknown:\n%s", out.String())
+	}
+}
+
+func TestShortImage(t *testing.T) {
+	for in, want := range map[string]string{
+		"registry.felis.svc:5000/felis/felis-api:v1.4.0":                                       "felis-api:v1.4.0",
+		"docker.io/library/postgres:17@sha256:0123456789abcdef0123":                            "postgres:17@0123456789ab",
+		"felis-operator@sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba98765432": "felis-operator@fedcba987654",
+		"busybox": "busybox",
+	} {
+		if got := shortImage(in); got != want {
+			t.Errorf("shortImage(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+// A node that is not ready, a Deployment without replicas or containers, and
+// lists the API server refuses each read as such.
+func TestStatusClusterEdges(t *testing.T) {
+	env := statusTestEnv(t)
+	env.cfg = &config.Config{K8s: config.K8sConfig{Namespace: "games"}}
+	env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}, Status: corev1.NodeStatus{
+			Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionFalse}, {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionTrue}}}},
+		&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-bare", Namespace: "felis"}},
+		&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "felis"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 4}}}},
+	).Build()
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	got := out.String()
+	for _, want := range []string{
+		"node:     felis-1 NotReady, k3s , , kernel \n",
+		"\nservers (namespace games): 0, 0 running, 0 players online\n\nbackups:\n",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("status lacks %q:\n%s", want, got)
+		}
+	}
+	if f := strings.Join(lineFields(got, "felis-bare"), " "); f != "felis-bare 0/1 ready - restarts 0" {
+		t.Errorf("row felis-bare = %q, want its one replica wanted, no image, and no pod of its own:\n%s", f, got)
+	}
+
+	failing := func(what client.ObjectList) {
+		env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).
+			WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
+				if fmt.Sprintf("%T", list) == fmt.Sprintf("%T", what) {
+					return errors.New("forbidden")
+				}
+				return c.List(ctx, list, opts...)
+			}}).Build()
+		out.Reset()
+		printStatus(context.Background(), env, &out)
+	}
+	for _, tc := range []struct {
+		list client.ObjectList
+		want string
+	}{
+		{&corev1.NodeList{}, "cluster:  unreachable (forbidden)\n\nproxy:"},
+		{&appsv1.DeploymentList{}, "\ncontrol plane (namespace felis):\n  cannot list it: forbidden\n\nproxy:"},
+		{&corev1.PodList{}, "\ncontrol plane (namespace felis):\n  cannot list it: forbidden\n\nproxy:"},
+		{&v1alpha1.MinecraftServerList{}, "\nservers (namespace games): cannot list them: forbidden\n\nbackups:"},
+	} {
+		failing(tc.list)
+		if !strings.Contains(out.String(), tc.want) {
+			t.Errorf("listing %T refused: status lacks %q:\n%s", tc.list, tc.want, out.String())
+		}
+	}
+}
+
+func TestStatusProxy(t *testing.T) {
+	env := statusTestEnv(t)
+	writeTestFile(t, filepath.Join(env.unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
+	env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active"})
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	env.w.proxyAddr = ln.Addr().String()
+	var out bytes.Buffer
+	statusProxy(context.Background(), env, &out)
+	if want := "\nproxy:    felis-velocity active, " + env.w.proxyAddr + " accepts connections\n"; out.String() != want {
+		t.Errorf("proxy listening: %q, want %q", out.String(), want)
+	}
+	ln.Close()
+	out.Reset()
+	statusProxy(context.Background(), env, &out)
+	if want := "\nproxy:    felis-velocity active, " + env.w.proxyAddr + " refuses connections (dial tcp " + env.w.proxyAddr + ": "; !strings.HasPrefix(out.String(), want) {
+		t.Errorf("proxy gone: %q, want it to start %q", out.String(), want)
+	}
+}
+
+func TestStatusBackups(t *testing.T) {
+	env := statusTestEnv(t)
+	status := func() string {
+		var out bytes.Buffer
+		statusBackups(env, &out)
+		return out.String()
+	}
+	for _, tc := range []struct {
+		what, dir, want string
+	}{
+		{"no -backup-dir", "", "  database: not checked (felis-watchdog.service names no -backup-dir)\n"},
+		{"an empty directory", t.TempDir(), "  database: none in %s\n"},
+		{"a file where the directory should be", filepath.Join(env.w.backupDir, dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily)), "  database: cannot read %s: "},
+	} {
+		env.w.backupDir = tc.dir
+		want := tc.want
+		if strings.Contains(want, "%s") {
+			want = fmt.Sprintf(want, tc.dir)
+		}
+		if got := status(); !strings.Contains(got, want) {
+			t.Errorf("%s: %q, want %q", tc.what, got, want)
+		}
+	}
+
+	for _, tc := range []struct {
+		what   string
+		status *offsite.Status
+		raw    string
+		want   string
+	}{
+		{"never synced", nil, "", "  off-site: never synced\n"},
+		{"never succeeded", &offsite.Status{LastAttempt: env.now.Add(-time.Hour), LastError: "403 Forbidden"}, "", "  off-site: never succeeded; last attempt 1h0m ago: 403 Forbidden\n"},
+		{"the last attempt failed", &offsite.Status{LastAttempt: env.now.Add(-30 * time.Minute), LastSuccess: env.now.Add(-26 * time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
+			"  off-site: last good sync 26h0m ago to felis-dr; the last attempt, 30m ago, failed: timeout\n"},
+		{"an error an earlier attempt left", &offsite.Status{LastAttempt: env.now.Add(-2 * time.Hour), LastSuccess: env.now.Add(-time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
+			"  off-site: last good sync 1h0m ago to felis-dr\n"},
+		{"an unreadable record", nil, "{", "  off-site: unexpected end of JSON input\n"},
+	} {
+		os.Remove(env.w.offsiteStatus)
+		switch {
+		case tc.status != nil:
+			if err := offsite.WriteStatus(env.w.offsiteStatus, *tc.status); err != nil {
+				t.Fatal(err)
+			}
+		case tc.raw != "":
+			writeTestFile(t, env.w.offsiteStatus, tc.raw, 0o600)
+		}
+		if got := status(); !strings.HasSuffix(got, tc.want) {
+			t.Errorf("%s: %q, want it to end %q", tc.what, got, tc.want)
+		}
+	}
+}
+
+func TestStatusHost(t *testing.T) {
+	env := statusTestEnv(t)
+	dir := t.TempDir()
+	env.w.diskPaths = "/nonexistent-felis-status-test," + dir + "," + dir
+	env.meminfo = filepath.Join(dir, "meminfo")
+	var out bytes.Buffer
+	statusHost(env, &out)
+	lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
+	if len(lines) != 3 || lines[0] != "" || lines[1] != "host:" || !strings.HasPrefix(lines[2], "  disk "+dir+": ") || !strings.HasSuffix(lines[2], "% free)") {
+		t.Errorf("host: %q, want one line for %s (a path on a disk already shown, and one that is not there, print none) and no memory line", lines, dir)
+	}
+
+	writeTestFile(t, env.meminfo, "MemTotal: 4096 kB\ngarbage\nMemAvailable: 1024 kB\n", 0o644)
+	if total, avail, ok := readMeminfo(env.meminfo); !ok || total != 4096*1024 || avail != 1024*1024 {
+		t.Errorf("readMeminfo = %d, %d, %v", total, avail, ok)
+	}
+	writeTestFile(t, env.meminfo, "MemAvailable: 1024 kB\n", 0o644)
+	if _, _, ok := readMeminfo(env.meminfo); ok {
+		t.Error("readMeminfo without MemTotal: ok")
+	}
+}
+
+func TestStatusWatchdog(t *testing.T) {
+	env := statusTestEnv(t)
+	status := func() string {
+		var out bytes.Buffer
+		statusWatchdog(context.Background(), env, &out)
+		return out.String()
+	}
+	run := env.run
+	env.run = func(ctx context.Context, name string, args ...string) ([]byte, error) {
+		if len(args) > 0 && args[0] == "show" {
+			return []byte("Result=success\nExecMainExitTimestamp=\n"), nil
+		}
+		return run(ctx, name, args...)
+	}
+	if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: map[string]*watchdog.Alert{
+		"proxy": {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: env.now.Add(-time.Hour), ClearedAt: env.now.Add(-time.Minute)},
+	}}); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := status(), "\nwatchdog:\n  timer active\n  alerts: none open\n"; got != want {
+		t.Errorf("a watchdog that has not run and has nothing open: %q, want %q", got, want)
+	}
+
+	writeTestFile(t, env.w.statePath, "{", 0o600)
+	if got := status(); !strings.Contains(got, "\n  alerts: unknown (") {
+		t.Errorf("an unreadable state: %q", got)
+	}
+
+	if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := status(), "\nwatchdog:\n  not installed: nothing checks this host\n"; got != want {
+		t.Errorf("no watchdog timer: %q, want %q", got, want)
+	}
+}
+
+// Rows print by name in whatever order the API server lists them, a server
+// the operator has not reconciled yet reads as dashes, and open alerts print
+// by key in whatever order the state's map yields them.
+func TestStatusOrder(t *testing.T) {
+	env := statusTestEnv(t)
+	objs := append(statusClusterObjects(),
+		&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-operator", Namespace: "felis"}},
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "fresh", Namespace: "minecraft"}})
+	env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
+		WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
+			// The fake lists by name; the other way round, then.
+			if err := c.List(ctx, list, opts...); err != nil {
+				return err
+			}
+			items, err := meta.ExtractList(list)
+			if err != nil {
+				return err
+			}
+			slices.Reverse(items)
+			return meta.SetList(list, items)
+		}}).Build()
+	var out bytes.Buffer
+	printStatus(context.Background(), env, &out)
+	var rows []string
+	for _, l := range strings.Split(out.String(), "\n") {
+		if f := strings.Fields(l); len(f) > 0 && (strings.HasPrefix(f[0], "felis-") || slices.Contains([]string{"creative", "fresh", "lobby", "survival"}, f[0])) {
+			rows = append(rows, strings.Join(f, " "))
+		}
+	}
+	want := []string{
+		"felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
+		"felis-operator 0/1 ready - restarts 0",
+		"creative - Stopped Stopped - none",
+		"fresh - - - - none",
+		"lobby lobby Running Starting - none",
+		"survival - Running Running 2/20 3h0m ago",
+	}
+	if !slices.Equal(rows, want) {
+		t.Errorf("rows %q, want %q:\n%s", rows, want, out.String())
+	}
+
+	alerts := map[string]*watchdog.Alert{}
+	for i, key := range []string{"a", "b", "c", "d"} {
+		alerts[key] = &watchdog.Alert{Finding: watchdog.Finding{Key: key, Severity: watchdog.Warning}, FirstSeen: env.now.Add(-time.Duration(i+1) * time.Minute)}
+	}
+	if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: alerts}); err != nil {
+		t.Fatal(err)
+	}
+	wantAlerts := "  alerts: 4 open (sudo felis doctor says where to look)\n" +
+		"    a (warning, seen 1m ago, not mailed yet)\n    b (warning, seen 2m ago, not mailed yet)\n" +
+		"    c (warning, seen 3m ago, not mailed yet)\n    d (warning, seen 4m ago, not mailed yet)\n"
+	// A map starts its walk at random: fifty walks all in order by chance
+	// is out of the question.
+	for range 50 {
+		out.Reset()
+		statusWatchdog(context.Background(), env, &out)
+		if !strings.HasSuffix(out.String(), wantAlerts) {
+			t.Fatalf("alerts %q, want them to end %q", out.String(), wantAlerts)
+		}
+	}
+}
+
+// A selector the API server would have refused picks no pods.
+func TestPodRestartsBadSelector(t *testing.T) {
+	pods := []corev1.Pod{{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 3}}}}}
+	if n := podRestarts(&metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "app", Operator: "Near"}}}, pods); n != 0 {
+		t.Errorf("podRestarts with a bad selector = %d, want 0", n)
+	}
+	if n := podRestarts(&metav1.LabelSelector{}, pods); n != 3 {
+		t.Errorf("podRestarts with a selector that picks all = %d, want 3", n)
+	}
+}

cmd/felis/supportbundle.go

@@ -0,0 +1,862 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+)
+
+// supportBundleDir is where felis support-bundle writes unless told otherwise.
+const supportBundleDir = "/var/lib/felis/support"
+
+// redacted stands in for every value the bundle leaves out.
+const redacted = "<redacted>"
+
+// minScrubLen is the shortest known secret value the bundle searches for: a
+// shorter one would blank ordinary words, and every secret the installer
+// generates is far longer.
+const minScrubLen = 8
+
+// hostSecretSources are the files on a Felis host that hold secrets; the
+// bundle reads them only to take each value out of what it collects.
+var hostSecretSources = secretSources{
+	envFiles:   []string{"/etc/felis/secrets.env", defaultOffsiteEnvFile},
+	valueFiles: []string{hostSMTPPasswordPath, hostUploadsS3AccessKeyPath, hostUploadsS3SecretKeyPath, defaultHeartbeatFile, "/opt/felis/velocity/forwarding.secret"},
+	propsFiles: []string{"/opt/felis/velocity/plugins/felis-link/felis-link.properties"},
+	tokenFiles: []string{"/var/lib/rancher/k3s/server/token", "/var/lib/rancher/k3s/server/agent-token"},
+}
+
+// cmdSupportBundle collects what someone helping with this host needs into
+// one tar.gz: felis status and felis doctor, the logs of the control plane,
+// the builds and the systemd units, the cluster's workloads and events, and a
+// summary of the configuration. It never collects a Secret, a ConfigMap, the
+// contents of a configuration file, the database or a world, and takes every
+// value of the host's secret files out of what it does collect. Game server
+// logs, which carry player names, addresses and chat, only with -server-logs.
+func cmdSupportBundle(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("support-bundle", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	outDir := fs.String("o", supportBundleDir, "directory to write the bundle to (created mode 0700 when missing)")
+	logLines := fs.Int64("log-lines", 2000, "lines kept from the end of each pod log and each unit's journal")
+	since := fs.Duration("since", 48*time.Hour, "how far back each unit's journal is read")
+	serverLogs := fs.Bool("server-logs", false, "also collect the game servers' own logs, which carry player names, IP addresses and chat")
+	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis support-bundle: run as root (sudo felis support-bundle): it reads root-only logs and state")
+		return 1
+	}
+	b := hostSupportBundle(*unitDir, *logLines, *since, *serverLogs)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	path, err := b.write(ctx, *outDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis support-bundle: %v\n", err)
+		return 1
+	}
+	size := int64(0)
+	if st, err := os.Stat(path); err == nil {
+		size = st.Size()
+	}
+	fmt.Fprintf(stdout, "wrote %s (%s, mode 0600)\n", path, humanSize(size))
+	fmt.Fprintln(stdout, "MANIFEST.txt inside says what it holds and what was taken out. Read it through before you send it anywhere:")
+	fmt.Fprintln(stdout, "redaction finds this host's known secrets and the common ways a secret is logged, and a secret logged another way stays in.")
+	if !*serverLogs {
+		fmt.Fprintln(stdout, "Game server logs are left out; -server-logs adds them (player names, IP addresses, chat).")
+	}
+	return 0
+}
+
+func humanSize(n int64) string {
+	switch {
+	case n >= 1<<20:
+		return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
+	case n >= 1<<10:
+		return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10))
+	}
+	return fmt.Sprintf("%d B", n)
+}
+
+// shortDuration is d as Duration.String writes it, less the zero minutes and
+// seconds after whole hours or minutes: 48h, and 90m as 1h30m.
+func shortDuration(d time.Duration) string {
+	s := d.String()
+	if strings.HasSuffix(s, "m0s") {
+		s = strings.TrimSuffix(s, "0s")
+	}
+	if strings.HasSuffix(s, "h0m") {
+		s = strings.TrimSuffix(s, "0m")
+	}
+	return s
+}
+
+// supportBundle is one collection and what it reads the host through.
+type supportBundle struct {
+	host    string
+	now     time.Time
+	unitDir string
+	// w is how felis-watchdog.service runs the watchdog, wErr why it could
+	// not be read (w then holds the defaults).
+	w          watchdogFlags
+	wErr       error
+	cfg        *config.Config // nil when cfgErr
+	cfgErr     error
+	cl         client.Client // nil when clErr
+	clErr      error
+	logs       func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error)
+	run        func(ctx context.Context, name string, args ...string) ([]byte, error)
+	backups    func(ctx context.Context) (map[string]time.Time, error)
+	doctor     func(ctx context.Context, out io.Writer)
+	secrets    secretSources
+	logLines   int64
+	since      time.Duration
+	serverLogs bool
+	// Host files read whole, and the directory whose listing is kept.
+	meminfo, osRelease, procVersion, stateDir string
+}
+
+func hostSupportBundle(unitDir string, logLines int64, since time.Duration, serverLogs bool) *supportBundle {
+	host, _ := os.Hostname()
+	b := &supportBundle{
+		host: host, now: time.Now(), unitDir: unitDir, run: hostCommand, secrets: hostSecretSources,
+		logLines: logLines, since: since, serverLogs: serverLogs,
+		meminfo: "/proc/meminfo", osRelease: "/etc/os-release", procVersion: "/proc/version", stateDir: "/etc/felis",
+	}
+	var found bool
+	b.w, found, b.wErr = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
+	if b.wErr == nil && !found {
+		b.wErr = fmt.Errorf("%s is not installed; read the watchdog's defaults", filepath.Join(unitDir, "felis-watchdog.service"))
+	}
+	if b.cfg, b.cfgErr = config.Load(b.w.cfgPath); b.cfgErr == nil {
+		cfg := b.cfg
+		b.backups = func(ctx context.Context) (map[string]time.Time, error) {
+			return newestWorldBackups(ctx, cfg.Database.URL)
+		}
+	} else {
+		b.cfg = nil
+	}
+	if b.cl, b.clErr = buildSystemServerClient(); b.clErr != nil {
+		b.cl = nil
+	} else if rc, err := hostRESTConfig(); err != nil {
+		b.clErr = err
+		b.cl = nil
+	} else if cs, err := kubernetes.NewForConfig(rc); err != nil {
+		b.clErr = err
+		b.cl = nil
+	} else {
+		limit := int64(8 << 20)
+		b.logs = func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error) {
+			return cs.CoreV1().Pods(ns).GetLogs(pod, &corev1.PodLogOptions{
+				Container: container, Previous: previous, Timestamps: true, TailLines: &logLines, LimitBytes: &limit,
+			}).DoRaw(ctx)
+		}
+	}
+	b.doctor = func(ctx context.Context, out io.Writer) {
+		runDoctor(ctx, doctorEnv{unitDir: unitDir, run: hostCommand, now: b.now, host: host}, out)
+	}
+	return b
+}
+
+// bundleWriter streams scrubbed files into the archive and keeps what went
+// wrong while collecting, for MANIFEST.txt.
+type bundleWriter struct {
+	tw     *tar.Writer
+	prefix string
+	now    time.Time
+	scrub  *scrubber
+	errs   []string
+}
+
+// logText adds a log, or a report that quotes errors: known secrets and
+// anything logged as one come out.
+func (bw *bundleWriter) logText(name string, data []byte) error {
+	return bw.add(name, bw.scrub.text(data))
+}
+
+// plain adds a file whose secrets were already taken out by structure (the
+// cluster's objects, the configuration summary) or that names none (the
+// release, disk use, addresses): known secret values still come out, and
+// nothing else is rewritten.
+func (bw *bundleWriter) plain(name string, data []byte) error {
+	return bw.add(name, bw.scrub.values(data))
+}
+
+func (bw *bundleWriter) add(name string, data []byte) error {
+	hdr := &tar.Header{Name: path.Join(bw.prefix, name), Mode: 0o600, Size: int64(len(data)), ModTime: bw.now, Typeflag: tar.TypeReg}
+	if err := bw.tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+	_, err := bw.tw.Write(data)
+	return err
+}
+
+func (bw *bundleWriter) failed(what string, err error) {
+	bw.errs = append(bw.errs, string(bw.scrub.text([]byte(fmt.Sprintf("%s: %v", what, err)))))
+}
+
+// write collects the bundle into dir and returns its path.
+func (b *supportBundle) write(ctx context.Context, dir string) (string, error) {
+	if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {
+		if err := os.MkdirAll(dir, 0o700); err != nil {
+			return "", err
+		}
+	}
+	stamp := b.now.UTC().Format("20060102T150405Z")
+	base := fmt.Sprintf("felis-support-%s-%s", safeName(b.host), stamp)
+	final := filepath.Join(dir, base+".tar.gz")
+	f, err := os.CreateTemp(dir, "."+base+".*.partial") // mode 0600
+	if err != nil {
+		return "", err
+	}
+	keep := false
+	defer func() {
+		if !keep {
+			f.Close()
+			os.Remove(f.Name())
+		}
+	}()
+	gz := gzip.NewWriter(f)
+	bw := &bundleWriter{tw: tar.NewWriter(gz), prefix: base, now: b.now, scrub: b.scrubber()}
+	if err := b.collect(ctx, bw); err != nil {
+		return "", err
+	}
+	if err := bw.tw.Close(); err != nil {
+		return "", err
+	}
+	if err := gz.Close(); err != nil {
+		return "", err
+	}
+	if err := f.Sync(); err != nil {
+		return "", err
+	}
+	if err := f.Close(); err != nil {
+		return "", err
+	}
+	if err := os.Rename(f.Name(), final); err != nil {
+		return "", err
+	}
+	keep = true
+	return final, nil
+}
+
+// safeName keeps a host name usable in a file name.
+func safeName(s string) string {
+	s = strings.Map(func(r rune) rune {
+		if r == '-' || r == '.' || r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
+			return r
+		}
+		return '_'
+	}, s)
+	if s == "" {
+		return "host"
+	}
+	return s
+}
+
+func (b *supportBundle) collect(ctx context.Context, bw *bundleWriter) error {
+	var buf bytes.Buffer
+	cmdVersion(nil, &buf, io.Discard)
+	for _, p := range []string{b.osRelease, b.procVersion} {
+		if raw, err := os.ReadFile(p); err == nil {
+			fmt.Fprintf(&buf, "\n# %s\n%s", p, raw)
+		}
+	}
+	if err := bw.plain("version.txt", buf.Bytes()); err != nil {
+		return err
+	}
+
+	buf.Reset()
+	switch {
+	case b.cfgErr != nil:
+		fmt.Fprintf(&buf, "felis status: the configuration did not load: %v\n", b.cfgErr)
+	default:
+		printStatus(ctx, statusEnv{
+			cfg: b.cfg, w: b.w, cl: b.cl, clErr: b.clErr, backups: b.backups, run: b.run,
+			unitDir: b.unitDir, meminfo: b.meminfo, host: b.host, now: b.now,
+		}, &buf)
+	}
+	if err := bw.logText("status.txt", buf.Bytes()); err != nil {
+		return err
+	}
+	buf.Reset()
+	b.doctor(ctx, &buf)
+	if err := bw.logText("doctor.txt", buf.Bytes()); err != nil {
+		return err
+	}
+	if b.cfg != nil {
+		if err := bw.plain("config.txt", configSummary(b.cfg, b.w.cfgPath)); err != nil {
+			return err
+		}
+	}
+	if err := b.collectHost(ctx, bw); err != nil {
+		return err
+	}
+	if err := b.collectJournal(ctx, bw); err != nil {
+		return err
+	}
+	if b.cl == nil {
+		bw.failed("cluster", b.clErr)
+	} else if err := b.collectCluster(ctx, bw); err != nil {
+		return err
+	}
+	return bw.add("MANIFEST.txt", b.manifest(bw))
+}
+
+func (b *supportBundle) collectHost(ctx context.Context, bw *bundleWriter) error {
+	commands := []struct {
+		name string
+		argv []string
+	}{
+		{"host/systemd-units.txt", []string{"systemctl", "list-units", "--all", "--no-pager", "--plain", "felis-*", "k3s.service"}},
+		{"host/systemd-timers.txt", []string{"systemctl", "list-timers", "--all", "--no-pager", "felis-*"}},
+		{"host/df.txt", []string{"df", "-h"}},
+		{"host/addresses.txt", []string{"ip", "-brief", "address"}},
+	}
+	for _, c := range commands {
+		out, err := b.run(ctx, c.argv[0], c.argv[1:]...)
+		if err != nil && len(out) == 0 {
+			bw.failed(strings.Join(c.argv, " "), err)
+			continue
+		}
+		if err := bw.plain(c.name, out); err != nil {
+			return err
+		}
+	}
+	if raw, err := os.ReadFile(b.meminfo); err == nil {
+		if err := bw.plain("host/meminfo.txt", raw); err != nil {
+			return err
+		}
+	}
+	listing, err := dirListing(b.stateDir)
+	if err != nil {
+		bw.failed("list "+b.stateDir, err)
+		return nil
+	}
+	return bw.plain("host/etc-felis.txt", listing)
+}
+
+// dirListing names every file under dir with its mode, size and time, and
+// holds nothing of what is in them.
+func dirListing(dir string) ([]byte, error) {
+	var buf bytes.Buffer
+	tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
+	fmt.Fprintf(tw, "# %s: names, modes, sizes and times only; no contents\n", dir)
+	err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(tw, "%s\t%d\t%s\t%s\n", info.Mode(), info.Size(), info.ModTime().UTC().Format(time.RFC3339), p)
+		return nil
+	})
+	tw.Flush()
+	return buf.Bytes(), err
+}
+
+func (b *supportBundle) collectJournal(ctx context.Context, bw *bundleWriter) error {
+	units, _ := filepath.Glob(filepath.Join(b.unitDir, "felis-*.service"))
+	if _, err := os.Stat(filepath.Join(b.unitDir, "k3s.service")); err == nil {
+		units = append(units, filepath.Join(b.unitDir, "k3s.service"))
+	}
+	since := "@" + strconv.FormatInt(b.now.Add(-b.since).Unix(), 10)
+	for _, u := range units {
+		unit := filepath.Base(u)
+		out, err := b.run(ctx, "journalctl", "-u", unit, "--since", since, "-n", strconv.FormatInt(b.logLines, 10), "--no-pager", "-o", "short-iso")
+		if err != nil && len(out) == 0 {
+			bw.failed("journalctl -u "+unit, err)
+			continue
+		}
+		if err := bw.logText("journal/"+strings.TrimSuffix(unit, ".service")+".log", out); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// bundleNamespaces are the namespaces whose objects and logs the bundle
+// collects: the control plane, the builds and the game servers.
+func (b *supportBundle) bundleNamespaces() (control, build, minecraft string) {
+	control, build, minecraft = b.w.controlNS, platform.DefaultBuildNamespace, platform.DefaultMinecraftNamespace
+	if b.cfg != nil {
+		if b.cfg.Registry.BuildNamespace != "" {
+			build = b.cfg.Registry.BuildNamespace
+		}
+		if b.cfg.K8s.Namespace != "" {
+			minecraft = b.cfg.K8s.Namespace
+		}
+	}
+	return control, build, minecraft
+}
+
+func (b *supportBundle) collectCluster(ctx context.Context, bw *bundleWriter) error {
+	control, build, minecraft := b.bundleNamespaces()
+	dump := func(name string, list client.ObjectList, opts ...client.ListOption) error {
+		if err := b.cl.List(ctx, list, opts...); err != nil {
+			bw.failed("list "+name, err)
+			return nil
+		}
+		redactList(list)
+		out, err := yaml.Marshal(list)
+		if err != nil {
+			bw.failed("encode "+name, err)
+			return nil
+		}
+		return bw.plain(name, out)
+	}
+	if err := dump("cluster/nodes.yaml", &corev1.NodeList{}); err != nil {
+		return err
+	}
+	if err := dump("cluster/persistentvolumes.yaml", &corev1.PersistentVolumeList{}); err != nil {
+		return err
+	}
+	if err := dump("cluster/minecraftservers.yaml", &v1alpha1.MinecraftServerList{}, client.InNamespace(minecraft)); err != nil {
+		return err
+	}
+	for _, ns := range []string{control, build, minecraft} {
+		for _, k := range []struct {
+			name string
+			list client.ObjectList
+		}{
+			{"pods", &corev1.PodList{}},
+			{"deployments", &appsv1.DeploymentList{}},
+			{"statefulsets", &appsv1.StatefulSetList{}},
+			{"jobs", &batchv1.JobList{}},
+			{"cronjobs", &batchv1.CronJobList{}},
+			{"services", &corev1.ServiceList{}},
+			{"persistentvolumeclaims", &corev1.PersistentVolumeClaimList{}},
+			{"networkpolicies", &networkingv1.NetworkPolicyList{}},
+			{"events", &corev1.EventList{}},
+		} {
+			if err := dump("cluster/"+ns+"/"+k.name+".yaml", k.list, client.InNamespace(ns)); err != nil {
+				return err
+			}
+		}
+	}
+
+	var all corev1.PodList
+	if err := b.cl.List(ctx, &all); err != nil {
+		bw.failed("list every pod", err)
+	} else if err := bw.plain("cluster/pods-all-namespaces.txt", podTable(all.Items, b.now)); err != nil {
+		return err
+	}
+
+	for _, ns := range []string{control, build, minecraft} {
+		var pods corev1.PodList
+		if err := b.cl.List(ctx, &pods, client.InNamespace(ns)); err != nil {
+			continue // already recorded by the dump above
+		}
+		for _, p := range pods.Items {
+			if err := b.collectPodLogs(ctx, bw, p, ns == minecraft && !b.serverLogs); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// collectPodLogs keeps the tail of each container's log, and of its previous
+// run when it restarted. initOnly keeps only the init containers: a game
+// server's own log carries player names, addresses and chat.
+func (b *supportBundle) collectPodLogs(ctx context.Context, bw *bundleWriter, p corev1.Pod, initOnly bool) error {
+	type ctr struct {
+		name     string
+		restarts int32
+	}
+	var ctrs []ctr
+	restarts := map[string]int32{}
+	for _, cs := range append(append([]corev1.ContainerStatus(nil), p.Status.InitContainerStatuses...), p.Status.ContainerStatuses...) {
+		restarts[cs.Name] = cs.RestartCount
+	}
+	for _, c := range p.Spec.InitContainers {
+		ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
+	}
+	if !initOnly {
+		for _, c := range p.Spec.Containers {
+			ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
+		}
+	}
+	for _, c := range ctrs {
+		for _, previous := range []bool{false, true} {
+			if previous && c.restarts == 0 {
+				continue
+			}
+			name := fmt.Sprintf("logs/%s/%s/%s.log", p.Namespace, p.Name, c.name)
+			if previous {
+				name = fmt.Sprintf("logs/%s/%s/%s.previous.log", p.Namespace, p.Name, c.name)
+			}
+			out, err := b.logs(ctx, p.Namespace, p.Name, c.name, previous)
+			if err != nil {
+				bw.failed(name, err)
+				continue
+			}
+			if err := bw.logText(name, out); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// podTable is every pod on the node, one line each.
+func podTable(pods []corev1.Pod, now time.Time) []byte {
+	sort.Slice(pods, func(i, j int) bool {
+		if pods[i].Namespace != pods[j].Namespace {
+			return pods[i].Namespace < pods[j].Namespace
+		}
+		return pods[i].Name < pods[j].Name
+	})
+	var buf bytes.Buffer
+	tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "NAMESPACE\tNAME\tPHASE\tREADY\tRESTARTS\tAGE")
+	for _, p := range pods {
+		var ready, restarts int32
+		for _, cs := range p.Status.ContainerStatuses {
+			if cs.Ready {
+				ready++
+			}
+			restarts += cs.RestartCount
+		}
+		age := "-"
+		if !p.CreationTimestamp.IsZero() {
+			age = now.Sub(p.CreationTimestamp.Time).Round(time.Minute).String()
+		}
+		fmt.Fprintf(tw, "%s\t%s\t%s\t%d/%d\t%d\t%s\n", p.Namespace, p.Name, p.Status.Phase, ready, len(p.Spec.Containers), restarts, age)
+	}
+	tw.Flush()
+	return buf.Bytes()
+}
+
+// redactList takes out of every object what the bundle must not carry: the
+// literal env values of pod specs and of MinecraftServers (valueFrom
+// references stay, naming the Secret without its contents), the
+// last-applied-configuration annotation that repeats them, and managedFields.
+func redactList(list client.ObjectList) {
+	items, err := meta.ExtractList(list)
+	if err != nil {
+		return
+	}
+	for _, it := range items {
+		if acc, err := meta.Accessor(it); err == nil {
+			acc.SetManagedFields(nil)
+			if ann := acc.GetAnnotations(); ann != nil {
+				delete(ann, corev1.LastAppliedConfigAnnotation)
+				acc.SetAnnotations(ann)
+			}
+		}
+		switch o := it.(type) {
+		case *corev1.Pod:
+			redactPodSpec(&o.Spec)
+		case *appsv1.Deployment:
+			redactPodSpec(&o.Spec.Template.Spec)
+		case *appsv1.StatefulSet:
+			redactPodSpec(&o.Spec.Template.Spec)
+		case *batchv1.Job:
+			redactPodSpec(&o.Spec.Template.Spec)
+		case *batchv1.CronJob:
+			redactPodSpec(&o.Spec.JobTemplate.Spec.Template.Spec)
+		case *v1alpha1.MinecraftServer:
+			for i := range o.Spec.Env {
+				if o.Spec.Env[i].Value != "" {
+					o.Spec.Env[i].Value = redacted
+				}
+			}
+		}
+	}
+}
+
+func redactPodSpec(s *corev1.PodSpec) {
+	blank := func(cs []corev1.Container) {
+		for i := range cs {
+			for j := range cs[i].Env {
+				if cs[i].Env[j].Value != "" {
+					cs[i].Env[j].Value = redacted
+				}
+			}
+		}
+	}
+	blank(s.InitContainers)
+	blank(s.Containers)
+	for i := range s.EphemeralContainers {
+		for j := range s.EphemeralContainers[i].Env {
+			if s.EphemeralContainers[i].Env[j].Value != "" {
+				s.EphemeralContainers[i].Env[j].Value = redacted
+			}
+		}
+	}
+}
+
+// configSummary is felis.toml without a single credential: the hostnames,
+// namespaces and which features are on. Fields are picked one by one, so a
+// field added later stays out until someone decides it is safe.
+func configSummary(c *config.Config, path string) []byte {
+	var buf bytes.Buffer
+	line := func(k string, v any) { fmt.Fprintf(&buf, "%-34s %v\n", k, v) }
+	fmt.Fprintf(&buf, "# a summary of %s; no password, key or token is in it\n", path)
+	line("server.root_domain", c.Server.RootDomain)
+	line("server.listen", c.Server.Listen)
+	db := "(unparsable)"
+	if u, err := url.Parse(c.Database.URL); err == nil {
+		db = u.Scheme + "://" + u.User.Username() + "@" + u.Host + u.Path
+	}
+	line("database.url (no password)", db)
+	line("database.deployment", c.Database.Deployment)
+	line("velocity.public_ip", c.Velocity.PublicIP)
+	line("velocity.game_port", c.Velocity.GamePort)
+	line("velocity.login_image", c.Velocity.LoginImage)
+	line("velocity.lobby_image", c.Velocity.LobbyImage)
+	line("auth.panel_hostname", c.Auth.PanelHostname)
+	line("auth.admin_hostname", c.Auth.AdminHostname)
+	line("auth.client_ip_header", c.Auth.ClientIPHeader)
+	line("k8s.namespace", c.K8s.Namespace)
+	line("k8s.egress_mode", c.K8s.EgressMode)
+	line("k8s.metallb_pool", c.K8s.MetalLBPool)
+	line("registry.url", c.Registry.URL)
+	line("registry.build_namespace", c.Registry.BuildNamespace)
+	line("registry.trivy_db_repository", c.Registry.TrivyDBRepository)
+	line("registry.build_user_namespaces", c.Registry.BuildUserNamespaces)
+	line("registry.build_runtime_class", c.Registry.BuildRuntimeClass)
+	line("registry.max_concurrent_builds", c.Registry.MaxConcurrentBuilds)
+	line("registry.user_uploads_context", c.Registry.UserUploadsContext)
+	line("archive.store", c.Archive.Store)
+	line("archive.local_path", c.Archive.LocalPath)
+	line("archive.retention", c.Archive.Retention)
+	line("archive.scheduled_every", c.Archive.ScheduledEvery)
+	line("archive.scheduled_keep", c.Archive.ScheduledKeep)
+	line("offsite (configured)", c.Offsite.Enabled())
+	if c.Offsite.Enabled() {
+		line("offsite.endpoint", c.Offsite.Endpoint)
+		line("offsite.bucket", c.Offsite.Bucket)
+		line("offsite.prefix", c.Offsite.Prefix)
+	}
+	line("smtp.host", c.SMTP.Host)
+	if c.SMTP.Host != "" {
+		line("smtp.port", c.SMTP.Port)
+		line("smtp.require_tls", c.SMTP.TLSRequired())
+		line("smtp.max_per_hour", c.SMTP.MaxPerHour)
+	}
+	for i, s := range c.AuthSources {
+		line(fmt.Sprintf("auth_source[%d]", i), s.Tag+" "+s.Prefix+" "+s.URL)
+	}
+	return buf.Bytes()
+}
+
+func (b *supportBundle) manifest(bw *bundleWriter) []byte {
+	control, build, minecraft := b.bundleNamespaces()
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "Felis support bundle\nhost %s, collected %s, felis %s\n\n", b.host, b.now.UTC().Format(time.RFC3339), resolvedVersion())
+	fmt.Fprintf(&buf, `What it holds:
+  status.txt, doctor.txt   felis status and felis doctor at collection time
+  version.txt              the release, the OS and the kernel
+  config.txt               a summary of felis.toml: hostnames, namespaces, which features are on
+  host/                    systemd units and timers, disk use, memory, addresses, and the names,
+                           modes, sizes and times of the files under %s (not what is in them)
+  journal/                 up to %d lines per Felis unit and k3s, from the last %s
+  logs/                    up to %d lines of each container of the pods in %s and %s, and of the
+                           init containers of the game server pods in %s; the previous run too
+                           where a container restarted
+  cluster/                 nodes, volumes, the MinecraftServers, and the pods, workloads, services,
+                           volume claims, network policies and events of %s, %s and %s
+`, b.stateDir, b.logLines, shortDuration(b.since), b.logLines, control, build, minecraft, control, build, minecraft)
+	if b.serverLogs {
+		fmt.Fprintln(&buf, "\nThe game servers' own logs are in logs/ (-server-logs): they carry player names, IP addresses and chat.")
+	} else {
+		fmt.Fprintln(&buf, "\nThe game servers' own logs are left out (they carry player names, IP addresses and chat; -server-logs adds them).")
+	}
+	fmt.Fprint(&buf, `
+Never collected: Kubernetes Secrets and ConfigMaps, what is in /etc/felis or any configuration
+file, the database, worlds, uploads.
+
+Taken out:
+`)
+	if len(bw.scrub.sources) > 0 {
+		fmt.Fprintf(&buf, "  - %d secret values, wherever they appear, read from:\n", len(bw.scrub.vals))
+		for _, s := range bw.scrub.sources {
+			fmt.Fprintf(&buf, "      %s\n", s)
+		}
+	} else {
+		fmt.Fprintln(&buf, "  - no secret file was found on this host to take values from")
+	}
+	fmt.Fprint(&buf, `  - passwords in URLs, private keys, Bearer and Basic credentials
+  - in logs and command output, whatever follows password=, secret=, token=, api_key=,
+    access_key=, private_key= or credentials= (and the same with a colon)
+  - every literal env value in pod specs and MinecraftServers (valueFrom references stay)
+
+Read it through before you send it anywhere: redaction finds this host's known secrets and the
+common ways a secret is logged, and a secret logged another way stays in.
+`)
+	if b.wErr != nil {
+		fmt.Fprintf(&buf, "\nThe watchdog's settings: %v\n", b.wErr)
+	}
+	if len(bw.errs) > 0 {
+		fmt.Fprintln(&buf, "\nNot collected:")
+		for _, e := range bw.errs {
+			fmt.Fprintf(&buf, "  - %s\n", e)
+		}
+	}
+	// Its own words name what is redacted, and would be redacted themselves;
+	// what it quotes was scrubbed as it was recorded.
+	return buf.Bytes()
+}
+
+// secretSources are files that hold secrets, by how each is laid out.
+type secretSources struct {
+	envFiles   []string // KEY=VALUE lines, every value a secret (a commented-out one too)
+	valueFiles []string // one secret, the whole file
+	propsFiles []string // key=value lines; the keys naming a token, secret, password or key hold one
+	tokenFiles []string // k3s join tokens: the whole token and its secret part after the last ':'
+}
+
+// scrubber takes secrets out of what the bundle collects.
+type scrubber struct {
+	vals    []string // longest first, so a secret that contains another goes whole
+	sources []string // the files vals came from
+}
+
+var (
+	pemPrivateKey = regexp.MustCompile(`(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?-----END [A-Z0-9 ]*PRIVATE KEY-----`)
+	urlUserinfo   = regexp.MustCompile(`([A-Za-z][A-Za-z0-9+.-]*://[^/\s:@]*:)[^/\s@]+@`)
+	authScheme    = regexp.MustCompile(`(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}`)
+	secretAssign  = regexp.MustCompile(`(?i)((?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|credentials?)[A-Za-z0-9_.-]*"?[ \t]*[=:][ \t]*"?)([^\s"',;&]{4,})`)
+	secretPropKey = regexp.MustCompile(`(?i)token|secret|password|key`)
+)
+
+func (b *supportBundle) scrubber() *scrubber {
+	s := &scrubber{}
+	s.readSources(b.secrets)
+	if b.cfg != nil {
+		if u, err := url.Parse(b.cfg.Database.URL); err == nil {
+			if pw, ok := u.User.Password(); ok {
+				s.add(pw)
+			}
+		}
+		for _, ref := range []string{
+			b.cfg.Velocity.ServiceTokenRef, b.cfg.SMTP.PasswordRef,
+			b.cfg.Offsite.AccessKeyRef, b.cfg.Offsite.SecretKeyRef, b.cfg.Offsite.KeyRef,
+			b.cfg.Registry.S3.AccessKeyRef, b.cfg.Registry.S3.SecretKeyRef,
+			b.cfg.Archive.S3.AccessKeyRef, b.cfg.Archive.S3.SecretKeyRef,
+		} {
+			if ref != "" {
+				s.add(os.Getenv(ref))
+			}
+		}
+	}
+	if st, err := watchdog.LoadState(watchdog.NewestState(b.w.statePath, b.w.fallbackState)); err == nil {
+		s.add(st.SMTPPassword)
+	}
+	return s
+}
+
+func (s *scrubber) add(v string) bool {
+	v = strings.TrimSpace(v)
+	if len(v) < minScrubLen {
+		return false
+	}
+	for _, have := range s.vals {
+		if have == v {
+			return true
+		}
+	}
+	s.vals = append(s.vals, v)
+	sort.SliceStable(s.vals, func(i, j int) bool { return len(s.vals[i]) > len(s.vals[j]) })
+	return true
+}
+
+func (s *scrubber) readSources(src secretSources) {
+	read := func(p string, take func(content string) bool) {
+		raw, err := os.ReadFile(p)
+		if err != nil {
+			return
+		}
+		if take(string(raw)) {
+			s.sources = append(s.sources, p)
+		}
+	}
+	keyValues := func(content string, keep func(key string) bool) bool {
+		found := false
+		for _, line := range strings.Split(content, "\n") {
+			k, v, ok := strings.Cut(line, "=")
+			if !ok || !keep(strings.TrimSpace(k)) {
+				continue
+			}
+			v = strings.TrimSpace(v)
+			if len(v) >= 2 && (v[0] == '\'' || v[0] == '"') && v[len(v)-1] == v[0] {
+				v = v[1 : len(v)-1]
+			}
+			found = s.add(v) || found
+		}
+		return found
+	}
+	for _, p := range src.envFiles {
+		read(p, func(c string) bool { return keyValues(c, func(string) bool { return true }) })
+	}
+	for _, p := range src.propsFiles {
+		read(p, func(c string) bool { return keyValues(c, secretPropKey.MatchString) })
+	}
+	for _, p := range src.valueFiles {
+		read(p, func(c string) bool { return s.add(c) })
+	}
+	for _, p := range src.tokenFiles {
+		read(p, func(c string) bool {
+			c = strings.TrimSpace(c)
+			whole := s.add(c)
+			part := false
+			if i := strings.LastIndex(c, ":"); i >= 0 {
+				part = s.add(c[i+1:])
+			}
+			return whole || part
+		})
+	}
+}
+
+// values takes every known secret value out of data.
+func (s *scrubber) values(data []byte) []byte {
+	t := pemPrivateKey.ReplaceAllString(string(data), "<redacted private key>")
+	for _, v := range s.vals {
+		t = strings.ReplaceAll(t, v, redacted)
+	}
+	t = urlUserinfo.ReplaceAllString(t, "${1}"+redacted+"@")
+	t = authScheme.ReplaceAllString(t, "${1} "+redacted)
+	return []byte(t)
+}
+
+// text is values, and whatever a log names as a secret as well.
+func (s *scrubber) text(data []byte) []byte {
+	return secretAssign.ReplaceAll(s.values(data), []byte("${1}"+redacted))
+}

cmd/felis/supportbundle_test.go

@@ -0,0 +1,433 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/watchdog"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+// The secrets a bundle host holds, each of which must be nowhere in the bundle.
+var plantedSecrets = map[string]string{
+	"secrets.env SERVICE_TOKEN":     "svc-token-planted-0a1b2c3d",
+	"secrets.env DB_PASSWORD":       "db-pass-planted-4e5f6a7b",
+	"offsite.env FELIS_OFFSITE_KEY": "offsite-key-planted+8c9d/0e1f=",
+	"smtp-password":                 "smtp-pass-planted-2a3b",
+	"felis-link.properties token":   "props-token-planted-4c5d",
+	"k3s token secret part":         "k3s-secret-part-planted-6e7f",
+	"watchdog state relay password": "cached-relay-pw-planted-8a9b",
+	"pod env literal":               "env-literal-planted-0c1d",
+	"deployment env literal":        "deploy-env-planted-2e3f",
+	"MinecraftServer env literal":   "cr-env-planted-4a5b",
+	"last-applied annotation":       "annotation-planted-6c7d",
+	"logged password":               "hunter2-planted-8e9f",
+	"logged bearer":                 "bearerplanted0a1b2c3d",
+	"URL password":                  "urlpass-planted-4e5f",
+	"token in an error":             "errtoken-planted-0f1e",
+	"felis.host.toml DB password":   "cfg-db-pass-planted-1c2d",
+	"service token by its env ref":  "env-ref-token-planted-3e4f",
+	"init container env literal":    "init-env-planted-5a6b",
+	"ephemeral container env":       "ephemeral-env-planted-7c8d",
+	"statefulset env literal":       "sts-env-planted-9e0f",
+	"job env literal":               "job-env-planted-1a2b",
+	"cronjob env literal":           "cronjob-env-planted-3c4d",
+	"commented-out offsite key":     "old-offsite-key-planted-5e6f",
+	"doctor quoting a password":     "doctor-pw-planted-7a8b",
+	"status quoting a password":     "status-pw-planted-9c0d",
+	"journal quoting a password":    "journal-pw-planted-1e2f",
+}
+
+// bundleHost is a Felis host for felis support-bundle: its secret files, its
+// watchdog unit and state, a cluster with a control-plane pod that restarted
+// and a game server, and logs and a journal that name secrets.
+func bundleHost(t *testing.T) *supportBundle {
+	t.Helper()
+	p := plantedSecrets
+	dir := t.TempDir()
+	etc := filepath.Join(dir, "etc-felis")
+	for _, d := range []string{etc, filepath.Join(dir, "systemd"), filepath.Join(dir, "k3s")} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeTestFile(t, filepath.Join(etc, "secrets.env"), "SERVICE_TOKEN="+p["secrets.env SERVICE_TOKEN"]+"\nDB_PASSWORD="+p["secrets.env DB_PASSWORD"]+"\nSHORT=abc\n", 0o600)
+	writeTestFile(t, filepath.Join(etc, "offsite.env"), "# before the rotation\n# FELIS_OFFSITE_KEY="+p["commented-out offsite key"]+"\nFELIS_OFFSITE_KEY='"+p["offsite.env FELIS_OFFSITE_KEY"]+"'\n", 0o600)
+	writeTestFile(t, filepath.Join(etc, "smtp-password"), p["smtp-password"]+"\n", 0o600)
+	writeTestFile(t, filepath.Join(etc, "felis-link.properties"), "api-base-url=http://10.43.0.10:8081\nservice-token="+p["felis-link.properties token"]+"\nroot-domain=games.example.org\n", 0o640)
+	writeTestFile(t, filepath.Join(dir, "k3s", "token"), "K10deadbeefcafe::server:"+p["k3s token secret part"]+"\n", 0o600)
+	cfgPath := filepath.Join(etc, "felis.host.toml")
+	writeTestFile(t, cfgPath, "[database]\nurl = \"postgres://felis:"+p["felis.host.toml DB password"]+"@127.0.0.1:1/felis?sslmode=disable&connect_timeout=1\"\n"+
+		"[server]\nroot_domain = \"games.example.org\"\n[archive]\nstore = \"tarLocal\"\n[k8s]\negress_mode = \"nodeport\"\n"+
+		"[velocity]\nservice_token_ref = \"FELIS_BUNDLE_TEST_SERVICE_TOKEN\"\n", 0o600)
+	t.Setenv("FELIS_BUNDLE_TEST_SERVICE_TOKEN", p["service token by its env ref"])
+	statePath := filepath.Join(dir, "state.json")
+	if err := watchdog.SaveState(statePath, &watchdog.State{SMTPPassword: p["watchdog state relay password"]}); err != nil {
+		t.Fatal(err)
+	}
+	unitDir := filepath.Join(dir, "systemd")
+	writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+cfgPath+" -state "+statePath+"\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "felis-offsite.service"), "[Unit]\n", 0o644)
+	writeTestFile(t, filepath.Join(unitDir, "k3s.service"), "[Unit]\n", 0o644)
+	meminfo := filepath.Join(dir, "meminfo")
+	writeTestFile(t, meminfo, "MemTotal: 8000000 kB\nMemAvailable: 2000000 kB\n", 0o644)
+
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var w watchdogFlags
+	w, _, err = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	w.diskPaths = "/nonexistent-felis-bundle-test"
+
+	secretEnv := corev1.EnvVar{Name: "FELIS_SMTP_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
+		LocalObjectReference: corev1.LocalObjectReference{Name: "felis-smtp"}, Key: "password"}}}
+	replicas := int32(1)
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "felis-api-7d9", Namespace: "felis",
+				Annotations:   map[string]string{corev1.LastAppliedConfigAnnotation: `{"env":"` + p["last-applied annotation"] + `"}`, "felis.lolicon.best/kept": "yes"},
+				ManagedFields: []metav1.ManagedFieldsEntry{{Manager: "kubectl-client-side-apply"}},
+			},
+			Spec: corev1.PodSpec{
+				InitContainers: []corev1.Container{{Name: "wait-db", Image: "busybox", Env: []corev1.EnvVar{{Name: "FELIS_INIT_SETTING", Value: p["init container env literal"]}}}},
+				Containers: []corev1.Container{{Name: "api", Image: "felis-api:v1", Env: []corev1.EnvVar{
+					{Name: "FELIS_PLAIN_SETTING", Value: p["pod env literal"]}, secretEnv,
+				}}},
+				EphemeralContainers: []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{
+					Name: "debug", Env: []corev1.EnvVar{{Name: "FELIS_DEBUG_SETTING", Value: p["ephemeral container env"]}}}}},
+			},
+			Status: corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 1, Ready: true}}},
+		},
+		&appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
+			Spec: appsv1.DeploymentSpec{Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "felis-api"}},
+				Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "api", Env: []corev1.EnvVar{
+					{Name: "FELIS_DEPLOY_SETTING", Value: p["deployment env literal"]},
+				}}}}}},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "survival-0", Namespace: "minecraft"},
+			Spec: corev1.PodSpec{
+				InitContainers: []corev1.Container{{Name: "prepare-data"}, {Name: "egress-gate"}},
+				Containers:     []corev1.Container{{Name: "server"}},
+			},
+			Status: corev1.PodStatus{Phase: corev1.PodRunning},
+		},
+		&v1alpha1.MinecraftServer{
+			ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
+			Spec:       v1alpha1.MinecraftServerSpec{Env: []v1alpha1.EnvVar{{Name: "DISCORD_WEBHOOK", Value: p["MinecraftServer env literal"]}}},
+		},
+		&appsv1.StatefulSet{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-postgres", Namespace: "felis"},
+			Spec:       appsv1.StatefulSetSpec{Template: envTemplate("FELIS_STS_SETTING", p["statefulset env literal"])},
+		},
+		&batchv1.Job{
+			ObjectMeta: metav1.ObjectMeta{Name: "build-1", Namespace: "felis-build"},
+			Spec:       batchv1.JobSpec{Template: envTemplate("FELIS_JOB_SETTING", p["job env literal"])},
+		},
+		&batchv1.CronJob{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-reaper", Namespace: "felis"},
+			Spec:       batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: envTemplate("FELIS_CRON_SETTING", p["cronjob env literal"])}}},
+		},
+		&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}},
+	).Build()
+
+	secretLog := fmt.Sprintf("started with SERVICE_TOKEN=%s\nlogin password=%s ok\nAuthorization: Bearer %s\ndial postgres://felis:%s@db:5432/felis\n",
+		p["secrets.env SERVICE_TOKEN"], p["logged password"], p["logged bearer"], p["URL password"])
+	return &supportBundle{
+		host: "felis-test", now: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC), unitDir: unitDir,
+		w: w, cfg: cfg, cl: cl,
+		logs: func(_ context.Context, ns, pod, container string, previous bool) ([]byte, error) {
+			if container == "wait-db" {
+				return nil, errors.New("container \"wait-db\" is waiting to start; token=" + p["token in an error"])
+			}
+			return []byte(fmt.Sprintf("log of %s/%s/%s previous=%v\n%s", ns, pod, container, previous, secretLog)), nil
+		},
+		run: func(_ context.Context, name string, args ...string) ([]byte, error) {
+			switch name {
+			case "journalctl":
+				return []byte("journal of " + args[1] + "\nFELIS_OFFSITE_KEY=" + p["offsite.env FELIS_OFFSITE_KEY"] + "\nrelay " + p["smtp-password"] + " refused\n" +
+					"relay login with the cached " + p["watchdog state relay password"] + "\ndatabase auth failed for " + p["felis.host.toml DB password"] +
+					"\nservice token " + p["service token by its env ref"] + " rejected\nnode joined with " + p["k3s token secret part"] + "\n" +
+					"old copies sealed with " + p["commented-out offsite key"] + "\nretrying with password=" + p["journal quoting a password"] + "\n"), nil
+			case "systemctl", "df", "ip":
+				return []byte(name + " output\n"), nil
+			}
+			return nil, errors.New("unexpected " + name)
+		},
+		backups: func(context.Context) (map[string]time.Time, error) {
+			return nil, errors.New("connect: password=" + p["status quoting a password"])
+		},
+		doctor: func(_ context.Context, out io.Writer) {
+			fmt.Fprintf(out, "doctor report; the k3s token K10deadbeefcafe::server:%s leaked here\nprobe said password=%s\n", p["k3s token secret part"], p["doctor quoting a password"])
+		},
+		secrets: secretSources{
+			envFiles:   []string{filepath.Join(etc, "secrets.env"), filepath.Join(etc, "offsite.env"), filepath.Join(etc, "absent.env")},
+			valueFiles: []string{filepath.Join(etc, "smtp-password"), filepath.Join(etc, "uploads-s3-secret-key")},
+			propsFiles: []string{filepath.Join(etc, "felis-link.properties")},
+			tokenFiles: []string{filepath.Join(dir, "k3s", "token")},
+		},
+		logLines: 500, since: 48 * time.Hour,
+		meminfo: meminfo, osRelease: filepath.Join(dir, "os-release"), procVersion: filepath.Join(dir, "version"), stateDir: etc,
+	}
+}
+
+// envTemplate is a pod template whose one container sets name to a literal.
+func envTemplate(name, value string) corev1.PodTemplateSpec {
+	return corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Env: []corev1.EnvVar{{Name: name, Value: value}}}}}}
+}
+
+// readBundle is every file in the bundle at path, by its name inside the
+// bundle's directory, and each one's mode.
+func readBundle(t *testing.T, path string) (map[string]string, map[string]int64) {
+	t.Helper()
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr := tar.NewReader(gz)
+	files, modes := map[string]string{}, map[string]int64{}
+	prefix := strings.TrimSuffix(filepath.Base(path), ".tar.gz") + "/"
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		name, ok := strings.CutPrefix(hdr.Name, prefix)
+		if !ok {
+			t.Errorf("%s is outside the bundle's directory %s", hdr.Name, prefix)
+		}
+		raw, err := io.ReadAll(tr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		files[name], modes[name] = string(raw), hdr.Mode
+	}
+	return files, modes
+}
+
+func TestSupportBundle(t *testing.T) {
+	b := bundleHost(t)
+	out := filepath.Join(t.TempDir(), "support")
+	path, err := b.write(context.Background(), out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := filepath.Join(out, "felis-support-felis-test-20260927T120000Z.tar.gz"); path != want {
+		t.Errorf("path %s, want %s", path, want)
+	}
+	for p, want := range map[string]os.FileMode{out: 0o700 | os.ModeDir, path: 0o600} {
+		if st, err := os.Stat(p); err != nil || st.Mode() != want {
+			t.Errorf("%s: mode %v (err %v), want %v", p, st.Mode(), err, want)
+		}
+	}
+	if left, _ := filepath.Glob(filepath.Join(out, ".*partial")); len(left) != 0 {
+		t.Errorf("left behind %v", left)
+	}
+
+	files, modes := readBundle(t, path)
+	var names []string
+	for n := range files {
+		names = append(names, n)
+		if modes[n] != 0o600 {
+			t.Errorf("%s: mode %o in the archive, want 600", n, modes[n])
+		}
+	}
+	sort.Strings(names)
+	want := []string{
+		"MANIFEST.txt",
+		"cluster/felis-build/cronjobs.yaml", "cluster/felis-build/deployments.yaml", "cluster/felis-build/events.yaml", "cluster/felis-build/jobs.yaml",
+		"cluster/felis-build/networkpolicies.yaml", "cluster/felis-build/persistentvolumeclaims.yaml", "cluster/felis-build/pods.yaml",
+		"cluster/felis-build/services.yaml", "cluster/felis-build/statefulsets.yaml",
+		"cluster/felis/cronjobs.yaml", "cluster/felis/deployments.yaml", "cluster/felis/events.yaml", "cluster/felis/jobs.yaml",
+		"cluster/felis/networkpolicies.yaml", "cluster/felis/persistentvolumeclaims.yaml", "cluster/felis/pods.yaml",
+		"cluster/felis/services.yaml", "cluster/felis/statefulsets.yaml",
+		"cluster/minecraft/cronjobs.yaml", "cluster/minecraft/deployments.yaml", "cluster/minecraft/events.yaml", "cluster/minecraft/jobs.yaml",
+		"cluster/minecraft/networkpolicies.yaml", "cluster/minecraft/persistentvolumeclaims.yaml", "cluster/minecraft/pods.yaml",
+		"cluster/minecraft/services.yaml", "cluster/minecraft/statefulsets.yaml",
+		"cluster/minecraftservers.yaml", "cluster/nodes.yaml", "cluster/persistentvolumes.yaml", "cluster/pods-all-namespaces.txt",
+		"config.txt", "doctor.txt",
+		"host/addresses.txt", "host/df.txt", "host/etc-felis.txt", "host/meminfo.txt", "host/systemd-timers.txt", "host/systemd-units.txt",
+		"journal/felis-offsite.log", "journal/felis-watchdog.log", "journal/k3s.log",
+		"logs/felis/felis-api-7d9/api.log", "logs/felis/felis-api-7d9/api.previous.log",
+		"logs/minecraft/survival-0/egress-gate.log", "logs/minecraft/survival-0/prepare-data.log",
+		"status.txt", "version.txt",
+	}
+	if strings.Join(names, "\n") != strings.Join(want, "\n") {
+		t.Errorf("bundle holds\n  %s\nwant\n  %s", strings.Join(names, "\n  "), strings.Join(want, "\n  "))
+	}
+
+	for what, secret := range plantedSecrets {
+		for n, body := range files {
+			if strings.Contains(body, secret) {
+				t.Errorf("%s (%s) is in %s:\n%s", what, secret, n, body)
+			}
+		}
+	}
+
+	// What was taken out leaves what a reader needs around it.
+	for name, wants := range map[string][]string{
+		"logs/felis/felis-api-7d9/api.previous.log": {"log of felis/felis-api-7d9/api previous=true\n", "SERVICE_TOKEN=<redacted>\n", "login password=<redacted> ok\n", "Authorization: Bearer <redacted>\n", "postgres://felis:<redacted>@db:5432/felis\n"},
+		"journal/k3s.log": {"journal of k3s.service\n", "FELIS_OFFSITE_KEY=<redacted>\n", "relay <redacted> refused\n",
+			"relay login with the cached <redacted>\n", "database auth failed for <redacted>\n", "service token <redacted> rejected\n", "node joined with <redacted>\n"},
+		"cluster/felis/statefulsets.yaml": {"name: FELIS_STS_SETTING\n"},
+		"cluster/felis/cronjobs.yaml":     {"name: FELIS_CRON_SETTING\n"},
+		"cluster/felis-build/jobs.yaml":   {"name: FELIS_JOB_SETTING\n"},
+		"cluster/felis/pods.yaml":         {"name: FELIS_PLAIN_SETTING\n        value: <redacted>\n", "name: FELIS_SMTP_PASSWORD\n        valueFrom:\n          secretKeyRef:\n            key: password\n            name: felis-smtp\n", "felis.lolicon.best/kept: \"yes\"", "name: FELIS_INIT_SETTING\n", "name: FELIS_DEBUG_SETTING\n"},
+		"cluster/felis/deployments.yaml":  {"name: FELIS_DEPLOY_SETTING\n            value: <redacted>\n"},
+		"cluster/minecraftservers.yaml":   {"name: DISCORD_WEBHOOK\n      value: <redacted>\n"},
+		"config.txt":                      {fmt.Sprintf("%-34s %s\n", "server.root_domain", "games.example.org"), fmt.Sprintf("%-34s %s\n", "database.url (no password)", "postgres://[email protected]:1/felis")},
+		"doctor.txt":                      {"the k3s token <redacted> leaked here\nprobe said password=<redacted>\n"},
+		"status.txt":                      {"  world backups unknown: connect: password=<redacted>\n"},
+		"host/etc-felis.txt":              {"secrets.env\n", "smtp-password\n", "felis-link.properties\n"},
+		"MANIFEST.txt": {
+			"The game servers' own logs are left out",
+			"  journal/                 up to 500 lines per Felis unit and k3s, from the last 48h\n",
+			"  - 11 secret values, wherever they appear, read from:\n",
+			"secrets.env\n", "offsite.env\n", "smtp-password\n", "felis-link.properties\n", "token\n",
+			"logs/felis/felis-api-7d9/wait-db.log: container \"wait-db\" is waiting to start; token=<redacted>\n",
+			// Its own words about what is redacted come through whole.
+			"  - passwords in URLs, private keys, Bearer and Basic credentials\n",
+			"access_key=, private_key= or credentials= (and the same with a colon)\n",
+			"Read it through before you send it anywhere",
+		},
+	} {
+		for _, w := range wants {
+			if !strings.Contains(files[name], w) {
+				t.Errorf("%s lacks %q:\n%s", name, w, files[name])
+			}
+		}
+	}
+	for _, gone := range []string{"managedFields", "last-applied-configuration"} {
+		if strings.Contains(files["cluster/felis/pods.yaml"], gone) {
+			t.Errorf("pods.yaml keeps %s:\n%s", gone, files["cluster/felis/pods.yaml"])
+		}
+	}
+	if strings.Contains(files["MANIFEST.txt"], "absent.env") || strings.Contains(files["MANIFEST.txt"], "uploads-s3-secret-key") {
+		t.Errorf("MANIFEST names files this host does not have:\n%s", files["MANIFEST.txt"])
+	}
+}
+
+// -server-logs adds the game servers' own logs, and says so.
+func TestSupportBundleServerLogs(t *testing.T) {
+	b := bundleHost(t)
+	b.serverLogs = true
+	path, err := b.write(context.Background(), t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	files, _ := readBundle(t, path)
+	if _, ok := files["logs/minecraft/survival-0/server.log"]; !ok {
+		t.Error("no server.log with -server-logs")
+	}
+	if !strings.Contains(files["MANIFEST.txt"], "The game servers' own logs are in logs/ (-server-logs)") {
+		t.Errorf("MANIFEST does not say server logs are in:\n%s", files["MANIFEST.txt"])
+	}
+}
+
+// With the cluster and the configuration gone the bundle still holds what the
+// host shows, and MANIFEST says what it could not collect.
+func TestSupportBundleWithTheClusterDown(t *testing.T) {
+	b := bundleHost(t)
+	b.cl, b.clErr = nil, errors.New("connection refused")
+	b.cfg, b.cfgErr = nil, errors.New("felis.host.toml: no such file")
+	b.wErr = errors.New("felis-watchdog.service is not installed; read the watchdog's defaults")
+	path, err := b.write(context.Background(), t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	files, _ := readBundle(t, path)
+	for _, n := range []string{"status.txt", "doctor.txt", "host/etc-felis.txt", "journal/k3s.log"} {
+		if _, ok := files[n]; !ok {
+			t.Errorf("no %s", n)
+		}
+	}
+	for n := range files {
+		if strings.HasPrefix(n, "cluster/") || strings.HasPrefix(n, "logs/") || n == "config.txt" {
+			t.Errorf("%s with no cluster and no configuration", n)
+		}
+	}
+	if !strings.Contains(files["status.txt"], "felis status: the configuration did not load: felis.host.toml: no such file") {
+		t.Errorf("status.txt:\n%s", files["status.txt"])
+	}
+	if !strings.Contains(files["MANIFEST.txt"], "\nThe watchdog's settings: felis-watchdog.service is not installed; read the watchdog's defaults\n") ||
+		!strings.Contains(files["MANIFEST.txt"], "  - cluster: connection refused\n") {
+		t.Errorf("MANIFEST.txt:\n%s", files["MANIFEST.txt"])
+	}
+}
+
+func TestShortDuration(t *testing.T) {
+	for d, want := range map[time.Duration]string{
+		48 * time.Hour:             "48h",
+		90 * time.Minute:           "1h30m",
+		45 * time.Minute:           "45m",
+		30 * time.Second:           "30s",
+		time.Hour + 30*time.Second: "1h0m30s",
+	} {
+		if got := shortDuration(d); got != want {
+			t.Errorf("shortDuration(%v) = %q, want %q", d, got, want)
+		}
+	}
+}
+
+func TestScrubber(t *testing.T) {
+	s := &scrubber{}
+	for _, v := range []string{"known-secret-value", "known-secret-value-longer", "short", "known-secret-value"} {
+		s.add(v)
+	}
+	if got := strings.Join(s.vals, ","); got != "known-secret-value-longer,known-secret-value" {
+		t.Errorf("kept %q; want each value once, longest first, none under %d characters", s.vals, minScrubLen)
+	}
+	for in, want := range map[string]string{
+		"a known-secret-value-longer b":                                     "a <redacted> b",
+		"a known-secret-value b":                                            "a <redacted> b",
+		"password=abcd1234 next":                                            "password=<redacted> next",
+		`{"token": "abcd1234"}`:                                             `{"token": "<redacted>"}`,
+		"SMTP_PASSWORD: s3cr3t!x":                                           "SMTP_PASSWORD: <redacted>",
+		"x-api-key=zzzz9999&q=1":                                            "x-api-key=<redacted>&q=1",
+		"Authorization: Basic dXNlcjpwYXNzd29yZA==":                         "Authorization: Basic <redacted>",
+		"s3://AKIA:secretpart123@bucket/key":                                "s3://AKIA:<redacted>@bucket/key",
+		"https://user@host/path":                                            "https://user@host/path",
+		"-----BEGIN EC PRIVATE KEY-----\nMHc\n-----END EC PRIVATE KEY-----": "<redacted private key>",
+		"tokens: 3": "tokens: 3",
+		"read the relay password (keeping the cached one)": "read the relay password (keeping the cached one)",
+		`secrets "felis-smtp" not found`:                   `secrets "felis-smtp" not found`,
+	} {
+		if got := string(s.text([]byte(in))); got != want {
+			t.Errorf("text(%q) = %q, want %q", in, got, want)
+		}
+	}
+	// Structured files keep what only looks like a secret.
+	if got := string(s.values([]byte("secretName: felis-forwarding-secret and known-secret-value"))); got != "secretName: felis-forwarding-secret and <redacted>" {
+		t.Errorf("values() = %q", got)
+	}
+}

cmd/felis/systemservers.go

@@ -1,20 +1,21 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"fmt"
-	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/naming"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -95,7 +96,7 @@ const felisLimboHealthPort int32 = 8080
 // fail-safes to readiness-only, so a login pod that has the URL/domain but not yet
 // the token is safe (it simply does not authenticate) rather than broken.
 const (
-	envAPIBaseURL    = "FELIS_API_BASE_URL"
+	envAPIBaseURL    = naming.EnvAPIBaseURL
 	envRootDomain    = "FELIS_ROOT_DOMAIN"
 	envPanelHostname = "FELIS_PANEL_HOSTNAME"
 	envLobbyServer   = "FELIS_LOBBY_SERVER"
@@ -241,6 +242,15 @@ func buildSystemServerClient() (client.Client, error) {
 	if err := v1alpha1.AddToScheme(scheme); err != nil {
 		return nil, err
 	}
+	cfg, err := hostRESTConfig()
+	if err != nil {
+		return nil, err
+	}
+	return client.New(cfg, client.Options{Scheme: scheme})
+}
+
+// hostRESTConfig is the cluster connection buildSystemServerClient describes.
+func hostRESTConfig() (*rest.Config, error) {
 	cfg, err := ctrl.GetConfig()
 	if err != nil {
 		cfg, err = clientcmd.BuildConfigFromFlags("", hostBootstrapKubeconfigPath)
@@ -248,17 +258,38 @@ func buildSystemServerClient() (client.Client, error) {
 			return nil, fmt.Errorf("no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and %s): %w", hostBootstrapKubeconfigPath, err)
 		}
 	}
-	return client.New(cfg, client.Options{Scheme: scheme})
+	return cfg, nil
 }
 
 // systemServerOutcome records what ensureSystemServers did with one service so
 // setup can report it without the provisioner deciding on the output format.
 type systemServerOutcome struct {
 	name      string
-	created   bool   // true = we created it this run
-	available bool   // true = the required object now exists
-	skipped   string // non-empty = why it was skipped (image unset / already exists)
-	err       error  // non-nil = create failed
+	created   bool     // true = we created it this run
+	updated   bool     // true = we refreshed an existing replica from the source
+	available bool     // true = the required object now exists
+	skipped   string   // non-empty = why it was skipped (image unset / already exists)
+	err       error    // non-nil = create failed
+	changes   []string // converge only: the fields this pass filled
+}
+
+// systemServerPlan is one system service in the provisioner's table: its name,
+// the image config gives it, and the pure builder for its desired CR.
+type systemServerPlan struct {
+	name  string
+	image string
+	build func(image, namespace string) (*v1alpha1.MinecraftServer, error)
+}
+
+// systemServerPlans is the single description of the login+lobby pair, shared by
+// ensureSystemServers (create-if-absent) and convergeSystemServers (field fill).
+func systemServerPlans(loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname string) []systemServerPlan {
+	return []systemServerPlan{
+		{name: naming.SystemLoginServer, image: loginImage, build: func(image, ns string) (*v1alpha1.MinecraftServer, error) {
+			return loginSystemServer(image, ns, apiBaseURL, rootDomain, panelHostname)
+		}},
+		{name: naming.SystemLobbyServer, image: lobbyImage, build: lobbySystemServer},
+	}
 }
 
 // ensureSystemServers idempotently creates the login and lobby system services.
@@ -269,17 +300,7 @@ type systemServerOutcome struct {
 // K8s client and namespace; this function performs no signal-handler or client
 // setup of its own.
 func ensureSystemServers(ctx context.Context, cl client.Client, namespace, loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname string) []systemServerOutcome {
-	type plan struct {
-		name  string
-		image string
-		build func(image, namespace string) (*v1alpha1.MinecraftServer, error)
-	}
-	plans := []plan{
-		{name: naming.SystemLoginServer, image: loginImage, build: func(image, ns string) (*v1alpha1.MinecraftServer, error) {
-			return loginSystemServer(image, ns, apiBaseURL, rootDomain, panelHostname)
-		}},
-		{name: naming.SystemLobbyServer, image: lobbyImage, build: lobbySystemServer},
-	}
+	plans := systemServerPlans(loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname)
 
 	outcomes := make([]systemServerOutcome, 0, len(plans))
 	for _, p := range plans {
@@ -379,118 +400,231 @@ var derivedSystemEnv = map[string]bool{
 // deliberate removal is indistinguishable from drift and re-adding it would fight the
 // operator every run.
 func refreshDerivedEnv(ctx context.Context, cl client.Client, existing, desired *v1alpha1.MinecraftServer) (bool, error) {
+	want := derivedEnvWanted(desired)
+	changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
+		changed := false
+		for i, e := range existing.Spec.Env {
+			if v, ok := want[e.Name]; ok && v != e.Value {
+				existing.Spec.Env[i].Value = v
+				changed = true
+			}
+		}
+		return changed
+	})
+	if err != nil {
+		return false, fmt.Errorf("refresh %s env: %w", existing.Name, err)
+	}
+	return changed, nil
+}
+
+// patchOnConflictRetry applies mutate to obj and sends only the difference, as a
+// merge patch that carries the resourceVersion obj was read at. The operator writes
+// status and felis-api patches spec.idle on these same objects, so a write can land
+// between setup's read and its patch: the apiserver then answers 409, and this
+// re-reads obj and runs mutate again on the fresh copy, up to retry.DefaultRetry's
+// five attempts. The pinned resourceVersion is what keeps a list field such as
+// spec.env safe — a merge patch replaces a list whole, and without the lock an
+// entry added concurrently would be dropped. mutate reports whether it changed
+// anything; nothing is sent when it did not. obj holds the stored object after.
+func patchOnConflictRetry(ctx context.Context, cl client.Client, obj client.Object, mutate func() bool) (bool, error) {
+	key := client.ObjectKeyFromObject(obj)
+	changed, reread := false, false
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		if reread {
+			if err := cl.Get(ctx, key, obj); err != nil {
+				return err
+			}
+		}
+		reread = true
+		base := obj.DeepCopyObject().(client.Object)
+		if changed = mutate(); !changed {
+			return nil
+		}
+		return cl.Patch(ctx, obj, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}))
+	})
+	return changed, err
+}
+
+// derivedEnvWanted maps the derived env keys of desired onto their values.
+func derivedEnvWanted(desired *v1alpha1.MinecraftServer) map[string]string {
 	want := make(map[string]string, len(derivedSystemEnv))
 	for _, e := range desired.Spec.Env {
 		if derivedSystemEnv[e.Name] {
 			want[e.Name] = e.Value
 		}
 	}
+	return want
+}
 
-	changed := false
-	for i, e := range existing.Spec.Env {
+// convergeSystemServers is the explicit convergence pass over already-installed
+// system servers (#1). ensureSystemServers is create-if-absent by design — an
+// existing CR is left alone so a re-run cannot clobber an operator's edits — and
+// that leaves no path for a field the DESIRED spec gained after the install:
+// spec.rcon (the lobby's write channel), spec.startup.healthHTTPPort (the login
+// gate's readiness probe), or a config-derived env key that did not exist yet.
+// Such fields sit at their zero value forever while re-running setup reports
+// success, which is exactly the reported "configuration updates never reach an
+// installed deployment" symptom.
+//
+// This pass fills exactly those zero-value fields and the config-derived env keys,
+// and nothing else: a field already holding a non-zero value is the operator's and
+// is never overwritten. It is an explicit command rather than an implicit step of
+// setup because some fills need an ordering only the operator knows — enabling
+// RCON or the HTTP readiness gate on a server whose image predates the listener
+// would hold that server in Starting until it was marked Failed. Rebuild (or
+// upgrade) the images first, then run this.
+func convergeSystemServers(ctx context.Context, cl client.Client, namespace, loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname string) []systemServerOutcome {
+	outcomes := make([]systemServerOutcome, 0, 2)
+	for _, p := range systemServerPlans(loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname) {
+		if p.image == "" {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, skipped: "image not configured"})
+			continue
+		}
+		desired, err := p.build(p.image, namespace)
+		if err != nil {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: err})
+			continue
+		}
+
+		var existing v1alpha1.MinecraftServer
+		switch err := cl.Get(ctx, client.ObjectKeyFromObject(desired), &existing); {
+		case apierrors.IsNotFound(err):
+			outcomes = append(outcomes, systemServerOutcome{name: p.name,
+				skipped: "not present — run `sudo felis setup` first"})
+			continue
+		case err != nil:
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: err})
+			continue
+		}
+		if existing.Labels[v1alpha1.LabelSystemRole] != p.name {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf(
+				"existing MinecraftServer %s/%s is not marked as the Felis %q system role; refusing to converge it",
+				namespace, p.name, p.name,
+			)})
+			continue
+		}
+
+		var changes []string
+		changed, err := patchOnConflictRetry(ctx, cl, &existing, func() bool {
+			changes = nil
+			if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
+				existing.Spec.Rcon = desired.Spec.Rcon
+				changes = append(changes, "spec.rcon")
+			}
+			if existing.Spec.Startup.HealthHTTPPort == 0 && desired.Spec.Startup.HealthHTTPPort != 0 {
+				existing.Spec.Startup.HealthHTTPPort = desired.Spec.Startup.HealthHTTPPort
+				changes = append(changes, "spec.startup.healthHTTPPort")
+			}
+			changes = append(changes, convergeDerivedEnv(&existing, desired)...)
+			return len(changes) > 0
+		})
+		if err != nil {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
+			continue
+		}
+		if !changed {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
+			continue
+		}
+		outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, updated: true, changes: changes})
+	}
+	return outcomes
+}
+
+// convergeDerivedEnv makes the config-derived env match the desired values: a key
+// whose value drifted is overwritten, and a key missing entirely is added. This is
+// the wider half of the same explicit pass — refreshDerivedEnv's present-only loop
+// can never introduce a NEW key, which is how a derived key added after an install
+// never reached it at all.
+func convergeDerivedEnv(existing, desired *v1alpha1.MinecraftServer) []string {
+	want := derivedEnvWanted(desired)
+	var changes []string
+	present := make(map[string]bool, len(existing.Spec.Env))
+	for i := range existing.Spec.Env {
+		e := &existing.Spec.Env[i]
+		present[e.Name] = true
 		if v, ok := want[e.Name]; ok && v != e.Value {
-			existing.Spec.Env[i].Value = v
-			changed = true
+			e.Value = v
+			changes = append(changes, "env "+e.Name)
 		}
 	}
-	if !changed {
-		return false, nil
+	for _, e := range desired.Spec.Env {
+		if !derivedSystemEnv[e.Name] || present[e.Name] {
+			continue
+		}
+		existing.Spec.Env = append(existing.Spec.Env, e)
+		changes = append(changes, "env "+e.Name)
 	}
-	if err := cl.Update(ctx, existing); err != nil {
-		return false, fmt.Errorf("refresh %s env: %w", existing.Name, err)
-	}
-	return true, nil
+	return changes
 }
 
-// The login gate is a hard prerequisite of the Owner bind, so setup waits for it
-// rather than racing it. The ceiling covers a cold image pull on a fresh node;
-// the poll is fast enough that a warm start feels immediate.
-const (
-	loginGateReadyTimeout = 5 * time.Minute
-	loginGatePollInterval = 3 * time.Second
-)
+// provisionSecretReplicas copies the Secrets workload pods mount from the control
+// namespace into the namespaces those pods run in. The proxy's felis-service-token
+// is not among them: it lives in the control namespace and on the host, and a copy
+// anywhere else would open every game route to whoever reads that namespace.
+func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, minecraftNS, buildNS string) []systemServerOutcome {
+	return []systemServerOutcome{
+		// refresh=true for both caller tokens: the control namespace holds the
+		// current value and `felis rotate-token` replaces it there, so a replica
+		// that differs is stale and the login gate would be turned away with it.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.LimboTokenSecretName, naming.ServiceTokenSecretKey, "limbo-token", "minecraft ns", true),
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
+		// refresh=true: felis-config is the rendered config, not a credential. The
+		// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
+		// must update it when the control plane's render has moved on (a stale copy
+		// e.g. keeps an old database URL after a credential rotation).
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-config", "felis.toml", "config", "minecraft ns", true),
+		// The reaper's pre-reap warning emails authenticate with the same relay
+		// password felis-api uses; the reaper pod runs in the minecraft namespace,
+		// where a secretKeyRef resolves only against a local mirror. Skipped while
+		// the relay is not configured yet — the "configure email" screen refreshes
+		// both mirrors when it applies.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-smtp", "password", "smtp", "minecraft ns", false),
+		// The build namespace needs the build token: the build Job's fetch
+		// initContainer reads the submission context from the internal face, and
+		// that is all this token opens. Best effort — a deployment that only
+		// installs the control plane simply never builds a user submission.
+		ensureSecretReplica(ctx, cl, controlNS, buildNS,
+			naming.BuildTokenSecretName, naming.ServiceTokenSecretKey, "build-token", "felis-build ns", true),
+	}
+}
 
-// awaitLoginGateReady blocks until the login system server reports status.ready.
+// ensureSecretReplica copies one Secret from the control namespace into a workload
+// namespace (minecraft — or the build namespace, whose fetch initContainer reads the
+// context from the felis-api internal face with the build token) so a pod can mount it
+// via secretKeyRef. A secretKeyRef is namespace-local, but those workloads do not run
+// beside the control plane — so without this replica the secretKeyRef would dangle and
+// wedge the pod in CreateContainerConfigError.
 //
-// The Owner claims their seat by JOINING the game and running /link, so the gate
-// being up is not a nicety — it is the precondition for the very next thing setup
-// asks of the operator. progress is called on each phase change so the caller can
-// show movement during a cold image pull; it may be nil.
-func awaitLoginGateReady(ctx context.Context, cl client.Client, namespace string, timeout, poll time.Duration, progress func(v1alpha1.Phase)) error {
-	key := client.ObjectKey{Namespace: namespace, Name: naming.SystemLoginServer}
-	deadline := time.Now().Add(timeout)
-	last := v1alpha1.Phase("")
-	for {
-		var ms v1alpha1.MinecraftServer
-		switch err := cl.Get(ctx, key, &ms); {
-		case err == nil:
-			if ms.Status.Ready {
-				return nil
-			}
-			if ms.Status.Phase != last {
-				last = ms.Status.Phase
-				if progress != nil {
-					progress(last)
-				}
-			}
-			// The operator only marks Failed once its OWN startup deadline has already
-			// elapsed, so Failed is a settled verdict rather than a transient — sitting
-			// out the rest of our timeout on top of it would only hide the reason.
-			if ms.Status.Phase == v1alpha1.PhaseFailed {
-				return fmt.Errorf("the login gate failed to start: %s", readyConditionMessage(&ms))
-			}
-		case !apierrors.IsNotFound(err):
-			return err
-		}
-		if !time.Now().Before(deadline) {
-			return fmt.Errorf("timed out after %s waiting for the login gate to become ready (last phase: %s)", timeout, phaseOrPending(last))
-		}
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-time.After(poll):
-		}
-	}
-}
-
-// readyConditionMessage is the operator's own account of why the gate is not
-// ready — far more useful to an operator than "phase: Failed".
-func readyConditionMessage(ms *v1alpha1.MinecraftServer) string {
-	if c := meta.FindStatusCondition(ms.Status.Conditions, v1alpha1.ConditionReady); c != nil && c.Message != "" {
-		return c.Message
-	}
-	return "no Ready condition was reported"
-}
-
-// phaseOrPending names the empty phase, which means the operator has not
-// reconciled the server yet (commonly: the operator itself is not running).
-func phaseOrPending(p v1alpha1.Phase) string {
-	if p == "" {
-		return "not yet reconciled — is the felis operator running?"
-	}
-	return string(p)
-}
-
-// ensureSecretReplica copies one Secret from the control namespace into the minecraft
-// namespace so a backend pod can mount it via secretKeyRef. A secretKeyRef is
-// namespace-local, but the backends run in the minecraft namespace while the sources
-// of truth live beside the control plane — so without this replica the operator's
-// injected secretKeyRef would dangle and wedge the pod in CreateContainerConfigError.
-//
-// Two Secrets need it, for different reasons: the service token (login only — it
-// authenticates the limbo plugin to the felis-api internal face) and the Velocity
-// modern-forwarding secret (every backend — it is how a backend knows a login really
-// came from the proxy, and so that the player's UUID is Mojang-verified rather than
-// offline-derived).
+// Several Secrets need it, for different reasons: the caller tokens of the login
+// limbo and the build Pod's context fetch (both authenticate to the felis-api
+// internal face, each with its own token),
+// the Velocity modern-forwarding secret (every backend — it is how a backend knows
+// a login really came from the proxy, and so that the player's UUID is Mojang-verified
+// rather than offline-derived), and the SMTP relay password (the reaper's pre-reap
+// warning emails; the felis-config mirror is what carries [smtp] into its pod).
 //
 // It is create-if-absent: an existing replica is left untouched so a hand-rotated
-// value in the minecraft namespace is never clobbered (to rotate, delete the replica
+// value in the workload namespace is never clobbered (to rotate, delete the replica
 // and re-run setup). Best-effort like the rest of the provisioner: a missing source or
 // a create failure degrades to a reported outcome, never a hard setup failure. It
 // copies only Type and Data — never labels/annotations/ownerRefs — so the replica
 // carries no accidental GC owner or managed-by lineage.
-func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label string) systemServerOutcome {
-	name := label + " (minecraft ns)"
+//
+// refreshExisting switches a replica to refresh-in-place from the control namespace.
+// The felis-config mirror uses it because that Secret is a rendered config and the
+// workload Jobs that mount it (backup/restore/fileedit) plus the reaper silently
+// misbehave on a stale copy — e.g. after a database credential rotation the control
+// plane moves on while every backup Job keeps failing auth. The caller tokens use it
+// because `felis rotate-token` replaces them in the control namespace, which makes a
+// differing replica stale by definition. The forwarding and SMTP Secrets keep the
+// never-overwrite rule.
+func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label, where string, refreshExisting bool) systemServerOutcome {
+	name := label + " (" + where + ")"
 	validate := func(secret *corev1.Secret, location, skipped string) systemServerOutcome {
 		if len(secret.Data[secretKey]) == 0 {
 			return systemServerOutcome{name: name, skipped: fmt.Sprintf(
@@ -498,6 +632,39 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 		}
 		return systemServerOutcome{name: name, available: true, skipped: skipped}
 	}
+	// refreshFromControl updates an existing replica from the control-namespace source
+	// when the rendered key differs. Only the felis-config mirror opts in.
+	refreshFromControl := func(existing *corev1.Secret) systemServerOutcome {
+		var src corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: controlNamespace, Name: secretName}, &src); err != nil {
+			if apierrors.IsNotFound(err) {
+				return systemServerOutcome{name: name, skipped: fmt.Sprintf(
+					"source Secret %s/%s not found — provision it (deploy/bootstrap.sh), then re-run setup",
+					controlNamespace, secretName)}
+			}
+			return systemServerOutcome{name: name, err: err}
+		}
+		if out := validate(&src, controlNamespace, ""); !out.available {
+			return out
+		}
+		changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
+			if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
+				return false
+			}
+			if existing.Data == nil {
+				existing.Data = map[string][]byte{}
+			}
+			existing.Data[secretKey] = src.Data[secretKey]
+			return true
+		})
+		if err != nil {
+			return systemServerOutcome{name: name, err: err}
+		}
+		if !changed {
+			return validate(existing, minecraftNamespace, "already current")
+		}
+		return systemServerOutcome{name: name, updated: true, available: true}
+	}
 	if controlNamespace == minecraftNamespace {
 		// Same namespace needs no replica, but the source still has to exist.
 		var existing corev1.Secret
@@ -516,6 +683,9 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 	var existing corev1.Secret
 	getErr := cl.Get(ctx, client.ObjectKey{Namespace: minecraftNamespace, Name: secretName}, &existing)
 	if getErr == nil {
+		if refreshExisting {
+			return refreshFromControl(&existing)
+		}
 		return validate(&existing, minecraftNamespace, "already exists")
 	}
 	if !apierrors.IsNotFound(getErr) {
@@ -544,30 +714,12 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 			if getErr := cl.Get(ctx, client.ObjectKey{Namespace: minecraftNamespace, Name: secretName}, &existing); getErr != nil {
 				return systemServerOutcome{name: name, err: getErr}
 			}
+			if refreshExisting {
+				return refreshFromControl(&existing)
+			}
 			return validate(&existing, minecraftNamespace, "already exists")
 		}
 		return systemServerOutcome{name: name, err: err}
 	}
 	return systemServerOutcome{name: name, created: true, available: true}
 }
-
-func requiredProvisioningError(outcomes []systemServerOutcome) error {
-	required := map[string]struct{}{
-		"service-token (minecraft ns)":     {},
-		"forwarding-secret (minecraft ns)": {},
-		naming.SystemLoginServer:           {},
-	}
-	for _, o := range outcomes {
-		if o.err != nil {
-			return fmt.Errorf("%s: %w", o.name, o.err)
-		}
-		if _, ok := required[o.name]; ok && !o.available {
-			reason := o.skipped
-			if reason == "" {
-				reason = "object was not created"
-			}
-			return fmt.Errorf("%s unavailable: %s", o.name, reason)
-		}
-	}
-	return nil
-}

cmd/felis/systemservers_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"strings"
 	"testing"
-	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/naming"
@@ -142,21 +141,21 @@ func TestLoginSystemServerEnv(t *testing.T) {
 // namespace (create-if-absent), so the operator's secretKeyRef on the backend pod
 // resolves. It must not overwrite an existing replica, and must degrade gracefully
 // when the source is missing or the namespaces coincide. Exercised here with the
-// service token; setup runs it a second time for the Velocity forwarding secret.
+// Velocity forwarding secret, which setup replicates in this never-overwrite mode.
 func TestEnsureSecretReplica(t *testing.T) {
 	scheme := newSystemServerScheme(t)
 	ctx := context.Background()
 
 	srcSecret := func() *corev1.Secret {
 		return &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "felis"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "felis"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("s3cr3t")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("s3cr3t")},
 		}
 	}
 	replicate := func(cl client.Client, controlNS, mcNS string) systemServerOutcome {
 		return ensureSecretReplica(ctx, cl, controlNS, mcNS,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token")
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false)
 	}
 
 	t.Run("replicates when absent", func(t *testing.T) {
@@ -166,19 +165,19 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want created", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "s3cr3t" {
-			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ServiceTokenSecretKey])
+		if string(replica.Data[naming.ForwardingSecretKey]) != "s3cr3t" {
+			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ForwardingSecretKey])
 		}
 	})
 
 	t.Run("does not overwrite existing replica", func(t *testing.T) {
 		existing := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("rotated")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("rotated")},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), existing).Build()
 		out := replicate(cl, "felis", "minecraft")
@@ -186,11 +185,11 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want skipped (not clobbered)", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "rotated" {
-			t.Error("existing replica was overwritten — a rotated token must survive")
+		if string(replica.Data[naming.ForwardingSecretKey]) != "rotated" {
+			t.Error("existing replica was overwritten — a hand-set value must survive")
 		}
 	})
 
@@ -204,22 +203,22 @@ func TestEnsureSecretReplica(t *testing.T) {
 
 	t.Run("rejects a source with an empty required key", func(t *testing.T) {
 		bad := srcSecret()
-		bad.Data[naming.ServiceTokenSecretKey] = nil
+		bad.Data[naming.ForwardingSecretKey] = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
 
 	t.Run("rejects an existing replica with an empty required key", func(t *testing.T) {
 		bad := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Data:       map[string][]byte{},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable existing replica", out)
 		}
 	})
@@ -245,92 +244,89 @@ func TestEnsureSecretReplica(t *testing.T) {
 		bad.Data = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "felis")
-		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
 }
 
-func TestRequiredProvisioningError(t *testing.T) {
-	ready := []systemServerOutcome{
-		{name: "service-token (minecraft ns)", available: true},
-		{name: "forwarding-secret (minecraft ns)", available: true},
-		{name: naming.SystemLoginServer, available: true},
-		{name: naming.SystemLobbyServer, skipped: "image not configured"},
-	}
-	if err := requiredProvisioningError(ready); err != nil {
-		t.Fatalf("ready outcomes: %v", err)
-	}
-
-	missing := append([]systemServerOutcome(nil), ready...)
-	missing[1] = systemServerOutcome{name: "forwarding-secret (minecraft ns)", skipped: "source missing"}
-	if err := requiredProvisioningError(missing); err == nil || !strings.Contains(err.Error(), "forwarding-secret") {
-		t.Fatalf("missing forwarding secret = %v, want named error", err)
-	}
-
-	failed := append([]systemServerOutcome(nil), ready...)
-	failed[3] = systemServerOutcome{name: naming.SystemLobbyServer, err: context.DeadlineExceeded}
-	if err := requiredProvisioningError(failed); err == nil || !strings.Contains(err.Error(), naming.SystemLobbyServer) {
-		t.Fatalf("lobby create failure = %v, want immediate named error", err)
-	}
-}
-
-// The Owner binds by joining the game, so setup blocks on the login gate rather
-// than racing it. What matters is that each ending is distinguishable: Ready
-// proceeds, Failed reports the operator's own reason instead of waiting out the
-// clock, and a gate that never appears (no operator reconciling it) times out
-// saying so rather than dropping the operator on a bind screen that cannot work.
-func TestAwaitLoginGateReady(t *testing.T) {
+// The felis-config mirror is the one replica that must refresh: it is a rendered
+// config, and a stale workload-side copy (backup/restore/fileedit Jobs, the reaper)
+// misbehaves silently — a rotated database credential keeps the control plane moving
+// while every backup Job keeps failing auth. Credential Secrets keep create-if-absent.
+func TestEnsureSecretReplicaRefresh(t *testing.T) {
 	scheme := newSystemServerScheme(t)
 	ctx := context.Background()
-
-	gate := func(mut func(*v1alpha1.MinecraftServer)) *v1alpha1.MinecraftServer {
-		ms := &v1alpha1.MinecraftServer{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.SystemLoginServer, Namespace: "minecraft"},
+	configSecret := func(ns, body string) *corev1.Secret {
+		return &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{"felis.toml": []byte(body)},
 		}
-		mut(ms)
-		return ms
+	}
+	refresh := func(cl client.Client) systemServerOutcome {
+		return ensureSecretReplica(ctx, cl, "felis", "minecraft",
+			"felis-config", "felis.toml", "config", "minecraft ns", true)
+	}
+	replicaBody := func(t *testing.T, cl client.Client) string {
+		t.Helper()
+		var got corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &got); err != nil {
+			t.Fatalf("get replica: %v", err)
+		}
+		return string(got.Data["felis.toml"])
 	}
 
-	t.Run("returns once the gate is ready", func(t *testing.T) {
-		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gate(func(ms *v1alpha1.MinecraftServer) {
-			ms.Status.Phase = v1alpha1.PhaseRunning
-			ms.Status.Ready = true
-		})).Build()
-		if err := awaitLoginGateReady(ctx, cl, "minecraft", time.Second, 10*time.Millisecond, nil); err != nil {
-			t.Fatalf("await: %v", err)
+	t.Run("refreshes a stale config replica", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("felis", "current"),
+			configSecret("minecraft", "stale"),
+		).Build()
+		out := refresh(cl)
+		if out.err != nil || !out.updated || !out.available {
+			t.Fatalf("outcome = %+v, want refreshed", out)
+		}
+		if got := replicaBody(t, cl); got != "current" {
+			t.Errorf("replica = %q, want current", got)
 		}
 	})
 
-	t.Run("fails fast on Failed, carrying the operator's reason", func(t *testing.T) {
-		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gate(func(ms *v1alpha1.MinecraftServer) {
-			ms.Status.Phase = v1alpha1.PhaseFailed
-			ms.Status.Conditions = []metav1.Condition{{
-				Type:               v1alpha1.ConditionReady,
-				Status:             metav1.ConditionFalse,
-				Reason:             "StartupTimeout",
-				Message:            "pod never became ready: ImagePullBackOff",
-				LastTransitionTime: metav1.Now(),
-			}}
-		})).Build()
-		start := time.Now()
-		err := awaitLoginGateReady(ctx, cl, "minecraft", time.Minute, 10*time.Millisecond, nil)
-		if err == nil {
-			t.Fatal("await: nil error, want failure")
-		}
-		if !strings.Contains(err.Error(), "ImagePullBackOff") {
-			t.Errorf("error = %q, want the operator's Ready-condition message", err)
-		}
-		if time.Since(start) > 5*time.Second {
-			t.Error("await sat out the full timeout on a settled Failed verdict")
+	t.Run("leaves a current config replica alone", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("felis", "same"),
+			configSecret("minecraft", "same"),
+		).Build()
+		out := refresh(cl)
+		if out.err != nil || out.updated || !out.available || out.skipped != "already current" {
+			t.Fatalf("outcome = %+v, want already current", out)
 		}
 	})
 
-	t.Run("times out when nothing ever reconciles the gate", func(t *testing.T) {
-		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
-		err := awaitLoginGateReady(ctx, cl, "minecraft", 30*time.Millisecond, 10*time.Millisecond, nil)
-		if err == nil || !strings.Contains(err.Error(), "timed out") {
-			t.Fatalf("await = %v, want a timeout", err)
+	t.Run("fills an empty-key replica", func(t *testing.T) {
+		empty := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: "minecraft"},
+			Data:       map[string][]byte{},
+		}
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("felis", "current"), empty).Build()
+		out := refresh(cl)
+		if out.err != nil || !out.updated {
+			t.Fatalf("outcome = %+v, want refreshed", out)
+		}
+		if got := replicaBody(t, cl); got != "current" {
+			t.Errorf("replica = %q, want current", got)
+		}
+	})
+
+	t.Run("missing source degrades to a skip", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("minecraft", "stale")).Build()
+		out := refresh(cl)
+		if out.err != nil || out.updated || out.available || out.skipped == "" {
+			t.Fatalf("outcome = %+v, want skipped (source missing)", out)
+		}
+		if got := replicaBody(t, cl); got != "stale" {
+			t.Errorf("replica = %q, want untouched stale", got)
 		}
 	})
 }
@@ -581,3 +577,62 @@ func TestEnsureSystemServersRefreshesDerivedEnv(t *testing.T) {
 		}
 	})
 }
+
+// Setup's replicas carry each workload its own caller token and nothing more: the
+// login gate gets felis-limbo-token in the minecraft namespace, the build Jobs get
+// felis-build-token in the build namespace, a replica left stale by a rotation is
+// brought up to date, and the proxy's felis-service-token is copied nowhere.
+func TestProvisionSecretReplicasCarryCallerTokens(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	secret := func(ns, name, key, val string) *corev1.Secret {
+		return &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{key: []byte(val)},
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		secret("felis", "felis-service-token", "token", "proxy-tok"),
+		secret("felis", "felis-limbo-token", "token", "limbo-new"),
+		secret("felis", "felis-build-token", "token", "build-tok"),
+		secret("felis", "felis-ops-token", "token", "ops-tok"),
+		secret("felis", naming.ForwardingSecretName, naming.ForwardingSecretKey, "fwd"),
+		// What a rotation leaves behind before setup runs again.
+		secret("minecraft", "felis-limbo-token", "token", "limbo-old"),
+	).Build()
+
+	outcomes := provisionSecretReplicas(ctx, cl, "felis", "minecraft", "felis-build")
+	for _, o := range outcomes {
+		if o.err != nil {
+			t.Fatalf("%s: %v", o.name, o.err)
+		}
+	}
+
+	read := func(ns, name string) (string, bool) {
+		var s corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+			return "", false
+		}
+		return string(s.Data["token"]), true
+	}
+	if got, _ := read("minecraft", "felis-limbo-token"); got != "limbo-new" {
+		t.Errorf("minecraft/felis-limbo-token = %q, want the rotated limbo-new", got)
+	}
+	if got, _ := read("felis-build", "felis-build-token"); got != "build-tok" {
+		t.Errorf("felis-build/felis-build-token = %q, want build-tok", got)
+	}
+	for _, ns := range []string{"minecraft", "felis-build"} {
+		for _, name := range []string{"felis-service-token", "felis-ops-token"} {
+			if _, ok := read(ns, name); ok {
+				t.Errorf("%s/%s was replicated; only the control namespace holds it", ns, name)
+			}
+		}
+	}
+	if _, ok := read("minecraft", "felis-build-token"); ok {
+		t.Error("the build token was copied into the minecraft namespace")
+	}
+	if _, ok := read("felis-build", "felis-limbo-token"); ok {
+		t.Error("the limbo token was copied into the build namespace")
+	}
+}