build(deps): bump actions/setup-java from 4.9.1 to 6.0.1

Bumps [actions/setup-java](https://github.com/actions/setup-java) from 4.9.1 to 6.0.1.
- [Release notes](https://github.com/actions/setup-java/releases)
- [Commits](https://github.com/actions/setup-java/compare/cf277c60eb25467037889841efdb72551f06f6c3...de7274f081f381c8f8158605e0321c36c376e2e6)

---
updated-dependencies:
- dependency-name: actions/setup-java
  dependency-version: 6.0.1
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>

.dockerignore

@@ -31,3 +31,14 @@ Dockerfile
 *.key
 felis
 felis.exe
+
+# macOS materializes extended attributes as ._<name> sidecars (BSD tar uploads,
+# Finder copies, network volumes) and leaves .DS_Store behind. Neither is
+# source, and one is actively harmful: a ._*.sql beside the migrations is
+# //go:embed-ed into the binary and makes every `felis migrate` fail
+# ("non-numeric version") — observed live on a Mac-staged tree. Same exposure
+# for any tree the other //go:embed patterns walk (deploy/, plugins/).
+._*
+**/._*
+.DS_Store
+**/.DS_Store

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# The workflows pin every action to a commit SHA, and every Dockerfile pins its base images
+# by digest. This keeps those pins moving: Dependabot reads the "# vX.Y.Z" comment next to
+# each action SHA, and the tag in front of each image digest, and opens a PR that bumps both
+# together.
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directories:
+      - /
+      - /deploy/limbo
+      - /deploy/lobby
+      - /deploy/paper
+    schedule:
+      interval: weekly
+    # A new major is a runtime change (Paper 26.x needs Java 25, Limbo's jar is Java 21
+    # bytecode), so only digests and minors are proposed; majors move by hand.
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/workflows/ci.yml

@@ -14,10 +14,14 @@
 # PR is what asks for the answer.
 name: ci
 
+#
+# release.yml calls this workflow (workflow_call) before it builds anything, so a tag passes
+# exactly these gates and there is one list of them.
 on:
   push:
     branches: [main]
   pull_request:
+  workflow_call:
 
 permissions:
   contents: read
@@ -30,9 +34,9 @@ jobs:
   go:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
           go-version-file: go.mod
 
@@ -43,12 +47,66 @@ jobs:
             echo "gofmt needed on:"; echo "$unformatted"; exit 1
           fi
       - run: go vet ./...
-      - run: go test ./...
+      # -race: felis-api and the operator are mostly goroutines (watchers, the
+      # registry pruner, the backup scheduler, the rate limiters).
+      - run: go test -race ./...
+      # The version is pinned here and bumped by hand; Dependabot does not read `go run`.
+      - name: staticcheck
+        run: go run honnef.co/go/tools/cmd/[email protected] ./...
+
+  # Separate from the go job so a newly published advisory reads as what it is. govulncheck
+  # exits non-zero only for vulnerable code this module can actually reach, standard
+  # library included: setup-go installs the newest patch of go.mod's Go line, so a finding
+  # there means the Dockerfile's golang digest (which ships the release) needs a bump too.
+  vuln:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go run golang.org/x/vuln/cmd/[email protected] ./...
+
+  # The business stores' SQL against a real PostgreSQL (internal/pgint): the unit suites run
+  # on fakes, and PGRepo drifted from them three times while those stayed green. 13 is the
+  # oldest server a supported distribution installs (EL9), 18 the newest (Arch).
+  pgint:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        postgres: ['13', '18']
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres }}
+        env:
+          POSTGRES_USER: felis
+          POSTGRES_PASSWORD: pgint
+          POSTGRES_DB: felis_pgint
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U felis -d felis_pgint"
+          --health-interval 2s
+          --health-timeout 5s
+          --health-retries 30
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go test -race -tags pgint -count=1 ./internal/pgint/
+        env:
+          FELIS_TEST_PG_URL: postgres://felis:pgint@localhost:5432/felis_pgint?sslmode=disable
 
   shell:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # bootstrap.sh is the only thing that ever runs on a fresh host, and nothing here can
       # run it — it wants root, a package manager and k3s. Syntax plus the extracted-block
@@ -66,12 +124,23 @@ jobs:
             esac
           done
 
+      # A pinned release rather than the runner image's copy, so a runner update cannot
+      # change what fails. Warnings and errors fail the job; style notes (info) do not.
+      - name: shellcheck
+        run: |
+          curl -fsSL -o shellcheck.tar.xz \
+            https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz
+          echo "8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198  shellcheck.tar.xz" | sha256sum -c
+          tar -xJf shellcheck.tar.xz
+          ./shellcheck-v0.11.0/shellcheck -S warning $(git ls-files '*.sh')
+
       - run: sh deploy/bootstrap_test.sh
+      - run: sh deploy/uninstall_test.sh
 
   panel:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # The Dockerfile's `FROM node:<major>` is the only place the panel's Node version is
       # declared — there is no .nvmrc and no engines field. Reading it here rather than
@@ -84,7 +153,7 @@ jobs:
           [ -n "$version" ] || { echo "Dockerfile has no 'FROM ... node:<major>' line"; exit 1; }
           echo "version=${version}" >> "$GITHUB_OUTPUT"
 
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: ${{ steps.node.outputs.version }}
           cache: npm
@@ -96,5 +165,71 @@ jobs:
       - run: npm test
         working-directory: panel
 
+      # `tsc -b`: tsconfig.json is a solution file (files: [] plus references), so a plain
+      # `tsc --noEmit` checked nothing and passed with type errors in the tree.
       - run: npm run typecheck
         working-directory: panel
+
+      # Rules of hooks and effect dependency lists, with --max-warnings 0.
+      - run: npm run lint
+        working-directory: panel
+
+      # openapi.gen.ts is generated from docs/openapi.yaml and checked in, so the
+      # compile-time parity in src/lib/types.parity.ts needs no generator in the build;
+      # a schema edit that was not regenerated fails here.
+      - name: openapi.gen.ts matches docs/openapi.yaml
+        working-directory: panel
+        run: |
+          npm run gen:api
+          git diff --exit-code -- src/lib/openapi.gen.ts
+
+      # Browser smoke over the mock-mode dev server, in the runner's installed Chrome
+      # (playwright.config.ts sets channel: chrome, so nothing is downloaded).
+      - run: npm run test:e2e
+        working-directory: panel
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: panel-playwright-report
+          path: |
+            panel/playwright-report
+            panel/test-results
+          retention-days: 7
+
+  plugins:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The other jobs never touch the Java layer: the plugin jars were only ever
+      # compiled by bootstrap on a live host, and the test mains under plugins/*/test
+      # were run by hand. JDK 25 is what the plugin build image runs (paper-api 26.x
+      # needs it); each module's wrapper brings the Gradle the image pins.
+      - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
+        with:
+          distribution: temurin
+          java-version: '25'
+
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
+
+      - run: bash plugins/test.sh
+
+  mods:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The three loader mods (Minecraft 1.20.1 / 1.20.4, Java-17 lines) compile
+      # through their vendored Gradle wrappers, which fetch their own Gradle. Until
+      # this job nothing ever built them: no install path touches them, and their
+      # gradlew scripts were committed without the exec bit, so the README's
+      # one-liners failed on a fresh clone.
+      - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
+
+      - run: bash plugins/test-mods.sh

.github/workflows/e2e.yml

@@ -0,0 +1,154 @@
+# deploy/bootstrap.sh end to end on a fresh Ubuntu 24.04 x86_64 runner: the paths every
+# host goes through, run for real instead of by hand on a VM.
+#
+#   install  a full install of this commit, then the same commit again (a rerun must
+#            converge without restarting what did not change)
+#   upgrade  the newest published release, then this commit on top of it; skipped until
+#            a release exists
+#
+# Each job builds the control plane, the game images and the proxy from scratch, about
+# half an hour of runner time, so this runs on pushes that touch what gets installed, by
+# hand, and weekly (a moving upstream: apt mirrors, k3s's install script, Adoptium).
+# deploy/e2e_check.sh holds the assertions.
+name: e2e
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'deploy/**'
+      - 'cmd/**'
+      - 'internal/**'
+      - 'panel/**'
+      - 'plugins/**'
+      - 'Dockerfile'
+      - 'go.mod'
+      - 'go.sum'
+      - '.github/workflows/e2e.yml'
+  workflow_dispatch:
+  schedule:
+    - cron: '23 4 * * 1'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  install:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The images, k3s and the JRE need more room than a stock runner leaves free.
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      # FELIS_SKIP_FETCH builds whatever sits in /opt/felis/src, the way the VM runs
+      # have always been done; the .git directory is what stamps the build. Root owns it
+      # so git, run as root by the installer, does not refuse it as dubious.
+      - name: Stage this commit as the installer's source
+        run: |
+          sudo mkdir -p /opt/felis
+          sudo cp -a "$GITHUB_WORKSPACE" /opt/felis/src
+          sudo chown -R root:root /opt/felis/src
+
+      - name: Install
+        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee install.log
+
+      - name: Check the install
+        run: sudo bash deploy/e2e_check.sh install
+
+      - name: Rerun the same commit
+        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee rerun.log
+
+      - name: Check the rerun
+        run: |
+          sudo bash deploy/e2e_check.sh rerun
+          grep -q 'felis-velocity unchanged; left running' rerun.log
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          k() { sudo -E /usr/local/bin/k3s kubectl "$@"; }
+          k get pods -A -o wide || true
+          k get events -A --sort-by=.lastTimestamp | tail -n 60 || true
+          for p in $(k get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" {print $1 "/" $2}'); do
+            k -n "${p%%/*}" describe pod "${p#*/}" | tail -n 40 || true
+            k -n "${p%%/*}" logs "${p#*/}" --all-containers --tail=60 || true
+          done
+          sudo journalctl -u k3s -u felis-velocity -u postgresql --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-install-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  upgrade:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - name: Find the newest release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          tag="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName 2>/dev/null || true)"
+          if [ -z "$tag" ]; then
+            echo "::notice::no published release yet; the upgrade path has nothing to start from"
+          fi
+          echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+
+      # The release's own installer, fetching the release's own binary: what a host that
+      # installed that release is running today.
+      - name: Install the newest release
+        if: steps.release.outputs.tag != ''
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          TOKEN: ${{ github.token }}
+        run: |
+          git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh
+          sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_REF="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log
+
+      - name: Check the release install
+        if: steps.release.outputs.tag != ''
+        run: sudo bash deploy/e2e_check.sh release
+
+      - name: Upgrade to this commit
+        if: steps.release.outputs.tag != ''
+        run: |
+          sudo rm -rf /opt/felis/src
+          sudo cp -a "$GITHUB_WORKSPACE" /opt/felis/src
+          sudo chown -R root:root /opt/felis/src
+          sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee upgrade.log
+
+      - name: Check the upgrade
+        if: steps.release.outputs.tag != ''
+        run: sudo bash deploy/e2e_check.sh upgrade
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo journalctl -u k3s -u felis-velocity -u postgresql --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-upgrade-logs
+          path: '*.log'
+          if-no-files-found: ignore

.github/workflows/release.yml

@@ -17,6 +17,16 @@
 # quietly ships a release whose panel is that placeholder. The Dockerfile runs the npm
 # build first, and is the same recipe bootstrap uses, so there is one way to build felis
 # rather than two that can drift.
+#
+# SHA256SUMS is a contract with bootstrap too: download_release_binary refuses a binary whose
+# hash is not listed there, BEFORE it runs it. A release without the file installs by source
+# build instead.
+#
+# The write token never meets the test suite: `gates` (ci.yml) and `build` run the tests,
+# Gradle and the Docker build (each of which executes third-party code) with a read-only
+# token, and `build` hands the binaries over as a workflow artifact; `publish` holds contents:write and runs only
+# pinned actions and gh. Every action is pinned to a commit SHA (the tag in the trailing
+# comment is for humans); .github/dependabot.yml proposes the bumps.
 name: release
 
 on:
@@ -24,28 +34,29 @@ on:
     tags: ['v*']
 
 permissions:
-  contents: write # gh release create/upload
+  contents: read
 
 jobs:
-  release:
+  # A tag that ships red is worse than a tag that fails to ship. These are ci.yml's gates,
+  # called rather than copied: Go (race, vet, staticcheck), govulncheck, the PostgreSQL
+  # contract suite, shellcheck and the bootstrap tests, the panel, and the Java layer the
+  # binary EMBEDS (bootstrap_asset.go ships the plugin sources, so a tag whose plugins do
+  # not compile turns every install of that release into a failed bootstrap).
+  gates:
+    uses: ./.github/workflows/ci.yml
+
+  build:
+    needs: gates
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-
-      # A tag that ships red is worse than a tag that fails to ship.
-      - run: go vet ./...
-      - run: go test ./...
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # Both architectures, because bootstrap's default release channel DOWNLOADS these
       # rather than compiling on the target host — an arm64 host with no asset silently
       # falls back to a slow source build. Neither stage is emulated: the Dockerfile pins
       # both build stages to $BUILDPLATFORM and the Go stage cross-compiles via TARGETARCH,
       # so the second architecture costs about a minute.
-      - uses: docker/setup-buildx-action@v3
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
 
       - name: Build the stamped binaries
         run: |
@@ -75,8 +86,67 @@ jobs:
           file ./felis-linux-arm64 | grep -q 'ARM aarch64' \
             || { echo "felis-linux-arm64 is not an arm64 ELF — TARGETARCH did not reach the go build"; exit 1; }
 
-      # --verify-tag refuses to invent a release for a tag that is not pushed. The upload
-      # fallback makes a re-run converge rather than failing on an existing release.
+      - name: Checksum the binaries
+        run: sha256sum felis-linux-amd64 felis-linux-arm64 | tee SHA256SUMS
+
+      # A CycloneDX SBOM per binary: the Go modules (and versions) linked into it, read
+      # from the build info the linker embeds.
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: felis-linux-amd64
+          format: cyclonedx-json
+          output-file: felis-linux-amd64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: felis-linux-arm64
+          format: cyclonedx-json
+          output-file: felis-linux-arm64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: release-assets
+          path: |
+            felis-linux-amd64
+            felis-linux-arm64
+            felis-linux-amd64.cdx.json
+            felis-linux-arm64.cdx.json
+            SHA256SUMS
+          if-no-files-found: error
+          retention-days: 7
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # gh release create/upload
+      id-token: write # the Sigstore certificate behind the provenance attestation
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: release-assets
+
+      # The artifact store sits between the two jobs, so check the handover too.
+      - run: sha256sum -c SHA256SUMS
+
+      # Signed SLSA provenance: which workflow run, commit and repository produced each
+      # binary. Check one with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`.
+      # GitHub only stores attestations for private repositories on Enterprise Cloud, and a
+      # failure here would block the release, so a private repository skips the step and
+      # relies on SHA256SUMS alone.
+      - name: Attest build provenance
+        if: ${{ !github.event.repository.private }}
+        uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0
+        with:
+          subject-path: |
+            felis-linux-amd64
+            felis-linux-arm64
+
+      # --verify-tag refuses to invent a release for a tag that is not pushed.
       #
       # The prerelease flag has to be passed explicitly: the trigger glob is v*, so v1.2.3-rc1
       # lands here too, and gh does not read semver out of the tag name. Published as a full
@@ -84,13 +154,33 @@ jobs:
       # channel installs from and `felis update` polls — so every fresh install would get the
       # RC binary and every deployed felis-api would error on the felis component until a
       # stable tag was cut. Flagged, GitHub keeps latest pointing at the last stable release.
+      #
+      # A re-run (the release already exists) uploads only what is missing and never
+      # replaces a published asset: hosts may already have installed it, and their
+      # SHA256SUMS check would start failing against a swapped file. An asset that is
+      # there with different bytes stops the job; cut a new tag instead.
       - name: Publish the release
         env:
           GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
         run: |
+          assets="felis-linux-amd64 felis-linux-arm64 felis-linux-amd64.cdx.json felis-linux-arm64.cdx.json SHA256SUMS"
           flags=""
           case "$GITHUB_REF_NAME" in *-*) flags="--prerelease" ;; esac
-          gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags \
-              ./felis-linux-amd64 ./felis-linux-arm64 \
-            || gh release upload "$GITHUB_REF_NAME" \
-                 ./felis-linux-amd64 ./felis-linux-arm64 --clobber
+          if ! gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
+            # shellcheck disable=SC2086 # word-splitting the list is the point
+            gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags $assets
+            exit 0
+          fi
+          # The REST payload's per-asset "digest" is GitHub's own sha256 of the stored file.
+          published="$(gh api "repos/${GH_REPO}/releases/tags/${GITHUB_REF_NAME}" --jq '.assets[] | "\(.name) \(.digest)"')"
+          for a in $assets; do
+            have="$(printf '%s\n' "$published" | awk -v n="$a" '$1 == n { print $2 }')"
+            want="sha256:$(sha256sum < "$a" | cut -d' ' -f1)"
+            if [ -z "$have" ]; then
+              gh release upload "$GITHUB_REF_NAME" "$a"
+            elif [ "$have" != "$want" ]; then
+              echo "::error::$a is already published with $have; this run built $want. Published assets are never replaced."
+              exit 1
+            fi
+          done

CONTRIBUTING.md

@@ -67,6 +67,20 @@ go test ./internal/api
 go test ./cmd/felis
 ```
 
+The hermetic suites run against in-memory fakes; the business stores' SQL is
+verified separately against a real Postgres, on a throwaway database whose name
+must contain `pgint` (the harness drops and recreates its schema and replays the
+embedded migrations):
+
+```bash
+FELIS_TEST_PG_URL='postgres://felis:***@127.0.0.1:5432/felis_pgint?sslmode=disable' \
+  go test -tags pgint ./internal/pgint/ -v
+```
+
+Run it after touching anything under `internal/api/pgrepo.go`, `internal/submit`,
+or `internal/build` that speaks SQL: the fakes encode the contract, and this
+suite exists to catch the drift between the fakes and the real queries.
+
 Build the CLI:
 
 ```bash

Dockerfile

@@ -21,14 +21,18 @@
 # minutes. The FINAL stage is deliberately NOT pinned — it must stay on the target platform
 # or the published arm64 image would carry amd64 layers. It contains only COPY, which
 # BuildKit performs itself, so it needs no QEMU either; adding a RUN there would.
-FROM --platform=$BUILDPLATFORM node:22-bookworm AS panel
+#
+# Every base image here and in deploy/{limbo,lobby,paper} is pinned by digest, so a rebuild
+# of one release uses the same bytes; .github/dependabot.yml proposes the bumps (tag and
+# digest together).
+FROM --platform=$BUILDPLATFORM node:22-bookworm@sha256:363e1587494626837fa7f9a23bdb453d13b0ff3c67c705c2805cfc69c2d2fad7 AS panel
 WORKDIR /panel
 COPY panel/package*.json ./
 RUN npm ci
 COPY panel/ ./
 RUN npm run build
 
-FROM --platform=$BUILDPLATFORM golang:1.26 AS build
+FROM --platform=$BUILDPLATFORM golang:1.26@sha256:6c2a5538f964f1c82f97ad14988bf05de100d922d159d0e398b54c7b0ca0c6c9 AS build
 WORKDIR /src
 ARG TARGETOS=linux
 ARG TARGETARCH
@@ -55,7 +59,7 @@ ARG FELIS_VERSION=dev
 RUN CGO_ENABLED=0 GOOS="$TARGETOS" GOARCH="${TARGETARCH:-$(go env GOARCH)}" \
     go build -trimpath -ldflags="-s -w -X main.version=${FELIS_VERSION}" -o /out/felis ./cmd/felis
 
-FROM gcr.io/distroless/static-debian12:nonroot
+FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
 ENV PATH=/usr/local/bin:/usr/bin:/bin
 COPY --chmod=0755 --from=build /out/felis /usr/local/bin/felis
 # distroless "nonroot" is uid 65532; the rendered PodSecurityContext pins

README.md

@@ -17,19 +17,20 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy,
 
 - **即开即玩**：玩家尝试连接时自动唤醒服务器，空闲后自动休眠，像游戏主机一样省资源。
 - **Web 控制面板**：浏览器中查看服务器状态、在线玩家与资源用量，管理备份与恢复。
-- **自动备份与恢复**：定时将世界打包存档，支持从任意备份点一键回滚。
-- **智慧回收**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间。
+- **备份与恢复**：一键把整服数据（世界、配置、插件/模组，即整个 /data 卷）打包进集群内的归档库，支持从任意备份点回滚；默认安装就已启用（归档 PVC 与路径由安装器一并生成）。
+- **控制面数据库备份**：账号、服务器归属、配额与存档索引所在的数据库每天自动备份，每次升级迁移前先快照，出错可用 `felis db restore` 整库原子回滚；面板「维护与备份」页显示备份是否新鲜（见 [故障排查 §16](docs/troubleshooting.md)）。
+- **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。过期备份无论是否开启都会每天清理。
 - **多核心支持**：兼容 Paper、Fabric、Forge、NeoForge，经由 Velocity 代理统一入口。
-- **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建并部署。
+- **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建；构建产物进入镜像白名单，可直接选用为服务器镜像完成部署。
 - **Passkey 登录**：支持指纹、面容、硬件密钥等无密码认证方式。
 - **零信任安全**：面板流量由 Cloudflare Access 保护，集群内 API 不暴露到公网。
 
 ## 使用方式
 
-在准备好的 Linux 主机上执行：
+在准备好的 Linux 主机上执行（已验证的发行版与架构见 [运维手册 §1](docs/operations.md#1-supported-hosts)：CentOS Stream 9 aarch64 实机验证，Ubuntu 24.04 x86_64 每次推送由 CI 跑全新安装、重跑与升级）：
 
 ```bash
-curl -fsSL https://raw.githubusercontent.com/MliroLirrorsIngenuity/Felis/main/deploy/bootstrap.sh | sudo bash
+curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
 ```
 
 脚本将自动安装 K3s、部署控制平面并启动设置向导。完成后浏览器访问已配置的域名进入控制面板即可使用。
@@ -41,7 +42,7 @@ curl -fsSL https://raw.githubusercontent.com/MliroLirrorsIngenuity/Felis/main/de
 > export FELIS_GITHUB_TOKEN=<对本仓库有读权限的 token>
 > printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \
 >   | curl -fsSL --config - -H "Accept: application/vnd.github.raw" \
->       https://api.github.com/repos/MliroLirrorsIngenuity/Felis/contents/deploy/bootstrap.sh \
+>       https://api.github.com/repos/FelisMC/Felis/contents/deploy/bootstrap.sh \
 >   | sudo -E bash
 > ```
 >

README_EN.md

@@ -17,23 +17,48 @@ Table of Contents
 
 - **Wake on Join**: Servers start automatically when a player connects, and stop when idle — like hibernate for your server.
 - **Web Dashboard**: Monitor server status, online players, and resource usage from your browser, with backup and restore management.
-- **Auto Backup & Restore**: Scheduled world backups with one-click rollback from any backup point.
-- **World Reaper**: Worlds idle for more than 15 days are automatically backed up and removed to free disk space.
+- **Backup & Restore**: One-click snapshots of a server's whole data volume (worlds, config, plugins/mods — the entire /data volume) into the cluster's archive store, with rollback from any backup point — enabled by default (the installer renders the archive PVC and its path).
+- **Control-plane database backups**: The database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migrates it; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows whether the newest backup is fresh (see [troubleshooting §16](docs/troubleshooting.md)).
+- **World Reaper** (opt in): Worlds idle for more than 15 days are automatically backed up and removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is ever deleted.
 - **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, federated behind a Velocity proxy.
-- **Modpack Submission**: Players submit custom modpacks; admin approval triggers automatic build and deployment.
+- **Modpack Submission**: Players submit custom modpacks; admin approval triggers an automatic build, and the result is whitelisted as a server image you can select to deploy.
 - **Passkey Login**: Passwordless authentication via fingerprint, face recognition, or hardware security keys.
 - **Zero Trust Security**: Panel traffic protected by Cloudflare Access; the internal API is never exposed to the internet.
 
 ## Getting Started
 
-On a prepared Linux host, run:
+On a prepared Linux host, run (the verified distributions and architectures are listed in
+[operations §1](docs/operations.md#1-supported-hosts): CentOS Stream 9 on aarch64 is verified
+on a real host, and Ubuntu 24.04 on x86_64 gets a fresh install, rerun and upgrade in CI on
+every push):
 
 ```bash
-curl -fsSL https://raw.githubusercontent.com/MliroLirrorsIngenuity/Felis/main/deploy/bootstrap.sh | sudo bash
+curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
 ```
 
 The script installs K3s, deploys the control plane, and launches a setup wizard. Once done, open your browser at the configured domain.
 
+> **This repository is currently private**, so the command above returns 404. Use the
+> credentialed form instead; the installer itself needs the same token to resolve and
+> download the release, so pass it through with `sudo -E`:
+>
+> ```bash
+> export FELIS_GITHUB_TOKEN=<a token with read access to this repository>
+> printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \
+>   | curl -fsSL --config - -H "Accept: application/vnd.github.raw" \
+>       https://api.github.com/repos/FelisMC/Felis/contents/deploy/bootstrap.sh \
+>   | sudo -E bash
+> ```
+>
+> The token reaches `curl --config -` over stdin instead of the command line: argv is
+> readable by any local user via `/proc`, and that is exactly why the installer's
+> internal `github_api` uses the same form.
+
+Rerunning this command is also how you upgrade felis-api to a newer version (`felis setup`
+cannot — it uses the binary already installed on the host). The rerun keeps the installed
+root domain but **not** the channel: if this host follows main, also
+`export FELIS_VERSION_BOOTSTRAP=dev`.
+
 ## Build from Source
 
 Felis is built with Go and Node.js:

bootstrap_asset.go

@@ -22,14 +22,17 @@ var bootstrapAssets embed.FS
 // developer's working tree carries gradle output (plugins/*/build, plugins/*/bin,
 // and for the modded loaders a decompiled Minecraft under build/) which would
 // otherwise be baked into every felis binary. Keep them explicit — add a source
-// directory here, never a parent.
+// directory here, never a parent. Each module's gradle/verification-metadata.xml rides
+// along, since Gradle builds unverified without it; the wrapper stays out, because the
+// image builds run the pinned build image's own gradle.
 //
+//go:embed deploy/game-stack.lock
 //go:embed deploy/limbo/Dockerfile deploy/limbo/entrypoint.sh
 //go:embed deploy/lobby/Dockerfile deploy/lobby/entrypoint.sh
 //go:embed deploy/paper/Dockerfile deploy/paper/entrypoint.sh
-//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src
-//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src
-//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src
+//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src plugins/limbo/gradle/verification-metadata.xml
+//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src plugins/paper/gradle/verification-metadata.xml
+//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src plugins/velocity/gradle/verification-metadata.xml
 //go:embed plugins/shared/src
 var gameStackAssets embed.FS
 

bootstrap_asset_test.go

@@ -1,7 +1,9 @@
 package felis
 
 import (
+	"encoding/xml"
 	"io/fs"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -205,6 +207,286 @@ func requireEmbedded(t *testing.T, path string) {
 	}
 }
 
+// The lock file is the install's only source of upstream builds, and bootstrap.sh reads it
+// with a strict KEY=value parser that dies on anything unexpected, so a malformed lock is a
+// failed install on every host. Check the shipped copy the same way here.
+func TestGameStackLockIsComplete(t *testing.T) {
+	lock := gameStackLock(t)
+	m := regexp.MustCompile(`GAME_STACK_LOCK_KEYS="([^"]*)"`).FindStringSubmatch(BootstrapScript())
+	if m == nil {
+		t.Fatal("bootstrap.sh no longer declares GAME_STACK_LOCK_KEYS")
+	}
+	keys := strings.Fields(m[1])
+	sha := regexp.MustCompile(`^[0-9a-f]{64}$`)
+	for _, k := range keys {
+		v, ok := lock[k]
+		if !ok || v == "" {
+			t.Errorf("game-stack.lock does not set %s", k)
+			continue
+		}
+		if strings.HasSuffix(k, "_SHA256") && !sha.MatchString(v) {
+			t.Errorf("%s=%q is not a lowercase sha256", k, v)
+		}
+		// A moving URL pins nothing: the digest check would start failing the day
+		// upstream publishes the next build.
+		if strings.HasSuffix(k, "_URL") && strings.Contains(v, "lastSuccessfulBuild") {
+			t.Errorf("%s names a moving build: %s", k, v)
+		}
+	}
+	for k := range lock {
+		if !strings.Contains(" "+m[1]+" ", " "+k+" ") {
+			t.Errorf("game-stack.lock sets %s, which bootstrap.sh refuses as an unknown key", k)
+		}
+	}
+	// Fill's URLs are content-addressed; a lock whose digest disagrees with its own URL
+	// was edited by hand and half-way.
+	for _, name := range []string{"PAPER", "VELOCITY"} {
+		if !strings.Contains(lock[name+"_JAR_URL"], "/objects/"+lock[name+"_JAR_SHA256"]+"/") {
+			t.Errorf("%s_JAR_SHA256 is not the digest in %s_JAR_URL", name, name)
+		}
+	}
+	if !strings.Contains(lock["LIMBO_JAR_URL"], "-"+lock["MC_VERSION"]+".jar") {
+		t.Errorf("LIMBO_JAR_URL %s is not a Minecraft %s build", lock["LIMBO_JAR_URL"], lock["MC_VERSION"])
+	}
+	if !strings.Contains(lock["PAPER_JAR_URL"], "/paper-"+lock["MC_VERSION"]+"-") {
+		t.Errorf("PAPER_JAR_URL %s is not a Minecraft %s build; the lobby would not speak the login gate's protocol", lock["PAPER_JAR_URL"], lock["MC_VERSION"])
+	}
+}
+
+// Each downloaded jar's digest is a build-arg bootstrap.sh passes and the Dockerfile must
+// both require and spend on the file it downloaded; docker only warns about an unknown
+// --build-arg, so a renamed arg would ship an unchecked jar.
+func TestGameStackDigestsReachTheImageBuilds(t *testing.T) {
+	script := BootstrapScript()
+	for _, c := range []struct{ dockerfile, arg, path string }{
+		{"deploy/limbo/Dockerfile", "LIMBO_JAR_SHA256", "/limbo/Limbo.jar"},
+		{"deploy/limbo/Dockerfile", "LIMBO_SCHEM_SHA256", "/limbo/spawn.schem"},
+		{"deploy/lobby/Dockerfile", "LUCKPERMS_JAR_SHA256", "/paper/plugins/LuckPerms.jar"},
+	} {
+		if !strings.Contains(script, "--build-arg "+c.arg+"=\"$"+c.arg+"\"") {
+			t.Errorf("bootstrap.sh never passes --build-arg %s", c.arg)
+		}
+		dockerfile := readGameStackFile(t, c.dockerfile)
+		if !strings.Contains(dockerfile, "ARG "+c.arg) {
+			t.Errorf("%s declares no ARG %s", c.dockerfile, c.arg)
+		}
+		if !strings.Contains(dockerfile, `echo "$`+c.arg+`  `+c.path+`" | sha256sum -c`) {
+			t.Errorf("%s never verifies %s against %s", c.dockerfile, c.path, c.arg)
+		}
+	}
+}
+
+// A base image named by tag alone is whatever the tag points at on build day.
+func TestDockerfileBaseImagesArePinnedByDigest(t *testing.T) {
+	root, err := os.ReadFile("Dockerfile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	files := map[string]string{"Dockerfile": string(root)}
+	for _, name := range []string{"deploy/limbo/Dockerfile", "deploy/lobby/Dockerfile", "deploy/paper/Dockerfile"} {
+		files[name] = readGameStackFile(t, name)
+	}
+	pinned := regexp.MustCompile(`^FROM (--platform=\S+ )?\S+:\S+@sha256:[0-9a-f]{64}( AS \S+)?$`)
+	for name, body := range files {
+		n := 0
+		for line := range strings.SplitSeq(body, "\n") {
+			if !strings.HasPrefix(line, "FROM ") {
+				continue
+			}
+			n++
+			if !pinned.MatchString(line) {
+				t.Errorf("%s: %q is not pinned by digest", name, line)
+			}
+		}
+		if n == 0 {
+			t.Errorf("%s has no FROM line", name)
+		}
+	}
+}
+
+// The plugin jars are built in three places the installer controls — the lobby and limbo
+// image builds and bootstrap's Velocity build — and through each module's wrapper by a
+// developer or CI. A tag alone is whatever it points at on build day, and two Gradle
+// versions are two chances for a build to pass in one place and break in the other, so
+// all of them run one image, pinned by digest, whose Gradle is the wrappers' Gradle.
+func TestPluginBuildsRunOnePinnedGradle(t *testing.T) {
+	sources := map[string]string{
+		"deploy/lobby/Dockerfile": readGameStackFile(t, "deploy/lobby/Dockerfile"),
+		"deploy/limbo/Dockerfile": readGameStackFile(t, "deploy/limbo/Dockerfile"),
+		"deploy/bootstrap.sh":     BootstrapScript(),
+	}
+	anyRef := regexp.MustCompile(`gradle:[\w.-]+(@sha256:\w+)?`)
+	pinned := regexp.MustCompile(`^gradle:(\d+\.\d+(?:\.\d+)?)-jdk\d+@sha256:[0-9a-f]{64}$`)
+	images := map[string]bool{}
+	gradle := ""
+	for name, body := range sources {
+		refs := anyRef.FindAllString(body, -1)
+		if len(refs) == 0 {
+			t.Errorf("%s names no gradle image", name)
+		}
+		for _, ref := range refs {
+			m := pinned.FindStringSubmatch(ref)
+			if m == nil {
+				t.Errorf("%s: %s is not a gradle image pinned by digest", name, ref)
+				continue
+			}
+			images[ref] = true
+			gradle = m[1]
+		}
+	}
+	if len(images) != 1 {
+		t.Fatalf("the plugin builds use %d different gradle images, want one: %v", len(images), images)
+	}
+	// bootstrap names the image once and has to spend it where it builds the jar.
+	if !strings.Contains(BootstrapScript(), `"$PLUGIN_BUILD_IMAGE" gradle --no-daemon clean build`) {
+		t.Error("build_velocity_plugin does not build in $PLUGIN_BUILD_IMAGE")
+	}
+
+	for _, module := range []string{"velocity", "paper", "limbo"} {
+		props := wrapperProperties(t, module)
+		if want := "gradle-" + gradle + "-bin.zip"; !strings.HasSuffix(props["distributionUrl"], "/"+want) {
+			t.Errorf("plugins/%s wrapper runs %s; the image builds run Gradle %s", module, props["distributionUrl"], gradle)
+		}
+	}
+	// The mods are no part of the install, but a wrapper without a checksum runs whatever
+	// the download handed it.
+	for _, module := range []string{"velocity", "paper", "limbo", "fabric", "forge", "neoforge"} {
+		if sum := wrapperProperties(t, module)["distributionSha256Sum"]; !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(sum) {
+			t.Errorf("plugins/%s wrapper pins no distribution sha256 (got %q)", module, sum)
+		}
+	}
+}
+
+// Each plugin compiles against the API of the exact build the install runs, and Gradle
+// checks those bytes against the module's verification file. Nothing but this test ties
+// the three to deploy/game-stack.lock: a lock refresh that leaves them behind builds the
+// lobby against yesterday's API, or fails every image build on a checksum the file does
+// not have.
+func TestPluginApisAreTheLockedBuilds(t *testing.T) {
+	lock := gameStackLock(t)
+
+	// Paper: paper-<mc>-<build>.jar runs; paper-api <mc>.build.<build>-<channel> compiles.
+	jar := regexp.MustCompile(`/paper-([^/]+)-(\d+)\.jar$`).FindStringSubmatch(lock["PAPER_JAR_URL"])
+	if jar == nil {
+		t.Fatalf("PAPER_JAR_URL %s does not name paper-<mc>-<build>.jar", lock["PAPER_JAR_URL"])
+	}
+	dep := regexp.MustCompile(`compileOnly 'io\.papermc\.paper:paper-api:([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/paper/build.gradle"))
+	if dep == nil {
+		t.Fatal("plugins/paper/build.gradle declares no paper-api dependency")
+	}
+	if !regexp.MustCompile(`^` + regexp.QuoteMeta(jar[1]+".build."+jar[2]) + `(-[a-z]+)?$`).MatchString(dep[1]) {
+		t.Errorf("paper-api %s is not the API of the locked server paper-%s-%s.jar", dep[1], jar[1], jar[2])
+	}
+	requireVerified(t, "paper", "io.papermc.paper", "paper-api", dep[1])
+
+	// Limbo: the lock's release, passed to the image build, which refuses to guess one.
+	limbo := lock["LIMBO_VERSION"]
+	if !strings.Contains(BootstrapScript(), `--build-arg LIMBO_VERSION="$LIMBO_VERSION"`) {
+		t.Error("bootstrap.sh does not pass the locked LIMBO_VERSION to the limbo image build")
+	}
+	dockerfile := readGameStackFile(t, "deploy/limbo/Dockerfile")
+	if !regexp.MustCompile(`(?m)^ARG LIMBO_VERSION$`).MatchString(dockerfile) ||
+		!strings.Contains(dockerfile, `if [ -z "${LIMBO_VERSION:-}" ]`) {
+		t.Error("deploy/limbo/Dockerfile does not require LIMBO_VERSION; a build without it would " +
+			"compile against a version nobody chose")
+	}
+	// LOOHP publishes the jar the login gate runs as the Limbo API artifact itself, so the
+	// checksum Gradle holds for it is the lock's: compiled-against and running are one file.
+	if got := requireVerified(t, "limbo", "com.loohp", "Limbo", limbo)["Limbo-"+limbo+".jar"]; got != lock["LIMBO_JAR_SHA256"] {
+		t.Errorf("verification-metadata.xml holds %q for Limbo-%s.jar; the login gate runs %s", got, limbo, lock["LIMBO_JAR_SHA256"])
+	}
+
+	// Velocity: the API default is the proxy the install runs.
+	api := regexp.MustCompile(`findProperty\('velocityApi'\) \?: '([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/velocity/build.gradle"))
+	if api == nil {
+		t.Fatal("plugins/velocity/build.gradle has no velocityApi default")
+	}
+	if api[1] != lock["VELOCITY_VERSION"] {
+		t.Errorf("velocity-api defaults to %s; the install runs Velocity %s", api[1], lock["VELOCITY_VERSION"])
+	}
+	requireVerified(t, "velocity", "com.velocitypowered", "velocity-api", api[1])
+}
+
+// requireVerified asserts the module's shipped verification file checks metadata and
+// pins group:name:version, and returns that component's artifact sha256s by file name.
+func requireVerified(t *testing.T, module, group, name, version string) map[string]string {
+	t.Helper()
+	path := "plugins/" + module + "/gradle/verification-metadata.xml"
+	var doc struct {
+		VerifyMetadata bool `xml:"configuration>verify-metadata"`
+		Components     []struct {
+			Group     string `xml:"group,attr"`
+			Name      string `xml:"name,attr"`
+			Version   string `xml:"version,attr"`
+			Artifacts []struct {
+				Name   string `xml:"name,attr"`
+				SHA256 []struct {
+					Value string `xml:"value,attr"`
+				} `xml:"sha256"`
+			} `xml:"artifact"`
+		} `xml:"components>component"`
+	}
+	if err := xml.Unmarshal([]byte(readGameStackFile(t, path)), &doc); err != nil {
+		t.Fatalf("%s: %v", path, err)
+	}
+	if !doc.VerifyMetadata {
+		t.Errorf("%s does not verify metadata; a swapped pom could redirect the graph", path)
+	}
+	for _, c := range doc.Components {
+		if c.Group != group || c.Name != name || c.Version != version {
+			continue
+		}
+		sums := map[string]string{}
+		for _, a := range c.Artifacts {
+			if len(a.SHA256) > 0 {
+				sums[a.Name] = a.SHA256[0].Value
+			}
+		}
+		if sums[name+"-"+version+".jar"] == "" {
+			t.Errorf("%s pins %s:%s:%s but no sha256 for its jar", path, group, name, version)
+		}
+		return sums
+	}
+	t.Errorf("%s has no checksum for %s:%s:%s; the build would refuse it", path, group, name, version)
+	return nil
+}
+
+// wrapperProperties reads a module's gradle-wrapper.properties off disk: the wrappers are
+// for developers and CI, and nothing embeds them.
+func wrapperProperties(t *testing.T, module string) map[string]string {
+	t.Helper()
+	b, err := os.ReadFile("plugins/" + module + "/gradle/wrapper/gradle-wrapper.properties")
+	if err != nil {
+		t.Fatal(err)
+	}
+	props := map[string]string{}
+	for line := range strings.SplitSeq(string(b), "\n") {
+		if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok && !strings.HasPrefix(k, "#") {
+			props[k] = strings.ReplaceAll(v, `\:`, ":")
+		}
+	}
+	return props
+}
+
+// gameStackLock parses the shipped deploy/game-stack.lock the way bootstrap.sh does.
+func gameStackLock(t *testing.T) map[string]string {
+	t.Helper()
+	lock := map[string]string{}
+	for line := range strings.SplitSeq(readGameStackFile(t, "deploy/game-stack.lock"), "\n") {
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			t.Fatalf("not a KEY=value line: %q", line)
+		}
+		lock[k] = v
+	}
+	return lock
+}
+
 func readGameStackFile(t *testing.T, name string) string {
 	t.Helper()
 	b, err := gameStackAssets.ReadFile(name)

cmd/felis/api.go

@@ -5,10 +5,14 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"felis.lolicon.best/internal/api"
@@ -17,18 +21,25 @@ import (
 	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/fileedit"
+	"felis.lolicon.best/internal/imagepin"
 	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/metrics"
+	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/panel"
 	"felis.lolicon.best/internal/passkey"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/reaper"
+	"felis.lolicon.best/internal/registryprune"
 	"felis.lolicon.best/internal/restore"
-	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/retention"
 	"felis.lolicon.best/internal/submit"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -53,10 +64,8 @@ func authSourcesFromConfig(configured []config.AuthSourceConfig) []api.AuthSourc
 }
 
 // cmdAPI runs felis-api: two listeners, two middleware chains (spec §7). The
-// internal face (service token) is fully wired. The external face is wired but
-// fails closed until an Access JWKS key function is configured — the verifier's
-// audience logic is unit-tested (internal/api), the JWKS source is a deployment
-// integration point.
+// internal face authenticates per-caller service tokens; the external face
+// authenticates the local session cookie.
 func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("api", flag.ContinueOnError)
 	fs.SetOutput(stderr)
@@ -79,9 +88,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	// Load already refused a malformed [audit] retention.
+	auditRetention, _ := cfg.Audit.RetentionPeriod()
+	if auditRetention == 0 {
+		fmt.Fprintln(stdout, "felis api: audit rows are kept forever ([audit] retention = \"forever\")")
+	} else {
+		fmt.Fprintf(stdout, "felis api: audit rows older than %d days are deleted ([audit] retention; export them first with felis db audit-export)\n", int(auditRetention/(24*time.Hour)))
+	}
+
 	ctx := ctrl.SetupSignalHandler()
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Before anything serves: an api on a schema it was not built for answers with
+	// errors, or writes rows the other version cannot read.
+	drv, err := openStore(ctx, cfg.Database.URL, false)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis api: open database: %v\n", err)
 		return 1
@@ -108,15 +127,24 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	token := os.Getenv("FELIS_SERVICE_TOKEN")
-	if token == "" {
-		fmt.Fprintln(stderr, "felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers")
+	metrics.SetBuildInfo("api", resolvedVersion())
+
+	internalAuth, err := internalCallerTokens(os.Getenv)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: internal face tokens: %v\n", err)
+		return 1
+	}
+	for _, ct := range naming.CallerTokens {
+		if internalAuth[api.Caller(ct.Caller)] == "" {
+			fmt.Fprintf(stderr, "felis api: warning: %s unset — the internal face turns the %s caller away\n", ct.APIEnv, ct.Caller)
+		}
 	}
 
 	// Email one-time codes go through the [smtp] relay when one is configured; the
 	// password is read from the env var password_ref names (default SMTPPasswordEnv,
 	// injected from the felis-smtp Secret). No [smtp] host ⇒ mailer stays nil and
-	// deliverOTP logs each code server-side (the pre-SMTP bootstrap posture).
+	// every door that mails a code answers 503 mail_unavailable: a code that is
+	// not mailed is never written anywhere else either.
 	var mailer api.OTPMailer
 	if cfg.SMTP.Host != "" {
 		passRef := cfg.SMTP.PasswordRef
@@ -127,26 +155,30 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		if cfg.SMTP.Username != "" && password == "" {
 			fmt.Fprintf(stderr, "felis api: warning: [smtp] username is set but credentials env %s is empty — OTP sends will fail AUTH\n", passRef)
 		}
-		mailer = &mail.SMTP{
-			Host:     cfg.SMTP.Host,
-			Port:     cfg.SMTP.Port,
-			From:     cfg.SMTP.From,
-			Username: cfg.SMTP.Username,
-			Password: password,
+		mailer = smtpRelay(cfg.SMTP, password)
+		if !cfg.SMTP.TLSRequired() {
+			fmt.Fprintf(stderr, "felis api: warning: [smtp] %s may be sent codes without TLS (require_tls off or a relay on this host)\n", cfg.SMTP.Host)
 		}
 	} else {
-		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email one-time codes are logged, not mailed")
+		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email sign-in and verification are off (503 mail_unavailable); sign in with a passkey, or run felis setup to add a relay")
 	}
 
 	// Build subsystem (spec §16): the weak-SA build Job runs in the configured
 	// build namespace and pushes to the internal registry. The build Pod never
 	// holds DB credentials — felis-api owns the PG store and admits scanned
 	// images, so the Builder is constructed here with both bindings.
+	buildCfg := buildConfig(cfg)
+	// The fetch initContainer runs THIS image's fetch-context entrypoint, so the
+	// build config carries the api's own image (the platform sets FELIS_IMAGE).
+	buildCfg.FelisImage = os.Getenv("FELIS_IMAGE")
+	buildJobs := build.NewK8sJobs(cl, buildCfg)
 	builder := &build.Builder{
 		Store:    build.NewPGStore(drv.DB()),
-		Jobs:   build.NewK8sJobs(cl, buildConfig(cfg)),
-		Config: buildConfig(cfg),
+		Jobs:     buildJobs,
+		Config:   buildCfg,
+		Outcomes: build.NewK8sOutcomes(clientset, buildCfg),
 	}
+	go probeBuildUserNamespaces(ctx, buildJobs, buildCfg, stderr)
 
 	// User-modpack approval lane (user-directed extension over §16; see
 	// internal/submit). An ordinary user may only SUBMIT a
@@ -159,14 +191,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// The blob upload transport is selected by the shape of user_uploads_context —
 	// the two backends the setup wizard chooses between. A local path wires
 	// LocalContextStore (the mounted uploads PVC); an s3:// base wires
-	// S3ContextStore when its credentials resolve. Either way the store's target is
-	// derived from the SAME config field the context ref uses, so the blob lands
-	// exactly where Kaniko's --context points. Anything else — or an s3:// base with
-	// no credentials configured — leaves Blobs nil so POST
+	// S3ContextStore when its credentials resolve. Anything else — or an s3:// base
+	// with no credentials configured — leaves Blobs nil so POST
 	// /me/submissions/{id}/context returns 503, honest like the restore executor
-	// when its PVC is not supplied. (Letting the sandboxed Kaniko build Pod READ the
-	// context — PVC mount for local, creds+egress for S3 — is a separate deployment
-	// integration.)
+	// when its PVC is not supplied.
+	//
+	// Reading the blob back is the API's job, not Kaniko's: the build Pod runs in
+	// another namespace and can neither mount the uploads PVC (a PVC does not cross
+	// namespaces) nor hold object-store credentials, so ContextBaseURL makes the
+	// derived context ref an internal-face URL that the build Job's fetch
+	// initContainer streams (cmd/felis fetch-context). The platform renders this
+	// address into the api Deployment (felis API base URL env); the fallback keeps
+	// a hand-rolled deployment working under the platform's default control
+	// namespace.
 	contextBase := cfg.Registry.UserUploadsContext
 	var blobs submit.Blobs
 	switch {
@@ -190,8 +227,24 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		Builds:         builder,
 		Registry:       cfg.Registry.URL,
 		ContextStore:   contextBase,
+		ContextBaseURL: internalAPIBaseURL(),
 		Blobs:          blobs,
 	}
+	if blobs != nil {
+		submissions.Parts = &submit.PartStore{Dir: uploadPartsDir(contextBase)}
+	}
+	if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
+		if n, err := parseByteSize(v); err != nil || n <= 0 {
+			fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
+		} else {
+			submissions.MaxStoredBytesTotal = n
+		}
+	}
+	if n, err := contextMaxBytes(cfg); err != nil {
+		fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 512Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
+	} else {
+		submissions.MaxContextBytes = n
+	}
 
 	// Restore subsystem (spec §7): the weak-SA restore Job mounts the target
 	// world PVC + the backup PVC and runs `felis restore`. It needs deployment-
@@ -246,32 +299,51 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// the same store the auth handlers write to, so a login and the next request
 	// agree on what local auth knows.
 	repo := api.NewPGRepo(drv.DB())
+	if err := api.RegisterStorePool(drv.DB()); err != nil {
+		fmt.Fprintf(stderr, "felis api: store pool metrics unavailable: %v\n", err)
+	}
 
+	// The owner's on-demand backup levers come from [archive], the same keys the
+	// backup Job and the reaper read. A malformed key leaves the defaults in
+	// place here; the reaper Job fails on it and names it.
+	rcfg, err := reaperConfig(cfg)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: %v; using the default backup limits\n", err)
+		rcfg = reaper.DefaultConfig()
+	}
+
+	serverCache, serversSynced, err := startServerCache(ctx, restCfg, scheme, cfg.K8s.Namespace, stderr)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err)
+		return 1
+	}
+	cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced)
+	jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace)
 	a := &api.API{
 		Repo:    repo,
-		Cluster: api.NewK8sCluster(cl, cfg.K8s.Namespace),
+		Cluster: cluster,
 		Console: api.NewK8sConsole(cl, cfg.K8s.Namespace),
 		Logs:    api.NewK8sLogStreamer(clientset, cfg.K8s.Namespace),
 		// Build-log stream (spec §16) is scoped to the BUILD namespace — the same
 		// value the Builder renders Jobs into — so it follows where build Pods run.
 		BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
-		Internal:    api.BearerTokenAuth{Token: token},
+		Internal:  internalAuth,
 		Builder:   builder,
+		Images:    imagePinner(cfg.Registry.URL),
 		Restorer:  restorer,
 		Backuper:  backuper,
+		JobStatus: jobStatus,
+		// A restore starts with a safety snapshot; settleRestoreChains starts the
+		// restore behind each one.
+		RestoreChains: jobStatus,
 		Files:         files,
 		Submissions:   submissions,
 		Mailer:        mailer,
-		// The external face is fronted by SessionAuth: it prefers a local session
-		// cookie (minted by the passwordless doors) and otherwise delegates to the
-		// Cloudflare-Access JWT verifier, so both auth models coexist on one face. The
-		// delegate's Keyfunc is intentionally nil — the JWT path fails closed until a
-		// JWKS-backed key function is wired (deployment integration point) — while the
-		// local session path is live the moment `felis breakGlass` flips
-		// local_auth_enabled on.
+		// The external face authenticates the local session cookie the sign-in doors
+		// mint, live once `felis breakGlass` flips local_auth_enabled on. Cloudflare
+		// Access, when the install sits behind it, is enforced at the edge only.
 		External: api.SessionAuth{
 			Repo:          repo,
-			Delegate:      api.AccessVerifier{Audience: cfg.Auth.AccessJWTAud},
 			RootDomain:    cfg.Server.RootDomain,
 			AdminHostname: cfg.Auth.AdminHostname,
 		},
@@ -279,12 +351,32 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		AdminHostname: cfg.Auth.AdminHostname,
 		PanelHostname: cfg.Auth.PanelHostname,
 		WakeCooldown:  30 * time.Second,
+		// An owner may start one backup per server per manual_cooldown, and none
+		// while the store is at max_local_bytes (data-durability-9).
+		BackupCooldown: rcfg.ManualCooldown,
+		BackupStoreCap: rcfg.MaxLocalBytes,
+		// The user-modpack lane's per-user throttles: a create spaces out
+		// review-queue rows, an upload spaces out (up to 1 GiB) context streams.
+		// Separate keys, so the normal create→upload sequence stays immediate.
+		SubmitCreateCooldown: 30 * time.Second,
+		SubmitUploadCooldown: 15 * time.Second,
 		// Bound concurrent console/build-log SSE streams per principal. Generous enough
 		// for legitimate multi-tab / multi-server watching, while capping how many
 		// upstream follow connections a single caller can tie up if their streams stall.
 		MaxStreamsPerPrincipal: 16,
+		// Public sign-in doors, per client address: a person signing in makes a
+		// handful of calls, so 20 at once refilled at 20 a minute never bites a
+		// real user and still turns a spray into a trickle. The client address
+		// is the edge's header when the install names one (config.AuthConfig).
+		AuthDoorLimit:  api.RateLimit{Burst: 20, PerMinute: 20},
+		ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(),
+		MailLimit:      mailLimit(cfg.SMTP.MaxPerHour),
+	}
+	if a.ClientIPHeader != "" {
+		fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader)
+	} else {
+		fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)")
 	}
-	fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)")
 
 	// Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned
 	// identity anchor (正版优先); config can only append namespace-rewritten third-party
@@ -325,7 +417,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain,
 		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname),
 		defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname),
-		resolvedVersion())
+		cfg.Velocity.GamePort, resolvedVersion())
 	internalSrv := newAPIServer(*internalAddr, a.InternalHandler())
 	externalSrv := newAPIServer(cfg.Server.Listen, externalHandler)
 
@@ -347,16 +439,25 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// and advance any whose Job has reached a terminal phase. GET on a build also
 	// reconciles it, but this loop converges builds nobody is polling.
 	go reconcileBuilds(ctx, builder, stderr)
+	go settleRestoreChains(ctx, a, stderr)
+
+	if pruner := registryPruner(cfg, builder.Store, cluster, stderr); pruner != nil {
+		go pruner.Loop(ctx, registryPruneInterval)
+	}
+	go reapRejectedContexts(ctx, submissions, stderr)
+	go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
+
+	servers := []*http.Server{internalSrv, externalSrv}
+	if httpsSrv != nil {
+		servers = append(servers, httpsSrv)
+	}
+	for _, srv := range servers {
+		srv.RegisterOnShutdown(a.CloseStreams)
+	}
 
 	select {
 	case <-ctx.Done():
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		_ = internalSrv.Shutdown(shutdownCtx)
-		_ = externalSrv.Shutdown(shutdownCtx)
-		if httpsSrv != nil {
-			_ = httpsSrv.Shutdown(shutdownCtx)
-		}
+		shutdownServers(servers, apiShutdownGrace, stderr)
 		return 0
 	case err := <-errc:
 		if err != nil && err != http.ErrServerClosed {
@@ -376,8 +477,30 @@ const (
 	// apiIdleTimeout caps how long a kept-alive connection may sit idle between
 	// requests before the server closes it, bounding idle-connection exhaustion.
 	apiIdleTimeout = 120 * time.Second
+	// apiShutdownGrace is how long the listeners drain after SIGTERM. The pod gets
+	// the Kubernetes default of 30s before SIGKILL; this leaves the rest for the
+	// process to exit.
+	apiShutdownGrace = 20 * time.Second
 )
 
+// shutdownServers drains every listener at once under one deadline: in turn, a
+// slow first listener would spend the time the others needed. Log streams end
+// through RegisterOnShutdown (API.CloseStreams); what is still running when the
+// deadline passes is cut off with the process.
+func shutdownServers(servers []*http.Server, grace time.Duration, stderr io.Writer) {
+	ctx, cancel := context.WithTimeout(context.Background(), grace)
+	defer cancel()
+	var wg sync.WaitGroup
+	for _, srv := range servers {
+		wg.Go(func() {
+			if err := srv.Shutdown(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: shutdown %s: %v\n", srv.Addr, err)
+			}
+		})
+	}
+	wg.Wait()
+}
+
 // newAPIServer builds an http.Server with hardened header/idle timeouts (gosec
 // G112) shared by all three felis-api listeners (internal, external, https).
 // WriteTimeout and ReadTimeout are deliberately LEFT UNSET: the external and https
@@ -401,9 +524,63 @@ func buildConfig(cfg *config.Config) build.Config {
 	return build.Config{
 		Namespace:   cfg.Registry.BuildNamespace,
 		RegistryURL: cfg.Registry.URL,
+		// Empty overrides fall back to the registry's copies of the tools
+		// (build.Tools), which felis mirror-build-tools keeps current.
+		KanikoImage: cfg.Registry.KanikoImage,
+		TrivyImage:  cfg.Registry.TrivyImage,
+		CPULimit:    cfg.Registry.BuildCPULimit,
+		MemLimit:    cfg.Registry.BuildMemLimit,
+		DiskLimit:   cfg.Registry.BuildDiskLimit,
+		// "auto" follows the startup probe (see probeBuildUserNamespaces).
+		UserNamespaces:        cfg.Registry.BuildUserNamespaces,
+		UserNamespacesProbe:   new(atomic.Bool),
+		RuntimeClass:          cfg.Registry.BuildRuntimeClass,
+		MaxConcurrent:         cfg.Registry.MaxConcurrentBuilds,
+		TrivyDBRepository:     cfg.Registry.TrivyDBRepository,
+		TrivyJavaDBRepository: cfg.Registry.TrivyJavaDBRepository,
+		ScanFailOn:            cfg.Registry.ScanFailOn,
+		ScanFailUnfixed:       cfg.Registry.ScanFailUnfixed,
+		ScanAccept:            cfg.Registry.ScanAccept,
+		// The submit lane's derived context URLs live here; the fetch step's
+		// service token goes nowhere else.
+		ContextOrigin: internalAPIBaseURL(),
 	}
 }
 
+// probeBuildUserNamespaces settles build_user_namespaces = "auto": one probe
+// pod with hostUsers: false tells whether this node's kernel and runtime can run
+// build pods in a user namespace. Builds submitted before it answers run without.
+func probeBuildUserNamespaces(ctx context.Context, jobs *build.K8sJobs, cfg build.Config, stderr io.Writer) {
+	if mode := cfg.UserNamespaces; mode != "" && mode != build.UserNamespacesAuto {
+		return
+	}
+	if cfg.FelisImage == "" {
+		fmt.Fprintln(stderr, "felis api: FELIS_IMAGE unset — build pods run without a user namespace")
+		return
+	}
+	ok, err := jobs.ProbeUserNamespaces(ctx, cfg.FelisImage)
+	cfg.UserNamespacesProbe.Store(ok)
+	switch {
+	case ok:
+		fmt.Fprintln(stderr, "felis api: build pods run in a user namespace (hostUsers: false)")
+	case err != nil:
+		fmt.Fprintf(stderr, "felis api: build pods run without a user namespace: the probe failed: %v\n", err)
+	default:
+		fmt.Fprintln(stderr, "felis api: build pods run without a user namespace: this node cannot start a pod with hostUsers: false")
+	}
+}
+
+// internalAPIBaseURL resolves the platform's internal-face base URL: the address
+// the platform rendered into this pod (felis API base URL env), or — for a
+// hand-rolled deployment that set none — the platform default control namespace,
+// the same fallback setup.go uses to hand the login gate its address.
+func internalAPIBaseURL() string {
+	if base := os.Getenv(naming.EnvAPIBaseURL); base != "" {
+		return base
+	}
+	return platform.InternalAPIBaseURL(platform.DefaultControlNamespace)
+}
+
 // uploadsSchemeRE matches a leading URL scheme like "s3://" or "gs://".
 var uploadsSchemeRE = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://`)
 
@@ -506,3 +683,247 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
 		}
 	}
 }
+
+// settleRestoreChains starts the restore behind each safety snapshot that has
+// finished (and gives up the one behind a snapshot that failed). The world stays
+// locked in between, so the interval is how long a finished snapshot keeps the
+// server down before its restore begins.
+func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
+	t := time.NewTicker(5 * time.Second)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := a.SettleRestoreChains(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: restore chains: %v\n", err)
+			}
+		}
+	}
+}
+
+// reapRejectedContexts deletes, once an hour, the uploaded contexts of
+// submissions rejected more than submit.RejectedContextRetention ago, and the
+// chunked uploads left untouched for submit.StalePartRetention. Without it a
+// rejected modpack or an abandoned upload keeps its bytes on the uploads store
+// (and against its submitter's budget) until an admin deletes the row.
+func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
+	t := time.NewTicker(time.Hour)
+	defer t.Stop()
+	for {
+		n, err := m.ReapRejected(ctx, submit.RejectedContextRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap rejected uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
+		}
+		n, err = m.ReapStaleParts(submit.StalePartRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap abandoned uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted %d abandoned chunked upload(s)\n", n)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// retentionInterval spaces the runs that delete spent sign-in rows and audit rows
+// past [audit] retention. The rows are spent for weeks before they go, so a few
+// runs a day keep the tables flat.
+const retentionInterval = 6 * time.Hour
+
+// registryPruneInterval spaces the registry pruner's runs. The registry-gc
+// sidecar sweeps once a day, so pruning more often only changes which sweep frees
+// a layer.
+const registryPruneInterval = 6 * time.Hour
+
+// registryPruner deletes the registry manifests nothing references
+// (internal/registryprune); the registry-gc sidecar frees their layers on its next
+// sweep. It acts as the gate's prune principal, whose token the api Deployment
+// injects from felis-registry-auth. Without the token the registry only grows,
+// which is said once here.
+func registryPruner(cfg *config.Config, store imageRefStore, servers serverLister, stderr io.Writer) *registryprune.Pruner {
+	if cfg.Registry.URL == "" {
+		return nil
+	}
+	token := os.Getenv(platform.RegistryPruneTokenEnv)
+	if token == "" {
+		fmt.Fprintf(stderr, "felis api: registry pruner disabled (%s unset) — images nothing uses are never deleted from the registry\n", platform.RegistryPruneTokenEnv)
+		return nil
+	}
+	static := append([]string{os.Getenv("FELIS_IMAGE")}, buildConfig(cfg).ToolRefs()...)
+	return &registryprune.Pruner{
+		Registry: &registryprune.Client{Endpoint: "http://" + cfg.Registry.URL, Token: token},
+		Host:     cfg.Registry.URL,
+		Refs: func(ctx context.Context) ([]string, error) {
+			return inUseImageRefs(ctx, store, servers, static)
+		},
+		Log: slog.New(slog.NewTextHandler(stderr, nil)),
+	}
+}
+
+type imageRefStore interface {
+	ListImages(ctx context.Context) ([]build.Image, error)
+	ListUnfinishedBuilds(ctx context.Context) ([]build.Build, error)
+}
+
+type serverLister interface {
+	ListServers(ctx context.Context) ([]api.ServerInfo, error)
+	PodImages(ctx context.Context) ([]string, error)
+}
+
+// inUseImageRefs lists every image reference the platform still depends on: the
+// whitelist (disabled rows too, an admin may enable them again), every server's
+// spec, the images the game pods run, builds still running, and the images the
+// control plane and the build Jobs run. Any source failing fails the whole list,
+// so the pruner never decides on a partial view.
+//
+// The pods matter for the felis image: a running server keeps the one it started
+// with across platform upgrades (operator.PodTemplateAnnotation), which after a
+// few releases is no longer among the newest tags the pruner keeps anyway, and
+// the pod needs it again whenever it is recreated.
+func inUseImageRefs(ctx context.Context, store imageRefStore, servers serverLister, static []string) ([]string, error) {
+	refs := append([]string(nil), static...)
+	images, err := store.ListImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("image whitelist: %w", err)
+	}
+	for _, img := range images {
+		refs = append(refs, img.ImageRef)
+	}
+	srvs, err := servers.ListServers(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("servers: %w", err)
+	}
+	for _, s := range srvs {
+		refs = append(refs, s.Image)
+	}
+	podImages, err := servers.PodImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("game pods: %w", err)
+	}
+	refs = append(refs, podImages...)
+	builds, err := store.ListUnfinishedBuilds(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("running builds: %w", err)
+	}
+	for _, b := range builds {
+		refs = append(refs, b.ImageRef)
+	}
+	return refs, nil
+}
+
+// mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket:
+// the hourly cap as the refill rate, with a quarter of it (at least 5) allowed
+// at once so a burst of real sign-ins is not queued behind the average.
+func mailLimit(perHour int) api.RateLimit {
+	if perHour <= 0 {
+		perHour = config.DefaultMailPerHour
+	}
+	return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60}
+}
+
+// imagePinner resolves a new server's image against the platform registry
+// through its in-cluster Service, the address its refs already spell. An install
+// without a registry has no platform-built images to pin.
+func imagePinner(registry string) api.ImagePinner {
+	if registry == "" {
+		return nil
+	}
+	return imagepin.Resolver{Registry: registry}
+}
+
+// internalCallerTokens reads each internal caller's token from the env var the
+// Deployment feeds it from (naming.CallerTokens). Two callers sharing a value
+// would make the caller ambiguous, so that refuses to start.
+func internalCallerTokens(getenv func(string) string) (api.CallerTokens, error) {
+	tokens := map[api.Caller]string{}
+	for _, ct := range naming.CallerTokens {
+		tokens[api.Caller(ct.Caller)] = strings.TrimSpace(getenv(ct.APIEnv))
+	}
+	return api.NewCallerTokens(tokens)
+}
+
+// smtpRelay is the relay [smtp] names, with the resolved password and the TLS
+// posture config.SMTPConfig.TLSRequired picks. felis api, the reaper and the
+// watchdog all send through it, so none can drift to a weaker posture.
+func smtpRelay(c config.SMTPConfig, password string) *mail.SMTP {
+	return &mail.SMTP{
+		Host:       c.Host,
+		Port:       c.Port,
+		From:       c.From,
+		Username:   c.Username,
+		Password:   password,
+		RequireTLS: c.TLSRequired(),
+	}
+}
+
+// startServerCache starts the informer that serves the api's fleet-wide
+// MinecraftServer reads (api.K8sCluster.WithServerCache): one watch on the
+// namespace instead of a full List per velocity pull, fleet page and wake. It
+// caches MinecraftServers only — ReaderFailOnMissingInformer turns any other read
+// through it into an error rather than a new informer the api's Role cannot back —
+// indexes spec.subdomain for GetBySubdomain, and drops managedFields to keep the
+// copy small. It returns without waiting: the reads block until the first list
+// lands and /readyz reports not-ready until then.
+func startServerCache(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, stderr io.Writer) (cache.Cache, func() bool, error) {
+	c, err := cache.New(cfg, cache.Options{
+		Scheme:                      scheme,
+		DefaultNamespaces:           map[string]cache.Config{namespace: {}},
+		DefaultTransform:            cache.TransformStripManagedFields(),
+		ReaderFailOnMissingInformer: true,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := c.IndexField(ctx, &v1alpha1.MinecraftServer{}, api.SubdomainIndex, api.SubdomainOf); err != nil {
+		return nil, nil, fmt.Errorf("index %s: %w", api.SubdomainIndex, err)
+	}
+	inf, err := c.GetInformer(ctx, &v1alpha1.MinecraftServer{})
+	if err != nil {
+		return nil, nil, err
+	}
+	go func() {
+		if err := c.Start(ctx); err != nil {
+			fmt.Fprintf(stderr, "felis api: MinecraftServer cache stopped: %v\n", err)
+		}
+	}()
+	return c, inf.HasSynced, nil
+}
+
+// uploadPartsDir is where chunked uploads are staged: beside a local store's
+// contexts, so the room check and the budget see one disk and a staged upload
+// survives an API restart; for an s3:// store, on the uploads volume the
+// platform mounts either way, or the pod's /tmp when run by hand without it.
+func uploadPartsDir(contextBase string) string {
+	if isLocalUploadsPath(contextBase) {
+		return filepath.Join(strings.TrimPrefix(contextBase, "file://"), ".parts")
+	}
+	if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
+		return filepath.Join(platform.UploadsLocalPath, ".parts")
+	}
+	return filepath.Join(os.TempDir(), "felis-upload-parts")
+}
+
+// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
+// package's own default (1 GiB). The Cloudflare edge refuses a single request
+// body over 100 MB, which the panel's chunked upload stays under, so the edge
+// does not lower the cap.
+func contextMaxBytes(cfg *config.Config) (int64, error) {
+	v := cfg.Registry.ContextMaxBytes
+	if v == "" {
+		return 0, nil
+	}
+	n, err := parseByteSize(v)
+	if err != nil || n <= 0 {
+		return 0, fmt.Errorf("not a positive size: %q", v)
+	}
+	return n, nil
+}

cmd/felis/api_test.go

@@ -1,9 +1,18 @@
 package main
 
 import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
 	"net/http"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
 )
 
@@ -44,6 +53,32 @@ func TestAuthSourcesFromConfig(t *testing.T) {
 	}
 }
 
+// TestBuildConfig_ProjectsOverrides pins the [registry] overrides reaching the
+// build subsystem: unset fields must stay EMPTY (the build package's compiled-in
+// defaults apply there, not here), and set fields must pass through verbatim —
+// an air-gapped install points these at its imported mirrors.
+func TestBuildConfig_ProjectsOverrides(t *testing.T) {
+	empty := buildConfig(&config.Config{})
+	if empty.KanikoImage != "" || empty.TrivyImage != "" || empty.CPULimit != "" || empty.MemLimit != "" {
+		t.Errorf("empty registry config must project empty overrides (defaults live in internal/build), got %+v", empty)
+	}
+	full := buildConfig(&config.Config{Registry: config.RegistryConfig{
+		URL:            "registry.felis.svc:5000",
+		BuildNamespace: "felis-build",
+		KanikoImage:    "reg/kaniko:v1",
+		TrivyImage:     "reg/trivy:v1",
+		BuildCPULimit:  "1",
+		BuildMemLimit:  "2Gi",
+	}})
+	if full.KanikoImage != "reg/kaniko:v1" || full.TrivyImage != "reg/trivy:v1" ||
+		full.CPULimit != "1" || full.MemLimit != "2Gi" {
+		t.Errorf("registry overrides did not reach build.Config: %+v", full)
+	}
+	if full.Namespace != "felis-build" || full.RegistryURL != "registry.felis.svc:5000" {
+		t.Errorf("namespace/registry url must keep projecting: %+v", full)
+	}
+}
+
 // TestNewAPIServerSetsHardenedTimeouts pins the gosec-G112 hardening on every
 // felis-api listener: the shared factory must bound the header and idle phases
 // (Slowloris + idle-connection exhaustion) while leaving WriteTimeout UNSET, because
@@ -65,3 +100,106 @@ func TestNewAPIServerSetsHardenedTimeouts(t *testing.T) {
 		t.Errorf("ReadTimeout = %v, want 0 (unset) so a slow SSE attach is not capped", srv.ReadTimeout)
 	}
 }
+
+// Every listener drains at once, and the shutdown hook (API.CloseStreams in
+// cmdAPI) runs on each: two listeners each holding a request that ends when
+// the hook fires take one hook's worth of time, well inside the deadline.
+func TestShutdownServersDrainsListenersTogether(t *testing.T) {
+	release := make(chan struct{})
+	var hooks int32
+	started := make(chan struct{}, 2)
+	var servers []*http.Server
+	for range 2 {
+		srv := newAPIServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			started <- struct{}{}
+			<-release
+		}))
+		srv.RegisterOnShutdown(func() {
+			if atomic.AddInt32(&hooks, 1) == 1 {
+				time.AfterFunc(100*time.Millisecond, func() { close(release) })
+			}
+		})
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			t.Fatal(err)
+		}
+		srv.Addr = l.Addr().String()
+		go func() { _ = srv.Serve(l) }()
+		go func() {
+			if resp, err := http.Get("http://" + srv.Addr); err == nil {
+				resp.Body.Close()
+			}
+		}()
+		servers = append(servers, srv)
+	}
+	<-started
+	<-started
+
+	begun := time.Now()
+	shutdownServers(servers, 5*time.Second, io.Discard)
+	if took := time.Since(begun); took > 2*time.Second {
+		t.Fatalf("shutdown took %v", took)
+	}
+	if got := atomic.LoadInt32(&hooks); got != 2 {
+		t.Fatalf("shutdown hook ran %d times, want once per listener", got)
+	}
+}
+
+type fakeRefStore struct {
+	images []build.Image
+	builds []build.Build
+	err    error
+}
+
+func (f fakeRefStore) ListImages(context.Context) ([]build.Image, error) { return f.images, f.err }
+func (f fakeRefStore) ListUnfinishedBuilds(context.Context) ([]build.Build, error) {
+	return f.builds, nil
+}
+
+type fakeServers struct {
+	list    []api.ServerInfo
+	pods    []string
+	podsErr error
+}
+
+func (f fakeServers) ListServers(context.Context) ([]api.ServerInfo, error) { return f.list, nil }
+func (f fakeServers) PodImages(context.Context) ([]string, error)           { return f.pods, f.podsErr }
+
+// The registry pruner deletes whatever this list does not name, so every source of
+// a reference has to be in it, and a failing source must fail the list.
+func TestInUseImageRefsCoversEverySource(t *testing.T) {
+	const reg = "registry.felis.svc:5000/"
+	store := fakeRefStore{
+		images: []build.Image{{ImageRef: reg + "modpacks/pack:*"}, {ImageRef: reg + "felis/paper:demo"}},
+		builds: []build.Build{{ImageRef: reg + "user-uploads/sub-9:latest"}},
+	}
+	servers := fakeServers{
+		list: []api.ServerInfo{{Name: "s1", Image: reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1)}},
+		pods: []string{reg + "felis/felis:v1.0.0"},
+	}
+	got, err := inUseImageRefs(context.Background(), store, servers, []string{reg + "felis/felis:b60"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		reg + "felis/felis:b60",
+		reg + "modpacks/pack:*", reg + "felis/paper:demo",
+		reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1),
+		reg + "felis/felis:v1.0.0",
+		reg + "user-uploads/sub-9:latest",
+	}
+	if fmt.Sprint(got) != fmt.Sprint(want) {
+		t.Fatalf("refs = %v\nwant %v", got, want)
+	}
+
+	servers.podsErr = errors.New("apiserver down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing pod list produced a reference list")
+	}
+	servers.podsErr = nil
+
+	store.err = errors.New("db down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing whitelist read produced a reference list")
+	}
+}

cmd/felis/api_tokens_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/api"
+)
+
+// felis-api maps each env var onto the caller the Deployment feeds it for, so the
+// build Job's token (FELIS_BUILD_TOKEN) authenticates as build and only as build.
+func TestInternalCallerTokensReadEachCallersEnv(t *testing.T) {
+	env := map[string]string{
+		"FELIS_SERVICE_TOKEN": "v-tok",
+		"FELIS_LIMBO_TOKEN":   "l-tok",
+		"FELIS_BUILD_TOKEN":   " b-tok\n",
+		"FELIS_OPS_TOKEN":     "o-tok",
+	}
+	auth, err := internalCallerTokens(func(k string) string { return env[k] })
+	if err != nil {
+		t.Fatal(err)
+	}
+	for tok, want := range map[string]api.Caller{"v-tok": api.CallerVelocity, "l-tok": api.CallerLimbo, "b-tok": api.CallerBuild, "o-tok": api.CallerOps} {
+		r := httptest.NewRequest("GET", "/", nil)
+		r.Header.Set("Authorization", "Bearer "+tok)
+		if got, err := auth.Authenticate(r); err != nil || got != want {
+			t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want)
+		}
+	}
+
+	// An install where the build namespace still holds a copy of the proxy's token
+	// would let that copy act as the proxy; the api refuses to start on it.
+	env["FELIS_BUILD_TOKEN"] = "v-tok"
+	if _, err := internalCallerTokens(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "same value") {
+		t.Fatalf("shared token: err = %v, want a same-value refusal", err)
+	}
+}

cmd/felis/apply.go

@@ -243,6 +243,19 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 			AutostartPolicy: policy,
 			Storage:         v1alpha1.StorageSpec{Size: storageQ.String()},
 			Resources:       corev1.ResourceRequirements{Limits: limits, Requests: requests},
+			// The rest matches what felis-api's create writes (K8sCluster.CreateServer):
+			// fall back to the login gate while stopped, RCON on (readiness, the
+			// player count and the console all ride it; the operator mints the
+			// password), and the default idle stop.
+			FallbackServer: naming.SystemLoginServer,
+			Rcon: v1alpha1.RconSpec{
+				Enabled: true,
+				SecretRef: v1alpha1.SecretKeyRef{
+					Name: naming.RconSecretName(req.Name),
+					Key:  naming.RconSecretKey,
+				},
+			},
+			Idle: v1alpha1.DefaultIdle(),
 		},
 	}, nil
 }

cmd/felis/apply_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
@@ -167,6 +168,18 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	if ms.Spec.Storage.Size != "20Gi" {
 		t.Errorf("Storage.Size = %q, want 20Gi", ms.Spec.Storage.Size)
 	}
+	// Same operational defaults as the API create path: without RCON the server
+	// never reports players and the console answers 503; without spec.idle it
+	// never stops on its own.
+	if !ms.Spec.Rcon.Enabled || ms.Spec.Rcon.SecretRef.Name != naming.RconSecretName("test-server") {
+		t.Errorf("Rcon = %+v, want enabled with the operator-minted secret", ms.Spec.Rcon)
+	}
+	if ms.Spec.Idle != v1alpha1.DefaultIdle() {
+		t.Errorf("Idle = %+v, want the default %+v", ms.Spec.Idle, v1alpha1.DefaultIdle())
+	}
+	if ms.Spec.FallbackServer != naming.SystemLoginServer {
+		t.Errorf("FallbackServer = %q, want the login gate", ms.Spec.FallbackServer)
+	}
 	mem, ok := ms.Spec.Resources.Limits[corev1.ResourceMemory]
 	if !ok {
 		t.Fatal("memory limit missing")

cmd/felis/backup.go

@@ -1,14 +1,17 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	"felis.lolicon.best/internal/backup"
+	"felis.lolicon.best/internal/backupjob"
 	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/reaper"
@@ -35,6 +38,8 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	server := fs.String("server", "", "server name whose world is being backed up")
 	formerOwner := fs.String("former-owner", "", "owner recorded on the backup row (empty for an unowned server)")
 	worldsRoot := fs.String("worlds-root", "/world", "mount path of the world PVC being archived")
+	reason := fs.String("reason", reasonManual, "world_backups reason: manual, or pre_restore for the safety snapshot in front of a restore")
+	protect := fs.String("protect", "", "backup id the prune must keep (the one a chained restore extracts)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -42,6 +47,11 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintln(stderr, "felis backup: --server is required")
 		return 2
 	}
+	keep, ok := map[string]int{reasonManual: -1, backupjob.ReasonPreRestore: preRestoreKeep}[*reason]
+	if !ok {
+		fmt.Fprintf(stderr, "felis backup: unknown --reason %q (manual or %s)\n", *reason, backupjob.ReasonPreRestore)
+		return 2
+	}
 
 	cfg, err := config.Load(*cfgPath)
 	if err != nil {
@@ -52,13 +62,16 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis backup: archive store %q is not implemented in this build (only tarLocal)\n", cfg.Archive.Store)
 		return 1
 	}
-	// Reuse the reaper's retention derivation so an on-demand backup expires on the
-	// same clock as an inactivity backup — one retention policy, not two.
+	// The [archive] parse the reaper uses; an on-demand backup takes its
+	// manual_retention and manual_keep.
 	rcfg, err := reaperConfig(cfg)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: %v\n", err)
 		return 1
 	}
+	if keep < 0 {
+		keep = rcfg.ManualKeep
+	}
 
 	// The world PVC is mounted directly at worldsRoot; the resolver returns it for
 	// any target, exactly as in cmdRestore. This is the same TarLocal the reaper
@@ -72,11 +85,23 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 
 	ctx := ctrl.SetupSignalHandler()
 
-	ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
+	// The archive store shares the node's disk with every world and the
+	// database: an owner's backup must not be what tips it into eviction.
+	if err := backup.CheckRoom(cfg.Archive.LocalPath, *worldsRoot, backup.MinFreeAfter); err != nil {
+		fmt.Fprintf(stderr, "felis backup: %v\n", err)
+		return 1
+	}
+
+	a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
 		return 1
 	}
+	ref, size := a.Ref, a.Size
+	if len(a.Skipped) > 0 {
+		fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",
+			len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))
+	}
 
 	drv, err := store.Open(ctx, cfg.Database.URL)
 	if err != nil {
@@ -91,10 +116,14 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		FormerOwner: *formerOwner,
 		BackupRef:   string(ref),
 		SizeBytes:   size,
-		Reason:      "manual",
-		ExpiresAt:   time.Now().Add(rcfg.Retention),
+		Reason:      *reason,
+		ExpiresAt:   time.Now().Add(rcfg.ManualRetention),
+
+		SHA256:         a.SHA256,
+		SkippedEntries: len(a.Skipped),
 	}
-	if err := reaper.NewPGStore(drv.DB()).InsertBackup(ctx, rec); err != nil {
+	st := reaper.NewPGStore(drv.DB())
+	if err := st.InsertBackup(ctx, rec); err != nil {
 		// The archive is written but unrecorded — an orphan the retention pass would
 		// never expire. Delete it so a failed backup leaves no leaked bytes, mirroring
 		// the reaper's archive-then-record atomicity.
@@ -107,9 +136,42 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	}
 
 	fmt.Fprintf(stdout, "felis backup: server=%s archived %d bytes to %s (backup %s)\n", *server, size, ref, rec.ID)
+	pruneBackups(ctx, st, archiver, *server, *reason, keep, *protect, stdout, stderr)
 	return 0
 }
 
+const (
+	reasonManual = "manual"
+	// preRestoreKeep is how many safety snapshots a server keeps: enough to walk
+	// back a couple of restores in a row, without every restore adding a world's
+	// worth of bytes for the full manual retention.
+	preRestoreKeep = 3
+)
+
+// pruneBackups keeps server's newest keep backups of this reason and removes the
+// rest, oldest first, so repeated backups of one world cannot fill the shared
+// archive store. protect is never removed: it is the backup a chained restore is
+// about to extract. The new backup is already recorded; a removal that fails is
+// reported and retried after the next backup.
+func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server, reason string, keep int, protect string, stdout, stderr io.Writer) {
+	excess, err := st.ExcessBackups(ctx, server, reason, keep, protect)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)
+		return
+	}
+	for _, b := range excess {
+		if err := archiver.Delete(ctx, backup.ArchiveRef(b.BackupRef)); err != nil {
+			fmt.Fprintf(stderr, "felis backup: remove older backup %s: %v\n", b.ID, err)
+			continue
+		}
+		if err := st.MarkBackupDeleted(ctx, b.ID, time.Now()); err != nil {
+			fmt.Fprintf(stderr, "felis backup: record the removal of %s: %v\n", b.ID, err)
+			continue
+		}
+		fmt.Fprintf(stdout, "felis backup: removed older %s backup %s of %s (keeping the newest %d)\n", reason, b.ID, server, keep)
+	}
+}
+
 // newBackupID mints a world_backups primary key, matching the reaper's "bk-"+hex
 // scheme so a manual and an inactivity backup are indistinguishable downstream.
 func newBackupID() string {

cmd/felis/backup_test.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+// The reason decides which backups the new one's prune may remove, so an
+// unknown one is refused before anything is archived.
+func TestBackupSubcommandRejectsUnknownReason(t *testing.T) {
+	var stderr bytes.Buffer
+	if code := cmdBackup([]string{"--server", "survival", "--reason", "inactive_15d"}, &bytes.Buffer{}, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2 (%s)", code, stderr.String())
+	}
+	if !strings.Contains(stderr.String(), "unknown --reason") {
+		t.Fatalf("stderr = %q", stderr.String())
+	}
+}

cmd/felis/backupnow.go

@@ -20,7 +20,7 @@ import (
 // backupnow is the break-glass "back up a world now" op (§B4 "Sync"). Unlike halt —
 // which writes the CRD directly — a backup needs felis-api's deployment coordinates
 // (FELIS_IMAGE / FELIS_BACKUP_PVC) to render the one-shot backup Job, so the console
-// cannot do it in-process. It POSTs the felis-api INTERNAL face (service-token auth)
+// cannot do it in-process. It POSTs the felis-api INTERNAL face (ops-token auth)
 // while the API is alive, and the API renders the Job and audits the action. This file
 // is the pure core (no bubbletea); tui_backupnow.go is the terminal glue.
 
@@ -33,7 +33,7 @@ type backupNowOutcome struct {
 
 // resolveInternalAPI reads the two things the on-node console needs to reach the
 // felis-api internal face: the felis-api-internal Service ClusterIP (the host's
-// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the service
+// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the ops
 // token. Both live in the control namespace.
 func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace string) (baseURL, token string, err error) {
 	var svc corev1.Service
@@ -45,13 +45,14 @@ func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace
 		return "", "", fmt.Errorf("%s Service has no ClusterIP yet", platform.APIInternalServiceName)
 	}
 
+	// The console's own token, which the api serves on the backup route alone.
 	var sec corev1.Secret
-	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.ServiceTokenSecretName}, &sec); err != nil {
-		return "", "", fmt.Errorf("get %s Secret: %w", naming.ServiceTokenSecretName, err)
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.OpsTokenSecretName}, &sec); err != nil {
+		return "", "", fmt.Errorf("get %s Secret (re-run the installer to create it): %w", naming.OpsTokenSecretName, err)
 	}
 	token = string(sec.Data[naming.ServiceTokenSecretKey])
 	if token == "" {
-		return "", "", fmt.Errorf("secret %s has no %s key", naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey)
+		return "", "", fmt.Errorf("secret %s has no %s key", naming.OpsTokenSecretName, naming.ServiceTokenSecretKey)
 	}
 
 	return fmt.Sprintf("http://%s:%d", ip, platform.APIInternalPort), token, nil
@@ -86,23 +87,31 @@ func requestBackup(ctx context.Context, hc *http.Client, baseURL, token, name, o
 
 // backupErrorFromResponse turns a non-202 into a human message. The well-known codes get
 // an operator-facing explanation; anything else falls back to the API's
-// {"error":{message}} body, then the bare status code.
+// {"error":{code,message}} body, then the bare status code.
 func backupErrorFromResponse(resp *http.Response) error {
-	switch resp.StatusCode {
-	case http.StatusConflict: // not_stopped
-		return fmt.Errorf("the server must be stopped before its world can be backed up — halt it first")
-	case http.StatusServiceUnavailable: // backup_unavailable
-		return fmt.Errorf("the backup subsystem is not configured on felis-api (FELIS_IMAGE / FELIS_BACKUP_PVC unset)")
-	case http.StatusNotFound:
-		return fmt.Errorf("no such server")
-	}
 	var e struct {
 		Error struct {
+			Code    string `json:"code"`
 			Message string `json:"message"`
 		} `json:"error"`
 	}
 	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
 	_ = json.Unmarshal(raw, &e)
+
+	switch resp.StatusCode {
+	case http.StatusConflict:
+		// Two refusals share 409: the stopped gate and the missing-world-volume
+		// gate. The body's code distinguishes them; a code-less body reads as the
+		// stopped gate (the only 409 before the volume gate existed), and any other
+		// coded 409 falls through to the API's own operator text.
+		if e.Error.Code == "" || e.Error.Code == "not_stopped" {
+			return fmt.Errorf("the server must be stopped before its world can be backed up — halt it first")
+		}
+	case http.StatusServiceUnavailable: // backup_unavailable
+		return fmt.Errorf("the backup subsystem is not configured on felis-api (FELIS_IMAGE / FELIS_BACKUP_PVC unset)")
+	case http.StatusNotFound:
+		return fmt.Errorf("no such server")
+	}
 	if e.Error.Message != "" {
 		return fmt.Errorf("felis-api: %s", e.Error.Message)
 	}
@@ -120,3 +129,19 @@ func performBackupNow(ctx context.Context, cl client.Client, controlNamespace, n
 	hc := &http.Client{Timeout: 10 * time.Second}
 	return requestBackup(ctx, hc, baseURL, token, name, osUser)
 }
+
+// backupPickable narrows the backup picker to servers the backup API can accept.
+// System servers (login/lobby) are excluded: they have no row in the servers
+// table and carry reserved names, so every attempt dies in name validation —
+// offering them would be a dead pick. The halt picker keeps them on purpose
+// (break-glass retains full power over system servers); only the API-backed
+// backup op cannot reach them.
+func backupPickable(servers []haltableServer) []haltableServer {
+	out := make([]haltableServer, 0, len(servers))
+	for _, s := range servers {
+		if !s.system {
+			out = append(out, s)
+		}
+	}
+	return out
+}

cmd/felis/backupnow_test.go

@@ -11,7 +11,6 @@ import (
 	"testing"
 	"time"
 
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 
 	corev1 "k8s.io/api/core/v1"
@@ -28,9 +27,15 @@ func internalAPIObjs(clusterIP, token string) []client.Object {
 			ObjectMeta: metav1.ObjectMeta{Name: platform.APIInternalServiceName, Namespace: bgControlNS},
 			Spec:       corev1.ServiceSpec{ClusterIP: clusterIP},
 		},
+		// The console presents the ops token; the proxy's felis-service-token sits
+		// beside it and must not be the one picked up.
 		&corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: bgControlNS},
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-ops-token", Namespace: bgControlNS},
+			Data:       map[string][]byte{"token": []byte(token)},
+		},
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-service-token", Namespace: bgControlNS},
+			Data:       map[string][]byte{"token": []byte("proxy-" + token)},
 		},
 	}
 }
@@ -114,16 +119,23 @@ func TestRequestBackup(t *testing.T) {
 	cases := []struct {
 		name   string
 		code   int
+		body   string // optional JSON error body
 		expect string
 	}{
-		{"409 not_stopped", http.StatusConflict, "must be stopped"},
-		{"503 backup_unavailable", http.StatusServiceUnavailable, "not configured"},
-		{"404 not found", http.StatusNotFound, "no such server"},
+		{"409 not_stopped", http.StatusConflict, "", "must be stopped"},
+		{"409 no_world_volume surfaces the API's own text", http.StatusConflict,
+			`{"error":{"code":"no_world_volume","message":"this server has no world volume yet — start it once to create it, then retry"}}`,
+			"no world volume yet"},
+		{"503 backup_unavailable", http.StatusServiceUnavailable, "", "not configured"},
+		{"404 not found", http.StatusNotFound, "", "no such server"},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(tc.code)
+				if tc.body != "" {
+					_, _ = io.WriteString(w, tc.body)
+				}
 			}))
 			defer srv.Close()
 			_, err := requestBackup(context.Background(), hc, srv.URL, "tok", "survival", "alice")
@@ -142,3 +154,25 @@ func TestRequestBackup(t *testing.T) {
 		}
 	})
 }
+
+// The Sync picker must not offer system servers: the backup API validates names
+// and resolves a servers-table row, so a lobby/login pick can only die in
+// validation — a dead choice in an emergency console.
+func TestBackupPickable(t *testing.T) {
+	got := backupPickable([]haltableServer{
+		{name: "lobby", phase: "Running", system: true},
+		{name: "login", phase: "Running", system: true},
+		{name: "test-one", phase: "Stopped"},
+	})
+	if len(got) != 1 || got[0].name != "test-one" || got[0].system {
+		t.Fatalf("backupPickable = %+v, want only the user server", got)
+	}
+
+	// Survivors keep their input order (the picker's cursor math depends on it).
+	got = backupPickable([]haltableServer{
+		{name: "alpha"}, {name: "login", system: true}, {name: "beta"},
+	})
+	if len(got) != 2 || got[0].name != "alpha" || got[1].name != "beta" {
+		t.Fatalf("backupPickable order = %+v, want [alpha beta]", got)
+	}
+}

cmd/felis/breakglass.go

@@ -76,12 +76,19 @@ type ownerStore interface {
 	AdminExists(ctx context.Context) (bool, error)
 	// UserByUsername loads a staff login projection.
 	UserByUsername(ctx context.Context, username string) (*api.StaffUser, error)
+	// OwnerUsername names the single active Owner seat, or "" when none exists.
+	// provisionOwner refuses to re-target anything but this username: with the
+	// seat occupied, a fresh name would mint a second owner row (UpsertOwner's
+	// insert arm) while the existing — possibly compromised — seat stays live,
+	// and no supported path can delete an owner row afterwards.
+	OwnerUsername(ctx context.Context) (string, error)
 	UpsertOwner(ctx context.Context, id, username, email string) error
 	// InsertOperator mints a NEW Operator staff account. Unlike UpsertOwner it is
 	// insert-only: a username already taken is a conflict (api.ErrConflict), never a
 	// silent reset, so adding an Operator can never clobber the Owner or an existing
-	// Operator. The row is role=admin, identical in shape to the Owner — Felis has no
-	// separate operator DB role (migration 0003: staff = role=admin).
+	// Operator. The row is role=admin — an Operator is staff BELOW the single
+	// role=owner identity (migration 0011 adds that role); the two are the only
+	// staff roles.
 	InsertOperator(ctx context.Context, id, username, email string) error
 	// CompleteOwnerSetup atomically consumes the in-game link code, creates or
 	// promotes the bound Owner, enables local auth, and stores the one-time setup
@@ -266,20 +273,31 @@ func authenticateAdmin(ctx context.Context, s ownerStore, username string) (matc
 	if err != nil {
 		return "", false, err
 	}
-	if u.Role != "admin" {
+	// Staff means admin OR owner: recovery attribution must accept the Owner (the
+	// primary break-glass identity), not just plain admins.
+	if u.Role != "admin" && u.Role != "owner" {
 		return "", false, nil
 	}
 	return u.Username, true, nil
 }
 
 // provisionOwner mints or resets the single Owner account direct-to-Postgres,
-// passwordless. The account is role=admin with no password — the Owner completes
+// passwordless. The account is role=owner with no password — the Owner completes
 // passwordless login setup via the web setup-token flow after `felis setup`.
+// With a seat already occupied the reset must name that seat (ownerSeatTakenError
+// otherwise): the upsert's insert arm would silently mint a SECOND owner, and
+// every owner row is undeletable through the panel, so the tier could never
+// converge back to one.
 func provisionOwner(ctx context.Context, s ownerStore, username, email string) error {
 	username = strings.TrimSpace(username)
 	if username == "" {
 		return errors.New("owner username is required")
 	}
+	if seat, err := s.OwnerUsername(ctx); err != nil {
+		return fmt.Errorf("check the owner seat: %w", err)
+	} else if seat != "" && seat != username {
+		return &ownerSeatTakenError{seat: seat}
+	}
 	id := newOwnerID()
 	if id == "" {
 		return errors.New("generate owner id: entropy source failed")
@@ -290,13 +308,26 @@ func provisionOwner(ctx context.Context, s ownerStore, username, email string) e
 	return nil
 }
 
-// provisionOperator mints a NEW Operator staff account direct-to-Postgres. Like the
-// Owner it is role=admin and passwordless — Felis has no separate operator DB role,
-// so an Operator is simply an additional staff admin (migration 0003). UNLIKE
-// provisionOwner, which upserts the single Owner and resets it on a username
-// conflict, this is insert-only: a username already taken returns api.ErrConflict
-// rather than overwriting a live account, so adding an Operator can never silently
-// clobber the Owner's or another Operator's account.
+// ownerSeatTakenError refuses an Owner reset that names anything but the
+// occupied seat, naming it so the operator can retype. Is reports
+// api.ErrConflict so the TUI's recoverable-error branch (shared with the
+// operator path's taken-name clash) routes back to the form instead of ending
+// the console.
+type ownerSeatTakenError struct{ seat string }
+
+func (e *ownerSeatTakenError) Error() string {
+	return fmt.Sprintf("an Owner already exists as %q — enter that username to reset the Owner", e.seat)
+}
+
+func (e *ownerSeatTakenError) Is(target error) bool { return target == api.ErrConflict }
+
+// provisionOperator mints a NEW Operator staff account direct-to-Postgres. It is
+// role=admin and passwordless — an additional staff admin below the single
+// role=owner identity (migrations 0003 + 0011). UNLIKE provisionOwner, which
+// upserts the single Owner and resets it on a username conflict, this is
+// insert-only: a username already taken returns api.ErrConflict rather than
+// overwriting a live account, so adding an Operator can never silently clobber
+// the Owner's or another Operator's account.
 func provisionOperator(ctx context.Context, s ownerStore, username, email string) error {
 	username = strings.TrimSpace(username)
 	if username == "" {
@@ -387,7 +418,7 @@ func newSetupToken() (raw, hash string, err error) {
 
 // performSetupMCBind is the `felis setup` Owner-establishment path: the operator
 // binds their Minecraft account via a one-time link code the login gate handed
-// them in-game, the bound user is promoted to role='admin' (passwordless Owner),
+// them in-game, the bound user is promoted to role='owner' (passwordless Owner),
 // local auth is enabled, and a one-time setup URL is minted for the first web
 // login where the Owner verifies email / enrolls a passkey. adminHostname is the
 // operator-console host the URL points at (op.console.<root>): the Owner is staff,

cmd/felis/breakglass_test.go

@@ -27,6 +27,7 @@ type fakeOwnerStore struct {
 	redeems   []redeemCall
 	users     map[string]*api.StaffUser // keyed by username
 	admins    bool                      // AdminExists answer
+	ownerSeat string                    // OwnerUsername answer: the occupied seat, "" when none
 
 	// CompleteOwnerSetup's success result. redeemUserID defaults to the fresh id
 	// the caller passes (the unlinked-UUID case) when left empty.
@@ -40,6 +41,7 @@ type fakeOwnerStore struct {
 	auditErr       error
 	userErr        error // non-not-found error from UserByUsername
 	adminErr       error
+	seatErr        error
 	redeemErr      error
 	createTokenErr error
 }
@@ -80,6 +82,15 @@ func (f *fakeOwnerStore) UserByUsername(_ context.Context, username string) (*ap
 	return nil, api.ErrNotFound
 }
 
+// OwnerUsername reports the single active Owner seat. Tests set ownerSeat; the
+// zero value models a fresh install where bootstrap is free to mint.
+func (f *fakeOwnerStore) OwnerUsername(_ context.Context) (string, error) {
+	if f.seatErr != nil {
+		return "", f.seatErr
+	}
+	return f.ownerSeat, nil
+}
+
 func (f *fakeOwnerStore) UpsertOwner(_ context.Context, id, username, email string) error {
 	if f.upsertErr != nil {
 		return f.upsertErr
@@ -201,6 +212,34 @@ func TestProvisionOwner(t *testing.T) {
 		}
 	})
 
+	t.Run("an occupied seat refuses any other username", func(t *testing.T) {
+		// The seat is the single owner row: upserting a fresh name would take the
+		// insert arm and mint a SECOND owner, while the existing seat — possibly the
+		// compromised account this reset was meant to replace — stays live, and no
+		// supported path can delete an owner row.
+		f := &fakeOwnerStore{ownerSeat: "seat-holder"}
+		err := provisionOwner(ctx, f, "someone-else", "")
+		if !errors.Is(err, api.ErrConflict) {
+			t.Fatalf("error = %v, want it to wrap api.ErrConflict so the TUI routes back to the form", err)
+		}
+		if !strings.Contains(err.Error(), `"seat-holder"`) {
+			t.Errorf("error = %q, want it to name the occupied seat", err)
+		}
+		if len(f.upserts) != 0 {
+			t.Errorf("want no write against an occupied seat, got %d", len(f.upserts))
+		}
+	})
+
+	t.Run("the occupied seat's own username still resets", func(t *testing.T) {
+		f := &fakeOwnerStore{ownerSeat: "seat-holder"}
+		if err := provisionOwner(ctx, f, "seat-holder", "[email protected]"); err != nil {
+			t.Fatalf("provisionOwner(reset): %v", err)
+		}
+		if len(f.upserts) != 1 || f.upserts[0].username != "seat-holder" || f.upserts[0].email != "[email protected]" {
+			t.Fatalf("want 1 reset upsert for the seat, got %+v", f.upserts)
+		}
+	})
+
 	t.Run("propagates a store error", func(t *testing.T) {
 		f := &fakeOwnerStore{upsertErr: errors.New("boom")}
 		if err := provisionOwner(ctx, f, "owner", ""); err == nil {
@@ -264,6 +303,16 @@ func TestAuthenticateAdmin(t *testing.T) {
 		}
 	})
 
+	t.Run("the owner role attributes like an admin", func(t *testing.T) {
+		owner := mkAdmin("root")
+		owner.Role = "owner" // the platform owner is staff too (migration 0011)
+		f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": owner}}
+		matched, ok, err := authenticateAdmin(ctx, f, "root")
+		if err != nil || !ok || matched != "root" {
+			t.Fatalf("authenticateAdmin(owner) = (%q, %v, %v), want (root, true, nil)", matched, ok, err)
+		}
+	})
+
 	t.Run("an unknown user is a non-match, not an error", func(t *testing.T) {
 		f := &fakeOwnerStore{}
 		_, ok, err := authenticateAdmin(ctx, f, "nobody")

cmd/felis/conflict_test.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// racingClient lands one concurrent write (race) on the stored object just before
+// setup's first write of it goes out, the way the operator's status update or
+// felis-api's idle patch can, and counts setup's writes.
+func racingClient(t *testing.T, race func(ctx context.Context, c client.WithWatch), objs ...client.Object) (client.Client, *int) {
+	t.Helper()
+	writes := 0
+	before := func(ctx context.Context, c client.WithWatch) {
+		writes++
+		if writes == 1 {
+			race(ctx, c)
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
+		WithStatusSubresource(&v1alpha1.MinecraftServer{}).
+		WithInterceptorFuncs(interceptor.Funcs{
+			Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
+				before(ctx, c)
+				return c.Update(ctx, obj, opts...)
+			},
+			Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+				before(ctx, c)
+				return c.Patch(ctx, obj, patch, opts...)
+			},
+		}).Build()
+	return cl, &writes
+}
+
+func staleLoginGate(t *testing.T) *v1alpha1.MinecraftServer {
+	t.Helper()
+	ms, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://old.internal:8081", "203.0.113.10.nip.io", "console.203.0.113.10.nip.io")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ms
+}
+
+func getLogin(t *testing.T, cl client.Client) *v1alpha1.MinecraftServer {
+	t.Helper()
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	return &ms
+}
+
+func envMap(ms *v1alpha1.MinecraftServer) map[string]string {
+	m := map[string]string{}
+	for _, e := range ms.Spec.Env {
+		m[e.Name] = e.Value
+	}
+	return m
+}
+
+// A hand edit that lands while setup refreshes the console hostnames costs setup a
+// re-read and a second write; both the edit and the refresh survive.
+func TestRefreshDerivedEnvRetriesAConcurrentEdit(t *testing.T) {
+	ctx := context.Background()
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		ms := getLogin(t, c)
+		ms.Spec.Env = append(ms.Spec.Env, v1alpha1.EnvVar{Name: "HAND_TUNED", Value: "1"})
+		if err := c.Update(ctx, ms); err != nil {
+			t.Fatal(err)
+		}
+	}, staleLoginGate(t))
+
+	desired, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatal(err)
+	}
+	refreshed, err := refreshDerivedEnv(ctx, cl, getLogin(t, cl), desired)
+	if err != nil || !refreshed {
+		t.Fatalf("refreshed=%v err=%v, want a refresh after the retry", refreshed, err)
+	}
+	env := envMap(getLogin(t, cl))
+	if env[envPanelHostname] != "console.mc.example.net" || env[envRootDomain] != "mc.example.net" {
+		t.Errorf("env = %v, want the new hostnames", env)
+	}
+	if env["HAND_TUNED"] != "1" {
+		t.Errorf("env = %v: the concurrent hand edit was dropped", env)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2 (one conflict, one retry)", *writes)
+	}
+}
+
+// converge reports each fill once even when a status write forced a retry.
+func TestConvergeRetriesAConcurrentStatusWrite(t *testing.T) {
+	ctx := context.Background()
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatal(err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Status.Phase = v1alpha1.PhaseRunning
+		if err := c.Status().Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, lobby)
+
+	var got systemServerOutcome
+	for _, o := range convergeSystemServers(ctx, cl, "minecraft", "", "reg/lobby:1",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
+		if o.name == naming.SystemLobbyServer {
+			got = o
+		}
+	}
+	if got.err != nil || !got.updated || !slices.Equal(got.changes, []string{"spec.rcon"}) {
+		t.Fatalf("lobby outcome = %+v, want spec.rcon filled once", got)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if !ms.Spec.Rcon.Enabled || ms.Status.Phase != v1alpha1.PhaseRunning {
+		t.Errorf("rcon.enabled=%v phase=%q, want the fill and the status write both kept", ms.Spec.Rcon.Enabled, ms.Status.Phase)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+// A replica refresh racing another writer keeps that writer's key.
+func TestSecretReplicaRefreshRetriesAConcurrentWrite(t *testing.T) {
+	secret := func(ns, body string) *corev1.Secret {
+		return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
+			Data: map[string][]byte{"felis.toml": []byte(body)}}
+	}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var s corev1.Secret
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+			t.Fatal(err)
+		}
+		s.Data["extra"] = []byte("x")
+		if err := c.Update(ctx, &s); err != nil {
+			t.Fatal(err)
+		}
+	}, secret("felis", "current"), secret("minecraft", "stale"))
+
+	out := ensureSecretReplica(context.Background(), cl, "felis", "minecraft",
+		"felis-config", "felis.toml", "config", "minecraft ns", true)
+	if out.err != nil || !out.updated {
+		t.Fatalf("outcome = %+v, want refreshed", out)
+	}
+	var s corev1.Secret
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+		t.Fatal(err)
+	}
+	if string(s.Data["felis.toml"]) != "current" || string(s.Data["extra"]) != "x" {
+		t.Errorf("replica data = %q, want felis.toml=current and extra=x", s.Data)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+const (
+	sysOldDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
+	sysNewDigest = "sha256:4444444444444444444444444444444444444444444444444444444444444444"
+)
+
+func systemPinRegistry(t *testing.T) imagepin.Resolver {
+	t.Helper()
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v2/felis/limbo/manifests/demo", "/v2/felis/lobby/manifests/demo":
+			w.Header().Set("Docker-Content-Digest", sysNewDigest)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(reg.Close)
+	return imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+}
+
+func systemServer(name, image string) *v1alpha1.MinecraftServer {
+	ms := &v1alpha1.MinecraftServer{}
+	ms.Name, ms.Namespace = name, "minecraft"
+	ms.Labels = map[string]string{v1alpha1.LabelSystemRole: name}
+	ms.Spec.Image = image
+	return ms
+}
+
+// The installer's --system pin moves a system server onto the build its tag names
+// now, from the bare tag or from an earlier digest, and is a no-op the second time.
+func TestPinSystemServerImage(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	limbo := defaultRegistryURL + "/felis/limbo:demo"
+	lobby := defaultRegistryURL + "/felis/lobby:demo"
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		systemServer(naming.SystemLoginServer, limbo),
+		systemServer(naming.SystemLobbyServer, lobby+"@"+sysOldDigest),
+	).Build()
+	image := func(name string) string {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		return ms.Spec.Image
+	}
+
+	for name, want := range map[string]string{
+		naming.SystemLoginServer: limbo + "@" + sysNewDigest,
+		naming.SystemLobbyServer: lobby + "@" + sysNewDigest,
+	} {
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || o.err != nil || !o.updated {
+			t.Fatalf("%s: outcome=%+v err=%v, want pinned", name, o, err)
+		}
+		if got := image(name); got != want {
+			t.Errorf("%s image = %q, want %q", name, got, want)
+		}
+		again, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || again.updated || again.skipped != "already runs "+want {
+			t.Errorf("%s second pass = %+v, %v; want already runs %s", name, again, err, want)
+		}
+	}
+}
+
+func TestPinSystemServerImageLeavesOthersAlone(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	pin := func(t *testing.T, ms *v1alpha1.MinecraftServer) (systemServerOutcome, string) {
+		t.Helper()
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(ms).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(ms), &got); err != nil {
+			t.Fatal(err)
+		}
+		return o, got.Spec.Image
+	}
+
+	t.Run("external image", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, "docker.io/example/limbo:1.2"))
+		if o.err != nil || o.updated || img != "docker.io/example/limbo:1.2" ||
+			o.skipped != "runs docker.io/example/limbo:1.2, which names no platform registry tag to follow; left alone" {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("digest without a tag", func(t *testing.T) {
+		ref := defaultRegistryURL + "/felis/limbo@" + sysOldDigest
+		o, img := pin(t, systemServer(naming.SystemLoginServer, ref))
+		if o.err != nil || o.updated || img != ref {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("not a system server", func(t *testing.T) {
+		ms := systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo")
+		ms.Labels = nil
+		o, img := pin(t, ms)
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:demo" {
+			t.Fatalf("outcome=%+v image=%q, want refused", o, img)
+		}
+	})
+	t.Run("tag the registry lost", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:gone"))
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:gone" {
+			t.Fatalf("outcome=%+v image=%q, want an error the installer falls back on", o, img)
+		}
+	})
+	t.Run("absent", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.skipped != "not present yet; sudo felis setup creates it" {
+			t.Fatalf("outcome=%+v err=%v, want a skip", o, err)
+		}
+	})
+	t.Run("retargeted while pinning", func(t *testing.T) {
+		cl, _ := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+			ms := getLogin(t, c)
+			ms.Spec.Image = "docker.io/example/limbo:1.2"
+			if err := c.Update(ctx, ms); err != nil {
+				t.Fatal(err)
+			}
+		}, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo"))
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.updated {
+			t.Fatalf("outcome=%+v err=%v, want nothing written", o, err)
+		}
+		if img := getLogin(t, cl).Spec.Image; img != "docker.io/example/limbo:1.2" {
+			t.Errorf("image = %q, want the admin's retarget kept", img)
+		}
+	})
+}
+
+// --system names one of the two system servers; anything else is a usage error
+// before any cluster is touched.
+func TestPinImagesSystemFlagTakesOnlySystemServers(t *testing.T) {
+	var stdout, stderr strings.Builder
+	if code := cmdPinImages([]string{"--system", "survival"}, &stdout, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2", code)
+	}
+	if got := stderr.String(); got != "felis pin-images: --system takes login or lobby, not \"survival\"\n" {
+		t.Errorf("stderr = %q", got)
+	}
+}
+
+// An idle setting the panel saves while converge fills the default is kept.
+func TestConvergeIdleKeepsAConcurrentPanelEdit(t *testing.T) {
+	ctx := context.Background()
+	srv := &v1alpha1.MinecraftServer{}
+	srv.Name, srv.Namespace = "survival", "minecraft"
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Spec.Idle = v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}
+		if err := c.Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, srv)
+
+	if out := convergeUserServerIdle(ctx, cl, "minecraft"); len(out) != 0 {
+		t.Fatalf("outcomes = %+v, want none: the server has a setting by the time converge writes", out)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if want := (v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}); ms.Spec.Idle != want {
+		t.Errorf("idle = %+v, want the panel's %+v", ms.Spec.Idle, want)
+	}
+	if *writes != 1 {
+		t.Errorf("writes = %d, want 1 (the conflicted attempt; the retry sends nothing)", *writes)
+	}
+}

cmd/felis/context_max_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+)
+
+func TestContextMaxBytes(t *testing.T) {
+	cases := []struct {
+		name    string
+		reg     config.RegistryConfig
+		auth    config.AuthConfig
+		want    int64
+		wantErr bool
+	}{
+		{name: "direct install keeps the package default", want: 0},
+		// The panel uploads in parts under the edge's 100 MB body limit, so the
+		// Cloudflare edge keeps the full default.
+		{name: "the Cloudflare edge keeps the package default", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 0},
+		{name: "CF-Connecting-IP keeps the package default", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 0},
+		{name: "explicit value behind the edge", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
+		{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
+		{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
+		{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := contextMaxBytes(&config.Config{Registry: tc.reg, Auth: tc.auth})
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
+			}
+			if got != tc.want {
+				t.Fatalf("contextMaxBytes = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestUploadPartsDir(t *testing.T) {
+	if got := uploadPartsDir("/var/lib/felis/uploads"); got != "/var/lib/felis/uploads/.parts" {
+		t.Errorf("local store: parts dir = %q, want beside the contexts", got)
+	}
+	if got := uploadPartsDir("file:///srv/uploads"); got != "/srv/uploads/.parts" {
+		t.Errorf("file:// store: parts dir = %q, want /srv/uploads/.parts", got)
+	}
+	// This machine has no /var/lib/felis/uploads mount, so an s3:// store falls
+	// back to the temp dir.
+	if _, err := os.Stat("/var/lib/felis/uploads"); err == nil {
+		t.Skip("/var/lib/felis/uploads exists here")
+	}
+	if got := uploadPartsDir("s3://bucket/uploads"); got != filepath.Join(os.TempDir(), "felis-upload-parts") {
+		t.Errorf("s3 store without the uploads mount: parts dir = %q", got)
+	}
+}

cmd/felis/converge.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// cmdConverge is the explicit convergence pass over already-installed system
+// servers (#1), plus the idle-stop default for user servers that predate it. Provisioning is create-if-absent, so a field the desired spec
+// gained after an install (spec.rcon, spec.startup.healthHTTPPort, a derived env
+// key) never reaches the existing CR — and nothing says so. This command fills
+// exactly those zero-value fields; see convergeSystemServers for the full contract
+// and why it is a separate, operator-timed step rather than part of setup.
+//
+// It reads the same host config as setup (the control plane's felis.toml) and
+// talks to the cluster with the local kubeconfig, so it must run as root on the
+// control-plane host.
+func cmdConverge(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("converge", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis converge: refused — converging needs the cluster credentials, so it must run as root (try: sudo felis converge)")
+		return 1
+	}
+
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis converge: %v\n", err)
+		fmt.Fprintln(stderr, "If this host was never installed, run `sudo felis setup` first.")
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis converge: %v\n", err)
+		return 1
+	}
+
+	controlNS := platform.DefaultControlNamespace
+	outcomes := convergeSystemServers(context.Background(), cl, cfg.K8s.Namespace,
+		cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage,
+		platform.InternalAPIBaseURL(controlNS), cfg.Server.RootDomain,
+		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
+
+	outcomes = append(outcomes, convergeUserServerIdle(context.Background(), cl, cfg.K8s.Namespace)...)
+
+	fmt.Fprintln(stdout, "felis converge: filling fields an installed server predates (operator-set values are never overwritten):")
+	exit := 0
+	for _, o := range outcomes {
+		switch {
+		case o.err != nil:
+			fmt.Fprintf(stdout, "  - %s: ERROR %v\n", o.name, o.err)
+			exit = 1
+		case len(o.changes) > 0:
+			fmt.Fprintf(stdout, "  - %s: updated (%s)\n", o.name, strings.Join(o.changes, ", "))
+		default:
+			fmt.Fprintf(stdout, "  - %s: %s\n", o.name, o.skipped)
+		}
+	}
+	return exit
+}
+
+// convergeUserServerIdle gives every user server that predates the idle default
+// (spec.idle entirely unset) the default idle stop. A server whose idle stop was
+// turned off keeps a duration on its spec, so it is not "unset" and is left
+// alone; system servers never idle out and are skipped. Servers that already
+// carry a value produce no line, so a converged fleet prints nothing here.
+func convergeUserServerIdle(ctx context.Context, cl client.Client, namespace string) []systemServerOutcome {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+			continue
+		}
+		// Re-checked on the copy each attempt reads: an idle setting the panel saved
+		// meanwhile is the user's, and the default must not land over it.
+		changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
+			if ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+				return false
+			}
+			ms.Spec.Idle = v1alpha1.DefaultIdle()
+			return true
+		})
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
+			continue
+		}
+		if !changed {
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{fmt.Sprintf("spec.idle (stop after %ds empty)", v1alpha1.DefaultEmptySecondsBeforeStop)}})
+	}
+	return out
+}

cmd/felis/converge_test.go

@@ -0,0 +1,231 @@
+package main
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+// converge is the explicit pass over an installed system server whose CR predates
+// a field the desired spec has since gained (#1). It must fill exactly the
+// zero-valued whitelist fields and the derived env, and must not touch anything a
+// non-zero value already occupies — that is the operator's.
+func TestConvergeSystemServersFillsPredatedFields(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+
+	// An old install: the lobby CR was created before the desired spec began
+	// rendering spec.rcon, and the login CR before the HTTP readiness gate existed.
+	// One derived env key is absent entirely (as if it were added later), and one
+	// hand-added env var plus a non-whitelisted spec field must survive.
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatalf("build lobby: %v", err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{}
+	lobby.Spec.JavaMemory = "999Mi"
+
+	login, err := loginSystemServer("reg/limbo:1", "minecraft",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatalf("build login: %v", err)
+	}
+	login.Spec.Startup.HealthHTTPPort = 0
+	kept := login.Spec.Env
+	login.Spec.Env = nil
+	for _, e := range kept {
+		if e.Name != envPanelHostname {
+			login.Spec.Env = append(login.Spec.Env, e)
+		}
+	}
+	login.Spec.Env = append(login.Spec.Env, v1alpha1.EnvVar{Name: "OPERATOR_TUNING", Value: "keep-me"})
+
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(lobby, login).Build()
+	outcomes := convergeSystemServers(ctx, cl, "minecraft", "reg/limbo:1", "reg/lobby:1",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+
+	byName := map[string]systemServerOutcome{}
+	for _, o := range outcomes {
+		if o.err != nil {
+			t.Fatalf("%s: unexpected error: %v", o.name, o.err)
+		}
+		byName[o.name] = o
+	}
+	lobbyOut := byName[naming.SystemLobbyServer]
+	if len(lobbyOut.changes) != 1 || lobbyOut.changes[0] != "spec.rcon" {
+		t.Errorf("lobby changes = %v, want [spec.rcon] (only the zero-valued field)", lobbyOut.changes)
+	}
+	loginOut := byName[naming.SystemLoginServer]
+	if !slices.Contains(loginOut.changes, "spec.startup.healthHTTPPort") || !slices.Contains(loginOut.changes, "env "+envPanelHostname) {
+		t.Errorf("login changes = %v, want the health port plus the missing derived env key", loginOut.changes)
+	}
+
+	var gotLobby v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &gotLobby); err != nil {
+		t.Fatalf("get lobby: %v", err)
+	}
+	if !gotLobby.Spec.Rcon.Enabled ||
+		gotLobby.Spec.Rcon.SecretRef.Name != naming.RconSecretName(naming.SystemLobbyServer) ||
+		gotLobby.Spec.Rcon.SecretRef.Key != naming.RconSecretKey {
+		t.Errorf("lobby rcon = %+v, want the desired block with the %s secret",
+			gotLobby.Spec.Rcon, naming.RconSecretName(naming.SystemLobbyServer))
+	}
+	if gotLobby.Spec.JavaMemory != "999Mi" {
+		t.Errorf("lobby javaMemory = %q, want 999Mi — converge fills new fields, it does not rewrite the spec", gotLobby.Spec.JavaMemory)
+	}
+
+	var gotLogin v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &gotLogin); err != nil {
+		t.Fatalf("get login: %v", err)
+	}
+	if gotLogin.Spec.Startup.HealthHTTPPort != felisLimboHealthPort {
+		t.Errorf("login healthHTTPPort = %d, want %d", gotLogin.Spec.Startup.HealthHTTPPort, felisLimboHealthPort)
+	}
+	env := map[string]string{}
+	for _, e := range gotLogin.Spec.Env {
+		env[e.Name] = e.Value
+	}
+	if env[envPanelHostname] != "console.mc.example.net" {
+		t.Errorf("%s was not added back: %q", envPanelHostname, env[envPanelHostname])
+	}
+	if env["OPERATOR_TUNING"] != "keep-me" {
+		t.Error("a hand-added env var was dropped; converge only touches config-derived names")
+	}
+}
+
+// A field already holding a non-zero value belongs to the operator: converge must
+// report "already converged" and write nothing.
+func TestConvergeSystemServersLeavesNonZeroFieldsAlone(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatalf("build lobby: %v", err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{
+		Enabled:   true,
+		SecretRef: v1alpha1.SecretKeyRef{Name: "operator-rotated", Key: "password"},
+	}
+	login, err := loginSystemServer("reg/limbo:1", "minecraft",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatalf("build login: %v", err)
+	}
+
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(lobby, login).Build()
+	for _, o := range convergeSystemServers(ctx, cl, "minecraft", "reg/limbo:1", "reg/lobby:1",
+		"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
+		if o.err != nil {
+			t.Fatalf("%s: unexpected error: %v", o.name, o.err)
+		}
+		if len(o.changes) != 0 || o.skipped != "already converged" {
+			t.Errorf("%s outcome = %+v, want already converged with no writes", o.name, o)
+		}
+	}
+	var got v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &got); err != nil {
+		t.Fatalf("get lobby: %v", err)
+	}
+	if got.Spec.Rcon.SecretRef.Name != "operator-rotated" {
+		t.Errorf("lobby rcon secretRef = %q — converge overwrote a field the operator had already set",
+			got.Spec.Rcon.SecretRef.Name)
+	}
+}
+
+// Guards: an absent CR is reported (creation is setup's job), a foreign CR is
+// refused rather than adopted, and an unset image skips like the provisioner does.
+func TestConvergeSystemServersGuards(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	run := func(cl client.Client, loginImage, lobbyImage string) []systemServerOutcome {
+		return convergeSystemServers(ctx, cl, "minecraft", loginImage, lobbyImage,
+			"http://felis-api.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	}
+
+	t.Run("absent CRs are reported, not created", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
+		for _, o := range run(cl, "reg/limbo:1", "reg/lobby:1") {
+			if o.err != nil {
+				t.Fatalf("%s: %v", o.name, o.err)
+			}
+			if o.created || !strings.Contains(o.skipped, "not present") {
+				t.Errorf("%s outcome = %+v, want a not-present skip", o.name, o)
+			}
+		}
+	})
+
+	t.Run("foreign CR is refused", func(t *testing.T) {
+		foreign := &v1alpha1.MinecraftServer{}
+		foreign.Name = naming.SystemLoginServer
+		foreign.Namespace = "minecraft"
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(foreign).Build()
+		out := run(cl, "reg/limbo:1", "")
+		if len(out) != 2 {
+			t.Fatalf("outcomes = %d, want 2", len(out))
+		}
+		if out[0].err == nil || !strings.Contains(out[0].err.Error(), "not marked") {
+			t.Fatalf("login error = %v, want an unmarked-name refusal", out[0].err)
+		}
+	})
+
+	t.Run("unset image skips", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).Build()
+		out := run(cl, "", "reg/lobby:1")
+		if out[0].skipped != "image not configured" {
+			t.Errorf("login skipped = %q, want %q", out[0].skipped, "image not configured")
+		}
+	})
+}
+
+// TestConvergeUserServerIdle fills the idle default only where spec.idle was
+// never set: a server whose idle stop was turned off (duration kept), one with
+// its own duration, and a system server all stay as they are.
+func TestConvergeUserServerIdle(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	mk := func(name string, idle v1alpha1.IdleSpec, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Idle = idle
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		mk("legacy", v1alpha1.IdleSpec{}, ""),
+		mk("off", v1alpha1.IdleSpec{EmptySecondsBeforeStop: 600}, ""),
+		mk("custom", v1alpha1.IdleSpec{AutoStopEnabled: true, EmptySecondsBeforeStop: 1800}, ""),
+		mk(naming.SystemLobbyServer, v1alpha1.IdleSpec{}, naming.SystemLobbyServer),
+	).Build()
+
+	outcomes := convergeUserServerIdle(ctx, cl, "minecraft")
+	if len(outcomes) != 1 || outcomes[0].name != "legacy" || outcomes[0].err != nil {
+		t.Fatalf("outcomes = %+v, want exactly one fill for legacy", outcomes)
+	}
+	want := map[string]v1alpha1.IdleSpec{
+		"legacy":                 v1alpha1.DefaultIdle(),
+		"off":                    {EmptySecondsBeforeStop: 600},
+		"custom":                 {AutoStopEnabled: true, EmptySecondsBeforeStop: 1800},
+		naming.SystemLobbyServer: {},
+	}
+	for name, idle := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Idle != idle {
+			t.Errorf("%s idle = %+v, want %+v", name, ms.Spec.Idle, idle)
+		}
+	}
+	if again := convergeUserServerIdle(ctx, cl, "minecraft"); len(again) != 0 {
+		t.Fatalf("second pass = %+v, want nothing to do", again)
+	}
+}

cmd/felis/db.go

@@ -0,0 +1,420 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/retention"
+)
+
+const dbUsage = `usage:
+  felis db backup  [-config path] [-dir dir] [-label daily|manual|...] [-keep n] [-state-dir dir]
+                   [-no-servers] [-metrics-file path]
+  felis db restore [-config path] [-dir dir] [-yes] [-force] [-no-safety-backup] <bundle>
+  felis db verify  [-dir dir] <bundle>
+  felis db list    [-dir dir]
+  felis db check   [-dir dir] [-max-age 26h]
+  felis db audit-export [-config path] [-since date] [-until date] [-out file]
+`
+
+// defaultKeep is how many bundles of a label a backup leaves behind. Manual
+// bundles are the operator's own and are never pruned.
+var defaultKeep = map[string]int{
+	dbbackup.LabelDaily:      14,
+	dbbackup.LabelPreMigrate: 10,
+	dbbackup.LabelPreRestore: 5,
+}
+
+// cmdDB implements `felis db`: logical backups of the control-plane database
+// together with the host state a rebuild needs (internal/dbbackup). The verb
+// comes first for the same reason as `felis migrate up`.
+func cmdDB(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("db "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, dbUsage) }
+	dir := fs.String("dir", dbbackup.DefaultDir, "bundle directory")
+	switch verb {
+	case "backup":
+		return dbBackup(fs, dir, rest, stdout, stderr)
+	case "restore":
+		return dbRestore(fs, dir, rest, stdout, stderr)
+	case "verify":
+		return dbVerify(fs, dir, rest, stdout, stderr)
+	case "list":
+		return dbList(fs, dir, rest, stdout, stderr)
+	case "check":
+		return dbCheck(fs, dir, rest, stdout, stderr)
+	case "audit-export":
+		return dbAuditExport(fs, rest, stdout, stderr)
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, dbUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis db: unknown verb %q\n%s", verb, dbUsage)
+	return 2
+}
+
+// parseWithArg parses flags that may sit on either side of one positional
+// argument (`restore -yes x.tar` and `restore x.tar -yes` both work) and
+// returns that argument.
+func parseWithArg(fs *flag.FlagSet, args []string) (string, bool) {
+	if err := fs.Parse(args); err != nil {
+		return "", false
+	}
+	if fs.NArg() == 0 {
+		return "", true
+	}
+	arg := fs.Arg(0)
+	if err := fs.Parse(fs.Args()[1:]); err != nil {
+		return "", false
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprintf(fs.Output(), "felis db: unexpected argument %q\n", fs.Arg(0))
+		return "", false
+	}
+	return arg, true
+}
+
+func dbDatabaseURL(path string) (string, error) {
+	cfg, err := config.Load(path)
+	if err != nil {
+		return "", err
+	}
+	return cfg.Database.URL, nil
+}
+
+func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never")
+	keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, manual all)")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`)
+	noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle")
+	metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	if *keep < 0 {
+		*keep = defaultKeep[*label]
+	}
+	o := dbbackup.BackupOptions{
+		DatabaseURL: url, Dir: *dir, Label: *label, Keep: *keep,
+		StateDir: *stateDir, Version: resolvedVersion(), Log: stderr,
+		MetricsFile: *metrics, Record: true,
+	}
+	if !*noServers {
+		o.ExportServers = exportMinecraftServers
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	path, err := dbbackup.Backup(ctx, o)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+	return 0
+}
+
+// resolveBundle accepts a path, or a bare bundle name looked up in dir.
+func resolveBundle(dir, arg string) string {
+	if strings.ContainsRune(arg, os.PathSeparator) {
+		return arg
+	}
+	if _, err := os.Stat(arg); err == nil {
+		return arg
+	}
+	return filepath.Join(dir, arg)
+}
+
+func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	yes := fs.Bool("yes", false, "replace the database's contents (required)")
+	force := fs.Bool("force", false, "restore even while other clients are connected")
+	noSafety := fs.Bool("no-safety-backup", false, "skip the bundle of the current database taken first")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, "host state directory for the safety bundle")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	if !*yes {
+		fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d).\n",
+			filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion)
+		fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.")
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
+	defer cancel()
+	_, safety, err := dbbackup.Restore(ctx, dbbackup.RestoreOptions{
+		DatabaseURL: url, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
+		Safety: dbbackup.BackupOptions{Keep: defaultKeep[dbbackup.LabelPreRestore], StateDir: *stateDir,
+			Version: resolvedVersion(), ExportServers: exportMinecraftServers},
+		Log: stderr,
+	})
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		if errors.Is(err, dbbackup.ErrClientsConnected) {
+			fmt.Fprintln(stderr, "  kubectl -n felis scale deployment felis-api felis-operator --replicas=0")
+		}
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db restore: restored %s (schema %d)\n", filepath.Base(bundle), m.SchemaVersion)
+	if safety != "" {
+		fmt.Fprintf(stdout, "  the database as it was before is in %s\n", safety)
+	}
+	// Nothing migrates at startup, so a control plane newer than the bundle needs
+	// its migrations re-applied; rolling back to the release that wrote the bundle
+	// must skip that, or the rollback is undone.
+	fmt.Fprintf(stdout, "  next: felis migrate up -config %s (skip it when rolling back to felis %s, which wrote this bundle)\n", *cfgPath, orUnknown(m.FelisVersion))
+	fmt.Fprintln(stdout, "        kubectl -n felis scale deployment felis-api felis-operator --replicas=1")
+	return 0
+}
+
+func dbVerify(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db verify: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "%s: ok\n  taken   %s (%s)\n  felis   %s\n  schema  %d\n  %s\n",
+		filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion, orUnknown(m.PGDumpVersion))
+	for _, f := range m.Files {
+		if f.Link != "" {
+			fmt.Fprintf(stdout, "  %-40s -> %s\n", f.Name, f.Link)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %-40s %d bytes\n", f.Name, f.Size)
+	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  (no MinecraftServer objects: %s)\n", m.ServersError)
+	}
+	return 0
+}
+
+func orUnknown(s string) string {
+	if s == "" {
+		return "unknown"
+	}
+	return s
+}
+
+func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	all, err := dbbackup.List(*dir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db list: %v\n", err)
+		return 1
+	}
+	if len(all) == 0 {
+		fmt.Fprintf(stdout, "no database backups in %s\n", *dir)
+		return 0
+	}
+	now := time.Now()
+	for _, b := range all {
+		fmt.Fprintf(stdout, "%-50s %-12s %10s  %s ago\n", b.Name, b.Label, humanBytes(b.Size), dbbackup.Age(now.Sub(b.Created)))
+	}
+	return 0
+}
+
+// dbAuditExport writes audit rows to a file (or stdout) as JSON lines, so an
+// install can keep them past [audit] retention, after which felis-api deletes them.
+func dbAuditExport(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	sinceFlag := fs.String("since", "", "first day (or RFC 3339 instant) to export, inclusive; empty starts at the oldest row")
+	untilFlag := fs.String("until", "", "day (or RFC 3339 instant) to stop before, exclusive; empty runs to the newest row")
+	out := fs.String("out", "", "file to write (created 0600, never overwritten); empty writes to stdout")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	since, err := parseExportBound(*sinceFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -since: %v\n", err)
+		return 2
+	}
+	until, err := parseExportBound(*untilFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -until: %v\n", err)
+		return 2
+	}
+	if !since.IsZero() && !until.IsZero() && !until.After(since) {
+		fmt.Fprintf(stderr, "felis db audit-export: -until %s is not after -since %s\n", *untilFlag, *sinceFlag)
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+		return 1
+	}
+	w := stdout
+	var f *os.File
+	if *out != "" {
+		if f, err = os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600); err != nil {
+			fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+			return 1
+		}
+		w = f
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	n, err := exportAudit(ctx, url, since, until, w)
+	if f != nil {
+		if cerr := f.Close(); err == nil {
+			err = cerr
+		}
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v (%d rows written)\n", err, n)
+		return 1
+	}
+	fmt.Fprintf(stderr, "felis db audit-export: %d audit rows written\n", n)
+	return 0
+}
+
+func exportAudit(ctx context.Context, url string, since, until time.Time, w io.Writer) (int, error) {
+	drv, err := openStore(ctx, url, false)
+	if err != nil {
+		return 0, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	return retention.ExportAudit(ctx, drv.DB(), since, until, w)
+}
+
+// parseExportBound reads a -since/-until value: a day (midnight UTC) or an
+// RFC 3339 instant; empty is an open bound.
+func parseExportBound(v string) (time.Time, error) {
+	if v == "" {
+		return time.Time{}, nil
+	}
+	if t, err := time.Parse(time.DateOnly, v); err == nil {
+		return t, nil
+	}
+	if t, err := time.Parse(time.RFC3339, v); err == nil {
+		return t.UTC(), nil
+	}
+	return time.Time{}, fmt.Errorf("%q is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)", v)
+}
+
+func humanBytes(n int64) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%d B", n)
+	}
+	div, exp := int64(unit), 0
+	for m := n / unit; m >= unit; m /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
+}
+
+// dbCheck is the freshness probe: exit 1 when the newest bundle is missing or
+// older than -max-age, for a monitor or the break-glass console to act on.
+func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest bundle")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	b, err := dbbackup.Check(*dir, *maxAge, time.Now())
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db check: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db check: ok, newest backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
+	return 0
+}
+
+// exportMinecraftServers reads every MinecraftServer through the host's k3s
+// kubectl and strips what the API server owns, so the result can be fed back
+// with `kubectl apply -f` on a rebuilt cluster.
+func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning
+	// on stderr must not end up inside the JSON.
+	cmd := exec.CommandContext(ctx, "k3s", "kubectl", "get", "minecraftservers.felis.lolicon.best", "-A", "-o", "json")
+	cmd.Env = append(os.Environ(), "KUBECONFIG="+hostBootstrapKubeconfigPath)
+	var errBuf strings.Builder
+	cmd.Stderr = &errBuf
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("k3s kubectl get minecraftservers: %w: %s", err, strings.TrimSpace(errBuf.String()))
+	}
+	return cleanServerList(out)
+}
+
+// cleanServerList drops status and the server-assigned metadata from a
+// `kubectl get -o json` List.
+func cleanServerList(raw []byte) ([]byte, error) {
+	var list struct {
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(raw, &list); err != nil {
+		return nil, fmt.Errorf("parse MinecraftServer list: %w", err)
+	}
+	for _, it := range list.Items {
+		delete(it, "status")
+		if md, ok := it["metadata"].(map[string]any); ok {
+			for _, k := range []string{"resourceVersion", "uid", "creationTimestamp", "generation", "managedFields", "selfLink"} {
+				delete(md, k)
+			}
+		}
+	}
+	if list.Items == nil {
+		list.Items = []map[string]any{}
+	}
+	return json.MarshalIndent(map[string]any{"apiVersion": "v1", "kind": "List", "items": list.Items}, "", "  ")
+}

cmd/felis/db_test.go

@@ -0,0 +1,185 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/store"
+)
+
+func TestDBUsage(t *testing.T) {
+	for _, args := range [][]string{{"db"}, {"db", "frobnicate"}, {"db", "restore"}, {"db", "verify"}, {"db", "backup", "extra"}} {
+		var out, errBuf bytes.Buffer
+		if code := run(args, &out, &errBuf); code != 2 {
+			t.Errorf("%v: exit %d, want 2", args, code)
+		}
+		if !strings.Contains(errBuf.String(), "felis db restore") {
+			t.Errorf("%v: no usage on stderr: %q", args, errBuf.String())
+		}
+	}
+}
+
+func TestDBRestoreNeedsYes(t *testing.T) {
+	// A bundle that does not exist fails verification (1) before -yes matters;
+	// the -yes gate itself is exercised against a real bundle in internal/dbbackup
+	// and on the VM. Here: the refusal path never reaches the config or database.
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "restore", "-dir", t.TempDir(), "missing.tar"}, &out, &errBuf); code != 1 {
+		t.Fatalf("exit %d, stderr %q", code, errBuf.String())
+	}
+}
+
+func TestParseWithArg(t *testing.T) {
+	for _, args := range [][]string{{"-yes", "b.tar"}, {"b.tar", "-yes"}} {
+		fs := flag.NewFlagSet("t", flag.ContinueOnError)
+		fs.SetOutput(io.Discard)
+		yes := fs.Bool("yes", false, "")
+		arg, ok := parseWithArg(fs, args)
+		if !ok || arg != "b.tar" || !*yes {
+			t.Errorf("%v -> %q ok=%v yes=%v", args, arg, ok, *yes)
+		}
+	}
+	fs := flag.NewFlagSet("t", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	if _, ok := parseWithArg(fs, []string{"a.tar", "b.tar"}); ok {
+		t.Error("two positional arguments accepted")
+	}
+}
+
+func TestResolveBundle(t *testing.T) {
+	if got := resolveBundle("/var/lib/felis/db-backups", "felis-db-x.tar"); got != "/var/lib/felis/db-backups/felis-db-x.tar" {
+		t.Errorf("bare name -> %s", got)
+	}
+	if got := resolveBundle("/var/lib/felis/db-backups", "/root/copy.tar"); got != "/root/copy.tar" {
+		t.Errorf("path -> %s", got)
+	}
+}
+
+func TestCleanServerList(t *testing.T) {
+	raw := `{"apiVersion":"v1","kind":"List","metadata":{"resourceVersion":""},"items":[{
+		"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer",
+		"metadata":{"name":"survival","namespace":"minecraft","uid":"u","resourceVersion":"42","generation":3,
+			"creationTimestamp":"2026-09-01T00:00:00Z","managedFields":[{}],"labels":{"a":"b"}},
+		"spec":{"desiredState":"Running"},"status":{"phase":"Running"}}]}`
+	out, err := cleanServerList([]byte(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Kind  string           `json:"kind"`
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatal(err)
+	}
+	if got.Kind != "List" || len(got.Items) != 1 {
+		t.Fatalf("got %s", out)
+	}
+	it := got.Items[0]
+	if _, ok := it["status"]; ok {
+		t.Error("status kept")
+	}
+	md := it["metadata"].(map[string]any)
+	for _, k := range []string{"uid", "resourceVersion", "generation", "creationTimestamp", "managedFields"} {
+		if _, ok := md[k]; ok {
+			t.Errorf("metadata.%s kept", k)
+		}
+	}
+	if md["name"] != "survival" || md["namespace"] != "minecraft" || md["labels"] == nil {
+		t.Errorf("identity lost: %v", md)
+	}
+	if it["spec"].(map[string]any)["desiredState"] != "Running" {
+		t.Error("spec lost")
+	}
+
+	empty, err := cleanServerList([]byte(`{"items":null}`))
+	if err != nil || !strings.Contains(string(empty), `"items": []`) {
+		t.Errorf("empty list -> %s, %v", empty, err)
+	}
+	if _, err := cleanServerList([]byte("Warning: x\n{")); err == nil {
+		t.Error("garbage parsed")
+	}
+}
+
+func TestHasPending(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}, {Version: 3}}
+	if hasPending(map[int]struct{}{1: {}, 2: {}, 3: {}}, ms) {
+		t.Error("fully applied reported pending")
+	}
+	if !hasPending(map[int]struct{}{1: {}, 2: {}}, ms) {
+		t.Error("missing 3 not reported")
+	}
+}
+
+type appliedDriver struct {
+	store.Driver
+	done map[int]struct{}
+}
+
+func (d appliedDriver) EnsureVersionTable(context.Context) error { return nil }
+func (d appliedDriver) AppliedVersions(context.Context) (map[int]struct{}, error) {
+	return d.done, nil
+}
+
+func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	// An unusable URL makes an attempted backup observable as an error without
+	// any PostgreSQL tooling.
+	const badURL = "not-a-url"
+	for _, tc := range []struct {
+		name    string
+		done    map[int]struct{}
+		attempt bool
+	}{
+		{"fresh database", map[int]struct{}{}, false},
+		{"up to date", map[int]struct{}{1: {}, 2: {}}, false},
+		{"pending on a populated database", map[int]struct{}{1: {}}, true},
+	} {
+		path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, badURL, t.TempDir(), io.Discard)
+		if attempted := err != nil; attempted != tc.attempt {
+			t.Errorf("%s: attempted = %v (err %v), want %v", tc.name, attempted, err, tc.attempt)
+		}
+		if path != "" {
+			t.Errorf("%s: path = %q", tc.name, path)
+		}
+	}
+}
+
+// audit-export takes a day or an RFC 3339 instant for each bound, and refuses a
+// malformed or inverted window before it opens the config or the database.
+func TestAuditExportBounds(t *testing.T) {
+	for _, tc := range []struct{ in, want string }{
+		{"", "0001-01-01T00:00:00Z"},
+		{"2026-01-31", "2026-01-31T00:00:00Z"},
+		{"2026-01-31T12:30:00+08:00", "2026-01-31T04:30:00Z"},
+	} {
+		got, err := parseExportBound(tc.in)
+		if err != nil || got.Format(time.RFC3339) != tc.want {
+			t.Errorf("parseExportBound(%q) = %v, %v; want %s", tc.in, got, err, tc.want)
+		}
+	}
+	if _, err := parseExportBound("31/01/2026"); err == nil || err.Error() != `"31/01/2026" is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)` {
+		t.Errorf("parseExportBound(31/01/2026) err = %v", err)
+	}
+	for _, tc := range []struct {
+		args    []string
+		wantErr string
+	}{
+		{[]string{"db", "audit-export", "-since", "yesterday"}, `felis db audit-export: -since: "yesterday" is neither`},
+		{[]string{"db", "audit-export", "-until", "2026-13-01"}, `felis db audit-export: -until: "2026-13-01" is neither`},
+		{[]string{"db", "audit-export", "-since", "2026-02-01", "-until", "2026-02-01"}, "felis db audit-export: -until 2026-02-01 is not after -since 2026-02-01"},
+		{[]string{"db", "audit-export", "extra"}, "felis db audit-export [-config path]"},
+	} {
+		var out, errBuf bytes.Buffer
+		code := run(append(tc.args, "-config", "/nonexistent/felis.toml"), &out, &errBuf)
+		if code != 2 || !strings.Contains(errBuf.String(), tc.wantErr) {
+			t.Errorf("%v: exit %d, stderr %q; want 2 and %q", tc.args, code, errBuf.String(), tc.wantErr)
+		}
+	}
+}

cmd/felis/egressgate.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"time"
+)
+
+// Vars so tests can shrink them. A dial that neither connects nor is refused
+// within egressDialTimeout counts as blocked: a policy that drops packets looks
+// exactly like that.
+var (
+	egressDialTimeout  = 500 * time.Millisecond
+	egressPollInterval = 200 * time.Millisecond
+)
+
+// cmdEgressGate is the first initContainer of every build pod. The pod's
+// NetworkPolicy is programmed asynchronously after the pod starts (live on k3s:
+// a build-labelled pod reached the internet and the Kubernetes API for its first
+// ~0.7 s), so the gate dials a destination the policy denies until it stops
+// answering, and only then lets the pod's next container, eventually the
+// untrusted Dockerfile, start.
+//
+// The default probe is the Kubernetes API Service, which the kubelet names in
+// every pod's environment and the build policy never admits. A probe that still
+// answers after --wait means the policy is not enforced at all (a CNI without
+// NetworkPolicy support, or k3s run with --disable-network-policy), and the
+// build fails closed.
+func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	probe := fs.String("probe", "", "host:port the build NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
+	wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the build is refused")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *probe == "" {
+		host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
+		if host == "" || port == "" {
+			fmt.Fprintln(stderr, "felis egress-gate: no --probe and no KUBERNETES_SERVICE_HOST/PORT to default to")
+			return 2
+		}
+		*probe = net.JoinHostPort(host, port)
+	}
+
+	start := time.Now()
+	for {
+		conn, err := net.DialTimeout("tcp", *probe, egressDialTimeout)
+		if err != nil {
+			fmt.Fprintf(stdout, "felis egress-gate: %s is unreachable after %s (%v); the egress lock is in effect\n",
+				*probe, time.Since(start).Round(time.Millisecond), err)
+			return 0
+		}
+		_ = conn.Close()
+		if time.Since(start) >= *wait {
+			fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s: the build namespace's NetworkPolicy is not enforced "+
+				"(a CNI without NetworkPolicy support, or k3s started with --disable-network-policy); refusing to run the build\n",
+				*probe, *wait)
+			return 1
+		}
+		time.Sleep(egressPollInterval)
+	}
+}

cmd/felis/egressgate_test.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"bytes"
+	"net"
+	"strings"
+	"testing"
+	"time"
+)
+
+func shrinkEgressGate(t *testing.T) {
+	t.Helper()
+	dial, poll := egressDialTimeout, egressPollInterval
+	egressDialTimeout, egressPollInterval = 200*time.Millisecond, 10*time.Millisecond
+	t.Cleanup(func() { egressDialTimeout, egressPollInterval = dial, poll })
+}
+
+// The gate holds while the probe answers and lets the pod go on once the policy
+// lands, which the test plays by closing the listener.
+func TestEgressGateWaitsForTheLock(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	accepted := make(chan struct{}, 100)
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+			accepted <- struct{}{}
+		}
+	}()
+	go func() {
+		for i := 0; i < 3; i++ {
+			<-accepted
+		}
+		_ = ln.Close()
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "10s"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if !strings.Contains(out.String(), "egress lock is in effect") {
+		t.Errorf("stdout = %q", out.String())
+	}
+}
+
+// A probe that keeps answering means no policy is enforced: the build must not run.
+func TestEgressGateRefusesAnOpenNetwork(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+		}
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "100ms"}, &out, &errb); code != 1 {
+		t.Fatalf("exit %d, want 1", code)
+	}
+	if !strings.Contains(errb.String(), "not enforced") {
+		t.Errorf("stderr = %q", errb.String())
+	}
+}
+
+func TestEgressGateDefaultsToTheKubernetesService(t *testing.T) {
+	shrinkEgressGate(t)
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	t.Setenv("KUBERNETES_SERVICE_PORT", "")
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate(nil, &out, &errb); code != 2 {
+		t.Fatalf("exit %d without a probe, want 2", code)
+	}
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	host, port, _ := net.SplitHostPort(ln.Addr().String())
+	_ = ln.Close() // closed: the lock reads as in effect at once
+	t.Setenv("KUBERNETES_SERVICE_HOST", host)
+	t.Setenv("KUBERNETES_SERVICE_PORT", port)
+	out.Reset()
+	if code := cmdEgressGate(nil, &out, &errb); code != 0 || !strings.Contains(out.String(), ln.Addr().String()) {
+		t.Fatalf("exit %d, stdout %q", code, out.String())
+	}
+}

cmd/felis/fetchcontext.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// cmdFetchContext is the in-Pod entrypoint the build Job's context-fetch
+// initContainer runs. It reads the blob the platform stored for a submission
+// from the felis-api INTERNAL face (with a bounded retry — see
+// fetchContextWithRetry) and extracts it into the shared emptyDir the Kaniko
+// container then builds from.
+//
+// Why this exists: the build Pod runs in the build namespace, where it can neither
+// mount the control-plane uploads PVC (a PVC does not cross namespaces) nor hold
+// object-store credentials, so the API that WROTE the blob is the transport. The
+// route is service-token-gated; the token arrives through a namespace-local Secret
+// mounted only into this initContainer, never into Kaniko's — so the untrusted
+// Dockerfile's build steps have no credential to read (their containers share no
+// environment, no PID namespace, and Kaniko itself mounts the context read-only).
+//
+// The extraction is deliberately paranoid: the tarball is attacker-controlled
+// input, so absolute paths, ".." escapes, links, and special files are refused
+// rather than sanitized. Kaniko treats the extracted tree as hostile regardless
+// (spec §16), but the pod's own filesystem still must not be written outside the
+// context directory it was given.
+func cmdFetchContext(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("fetch-context", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	url := fs.String("url", "", "internal-face URL of the submission's build-context tarball")
+	out := fs.String("out", "/context", "directory to extract the build context into")
+	want := fs.String("sha256", "", "refuse the context unless the tarball's sha256 is this lowercase hex digest")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *want != "" && !build.IsSHA256Hex(*want) {
+		fmt.Fprintf(stderr, "felis fetch-context: --sha256 %q is not a lowercase hex sha256\n", *want)
+		return 2
+	}
+	if *url == "" {
+		fmt.Fprintln(stderr, "felis fetch-context: --url is required")
+		return 2
+	}
+	token := os.Getenv("FELIS_SERVICE_TOKEN")
+	if token == "" {
+		fmt.Fprintln(stderr, "felis fetch-context: FELIS_SERVICE_TOKEN is empty — the internal face rejects anonymous reads")
+		return 2
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	// Validate the URL once up front: a bad one is a usage error (2), not
+	// something to sit in the retry loop.
+	if _, err := http.NewRequest(http.MethodGet, *url, nil); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: bad --url: %v\n", err)
+		return 2
+	}
+	// No overall client timeout: a legitimate modpack context can be large and the
+	// Job's activeDeadlineSeconds is the real bound. The header timeout catches a
+	// wedged endpoint without capping a healthy download.
+	// Redirects are refused: the request carries the service token, and the
+	// internal face never redirects, so a 3xx is someone steering the token.
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: time.Minute},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	resp, err := fetchContextWithRetry(ctx, client, *url, token, stderr)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	defer resp.Body.Close()
+
+	h := sha256.New()
+	body := io.TeeReader(resp.Body, h)
+	if err := extractTarGz(body, *out); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	if *want == "" {
+		return 0
+	}
+	// The tar end marker comes before the gzip trailer and whatever follows it,
+	// so read to EOF: the digest must cover every byte the blob holds. The blob
+	// itself is size-capped at upload, which bounds this read.
+	if _, err := io.Copy(io.Discard, io.LimitReader(body, maxContextBytes)); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	if got := hex.EncodeToString(h.Sum(nil)); got != *want {
+		// The init container failing is what keeps Kaniko from ever starting on
+		// the extracted tree.
+		fmt.Fprintf(stderr, "felis fetch-context: the context's sha256 is %s, the approved digest is %s: it changed after approval; refusing to build\n", got, *want)
+		return 1
+	}
+	return 0
+}
+
+// fetchRetryInterval/fetchRetryWindow bound how long the fetch waits out a
+// control-plane blip before giving up. The api pod being replaced is a normal
+// event (rollout, eviction, a chaos drill), and without a retry one refused
+// dial turns it into a failed build: BackoffLimit=0 gives the Job no second
+// Pod, so the terminal verdict costs a manual re-approval — the live drill hit
+// exactly this (context-fetch exit 1 on `connect: connection refused` while
+// the api pod rolled; the new pod was serving 11 seconds later and the same
+// 198-byte blob). The window is tiny next to the Job's 30-minute
+// activeDeadline; a 4xx (missing blob, rejected token) still fails fast.
+//
+// Vars, not consts, so tests can shrink the window.
+var (
+	fetchRetryInterval = 3 * time.Second
+	fetchRetryWindow   = 45 * time.Second
+)
+
+// fetchContextWithRetry GETs the context tarball, retrying transport failures
+// and 5xx responses until fetchRetryWindow runs out. A 4xx is an answer, not a
+// blip — retrying it only delays the honest error.
+func fetchContextWithRetry(ctx context.Context, client *http.Client, url, token string, stderr io.Writer) (*http.Response, error) {
+	deadline := time.Now().Add(fetchRetryWindow)
+	for {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+		if err != nil {
+			return nil, fmt.Errorf("bad --url: %w", err)
+		}
+		req.Header.Set("Authorization", "Bearer "+token)
+
+		resp, err := client.Do(req)
+		if err == nil && resp.StatusCode == http.StatusOK {
+			return resp, nil
+		}
+		if err == nil {
+			status := resp.Status
+			_ = resp.Body.Close()
+			err = fmt.Errorf("GET returned %s", status)
+			if resp.StatusCode < 500 {
+				return nil, err
+			}
+		}
+		if ctx.Err() != nil {
+			return nil, fmt.Errorf("GET failed: %w", err)
+		}
+		if time.Now().After(deadline) {
+			return nil, fmt.Errorf("GET failed (retried for %s): %w", fetchRetryWindow, err)
+		}
+		fmt.Fprintf(stderr, "felis fetch-context: %v; retrying (the internal face may be restarting)\n", err)
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("GET failed: %w", err)
+		case <-time.After(fetchRetryInterval):
+		}
+	}
+}
+
+// maxContextBytes / maxContextEntries bound what one context may expand to. The
+// compressed upload is capped at 1 GiB, but gzip turns that into hundreds of GiB
+// or millions of empty files, and the emptyDir's 4 GiB sizeLimit is only
+// enforced by the kubelet's periodic sweep, after the disk has filled. The byte
+// cap matches that sizeLimit; the entry cap is far above any real modpack (a
+// large one is a few thousand files) and far below an inode exhaustion.
+//
+// Vars, not consts, so tests can shrink them.
+var (
+	maxContextBytes   int64 = 4 << 30
+	maxContextEntries       = 200_000
+)
+
+// extractTarGz streams a gzip'd tarball into root, creating directories as
+// needed. Every entry is vetted BEFORE anything is written: a path that is
+// absolute or escapes root (via ".."), a link (symlink or hardlink), or any
+// special file kind aborts the whole extraction. Refusing rather than skipping is
+// deliberate — a context that needs one of those constructs is not a context this
+// transport carries, and silently dropping entries would build from a corpus the
+// submitter did not upload. The whole extraction is also bounded by
+// maxContextBytes and maxContextEntries.
+func extractTarGz(r io.Reader, root string) error {
+	if err := os.MkdirAll(root, 0o755); err != nil {
+		return fmt.Errorf("create context dir: %w", err)
+	}
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return fmt.Errorf("context is not a valid gzip tarball: %w", err)
+	}
+	defer zr.Close()
+	tr := tar.NewReader(zr)
+	var written int64
+	entries := 0
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("read context tarball: %w", err)
+		}
+		if entries++; entries > maxContextEntries {
+			return fmt.Errorf("the build context has more than %d entries", maxContextEntries)
+		}
+		name := filepath.Clean(hdr.Name)
+		if name == "." {
+			continue
+		}
+		// The zip-slip guard: reject, never rewrite. filepath.Clean collapses any
+		// "a/../../b", so these two checks are sufficient once Clean has run.
+		if filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) {
+			return fmt.Errorf("context entry %q escapes the context directory", hdr.Name)
+		}
+		target := filepath.Join(root, name)
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0o755); err != nil {
+				return fmt.Errorf("create %q: %w", name, err)
+			}
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+				return fmt.Errorf("create parent of %q: %w", name, err)
+			}
+			mode := os.FileMode(0o644)
+			if hdr.FileInfo().Mode()&0o111 != 0 {
+				mode = 0o755 // preserve executability (entrypoint scripts), nothing else
+			}
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
+			if err != nil {
+				return fmt.Errorf("create %q: %w", name, err)
+			}
+			n, err := io.Copy(f, io.LimitReader(tr, maxContextBytes-written+1))
+			written += n
+			if err != nil {
+				_ = f.Close()
+				return fmt.Errorf("write %q: %w", name, err)
+			}
+			if written > maxContextBytes {
+				_ = f.Close()
+				return fmt.Errorf("the build context expands past %d bytes", maxContextBytes)
+			}
+			if err := f.Close(); err != nil {
+				return fmt.Errorf("close %q: %w", name, err)
+			}
+		default:
+			return fmt.Errorf("context entry %q has unsupported type %q (links and special files are refused)", hdr.Name, string(hdr.Typeflag))
+		}
+	}
+}

cmd/felis/fetchcontext_test.go

@@ -0,0 +1,404 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+type tarEntry struct {
+	name     string
+	body     string
+	mode     int64
+	typ      byte
+	linkname string
+}
+
+// tgzBody builds an in-memory .tar.gz from entries, preserving each entry's type
+// and mode so the tests can exercise the guards with exactly the bytes an
+// attacker could upload.
+func tgzBody(t *testing.T, entries ...tarEntry) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zw)
+	for _, e := range entries {
+		typ := e.typ
+		if typ == 0 {
+			typ = tar.TypeReg
+		}
+		mode := e.mode
+		if mode == 0 {
+			mode = 0o644
+		}
+		hdr := &tar.Header{Name: e.name, Typeflag: typ, Mode: mode, Size: int64(len(e.body))}
+		if typ == tar.TypeSymlink {
+			hdr.Linkname = e.linkname
+			hdr.Size = 0
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatalf("write header %q: %v", e.name, err)
+		}
+		if hdr.Size > 0 {
+			if _, err := tw.Write([]byte(e.body)); err != nil {
+				t.Fatalf("write body %q: %v", e.name, err)
+			}
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatalf("close tar: %v", err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatalf("close gzip: %v", err)
+	}
+	return buf.Bytes()
+}
+
+// A normal context extracts with its tree intact, and the executable bit that
+// modpack entrypoints rely on survives.
+func TestExtractTarGzRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	body := tgzBody(t,
+		tarEntry{name: "Dockerfile", body: "FROM scratch\n"},
+		tarEntry{name: "mods/example.jar", body: "jar-bytes"},
+		tarEntry{name: "start.sh", body: "#!/bin/sh\n", mode: 0o755},
+		tarEntry{name: "mods/", typ: tar.TypeDir, mode: 0o755},
+	)
+	if err := extractTarGz(bytes.NewReader(body), dir); err != nil {
+		t.Fatalf("extract: %v", err)
+	}
+	for name, want := range map[string]string{
+		"Dockerfile":       "FROM scratch\n",
+		"mods/example.jar": "jar-bytes",
+	} {
+		got, err := os.ReadFile(filepath.Join(dir, name))
+		if err != nil || string(got) != want {
+			t.Fatalf("%s = (%q, %v), want %q", name, got, err, want)
+		}
+	}
+	fi, err := os.Stat(filepath.Join(dir, "start.sh"))
+	if err != nil || fi.Mode()&0o111 == 0 {
+		t.Fatalf("entrypoint script lost its exec bit: %v (%v)", fi, err)
+	}
+}
+
+// The guards: "..", absolute paths, symlinks, and special files are refused whole
+// — nothing escapes, and nothing is silently skipped.
+func TestExtractTarGzRefusesEscapes(t *testing.T) {
+	cases := []struct {
+		name    string
+		entries []tarEntry
+	}{
+		{"dotdot", []tarEntry{{name: "../outside", body: "x"}}},
+		{"nested dotdot", []tarEntry{{name: "a/../../outside", body: "x"}}},
+		{"absolute", []tarEntry{{name: "/etc/outside", body: "x"}}},
+		{"symlink", []tarEntry{{name: "link", typ: tar.TypeSymlink, linkname: "/etc"}}},
+		{"hardlink", []tarEntry{{name: "hard", typ: tar.TypeLink, linkname: "somewhere"}}},
+		{"device", []tarEntry{{name: "dev", typ: tar.TypeChar}}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			if err := extractTarGz(bytes.NewReader(tgzBody(t, tc.entries...)), dir); err == nil {
+				t.Fatal("extract accepted a hostile entry, want an error")
+			}
+			// Nothing may have been written outside the target (or at all).
+			entries, _ := os.ReadDir(dir)
+			if len(entries) != 0 {
+				t.Fatalf("hostile archive left %d entries behind", len(entries))
+			}
+		})
+	}
+}
+
+// A context that expands past the byte or entry cap is refused, however small
+// it was compressed: gzip bombs and inode floods stop at the cap.
+func TestExtractTarGzCapsExpansion(t *testing.T) {
+	bytesCap, entriesCap := maxContextBytes, maxContextEntries
+	t.Cleanup(func() { maxContextBytes, maxContextEntries = bytesCap, entriesCap })
+	maxContextBytes, maxContextEntries = 1000, 5
+
+	fits := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 400)})
+	if err := extractTarGz(bytes.NewReader(fits), t.TempDir()); err != nil {
+		t.Fatalf("a context exactly at the byte cap: %v", err)
+	}
+	big := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 401)})
+	if err := extractTarGz(bytes.NewReader(big), t.TempDir()); err == nil || !strings.Contains(err.Error(), "expands past") {
+		t.Fatalf("one byte over the cap: err = %v", err)
+	}
+	var many []tarEntry
+	for i := 0; i < 6; i++ {
+		many = append(many, tarEntry{name: "d" + string(rune('0'+i)) + "/", typ: tar.TypeDir})
+	}
+	if err := extractTarGz(bytes.NewReader(tgzBody(t, many...)), t.TempDir()); err == nil || !strings.Contains(err.Error(), "entries") {
+		t.Fatalf("six entries over a cap of five: err = %v", err)
+	}
+}
+
+// The command end to end: it dials the URL with the bearer token from the
+// environment, and refuses to run without it (the internal face would 401
+// anyway; failing at parse time is the honest earlier error).
+func TestCmdFetchContextFetchAndExtract(t *testing.T) {
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer test-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		w.Header().Set("Content-Type", "application/gzip")
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + dir}, io.Discard, io.Discard); code != 0 {
+		t.Fatalf("cmdFetchContext exit = %d, want 0", code)
+	}
+	if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
+		t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
+	}
+
+	// No token: refuse before dialing.
+	t.Setenv("FELIS_SERVICE_TOKEN", "")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 2 {
+		t.Fatalf("missing token exit = %d, want 2 (stderr %q)", code, stderr.String())
+	}
+
+	// A non-200 answer (e.g. the route's 404 for a never-uploaded context) fails.
+	srv404 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv404.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	if code := cmdFetchContext([]string{"--url=" + srv404.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, io.Discard); code != 1 {
+		t.Fatalf("404 exit = %d, want 1", code)
+	}
+}
+
+// With --sha256 the fetch refuses any bytes but the approved ones, including
+// a tarball that extracts cleanly: that is exactly the context an uploader
+// swapped in after the review.
+func TestCmdFetchContextChecksDigest(t *testing.T) {
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	sum := sha256.Sum256(body)
+	good := hex.EncodeToString(sum[:])
+	args := func(digest string) []string {
+		return []string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir(), "--sha256=" + digest}
+	}
+
+	if code := cmdFetchContext(args(good), io.Discard, io.Discard); code != 0 {
+		t.Fatalf("matching digest exit = %d, want 0", code)
+	}
+	var stderr bytes.Buffer
+	other := strings.Repeat("0", 64)
+	if code := cmdFetchContext(args(other), io.Discard, &stderr); code != 1 || !strings.Contains(stderr.String(), "changed after approval") {
+		t.Fatalf("mismatched digest exit = %d, stderr %q; want 1 naming the change", code, stderr.String())
+	}
+	stderr.Reset()
+	if code := cmdFetchContext(args("ABC"), io.Discard, &stderr); code != 2 {
+		t.Fatalf("malformed digest exit = %d, want 2 (stderr %q)", code, stderr.String())
+	}
+
+	// Bytes after the tar end marker still count: appending to an approved blob
+	// must change what the fetch accepts.
+	padded := append(append([]byte{}, body...), "trailing"...)
+	srvPadded := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(padded)
+	}))
+	defer srvPadded.Close()
+	if code := cmdFetchContext([]string{"--url=" + srvPadded.URL + "/c", "--out=" + t.TempDir(), "--sha256=" + good}, io.Discard, io.Discard); code != 1 {
+		t.Fatalf("padded blob exit = %d, want 1", code)
+	}
+}
+
+// The request carries the service token, so a redirect is a failure: the token
+// never follows it to another host (build-supply-chain-13).
+func TestCmdFetchContextRefusesRedirects(t *testing.T) {
+	var leaked bool
+	elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		leaked = true
+	}))
+	defer elsewhere.Close()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, elsewhere.URL+"/steal", http.StatusFound)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/c", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("redirect exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if leaked {
+		t.Fatal("the fetch followed the redirect")
+	}
+}
+
+// A body that is not a gzip tarball must fail the extraction rather than produce
+// an empty (or partial) context Kaniko would then try to build.
+func TestExtractTarGzRejectsNonGzip(t *testing.T) {
+	dir := t.TempDir()
+	err := extractTarGz(strings.NewReader("not a tarball"), dir)
+	if err == nil || !strings.Contains(err.Error(), "gzip") {
+		t.Fatalf("err = %v, want a gzip complaint", err)
+	}
+}
+
+// shrinkFetchWindow swaps the retry knobs for a faster test and restores them
+// afterwards, so no test leaks a tiny window into another.
+func shrinkFetchWindow(t *testing.T, interval, window time.Duration) {
+	t.Helper()
+	oldInterval, oldWindow := fetchRetryInterval, fetchRetryWindow
+	fetchRetryInterval, fetchRetryWindow = interval, window
+	t.Cleanup(func() { fetchRetryInterval, fetchRetryWindow = oldInterval, oldWindow })
+}
+
+// A control-plane blip mid-fetch is survived: a 5xx on the first attempt is
+// retried and the second attempt's tarball extracts. This walks back the live
+// drill's failure, where the api pod rolled mid-fetch and the single attempt
+// died, failing the build Job.
+func TestFetchContextRetriesThroughBlip(t *testing.T) {
+	shrinkFetchWindow(t, 10*time.Millisecond, time.Second)
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			w.WriteHeader(http.StatusBadGateway) // the port is up, the API is not
+			return
+		}
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + dir}, io.Discard, &stderr); code != 0 {
+		t.Fatalf("exit = %d, want 0 (stderr %q)", code, stderr.String())
+	}
+	if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
+		t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
+	}
+	if !strings.Contains(stderr.String(), "retrying") {
+		t.Fatalf("stderr %q does not mention the retry", stderr.String())
+	}
+}
+
+// The live drill's exact shape: the dial itself is refused (the api pod is
+// gone and no endpoint answers). A refused dial is retried like any other
+// transport failure, and once the face is back the fetch completes.
+func TestFetchContextRetriesRefusedDial(t *testing.T) {
+	shrinkFetchWindow(t, 10*time.Millisecond, 5*time.Second)
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+
+	// Borrow a listen address, then close it: the first attempts dial into a
+	// refused connection, exactly like a restarting control plane.
+	probe := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+	addr := strings.TrimPrefix(probe.URL, "http://")
+	probe.Close()
+
+	dir := t.TempDir()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	// Start the fetch; while the retry loop burns refused dials, bring the same
+	// address back.
+	result := make(chan int, 1)
+	go func() {
+		result <- cmdFetchContext([]string{"--url=http://" + addr + "/sub-1/context", "--out=" + dir}, io.Discard, &stderr)
+	}()
+	time.Sleep(100 * time.Millisecond) // let a handful of dials be refused
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		t.Fatalf("rebind %s: %v", addr, err)
+	}
+	back := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer test-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		_, _ = w.Write(body)
+	})}
+	defer back.Close()
+	go func() { _ = back.Serve(ln) }()
+
+	code := <-result
+	if code != 0 {
+		t.Fatalf("exit = %d, want 0 (stderr %q)", code, stderr.String())
+	}
+	if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
+		t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
+	}
+	if !strings.Contains(stderr.String(), "retrying") {
+		t.Fatalf("stderr %q does not mention the retry", stderr.String())
+	}
+}
+
+// A 4xx is an answer, not a blip: a missing/never-uploaded context fails
+// immediately — no retry loop burns the build's deadline on a terminal error.
+func TestFetchContextDoesNotRetry4xx(t *testing.T) {
+	shrinkFetchWindow(t, 5*time.Millisecond, 200*time.Millisecond)
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if got := atomic.LoadInt32(&calls); got != 1 {
+		t.Fatalf("server saw %d attempts, want exactly 1", got)
+	}
+	if strings.Contains(stderr.String(), "retrying") {
+		t.Fatalf("stderr %q mentions a retry for a terminal 4xx", stderr.String())
+	}
+}
+
+// The retry is bounded: an internal face that stays down does not hang the
+// build pod; the window runs out and the fetch reports the exhausted retries.
+func TestFetchContextGivesUpAfterWindow(t *testing.T) {
+	shrinkFetchWindow(t, 5*time.Millisecond, 60*time.Millisecond)
+	var calls int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
+	defer srv.Close() // the face is up but never healthy: 503 forever
+
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	start := time.Now()
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if elapsed := time.Since(start); elapsed > 5*time.Second {
+		t.Fatalf("gave up after %v; the window is supposed to bound it", elapsed)
+	}
+	if got := atomic.LoadInt32(&calls); got < 2 {
+		t.Fatalf("server saw %d attempts, want at least one retry", got)
+	}
+	if !strings.Contains(stderr.String(), "retried for") {
+		t.Fatalf("stderr %q does not report the exhausted retry window", stderr.String())
+	}
+}

cmd/felis/files.go

@@ -19,7 +19,7 @@ import (
 // Like cmdRestore it deliberately holds NO database credentials and never calls
 // config.Load: felis-api made the authorization decision (the caller owns this
 // server, and the server is stopped so the RWO world volume is free); this process
-// is the unprivileged hands that touch bytes. Its entire input is the three flags
+// is the unprivileged hands that touch bytes. Its entire input is the flags
 // below plus, for a write, one environment variable. Every isolation guarantee
 // lives in the Pod spec (internal/fileedit/jobspec.go), and the path-containment
 // guarantee lives in fileedit.Execute, which resolves the path through os.Root and
@@ -37,6 +37,7 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	op := fs.String("op", "", "operation: list, read, or write")
 	path := fs.String("path", "", "path to operate on, relative to the world root (empty = the root itself)")
 	worldsRoot := fs.String("worlds-root", "/data", "mount path of the world PVC; every path resolves under it")
+	expect := fs.String("expect-sha256", "", "write only: refuse unless the file's current SHA-256 (hex) is this")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -67,7 +68,7 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 		content = decoded
 	}
 
-	res, err := fileedit.Execute(*worldsRoot, *op, *path, content)
+	res, err := fileedit.Execute(*worldsRoot, *op, *path, content, *expect)
 	if err != nil {
 		// The operation could not be attempted — infrastructure, not caller fault.
 		fmt.Fprintf(stderr, "felis files: %v\n", err)

cmd/felis/initforwarding.go

@@ -20,18 +20,14 @@ const forwardingSecretEnv = "FELIS_FORWARDING_SECRET"
 // config/ and server.properties live under it.
 const defaultForwardingDataDir = "/data"
 
-// fwd*Mode make the written config readable AND rewritable by the main server
-// container, whose UID we do not control (an arbitrary user image). The
-// initContainer runs as root (see buildStatefulSet) so it can write into a data
-// volume of unknown ownership; 0666/0777 then let a non-root Paper rewrite the
-// same files on boot.
-//
-// This relies on the initContainer running as root to write into a volume of
-// unknown ownership; that is how the operator schedules it. If that ever changes,
-// give the server pod an fsGroup so the shared volume is group-writable instead.
+// fwd*Mode are the modes the written config lands with. The initContainer runs as
+// the same uid as the server container (naming.GameUID, pinned by the operator in
+// the pod securityContext) after the prepare-data initContainer has handed the
+// whole volume to that uid, so owner read/write is all the server needs to rewrite
+// these files on boot and nothing else on the node gets write access to them.
 const (
-	fwdFileMode os.FileMode = 0o666
-	fwdDirMode  os.FileMode = 0o777
+	fwdFileMode os.FileMode = 0o644
+	fwdDirMode  os.FileMode = 0o755
 )
 
 // cmdInitForwarding is the felis-image initContainer entrypoint that makes an
@@ -96,9 +92,8 @@ func writePaperGlobal(dataDir, secret string) error {
 	if err := os.MkdirAll(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("create %s: %w", dir, err)
 	}
-	// MkdirAll honours the process umask (root's is typically 022 → 0755); chmod
-	// does not, and a non-root main container must be able to place/replace the
-	// file in this directory on boot.
+	// MkdirAll honours the process umask; chmod does not, so a directory an older
+	// release left at 0777 is brought back to fwdDirMode here.
 	if err := os.Chmod(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("chmod %s: %w", dir, err)
 	}
@@ -188,8 +183,8 @@ func upsertProperty(content []byte, key, value string) []byte {
 }
 
 // writeFileMode writes data then forces the mode, since WriteFile honours the
-// umask (root's is typically 022 → 0644) but a non-root main container must be
-// able to rewrite these files on boot.
+// umask and leaves an existing file's mode alone: a file an older release wrote
+// world-writable (0666) is tightened back to fwdFileMode on the next boot.
 func writeFileMode(path string, data []byte) error {
 	if err := os.WriteFile(path, data, fwdFileMode); err != nil {
 		return fmt.Errorf("write %s: %w", path, err)

cmd/felis/initforwarding_test.go

@@ -164,8 +164,9 @@ func TestUpsertPropertyAppends(t *testing.T) {
 	}
 }
 
-// The written files must be group/world writable so a non-root main container can
-// rewrite them. chmod semantics are POSIX-only, so this asserts on non-Windows.
+// The written files land at fwdFileMode: owner-writable for the game uid the init
+// shares with the server container, and no longer world-writable. chmod semantics
+// are POSIX-only, so this asserts on non-Windows.
 func TestWriteForwardingFileModes(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("POSIX file modes not represented on Windows")

cmd/felis/initvolume.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"syscall"
+
+	"felis.lolicon.best/internal/naming"
+)
+
+// cmdInitVolume is the felis-image `prepare-data` initContainer entrypoint: it
+// hands every entry of a server's world volume to the game uid/gid before the
+// server container starts. The operator runs the server itself as naming.GameUID,
+// so a world written by an earlier release (whose server ran as root), a restore
+// Job (which extracts as root), or a storage provisioner that creates the volume
+// root-owned would otherwise leave files the server cannot write — a world that
+// boots and then fails every save.
+//
+// fsGroup covers only part of this: kubelet applies it to volume types that
+// support ownership management, and a k3s local-path PV is a hostPath underneath,
+// which it skips. A walk from inside the pod works for every volume type.
+//
+// Only mismatched entries are touched, so a volume already owned by the game uid
+// costs one lstat per entry and no writes. The walk runs inside an os.Root at the
+// data dir and uses lchown, so a symlink a plugin planted is re-owned as a link
+// and never followed out of the volume.
+//
+// A single entry that cannot be chowned is reported and skipped: failing the pod
+// over one odd file would keep the whole server down, while the server itself
+// reports the one file it cannot write. Only an unreadable data dir fails.
+func cmdInitVolume(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("init-volume", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	dataDir := fs.String("data", defaultForwardingDataDir, "world volume mount to hand to the game uid")
+	uid := fs.Int64("uid", naming.GameUID, "owner uid for every entry")
+	gid := fs.Int64("gid", naming.GameGID, "owner gid for every entry")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	root, err := os.OpenRoot(*dataDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: open %s: %v\n", *dataDir, err)
+		return 1
+	}
+	defer root.Close()
+	res, err := chownTree(root, int(*uid), int(*gid), root.Lchown)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: %v\n", err)
+		return 1
+	}
+	for _, f := range res.failures {
+		fmt.Fprintf(stderr, "felis init-volume: %s\n", f)
+	}
+	fmt.Fprintf(stdout, "felis init-volume: %d entries checked, %d handed to %d:%d, %d failed\n",
+		res.checked, res.changed, *uid, *gid, len(res.failures))
+	return 0
+}
+
+// chownResult tallies one walk; failures is capped so a volume of thousands of
+// unownable files cannot flood the pod log.
+type chownResult struct {
+	checked  int
+	changed  int
+	failures []string
+}
+
+const maxReportedChownFailures = 20
+
+// chownTree walks root and calls chown on every entry (the root dir included)
+// whose owner is not uid:gid. It returns an error only when the root itself
+// cannot be read; per-entry failures are collected in the result.
+func chownTree(root *os.Root, uid, gid int, chown func(name string, uid, gid int) error) (chownResult, error) {
+	var res chownResult
+	fail := func(name string, err error) {
+		if len(res.failures) < maxReportedChownFailures {
+			res.failures = append(res.failures, fmt.Sprintf("%s: %v", name, err))
+		} else if len(res.failures) == maxReportedChownFailures {
+			res.failures = append(res.failures, "further failures not listed")
+		}
+	}
+	err := fs.WalkDir(root.FS(), ".", func(name string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			if name == "." {
+				return walkErr
+			}
+			fail(name, walkErr)
+			// A directory that cannot be listed is skipped as a whole; a file
+			// error has nothing below it to skip.
+			if d != nil && d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		res.checked++
+		info, err := d.Info()
+		if err != nil {
+			fail(name, err)
+			return nil
+		}
+		if st, ok := info.Sys().(*syscall.Stat_t); ok && int(st.Uid) == uid && int(st.Gid) == gid {
+			return nil
+		}
+		if err := chown(name, uid, gid); err != nil {
+			if !errors.Is(err, fs.ErrNotExist) { // gone mid-walk: nothing left to own
+				fail(name, err)
+			}
+			return nil
+		}
+		res.changed++
+		return nil
+	})
+	return res, err
+}

cmd/felis/initvolume_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strconv"
+	"testing"
+)
+
+// openTree builds a small world under a temp dir: nested dirs, a file, and a
+// symlink pointing out of the volume that the walk must not follow.
+func openTree(t *testing.T) *os.Root {
+	t.Helper()
+	dir := t.TempDir()
+	for _, d := range []string{"world/region", "plugins"} {
+		if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, f := range []string{"level.dat", "world/region/r.0.0.mca"} {
+		if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	outside := t.TempDir()
+	if err := os.Symlink(outside, filepath.Join(dir, "plugins", "escape")); err != nil {
+		t.Fatal(err)
+	}
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { root.Close() })
+	return root
+}
+
+// Every entry owned by someone else is handed over, the root dir included, and a
+// symlink is re-owned as a link rather than walked into.
+func TestChownTreeHandsOverMismatchedEntries(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	var got []string
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		got = append(got, name)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	want := []string{".", "level.dat", "plugins", "plugins/escape", "world", "world/region", "world/region/r.0.0.mca"}
+	slices.Sort(got)
+	if !slices.Equal(got, want) {
+		t.Errorf("chowned %v, want %v", got, want)
+	}
+	if res.changed != len(want) || res.checked != len(want) || len(res.failures) != 0 {
+		t.Errorf("result = %+v, want %d checked and changed, no failures", res, len(want))
+	}
+}
+
+// A volume already owned by the game uid costs no chown at all: this is the steady
+// state every restart after the first one hits.
+func TestChownTreeSkipsMatchingOwner(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	calls := 0
+	res, err := chownTree(root, os.Getuid(), os.Getgid(), func(string, int, int) error {
+		calls++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if calls != 0 || res.changed != 0 {
+		t.Errorf("chown called %d times on an already-owned tree (result %+v)", calls, res)
+	}
+}
+
+// One entry that refuses the chown is reported and the walk carries on: a single
+// odd file must not keep the whole server from starting.
+func TestChownTreeContinuesPastFailures(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		if name == "level.dat" {
+			return os.ErrPermission
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if len(res.failures) != 1 || res.changed != 6 {
+		t.Errorf("result = %+v, want 1 failure and 6 changed", res)
+	}
+}
+
+// Against a real directory the owner already matches, so the command succeeds
+// without needing CAP_CHOWN — the path every test runner (non-root) can take.
+func TestCmdInitVolumeOwnedTree(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "server.properties"), []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	var out, errb bytes.Buffer
+	code := cmdInitVolume([]string{"--data", dir, "--uid", strconv.Itoa(os.Getuid()), "--gid", strconv.Itoa(os.Getgid())}, &out, &errb)
+	if code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, errb.String())
+	}
+	if code := cmdInitVolume([]string{"--data", filepath.Join(dir, "missing")}, &out, &errb); code != 1 {
+		t.Errorf("missing data dir exit = %d, want 1", code)
+	}
+}

cmd/felis/manifests.go

@@ -26,8 +26,10 @@ func (m *multiFlag) Set(v string) error {
 // felis-reaper identity only when the retention reaper is enabled, gated with
 // its CronJob), the weak build/restore Job SAs, the build/minecraft
 // NetworkPolicies, and the running control-plane workloads (felis-api/operator
-// Deployments + the in-cluster registry Deployment/Service/PVC) — as a single
-// multi-document YAML stream on stdout, ready for `kubectl apply -f -`.
+// Deployments + the in-cluster registry Deployment/Service/PVC + the
+// world-archive PVC that backs backup/restore, unless --backup-pvc is emptied)
+// — as a single multi-document YAML stream on stdout, ready for
+// `kubectl apply -f -`.
 //
 // It is a pure renderer: it never contacts a cluster and holds no credentials.
 // --velocity-cidr records the proxy host addresses allowed by the game NetworkPolicy.
@@ -43,14 +45,22 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	registryPort := fs.Int("registry-port", 5000, "port the in-cluster registry listens on")
 	panelNodePort := fs.Int("panel-node-port", int(platform.DefaultPanelNodePort), "NodePort that exposes the built-in HTTPS panel/API origin")
 	felisImage := fs.String("felis-image", "", "container image the felis-api/operator Deployments run, also passed through as FELIS_IMAGE (REQUIRED)")
-	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry:2)")
-	backupPVC := fs.String("backup-pvc", "", "name of the backup PVC advertised to the restore executor via FELIS_BACKUP_PVC (default none = restore endpoint returns 503)")
-	worldsHostPath := fs.String("worlds-host-path", "", "node directory under which each world PVC is visible as <path>/<pvc>; enables the reaper CronJob (requires --backup-pvc and --archive-local-path)")
-	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
+	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry 2.8.3, pinned by digest)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", "name of the world-archive PVC this bundle renders in the Minecraft namespace and advertises to the backup/restore executors via FELIS_BACKUP_PVC (default: felis-backups; pass an empty value to render none, leaving backup/restore answering 503)")
+	worldsHostPath := fs.String("worlds-host-path", "", "node directory the reaper reads worlds from: each world PVC resolves as <path>/<pvc>, or as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (k3s storage root: /var/lib/rancher/k3s/storage); enables the reaper CronJob (requires --archive-local-path and a non-empty --backup-pvc)")
+	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path. With the backup PVC alone it renders the retention-only CronJob, which deletes backups past their expiry and never touches a world")
+	registryStorage := fs.String("registry-storage", "", "capacity the registry PVC requests (default 10Gi; k3s local-path does not enforce it)")
+	uploadsStorage := fs.String("uploads-storage", "", "capacity the uploads PVC requests (default 5Gi; k3s local-path does not enforce it)")
+	backupStorage := fs.String("backup-storage", "", "capacity the world-archive PVC requests (default 10Gi; k3s local-path does not enforce it)")
+	reaperNode := fs.String("reaper-node", "", "node that holds --worlds-host-path: pins the reaper CronJob's pod there via nodeSelector kubernetes.io/hostname (multi-node clusters need this, or the reaper may schedule where the hostPath is empty)")
 	var velocityCIDRs multiFlag
 	fs.Var(&velocityCIDRs, "velocity-cidr", "CIDR of a Velocity proxy host allowed to reach game port 25565 (repeatable, REQUIRED)")
 	var packageCIDRs multiFlag
 	fs.Var(&packageCIDRs, "package-cidr", "CIDR of a package mirror build Pods may reach (repeatable; default none = no internet egress)")
+	var serverDenyCIDRs multiFlag
+	fs.Var(&serverDenyCIDRs, "server-egress-deny-cidr", "extra CIDR game server pods may never reach, e.g. the node's public address (repeatable)")
+	var serverAllowCIDRs multiFlag
+	fs.Var(&serverAllowCIDRs, "server-egress-allow-cidr", "private CIDR game server pods may reach despite the private-range block, e.g. a LAN database (repeatable)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -71,7 +81,9 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"(the felis-api/operator Deployments run it and it is passed through as FELIS_IMAGE, e.g. --felis-image registry.felis.svc:5000/felis:v1)")
 		return 2
 	}
-	for _, cidr := range append(append([]string{}, velocityCIDRs...), packageCIDRs...) {
+	allCIDRs := append(append([]string{}, velocityCIDRs...), packageCIDRs...)
+	allCIDRs = append(append(allCIDRs, serverDenyCIDRs...), serverAllowCIDRs...)
+	for _, cidr := range allCIDRs {
 		if _, _, err := net.ParseCIDR(cidr); err != nil {
 			fmt.Fprintf(stderr, "felis manifests: invalid CIDR %q: %v\n", cidr, err)
 			return 2
@@ -81,39 +93,70 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis manifests: --panel-node-port must be in Kubernetes NodePort range 30000-32767 (got %d)\n", *panelNodePort)
 		return 2
 	}
+	// The node pin exists only for the reaper's hostPath: naming a node without the
+	// worlds root would be silently dropped (no CronJob renders), so fail loud like
+	// the storage-trio check below.
+	if *reaperNode != "" && *worldsHostPath == "" {
+		fmt.Fprintln(stderr, "felis manifests: --reaper-node requires --worlds-host-path "+
+			"(it pins the reaper CronJob, which renders only with the retention storage trio)")
+		return 2
+	}
 
-	// Retention/reaper rendering is opt-in and needs all three storage coordinates
-	// together: where worlds live (to read+archive them), the backup PVC (to write
-	// archives into), and the path it is mounted at (which MUST equal felis.toml
-	// [archive] local_path so tarLocal's absolute archive refs resolve). A partial
-	// configuration is almost certainly an operator mistake, so fail loud rather than
-	// silently drop retention. Asking for it without the other two is rejected; an
-	// empty trio renders the bundle WITHOUT the reaper and says so.
+	// Retention/reaper rendering is opt-in and needs a storage topology together:
+	// where worlds live (to read+archive them), a backup PVC (to write archives
+	// into — rendered from --backup-pvc), and the path it is mounted at (which MUST
+	// equal felis.toml [archive] local_path so tarLocal's absolute archive refs
+	// resolve). A partial configuration is almost certainly an operator mistake, so
+	// fail loud rather than silently drop retention or render a reaper with nowhere
+	// to write. The backup PVC itself defaults to felis-backups (it is what makes a
+	// default install's backup endpoint work at all); retention additionally needs
+	// --worlds-host-path.
 	if *worldsHostPath != "" {
 		if *backupPVC == "" || *archiveLocalPath == "" {
 			fmt.Fprintln(stderr, "felis manifests: --worlds-host-path enables the reaper CronJob and requires "+
-				"--backup-pvc and --archive-local-path too (--archive-local-path must equal felis.toml [archive] local_path)")
+				"--archive-local-path (must equal felis.toml [archive] local_path) and a non-empty --backup-pvc "+
+				"(the archive store; default felis-backups)")
 			return 2
 		}
-		// The reaper WILL render. Two deployment preconditions this generator cannot
-		// check would SILENTLY turn retention into a no-op if unmet — surface them as
+		// The reaper WILL render. Two deployment facts this generator cannot check
+		// would silently turn retention into a no-op if unmet — surface them as
 		// loudly as the fail-closed cases above, so an operator is never left with a
-		// reaper that reaps an empty directory. (Both are also in the WorldsHostPath
-		// flag/field docs, but nobody deploying from stdout reads those.)
+		// reaper that reaps nothing. (Both are also in the WorldsHostPath flag/field
+		// docs, but nobody deploying from stdout reads those.)
+		pin := "the CronJob sets NO nodeSelector: a single-node starter pins it to the worlds implicitly, but on a " +
+			"multi-node cluster you MUST pass --reaper-node <name> (or add a nodeSelector) for the node holding the " +
+			"worlds, or the reaper may schedule where the hostPath is empty"
+		if *reaperNode != "" {
+			pin = fmt.Sprintf("the CronJob and its worlds-root PV are pinned to node %q via kubernetes.io/hostname — "+
+				"keep this pointed at the node that actually holds the world volumes", *reaperNode)
+		}
 		fmt.Fprintf(stderr, "felis manifests: note: rendering the retention reaper CronJob (worlds hostPath %q). "+
-			"Two preconditions are NOT verified here:\n"+
-			"  - each world PVC must be visible at %s/<pvc> on the node: a stock local-path-provisioner lays "+
-			"volumes under PV-name paths (.../pvc-<uuid>_<ns>_<pvc>/), so unless the worlds StorageClass is "+
-			"arranged to expose <path>/<pvc>, the reaper tars an empty directory;\n"+
-			"  - the CronJob sets NO nodeSelector: a single-node starter pins it to the worlds implicitly, but "+
-			"on a multi-node cluster you MUST add a nodeSelector for the node holding the worlds, or the reaper "+
-			"may schedule where the hostPath is empty.\n", *worldsHostPath, *worldsHostPath)
+			"These points are NOT verified here:\n"+
+			"  - the node's world volumes must actually live below %s: the reaper resolves a world as "+
+			"%s/<pvc>, then as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (what k3s "+
+			"writes under /var/lib/rancher/k3s/storage). Any other provisioner needs its volumes exposed as "+
+			"<path>/<pvc>, or each candidate's archive fails and the world is preserved;\n"+
+			"  - %s.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, pin)
 	} else {
-		fmt.Fprintln(stderr, "felis manifests: note: retention reaper CronJob not rendered "+
-			"(pass --worlds-host-path, --backup-pvc and --archive-local-path to enable it)")
+		switch {
+		case *archiveLocalPath != "" && *backupPVC == "":
+			fmt.Fprintln(stderr, "felis manifests: --archive-local-path names where the backup PVC is mounted, "+
+				"but --backup-pvc is empty (no archive store renders); drop one or the other")
+			return 2
+		case *archiveLocalPath != "":
+			fmt.Fprintln(stderr, "felis manifests: note: rendering the reaper CronJob retention-only: backups past "+
+				"their expiry are deleted daily, idle worlds are never archived or deleted "+
+				"(pass --worlds-host-path to reap them too)")
+		case *backupPVC != "":
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered: backups are never expired, so "+
+				"the archive store only grows until the disk fills (pass --archive-local-path, equal to felis.toml "+
+				"[archive] local_path, for the retention-only CronJob, and --worlds-host-path as well to reap idle worlds)")
+		default:
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered (backups are disabled)")
+		}
 	}
 
-	out, err := platform.RenderYAML(platform.Params{
+	params := platform.Params{
 		ControlNamespace:   *controlNS,
 		MinecraftNamespace: *minecraftNS,
 		BuildNamespace:     *buildNS,
@@ -124,10 +167,23 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 		RegistryImage:      *registryImage,
 		BackupPVC:          *backupPVC,
 		WorldsHostPath:     *worldsHostPath,
+		ReaperNode:         *reaperNode,
 		ArchiveLocalPath:   *archiveLocalPath,
 		VelocityCIDRs:      []string(velocityCIDRs),
 		PackageSourceCIDRs: []string(packageCIDRs),
-	})
+
+		ServerEgressDenyCIDRs:  []string(serverDenyCIDRs),
+		ServerEgressAllowCIDRs: []string(serverAllowCIDRs),
+
+		RegistryStorage: *registryStorage,
+		UploadsStorage:  *uploadsStorage,
+		BackupStorage:   *backupStorage,
+	}
+	if err := params.Validate(); err != nil {
+		fmt.Fprintf(stderr, "felis manifests: %v\n", err)
+		return 2
+	}
+	out, err := platform.RenderYAML(params)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis manifests: render: %v\n", err)
 		return 1

cmd/felis/manifests_test.go

@@ -77,6 +77,10 @@ func TestManifestsRendersBundle(t *testing.T) {
 		"10.0.0.5/32",
 		// The felis image flows through to the Deployments.
 		"registry.felis.svc:5000/felis:v1",
+		// Backup works out of the box: the archive PVC renders and the api gets
+		// the env that wires the backup/restore executors to it.
+		"name: felis-backups",
+		"name: FELIS_BACKUP_PVC",
 	} {
 		if !strings.Contains(text, want) {
 			t.Errorf("rendered bundle missing %q", want)
@@ -86,24 +90,29 @@ func TestManifestsRendersBundle(t *testing.T) {
 		t.Error("rendered bundle must not contain ClusterRole/ClusterRoleBinding")
 	}
 	// Without the retention flags, the reaper CronJob is not rendered and the
-	// generator says so on stderr.
+	// generator says so on stderr, naming what that leaves: backups that never
+	// expire.
 	if strings.Contains(text, "kind: CronJob") {
-		t.Error("no reaper CronJob must render without --worlds-host-path")
+		t.Error("no reaper CronJob must render without --archive-local-path")
 	}
-	if !strings.Contains(errBuf.String(), "not rendered") {
-		t.Errorf("expected a 'reaper not rendered' notice on stderr, got %q", errBuf.String())
+	if !strings.Contains(errBuf.String(), "not rendered") || !strings.Contains(errBuf.String(), "never expired") {
+		t.Errorf("expected a 'reaper not rendered, backups never expired' notice on stderr, got %q", errBuf.String())
 	}
 }
 
-// TestManifestsReaperRequiresTrio proves --worlds-host-path is a fail-loud opt-in:
-// asking for the reaper without the backup PVC and its mount path (which must equal
-// [archive] local_path) is rejected rather than silently dropping retention.
-func TestManifestsReaperRequiresTrio(t *testing.T) {
+// TestManifestsReaperRequiresStorage proves --worlds-host-path is a fail-loud
+// opt-in: asking for the reaper without a writable archive store (the backup PVC,
+// which defaults to felis-backups but can be emptied) and its mount path (which
+// must equal [archive] local_path) is rejected rather than silently dropping
+// retention or deleting worlds it could not archive first.
+func TestManifestsReaperRequiresStorage(t *testing.T) {
 	base := []string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32", "--worlds-host-path", "/var/lib/felis/worlds"}
 	for _, extra := range [][]string{
-		{},                                   // neither backup-pvc nor archive-local-path
-		{"--backup-pvc", "felis-backups"},    // missing archive-local-path
-		{"--archive-local-path", "/backups"}, // missing backup-pvc
+		{},                        // missing archive-local-path (backup-pvc defaults)
+		{"--backup-pvc", "other"}, // still missing archive-local-path
+		// A reaper with no archive store would have nowhere to write the archive
+		// it must verify before deleting a world; emptying the PVC is rejected.
+		{"--archive-local-path", "/backups", "--backup-pvc="},
 	} {
 		var out, errBuf bytes.Buffer
 		code := run(append(append([]string{}, base...), extra...), &out, &errBuf)
@@ -119,6 +128,57 @@ func TestManifestsReaperRequiresTrio(t *testing.T) {
 	}
 }
 
+// TestManifestsBackupPVCOptOut proves --backup-pvc= renders a bundle with no
+// archive store at all: no PVC and no FELIS_BACKUP_PVC env, so backup/restore
+// answer 503 instead of pointing Jobs at a claim nobody provisions.
+func TestManifestsBackupPVCOptOut(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test",
+		"--velocity-cidr", "10.0.0.5/32", "--backup-pvc="}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	for _, absent := range []string{"felis-backups", "FELIS_BACKUP_PVC"} {
+		if strings.Contains(out.String(), absent) {
+			t.Errorf("--backup-pvc= bundle must not contain %q", absent)
+		}
+	}
+}
+
+// TestManifestsRendersRetentionOnly: the archive store without a worlds root
+// still gets the daily CronJob, retention-only, so backups past their expiry
+// leave the store on an install that never reaps a world; the operator is told
+// which of the two it got. An archive path with the store switched off is a
+// mistake and fails loud.
+func TestManifestsRendersRetentionOnly(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	text := out.String()
+	for _, want := range []string{"kind: CronJob", "name: felis-reaper", "--retention-only", "claimName: felis-backups"} {
+		if !strings.Contains(text, want) {
+			t.Errorf("retention-only bundle missing %q", want)
+		}
+	}
+	if strings.Contains(text, "kind: PersistentVolume\n") || strings.Contains(text, "--worlds-root") {
+		t.Error("a retention-only bundle must not reach for a worlds root")
+	}
+	if !strings.Contains(errBuf.String(), "retention-only") {
+		t.Errorf("stderr must say the CronJob is retention-only, got %q", errBuf.String())
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives", "--backup-pvc="}, &out, &errBuf)
+	if code != 2 || out.Len() != 0 || !strings.Contains(errBuf.String(), "--backup-pvc is empty") {
+		t.Errorf("archive path without an archive store: exit=%d out=%d bytes stderr=%q, want a fail-loud 2", code, out.Len(), errBuf.String())
+	}
+}
+
 // TestManifestsRendersReaper proves the happy path with the full retention trio:
 // a batch/v1 CronJob is emitted, named felis-reaper, mounting the backup PVC at the
 // supplied archive path.
@@ -150,9 +210,71 @@ func TestManifestsRendersReaper(t *testing.T) {
 	// this generator cannot verify (else a misarranged hostPath silently no-ops
 	// retention): the <path>/<pvc> arrangement-dependency and the multi-node
 	// nodeSelector hazard.
-	for _, want := range []string{"local-path-provisioner", "nodeSelector"} {
+	for _, want := range []string{"local-path", "nodeSelector"} {
 		if !strings.Contains(errBuf.String(), want) {
 			t.Errorf("reaper render must warn operators about %q on stderr, got %q", want, errBuf.String())
 		}
 	}
 }
+
+// TestManifestsReaperNodePin: --reaper-node pins the rendered CronJob's pod via
+// kubernetes.io/hostname and replaces the "no nodeSelector" hazard note with the
+// pin confirmation; using it without the worlds root is a fail-loud 2.
+func TestManifestsReaperNodePin(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{
+		"manifests",
+		"--felis-image", "registry.felis.svc:5000/felis:v1",
+		"--velocity-cidr", "10.0.0.5/32",
+		"--worlds-host-path", "/var/lib/felis/worlds",
+		"--archive-local-path", "/backups",
+		"--reaper-node", "node-a",
+	}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	for _, want := range []string{
+		"kubernetes.io/hostname: node-a",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("pinned render missing %q", want)
+		}
+	}
+	if !strings.Contains(errBuf.String(), "node-a") {
+		t.Errorf("stderr must confirm the pin, got %q", errBuf.String())
+	}
+
+	var out2, err2 bytes.Buffer
+	if code := run([]string{
+		"manifests",
+		"--felis-image", "registry.felis.svc:5000/felis:v1",
+		"--velocity-cidr", "10.0.0.5/32",
+		"--reaper-node", "node-a",
+	}, &out2, &err2); code != 2 {
+		t.Errorf("--reaper-node without --worlds-host-path: exit = %d, want 2", code)
+	}
+}
+
+// TestManifestsStorageSizes proves the PVC size flags reach the rendered claims
+// and a size the API server would reject fails before anything is applied.
+func TestManifestsStorageSizes(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "40Gi", "--uploads-storage", "8Gi"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, stderr = %s", code, errBuf.String())
+	}
+	for _, want := range []string{"storage: 40Gi", "storage: 8Gi"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("bundle lacks %q", want)
+		}
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "lots"}, &out, &errBuf)
+	if code == 0 || !strings.Contains(errBuf.String(), "registry storage") {
+		t.Errorf("--registry-storage lots: exit %d, stderr %q; want a refusal naming the flag", code, errBuf.String())
+	}
+}

cmd/felis/migrate.go

@@ -7,15 +7,24 @@ import (
 	"io"
 
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
 // cmdMigrate implements `felis migrate up`: load config, open the database, and
 // apply every pending embedded migration under the advisory lock (spec §6).
+//
+// Migrations only roll forward, and some drop data (0017_drop_password), so a
+// database that already holds a schema and has migrations pending is bundled
+// first (internal/dbbackup, label pre-migrate). A failed snapshot stops the
+// upgrade; -no-backup is the explicit way past it, e.g. for an external
+// database whose server is newer than the host's pg_dump.
 func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("migrate", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	backupDir := fs.String("backup-dir", dbbackup.DefaultDir, "where the pre-migration snapshot goes")
+	noBackup := fs.Bool("no-backup", false, "apply pending migrations without snapshotting the database first")
 	// The "up" verb precedes any flags (felis migrate up -config path). Go's
 	// flag.Parse stops at the first non-flag token and would never see a flag
 	// placed after "up", silently falling back to the default -config. Pull the
@@ -48,6 +57,18 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	if !*noBackup {
+		path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database.URL, *backupDir, stderr)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis migrate: pre-migration backup failed, nothing applied: %v\n", err)
+			fmt.Fprintln(stderr, "  fix the backup, or re-run with -no-backup to migrate without one")
+			return 1
+		}
+		if path != "" {
+			fmt.Fprintf(stdout, "felis migrate: database snapshot %s\n", path)
+		}
+	}
+
 	applied, err := store.Up(ctx, drv, migrations)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis migrate: %v\n", err)
@@ -60,3 +81,56 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// preMigrateBackup bundles the database when it already carries a schema and
+// some of migrations are not applied yet, and returns the bundle's path ("" when
+// there was nothing to protect: a fresh database, or nothing pending).
+func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, dbURL, dir string, log io.Writer) (string, error) {
+	if err := drv.EnsureVersionTable(ctx); err != nil {
+		return "", fmt.Errorf("ensure version table: %w", err)
+	}
+	done, err := drv.AppliedVersions(ctx)
+	if err != nil {
+		return "", fmt.Errorf("read applied versions: %w", err)
+	}
+	if len(done) == 0 || !hasPending(done, migrations) {
+		return "", nil
+	}
+	return dbbackup.Backup(ctx, dbbackup.BackupOptions{
+		DatabaseURL: dbURL, Dir: dir, Label: dbbackup.LabelPreMigrate,
+		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir,
+		Version: resolvedVersion(), Log: log, Record: true,
+	})
+}
+
+func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
+	for _, m := range migrations {
+		if _, ok := done[m.Version]; !ok {
+			return true
+		}
+	}
+	return false
+}
+
+// openStore opens the business database for a command that reads and writes its
+// tables, and refuses one whose schema this build was not written against: a newer
+// Felis migrated it (a rolled-back binary), or, unless allowPending, migrations this
+// build embeds have not run yet (a binary swapped in ahead of `felis migrate up`).
+func openStore(ctx context.Context, url string, allowPending bool) (*store.PostgresDriver, error) {
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	s, err := store.ReadSchema(ctx, drv)
+	if err == nil {
+		err = s.Err()
+		if allowPending {
+			err = s.Newer()
+		}
+	}
+	if err != nil {
+		drv.Close()
+		return nil, err
+	}
+	return drv, nil
+}

cmd/felis/mirrortools.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/build"
+	"felis.lolicon.best/internal/imagepush"
+)
+
+// defaultBuildToolsStatus is where mirror-build-tools records its last run; the
+// watchdog reads it to tell a vulnerability DB that stopped refreshing.
+const defaultBuildToolsStatus = "/var/lib/felis/build-tools/status.json"
+
+// cmdMirrorBuildTools copies the build lane's tools (build.Tools: the kaniko and
+// trivy images, Trivy's vulnerability and Java DBs) from upstream into the
+// platform registry, where build Jobs pull them. deploy/bootstrap.sh runs it at
+// install and from felis-build-tools.timer twice a day, which is what keeps the
+// DBs fresh; a root shell can run it the same way to refresh now.
+//
+// It writes as the platform principal through the node's loopback hostPort, the
+// same way the installer pushes, reading the token from the environment or from
+// /etc/felis/secrets.env.
+func cmdMirrorBuildTools(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("mirror-build-tools", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	endpoint := fs.String("endpoint", "127.0.0.1:5000", "host[:port] of the registry to write to (plain HTTP)")
+	only := fs.String("only", "", "comma-separated tool names to copy (default: all of "+toolNames()+")")
+	status := fs.String("status", defaultBuildToolsStatus, `file to record the run in ("" records nothing)`)
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	platformFlag := fs.String("platform", "", "os/arch of the images to copy (default: this machine's)")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	tools, err := selectTools(*only)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: "http", Username: user, Password: pass, Log: stdout}
+	src := &imagepush.Source{Platform: *platformFlag}
+	started := time.Now()
+	var failed []string
+	for _, t := range tools {
+		dst := strings.TrimSuffix(*endpoint, "/") + "/" + t.Mirror
+		if _, err := p.Mirror(ctx, src, t.Source, dst); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: %s: %v\n", t.Name, err)
+			failed = append(failed, t.Name+": "+err.Error())
+		}
+	}
+	if *status != "" {
+		st, err := imagepush.ReadMirrorStatus(*status)
+		if err != nil || st == nil {
+			st = &imagepush.MirrorStatus{}
+		}
+		st.LastAttempt = started
+		st.LastError = strings.Join(failed, "; ")
+		if len(failed) == 0 {
+			st.LastSuccess = started
+		}
+		if err := imagepush.WriteMirrorStatus(*status, *st); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: record %s: %v\n", *status, err)
+		}
+	}
+	if len(failed) > 0 {
+		return 1
+	}
+	return 0
+}
+
+func toolNames() string {
+	var names []string
+	for _, t := range build.Tools {
+		names = append(names, t.Name)
+	}
+	return strings.Join(names, ",")
+}
+
+func selectTools(only string) ([]build.Tool, error) {
+	if only == "" {
+		return build.Tools, nil
+	}
+	var out []build.Tool
+	for _, name := range strings.Split(only, ",") {
+		name = strings.TrimSpace(name)
+		found := false
+		for _, t := range build.Tools {
+			if t.Name == name {
+				out = append(out, t)
+				found = true
+			}
+		}
+		if !found {
+			return nil, fmt.Errorf("unknown tool %q (known: %s)", name, toolNames())
+		}
+	}
+	return out, nil
+}

cmd/felis/offsite.go

@@ -0,0 +1,653 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const offsiteUsage = `usage:
+  felis offsite sync         [-config path] [-archive-dir dir] [-db-dir dir] [-registry host:port|off]
+                             [-uploads-dir dir] [-status-file path]
+  felis offsite status       [-config path] [-status-file path]
+  felis offsite list         [-config path]
+  felis offsite fetch-db     [-config path | -endpoint url -bucket name [-region r] [-prefix p]]
+                             [-dir dir] latest|<bundle>
+  felis offsite fetch-worlds [-config path] [-archive-dir dir]
+  felis offsite fetch-images [-config path] [-registry host:port] [-at version]
+  felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
+  felis offsite keygen
+
+Every verb but keygen reads the bucket credentials and the encryption key from
+the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY,
+FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
+-env-file (default /etc/felis/offsite.env).
+`
+
+// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
+// felis-offsite.service unit loads it as its EnvironmentFile.
+const defaultOffsiteEnvFile = "/etc/felis/offsite.env"
+
+// cmdOffsite implements `felis offsite`: the off-site copy of the world
+// archives, the database bundles, the registry's user images and the
+// submission uploads (internal/offsite). felis-offsite.timer runs `sync`
+// hourly on the host; the fetch verbs are the way back after the node is lost
+// (docs/troubleshooting.md §16).
+func cmdOffsite(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("offsite "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, offsiteUsage) }
+	switch verb {
+	case "sync":
+		return offsiteSync(fs, rest, stdout, stderr)
+	case "status":
+		return offsiteStatus(fs, rest, stdout, stderr)
+	case "list":
+		return offsiteList(fs, rest, stdout, stderr)
+	case "fetch-db":
+		return offsiteFetchDB(fs, rest, stdout, stderr)
+	case "fetch-worlds":
+		return offsiteFetchWorlds(fs, rest, stdout, stderr)
+	case "fetch-images":
+		return offsiteFetchImages(fs, rest, stdout, stderr)
+	case "fetch-uploads":
+		return offsiteFetchUploads(fs, rest, stdout, stderr)
+	case "keygen":
+		k, err := offsite.NewKey()
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite keygen: %v\n", err)
+			return 1
+		}
+		fmt.Fprintln(stdout, k)
+		return 0
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, offsiteUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis offsite: unknown verb %q\n%s", verb, offsiteUsage)
+	return 2
+}
+
+// offsiteEnv is the resolved [offsite] binding: the bucket and the key.
+type offsiteEnv struct {
+	cfg    config.OffsiteConfig
+	bucket *offsite.S3
+	key    []byte
+}
+
+// loadEnvFile sets each KEY=VALUE of path that is not already in the
+// environment, so a root shell runs a command the same way its unit does.
+// A missing file is not an error.
+func loadEnvFile(path string) error {
+	if path == "" {
+		return nil
+	}
+	f, err := os.Open(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		k = strings.TrimSpace(strings.TrimPrefix(k, "export "))
+		v = strings.TrimSpace(v)
+		if len(v) >= 2 && (v[0] == '"' || v[0] == '\'') && v[len(v)-1] == v[0] {
+			v = v[1 : len(v)-1]
+		}
+		if os.Getenv(k) == "" {
+			os.Setenv(k, v)
+		}
+	}
+	return sc.Err()
+}
+
+// resolveOffsite builds the bucket client and parses the key for c.
+func resolveOffsite(c config.OffsiteConfig) (*offsiteEnv, error) {
+	if !c.Enabled() {
+		return nil, errors.New("no [offsite] bucket is configured (docs/troubleshooting.md §16, \"Keep a copy somewhere else\")")
+	}
+	need := func(ref, what string) (string, error) {
+		v := os.Getenv(ref)
+		if v == "" {
+			return "", fmt.Errorf("%s: environment variable %s is empty (set it, or put it in %s)", what, ref, defaultOffsiteEnvFile)
+		}
+		return v, nil
+	}
+	ak, err := need(c.AccessKeyRef, "access key")
+	if err != nil {
+		return nil, err
+	}
+	sk, err := need(c.SecretKeyRef, "secret key")
+	if err != nil {
+		return nil, err
+	}
+	rawKey, err := need(c.KeyRef, "encryption key")
+	if err != nil {
+		return nil, err
+	}
+	key, err := offsite.ParseKey(rawKey)
+	if err != nil {
+		return nil, err
+	}
+	b, err := offsite.NewS3(offsite.S3Config{
+		Endpoint: c.Endpoint, Region: c.Region, Bucket: c.Bucket, Prefix: c.Prefix,
+		AccessKey: ak, SecretKey: sk,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &offsiteEnv{cfg: c, bucket: b, key: key}, nil
+}
+
+// loadOffsite loads felis.toml and the env file and resolves [offsite].
+func loadOffsite(cfgPath, envFile string) (*config.Config, *offsiteEnv, error) {
+	if err := loadEnvFile(envFile); err != nil {
+		return nil, nil, fmt.Errorf("read %s: %w", envFile, err)
+	}
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	env, err := resolveOffsite(cfg.Offsite)
+	if err != nil {
+		return cfg, nil, err
+	}
+	return cfg, env, nil
+}
+
+func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
+	dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
+	registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC through the cluster)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, `the submission uploads PVC, in the control-plane namespace ("" copies no uploads)`)
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "where the result of this run is recorded for the watchdog and `status`")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	st := offsite.Status{
+		LastAttempt: time.Now().UTC(), Endpoint: env.cfg.Endpoint, Bucket: env.cfg.Bucket,
+		Prefix: env.cfg.Prefix, KeyID: offsite.KeyID(env.key),
+	}
+	if prev, _ := offsite.ReadStatus(*statusFile); prev != nil {
+		st.LastSuccess = prev.LastSuccess
+	}
+	res, err := runOffsiteSync(cfg, env, offsiteSources{
+		archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir,
+		registry:   offsiteRegistryEndpoint(*registry, cfg.Registry),
+		uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
+	}, stderr)
+	st.Result = res
+	if err != nil {
+		st.LastError = err.Error()
+	} else {
+		st.LastSuccess = st.LastAttempt
+	}
+	if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
+	}
+	fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n",
+		res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
+		res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
+		res.UploadsUploaded, res.UploadObjectsPruned,
+		res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes),
+		res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes))
+	for _, m := range res.WorldsMissing {
+		fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
+	}
+	for _, m := range res.ImagesIncomplete {
+		fmt.Fprintf(stderr, "felis offsite sync: registry image not whole: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// offsiteSources is where one sync pass reads from: the world archive volume
+// (archiveDir, or the backupPVC's directory), the bundle directory, the
+// registry's loopback endpoint and the uploads volume (uploadsDir, or the
+// uploadsPVC's directory). An empty source is skipped.
+type offsiteSources struct {
+	archiveDir, backupPVC  string
+	dbDir                  string
+	registry               string
+	uploadsDir, uploadsPVC string
+}
+
+func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, log io.Writer) (offsite.Result, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Minute)
+	defer cancel()
+	checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
+	err := env.bucket.Check(checkCtx)
+	checkCancel()
+	if err != nil {
+		return offsite.Result{}, err
+	}
+	archiveDir, uploadsDir := src.archiveDir, src.uploadsDir
+	if archiveDir == "" && src.backupPVC != "" {
+		dir, err := resolveVolumeDir(ctx, cfg.K8s.Namespace, src.backupPVC, archiveVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		archiveDir = dir
+	}
+	// An s3:// uploads store is off the host already; only a local one, on
+	// the uploads PVC, needs the copy.
+	if uploadsDir == "" && src.uploadsPVC != "" && isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+		dir, err := resolveVolumeDir(ctx, platform.DefaultControlNamespace, src.uploadsPVC, uploadsVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		uploadsDir = dir
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		return offsite.Result{}, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	s := &offsite.Syncer{
+		Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key,
+		ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Log: log,
+	}
+	if src.registry != "" {
+		s.Images = newRegistryImages(src.registry)
+		s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL)
+	}
+	return s.Run(ctx)
+}
+
+// volumeKind names a PVC the off-site copy reads or restores, for messages,
+// with the flag that bypasses finding it through the cluster.
+type volumeKind struct{ what, dirFlag, empty string }
+
+var (
+	archiveVolume = volumeKind{"archive volume", "-archive-dir", "no world has been archived"}
+	uploadsVolume = volumeKind{"uploads volume", "-uploads-dir", "no modpack has been uploaded"}
+)
+
+// resolveVolumeDir finds the host directory behind a PVC: a local-path volume
+// is a directory on this node. A PVC still waiting for its first consumer
+// holds nothing yet: without bind that is "" (nothing to copy), with bind it
+// is bound first, for a fetch to write into.
+func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind, bind bool, log io.Writer) (string, error) {
+	if ns == "" {
+		ns = platform.DefaultMinecraftNamespace
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		return "", fmt.Errorf("reach the cluster to find the %s (or pass %s): %w", kind.what, kind.dirFlag, err)
+	}
+	var pvc corev1.PersistentVolumeClaim
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+		return "", fmt.Errorf("%s %s/%s: %w", kind.what, ns, pvcName, err)
+	}
+	if pvc.Spec.VolumeName == "" {
+		if !bind {
+			fmt.Fprintf(log, "felis offsite: %s %s/%s is not bound yet; %s\n", kind.what, ns, pvcName, kind.empty)
+			return "", nil
+		}
+		if err := bindVolume(ctx, cl, ns, pvcName, kind, log); err != nil {
+			return "", err
+		}
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+			return "", err
+		}
+	}
+	var pv corev1.PersistentVolume
+	if err := cl.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil {
+		return "", fmt.Errorf("%s %s: %w", kind.what, pvc.Spec.VolumeName, err)
+	}
+	var dir string
+	switch {
+	case pv.Spec.Local != nil:
+		dir = pv.Spec.Local.Path
+	case pv.Spec.HostPath != nil:
+		dir = pv.Spec.HostPath.Path
+	default:
+		return "", fmt.Errorf("%s %s is not a directory on a node (local or hostPath); pass %s with where it is mounted on this host", kind.what, pv.Name, kind.dirFlag)
+	}
+	if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
+		return "", fmt.Errorf("%s %s is %s on its node, which is not a directory here; run this on the node that holds it, or pass %s", kind.what, pv.Name, dir, kind.dirFlag)
+	}
+	return dir, nil
+}
+
+// bindVolume runs a pod that mounts the PVC and exits, which is what makes a
+// WaitForFirstConsumer volume (k3s local-path) get provisioned. The pod uses
+// the control plane's own image, which every install already has.
+func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind volumeKind, log io.Writer) error {
+	var api appsv1.Deployment
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: platform.DefaultControlNamespace, Name: "felis-api"}, &api); err != nil {
+		return fmt.Errorf("find the felis image to bind the %s with: %w", kind.what, err)
+	}
+	if len(api.Spec.Template.Spec.Containers) == 0 {
+		return errors.New("felis-api has no container to take the image from")
+	}
+	image := api.Spec.Template.Spec.Containers[0].Image
+	pod := platform.VolumeBinderPod(ns, pvcName, image)
+	if err := cl.Create(ctx, pod); err != nil {
+		return fmt.Errorf("start a pod to bind the %s: %w", kind.what, err)
+	}
+	fmt.Fprintf(log, "felis offsite: binding the %s %s/%s (pod %s)\n", kind.what, ns, pvcName, pod.Name)
+	defer func() {
+		_ = cl.Delete(context.Background(), pod, client.PropagationPolicy(metav1.DeletePropagationBackground))
+	}()
+	deadline := time.Now().Add(3 * time.Minute)
+	for time.Now().Before(deadline) {
+		var pvc corev1.PersistentVolumeClaim
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err == nil && pvc.Spec.VolumeName != "" && pvc.Status.Phase == corev1.ClaimBound {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+	return fmt.Errorf("the %s %s/%s did not bind within 3 minutes; see kubectl -n %s describe pod %s", kind.what, ns, pvcName, ns, pod.Name)
+}
+
+func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if !cfg.Offsite.Enabled() {
+		fmt.Fprintln(stdout, "off-site copy: not configured. World archives, database bundles, user images and uploaded modpacks exist on this machine only.")
+		fmt.Fprintln(stdout, "See docs/troubleshooting.md §16, \"Keep a copy somewhere else\".")
+		return 1
+	}
+	o := cfg.Offsite
+	fmt.Fprintf(stdout, "bucket:       %s at %s", o.Bucket, o.Endpoint)
+	if o.Prefix != "" {
+		fmt.Fprintf(stdout, ", prefix %s", o.Prefix)
+	}
+	fmt.Fprintln(stdout)
+	st, err := offsite.ReadStatus(*statusFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if st == nil {
+		fmt.Fprintln(stdout, "last sync:    never (sudo systemctl start felis-offsite.service)")
+		return 1
+	}
+	now := time.Now()
+	fmt.Fprintf(stdout, "key id:       %s\n", st.KeyID)
+	fmt.Fprintf(stdout, "last attempt: %s (%s ago)\n", st.LastAttempt.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastAttempt)))
+	if st.LastSuccess.IsZero() {
+		fmt.Fprintln(stdout, "last success: never")
+	} else {
+		fmt.Fprintf(stdout, "last success: %s (%s ago)\n", st.LastSuccess.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastSuccess)))
+	}
+	if st.LastError != "" {
+		fmt.Fprintf(stdout, "last error:   %s\n", st.LastError)
+	}
+	r := st.Result
+	fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
+		r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
+	if r.ImageIndex != "" {
+		fmt.Fprintf(stdout, "images:       %d in %d repositories (%s), registry index %s\n",
+			r.Images, r.ImageRepos, offsite.HumanBytes(r.RemoteImageBytes), r.ImageIndex)
+	} else {
+		fmt.Fprintln(stdout, "images:       not copied (no in-cluster registry, or no sync has reached it yet)")
+	}
+	if r.UploadIndex != "" {
+		fmt.Fprintf(stdout, "uploads:      %d submission contexts (%s), uploads index %s\n",
+			r.Uploads, offsite.HumanBytes(r.RemoteUploadBytes), r.UploadIndex)
+	} else {
+		fmt.Fprintln(stdout, "uploads:      not copied (an s3:// uploads store, or no sync has reached the volume yet)")
+	}
+	fmt.Fprintf(stdout, "waiting:      %d world archives not yet copied\n", r.WorldsPending)
+	for _, m := range r.WorldsMissing {
+		fmt.Fprintf(stdout, "missing:      %s is recorded but not on the volume\n", m)
+	}
+	for _, m := range r.ImagesIncomplete {
+		fmt.Fprintf(stdout, "not whole:    %s\n", m)
+	}
+	if st.LastSuccess.IsZero() || now.Sub(st.LastSuccess) > offsite.StaleAfter {
+		fmt.Fprintf(stdout, "\nThe last successful sync is older than %s: journalctl -u felis-offsite -n 50\n", dbbackup.Age(offsite.StaleAfter))
+		return 1
+	}
+	return 0
+}
+
+func orNone(s string) string {
+	if s == "" {
+		return "none"
+	}
+	return s
+}
+
+func offsiteList(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	return printOffsiteList(env, stdout, stderr)
+}
+
+func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	bundles, err := offsite.ListDB(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	worlds, err := env.bucket.List(ctx, "worlds/")
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "database bundles (%d, newest first):\n", len(bundles))
+	for _, b := range bundles {
+		fmt.Fprintf(stdout, "  %s  %s\n", b.Key, offsite.HumanBytes(b.Size))
+	}
+	var total int64
+	for _, w := range worlds {
+		total += w.Size
+	}
+	fmt.Fprintf(stdout, "world archives: %d (%s)\n", len(worlds), offsite.HumanBytes(total))
+	versions, err := offsite.ImageIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "registry index versions (%d, newest first; restore one with fetch-images -at):\n", len(versions))
+	for i := len(versions) - 1; i >= 0; i-- {
+		x, err := offsite.LoadImageIndex(ctx, env.bucket, env.key, versions[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", versions[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d images in %d repositories\n", versions[i], x.Images(), len(x.Repositories))
+	}
+	uploads, err := offsite.UploadIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "uploads index versions (%d, newest first; restore one with fetch-uploads -at):\n", len(uploads))
+	for i := len(uploads) - 1; i >= 0; i-- {
+		x, err := offsite.LoadUploadIndex(ctx, env.bucket, env.key, uploads[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", uploads[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d submission contexts (%s)\n", uploads[i], len(x.Contexts), offsite.HumanBytes(x.Bytes()))
+	}
+	return 0
+}
+
+func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	endpoint := fs.String("endpoint", "", "bucket endpoint, when there is no felis.toml")
+	bucket := fs.String("bucket", "", "bucket name, when there is no felis.toml")
+	region := fs.String("region", "", "bucket region, when there is no felis.toml")
+	prefix := fs.String("prefix", "", "key prefix, when there is no felis.toml")
+	dir := fs.String("dir", dbbackup.DefaultDir, "directory to write the bundle to")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	if err := loadEnvFile(*envFile); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: read %s: %v\n", *envFile, err)
+		return 1
+	}
+	var oc config.OffsiteConfig
+	if *bucket != "" {
+		oc = config.OffsiteConfig{
+			Endpoint: *endpoint, Bucket: *bucket, Region: *region, Prefix: *prefix,
+			AccessKeyRef: config.DefaultOffsiteAccessKeyEnv, SecretKeyRef: config.DefaultOffsiteSecretKeyEnv,
+			KeyRef: config.DefaultOffsiteKeyEnv,
+		}
+	} else {
+		cfg, err := config.Load(*cfgPath)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v (on a host with no install yet, pass -endpoint and -bucket)\n", err)
+			return 1
+		}
+		oc = cfg.Offsite
+	}
+	env, err := resolveOffsite(oc)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	name := arg
+	if name == "latest" {
+		bundles, err := offsite.ListDB(ctx, env.bucket)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+			return 1
+		}
+		if len(bundles) == 0 {
+			fmt.Fprintln(stderr, "felis offsite fetch-db: the bucket holds no database bundle")
+			return 1
+		}
+		name = bundles[0].Key
+	}
+	if _, _, ok := dbbackup.ParseBundleName(name); !ok {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %q is not a bundle name (felis-db-<stamp>-<label>.tar); see `felis offsite list`\n", name)
+		return 2
+	}
+	if err := os.MkdirAll(*dir, 0o700); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	dst := filepath.Join(*dir, name)
+	if err := offsite.FetchObject(ctx, env.bucket, env.key, offsite.DBKey(name), dst, 0o600); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	if _, err := dbbackup.Verify(dst); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: fetched %s but it does not verify: %v\n", dst, err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-db: wrote %s (verified)\n", dst)
+	return 0
+}
+
+func offsiteFetchWorlds(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC, binding it if needed)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", "the world archive PVC, in the [k8s] namespace")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	dir := *archiveDir
+	if dir == "" {
+		if dir, err = resolveVolumeDir(ctx, cfg.K8s.Namespace, *backupPVC, archiveVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+			return 1
+		}
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: open database: %v\n", err)
+		return 1
+	}
+	defer drv.Close()
+	res, err := offsite.FetchWorlds(ctx, env.bucket, offsite.PGCatalog{DB: drv.DB()}, env.key, dir, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-worlds: %d recorded archives, %d fetched into %s, %d with no copy in the bucket\n",
+		res.Present, len(res.Fetched), dir, len(res.Missing))
+	for _, m := range res.Missing {
+		fmt.Fprintf(stdout, "  no off-site copy: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_images.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/registrygate"
+	"felis.lolicon.best/internal/registryprune"
+)
+
+// registryImages is the platform registry as the off-site copy sees it: read
+// anonymously through the gate (the catalog, its manifest index, manifests and
+// blobs) and, for a restore, written as the platform principal. Both go through
+// the node's loopback hostPort, the way the installer pushes.
+type registryImages struct {
+	host   string
+	index  *registryprune.Client
+	source *imagepush.Source
+	pusher *imagepush.Pusher
+}
+
+func newRegistryImages(endpoint string) *registryImages {
+	return &registryImages{
+		host:   endpoint,
+		index:  &registryprune.Client{Endpoint: "http://" + endpoint},
+		source: &imagepush.Source{Scheme: "http"},
+	}
+}
+
+func (r *registryImages) Repositories(ctx context.Context) ([]string, error) {
+	return r.index.Repositories(ctx)
+}
+
+func (r *registryImages) Revisions(ctx context.Context, repo string) ([]string, map[string]string, error) {
+	idx, err := r.index.Index(ctx, repo)
+	if err != nil {
+		return nil, nil, err
+	}
+	digests := make([]string, 0, len(idx.Revisions))
+	for _, rev := range idx.Revisions {
+		digests = append(digests, rev.Digest)
+	}
+	return digests, idx.Tags, nil
+}
+
+func (r *registryImages) Manifest(ctx context.Context, repo, digest string) ([]byte, string, error) {
+	body, mt, err := r.source.Manifest(ctx, r.host, repo, digest)
+	return body, mt, registryGone(err)
+}
+
+func (r *registryImages) Blob(ctx context.Context, repo, digest string) (io.ReadCloser, error) {
+	rc, err := r.source.Blob(ctx, r.host, repo, digest)
+	return rc, registryGone(err)
+}
+
+func (r *registryImages) PutBlob(ctx context.Context, repo, digest string, size int64, open func() (io.ReadCloser, error)) error {
+	return r.pusher.UploadBlob(ctx, r.host, repo, digest, size, open)
+}
+
+func (r *registryImages) PutManifest(ctx context.Context, repo, reference, mediaType string, body []byte) error {
+	_, err := r.pusher.PutManifest(ctx, r.host, repo, reference, mediaType, body)
+	return err
+}
+
+// imagePins lists, per repository of the registry refs spell as host, the
+// digests the MinecraftServers' specs and the image whitelist pin: what a
+// restored database and its servers will ask the registry for.
+func imagePins(db *sql.DB, host string) func(ctx context.Context) (map[string][]string, error) {
+	return func(ctx context.Context) (map[string][]string, error) {
+		cl, err := buildSystemServerClient()
+		if err != nil {
+			return nil, fmt.Errorf("reach the cluster: %w", err)
+		}
+		var servers v1alpha1.MinecraftServerList
+		if err := cl.List(ctx, &servers); err != nil {
+			return nil, fmt.Errorf("list MinecraftServers: %w", err)
+		}
+		refs := make([]string, 0, len(servers.Items))
+		for _, s := range servers.Items {
+			refs = append(refs, s.Spec.Image)
+		}
+		rows, err := db.QueryContext(ctx, `SELECT image_ref FROM image_whitelist`)
+		if err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var ref string
+			if err := rows.Scan(&ref); err != nil {
+				return nil, fmt.Errorf("read the image whitelist: %w", err)
+			}
+			refs = append(refs, ref)
+		}
+		if err := rows.Err(); err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		pins := map[string][]string{}
+		for _, ref := range refs {
+			repo, _, digest, ok := registryprune.ParseRef(ref, host)
+			if ok && digest != "" && !slices.Contains(pins[repo], digest) {
+				pins[repo] = append(pins[repo], digest)
+			}
+		}
+		return pins, nil
+	}
+}
+
+// registryGone marks a 404 as a manifest or blob the registry no longer holds.
+func registryGone(err error) error {
+	var se *imagepush.StatusError
+	if errors.As(err, &se) && se.Code == http.StatusNotFound {
+		return fmt.Errorf("%w: %v", offsite.ErrImageGone, err)
+	}
+	return err
+}
+
+// offsiteRegistryEndpoint is where the host reaches the registry whose images
+// the off-site copy covers: flag when given ("off" for none), otherwise the
+// loopback hostPort of the in-cluster registry [registry] url names. A
+// registry outside the cluster is not this install's to copy.
+func offsiteRegistryEndpoint(flag string, reg config.RegistryConfig) string {
+	switch flag {
+	case "off":
+		return ""
+	case "":
+	default:
+		return flag
+	}
+	host, _, _ := strings.Cut(reg.URL, "/")
+	name, _, _ := strings.Cut(host, ":")
+	if strings.HasSuffix(name, ".svc") || strings.HasSuffix(name, ".svc.cluster.local") {
+		return loopbackEndpoint(host)
+	}
+	return ""
+}
+
+// registryWriteCredential is the credential a host-side push uses:
+// FELIS_REGISTRY_USERNAME/PASSWORD when set, otherwise the platform principal
+// with REGISTRY_PLATFORM_TOKEN from the installer's secrets file.
+func registryWriteCredential(secrets string) (string, string, error) {
+	if err := loadEnvFile(secrets); err != nil {
+		return "", "", fmt.Errorf("read %s: %w", secrets, err)
+	}
+	user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if pass == "" {
+		user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
+	}
+	if pass == "" {
+		return "", "", errors.New("no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
+	}
+	return user, pass, nil
+}
+
+func offsiteFetchImages(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	registry := fs.String("registry", "", "host[:port] of the registry to push into (default: the loopback hostPort of the in-cluster registry)")
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	at := fs.String("at", "", "registry index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	endpoint := offsiteRegistryEndpoint(*registry, cfg.Registry)
+	if endpoint == "" {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: [registry] url %q is not the in-cluster registry; pass -registry host:port\n", cfg.Registry.URL)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 2
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseImageIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-images: restoring registry index %s (%d repositories, %d images) into %s\n",
+		stamp, len(idx.Repositories), idx.Images(), endpoint)
+	target := newRegistryImages(endpoint)
+	target.pusher = &imagepush.Pusher{Scheme: "http", Username: user, Password: pass}
+	res, err := offsite.FetchImages(ctx, env.bucket, env.key, idx, target, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-images: %d of %d repositories restored, %d images, %d tags, %d blobs pushed (%s)\n",
+		res.Repositories, len(idx.Repositories), res.Manifests, res.Tags, res.BlobsPushed, offsite.HumanBytes(res.BytesPushed))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v (a second run pushes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_test.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+)
+
+func TestLoadOffsiteEnvFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "offsite.env")
+	body := `# written by bootstrap
+FELIS_OFFSITE_ACCESS_KEY=AKIA123
+export FELIS_OFFSITE_SECRET_KEY="se=cret"
+FELIS_OFFSITE_KEY='k'
+
+not a line
+`
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "from-the-shell")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "")
+	t.Setenv("FELIS_OFFSITE_KEY", "")
+	if err := loadEnvFile(path); err != nil {
+		t.Fatal(err)
+	}
+	for k, want := range map[string]string{
+		"FELIS_OFFSITE_ACCESS_KEY": "from-the-shell", // the environment wins
+		"FELIS_OFFSITE_SECRET_KEY": "se=cret",
+		"FELIS_OFFSITE_KEY":        "k",
+	} {
+		if got := os.Getenv(k); got != want {
+			t.Errorf("%s = %q, want %q", k, got, want)
+		}
+	}
+	if err := loadEnvFile(filepath.Join(t.TempDir(), "absent")); err != nil {
+		t.Errorf("a missing env file is not an error: %v", err)
+	}
+}
+
+func TestResolveOffsiteNamesTheMissingVariable(t *testing.T) {
+	c := config.OffsiteConfig{
+		Endpoint: "https://s3.example", Bucket: "b",
+		AccessKeyRef: "T_AK", SecretKeyRef: "T_SK", KeyRef: "T_KEY",
+	}
+	t.Setenv("T_AK", "ak")
+	t.Setenv("T_SK", "sk")
+	t.Setenv("T_KEY", "")
+	if _, err := resolveOffsite(c); err == nil || !strings.Contains(err.Error(), "T_KEY") {
+		t.Fatalf("err = %v, want it to name T_KEY", err)
+	}
+	t.Setenv("T_KEY", "not base64 at all")
+	if _, err := resolveOffsite(c); err == nil {
+		t.Fatal("a malformed key was accepted")
+	}
+	key, _ := offsite.NewKey()
+	t.Setenv("T_KEY", key)
+	env, err := resolveOffsite(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(env.key) != offsite.KeySize {
+		t.Fatalf("key is %d bytes", len(env.key))
+	}
+	if _, err := resolveOffsite(config.OffsiteConfig{}); err == nil {
+		t.Fatal("an unconfigured [offsite] resolved")
+	}
+}
+
+func TestOffsiteKeygen(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := cmdOffsite([]string{"keygen"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if _, err := offsite.ParseKey(strings.TrimSpace(out.String())); err != nil {
+		t.Fatalf("keygen printed %q: %v", out.String(), err)
+	}
+}
+
+func TestOffsiteFetchDBRejectsOddNames(t *testing.T) {
+	key, _ := offsite.NewKey()
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "ak")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "sk")
+	t.Setenv("FELIS_OFFSITE_KEY", key)
+	var out, errb bytes.Buffer
+	code := cmdOffsite([]string{"fetch-db", "-env-file", "", "-endpoint", "http://127.0.0.1:1", "-bucket", "b",
+		"-dir", t.TempDir(), "../../etc/shadow"}, &out, &errb)
+	if code != 2 || !strings.Contains(errb.String(), "not a bundle name") {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+}
+
+func TestOffsiteRegistryEndpoint(t *testing.T) {
+	for _, tc := range []struct{ flag, url, want string }{
+		{"", "registry.felis.svc:5000", "127.0.0.1:5000"},
+		{"", "registry.felis.svc.cluster.local:5001", "127.0.0.1:5001"},
+		{"", "ghcr.io/acme", ""},
+		{"", "", ""},
+		{"off", "registry.felis.svc:5000", ""},
+		{"10.0.0.5:5000", "ghcr.io/acme", "10.0.0.5:5000"},
+	} {
+		if got := offsiteRegistryEndpoint(tc.flag, config.RegistryConfig{URL: tc.url}); got != tc.want {
+			t.Errorf("offsiteRegistryEndpoint(%q, %q) = %q, want %q", tc.flag, tc.url, got, tc.want)
+		}
+	}
+}
+
+func TestRegistryGoneMarksNotFound(t *testing.T) {
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 404}); !errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("404 = %v, want ErrImageGone", err)
+	}
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 503}); errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("503 = %v, want it kept an ordinary failure", err)
+	}
+}

cmd/felis/offsite_uploads.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+)
+
+func offsiteFetchUploads(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC, binding it if needed)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, "the submission uploads PVC, in the control-plane namespace")
+	at := fs.String("at", "", "uploads index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseUploadIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	dir := *uploadsDir
+	if dir == "" {
+		if !isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: [registry] user_uploads_context %q is not the uploads volume; pass -uploads-dir to restore into a directory anyway\n", cfg.Registry.UserUploadsContext)
+			return 2
+		}
+		if dir, err = resolveVolumeDir(ctx, platform.DefaultControlNamespace, *uploadsPVC, uploadsVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+			return 1
+		}
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: restoring uploads index %s (%d submission contexts, %s) into %s\n",
+		stamp, len(idx.Contexts), offsite.HumanBytes(idx.Bytes()), dir)
+	res, err := offsite.FetchUploads(ctx, env.bucket, env.key, idx, dir, platform.ControlPlaneUID, platform.ControlPlaneUID, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: %d written (%s), %d already in place, %d failed\n",
+		res.Written, offsite.HumanBytes(res.Bytes), res.Present, len(res.Failures))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v (a second run writes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/operator.go

@@ -1,19 +1,26 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"log/slog"
+	"net/http"
 	"os"
+	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	felismetrics "felis.lolicon.best/internal/metrics"
 	"felis.lolicon.best/internal/operator"
+	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 )
@@ -25,6 +32,11 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	fs := flag.NewFlagSet("operator", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	metricsAddr := fs.String("metrics-bind-address", ":8080", "address the metric endpoint binds to")
+	// healthAddr serves the manager's health endpoints (/healthz, /readyz) that the
+	// Deployment's probes dial. Without it the operator pod would carry no probe at
+	// all, and a wedged manager would keep its endpoint forever. It must differ from
+	// metricsAddr: the metrics server owns :8080.
+	healthAddr := fs.String("health-probe-bind-address", ":8081", "address the health probe endpoint binds to")
 	// namespace MUST equal the [k8s] namespace felis-api is configured with, and
 	// the deployment manifests (felis manifests) render both from one value. It
 	// scopes the manager's cache (informers) to a single namespace so the operator
@@ -42,9 +54,16 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(v1alpha1.AddToScheme(scheme))
 
+	// controller-runtime logs through its own logr sink; without one, its first
+	// reconcile prints "log.SetLogger(...) was never called" ATTACHED TO A FULL
+	// GOROUTINE STACK — pure noise, not signal. Route it to slog's default handler
+	// so its messages appear as ordinary stderr lines.
+	ctrl.SetLogger(logr.FromSlogHandler(slog.Default().Handler()))
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsserver.Options{BindAddress: *metricsAddr},
+		HealthProbeBindAddress: *healthAddr,
 		// Scope every informer to the single watched namespace. Without this the
 		// cached client (mgr.GetClient) would LIST/WATCH cluster-wide, which a
 		// namespaced Role cannot grant — the operator would fail closed at runtime
@@ -60,6 +79,23 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	fmt.Fprintf(stderr, "felis operator: watching namespace %q\n", *namespace)
 
+	// /healthz fails while a reconcile pass has been stuck past its limit, so the
+	// liveness probe restarts an operator whose workers are wedged (a Pod whose
+	// process answers but no server starts or stops). /readyz waits for the
+	// informer caches: until they sync the operator acts on nothing, and one that
+	// never syncs (lost RBAC, an unreachable API) never reports Available.
+	// A dependency hiccup fails neither: the caches ride through API blips, and
+	// each pass is bounded well inside the stuck limit.
+	watch := &operator.ReconcileWatch{}
+	if err := mgr.AddHealthzCheck("reconcile", watch.Check); err != nil {
+		fmt.Fprintf(stderr, "felis operator: register healthz check: %v\n", err)
+		return 1
+	}
+	if err := mgr.AddReadyzCheck("informers", cacheSynced(mgr.GetCache())); err != nil {
+		fmt.Fprintf(stderr, "felis operator: register readyz check: %v\n", err)
+		return 1
+	}
+
 	// Publish the named felis_* metrics (spec §23) on the endpoint the manager
 	// already serves (metricsAddr). controller-runtime's metrics server exposes
 	// its global Registry, so registering into it is all that is needed for
@@ -69,6 +105,7 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis operator: register metrics: %v\n", err)
 		return 1
 	}
+	felismetrics.SetBuildInfo("operator", resolvedVersion())
 
 	r := &operator.Reconciler{
 		Client: mgr.GetClient(),
@@ -78,6 +115,14 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		// injects into user servers. The Deployment passes it as FELIS_IMAGE (see
 		// platform.OperatorDeployment); absent, that injection is simply skipped.
 		FelisImage: os.Getenv("FELIS_IMAGE"),
+		// Uncached: the maintenance-lock check lists Jobs only when a server is
+		// about to start, which does not justify a namespace-wide Job informer.
+		Jobs: mgr.GetAPIReader(),
+		// Uncached too: RCON Secrets are read by name, so the Role grants
+		// secrets:get without the list/watch an informer would need.
+		Secrets:  mgr.GetAPIReader(),
+		Recorder: mgr.GetEventRecorderFor("felis-operator"),
+		Watch:    watch,
 	}
 	if err := r.SetupWithManager(mgr); err != nil {
 		fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
@@ -99,3 +144,18 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// cacheSynced is a readyz check that passes once every informer the manager
+// started has synced. It waits at most a second, well inside the probe timeout.
+func cacheSynced(c interface {
+	WaitForCacheSync(ctx context.Context) bool
+}) healthz.Checker {
+	return func(req *http.Request) error {
+		ctx, cancel := context.WithTimeout(req.Context(), time.Second)
+		defer cancel()
+		if !c.WaitForCacheSync(ctx) {
+			return errors.New("informer caches not synced")
+		}
+		return nil
+	}
+}

cmd/felis/operator_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"context"
+	"net/http/httptest"
+	"testing"
+)
+
+type fakeCache bool
+
+func (f fakeCache) WaitForCacheSync(ctx context.Context) bool {
+	if !f {
+		<-ctx.Done()
+	}
+	return bool(f)
+}
+
+// TestCacheSynced: the operator reports ready only once its informers synced,
+// and a check against caches that never sync returns within its own deadline.
+func TestCacheSynced(t *testing.T) {
+	req := httptest.NewRequest("GET", "/readyz", nil)
+	if err := cacheSynced(fakeCache(true))(req); err != nil {
+		t.Errorf("synced: %v", err)
+	}
+	if err := cacheSynced(fakeCache(false))(req); err == nil {
+		t.Error("unsynced caches reported ready")
+	}
+}

cmd/felis/pinimages.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// defaultRegistryURL is the [registry] url every install uses; deploy/bootstrap.sh
+// spells the same value as REGISTRY_URL.
+const defaultRegistryURL = "registry.felis.svc:5000"
+
+// cmdPinImages pins every user server whose spec.image still names a mutable tag
+// in the platform registry to the digest that tag names now (internal/imagepin).
+// felis-api pins on create, so this covers the servers created before it did.
+//
+// deploy/bootstrap.sh runs it before it rebuilds the game images and pushes them
+// over the same tags: run after the push, it would pin those servers to the new
+// build, which is exactly the silent Minecraft upgrade pinning exists to stop.
+// It reaches the registry through the node's loopback hostPort, the same way the
+// installer pushes.
+func cmdPinImages(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("pin-images", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	namespace := fs.String("namespace", platform.DefaultMinecraftNamespace, "namespace the MinecraftServers live in")
+	registry := fs.String("registry", defaultRegistryURL, "registry host[:port] the image refs spell")
+	endpoint := fs.String("endpoint", "", "host[:port] to reach the registry at (default: 127.0.0.1 on the registry's port, its hostPort on this node)")
+	system := fs.String("system", "", "pin this system server ("+naming.SystemLoginServer+" or "+naming.SystemLobbyServer+") to the build its tag names now, instead of the user servers")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if *system != "" && *system != naming.SystemLoginServer && *system != naming.SystemLobbyServer {
+		fmt.Fprintf(stderr, "felis pin-images: --system takes %s or %s, not %q\n", naming.SystemLoginServer, naming.SystemLobbyServer, *system)
+		return 2
+	}
+	if *endpoint == "" {
+		*endpoint = loopbackEndpoint(*registry)
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	if *system != "" {
+		return reportSystemPin(ctx, cl, *namespace, *system, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint}, stdout, stderr)
+	}
+	outcomes, err := pinUserServerImages(ctx, cl, *namespace, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint})
+	if meta.IsNoMatchError(err) {
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	if len(outcomes) == 0 {
+		fmt.Fprintln(stdout, "felis pin-images: every user server already runs a pinned image")
+		return 0
+	}
+	fmt.Fprintln(stdout, "felis pin-images: pinning user servers to the build their tag names now:")
+	exit := 0
+	for _, o := range outcomes {
+		if o.err != nil {
+			fmt.Fprintf(stdout, "  - %s: ERROR %v\n", o.name, o.err)
+			exit = 1
+			continue
+		}
+		fmt.Fprintf(stdout, "  - %s: %s\n", o.name, strings.Join(o.changes, ", "))
+	}
+	return exit
+}
+
+// reportSystemPin runs pinSystemServerImage for `felis pin-images --system` and
+// prints what it did. Only a failed pin exits non-zero: the installer falls back
+// to restarting the pod on its tag then.
+func reportSystemPin(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver, stdout, stderr io.Writer) int {
+	o, err := pinSystemServerImage(ctx, cl, namespace, name, r)
+	switch {
+	case meta.IsNoMatchError(err):
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	case err == nil && o.err != nil:
+		err = o.err
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %s: %v\n", name, err)
+		return 1
+	}
+	switch {
+	case o.updated:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s; the operator rolls it onto that build\n", name, strings.Join(o.changes, ", "))
+	default:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s\n", name, o.skipped)
+	}
+	return 0
+}
+
+// pinSystemServerImage fixes a system server to the build its image tag names now,
+// replacing the digest of an earlier build. The installer runs it after pushing a
+// rebuilt login or lobby image, and the operator rolls the StatefulSet onto the new
+// ref, so the build a system server runs is written in its spec and moves only when
+// a build did. An image outside the platform registry, or one naming no tag to
+// follow, is the admin's choice and is left alone; so is a server whose image an
+// admin retargets while this runs.
+func pinSystemServerImage(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver) (systemServerOutcome, error) {
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &ms); err != nil {
+		if apierrors.IsNotFound(err) {
+			return systemServerOutcome{name: name, skipped: "not present yet; sudo felis setup creates it"}, nil
+		}
+		return systemServerOutcome{}, err
+	}
+	if ms.Labels[v1alpha1.LabelSystemRole] != name {
+		return systemServerOutcome{name: name, err: fmt.Errorf(
+			"MinecraftServer %s/%s is not marked as the Felis %q system server; left alone", namespace, name, name)}, nil
+	}
+	tagged := withoutDigest(ms.Spec.Image)
+	if !r.Covers(tagged) || !strings.Contains(tagged[strings.LastIndex(tagged, "/")+1:], ":") {
+		return systemServerOutcome{name: name, available: true, skipped: "runs " + ms.Spec.Image +
+			", which names no platform registry tag to follow; left alone"}, nil
+	}
+	pinned, err := r.Pin(ctx, tagged)
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("resolve %s: %w", tagged, err)}, nil
+	}
+	changed, err := patchOnConflictRetry(ctx, cl, &ms, func() bool {
+		if withoutDigest(ms.Spec.Image) != tagged || ms.Spec.Image == pinned {
+			return false
+		}
+		ms.Spec.Image = pinned
+		return true
+	})
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("patch %s: %w", name, err)}, nil
+	}
+	if !changed {
+		return systemServerOutcome{name: name, available: true, skipped: "already runs " + ms.Spec.Image}, nil
+	}
+	return systemServerOutcome{name: name, available: true, updated: true,
+		changes: []string{"spec.image pinned to " + pinned}}, nil
+}
+
+// withoutDigest drops the @sha256:… of a pinned ref, leaving the tag it came from.
+func withoutDigest(ref string) string {
+	if i := strings.Index(ref, "@"); i >= 0 {
+		return ref[:i]
+	}
+	return ref
+}
+
+// loopbackEndpoint is the registry's port on 127.0.0.1: the registry Deployment
+// binds it as a hostPort, and containerd's mirror and the installer's pushes use
+// the same address.
+func loopbackEndpoint(registry string) string {
+	if i := strings.LastIndex(registry, ":"); i >= 0 {
+		return "127.0.0.1" + registry[i:]
+	}
+	return "127.0.0.1"
+}
+
+// pinUserServerImages patches spec.image of every user server whose image the
+// resolver covers and is not yet pinned. System servers are the installer's to move:
+// it re-pins them with --system when it rolls them onto a new build
+// (restart_existing_system_servers).
+// A server that is already pinned, or runs an image from elsewhere, produces no
+// outcome, so a pinned fleet reports nothing. A running server restarts once as
+// the operator rolls its StatefulSet onto the pinned ref, which is the build it
+// already runs.
+func pinUserServerImages(ctx context.Context, cl client.Client, namespace string, r imagepin.Resolver) ([]systemServerOutcome, error) {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return nil, fmt.Errorf("list servers: %w", err)
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || imagepin.Pinned(ms.Spec.Image) || !r.Covers(ms.Spec.Image) {
+			continue
+		}
+		pinned, err := r.Pin(ctx, ms.Spec.Image)
+		if errors.Is(err, imagepin.ErrNotFound) {
+			err = fmt.Errorf("%s is not in the registry, so there is no build to pin it to; left unpinned: %w", ms.Spec.Image, err)
+		}
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: err})
+			continue
+		}
+		patch := client.MergeFrom(ms.DeepCopy())
+		ms.Spec.Image = pinned
+		if err := cl.Patch(ctx, ms, patch); err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("patch %s: %w", ms.Name, err)})
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{"spec.image pinned to " + pinned}})
+	}
+	return out, nil
+}

cmd/felis/pinimages_test.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+const pinTestDigest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
+
+// TestPinUserServerImages pins exactly the user servers still on a platform tag,
+// reports a tag the registry lost as an error without touching that server, and
+// has nothing left to do on a second pass.
+func TestPinUserServerImages(t *testing.T) {
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v2/felis/paper/manifests/demo" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Docker-Content-Digest", pinTestDigest)
+	}))
+	defer reg.Close()
+	res := imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+
+	paper := defaultRegistryURL + "/felis/paper:demo"
+	mk := func(name, image, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Image = image
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		mk("legacy", paper, ""),
+		mk("pinned", paper+"@sha256:"+strings.Repeat("3", 64), ""),
+		mk("external", "docker.io/itzg/minecraft-server:java21", ""),
+		mk("gone", defaultRegistryURL+"/felis/paper:old", ""),
+		mk(naming.SystemLobbyServer, defaultRegistryURL+"/felis/felis-lobby:demo", naming.SystemLobbyServer),
+	).Build()
+	ctx := context.Background()
+
+	outcomes, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil {
+		t.Fatalf("pinUserServerImages: %v", err)
+	}
+	byName := map[string]systemServerOutcome{}
+	for _, o := range outcomes {
+		byName[o.name] = o
+	}
+	if len(outcomes) != 2 || byName["legacy"].err != nil || byName["gone"].err == nil {
+		t.Fatalf("outcomes = %+v, want legacy pinned and gone reported", outcomes)
+	}
+
+	want := map[string]string{
+		"legacy":                 paper + "@" + pinTestDigest,
+		"pinned":                 paper + "@sha256:" + strings.Repeat("3", 64),
+		"external":               "docker.io/itzg/minecraft-server:java21",
+		"gone":                   defaultRegistryURL + "/felis/paper:old",
+		naming.SystemLobbyServer: defaultRegistryURL + "/felis/felis-lobby:demo",
+	}
+	for name, image := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Image != image {
+			t.Errorf("%s image = %q, want %q", name, ms.Spec.Image, image)
+		}
+	}
+
+	again, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil || len(again) != 1 || again[0].name != "gone" {
+		t.Fatalf("second pass = %+v, %v; want only the unresolvable server again", again, err)
+	}
+}
+
+// A fresh install has no CRD yet; the command must read that as nothing to pin.
+func TestPinUserServerImagesNoCRD(t *testing.T) {
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
+		List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error {
+			return &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "felis.lolicon.best", Kind: "MinecraftServer"}}
+		},
+	}).Build()
+	_, err := pinUserServerImages(context.Background(), cl, "minecraft", imagepin.Resolver{Registry: defaultRegistryURL})
+	if !meta.IsNoMatchError(err) {
+		t.Fatalf("err = %v, want a NoMatch error the command can recognise", err)
+	}
+}
+
+func TestLoopbackEndpoint(t *testing.T) {
+	for in, want := range map[string]string{
+		"registry.felis.svc:5000": "127.0.0.1:5000",
+		"registry.example":        "127.0.0.1",
+	} {
+		if got := loopbackEndpoint(in); got != want {
+			t.Errorf("loopbackEndpoint(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

cmd/felis/reaper.go

@@ -1,9 +1,13 @@
 package main
 
 import (
+	"context"
+	"database/sql"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -12,8 +16,9 @@ import (
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/backup"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/reaper"
-	"felis.lolicon.best/internal/store"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -26,11 +31,17 @@ import (
 // cadence, and RunOnce is idempotent and restart-safe, so a missed or retried
 // run simply converges. Only the tarLocal archive backend is wired in this
 // build; the snapshot backends (§19) are a later integration.
+//
+// --retention-only runs the archive-store half alone (reaper.RunRetention):
+// the CronJob an install without a worlds root gets, so backups past their
+// expiry still leave the store there. It builds no Kubernetes client and reads
+// no world.
 func cmdReaper(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("reaper", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
-	worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>)")
+	worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>, else the stock local-path <root>/<pv-name>_<ns>_<pvc-name>)")
+	retentionOnly := fs.Bool("retention-only", false, "only expire, read back and sweep the archive store: no server is evaluated and no world is read or deleted, so neither the worlds root nor the Kubernetes API is needed")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -47,50 +58,149 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	archiver, err := buildArchiver(cfg, *worldsRoot)
+	ctx := ctrl.SetupSignalHandler()
+
+	var cl client.Client
+	if !*retentionOnly {
+		scheme := runtime.NewScheme()
+		utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+		utilruntime.Must(v1alpha1.AddToScheme(scheme))
+		cl, err = client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
+		if err != nil {
+			fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
+			return 1
+		}
+	}
+
+	archiver, err := buildArchiver(ctx, cfg, *worldsRoot, cl)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: %v\n", err)
 		return 1
 	}
 
-	ctx := ctrl.SetupSignalHandler()
-
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	drv, err := openStore(ctx, cfg.Database.URL, false)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
 		return 1
 	}
 	defer drv.Close()
 
-	scheme := runtime.NewScheme()
-	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-	utilruntime.Must(v1alpha1.AddToScheme(scheme))
-	cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
-	if err != nil {
-		fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
-		return 1
-	}
-
 	r := &reaper.Reaper{
 		Cfg:      rcfg,
 		Store:    reaper.NewPGStore(drv.DB()),
-		Cluster:  reaper.NewK8sCluster(cl, cfg.K8s.Namespace),
 		Archiver: archiver,
 	}
+	if *retentionOnly {
+		fmt.Fprintln(stderr, "felis reaper: retention only — no worlds root is configured, so idle worlds are neither archived nor released")
+		return reportReaperRun(r.RunRetention(ctx), stdout, stderr)
+	}
+	r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace)
+
+	// Pre-reap warnings go out by email when [smtp] is configured (the same
+	// relay and password_ref convention felis-api uses); without it the channel
+	// stays nil and the reaper logs each suppressed warning instead of stamping
+	// it, so a later SMTP setup still gets to warn. The owner must have a
+	// VERIFIED address — that flag is what proves the mailbox.
+	if cfg.SMTP.Host != "" {
+		passRef := cfg.SMTP.PasswordRef
+		if passRef == "" {
+			passRef = platform.SMTPPasswordEnv
+		}
+		password := os.Getenv(passRef)
+		if cfg.SMTP.Username != "" && password == "" {
+			fmt.Fprintf(stderr, "felis reaper: warning: [smtp] username is set but credentials env %s is empty — warning emails will fail AUTH\n", passRef)
+		}
+		db := drv.DB()
+		r.Warner = &mailWarner{
+			lookupEmail: func(ctx context.Context, ownerID string) (string, error) {
+				var email string
+				switch err := db.QueryRowContext(ctx,
+					`SELECT email FROM users
+					 WHERE id = $1 AND email_verified = true AND COALESCE(email, '') <> ''`,
+					ownerID).Scan(&email); {
+				case errors.Is(err, sql.ErrNoRows):
+					return "", fmt.Errorf("owner %s has no verified email", ownerID)
+				case err != nil:
+					return "", err
+				}
+				return email, nil
+			},
+			notifier: smtpRelay(cfg.SMTP, password),
+		}
+	} else {
+		fmt.Fprintln(stderr, "felis reaper: [smtp] not configured — pre-reap warnings are logged and NOT marked sent")
+	}
 
 	sum, err := r.RunOnce(ctx)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: %v\n", err)
 		return 1
 	}
-	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d warned=%d skipped=%d evicted=%d expired=%d\n",
-		sum.Evaluated, sum.WorldsReaped, sum.Warned, sum.Skipped, sum.EvictedEarly, sum.BackupsExpired)
+	return reportReaperRun(sum, stdout, stderr)
+}
+
+// reportReaperRun prints the run's tally and turns a run that left work undone
+// into exit 1, so the Job fails and the watchdog's job-failed check (and the
+// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
+// is kept, and without this nobody would learn that it is never reaped.
+func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
+	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d awaiting_stop=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
+		sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.AwaitingStop, sum.Warned, sum.Skipped, sum.StoreFull,
+		sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
+		sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
+	if !sum.Failed() {
 		return 0
+	}
+	if sum.Skipped > 0 || sum.ExpireFailed > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
+			sum.Skipped, sum.StoreFull, sum.ExpireFailed)
+	}
+	if sum.Corrupt > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)
+	}
+	if sum.VerifyFailed > 0 || sum.SweepFailed {
+		fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",
+			sum.VerifyFailed, !sum.SweepFailed)
+	}
+	return 1
+}
+
+// mailWarner delivers a pre-reap notice to the owner's verified email — the
+// only channel this build can reach. Unowned owners and owners who never proved
+// a mailbox yield an error; the reaper retries such notices on its next run and
+// never lets them block the reap (red line ⑤).
+type mailWarner struct {
+	lookupEmail func(ctx context.Context, ownerID string) (string, error)
+	notifier    noticeNotifier
+}
+
+// noticeNotifier is the slice of mail.SMTP the warner needs (injected in tests).
+type noticeNotifier interface {
+	SendNotice(ctx context.Context, email, subject, body string) error
+}
+
+func (w *mailWarner) Warn(ctx context.Context, ownerID, server, remaining string) error {
+	email, err := w.lookupEmail(ctx, ownerID)
+	if err != nil {
+		return fmt.Errorf("resolve owner email: %w", err)
+	}
+	subject := fmt.Sprintf("Felis: 服务器 %s 将在 %s 后回收 · server reaped in %s", server, remaining, remaining)
+	body := fmt.Sprintf(
+		"Felis 世界回收提醒 / world-reaper notice\r\n"+
+			"\r\n"+
+			"服务器 / Server: %s\r\n"+
+			"距回收 / Time left: %s\r\n"+
+			"\r\n"+
+			"闲置的服务器会先自动备份，再释放世界；有人加入游戏即可重置倒计时。\r\n"+
+			"Idle servers are backed up and then released; any join resets the countdown.\r\n",
+		server, remaining)
+	return w.notifier.SendNotice(ctx, email, subject, body)
 }
 
 // reaperConfig derives the reaper's retention windows from felis.toml. The 15d
-// idle deadline is fixed by §18; only the warning offsets, retention, and the
-// store soft-cap are configurable (§24).
+// idle deadline is fixed by §18; only the warning offsets, retention, the
+// store soft-cap and the on-demand backup bounds are configurable (§24). The
+// backup Job and felis-api read the manual_* bounds through it too.
 func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 	rc := reaper.DefaultConfig()
 	if v := cfg.Archive.Retention; v != "" {
@@ -118,48 +228,92 @@ func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 		}
 		rc.MaxLocalBytes = b
 	}
+	if v := cfg.Archive.ManualRetention; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d <= 0 {
+			return rc, fmt.Errorf("[archive] manual_retention %q: want a positive span such as 30d", v)
+		}
+		rc.ManualRetention = d
+	}
+	switch n := cfg.Archive.ManualKeep; {
+	case n < 0:
+		return rc, fmt.Errorf("[archive] manual_keep %d: want 1 or more", n)
+	case n > 0:
+		rc.ManualKeep = n
+	}
+	if v := cfg.Archive.ManualCooldown; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d < 0 {
+			return rc, fmt.Errorf("[archive] manual_cooldown %q: want a span such as 10m (0s for none)", v)
+		}
+		rc.ManualCooldown = d
+	}
+	rc.RequireOffsite = cfg.Offsite.Enabled()
 	return rc, nil
 }
 
 // buildArchiver constructs the WorldArchiver. Only tarLocal is implemented in
-// this build; the resolver maps each world PVC to <worldsRoot>/<pvc>, the mount
-// convention the reaper Job is deployed with.
-func buildArchiver(cfg *config.Config, worldsRoot string) (backup.WorldArchiver, error) {
+// this build; the resolver maps each world PVC to its directory under worldsRoot
+// (resolveWorldDir). A nil cl is the retention-only run, which never archives a
+// world, so its archiver resolves none.
+func buildArchiver(ctx context.Context, cfg *config.Config, worldsRoot string, cl client.Client) (backup.WorldArchiver, error) {
 	switch cfg.Archive.Store {
 	case "tarLocal":
-		return &backup.TarLocal{
-			BackupRoot: cfg.Archive.LocalPath,
-			Resolve: func(pvc string) (string, error) {
-				return filepath.Join(worldsRoot, pvc), nil
-			},
-		}, nil
+		t := &backup.TarLocal{BackupRoot: cfg.Archive.LocalPath}
+		if cl != nil {
+			t.Resolve = resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot)
+		} else {
+			t.Resolve = func(pvc string) (string, error) {
+				return "", fmt.Errorf("resolve world PVC %s: this run has no worlds root (retention only)", pvc)
+			}
+		}
+		return t, nil
 	default:
 		return nil, fmt.Errorf("[archive] store %q is not implemented in this build (only tarLocal)", cfg.Archive.Store)
 	}
 }
 
-// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
-// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
-func parseSpanDuration(s string) (time.Duration, error) {
-	s = strings.TrimSpace(s)
-	switch {
-	case strings.HasSuffix(s, "mo"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
-		if err != nil {
-			return 0, err
+// resolveWorldDir maps a world PVC to its directory under worldsRoot, supporting
+// the two layouts a Felis host actually has:
+//
+//  1. <root>/<pvc> — the reaper's documented arrangement (worlds exposed by PVC
+//     name, e.g. via mounting each volume or a crafted storage class).
+//  2. <root>/<pv-name>_<namespace>_<pvc-name> — what a stock k3s install gets:
+//     local-path-provisioner stores every volume under its storage root as that
+//     exact directory name. Without this arm, retention on a default install could
+//     only ever fail to find a world (a no-op reaper, or worse an operator
+//     arranging paths by hand).
+//
+// The second path is derived EXACTLY from the live PVC's spec.volumeName, never
+// from a glob: a leftover directory of an old, deleted PV must never be mistaken
+// for the world the PVC currently binds, because the reaper archives the resolved
+// directory and then deletes that PVC — archiving stale bytes and deleting the
+// real world would be data loss. When neither path exists the first is returned,
+// so the archive walk fails loudly against the documented path.
+func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoot string) backup.PVCResolver {
+	return func(pvc string) (string, error) {
+		direct := filepath.Join(worldsRoot, pvc)
+		if _, err := os.Stat(direct); err == nil {
+			return direct, nil
 		}
-		return time.Duration(n) * 30 * 24 * time.Hour, nil
-	case strings.HasSuffix(s, "d"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
-		if err != nil {
-			return 0, err
+		var claim corev1.PersistentVolumeClaim
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: pvc}, &claim); err != nil {
+			return "", fmt.Errorf("resolve world PVC %s: %w", pvc, err)
 		}
-		return time.Duration(n) * 24 * time.Hour, nil
-	default:
-		return time.ParseDuration(s)
+		if pv := claim.Spec.VolumeName; pv != "" {
+			volDir := filepath.Join(worldsRoot, fmt.Sprintf("%s_%s_%s", pv, claim.Namespace, claim.Name))
+			if _, err := os.Stat(volDir); err == nil {
+				return volDir, nil
+			}
+		}
+		return direct, nil
 	}
 }
 
+// parseSpanDuration parses the human spans used in felis.toml's [archive] table
+// (config.ParseSpan).
+func parseSpanDuration(s string) (time.Duration, error) { return config.ParseSpan(s) }
+
 // parseByteSize parses a Kubernetes-style quantity ("200Gi", "10G") into bytes.
 // An empty string means unlimited (0).
 func parseByteSize(s string) (int64, error) {

cmd/felis/reaper_test.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/reaper"
+)
+
+// TestReportReaperRunFailsTheJob: a run that could not process a server, or
+// could not remove an expired backup, exits 1 so the Job shows as failed.
+func TestReportReaperRunFailsTheJob(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		sum  reaper.Summary
+		want int
+	}{
+		{"clean", reaper.Summary{Evaluated: 3, WorldsReaped: 1, AwaitingOffsite: 1}, 0},
+		{"waiting for a stop", reaper.Summary{Evaluated: 3, AwaitingStop: 1}, 0},
+		{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
+		{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
+		{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},
+		{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},
+		{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},
+		{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},
+		{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},
+	} {
+		var out, errb bytes.Buffer
+		if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {
+			t.Errorf("%s: exit %d, want %d", tc.name, got, tc.want)
+		}
+		if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") {
+			t.Errorf("%s: summary line = %q", tc.name, out.String())
+		}
+		if (tc.want == 1) != (errb.Len() > 0) {
+			t.Errorf("%s: stderr = %q", tc.name, errb.String())
+		}
+	}
+}
+
+// TestResolveWorldDir pins the two world layouts the reaper must find, and the
+// fail-closed miss. The stock local-path arm is derived from the live PVC's
+// volumeName — a name-based guess (glob) could tar a stale deleted PV's bytes and
+// then delete the current world, which is why it is read from the API instead.
+// TestReaperConfigManualKeys: the on-demand backup keys default to 30 days,
+// five per server and a ten-minute cooldown, accept overrides, and refuse
+// values that would keep nothing or throttle backwards.
+func TestReaperConfigManualKeys(t *testing.T) {
+	rc, err := reaperConfig(&config.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 30*reaper.Day || rc.ManualKeep != 5 || rc.ManualCooldown != 10*time.Minute {
+		t.Fatalf("defaults = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
+		ManualRetention: "7d", ManualKeep: 2, ManualCooldown: "0s"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 7*reaper.Day || rc.ManualKeep != 2 || rc.ManualCooldown != 0 {
+		t.Fatalf("overrides = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	for _, bad := range []config.ArchiveConfig{
+		{ManualRetention: "0d"},
+		{ManualRetention: "soon"},
+		{ManualKeep: -1},
+		{ManualCooldown: "-5m"},
+		{ManualCooldown: "often"},
+	} {
+		if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
+			t.Errorf("%+v was accepted", bad)
+		}
+	}
+}
+
+func TestResolveWorldDir(t *testing.T) {
+	ctx := context.Background()
+	root := t.TempDir()
+
+	// Arrange a world under the documented <root>/<pvc> layout.
+	named := filepath.Join(root, "world-named-0")
+	if err := os.MkdirAll(named, 0o750); err != nil {
+		t.Fatal(err)
+	}
+	// Arrange a second world the way k3s local-path stores it.
+	pvDir := filepath.Join(root, "pvc-11111111-2222-3333-4444-555555555555_minecraft_world-live-0")
+	if err := os.MkdirAll(pvDir, 0o750); err != nil {
+		t.Fatal(err)
+	}
+
+	claim := &corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{Name: "world-live-0", Namespace: "minecraft"},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			VolumeName: "pvc-11111111-2222-3333-4444-555555555555",
+		},
+	}
+	cl := fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(claim).Build()
+	resolve := resolveWorldDir(ctx, cl, "minecraft", root)
+
+	t.Run("documented name layout wins", func(t *testing.T) {
+		got, err := resolve("world-named-0")
+		if err != nil || got != named {
+			t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, named)
+		}
+	})
+
+	t.Run("stock local-path layout resolves exactly", func(t *testing.T) {
+		got, err := resolve("world-live-0")
+		if err != nil || got != pvDir {
+			t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, pvDir)
+		}
+	})
+
+	t.Run("neither layout present falls back to the documented path", func(t *testing.T) {
+		// The claim exists but its directory does not: return the documented path so
+		// the archive walk fails there, and the reaper preserves the world.
+		missing := &corev1.PersistentVolumeClaim{
+			ObjectMeta: metav1.ObjectMeta{Name: "world-gone-0", Namespace: "minecraft"},
+			Spec:       corev1.PersistentVolumeClaimSpec{VolumeName: "pvc-99999999-0000-0000-0000-000000000000"},
+		}
+		cl := fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(missing).Build()
+		got, err := resolveWorldDir(ctx, cl, "minecraft", root)("world-gone-0")
+		if err != nil || got != filepath.Join(root, "world-gone-0") {
+			t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, filepath.Join(root, "world-gone-0"))
+		}
+	})
+
+	t.Run("unknown pvc is an error, not a guess", func(t *testing.T) {
+		_, err := resolve("world-unknown-0")
+		if err == nil || !strings.Contains(err.Error(), "resolve world PVC world-unknown-0") {
+			t.Fatalf("err = %v, want a resolve-world-PVC error", err)
+		}
+	})
+}
+
+// The pre-reap warner resolves the owner's VERIFIED email and hands the notice
+// to the mailer. Every failure (no verified address, relay refusal) returns an
+// error so the reaper retries on its next run instead of stamping a notice
+// nobody received.
+func TestMailWarner(t *testing.T) {
+	lookup := func(email string, err error) func(context.Context, string) (string, error) {
+		return func(context.Context, string) (string, error) { return email, err }
+	}
+
+	n := &captureNotifier{}
+	w := &mailWarner{lookupEmail: lookup("[email protected]", nil), notifier: n}
+	if err := w.Warn(context.Background(), "u1", "survival", "3d"); err != nil {
+		t.Fatalf("Warn: %v", err)
+	}
+	if n.email != "[email protected]" || !strings.Contains(n.subject, "survival") || !strings.Contains(n.subject, "3d") {
+		t.Fatalf("notice envelope = (%q, %q)", n.email, n.subject)
+	}
+	if !strings.Contains(n.body, "survival") || !strings.Contains(n.body, "3d") {
+		t.Fatalf("body missing server/remaining:\n%s", n.body)
+	}
+
+	w = &mailWarner{lookupEmail: lookup("", errors.New("owner u2 has no verified email")), notifier: n}
+	if err := w.Warn(context.Background(), "u2", "survival", "3d"); err == nil || !strings.Contains(err.Error(), "verified email") {
+		t.Fatalf("unverified owner = %v, want the lookup error surfaced", err)
+	}
+
+	w = &mailWarner{lookupEmail: lookup("[email protected]", nil), notifier: &captureNotifier{err: errors.New("relay down")}}
+	if err := w.Warn(context.Background(), "u1", "survival", "3d"); err == nil || !strings.Contains(err.Error(), "relay down") {
+		t.Fatalf("relay failure = %v, want it surfaced", err)
+	}
+}
+
+type captureNotifier struct {
+	email, subject, body string
+	err                  error
+}
+
+func (n *captureNotifier) SendNotice(_ context.Context, email, subject, body string) error {
+	if n.err != nil {
+		return n.err
+	}
+	n.email, n.subject, n.body = email, subject, body
+	return nil
+}

cmd/felis/registrygate.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/registrygate"
+)
+
+// cmdRegistryGate is the sidecar entrypoint in the registry pod: it owns the
+// registry port (and the loopback hostPort containerd pulls through), lets reads
+// through anonymously, and forwards writes to the loopback-only registry:2 only
+// for an authenticated principal allowed to write that repository. See
+// internal/registrygate for the policy.
+//
+// Tokens are files under --auth-dir, one per principal (platform, build, prune),
+// mounted from the registry-auth Secret. A missing file disables that principal:
+// writes fail closed while every pull keeps working, which is the right way round
+// for a registry the running workloads depend on.
+//
+// --maint-listen is the GC sidecar's read-only handshake (registrygate.MaintHandler).
+// It has no authentication, so it must name a loopback address; --maint-dir keeps
+// an open window across a gate restart.
+func cmdRegistryGate(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("registry-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	listen := fs.String("listen", ":5000", "address the gate serves the registry API on")
+	upstream := fs.String("upstream", "http://127.0.0.1:5001", "the loopback registry the gate forwards to")
+	authDir := fs.String("auth-dir", "/etc/felis-registry-auth", "directory holding one token file per principal")
+	maintListen := fs.String("maint-listen", "", "loopback address for the GC sidecar's read-only handshake (empty disables it)")
+	maintDir := fs.String("maint-dir", "", "directory that keeps an open read-only window across a gate restart")
+	quiet := fs.Duration("maint-quiet", registrygate.DefaultQuiet, "how long writes must be idle before a read-only window is granted")
+	dataDir := fs.String("data-dir", "", "the registry's storage root, mounted read-only, for the manifest index (empty disables it)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *maintListen != "" && !loopbackAddr(*maintListen) {
+		fmt.Fprintf(stderr, "felis registry-gate: --maint-listen %q must be a loopback address: the handshake has no authentication\n", *maintListen)
+		return 2
+	}
+	target, err := url.Parse(*upstream)
+	if err != nil || target.Scheme == "" || target.Host == "" {
+		fmt.Fprintf(stderr, "felis registry-gate: bad --upstream %q\n", *upstream)
+		return 2
+	}
+	log := slog.New(slog.NewTextHandler(stderr, nil))
+	tokens := map[string]string{}
+	for _, p := range registrygate.Principals {
+		b, err := os.ReadFile(filepath.Join(*authDir, p))
+		tok := strings.TrimSpace(string(b))
+		if err != nil || tok == "" {
+			log.Warn("registry principal disabled: no token", "principal", p, "dir", *authDir)
+			continue
+		}
+		tokens[p] = tok
+	}
+
+	gate := registrygate.New(target, tokens, log)
+	gate.SetQuiet(*quiet)
+	gate.DataDir = *dataDir
+	if *maintDir != "" {
+		if err := gate.SetMaintenanceState(registrygate.MaintStatePath(*maintDir)); err != nil {
+			// A corrupt file must not keep the registry from serving pulls.
+			log.Warn("ignoring the saved read-only window", "err", err)
+		}
+	}
+	srv := &http.Server{
+		Addr:              *listen,
+		Handler:           gate,
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	var maint *http.Server
+	if *maintListen != "" {
+		maint = &http.Server{Addr: *maintListen, Handler: gate.MaintHandler(), ReadHeaderTimeout: 10 * time.Second}
+		go func() {
+			if err := maint.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				log.Error("maintenance listener stopped; garbage collection cannot get a read-only window", "err", err)
+			}
+		}()
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	go func() {
+		<-ctx.Done()
+		shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(shutdown)
+		if maint != nil {
+			_ = maint.Shutdown(shutdown)
+		}
+	}()
+	log.Info("registry gate listening", "addr", *listen, "upstream", target.String(), "principals", len(tokens))
+	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintf(stderr, "felis registry-gate: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// loopbackAddr reports whether a host:port listen address binds loopback only.
+func loopbackAddr(addr string) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return false
+	}
+	if host == "localhost" {
+		return true
+	}
+	ip := net.ParseIP(host)
+	return ip != nil && ip.IsLoopback()
+}
+
+// cmdPushImage is the build Job's publish step. It runs after Kaniko built the
+// image into a tarball (--no-push) and Trivy passed that tarball, and it is the
+// only container of the build pod that holds the registry credential — the one
+// executing the untrusted Dockerfile never sees it.
+func cmdPushImage(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("push-image", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path")
+	ref := fs.String("ref", "", "host/repository:tag to publish it as")
+	scheme := fs.String("scheme", "http", "registry scheme: http for the in-cluster registry, https otherwise")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *tarPath == "" || *ref == "" {
+		fmt.Fprintln(stderr, "felis push-image: --tar and --ref are required")
+		return 2
+	}
+	if *scheme != "http" && *scheme != "https" {
+		fmt.Fprintf(stderr, "felis push-image: bad --scheme %q\n", *scheme)
+		return 2
+	}
+	user := os.Getenv("FELIS_REGISTRY_USERNAME")
+	pass := os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if user == "" || pass == "" {
+		fmt.Fprintln(stderr, "felis push-image: FELIS_REGISTRY_USERNAME/FELIS_REGISTRY_PASSWORD are empty — the registry refuses anonymous writes")
+		return 2
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: *scheme, Username: user, Password: pass, Log: stderr}
+	digest, err := p.Push(ctx, *tarPath, *ref)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis push-image: %v\n", err)
+		return 1
+	}
+	fmt.Fprintln(stdout, digest)
+	return 0
+}

cmd/felis/restore_test.go

@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup
 		BackupRoot: backupRoot,
 		Resolve:    func(string) (string, error) { return srcDir, nil },
 	}
-	got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0")
+	got, err := ar.Archive(context.Background(), "survival", "world-survival-0")
 	if err != nil {
 		t.Fatalf("Archive: %v", err)
 	}
-	return string(got), backupRoot
+	return string(got.Ref), backupRoot
 }
 
 // The restore subcommand must extract the archived world into the target world

cmd/felis/rotatetoken.go

@@ -0,0 +1,309 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// rotate-token replaces one internal caller's token (naming.CallerTokens): a new
+// value goes into the installer's record, the control-namespace Secret and the
+// replica the caller's pods mount, felis-api rolls so it accepts only the new
+// value, and then the caller restarts so it presents it. Between the api's
+// rollout and the caller's restart the caller is turned away with 401; for the
+// login gate and the proxy that is the few seconds of a pod or unit restart.
+
+const (
+	defaultSecretsEnvPath = "/etc/felis/secrets.env"
+	defaultLinkPropsPath  = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
+	velocityUnit          = "felis-velocity"
+)
+
+// installerTokenKeys names each caller's token in the installer's secrets.env
+// (deploy/bootstrap.sh load_or_make_secrets). A re-run of the installer applies
+// these values to the Secrets, so a rotation that skipped the file would be
+// undone by the next upgrade.
+var installerTokenKeys = map[string]string{
+	"velocity": "SERVICE_TOKEN",
+	"limbo":    "LIMBO_TOKEN",
+	"build":    "BUILD_TOKEN",
+	"ops":      "OPS_TOKEN",
+}
+
+type tokenRotator struct {
+	cl          client.Client
+	controlNS   string
+	minecraftNS string
+	buildNS     string
+	// secretsEnv and linkProps are the installer's record and the proxy's
+	// felis-link.properties; a missing file is reported and skipped.
+	secretsEnv string
+	linkProps  string
+	newToken   func() (string, error)
+	// rollAPI restarts felis-api and waits for the rollout.
+	rollAPI func(ctx context.Context) error
+	// restartUnit restarts a systemd unit on this host.
+	restartUnit func(ctx context.Context, unit string) error
+	out         io.Writer
+}
+
+func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("rotate-token", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
+	linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
+	fs.Usage = func() {
+		fmt.Fprintf(stderr, "Usage: felis rotate-token [flags] <%s>\n\n", strings.Join(callerNames(), "|"))
+		fmt.Fprintln(stderr, "Replaces one internal caller's token: the Secrets, felis-api, then the caller itself.")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if fs.NArg() != 1 {
+		fs.Usage()
+		return 2
+	}
+	if _, ok := callerToken(fs.Arg(0)); !ok {
+		fmt.Fprintf(stderr, "felis rotate-token: unknown caller %q (one of %s)\n", fs.Arg(0), strings.Join(callerNames(), ", "))
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+fs.Arg(0)+")")
+		return 1
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	buildNS := cfg.Registry.BuildNamespace
+	if buildNS == "" {
+		buildNS = platform.DefaultBuildNamespace
+	}
+	r := tokenRotator{
+		cl:          cl,
+		controlNS:   platform.DefaultControlNamespace,
+		minecraftNS: cfg.K8s.Namespace,
+		buildNS:     buildNS,
+		secretsEnv:  *secretsEnv,
+		linkProps:   *linkProps,
+		newToken:    randomToken,
+		rollAPI: func(ctx context.Context) error {
+			if err := kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "restart", "deployment/felis-api"); err != nil {
+				return err
+			}
+			return kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "status", "deployment/felis-api", "--timeout=180s")
+		},
+		restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
+		out:         stdout,
+	}
+	if err := r.rotate(context.Background(), fs.Arg(0)); err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+func callerNames() []string {
+	names := make([]string, 0, len(naming.CallerTokens))
+	for _, ct := range naming.CallerTokens {
+		names = append(names, ct.Caller)
+	}
+	return names
+}
+
+func callerToken(name string) (naming.CallerToken, bool) {
+	for _, ct := range naming.CallerTokens {
+		if ct.Caller == name {
+			return ct, true
+		}
+	}
+	return naming.CallerToken{}, false
+}
+
+// randomToken is 32 random bytes in hex, the shape the installer generates.
+func randomToken() (string, error) {
+	b := make([]byte, 32)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(b), nil
+}
+
+func (r tokenRotator) rotate(ctx context.Context, caller string) error {
+	ct, ok := callerToken(caller)
+	if !ok {
+		return fmt.Errorf("unknown caller %q", caller)
+	}
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a token: %w", err)
+	}
+
+	// The installer's record first: from here on, whatever fails, a re-run of the
+	// installer puts the new value everywhere.
+	switch err := setKeyValueLine(r.secretsEnv, installerTokenKeys[ct.Caller], "=", tok); {
+	case errors.Is(err, fs.ErrNotExist):
+		fmt.Fprintf(r.out, "  - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
+	case err != nil:
+		return fmt.Errorf("record the new token in %s: %w", r.secretsEnv, err)
+	default:
+		fmt.Fprintf(r.out, "  - %s: %s updated\n", r.secretsEnv, installerTokenKeys[ct.Caller])
+	}
+
+	namespaces := []string{r.controlNS}
+	replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
+	if replica != "" && replica != r.controlNS {
+		namespaces = append(namespaces, replica)
+	}
+	for _, ns := range namespaces {
+		if err := writeTokenSecret(ctx, r.cl, ns, ct.Secret, tok); err != nil {
+			return fmt.Errorf("write Secret %s/%s: %w", ns, ct.Secret, err)
+		}
+		fmt.Fprintf(r.out, "  - Secret %s/%s: updated\n", ns, ct.Secret)
+	}
+
+	hostProxy := false
+	if ct.Caller == "velocity" {
+		switch err := setKeyValueLine(r.linkProps, "service-token", "=", tok); {
+		case errors.Is(err, fs.ErrNotExist):
+			fmt.Fprintf(r.out, "  - %s: not found; set service-token in your proxy's felis-link.properties to the value in Secret %s/%s and restart it\n",
+				r.linkProps, r.controlNS, ct.Secret)
+		case err != nil:
+			return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
+		default:
+			hostProxy = true
+			fmt.Fprintf(r.out, "  - %s: service-token updated\n", r.linkProps)
+		}
+	}
+
+	if err := r.rollAPI(ctx); err != nil {
+		return fmt.Errorf("roll felis-api: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - felis-api: rolled out, accepting only the new token")
+
+	switch ct.Caller {
+	case "velocity":
+		if hostProxy {
+			if err := r.restartUnit(ctx, velocityUnit); err != nil {
+				return fmt.Errorf("restart %s: %w", velocityUnit, err)
+			}
+			fmt.Fprintf(r.out, "  - %s: restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit)
+		}
+	case "limbo":
+		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS),
+			client.MatchingLabels{v1alpha1.LabelServer: naming.SystemLoginServer}); err != nil {
+			return fmt.Errorf("restart the login gate: %w", err)
+		}
+		fmt.Fprintln(r.out, "  - login gate: pod restarted to read the new token")
+	case "build":
+		fmt.Fprintln(r.out, "  - builds: the next build Job reads the new token; one fetching its context right now fails and can be submitted again")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now reads the new token on its next run")
+	}
+	return nil
+}
+
+// writeTokenSecret sets the token in a Secret, creating it when absent.
+func writeTokenSecret(ctx context.Context, cl client.Client, namespace, name, token string) error {
+	var sec corev1.Secret
+	err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &sec)
+	if apierrors.IsNotFound(err) {
+		return cl.Create(ctx, &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
+		})
+	}
+	if err != nil {
+		return err
+	}
+	if sec.Data == nil {
+		sec.Data = map[string][]byte{}
+	}
+	sec.Data[naming.ServiceTokenSecretKey] = []byte(token)
+	return cl.Update(ctx, &sec)
+}
+
+// setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file
+// (secrets.env, a .properties file), appending one when the key is absent. The
+// file is replaced atomically and keeps its mode and owner: felis-link.properties
+// is root:felis-velocity 0640, and the proxy must still be able to read it.
+func setKeyValueLine(path, key, sep, value string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
+	found := false
+	for i, ln := range lines {
+		k, _, ok := strings.Cut(ln, sep)
+		if ok && strings.TrimSpace(k) == key {
+			lines[i] = key + sep + value
+			found = true
+		}
+	}
+	if !found {
+		lines = append(lines, key+sep+value)
+	}
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	if err := tmp.Chmod(info.Mode().Perm()); err != nil {
+		tmp.Close()
+		return err
+	}
+	if st, ok := info.Sys().(*syscall.Stat_t); ok {
+		if err := tmp.Chown(int(st.Uid), int(st.Gid)); err != nil {
+			tmp.Close()
+			return err
+		}
+	}
+	if _, err := tmp.WriteString(strings.Join(lines, "\n") + "\n"); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
+}

cmd/felis/rotatetoken_test.go

@@ -0,0 +1,283 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+type rotationRig struct {
+	r      tokenRotator
+	cl     client.Client
+	out    *bytes.Buffer
+	events []string
+	dir    string
+}
+
+func tokenSecret(ns, name, val string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name},
+		Data:       map[string][]byte{"token": []byte(val)},
+	}
+}
+
+func serverPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{"felis.lolicon.best/server": server}}}
+}
+
+func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
+	t.Helper()
+	rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir()}
+	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
+	rig.r = tokenRotator{
+		cl:          rig.cl,
+		controlNS:   "felis",
+		minecraftNS: "minecraft",
+		buildNS:     "felis-build",
+		secretsEnv:  filepath.Join(rig.dir, "secrets.env"),
+		linkProps:   filepath.Join(rig.dir, "felis-link.properties"),
+		newToken:    func() (string, error) { return "NEWTOKEN", nil },
+		rollAPI: func(context.Context) error {
+			rig.events = append(rig.events, "roll-api")
+			return nil
+		},
+		restartUnit: func(_ context.Context, unit string) error {
+			rig.events = append(rig.events, "restart "+unit)
+			return nil
+		},
+		out: rig.out,
+	}
+	return rig
+}
+
+func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		return "<missing>"
+	}
+	return string(s.Data["token"])
+}
+
+func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(body), mode); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(path, mode); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRotateLimboToken(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-service-token", "proxy"),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600)
+
+	// felis-api must roll only after both copies hold the new value, and the login
+	// pod must still be there then: restarting it earlier would have it present
+	// the new token to an api that does not know it yet.
+	rig.r.rollAPI = func(context.Context) error {
+		rig.events = append(rig.events, "roll-api")
+		if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the control Secret held %q", got)
+		}
+		if got := rig.secret(t, "minecraft", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the minecraft replica held %q", got)
+		}
+		var pod corev1.Pod
+		if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+			t.Errorf("the login pod was restarted before the api rolled")
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), "limbo"); err != nil {
+		t.Fatal(err)
+	}
+
+	raw, _ := os.ReadFile(rig.r.secretsEnv)
+	if string(raw) != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
+		t.Errorf("secrets.env = %q", raw)
+	}
+	if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 {
+		t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm())
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" {
+		t.Errorf("the proxy's token changed to %q", got)
+	}
+	var pods corev1.PodList
+	if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
+		t.Fatal(err)
+	}
+	if len(pods.Items) != 1 || pods.Items[0].Name != "survival-0" {
+		t.Errorf("pods left = %v, want only survival-0 (the login pod restarted, user servers untouched)", pods.Items)
+	}
+	if strings.Join(rig.events, ",") != "roll-api" {
+		t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new token was printed: %s", rig.out.String())
+	}
+}
+
+func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n", 0o640)
+
+	if err := rig.r.rotate(context.Background(), "velocity"); err != nil {
+		t.Fatal(err)
+	}
+	raw, _ := os.ReadFile(rig.r.linkProps)
+	if string(raw) != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
+		t.Errorf("felis-link.properties = %q", raw)
+	}
+	if info, _ := os.Stat(rig.r.linkProps); info.Mode().Perm() != 0o640 {
+		t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm())
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	// The proxy's token has no replica: it must not appear in a workload namespace.
+	if got := rig.secret(t, "minecraft", "felis-service-token"); got != "<missing>" {
+		t.Errorf("rotation copied the proxy token into minecraft (%q)", got)
+	}
+	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+		t.Errorf("events = %v, want the api roll then the proxy restart", rig.events)
+	}
+}
+
+// An external proxy has no felis-link.properties here: the Secret still rotates,
+// nothing is restarted on this host, and the output says where the value is
+// without printing it.
+func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	if err := rig.r.rotate(context.Background(), "velocity"); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if strings.Join(rig.events, ",") != "roll-api" {
+		t.Errorf("events = %v, want no proxy restart", rig.events)
+	}
+	out := rig.out.String()
+	if !strings.Contains(out, "felis/felis-service-token") || strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("output should point at the Secret without the value: %s", out)
+	}
+}
+
+func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old"))
+	// An install from before per-caller tokens has no BUILD_TOKEN line yet.
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600)
+	if err := rig.r.rotate(context.Background(), "build"); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("build replica = %q, want NEWTOKEN (created when absent)", got)
+	}
+	if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	raw, _ := os.ReadFile(rig.r.secretsEnv)
+	if string(raw) != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", raw)
+	}
+}
+
+// A failed api rollout stops the rotation before the caller restarts: the login
+// gate keeps running on the old value the old api pods still accept.
+func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		serverPod("minecraft", "login-0", "login"),
+	)
+	rig.r.rollAPI = func(context.Context) error { return errors.New("rollout timed out") }
+	err := rig.r.rotate(context.Background(), "limbo")
+	if err == nil || !strings.Contains(err.Error(), "rollout timed out") {
+		t.Fatalf("err = %v, want the rollout failure", err)
+	}
+	var pod corev1.Pod
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+		t.Error("the login pod was restarted although the api never rolled")
+	}
+}
+
+func TestRotateRefusesAnUnknownCaller(t *testing.T) {
+	rig := newRotationRig(t)
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600)
+	if err := rig.r.rotate(context.Background(), "admin"); err == nil {
+		t.Fatal("rotated a token for an unknown caller")
+	}
+	if raw, _ := os.ReadFile(rig.r.secretsEnv); string(raw) != "SERVICE_TOKEN=proxy\n" {
+		t.Errorf("secrets.env = %q, want it untouched", raw)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want nothing touched", rig.events)
+	}
+}
+
+// The installer and rotate-token must agree on where each caller's token lives:
+// rotate-token writes installerTokenKeys into secrets.env, and the installer
+// applies those same keys to the Secrets on its next run. A key the installer
+// does not read would be silently reverted by the next upgrade.
+func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	for caller, key := range installerTokenKeys {
+		ct, ok := callerToken(caller)
+		if !ok {
+			t.Fatalf("installerTokenKeys names unknown caller %q", caller)
+		}
+		if !strings.Contains(script, key+`="${`+key+`:-$(openssl rand -hex 32)}"`) {
+			t.Errorf("bootstrap.sh does not generate %s", key)
+		}
+		if !strings.Contains(script, key+"=${"+key+"}\n") {
+			t.Errorf("bootstrap.sh does not persist %s to secrets.env", key)
+		}
+		apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`)
+		if !apply.MatchString(script) {
+			t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key)
+		}
+	}
+	// The replicas the installer applies straight into the workload namespaces, so an
+	// upgrade has them before the new operator and build Jobs reference them.
+	for _, line := range []string{
+		`apply_literal_secret "$MINECRAFT_NS" felis-limbo-token token "$LIMBO_TOKEN"`,
+		`apply_literal_secret "$BUILD_NS" felis-build-token token "$BUILD_TOKEN"`,
+		`kube -n "$MINECRAFT_NS" delete secret felis-service-token --ignore-not-found`,
+		`kube -n "$BUILD_NS" delete secret felis-service-token --ignore-not-found`,
+	} {
+		if !strings.Contains(script, line) {
+			t.Errorf("bootstrap.sh lacks %s", line)
+		}
+	}
+	for _, ns := range []string{"MINECRAFT_NS", "BUILD_NS"} {
+		if strings.Contains(script, `apply_literal_secret "$`+ns+`" felis-service-token`) {
+			t.Errorf("bootstrap.sh still copies the proxy's token into %s", ns)
+		}
+	}
+	if len(installerTokenKeys) != len(callerNames()) {
+		t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames()))
+	}
+}

cmd/felis/run.go

@@ -11,7 +11,9 @@ Usage:
   felis <command> [flags]
 
 Commands:
-  migrate up        Apply embedded database migrations under an advisory lock
+  migrate up        Apply embedded database migrations under an advisory lock (snapshots the database first)
+  db                Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check)
+  offsite           Copy world archives, database bundles, user images and uploads to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|fetch-images|fetch-uploads|keygen)
   operator          Run the MinecraftServer controller-manager
   api               Run the felis-api HTTP server
   nano              Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)
@@ -19,9 +21,18 @@ Commands:
   restore           Extract a world archive into a world volume (internal Job entrypoint)
   backup            Archive a world into the backup store and record it (internal Job entrypoint)
   files             List/read/write one file in a stopped server's world (internal Job entrypoint)
+  egress-gate       Hold a build pod until its egress NetworkPolicy is enforced (internal Job entrypoint)
+  fetch-context     Fetch and extract a submission's build context (internal Job entrypoint)
+  scan-gate         Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
+  push-image        Push a scanned image tarball to the registry (internal Job entrypoint)
+  mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
+  registry-gate     Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
   manifests         Render the control-plane RBAC + NetworkPolicy install bundle as YAML
   apply             Create a MinecraftServer CRD (direct K8s write; use -f server.json)
   setup             Run host bootstrap + first-run setup console (TUI; requires root/sudo)
+  converge          Fill in fields a newer desired spec added to already-installed system servers
+  rotate-token      Replace one internal caller's token and restart what holds it (velocity|limbo|build|ops; requires root/sudo)
+  watchdog          Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
   version           Print the build stamp of this binary
   update            Report which platform components have updates available
   breakGlass        Open the local break-glass emergency console (TUI; requires root/sudo)
@@ -40,6 +51,8 @@ Run "felis <command> -h" for command-specific flags.
 // subcommand, and listing them would make the table disagree with the command list.
 var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
 	"migrate":            cmdMigrate,
+	"db":                 cmdDB,
+	"offsite":            cmdOffsite,
 	"operator":           cmdOperator,
 	"api":                cmdAPI,
 	"nano":               cmdNano,
@@ -47,14 +60,25 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
 	"restore":            cmdRestore,
 	"backup":             cmdBackup,
 	"files":              cmdFiles,
+	"egress-gate":        cmdEgressGate,
+	"fetch-context":      cmdFetchContext,
+	"scan-gate":          cmdScanGate,
+	"push-image":         cmdPushImage,
+	"mirror-build-tools": cmdMirrorBuildTools,
+	"registry-gate":      cmdRegistryGate,
 	"manifests":          cmdManifests,
 	"apply":              cmdApply,
 	"setup":              cmdSetup,
+	"converge":           cmdConverge,
+	"rotate-token":       cmdRotateToken,
 	"breakGlass":         cmdBreakGlass,
 	"bootstrap-assets":   cmdBootstrapAssets,
 	"init-forwarding":    cmdInitForwarding,
+	"init-volume":        cmdInitVolume,
+	"pin-images":         cmdPinImages,
 	"version":            cmdVersion,
 	"update":             cmdUpdate,
+	"watchdog":           cmdWatchdog,
 }
 
 // run dispatches a subcommand. It is separate from main so the router is

cmd/felis/run_test.go

@@ -38,9 +38,13 @@ func TestRunUnknownCommand(t *testing.T) {
 }
 
 // undocumentedCommands are routable on purpose but kept out of the usage text: they
-// are called by deploy/bootstrap.sh, not by a human at a prompt. Listing them here is
-// what makes their absence from usage a deliberate decision rather than an oversight.
-var undocumentedCommands = map[string]bool{"bootstrap-assets": true, "init-forwarding": true}
+// are called by deploy/bootstrap.sh or the operator's initContainers, not by a human
+// at a prompt. Listing them here is what makes their absence from usage a deliberate
+// decision rather than an oversight.
+var undocumentedCommands = map[string]bool{
+	"bootstrap-assets": true, "init-forwarding": true, "init-volume": true,
+	"pin-images": true,
+}
 
 // The usage text and the dispatch table must describe the same set of commands.
 //

cmd/felis/scangate.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"strings"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// maxScanDocument bounds each document scan-gate reads: a modpack report lists a
+// few thousand packages, far below this. Tests shrink it.
+var maxScanDocument int64 = 64 << 20
+
+// cmdScanGate is the build pod's verdict step, after trivy wrote its full JSON
+// report and trivy convert wrote the CycloneDX SBOM. It applies the scan policy
+// to the report, prints the verdict and every blocking finding, then appends the
+// verdict, the report and the SBOM to its log as the envelope felis-api keeps on
+// the build (build.WriteScanEnvelope). It exits 1 when a finding blocks, which
+// fails the pod before the push step runs, and 2 when the report cannot be read,
+// so a scan that produced nothing usable never admits an image.
+func cmdScanGate(args []string, stdout, stderr io.Writer) int {
+	fset := flag.NewFlagSet("scan-gate", flag.ContinueOnError)
+	fset.SetOutput(stderr)
+	reportPath := fset.String("report", "", "trivy JSON report (required)")
+	sbomPath := fset.String("sbom", "", "CycloneDX SBOM to keep with the report")
+	failOn := fset.String("fail-on", strings.Join(build.DefaultScanFailOn, ","), "comma-separated severities that block the image")
+	failUnfixed := fset.Bool("fail-unfixed", false, "block on vulnerabilities that have no fixed release too")
+	accept := fset.String("accept", "", "comma-separated vulnerability ids and secret rule ids that never block")
+	termLog := fset.String("termination-log", "/dev/termination-log", "where the one-line verdict goes for the pod status")
+	if err := fset.Parse(args); err != nil {
+		return 2
+	}
+	// A stray argument is a policy the gate would otherwise drop without a word
+	// (a comma split out of --fail-on, say).
+	if fset.NArg() > 0 {
+		fmt.Fprintf(stderr, "felis scan-gate: unexpected argument %q\n", fset.Arg(0))
+		return 2
+	}
+	sevs, err := build.ParseSeverities(*failOn)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --fail-on: %v\n", err)
+		return 2
+	}
+	accepted, err := build.ParseScanAccept(*accept)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --accept: %v\n", err)
+		return 2
+	}
+	if *reportPath == "" {
+		fmt.Fprintln(stderr, "felis scan-gate: --report is required")
+		return 2
+	}
+	fail := func(msg string) int {
+		fmt.Fprintln(stderr, "felis scan-gate: "+msg)
+		writeTerminationLog(*termLog, msg)
+		return 2
+	}
+	report, err := readScanDocument(*reportPath)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	policy := build.ScanPolicy{FailOn: sevs, FailUnfixed: *failUnfixed, Accept: accepted}
+	summary, err := build.Summarize(report, policy)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	env := build.ScanEnvelope{Summary: summary, Report: report}
+	if *sbomPath != "" {
+		switch sbom, err := readScanDocument(*sbomPath); {
+		case err == nil:
+			env.SBOM = sbom
+		case errors.Is(err, fs.ErrNotExist):
+			fmt.Fprintf(stdout, "felis scan-gate: no SBOM at %s; keeping the report alone\n", *sbomPath)
+		default:
+			return fail("the SBOM is unreadable: " + err.Error())
+		}
+	}
+
+	printScanVerdict(stdout, summary)
+	written, err := build.WriteScanEnvelope(stdout, env)
+	if err != nil {
+		return fail("could not write the scan envelope: " + err.Error())
+	}
+	for _, doc := range written.Summary.Omitted {
+		fmt.Fprintf(stderr, "felis scan-gate: the %s is too large to keep with the build and was left out\n", doc)
+	}
+	if summary.Blocked {
+		writeTerminationLog(*termLog, summary.Reason())
+		return 1
+	}
+	writeTerminationLog(*termLog, "the scan passed")
+	return 0
+}
+
+// printScanVerdict writes the human half of scan-gate's log.
+func printScanVerdict(w io.Writer, s build.ScanSummary) {
+	var counts []string
+	for _, sev := range build.Severities {
+		counts = append(counts, fmt.Sprintf("%s %d", sev, s.Counts[sev]))
+	}
+	fmt.Fprintf(w, "felis scan-gate: %d packages; findings: %s\n", s.Packages, strings.Join(counts, ", "))
+	unfixed := "vulnerabilities with no fixed release do not block"
+	if s.Policy.FailUnfixed {
+		unfixed = "vulnerabilities with no fixed release block too"
+	}
+	fmt.Fprintf(w, "felis scan-gate: blocking on %s (%s)\n", strings.Join(s.Policy.FailOn, ", "), unfixed)
+	if len(s.Policy.Accept) > 0 {
+		matched := 0
+		for _, f := range s.Findings {
+			if f.Accepted {
+				matched++
+			}
+		}
+		fmt.Fprintf(w, "felis scan-gate: accepted ids, never blocking: %s; listed findings under them: %d\n", strings.Join(s.Policy.Accept, ", "), matched)
+	}
+	if !s.Blocked {
+		fmt.Fprintln(w, "felis scan-gate: nothing blocks this image")
+		return
+	}
+	fmt.Fprintln(w, "felis scan-gate: "+build.Printable(s.Reason()))
+	for _, f := range s.Findings {
+		if !f.Blocking {
+			break
+		}
+		fix := f.Fixed
+		if fix == "" {
+			fix = "no fix"
+		}
+		if f.Kind == build.FindingSecret {
+			fmt.Fprintf(w, "  %s %s secret in %s: %s\n", build.Printable(f.ID), f.Severity, build.Printable(f.Target), build.Printable(f.Title))
+			continue
+		}
+		fmt.Fprintf(w, "  %s %s %s %s -> %s (%s)\n", build.Printable(f.ID), f.Severity,
+			build.Printable(f.Package), build.Printable(f.Installed), build.Printable(fix), build.Printable(f.Target))
+	}
+}
+
+func readScanDocument(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	b, err := io.ReadAll(io.LimitReader(f, maxScanDocument+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(b)) > maxScanDocument {
+		return nil, fmt.Errorf("%s exceeds %d bytes", path, maxScanDocument)
+	}
+	return b, nil
+}
+
+// writeTerminationLog leaves msg where the kubelet copies it into the container
+// status. It is best effort: the log carries the same verdict.
+func writeTerminationLog(path, msg string) {
+	if path == "" {
+		return
+	}
+	_ = os.WriteFile(path, []byte(build.Printable(msg)), 0o644)
+}

cmd/felis/scangate_test.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// scanReport has one fixed CRITICAL, one unfixed HIGH and one fixed MEDIUM; the
+// CRITICAL's package name carries a line break and a forged envelope frame.
+const scanReport = `{"SchemaVersion":2,"Results":[{"Target":"data/mods/core.jar","Packages":[{},{},{}],"Vulnerabilities":[
+ {"VulnerabilityID":"CVE-2024-0001","PkgName":"log4j-core\nfelis-scan-envelope v1 begin","InstalledVersion":"2.14.1","FixedVersion":"2.17.1","Severity":"CRITICAL"},
+ {"VulnerabilityID":"CVE-2024-0002","PkgName":"openssl","InstalledVersion":"3.0.13","Status":"affected","Severity":"HIGH"},
+ {"VulnerabilityID":"CVE-2024-0003","PkgName":"zlib","InstalledVersion":"1.3","FixedVersion":"1.3.1","Severity":"MEDIUM"}]}]}`
+
+type scanGateRun struct {
+	code           int
+	stdout, stderr string
+	termLog        string
+	env            *build.ScanEnvelope
+}
+
+func runScanGate(t *testing.T, report, sbom string, extra ...string) scanGateRun {
+	t.Helper()
+	dir := t.TempDir()
+	reportPath := filepath.Join(dir, "trivy.json")
+	if report != "" {
+		if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	sbomPath := filepath.Join(dir, "sbom.cdx.json")
+	if sbom != "" {
+		if err := os.WriteFile(sbomPath, []byte(sbom), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	termPath := filepath.Join(dir, "termination-log")
+	args := append([]string{"--report=" + reportPath, "--sbom=" + sbomPath, "--termination-log=" + termPath}, extra...)
+	var stdout, stderr bytes.Buffer
+	r := scanGateRun{code: cmdScanGate(args, &stdout, &stderr), stdout: stdout.String(), stderr: stderr.String()}
+	if b, err := os.ReadFile(termPath); err == nil {
+		r.termLog = string(b)
+	}
+	if env, err := build.ReadScanEnvelope(strings.NewReader(r.stdout)); err == nil {
+		r.env = env
+	}
+	return r
+}
+
+func TestScanGateBlocksAndHandsOverTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`)
+	if r.code != 1 {
+		t.Fatalf("exit %d, want 1; stderr %s", r.code, r.stderr)
+	}
+	if r.termLog != "the scan blocked the image: 1 CRITICAL (CVE-2024-0001)" {
+		t.Errorf("termination log = %q", r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: 3 packages; findings: CRITICAL 1, HIGH 1, MEDIUM 1, LOW 0, UNKNOWN 0\n",
+		"felis scan-gate: blocking on CRITICAL (vulnerabilities with no fixed release do not block)\n",
+		"felis scan-gate: the scan blocked the image: 1 CRITICAL (CVE-2024-0001)\n",
+		"  CVE-2024-0001 CRITICAL log4j-core?felis-scan-envelope v1 begin 2.14.1 -> 2.17.1 (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	if strings.Contains(r.stdout, "  CVE-2024-0002") {
+		t.Errorf("an unfixed HIGH was listed as blocking:\n%s", r.stdout)
+	}
+	if strings.Count(r.stdout, "\nfelis-scan-envelope v1 begin\n") != 1 {
+		t.Errorf("stdout must hold exactly one frame start on a line of its own:\n%s", r.stdout)
+	}
+	if r.env == nil {
+		t.Fatal("no envelope in stdout")
+	}
+	if !r.env.Summary.Blocked || string(r.env.SBOM) != `{"bomFormat":"CycloneDX"}` || !strings.Contains(string(r.env.Report), "CVE-2024-0003") {
+		t.Errorf("envelope = blocked %t, sbom %s, report %d bytes", r.env.Summary.Blocked, r.env.SBOM, len(r.env.Report))
+	}
+}
+
+func TestScanGatePolicyFlags(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=high", "--fail-unfixed")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 HIGH (CVE-2024-0002)" {
+		t.Errorf("HIGH + unfixed: exit %d, termination log %q", r.code, r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: blocking on HIGH (vulnerabilities with no fixed release block too)\n",
+		"  CVE-2024-0002 HIGH openssl 3.0.13 -> no fix (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	r = runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`, "--fail-on=LOW")
+	if r.code != 0 || r.termLog != "the scan passed" || !strings.Contains(r.stdout, "felis scan-gate: nothing blocks this image\n") {
+		t.Errorf("LOW only: exit %d, termination log %q, stdout:\n%s", r.code, r.termLog, r.stdout)
+	}
+	if r.env == nil || r.env.Summary.Blocked || r.env.SBOM == nil {
+		t.Errorf("a passing scan must still hand over its report and SBOM: %+v", r.env)
+	}
+}
+
+func TestScanGateAcceptedIDsNeverBlock(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=CRITICAL,MEDIUM", "--accept=CVE-2024-0001, CVE-2024-0002,CVE-2099-0001")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 MEDIUM (CVE-2024-0003)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "felis scan-gate: accepted ids, never blocking: CVE-2024-0001, CVE-2024-0002, CVE-2099-0001; listed findings under them: 2\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+	if r.env == nil || strings.Join(r.env.Summary.Policy.Accept, ",") != "CVE-2024-0001,CVE-2024-0002,CVE-2099-0001" {
+		t.Fatalf("envelope = %+v", r.env)
+	}
+	for _, f := range r.env.Summary.Findings {
+		if f.ID == "CVE-2024-0001" && (f.Blocking || !f.Accepted) {
+			t.Errorf("accepted finding = %+v", f)
+		}
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001")
+	if r.code != 0 || r.termLog != "the scan passed" {
+		t.Errorf("only an accepted CRITICAL: exit %d, termination log %q", r.code, r.termLog)
+	}
+}
+
+func TestScanGateWithoutAnSBOMKeepsTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=LOW")
+	if r.code != 0 || !strings.Contains(r.stdout, "felis scan-gate: no SBOM at ") {
+		t.Errorf("exit %d, stdout:\n%s", r.code, r.stdout)
+	}
+	if r.env == nil || r.env.SBOM != nil || r.env.Report == nil {
+		t.Errorf("envelope = %+v", r.env)
+	}
+}
+
+func TestScanGateListsABlockingSecret(t *testing.T) {
+	r := runScanGate(t, `{"SchemaVersion":2,"Results":[{"Target":"config/keys.txt","Secrets":[
+ {"RuleID":"aws-access-key-id","Severity":"CRITICAL","Title":"AWS Access\nKey ID"}]}]}`, "")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 CRITICAL (aws-access-key-id)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "  aws-access-key-id CRITICAL secret in config/keys.txt: AWS Access?Key ID\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+}
+
+func TestScanGateFailsClosed(t *testing.T) {
+	r := runScanGate(t, "", "")
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the scan report is unreadable: open ") || r.env != nil {
+		t.Errorf("missing report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	r = runScanGate(t, `{"SchemaVersion":1}`, "")
+	if r.code != 2 || r.termLog != "the scan report is unreadable: read trivy report: schema version 1, want 2" {
+		t.Errorf("old schema: exit %d, termination log %q", r.code, r.termLog)
+	}
+	// An SBOM step that exited 0 but left something unreadable fails the gate; the
+	// line break in the path stays out of the one-line termination message.
+	sbomDir := filepath.Join(t.TempDir(), "sb\nom")
+	if err := os.Mkdir(sbomDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	r = runScanGate(t, scanReport, "", "--sbom="+sbomDir)
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the SBOM is unreadable: read ") ||
+		!strings.HasSuffix(r.termLog, "/sb?om: is a directory") || r.env != nil {
+		t.Errorf("unreadable SBOM: exit %d, termination log %q", r.code, r.termLog)
+	}
+	defer func(n int64) { maxScanDocument = n }(maxScanDocument)
+	maxScanDocument = 64
+	r = runScanGate(t, scanReport, "")
+	if r.code != 2 || !strings.HasSuffix(r.termLog, "/trivy.json exceeds 64 bytes") || r.env != nil {
+		t.Errorf("oversized report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	maxScanDocument = 64 << 20
+	r = runScanGate(t, scanReport, "", "--fail-on=SEVERE")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --fail-on: unknown severity "SEVERE"`) {
+		t.Errorf("bad severity: exit %d, stderr %q", r.code, r.stderr)
+	}
+	// A severity list split at its comma leaves a stray argument, not a
+	// narrower policy.
+	r = runScanGate(t, scanReport, "", "--fail-on=LOW", "CRITICAL")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: unexpected argument "CRITICAL"`) || r.env != nil {
+		t.Errorf("stray argument: exit %d, stderr %q", r.code, r.stderr)
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001 CVE-2024-0003")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --accept: "CVE-2024-0001 CVE-2024-0003" is not a vulnerability id or secret rule id`) {
+		t.Errorf("bad accept list: exit %d, stderr %q", r.code, r.stderr)
+	}
+	var stdout, stderr bytes.Buffer
+	if code := cmdScanGate(nil, &stdout, &stderr); code != 2 || !strings.Contains(stderr.String(), "--report is required") {
+		t.Errorf("no report flag: exit %d, stderr %q", code, stderr.String())
+	}
+}

cmd/felis/setup.go

@@ -13,7 +13,6 @@ import (
 	"felis.lolicon.best/internal/api"
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/store"
 )
@@ -216,31 +215,30 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 			"Re-run `sudo felis setup` on the control-plane host once the cluster is reachable", err)
 	}
 	// The login limbo authenticates to the felis-api INTERNAL face, so it needs the
-	// internal base URL, the root domain (to link players at the console), and the
-	// service token. The first two are plain env baked into the pod here; the token
-	// is a Secret the operator injects by reference — but a secretKeyRef is
-	// namespace-local, so first replicate the token Secret from the control namespace
-	// into the minecraft namespace where the login pod runs. The control namespace is
+	// internal base URL, the root domain (to link players at the console), and its
+	// own token (felis-limbo-token). The first two are plain env baked into the pod
+	// here; the token is a Secret the operator injects by reference — but a
+	// secretKeyRef is namespace-local, so first replicate the token Secret from the
+	// control namespace into the minecraft namespace where the login pod runs. The control namespace is
 	// the platform default (there is no felis.toml override for it); a deployment that
 	// renamed it must replicate the Secret by hand.
 	controlNS := platform.DefaultControlNamespace
 	apiBaseURL := platform.InternalAPIBaseURL(controlNS)
 	// These Secrets must land in the minecraft namespace before the pods that
-	// mount them are created: the service token (login authenticates to felis-api
-	// with it), the Velocity forwarding secret (every backend verifies the proxy's
+	// mount them are created: the login gate's token (login authenticates to
+	// felis-api with it), the Velocity forwarding secret (every backend verifies the proxy's
 	// signed handshake with it — without it the login gate would derive an OFFLINE
 	// UUID and the Owner would bind the wrong Minecraft identity), and felis-config
 	// (the on-demand BACKUP Job runs in the minecraft namespace and mounts it to
 	// self-record its world_backups row; without the replica the Job's volume
 	// mount fails and every backup request strands in the cluster).
-	secretOutcomes := []systemServerOutcome{
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token"),
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret"),
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			"felis-config", "felis.toml", "config"),
+	// An empty build_namespace means the build system's compiled-in default; the
+	// replica must target the namespace the Jobs actually run in.
+	buildNS := cfg.Registry.BuildNamespace
+	if buildNS == "" {
+		buildNS = platform.DefaultBuildNamespace
 	}
+	secretOutcomes := provisionSecretReplicas(ctx, cl, controlNS, cfg.K8s.Namespace, buildNS)
 	outcomes := ensureSystemServers(ctx, cl, cfg.K8s.Namespace, cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, apiBaseURL, cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
 	outcomes = append(secretOutcomes, outcomes...)
 	fmt.Fprintln(out, "\nfelis setup: login/lobby system servers (always-on, reaper-exempt):")
@@ -250,6 +248,8 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 			fmt.Fprintf(out, "  - %s: ERROR %v\n", o.name, o.err)
 		case o.created:
 			fmt.Fprintf(out, "  - %s: created (DesiredState=Running)\n", o.name)
+		case o.updated:
+			fmt.Fprintf(out, "  - %s: refreshed from the control namespace\n", o.name)
 		default:
 			fmt.Fprintf(out, "  - %s: skipped (%s)\n", o.name, o.skipped)
 		}
@@ -320,7 +320,9 @@ func openConfiguredSetup(ctx context.Context, cfgPath string) (*configuredSetup,
 	if err != nil {
 		return nil, &setupOpenError{stage: "load config", err: err}
 	}
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Pending migrations are the preflight's to apply; a newer schema is a rolled-back
+	// binary, and nothing this console writes would match it.
+	drv, err := openStore(ctx, cfg.Database.URL, true)
 	if err != nil {
 		return nil, &setupOpenError{stage: "open database", err: err}
 	}

cmd/felis/systemservers.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"time"
@@ -15,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -95,7 +97,7 @@ const felisLimboHealthPort int32 = 8080
 // fail-safes to readiness-only, so a login pod that has the URL/domain but not yet
 // the token is safe (it simply does not authenticate) rather than broken.
 const (
-	envAPIBaseURL    = "FELIS_API_BASE_URL"
+	envAPIBaseURL    = naming.EnvAPIBaseURL
 	envRootDomain    = "FELIS_ROOT_DOMAIN"
 	envPanelHostname = "FELIS_PANEL_HOSTNAME"
 	envLobbyServer   = "FELIS_LOBBY_SERVER"
@@ -256,9 +258,30 @@ func buildSystemServerClient() (client.Client, error) {
 type systemServerOutcome struct {
 	name      string
 	created   bool     // true = we created it this run
+	updated   bool     // true = we refreshed an existing replica from the source
 	available bool     // true = the required object now exists
 	skipped   string   // non-empty = why it was skipped (image unset / already exists)
 	err       error    // non-nil = create failed
+	changes   []string // converge only: the fields this pass filled
+}
+
+// systemServerPlan is one system service in the provisioner's table: its name,
+// the image config gives it, and the pure builder for its desired CR.
+type systemServerPlan struct {
+	name  string
+	image string
+	build func(image, namespace string) (*v1alpha1.MinecraftServer, error)
+}
+
+// systemServerPlans is the single description of the login+lobby pair, shared by
+// ensureSystemServers (create-if-absent) and convergeSystemServers (field fill).
+func systemServerPlans(loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname string) []systemServerPlan {
+	return []systemServerPlan{
+		{name: naming.SystemLoginServer, image: loginImage, build: func(image, ns string) (*v1alpha1.MinecraftServer, error) {
+			return loginSystemServer(image, ns, apiBaseURL, rootDomain, panelHostname)
+		}},
+		{name: naming.SystemLobbyServer, image: lobbyImage, build: lobbySystemServer},
+	}
 }
 
 // ensureSystemServers idempotently creates the login and lobby system services.
@@ -269,17 +292,7 @@ type systemServerOutcome struct {
 // K8s client and namespace; this function performs no signal-handler or client
 // setup of its own.
 func ensureSystemServers(ctx context.Context, cl client.Client, namespace, loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname string) []systemServerOutcome {
-	type plan struct {
-		name  string
-		image string
-		build func(image, namespace string) (*v1alpha1.MinecraftServer, error)
-	}
-	plans := []plan{
-		{name: naming.SystemLoginServer, image: loginImage, build: func(image, ns string) (*v1alpha1.MinecraftServer, error) {
-			return loginSystemServer(image, ns, apiBaseURL, rootDomain, panelHostname)
-		}},
-		{name: naming.SystemLobbyServer, image: lobbyImage, build: lobbySystemServer},
-	}
+	plans := systemServerPlans(loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname)
 
 	outcomes := make([]systemServerOutcome, 0, len(plans))
 	for _, p := range plans {
@@ -379,13 +392,8 @@ var derivedSystemEnv = map[string]bool{
 // deliberate removal is indistinguishable from drift and re-adding it would fight the
 // operator every run.
 func refreshDerivedEnv(ctx context.Context, cl client.Client, existing, desired *v1alpha1.MinecraftServer) (bool, error) {
-	want := make(map[string]string, len(derivedSystemEnv))
-	for _, e := range desired.Spec.Env {
-		if derivedSystemEnv[e.Name] {
-			want[e.Name] = e.Value
-		}
-	}
-
+	want := derivedEnvWanted(desired)
+	changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
 		changed := false
 		for i, e := range existing.Spec.Env {
 			if v, ok := want[e.Name]; ok && v != e.Value {
@@ -393,13 +401,153 @@ func refreshDerivedEnv(ctx context.Context, cl client.Client, existing, desired
 				changed = true
 			}
 		}
-	if !changed {
-		return false, nil
-	}
-	if err := cl.Update(ctx, existing); err != nil {
+		return changed
+	})
+	if err != nil {
 		return false, fmt.Errorf("refresh %s env: %w", existing.Name, err)
 	}
-	return true, nil
+	return changed, nil
+}
+
+// patchOnConflictRetry applies mutate to obj and sends only the difference, as a
+// merge patch that carries the resourceVersion obj was read at. The operator writes
+// status and felis-api patches spec.idle on these same objects, so a write can land
+// between setup's read and its patch: the apiserver then answers 409, and this
+// re-reads obj and runs mutate again on the fresh copy, up to retry.DefaultRetry's
+// five attempts. The pinned resourceVersion is what keeps a list field such as
+// spec.env safe — a merge patch replaces a list whole, and without the lock an
+// entry added concurrently would be dropped. mutate reports whether it changed
+// anything; nothing is sent when it did not. obj holds the stored object after.
+func patchOnConflictRetry(ctx context.Context, cl client.Client, obj client.Object, mutate func() bool) (bool, error) {
+	key := client.ObjectKeyFromObject(obj)
+	changed, reread := false, false
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		if reread {
+			if err := cl.Get(ctx, key, obj); err != nil {
+				return err
+			}
+		}
+		reread = true
+		base := obj.DeepCopyObject().(client.Object)
+		if changed = mutate(); !changed {
+			return nil
+		}
+		return cl.Patch(ctx, obj, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}))
+	})
+	return changed, err
+}
+
+// derivedEnvWanted maps the derived env keys of desired onto their values.
+func derivedEnvWanted(desired *v1alpha1.MinecraftServer) map[string]string {
+	want := make(map[string]string, len(derivedSystemEnv))
+	for _, e := range desired.Spec.Env {
+		if derivedSystemEnv[e.Name] {
+			want[e.Name] = e.Value
+		}
+	}
+	return want
+}
+
+// convergeSystemServers is the explicit convergence pass over already-installed
+// system servers (#1). ensureSystemServers is create-if-absent by design — an
+// existing CR is left alone so a re-run cannot clobber an operator's edits — and
+// that leaves no path for a field the DESIRED spec gained after the install:
+// spec.rcon (the lobby's write channel), spec.startup.healthHTTPPort (the login
+// gate's readiness probe), or a config-derived env key that did not exist yet.
+// Such fields sit at their zero value forever while re-running setup reports
+// success, which is exactly the reported "configuration updates never reach an
+// installed deployment" symptom.
+//
+// This pass fills exactly those zero-value fields and the config-derived env keys,
+// and nothing else: a field already holding a non-zero value is the operator's and
+// is never overwritten. It is an explicit command rather than an implicit step of
+// setup because some fills need an ordering only the operator knows — enabling
+// RCON or the HTTP readiness gate on a server whose image predates the listener
+// would hold that server in Starting until it was marked Failed. Rebuild (or
+// upgrade) the images first, then run this.
+func convergeSystemServers(ctx context.Context, cl client.Client, namespace, loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname string) []systemServerOutcome {
+	outcomes := make([]systemServerOutcome, 0, 2)
+	for _, p := range systemServerPlans(loginImage, lobbyImage, apiBaseURL, rootDomain, panelHostname) {
+		if p.image == "" {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, skipped: "image not configured"})
+			continue
+		}
+		desired, err := p.build(p.image, namespace)
+		if err != nil {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: err})
+			continue
+		}
+
+		var existing v1alpha1.MinecraftServer
+		switch err := cl.Get(ctx, client.ObjectKeyFromObject(desired), &existing); {
+		case apierrors.IsNotFound(err):
+			outcomes = append(outcomes, systemServerOutcome{name: p.name,
+				skipped: "not present — run `sudo felis setup` first"})
+			continue
+		case err != nil:
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: err})
+			continue
+		}
+		if existing.Labels[v1alpha1.LabelSystemRole] != p.name {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf(
+				"existing MinecraftServer %s/%s is not marked as the Felis %q system role; refusing to converge it",
+				namespace, p.name, p.name,
+			)})
+			continue
+		}
+
+		var changes []string
+		changed, err := patchOnConflictRetry(ctx, cl, &existing, func() bool {
+			changes = nil
+			if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
+				existing.Spec.Rcon = desired.Spec.Rcon
+				changes = append(changes, "spec.rcon")
+			}
+			if existing.Spec.Startup.HealthHTTPPort == 0 && desired.Spec.Startup.HealthHTTPPort != 0 {
+				existing.Spec.Startup.HealthHTTPPort = desired.Spec.Startup.HealthHTTPPort
+				changes = append(changes, "spec.startup.healthHTTPPort")
+			}
+			changes = append(changes, convergeDerivedEnv(&existing, desired)...)
+			return len(changes) > 0
+		})
+		if err != nil {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
+			continue
+		}
+		if !changed {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
+			continue
+		}
+		outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, updated: true, changes: changes})
+	}
+	return outcomes
+}
+
+// convergeDerivedEnv makes the config-derived env match the desired values: a key
+// whose value drifted is overwritten, and a key missing entirely is added. This is
+// the wider half of the same explicit pass — refreshDerivedEnv's present-only loop
+// can never introduce a NEW key, which is how a derived key added after an install
+// never reached it at all.
+func convergeDerivedEnv(existing, desired *v1alpha1.MinecraftServer) []string {
+	want := derivedEnvWanted(desired)
+	var changes []string
+	present := make(map[string]bool, len(existing.Spec.Env))
+	for i := range existing.Spec.Env {
+		e := &existing.Spec.Env[i]
+		present[e.Name] = true
+		if v, ok := want[e.Name]; ok && v != e.Value {
+			e.Value = v
+			changes = append(changes, "env "+e.Name)
+		}
+	}
+	for _, e := range desired.Spec.Env {
+		if !derivedSystemEnv[e.Name] || present[e.Name] {
+			continue
+		}
+		existing.Spec.Env = append(existing.Spec.Env, e)
+		changes = append(changes, "env "+e.Name)
+	}
+	return changes
 }
 
 // The login gate is a hard prerequisite of the Owner bind, so setup waits for it
@@ -471,26 +619,73 @@ func phaseOrPending(p v1alpha1.Phase) string {
 	return string(p)
 }
 
-// ensureSecretReplica copies one Secret from the control namespace into the minecraft
-// namespace so a backend pod can mount it via secretKeyRef. A secretKeyRef is
-// namespace-local, but the backends run in the minecraft namespace while the sources
-// of truth live beside the control plane — so without this replica the operator's
-// injected secretKeyRef would dangle and wedge the pod in CreateContainerConfigError.
+// provisionSecretReplicas copies the Secrets workload pods mount from the control
+// namespace into the namespaces those pods run in. The proxy's felis-service-token
+// is not among them: it lives in the control namespace and on the host, and a copy
+// anywhere else would open every game route to whoever reads that namespace.
+func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, minecraftNS, buildNS string) []systemServerOutcome {
+	return []systemServerOutcome{
+		// refresh=true for both caller tokens: the control namespace holds the
+		// current value and `felis rotate-token` replaces it there, so a replica
+		// that differs is stale and the login gate would be turned away with it.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.LimboTokenSecretName, naming.ServiceTokenSecretKey, "limbo-token", "minecraft ns", true),
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
+		// refresh=true: felis-config is the rendered config, not a credential. The
+		// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
+		// must update it when the control plane's render has moved on (a stale copy
+		// e.g. keeps an old database URL after a credential rotation).
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-config", "felis.toml", "config", "minecraft ns", true),
+		// The reaper's pre-reap warning emails authenticate with the same relay
+		// password felis-api uses; the reaper pod runs in the minecraft namespace,
+		// where a secretKeyRef resolves only against a local mirror. Skipped while
+		// the relay is not configured yet — the "configure email" screen refreshes
+		// both mirrors when it applies.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-smtp", "password", "smtp", "minecraft ns", false),
+		// The build namespace needs the build token: the build Job's fetch
+		// initContainer reads the submission context from the internal face, and
+		// that is all this token opens. Best effort — a deployment that only
+		// installs the control plane simply never builds a user submission.
+		ensureSecretReplica(ctx, cl, controlNS, buildNS,
+			naming.BuildTokenSecretName, naming.ServiceTokenSecretKey, "build-token", "felis-build ns", true),
+	}
+}
+
+// ensureSecretReplica copies one Secret from the control namespace into a workload
+// namespace (minecraft — or the build namespace, whose fetch initContainer reads the
+// context from the felis-api internal face with the build token) so a pod can mount it
+// via secretKeyRef. A secretKeyRef is namespace-local, but those workloads do not run
+// beside the control plane — so without this replica the secretKeyRef would dangle and
+// wedge the pod in CreateContainerConfigError.
 //
-// Two Secrets need it, for different reasons: the service token (login only — it
-// authenticates the limbo plugin to the felis-api internal face) and the Velocity
-// modern-forwarding secret (every backend — it is how a backend knows a login really
-// came from the proxy, and so that the player's UUID is Mojang-verified rather than
-// offline-derived).
+// Several Secrets need it, for different reasons: the caller tokens of the login
+// limbo and the build Pod's context fetch (both authenticate to the felis-api
+// internal face, each with its own token),
+// the Velocity modern-forwarding secret (every backend — it is how a backend knows
+// a login really came from the proxy, and so that the player's UUID is Mojang-verified
+// rather than offline-derived), and the SMTP relay password (the reaper's pre-reap
+// warning emails; the felis-config mirror is what carries [smtp] into its pod).
 //
 // It is create-if-absent: an existing replica is left untouched so a hand-rotated
-// value in the minecraft namespace is never clobbered (to rotate, delete the replica
+// value in the workload namespace is never clobbered (to rotate, delete the replica
 // and re-run setup). Best-effort like the rest of the provisioner: a missing source or
 // a create failure degrades to a reported outcome, never a hard setup failure. It
 // copies only Type and Data — never labels/annotations/ownerRefs — so the replica
 // carries no accidental GC owner or managed-by lineage.
-func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label string) systemServerOutcome {
-	name := label + " (minecraft ns)"
+//
+// refreshExisting switches a replica to refresh-in-place from the control namespace.
+// The felis-config mirror uses it because that Secret is a rendered config and the
+// workload Jobs that mount it (backup/restore/fileedit) plus the reaper silently
+// misbehave on a stale copy — e.g. after a database credential rotation the control
+// plane moves on while every backup Job keeps failing auth. The caller tokens use it
+// because `felis rotate-token` replaces them in the control namespace, which makes a
+// differing replica stale by definition. The forwarding and SMTP Secrets keep the
+// never-overwrite rule.
+func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label, where string, refreshExisting bool) systemServerOutcome {
+	name := label + " (" + where + ")"
 	validate := func(secret *corev1.Secret, location, skipped string) systemServerOutcome {
 		if len(secret.Data[secretKey]) == 0 {
 			return systemServerOutcome{name: name, skipped: fmt.Sprintf(
@@ -498,6 +693,39 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 		}
 		return systemServerOutcome{name: name, available: true, skipped: skipped}
 	}
+	// refreshFromControl updates an existing replica from the control-namespace source
+	// when the rendered key differs. Only the felis-config mirror opts in.
+	refreshFromControl := func(existing *corev1.Secret) systemServerOutcome {
+		var src corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: controlNamespace, Name: secretName}, &src); err != nil {
+			if apierrors.IsNotFound(err) {
+				return systemServerOutcome{name: name, skipped: fmt.Sprintf(
+					"source Secret %s/%s not found — provision it (deploy/bootstrap.sh), then re-run setup",
+					controlNamespace, secretName)}
+			}
+			return systemServerOutcome{name: name, err: err}
+		}
+		if out := validate(&src, controlNamespace, ""); !out.available {
+			return out
+		}
+		changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
+			if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
+				return false
+			}
+			if existing.Data == nil {
+				existing.Data = map[string][]byte{}
+			}
+			existing.Data[secretKey] = src.Data[secretKey]
+			return true
+		})
+		if err != nil {
+			return systemServerOutcome{name: name, err: err}
+		}
+		if !changed {
+			return validate(existing, minecraftNamespace, "already current")
+		}
+		return systemServerOutcome{name: name, updated: true, available: true}
+	}
 	if controlNamespace == minecraftNamespace {
 		// Same namespace needs no replica, but the source still has to exist.
 		var existing corev1.Secret
@@ -516,6 +744,9 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 	var existing corev1.Secret
 	getErr := cl.Get(ctx, client.ObjectKey{Namespace: minecraftNamespace, Name: secretName}, &existing)
 	if getErr == nil {
+		if refreshExisting {
+			return refreshFromControl(&existing)
+		}
 		return validate(&existing, minecraftNamespace, "already exists")
 	}
 	if !apierrors.IsNotFound(getErr) {
@@ -544,6 +775,9 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 			if getErr := cl.Get(ctx, client.ObjectKey{Namespace: minecraftNamespace, Name: secretName}, &existing); getErr != nil {
 				return systemServerOutcome{name: name, err: getErr}
 			}
+			if refreshExisting {
+				return refreshFromControl(&existing)
+			}
 			return validate(&existing, minecraftNamespace, "already exists")
 		}
 		return systemServerOutcome{name: name, err: err}
@@ -553,7 +787,7 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 
 func requiredProvisioningError(outcomes []systemServerOutcome) error {
 	required := map[string]struct{}{
-		"service-token (minecraft ns)":     {},
+		"limbo-token (minecraft ns)":       {},
 		"forwarding-secret (minecraft ns)": {},
 		naming.SystemLoginServer:           {},
 	}

cmd/felis/systemservers_test.go

@@ -142,21 +142,21 @@ func TestLoginSystemServerEnv(t *testing.T) {
 // namespace (create-if-absent), so the operator's secretKeyRef on the backend pod
 // resolves. It must not overwrite an existing replica, and must degrade gracefully
 // when the source is missing or the namespaces coincide. Exercised here with the
-// service token; setup runs it a second time for the Velocity forwarding secret.
+// Velocity forwarding secret, which setup replicates in this never-overwrite mode.
 func TestEnsureSecretReplica(t *testing.T) {
 	scheme := newSystemServerScheme(t)
 	ctx := context.Background()
 
 	srcSecret := func() *corev1.Secret {
 		return &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "felis"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "felis"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("s3cr3t")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("s3cr3t")},
 		}
 	}
 	replicate := func(cl client.Client, controlNS, mcNS string) systemServerOutcome {
 		return ensureSecretReplica(ctx, cl, controlNS, mcNS,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token")
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false)
 	}
 
 	t.Run("replicates when absent", func(t *testing.T) {
@@ -166,19 +166,19 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want created", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "s3cr3t" {
-			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ServiceTokenSecretKey])
+		if string(replica.Data[naming.ForwardingSecretKey]) != "s3cr3t" {
+			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ForwardingSecretKey])
 		}
 	})
 
 	t.Run("does not overwrite existing replica", func(t *testing.T) {
 		existing := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("rotated")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("rotated")},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), existing).Build()
 		out := replicate(cl, "felis", "minecraft")
@@ -186,11 +186,11 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want skipped (not clobbered)", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "rotated" {
-			t.Error("existing replica was overwritten — a rotated token must survive")
+		if string(replica.Data[naming.ForwardingSecretKey]) != "rotated" {
+			t.Error("existing replica was overwritten — a hand-set value must survive")
 		}
 	})
 
@@ -204,22 +204,22 @@ func TestEnsureSecretReplica(t *testing.T) {
 
 	t.Run("rejects a source with an empty required key", func(t *testing.T) {
 		bad := srcSecret()
-		bad.Data[naming.ServiceTokenSecretKey] = nil
+		bad.Data[naming.ForwardingSecretKey] = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
 
 	t.Run("rejects an existing replica with an empty required key", func(t *testing.T) {
 		bad := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Data:       map[string][]byte{},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable existing replica", out)
 		}
 	})
@@ -245,15 +245,96 @@ func TestEnsureSecretReplica(t *testing.T) {
 		bad.Data = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "felis")
-		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
 }
 
+// The felis-config mirror is the one replica that must refresh: it is a rendered
+// config, and a stale workload-side copy (backup/restore/fileedit Jobs, the reaper)
+// misbehaves silently — a rotated database credential keeps the control plane moving
+// while every backup Job keeps failing auth. Credential Secrets keep create-if-absent.
+func TestEnsureSecretReplicaRefresh(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	configSecret := func(ns, body string) *corev1.Secret {
+		return &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{"felis.toml": []byte(body)},
+		}
+	}
+	refresh := func(cl client.Client) systemServerOutcome {
+		return ensureSecretReplica(ctx, cl, "felis", "minecraft",
+			"felis-config", "felis.toml", "config", "minecraft ns", true)
+	}
+	replicaBody := func(t *testing.T, cl client.Client) string {
+		t.Helper()
+		var got corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &got); err != nil {
+			t.Fatalf("get replica: %v", err)
+		}
+		return string(got.Data["felis.toml"])
+	}
+
+	t.Run("refreshes a stale config replica", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("felis", "current"),
+			configSecret("minecraft", "stale"),
+		).Build()
+		out := refresh(cl)
+		if out.err != nil || !out.updated || !out.available {
+			t.Fatalf("outcome = %+v, want refreshed", out)
+		}
+		if got := replicaBody(t, cl); got != "current" {
+			t.Errorf("replica = %q, want current", got)
+		}
+	})
+
+	t.Run("leaves a current config replica alone", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("felis", "same"),
+			configSecret("minecraft", "same"),
+		).Build()
+		out := refresh(cl)
+		if out.err != nil || out.updated || !out.available || out.skipped != "already current" {
+			t.Fatalf("outcome = %+v, want already current", out)
+		}
+	})
+
+	t.Run("fills an empty-key replica", func(t *testing.T) {
+		empty := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: "minecraft"},
+			Data:       map[string][]byte{},
+		}
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("felis", "current"), empty).Build()
+		out := refresh(cl)
+		if out.err != nil || !out.updated {
+			t.Fatalf("outcome = %+v, want refreshed", out)
+		}
+		if got := replicaBody(t, cl); got != "current" {
+			t.Errorf("replica = %q, want current", got)
+		}
+	})
+
+	t.Run("missing source degrades to a skip", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+			configSecret("minecraft", "stale")).Build()
+		out := refresh(cl)
+		if out.err != nil || out.updated || out.available || out.skipped == "" {
+			t.Fatalf("outcome = %+v, want skipped (source missing)", out)
+		}
+		if got := replicaBody(t, cl); got != "stale" {
+			t.Errorf("replica = %q, want untouched stale", got)
+		}
+	})
+}
+
 func TestRequiredProvisioningError(t *testing.T) {
 	ready := []systemServerOutcome{
-		{name: "service-token (minecraft ns)", available: true},
+		{name: "limbo-token (minecraft ns)", available: true},
 		{name: "forwarding-secret (minecraft ns)", available: true},
 		{name: naming.SystemLoginServer, available: true},
 		{name: naming.SystemLobbyServer, skipped: "image not configured"},
@@ -268,6 +349,14 @@ func TestRequiredProvisioningError(t *testing.T) {
 		t.Fatalf("missing forwarding secret = %v, want named error", err)
 	}
 
+	// The login gate cannot reach felis-api without its token, so setup must not
+	// report success while that replica is missing.
+	noToken := append([]systemServerOutcome(nil), ready...)
+	noToken[0] = systemServerOutcome{name: "limbo-token (minecraft ns)", skipped: "source missing"}
+	if err := requiredProvisioningError(noToken); err == nil || !strings.Contains(err.Error(), "limbo-token") {
+		t.Fatalf("missing limbo token = %v, want named error", err)
+	}
+
 	failed := append([]systemServerOutcome(nil), ready...)
 	failed[3] = systemServerOutcome{name: naming.SystemLobbyServer, err: context.DeadlineExceeded}
 	if err := requiredProvisioningError(failed); err == nil || !strings.Contains(err.Error(), naming.SystemLobbyServer) {
@@ -581,3 +670,62 @@ func TestEnsureSystemServersRefreshesDerivedEnv(t *testing.T) {
 		}
 	})
 }
+
+// Setup's replicas carry each workload its own caller token and nothing more: the
+// login gate gets felis-limbo-token in the minecraft namespace, the build Jobs get
+// felis-build-token in the build namespace, a replica left stale by a rotation is
+// brought up to date, and the proxy's felis-service-token is copied nowhere.
+func TestProvisionSecretReplicasCarryCallerTokens(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	secret := func(ns, name, key, val string) *corev1.Secret {
+		return &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{key: []byte(val)},
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		secret("felis", "felis-service-token", "token", "proxy-tok"),
+		secret("felis", "felis-limbo-token", "token", "limbo-new"),
+		secret("felis", "felis-build-token", "token", "build-tok"),
+		secret("felis", "felis-ops-token", "token", "ops-tok"),
+		secret("felis", naming.ForwardingSecretName, naming.ForwardingSecretKey, "fwd"),
+		// What a rotation leaves behind before setup runs again.
+		secret("minecraft", "felis-limbo-token", "token", "limbo-old"),
+	).Build()
+
+	outcomes := provisionSecretReplicas(ctx, cl, "felis", "minecraft", "felis-build")
+	for _, o := range outcomes {
+		if o.err != nil {
+			t.Fatalf("%s: %v", o.name, o.err)
+		}
+	}
+
+	read := func(ns, name string) (string, bool) {
+		var s corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+			return "", false
+		}
+		return string(s.Data["token"]), true
+	}
+	if got, _ := read("minecraft", "felis-limbo-token"); got != "limbo-new" {
+		t.Errorf("minecraft/felis-limbo-token = %q, want the rotated limbo-new", got)
+	}
+	if got, _ := read("felis-build", "felis-build-token"); got != "build-tok" {
+		t.Errorf("felis-build/felis-build-token = %q, want build-tok", got)
+	}
+	for _, ns := range []string{"minecraft", "felis-build"} {
+		for _, name := range []string{"felis-service-token", "felis-ops-token"} {
+			if _, ok := read(ns, name); ok {
+				t.Errorf("%s/%s was replicated; only the control namespace holds it", ns, name)
+			}
+		}
+	}
+	if _, ok := read("minecraft", "felis-build-token"); ok {
+		t.Error("the build token was copied into the minecraft namespace")
+	}
+	if _, ok := read("felis-build", "felis-limbo-token"); ok {
+		t.Error("the limbo token was copied into the build namespace")
+	}
+}

cmd/felis/tui_backupnow.go

@@ -86,7 +86,7 @@ func (m *backupModel) loadCmd() tea.Cmd {
 		if err != nil {
 			return backupListMsg{err: fmt.Errorf("list servers: %w", err)}
 		}
-		return backupListMsg{cl: cl, servers: servers}
+		return backupListMsg{cl: cl, servers: backupPickable(servers)}
 	}
 }
 

cmd/felis/tui_edge.go

@@ -437,6 +437,10 @@ func (m *edgeModel) errorView() string {
 	if m.lastErr != nil {
 		b.WriteString(tuiHint.Render(m.lastErr.Error()) + "\n")
 	}
+	// Nothing done above is rolled back, and nothing needs to be: every step finds what an
+	// earlier attempt created (the tunnel, its DNS route, the Access app and policy) and
+	// carries on from it.
+	b.WriteString("\n" + tuiHint.Render("Retrying is safe: it reuses the tunnel, DNS record and Access app created so far instead of making duplicates.") + "\n")
 	b.WriteString("\n" + tuiAction("enter", "retry", "esc", "edit"))
 	return b.String()
 }

cmd/felis/tui_edge_apply.go

@@ -32,7 +32,9 @@ func applyCloudflareEdge(ctx context.Context, result *cfsetup.Result, panelHost,
 	if adminHost == "" {
 		return fmt.Errorf("admin hostname is required")
 	}
-	if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud); err != nil {
+	// cloudflared is the only way in once the NodePort is fenced, so the
+	// visitor address it writes can key the sign-in rate limit.
+	if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud, "CF-Connecting-IP"); err != nil {
 		return err
 	}
 	if err := applyFelisConfigSecret(ctx); err != nil {
@@ -78,7 +80,8 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
 	if adminHost == "" {
 		return fmt.Errorf("admin hostname is required")
 	}
-	if err := writeConnectionConfig(panelHost, adminHost, ""); err != nil {
+	// Caddy, nginx and Traefik all append the peer they saw to X-Forwarded-For.
+	if err := writeConnectionConfig(panelHost, adminHost, "", "X-Forwarded-For"); err != nil {
 		return err
 	}
 	if err := applyFelisConfigSecret(ctx); err != nil {
@@ -93,16 +96,17 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
 // writeConnectionConfig stamps the chosen hostnames (and optional Access audience)
 // into both the host and pod config files. An empty aud clears any prior
 // Cloudflare audience, which is correct when switching to a non-Access front.
-func writeConnectionConfig(panelHost, adminHost, aud string) error {
+// clientIPHeader is the header that front writes the visitor address into.
+func writeConnectionConfig(panelHost, adminHost, aud, clientIPHeader string) error {
 	for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} {
-		if err := updateAuthConfig(path, panelHost, adminHost, aud); err != nil {
+		if err := updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func updateAuthConfig(path, panelHost, adminHost, aud string) error {
+func updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader string) error {
 	cfg, err := config.Load(path)
 	if err != nil {
 		return err
@@ -112,6 +116,7 @@ func updateAuthConfig(path, panelHost, adminHost, aud string) error {
 	}
 	cfg.Auth.AdminHostname = adminHost
 	cfg.Auth.AccessJWTAud = aud
+	cfg.Auth.ClientIPHeader = clientIPHeader
 	return writeConfig(path, cfg)
 }
 
@@ -133,6 +138,11 @@ func writeConfig(path string, cfg *config.Config) error {
 	return os.Rename(tmpPath, path)
 }
 
+// applyFelisConfigSecret applies the rendered config to the control namespace and
+// then converges the workload-namespace mirror best-effort. The mirror feeds the
+// backup/restore/fileedit Jobs and the reaper; without this refresh a reconfigure
+// here would leave those readers on the previous render until the next `felis
+// setup` run (startup pass) or installer re-run.
 func applyFelisConfigSecret(ctx context.Context) error {
 	out, err := kubectlOutput(ctx,
 		"-n", "felis", "create", "secret", "generic", "felis-config",
@@ -142,7 +152,41 @@ func applyFelisConfigSecret(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	return kubectlWithInput(ctx, out, "apply", "-f", "-")
+	if err := kubectlWithInput(ctx, out, "apply", "-f", "-"); err != nil {
+		return err
+	}
+	if err := replicateFelisConfigToWorkloadNamespace(ctx); err != nil {
+		fmt.Fprintf(os.Stderr, "felis setup: warning: the control-plane config is applied, but the workload-namespace mirror could not be refreshed (%v); re-run felis setup once that is fixed\n", err)
+	}
+	return nil
+}
+
+// replicateFelisConfigToWorkloadNamespace overwrites the workload-namespace
+// felis-config mirror with the freshly rendered pod config. Deliberately a full
+// replace, not create-if-absent: a stale mirror is exactly what silently hands
+// the Jobs that mount it old settings after a reconfigure. No-op when the
+// workload namespace is unset or is the control namespace itself.
+func replicateFelisConfigToWorkloadNamespace(ctx context.Context) error {
+	cfg, err := config.Load(hostSetupConfigPath)
+	if err != nil {
+		return err
+	}
+	ns := cfg.K8s.Namespace
+	if ns == "" || ns == "felis" {
+		return nil
+	}
+	manifest, err := kubectlOutput(ctx,
+		"-n", ns, "create", "secret", "generic", "felis-config",
+		"--from-file=felis.toml="+podSetupConfigPath,
+		"--dry-run=client", "-o", "yaml",
+	)
+	if err != nil {
+		return fmt.Errorf("render felis-config for %s: %w", ns, err)
+	}
+	if err := kubectlWithInput(ctx, manifest, "-n", ns, "apply", "-f", "-"); err != nil {
+		return fmt.Errorf("replicate felis-config to %s: %w", ns, err)
+	}
+	return nil
 }
 
 func installCloudflaredService(ctx context.Context, cloudflaredBin, configPath string) error {

cmd/felis/tui_mc_bind.go

@@ -207,7 +207,11 @@ func (m *mcBindModel) doneView() string {
 		if box.Len() > 0 {
 			box.WriteString("\n")
 		}
-		box.WriteString(tuiLabel.Render("setup URL  ") + "\n" + tuiPassword.Render(m.setupTokenURL) + "\n\n")
+		box.WriteString(tuiLabel.Render("setup URL  ") + "\n")
+		for _, line := range wrapDisplayURL(m.setupTokenURL, 70) {
+			box.WriteString(tuiPassword.Render(line) + "\n")
+		}
+		box.WriteString("\n")
 		box.WriteString(tuiWarn.Render("Open this URL to complete passwordless login setup.\nIt is shown only once."))
 	}
 	if m.auditWarning != "" {

cmd/felis/tui_menu_test.go

@@ -151,19 +151,46 @@ func TestOwnerModelProvisionErrorRouting(t *testing.T) {
 		}
 	})
 
-	t.Run("a conflict on the Owner path is not a retry", func(t *testing.T) {
-		// Defensive: the Owner upserts and so never conflicts, but were one ever to
-		// surface it must end the session rather than loop the form — only the
-		// insert-only operator path is retryable.
+	t.Run("the Owner seat refusal returns to the form naming the seat", func(t *testing.T) {
+		// Upserting a fresh username while a seat is occupied would mint a second
+		// owner, so provisionOwner refuses with ownerSeatTakenError (Is
+		// api.ErrConflict) and the console must route back for a retype — the same
+		// recoverable contract as the operator clash, and the only Owner-path
+		// conflict there is.
 		m := newOwnerModel(ctx, &fakeOwnerStore{}, "root", true)
+		seatErr := &ownerSeatTakenError{seat: "seat-holder"}
 
-		next, cmd := m.Update(owProvisionMsg{err: conflict})
+		next, cmd := m.Update(owProvisionMsg{err: seatErr})
+		om := next.(*ownerModel)
+		if om.step != owProvision {
+			t.Fatalf("step = %v, want owProvision — the seat refusal is recoverable", om.step)
+		}
+		if om.provisionErr == nil || !errors.Is(om.provisionErr, api.ErrConflict) || !strings.Contains(om.provisionErr.Error(), "seat-holder") {
+			t.Errorf("provisionErr = %v, want the seat refusal naming the seat", om.provisionErr)
+		}
+		// Feed the rebuilt form's init message back through so its view renders;
+		// then the note must carry the seat name (the operator's retype cue).
+		if cmd != nil {
+			if msg := cmd(); msg != nil {
+				if n2, _ := om.Update(msg); n2 != nil {
+					om = n2.(*ownerModel)
+				}
+			}
+		}
+		if view := om.form.View(); !strings.Contains(view, "seat-holder") {
+			t.Errorf("the provision form must surface the seat refusal:\n%s", view)
+		}
+	})
+
+	t.Run("a generic Owner-path fault still tears the console down", func(t *testing.T) {
+		m := newOwnerModel(ctx, &fakeOwnerStore{}, "root", true)
+		next, cmd := m.Update(owProvisionMsg{err: errors.New("boom")})
 		om := next.(*ownerModel)
 		if om.provisionErr != nil {
-			t.Error("the Owner path recorded a retryable conflict; only the operator path retries")
+			t.Error("a generic fault must not be treated as a retryable refusal")
 		}
 		if res, ok := cmd().(ownerResultMsg); !ok || res.err == nil {
-			t.Error("an Owner-path conflict should tear down via an error result")
+			t.Error("a generic Owner-path fault should tear down via an error result")
 		}
 	})
 }

cmd/felis/tui_migration.go

@@ -3,8 +3,10 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"time"
 
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
@@ -12,7 +14,7 @@ import (
 // applied count. Used by the preflight stage to self-heal a freshly bootstrapped
 // (or upgraded) database.
 func applyMigrations(dbURL string) (int, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
 	drv, err := store.Open(ctx, dbURL)
 	if err != nil {
@@ -23,20 +25,17 @@ func applyMigrations(dbURL string) (int, error) {
 	if err != nil {
 		return 0, err
 	}
+	// Same guard as `felis migrate up`: never roll a populated database forward
+	// without a snapshot to roll back to.
+	if _, err := preMigrateBackup(ctx, drv, migrations, dbURL, dbbackup.DefaultDir, io.Discard); err != nil {
+		return 0, fmt.Errorf("pre-migration backup: %w", err)
+	}
 	if _, err := store.Up(ctx, drv, migrations); err != nil {
 		return 0, err
 	}
-	applied, err := drv.AppliedVersions(ctx)
+	s, err := store.ReadSchema(ctx, drv)
 	if err != nil {
 		return 0, err
 	}
-	return len(applied), nil
-}
-
-func totalMigrations() (int, error) {
-	migrations, err := store.LoadMigrations()
-	if err != nil {
-		return 0, err
-	}
-	return len(migrations), nil
+	return s.Applied, nil
 }

cmd/felis/tui_owner.go

@@ -174,12 +174,13 @@ func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 
 	case owProvisionMsg:
 		if msg.err != nil {
-			// A taken Operator username is the expected, recoverable outcome of the
-			// insert-only operator path (refusing the clash is the whole reason it is
-			// insert-only, not an upsert). Route back to the form with a note so the
-			// operator can pick another name, rather than tearing down the console —
-			// any other error is a genuine fault and still ends the session.
-			if m.operation == bgAddOperator && errors.Is(msg.err, api.ErrConflict) {
+			// api.ErrConflict marks the two recoverable refusals: a taken Operator
+			// username (insert-only clash) and an Owner reset naming anything but the
+			// occupied seat (ownerSeatTakenError Is ErrConflict). Route back to the
+			// form with a note so the operator can retype, rather than tearing down
+			// the console — any other error is a genuine fault and still ends the
+			// session.
+			if errors.Is(msg.err, api.ErrConflict) {
 				m.provisionErr = msg.err
 				m.step = owProvision
 				m.form = m.sized(m.buildProvisionForm())
@@ -364,9 +365,15 @@ func (m *ownerModel) buildProvisionForm() *huh.Form {
 		}
 	}
 	if m.provisionErr != nil {
-		// The only error routed back to this form is a username clash on the insert-only
-		// operator path; show a concrete prompt to choose another name.
-		desc = "That username is already taken — choose a different one.\n\n" + desc
+		// Recoverable refusals routed back here: the seat refusal already names the
+		// username to enter, so show it verbatim; the operator-name clash gets the
+		// generic retry prompt.
+		note := "That username is already taken — choose a different one."
+		var seatErr *ownerSeatTakenError
+		if errors.As(m.provisionErr, &seatErr) {
+			note = seatErr.Error()
+		}
+		desc = note + "\n\n" + desc
 	}
 
 	fields := []huh.Field{
@@ -420,7 +427,11 @@ func (m *ownerModel) doneView() string {
 	var box strings.Builder
 	box.WriteString(tuiLabel.Render("username  ") + m.username + "\n")
 	if m.setupTokenURL != "" {
-		box.WriteString("\n" + tuiLabel.Render("setup URL  ") + "\n" + tuiPassword.Render(m.setupTokenURL) + "\n\n")
+		box.WriteString("\n" + tuiLabel.Render("setup URL  ") + "\n")
+		for _, line := range wrapDisplayURL(m.setupTokenURL, 70) {
+			box.WriteString(tuiPassword.Render(line) + "\n")
+		}
+		box.WriteString("\n")
 		box.WriteString(tuiWarn.Render("Open this URL to complete passwordless login setup. It is shown only once."))
 	}
 	if m.auditWarning != "" {

cmd/felis/tui_postgres.go

@@ -72,20 +72,14 @@ func parseDBURL(url string) (dbCfg, error) {
 	}, nil
 }
 
-func countMigrations(dbURL string) (int, error) {
+// readSchema lines the database's recorded migrations up with this build's.
+func readSchema(dbURL string) (store.Schema, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	drv, err := store.Open(ctx, dbURL)
 	if err != nil {
-		return 0, err
+		return store.Schema{}, err
 	}
 	defer drv.Close()
-	if err := drv.EnsureVersionTable(ctx); err != nil {
-		return 0, err
-	}
-	applied, err := drv.AppliedVersions(ctx)
-	if err != nil {
-		return 0, err
-	}
-	return len(applied), nil
+	return store.ReadSchema(ctx, drv)
 }

cmd/felis/tui_preflight.go

@@ -46,6 +46,7 @@ type pfDBMsg struct{ err error }
 type pfMigCheckMsg struct {
 	applied int
 	total   int
+	pending bool
 	err     error
 }
 
@@ -84,7 +85,7 @@ func (m *preflightModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			return m, nil
 		}
 		m.applied, m.total = msg.applied, msg.total
-		if msg.applied < msg.total {
+		if msg.pending {
 			m.state = pfApplyMig
 			return m, m.applyMigrations()
 		}
@@ -204,12 +205,14 @@ func (m *preflightModel) checkDB() tea.Cmd {
 
 func (m *preflightModel) checkMigrations() tea.Cmd {
 	return func() tea.Msg {
-		applied, err := countMigrations(m.dbURL)
-		if err != nil {
-			return pfMigCheckMsg{err: err}
+		// The sets, not their sizes: a database a newer release migrated can hold as
+		// many rows as this build has migrations, and must stop here rather than be
+		// "healed" by an older binary.
+		s, err := readSchema(m.dbURL)
+		if err == nil {
+			err = s.Newer()
 		}
-		total, err := totalMigrations()
-		return pfMigCheckMsg{applied: applied, total: total, err: err}
+		return pfMigCheckMsg{applied: s.Applied, total: s.Total, pending: len(s.Pending) > 0, err: err}
 	}
 }
 

cmd/felis/tui_root.go

@@ -553,6 +553,7 @@ func (m *rootModel) showSummary() (tea.Model, tea.Cmd) {
 		storageLabel:  m.result.storageDetail,
 		routedHosts:   routed,
 		localHint:     m.result.connectMethod == connectLocal,
+		alreadySetUp:  m.result.alreadySetUp,
 	})
 }
 

cmd/felis/tui_root_test.go

@@ -249,6 +249,34 @@ func TestRootReconfigureSMTP(t *testing.T) {
 	}
 }
 
+// TestRootReconfigureStorageKeepsStatusFraming locks the same rule for the
+// "change storage" path: on a re-run, completing it must land back on the
+// alreadySetUp status framing (with the updated recap), not "Setup complete."
+func TestRootReconfigureStorageKeepsStatusFraming(t *testing.T) {
+	m := newTestRoot(true, consoleModeSetup, "")
+	m = drive(t, m, preflightDoneMsg{})
+	if _, ok := m.screen.(*summaryModel); !ok {
+		t.Fatalf("re-run after preflight, screen = %T, want *summaryModel", m.screen)
+	}
+
+	m = drive(t, m, reconfigureStorageMsg{})
+	if _, ok := m.screen.(*storageChooserModel); !ok {
+		t.Fatalf("reconfigure-storage screen = %T, want *storageChooserModel", m.screen)
+	}
+
+	m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"})
+	sum, ok := m.screen.(*summaryModel)
+	if !ok {
+		t.Fatalf("after reconfigure-storage, screen = %T, want *summaryModel", m.screen)
+	}
+	if !sum.alreadySetUp {
+		t.Fatalf("after reconfigure-storage, summary should keep the alreadySetUp framing")
+	}
+	if sum.storageLabel != "local disk · /var/lib/felis/uploads" {
+		t.Fatalf("storageLabel = %q, want the updated recap", sum.storageLabel)
+	}
+}
+
 func TestRootRerunLandsOnStatus(t *testing.T) {
 	// adminExists at start of a setup run = re-run: preflight should skip straight
 	// to the "manage in panel" status screen, never touching owner/connect.

cmd/felis/tui_smtp.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/mail"
 	"felis.lolicon.best/internal/platform"
 
 	"github.com/charmbracelet/bubbles/spinner"
@@ -310,24 +310,22 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
 	if err != nil {
 		return fmt.Errorf("port %q is not a number", in.port)
 	}
-	relay := &mail.SMTP{Host: in.host, Port: port, From: in.from, Username: in.username, Password: in.password}
-	if err := relay.Ping(ctx); err != nil {
+	var prev config.SMTPConfig
+	if cur, err := config.Load(hostSetupConfigPath); err == nil {
+		prev = cur.SMTP
+	}
+	// Ping under the posture felis api will send with, so a relay without
+	// STARTTLS is turned down here rather than at a player's first code.
+	if err := smtpRelay(setupSMTPConfig(in, port, prev), in.password).Ping(ctx); err != nil {
 		return err
 	}
 
-	smtpCfg := config.SMTPConfig{
-		Host:        in.host,
-		Port:        port,
-		From:        in.from,
-		Username:    in.username,
-		PasswordRef: platform.SMTPPasswordEnv,
-	}
 	for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} {
 		cfg, err := config.Load(path)
 		if err != nil {
 			return err
 		}
-		cfg.SMTP = smtpCfg
+		cfg.SMTP = setupSMTPConfig(in, port, cfg.SMTP)
 		if err := writeConfig(path, cfg); err != nil {
 			return err
 		}
@@ -338,20 +336,46 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
 	if err := applyFelisConfigSecret(ctx); err != nil {
 		return err
 	}
+	// Refresh the workload-namespace copies too (the reaper's warning path): the
+	// OTP path is already live in the control namespace, so a replica miss is
+	// reported but not fatal.
+	if err := replicateSMTPToWorkloadNamespace(ctx, in.password); err != nil {
+		fmt.Fprintf(os.Stderr, "felis setup: warning: email is configured, but refreshing the workload copies failed (pre-reap warning emails may stay suppressed): %v\n", err)
+	}
 	if err := kubectl(ctx, "-n", "felis", "rollout", "restart", "deployment/felis-api"); err != nil {
 		return err
 	}
 	return kubectl(ctx, "-n", "felis", "rollout", "status", "deployment/felis-api", "--timeout=180s")
 }
 
-// applySMTPSecret creates (or replaces) the felis-smtp Secret the felis-api
-// Deployment injects the relay password from. Rendered in-process and piped to
+// setupSMTPConfig is the [smtp] block this screen writes: the relay it just
+// proved, plus the keys only an operator sets by hand (require_tls,
+// max_per_hour), carried over from the block it replaces so reconfiguring the
+// relay does not quietly reset them.
+func setupSMTPConfig(in smtpInputs, port int, prev config.SMTPConfig) config.SMTPConfig {
+	return config.SMTPConfig{
+		Host:        in.host,
+		Port:        port,
+		From:        in.from,
+		Username:    in.username,
+		PasswordRef: platform.SMTPPasswordEnv,
+		MaxPerHour:  prev.MaxPerHour,
+		RequireTLS:  prev.RequireTLS,
+	}
+}
+
+// smtpSecretManifest renders the felis-smtp Secret for the given namespace, the
+// one the receiving Deployment/CronJob resolves its secretKeyRef against (felis
+// for felis-api, the workload namespace for the reaper's mirror). The namespace
+// must be IN the manifest: kubectl rejects a manifest whose namespace conflicts
+// with -n, so leaving the control namespace hardcoded made every workload-ns
+// replica fail before it started. Rendered in-process and piped to
 // `kubectl apply` — the password is never a command-line arg, so it never
 // appears in the host process table.
-func applySMTPSecret(ctx context.Context, password string) error {
+func smtpSecretManifest(password, namespace string) ([]byte, error) {
 	secret := &corev1.Secret{
 		TypeMeta:   metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"},
-		ObjectMeta: metav1.ObjectMeta{Name: platform.SMTPSecretName, Namespace: "felis"},
+		ObjectMeta: metav1.ObjectMeta{Name: platform.SMTPSecretName, Namespace: namespace},
 		Type:       corev1.SecretTypeOpaque,
 		StringData: map[string]string{
 			platform.SMTPSecretPasswordKey: password,
@@ -359,7 +383,42 @@ func applySMTPSecret(ctx context.Context, password string) error {
 	}
 	manifest, err := yaml.Marshal(secret)
 	if err != nil {
-		return fmt.Errorf("render smtp secret: %w", err)
+		return nil, fmt.Errorf("render smtp secret: %w", err)
+	}
+	return manifest, nil
+}
+
+func applySMTPSecret(ctx context.Context, password string) error {
+	manifest, err := smtpSecretManifest(password, "felis")
+	if err != nil {
+		return err
 	}
 	return kubectlWithInput(ctx, manifest, "apply", "-f", "-")
 }
+
+// replicateSMTPToWorkloadNamespace refreshes the workload-namespace (minecraft)
+// copy of felis-smtp after email is reconfigured. The reaper's CronJob runs
+// there and resolves the password by local reference — a secretKeyRef is
+// namespace-local — so without this refresh a later SMTP change would never
+// reach the pre-reap warning emails. Deliberately OVERWRITES: this is a mirror
+// of the control-namespace source, and a stale mirror is exactly the failure
+// this closes. The felis-config mirror rides along in applyFelisConfigSecret,
+// which every apply path refreshes.
+func replicateSMTPToWorkloadNamespace(ctx context.Context, password string) error {
+	cfg, err := config.Load(hostSetupConfigPath)
+	if err != nil {
+		return err
+	}
+	ns := cfg.K8s.Namespace
+	if ns == "" || ns == "felis" {
+		return nil
+	}
+	smtpManifest, err := smtpSecretManifest(password, ns)
+	if err != nil {
+		return err
+	}
+	if err := kubectlWithInput(ctx, smtpManifest, "-n", ns, "apply", "-f", "-"); err != nil {
+		return fmt.Errorf("replicate %s to %s: %w", platform.SMTPSecretName, ns, err)
+	}
+	return nil
+}

cmd/felis/tui_smtp_test.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"sigs.k8s.io/yaml"
+)
+
+// TestSMTPSecretManifestCarriesTargetNamespace pins the fix for the
+// workload-namespace replica: kubectl refuses a manifest whose namespace
+// conflicts with -n ("the namespace from the provided object ... does not
+// match"), so the mirror must render felis-smtp with the TARGET namespace —
+// otherwise the "configure email" refresh fails on the first apply and the
+// felis-config mirror never runs at all.
+func TestSMTPSecretManifestCarriesTargetNamespace(t *testing.T) {
+	for _, ns := range []string{"felis", "minecraft"} {
+		b, err := smtpSecretManifest("pw", ns)
+		if err != nil {
+			t.Fatalf("render for %s: %v", ns, err)
+		}
+		var got struct {
+			Metadata struct {
+				Namespace string `json:"namespace"`
+			} `json:"metadata"`
+		}
+		if err := yaml.Unmarshal(b, &got); err != nil {
+			t.Fatalf("unmarshal for %s: %v", ns, err)
+		}
+		if got.Metadata.Namespace != ns {
+			t.Fatalf("manifest namespace = %q, want %q", got.Metadata.Namespace, ns)
+		}
+		if !strings.Contains(string(b), "name: felis-smtp") {
+			t.Fatalf("manifest must still name felis-smtp: %s", b)
+		}
+	}
+}
+
+// TestSMTPRelayCarriesTLSPosture: the relay felis api, the reaper and the
+// watchdog send through refuses plaintext for a remote host and allows it for
+// one on this host, as [smtp] says.
+func TestSMTPRelayCarriesTLSPosture(t *testing.T) {
+	remote := smtpRelay(config.SMTPConfig{Host: "smtp.example.net", Port: 587, From: "[email protected]", Username: "felis"}, "pw")
+	want := &mail.SMTP{Host: "smtp.example.net", Port: 587, From: "[email protected]", Username: "felis", Password: "pw", RequireTLS: true}
+	if !reflect.DeepEqual(remote, want) {
+		t.Errorf("remote relay = %+v, want %+v", remote, want)
+	}
+	if local := smtpRelay(config.SMTPConfig{Host: "127.0.0.1", Port: 25, From: "[email protected]"}, ""); local.RequireTLS {
+		t.Error("a relay on this host must not require TLS by default")
+	}
+}
+
+// TestSetupSMTPConfigKeepsHandSetKeys: re-running the email screen replaces the
+// relay but keeps require_tls and max_per_hour, which only an operator sets.
+func TestSetupSMTPConfigKeepsHandSetKeys(t *testing.T) {
+	off := false
+	prev := config.SMTPConfig{Host: "old.example.net", Port: 25, From: "[email protected]", MaxPerHour: 500, RequireTLS: &off}
+	in := smtpInputs{host: "smtp.example.net", from: "[email protected]", username: "felis"}
+	got := setupSMTPConfig(in, 465, prev)
+	if got.Host != "smtp.example.net" || got.Port != 465 || got.From != "[email protected]" ||
+		got.Username != "felis" || got.PasswordRef != "FELIS_SMTP_PASSWORD" {
+		t.Errorf("relay fields = %+v", got)
+	}
+	if got.MaxPerHour != 500 {
+		t.Errorf("max_per_hour = %d, want 500", got.MaxPerHour)
+	}
+	if got.RequireTLS == nil || *got.RequireTLS {
+		t.Errorf("require_tls = %v, want the operator's false", got.RequireTLS)
+	}
+	if fresh := setupSMTPConfig(in, 587, config.SMTPConfig{}); fresh.RequireTLS != nil || fresh.MaxPerHour != 0 {
+		t.Errorf("first setup = %+v, want require_tls and max_per_hour unset", fresh)
+	}
+}

cmd/felis/tui_widgets.go

@@ -29,6 +29,33 @@ func tuiSeparator() string {
 	return tuiHint.Render(strings.Repeat("─", 70))
 }
 
+// wrapDisplayURL breaks a long URL into lines no wider than width so the TUI
+// renderer never truncates it on a narrow terminal — the one-time setup URL
+// carries a 43-char token and overruns 80 columns. It prefers breaking right
+// after a '=' or '/' inside the window (the token then lands on its own line)
+// and hard-wraps only when no boundary is available. Lines concatenate back to
+// the original string.
+func wrapDisplayURL(u string, width int) []string {
+	if width <= 0 {
+		width = 70
+	}
+	var lines []string
+	for len(u) > width {
+		cut := width
+		if i := strings.LastIndexByte(u[:width], '='); i >= 0 && i >= width/2 {
+			cut = i + 1
+		} else if i := strings.LastIndexByte(u[:width], '/'); i >= 0 && i >= width/2 {
+			cut = i + 1
+		}
+		lines = append(lines, u[:cut])
+		u = u[cut:]
+	}
+	if u != "" {
+		lines = append(lines, u)
+	}
+	return lines
+}
+
 // tuiStepRail renders a breadcrumb of wizard stages. Steps before `current`
 // render as done, `current` is highlighted, and later steps are dimmed.
 func tuiStepRail(steps []string, current int) string {

cmd/felis/tui_widgets_test.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestWrapDisplayURL(t *testing.T) {
+	u := "https://op.console.example.net/setup?token=" + strings.Repeat("A", 43)
+	lines := wrapDisplayURL(u, 70)
+	if got := strings.Join(lines, ""); got != u {
+		t.Fatalf("concatenated lines = %q, want the original URL back", got)
+	}
+	for i, l := range lines {
+		if len(l) > 70 {
+			t.Errorf("line %d is %d cols wide: %q", i, len(l), l)
+		}
+	}
+	if len(lines) < 2 || !strings.HasSuffix(lines[0], "token=") {
+		t.Fatalf("want the first line to end at the 'token=' boundary, got %q", lines)
+	}
+	short := "https://a/b"
+	if got := wrapDisplayURL(short, 70); len(got) != 1 || got[0] != short {
+		t.Errorf("short URL should pass through unsplit, got %q", got)
+	}
+}

cmd/felis/update.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -9,6 +11,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/jackc/pgx/v5"
+
+	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/updater"
 	"felis.lolicon.best/internal/updates"
 )
@@ -24,8 +29,9 @@ const updateTimeout = 60 * time.Second
 // The apply side is deliberately NOT implemented in this command. Every component
 // here is installed by deploy/bootstrap.sh, which is idempotent, already handles the
 // parts that are easy to get wrong (Velocity's pinned MINOR, the atomic jar install,
-// the k3s image re-import that a byte-identical StatefulSet template will not
-// trigger on its own), and is the path that gets exercised on every install. A
+// the image re-import + registry push that a byte-identical StatefulSet template
+// will not trigger on its own), and is the path that gets exercised on every
+// install. A
 // second installer living in this file would duplicate that policy, could drift from
 // it silently, and would be reachable only on a live node where a mistake takes the
 // proxy or the control plane down. So `felis update` reports, and hands the operator
@@ -33,6 +39,8 @@ const updateTimeout = 60 * time.Second
 type updateTarget struct {
 	// selector is the flag name without dashes.
 	selector string
+	// help is the flag's usage line.
+	help string
 	// component is the updates planner's name for this piece, or "" when the planner
 	// deliberately does not track it (Minecraft, which is pinned).
 	component string
@@ -40,8 +48,29 @@ type updateTarget struct {
 	note string
 	// command is the exact, already-tested way to apply it.
 	command string
+	// installer marks a command that re-runs the installer, which the trailer explains.
+	installer bool
 }
 
+// installerRerun is the tested apply path for every selector Felis installs: re-run the
+// installer. It is idempotent, and it is the only path that fetches a newer version --
+// `felis setup` skips its host-bootstrap phase on a completed install (all four install
+// markers already exist), so there it opens the config console and moves no component,
+// and even on the bootstrap path it re-images felis-api from the binary setup is already
+// running (FELIS_BOOTSTRAP_BINARY), which looks like an update and changes nothing.
+//
+// The URL is the one-liner both READMEs hand out, read at a tag rather than main: the
+// script's release channel installs the newest release's binary, and main can carry
+// installer changes that binary was never tested with. installerRef picks the tag and
+// renderApplyGuidance substitutes it for {ref}. While the repo is private the URL
+// answers 404 (raw.githubusercontent.com hides private repos), which is why the trailer
+// below points at the README's token'd form for that case.
+const installerRerun = "curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/{ref}/deploy/bootstrap.sh | sudo bash"
+
+// installerRerunDeps is the same re-run with FELIS_UPGRADE_DEPS=1, which lets it move an
+// installed k3s and cloudflared to the versions the release pins.
+const installerRerunDeps = "curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/{ref}/deploy/bootstrap.sh | sudo FELIS_UPGRADE_DEPS=1 bash"
+
 // updateTargets is the selector table. panel and plugins both resolve to felis-api
 // because they are not separately versioned: the panel is compiled into the felis
 // binary with //go:embed, and the plugin jars are built from this same repo in the
@@ -49,24 +78,62 @@ type updateTarget struct {
 var updateTargets = []updateTarget{
 	{
 		selector:  "panel",
+		help:      "select the panel + control plane (felis-api)",
 		component: "felis-api",
 		note:      "the panel is embedded in the felis binary (//go:embed), so updating it means rebuilding the felis image and rolling felis-api",
-		command:   "sudo felis setup",
+		command:   installerRerun,
+		installer: true,
 	},
 	{
 		selector:  "velocity",
+		help:      "select the Velocity proxy",
 		component: "velocity",
-		note:      "re-runs install_velocity: newest BUILD of the pinned minor (FELIS_VELOCITY_VERSION), atomic jar install, then restarts felis-velocity",
-		command:   "sudo felis setup",
+		note:      "re-runs install_velocity: the build the release pins in deploy/game-stack.lock (FELIS_VELOCITY_VERSION=<minor> takes that minor's newest build instead), sha256-checked, atomic jar install, then restarts felis-velocity only if the jar or its config changed",
+		command:   installerRerun,
+		installer: true,
 	},
 	{
 		selector:  "plugins",
+		help:      "select the Felis plugin jars (velocity/paper/limbo)",
 		component: "felis-api",
-		note:      "felis-velocity.jar is a host-file swap, but felis-paper.jar and felis-limbo.jar are baked into the lobby/limbo images and need a rebuild + k3s image re-import",
-		command:   "sudo felis setup",
+		note:      "felis-velocity.jar is a host-file swap, but felis-paper.jar and felis-limbo.jar are baked into the lobby/limbo images and need a rebuild + re-mirror into the in-cluster registry (the installer re-run does both)",
+		command:   installerRerun,
+		installer: true,
+	},
+	{
+		selector:  "k3s",
+		help:      "select k3s",
+		component: "k3s",
+		note:      "FELIS_UPGRADE_DEPS=1 moves k3s to the version the Felis release pins, which can trail the newest upstream; it moves one minor version at a time and refuses a larger jump. Running game servers keep running while k3s restarts",
+		command:   installerRerunDeps,
+		installer: true,
+	},
+	{
+		selector:  "cloudflared",
+		help:      "select cloudflared",
+		component: "cloudflared",
+		note:      "FELIS_UPGRADE_DEPS=1 swaps the binary for the sha256-pinned build the Felis release names and restarts cloudflared-felis; the panel's tunnel drops for a few seconds",
+		command:   installerRerunDeps,
+		installer: true,
+	},
+	{
+		selector:  "jre",
+		help:      "select the Temurin JRE Velocity runs on",
+		component: "jre",
+		note:      "the installer installs the Temurin build the Felis release pins (sha256-checked) and restarts felis-velocity when it changed; a newer upstream build reaches the host with a release that pins it",
+		command:   installerRerun,
+		installer: true,
+	},
+	{
+		selector:  "postgres",
+		help:      "select PostgreSQL",
+		component: "postgresql",
+		note:      "PostgreSQL comes from the distribution's packages, so a minor release is a package update followed by a restart (a few seconds without the API). A new major needs pg_upgrade first: docs/operations.md §4",
+		command:   "sudo dnf upgrade 'postgresql*' || sudo apt-get install --only-upgrade 'postgresql*'; sudo systemctl restart postgresql",
 	},
 	{
 		selector:  "mc",
+		help:      "select Minecraft (pinned; reported only)",
 		component: "", // never tracked: see the pin note below
 		note:      "Minecraft is pinned by policy (\"能不动的就别动\") and Felis never proposes a version change for it. A server's version is a property of that server's image — change it on the server, not through a platform update",
 		command:   "",
@@ -76,26 +143,29 @@ var updateTargets = []updateTarget{
 // cmdUpdate reports what can be updated and what is already current.
 //
 // Bare `felis update` prints the status of every tracked component. Selector flags
-// (--panel/--velocity/--mc/--plugins/--all) narrow that report to the components
+// (--panel/--velocity/--plugins/--k3s/--cloudflared/--jre/--postgres/--mc/--all) narrow that report to the components
 // they name AND print how to apply each one. --force additionally prints the apply
 // instruction for a selected component that is already up to date, for the
 // reinstall/repair case.
 //
 // It never applies anything and never mutates the node, so unlike setup/breakGlass
-// it needs no root. The versions it reads come from this host: k3s and cloudflared
-// answer `--version`, Velocity's version is read out of the installed jar's
-// manifest, and felis-api's is this binary's own build stamp — the same value
-// `felis version` prints, which is what the user asked to be the source of truth.
+// it needs no root. The versions it reads come from this host: k3s, cloudflared and
+// PostgreSQL answer `--version`, Velocity's version is read out of the installed jar's
+// manifest, the JRE's out of its release file, and felis-api's is this binary's own
+// build stamp — the same value `felis version` prints, which is what the user asked
+// to be the source of truth.
 func cmdUpdate(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("update", flag.ContinueOnError)
 	fs.SetOutput(stderr)
-	panel := fs.Bool("panel", false, "select the panel + control plane (felis-api)")
-	velocity := fs.Bool("velocity", false, "select the Velocity proxy")
-	mc := fs.Bool("mc", false, "select Minecraft (pinned; reported only)")
-	plugins := fs.Bool("plugins", false, "select the Felis plugin jars (velocity/paper/limbo)")
+	flags := map[string]*bool{}
+	for _, t := range updateTargets {
+		flags[t.selector] = fs.Bool(t.selector, false, t.help)
+	}
 	all := fs.Bool("all", false, "select every component above")
 	force := fs.Bool("force", false, "print the apply command for a selected component even when it is already up to date")
 	velocityJar := fs.String("velocity-jar", updater.DefaultVelocityJarPath, "path to the installed Velocity jar to read the current version from")
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml, read for the maintenance window the panel stores")
+	record := fs.Bool("record", false, "also store this check for the panel's Updates page (felis-update-check.timer runs it daily)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -105,8 +175,8 @@ func cmdUpdate(args []string, stdout, stderr io.Writer) int {
 	}
 
 	selected := map[string]bool{}
-	for sel, on := range map[string]bool{"panel": *panel, "velocity": *velocity, "mc": *mc, "plugins": *plugins} {
-		if on || *all {
+	for sel, on := range flags {
+		if *on || *all {
 			selected[sel] = true
 		}
 	}
@@ -114,9 +184,10 @@ func cmdUpdate(args []string, stdout, stderr io.Writer) int {
 	ctx, cancel := context.WithTimeout(context.Background(), updateTimeout)
 	defer cancel()
 
+	src := updater.NewRoutingSource(updater.Topology())
 	rn := &updater.Runner{
 		Gatherer: updater.NewHostGatherer(resolvedVersion(), *velocityJar),
-		Source:   updater.NewRoutingSource(updater.Topology()),
+		Source:   src,
 		// Notifier and Applier stay nil on purpose: a human typing this command IS the
 		// notification, and nothing here applies. The zero Window below means every
 		// Scheduled component degrades to a notify, so the report can never claim an
@@ -128,13 +199,104 @@ func cmdUpdate(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	now := time.Now()
+	win, winErr := readUpdateWindow(ctx, *cfgPath)
+	fmt.Fprint(stdout, renderWindowLine(win, winErr, now))
 	fmt.Fprint(stdout, renderUpdateReport(res, selected))
+	fmt.Fprint(stdout, renderNotes(src.Notes(), selected))
 	if len(selected) > 0 {
+		if winErr == nil && !win.Start.IsZero() && !win.Contains(now) {
+			fmt.Fprint(stdout, "Warning: this is outside the maintenance window; the commands below take effect as soon as you run them.\n")
+		}
 		fmt.Fprint(stdout, renderApplyGuidance(res, selected, *force))
 	}
+	if *record {
+		// A fresh context: the discovery pass may have spent most of updateTimeout.
+		rctx, rcancel := context.WithTimeout(context.Background(), updateWindowTimeout)
+		defer rcancel()
+		if err := recordUpdateStatus(rctx, *cfgPath, buildStatusReport(res, src.Notes(), resolvedVersion(), now)); err != nil {
+			fmt.Fprintf(stderr, "felis update: record the check for the panel: %v\n", err)
+			return 1
+		}
+		fmt.Fprint(stdout, "Recorded this check for the panel's Updates page.\n")
+	}
 	return 0
 }
 
+// buildStatusReport turns one run into the record the panel shows: every planned
+// component in plan order, then each component whose installed version could not
+// be read, by name. A component the feed could not answer for is StateUnknown with
+// the reason, never StateCurrent: the panel must not call a component current when
+// nobody could check.
+func buildStatusReport(res updater.Result, notes map[string]string, felis string, now time.Time) updates.StatusReport {
+	selectorOf := map[string]string{}
+	for _, t := range updateTargets {
+		if t.component != "" && selectorOf[t.component] == "" {
+			selectorOf[t.component] = t.selector
+		}
+	}
+	rep := updates.StatusReport{CheckedAt: now.UTC(), Felis: felis, Components: []updates.ComponentStatus{}}
+	for _, a := range res.RunResult.Plan {
+		cs := updates.ComponentStatus{
+			Name:     a.Component,
+			Current:  a.Current.String(),
+			Selector: selectorOf[a.Component],
+			Note:     notes[a.Component],
+		}
+		switch {
+		case a.Kind == updates.ActionPinned:
+			cs.State = updates.StatePinned
+		case a.Kind == updates.ActionNotify || a.Kind == updates.ActionApply:
+			cs.State = updates.StateAvailable
+			cs.Latest = a.Latest.String()
+		case a.LatestKnown:
+			cs.State = updates.StateCurrent
+		default:
+			cs.State = updates.StateUnknown
+			if err := res.RunResult.SourceErrors[a.Component]; err != nil {
+				cs.Error = err.Error()
+			}
+		}
+		rep.Components = append(rep.Components, cs)
+	}
+	names := make([]string, 0, len(res.GatherErrors))
+	for name := range res.GatherErrors {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		rep.Components = append(rep.Components, updates.ComponentStatus{
+			Name:     name,
+			State:    updates.StateUnreadable,
+			Selector: selectorOf[name],
+			Note:     notes[name],
+			Error:    res.GatherErrors[name].Error(),
+		})
+	}
+	return rep
+}
+
+// recordUpdateStatus upserts rep into platform_settings[updates.StatusKey], the
+// row the API serves to the panel's Updates page.
+func recordUpdateStatus(ctx context.Context, cfgPath string, rep updates.StatusReport) error {
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return err
+	}
+	v, err := json.Marshal(rep)
+	if err != nil {
+		return err
+	}
+	conn, err := pgx.Connect(ctx, cfg.Database.URL)
+	if err != nil {
+		return err
+	}
+	defer conn.Close(context.Background())
+	_, err = conn.Exec(ctx, `INSERT INTO platform_settings (key, value) VALUES ($1, $2::jsonb)
+		ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`, updates.StatusKey, string(v))
+	return err
+}
+
 // renderUpdateReport renders the component status table. With no selectors it shows
 // every tracked component; with selectors it shows only the components those
 // selectors name, so `felis update --velocity` is a focused answer rather than the
@@ -183,6 +345,23 @@ func renderUpdateReport(res updater.Result, selected map[string]bool) string {
 	return b.String()
 }
 
+// renderNotes prints what the release lookups learned beyond the versions (today: a
+// PostgreSQL major past its end of life), for the components the selectors show.
+func renderNotes(notes map[string]string, selected map[string]bool) string {
+	names := make([]string, 0, len(notes))
+	for name := range notes {
+		if len(selected) == 0 || selectedCovers(selected, name) {
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	var b strings.Builder
+	for _, name := range names {
+		fmt.Fprintf(&b, "%-13s note: %s\n", name, notes[name])
+	}
+	return b.String()
+}
+
 // writeErrs appends one explanatory line per failed component, in a stable order so
 // the output does not shuffle between runs, honouring the active selector filter.
 func writeErrs(b *strings.Builder, label string, errs map[string]error, selected map[string]bool) {
@@ -218,8 +397,7 @@ func renderApplyGuidance(res updater.Result, selected map[string]bool, force boo
 	}
 
 	var b strings.Builder
-	var offeredCommand bool
-	var offeredFelisAPI bool
+	var offeredInstaller bool
 	for _, t := range updateTargets {
 		if !selected[t.selector] {
 			continue
@@ -246,30 +424,104 @@ func renderApplyGuidance(res updater.Result, selected map[string]bool, force boo
 			// component, and reinstalling the current release is a valid repair action.
 			fmt.Fprintf(&b, "  note: cannot tell whether %s is current — its latest version could not be discovered (see above); this reinstalls it either way\n", t.component)
 		}
-		fmt.Fprintf(&b, "  run: %s\n", t.command)
-		offeredCommand = true
-		offeredFelisAPI = offeredFelisAPI || t.component == "felis-api"
+		fmt.Fprintf(&b, "  run: %s\n", strings.ReplaceAll(t.command, "{ref}", installerRef(byComponent)))
+		offeredInstaller = offeredInstaller || t.installer
 	}
 	// Only explain the command when one was actually offered; a --mc-only run has
 	// nothing to run and the trailer would be a non-sequitur.
-	if offeredCommand {
-		b.WriteString("\nfelis setup is idempotent and re-runs the installer that owns these components;\nit does not reinstall what is already current. Restart game servers afterwards.\n")
-	}
-	// Scoped to felis-api because it is the only component setup cannot move forward.
-	// velocity is fine: install_velocity re-resolves the newest build of the pinned minor
-	// on every run. But setup hands deploy/bootstrap.sh the binary it is itself running
-	// (FELIS_BOOTSTRAP_BINARY), and that arm skips the release lookup entirely, so it
-	// rebuilds the image and rolls the deployment from the SAME binary -- a run that looks
-	// like a successful update and leaves the version unchanged.
 	//
-	// The installer is the only thing that moves felis-api. It is safe to point at now
-	// that detect_node_ip reuses the installed root domain, so what is left to warn about
-	// is the channel: FELIS_VERSION_BOOTSTRAP is not persisted anywhere and defaults to
-	// release, so a bare re-run on a host tracking main quietly moves it onto releases.
-	// That is a channel change, not a broken install, which is why it is one clause and
-	// not a paragraph.
-	if offeredFelisAPI {
-		b.WriteString("\nfelis-api (panel, plugins) is the exception: setup re-images it from the felis binary\nalready on this host, so it cannot install a NEWER felis-api. Re-run the bootstrap\ninstaller for that -- it keeps this install's root domain. It does default to the\nrelease channel, so pass FELIS_VERSION_BOOTSTRAP=dev if this host tracks main.\n")
+	// One trailer serves every selector now: setup is not an apply path at all on a
+	// completed install (shouldRunHostBootstrapBeforeConfig only enters the host
+	// bootstrap while an install marker is missing), so the installer re-run is the one
+	// worked path for every component Felis installs and there is no per-component exception left
+	// to scope. Two caveats stay because following the advice without them bites real
+	// hosts: the channel is not persisted anywhere (a bare re-run on a main host quietly
+	// moves it onto releases), and the private repo's one-liner needs the read token
+	// back in the environment before it can resolve anything.
+	if offeredInstaller {
+		b.WriteString("\nRe-running the installer applies each installer command above: it fetches the newest version on\nthe channel in effect and re-applies the bundle (release is the default). The channel\nis not persisted, so pass FELIS_VERSION_BOOTSTRAP=dev if this host tracks main. While\nthis repo is private, the one-liner above 404s without a token; the README's install\nsection has the token'd form that works. felis setup is not this path: on a completed\ninstall it opens the config console and installs nothing newer. Restart game servers\nafterwards.\n")
 	}
 	return b.String()
 }
+
+// installerRef is the git ref the installer re-run reads bootstrap.sh from: the newest
+// stable felis release when the feed answered, which is the release that script then
+// installs; else the release this host runs; main only when neither is a release tag.
+func installerRef(byComponent map[string]updates.Action) string {
+	a, ok := byComponent["felis-api"]
+	if !ok {
+		return "main"
+	}
+	if a.LatestKnown && isReleaseTag(a.Latest) {
+		return a.Latest.String()
+	}
+	if isReleaseTag(a.Current) {
+		return a.Current.String()
+	}
+	return "main"
+}
+
+// isReleaseTag reports whether v was read from a stable vX.Y.Z tag, the only refs
+// release.yml publishes a binary for. A source build stamps v0.0.0+g<commit>, which
+// names no tag, so build metadata disqualifies a version too.
+func isReleaseTag(v updates.Version) bool {
+	s := v.String()
+	if !strings.HasPrefix(s, "v") || v.IsPrerelease() || strings.Contains(s, "+") {
+		return false
+	}
+	_, err := updates.Parse(s)
+	return err == nil
+}
+
+// updateWindowTimeout bounds the maintenance-window read, so an unreachable
+// database costs the report a line and never the report itself.
+const updateWindowTimeout = 3 * time.Second
+
+// readUpdateWindow reads the maintenance window the panel stores
+// (platform_settings "update_window"). Felis applies nothing on its own: this
+// command is the window's consumer, showing it and warning before an apply
+// outside it. A missing row is an unset window.
+func readUpdateWindow(ctx context.Context, cfgPath string) (updates.Window, error) {
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return updates.Window{}, err
+	}
+	ctx, cancel := context.WithTimeout(ctx, updateWindowTimeout)
+	defer cancel()
+	conn, err := pgx.Connect(ctx, cfg.Database.URL)
+	if err != nil {
+		return updates.Window{}, err
+	}
+	defer conn.Close(context.Background())
+	var raw []byte
+	err = conn.QueryRow(ctx, `SELECT value FROM platform_settings WHERE key = 'update_window'`).Scan(&raw)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return updates.Window{}, nil
+	}
+	if err != nil {
+		return updates.Window{}, err
+	}
+	var w updates.Window
+	if err := json.Unmarshal(raw, &w); err != nil {
+		return updates.Window{}, fmt.Errorf("stored window: %w", err)
+	}
+	return w, nil
+}
+
+// renderWindowLine is the report's first line: where now sits against the
+// maintenance window.
+func renderWindowLine(w updates.Window, err error, now time.Time) string {
+	const layout = "2006-01-02 15:04 MST"
+	switch {
+	case err != nil:
+		return fmt.Sprintf("Maintenance window: unknown (%v).\n", err)
+	case w.Start.IsZero() || w.End.IsZero():
+		return "Maintenance window: not set; apply whenever suits you.\n"
+	case w.Contains(now):
+		return fmt.Sprintf("Maintenance window: open now, until %s.\n", w.End.Local().Format(layout))
+	case now.Before(w.Start):
+		return fmt.Sprintf("Maintenance window: opens %s, until %s. Felis applies nothing on its own; run the apply commands inside it.\n", w.Start.Local().Format(layout), w.End.Local().Format(layout))
+	default:
+		return fmt.Sprintf("Maintenance window: ended %s; set a new one in the panel before applying.\n", w.End.Local().Format(layout))
+	}
+}

cmd/felis/update_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	"felis.lolicon.best/internal/updater"
 	"felis.lolicon.best/internal/updates"
@@ -107,7 +109,7 @@ func TestApplyGuidanceMinecraftOffersNoCommand(t *testing.T) {
 	if !strings.Contains(out, "pinned by policy") {
 		t.Fatalf("want the pin explained:\n%s", out)
 	}
-	if strings.Contains(out, "run:") || strings.Contains(out, "felis setup is idempotent") {
+	if strings.Contains(out, "run:") || strings.Contains(out, "Re-running the installer") {
 		t.Fatalf("--mc must offer no command and no command trailer:\n%s", out)
 	}
 }
@@ -133,42 +135,199 @@ func TestUpdateTargetsMatchTopology(t *testing.T) {
 	}
 }
 
-// `sudo felis setup` is the right answer for velocity and the wrong one for felis-api,
-// so the caveat has to be scoped rather than appended to every run. setup hands
-// bootstrap the binary it is already running, and that arm skips the release lookup:
-// the run rebuilds the image and rolls the deployment off the SAME binary, which looks
-// like a successful update and changes nothing. install_velocity, by contrast, really
-// does re-resolve the newest build on every run.
-func TestApplyGuidanceScopesTheFelisAPICaveat(t *testing.T) {
-	const caveat = "cannot install a NEWER felis-api"
-
+// Re-running the installer is the one apply path this table may hand out. setup is NOT an
+// updater on a completed install -- its host-bootstrap phase only runs while an install
+// marker is missing, so it opens the config console and moves no component -- and even on
+// the bootstrap path it re-images felis-api from the binary setup is already running. The
+// table used to answer with "sudo felis setup" and scope a felis-api-only exception; both
+// taught a model that does not survive contact with an installed host.
+func TestApplyGuidancePointsEveryComponentAtTheInstaller(t *testing.T) {
 	api := renderApplyGuidance(
 		planResult([]updates.Action{{Component: "felis-api", Kind: updates.ActionNotify, LatestKnown: true}}),
 		map[string]bool{"panel": true}, false)
-	if !strings.Contains(api, caveat) {
-		t.Fatalf("--panel resolves to felis-api and must carry the caveat:\n%s", api)
+	for _, want := range []string{"deploy/bootstrap.sh", "FELIS_VERSION_BOOTSTRAP=dev", "felis setup is not this path"} {
+		if !strings.Contains(api, want) {
+			t.Fatalf("--panel guidance missing %q:\n%s", want, api)
 		}
-	// Naming the installer obliges us to name what a bare re-run still changes. The domain
-	// is handled -- detect_node_ip reuses the installed one -- but the channel is not
-	// persisted at all and defaults to release, so a host tracking main gets moved onto
-	// releases by following this advice.
-	if !strings.Contains(api, "FELIS_VERSION_BOOTSTRAP=dev") {
-		t.Fatalf("pointing at the installer without the channel caveat misleads a dev host:\n%s", api)
+	}
+	if strings.Contains(api, "run: sudo felis setup") {
+		t.Fatalf("setup must never be offered as the apply command:\n%s", api)
 	}
 
+	// The same path serves velocity; a scoped caveat would re-teach the old model that
+	// setup fixes velocity.
 	vel := renderApplyGuidance(
 		planResult([]updates.Action{{Component: "velocity", Kind: updates.ActionNotify, LatestKnown: true}}),
 		map[string]bool{"velocity": true}, false)
-	if strings.Contains(vel, caveat) {
-		t.Fatalf("velocity IS fixed by setup; the caveat would misdirect the operator:\n%s", vel)
-	}
-	if !strings.Contains(vel, "felis setup is idempotent") {
-		t.Fatalf("velocity still wants the ordinary trailer:\n%s", vel)
+	if !strings.Contains(vel, "run: curl -fsSL") || !strings.Contains(vel, "felis setup is not this path") {
+		t.Fatalf("velocity gets the same installer path:\n%s", vel)
 	}
 
 	// --mc offers no command at all, so neither trailer belongs.
 	mc := renderApplyGuidance(planResult(nil), map[string]bool{"mc": true}, true)
-	if strings.Contains(mc, caveat) {
-		t.Fatalf("--mc offers no command; the caveat is a non-sequitur:\n%s", mc)
+	if strings.Contains(mc, "deploy/bootstrap.sh") || strings.Contains(mc, "FELIS_VERSION_BOOTSTRAP") {
+		t.Fatalf("--mc offers no command; the trailer is a non-sequitur:\n%s", mc)
+	}
+}
+
+// The re-run reads bootstrap.sh at the tag whose binary it installs. main can carry
+// installer changes no release was tested with.
+func TestApplyGuidanceReadsTheInstallerAtTheReleaseTag(t *testing.T) {
+	v := func(s string) updates.Version {
+		t.Helper()
+		out, err := updates.Parse(s)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return out
+	}
+	cases := []struct {
+		name string
+		api  []updates.Action
+		want string
+	}{
+		{"latest known", []updates.Action{{Component: "felis-api", Kind: updates.ActionNotify, Current: v("v1.3.0"), Latest: v("v1.4.0"), LatestKnown: true}}, "/FelisMC/Felis/v1.4.0/deploy/bootstrap.sh"},
+		{"latest unknown", []updates.Action{{Component: "felis-api", Kind: updates.ActionNone, Current: v("v1.3.0")}}, "/FelisMC/Felis/v1.3.0/deploy/bootstrap.sh"},
+		{"prerelease latest", []updates.Action{{Component: "felis-api", Kind: updates.ActionNone, Current: v("v1.3.0"), Latest: v("v1.4.0-rc.1"), LatestKnown: true}}, "/FelisMC/Felis/v1.3.0/deploy/bootstrap.sh"},
+		{"nothing known", nil, "/FelisMC/Felis/main/deploy/bootstrap.sh"},
+	}
+	for _, c := range cases {
+		out := renderApplyGuidance(planResult(c.api), map[string]bool{"velocity": true, "panel": true}, true)
+		if !strings.Contains(out, c.want) {
+			t.Errorf("%s: want %q in:\n%s", c.name, c.want, out)
+		}
+		if strings.Contains(out, "{ref}") {
+			t.Errorf("%s: placeholder left in:\n%s", c.name, out)
+		}
+	}
+}
+
+// Every selector in the table is a flag: the FlagSet is built from the table.
+func TestUpdateSelectorsAreFlags(t *testing.T) {
+	var out, errb strings.Builder
+	if code := cmdUpdate([]string{"-h"}, &out, &errb); code != 2 {
+		t.Fatalf("-h exit = %d, want 2", code)
+	}
+	for _, target := range updateTargets {
+		if !strings.Contains(errb.String(), "-"+target.selector+"\n") {
+			t.Errorf("usage has no -%s flag:\n%s", target.selector, errb.String())
+		}
+	}
+}
+
+// k3s and cloudflared move only when the re-run is told to; PostgreSQL is the package
+// manager's, so its guidance carries no installer trailer.
+func TestApplyGuidanceForHostDependencies(t *testing.T) {
+	notify := func(c string) updater.Result {
+		return planResult([]updates.Action{{Component: c, Kind: updates.ActionNotify, LatestKnown: true}})
+	}
+	for _, sel := range []string{"k3s", "cloudflared"} {
+		out := renderApplyGuidance(notify(sel), map[string]bool{sel: true}, false)
+		if !strings.Contains(out, "sudo FELIS_UPGRADE_DEPS=1 bash") || !strings.Contains(out, "Re-running the installer") {
+			t.Errorf("--%s guidance must re-run the installer with FELIS_UPGRADE_DEPS=1:\n%s", sel, out)
+		}
+	}
+	jre := renderApplyGuidance(notify("jre"), map[string]bool{"jre": true}, false)
+	if !strings.Contains(jre, "| sudo bash") || strings.Contains(jre, "FELIS_UPGRADE_DEPS") {
+		t.Errorf("--jre guidance is the plain installer re-run:\n%s", jre)
+	}
+	pg := renderApplyGuidance(notify("postgresql"), map[string]bool{"postgres": true}, false)
+	if !strings.Contains(pg, "apt-get install --only-upgrade") || strings.Contains(pg, "Re-running the installer") {
+		t.Errorf("--postgres guidance is the package manager, without the installer trailer:\n%s", pg)
+	}
+}
+
+func TestRenderNotesHonoursSelectors(t *testing.T) {
+	notes := map[string]string{"postgresql": "PostgreSQL 13 reached end of life on 2025-11-13"}
+	if out := renderNotes(notes, nil); !strings.Contains(out, "postgresql") || !strings.Contains(out, "note: PostgreSQL 13 reached end of life") {
+		t.Errorf("unfiltered notes = %q", out)
+	}
+	if out := renderNotes(notes, map[string]bool{"postgres": true}); !strings.Contains(out, "end of life") {
+		t.Errorf("--postgres must show its note, got %q", out)
+	}
+	if out := renderNotes(notes, map[string]bool{"velocity": true}); out != "" {
+		t.Errorf("--velocity must not show the postgresql note, got %q", out)
+	}
+}
+
+// A source build's v0.0.0+g<commit> names no tag, so the installer one-liner has to
+// fall back to main instead of a 404ing ref.
+func TestInstallerRefNamesATag(t *testing.T) {
+	v := func(s string) updates.Version {
+		t.Helper()
+		x, err := updates.Parse(s)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return x
+	}
+	cases := []struct {
+		name string
+		api  updates.Action
+		want string
+	}{
+		{"newest release", updates.Action{Current: v("v1.2.0"), Latest: v("v1.3.0"), LatestKnown: true}, "v1.3.0"},
+		{"feed down, host on a release", updates.Action{Current: v("v1.2.0")}, "v1.2.0"},
+		{"source build", updates.Action{Current: v("v0.0.0+gunknown")}, "main"},
+		{"source build with commit", updates.Action{Current: v("v0.0.0+g1a2b3c4")}, "main"},
+		{"prerelease", updates.Action{Current: v("v1.3.0-rc.1")}, "main"},
+	}
+	for _, c := range cases {
+		if got := installerRef(map[string]updates.Action{"felis-api": c.api}); got != c.want {
+			t.Errorf("%s: installerRef = %q, want %q", c.name, got, c.want)
+		}
+	}
+	if got := installerRef(nil); got != "main" {
+		t.Errorf("no felis-api row: installerRef = %q, want main", got)
+	}
+}
+
+func mustVersion(t *testing.T, s string) updates.Version {
+	t.Helper()
+	v, err := updates.Parse(s)
+	if err != nil {
+		t.Fatalf("parse %q: %v", s, err)
+	}
+	return v
+}
+
+// The record the panel shows keeps every component with a state that cannot be
+// mistaken: a feed failure is "unknown" with its reason, an unreadable install is
+// listed after the plan, and each row carries the selector that prints its apply.
+func TestBuildStatusReport(t *testing.T) {
+	res := planResult([]updates.Action{
+		{Component: "felis-api", Current: mustVersion(t, "v0.4.0"), Latest: mustVersion(t, "v0.5.0"), LatestKnown: true, Kind: updates.ActionNotify},
+		{Component: "velocity", Current: mustVersion(t, "3.4.0"), Latest: mustVersion(t, "3.4.0"), LatestKnown: true, Kind: updates.ActionNone},
+		{Component: "k3s", Current: mustVersion(t, "v1.36.2+k3s1"), Kind: updates.ActionNone},
+		{Component: "cloudflared", Current: mustVersion(t, "2026.6.1"), Latest: mustVersion(t, "2026.9.0"), LatestKnown: true, Kind: updates.ActionApply},
+		{Component: "mc-lobby", Current: mustVersion(t, "1.21.4"), Kind: updates.ActionPinned},
+	})
+	res.RunResult.SourceErrors["k3s"] = errors.New("github: HTTP 403")
+	res.GatherErrors["postgresql"] = errors.New("psql: not found")
+	res.GatherErrors["jre"] = errors.New("release file missing")
+	notes := map[string]string{"postgresql": "PostgreSQL 13 is past its end of life", "velocity": "pinned minor 3.4"}
+	now := time.Date(2026, 9, 25, 3, 4, 5, 0, time.FixedZone("CST", 8*3600))
+
+	b, err := json.Marshal(buildStatusReport(res, notes, "v0.4.0", now))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := `{"checked_at":"2026-09-24T19:04:05Z","felis":"v0.4.0","components":[` +
+		`{"name":"felis-api","current":"v0.4.0","latest":"v0.5.0","state":"available","selector":"panel"},` +
+		`{"name":"velocity","current":"3.4.0","state":"current","selector":"velocity","note":"pinned minor 3.4"},` +
+		`{"name":"k3s","current":"v1.36.2+k3s1","state":"unknown","selector":"k3s","error":"github: HTTP 403"},` +
+		`{"name":"cloudflared","current":"2026.6.1","latest":"2026.9.0","state":"available","selector":"cloudflared"},` +
+		`{"name":"mc-lobby","current":"1.21.4","state":"pinned"},` +
+		`{"name":"jre","state":"unreadable","selector":"jre","error":"release file missing"},` +
+		`{"name":"postgresql","state":"unreadable","selector":"postgres","note":"PostgreSQL 13 is past its end of life","error":"psql: not found"}]}`
+	if string(b) != want {
+		t.Errorf("status report =\n%s\nwant\n%s", b, want)
+	}
+
+	// Nothing tracked still records an empty list, so the panel can tell "checked,
+	// nothing to show" from a report that never arrived.
+	b, _ = json.Marshal(buildStatusReport(planResult(nil), nil, "v0.4.0", now))
+	if !strings.Contains(string(b), `"components":[]`) {
+		t.Errorf("an empty check = %s, want an empty components list", b)
 	}
 }

cmd/felis/update_window_test.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/updates"
+)
+
+func TestRenderWindowLinePlacesNowAgainstTheWindow(t *testing.T) {
+	prev := time.Local
+	time.Local = time.FixedZone("CST", 8*3600)
+	t.Cleanup(func() { time.Local = prev })
+
+	w := updates.Window{
+		Start: time.Date(2026, 9, 26, 18, 0, 0, 0, time.UTC),
+		End:   time.Date(2026, 9, 26, 20, 0, 0, 0, time.UTC),
+	}
+	cases := []struct {
+		name string
+		w    updates.Window
+		err  error
+		now  time.Time
+		want string
+	}{
+		{"unreadable", updates.Window{}, errors.New("connection refused"), w.Start, "Maintenance window: unknown (connection refused).\n"},
+		{"unset", updates.Window{}, nil, w.Start, "Maintenance window: not set; apply whenever suits you.\n"},
+		{"half set", updates.Window{Start: w.Start}, nil, w.Start, "Maintenance window: not set; apply whenever suits you.\n"},
+		{"at the opening instant", w, nil, w.Start, "Maintenance window: open now, until 2026-09-27 04:00 CST.\n"},
+		{"before", w, nil, w.Start.Add(-time.Minute), "Maintenance window: opens 2026-09-27 02:00 CST, until 2026-09-27 04:00 CST. Felis applies nothing on its own; run the apply commands inside it.\n"},
+		{"at the closing instant", w, nil, w.End, "Maintenance window: ended 2026-09-27 04:00 CST; set a new one in the panel before applying.\n"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := renderWindowLine(tc.w, tc.err, tc.now); got != tc.want {
+				t.Fatalf("got  %q\nwant %q", got, tc.want)
+			}
+		})
+	}
+}

cmd/felis/watchdog.go

@@ -0,0 +1,268 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/watchdog"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// proxyFor is how long the game proxy may refuse connections before it is
+// mailed: a restart takes seconds.
+const proxyFor = 3 * time.Minute
+
+// cmdWatchdog runs one pass of the platform watchdog (internal/watchdog): it
+// checks the cluster, PostgreSQL, the game proxy, the database backups and the
+// host, prints every finding, and mails the platform owners what came due.
+// deploy/bootstrap.sh runs it every two minutes from felis-watchdog.timer.
+func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
+	statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")
+	quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
+	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
+	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/postgresql,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
+	proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)
+	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")
+	offsiteStatus := fs.String("offsite-status", offsite.DefaultStatusFile, "the record `felis offsite sync` leaves, checked when [offsite] is configured")
+	toolsStatus := fs.String("build-tools-status", defaultBuildToolsStatus, "the record `felis mirror-build-tools` leaves, checked when builds scan against the registry's DB copy")
+	dryRun := fs.Bool("dry-run", false, "print every finding and the mail that is due; send nothing and keep the state as it was")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis watchdog: %v\n", err)
+		return 1
+	}
+	state, err := watchdog.LoadState(*statePath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis watchdog: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+	now := time.Now()
+
+	var report watchdog.Report
+	add := func(f *watchdog.Finding) {
+		if f != nil {
+			report.Findings = append(report.Findings, *f)
+		}
+	}
+
+	// The cluster: one unreachable API server stands in for every check behind it.
+	minecraftNS := cfg.K8s.Namespace
+	if minecraftNS == "" {
+		minecraftNS = platform.DefaultMinecraftNamespace
+	}
+	cl, err := buildSystemServerClient()
+	var found []watchdog.Finding
+	if err == nil {
+		found, err = watchdog.Cluster{Client: cl, ControlNamespace: *controlNS, MinecraftNamespace: minecraftNS}.Check(ctx, now)
+	}
+	if err != nil {
+		f := watchdog.KubeAPIDown(err)
+		add(&f)
+		report.Unknown = append(report.Unknown, watchdog.ClusterPrefixes...)
+	} else {
+		report.Findings = append(report.Findings, found...)
+		if cfg.SMTP.Host != "" {
+			refreshSMTPPassword(ctx, cl, *controlNS, state, stderr)
+		}
+	}
+
+	if recipients, err := ownerEmails(ctx, cfg.Database.URL); err != nil {
+		f := watchdog.PostgresDown(err)
+		add(&f)
+	} else {
+		state.Recipients = recipients
+	}
+
+	if *proxyAddr != "" {
+		add(proxyFinding(ctx, *proxyAddr))
+	}
+	if *backupDir != "" {
+		add(watchdog.BackupFinding(*backupDir, now))
+	}
+	if cfg.Offsite.Enabled() {
+		add(watchdog.OffsiteFinding(*offsiteStatus, now))
+	}
+	if usesMirroredScanDB(cfg) {
+		add(watchdog.ScanDBFinding(*toolsStatus, now))
+	}
+	report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(*diskPaths))...)
+	add(watchdog.MemoryFinding("/proc/meminfo"))
+
+	if len(report.Findings) == 0 {
+		fmt.Fprintln(stdout, "felis watchdog: every check passed")
+	}
+	for _, f := range report.Findings {
+		fmt.Fprintf(stdout, "felis watchdog: [%s] %s: %s\n", f.Severity, f.Key, f.SummaryEN)
+	}
+
+	plan := state.Observe(report, now)
+	host, _ := os.Hostname()
+	subject, body := plan.Message(host, now)
+	if *dryRun {
+		if plan.Empty() {
+			fmt.Fprintln(stdout, "felis watchdog: nothing is due to be mailed")
+		} else {
+			fmt.Fprintf(stdout, "felis watchdog: due to be mailed to %s:\nSubject: %s\n\n%s", strings.Join(state.Recipients, ", "), subject, strings.ReplaceAll(body, "\r\n", "\n"))
+		}
+		return 0
+	}
+
+	save := func() int {
+		if err := watchdog.SaveState(*statePath, state); err != nil {
+			fmt.Fprintf(stderr, "felis watchdog: save state: %v\n", err)
+			return 1
+		}
+		return 0
+	}
+	if plan.Empty() {
+		return save()
+	}
+	if until := watchdog.QuietUntil(*quietPath); now.Before(until) {
+		fmt.Fprintf(stdout, "felis watchdog: quiet until %s (installer running); holding this mail: %s\n", until.UTC().Format(time.RFC3339), subject)
+		return save()
+	}
+	switch {
+	case cfg.SMTP.Host == "":
+		fmt.Fprintf(stdout, "felis watchdog: no [smtp] relay configured, so this is logged only: %s\n", subject)
+	case len(state.Recipients) == 0:
+		fmt.Fprintf(stdout, "felis watchdog: no owner account has a verified email, so this is logged only: %s\n", subject)
+	default:
+		if err := sendAlert(ctx, cfg, state, subject, body); err != nil {
+			// Not committed: the same alerts come due again next run.
+			fmt.Fprintf(stderr, "felis watchdog: mail %q: %v\n", subject, err)
+			save()
+			return 1
+		}
+		fmt.Fprintf(stdout, "felis watchdog: mailed %s: %s\n", strings.Join(state.Recipients, ", "), subject)
+	}
+	state.Commit(plan, now)
+	return save()
+}
+
+// usesMirroredScanDB reports whether build scans read the vulnerability DB copy
+// felis mirror-build-tools keeps in the platform registry: the default, or an
+// explicit trivy_db_repository under the registry's mirror/.
+func usesMirroredScanDB(cfg *config.Config) bool {
+	if cfg.Registry.URL == "" {
+		return false
+	}
+	repo := cfg.Registry.TrivyDBRepository
+	return repo == "" || strings.HasPrefix(repo, cfg.Registry.URL+"/mirror/")
+}
+
+// refreshSMTPPassword caches the relay password from the felis-smtp Secret, or
+// forgets it when the Secret is gone (a relay without AUTH). An env var named by
+// [smtp] password_ref, when set, wins at send time instead.
+func refreshSMTPPassword(ctx context.Context, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) {
+	var sec corev1.Secret
+	err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: platform.SMTPSecretName}, &sec)
+	switch {
+	case apierrors.IsNotFound(err):
+		state.SMTPPassword = ""
+	case err != nil:
+		fmt.Fprintf(stderr, "felis watchdog: read %s/%s (keeping the cached relay password): %v\n", ns, platform.SMTPSecretName, err)
+	default:
+		state.SMTPPassword = string(sec.Data[platform.SMTPSecretPasswordKey])
+	}
+}
+
+// ownerEmails pings PostgreSQL and returns the verified addresses of the
+// enabled owner accounts, the people who can act on an alert.
+func ownerEmails(ctx context.Context, url string) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	defer drv.Close()
+	rows, err := drv.DB().QueryContext(ctx,
+		`SELECT email FROM users
+		 WHERE role = 'owner' AND email_verified AND COALESCE(email, '') <> ''
+		   AND NOT disabled AND deleted_at IS NULL
+		 ORDER BY email`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []string
+	for rows.Next() {
+		var email sql.NullString
+		if err := rows.Scan(&email); err != nil {
+			return nil, err
+		}
+		out = append(out, email.String)
+	}
+	return out, rows.Err()
+}
+
+// proxyFinding dials the game proxy; players reach every server through it.
+func proxyFinding(ctx context.Context, addr string) *watchdog.Finding {
+	d := net.Dialer{Timeout: 5 * time.Second}
+	conn, err := d.DialContext(ctx, "tcp", addr)
+	if err == nil {
+		conn.Close()
+		return nil
+	}
+	return &watchdog.Finding{
+		Key: "proxy", Severity: watchdog.Critical, For: proxyFor,
+		Summary:   fmt.Sprintf("游戏代理 %s 无法连接：玩家进不了任何服务器", addr),
+		SummaryEN: fmt.Sprintf("the game proxy at %s refuses connections: players cannot reach any server", addr),
+		Hint:      fmt.Sprintf("systemctl status felis-velocity; journalctl -u felis-velocity -n 200 (%v)", err),
+	}
+}
+
+// sendAlert mails subject/body to every recipient; it fails only when no
+// recipient got it.
+func sendAlert(ctx context.Context, cfg *config.Config, state *watchdog.State, subject, body string) error {
+	password := state.SMTPPassword
+	if ref := cfg.SMTP.PasswordRef; ref != "" && os.Getenv(ref) != "" {
+		password = os.Getenv(ref)
+	}
+	relay := smtpRelay(cfg.SMTP, password)
+	var errs []error
+	for _, to := range state.Recipients {
+		if err := relay.SendNotice(ctx, to, subject, body); err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", to, err))
+		}
+	}
+	if len(errs) == len(state.Recipients) {
+		return errors.Join(errs...)
+	}
+	return nil
+}
+
+func splitList(s string) []string {
+	var out []string
+	for _, p := range strings.Split(s, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}

cmd/felis/watchdog_test.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"context"
+	"net"
+	"strings"
+	"testing"
+)
+
+// TestProxyFinding: a listening proxy is healthy; a closed port is the critical
+// "players cannot reach any server" finding.
+func TestProxyFinding(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := ln.Addr().String()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			c.Close()
+		}
+	}()
+	if f := proxyFinding(context.Background(), addr); f != nil {
+		t.Fatalf("listening proxy reported: %+v", f)
+	}
+	ln.Close()
+	f := proxyFinding(context.Background(), addr)
+	if f == nil || f.Key != "proxy" || !strings.Contains(f.SummaryEN, addr) {
+		t.Fatalf("closed proxy = %+v, want the proxy finding", f)
+	}
+}
+
+func TestSplitList(t *testing.T) {
+	got := splitList(" /, /var/lib/felis ,,")
+	if strings.Join(got, "|") != "/|/var/lib/felis" {
+		t.Fatalf("splitList = %q", got)
+	}
+}

deploy/alerts/felis-alerts.yaml

@@ -0,0 +1,284 @@
+# Felis alert rules — plain Prometheus format (also the promtool-tested source
+# for felis-prometheusrule.yaml). See docs/troubleshooting.md §14 for scraping
+# and loading instructions. These are for a deployment that brings its own
+# Prometheus; every install already runs `felis watchdog` on the host, which
+# checks the same conditions without one and mails the owners (§14).
+#
+# felis_* series come from these processes:
+#   - felis-operator-metrics Service :8080  → felis_servers_total, felis_server_phase,
+#                                             felis_start_duration_seconds,
+#                                             felis_build_info{component="operator"},
+#                                             controller_runtime_*, workqueue_*
+#   - felis-api-internal Service :8081      → felis_image_build_failures_total,
+#                                             felis_mail_total, felis_rate_limited_total,
+#                                             felis_auth_otp_lockouts_total,
+#                                             felis_auth_failures_total,
+#                                             felis_audit_write_failures_total,
+#                                             felis_build_info{component="api"}
+#   - node-exporter textfile collector      → felis_db_backup_* (felis-db-backup.timer)
+# node_* / kube_* series come from node-exporter / kube-state-metrics.
+groups:
+  - name: felis.rules
+    rules:
+      - alert: FelisImageBuildFailures
+        expr: increase(felis_image_build_failures_total[6h]) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "modpack/image build failed in the last 6h"
+          description: >-
+            felis_image_build_failures_total increased. Inspect the failed build Job
+            (kubectl logs -n felis-build job/<build-job>); the same error text is on
+            GET /api/v1/images/build/{id} and in the submitter's row in the panel.
+      - alert: FelisSlowServerStarts
+        expr: histogram_quantile(0.9, sum by (le) (rate(felis_start_duration_seconds_bucket[30m]))) > 300
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "p90 server start time exceeds 5 minutes"
+          description: >-
+            Starts regularly take over five minutes (felis_start_duration_seconds,
+            observed when readiness is first reached). A start that never completes
+            records nothing — cross-check desiredState=Running servers with no ready
+            phase (troubleshooting §1).
+  - name: felis.platform.rules
+    rules:
+      - alert: FelisOperatorDown
+        expr: absent(felis_build_info{component="operator"})
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "felis-operator is down or not scraped"
+          description: >-
+            No felis_build_info{component="operator"} series for 10 minutes. Without
+            the operator no server starts, stops or recovers. Check
+            `kubectl -n felis get deploy felis-operator` and its log; if the pod is
+            healthy, the felis-operator-metrics Service is not being scraped
+            (troubleshooting §14).
+      - alert: FelisAPIDown
+        expr: absent(felis_build_info{component="api"})
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "felis-api is down or not scraped"
+          description: >-
+            No felis_build_info{component="api"} series for 10 minutes. The panel,
+            sign-in and the proxy's player lookups all go through felis-api. Check
+            `kubectl -n felis get deploy felis-api` and its log; if the pod is
+            healthy, the felis-api-internal Service is not being scraped
+            (troubleshooting §14).
+      - alert: FelisLoginGateDown
+        expr: felis_server_phase{role="login",desired="Running",phase!="Running"} == 1
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "the login gate {{ $labels.server }} is {{ $labels.phase }}"
+          description: >-
+            Every player connection passes through the login server first, so no one
+            can join. The MinecraftServer's conditions carry the reason:
+            `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+            (troubleshooting §1, §2).
+      - alert: FelisSystemServerDown
+        expr: felis_server_phase{role!="",role!="login",desired="Running",phase!="Running"} == 1
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "system server {{ $labels.server }} ({{ $labels.role }}) is {{ $labels.phase }}"
+          description: >-
+            Players who sign in are sent to the lobby; while it is down they stay at
+            the gate. `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+            shows the reason (troubleshooting §1, §2).
+      - alert: FelisServerFailed
+        expr: felis_server_phase{role="",phase="Failed"} == 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "server {{ $labels.server }} is Failed"
+          description: >-
+            The operator gave up on this server (a crash loop, an image that will not
+            pull, a world volume that will not mount). Its conditions carry the
+            reason: `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+            (troubleshooting §2).
+      - alert: FelisReconcileErrors
+        expr: sum(increase(controller_runtime_reconcile_errors_total{controller="minecraftserver"}[15m])) > 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "the operator failed over 10 reconciles in 15 minutes"
+          description: >-
+            Server changes are being retried instead of applied. The felis-operator
+            log names each failing server and its error.
+      - alert: FelisReconcileStuck
+        expr: max(workqueue_longest_running_processor_seconds{name="minecraftserver"}) > 300
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "an operator reconcile has been running for over 5 minutes"
+          description: >-
+            Each reconcile is bounded at 3 minutes, so this one is ignoring its
+            deadline and holding a worker. The liveness probe restarts the operator
+            once a pass passes 10 minutes; the log from before the restart shows
+            where it hung.
+  - name: felis.jobs.rules
+    rules:
+      - alert: FelisWorldJobFailed
+        expr: kube_job_failed{namespace="minecraft",condition="true"} == 1
+        labels:
+          severity: warning
+        annotations:
+          summary: "Job {{ $labels.job_name }} failed"
+          description: >-
+            A world backup, restore or reaper run failed; after a failed backup that
+            world's newest archive is older than planned.
+            `kubectl -n minecraft logs job/{{ $labels.job_name }}` has the error
+            (troubleshooting §10).
+      - alert: FelisReaperStale
+        expr: time() - kube_cronjob_status_last_successful_time{namespace="minecraft",cronjob="felis-reaper"} > 26 * 3600
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "the world reaper has not succeeded in over 26h"
+          description: >-
+            felis-reaper runs daily; idle worlds are neither backed up nor reclaimed
+            while it fails. `kubectl -n minecraft get jobs --sort-by=.metadata.creationTimestamp`
+            lists its runs, and the newest one's log
+            shows why (troubleshooting §10).
+  - name: felis.node.rules
+    rules:
+      - alert: FelisNodeDiskSpaceLow
+        expr: >-
+          node_filesystem_avail_bytes{fstype=~"ext4|xfs|btrfs"}
+          / node_filesystem_size_bytes{fstype=~"ext4|xfs|btrfs"} < 0.15
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "node filesystem {{ $labels.mountpoint }} below 15% available"
+          description: >-
+            Sustained disk pressure evicts game pods and garbage-collects images
+            (troubleshooting §13b). Free space before kubelet raises DiskPressure.
+      - alert: FelisNodeDiskPressure
+        expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "kubelet reports DiskPressure on {{ $labels.node }}"
+          description: >-
+            The eviction chain is in progress: control-plane pods hold
+            system-cluster-critical and survive, game pods do not. Free disk now
+            (troubleshooting §13b).
+      - alert: FelisNodeMemoryLow
+        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.10
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "node memory available below 10% for 15m"
+          description: >-
+            PostgreSQL, the control plane, the registry and game servers share one
+            node; sustained memory pressure risks OOM kills.
+  - name: felis.backup.rules
+    rules:
+      - alert: FelisDBBackupStale
+        expr: time() - max(felis_db_backup_last_success_timestamp_seconds) > 26 * 3600
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "no control-plane database backup in over 26h"
+          description: >-
+            felis-db-backup.timer runs daily; the newest bundle is more than a day
+            old. Read `journalctl -u felis-db-backup` on the host, then take one now
+            with `sudo felis db backup` (troubleshooting §16).
+      - alert: FelisDBBackupMetricMissing
+        expr: absent(felis_db_backup_last_success_timestamp_seconds)
+        for: 2h
+        labels:
+          severity: warning
+        annotations:
+          summary: "database backup freshness is not being scraped"
+          description: >-
+            No felis_db_backup_last_success_timestamp_seconds series, so
+            FelisDBBackupStale cannot fire. Point node-exporter's
+            --collector.textfile.directory at the directory of
+            FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
+            (troubleshooting §16).
+  - name: felis.auth.rules
+    rules:
+      - alert: FelisMailBudgetExhausted
+        expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "the install-wide mail budget refused mail"
+          description: >-
+            felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
+            spent, and every sign-in code is refused with 429 mail_rate_limited until
+            it refills. Check felis_rate_limited_total for a flood before raising the
+            budget (troubleshooting §17).
+      - alert: FelisMailDeliveryFailing
+        expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "the SMTP relay refused mail in the last 15m"
+          description: >-
+            felis_mail_total{result="failed"} increased: sign-in codes are not being
+            delivered (502 mail_undeliverable). The relay's reason is in the
+            felis-api log (troubleshooting §17).
+      - alert: FelisSignInFlood
+        expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "sign-in doors refusing over 10 requests a minute"
+          description: >-
+            The per-address sign-in limit has been refusing callers for 10 minutes.
+            A script is hammering the auth doors; if real users report rate_limited
+            at once instead, [auth] client_ip_header is missing and everyone shares
+            the proxy's address (troubleshooting §17).
+      - alert: FelisOTPAccountLocked
+        expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "an account's email-code sign-in locked after 10 wrong codes"
+          description: >-
+            Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}).
+            The audit log names the account (action auth.otp.locked); the owner was
+            mailed. Unless they fumbled codes, someone is guessing at it
+            (troubleshooting §17).
+      - alert: FelisSignInFailures
+        expr: sum(increase(felis_auth_failures_total[15m])) > 30
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "over 30 refused sign-ins in 15 minutes"
+          description: >-
+            Wrong codes, unknown addresses or bad passkey assertions well above people
+            mistyping: someone is guessing or enumerating. `sum by (door, reason)
+            (increase(felis_auth_failures_total[15m]))` shows where; the audit rows
+            (action auth.<door>.failed) carry each caller's client_ip (troubleshooting §17).
+      - alert: FelisAuditWriteFailing
+        expr: increase(felis_audit_write_failures_total[15m]) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "felis-api failed to write audit rows"
+          description: >-
+            The actions went through but their audit rows were lost. The felis-api log
+            names each lost row (`audit: lost ...`); the usual cause is PostgreSQL
+            being unreachable or out of disk.

deploy/alerts/felis-alerts_test.yml

@@ -0,0 +1,509 @@
+# promtool unit tests: `promtool test rules felis-alerts_test.yml`
+# Proves every shipped rule actually fires on its target condition (and stays
+# silent before it).
+rule_files:
+  - felis-alerts.yaml
+evaluation_interval: 1m
+tests:
+  - name: build failure and slow starts
+    interval: 1m
+    input_series:
+      # counter: quiet for 5m, then one failure per step.
+      - series: 'felis_image_build_failures_total'
+        values: '0x5 1x15'
+      # histogram: all observations land in the (300,600] bucket.
+      - series: 'felis_start_duration_seconds_bucket{le="120"}'
+        values: '0x22'
+      - series: 'felis_start_duration_seconds_bucket{le="300"}'
+        values: '0x22'
+      - series: 'felis_start_duration_seconds_bucket{le="600"}'
+        values: '0+10x21'
+      - series: 'felis_start_duration_seconds_bucket{le="+Inf"}'
+        values: '0+10x21'
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: FelisImageBuildFailures
+        exp_alerts: []
+      - eval_time: 20m
+        alertname: FelisImageBuildFailures
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "modpack/image build failed in the last 6h"
+              description: >-
+                felis_image_build_failures_total increased. Inspect the failed build Job
+                (kubectl logs -n felis-build job/<build-job>); the same error text is on
+                GET /api/v1/images/build/{id} and in the submitter's row in the panel.
+      - eval_time: 20m
+        alertname: FelisSlowServerStarts
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "p90 server start time exceeds 5 minutes"
+              description: >-
+                Starts regularly take over five minutes (felis_start_duration_seconds,
+                observed when readiness is first reached). A start that never completes
+                records nothing — cross-check desiredState=Running servers with no ready
+                phase (troubleshooting §1).
+  - name: node disk and memory thresholds
+    interval: 1m
+    input_series:
+      - series: 'node_filesystem_avail_bytes{device="/dev/vda1",fstype="xfs",instance="node1",job="node-exporter",mountpoint="/"}'
+        values: '10x26'
+      - series: 'node_filesystem_size_bytes{device="/dev/vda1",fstype="xfs",instance="node1",job="node-exporter",mountpoint="/"}'
+        values: '100x26'
+      - series: 'kube_node_status_condition{condition="DiskPressure",node="n1",status="true"}'
+        values: '0x4 1x22'
+      - series: 'node_memory_MemAvailable_bytes{instance="node1",job="node-exporter"}'
+        values: '5x26'
+      - series: 'node_memory_MemTotal_bytes{instance="node1",job="node-exporter"}'
+        values: '100x26'
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: FelisNodeDiskPressure
+        exp_alerts: []
+      - eval_time: 20m
+        alertname: FelisNodeDiskSpaceLow
+        exp_alerts:
+          - exp_labels:
+              device: /dev/vda1
+              fstype: xfs
+              instance: node1
+              job: node-exporter
+              mountpoint: /
+              severity: warning
+            exp_annotations:
+              summary: "node filesystem / below 15% available"
+              description: >-
+                Sustained disk pressure evicts game pods and garbage-collects images
+                (troubleshooting §13b). Free space before kubelet raises DiskPressure.
+      - eval_time: 20m
+        alertname: FelisNodeDiskPressure
+        exp_alerts:
+          - exp_labels:
+              condition: DiskPressure
+              node: n1
+              status: "true"
+              severity: critical
+            exp_annotations:
+              summary: "kubelet reports DiskPressure on n1"
+              description: >-
+                The eviction chain is in progress: control-plane pods hold
+                system-cluster-critical and survive, game pods do not. Free disk now
+                (troubleshooting §13b).
+      - eval_time: 20m
+        alertname: FelisNodeMemoryLow
+        exp_alerts:
+          - exp_labels:
+              instance: node1
+              job: node-exporter
+              severity: warning
+            exp_annotations:
+              summary: "node memory available below 10% for 15m"
+              description: >-
+                PostgreSQL, the control plane, the registry and game servers share one
+                node; sustained memory pressure risks OOM kills.
+  - name: database backup freshness
+    interval: 1m
+    input_series:
+      # The newest bundle was taken at t=0 and none since.
+      - series: 'felis_db_backup_last_success_timestamp_seconds{instance="node1",job="node-exporter",label="daily"}'
+        values: '0x1630'
+    alert_rule_test:
+      - eval_time: 25h
+        alertname: FelisDBBackupStale
+        exp_alerts: []
+      - eval_time: 27h
+        alertname: FelisDBBackupStale
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+            exp_annotations:
+              summary: "no control-plane database backup in over 26h"
+              description: >-
+                felis-db-backup.timer runs daily; the newest bundle is more than a day
+                old. Read `journalctl -u felis-db-backup` on the host, then take one now
+                with `sudo felis db backup` (troubleshooting §16).
+      - eval_time: 27h
+        alertname: FelisDBBackupMetricMissing
+        exp_alerts: []
+  - name: database backup freshness not scraped
+    interval: 1m
+    input_series:
+      - series: 'up{job="node-exporter"}'
+        values: '1x200'
+    alert_rule_test:
+      - eval_time: 1h
+        alertname: FelisDBBackupMetricMissing
+        exp_alerts: []
+      - eval_time: 3h
+        alertname: FelisDBBackupMetricMissing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "database backup freshness is not being scraped"
+              description: >-
+                No felis_db_backup_last_success_timestamp_seconds series, so
+                FelisDBBackupStale cannot fire. Point node-exporter's
+                --collector.textfile.directory at the directory of
+                FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
+                (troubleshooting §16).
+  - name: sign-in mail budget and relay
+    interval: 1m
+    input_series:
+      # Created at zero on start; the budget refuses one mail at t=3m.
+      - series: 'felis_mail_total{kind="otp",result="throttled",job="felis-api"}'
+        values: '0 0 0 1x30'
+      - series: 'felis_mail_total{kind="otp",result="failed",job="felis-api"}'
+        values: '0x33'
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: FelisMailBudgetExhausted
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: FelisMailBudgetExhausted
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "the install-wide mail budget refused mail"
+              description: >-
+                felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
+                spent, and every sign-in code is refused with 429 mail_rate_limited until
+                it refills. Check felis_rate_limited_total for a flood before raising the
+                budget (troubleshooting §17).
+      - eval_time: 5m
+        alertname: FelisMailDeliveryFailing
+        exp_alerts: []
+  - name: sign-in flood
+    interval: 1m
+    input_series:
+      # 30 refusals a minute from t=0; a lone refused script at 2/min stays quiet.
+      - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}'
+        values: '0+30x40'
+    alert_rule_test:
+      - eval_time: 10m
+        alertname: FelisSignInFlood
+        exp_alerts: []
+      - eval_time: 20m
+        alertname: FelisSignInFlood
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "sign-in doors refusing over 10 requests a minute"
+              description: >-
+                The per-address sign-in limit has been refusing callers for 10 minutes.
+                A script is hammering the auth doors; if real users report rate_limited
+                at once instead, [auth] client_ip_header is missing and everyone shares
+                the proxy's address (troubleshooting §17).
+  - name: sign-in trickle stays quiet
+    interval: 1m
+    input_series:
+      - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}'
+        values: '0+2x40'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: FelisSignInFlood
+        exp_alerts: []
+  - name: account email-code lock
+    interval: 1m
+    input_series:
+      - series: 'felis_auth_otp_lockouts_total{purpose="login_email",job="felis-api"}'
+        values: '0 0 1x90'
+      - series: 'felis_auth_otp_lockouts_total{purpose="op_login",job="felis-api"}'
+        values: '0x92'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: FelisOTPAccountLocked
+        exp_alerts: []
+      - eval_time: 10m
+        alertname: FelisOTPAccountLocked
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              purpose: login_email
+            exp_annotations:
+              summary: "an account's email-code sign-in locked after 10 wrong codes"
+              description: >-
+                Someone entered 10 wrong codes for one account within 24h (login_email).
+                The audit log names the account (action auth.otp.locked); the owner was
+                mailed. Unless they fumbled codes, someone is guessing at it
+                (troubleshooting §17).
+      - eval_time: 90m
+        alertname: FelisOTPAccountLocked
+        exp_alerts: []
+  - name: sign-in failure rate
+    interval: 1m
+    input_series:
+      # Two doors failing at 3/min between them from t=0.
+      - series: 'felis_auth_failures_total{door="login_email",reason="bad_code",job="felis-api"}'
+        values: '0+2x40'
+      - series: 'felis_auth_failures_total{door="op_login",reason="no_account",job="felis-api"}'
+        values: '0+1x40'
+    alert_rule_test:
+      - eval_time: 8m
+        alertname: FelisSignInFailures
+        exp_alerts: []
+      - eval_time: 25m
+        alertname: FelisSignInFailures
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "over 30 refused sign-ins in 15 minutes"
+              description: >-
+                Wrong codes, unknown addresses or bad passkey assertions well above people
+                mistyping: someone is guessing or enumerating. `sum by (door, reason)
+                (increase(felis_auth_failures_total[15m]))` shows where; the audit rows
+                (action auth.<door>.failed) carry each caller's client_ip (troubleshooting §17).
+  - name: people mistyping stays quiet
+    interval: 1m
+    input_series:
+      - series: 'felis_auth_failures_total{door="login_email",reason="bad_code",job="felis-api"}'
+        values: '0 0 1 1 2 2 3 3 4 4 5x30'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: FelisSignInFailures
+        exp_alerts: []
+  - name: audit rows lost
+    interval: 1m
+    input_series:
+      - series: 'felis_audit_write_failures_total{job="felis-api",instance="api-0"}'
+        values: '0 0 0 2x20'
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: FelisAuditWriteFailing
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: FelisAuditWriteFailing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              job: felis-api
+              instance: api-0
+            exp_annotations:
+              summary: "felis-api failed to write audit rows"
+              description: >-
+                The actions went through but their audit rows were lost. The felis-api log
+                names each lost row (`audit: lost ...`); the usual cause is PostgreSQL
+                being unreachable or out of disk.
+  - name: operator and api presence
+    interval: 1m
+    input_series:
+      - series: 'felis_build_info{component="operator",version="v1",job="felis-operator",instance="op-0"}'
+        values: '1x30'
+      # felis-api stops being scraped after 5m; the series goes stale 5m later.
+      - series: 'felis_build_info{component="api",version="v1",job="felis-api",instance="api-0"}'
+        values: '1x5'
+    alert_rule_test:
+      - eval_time: 25m
+        alertname: FelisOperatorDown
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: FelisAPIDown
+        exp_alerts: []
+      - eval_time: 25m
+        alertname: FelisAPIDown
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              component: api
+            exp_annotations:
+              summary: "felis-api is down or not scraped"
+              description: >-
+                No felis_build_info{component="api"} series for 10 minutes. The panel,
+                sign-in and the proxy's player lookups all go through felis-api. Check
+                `kubectl -n felis get deploy felis-api` and its log; if the pod is
+                healthy, the felis-api-internal Service is not being scraped
+                (troubleshooting §14).
+  - name: operator never scraped
+    interval: 1m
+    input_series:
+      - series: 'felis_build_info{component="api",version="v1",job="felis-api",instance="api-0"}'
+        values: '1x30'
+    alert_rule_test:
+      - eval_time: 5m
+        alertname: FelisOperatorDown
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: FelisOperatorDown
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              component: operator
+            exp_annotations:
+              summary: "felis-operator is down or not scraped"
+              description: >-
+                No felis_build_info{component="operator"} series for 10 minutes. Without
+                the operator no server starts, stops or recovers. Check
+                `kubectl -n felis get deploy felis-operator` and its log; if the pod is
+                healthy, the felis-operator-metrics Service is not being scraped
+                (troubleshooting §14).
+  - name: system and user servers down
+    interval: 1m
+    input_series:
+      - series: 'felis_server_phase{server="login",role="login",phase="Starting",desired="Running"}'
+        values: '1x20'
+      - series: 'felis_server_phase{server="lobby",role="lobby",phase="Failed",desired="Running"}'
+        values: '1x20'
+      # Stopped on purpose: not an outage.
+      - series: 'felis_server_phase{server="lobby2",role="lobby",phase="Stopped",desired="Stopped"}'
+        values: '1x20'
+      # A user server carries no role label (the operator publishes role="").
+      - series: 'felis_server_phase{server="survival",phase="Failed",desired="Running"}'
+        values: '1x20'
+      - series: 'felis_server_phase{server="creative",phase="Running",desired="Running"}'
+        values: '1x20'
+    alert_rule_test:
+      - eval_time: 9m
+        alertname: FelisLoginGateDown
+        exp_alerts: []
+      - eval_time: 11m
+        alertname: FelisLoginGateDown
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              server: login
+              role: login
+              phase: Starting
+              desired: Running
+            exp_annotations:
+              summary: "the login gate login is Starting"
+              description: >-
+                Every player connection passes through the login server first, so no one
+                can join. The MinecraftServer's conditions carry the reason:
+                `kubectl -n minecraft describe minecraftserver login`
+                (troubleshooting §1, §2).
+      - eval_time: 11m
+        alertname: FelisSystemServerDown
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              server: lobby
+              role: lobby
+              phase: Failed
+              desired: Running
+            exp_annotations:
+              summary: "system server lobby (lobby) is Failed"
+              description: >-
+                Players who sign in are sent to the lobby; while it is down they stay at
+                the gate. `kubectl -n minecraft describe minecraftserver lobby`
+                shows the reason (troubleshooting §1, §2).
+      - eval_time: 3m
+        alertname: FelisServerFailed
+        exp_alerts: []
+      - eval_time: 6m
+        alertname: FelisServerFailed
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              server: survival
+              phase: Failed
+              desired: Running
+            exp_annotations:
+              summary: "server survival is Failed"
+              description: >-
+                The operator gave up on this server (a crash loop, an image that will not
+                pull, a world volume that will not mount). Its conditions carry the
+                reason: `kubectl -n minecraft describe minecraftserver survival`
+                (troubleshooting §2).
+  - name: operator reconcile errors and a stuck pass
+    interval: 1m
+    input_series:
+      # Two failed reconciles a minute from 6m on.
+      - series: 'controller_runtime_reconcile_errors_total{controller="minecraftserver",job="felis-operator"}'
+        values: '0x5 0+2x20'
+      # One pass that started at 5m and never returns.
+      - series: 'workqueue_longest_running_processor_seconds{name="minecraftserver",controller="minecraftserver",job="felis-operator"}'
+        values: '0x5 60+60x20'
+    alert_rule_test:
+      - eval_time: 5m
+        alertname: FelisReconcileErrors
+        exp_alerts: []
+      - eval_time: 25m
+        alertname: FelisReconcileErrors
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "the operator failed over 10 reconciles in 15 minutes"
+              description: >-
+                Server changes are being retried instead of applied. The felis-operator
+                log names each failing server and its error.
+      - eval_time: 12m
+        alertname: FelisReconcileStuck
+        exp_alerts: []
+      - eval_time: 20m
+        alertname: FelisReconcileStuck
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+            exp_annotations:
+              summary: "an operator reconcile has been running for over 5 minutes"
+              description: >-
+                Each reconcile is bounded at 3 minutes, so this one is ignoring its
+                deadline and holding a worker. The liveness probe restarts the operator
+                once a pass passes 10 minutes; the log from before the restart shows
+                where it hung.
+  - name: world job failures and a late reaper
+    interval: 1m
+    input_series:
+      - series: 'kube_job_failed{namespace="minecraft",job_name="backup-survival-abc",condition="true"}'
+        values: '0x2 1x10'
+      - series: 'kube_job_failed{namespace="minecraft",job_name="backup-survival-abc",condition="false"}'
+        values: '1x2 0x10'
+      # A build Job in another namespace is FelisImageBuildFailures' business.
+      - series: 'kube_job_failed{namespace="felis-build",job_name="build-x",condition="true"}'
+        values: '1x12'
+      # Evaluation starts at the epoch, so "over a day ago" is a negative timestamp.
+      - series: 'kube_cronjob_status_last_successful_time{namespace="minecraft",cronjob="felis-reaper"}'
+        values: '-100000x30'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: FelisWorldJobFailed
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: FelisWorldJobFailed
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              namespace: minecraft
+              job_name: backup-survival-abc
+              condition: "true"
+            exp_annotations:
+              summary: "Job backup-survival-abc failed"
+              description: >-
+                A world backup, restore or reaper run failed; after a failed backup that
+                world's newest archive is older than planned.
+                `kubectl -n minecraft logs job/backup-survival-abc` has the error
+                (troubleshooting §10).
+      - eval_time: 5m
+        alertname: FelisReaperStale
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: FelisReaperStale
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              namespace: minecraft
+              cronjob: felis-reaper
+            exp_annotations:
+              summary: "the world reaper has not succeeded in over 26h"
+              description: >-
+                felis-reaper runs daily; idle worlds are neither backed up nor reclaimed
+                while it fails. `kubectl -n minecraft get jobs --sort-by=.metadata.creationTimestamp`
+                lists its runs, and the newest one's log
+                shows why (troubleshooting §10).
+  - name: a reaper that ran yesterday stays quiet
+    interval: 1m
+    input_series:
+      - series: 'kube_cronjob_status_last_successful_time{namespace="minecraft",cronjob="felis-reaper"}'
+        values: '-50000x30'
+    alert_rule_test:
+      - eval_time: 25m
+        alertname: FelisReaperStale
+        exp_alerts: []

deploy/alerts/felis-prometheusrule.yaml

@@ -0,0 +1,278 @@
+# prometheus-operator twin of felis-alerts.yaml (kube-prometheus-stack loads
+# rules through the PrometheusRule CRD, not rule_files). The plain file is the
+# promtool-tested source; keep the groups in sync.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: felis-alerts
+  namespace: monitoring
+  labels:
+    # Change to match your stack's ruleSelector (kube-prometheus-stack's
+    # default selects on the Helm release name).
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: felis.rules
+      rules:
+        - alert: FelisImageBuildFailures
+          expr: increase(felis_image_build_failures_total[6h]) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "modpack/image build failed in the last 6h"
+            description: >-
+              felis_image_build_failures_total increased. Inspect the failed build Job
+              (kubectl logs -n felis-build job/<build-job>); the same error text is on
+              GET /api/v1/images/build/{id} and in the submitter's row in the panel.
+        - alert: FelisSlowServerStarts
+          expr: histogram_quantile(0.9, sum by (le) (rate(felis_start_duration_seconds_bucket[30m]))) > 300
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "p90 server start time exceeds 5 minutes"
+            description: >-
+              Starts regularly take over five minutes (felis_start_duration_seconds,
+              observed when readiness is first reached). A start that never completes
+              records nothing — cross-check desiredState=Running servers with no ready
+              phase (troubleshooting §1).
+    - name: felis.platform.rules
+      rules:
+        - alert: FelisOperatorDown
+          expr: absent(felis_build_info{component="operator"})
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "felis-operator is down or not scraped"
+            description: >-
+              No felis_build_info{component="operator"} series for 10 minutes. Without
+              the operator no server starts, stops or recovers. Check
+              `kubectl -n felis get deploy felis-operator` and its log; if the pod is
+              healthy, the felis-operator-metrics Service is not being scraped
+              (troubleshooting §14).
+        - alert: FelisAPIDown
+          expr: absent(felis_build_info{component="api"})
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "felis-api is down or not scraped"
+            description: >-
+              No felis_build_info{component="api"} series for 10 minutes. The panel,
+              sign-in and the proxy's player lookups all go through felis-api. Check
+              `kubectl -n felis get deploy felis-api` and its log; if the pod is
+              healthy, the felis-api-internal Service is not being scraped
+              (troubleshooting §14).
+        - alert: FelisLoginGateDown
+          expr: felis_server_phase{role="login",desired="Running",phase!="Running"} == 1
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "the login gate {{ $labels.server }} is {{ $labels.phase }}"
+            description: >-
+              Every player connection passes through the login server first, so no one
+              can join. The MinecraftServer's conditions carry the reason:
+              `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+              (troubleshooting §1, §2).
+        - alert: FelisSystemServerDown
+          expr: felis_server_phase{role!="",role!="login",desired="Running",phase!="Running"} == 1
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "system server {{ $labels.server }} ({{ $labels.role }}) is {{ $labels.phase }}"
+            description: >-
+              Players who sign in are sent to the lobby; while it is down they stay at
+              the gate. `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+              shows the reason (troubleshooting §1, §2).
+        - alert: FelisServerFailed
+          expr: felis_server_phase{role="",phase="Failed"} == 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "server {{ $labels.server }} is Failed"
+            description: >-
+              The operator gave up on this server (a crash loop, an image that will not
+              pull, a world volume that will not mount). Its conditions carry the
+              reason: `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+              (troubleshooting §2).
+        - alert: FelisReconcileErrors
+          expr: sum(increase(controller_runtime_reconcile_errors_total{controller="minecraftserver"}[15m])) > 10
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "the operator failed over 10 reconciles in 15 minutes"
+            description: >-
+              Server changes are being retried instead of applied. The felis-operator
+              log names each failing server and its error.
+        - alert: FelisReconcileStuck
+          expr: max(workqueue_longest_running_processor_seconds{name="minecraftserver"}) > 300
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "an operator reconcile has been running for over 5 minutes"
+            description: >-
+              Each reconcile is bounded at 3 minutes, so this one is ignoring its
+              deadline and holding a worker. The liveness probe restarts the operator
+              once a pass passes 10 minutes; the log from before the restart shows
+              where it hung.
+    - name: felis.jobs.rules
+      rules:
+        - alert: FelisWorldJobFailed
+          expr: kube_job_failed{namespace="minecraft",condition="true"} == 1
+          labels:
+            severity: warning
+          annotations:
+            summary: "Job {{ $labels.job_name }} failed"
+            description: >-
+              A world backup, restore or reaper run failed; after a failed backup that
+              world's newest archive is older than planned.
+              `kubectl -n minecraft logs job/{{ $labels.job_name }}` has the error
+              (troubleshooting §10).
+        - alert: FelisReaperStale
+          expr: time() - kube_cronjob_status_last_successful_time{namespace="minecraft",cronjob="felis-reaper"} > 26 * 3600
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "the world reaper has not succeeded in over 26h"
+            description: >-
+              felis-reaper runs daily; idle worlds are neither backed up nor reclaimed
+              while it fails. `kubectl -n minecraft get jobs --sort-by=.metadata.creationTimestamp`
+              lists its runs, and the newest one's log
+              shows why (troubleshooting §10).
+    - name: felis.node.rules
+      rules:
+        - alert: FelisNodeDiskSpaceLow
+          expr: >-
+            node_filesystem_avail_bytes{fstype=~"ext4|xfs|btrfs"}
+            / node_filesystem_size_bytes{fstype=~"ext4|xfs|btrfs"} < 0.15
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "node filesystem {{ $labels.mountpoint }} below 15% available"
+            description: >-
+              Sustained disk pressure evicts game pods and garbage-collects images
+              (troubleshooting §13b). Free space before kubelet raises DiskPressure.
+        - alert: FelisNodeDiskPressure
+          expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "kubelet reports DiskPressure on {{ $labels.node }}"
+            description: >-
+              The eviction chain is in progress: control-plane pods hold
+              system-cluster-critical and survive, game pods do not. Free disk now
+              (troubleshooting §13b).
+        - alert: FelisNodeMemoryLow
+          expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.10
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "node memory available below 10% for 15m"
+            description: >-
+              PostgreSQL, the control plane, the registry and game servers share one
+              node; sustained memory pressure risks OOM kills.
+    - name: felis.backup.rules
+      rules:
+        - alert: FelisDBBackupStale
+          expr: time() - max(felis_db_backup_last_success_timestamp_seconds) > 26 * 3600
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "no control-plane database backup in over 26h"
+            description: >-
+              felis-db-backup.timer runs daily; the newest bundle is more than a day
+              old. Read `journalctl -u felis-db-backup` on the host, then take one now
+              with `sudo felis db backup` (troubleshooting §16).
+        - alert: FelisDBBackupMetricMissing
+          expr: absent(felis_db_backup_last_success_timestamp_seconds)
+          for: 2h
+          labels:
+            severity: warning
+          annotations:
+            summary: "database backup freshness is not being scraped"
+            description: >-
+              No felis_db_backup_last_success_timestamp_seconds series, so
+              FelisDBBackupStale cannot fire. Point node-exporter's
+              --collector.textfile.directory at the directory of
+              FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
+              (troubleshooting §16).
+    - name: felis.auth.rules
+      rules:
+        - alert: FelisMailBudgetExhausted
+          expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "the install-wide mail budget refused mail"
+            description: >-
+              felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
+              spent, and every sign-in code is refused with 429 mail_rate_limited until
+              it refills. Check felis_rate_limited_total for a flood before raising the
+              budget (troubleshooting §17).
+        - alert: FelisMailDeliveryFailing
+          expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "the SMTP relay refused mail in the last 15m"
+            description: >-
+              felis_mail_total{result="failed"} increased: sign-in codes are not being
+              delivered (502 mail_undeliverable). The relay's reason is in the
+              felis-api log (troubleshooting §17).
+        - alert: FelisSignInFlood
+          expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "sign-in doors refusing over 10 requests a minute"
+            description: >-
+              The per-address sign-in limit has been refusing callers for 10 minutes.
+              A script is hammering the auth doors; if real users report rate_limited
+              at once instead, [auth] client_ip_header is missing and everyone shares
+              the proxy's address (troubleshooting §17).
+        - alert: FelisOTPAccountLocked
+          expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "an account's email-code sign-in locked after 10 wrong codes"
+            description: >-
+              Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}).
+              The audit log names the account (action auth.otp.locked); the owner was
+              mailed. Unless they fumbled codes, someone is guessing at it
+              (troubleshooting §17).
+        - alert: FelisSignInFailures
+          expr: sum(increase(felis_auth_failures_total[15m])) > 30
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "over 30 refused sign-ins in 15 minutes"
+            description: >-
+              Wrong codes, unknown addresses or bad passkey assertions well above people
+              mistyping: someone is guessing or enumerating. `sum by (door, reason)
+              (increase(felis_auth_failures_total[15m]))` shows where; the audit rows
+              (action auth.<door>.failed) carry each caller's client_ip (troubleshooting §17).
+        - alert: FelisAuditWriteFailing
+          expr: increase(felis_audit_write_failures_total[15m]) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "felis-api failed to write audit rows"
+            description: >-
+              The actions went through but their audit rows were lost. The felis-api log
+              names each lost row (`audit: lost ...`); the usual cause is PostgreSQL
+              being unreachable or out of disk.

deploy/crd/felis.lolicon.best_minecraftservers.yaml

@@ -101,6 +101,8 @@ spec:
                       EmptySecondsBeforeStop is how long the server may sit empty before the
                       operator scales it down.
                     format: int32
+                    maximum: 604800
+                    minimum: 0
                     type: integer
                 type: object
               image:
@@ -123,14 +125,12 @@ spec:
               lifecycle:
                 description: Lifecycle tunes graceful shutdown (spec §7).
                 properties:
-                  preStopSaveAndStop:
-                    description: PreStopSaveAndStop enables the operator-injected
-                      RCON save+stop preStop.
-                    type: boolean
                   terminationGracePeriodSeconds:
                     description: TerminationGracePeriodSeconds is the pod grace period
                       (default 300).
                     format: int64
+                    maximum: 3600
+                    minimum: 0
                     type: integer
                 type: object
               motd:
@@ -163,6 +163,8 @@ spec:
                   port:
                     description: Port is the RCON TCP port (default 25575).
                     format: int32
+                    maximum: 65535
+                    minimum: 0
                     type: integer
                   secretRef:
                     description: SecretRef points at the Secret holding the RCON password.
@@ -176,6 +178,10 @@ spec:
                     - name
                     type: object
                 type: object
+                x-kubernetes-validations:
+                - message: the allow-rcon NetworkPolicy admits only port 25575; leave
+                    port unset
+                  rule: '!has(self.port) || self.port == 0 || self.port == 25575'
               reaperExempt:
                 description: ReaperExempt opts this server out of the world reaper
                   entirely (spec §18).
@@ -254,16 +260,22 @@ spec:
                       (notably LOOHP/Limbo) where the felis-limbo plugin reports true
                       readiness only after the first server tick.
                     format: int32
+                    maximum: 65535
+                    minimum: 0
                     type: integer
                   readinessTimeoutSeconds:
                     description: ReadinessTimeoutSeconds is the budget for the first
                       successful RCON probe.
                     format: int32
+                    maximum: 86400
+                    minimum: 0
                     type: integer
                   timeoutSeconds:
                     description: TimeoutSeconds is the overall budget before the server
                       is marked Failed.
                     format: int32
+                    maximum: 86400
+                    minimum: 0
                     type: integer
                 type: object
               storage:
@@ -289,6 +301,13 @@ spec:
           status:
             description: MinecraftServerStatus is the observed state (spec §4 status.*).
             properties:
+              autoRestarts:
+                description: |-
+                  AutoRestarts counts how often the operator recreated the pod of a start
+                  that timed out (at most 3, with a doubling backoff); reaching Ready or
+                  stopping resets it.
+                format: int32
+                type: integer
               conditions:
                 description: Conditions are the standard metav1 conditions (Ready,
                   RconReached, ...).
@@ -347,6 +366,14 @@ spec:
                   - type
                   type: object
                 type: array
+              emptySince:
+                description: |-
+                  EmptySince is when the operator first observed 0 online players during
+                  a Running phase (spec §8 idle auto-stop). It is reset when a player joins
+                  or the server stops, so the empty-duration counter starts fresh each time
+                  the server becomes unoccupied.
+                format: date-time
+                type: string
               endpoint:
                 description: Endpoint is where the proxy should route traffic.
                 properties:
@@ -357,6 +384,11 @@ spec:
                     description: Mode is "direct" or "fallback".
                     type: string
                 type: object
+              lastAutoRestartAt:
+                description: LastAutoRestartAt is when the operator last recreated
+                  the pod.
+                format: date-time
+                type: string
               liveMotd:
                 description: LiveMotd is the MOTD currently advertised for the active
                   phase.

deploy/demo-up.sh

@@ -1,30 +1,28 @@
 #!/bin/bash
 # demo-up.sh — one-shot Felis demo bring-up.
 #
-# Collapses the four manual steps (bootstrap -> build/import limbo+lobby images ->
-# edit felis.toml -> felis setup) into a single command:
-#
 #     sudo bash deploy/demo-up.sh
 #
-# It ends by exec'ing the interactive `felis setup` TUI (create the Owner account) —
-# that human step is the only thing this script cannot do for you.
+# Every piece a demo box needs — base platform (k3s + felis + docker + cloudflared +
+# control plane), the limbo/lobby/paper images, the felis-velocity proxy plugin, and
+# the [velocity] wiring in felis.host.toml — is built by deploy/bootstrap.sh. This
+# wrapper adds only the one step the installer cannot do: the interactive
+# `felis setup` TUI that creates the Owner account.
 #
-# Image source, in order of preference:
-#   1. Prebuilt tars at deploy/images/felis-limbo.tar + felis-lobby.tar (imported as-is).
-#   2. Otherwise built on this host with docker, resolving the LOOHP/Limbo CI jar and
-#      the latest stable Paper jar automatically. Override any of:
-#        LIMBO_JAR_URL LIMBO_SCHEM_URL LIMBO_VERSION PAPER_JAR_URL PAPER_JAR_SHA256
-#        PAPER_MC_VERSION
+# It used to rebuild the game images here with its own copy of that logic, written
+# before bootstrap grew the job. The copy drifted: it pinned Paper 1.21.8 while the
+# installer derives one version from the Limbo login gate (both hops of a login must
+# speak one protocol), never built felis-velocity.jar (so the proxy it wired had
+# nowhere to route), and left docker running. The installer is the single origin.
 #
-# Toggles: SKIP_BOOTSTRAP=1 (base already up), SKIP_SETUP=1 (stop before the TUI).
+# Toggles: SKIP_BOOTSTRAP=1 (base + game stack already installed by a full bootstrap),
+#          SKIP_SETUP=1 (stop before the TUI).
 set -Eeuo pipefail
 
 SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 STATE_DIR=/etc/felis
 HOST_TOML="$STATE_DIR/felis.host.toml"
-IMG_DIR="$SRC_DIR/deploy/images"
-LIMBO_IMAGE="felis-limbo:demo"
-LOBBY_IMAGE="felis-lobby:demo"
+PLUGIN_JAR=/opt/felis/velocity/plugins/felis-velocity.jar
 K3S=/usr/local/bin/k3s
 FELIS=/usr/local/bin/felis
 
@@ -33,11 +31,11 @@ die() { printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
 
 [ "$(id -u)" -eq 0 ] || die "run as root (sudo bash deploy/demo-up.sh)"
 
-# 1. base platform (k3s + felis + docker + cloudflared + control plane) ----------
+# 1. base platform + full game stack (deploy/bootstrap.sh) -----------------------
 if [ "${SKIP_BOOTSTRAP:-0}" = 1 ]; then
   log "SKIP_BOOTSTRAP=1 — assuming the base platform is already up"
 else
-  log "bringing up the base platform (deploy/bootstrap.sh)"
+  log "bringing up the base platform and the game stack (deploy/bootstrap.sh)"
   bash "$SRC_DIR/deploy/bootstrap.sh"
 fi
 
@@ -46,83 +44,19 @@ command -v "$K3S"   >/dev/null 2>&1 || K3S=k3s
 command -v "$K3S"   >/dev/null 2>&1 || die "k3s not found — did bootstrap complete?"
 command -v "$FELIS" >/dev/null 2>&1 || die "felis not found — did bootstrap complete?"
 
-# 2. get the two game images into k3s containerd --------------------------------
-if [ -f "$IMG_DIR/felis-limbo.tar" ] && [ -f "$IMG_DIR/felis-lobby.tar" ]; then
-  log "importing prebuilt image tars from $IMG_DIR"
-  "$K3S" ctr images import "$IMG_DIR/felis-limbo.tar"
-  "$K3S" ctr images import "$IMG_DIR/felis-lobby.tar"
-  # Optional: the plain-Paper recommended base, if a tar was staged for it.
-  [ -f "$IMG_DIR/felis-paper.tar" ] && "$K3S" ctr images import "$IMG_DIR/felis-paper.tar"
-else
-  log "no prebuilt tars in $IMG_DIR — building on this host with docker"
-  command -v docker >/dev/null 2>&1 || die "docker not found; cannot build images"
-
-  rel=$(curl -fsSL --max-time 30 "https://ci.loohpjames.com/job/Limbo/lastSuccessfulBuild/api/json" \
-          | grep -oE 'target/Limbo-[0-9][^"]+\.jar' | head -1) || true
-  : "${LIMBO_JAR_URL:=https://ci.loohpjames.com/job/Limbo/lastSuccessfulBuild/artifact/$rel}"
-  : "${LIMBO_SCHEM_URL:=https://ci.loohpjames.com/job/Limbo/lastSuccessfulBuild/artifact/spawn.schem}"
-  : "${LIMBO_VERSION:=$(basename "$rel" | sed -E 's/^Limbo-//; s/\.jar$//; s/-[0-9]+\.[0-9]+$//')}"
-  [ -n "$rel" ] || [ -n "${LIMBO_JAR_URL##*artifact/}" ] || die "could not resolve the Limbo jar; set LIMBO_JAR_URL"
-  log "building $LIMBO_IMAGE (Limbo $LIMBO_VERSION)"
-  docker build -f "$SRC_DIR/deploy/limbo/Dockerfile" \
-    --build-arg LIMBO_JAR_URL="$LIMBO_JAR_URL" \
-    --build-arg LIMBO_SCHEM_URL="$LIMBO_SCHEM_URL" \
-    --build-arg LIMBO_VERSION="$LIMBO_VERSION" \
-    -t "$LIMBO_IMAGE" "$SRC_DIR"
-  docker save "$LIMBO_IMAGE" | "$K3S" ctr images import -
-
-  : "${PAPER_MC_VERSION:=1.21.8}"
-  : "${PAPER_JAR_URL:=$(curl -fsSL --max-time 30 "https://fill.papermc.io/v3/projects/paper/versions/${PAPER_MC_VERSION}/builds/latest" | grep -oE 'https://fill-data\.papermc\.io/[^"]+\.jar' | head -1)}"
-  [ -n "$PAPER_JAR_URL" ] || die "could not resolve the Paper jar; set PAPER_JAR_URL"
-  # Both Dockerfiles require the jar's digest. The fill-data URL is content-addressed
-  # (the objects/ path segment IS the sha256), so it is derived rather than asked for;
-  # a mirror override carries no such segment and must bring its own digest.
-  if [ -z "${PAPER_JAR_SHA256:-}" ]; then
-    sha="${PAPER_JAR_URL#*/objects/}"
-    sha="${sha%%/*}"
-    case "$sha" in
-      *[!0-9a-f]*|"") sha="" ;;
-    esac
-    if [ "${#sha}" -ne 64 ]; then
-      die "cannot derive the Paper jar sha256 from PAPER_JAR_URL (not a content-addressed fill-data URL); set PAPER_JAR_SHA256"
-    fi
-    PAPER_JAR_SHA256="$sha"
-  fi
-  log "building $LOBBY_IMAGE (Paper $PAPER_MC_VERSION)"
-  docker build -f "$SRC_DIR/deploy/lobby/Dockerfile" \
-    --build-arg PAPER_JAR_URL="$PAPER_JAR_URL" \
-    --build-arg PAPER_JAR_SHA256="$PAPER_JAR_SHA256" \
-    -t "$LOBBY_IMAGE" "$SRC_DIR"
-  docker save "$LOBBY_IMAGE" | "$K3S" ctr images import -
-
-  # Plain Paper recommended base — same PAPER_JAR_URL, no plugins, no secret gate.
-  : "${PAPER_IMAGE:=felis-paper:demo}"
-  log "building $PAPER_IMAGE (plain Paper $PAPER_MC_VERSION, forwarding via the operator initContainer)"
-  docker build -f "$SRC_DIR/deploy/paper/Dockerfile" \
-    --build-arg PAPER_JAR_URL="$PAPER_JAR_URL" \
-    --build-arg PAPER_JAR_SHA256="$PAPER_JAR_SHA256" \
-    -t "$PAPER_IMAGE" "$SRC_DIR"
-  docker save "$PAPER_IMAGE" | "$K3S" ctr images import -
-fi
-
-# 3. wire the images into the config `felis setup` reads ------------------------
-log "wiring [velocity] images into $HOST_TOML"
+# SKIP_BOOTSTRAP=1 trusts an earlier run to be complete. Check that it actually left
+# the full stack behind: a base from before the game-stack installer, or one whose
+# pieces were pruned by hand, must fail here with a pointer — not present as a proxy
+# that accepts logins and routes nowhere, with nothing in any log to say why.
 [ -f "$HOST_TOML" ] || die "missing $HOST_TOML — did bootstrap run?"
-if grep -q '^\[velocity\]' "$HOST_TOML"; then
-  echo "  [velocity] table already present — leaving it untouched"
-else
-  cat >> "$HOST_TOML" <<EOF
+grep -q '^\[velocity\]' "$HOST_TOML" \
+  || die "$HOST_TOML has no [velocity] section — re-run the installer without SKIP_BOOTSTRAP so the system servers get wired"
+[ -f "$PLUGIN_JAR" ] \
+  || die "$PLUGIN_JAR missing — this base did not finish the full installer, and a proxy without it silently routes nothing; re-run the installer without SKIP_BOOTSTRAP"
 
-[velocity]
-login_image = "$LIMBO_IMAGE"
-lobby_image = "$LOBBY_IMAGE"
-EOF
-  echo "  appended login_image=$LIMBO_IMAGE / lobby_image=$LOBBY_IMAGE"
-fi
-
-# 4. interactive Owner creation + system-server provisioning -------------------
+# 2. interactive Owner creation + system-server provisioning --------------------
 if [ "${SKIP_SETUP:-0}" = 1 ]; then
-  log "SKIP_SETUP=1 — base + images + config ready. Finish with:  sudo felis setup"
+  log "SKIP_SETUP=1 — base + game stack ready. Finish with:  sudo felis setup"
 else
   log "launching 'felis setup' — create the Owner account (this is the only interactive step)"
   exec "$FELIS" setup

deploy/e2e_check.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# Checks a host that deploy/bootstrap.sh just installed (or re-ran on). The e2e workflow
+# runs it on a fresh GitHub runner after each of its two installer runs:
+#
+#   sudo bash deploy/e2e_check.sh install   # after the first run
+#   sudo bash deploy/e2e_check.sh rerun     # after the same commit ran again
+#   sudo bash deploy/e2e_check.sh release   # after the newest release installed
+#   sudo bash deploy/e2e_check.sh upgrade   # after this commit ran over a release
+#
+# It asks what an operator's first minutes ask: the binary runs, the control plane is
+# rolled out and ready, the panel answers on its NodePort, the proxy answers a Minecraft
+# status ping, and the host timers are there. A rerun must also leave the proxy running
+# (it restarts only when what it runs changed) and keep every earlier answer.
+set -euo pipefail
+
+phase="${1:?usage: e2e_check.sh install|rerun|release|upgrade}"
+export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+KUBECTL=(/usr/local/bin/k3s kubectl)
+PID_FILE=/var/tmp/felis-e2e-velocity.pid
+fails=0
+
+pass() { printf 'PASS %s\n' "$*"; }
+fail() { printf 'FAIL %s\n' "$*"; fails=$((fails + 1)); }
+check() { # label command...
+  local label="$1"
+  shift
+  if "$@"; then pass "$label"; else fail "$label"; fi
+}
+
+check "felis version runs" sh -c '/usr/local/bin/felis version | grep -q "^felis "'
+
+for d in felis-api felis-operator registry; do
+  check "deployment ${d} is rolled out" "${KUBECTL[@]}" -n felis rollout status "deploy/${d}" --timeout=180s
+done
+
+node_ip="$(ip -4 route get 1.1.1.1 | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}')"
+check "the panel serves its page on the NodePort" \
+  sh -c "curl -skf --retry 10 --retry-delay 3 --retry-all-errors https://${node_ip}:30443/ | grep -qi '<html'"
+# Readiness lives on the internal face only; a Service ClusterIP routes from the node.
+internal="$("${KUBECTL[@]}" -n felis get svc felis-api-internal -o jsonpath='{.spec.clusterIP}:{.spec.ports[0].port}')"
+check "felis-api is ready (database and cluster reachable)" \
+  curl -sf --retry 10 --retry-delay 3 --retry-all-errors -o /dev/null "http://${internal}/readyz"
+
+for unit in k3s postgresql felis-velocity; do
+  check "${unit} is active" systemctl is-active --quiet "$unit"
+done
+# A release may predate a timer; what this commit installs has them all.
+if [ "$phase" != release ]; then
+  for timer in felis-db-backup.timer felis-watchdog.timer felis-update-check.timer; do
+    check "${timer} is scheduled" systemctl is-enabled --quiet "$timer"
+  done
+fi
+
+# A status ping is the proxy's own answer (ping passthrough is off), so it proves the JRE,
+# Velocity and its config without a login gate or a Mojang account.
+ping_proxy() {
+  python3 - <<'EOF'
+import json, socket, struct, sys
+
+def varint(n):
+    out = b""
+    n &= 0xFFFFFFFF
+    while True:
+        b, n = n & 0x7F, n >> 7
+        if n:
+            out += bytes([b | 0x80])
+        else:
+            return out + bytes([b])
+
+def read_varint(s):
+    n = 0
+    for i in range(5):
+        b = s.recv(1)
+        if not b:
+            raise EOFError("connection closed")
+        n |= (b[0] & 0x7F) << (7 * i)
+        if not b[0] & 0x80:
+            return n
+    raise ValueError("varint too long")
+
+host, port = "127.0.0.1", 25565
+s = socket.create_connection((host, port), timeout=10)
+hs = varint(0) + varint(767) + varint(len(host)) + host.encode() + struct.pack(">H", port) + varint(1)
+s.sendall(varint(len(hs)) + hs + varint(1) + varint(0))
+read_varint(s)
+read_varint(s)
+size = read_varint(s)
+data = b""
+while len(data) < size:
+    chunk = s.recv(size - len(data))
+    if not chunk:
+        raise EOFError("short status response")
+    data += chunk
+print(json.loads(data)["version"]["name"])
+EOF
+}
+# The installer returns once the unit is started; the JVM binds the port a few
+# seconds later.
+for _ in $(seq 30); do
+  if version="$(ping_proxy 2>&1)"; then
+    break
+  fi
+  sleep 2
+done
+if version="$(ping_proxy 2>&1)"; then
+  pass "the proxy answers a status ping (${version})"
+else
+  fail "the proxy answers a status ping: ${version}"
+fi
+
+pid="$(systemctl show -p MainPID --value felis-velocity)"
+case "$phase" in
+  install | release) printf '%s\n' "$pid" > "$PID_FILE" ;;
+  rerun)
+    if [ "$pid" = "$(cat "$PID_FILE" 2>/dev/null)" ]; then
+      pass "the rerun left the proxy running (pid ${pid})"
+    else
+      fail "the rerun restarted the proxy (pid $(cat "$PID_FILE" 2>/dev/null || echo '?') -> ${pid}) though nothing it runs changed"
+    fi
+    ;;
+  upgrade) ;; # a new release may well change what the proxy runs
+  *) fail "unknown phase ${phase}" ;;
+esac
+
+if [ "$fails" -eq 0 ]; then
+  echo "ALL PASS (${phase})"
+else
+  echo "${fails} FAILED (${phase})"
+fi
+exit "$fails"

deploy/game-stack.lock

@@ -0,0 +1,23 @@
+# The upstream builds this release installs. deploy/bootstrap.sh reads this file (the
+# default FELIS_GAME_STACK=pinned), downloads exactly these artifacts and refuses any whose
+# sha256 differs, so every host installing one release gets the same login gate, lobby,
+# plain-Paper image and proxy, and a rerun rebuilds nothing that did not change.
+#
+# MC_VERSION is the protocol the whole stack speaks: Limbo speaks exactly one, and Paper
+# follows it so a client that passes the login gate can also reach the lobby.
+#
+# Refresh with deploy/update-game-stack-lock.sh, which resolves upstream's newest builds and
+# hashes them. Plain KEY=value lines only; bootstrap reads it without evaluating it.
+MC_VERSION=26.3
+LIMBO_VERSION=2026.0.3-ALPHA
+LIMBO_JAR_URL=https://ci.loohpjames.com/job/Limbo/76/artifact/target/Limbo-2026.0.3-ALPHA-26.3.jar
+LIMBO_JAR_SHA256=a2de91fcaa2255aed8a111b8786f370213423c7798eee778c00d016d83aa65d2
+LIMBO_SCHEM_URL=https://ci.loohpjames.com/job/Limbo/76/artifact/spawn.schem
+LIMBO_SCHEM_SHA256=70c85dae2db157971ef513e318820c5a5e10a96b813b70e21f8af2b632cbcbc7
+PAPER_JAR_URL=https://fill-data.papermc.io/v1/objects/49399919246cbf443efc8507447dc948eb7477c41be560b0e87e2a455aff824a/paper-26.3-40.jar
+PAPER_JAR_SHA256=49399919246cbf443efc8507447dc948eb7477c41be560b0e87e2a455aff824a
+LUCKPERMS_JAR_URL=https://download.luckperms.net/1672/bukkit/loader/LuckPerms-Bukkit-5.5.85.jar
+LUCKPERMS_JAR_SHA256=dc637ce18f48d3b75a7ffd1784b85be16a627359090dfe5adf15dab6d145dc7d
+VELOCITY_VERSION=3.5.1
+VELOCITY_JAR_URL=https://fill-data.papermc.io/v1/objects/b4e3164df5377346854dc6cb9e6a78022b1946ff69e89676313f5f6f1c6f0fb3/velocity-3.5.1-615.jar
+VELOCITY_JAR_SHA256=b4e3164df5377346854dc6cb9e6a78022b1946ff69e89676313f5f6f1c6f0fb3

deploy/limbo/Dockerfile

@@ -10,10 +10,11 @@
 # There is no bundled server.properties — Limbo writes a default on first run.
 # So the runtime is assembled from those two URLs (not a zip) via --build-arg:
 #
+#   . <(grep '^LIMBO_' deploy/game-stack.lock)
 #   docker build -f deploy/limbo/Dockerfile \
-#     --build-arg LIMBO_JAR_URL=https://ci.loohpjames.com/job/Limbo/<n>/artifact/target/Limbo-<ver>.jar \
-#     --build-arg LIMBO_SCHEM_URL=https://ci.loohpjames.com/job/Limbo/<n>/artifact/spawn.schem \
-#     --build-arg LIMBO_VERSION=<maven-api-version> \
+#     --build-arg LIMBO_JAR_URL="$LIMBO_JAR_URL" --build-arg LIMBO_JAR_SHA256="$LIMBO_JAR_SHA256" \
+#     --build-arg LIMBO_SCHEM_URL="$LIMBO_SCHEM_URL" --build-arg LIMBO_SCHEM_SHA256="$LIMBO_SCHEM_SHA256" \
+#     --build-arg LIMBO_VERSION="$LIMBO_VERSION" \
 #     -t felis-limbo:demo .
 #
 # Note LIMBO_VERSION (the maven API version the plugin compiles against, e.g.
@@ -28,43 +29,59 @@
 #      overridable via FELIS_HEALTH_PORT) and returns 200 only after the first tick.
 
 # ---- build the felis-limbo plugin jar ----
-# gradle:*-jdk21 — an official Gradle image on JDK 21. JDK 21 is required because
-# current LOOHP/Limbo releases ship Java 21 API classes (class-file major 65); a
-# JDK 17 fails to read them with "wrong version 65.0, should be 61.0". The image
-# also provides the `gradle` binary (this tree vendors no Gradle wrapper).
-# build.gradle still targets release 17 bytecode so the plugin loads on Java 17+.
-FROM gradle:8.14-jdk21 AS plugin
+# The same pinned Gradle image as the lobby build (see deploy/lobby/Dockerfile). Its JDK
+# must be >= 21 because current LOOHP/Limbo releases ship Java 21 API classes
+# (class-file major 65); a JDK 17 fails to read them with "wrong version 65.0, should be
+# 61.0". build.gradle still targets release 17 bytecode so the plugin loads on Java 17+.
+FROM gradle:9.8.0-jdk25@sha256:2b2fc1b1dfc3604a2acc916839f36eb5ee48fd7f232427fc5faca224c73bcb01 AS plugin
 WORKDIR /src
 # Copy what the limbo module needs: its own tree plus the shared link core it
 # srcDir-includes (../shared/src/main/java → /src/plugins/shared/src/main/java), so
 # the account-link client + config loader compile straight into the jar.
 COPY plugins/limbo/ ./plugins/limbo/
 COPY plugins/shared/ ./plugins/shared/
-ARG LIMBO_VERSION=+
-RUN cd plugins/limbo \
-    && (test -x ./gradlew && ./gradlew --no-daemon -PlimboVersion="$LIMBO_VERSION" build \
-        || gradle --no-daemon -PlimboVersion="$LIMBO_VERSION" build) \
+# The Limbo API release to compile against: deploy/game-stack.lock's LIMBO_VERSION, which
+# bootstrap passes. Required — the API is checked against the checksum
+# plugins/limbo/gradle/verification-metadata.xml holds for that release.
+ARG LIMBO_VERSION
+RUN if [ -z "${LIMBO_VERSION:-}" ]; then \
+        echo "LIMBO_VERSION is required (deploy/game-stack.lock)" >&2; exit 1; \
+    fi \
+    && cd plugins/limbo \
+    && gradle --no-daemon -PlimboVersion="$LIMBO_VERSION" build \
     && cp build/libs/*.jar /felis-limbo.jar
 
 # ---- assemble the runtime ----
 # 21-jre: the Limbo jar is Java 21 bytecode (class-file major 65), so a Java 17
 # JRE cannot run it (UnsupportedClassVersionError). A 21 JRE also runs the
 # plugin's release-17 bytecode fine.
-FROM eclipse-temurin:21-jre
+FROM eclipse-temurin:21-jre@sha256:49e21e16e3c86eb7816a44a67549910ed090fbeb40c29c525d58bf5e02e91b0f
 ARG LIMBO_JAR_URL
+ARG LIMBO_JAR_SHA256
 ARG LIMBO_SCHEM_URL
+ARG LIMBO_SCHEM_SHA256
 WORKDIR /limbo
 # Pull the two loose LOOHP/Limbo CI artifacts: the server jar (required, saved as
-# Limbo.jar) and the default spawn schematic (optional). Fail loudly if the jar
-# URL was not supplied.
+# Limbo.jar) and the default spawn schematic (optional). Each is checked against the
+# digest deploy/game-stack.lock names (bootstrap.sh passes it): the login gate is the
+# first thing every player's connection reaches, and Limbo's CI publishes no digest of
+# its own.
 RUN set -eu; \
     if [ -z "${LIMBO_JAR_URL:-}" ]; then \
       echo "ERROR: --build-arg LIMBO_JAR_URL=<Limbo server jar> is required" >&2; exit 1; \
     fi; \
+    if [ -z "${LIMBO_JAR_SHA256:-}" ]; then \
+      echo "ERROR: --build-arg LIMBO_JAR_SHA256=<Limbo jar sha256> is required" >&2; exit 1; \
+    fi; \
+    if [ -n "${LIMBO_SCHEM_URL:-}" ] && [ -z "${LIMBO_SCHEM_SHA256:-}" ]; then \
+      echo "ERROR: --build-arg LIMBO_SCHEM_SHA256=<spawn.schem sha256> is required with LIMBO_SCHEM_URL" >&2; exit 1; \
+    fi; \
     apt-get update && apt-get install -y --no-install-recommends curl ca-certificates; \
     curl -fSL "$LIMBO_JAR_URL" -o /limbo/Limbo.jar; \
+    echo "$LIMBO_JAR_SHA256  /limbo/Limbo.jar" | sha256sum -c; \
     if [ -n "${LIMBO_SCHEM_URL:-}" ]; then \
       curl -fSL "$LIMBO_SCHEM_URL" -o /limbo/spawn.schem; \
+      echo "$LIMBO_SCHEM_SHA256  /limbo/spawn.schem" | sha256sum -c; \
     fi; \
     apt-get purge -y curl && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*; \
     mkdir -p /limbo/plugins
@@ -78,6 +95,13 @@ COPY deploy/limbo/entrypoint.sh /usr/local/bin/felis-entrypoint.sh
 # The operator mounts the world PVC at /data. Runtime state lives there; /limbo
 # remains the immutable image seed copied into the volume by the entrypoint.
 WORKDIR /data
+# Run as the game uid (naming.GameUID in the Go tree). The operator pins the same uid in
+# the pod securityContext whatever USER an image declares; declaring it here as well
+# keeps a plain `docker run` of this image off root, and chowning the empty /data seed
+# lets that run write its world. The jar seed above stays root-owned and read-only to
+# the server.
+RUN chown 1000:1000 /data
+USER 1000:1000
 
 ENV FELIS_HEALTH_PORT=8080
 # FELIS_GAME_PORT is the port the entrypoint pins Limbo to; it MUST equal the operator's

deploy/limbo/README.md

@@ -90,14 +90,21 @@ docker build -f deploy/limbo/Dockerfile \
   version `2026.0.2-ALPHA` (the `-26.2` CI qualifier is not published to the
   maven repo).
 
-Import into k3s and point config at it:
+Publish it into the cluster's registry and point config at it. On the node
+itself (docker treats `127.0.0.1` as insecure by default):
 
 ```
-docker save felis-limbo:demo | sudo k3s ctr images import -
-# felis.toml → [velocity] login_image = "felis-limbo:demo"
+docker tag  felis-limbo:demo 127.0.0.1:5000/felis/limbo:demo
+docker push 127.0.0.1:5000/felis/limbo:demo
+# felis.toml → [velocity] login_image = "registry.felis.svc:5000/felis/limbo:demo"
 sudo felis setup
 ```
 
+The registry keys a repository by the path after the host, so pushing through a
+`kubectl -n felis port-forward svc/registry 5000:5000` from another machine is
+equivalent. Hosting the image in the registry (rather than only importing it
+into containerd) is what lets kubelet re-pull it after an image GC.
+
 ## Ports (handled for you)
 
 The entrypoint (`deploy/limbo/entrypoint.sh`) pins Limbo's `server-port` to
@@ -125,10 +132,13 @@ set them by hand:
   `spec.env` by `felis setup` (`cmd/felis` derives the internal API URL from the
   control namespace — the platform default `felis`; a renamed control namespace must
   be reflected by hand — and the root domain from `felis.toml`).
-- `FELIS_SERVICE_TOKEN` is a **secret**, so it is never written into the CRD. `felis
-  setup` replicates the `felis-service-token` Secret from the control namespace into
-  the minecraft namespace, and the operator injects it into the `login` pod (only)
-  via a `secretKeyRef`, keyed off the reserved `login` name. Until the token is
+- `FELIS_SERVICE_TOKEN` is a **secret**, so it is never written into the CRD. The
+  login gate has its own internal-API token, `felis-limbo-token`, which may only mint
+  link codes, poll link status and check the blacklist. The installer applies it into
+  the minecraft namespace (and `felis setup` refreshes that replica from the control
+  namespace), and the operator injects it into the `login` pod (only) as
+  `FELIS_SERVICE_TOKEN` via a `secretKeyRef`, keyed off the reserved `login` name.
+  `sudo felis rotate-token limbo` replaces it and restarts the pod. Until the token is
   present the plugin fail-safes to readiness-only, so the gate is never broken — it
   simply does not authenticate yet.
 - **Service:** the login pod dials `FELIS_API_BASE_URL`, which resolves to the
@@ -136,11 +146,14 @@ set them by hand:
   pod's internal port 8081. That Service is deliberately separate from the external
   NodePort `felis-api` (443) so the no-Zero-Trust internal face is never published on
   a node's external IP.
-- **NetworkPolicy:** none is required today — neither the minecraft-namespace egress
-  nor the control-namespace ingress is policy-locked, so the login pod's call to the
-  API internal port is reachable. If a future deployment adds a minecraft egress lock
-  or a control-namespace ingress fence, it must also open the login-pod →
-  felis-api-internal (8081) path.
+- **NetworkPolicy:** the minecraft namespace is egress-locked
+  (`felis-server-egress`: DNS plus the public internet, every private range
+  excluded), so the internal API is unreachable from a game server by default.
+  `felis-login-to-internal-api` opens exactly the login pod → felis-api (8081) path,
+  selecting on the reserved `login` name AND the setup-owned
+  `felis.lolicon.best/system-role=login` label the operator copies onto the pod — the
+  same pair that decides who receives `FELIS_SERVICE_TOKEN`, so a user server cannot
+  match it by picking a name.
 
 The Velocity gate/lobby wiring is printed by `felis setup` and enforces the
 invariant: fresh connections hit `login` first, and only an authenticated release

deploy/lobby/Dockerfile

@@ -5,13 +5,13 @@
 # the POST-auth /menu hub: it is reached only when the login gate transfers an
 # authenticated player onward, and it must never be a fallback target.
 #
-# Build (deploy/bootstrap.sh does this for you; the PAPER_JAR_URL comes from PaperMC's
-# Fill v3 API — api.papermc.io v2 has returned HTTP 410 since 2026-07-01):
+# Build (deploy/bootstrap.sh does this for you, with the URLs and digests
+# deploy/game-stack.lock names):
+#   . <(grep -E '^(PAPER|LUCKPERMS)_' deploy/game-stack.lock)
 #   docker build -f deploy/lobby/Dockerfile \
-#     --build-arg PAPER_JAR_URL=https://fill-data.papermc.io/v1/objects/<sha>/paper-26.2-<build>.jar \
-#     --build-arg PAPER_JAR_SHA256=<that same sha — the objects/ path segment> \
-#     --build-arg LUCKPERMS_JAR_URL="$(curl -fsSL https://metadata.luckperms.net/data/all \
-#       | grep -o 'https://download.luckperms.net/[^"]*/bukkit/loader/[^"]*\.jar')" \
+#     --build-arg PAPER_JAR_URL="$PAPER_JAR_URL" --build-arg PAPER_JAR_SHA256="$PAPER_JAR_SHA256" \
+#     --build-arg LUCKPERMS_JAR_URL="$LUCKPERMS_JAR_URL" \
+#     --build-arg LUCKPERMS_JAR_SHA256="$LUCKPERMS_JAR_SHA256" \
 #     -t felis-lobby:demo .
 #   docker save felis-lobby:demo | sudo k3s ctr images import -
 #   # felis.toml → [velocity] lobby_image = "felis-lobby:demo"
@@ -25,23 +25,26 @@
 # Secret) and refuses to start without it: a lobby that cannot verify the proxy's signed
 # handshake would trust an offline, forgeable UUID.
 
-# ---- build the felis-paper plugin jar (Paper API is Java 21) ----
-# gradle:8.14-jdk21 — an official Gradle image on JDK 21 (this tree vendors no Gradle
-# wrapper, and a bare JDK image ships no `gradle`). JDK 21 matches the Paper API.
-FROM gradle:8.14-jdk21 AS plugin
+# ---- build the felis-paper plugin jar (Paper API is Java 25) ----
+# The official Gradle image on JDK 25, the Gradle version plugins/*/gradle/wrapper pins.
+# The limbo Dockerfile and bootstrap's velocity build use this same image, digest and all
+# (bootstrap_asset_test.go holds the three together). The image's own `gradle` runs the
+# build rather than the module's wrapper, which would download the same distribution
+# again on every image build. The build checks every dependency against
+# plugins/paper/gradle/verification-metadata.xml and fails on a mismatch.
+FROM gradle:9.8.0-jdk25@sha256:2b2fc1b1dfc3604a2acc916839f36eb5ee48fd7f232427fc5faca224c73bcb01 AS plugin
 WORKDIR /src
 COPY plugins/paper/ ./plugins/paper/
 COPY plugins/shared/ ./plugins/shared/
 RUN cd plugins/paper \
-    && (test -x ./gradlew && ./gradlew --no-daemon build \
-        || gradle --no-daemon build) \
+    && gradle --no-daemon build \
     && cp build/libs/*.jar /felis-paper.jar
 
 # ---- assemble the runtime ----
 # 25-jre, not 21: Paper 26.2 declares `java.version.minimum = 25` (PaperMC Fill v3,
 # GET /v3/projects/paper/versions/26.2) and refuses to boot on anything older. A 25 JRE
 # also runs the plugin's Java-21 bytecode, so only the runtime moves.
-FROM eclipse-temurin:25-jre
+FROM eclipse-temurin:25-jre@sha256:bb036ed6cfdc57e3da7c22634d15f1b840d2caf76183861c80e81ca4b5104abb
 ARG PAPER_JAR_URL
 # Required alongside the URL: Fill's URLs are content-addressed, but nothing enforces
 # that shape at build time. Checking the digest after the download turns a truncated or
@@ -51,10 +54,12 @@ ARG PAPER_JAR_SHA256
 # (internal/api/handlers_access.go) issues `lp user ...` over RCON, so a lobby built
 # without it answers every grant with "Unknown command" — a failure the operator only
 # discovers in production, because the server itself starts and runs perfectly well.
-# Failing the build is the cheap place to notice. Resolved by URL rather than pinned
-# here for the same reason PAPER_JAR_URL is: bootstrap.sh asks upstream for the current
-# build, so this file does not go stale on every LuckPerms release.
+# Failing the build is the cheap place to notice. Passed in rather than pinned here for
+# the same reason PAPER_JAR_URL is: deploy/game-stack.lock names the build, so this file
+# does not change on every LuckPerms release. The digest is required like Paper's; the
+# jar runs inside the lobby with the server's full permissions.
 ARG LUCKPERMS_JAR_URL
+ARG LUCKPERMS_JAR_SHA256
 WORKDIR /paper
 RUN set -eu; \
     if [ -z "${PAPER_JAR_URL:-}" ]; then \
@@ -66,11 +71,15 @@ RUN set -eu; \
     if [ -z "${LUCKPERMS_JAR_URL:-}" ]; then \
       echo "ERROR: --build-arg LUCKPERMS_JAR_URL=<luckperms bukkit jar> is required" >&2; exit 1; \
     fi; \
+    if [ -z "${LUCKPERMS_JAR_SHA256:-}" ]; then \
+      echo "ERROR: --build-arg LUCKPERMS_JAR_SHA256=<luckperms jar sha256> is required" >&2; exit 1; \
+    fi; \
     apt-get update && apt-get install -y --no-install-recommends curl ca-certificates; \
     mkdir -p /paper/plugins; \
     curl -fSL "$PAPER_JAR_URL" -o /paper/paper.jar; \
     echo "$PAPER_JAR_SHA256  /paper/paper.jar" | sha256sum -c; \
     curl -fSL "$LUCKPERMS_JAR_URL" -o /paper/plugins/LuckPerms.jar; \
+    echo "$LUCKPERMS_JAR_SHA256  /paper/plugins/LuckPerms.jar" | sha256sum -c; \
     apt-get purge -y curl && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*; \
     echo "eula=true" > /paper/eula.txt
 COPY --from=plugin /felis-paper.jar /paper/plugins/felis-paper.jar
@@ -82,6 +91,13 @@ COPY deploy/lobby/entrypoint.sh /usr/local/bin/felis-entrypoint.sh
 # The operator mounts the world PVC at /data. Runtime state lives there; /paper
 # remains the immutable image seed copied into the volume by the entrypoint.
 WORKDIR /data
+# Run as the game uid (naming.GameUID in the Go tree). The operator pins the same uid in
+# the pod securityContext whatever USER an image declares; declaring it here as well
+# keeps a plain `docker run` of this image off root, and chowning the empty /data seed
+# lets that run write its world. The jar seed above stays root-owned and read-only to
+# the server.
+RUN chown 1000:1000 /data
+USER 1000:1000
 
 # FELIS_GAME_PORT is the port the entrypoint pins Paper to; it MUST equal the operator's
 # GamePort (internal/operator/builders.go). Default 25565 — override only in lockstep

deploy/lobby/README.md

@@ -31,8 +31,12 @@ docker build -f deploy/lobby/Dockerfile \
   --build-arg PAPER_JAR_URL=https://<mirror>/paper-1.21.x-<build>.jar \
   --build-arg PAPER_JAR_SHA256=<sha256 of that jar> \
   -t felis-lobby:demo .
-docker save felis-lobby:demo | sudo k3s ctr images import -
-# felis.toml → [velocity] lobby_image = "felis-lobby:demo"
+# Publish into the cluster's registry (on the node; docker treats 127.0.0.1 as
+# insecure by default — or through a `kubectl -n felis port-forward svc/registry
+# 5000:5000`, which is equivalent: only the path after the host matters).
+docker tag  felis-lobby:demo 127.0.0.1:5000/felis/lobby:demo
+docker push 127.0.0.1:5000/felis/lobby:demo
+# felis.toml → [velocity] lobby_image = "registry.felis.svc:5000/felis/lobby:demo"
 sudo felis setup
 ```
 

deploy/paper/Dockerfile

@@ -5,7 +5,7 @@
 # internal/store/migrations/0019_recommended_paper.sql). It is NOT a system server: it
 # carries no felis-paper /menu plugin, no LuckPerms, and no forwarding-secret gate.
 #
-# It writes NO Velocity forwarding config itself. The operator injects a root
+# It writes NO Velocity forwarding config itself. The operator injects a
 # `felis init-forwarding` initContainer into every USER server (internal/operator/
 # builders.go: buildStatefulSet) that writes config/paper-global.yml + server.properties
 # online-mode=false onto the /data PVC before this container starts. That external step is
@@ -14,11 +14,11 @@
 # image a user brings is made joinable the same way. If the initContainer is absent (no
 # FELIS_IMAGE configured) Paper boots as a standalone online server: degraded, not broken.
 #
-# Build (deploy/bootstrap.sh does this for you; PAPER_JAR_URL comes from PaperMC's Fill v3
-# API — the SAME url the lobby build resolves, so this reuses it and adds no new dependency):
+# Build (deploy/bootstrap.sh does this for you, with the SAME Paper build the lobby uses —
+# deploy/game-stack.lock names it — so this adds no new dependency):
+#   . <(grep '^PAPER_' deploy/game-stack.lock)
 #   docker build -f deploy/paper/Dockerfile \
-#     --build-arg PAPER_JAR_URL=https://fill-data.papermc.io/v1/objects/<sha>/paper-<ver>-<build>.jar \
-#     --build-arg PAPER_JAR_SHA256=<that same sha — the objects/ path segment> \
+#     --build-arg PAPER_JAR_URL="$PAPER_JAR_URL" --build-arg PAPER_JAR_SHA256="$PAPER_JAR_SHA256" \
 #     -t felis-paper:demo .
 #   docker save felis-paper:demo | sudo k3s ctr images import -
 #   # felis.toml → recommended via 0019_recommended_paper.sql (no [velocity] key points here)
@@ -29,7 +29,7 @@
 
 # 25-jre, not 21: Paper 26.2 declares java.version.minimum=25 (PaperMC Fill v3) and refuses
 # to boot on anything older.
-FROM eclipse-temurin:25-jre
+FROM eclipse-temurin:25-jre@sha256:bb036ed6cfdc57e3da7c22634d15f1b840d2caf76183861c80e81ca4b5104abb
 ARG PAPER_JAR_URL
 # Required alongside the URL: Fill's URLs are content-addressed, but nothing enforces
 # that shape at build time. Checking the digest after the download turns a truncated or
@@ -54,6 +54,13 @@ COPY deploy/paper/entrypoint.sh /usr/local/bin/felis-entrypoint.sh
 # on the PVC. /paper stays the immutable image seed: the jar is never copied onto the
 # volume, so the panel file editor (which sees only /data) cannot tamper with it.
 WORKDIR /data
+# Run as the game uid (naming.GameUID in the Go tree). The operator pins the same uid in
+# the pod securityContext whatever USER an image declares; declaring it here as well
+# keeps a plain `docker run` of this image off root, and chowning the empty /data seed
+# lets that run write its world. The jar seed above stays root-owned and read-only to
+# the server.
+RUN chown 1000:1000 /data
+USER 1000:1000
 
 # FELIS_GAME_PORT is the port the entrypoint pins Paper to; it MUST equal the operator's
 # GamePort (internal/operator/builders.go). Default 25565 — override only in lockstep with

deploy/paper/entrypoint.sh

@@ -3,7 +3,7 @@
 #
 # A plain Paper backend for a user's OWN world — NOT a system server. Unlike deploy/limbo
 # and deploy/lobby it writes no Velocity forwarding config and has no secret gate: the
-# operator injects a root `felis init-forwarding` initContainer that writes
+# operator injects a `felis init-forwarding` initContainer that writes
 # config/paper-global.yml + server.properties online-mode=false onto /data BEFORE this
 # container starts, so forwarding is configured externally and this stays a drop-in Paper
 # image. With no initContainer (no FELIS_IMAGE) Paper just boots standalone-online —

deploy/uninstall.sh

@@ -0,0 +1,428 @@
+#!/usr/bin/env bash
+# Removes what deploy/bootstrap.sh installed on this host.
+#
+#   sudo bash deploy/uninstall.sh                 # remove Felis, keep the data
+#   sudo bash deploy/uninstall.sh --purge         # remove the data too
+#   curl -fsSL <raw-url>/deploy/uninstall.sh | sudo bash -s -- --yes
+#
+# Options:
+#   --purge       also drop the felis database and role, and delete /etc/felis and
+#                 /var/lib/felis (the database bundles, and anything an earlier keep-data
+#                 run set aside). Asks for the word "purge" unless --yes is given.
+#   --keep-k3s    leave k3s installed and remove only Felis's namespaces and CRD.
+#   --remove-k3s  run k3s's own uninstaller even when other workloads live in the cluster.
+#   --no-backup   skip the final database bundle keep-data mode takes first.
+#   --yes         do not ask.
+#
+# Keep-data mode (the default) first takes a database bundle (`felis db backup -label
+# manual`) and stops if that fails. It leaves PostgreSQL's felis database, /etc/felis (the
+# secrets, felis.toml, offsite.env) and /var/lib/felis in place. The world, archive,
+# registry and upload volumes live under k3s's storage directory, which k3s's uninstaller
+# deletes, so they are moved to /var/lib/felis/retained/k3s-storage-<UTC stamp> first; with
+# --keep-k3s their PersistentVolumes are switched to Retain before the namespaces go.
+# docs/operations.md walks through reinstalling on top of what is left.
+#
+# Either mode leaves packages alone (Docker, PostgreSQL, git and the rest), and the swap
+# file a low-memory host got: other software may use them. docs/operations.md lists the
+# package commands for a bare host.
+set -Eeuo pipefail
+
+STATE_DIR="${STATE_DIR:-/etc/felis}"
+DATA_DIR="${DATA_DIR:-/var/lib/felis}"
+RETAIN_DIR="${DATA_DIR}/retained"
+HOST_BIN="${HOST_BIN:-/usr/local/bin/felis}"
+OPT_DIR="${OPT_DIR:-/opt/felis}"
+UNIT_DIR="${UNIT_DIR:-/etc/systemd/system}"
+K3S_BIN_DIR="${K3S_BIN_DIR:-/usr/local/bin}"
+K3S_STORAGE="${K3S_STORAGE:-/var/lib/rancher/k3s/storage}"
+K3S_REGISTRIES="${K3S_REGISTRIES:-/etc/rancher/k3s/registries.yaml}"
+export KUBECONFIG="${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}"
+CLOUDFLARED_BIN="${CLOUDFLARED_BIN:-/usr/local/bin/cloudflared}"
+VELOCITY_USER="felis-velocity"
+DB_NAME="felis"
+DB_USER="felis"
+POD_CIDR="10.42.0.0/16"
+SERVICE_CIDR="10.43.0.0/16"
+FELIS_PANEL_NODEPORT="${FELIS_PANEL_NODEPORT:-30443}"
+CONFIRM_TTY="${CONFIRM_TTY:-/dev/tty}"
+FELIS_NAMESPACES=(felis minecraft felis-build)
+FELIS_CRD="minecraftservers.felis.lolicon.best"
+# Every unit the installer and `felis setup` write. Timers first, so none fires into a
+# service that is already gone.
+FELIS_UNITS=(
+  felis-db-backup.timer felis-watchdog.timer felis-offsite.timer felis-build-tools.timer felis-update-check.timer
+  felis-db-backup.service felis-watchdog.service felis-offsite.service felis-build-tools.service felis-update-check.service
+  felis-velocity.service felis-nano.service cloudflared-felis.service
+  felis-postgres-firewall.service
+)
+
+PURGE=0
+K3S_MODE=auto
+BACKUP=1
+ASSUME_YES=0
+TUNNEL_CONFIG=""
+
+log()  { printf '\033[1;36m[felis]\033[0m %s\n' "$*"; }
+ok()   { printf '\033[1;32m[ ok ]\033[0m %s\n' "$*"; }
+warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
+die()  { printf '\033[1;31m[fail]\033[0m %s\n' "$*" >&2; exit 1; }
+
+parse_args() {
+  while [ $# -gt 0 ]; do
+    case "$1" in
+      --purge) PURGE=1 ;;
+      --keep-k3s) K3S_MODE=keep ;;
+      --remove-k3s) K3S_MODE=remove ;;
+      --no-backup) BACKUP=0 ;;
+      --yes|-y) ASSUME_YES=1 ;;
+      -h|--help) printf 'usage: uninstall.sh [--purge] [--keep-k3s|--remove-k3s] [--no-backup] [--yes]\n'; exit 0 ;;
+      *) die "unknown option: $1 (see --help)" ;;
+    esac
+    shift
+  done
+}
+
+kube() { "${K3S_BIN_DIR}/k3s" kubectl "$@"; }
+k3s_present() { [ -x "${K3S_BIN_DIR}/k3s" ]; }
+
+# foreign_namespaces prints the namespaces that are neither k3s's own nor Felis's, one per
+# line. A cluster with none of them exists for Felis alone, and removing k3s takes nothing
+# else with it.
+foreign_namespaces() {
+  kube get namespaces -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' \
+    | awk '$0 != "" && $0 != "default" && $0 !~ /^kube-/ && $0 != "felis" && $0 != "minecraft" && $0 != "felis-build"'
+}
+
+# decide_k3s turns K3S_MODE=auto into keep or remove.
+decide_k3s() {
+  k3s_present || { K3S_MODE=absent; return 0; }
+  [ "$K3S_MODE" = auto ] || return 0
+  local others
+  if ! others="$(foreign_namespaces)"; then
+    die "k3s does not answer, so this cannot tell whether it runs anything besides Felis; start it (systemctl start k3s) or pass --keep-k3s or --remove-k3s"
+  fi
+  if [ -z "$others" ]; then
+    K3S_MODE=remove
+  else
+    K3S_MODE=keep
+    log "k3s also runs namespaces Felis did not create ($(printf '%s' "$others" | tr '\n' ' ')); leaving k3s installed"
+  fi
+}
+
+confirm() {
+  [ "$ASSUME_YES" = 1 ] && return 0
+  local want="yes" answer=""
+  [ "$PURGE" = 1 ] && want="purge"
+  # A piped script has no stdin to read from; the terminal is asked directly.
+  { exec 3<"$CONFIRM_TTY" 4>>"$CONFIRM_TTY"; } 2>/dev/null || die "no terminal to confirm on; re-run with --yes"
+  printf 'Type "%s" to continue: ' "$want" >&4
+  read -r answer <&3 || true
+  exec 3<&- 4>&-
+  [ "$answer" = "$want" ] || die "not confirmed; nothing was changed"
+}
+
+print_plan() {
+  log "this will remove from $(uname -n):"
+  log "  the felis-* systemd units, cloudflared-felis.service, the ${VELOCITY_USER} user,"
+  log "  ${OPT_DIR}, ${HOST_BIN}, the felis_postgres and felis_edge nftables tables and the firewalld openings"
+  case "$K3S_MODE" in
+    remove) log "  k3s, with everything in it (${K3S_BIN_DIR}/k3s-uninstall.sh)" ;;
+    keep) log "  Felis's namespaces (${FELIS_NAMESPACES[*]}) and the ${FELIS_CRD} CRD; k3s stays" ;;
+    absent) ;;
+  esac
+  if [ "$PURGE" = 1 ]; then
+    log "  PURGE: the ${DB_NAME} database and role, ${STATE_DIR} (secrets), ${DATA_DIR} (database bundles"
+    log "  and anything set aside before), every world and archive, the Felis images and Docker's build cache"
+  else
+    [ "$BACKUP" = 1 ] && log "  after a final database bundle into ${DATA_DIR}/db-backups"
+    log "  kept: the ${DB_NAME} database, ${STATE_DIR}, ${DATA_DIR}; the volumes move to ${RETAIN_DIR}/"
+  fi
+}
+
+final_backup() {
+  [ "$PURGE" = 0 ] && [ "$BACKUP" = 1 ] || return 0
+  [ -x "$HOST_BIN" ] && [ -r "${STATE_DIR}/felis.host.toml" ] || {
+    warn "no ${HOST_BIN} or ${STATE_DIR}/felis.host.toml; skipping the final database bundle"
+    return 0
+  }
+  log "taking a final database bundle"
+  "$HOST_BIN" db backup -config "${STATE_DIR}/felis.host.toml" -label manual \
+    || die "the final database bundle failed, so nothing was removed. Fix the database (sudo felis db check), or pass --no-backup to go on without one"
+  ok "database bundle written to ${DATA_DIR}/db-backups"
+}
+
+# game_port reads the proxy's port from velocity.toml before /opt/felis goes.
+game_port() {
+  local toml="${OPT_DIR}/velocity/velocity.toml" port=""
+  [ -r "$toml" ] && port="$(sed -n 's/^bind *= *"[^"]*:\([0-9][0-9]*\)".*/\1/p' "$toml" | head -n 1)"
+  printf '%s\n' "${FELIS_GAME_PORT:-${port:-25565}}"
+}
+
+# tunnel_config reads the cloudflared config felis setup pointed its unit at (by default
+# /etc/felis/cloudflared.yml) before the unit goes.
+tunnel_config() {
+  local unit="${UNIT_DIR}/cloudflared-felis.service" path=""
+  [ -r "$unit" ] && path="$(sed -n 's/^ExecStart=.* --config \([^ ]*\) tunnel run$/\1/p' "$unit" | head -n 1)"
+  printf '%s\n' "${path:-${STATE_DIR}/cloudflared.yml}"
+}
+
+# nano_port reads felis-nano's port from its unit before the unit goes.
+nano_port() {
+  local unit="${UNIT_DIR}/felis-nano.service"
+  [ -r "$unit" ] || return 0
+  sed -n 's/^ExecStart=.* -listen [^ ]*:\([0-9][0-9]*\).*$/\1/p' "$unit" | head -n 1
+}
+
+remove_units() {
+  local unit removed=0
+  for unit in "${FELIS_UNITS[@]}"; do
+    [ -f "${UNIT_DIR}/${unit}" ] || continue
+    systemctl disable --now "$unit" >/dev/null 2>&1 || systemctl stop "$unit" >/dev/null 2>&1 || true
+    rm -f "${UNIT_DIR}/${unit}"
+    removed=$((removed + 1))
+  done
+  systemctl daemon-reload
+  systemctl reset-failed >/dev/null 2>&1 || true
+  ok "${removed} systemd unit(s) removed"
+}
+
+remove_nft_tables() {
+  command -v nft >/dev/null 2>&1 || return 0
+  local t
+  for t in felis_postgres felis_edge; do
+    if nft list table inet "$t" >/dev/null 2>&1; then
+      nft delete table inet "$t"
+      ok "nftables table inet ${t} removed"
+    fi
+  done
+}
+
+# remove_firewalld_rules takes back what configure_k3s_firewall, configure_velocity_firewall
+# and the nano setup opened. The k3s ones stay when k3s does.
+remove_firewalld_rules() { # game-port nano-port
+  command -v firewall-cmd >/dev/null 2>&1 || return 0
+  systemctl is-active --quiet firewalld || return 0
+  local game="$1" nano="$2" port rule changed=0
+  local ports=("${game}/tcp" "${FELIS_PANEL_NODEPORT}/tcp")
+  [ -n "$nano" ] && ports+=("${nano}/tcp")
+  [ "$K3S_MODE" = remove ] && ports+=("6443/tcp")
+  for port in "${ports[@]}"; do
+    if firewall-cmd --permanent --query-port="$port" >/dev/null 2>&1; then
+      firewall-cmd --permanent --remove-port="$port" >/dev/null
+      changed=1
+    fi
+  done
+  if [ -n "$nano" ]; then
+    while IFS= read -r rule; do
+      case "$rule" in
+        *"port=\"${nano}\""*) firewall-cmd --permanent --remove-rich-rule="$rule" >/dev/null; changed=1 ;;
+      esac
+    done < <(firewall-cmd --permanent --list-rich-rules 2>/dev/null)
+  fi
+  if [ "$K3S_MODE" = remove ]; then
+    local cidr
+    for cidr in "$POD_CIDR" "$SERVICE_CIDR"; do
+      if firewall-cmd --permanent --zone=trusted --query-source="$cidr" >/dev/null 2>&1; then
+        firewall-cmd --permanent --zone=trusted --remove-source="$cidr" >/dev/null
+        changed=1
+      fi
+    done
+  fi
+  if [ "$changed" = 1 ]; then
+    firewall-cmd --reload >/dev/null
+    ok "firewalld openings removed"
+  fi
+}
+
+# retain_volumes_in_cluster keeps every volume Felis's claims are bound to when the
+# namespaces go: local-path deletes a Delete-policy volume's directory with its claim.
+retain_volumes_in_cluster() {
+  local pv
+  while IFS= read -r pv; do
+    [ -n "$pv" ] || continue
+    kube patch pv "$pv" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' >/dev/null
+  done < <(kube get pv -o 'jsonpath={range .items[*]}{.metadata.name} {.spec.claimRef.namespace}{"\n"}{end}' \
+    | awk '$2 == "felis" || $2 == "minecraft" || $2 == "felis-build" { print $1 }')
+  ok "Felis's volumes set to Retain; their directories stay under ${K3S_STORAGE}"
+}
+
+remove_from_cluster() {
+  [ "$PURGE" = 1 ] || retain_volumes_in_cluster
+  log "deleting Felis's namespaces and CRD"
+  # The operator is part of what goes, so nothing would clear a MinecraftServer finalizer
+  # and the minecraft namespace would stay Terminating. Drop them first.
+  local s
+  while IFS= read -r s; do
+    [ -n "$s" ] || continue
+    kube -n minecraft patch minecraftserver "$s" --type=merge -p '{"metadata":{"finalizers":null}}' >/dev/null 2>&1 || true
+  done < <(kube -n minecraft get minecraftservers -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
+  kube delete namespace "${FELIS_NAMESPACES[@]}" --ignore-not-found --wait=true --timeout=300s >/dev/null \
+    || warn "a namespace is still terminating; check: k3s kubectl get namespaces"
+  kube delete crd "$FELIS_CRD" --ignore-not-found >/dev/null || true
+  if [ "$PURGE" = 1 ]; then
+    local pv
+    while IFS= read -r pv; do
+      [ -n "$pv" ] && kube delete pv "$pv" --ignore-not-found >/dev/null
+    done < <(kube get pv -o 'jsonpath={range .items[*]}{.metadata.name} {.spec.claimRef.namespace}{"\n"}{end}' \
+      | awk '$2 == "felis" || $2 == "minecraft" || $2 == "felis-build" { print $1 }')
+  fi
+  if [ -f "$K3S_REGISTRIES" ] && grep -q 'registry\.felis\.svc' "$K3S_REGISTRIES"; then
+    rm -f "$K3S_REGISTRIES"
+    systemctl restart k3s
+  fi
+  ok "Felis removed from the cluster; k3s stays"
+}
+
+remove_k3s() {
+  local stamp
+  if [ "$PURGE" = 0 ] && [ -d "$K3S_STORAGE" ]; then
+    # k3s-killall.sh stops every pod and unmounts their volumes, so nothing is writing a
+    # world while it moves.
+    "${K3S_BIN_DIR}/k3s-killall.sh" >/dev/null 2>&1 || systemctl stop k3s
+    stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+    install -d -m 0700 "$RETAIN_DIR"
+    mv "$K3S_STORAGE" "${RETAIN_DIR}/k3s-storage-${stamp}"
+    ok "volumes moved to ${RETAIN_DIR}/k3s-storage-${stamp}"
+  fi
+  if [ -x "${K3S_BIN_DIR}/k3s-uninstall.sh" ]; then
+    log "running k3s-uninstall.sh"
+    "${K3S_BIN_DIR}/k3s-uninstall.sh" >/dev/null 2>&1 || warn "k3s-uninstall.sh reported an error; check /var/lib/rancher and /etc/rancher"
+    ok "k3s removed"
+  else
+    warn "k3s is at ${K3S_BIN_DIR}/k3s but ${K3S_BIN_DIR}/k3s-uninstall.sh is missing; remove k3s by hand"
+  fi
+}
+
+# remove_cloudflared_binary deletes the binary the installer put in /usr/local/bin, unless
+# a unit other than Felis's still runs it.
+remove_cloudflared_binary() {
+  [ -x "$CLOUDFLARED_BIN" ] || return 0
+  local others
+  others="$(grep -ls "$CLOUDFLARED_BIN" "${UNIT_DIR}"/*.service /lib/systemd/system/*.service /usr/lib/systemd/system/*.service 2>/dev/null || true)"
+  if [ -n "$others" ]; then
+    log "leaving ${CLOUDFLARED_BIN}: $(printf '%s' "$others" | tr '\n' ' ')uses it"
+    return 0
+  fi
+  rm -f "$CLOUDFLARED_BIN"
+  ok "${CLOUDFLARED_BIN} removed"
+}
+
+remove_host_files() {
+  if id "$VELOCITY_USER" >/dev/null 2>&1; then
+    userdel "$VELOCITY_USER" >/dev/null 2>&1 || warn "could not remove the ${VELOCITY_USER} user"
+  fi
+  rm -rf "$OPT_DIR"
+  rm -f "$HOST_BIN" "${HOST_BIN}.new" "${HOST_BIN}.prev"
+  remove_cloudflared_binary
+  if [ "$PURGE" = 0 ]; then
+    # What describes the removed install goes; what a reinstall reuses stays. Without
+    # bootstrap.done the next run takes the first-install path.
+    rm -f "${STATE_DIR}/bootstrap.done" "${STATE_DIR}/system-server-images" \
+      "${STATE_DIR}/velocity.fingerprint" "${STATE_DIR}/previous-felis-image"
+  fi
+  ok "${OPT_DIR} and ${HOST_BIN} removed"
+}
+
+as_postgres() { (cd / && runuser -u postgres -- "$@"); }
+
+# remove_hba_block drops the block write_pg_hba_block maintains, and nothing else.
+remove_hba_block() { # file
+  local tmp
+  tmp="$(mktemp)"
+  awk '
+    $0 == "# BEGIN FELIS MANAGED HBA" { skip = 1; next }
+    $0 == "# END FELIS MANAGED HBA" { skip = 0; blank = 1; next }
+    blank && $0 == "" { blank = 0; next }
+    { blank = 0 }
+    !skip { print }
+  ' "$1" > "$tmp"
+  cat "$tmp" > "$1"
+  rm -f "$tmp"
+}
+
+purge_database() {
+  [ "$PURGE" = 1 ] || return 0
+  if ! systemctl is-active --quiet postgresql 2>/dev/null; then
+    warn "PostgreSQL is not running; the ${DB_NAME} database and role are left in it"
+    return 0
+  fi
+  local hba
+  hba="$(as_postgres psql -tAc 'SHOW hba_file;' 2>/dev/null || true)"
+  as_postgres psql -v ON_ERROR_STOP=1 -q <<SQL
+SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${DB_NAME}' AND pid <> pg_backend_pid();
+DROP DATABASE IF EXISTS ${DB_NAME};
+DROP ROLE IF EXISTS ${DB_USER};
+ALTER SYSTEM RESET listen_addresses;
+SQL
+  [ -n "$hba" ] && [ -f "$hba" ] && remove_hba_block "$hba"
+  systemctl restart postgresql
+  ok "database and role '${DB_NAME}' dropped; PostgreSQL listens on its default address again"
+}
+
+purge_images() {
+  [ "$PURGE" = 1 ] || return 0
+  command -v docker >/dev/null 2>&1 || return 0
+  # The installer stops Docker after its builds; start it just long enough to clean up.
+  local was_active=1 refs
+  systemctl is-active --quiet docker || { was_active=0; systemctl start docker >/dev/null 2>&1 || return 0; }
+  refs="$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -E '^(registry\.felis\.svc:5000/felis/|felis/)' || true)"
+  if [ -n "$refs" ]; then
+    # shellcheck disable=SC2086 # one ref per word
+    docker image rm -f $refs >/dev/null 2>&1 || true
+  fi
+  docker builder prune -af >/dev/null 2>&1 || true
+  [ "$was_active" = 1 ] || systemctl stop docker docker.socket >/dev/null 2>&1 || true
+  ok "Felis images and Docker's build cache removed"
+}
+
+purge_state() {
+  [ "$PURGE" = 1 ] || return 0
+  # felis setup's tunnel credentials sit beside cloudflared's login (cert.pem, which stays:
+  # it is the Cloudflare account's, not Felis's). The tunnel itself lives on in the account
+  # until it is deleted there (docs/operations.md).
+  local cred=""
+  [ -r "$TUNNEL_CONFIG" ] \
+    && cred="$(sed -n 's/^credentials-file: *"\{0,1\}\([^"]*\)"\{0,1\} *$/\1/p' "$TUNNEL_CONFIG" | head -n 1)"
+  if [ -n "$cred" ]; then rm -f "$cred"; fi
+  rm -f "$TUNNEL_CONFIG"
+  rm -rf "$STATE_DIR" "$DATA_DIR"
+  ok "${STATE_DIR} and ${DATA_DIR} removed"
+}
+
+main() {
+  parse_args "$@"
+  [ "$(id -u)" = 0 ] || die "run as root: sudo bash $0"
+  [ -e "$STATE_DIR" ] || [ -e "$HOST_BIN" ] || [ -e "$OPT_DIR" ] \
+    || die "no Felis install here (${STATE_DIR}, ${HOST_BIN} and ${OPT_DIR} are all absent)"
+  decide_k3s
+  print_plan
+  confirm
+  final_backup
+
+  local game nano
+  game="$(game_port)"
+  nano="$(nano_port)"
+  TUNNEL_CONFIG="$(tunnel_config)"
+  remove_units
+  case "$K3S_MODE" in
+    remove) remove_k3s ;;
+    keep) remove_from_cluster ;;
+  esac
+  remove_nft_tables
+  remove_firewalld_rules "$game" "$nano"
+  remove_host_files
+  purge_database
+  purge_images
+  purge_state
+
+  if [ "$PURGE" = 1 ]; then
+    ok "Felis is gone from this host"
+  else
+    ok "Felis is removed; the data stays in the ${DB_NAME} database, ${STATE_DIR} and ${DATA_DIR}"
+    log "reinstalling reuses it: see docs/operations.md, \"Reinstall on top of kept data\""
+  fi
+}
+
+if [ "${FELIS_UNINSTALL_SOURCED:-0}" != 1 ]; then
+  main "$@"
+fi

deploy/uninstall_test.sh

@@ -0,0 +1,221 @@
+#!/bin/sh
+# Checks for deploy/uninstall.sh. Run it as: sh deploy/uninstall_test.sh
+#
+# The script is sourced with FELIS_UNINSTALL_SOURCED=1, every host path pointed into a
+# scratch directory, and the commands that would change the host (systemctl, k3s, nft,
+# firewall-cmd, runuser, docker, userdel) replaced by stubs that log their arguments.
+set -u
+
+US="${1:-$(dirname "$0")/uninstall.sh}"
+[ -f "$US" ] || { echo "no such script: $US"; exit 1; }
+fails=0
+
+expect() { # label needle haystack
+  case "$3" in
+    *"$2"*) echo "PASS $1" ;;
+    *) echo "FAIL $1: expected <$2> in:"; echo "$3"; fails=$((fails + 1)) ;;
+  esac
+}
+refute() { # label needle haystack
+  case "$3" in
+    *"$2"*) echo "FAIL $1: did not expect <$2> in:"; echo "$3"; fails=$((fails + 1)) ;;
+    *) echo "PASS $1" ;;
+  esac
+}
+
+root="$(mktemp -d)"
+trap 'rm -rf "$root"' EXIT
+
+# fresh_host lays out what an install leaves: units, state, /opt/felis, the host binary, a
+# k3s with its uninstaller and a volume, a tunnel config and its credentials.
+fresh_host() {
+  rm -rf "$root/h"
+  mkdir -p "$root/h/units" "$root/h/etc" "$root/h/data/db-backups" "$root/h/opt/velocity" \
+    "$root/h/bin" "$root/h/storage/pvc-1_minecraft_world-a-0" "$root/h/cf"
+  for u in felis-db-backup.timer felis-db-backup.service felis-velocity.service felis-postgres-firewall.service; do
+    printf '[Unit]\n' > "$root/h/units/$u"
+  done
+  printf '[Service]\nExecStart=/usr/local/bin/cloudflared --config %s tunnel run\n' "$root/h/etc/cloudflared.yml" \
+    > "$root/h/units/cloudflared-felis.service"
+  printf 'tunnel: abc\ncredentials-file: %s\n' "$root/h/cf/abc.json" > "$root/h/etc/cloudflared.yml"
+  printf '{}\n' > "$root/h/cf/abc.json"
+  printf 'bind = "0.0.0.0:25577"\n' > "$root/h/opt/velocity/velocity.toml"
+  for f in secrets.env felis.host.toml bootstrap.done system-server-images velocity.fingerprint; do
+    printf 'x\n' > "$root/h/etc/$f"
+  done
+  printf 'world\n' > "$root/h/storage/pvc-1_minecraft_world-a-0/level.dat"
+  for b in felis k3s k3s-killall.sh k3s-uninstall.sh; do
+    printf '#!/bin/sh\necho "RUN %s $*" >> "%s"\n' "$b" "$root/calls" > "$root/h/bin/$b"
+    chmod +x "$root/h/bin/$b"
+  done
+  cat > "$root/h/hba.conf" <<'EOF'
+# BEGIN FELIS MANAGED HBA
+# Felis rules must precede distro defaults such as 127.0.0.1 ident.
+host felis felis 127.0.0.1/32 scram-sha-256
+# END FELIS MANAGED HBA
+
+local all all peer
+host all all 127.0.0.1/32 ident
+EOF
+  : > "$root/calls"
+}
+
+# run_uninstall <namespaces> <args...>: runs main with the stubs; <namespaces> is what
+# `kubectl get namespaces` answers, or "down" for a k3s that does not answer.
+run_uninstall() {
+  ns="$1"; shift
+  NS="$ns" ROOT="$root" STATE_DIR="$root/h/etc" DATA_DIR="$root/h/data" HOST_BIN="$root/h/bin/felis" \
+  OPT_DIR="$root/h/opt" UNIT_DIR="$root/h/units" K3S_BIN_DIR="$root/h/bin" \
+  K3S_STORAGE="$root/h/storage" K3S_REGISTRIES="$root/h/registries.yaml" \
+  CLOUDFLARED_BIN="$root/h/no-cloudflared" FELIS_UNINSTALL_SOURCED=1 bash -c '
+    set -Eeuo pipefail
+    . "$0"
+    calls="$ROOT/calls"
+    id() { if [ "${1:-}" = -u ]; then echo 0; else echo "ID $*" >> "$calls"; fi; }
+    systemctl() {
+      echo "SYSTEMCTL $*" >> "$calls"
+      case "$*" in "is-active --quiet firewalld") return 1 ;; esac
+      return 0
+    }
+    kube() {
+      echo "KUBE $*" >> "$calls"
+      case "$*" in
+        "get namespaces"*) [ "$NS" = down ] && return 1; printf "%s\n" $NS ;;
+        "get pv"*) printf "pvc-1 minecraft\npvc-9 other\n" ;;
+      esac
+    }
+    nft() { echo "NFT $*" >> "$calls"; return 1; }
+    runuser() {
+      shift 3
+      case "$*" in
+        *"SHOW hba_file"*) echo "$ROOT/h/hba.conf" ;;
+        *) echo "PSQL $* $(cat)" >> "$calls" ;;
+      esac
+    }
+    docker() { echo "DOCKER $*" >> "$calls"; }
+    userdel() { echo "USERDEL $*" >> "$calls"; }
+    uname() { echo testhost; }
+    main "$@"' "$US" "$@" 2>&1
+}
+
+# --- keep-data, a cluster that runs only Felis -------------------------------------------
+fresh_host
+out="$(run_uninstall "default kube-system felis minecraft felis-build" --yes)"
+calls="$(cat "$root/calls")"
+expect "keep-data takes a final bundle first" "RUN felis db backup -config $root/h/etc/felis.host.toml -label manual" "$calls"
+expect "a Felis-only cluster is removed with k3s's uninstaller" "RUN k3s-uninstall.sh" "$calls"
+expect "the pods are stopped before the volumes move" "RUN k3s-killall.sh" "$calls"
+kept="$(ls "$root/h/data/retained" 2>/dev/null)"
+expect "the volumes are set aside before k3s deletes them" "k3s-storage-" "$kept"
+[ -f "$root/h/data/retained/$kept/pvc-1_minecraft_world-a-0/level.dat" ] \
+  && echo "PASS a world survives the uninstall" \
+  || { echo "FAIL the world did not survive: $(ls -R "$root/h/data")"; fails=$((fails + 1)); }
+[ -f "$root/h/etc/secrets.env" ] && [ -f "$root/h/etc/felis.host.toml" ] \
+  && echo "PASS the secrets and felis.toml stay" \
+  || { echo "FAIL keep-data removed the secrets"; fails=$((fails + 1)); }
+[ ! -e "$root/h/etc/bootstrap.done" ] && [ ! -e "$root/h/etc/velocity.fingerprint" ] \
+  && echo "PASS the markers of the removed install go, so a reinstall starts fresh" \
+  || { echo "FAIL bootstrap.done or the proxy fingerprint was left"; fails=$((fails + 1)); }
+refute "keep-data leaves the database alone" "DROP DATABASE" "$calls"
+[ ! -e "$root/h/opt" ] && [ ! -e "$root/h/bin/felis" ] \
+  && echo "PASS /opt/felis and the host binary are removed" \
+  || { echo "FAIL /opt/felis or the host binary is still there"; fails=$((fails + 1)); }
+[ -z "$(ls "$root/h/units")" ] && echo "PASS every Felis unit file is removed" \
+  || { echo "FAIL units left: $(ls "$root/h/units")"; fails=$((fails + 1)); }
+expect "the timers are disabled" "SYSTEMCTL disable --now felis-db-backup.timer" "$calls"
+expect "the velocity user is removed" "USERDEL felis-velocity" "$calls"
+expect "the run ends pointing at the reinstall steps" "Reinstall on top of kept data" "$out"
+
+# --- a failed final bundle stops everything ------------------------------------------------
+fresh_host
+printf '#!/bin/sh\necho "RUN felis $*" >> "%s"\nexit 1\n' "$root/calls" > "$root/h/bin/felis"
+out="$(run_uninstall "default felis minecraft" --yes)"
+expect "a failed bundle is fatal" "the final database bundle failed, so nothing was removed" "$out"
+[ -d "$root/h/opt" ] && [ -f "$root/h/units/felis-velocity.service" ] \
+  && echo "PASS nothing is removed when the bundle fails" \
+  || { echo "FAIL the uninstall went on after the bundle failed"; fails=$((fails + 1)); }
+run_uninstall "default felis minecraft" --yes --no-backup >/dev/null
+[ ! -d "$root/h/opt" ] && echo "PASS --no-backup goes on without one" \
+  || { echo "FAIL --no-backup did not remove anything"; fails=$((fails + 1)); }
+
+# --- a shared cluster --------------------------------------------------------------------
+fresh_host
+out="$(run_uninstall "default kube-system felis minecraft felis-build shop" --yes)"
+calls="$(cat "$root/calls")"
+refute "a cluster that runs something else keeps k3s" "RUN k3s-uninstall.sh" "$calls"
+expect "the reason names the other namespace" "shop" "$out"
+expect "Felis's volumes are retained before their claims go" 'KUBE patch pv pvc-1 -p {"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' "$calls"
+refute "a volume of another namespace is not touched" "patch pv pvc-9" "$calls"
+expect "Felis's namespaces are deleted" "KUBE delete namespace felis minecraft felis-build" "$calls"
+expect "the CRD is deleted" "KUBE delete crd minecraftservers.felis.lolicon.best" "$calls"
+
+out="$(fresh_host; run_uninstall down --yes)"
+expect "a k3s that does not answer stops the run" "k3s does not answer" "$out"
+fresh_host
+run_uninstall down --yes --keep-k3s >/dev/null
+calls="$(cat "$root/calls")"
+refute "--keep-k3s never runs k3s's uninstaller" "RUN k3s-uninstall.sh" "$calls"
+
+# --- purge -------------------------------------------------------------------------------
+fresh_host
+out="$(run_uninstall "default felis minecraft" --purge --yes)"
+calls="$(cat "$root/calls")"
+refute "purge takes no bundle" "db backup" "$calls"
+expect "purge drops the database" "DROP DATABASE IF EXISTS felis;" "$calls"
+expect "purge drops the role" "DROP ROLE IF EXISTS felis;" "$calls"
+expect "purge puts listen_addresses back" "ALTER SYSTEM RESET listen_addresses;" "$calls"
+[ ! -e "$root/h/etc" ] && [ ! -e "$root/h/data" ] && echo "PASS purge removes /etc/felis and /var/lib/felis" \
+  || { echo "FAIL purge left state behind"; fails=$((fails + 1)); }
+[ ! -e "$root/h/cf/abc.json" ] && echo "PASS purge removes the tunnel's credentials file" \
+  || { echo "FAIL the tunnel credentials are still there"; fails=$((fails + 1)); }
+[ ! -e "$root/h/data/retained" ] && echo "PASS purge does not set the volumes aside" \
+  || { echo "FAIL purge kept the volumes"; fails=$((fails + 1)); }
+hba="$(cat "$root/h/hba.conf")"
+refute "the Felis block leaves pg_hba.conf" "FELIS MANAGED" "$hba"
+expect "the distro's own rules stay" "host all all 127.0.0.1/32 ident" "$hba"
+case "$hba" in
+  "local all all peer"*) echo "PASS no blank line is left where the block was" ;;
+  *) echo "FAIL pg_hba.conf starts with: $(printf '%s' "$hba" | head -n 1)"; fails=$((fails + 1)) ;;
+esac
+expect "purge cleans Docker's build cache" "DOCKER builder prune -af" "$calls"
+
+# --- the pieces read before they are removed ---------------------------------------------
+fresh_host
+lib() { STATE_DIR="$root/h/etc" OPT_DIR="$root/h/opt" UNIT_DIR="$root/h/units" FELIS_UNINSTALL_SOURCED=1 \
+  bash -c '. "$0"; '"$1" "$US"; }
+expect "the game port comes from velocity.toml" "25577" "$(lib game_port)"
+expect "FELIS_GAME_PORT overrides it" "25599" "$(FELIS_GAME_PORT=25599 lib game_port)"
+rm "$root/h/opt/velocity/velocity.toml"
+expect "without velocity.toml the port is the default" "25565" "$(lib game_port)"
+printf '[Service]\nExecStart=/usr/local/bin/felis nano -listen 10.0.0.5:8082 -config x\n' > "$root/h/units/felis-nano.service"
+expect "the nano port comes from its unit" "8082" "$(lib nano_port)"
+expect "the tunnel config comes from the cloudflared unit" "$root/h/etc/cloudflared.yml" "$(lib tunnel_config)"
+rm "$root/h/units/cloudflared-felis.service"
+expect "without the unit the tunnel config is the default path" "$root/h/etc/cloudflared.yml" "$(lib tunnel_config)"
+
+out="$(FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; parse_args --bogus' "$US" 2>&1)"
+expect "an unknown option is refused" "unknown option: --bogus" "$out"
+printf 'nope\n' > "$root/tty"
+out="$(CONFIRM_TTY="$root/tty" FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; confirm; echo WENT ON' "$US" 2>&1)"
+expect "any answer but the word stops it" "not confirmed; nothing was changed" "$out"
+printf 'yes\n' > "$root/tty"
+out="$(CONFIRM_TTY="$root/tty" FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; confirm; echo WENT ON' "$US" 2>&1)"
+expect "yes goes on" "WENT ON" "$out"
+out="$(CONFIRM_TTY="$root/tty" FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; PURGE=1; confirm; echo WENT ON' "$US" 2>&1)"
+expect "a purge wants the word purge" "not confirmed" "$out"
+out="$(CONFIRM_TTY="$root/no-tty/x" FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; confirm; echo WENT ON' "$US" 2>&1)"
+expect "with no terminal it asks for --yes" "no terminal to confirm on; re-run with --yes" "$out"
+
+# Every unit the installer writes is one the uninstaller removes: a timer left behind
+# keeps firing a felis binary that is gone.
+units="$(awk '/^FELIS_UNITS=\(/ { f = 1; next } f && /^\)/ { f = 0 } f' "$US")"
+for u in $(sed -n 's|^[A-Z_]*="/etc/systemd/system/\([^"]*\)"$|\1|p' "$(dirname "$US")/bootstrap.sh"); do
+  expect "the uninstaller removes $u" " $u" " $(printf '%s' "$units" | tr '\n' ' ')"
+done
+
+if [ "$fails" -eq 0 ]; then
+  echo "ALL PASS"
+else
+  echo "$fails FAILED"
+fi
+exit "$fails"