build(deps): bump actions/setup-java from 4.9.1 to 6.0.1

Bumps [actions/setup-java](https://github.com/actions/setup-java) from 4.9.1 to 6.0.1.
- [Release notes](https://github.com/actions/setup-java/releases)
- [Commits](https://github.com/actions/setup-java/compare/cf277c60eb25467037889841efdb72551f06f6c3...de7274f081f381c8f8158605e0321c36c376e2e6)

---
updated-dependencies:
- dependency-name: actions/setup-java
  dependency-version: 6.0.1
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# The workflows pin every action to a commit SHA, and every Dockerfile pins its base images
+# by digest. This keeps those pins moving: Dependabot reads the "# vX.Y.Z" comment next to
+# each action SHA, and the tag in front of each image digest, and opens a PR that bumps both
+# together.
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directories:
+      - /
+      - /deploy/limbo
+      - /deploy/lobby
+      - /deploy/paper
+    schedule:
+      interval: weekly
+    # A new major is a runtime change (Paper 26.x needs Java 25, Limbo's jar is Java 21
+    # bytecode), so only digests and minors are proposed; majors move by hand.
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/workflows/ci.yml

@@ -14,10 +14,14 @@
 # PR is what asks for the answer.
 name: ci
 
+#
+# release.yml calls this workflow (workflow_call) before it builds anything, so a tag passes
+# exactly these gates and there is one list of them.
 on:
   push:
     branches: [main]
   pull_request:
+  workflow_call:
 
 permissions:
   contents: read
@@ -30,9 +34,9 @@ jobs:
   go:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
           go-version-file: go.mod
 
@@ -43,12 +47,66 @@ jobs:
             echo "gofmt needed on:"; echo "$unformatted"; exit 1
           fi
       - run: go vet ./...
-      - run: go test ./...
+      # -race: felis-api and the operator are mostly goroutines (watchers, the
+      # registry pruner, the backup scheduler, the rate limiters).
+      - run: go test -race ./...
+      # The version is pinned here and bumped by hand; Dependabot does not read `go run`.
+      - name: staticcheck
+        run: go run honnef.co/go/tools/cmd/[email protected] ./...
+
+  # Separate from the go job so a newly published advisory reads as what it is. govulncheck
+  # exits non-zero only for vulnerable code this module can actually reach, standard
+  # library included: setup-go installs the newest patch of go.mod's Go line, so a finding
+  # there means the Dockerfile's golang digest (which ships the release) needs a bump too.
+  vuln:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go run golang.org/x/vuln/cmd/[email protected] ./...
+
+  # The business stores' SQL against a real PostgreSQL (internal/pgint): the unit suites run
+  # on fakes, and PGRepo drifted from them three times while those stayed green. 13 is the
+  # oldest server a supported distribution installs (EL9), 18 the newest (Arch).
+  pgint:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        postgres: ['13', '18']
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres }}
+        env:
+          POSTGRES_USER: felis
+          POSTGRES_PASSWORD: pgint
+          POSTGRES_DB: felis_pgint
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U felis -d felis_pgint"
+          --health-interval 2s
+          --health-timeout 5s
+          --health-retries 30
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: go.mod
+
+      - run: go test -race -tags pgint -count=1 ./internal/pgint/
+        env:
+          FELIS_TEST_PG_URL: postgres://felis:pgint@localhost:5432/felis_pgint?sslmode=disable
 
   shell:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # bootstrap.sh is the only thing that ever runs on a fresh host, and nothing here can
       # run it — it wants root, a package manager and k3s. Syntax plus the extracted-block
@@ -66,12 +124,23 @@ jobs:
             esac
           done
 
+      # A pinned release rather than the runner image's copy, so a runner update cannot
+      # change what fails. Warnings and errors fail the job; style notes (info) do not.
+      - name: shellcheck
+        run: |
+          curl -fsSL -o shellcheck.tar.xz \
+            https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz
+          echo "8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198  shellcheck.tar.xz" | sha256sum -c
+          tar -xJf shellcheck.tar.xz
+          ./shellcheck-v0.11.0/shellcheck -S warning $(git ls-files '*.sh')
+
       - run: sh deploy/bootstrap_test.sh
+      - run: sh deploy/uninstall_test.sh
 
   panel:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # The Dockerfile's `FROM node:<major>` is the only place the panel's Node version is
       # declared — there is no .nvmrc and no engines field. Reading it here rather than
@@ -84,7 +153,7 @@ jobs:
           [ -n "$version" ] || { echo "Dockerfile has no 'FROM ... node:<major>' line"; exit 1; }
           echo "version=${version}" >> "$GITHUB_OUTPUT"
 
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: ${{ steps.node.outputs.version }}
           cache: npm
@@ -96,44 +165,71 @@ jobs:
       - run: npm test
         working-directory: panel
 
+      # `tsc -b`: tsconfig.json is a solution file (files: [] plus references), so a plain
+      # `tsc --noEmit` checked nothing and passed with type errors in the tree.
       - run: npm run typecheck
         working-directory: panel
 
+      # Rules of hooks and effect dependency lists, with --max-warnings 0.
+      - run: npm run lint
+        working-directory: panel
+
+      # openapi.gen.ts is generated from docs/openapi.yaml and checked in, so the
+      # compile-time parity in src/lib/types.parity.ts needs no generator in the build;
+      # a schema edit that was not regenerated fails here.
+      - name: openapi.gen.ts matches docs/openapi.yaml
+        working-directory: panel
+        run: |
+          npm run gen:api
+          git diff --exit-code -- src/lib/openapi.gen.ts
+
+      # Browser smoke over the mock-mode dev server, in the runner's installed Chrome
+      # (playwright.config.ts sets channel: chrome, so nothing is downloaded).
+      - run: npm run test:e2e
+        working-directory: panel
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: panel-playwright-report
+          path: |
+            panel/playwright-report
+            panel/test-results
+          retention-days: 7
+
   plugins:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # The other jobs never touch the Java layer: the plugin jars were only ever
-      # compiled by bootstrap on a live host, and the three test mains under
+      # compiled by bootstrap on a live host, and the test mains under plugins/*/test
-      # plugins/*/test were run by hand. JDK 21 plus the Gradle major the plugin
+      # were run by hand. JDK 25 is what the plugin build image runs (paper-api 26.x
-      # Dockerfiles pin (8.14) is that same toolchain, in CI.
+      # needs it); each module's wrapper brings the Gradle the image pins.
-      - uses: actions/setup-java@v4
+      - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
           distribution: temurin
-          java-version: '21'
+          java-version: '25'
 
-      - uses: gradle/actions/setup-gradle@v4
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
-        with:
-          gradle-version: '8.14'
 
       - run: bash plugins/test.sh
 
   mods:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       # The three loader mods (Minecraft 1.20.1 / 1.20.4, Java-17 lines) compile
       # through their vendored Gradle wrappers, which fetch their own Gradle. Until
       # this job nothing ever built them: no install path touches them, and their
       # gradlew scripts were committed without the exec bit, so the README's
       # one-liners failed on a fresh clone.
-      - uses: actions/setup-java@v4
+      - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
         with:
           distribution: temurin
           java-version: '17'
 
-      - uses: gradle/actions/setup-gradle@v4
+      - uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
 
       - run: bash plugins/test-mods.sh

.github/workflows/e2e.yml

@@ -0,0 +1,154 @@
+# deploy/bootstrap.sh end to end on a fresh Ubuntu 24.04 x86_64 runner: the paths every
+# host goes through, run for real instead of by hand on a VM.
+#
+#   install  a full install of this commit, then the same commit again (a rerun must
+#            converge without restarting what did not change)
+#   upgrade  the newest published release, then this commit on top of it; skipped until
+#            a release exists
+#
+# Each job builds the control plane, the game images and the proxy from scratch, about
+# half an hour of runner time, so this runs on pushes that touch what gets installed, by
+# hand, and weekly (a moving upstream: apt mirrors, k3s's install script, Adoptium).
+# deploy/e2e_check.sh holds the assertions.
+name: e2e
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'deploy/**'
+      - 'cmd/**'
+      - 'internal/**'
+      - 'panel/**'
+      - 'plugins/**'
+      - 'Dockerfile'
+      - 'go.mod'
+      - 'go.sum'
+      - '.github/workflows/e2e.yml'
+  workflow_dispatch:
+  schedule:
+    - cron: '23 4 * * 1'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  install:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      # The images, k3s and the JRE need more room than a stock runner leaves free.
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      # FELIS_SKIP_FETCH builds whatever sits in /opt/felis/src, the way the VM runs
+      # have always been done; the .git directory is what stamps the build. Root owns it
+      # so git, run as root by the installer, does not refuse it as dubious.
+      - name: Stage this commit as the installer's source
+        run: |
+          sudo mkdir -p /opt/felis
+          sudo cp -a "$GITHUB_WORKSPACE" /opt/felis/src
+          sudo chown -R root:root /opt/felis/src
+
+      - name: Install
+        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee install.log
+
+      - name: Check the install
+        run: sudo bash deploy/e2e_check.sh install
+
+      - name: Rerun the same commit
+        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee rerun.log
+
+      - name: Check the rerun
+        run: |
+          sudo bash deploy/e2e_check.sh rerun
+          grep -q 'felis-velocity unchanged; left running' rerun.log
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          k() { sudo -E /usr/local/bin/k3s kubectl "$@"; }
+          k get pods -A -o wide || true
+          k get events -A --sort-by=.lastTimestamp | tail -n 60 || true
+          for p in $(k get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" {print $1 "/" $2}'); do
+            k -n "${p%%/*}" describe pod "${p#*/}" | tail -n 40 || true
+            k -n "${p%%/*}" logs "${p#*/}" --all-containers --tail=60 || true
+          done
+          sudo journalctl -u k3s -u felis-velocity -u postgresql --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-install-logs
+          path: '*.log'
+          if-no-files-found: ignore
+
+  upgrade:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0
+
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - name: Find the newest release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          tag="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName 2>/dev/null || true)"
+          if [ -z "$tag" ]; then
+            echo "::notice::no published release yet; the upgrade path has nothing to start from"
+          fi
+          echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+
+      # The release's own installer, fetching the release's own binary: what a host that
+      # installed that release is running today.
+      - name: Install the newest release
+        if: steps.release.outputs.tag != ''
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          TOKEN: ${{ github.token }}
+        run: |
+          git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh
+          sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_REF="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log
+
+      - name: Check the release install
+        if: steps.release.outputs.tag != ''
+        run: sudo bash deploy/e2e_check.sh release
+
+      - name: Upgrade to this commit
+        if: steps.release.outputs.tag != ''
+        run: |
+          sudo rm -rf /opt/felis/src
+          sudo cp -a "$GITHUB_WORKSPACE" /opt/felis/src
+          sudo chown -R root:root /opt/felis/src
+          sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee upgrade.log
+
+      - name: Check the upgrade
+        if: steps.release.outputs.tag != ''
+        run: sudo bash deploy/e2e_check.sh upgrade
+
+      - name: Diagnostics
+        if: failure()
+        run: |
+          export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+          sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
+          sudo journalctl -u k3s -u felis-velocity -u postgresql --no-pager -n 120 || true
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: e2e-upgrade-logs
+          path: '*.log'
+          if-no-files-found: ignore

.github/workflows/release.yml

@@ -17,6 +17,16 @@
 # quietly ships a release whose panel is that placeholder. The Dockerfile runs the npm
 # build first, and is the same recipe bootstrap uses, so there is one way to build felis
 # rather than two that can drift.
+#
+# SHA256SUMS is a contract with bootstrap too: download_release_binary refuses a binary whose
+# hash is not listed there, BEFORE it runs it. A release without the file installs by source
+# build instead.
+#
+# The write token never meets the test suite: `gates` (ci.yml) and `build` run the tests,
+# Gradle and the Docker build (each of which executes third-party code) with a read-only
+# token, and `build` hands the binaries over as a workflow artifact; `publish` holds contents:write and runs only
+# pinned actions and gh. Every action is pinned to a commit SHA (the tag in the trailing
+# comment is for humans); .github/dependabot.yml proposes the bumps.
 name: release
 
 on:
@@ -24,53 +34,29 @@ on:
     tags: ['v*']
 
 permissions:
-  contents: write # gh release create/upload
+  contents: read
 
 jobs:
-  release:
+  # A tag that ships red is worse than a tag that fails to ship. These are ci.yml's gates,
+  # called rather than copied: Go (race, vet, staticcheck), govulncheck, the PostgreSQL
+  # contract suite, shellcheck and the bootstrap tests, the panel, and the Java layer the
+  # binary EMBEDS (bootstrap_asset.go ships the plugin sources, so a tag whose plugins do
+  # not compile turns every install of that release into a failed bootstrap).
+  gates:
+    uses: ./.github/workflows/ci.yml
+
+  build:
+    needs: gates
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-
-      # A tag that ships red is worse than a tag that fails to ship.
-      - run: go vet ./...
-      - run: go test ./...
-
-      # The same reason, for the Java layer the binary EMBEDS: the release asset is
-      # the tree's plugin sources (bootstrap_asset.go), and a tag whose plugins don't
-      # compile turns every install of that release into a failed bootstrap. JDK 21
-      # gates the install-time plugins + codec/invite tests; JDK 17 gates the loader
-      # mods (their vendored wrappers fetch their own Gradle).
-      - uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: '21'
-
-      - uses: gradle/actions/setup-gradle@v4
-        with:
-          gradle-version: '8.14'
-
-      - run: bash plugins/test.sh
-
-      - uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: '17'
-
-      - uses: gradle/actions/setup-gradle@v4
-
-      - run: bash plugins/test-mods.sh
 
       # Both architectures, because bootstrap's default release channel DOWNLOADS these
       # rather than compiling on the target host — an arm64 host with no asset silently
       # falls back to a slow source build. Neither stage is emulated: the Dockerfile pins
       # both build stages to $BUILDPLATFORM and the Go stage cross-compiles via TARGETARCH,
       # so the second architecture costs about a minute.
-      - uses: docker/setup-buildx-action@v3
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
 
       - name: Build the stamped binaries
         run: |
@@ -100,8 +86,67 @@ jobs:
           file ./felis-linux-arm64 | grep -q 'ARM aarch64' \
             || { echo "felis-linux-arm64 is not an arm64 ELF — TARGETARCH did not reach the go build"; exit 1; }
 
-      # --verify-tag refuses to invent a release for a tag that is not pushed. The upload
+      - name: Checksum the binaries
-      # fallback makes a re-run converge rather than failing on an existing release.
+        run: sha256sum felis-linux-amd64 felis-linux-arm64 | tee SHA256SUMS
+
+      # A CycloneDX SBOM per binary: the Go modules (and versions) linked into it, read
+      # from the build info the linker embeds.
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: felis-linux-amd64
+          format: cyclonedx-json
+          output-file: felis-linux-amd64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+      - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
+        with:
+          file: felis-linux-arm64
+          format: cyclonedx-json
+          output-file: felis-linux-arm64.cdx.json
+          upload-artifact: false
+          upload-release-assets: false
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: release-assets
+          path: |
+            felis-linux-amd64
+            felis-linux-arm64
+            felis-linux-amd64.cdx.json
+            felis-linux-arm64.cdx.json
+            SHA256SUMS
+          if-no-files-found: error
+          retention-days: 7
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # gh release create/upload
+      id-token: write # the Sigstore certificate behind the provenance attestation
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: release-assets
+
+      # The artifact store sits between the two jobs, so check the handover too.
+      - run: sha256sum -c SHA256SUMS
+
+      # Signed SLSA provenance: which workflow run, commit and repository produced each
+      # binary. Check one with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`.
+      # GitHub only stores attestations for private repositories on Enterprise Cloud, and a
+      # failure here would block the release, so a private repository skips the step and
+      # relies on SHA256SUMS alone.
+      - name: Attest build provenance
+        if: ${{ !github.event.repository.private }}
+        uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0
+        with:
+          subject-path: |
+            felis-linux-amd64
+            felis-linux-arm64
+
+      # --verify-tag refuses to invent a release for a tag that is not pushed.
       #
       # The prerelease flag has to be passed explicitly: the trigger glob is v*, so v1.2.3-rc1
       # lands here too, and gh does not read semver out of the tag name. Published as a full
@@ -109,13 +154,33 @@ jobs:
       # channel installs from and `felis update` polls — so every fresh install would get the
       # RC binary and every deployed felis-api would error on the felis component until a
       # stable tag was cut. Flagged, GitHub keeps latest pointing at the last stable release.
+      #
+      # A re-run (the release already exists) uploads only what is missing and never
+      # replaces a published asset: hosts may already have installed it, and their
+      # SHA256SUMS check would start failing against a swapped file. An asset that is
+      # there with different bytes stops the job; cut a new tag instead.
       - name: Publish the release
         env:
           GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
         run: |
+          assets="felis-linux-amd64 felis-linux-arm64 felis-linux-amd64.cdx.json felis-linux-arm64.cdx.json SHA256SUMS"
           flags=""
           case "$GITHUB_REF_NAME" in *-*) flags="--prerelease" ;; esac
-          gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags \
+          if ! gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
-              ./felis-linux-amd64 ./felis-linux-arm64 \
+            # shellcheck disable=SC2086 # word-splitting the list is the point
-            || gh release upload "$GITHUB_REF_NAME" \
+            gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes $flags $assets
-                 ./felis-linux-amd64 ./felis-linux-arm64 --clobber
+            exit 0
+          fi
+          # The REST payload's per-asset "digest" is GitHub's own sha256 of the stored file.
+          published="$(gh api "repos/${GH_REPO}/releases/tags/${GITHUB_REF_NAME}" --jq '.assets[] | "\(.name) \(.digest)"')"
+          for a in $assets; do
+            have="$(printf '%s\n' "$published" | awk -v n="$a" '$1 == n { print $2 }')"
+            want="sha256:$(sha256sum < "$a" | cut -d' ' -f1)"
+            if [ -z "$have" ]; then
+              gh release upload "$GITHUB_REF_NAME" "$a"
+            elif [ "$have" != "$want" ]; then
+              echo "::error::$a is already published with $have; this run built $want. Published assets are never replaced."
+              exit 1
+            fi
+          done

AUDIT-2026-09-22.md

@@ -132,6 +132,8 @@ IPv6-only 接入（`ssh -6 -i ~/.ssh/id_ed25519 root@fdb2:2c26:f4e4:0:21c:42ff:f
 
 第四十六批追加：#75（提交上传面无上限——pending 无个数上限、无存储预算、create/upload 无节流；一个账号可无限堆积上下文刷爆 uploads PVC ①）；#76（提交无撤回/管理员删除路径——提交者无法自救、运维无法回收占用 ①）；#77（未完成引导的会话触发受保护操作 → 面板显示"无权执行此操作"而非送往 /setup；tracker #8 ①）；#78（create-if-absent 使新增 CR 字段在已装机永不落地——lobby 无 RCON 故控制台死、玩家数恒 0；tracker #1 ②）；#79（troubleshooting [INERT] 图例指向已不存在字段 ④ 文档）。
 
+第四十七批追加：无新缺陷——首个稳定 tag `v0.1.0` 的发布链与 release 安装/升级通道全实弹（release 二进制下载 → 无 checkout 全量安装 2m17s 零 fail/零 warn → `felis update` 双向报告；证据见该批节）。
+
 ## 已验证事实（正向清单）
 
 - 安装→hook 发码→Owner 绑定→passkey（虚拟认证器）→面板管理员全链路 ✅
@@ -899,6 +901,34 @@ IPv6-only 接入（`ssh -6 -i ~/.ssh/id_ed25519 root@fdb2:2c26:f4e4:0:21c:42ff:f
 - CI：`43699b4` success；`c57daaf` 被后一 commit 的并发策略取消（同分支 cancel-in-progress），其树被 `b9ebc87` 的 success 完整覆盖；`b9ebc87` success。
 - tracker 收编：**关** #10/#20/#21/#22（#20→`d9246dd`、#21→`f5a76cf`、#22→`3f2b28d`、#10→`23792d6`，均附证据评论）+ **关** #1/#8（本轮实现并真机验证）；**注记**（保持打开作老装机待办）#2/#3/#4；**留存** #9/#12/#13/#15（真增强，超出生产可用主干，未动）。
 
+### 本轮新增真机证据（第四十七批：首个稳定版 `v0.1.0` 发布链全实弹 + release 通道无 checkout 全量安装 + `felis update` 双向报告）
+
+**零、现场**：切首个稳定 tag `v0.1.0` → `b9c97ff`（annotated），release run `35950722509` 全绿，产物 `felis-linux-amd64` 61,378,722 B / `felis-linux-arm64` 57,344,162 B；`GET /releases/latest` 现解析到 `v0.1.0`（`prerelease=false`），`v0.1.0-rc1` 保持 Pre-release。VM 侧事件前快照 `/root/probe77/pre77/`（host toml + deploys + crs + `hostbin.sha`=`106c4c9e…`）；引导函数副本 `/root/probe77/bootstrap-funcs.sh`（删 `main "$@"` 行、可 source）与完整版 `/root/probe77/bootstrap-full.sh`（sha256 `89d0181d…` = 仓库 `deploy/bootstrap.sh` 逐字节）。
+
+**一、Stage 1 — release 二进制下载（真实 github_api + asset + 原子安装）**：
+- `resolve_install_ref`（release 通道）→ `REF=v0.1.0`；
+- `download_release_binary`：宿主二进制 `felis v0.0.0+gunknown`（sha `106c4c9e…`）→ `felis v0.1.0`（sha `a64f32c5…`，57,344,162 B 与 arm64 资产一致）；
+- 复跑收敛：`host binary is already v0.1.0; skipping the download`（零 API、零下载）。留档 `stage1.log`、`stage1-release-download.sh`。
+
+**二、Stage 2 — 无 checkout 全量安装（真实 `curl|bash` 形态；本批主线）**：
+- 配方：`systemctl stop felis-velocity` → `mv /opt/felis/src src.bak47`（全程无 checkout）→ `bash < bootstrap-full.sh`（stdin 形态），`FELIS_IMAGE=…/felis/felis:v0.1.0`、`FELIS_WORLDS_HOST_PATH=/var/lib/rancher/k3s/storage`（对齐在册 reaper 形态）；
+- 结果：**rc=0、2m17s、`[fail]`=0、零 `[warn]`**（留档 `stage2.log`、`stage2-run.sh`）。
+- 关键路径逐条为真：`use_release_binary` 命中 → 跳过下载（`HAVE_PREBUILT_BINARY=1`）→ `build_image_from_binary`（宿主二进制裹 distroless 镜像并导入 k3s）→ **`game_stack_source` 走 `unpacking the embedded game-stack sources (no checkout on this host)`**（二进制内嵌 tar 解包）→ limbo/lobby/paper 构建（Paper 26.3-38）→ 四镜像全 mirror 进内部 registry（实测 tags：`felis` 增 `v0.1.0`、limbo/lobby/paper = `demo`）→ api/operator 收敛到 `felis:v0.1.0` 且 rollout 全绿 → **minecraft ns `felis-reaper` CronJob 模板同步为 `felis:v0.1.0`** → 既有系统服 login/lobby 滚到新镜像 Running。
+- 数据面复验：`users=16`、`servers=2`（resolvecheck/test-one）不变；面板 `https 200`；`/etc/felis/felis.host.toml` 与事件前快照**逐字节一致**（手改保留承诺实测）；`bootstrap.done` 更新至 `03:32:18Z`。
+- 收尾：`src.bak47` 复原为 `/opt/felis/src`；docker 停回 inactive。
+
+**三、Stage 3 — `felis update` 报告（双向对照）**：
+- `v0.1.0-rc1` 二进制：`felis-api v0.1.0-rc1 -> v0.1.0 update available (notify)`；velocity `3.5.1 -> 4.2.0 (notify)`；附 apply 命令（重跑安装器 + 私仓 token 指引文案）；
+- 现装 `v0.1.0` 二进制：`felis-api v0.1.0 up to date`——锤实「felis-api 已装版本 = 运行中二进制自身版本」（panel 内嵌，无独立版本可探）；
+- velocity minor（3.5.x→4.2.0）按设计走 notify；installer 只取 pinned minor 的最新 build。
+
+**四、运维注记（非缺陷）**
+- pg_hba `felis_pgint` 行又被重装清掉（第三轮）——**根因定位**：前两轮的补回位置落在 `# BEGIN/END FELIS MANAGED HBA` **块内**，重写段的 skip 对块内照删；本轮改为插在 `# END FELIS MANAGED HBA` 之后（块外，清理条件 `$2==felis` 不命中），下轮重装可复检；速查已注明正确插入位。
+- 控制面镜像引用：`auditfix77` → `felis:v0.1.0`（release 二进制包裹）；回退面 = 旧 `auditfix77` 仍在宿主 docker 与 registry；registry 的 `v0.1.0` tag 保留为 kubelet 回拉源。
+- 演练后 VM 复位：docker inactive、k3s/felis-velocity active、test-one 仍 Stopped。
+
+**五、结论**：stable release 通道端到端闭合——「tag → CI 双资产 → `/releases/latest` → 无 checkout 全量安装（embedded 资产 + registry mirror）→ 控制面/游戏栈全绿 → `felis update` 报告」全链实弹，无新缺陷。
+
 ## 结论：离"生产可用"还差什么（按优先级）
 
 1. ~~构建链路的上下文通道~~ ✅ **已修**（`f79e5eb`/`02fd2de`，真机全链路含拉回校验；Trivy DB 需按 §8e 镜像一次）。
@@ -906,7 +936,7 @@ IPv6-only 接入（`ssh -6 -i ~/.ssh/id_ed25519 root@fdb2:2c26:f4e4:0:21c:42ff:f
 3. ~~PG 级契约测试~~ ✅ **已落地**（`2a55a0d`）：`internal/pgint`（`-tags pgint`，需 `FELIS_TEST_PG_URL` 指向名字含 `pgint` 的库，harness 会 drop schema + 重放真实迁移）已覆盖会话/OTP/op-login/绑定码/submission/build/owner 角色；**首跑即抓到 #20**（索引与 ErrEmailTaken 从未存在）。运行方式见 CONTRIBUTING.md。
 4. ~~面板把 /jobs 显示出来~~ ✅ **已完成**（`97a64c8`，备份页「最近操作」卡，正是它把 #35 暴露出来的）。
 5. ~~多节点回收~~ ✅ **已修**（`daf7602`：`--reaper-node` → CronJob pod `kubernetes.io/hostname` nodeSelector，真机 render/dry-run/收敛 diff 三连；单节点部署不传即维持原状）。附带核销缺陷 #38（渲染提示里过期的 uid-1000/setfacl 指导）。
-6. ~~告警~~ ✅ **已完成**（`94f71ee` + `43df08b` + 第二十七批实弹演练：真实构建失败 → pending → 08:33:14Z firing；规则随 `deploy/alerts/` 交付）。
+6. ~~告警~~ ✅ **已完成**（`94f71ee` + `43df08b` + 第二十七批实弹演练：真实构建失败 → pending → 08:33:14Z firing；规则随 `deploy/alerts/` 交付）。**2026-09-24 补齐无 Prometheus 的告警面**（`d17524c`）：主机 `felis-watchdog.timer` 每 2 分钟巡检控制面/登录门/系统服/失败服/Job/reaper/节点/PG/代理/DB 备份/磁盘/内存，按持续时长门限给平台所有者发邮件；operator 增加 `felis_server_phase`、`felis_build_info`、卡死存活探针，规则新增 `felis.platform.rules`/`felis.jobs.rules`（promtool 22 条全过）。真机：felis-api 缩到 0 → 5 分钟后告警邮件落地（"Felis 严重告警…1 项异常"），恢复 10 分钟后收到恢复邮件。
 7. ~~玩家可见的构建结果~~ ✅ **已修**（`72c4aa3`，缺陷 #36：列表路由附 `build_status`/`build_error`，面板「我的提交」展开行呈现，真机双例验证）。
 8. ~~面板文件编辑器入口~~ ✅ **已补**（`0a36b3f`，缺口补齐，真机 CDP 全链）。
 9. ~~fleet 系统服务死操作~~ ✅ **已修**（`2f90851`，缺陷 #37）。
@@ -932,6 +962,8 @@ IPv6-only 接入（`ssh -6 -i ~/.ssh/id_ed25519 root@fdb2:2c26:f4e4:0:21c:42ff:f
 29. ~~未完成引导的导航（tracker #8）~~ ✅ **已修并真机闭环**（第四十六批 #77：`403 setup_required` → 自动 `/setup`；CDP 复验）。
 30. ~~已装机系统服的新增 CR 字段（tracker #1）~~ ✅ **已修并真机闭环**（第四十六批 #78：`sudo felis converge`——零值才填、非零不覆写；剥字段→填回→幂等三连真机过）。
 31. ~~troubleshooting `[INERT]` 图例~~ ✅ **已修**（第四十六批 #79，文档级）。
+32. ~~稳定版发布与 release 安装/升级通道~~ ✅ **已实证**（第四十七批：tag `v0.1.0`（`b9c97ff`）+ release run `35950722509` 双资产、`/releases/latest` 解析到 v0.1.0；无 checkout 全量安装 2m17s / 零 fail / 零 warn、registry mirror 四镜像、reaper CronJob 对齐 `felis:v0.1.0`；`felis update` 双向报告〔rc1→v0.1.0 notify / v0.1.0 up to date〕）。
+33. ~~世界归档与数据库备份只在本机~~ ✅ **已修并真机闭环**（2026-09-24：`felis offsite` + `felis-offsite.timer` 每小时把世界归档与 DB bundle 以 AES-256-GCM 分段加密推到 S3 兼容桶，`world_backups.offsite_at` 记账〔迁移 0024〕；配了 `[offsite]` 后 reaper 只在归档的异地副本确认后才删 PVC；watchdog 12 小时无成功同步即告警；安装器 `FELIS_OFFSITE_*` 生成密钥并在收尾横幅要求离机保存。真机（MinIO）：首次同步 8 世界 1.2 GiB + 12 bundle、两条"记录在册但盘上已无"的历史归档如实列出；`fetch-db latest` sha256 与本机一致、错误密钥拒绝且不留半成品；`fetch-worlds` 取回被挪走的归档 sha256 一致；reaper 演练〔20 天空闲世界〕第一轮 `awaiting_offsite=1` 且 PVC 保留 → 同步后第二轮 `reaped=1`、PVC 删除、所有权释放；watchdog 把 status 改成 35 小时前 → `[warning] offsite`。演练装置已清理）。
 
 ## 剩余待演练队列（截至第四十三批）
 
@@ -978,3 +1010,4 @@ IPv6-only 接入（`ssh -6 -i ~/.ssh/id_ed25519 root@fdb2:2c26:f4e4:0:21c:42ff:f
 - 第四十四批工具与留档：200MiB 上下文构造 = 5×40MiB `head -c 41943040 /dev/urandom` + `Dockerfile`（`FROM scratch` + `COPY payload /payload`），`tar -czf /root/bigctx.tar.gz Dockerfile payload`；上传 = `curl -sk -b /tmp/owner-jar43.txt -X POST -H "Content-Type: application/gzip" --data-binary @/root/bigctx.tar.gz https://127.0.0.1:30443/api/v1/me/submissions/<id>/context`；采样 = `k3s crictl stats`（此版 **无 `--no-trunc`**，列解析取 `$2=NAME $4=MEM`）；docker 缓存清理 = `systemctl start docker && docker builder prune -af && systemctl stop docker`；留档 `/root/stats44.log`、`/root/stats44-build.log`、提交 `sub-cc160b3ad19080f9`、镜像 `e2e/conc-b44-{1,2,3}`。
 - 第四十五批工具与留档：从平台底座构建的配方 = 提交上下文含 `Dockerfile`（`FROM registry.felis.svc:5000/felis/paper:demo` + `COPY marker.txt /felis-probe-marker.txt`）；装服验证 = `PATCH /api/v1/servers/test-one {"image":"registry.felis.svc:5000/user-uploads/<sub>:latest"}` → `POST /servers/test-one/wake` → `kubectl -n minecraft exec test-one-0 -- cat /felis-probe-marker.txt`；trivy 双 DB 镜像 = `mirror/trivy-db:2` + `mirror/trivy-java-db:1`（Java DB digest `5766dfbb…`；两键在 `/etc/felis/felis.{host,pod}.toml` 与 felis-config Secret 双副本）；`bootstrap_test.sh` 需在 Linux 跑（VM 留档 `/root/btest72/`，应 140 PASS）；本批提交样本 `sub-{b45d416f4af811ef(无镜像),83f677e41acd80c3,171e8f967f0de3bc,c006cbd633317ccb,cc160b3ad19080f9}`；留档 `/root/probe44/`、`/root/probe44.tar.gz`、`/root/probe44*.sid`。
 - 第四十六批工具与留档：控制面三处 = `registry.felis.svc:5000/felis/felis:auditfix77`（宿主 docker 源 `10.43.182.43:5000/felis/felis:auditfix77`，`v0.0.0+fix77`；api/operator + minecraft ns `felis-reaper` CronJob + api/operator `FELIS_IMAGE` 四处成对改）；`/opt/felis/src` = `b9ebc87` 快照（上一版 `src.bak46`）；宿主 drill 二进制 `/root/felis-fix77.bin`（Mac 侧 `CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=v0.0.0+fix77" ./cmd/felis`；scp 到 IPv6 主机时主机位必须写 `root@[fdb2:…]` 方括号）；#75/#76/#77/#78 留档 `/root/probe77/`（`create-lane.log`、`lane2.log`、`lane3.log`〔含玩家会话矩阵 + 分级 ZT 对照 + curl 7.76 在 `-H Host:` 下**不发 jar cookie**的坑——host 路由 drill 用显式 `-H "Cookie: felis_session=…"`〕、`converge-1.log`/`converge-2.log`、`crs-before.yaml`、各步请求/响应 json）；面板 CDP 截图（Mac）：`/tmp/setup8-redirect.png`、`/tmp/withdraw-{1,2}-*.png`、`/tmp/admdelete-{1,2}-*.png`，脚本 `/tmp/cdp-setup8.js`、`/tmp/cdp-withdraw76.js`、`/tmp/cdp-admdelete76.js`；锁定会话铸法 = `sha256('drill-locked-77')` 直插 `sessions`（用户 `3f2c1b0a-…`）；玩家会话铸法 = `[email protected]` 邮件 OTP（码在 felis-api 日志 `no Mailer configured` 行；`edge-dis2` 是软删死账号，登录门按设计排除）；pgint 前置：`pg_hba` 需 `host felis_pgint felis 127.0.0.1/32 scram-sha-256`（本批第二次丢失并补回）；发布链 smoke = tag `v0.1.0-rc1`（release run `35949621233`），prerelease 不移动 `/releases/latest`——无 stable 时 bootstrap 默认通道给出显式指引后 die、`felis update` 404 报错可读；首个稳定 tag 是产物决定（未代拍板）。
+- 第四十七批工具与留档：stable tag `v0.1.0` = `b9c97ff`（annotated；release run `35950722509`；资产 amd64 61,378,722 B / arm64 57,344,162 B；`/releases/latest` = v0.1.0、rc1 保持 prerelease）。VM `/root/probe77/`：`stage1-release-download.sh`+`stage1.log`（REF=v0.1.0；host bin sha `106c4c9e…`→`a64f32c5…`）、`stage2-run.sh`+`stage2.log`（无 checkout 全量安装 rc=0/2m17s/零 fail warn；"unpacking the embedded game-stack sources" 行）、`bootstrap-funcs.sh`（可 source 副本）、`bootstrap-full.sh`（完整版，sha `89d0181d…`）、`pre77/`（事件前快照）；Stage 3 = `/root/felis-rc1.bin update --all`（rc1→v0.1.0）对照现装 `felis update --all`（v0.1.0 up to date）；"无 checkout 主机"演练配方 = 停 felis-velocity → `mv /opt/felis/src src.bak47` → `bash < bootstrap-full.sh` → 复原 mv；pg_hba `felis_pgint` 行**必须插在 `# END FELIS MANAGED HBA` 之后**（块内会在重装时被重写段删除——三轮同根因；本轮已按块外补回）。

Dockerfile

@@ -21,14 +21,18 @@
 # minutes. The FINAL stage is deliberately NOT pinned — it must stay on the target platform
 # or the published arm64 image would carry amd64 layers. It contains only COPY, which
 # BuildKit performs itself, so it needs no QEMU either; adding a RUN there would.
-FROM --platform=$BUILDPLATFORM node:22-bookworm AS panel
+#
+# Every base image here and in deploy/{limbo,lobby,paper} is pinned by digest, so a rebuild
+# of one release uses the same bytes; .github/dependabot.yml proposes the bumps (tag and
+# digest together).
+FROM --platform=$BUILDPLATFORM node:22-bookworm@sha256:363e1587494626837fa7f9a23bdb453d13b0ff3c67c705c2805cfc69c2d2fad7 AS panel
 WORKDIR /panel
 COPY panel/package*.json ./
 RUN npm ci
 COPY panel/ ./
 RUN npm run build
 
-FROM --platform=$BUILDPLATFORM golang:1.26 AS build
+FROM --platform=$BUILDPLATFORM golang:1.26@sha256:6c2a5538f964f1c82f97ad14988bf05de100d922d159d0e398b54c7b0ca0c6c9 AS build
 WORKDIR /src
 ARG TARGETOS=linux
 ARG TARGETARCH
@@ -55,7 +59,7 @@ ARG FELIS_VERSION=dev
 RUN CGO_ENABLED=0 GOOS="$TARGETOS" GOARCH="${TARGETARCH:-$(go env GOARCH)}" \
     go build -trimpath -ldflags="-s -w -X main.version=${FELIS_VERSION}" -o /out/felis ./cmd/felis
 
-FROM gcr.io/distroless/static-debian12:nonroot
+FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
 ENV PATH=/usr/local/bin:/usr/bin:/bin
 COPY --chmod=0755 --from=build /out/felis /usr/local/bin/felis
 # distroless "nonroot" is uid 65532; the rendered PodSecurityContext pins

README.md

@@ -18,7 +18,8 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy,
 - **即开即玩**：玩家尝试连接时自动唤醒服务器，空闲后自动休眠，像游戏主机一样省资源。
 - **Web 控制面板**：浏览器中查看服务器状态、在线玩家与资源用量，管理备份与恢复。
 - **备份与恢复**：一键把整服数据（世界、配置、插件/模组，即整个 /data 卷）打包进集群内的归档库，支持从任意备份点回滚；默认安装就已启用（归档 PVC 与路径由安装器一并生成）。
-- **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。
+- **控制面数据库备份**：账号、服务器归属、配额与存档索引所在的数据库每天自动备份，每次升级迁移前先快照，出错可用 `felis db restore` 整库原子回滚；面板「维护与备份」页显示备份是否新鲜（见 [故障排查 §16](docs/troubleshooting.md)）。
+- **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。过期备份无论是否开启都会每天清理。
 - **多核心支持**：兼容 Paper、Fabric、Forge、NeoForge，经由 Velocity 代理统一入口。
 - **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建；构建产物进入镜像白名单，可直接选用为服务器镜像完成部署。
 - **Passkey 登录**：支持指纹、面容、硬件密钥等无密码认证方式。
@@ -26,7 +27,7 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy,
 
 ## 使用方式
 
-在准备好的 Linux 主机上执行：
+在准备好的 Linux 主机上执行（已验证的发行版与架构见 [运维手册 §1](docs/operations.md#1-supported-hosts)：CentOS Stream 9 aarch64 实机验证，Ubuntu 24.04 x86_64 每次推送由 CI 跑全新安装、重跑与升级）：
 
 ```bash
 curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash

README_EN.md

@@ -18,6 +18,7 @@ Table of Contents
 - **Wake on Join**: Servers start automatically when a player connects, and stop when idle — like hibernate for your server.
 - **Web Dashboard**: Monitor server status, online players, and resource usage from your browser, with backup and restore management.
 - **Backup & Restore**: One-click snapshots of a server's whole data volume (worlds, config, plugins/mods — the entire /data volume) into the cluster's archive store, with rollback from any backup point — enabled by default (the installer renders the archive PVC and its path).
+- **Control-plane database backups**: The database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migrates it; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows whether the newest backup is fresh (see [troubleshooting §16](docs/troubleshooting.md)).
 - **World Reaper** (opt in): Worlds idle for more than 15 days are automatically backed up and removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is ever deleted.
 - **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, federated behind a Velocity proxy.
 - **Modpack Submission**: Players submit custom modpacks; admin approval triggers an automatic build, and the result is whitelisted as a server image you can select to deploy.
@@ -26,7 +27,10 @@ Table of Contents
 
 ## Getting Started
 
-On a prepared Linux host, run:
+On a prepared Linux host, run (the verified distributions and architectures are listed in
+[operations §1](docs/operations.md#1-supported-hosts): CentOS Stream 9 on aarch64 is verified
+on a real host, and Ubuntu 24.04 on x86_64 gets a fresh install, rerun and upgrade in CI on
+every push):
 
 ```bash
 curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash

bootstrap_asset.go

@@ -22,14 +22,17 @@ var bootstrapAssets embed.FS
 // developer's working tree carries gradle output (plugins/*/build, plugins/*/bin,
 // and for the modded loaders a decompiled Minecraft under build/) which would
 // otherwise be baked into every felis binary. Keep them explicit — add a source
-// directory here, never a parent.
+// directory here, never a parent. Each module's gradle/verification-metadata.xml rides
+// along, since Gradle builds unverified without it; the wrapper stays out, because the
+// image builds run the pinned build image's own gradle.
 //
+//go:embed deploy/game-stack.lock
 //go:embed deploy/limbo/Dockerfile deploy/limbo/entrypoint.sh
 //go:embed deploy/lobby/Dockerfile deploy/lobby/entrypoint.sh
 //go:embed deploy/paper/Dockerfile deploy/paper/entrypoint.sh
-//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src
+//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src plugins/limbo/gradle/verification-metadata.xml
-//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src
+//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src plugins/paper/gradle/verification-metadata.xml
-//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src
+//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src plugins/velocity/gradle/verification-metadata.xml
 //go:embed plugins/shared/src
 var gameStackAssets embed.FS
 

bootstrap_asset_test.go

@@ -1,7 +1,9 @@
 package felis
 
 import (
+	"encoding/xml"
 	"io/fs"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -205,6 +207,286 @@ func requireEmbedded(t *testing.T, path string) {
 	}
 }
 
+// The lock file is the install's only source of upstream builds, and bootstrap.sh reads it
+// with a strict KEY=value parser that dies on anything unexpected, so a malformed lock is a
+// failed install on every host. Check the shipped copy the same way here.
+func TestGameStackLockIsComplete(t *testing.T) {
+	lock := gameStackLock(t)
+	m := regexp.MustCompile(`GAME_STACK_LOCK_KEYS="([^"]*)"`).FindStringSubmatch(BootstrapScript())
+	if m == nil {
+		t.Fatal("bootstrap.sh no longer declares GAME_STACK_LOCK_KEYS")
+	}
+	keys := strings.Fields(m[1])
+	sha := regexp.MustCompile(`^[0-9a-f]{64}$`)
+	for _, k := range keys {
+		v, ok := lock[k]
+		if !ok || v == "" {
+			t.Errorf("game-stack.lock does not set %s", k)
+			continue
+		}
+		if strings.HasSuffix(k, "_SHA256") && !sha.MatchString(v) {
+			t.Errorf("%s=%q is not a lowercase sha256", k, v)
+		}
+		// A moving URL pins nothing: the digest check would start failing the day
+		// upstream publishes the next build.
+		if strings.HasSuffix(k, "_URL") && strings.Contains(v, "lastSuccessfulBuild") {
+			t.Errorf("%s names a moving build: %s", k, v)
+		}
+	}
+	for k := range lock {
+		if !strings.Contains(" "+m[1]+" ", " "+k+" ") {
+			t.Errorf("game-stack.lock sets %s, which bootstrap.sh refuses as an unknown key", k)
+		}
+	}
+	// Fill's URLs are content-addressed; a lock whose digest disagrees with its own URL
+	// was edited by hand and half-way.
+	for _, name := range []string{"PAPER", "VELOCITY"} {
+		if !strings.Contains(lock[name+"_JAR_URL"], "/objects/"+lock[name+"_JAR_SHA256"]+"/") {
+			t.Errorf("%s_JAR_SHA256 is not the digest in %s_JAR_URL", name, name)
+		}
+	}
+	if !strings.Contains(lock["LIMBO_JAR_URL"], "-"+lock["MC_VERSION"]+".jar") {
+		t.Errorf("LIMBO_JAR_URL %s is not a Minecraft %s build", lock["LIMBO_JAR_URL"], lock["MC_VERSION"])
+	}
+	if !strings.Contains(lock["PAPER_JAR_URL"], "/paper-"+lock["MC_VERSION"]+"-") {
+		t.Errorf("PAPER_JAR_URL %s is not a Minecraft %s build; the lobby would not speak the login gate's protocol", lock["PAPER_JAR_URL"], lock["MC_VERSION"])
+	}
+}
+
+// Each downloaded jar's digest is a build-arg bootstrap.sh passes and the Dockerfile must
+// both require and spend on the file it downloaded; docker only warns about an unknown
+// --build-arg, so a renamed arg would ship an unchecked jar.
+func TestGameStackDigestsReachTheImageBuilds(t *testing.T) {
+	script := BootstrapScript()
+	for _, c := range []struct{ dockerfile, arg, path string }{
+		{"deploy/limbo/Dockerfile", "LIMBO_JAR_SHA256", "/limbo/Limbo.jar"},
+		{"deploy/limbo/Dockerfile", "LIMBO_SCHEM_SHA256", "/limbo/spawn.schem"},
+		{"deploy/lobby/Dockerfile", "LUCKPERMS_JAR_SHA256", "/paper/plugins/LuckPerms.jar"},
+	} {
+		if !strings.Contains(script, "--build-arg "+c.arg+"=\"$"+c.arg+"\"") {
+			t.Errorf("bootstrap.sh never passes --build-arg %s", c.arg)
+		}
+		dockerfile := readGameStackFile(t, c.dockerfile)
+		if !strings.Contains(dockerfile, "ARG "+c.arg) {
+			t.Errorf("%s declares no ARG %s", c.dockerfile, c.arg)
+		}
+		if !strings.Contains(dockerfile, `echo "$`+c.arg+`  `+c.path+`" | sha256sum -c`) {
+			t.Errorf("%s never verifies %s against %s", c.dockerfile, c.path, c.arg)
+		}
+	}
+}
+
+// A base image named by tag alone is whatever the tag points at on build day.
+func TestDockerfileBaseImagesArePinnedByDigest(t *testing.T) {
+	root, err := os.ReadFile("Dockerfile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	files := map[string]string{"Dockerfile": string(root)}
+	for _, name := range []string{"deploy/limbo/Dockerfile", "deploy/lobby/Dockerfile", "deploy/paper/Dockerfile"} {
+		files[name] = readGameStackFile(t, name)
+	}
+	pinned := regexp.MustCompile(`^FROM (--platform=\S+ )?\S+:\S+@sha256:[0-9a-f]{64}( AS \S+)?$`)
+	for name, body := range files {
+		n := 0
+		for line := range strings.SplitSeq(body, "\n") {
+			if !strings.HasPrefix(line, "FROM ") {
+				continue
+			}
+			n++
+			if !pinned.MatchString(line) {
+				t.Errorf("%s: %q is not pinned by digest", name, line)
+			}
+		}
+		if n == 0 {
+			t.Errorf("%s has no FROM line", name)
+		}
+	}
+}
+
+// The plugin jars are built in three places the installer controls — the lobby and limbo
+// image builds and bootstrap's Velocity build — and through each module's wrapper by a
+// developer or CI. A tag alone is whatever it points at on build day, and two Gradle
+// versions are two chances for a build to pass in one place and break in the other, so
+// all of them run one image, pinned by digest, whose Gradle is the wrappers' Gradle.
+func TestPluginBuildsRunOnePinnedGradle(t *testing.T) {
+	sources := map[string]string{
+		"deploy/lobby/Dockerfile": readGameStackFile(t, "deploy/lobby/Dockerfile"),
+		"deploy/limbo/Dockerfile": readGameStackFile(t, "deploy/limbo/Dockerfile"),
+		"deploy/bootstrap.sh":     BootstrapScript(),
+	}
+	anyRef := regexp.MustCompile(`gradle:[\w.-]+(@sha256:\w+)?`)
+	pinned := regexp.MustCompile(`^gradle:(\d+\.\d+(?:\.\d+)?)-jdk\d+@sha256:[0-9a-f]{64}$`)
+	images := map[string]bool{}
+	gradle := ""
+	for name, body := range sources {
+		refs := anyRef.FindAllString(body, -1)
+		if len(refs) == 0 {
+			t.Errorf("%s names no gradle image", name)
+		}
+		for _, ref := range refs {
+			m := pinned.FindStringSubmatch(ref)
+			if m == nil {
+				t.Errorf("%s: %s is not a gradle image pinned by digest", name, ref)
+				continue
+			}
+			images[ref] = true
+			gradle = m[1]
+		}
+	}
+	if len(images) != 1 {
+		t.Fatalf("the plugin builds use %d different gradle images, want one: %v", len(images), images)
+	}
+	// bootstrap names the image once and has to spend it where it builds the jar.
+	if !strings.Contains(BootstrapScript(), `"$PLUGIN_BUILD_IMAGE" gradle --no-daemon clean build`) {
+		t.Error("build_velocity_plugin does not build in $PLUGIN_BUILD_IMAGE")
+	}
+
+	for _, module := range []string{"velocity", "paper", "limbo"} {
+		props := wrapperProperties(t, module)
+		if want := "gradle-" + gradle + "-bin.zip"; !strings.HasSuffix(props["distributionUrl"], "/"+want) {
+			t.Errorf("plugins/%s wrapper runs %s; the image builds run Gradle %s", module, props["distributionUrl"], gradle)
+		}
+	}
+	// The mods are no part of the install, but a wrapper without a checksum runs whatever
+	// the download handed it.
+	for _, module := range []string{"velocity", "paper", "limbo", "fabric", "forge", "neoforge"} {
+		if sum := wrapperProperties(t, module)["distributionSha256Sum"]; !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(sum) {
+			t.Errorf("plugins/%s wrapper pins no distribution sha256 (got %q)", module, sum)
+		}
+	}
+}
+
+// Each plugin compiles against the API of the exact build the install runs, and Gradle
+// checks those bytes against the module's verification file. Nothing but this test ties
+// the three to deploy/game-stack.lock: a lock refresh that leaves them behind builds the
+// lobby against yesterday's API, or fails every image build on a checksum the file does
+// not have.
+func TestPluginApisAreTheLockedBuilds(t *testing.T) {
+	lock := gameStackLock(t)
+
+	// Paper: paper-<mc>-<build>.jar runs; paper-api <mc>.build.<build>-<channel> compiles.
+	jar := regexp.MustCompile(`/paper-([^/]+)-(\d+)\.jar$`).FindStringSubmatch(lock["PAPER_JAR_URL"])
+	if jar == nil {
+		t.Fatalf("PAPER_JAR_URL %s does not name paper-<mc>-<build>.jar", lock["PAPER_JAR_URL"])
+	}
+	dep := regexp.MustCompile(`compileOnly 'io\.papermc\.paper:paper-api:([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/paper/build.gradle"))
+	if dep == nil {
+		t.Fatal("plugins/paper/build.gradle declares no paper-api dependency")
+	}
+	if !regexp.MustCompile(`^` + regexp.QuoteMeta(jar[1]+".build."+jar[2]) + `(-[a-z]+)?$`).MatchString(dep[1]) {
+		t.Errorf("paper-api %s is not the API of the locked server paper-%s-%s.jar", dep[1], jar[1], jar[2])
+	}
+	requireVerified(t, "paper", "io.papermc.paper", "paper-api", dep[1])
+
+	// Limbo: the lock's release, passed to the image build, which refuses to guess one.
+	limbo := lock["LIMBO_VERSION"]
+	if !strings.Contains(BootstrapScript(), `--build-arg LIMBO_VERSION="$LIMBO_VERSION"`) {
+		t.Error("bootstrap.sh does not pass the locked LIMBO_VERSION to the limbo image build")
+	}
+	dockerfile := readGameStackFile(t, "deploy/limbo/Dockerfile")
+	if !regexp.MustCompile(`(?m)^ARG LIMBO_VERSION$`).MatchString(dockerfile) ||
+		!strings.Contains(dockerfile, `if [ -z "${LIMBO_VERSION:-}" ]`) {
+		t.Error("deploy/limbo/Dockerfile does not require LIMBO_VERSION; a build without it would " +
+			"compile against a version nobody chose")
+	}
+	// LOOHP publishes the jar the login gate runs as the Limbo API artifact itself, so the
+	// checksum Gradle holds for it is the lock's: compiled-against and running are one file.
+	if got := requireVerified(t, "limbo", "com.loohp", "Limbo", limbo)["Limbo-"+limbo+".jar"]; got != lock["LIMBO_JAR_SHA256"] {
+		t.Errorf("verification-metadata.xml holds %q for Limbo-%s.jar; the login gate runs %s", got, limbo, lock["LIMBO_JAR_SHA256"])
+	}
+
+	// Velocity: the API default is the proxy the install runs.
+	api := regexp.MustCompile(`findProperty\('velocityApi'\) \?: '([^']+)'`).
+		FindStringSubmatch(readGameStackFile(t, "plugins/velocity/build.gradle"))
+	if api == nil {
+		t.Fatal("plugins/velocity/build.gradle has no velocityApi default")
+	}
+	if api[1] != lock["VELOCITY_VERSION"] {
+		t.Errorf("velocity-api defaults to %s; the install runs Velocity %s", api[1], lock["VELOCITY_VERSION"])
+	}
+	requireVerified(t, "velocity", "com.velocitypowered", "velocity-api", api[1])
+}
+
+// requireVerified asserts the module's shipped verification file checks metadata and
+// pins group:name:version, and returns that component's artifact sha256s by file name.
+func requireVerified(t *testing.T, module, group, name, version string) map[string]string {
+	t.Helper()
+	path := "plugins/" + module + "/gradle/verification-metadata.xml"
+	var doc struct {
+		VerifyMetadata bool `xml:"configuration>verify-metadata"`
+		Components     []struct {
+			Group     string `xml:"group,attr"`
+			Name      string `xml:"name,attr"`
+			Version   string `xml:"version,attr"`
+			Artifacts []struct {
+				Name   string `xml:"name,attr"`
+				SHA256 []struct {
+					Value string `xml:"value,attr"`
+				} `xml:"sha256"`
+			} `xml:"artifact"`
+		} `xml:"components>component"`
+	}
+	if err := xml.Unmarshal([]byte(readGameStackFile(t, path)), &doc); err != nil {
+		t.Fatalf("%s: %v", path, err)
+	}
+	if !doc.VerifyMetadata {
+		t.Errorf("%s does not verify metadata; a swapped pom could redirect the graph", path)
+	}
+	for _, c := range doc.Components {
+		if c.Group != group || c.Name != name || c.Version != version {
+			continue
+		}
+		sums := map[string]string{}
+		for _, a := range c.Artifacts {
+			if len(a.SHA256) > 0 {
+				sums[a.Name] = a.SHA256[0].Value
+			}
+		}
+		if sums[name+"-"+version+".jar"] == "" {
+			t.Errorf("%s pins %s:%s:%s but no sha256 for its jar", path, group, name, version)
+		}
+		return sums
+	}
+	t.Errorf("%s has no checksum for %s:%s:%s; the build would refuse it", path, group, name, version)
+	return nil
+}
+
+// wrapperProperties reads a module's gradle-wrapper.properties off disk: the wrappers are
+// for developers and CI, and nothing embeds them.
+func wrapperProperties(t *testing.T, module string) map[string]string {
+	t.Helper()
+	b, err := os.ReadFile("plugins/" + module + "/gradle/wrapper/gradle-wrapper.properties")
+	if err != nil {
+		t.Fatal(err)
+	}
+	props := map[string]string{}
+	for line := range strings.SplitSeq(string(b), "\n") {
+		if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok && !strings.HasPrefix(k, "#") {
+			props[k] = strings.ReplaceAll(v, `\:`, ":")
+		}
+	}
+	return props
+}
+
+// gameStackLock parses the shipped deploy/game-stack.lock the way bootstrap.sh does.
+func gameStackLock(t *testing.T) map[string]string {
+	t.Helper()
+	lock := map[string]string{}
+	for line := range strings.SplitSeq(readGameStackFile(t, "deploy/game-stack.lock"), "\n") {
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			t.Fatalf("not a KEY=value line: %q", line)
+		}
+		lock[k] = v
+	}
+	return lock
+}
+
 func readGameStackFile(t *testing.T, name string) string {
 	t.Helper()
 	b, err := gameStackAssets.ReadFile(name)

cmd/felis/api.go

@@ -5,10 +5,14 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"felis.lolicon.best/internal/api"
@@ -17,19 +21,25 @@ import (
 	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/fileedit"
+	"felis.lolicon.best/internal/imagepin"
 	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/metrics"
 	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/panel"
 	"felis.lolicon.best/internal/passkey"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/reaper"
+	"felis.lolicon.best/internal/registryprune"
 	"felis.lolicon.best/internal/restore"
-	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/retention"
 	"felis.lolicon.best/internal/submit"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -54,10 +64,8 @@ func authSourcesFromConfig(configured []config.AuthSourceConfig) []api.AuthSourc
 }
 
 // cmdAPI runs felis-api: two listeners, two middleware chains (spec §7). The
-// internal face (service token) is fully wired. The external face is wired but
+// internal face authenticates per-caller service tokens; the external face
-// fails closed until an Access JWKS key function is configured — the verifier's
+// authenticates the local session cookie.
-// audience logic is unit-tested (internal/api), the JWKS source is a deployment
-// integration point.
 func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("api", flag.ContinueOnError)
 	fs.SetOutput(stderr)
@@ -80,9 +88,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	// Load already refused a malformed [audit] retention.
+	auditRetention, _ := cfg.Audit.RetentionPeriod()
+	if auditRetention == 0 {
+		fmt.Fprintln(stdout, "felis api: audit rows are kept forever ([audit] retention = \"forever\")")
+	} else {
+		fmt.Fprintf(stdout, "felis api: audit rows older than %d days are deleted ([audit] retention; export them first with felis db audit-export)\n", int(auditRetention/(24*time.Hour)))
+	}
+
 	ctx := ctrl.SetupSignalHandler()
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Before anything serves: an api on a schema it was not built for answers with
+	// errors, or writes rows the other version cannot read.
+	drv, err := openStore(ctx, cfg.Database.URL, false)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis api: open database: %v\n", err)
 		return 1
@@ -109,15 +127,24 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	token := os.Getenv("FELIS_SERVICE_TOKEN")
+	metrics.SetBuildInfo("api", resolvedVersion())
-	if token == "" {
+
-		fmt.Fprintln(stderr, "felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers")
+	internalAuth, err := internalCallerTokens(os.Getenv)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: internal face tokens: %v\n", err)
+		return 1
+	}
+	for _, ct := range naming.CallerTokens {
+		if internalAuth[api.Caller(ct.Caller)] == "" {
+			fmt.Fprintf(stderr, "felis api: warning: %s unset — the internal face turns the %s caller away\n", ct.APIEnv, ct.Caller)
+		}
 	}
 
 	// Email one-time codes go through the [smtp] relay when one is configured; the
 	// password is read from the env var password_ref names (default SMTPPasswordEnv,
 	// injected from the felis-smtp Secret). No [smtp] host ⇒ mailer stays nil and
-	// deliverOTP logs each code server-side (the pre-SMTP bootstrap posture).
+	// every door that mails a code answers 503 mail_unavailable: a code that is
+	// not mailed is never written anywhere else either.
 	var mailer api.OTPMailer
 	if cfg.SMTP.Host != "" {
 		passRef := cfg.SMTP.PasswordRef
@@ -128,15 +155,12 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		if cfg.SMTP.Username != "" && password == "" {
 			fmt.Fprintf(stderr, "felis api: warning: [smtp] username is set but credentials env %s is empty — OTP sends will fail AUTH\n", passRef)
 		}
-		mailer = &mail.SMTP{
+		mailer = smtpRelay(cfg.SMTP, password)
-			Host:     cfg.SMTP.Host,
+		if !cfg.SMTP.TLSRequired() {
-			Port:     cfg.SMTP.Port,
+			fmt.Fprintf(stderr, "felis api: warning: [smtp] %s may be sent codes without TLS (require_tls off or a relay on this host)\n", cfg.SMTP.Host)
-			From:     cfg.SMTP.From,
-			Username: cfg.SMTP.Username,
-			Password: password,
 		}
 	} else {
-		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email one-time codes are logged, not mailed")
+		fmt.Fprintln(stderr, "felis api: [smtp] not configured — email sign-in and verification are off (503 mail_unavailable); sign in with a passkey, or run felis setup to add a relay")
 	}
 
 	// Build subsystem (spec §16): the weak-SA build Job runs in the configured
@@ -147,11 +171,14 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// The fetch initContainer runs THIS image's fetch-context entrypoint, so the
 	// build config carries the api's own image (the platform sets FELIS_IMAGE).
 	buildCfg.FelisImage = os.Getenv("FELIS_IMAGE")
+	buildJobs := build.NewK8sJobs(cl, buildCfg)
 	builder := &build.Builder{
 		Store:    build.NewPGStore(drv.DB()),
-		Jobs:   build.NewK8sJobs(cl, buildCfg),
+		Jobs:     buildJobs,
 		Config:   buildCfg,
+		Outcomes: build.NewK8sOutcomes(clientset, buildCfg),
 	}
+	go probeBuildUserNamespaces(ctx, buildJobs, buildCfg, stderr)
 
 	// User-modpack approval lane (user-directed extension over §16; see
 	// internal/submit). An ordinary user may only SUBMIT a
@@ -203,6 +230,21 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		ContextBaseURL: internalAPIBaseURL(),
 		Blobs:          blobs,
 	}
+	if blobs != nil {
+		submissions.Parts = &submit.PartStore{Dir: uploadPartsDir(contextBase)}
+	}
+	if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
+		if n, err := parseByteSize(v); err != nil || n <= 0 {
+			fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
+		} else {
+			submissions.MaxStoredBytesTotal = n
+		}
+	}
+	if n, err := contextMaxBytes(cfg); err != nil {
+		fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 512Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
+	} else {
+		submissions.MaxContextBytes = n
+	}
 
 	// Restore subsystem (spec §7): the weak-SA restore Job mounts the target
 	// world PVC + the backup PVC and runs `felis restore`. It needs deployment-
@@ -257,33 +299,51 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// the same store the auth handlers write to, so a login and the next request
 	// agree on what local auth knows.
 	repo := api.NewPGRepo(drv.DB())
+	if err := api.RegisterStorePool(drv.DB()); err != nil {
+		fmt.Fprintf(stderr, "felis api: store pool metrics unavailable: %v\n", err)
+	}
 
+	// The owner's on-demand backup levers come from [archive], the same keys the
+	// backup Job and the reaper read. A malformed key leaves the defaults in
+	// place here; the reaper Job fails on it and names it.
+	rcfg, err := reaperConfig(cfg)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: %v; using the default backup limits\n", err)
+		rcfg = reaper.DefaultConfig()
+	}
+
+	serverCache, serversSynced, err := startServerCache(ctx, restCfg, scheme, cfg.K8s.Namespace, stderr)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err)
+		return 1
+	}
+	cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced)
+	jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace)
 	a := &api.API{
 		Repo:    repo,
-		Cluster: api.NewK8sCluster(cl, cfg.K8s.Namespace),
+		Cluster: cluster,
 		Console: api.NewK8sConsole(cl, cfg.K8s.Namespace),
 		Logs:    api.NewK8sLogStreamer(clientset, cfg.K8s.Namespace),
 		// Build-log stream (spec §16) is scoped to the BUILD namespace — the same
 		// value the Builder renders Jobs into — so it follows where build Pods run.
 		BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
-		Internal:    api.BearerTokenAuth{Token: token},
+		Internal:  internalAuth,
 		Builder:   builder,
+		Images:    imagePinner(cfg.Registry.URL),
 		Restorer:  restorer,
 		Backuper:  backuper,
-		JobStatus:   api.NewK8sJobStatus(cl, cfg.K8s.Namespace),
+		JobStatus: jobStatus,
+		// A restore starts with a safety snapshot; settleRestoreChains starts the
+		// restore behind each one.
+		RestoreChains: jobStatus,
 		Files:         files,
 		Submissions:   submissions,
 		Mailer:        mailer,
-		// The external face is fronted by SessionAuth: it prefers a local session
+		// The external face authenticates the local session cookie the sign-in doors
-		// cookie (minted by the passwordless doors) and otherwise delegates to the
+		// mint, live once `felis breakGlass` flips local_auth_enabled on. Cloudflare
-		// Cloudflare-Access JWT verifier, so both auth models coexist on one face. The
+		// Access, when the install sits behind it, is enforced at the edge only.
-		// delegate's Keyfunc is intentionally nil — the JWT path fails closed until a
-		// JWKS-backed key function is wired (deployment integration point) — while the
-		// local session path is live the moment `felis breakGlass` flips
-		// local_auth_enabled on.
 		External: api.SessionAuth{
 			Repo:          repo,
-			Delegate:      api.AccessVerifier{Audience: cfg.Auth.AccessJWTAud},
 			RootDomain:    cfg.Server.RootDomain,
 			AdminHostname: cfg.Auth.AdminHostname,
 		},
@@ -291,6 +351,10 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		AdminHostname: cfg.Auth.AdminHostname,
 		PanelHostname: cfg.Auth.PanelHostname,
 		WakeCooldown:  30 * time.Second,
+		// An owner may start one backup per server per manual_cooldown, and none
+		// while the store is at max_local_bytes (data-durability-9).
+		BackupCooldown: rcfg.ManualCooldown,
+		BackupStoreCap: rcfg.MaxLocalBytes,
 		// The user-modpack lane's per-user throttles: a create spaces out
 		// review-queue rows, an upload spaces out (up to 1 GiB) context streams.
 		// Separate keys, so the normal create→upload sequence stays immediate.
@@ -300,8 +364,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		// for legitimate multi-tab / multi-server watching, while capping how many
 		// upstream follow connections a single caller can tie up if their streams stall.
 		MaxStreamsPerPrincipal: 16,
+		// Public sign-in doors, per client address: a person signing in makes a
+		// handful of calls, so 20 at once refilled at 20 a minute never bites a
+		// real user and still turns a spray into a trickle. The client address
+		// is the edge's header when the install names one (config.AuthConfig).
+		AuthDoorLimit:  api.RateLimit{Burst: 20, PerMinute: 20},
+		ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(),
+		MailLimit:      mailLimit(cfg.SMTP.MaxPerHour),
+	}
+	if a.ClientIPHeader != "" {
+		fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader)
+	} else {
+		fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)")
 	}
-	fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)")
 
 	// Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned
 	// identity anchor (正版优先); config can only append namespace-rewritten third-party
@@ -342,7 +417,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain,
 		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname),
 		defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname),
-		resolvedVersion())
+		cfg.Velocity.GamePort, resolvedVersion())
 	internalSrv := newAPIServer(*internalAddr, a.InternalHandler())
 	externalSrv := newAPIServer(cfg.Server.Listen, externalHandler)
 
@@ -364,16 +439,25 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// and advance any whose Job has reached a terminal phase. GET on a build also
 	// reconciles it, but this loop converges builds nobody is polling.
 	go reconcileBuilds(ctx, builder, stderr)
+	go settleRestoreChains(ctx, a, stderr)
+
+	if pruner := registryPruner(cfg, builder.Store, cluster, stderr); pruner != nil {
+		go pruner.Loop(ctx, registryPruneInterval)
+	}
+	go reapRejectedContexts(ctx, submissions, stderr)
+	go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
+
+	servers := []*http.Server{internalSrv, externalSrv}
+	if httpsSrv != nil {
+		servers = append(servers, httpsSrv)
+	}
+	for _, srv := range servers {
+		srv.RegisterOnShutdown(a.CloseStreams)
+	}
 
 	select {
 	case <-ctx.Done():
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		shutdownServers(servers, apiShutdownGrace, stderr)
-		defer cancel()
-		_ = internalSrv.Shutdown(shutdownCtx)
-		_ = externalSrv.Shutdown(shutdownCtx)
-		if httpsSrv != nil {
-			_ = httpsSrv.Shutdown(shutdownCtx)
-		}
 		return 0
 	case err := <-errc:
 		if err != nil && err != http.ErrServerClosed {
@@ -393,8 +477,30 @@ const (
 	// apiIdleTimeout caps how long a kept-alive connection may sit idle between
 	// requests before the server closes it, bounding idle-connection exhaustion.
 	apiIdleTimeout = 120 * time.Second
+	// apiShutdownGrace is how long the listeners drain after SIGTERM. The pod gets
+	// the Kubernetes default of 30s before SIGKILL; this leaves the rest for the
+	// process to exit.
+	apiShutdownGrace = 20 * time.Second
 )
 
+// shutdownServers drains every listener at once under one deadline: in turn, a
+// slow first listener would spend the time the others needed. Log streams end
+// through RegisterOnShutdown (API.CloseStreams); what is still running when the
+// deadline passes is cut off with the process.
+func shutdownServers(servers []*http.Server, grace time.Duration, stderr io.Writer) {
+	ctx, cancel := context.WithTimeout(context.Background(), grace)
+	defer cancel()
+	var wg sync.WaitGroup
+	for _, srv := range servers {
+		wg.Go(func() {
+			if err := srv.Shutdown(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: shutdown %s: %v\n", srv.Addr, err)
+			}
+		})
+	}
+	wg.Wait()
+}
+
 // newAPIServer builds an http.Server with hardened header/idle timeouts (gosec
 // G112) shared by all three felis-api listeners (internal, external, https).
 // WriteTimeout and ReadTimeout are deliberately LEFT UNSET: the external and https
@@ -418,18 +524,49 @@ func buildConfig(cfg *config.Config) build.Config {
 	return build.Config{
 		Namespace:   cfg.Registry.BuildNamespace,
 		RegistryURL: cfg.Registry.URL,
-		// Empty overrides fall back to the build package's defaults, so an
+		// Empty overrides fall back to the registry's copies of the tools
-		// install that has not imported kaniko/trivy keeps the compiled-in refs
+		// (build.Tools), which felis mirror-build-tools keeps current.
-		// (and fails loudly on pull rather than silently building with the wrong
-		// image).
 		KanikoImage: cfg.Registry.KanikoImage,
 		TrivyImage:  cfg.Registry.TrivyImage,
 		CPULimit:    cfg.Registry.BuildCPULimit,
 		MemLimit:    cfg.Registry.BuildMemLimit,
-		// Empty keeps Trivy's own default; an install with builds points this at
+		DiskLimit:   cfg.Registry.BuildDiskLimit,
-		// the internal DB mirror (see config.RegistryConfig.TrivyDBRepository).
+		// "auto" follows the startup probe (see probeBuildUserNamespaces).
+		UserNamespaces:        cfg.Registry.BuildUserNamespaces,
+		UserNamespacesProbe:   new(atomic.Bool),
+		RuntimeClass:          cfg.Registry.BuildRuntimeClass,
+		MaxConcurrent:         cfg.Registry.MaxConcurrentBuilds,
 		TrivyDBRepository:     cfg.Registry.TrivyDBRepository,
 		TrivyJavaDBRepository: cfg.Registry.TrivyJavaDBRepository,
+		ScanFailOn:            cfg.Registry.ScanFailOn,
+		ScanFailUnfixed:       cfg.Registry.ScanFailUnfixed,
+		ScanAccept:            cfg.Registry.ScanAccept,
+		// The submit lane's derived context URLs live here; the fetch step's
+		// service token goes nowhere else.
+		ContextOrigin: internalAPIBaseURL(),
+	}
+}
+
+// probeBuildUserNamespaces settles build_user_namespaces = "auto": one probe
+// pod with hostUsers: false tells whether this node's kernel and runtime can run
+// build pods in a user namespace. Builds submitted before it answers run without.
+func probeBuildUserNamespaces(ctx context.Context, jobs *build.K8sJobs, cfg build.Config, stderr io.Writer) {
+	if mode := cfg.UserNamespaces; mode != "" && mode != build.UserNamespacesAuto {
+		return
+	}
+	if cfg.FelisImage == "" {
+		fmt.Fprintln(stderr, "felis api: FELIS_IMAGE unset — build pods run without a user namespace")
+		return
+	}
+	ok, err := jobs.ProbeUserNamespaces(ctx, cfg.FelisImage)
+	cfg.UserNamespacesProbe.Store(ok)
+	switch {
+	case ok:
+		fmt.Fprintln(stderr, "felis api: build pods run in a user namespace (hostUsers: false)")
+	case err != nil:
+		fmt.Fprintf(stderr, "felis api: build pods run without a user namespace: the probe failed: %v\n", err)
+	default:
+		fmt.Fprintln(stderr, "felis api: build pods run without a user namespace: this node cannot start a pod with hostUsers: false")
 	}
 }
 
@@ -546,3 +683,247 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
 		}
 	}
 }
+
+// settleRestoreChains starts the restore behind each safety snapshot that has
+// finished (and gives up the one behind a snapshot that failed). The world stays
+// locked in between, so the interval is how long a finished snapshot keeps the
+// server down before its restore begins.
+func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
+	t := time.NewTicker(5 * time.Second)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := a.SettleRestoreChains(ctx); err != nil {
+				fmt.Fprintf(stderr, "felis api: restore chains: %v\n", err)
+			}
+		}
+	}
+}
+
+// reapRejectedContexts deletes, once an hour, the uploaded contexts of
+// submissions rejected more than submit.RejectedContextRetention ago, and the
+// chunked uploads left untouched for submit.StalePartRetention. Without it a
+// rejected modpack or an abandoned upload keeps its bytes on the uploads store
+// (and against its submitter's budget) until an admin deletes the row.
+func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
+	t := time.NewTicker(time.Hour)
+	defer t.Stop()
+	for {
+		n, err := m.ReapRejected(ctx, submit.RejectedContextRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap rejected uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
+		}
+		n, err = m.ReapStaleParts(submit.StalePartRetention)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis api: reap abandoned uploads: %v\n", err)
+		}
+		if n > 0 {
+			fmt.Fprintf(stderr, "felis api: deleted %d abandoned chunked upload(s)\n", n)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// retentionInterval spaces the runs that delete spent sign-in rows and audit rows
+// past [audit] retention. The rows are spent for weeks before they go, so a few
+// runs a day keep the tables flat.
+const retentionInterval = 6 * time.Hour
+
+// registryPruneInterval spaces the registry pruner's runs. The registry-gc
+// sidecar sweeps once a day, so pruning more often only changes which sweep frees
+// a layer.
+const registryPruneInterval = 6 * time.Hour
+
+// registryPruner deletes the registry manifests nothing references
+// (internal/registryprune); the registry-gc sidecar frees their layers on its next
+// sweep. It acts as the gate's prune principal, whose token the api Deployment
+// injects from felis-registry-auth. Without the token the registry only grows,
+// which is said once here.
+func registryPruner(cfg *config.Config, store imageRefStore, servers serverLister, stderr io.Writer) *registryprune.Pruner {
+	if cfg.Registry.URL == "" {
+		return nil
+	}
+	token := os.Getenv(platform.RegistryPruneTokenEnv)
+	if token == "" {
+		fmt.Fprintf(stderr, "felis api: registry pruner disabled (%s unset) — images nothing uses are never deleted from the registry\n", platform.RegistryPruneTokenEnv)
+		return nil
+	}
+	static := append([]string{os.Getenv("FELIS_IMAGE")}, buildConfig(cfg).ToolRefs()...)
+	return &registryprune.Pruner{
+		Registry: &registryprune.Client{Endpoint: "http://" + cfg.Registry.URL, Token: token},
+		Host:     cfg.Registry.URL,
+		Refs: func(ctx context.Context) ([]string, error) {
+			return inUseImageRefs(ctx, store, servers, static)
+		},
+		Log: slog.New(slog.NewTextHandler(stderr, nil)),
+	}
+}
+
+type imageRefStore interface {
+	ListImages(ctx context.Context) ([]build.Image, error)
+	ListUnfinishedBuilds(ctx context.Context) ([]build.Build, error)
+}
+
+type serverLister interface {
+	ListServers(ctx context.Context) ([]api.ServerInfo, error)
+	PodImages(ctx context.Context) ([]string, error)
+}
+
+// inUseImageRefs lists every image reference the platform still depends on: the
+// whitelist (disabled rows too, an admin may enable them again), every server's
+// spec, the images the game pods run, builds still running, and the images the
+// control plane and the build Jobs run. Any source failing fails the whole list,
+// so the pruner never decides on a partial view.
+//
+// The pods matter for the felis image: a running server keeps the one it started
+// with across platform upgrades (operator.PodTemplateAnnotation), which after a
+// few releases is no longer among the newest tags the pruner keeps anyway, and
+// the pod needs it again whenever it is recreated.
+func inUseImageRefs(ctx context.Context, store imageRefStore, servers serverLister, static []string) ([]string, error) {
+	refs := append([]string(nil), static...)
+	images, err := store.ListImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("image whitelist: %w", err)
+	}
+	for _, img := range images {
+		refs = append(refs, img.ImageRef)
+	}
+	srvs, err := servers.ListServers(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("servers: %w", err)
+	}
+	for _, s := range srvs {
+		refs = append(refs, s.Image)
+	}
+	podImages, err := servers.PodImages(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("game pods: %w", err)
+	}
+	refs = append(refs, podImages...)
+	builds, err := store.ListUnfinishedBuilds(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("running builds: %w", err)
+	}
+	for _, b := range builds {
+		refs = append(refs, b.ImageRef)
+	}
+	return refs, nil
+}
+
+// mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket:
+// the hourly cap as the refill rate, with a quarter of it (at least 5) allowed
+// at once so a burst of real sign-ins is not queued behind the average.
+func mailLimit(perHour int) api.RateLimit {
+	if perHour <= 0 {
+		perHour = config.DefaultMailPerHour
+	}
+	return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60}
+}
+
+// imagePinner resolves a new server's image against the platform registry
+// through its in-cluster Service, the address its refs already spell. An install
+// without a registry has no platform-built images to pin.
+func imagePinner(registry string) api.ImagePinner {
+	if registry == "" {
+		return nil
+	}
+	return imagepin.Resolver{Registry: registry}
+}
+
+// internalCallerTokens reads each internal caller's token from the env var the
+// Deployment feeds it from (naming.CallerTokens). Two callers sharing a value
+// would make the caller ambiguous, so that refuses to start.
+func internalCallerTokens(getenv func(string) string) (api.CallerTokens, error) {
+	tokens := map[api.Caller]string{}
+	for _, ct := range naming.CallerTokens {
+		tokens[api.Caller(ct.Caller)] = strings.TrimSpace(getenv(ct.APIEnv))
+	}
+	return api.NewCallerTokens(tokens)
+}
+
+// smtpRelay is the relay [smtp] names, with the resolved password and the TLS
+// posture config.SMTPConfig.TLSRequired picks. felis api, the reaper and the
+// watchdog all send through it, so none can drift to a weaker posture.
+func smtpRelay(c config.SMTPConfig, password string) *mail.SMTP {
+	return &mail.SMTP{
+		Host:       c.Host,
+		Port:       c.Port,
+		From:       c.From,
+		Username:   c.Username,
+		Password:   password,
+		RequireTLS: c.TLSRequired(),
+	}
+}
+
+// startServerCache starts the informer that serves the api's fleet-wide
+// MinecraftServer reads (api.K8sCluster.WithServerCache): one watch on the
+// namespace instead of a full List per velocity pull, fleet page and wake. It
+// caches MinecraftServers only — ReaderFailOnMissingInformer turns any other read
+// through it into an error rather than a new informer the api's Role cannot back —
+// indexes spec.subdomain for GetBySubdomain, and drops managedFields to keep the
+// copy small. It returns without waiting: the reads block until the first list
+// lands and /readyz reports not-ready until then.
+func startServerCache(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, stderr io.Writer) (cache.Cache, func() bool, error) {
+	c, err := cache.New(cfg, cache.Options{
+		Scheme:                      scheme,
+		DefaultNamespaces:           map[string]cache.Config{namespace: {}},
+		DefaultTransform:            cache.TransformStripManagedFields(),
+		ReaderFailOnMissingInformer: true,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := c.IndexField(ctx, &v1alpha1.MinecraftServer{}, api.SubdomainIndex, api.SubdomainOf); err != nil {
+		return nil, nil, fmt.Errorf("index %s: %w", api.SubdomainIndex, err)
+	}
+	inf, err := c.GetInformer(ctx, &v1alpha1.MinecraftServer{})
+	if err != nil {
+		return nil, nil, err
+	}
+	go func() {
+		if err := c.Start(ctx); err != nil {
+			fmt.Fprintf(stderr, "felis api: MinecraftServer cache stopped: %v\n", err)
+		}
+	}()
+	return c, inf.HasSynced, nil
+}
+
+// uploadPartsDir is where chunked uploads are staged: beside a local store's
+// contexts, so the room check and the budget see one disk and a staged upload
+// survives an API restart; for an s3:// store, on the uploads volume the
+// platform mounts either way, or the pod's /tmp when run by hand without it.
+func uploadPartsDir(contextBase string) string {
+	if isLocalUploadsPath(contextBase) {
+		return filepath.Join(strings.TrimPrefix(contextBase, "file://"), ".parts")
+	}
+	if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
+		return filepath.Join(platform.UploadsLocalPath, ".parts")
+	}
+	return filepath.Join(os.TempDir(), "felis-upload-parts")
+}
+
+// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
+// package's own default (1 GiB). The Cloudflare edge refuses a single request
+// body over 100 MB, which the panel's chunked upload stays under, so the edge
+// does not lower the cap.
+func contextMaxBytes(cfg *config.Config) (int64, error) {
+	v := cfg.Registry.ContextMaxBytes
+	if v == "" {
+		return 0, nil
+	}
+	n, err := parseByteSize(v)
+	if err != nil || n <= 0 {
+		return 0, fmt.Errorf("not a positive size: %q", v)
+	}
+	return n, nil
+}

cmd/felis/api_test.go

@@ -1,9 +1,18 @@
 package main
 
 import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
 	"net/http"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"felis.lolicon.best/internal/api"
+	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
 )
 
@@ -91,3 +100,106 @@ func TestNewAPIServerSetsHardenedTimeouts(t *testing.T) {
 		t.Errorf("ReadTimeout = %v, want 0 (unset) so a slow SSE attach is not capped", srv.ReadTimeout)
 	}
 }
+
+// Every listener drains at once, and the shutdown hook (API.CloseStreams in
+// cmdAPI) runs on each: two listeners each holding a request that ends when
+// the hook fires take one hook's worth of time, well inside the deadline.
+func TestShutdownServersDrainsListenersTogether(t *testing.T) {
+	release := make(chan struct{})
+	var hooks int32
+	started := make(chan struct{}, 2)
+	var servers []*http.Server
+	for range 2 {
+		srv := newAPIServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			started <- struct{}{}
+			<-release
+		}))
+		srv.RegisterOnShutdown(func() {
+			if atomic.AddInt32(&hooks, 1) == 1 {
+				time.AfterFunc(100*time.Millisecond, func() { close(release) })
+			}
+		})
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			t.Fatal(err)
+		}
+		srv.Addr = l.Addr().String()
+		go func() { _ = srv.Serve(l) }()
+		go func() {
+			if resp, err := http.Get("http://" + srv.Addr); err == nil {
+				resp.Body.Close()
+			}
+		}()
+		servers = append(servers, srv)
+	}
+	<-started
+	<-started
+
+	begun := time.Now()
+	shutdownServers(servers, 5*time.Second, io.Discard)
+	if took := time.Since(begun); took > 2*time.Second {
+		t.Fatalf("shutdown took %v", took)
+	}
+	if got := atomic.LoadInt32(&hooks); got != 2 {
+		t.Fatalf("shutdown hook ran %d times, want once per listener", got)
+	}
+}
+
+type fakeRefStore struct {
+	images []build.Image
+	builds []build.Build
+	err    error
+}
+
+func (f fakeRefStore) ListImages(context.Context) ([]build.Image, error) { return f.images, f.err }
+func (f fakeRefStore) ListUnfinishedBuilds(context.Context) ([]build.Build, error) {
+	return f.builds, nil
+}
+
+type fakeServers struct {
+	list    []api.ServerInfo
+	pods    []string
+	podsErr error
+}
+
+func (f fakeServers) ListServers(context.Context) ([]api.ServerInfo, error) { return f.list, nil }
+func (f fakeServers) PodImages(context.Context) ([]string, error)           { return f.pods, f.podsErr }
+
+// The registry pruner deletes whatever this list does not name, so every source of
+// a reference has to be in it, and a failing source must fail the list.
+func TestInUseImageRefsCoversEverySource(t *testing.T) {
+	const reg = "registry.felis.svc:5000/"
+	store := fakeRefStore{
+		images: []build.Image{{ImageRef: reg + "modpacks/pack:*"}, {ImageRef: reg + "felis/paper:demo"}},
+		builds: []build.Build{{ImageRef: reg + "user-uploads/sub-9:latest"}},
+	}
+	servers := fakeServers{
+		list: []api.ServerInfo{{Name: "s1", Image: reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1)}},
+		pods: []string{reg + "felis/felis:v1.0.0"},
+	}
+	got, err := inUseImageRefs(context.Background(), store, servers, []string{reg + "felis/felis:b60"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		reg + "felis/felis:b60",
+		reg + "modpacks/pack:*", reg + "felis/paper:demo",
+		reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1),
+		reg + "felis/felis:v1.0.0",
+		reg + "user-uploads/sub-9:latest",
+	}
+	if fmt.Sprint(got) != fmt.Sprint(want) {
+		t.Fatalf("refs = %v\nwant %v", got, want)
+	}
+
+	servers.podsErr = errors.New("apiserver down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing pod list produced a reference list")
+	}
+	servers.podsErr = nil
+
+	store.err = errors.New("db down")
+	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {
+		t.Fatal("a failing whitelist read produced a reference list")
+	}
+}

cmd/felis/api_tokens_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/api"
+)
+
+// felis-api maps each env var onto the caller the Deployment feeds it for, so the
+// build Job's token (FELIS_BUILD_TOKEN) authenticates as build and only as build.
+func TestInternalCallerTokensReadEachCallersEnv(t *testing.T) {
+	env := map[string]string{
+		"FELIS_SERVICE_TOKEN": "v-tok",
+		"FELIS_LIMBO_TOKEN":   "l-tok",
+		"FELIS_BUILD_TOKEN":   " b-tok\n",
+		"FELIS_OPS_TOKEN":     "o-tok",
+	}
+	auth, err := internalCallerTokens(func(k string) string { return env[k] })
+	if err != nil {
+		t.Fatal(err)
+	}
+	for tok, want := range map[string]api.Caller{"v-tok": api.CallerVelocity, "l-tok": api.CallerLimbo, "b-tok": api.CallerBuild, "o-tok": api.CallerOps} {
+		r := httptest.NewRequest("GET", "/", nil)
+		r.Header.Set("Authorization", "Bearer "+tok)
+		if got, err := auth.Authenticate(r); err != nil || got != want {
+			t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want)
+		}
+	}
+
+	// An install where the build namespace still holds a copy of the proxy's token
+	// would let that copy act as the proxy; the api refuses to start on it.
+	env["FELIS_BUILD_TOKEN"] = "v-tok"
+	if _, err := internalCallerTokens(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "same value") {
+		t.Fatalf("shared token: err = %v, want a same-value refusal", err)
+	}
+}

cmd/felis/apply.go

@@ -243,6 +243,19 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
 			AutostartPolicy: policy,
 			Storage:         v1alpha1.StorageSpec{Size: storageQ.String()},
 			Resources:       corev1.ResourceRequirements{Limits: limits, Requests: requests},
+			// The rest matches what felis-api's create writes (K8sCluster.CreateServer):
+			// fall back to the login gate while stopped, RCON on (readiness, the
+			// player count and the console all ride it; the operator mints the
+			// password), and the default idle stop.
+			FallbackServer: naming.SystemLoginServer,
+			Rcon: v1alpha1.RconSpec{
+				Enabled: true,
+				SecretRef: v1alpha1.SecretKeyRef{
+					Name: naming.RconSecretName(req.Name),
+					Key:  naming.RconSecretKey,
+				},
+			},
+			Idle: v1alpha1.DefaultIdle(),
 		},
 	}, nil
 }

cmd/felis/apply_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/naming"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
@@ -167,6 +168,18 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
 	if ms.Spec.Storage.Size != "20Gi" {
 		t.Errorf("Storage.Size = %q, want 20Gi", ms.Spec.Storage.Size)
 	}
+	// Same operational defaults as the API create path: without RCON the server
+	// never reports players and the console answers 503; without spec.idle it
+	// never stops on its own.
+	if !ms.Spec.Rcon.Enabled || ms.Spec.Rcon.SecretRef.Name != naming.RconSecretName("test-server") {
+		t.Errorf("Rcon = %+v, want enabled with the operator-minted secret", ms.Spec.Rcon)
+	}
+	if ms.Spec.Idle != v1alpha1.DefaultIdle() {
+		t.Errorf("Idle = %+v, want the default %+v", ms.Spec.Idle, v1alpha1.DefaultIdle())
+	}
+	if ms.Spec.FallbackServer != naming.SystemLoginServer {
+		t.Errorf("FallbackServer = %q, want the login gate", ms.Spec.FallbackServer)
+	}
 	mem, ok := ms.Spec.Resources.Limits[corev1.ResourceMemory]
 	if !ok {
 		t.Fatal("memory limit missing")

cmd/felis/backup.go

@@ -1,14 +1,17 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	"felis.lolicon.best/internal/backup"
+	"felis.lolicon.best/internal/backupjob"
 	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/reaper"
@@ -35,6 +38,8 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	server := fs.String("server", "", "server name whose world is being backed up")
 	formerOwner := fs.String("former-owner", "", "owner recorded on the backup row (empty for an unowned server)")
 	worldsRoot := fs.String("worlds-root", "/world", "mount path of the world PVC being archived")
+	reason := fs.String("reason", reasonManual, "world_backups reason: manual, or pre_restore for the safety snapshot in front of a restore")
+	protect := fs.String("protect", "", "backup id the prune must keep (the one a chained restore extracts)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -42,6 +47,11 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintln(stderr, "felis backup: --server is required")
 		return 2
 	}
+	keep, ok := map[string]int{reasonManual: -1, backupjob.ReasonPreRestore: preRestoreKeep}[*reason]
+	if !ok {
+		fmt.Fprintf(stderr, "felis backup: unknown --reason %q (manual or %s)\n", *reason, backupjob.ReasonPreRestore)
+		return 2
+	}
 
 	cfg, err := config.Load(*cfgPath)
 	if err != nil {
@@ -52,13 +62,16 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis backup: archive store %q is not implemented in this build (only tarLocal)\n", cfg.Archive.Store)
 		return 1
 	}
-	// Reuse the reaper's retention derivation so an on-demand backup expires on the
+	// The [archive] parse the reaper uses; an on-demand backup takes its
-	// same clock as an inactivity backup — one retention policy, not two.
+	// manual_retention and manual_keep.
 	rcfg, err := reaperConfig(cfg)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: %v\n", err)
 		return 1
 	}
+	if keep < 0 {
+		keep = rcfg.ManualKeep
+	}
 
 	// The world PVC is mounted directly at worldsRoot; the resolver returns it for
 	// any target, exactly as in cmdRestore. This is the same TarLocal the reaper
@@ -72,11 +85,23 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 
 	ctx := ctrl.SetupSignalHandler()
 
-	ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
+	// The archive store shares the node's disk with every world and the
+	// database: an owner's backup must not be what tips it into eviction.
+	if err := backup.CheckRoom(cfg.Archive.LocalPath, *worldsRoot, backup.MinFreeAfter); err != nil {
+		fmt.Fprintf(stderr, "felis backup: %v\n", err)
+		return 1
+	}
+
+	a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
 		return 1
 	}
+	ref, size := a.Ref, a.Size
+	if len(a.Skipped) > 0 {
+		fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",
+			len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))
+	}
 
 	drv, err := store.Open(ctx, cfg.Database.URL)
 	if err != nil {
@@ -91,10 +116,14 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 		FormerOwner: *formerOwner,
 		BackupRef:   string(ref),
 		SizeBytes:   size,
-		Reason:      "manual",
+		Reason:      *reason,
-		ExpiresAt:   time.Now().Add(rcfg.Retention),
+		ExpiresAt:   time.Now().Add(rcfg.ManualRetention),
+
+		SHA256:         a.SHA256,
+		SkippedEntries: len(a.Skipped),
 	}
-	if err := reaper.NewPGStore(drv.DB()).InsertBackup(ctx, rec); err != nil {
+	st := reaper.NewPGStore(drv.DB())
+	if err := st.InsertBackup(ctx, rec); err != nil {
 		// The archive is written but unrecorded — an orphan the retention pass would
 		// never expire. Delete it so a failed backup leaves no leaked bytes, mirroring
 		// the reaper's archive-then-record atomicity.
@@ -107,9 +136,42 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
 	}
 
 	fmt.Fprintf(stdout, "felis backup: server=%s archived %d bytes to %s (backup %s)\n", *server, size, ref, rec.ID)
+	pruneBackups(ctx, st, archiver, *server, *reason, keep, *protect, stdout, stderr)
 	return 0
 }
 
+const (
+	reasonManual = "manual"
+	// preRestoreKeep is how many safety snapshots a server keeps: enough to walk
+	// back a couple of restores in a row, without every restore adding a world's
+	// worth of bytes for the full manual retention.
+	preRestoreKeep = 3
+)
+
+// pruneBackups keeps server's newest keep backups of this reason and removes the
+// rest, oldest first, so repeated backups of one world cannot fill the shared
+// archive store. protect is never removed: it is the backup a chained restore is
+// about to extract. The new backup is already recorded; a removal that fails is
+// reported and retried after the next backup.
+func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server, reason string, keep int, protect string, stdout, stderr io.Writer) {
+	excess, err := st.ExcessBackups(ctx, server, reason, keep, protect)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)
+		return
+	}
+	for _, b := range excess {
+		if err := archiver.Delete(ctx, backup.ArchiveRef(b.BackupRef)); err != nil {
+			fmt.Fprintf(stderr, "felis backup: remove older backup %s: %v\n", b.ID, err)
+			continue
+		}
+		if err := st.MarkBackupDeleted(ctx, b.ID, time.Now()); err != nil {
+			fmt.Fprintf(stderr, "felis backup: record the removal of %s: %v\n", b.ID, err)
+			continue
+		}
+		fmt.Fprintf(stdout, "felis backup: removed older %s backup %s of %s (keeping the newest %d)\n", reason, b.ID, server, keep)
+	}
+}
+
 // newBackupID mints a world_backups primary key, matching the reaper's "bk-"+hex
 // scheme so a manual and an inactivity backup are indistinguishable downstream.
 func newBackupID() string {

cmd/felis/backup_test.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+// The reason decides which backups the new one's prune may remove, so an
+// unknown one is refused before anything is archived.
+func TestBackupSubcommandRejectsUnknownReason(t *testing.T) {
+	var stderr bytes.Buffer
+	if code := cmdBackup([]string{"--server", "survival", "--reason", "inactive_15d"}, &bytes.Buffer{}, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2 (%s)", code, stderr.String())
+	}
+	if !strings.Contains(stderr.String(), "unknown --reason") {
+		t.Fatalf("stderr = %q", stderr.String())
+	}
+}

cmd/felis/backupnow.go

@@ -20,7 +20,7 @@ import (
 // backupnow is the break-glass "back up a world now" op (§B4 "Sync"). Unlike halt —
 // which writes the CRD directly — a backup needs felis-api's deployment coordinates
 // (FELIS_IMAGE / FELIS_BACKUP_PVC) to render the one-shot backup Job, so the console
-// cannot do it in-process. It POSTs the felis-api INTERNAL face (service-token auth)
+// cannot do it in-process. It POSTs the felis-api INTERNAL face (ops-token auth)
 // while the API is alive, and the API renders the Job and audits the action. This file
 // is the pure core (no bubbletea); tui_backupnow.go is the terminal glue.
 
@@ -33,7 +33,7 @@ type backupNowOutcome struct {
 
 // resolveInternalAPI reads the two things the on-node console needs to reach the
 // felis-api internal face: the felis-api-internal Service ClusterIP (the host's
-// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the service
+// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the ops
 // token. Both live in the control namespace.
 func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace string) (baseURL, token string, err error) {
 	var svc corev1.Service
@@ -45,13 +45,14 @@ func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace
 		return "", "", fmt.Errorf("%s Service has no ClusterIP yet", platform.APIInternalServiceName)
 	}
 
+	// The console's own token, which the api serves on the backup route alone.
 	var sec corev1.Secret
-	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.ServiceTokenSecretName}, &sec); err != nil {
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.OpsTokenSecretName}, &sec); err != nil {
-		return "", "", fmt.Errorf("get %s Secret: %w", naming.ServiceTokenSecretName, err)
+		return "", "", fmt.Errorf("get %s Secret (re-run the installer to create it): %w", naming.OpsTokenSecretName, err)
 	}
 	token = string(sec.Data[naming.ServiceTokenSecretKey])
 	if token == "" {
-		return "", "", fmt.Errorf("secret %s has no %s key", naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey)
+		return "", "", fmt.Errorf("secret %s has no %s key", naming.OpsTokenSecretName, naming.ServiceTokenSecretKey)
 	}
 
 	return fmt.Sprintf("http://%s:%d", ip, platform.APIInternalPort), token, nil

cmd/felis/backupnow_test.go

@@ -11,7 +11,6 @@ import (
 	"testing"
 	"time"
 
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 
 	corev1 "k8s.io/api/core/v1"
@@ -28,9 +27,15 @@ func internalAPIObjs(clusterIP, token string) []client.Object {
 			ObjectMeta: metav1.ObjectMeta{Name: platform.APIInternalServiceName, Namespace: bgControlNS},
 			Spec:       corev1.ServiceSpec{ClusterIP: clusterIP},
 		},
+		// The console presents the ops token; the proxy's felis-service-token sits
+		// beside it and must not be the one picked up.
 		&corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: bgControlNS},
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-ops-token", Namespace: bgControlNS},
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
+			Data:       map[string][]byte{"token": []byte(token)},
+		},
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: "felis-service-token", Namespace: bgControlNS},
+			Data:       map[string][]byte{"token": []byte("proxy-" + token)},
 		},
 	}
 }

cmd/felis/conflict_test.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+// racingClient lands one concurrent write (race) on the stored object just before
+// setup's first write of it goes out, the way the operator's status update or
+// felis-api's idle patch can, and counts setup's writes.
+func racingClient(t *testing.T, race func(ctx context.Context, c client.WithWatch), objs ...client.Object) (client.Client, *int) {
+	t.Helper()
+	writes := 0
+	before := func(ctx context.Context, c client.WithWatch) {
+		writes++
+		if writes == 1 {
+			race(ctx, c)
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
+		WithStatusSubresource(&v1alpha1.MinecraftServer{}).
+		WithInterceptorFuncs(interceptor.Funcs{
+			Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
+				before(ctx, c)
+				return c.Update(ctx, obj, opts...)
+			},
+			Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+				before(ctx, c)
+				return c.Patch(ctx, obj, patch, opts...)
+			},
+		}).Build()
+	return cl, &writes
+}
+
+func staleLoginGate(t *testing.T) *v1alpha1.MinecraftServer {
+	t.Helper()
+	ms, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://old.internal:8081", "203.0.113.10.nip.io", "console.203.0.113.10.nip.io")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ms
+}
+
+func getLogin(t *testing.T, cl client.Client) *v1alpha1.MinecraftServer {
+	t.Helper()
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	return &ms
+}
+
+func envMap(ms *v1alpha1.MinecraftServer) map[string]string {
+	m := map[string]string{}
+	for _, e := range ms.Spec.Env {
+		m[e.Name] = e.Value
+	}
+	return m
+}
+
+// A hand edit that lands while setup refreshes the console hostnames costs setup a
+// re-read and a second write; both the edit and the refresh survive.
+func TestRefreshDerivedEnvRetriesAConcurrentEdit(t *testing.T) {
+	ctx := context.Background()
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		ms := getLogin(t, c)
+		ms.Spec.Env = append(ms.Spec.Env, v1alpha1.EnvVar{Name: "HAND_TUNED", Value: "1"})
+		if err := c.Update(ctx, ms); err != nil {
+			t.Fatal(err)
+		}
+	}, staleLoginGate(t))
+
+	desired, err := loginSystemServer("felis-limbo:demo", "minecraft",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
+	if err != nil {
+		t.Fatal(err)
+	}
+	refreshed, err := refreshDerivedEnv(ctx, cl, getLogin(t, cl), desired)
+	if err != nil || !refreshed {
+		t.Fatalf("refreshed=%v err=%v, want a refresh after the retry", refreshed, err)
+	}
+	env := envMap(getLogin(t, cl))
+	if env[envPanelHostname] != "console.mc.example.net" || env[envRootDomain] != "mc.example.net" {
+		t.Errorf("env = %v, want the new hostnames", env)
+	}
+	if env["HAND_TUNED"] != "1" {
+		t.Errorf("env = %v: the concurrent hand edit was dropped", env)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2 (one conflict, one retry)", *writes)
+	}
+}
+
+// converge reports each fill once even when a status write forced a retry.
+func TestConvergeRetriesAConcurrentStatusWrite(t *testing.T) {
+	ctx := context.Background()
+	lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
+	if err != nil {
+		t.Fatal(err)
+	}
+	lobby.Spec.Rcon = v1alpha1.RconSpec{}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Status.Phase = v1alpha1.PhaseRunning
+		if err := c.Status().Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, lobby)
+
+	var got systemServerOutcome
+	for _, o := range convergeSystemServers(ctx, cl, "minecraft", "", "reg/lobby:1",
+		"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
+		if o.name == naming.SystemLobbyServer {
+			got = o
+		}
+	}
+	if got.err != nil || !got.updated || !slices.Equal(got.changes, []string{"spec.rcon"}) {
+		t.Fatalf("lobby outcome = %+v, want spec.rcon filled once", got)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if !ms.Spec.Rcon.Enabled || ms.Status.Phase != v1alpha1.PhaseRunning {
+		t.Errorf("rcon.enabled=%v phase=%q, want the fill and the status write both kept", ms.Spec.Rcon.Enabled, ms.Status.Phase)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+// A replica refresh racing another writer keeps that writer's key.
+func TestSecretReplicaRefreshRetriesAConcurrentWrite(t *testing.T) {
+	secret := func(ns, body string) *corev1.Secret {
+		return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
+			Data: map[string][]byte{"felis.toml": []byte(body)}}
+	}
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var s corev1.Secret
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+			t.Fatal(err)
+		}
+		s.Data["extra"] = []byte("x")
+		if err := c.Update(ctx, &s); err != nil {
+			t.Fatal(err)
+		}
+	}, secret("felis", "current"), secret("minecraft", "stale"))
+
+	out := ensureSecretReplica(context.Background(), cl, "felis", "minecraft",
+		"felis-config", "felis.toml", "config", "minecraft ns", true)
+	if out.err != nil || !out.updated {
+		t.Fatalf("outcome = %+v, want refreshed", out)
+	}
+	var s corev1.Secret
+	if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
+		t.Fatal(err)
+	}
+	if string(s.Data["felis.toml"]) != "current" || string(s.Data["extra"]) != "x" {
+		t.Errorf("replica data = %q, want felis.toml=current and extra=x", s.Data)
+	}
+	if *writes != 2 {
+		t.Errorf("writes = %d, want 2", *writes)
+	}
+}
+
+const (
+	sysOldDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
+	sysNewDigest = "sha256:4444444444444444444444444444444444444444444444444444444444444444"
+)
+
+func systemPinRegistry(t *testing.T) imagepin.Resolver {
+	t.Helper()
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v2/felis/limbo/manifests/demo", "/v2/felis/lobby/manifests/demo":
+			w.Header().Set("Docker-Content-Digest", sysNewDigest)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(reg.Close)
+	return imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+}
+
+func systemServer(name, image string) *v1alpha1.MinecraftServer {
+	ms := &v1alpha1.MinecraftServer{}
+	ms.Name, ms.Namespace = name, "minecraft"
+	ms.Labels = map[string]string{v1alpha1.LabelSystemRole: name}
+	ms.Spec.Image = image
+	return ms
+}
+
+// The installer's --system pin moves a system server onto the build its tag names
+// now, from the bare tag or from an earlier digest, and is a no-op the second time.
+func TestPinSystemServerImage(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	limbo := defaultRegistryURL + "/felis/limbo:demo"
+	lobby := defaultRegistryURL + "/felis/lobby:demo"
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		systemServer(naming.SystemLoginServer, limbo),
+		systemServer(naming.SystemLobbyServer, lobby+"@"+sysOldDigest),
+	).Build()
+	image := func(name string) string {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		return ms.Spec.Image
+	}
+
+	for name, want := range map[string]string{
+		naming.SystemLoginServer: limbo + "@" + sysNewDigest,
+		naming.SystemLobbyServer: lobby + "@" + sysNewDigest,
+	} {
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || o.err != nil || !o.updated {
+			t.Fatalf("%s: outcome=%+v err=%v, want pinned", name, o, err)
+		}
+		if got := image(name); got != want {
+			t.Errorf("%s image = %q, want %q", name, got, want)
+		}
+		again, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
+		if err != nil || again.updated || again.skipped != "already runs "+want {
+			t.Errorf("%s second pass = %+v, %v; want already runs %s", name, again, err, want)
+		}
+	}
+}
+
+func TestPinSystemServerImageLeavesOthersAlone(t *testing.T) {
+	ctx := context.Background()
+	res := systemPinRegistry(t)
+	pin := func(t *testing.T, ms *v1alpha1.MinecraftServer) (systemServerOutcome, string) {
+		t.Helper()
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(ms).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(ms), &got); err != nil {
+			t.Fatal(err)
+		}
+		return o, got.Spec.Image
+	}
+
+	t.Run("external image", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, "docker.io/example/limbo:1.2"))
+		if o.err != nil || o.updated || img != "docker.io/example/limbo:1.2" ||
+			o.skipped != "runs docker.io/example/limbo:1.2, which names no platform registry tag to follow; left alone" {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("digest without a tag", func(t *testing.T) {
+		ref := defaultRegistryURL + "/felis/limbo@" + sysOldDigest
+		o, img := pin(t, systemServer(naming.SystemLoginServer, ref))
+		if o.err != nil || o.updated || img != ref {
+			t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
+		}
+	})
+	t.Run("not a system server", func(t *testing.T) {
+		ms := systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo")
+		ms.Labels = nil
+		o, img := pin(t, ms)
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:demo" {
+			t.Fatalf("outcome=%+v image=%q, want refused", o, img)
+		}
+	})
+	t.Run("tag the registry lost", func(t *testing.T) {
+		o, img := pin(t, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:gone"))
+		if o.err == nil || img != defaultRegistryURL+"/felis/limbo:gone" {
+			t.Fatalf("outcome=%+v image=%q, want an error the installer falls back on", o, img)
+		}
+	})
+	t.Run("absent", func(t *testing.T) {
+		cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).Build()
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.skipped != "not present yet; sudo felis setup creates it" {
+			t.Fatalf("outcome=%+v err=%v, want a skip", o, err)
+		}
+	})
+	t.Run("retargeted while pinning", func(t *testing.T) {
+		cl, _ := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+			ms := getLogin(t, c)
+			ms.Spec.Image = "docker.io/example/limbo:1.2"
+			if err := c.Update(ctx, ms); err != nil {
+				t.Fatal(err)
+			}
+		}, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo"))
+		o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
+		if err != nil || o.err != nil || o.updated {
+			t.Fatalf("outcome=%+v err=%v, want nothing written", o, err)
+		}
+		if img := getLogin(t, cl).Spec.Image; img != "docker.io/example/limbo:1.2" {
+			t.Errorf("image = %q, want the admin's retarget kept", img)
+		}
+	})
+}
+
+// --system names one of the two system servers; anything else is a usage error
+// before any cluster is touched.
+func TestPinImagesSystemFlagTakesOnlySystemServers(t *testing.T) {
+	var stdout, stderr strings.Builder
+	if code := cmdPinImages([]string{"--system", "survival"}, &stdout, &stderr); code != 2 {
+		t.Fatalf("exit = %d, want 2", code)
+	}
+	if got := stderr.String(); got != "felis pin-images: --system takes login or lobby, not \"survival\"\n" {
+		t.Errorf("stderr = %q", got)
+	}
+}
+
+// An idle setting the panel saves while converge fills the default is kept.
+func TestConvergeIdleKeepsAConcurrentPanelEdit(t *testing.T) {
+	ctx := context.Background()
+	srv := &v1alpha1.MinecraftServer{}
+	srv.Name, srv.Namespace = "survival", "minecraft"
+	cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
+		var ms v1alpha1.MinecraftServer
+		if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+			t.Fatal(err)
+		}
+		ms.Spec.Idle = v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}
+		if err := c.Update(ctx, &ms); err != nil {
+			t.Fatal(err)
+		}
+	}, srv)
+
+	if out := convergeUserServerIdle(ctx, cl, "minecraft"); len(out) != 0 {
+		t.Fatalf("outcomes = %+v, want none: the server has a setting by the time converge writes", out)
+	}
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
+		t.Fatal(err)
+	}
+	if want := (v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}); ms.Spec.Idle != want {
+		t.Errorf("idle = %+v, want the panel's %+v", ms.Spec.Idle, want)
+	}
+	if *writes != 1 {
+		t.Errorf("writes = %d, want 1 (the conflicted attempt; the retry sends nothing)", *writes)
+	}
+}

cmd/felis/context_max_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+)
+
+func TestContextMaxBytes(t *testing.T) {
+	cases := []struct {
+		name    string
+		reg     config.RegistryConfig
+		auth    config.AuthConfig
+		want    int64
+		wantErr bool
+	}{
+		{name: "direct install keeps the package default", want: 0},
+		// The panel uploads in parts under the edge's 100 MB body limit, so the
+		// Cloudflare edge keeps the full default.
+		{name: "the Cloudflare edge keeps the package default", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 0},
+		{name: "CF-Connecting-IP keeps the package default", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 0},
+		{name: "explicit value behind the edge", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
+		{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
+		{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
+		{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := contextMaxBytes(&config.Config{Registry: tc.reg, Auth: tc.auth})
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
+			}
+			if got != tc.want {
+				t.Fatalf("contextMaxBytes = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestUploadPartsDir(t *testing.T) {
+	if got := uploadPartsDir("/var/lib/felis/uploads"); got != "/var/lib/felis/uploads/.parts" {
+		t.Errorf("local store: parts dir = %q, want beside the contexts", got)
+	}
+	if got := uploadPartsDir("file:///srv/uploads"); got != "/srv/uploads/.parts" {
+		t.Errorf("file:// store: parts dir = %q, want /srv/uploads/.parts", got)
+	}
+	// This machine has no /var/lib/felis/uploads mount, so an s3:// store falls
+	// back to the temp dir.
+	if _, err := os.Stat("/var/lib/felis/uploads"); err == nil {
+		t.Skip("/var/lib/felis/uploads exists here")
+	}
+	if got := uploadPartsDir("s3://bucket/uploads"); got != filepath.Join(os.TempDir(), "felis-upload-parts") {
+		t.Errorf("s3 store without the uploads mount: parts dir = %q", got)
+	}
+}

cmd/felis/converge.go

@@ -9,12 +9,14 @@ import (
 	"os"
 	"strings"
 
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/platform"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // cmdConverge is the explicit convergence pass over already-installed system
-// servers (#1). Provisioning is create-if-absent, so a field the desired spec
+// servers (#1), plus the idle-stop default for user servers that predate it. Provisioning is create-if-absent, so a field the desired spec
 // gained after an install (spec.rcon, spec.startup.healthHTTPPort, a derived env
 // key) never reaches the existing CR — and nothing says so. This command fills
 // exactly those zero-value fields; see convergeSystemServers for the full contract
@@ -56,7 +58,9 @@ func cmdConverge(args []string, stdout, stderr io.Writer) int {
 		platform.InternalAPIBaseURL(controlNS), cfg.Server.RootDomain,
 		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
 
-	fmt.Fprintln(stdout, "felis converge: filling fields an installed system server predates (operator-set values are never overwritten):")
+	outcomes = append(outcomes, convergeUserServerIdle(context.Background(), cl, cfg.K8s.Namespace)...)
+
+	fmt.Fprintln(stdout, "felis converge: filling fields an installed server predates (operator-set values are never overwritten):")
 	exit := 0
 	for _, o := range outcomes {
 		switch {
@@ -71,3 +75,41 @@ func cmdConverge(args []string, stdout, stderr io.Writer) int {
 	}
 	return exit
 }
+
+// convergeUserServerIdle gives every user server that predates the idle default
+// (spec.idle entirely unset) the default idle stop. A server whose idle stop was
+// turned off keeps a duration on its spec, so it is not "unset" and is left
+// alone; system servers never idle out and are skipped. Servers that already
+// carry a value produce no line, so a converged fleet prints nothing here.
+func convergeUserServerIdle(ctx context.Context, cl client.Client, namespace string) []systemServerOutcome {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+			continue
+		}
+		// Re-checked on the copy each attempt reads: an idle setting the panel saved
+		// meanwhile is the user's, and the default must not land over it.
+		changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
+			if ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
+				return false
+			}
+			ms.Spec.Idle = v1alpha1.DefaultIdle()
+			return true
+		})
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
+			continue
+		}
+		if !changed {
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{fmt.Sprintf("spec.idle (stop after %ds empty)", v1alpha1.DefaultEmptySecondsBeforeStop)}})
+	}
+	return out
+}

cmd/felis/converge_test.go

@@ -183,3 +183,49 @@ func TestConvergeSystemServersGuards(t *testing.T) {
 		}
 	})
 }
+
+// TestConvergeUserServerIdle fills the idle default only where spec.idle was
+// never set: a server whose idle stop was turned off (duration kept), one with
+// its own duration, and a system server all stay as they are.
+func TestConvergeUserServerIdle(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	mk := func(name string, idle v1alpha1.IdleSpec, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Idle = idle
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		mk("legacy", v1alpha1.IdleSpec{}, ""),
+		mk("off", v1alpha1.IdleSpec{EmptySecondsBeforeStop: 600}, ""),
+		mk("custom", v1alpha1.IdleSpec{AutoStopEnabled: true, EmptySecondsBeforeStop: 1800}, ""),
+		mk(naming.SystemLobbyServer, v1alpha1.IdleSpec{}, naming.SystemLobbyServer),
+	).Build()
+
+	outcomes := convergeUserServerIdle(ctx, cl, "minecraft")
+	if len(outcomes) != 1 || outcomes[0].name != "legacy" || outcomes[0].err != nil {
+		t.Fatalf("outcomes = %+v, want exactly one fill for legacy", outcomes)
+	}
+	want := map[string]v1alpha1.IdleSpec{
+		"legacy":                 v1alpha1.DefaultIdle(),
+		"off":                    {EmptySecondsBeforeStop: 600},
+		"custom":                 {AutoStopEnabled: true, EmptySecondsBeforeStop: 1800},
+		naming.SystemLobbyServer: {},
+	}
+	for name, idle := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Idle != idle {
+			t.Errorf("%s idle = %+v, want %+v", name, ms.Spec.Idle, idle)
+		}
+	}
+	if again := convergeUserServerIdle(ctx, cl, "minecraft"); len(again) != 0 {
+		t.Fatalf("second pass = %+v, want nothing to do", again)
+	}
+}

cmd/felis/db.go

@@ -0,0 +1,420 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/retention"
+)
+
+const dbUsage = `usage:
+  felis db backup  [-config path] [-dir dir] [-label daily|manual|...] [-keep n] [-state-dir dir]
+                   [-no-servers] [-metrics-file path]
+  felis db restore [-config path] [-dir dir] [-yes] [-force] [-no-safety-backup] <bundle>
+  felis db verify  [-dir dir] <bundle>
+  felis db list    [-dir dir]
+  felis db check   [-dir dir] [-max-age 26h]
+  felis db audit-export [-config path] [-since date] [-until date] [-out file]
+`
+
+// defaultKeep is how many bundles of a label a backup leaves behind. Manual
+// bundles are the operator's own and are never pruned.
+var defaultKeep = map[string]int{
+	dbbackup.LabelDaily:      14,
+	dbbackup.LabelPreMigrate: 10,
+	dbbackup.LabelPreRestore: 5,
+}
+
+// cmdDB implements `felis db`: logical backups of the control-plane database
+// together with the host state a rebuild needs (internal/dbbackup). The verb
+// comes first for the same reason as `felis migrate up`.
+func cmdDB(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("db "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, dbUsage) }
+	dir := fs.String("dir", dbbackup.DefaultDir, "bundle directory")
+	switch verb {
+	case "backup":
+		return dbBackup(fs, dir, rest, stdout, stderr)
+	case "restore":
+		return dbRestore(fs, dir, rest, stdout, stderr)
+	case "verify":
+		return dbVerify(fs, dir, rest, stdout, stderr)
+	case "list":
+		return dbList(fs, dir, rest, stdout, stderr)
+	case "check":
+		return dbCheck(fs, dir, rest, stdout, stderr)
+	case "audit-export":
+		return dbAuditExport(fs, rest, stdout, stderr)
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, dbUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis db: unknown verb %q\n%s", verb, dbUsage)
+	return 2
+}
+
+// parseWithArg parses flags that may sit on either side of one positional
+// argument (`restore -yes x.tar` and `restore x.tar -yes` both work) and
+// returns that argument.
+func parseWithArg(fs *flag.FlagSet, args []string) (string, bool) {
+	if err := fs.Parse(args); err != nil {
+		return "", false
+	}
+	if fs.NArg() == 0 {
+		return "", true
+	}
+	arg := fs.Arg(0)
+	if err := fs.Parse(fs.Args()[1:]); err != nil {
+		return "", false
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprintf(fs.Output(), "felis db: unexpected argument %q\n", fs.Arg(0))
+		return "", false
+	}
+	return arg, true
+}
+
+func dbDatabaseURL(path string) (string, error) {
+	cfg, err := config.Load(path)
+	if err != nil {
+		return "", err
+	}
+	return cfg.Database.URL, nil
+}
+
+func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never")
+	keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, manual all)")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`)
+	noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle")
+	metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	if *keep < 0 {
+		*keep = defaultKeep[*label]
+	}
+	o := dbbackup.BackupOptions{
+		DatabaseURL: url, Dir: *dir, Label: *label, Keep: *keep,
+		StateDir: *stateDir, Version: resolvedVersion(), Log: stderr,
+		MetricsFile: *metrics, Record: true,
+	}
+	if !*noServers {
+		o.ExportServers = exportMinecraftServers
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	path, err := dbbackup.Backup(ctx, o)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db backup: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
+	return 0
+}
+
+// resolveBundle accepts a path, or a bare bundle name looked up in dir.
+func resolveBundle(dir, arg string) string {
+	if strings.ContainsRune(arg, os.PathSeparator) {
+		return arg
+	}
+	if _, err := os.Stat(arg); err == nil {
+		return arg
+	}
+	return filepath.Join(dir, arg)
+}
+
+func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	yes := fs.Bool("yes", false, "replace the database's contents (required)")
+	force := fs.Bool("force", false, "restore even while other clients are connected")
+	noSafety := fs.Bool("no-safety-backup", false, "skip the bundle of the current database taken first")
+	stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, "host state directory for the safety bundle")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	if !*yes {
+		fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d).\n",
+			filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion)
+		fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.")
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
+	defer cancel()
+	_, safety, err := dbbackup.Restore(ctx, dbbackup.RestoreOptions{
+		DatabaseURL: url, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
+		Safety: dbbackup.BackupOptions{Keep: defaultKeep[dbbackup.LabelPreRestore], StateDir: *stateDir,
+			Version: resolvedVersion(), ExportServers: exportMinecraftServers},
+		Log: stderr,
+	})
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db restore: %v\n", err)
+		if errors.Is(err, dbbackup.ErrClientsConnected) {
+			fmt.Fprintln(stderr, "  kubectl -n felis scale deployment felis-api felis-operator --replicas=0")
+		}
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db restore: restored %s (schema %d)\n", filepath.Base(bundle), m.SchemaVersion)
+	if safety != "" {
+		fmt.Fprintf(stdout, "  the database as it was before is in %s\n", safety)
+	}
+	// Nothing migrates at startup, so a control plane newer than the bundle needs
+	// its migrations re-applied; rolling back to the release that wrote the bundle
+	// must skip that, or the rollback is undone.
+	fmt.Fprintf(stdout, "  next: felis migrate up -config %s (skip it when rolling back to felis %s, which wrote this bundle)\n", *cfgPath, orUnknown(m.FelisVersion))
+	fmt.Fprintln(stdout, "        kubectl -n felis scale deployment felis-api felis-operator --replicas=1")
+	return 0
+}
+
+func dbVerify(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	bundle := resolveBundle(*dir, arg)
+	m, err := dbbackup.Verify(bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db verify: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "%s: ok\n  taken   %s (%s)\n  felis   %s\n  schema  %d\n  %s\n",
+		filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion, orUnknown(m.PGDumpVersion))
+	for _, f := range m.Files {
+		if f.Link != "" {
+			fmt.Fprintf(stdout, "  %-40s -> %s\n", f.Name, f.Link)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %-40s %d bytes\n", f.Name, f.Size)
+	}
+	if m.ServersError != "" {
+		fmt.Fprintf(stdout, "  (no MinecraftServer objects: %s)\n", m.ServersError)
+	}
+	return 0
+}
+
+func orUnknown(s string) string {
+	if s == "" {
+		return "unknown"
+	}
+	return s
+}
+
+func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	all, err := dbbackup.List(*dir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db list: %v\n", err)
+		return 1
+	}
+	if len(all) == 0 {
+		fmt.Fprintf(stdout, "no database backups in %s\n", *dir)
+		return 0
+	}
+	now := time.Now()
+	for _, b := range all {
+		fmt.Fprintf(stdout, "%-50s %-12s %10s  %s ago\n", b.Name, b.Label, humanBytes(b.Size), dbbackup.Age(now.Sub(b.Created)))
+	}
+	return 0
+}
+
+// dbAuditExport writes audit rows to a file (or stdout) as JSON lines, so an
+// install can keep them past [audit] retention, after which felis-api deletes them.
+func dbAuditExport(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	sinceFlag := fs.String("since", "", "first day (or RFC 3339 instant) to export, inclusive; empty starts at the oldest row")
+	untilFlag := fs.String("until", "", "day (or RFC 3339 instant) to stop before, exclusive; empty runs to the newest row")
+	out := fs.String("out", "", "file to write (created 0600, never overwritten); empty writes to stdout")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if fs.NArg() > 0 {
+		fmt.Fprint(stderr, dbUsage)
+		return 2
+	}
+	since, err := parseExportBound(*sinceFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -since: %v\n", err)
+		return 2
+	}
+	until, err := parseExportBound(*untilFlag)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: -until: %v\n", err)
+		return 2
+	}
+	if !since.IsZero() && !until.IsZero() && !until.After(since) {
+		fmt.Fprintf(stderr, "felis db audit-export: -until %s is not after -since %s\n", *untilFlag, *sinceFlag)
+		return 2
+	}
+	url, err := dbDatabaseURL(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+		return 1
+	}
+	w := stdout
+	var f *os.File
+	if *out != "" {
+		if f, err = os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600); err != nil {
+			fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
+			return 1
+		}
+		w = f
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	n, err := exportAudit(ctx, url, since, until, w)
+	if f != nil {
+		if cerr := f.Close(); err == nil {
+			err = cerr
+		}
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db audit-export: %v (%d rows written)\n", err, n)
+		return 1
+	}
+	fmt.Fprintf(stderr, "felis db audit-export: %d audit rows written\n", n)
+	return 0
+}
+
+func exportAudit(ctx context.Context, url string, since, until time.Time, w io.Writer) (int, error) {
+	drv, err := openStore(ctx, url, false)
+	if err != nil {
+		return 0, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	return retention.ExportAudit(ctx, drv.DB(), since, until, w)
+}
+
+// parseExportBound reads a -since/-until value: a day (midnight UTC) or an
+// RFC 3339 instant; empty is an open bound.
+func parseExportBound(v string) (time.Time, error) {
+	if v == "" {
+		return time.Time{}, nil
+	}
+	if t, err := time.Parse(time.DateOnly, v); err == nil {
+		return t, nil
+	}
+	if t, err := time.Parse(time.RFC3339, v); err == nil {
+		return t.UTC(), nil
+	}
+	return time.Time{}, fmt.Errorf("%q is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)", v)
+}
+
+func humanBytes(n int64) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%d B", n)
+	}
+	div, exp := int64(unit), 0
+	for m := n / unit; m >= unit; m /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
+}
+
+// dbCheck is the freshness probe: exit 1 when the newest bundle is missing or
+// older than -max-age, for a monitor or the break-glass console to act on.
+func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
+	maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest bundle")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	b, err := dbbackup.Check(*dir, *maxAge, time.Now())
+	if err != nil {
+		fmt.Fprintf(stderr, "felis db check: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis db check: ok, newest backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
+	return 0
+}
+
+// exportMinecraftServers reads every MinecraftServer through the host's k3s
+// kubectl and strips what the API server owns, so the result can be fed back
+// with `kubectl apply -f` on a rebuilt cluster.
+func exportMinecraftServers(ctx context.Context) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning
+	// on stderr must not end up inside the JSON.
+	cmd := exec.CommandContext(ctx, "k3s", "kubectl", "get", "minecraftservers.felis.lolicon.best", "-A", "-o", "json")
+	cmd.Env = append(os.Environ(), "KUBECONFIG="+hostBootstrapKubeconfigPath)
+	var errBuf strings.Builder
+	cmd.Stderr = &errBuf
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("k3s kubectl get minecraftservers: %w: %s", err, strings.TrimSpace(errBuf.String()))
+	}
+	return cleanServerList(out)
+}
+
+// cleanServerList drops status and the server-assigned metadata from a
+// `kubectl get -o json` List.
+func cleanServerList(raw []byte) ([]byte, error) {
+	var list struct {
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(raw, &list); err != nil {
+		return nil, fmt.Errorf("parse MinecraftServer list: %w", err)
+	}
+	for _, it := range list.Items {
+		delete(it, "status")
+		if md, ok := it["metadata"].(map[string]any); ok {
+			for _, k := range []string{"resourceVersion", "uid", "creationTimestamp", "generation", "managedFields", "selfLink"} {
+				delete(md, k)
+			}
+		}
+	}
+	if list.Items == nil {
+		list.Items = []map[string]any{}
+	}
+	return json.MarshalIndent(map[string]any{"apiVersion": "v1", "kind": "List", "items": list.Items}, "", "  ")
+}

cmd/felis/db_test.go

@@ -0,0 +1,185 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/store"
+)
+
+func TestDBUsage(t *testing.T) {
+	for _, args := range [][]string{{"db"}, {"db", "frobnicate"}, {"db", "restore"}, {"db", "verify"}, {"db", "backup", "extra"}} {
+		var out, errBuf bytes.Buffer
+		if code := run(args, &out, &errBuf); code != 2 {
+			t.Errorf("%v: exit %d, want 2", args, code)
+		}
+		if !strings.Contains(errBuf.String(), "felis db restore") {
+			t.Errorf("%v: no usage on stderr: %q", args, errBuf.String())
+		}
+	}
+}
+
+func TestDBRestoreNeedsYes(t *testing.T) {
+	// A bundle that does not exist fails verification (1) before -yes matters;
+	// the -yes gate itself is exercised against a real bundle in internal/dbbackup
+	// and on the VM. Here: the refusal path never reaches the config or database.
+	var out, errBuf bytes.Buffer
+	if code := run([]string{"db", "restore", "-dir", t.TempDir(), "missing.tar"}, &out, &errBuf); code != 1 {
+		t.Fatalf("exit %d, stderr %q", code, errBuf.String())
+	}
+}
+
+func TestParseWithArg(t *testing.T) {
+	for _, args := range [][]string{{"-yes", "b.tar"}, {"b.tar", "-yes"}} {
+		fs := flag.NewFlagSet("t", flag.ContinueOnError)
+		fs.SetOutput(io.Discard)
+		yes := fs.Bool("yes", false, "")
+		arg, ok := parseWithArg(fs, args)
+		if !ok || arg != "b.tar" || !*yes {
+			t.Errorf("%v -> %q ok=%v yes=%v", args, arg, ok, *yes)
+		}
+	}
+	fs := flag.NewFlagSet("t", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	if _, ok := parseWithArg(fs, []string{"a.tar", "b.tar"}); ok {
+		t.Error("two positional arguments accepted")
+	}
+}
+
+func TestResolveBundle(t *testing.T) {
+	if got := resolveBundle("/var/lib/felis/db-backups", "felis-db-x.tar"); got != "/var/lib/felis/db-backups/felis-db-x.tar" {
+		t.Errorf("bare name -> %s", got)
+	}
+	if got := resolveBundle("/var/lib/felis/db-backups", "/root/copy.tar"); got != "/root/copy.tar" {
+		t.Errorf("path -> %s", got)
+	}
+}
+
+func TestCleanServerList(t *testing.T) {
+	raw := `{"apiVersion":"v1","kind":"List","metadata":{"resourceVersion":""},"items":[{
+		"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer",
+		"metadata":{"name":"survival","namespace":"minecraft","uid":"u","resourceVersion":"42","generation":3,
+			"creationTimestamp":"2026-09-01T00:00:00Z","managedFields":[{}],"labels":{"a":"b"}},
+		"spec":{"desiredState":"Running"},"status":{"phase":"Running"}}]}`
+	out, err := cleanServerList([]byte(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Kind  string           `json:"kind"`
+		Items []map[string]any `json:"items"`
+	}
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatal(err)
+	}
+	if got.Kind != "List" || len(got.Items) != 1 {
+		t.Fatalf("got %s", out)
+	}
+	it := got.Items[0]
+	if _, ok := it["status"]; ok {
+		t.Error("status kept")
+	}
+	md := it["metadata"].(map[string]any)
+	for _, k := range []string{"uid", "resourceVersion", "generation", "creationTimestamp", "managedFields"} {
+		if _, ok := md[k]; ok {
+			t.Errorf("metadata.%s kept", k)
+		}
+	}
+	if md["name"] != "survival" || md["namespace"] != "minecraft" || md["labels"] == nil {
+		t.Errorf("identity lost: %v", md)
+	}
+	if it["spec"].(map[string]any)["desiredState"] != "Running" {
+		t.Error("spec lost")
+	}
+
+	empty, err := cleanServerList([]byte(`{"items":null}`))
+	if err != nil || !strings.Contains(string(empty), `"items": []`) {
+		t.Errorf("empty list -> %s, %v", empty, err)
+	}
+	if _, err := cleanServerList([]byte("Warning: x\n{")); err == nil {
+		t.Error("garbage parsed")
+	}
+}
+
+func TestHasPending(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}, {Version: 3}}
+	if hasPending(map[int]struct{}{1: {}, 2: {}, 3: {}}, ms) {
+		t.Error("fully applied reported pending")
+	}
+	if !hasPending(map[int]struct{}{1: {}, 2: {}}, ms) {
+		t.Error("missing 3 not reported")
+	}
+}
+
+type appliedDriver struct {
+	store.Driver
+	done map[int]struct{}
+}
+
+func (d appliedDriver) EnsureVersionTable(context.Context) error { return nil }
+func (d appliedDriver) AppliedVersions(context.Context) (map[int]struct{}, error) {
+	return d.done, nil
+}
+
+func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
+	ms := []store.Migration{{Version: 1}, {Version: 2}}
+	// An unusable URL makes an attempted backup observable as an error without
+	// any PostgreSQL tooling.
+	const badURL = "not-a-url"
+	for _, tc := range []struct {
+		name    string
+		done    map[int]struct{}
+		attempt bool
+	}{
+		{"fresh database", map[int]struct{}{}, false},
+		{"up to date", map[int]struct{}{1: {}, 2: {}}, false},
+		{"pending on a populated database", map[int]struct{}{1: {}}, true},
+	} {
+		path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, badURL, t.TempDir(), io.Discard)
+		if attempted := err != nil; attempted != tc.attempt {
+			t.Errorf("%s: attempted = %v (err %v), want %v", tc.name, attempted, err, tc.attempt)
+		}
+		if path != "" {
+			t.Errorf("%s: path = %q", tc.name, path)
+		}
+	}
+}
+
+// audit-export takes a day or an RFC 3339 instant for each bound, and refuses a
+// malformed or inverted window before it opens the config or the database.
+func TestAuditExportBounds(t *testing.T) {
+	for _, tc := range []struct{ in, want string }{
+		{"", "0001-01-01T00:00:00Z"},
+		{"2026-01-31", "2026-01-31T00:00:00Z"},
+		{"2026-01-31T12:30:00+08:00", "2026-01-31T04:30:00Z"},
+	} {
+		got, err := parseExportBound(tc.in)
+		if err != nil || got.Format(time.RFC3339) != tc.want {
+			t.Errorf("parseExportBound(%q) = %v, %v; want %s", tc.in, got, err, tc.want)
+		}
+	}
+	if _, err := parseExportBound("31/01/2026"); err == nil || err.Error() != `"31/01/2026" is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)` {
+		t.Errorf("parseExportBound(31/01/2026) err = %v", err)
+	}
+	for _, tc := range []struct {
+		args    []string
+		wantErr string
+	}{
+		{[]string{"db", "audit-export", "-since", "yesterday"}, `felis db audit-export: -since: "yesterday" is neither`},
+		{[]string{"db", "audit-export", "-until", "2026-13-01"}, `felis db audit-export: -until: "2026-13-01" is neither`},
+		{[]string{"db", "audit-export", "-since", "2026-02-01", "-until", "2026-02-01"}, "felis db audit-export: -until 2026-02-01 is not after -since 2026-02-01"},
+		{[]string{"db", "audit-export", "extra"}, "felis db audit-export [-config path]"},
+	} {
+		var out, errBuf bytes.Buffer
+		code := run(append(tc.args, "-config", "/nonexistent/felis.toml"), &out, &errBuf)
+		if code != 2 || !strings.Contains(errBuf.String(), tc.wantErr) {
+			t.Errorf("%v: exit %d, stderr %q; want 2 and %q", tc.args, code, errBuf.String(), tc.wantErr)
+		}
+	}
+}

cmd/felis/egressgate.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"time"
+)
+
+// Vars so tests can shrink them. A dial that neither connects nor is refused
+// within egressDialTimeout counts as blocked: a policy that drops packets looks
+// exactly like that.
+var (
+	egressDialTimeout  = 500 * time.Millisecond
+	egressPollInterval = 200 * time.Millisecond
+)
+
+// cmdEgressGate is the first initContainer of every build pod. The pod's
+// NetworkPolicy is programmed asynchronously after the pod starts (live on k3s:
+// a build-labelled pod reached the internet and the Kubernetes API for its first
+// ~0.7 s), so the gate dials a destination the policy denies until it stops
+// answering, and only then lets the pod's next container, eventually the
+// untrusted Dockerfile, start.
+//
+// The default probe is the Kubernetes API Service, which the kubelet names in
+// every pod's environment and the build policy never admits. A probe that still
+// answers after --wait means the policy is not enforced at all (a CNI without
+// NetworkPolicy support, or k3s run with --disable-network-policy), and the
+// build fails closed.
+func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	probe := fs.String("probe", "", "host:port the build NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
+	wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the build is refused")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *probe == "" {
+		host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
+		if host == "" || port == "" {
+			fmt.Fprintln(stderr, "felis egress-gate: no --probe and no KUBERNETES_SERVICE_HOST/PORT to default to")
+			return 2
+		}
+		*probe = net.JoinHostPort(host, port)
+	}
+
+	start := time.Now()
+	for {
+		conn, err := net.DialTimeout("tcp", *probe, egressDialTimeout)
+		if err != nil {
+			fmt.Fprintf(stdout, "felis egress-gate: %s is unreachable after %s (%v); the egress lock is in effect\n",
+				*probe, time.Since(start).Round(time.Millisecond), err)
+			return 0
+		}
+		_ = conn.Close()
+		if time.Since(start) >= *wait {
+			fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s: the build namespace's NetworkPolicy is not enforced "+
+				"(a CNI without NetworkPolicy support, or k3s started with --disable-network-policy); refusing to run the build\n",
+				*probe, *wait)
+			return 1
+		}
+		time.Sleep(egressPollInterval)
+	}
+}

cmd/felis/egressgate_test.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"bytes"
+	"net"
+	"strings"
+	"testing"
+	"time"
+)
+
+func shrinkEgressGate(t *testing.T) {
+	t.Helper()
+	dial, poll := egressDialTimeout, egressPollInterval
+	egressDialTimeout, egressPollInterval = 200*time.Millisecond, 10*time.Millisecond
+	t.Cleanup(func() { egressDialTimeout, egressPollInterval = dial, poll })
+}
+
+// The gate holds while the probe answers and lets the pod go on once the policy
+// lands, which the test plays by closing the listener.
+func TestEgressGateWaitsForTheLock(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	accepted := make(chan struct{}, 100)
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+			accepted <- struct{}{}
+		}
+	}()
+	go func() {
+		for i := 0; i < 3; i++ {
+			<-accepted
+		}
+		_ = ln.Close()
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "10s"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if !strings.Contains(out.String(), "egress lock is in effect") {
+		t.Errorf("stdout = %q", out.String())
+	}
+}
+
+// A probe that keeps answering means no policy is enforced: the build must not run.
+func TestEgressGateRefusesAnOpenNetwork(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+		}
+	}()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "100ms"}, &out, &errb); code != 1 {
+		t.Fatalf("exit %d, want 1", code)
+	}
+	if !strings.Contains(errb.String(), "not enforced") {
+		t.Errorf("stderr = %q", errb.String())
+	}
+}
+
+func TestEgressGateDefaultsToTheKubernetesService(t *testing.T) {
+	shrinkEgressGate(t)
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	t.Setenv("KUBERNETES_SERVICE_PORT", "")
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate(nil, &out, &errb); code != 2 {
+		t.Fatalf("exit %d without a probe, want 2", code)
+	}
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	host, port, _ := net.SplitHostPort(ln.Addr().String())
+	_ = ln.Close() // closed: the lock reads as in effect at once
+	t.Setenv("KUBERNETES_SERVICE_HOST", host)
+	t.Setenv("KUBERNETES_SERVICE_PORT", port)
+	out.Reset()
+	if code := cmdEgressGate(nil, &out, &errb); code != 0 || !strings.Contains(out.String(), ln.Addr().String()) {
+		t.Fatalf("exit %d, stdout %q", code, out.String())
+	}
+}

cmd/felis/fetchcontext.go

@@ -4,6 +4,8 @@ import (
 	"archive/tar"
 	"compress/gzip"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"flag"
 	"fmt"
@@ -15,6 +17,8 @@ import (
 	"strings"
 	"syscall"
 	"time"
+
+	"felis.lolicon.best/internal/build"
 )
 
 // cmdFetchContext is the in-Pod entrypoint the build Job's context-fetch
@@ -41,9 +45,14 @@ func cmdFetchContext(args []string, _, stderr io.Writer) int {
 	fs.SetOutput(stderr)
 	url := fs.String("url", "", "internal-face URL of the submission's build-context tarball")
 	out := fs.String("out", "/context", "directory to extract the build context into")
+	want := fs.String("sha256", "", "refuse the context unless the tarball's sha256 is this lowercase hex digest")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
+	if *want != "" && !build.IsSHA256Hex(*want) {
+		fmt.Fprintf(stderr, "felis fetch-context: --sha256 %q is not a lowercase hex sha256\n", *want)
+		return 2
+	}
 	if *url == "" {
 		fmt.Fprintln(stderr, "felis fetch-context: --url is required")
 		return 2
@@ -66,7 +75,12 @@ func cmdFetchContext(args []string, _, stderr io.Writer) int {
 	// No overall client timeout: a legitimate modpack context can be large and the
 	// Job's activeDeadlineSeconds is the real bound. The header timeout catches a
 	// wedged endpoint without capping a healthy download.
-	client := &http.Client{Transport: &http.Transport{ResponseHeaderTimeout: time.Minute}}
+	// Redirects are refused: the request carries the service token, and the
+	// internal face never redirects, so a 3xx is someone steering the token.
+	client := &http.Client{
+		Transport:     &http.Transport{ResponseHeaderTimeout: time.Minute},
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
 	resp, err := fetchContextWithRetry(ctx, client, *url, token, stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
@@ -74,10 +88,28 @@ func cmdFetchContext(args []string, _, stderr io.Writer) int {
 	}
 	defer resp.Body.Close()
 
-	if err := extractTarGz(resp.Body, *out); err != nil {
+	h := sha256.New()
+	body := io.TeeReader(resp.Body, h)
+	if err := extractTarGz(body, *out); err != nil {
 		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
 		return 1
 	}
+	if *want == "" {
+		return 0
+	}
+	// The tar end marker comes before the gzip trailer and whatever follows it,
+	// so read to EOF: the digest must cover every byte the blob holds. The blob
+	// itself is size-capped at upload, which bounds this read.
+	if _, err := io.Copy(io.Discard, io.LimitReader(body, maxContextBytes)); err != nil {
+		fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
+		return 1
+	}
+	if got := hex.EncodeToString(h.Sum(nil)); got != *want {
+		// The init container failing is what keeps Kaniko from ever starting on
+		// the extracted tree.
+		fmt.Fprintf(stderr, "felis fetch-context: the context's sha256 is %s, the approved digest is %s: it changed after approval; refusing to build\n", got, *want)
+		return 1
+	}
 	return 0
 }
 
@@ -136,13 +168,27 @@ func fetchContextWithRetry(ctx context.Context, client *http.Client, url, token
 	}
 }
 
+// maxContextBytes / maxContextEntries bound what one context may expand to. The
+// compressed upload is capped at 1 GiB, but gzip turns that into hundreds of GiB
+// or millions of empty files, and the emptyDir's 4 GiB sizeLimit is only
+// enforced by the kubelet's periodic sweep, after the disk has filled. The byte
+// cap matches that sizeLimit; the entry cap is far above any real modpack (a
+// large one is a few thousand files) and far below an inode exhaustion.
+//
+// Vars, not consts, so tests can shrink them.
+var (
+	maxContextBytes   int64 = 4 << 30
+	maxContextEntries       = 200_000
+)
+
 // extractTarGz streams a gzip'd tarball into root, creating directories as
 // needed. Every entry is vetted BEFORE anything is written: a path that is
 // absolute or escapes root (via ".."), a link (symlink or hardlink), or any
 // special file kind aborts the whole extraction. Refusing rather than skipping is
 // deliberate — a context that needs one of those constructs is not a context this
 // transport carries, and silently dropping entries would build from a corpus the
-// submitter did not upload.
+// submitter did not upload. The whole extraction is also bounded by
+// maxContextBytes and maxContextEntries.
 func extractTarGz(r io.Reader, root string) error {
 	if err := os.MkdirAll(root, 0o755); err != nil {
 		return fmt.Errorf("create context dir: %w", err)
@@ -153,6 +199,8 @@ func extractTarGz(r io.Reader, root string) error {
 	}
 	defer zr.Close()
 	tr := tar.NewReader(zr)
+	var written int64
+	entries := 0
 	for {
 		hdr, err := tr.Next()
 		if errors.Is(err, io.EOF) {
@@ -161,6 +209,9 @@ func extractTarGz(r io.Reader, root string) error {
 		if err != nil {
 			return fmt.Errorf("read context tarball: %w", err)
 		}
+		if entries++; entries > maxContextEntries {
+			return fmt.Errorf("the build context has more than %d entries", maxContextEntries)
+		}
 		name := filepath.Clean(hdr.Name)
 		if name == "." {
 			continue
@@ -176,7 +227,7 @@ func extractTarGz(r io.Reader, root string) error {
 			if err := os.MkdirAll(target, 0o755); err != nil {
 				return fmt.Errorf("create %q: %w", name, err)
 			}
-		case tar.TypeReg, tar.TypeRegA:
+		case tar.TypeReg:
 			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
 				return fmt.Errorf("create parent of %q: %w", name, err)
 			}
@@ -188,10 +239,16 @@ func extractTarGz(r io.Reader, root string) error {
 			if err != nil {
 				return fmt.Errorf("create %q: %w", name, err)
 			}
-			if _, err := io.Copy(f, tr); err != nil {
+			n, err := io.Copy(f, io.LimitReader(tr, maxContextBytes-written+1))
+			written += n
+			if err != nil {
 				_ = f.Close()
 				return fmt.Errorf("write %q: %w", name, err)
 			}
+			if written > maxContextBytes {
+				_ = f.Close()
+				return fmt.Errorf("the build context expands past %d bytes", maxContextBytes)
+			}
 			if err := f.Close(); err != nil {
 				return fmt.Errorf("close %q: %w", name, err)
 			}

cmd/felis/fetchcontext_test.go

@@ -4,6 +4,8 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
 	"io"
 	"net"
 	"net/http"
@@ -121,6 +123,30 @@ func TestExtractTarGzRefusesEscapes(t *testing.T) {
 	}
 }
 
+// A context that expands past the byte or entry cap is refused, however small
+// it was compressed: gzip bombs and inode floods stop at the cap.
+func TestExtractTarGzCapsExpansion(t *testing.T) {
+	bytesCap, entriesCap := maxContextBytes, maxContextEntries
+	t.Cleanup(func() { maxContextBytes, maxContextEntries = bytesCap, entriesCap })
+	maxContextBytes, maxContextEntries = 1000, 5
+
+	fits := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 400)})
+	if err := extractTarGz(bytes.NewReader(fits), t.TempDir()); err != nil {
+		t.Fatalf("a context exactly at the byte cap: %v", err)
+	}
+	big := tgzBody(t, tarEntry{name: "a", body: strings.Repeat("x", 600)}, tarEntry{name: "b", body: strings.Repeat("y", 401)})
+	if err := extractTarGz(bytes.NewReader(big), t.TempDir()); err == nil || !strings.Contains(err.Error(), "expands past") {
+		t.Fatalf("one byte over the cap: err = %v", err)
+	}
+	var many []tarEntry
+	for i := 0; i < 6; i++ {
+		many = append(many, tarEntry{name: "d" + string(rune('0'+i)) + "/", typ: tar.TypeDir})
+	}
+	if err := extractTarGz(bytes.NewReader(tgzBody(t, many...)), t.TempDir()); err == nil || !strings.Contains(err.Error(), "entries") {
+		t.Fatalf("six entries over a cap of five: err = %v", err)
+	}
+}
+
 // The command end to end: it dials the URL with the bearer token from the
 // environment, and refuses to run without it (the internal face would 401
 // anyway; failing at parse time is the honest earlier error).
@@ -163,6 +189,69 @@ func TestCmdFetchContextFetchAndExtract(t *testing.T) {
 	}
 }
 
+// With --sha256 the fetch refuses any bytes but the approved ones, including
+// a tarball that extracts cleanly: that is exactly the context an uploader
+// swapped in after the review.
+func TestCmdFetchContextChecksDigest(t *testing.T) {
+	body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	sum := sha256.Sum256(body)
+	good := hex.EncodeToString(sum[:])
+	args := func(digest string) []string {
+		return []string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir(), "--sha256=" + digest}
+	}
+
+	if code := cmdFetchContext(args(good), io.Discard, io.Discard); code != 0 {
+		t.Fatalf("matching digest exit = %d, want 0", code)
+	}
+	var stderr bytes.Buffer
+	other := strings.Repeat("0", 64)
+	if code := cmdFetchContext(args(other), io.Discard, &stderr); code != 1 || !strings.Contains(stderr.String(), "changed after approval") {
+		t.Fatalf("mismatched digest exit = %d, stderr %q; want 1 naming the change", code, stderr.String())
+	}
+	stderr.Reset()
+	if code := cmdFetchContext(args("ABC"), io.Discard, &stderr); code != 2 {
+		t.Fatalf("malformed digest exit = %d, want 2 (stderr %q)", code, stderr.String())
+	}
+
+	// Bytes after the tar end marker still count: appending to an approved blob
+	// must change what the fetch accepts.
+	padded := append(append([]byte{}, body...), "trailing"...)
+	srvPadded := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(padded)
+	}))
+	defer srvPadded.Close()
+	if code := cmdFetchContext([]string{"--url=" + srvPadded.URL + "/c", "--out=" + t.TempDir(), "--sha256=" + good}, io.Discard, io.Discard); code != 1 {
+		t.Fatalf("padded blob exit = %d, want 1", code)
+	}
+}
+
+// The request carries the service token, so a redirect is a failure: the token
+// never follows it to another host (build-supply-chain-13).
+func TestCmdFetchContextRefusesRedirects(t *testing.T) {
+	var leaked bool
+	elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		leaked = true
+	}))
+	defer elsewhere.Close()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, elsewhere.URL+"/steal", http.StatusFound)
+	}))
+	defer srv.Close()
+	t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
+	var stderr bytes.Buffer
+	if code := cmdFetchContext([]string{"--url=" + srv.URL + "/c", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 1 {
+		t.Fatalf("redirect exit = %d, want 1 (stderr %q)", code, stderr.String())
+	}
+	if leaked {
+		t.Fatal("the fetch followed the redirect")
+	}
+}
+
 // A body that is not a gzip tarball must fail the extraction rather than produce
 // an empty (or partial) context Kaniko would then try to build.
 func TestExtractTarGzRejectsNonGzip(t *testing.T) {

cmd/felis/files.go

@@ -19,7 +19,7 @@ import (
 // Like cmdRestore it deliberately holds NO database credentials and never calls
 // config.Load: felis-api made the authorization decision (the caller owns this
 // server, and the server is stopped so the RWO world volume is free); this process
-// is the unprivileged hands that touch bytes. Its entire input is the three flags
+// is the unprivileged hands that touch bytes. Its entire input is the flags
 // below plus, for a write, one environment variable. Every isolation guarantee
 // lives in the Pod spec (internal/fileedit/jobspec.go), and the path-containment
 // guarantee lives in fileedit.Execute, which resolves the path through os.Root and
@@ -37,6 +37,7 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 	op := fs.String("op", "", "operation: list, read, or write")
 	path := fs.String("path", "", "path to operate on, relative to the world root (empty = the root itself)")
 	worldsRoot := fs.String("worlds-root", "/data", "mount path of the world PVC; every path resolves under it")
+	expect := fs.String("expect-sha256", "", "write only: refuse unless the file's current SHA-256 (hex) is this")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -67,7 +68,7 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
 		content = decoded
 	}
 
-	res, err := fileedit.Execute(*worldsRoot, *op, *path, content)
+	res, err := fileedit.Execute(*worldsRoot, *op, *path, content, *expect)
 	if err != nil {
 		// The operation could not be attempted — infrastructure, not caller fault.
 		fmt.Fprintf(stderr, "felis files: %v\n", err)

cmd/felis/initforwarding.go

@@ -20,18 +20,14 @@ const forwardingSecretEnv = "FELIS_FORWARDING_SECRET"
 // config/ and server.properties live under it.
 const defaultForwardingDataDir = "/data"
 
-// fwd*Mode make the written config readable AND rewritable by the main server
+// fwd*Mode are the modes the written config lands with. The initContainer runs as
-// container, whose UID we do not control (an arbitrary user image). The
+// the same uid as the server container (naming.GameUID, pinned by the operator in
-// initContainer runs as root (see buildStatefulSet) so it can write into a data
+// the pod securityContext) after the prepare-data initContainer has handed the
-// volume of unknown ownership; 0666/0777 then let a non-root Paper rewrite the
+// whole volume to that uid, so owner read/write is all the server needs to rewrite
-// same files on boot.
+// these files on boot and nothing else on the node gets write access to them.
-//
-// This relies on the initContainer running as root to write into a volume of
-// unknown ownership; that is how the operator schedules it. If that ever changes,
-// give the server pod an fsGroup so the shared volume is group-writable instead.
 const (
-	fwdFileMode os.FileMode = 0o666
+	fwdFileMode os.FileMode = 0o644
-	fwdDirMode  os.FileMode = 0o777
+	fwdDirMode  os.FileMode = 0o755
 )
 
 // cmdInitForwarding is the felis-image initContainer entrypoint that makes an
@@ -96,9 +92,8 @@ func writePaperGlobal(dataDir, secret string) error {
 	if err := os.MkdirAll(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("create %s: %w", dir, err)
 	}
-	// MkdirAll honours the process umask (root's is typically 022 → 0755); chmod
+	// MkdirAll honours the process umask; chmod does not, so a directory an older
-	// does not, and a non-root main container must be able to place/replace the
+	// release left at 0777 is brought back to fwdDirMode here.
-	// file in this directory on boot.
 	if err := os.Chmod(dir, fwdDirMode); err != nil {
 		return fmt.Errorf("chmod %s: %w", dir, err)
 	}
@@ -188,8 +183,8 @@ func upsertProperty(content []byte, key, value string) []byte {
 }
 
 // writeFileMode writes data then forces the mode, since WriteFile honours the
-// umask (root's is typically 022 → 0644) but a non-root main container must be
+// umask and leaves an existing file's mode alone: a file an older release wrote
-// able to rewrite these files on boot.
+// world-writable (0666) is tightened back to fwdFileMode on the next boot.
 func writeFileMode(path string, data []byte) error {
 	if err := os.WriteFile(path, data, fwdFileMode); err != nil {
 		return fmt.Errorf("write %s: %w", path, err)

cmd/felis/initforwarding_test.go

@@ -164,8 +164,9 @@ func TestUpsertPropertyAppends(t *testing.T) {
 	}
 }
 
-// The written files must be group/world writable so a non-root main container can
+// The written files land at fwdFileMode: owner-writable for the game uid the init
-// rewrite them. chmod semantics are POSIX-only, so this asserts on non-Windows.
+// shares with the server container, and no longer world-writable. chmod semantics
+// are POSIX-only, so this asserts on non-Windows.
 func TestWriteForwardingFileModes(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("POSIX file modes not represented on Windows")

cmd/felis/initvolume.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"syscall"
+
+	"felis.lolicon.best/internal/naming"
+)
+
+// cmdInitVolume is the felis-image `prepare-data` initContainer entrypoint: it
+// hands every entry of a server's world volume to the game uid/gid before the
+// server container starts. The operator runs the server itself as naming.GameUID,
+// so a world written by an earlier release (whose server ran as root), a restore
+// Job (which extracts as root), or a storage provisioner that creates the volume
+// root-owned would otherwise leave files the server cannot write — a world that
+// boots and then fails every save.
+//
+// fsGroup covers only part of this: kubelet applies it to volume types that
+// support ownership management, and a k3s local-path PV is a hostPath underneath,
+// which it skips. A walk from inside the pod works for every volume type.
+//
+// Only mismatched entries are touched, so a volume already owned by the game uid
+// costs one lstat per entry and no writes. The walk runs inside an os.Root at the
+// data dir and uses lchown, so a symlink a plugin planted is re-owned as a link
+// and never followed out of the volume.
+//
+// A single entry that cannot be chowned is reported and skipped: failing the pod
+// over one odd file would keep the whole server down, while the server itself
+// reports the one file it cannot write. Only an unreadable data dir fails.
+func cmdInitVolume(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("init-volume", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	dataDir := fs.String("data", defaultForwardingDataDir, "world volume mount to hand to the game uid")
+	uid := fs.Int64("uid", naming.GameUID, "owner uid for every entry")
+	gid := fs.Int64("gid", naming.GameGID, "owner gid for every entry")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	root, err := os.OpenRoot(*dataDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: open %s: %v\n", *dataDir, err)
+		return 1
+	}
+	defer root.Close()
+	res, err := chownTree(root, int(*uid), int(*gid), root.Lchown)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis init-volume: %v\n", err)
+		return 1
+	}
+	for _, f := range res.failures {
+		fmt.Fprintf(stderr, "felis init-volume: %s\n", f)
+	}
+	fmt.Fprintf(stdout, "felis init-volume: %d entries checked, %d handed to %d:%d, %d failed\n",
+		res.checked, res.changed, *uid, *gid, len(res.failures))
+	return 0
+}
+
+// chownResult tallies one walk; failures is capped so a volume of thousands of
+// unownable files cannot flood the pod log.
+type chownResult struct {
+	checked  int
+	changed  int
+	failures []string
+}
+
+const maxReportedChownFailures = 20
+
+// chownTree walks root and calls chown on every entry (the root dir included)
+// whose owner is not uid:gid. It returns an error only when the root itself
+// cannot be read; per-entry failures are collected in the result.
+func chownTree(root *os.Root, uid, gid int, chown func(name string, uid, gid int) error) (chownResult, error) {
+	var res chownResult
+	fail := func(name string, err error) {
+		if len(res.failures) < maxReportedChownFailures {
+			res.failures = append(res.failures, fmt.Sprintf("%s: %v", name, err))
+		} else if len(res.failures) == maxReportedChownFailures {
+			res.failures = append(res.failures, "further failures not listed")
+		}
+	}
+	err := fs.WalkDir(root.FS(), ".", func(name string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			if name == "." {
+				return walkErr
+			}
+			fail(name, walkErr)
+			// A directory that cannot be listed is skipped as a whole; a file
+			// error has nothing below it to skip.
+			if d != nil && d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		res.checked++
+		info, err := d.Info()
+		if err != nil {
+			fail(name, err)
+			return nil
+		}
+		if st, ok := info.Sys().(*syscall.Stat_t); ok && int(st.Uid) == uid && int(st.Gid) == gid {
+			return nil
+		}
+		if err := chown(name, uid, gid); err != nil {
+			if !errors.Is(err, fs.ErrNotExist) { // gone mid-walk: nothing left to own
+				fail(name, err)
+			}
+			return nil
+		}
+		res.changed++
+		return nil
+	})
+	return res, err
+}

cmd/felis/initvolume_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strconv"
+	"testing"
+)
+
+// openTree builds a small world under a temp dir: nested dirs, a file, and a
+// symlink pointing out of the volume that the walk must not follow.
+func openTree(t *testing.T) *os.Root {
+	t.Helper()
+	dir := t.TempDir()
+	for _, d := range []string{"world/region", "plugins"} {
+		if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, f := range []string{"level.dat", "world/region/r.0.0.mca"} {
+		if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	outside := t.TempDir()
+	if err := os.Symlink(outside, filepath.Join(dir, "plugins", "escape")); err != nil {
+		t.Fatal(err)
+	}
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { root.Close() })
+	return root
+}
+
+// Every entry owned by someone else is handed over, the root dir included, and a
+// symlink is re-owned as a link rather than walked into.
+func TestChownTreeHandsOverMismatchedEntries(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	var got []string
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		got = append(got, name)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	want := []string{".", "level.dat", "plugins", "plugins/escape", "world", "world/region", "world/region/r.0.0.mca"}
+	slices.Sort(got)
+	if !slices.Equal(got, want) {
+		t.Errorf("chowned %v, want %v", got, want)
+	}
+	if res.changed != len(want) || res.checked != len(want) || len(res.failures) != 0 {
+		t.Errorf("result = %+v, want %d checked and changed, no failures", res, len(want))
+	}
+}
+
+// A volume already owned by the game uid costs no chown at all: this is the steady
+// state every restart after the first one hits.
+func TestChownTreeSkipsMatchingOwner(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	calls := 0
+	res, err := chownTree(root, os.Getuid(), os.Getgid(), func(string, int, int) error {
+		calls++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if calls != 0 || res.changed != 0 {
+		t.Errorf("chown called %d times on an already-owned tree (result %+v)", calls, res)
+	}
+}
+
+// One entry that refuses the chown is reported and the walk carries on: a single
+// odd file must not keep the whole server from starting.
+func TestChownTreeContinuesPastFailures(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	root := openTree(t)
+	res, err := chownTree(root, os.Getuid()+1, os.Getgid(), func(name string, uid, gid int) error {
+		if name == "level.dat" {
+			return os.ErrPermission
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("chownTree: %v", err)
+	}
+	if len(res.failures) != 1 || res.changed != 6 {
+		t.Errorf("result = %+v, want 1 failure and 6 changed", res)
+	}
+}
+
+// Against a real directory the owner already matches, so the command succeeds
+// without needing CAP_CHOWN — the path every test runner (non-root) can take.
+func TestCmdInitVolumeOwnedTree(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX ownership not represented on Windows")
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "server.properties"), []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	var out, errb bytes.Buffer
+	code := cmdInitVolume([]string{"--data", dir, "--uid", strconv.Itoa(os.Getuid()), "--gid", strconv.Itoa(os.Getgid())}, &out, &errb)
+	if code != 0 {
+		t.Fatalf("exit %d, stderr %q", code, errb.String())
+	}
+	if code := cmdInitVolume([]string{"--data", filepath.Join(dir, "missing")}, &out, &errb); code != 1 {
+		t.Errorf("missing data dir exit = %d, want 1", code)
+	}
+}

cmd/felis/manifests.go

@@ -45,15 +45,22 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	registryPort := fs.Int("registry-port", 5000, "port the in-cluster registry listens on")
 	panelNodePort := fs.Int("panel-node-port", int(platform.DefaultPanelNodePort), "NodePort that exposes the built-in HTTPS panel/API origin")
 	felisImage := fs.String("felis-image", "", "container image the felis-api/operator Deployments run, also passed through as FELIS_IMAGE (REQUIRED)")
-	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry:2)")
+	registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry 2.8.3, pinned by digest)")
 	backupPVC := fs.String("backup-pvc", "felis-backups", "name of the world-archive PVC this bundle renders in the Minecraft namespace and advertises to the backup/restore executors via FELIS_BACKUP_PVC (default: felis-backups; pass an empty value to render none, leaving backup/restore answering 503)")
 	worldsHostPath := fs.String("worlds-host-path", "", "node directory the reaper reads worlds from: each world PVC resolves as <path>/<pvc>, or as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (k3s storage root: /var/lib/rancher/k3s/storage); enables the reaper CronJob (requires --archive-local-path and a non-empty --backup-pvc)")
-	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
+	archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path. With the backup PVC alone it renders the retention-only CronJob, which deletes backups past their expiry and never touches a world")
+	registryStorage := fs.String("registry-storage", "", "capacity the registry PVC requests (default 10Gi; k3s local-path does not enforce it)")
+	uploadsStorage := fs.String("uploads-storage", "", "capacity the uploads PVC requests (default 5Gi; k3s local-path does not enforce it)")
+	backupStorage := fs.String("backup-storage", "", "capacity the world-archive PVC requests (default 10Gi; k3s local-path does not enforce it)")
 	reaperNode := fs.String("reaper-node", "", "node that holds --worlds-host-path: pins the reaper CronJob's pod there via nodeSelector kubernetes.io/hostname (multi-node clusters need this, or the reaper may schedule where the hostPath is empty)")
 	var velocityCIDRs multiFlag
 	fs.Var(&velocityCIDRs, "velocity-cidr", "CIDR of a Velocity proxy host allowed to reach game port 25565 (repeatable, REQUIRED)")
 	var packageCIDRs multiFlag
 	fs.Var(&packageCIDRs, "package-cidr", "CIDR of a package mirror build Pods may reach (repeatable; default none = no internet egress)")
+	var serverDenyCIDRs multiFlag
+	fs.Var(&serverDenyCIDRs, "server-egress-deny-cidr", "extra CIDR game server pods may never reach, e.g. the node's public address (repeatable)")
+	var serverAllowCIDRs multiFlag
+	fs.Var(&serverAllowCIDRs, "server-egress-allow-cidr", "private CIDR game server pods may reach despite the private-range block, e.g. a LAN database (repeatable)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -74,7 +81,9 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"(the felis-api/operator Deployments run it and it is passed through as FELIS_IMAGE, e.g. --felis-image registry.felis.svc:5000/felis:v1)")
 		return 2
 	}
-	for _, cidr := range append(append([]string{}, velocityCIDRs...), packageCIDRs...) {
+	allCIDRs := append(append([]string{}, velocityCIDRs...), packageCIDRs...)
+	allCIDRs = append(append(allCIDRs, serverDenyCIDRs...), serverAllowCIDRs...)
+	for _, cidr := range allCIDRs {
 		if _, _, err := net.ParseCIDR(cidr); err != nil {
 			fmt.Fprintf(stderr, "felis manifests: invalid CIDR %q: %v\n", cidr, err)
 			return 2
@@ -118,8 +127,8 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"multi-node cluster you MUST pass --reaper-node <name> (or add a nodeSelector) for the node holding the " +
 			"worlds, or the reaper may schedule where the hostPath is empty"
 		if *reaperNode != "" {
-			pin = fmt.Sprintf("the CronJob is pinned to node %q via kubernetes.io/hostname — keep this pointed at the "+
+			pin = fmt.Sprintf("the CronJob and its worlds-root PV are pinned to node %q via kubernetes.io/hostname — "+
-				"node that actually holds the world volumes", *reaperNode)
+				"keep this pointed at the node that actually holds the world volumes", *reaperNode)
 		}
 		fmt.Fprintf(stderr, "felis manifests: note: rendering the retention reaper CronJob (worlds hostPath %q). "+
 			"These points are NOT verified here:\n"+
@@ -129,11 +138,25 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"<path>/<pvc>, or each candidate's archive fails and the world is preserved;\n"+
 			"  - %s.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, pin)
 	} else {
-		fmt.Fprintln(stderr, "felis manifests: note: retention reaper CronJob not rendered "+
+		switch {
-			"(pass --worlds-host-path and --archive-local-path — the archive PVC defaults to felis-backups — to enable it)")
+		case *archiveLocalPath != "" && *backupPVC == "":
+			fmt.Fprintln(stderr, "felis manifests: --archive-local-path names where the backup PVC is mounted, "+
+				"but --backup-pvc is empty (no archive store renders); drop one or the other")
+			return 2
+		case *archiveLocalPath != "":
+			fmt.Fprintln(stderr, "felis manifests: note: rendering the reaper CronJob retention-only: backups past "+
+				"their expiry are deleted daily, idle worlds are never archived or deleted "+
+				"(pass --worlds-host-path to reap them too)")
+		case *backupPVC != "":
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered: backups are never expired, so "+
+				"the archive store only grows until the disk fills (pass --archive-local-path, equal to felis.toml "+
+				"[archive] local_path, for the retention-only CronJob, and --worlds-host-path as well to reap idle worlds)")
+		default:
+			fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered (backups are disabled)")
+		}
 	}
 
-	out, err := platform.RenderYAML(platform.Params{
+	params := platform.Params{
 		ControlNamespace:   *controlNS,
 		MinecraftNamespace: *minecraftNS,
 		BuildNamespace:     *buildNS,
@@ -148,7 +171,19 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 		ArchiveLocalPath:   *archiveLocalPath,
 		VelocityCIDRs:      []string(velocityCIDRs),
 		PackageSourceCIDRs: []string(packageCIDRs),
-	})
+
+		ServerEgressDenyCIDRs:  []string(serverDenyCIDRs),
+		ServerEgressAllowCIDRs: []string(serverAllowCIDRs),
+
+		RegistryStorage: *registryStorage,
+		UploadsStorage:  *uploadsStorage,
+		BackupStorage:   *backupStorage,
+	}
+	if err := params.Validate(); err != nil {
+		fmt.Fprintf(stderr, "felis manifests: %v\n", err)
+		return 2
+	}
+	out, err := platform.RenderYAML(params)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis manifests: render: %v\n", err)
 		return 1

cmd/felis/manifests_test.go

@@ -90,12 +90,13 @@ func TestManifestsRendersBundle(t *testing.T) {
 		t.Error("rendered bundle must not contain ClusterRole/ClusterRoleBinding")
 	}
 	// Without the retention flags, the reaper CronJob is not rendered and the
-	// generator says so on stderr.
+	// generator says so on stderr, naming what that leaves: backups that never
+	// expire.
 	if strings.Contains(text, "kind: CronJob") {
-		t.Error("no reaper CronJob must render without --worlds-host-path")
+		t.Error("no reaper CronJob must render without --archive-local-path")
 	}
-	if !strings.Contains(errBuf.String(), "not rendered") {
+	if !strings.Contains(errBuf.String(), "not rendered") || !strings.Contains(errBuf.String(), "never expired") {
-		t.Errorf("expected a 'reaper not rendered' notice on stderr, got %q", errBuf.String())
+		t.Errorf("expected a 'reaper not rendered, backups never expired' notice on stderr, got %q", errBuf.String())
 	}
 }
 
@@ -144,6 +145,40 @@ func TestManifestsBackupPVCOptOut(t *testing.T) {
 	}
 }
 
+// TestManifestsRendersRetentionOnly: the archive store without a worlds root
+// still gets the daily CronJob, retention-only, so backups past their expiry
+// leave the store on an install that never reaps a world; the operator is told
+// which of the two it got. An archive path with the store switched off is a
+// mistake and fails loud.
+func TestManifestsRendersRetentionOnly(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
+	}
+	text := out.String()
+	for _, want := range []string{"kind: CronJob", "name: felis-reaper", "--retention-only", "claimName: felis-backups"} {
+		if !strings.Contains(text, want) {
+			t.Errorf("retention-only bundle missing %q", want)
+		}
+	}
+	if strings.Contains(text, "kind: PersistentVolume\n") || strings.Contains(text, "--worlds-root") {
+		t.Error("a retention-only bundle must not reach for a worlds root")
+	}
+	if !strings.Contains(errBuf.String(), "retention-only") {
+		t.Errorf("stderr must say the CronJob is retention-only, got %q", errBuf.String())
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--archive-local-path", "/var/lib/felis/archives", "--backup-pvc="}, &out, &errBuf)
+	if code != 2 || out.Len() != 0 || !strings.Contains(errBuf.String(), "--backup-pvc is empty") {
+		t.Errorf("archive path without an archive store: exit=%d out=%d bytes stderr=%q, want a fail-loud 2", code, out.Len(), errBuf.String())
+	}
+}
+
 // TestManifestsRendersReaper proves the happy path with the full retention trio:
 // a batch/v1 CronJob is emitted, named felis-reaper, mounting the backup PVC at the
 // supplied archive path.
@@ -219,3 +254,27 @@ func TestManifestsReaperNodePin(t *testing.T) {
 		t.Errorf("--reaper-node without --worlds-host-path: exit = %d, want 2", code)
 	}
 }
+
+// TestManifestsStorageSizes proves the PVC size flags reach the rendered claims
+// and a size the API server would reject fails before anything is applied.
+func TestManifestsStorageSizes(t *testing.T) {
+	var out, errBuf bytes.Buffer
+	code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "40Gi", "--uploads-storage", "8Gi"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("exit code = %d, stderr = %s", code, errBuf.String())
+	}
+	for _, want := range []string{"storage: 40Gi", "storage: 8Gi"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("bundle lacks %q", want)
+		}
+	}
+
+	out.Reset()
+	errBuf.Reset()
+	code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
+		"--registry-storage", "lots"}, &out, &errBuf)
+	if code == 0 || !strings.Contains(errBuf.String(), "registry storage") {
+		t.Errorf("--registry-storage lots: exit %d, stderr %q; want a refusal naming the flag", code, errBuf.String())
+	}
+}

cmd/felis/migrate.go

@@ -7,15 +7,24 @@ import (
 	"io"
 
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
 // cmdMigrate implements `felis migrate up`: load config, open the database, and
 // apply every pending embedded migration under the advisory lock (spec §6).
+//
+// Migrations only roll forward, and some drop data (0017_drop_password), so a
+// database that already holds a schema and has migrations pending is bundled
+// first (internal/dbbackup, label pre-migrate). A failed snapshot stops the
+// upgrade; -no-backup is the explicit way past it, e.g. for an external
+// database whose server is newer than the host's pg_dump.
 func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("migrate", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	backupDir := fs.String("backup-dir", dbbackup.DefaultDir, "where the pre-migration snapshot goes")
+	noBackup := fs.Bool("no-backup", false, "apply pending migrations without snapshotting the database first")
 	// The "up" verb precedes any flags (felis migrate up -config path). Go's
 	// flag.Parse stops at the first non-flag token and would never see a flag
 	// placed after "up", silently falling back to the default -config. Pull the
@@ -48,6 +57,18 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	if !*noBackup {
+		path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database.URL, *backupDir, stderr)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis migrate: pre-migration backup failed, nothing applied: %v\n", err)
+			fmt.Fprintln(stderr, "  fix the backup, or re-run with -no-backup to migrate without one")
+			return 1
+		}
+		if path != "" {
+			fmt.Fprintf(stdout, "felis migrate: database snapshot %s\n", path)
+		}
+	}
+
 	applied, err := store.Up(ctx, drv, migrations)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis migrate: %v\n", err)
@@ -60,3 +81,56 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// preMigrateBackup bundles the database when it already carries a schema and
+// some of migrations are not applied yet, and returns the bundle's path ("" when
+// there was nothing to protect: a fresh database, or nothing pending).
+func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, dbURL, dir string, log io.Writer) (string, error) {
+	if err := drv.EnsureVersionTable(ctx); err != nil {
+		return "", fmt.Errorf("ensure version table: %w", err)
+	}
+	done, err := drv.AppliedVersions(ctx)
+	if err != nil {
+		return "", fmt.Errorf("read applied versions: %w", err)
+	}
+	if len(done) == 0 || !hasPending(done, migrations) {
+		return "", nil
+	}
+	return dbbackup.Backup(ctx, dbbackup.BackupOptions{
+		DatabaseURL: dbURL, Dir: dir, Label: dbbackup.LabelPreMigrate,
+		Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir,
+		Version: resolvedVersion(), Log: log, Record: true,
+	})
+}
+
+func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
+	for _, m := range migrations {
+		if _, ok := done[m.Version]; !ok {
+			return true
+		}
+	}
+	return false
+}
+
+// openStore opens the business database for a command that reads and writes its
+// tables, and refuses one whose schema this build was not written against: a newer
+// Felis migrated it (a rolled-back binary), or, unless allowPending, migrations this
+// build embeds have not run yet (a binary swapped in ahead of `felis migrate up`).
+func openStore(ctx context.Context, url string, allowPending bool) (*store.PostgresDriver, error) {
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	s, err := store.ReadSchema(ctx, drv)
+	if err == nil {
+		err = s.Err()
+		if allowPending {
+			err = s.Newer()
+		}
+	}
+	if err != nil {
+		drv.Close()
+		return nil, err
+	}
+	return drv, nil
+}

cmd/felis/mirrortools.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/build"
+	"felis.lolicon.best/internal/imagepush"
+)
+
+// defaultBuildToolsStatus is where mirror-build-tools records its last run; the
+// watchdog reads it to tell a vulnerability DB that stopped refreshing.
+const defaultBuildToolsStatus = "/var/lib/felis/build-tools/status.json"
+
+// cmdMirrorBuildTools copies the build lane's tools (build.Tools: the kaniko and
+// trivy images, Trivy's vulnerability and Java DBs) from upstream into the
+// platform registry, where build Jobs pull them. deploy/bootstrap.sh runs it at
+// install and from felis-build-tools.timer twice a day, which is what keeps the
+// DBs fresh; a root shell can run it the same way to refresh now.
+//
+// It writes as the platform principal through the node's loopback hostPort, the
+// same way the installer pushes, reading the token from the environment or from
+// /etc/felis/secrets.env.
+func cmdMirrorBuildTools(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("mirror-build-tools", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	endpoint := fs.String("endpoint", "127.0.0.1:5000", "host[:port] of the registry to write to (plain HTTP)")
+	only := fs.String("only", "", "comma-separated tool names to copy (default: all of "+toolNames()+")")
+	status := fs.String("status", defaultBuildToolsStatus, `file to record the run in ("" records nothing)`)
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	platformFlag := fs.String("platform", "", "os/arch of the images to copy (default: this machine's)")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	tools, err := selectTools(*only)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
+		return 2
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: "http", Username: user, Password: pass, Log: stdout}
+	src := &imagepush.Source{Platform: *platformFlag}
+	started := time.Now()
+	var failed []string
+	for _, t := range tools {
+		dst := strings.TrimSuffix(*endpoint, "/") + "/" + t.Mirror
+		if _, err := p.Mirror(ctx, src, t.Source, dst); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: %s: %v\n", t.Name, err)
+			failed = append(failed, t.Name+": "+err.Error())
+		}
+	}
+	if *status != "" {
+		st, err := imagepush.ReadMirrorStatus(*status)
+		if err != nil || st == nil {
+			st = &imagepush.MirrorStatus{}
+		}
+		st.LastAttempt = started
+		st.LastError = strings.Join(failed, "; ")
+		if len(failed) == 0 {
+			st.LastSuccess = started
+		}
+		if err := imagepush.WriteMirrorStatus(*status, *st); err != nil {
+			fmt.Fprintf(stderr, "felis mirror-build-tools: record %s: %v\n", *status, err)
+		}
+	}
+	if len(failed) > 0 {
+		return 1
+	}
+	return 0
+}
+
+func toolNames() string {
+	var names []string
+	for _, t := range build.Tools {
+		names = append(names, t.Name)
+	}
+	return strings.Join(names, ",")
+}
+
+func selectTools(only string) ([]build.Tool, error) {
+	if only == "" {
+		return build.Tools, nil
+	}
+	var out []build.Tool
+	for _, name := range strings.Split(only, ",") {
+		name = strings.TrimSpace(name)
+		found := false
+		for _, t := range build.Tools {
+			if t.Name == name {
+				out = append(out, t)
+				found = true
+			}
+		}
+		if !found {
+			return nil, fmt.Errorf("unknown tool %q (known: %s)", name, toolNames())
+		}
+	}
+	return out, nil
+}

cmd/felis/offsite.go

@@ -0,0 +1,653 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const offsiteUsage = `usage:
+  felis offsite sync         [-config path] [-archive-dir dir] [-db-dir dir] [-registry host:port|off]
+                             [-uploads-dir dir] [-status-file path]
+  felis offsite status       [-config path] [-status-file path]
+  felis offsite list         [-config path]
+  felis offsite fetch-db     [-config path | -endpoint url -bucket name [-region r] [-prefix p]]
+                             [-dir dir] latest|<bundle>
+  felis offsite fetch-worlds [-config path] [-archive-dir dir]
+  felis offsite fetch-images [-config path] [-registry host:port] [-at version]
+  felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
+  felis offsite keygen
+
+Every verb but keygen reads the bucket credentials and the encryption key from
+the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY,
+FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
+-env-file (default /etc/felis/offsite.env).
+`
+
+// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
+// felis-offsite.service unit loads it as its EnvironmentFile.
+const defaultOffsiteEnvFile = "/etc/felis/offsite.env"
+
+// cmdOffsite implements `felis offsite`: the off-site copy of the world
+// archives, the database bundles, the registry's user images and the
+// submission uploads (internal/offsite). felis-offsite.timer runs `sync`
+// hourly on the host; the fetch verbs are the way back after the node is lost
+// (docs/troubleshooting.md §16).
+func cmdOffsite(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	verb, rest := args[0], args[1:]
+	fs := flag.NewFlagSet("offsite "+verb, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, offsiteUsage) }
+	switch verb {
+	case "sync":
+		return offsiteSync(fs, rest, stdout, stderr)
+	case "status":
+		return offsiteStatus(fs, rest, stdout, stderr)
+	case "list":
+		return offsiteList(fs, rest, stdout, stderr)
+	case "fetch-db":
+		return offsiteFetchDB(fs, rest, stdout, stderr)
+	case "fetch-worlds":
+		return offsiteFetchWorlds(fs, rest, stdout, stderr)
+	case "fetch-images":
+		return offsiteFetchImages(fs, rest, stdout, stderr)
+	case "fetch-uploads":
+		return offsiteFetchUploads(fs, rest, stdout, stderr)
+	case "keygen":
+		k, err := offsite.NewKey()
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite keygen: %v\n", err)
+			return 1
+		}
+		fmt.Fprintln(stdout, k)
+		return 0
+	case "-h", "--help", "help":
+		fmt.Fprint(stdout, offsiteUsage)
+		return 0
+	}
+	fmt.Fprintf(stderr, "felis offsite: unknown verb %q\n%s", verb, offsiteUsage)
+	return 2
+}
+
+// offsiteEnv is the resolved [offsite] binding: the bucket and the key.
+type offsiteEnv struct {
+	cfg    config.OffsiteConfig
+	bucket *offsite.S3
+	key    []byte
+}
+
+// loadEnvFile sets each KEY=VALUE of path that is not already in the
+// environment, so a root shell runs a command the same way its unit does.
+// A missing file is not an error.
+func loadEnvFile(path string) error {
+	if path == "" {
+		return nil
+	}
+	f, err := os.Open(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		k = strings.TrimSpace(strings.TrimPrefix(k, "export "))
+		v = strings.TrimSpace(v)
+		if len(v) >= 2 && (v[0] == '"' || v[0] == '\'') && v[len(v)-1] == v[0] {
+			v = v[1 : len(v)-1]
+		}
+		if os.Getenv(k) == "" {
+			os.Setenv(k, v)
+		}
+	}
+	return sc.Err()
+}
+
+// resolveOffsite builds the bucket client and parses the key for c.
+func resolveOffsite(c config.OffsiteConfig) (*offsiteEnv, error) {
+	if !c.Enabled() {
+		return nil, errors.New("no [offsite] bucket is configured (docs/troubleshooting.md §16, \"Keep a copy somewhere else\")")
+	}
+	need := func(ref, what string) (string, error) {
+		v := os.Getenv(ref)
+		if v == "" {
+			return "", fmt.Errorf("%s: environment variable %s is empty (set it, or put it in %s)", what, ref, defaultOffsiteEnvFile)
+		}
+		return v, nil
+	}
+	ak, err := need(c.AccessKeyRef, "access key")
+	if err != nil {
+		return nil, err
+	}
+	sk, err := need(c.SecretKeyRef, "secret key")
+	if err != nil {
+		return nil, err
+	}
+	rawKey, err := need(c.KeyRef, "encryption key")
+	if err != nil {
+		return nil, err
+	}
+	key, err := offsite.ParseKey(rawKey)
+	if err != nil {
+		return nil, err
+	}
+	b, err := offsite.NewS3(offsite.S3Config{
+		Endpoint: c.Endpoint, Region: c.Region, Bucket: c.Bucket, Prefix: c.Prefix,
+		AccessKey: ak, SecretKey: sk,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &offsiteEnv{cfg: c, bucket: b, key: key}, nil
+}
+
+// loadOffsite loads felis.toml and the env file and resolves [offsite].
+func loadOffsite(cfgPath, envFile string) (*config.Config, *offsiteEnv, error) {
+	if err := loadEnvFile(envFile); err != nil {
+		return nil, nil, fmt.Errorf("read %s: %w", envFile, err)
+	}
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	env, err := resolveOffsite(cfg.Offsite)
+	if err != nil {
+		return cfg, nil, err
+	}
+	return cfg, env, nil
+}
+
+func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
+	dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
+	registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC through the cluster)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, `the submission uploads PVC, in the control-plane namespace ("" copies no uploads)`)
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "where the result of this run is recorded for the watchdog and `status`")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	st := offsite.Status{
+		LastAttempt: time.Now().UTC(), Endpoint: env.cfg.Endpoint, Bucket: env.cfg.Bucket,
+		Prefix: env.cfg.Prefix, KeyID: offsite.KeyID(env.key),
+	}
+	if prev, _ := offsite.ReadStatus(*statusFile); prev != nil {
+		st.LastSuccess = prev.LastSuccess
+	}
+	res, err := runOffsiteSync(cfg, env, offsiteSources{
+		archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir,
+		registry:   offsiteRegistryEndpoint(*registry, cfg.Registry),
+		uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
+	}, stderr)
+	st.Result = res
+	if err != nil {
+		st.LastError = err.Error()
+	} else {
+		st.LastSuccess = st.LastAttempt
+	}
+	if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
+	}
+	fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n",
+		res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
+		res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
+		res.UploadsUploaded, res.UploadObjectsPruned,
+		res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes),
+		res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes))
+	for _, m := range res.WorldsMissing {
+		fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
+	}
+	for _, m := range res.ImagesIncomplete {
+		fmt.Fprintf(stderr, "felis offsite sync: registry image not whole: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// offsiteSources is where one sync pass reads from: the world archive volume
+// (archiveDir, or the backupPVC's directory), the bundle directory, the
+// registry's loopback endpoint and the uploads volume (uploadsDir, or the
+// uploadsPVC's directory). An empty source is skipped.
+type offsiteSources struct {
+	archiveDir, backupPVC  string
+	dbDir                  string
+	registry               string
+	uploadsDir, uploadsPVC string
+}
+
+func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, log io.Writer) (offsite.Result, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Minute)
+	defer cancel()
+	checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
+	err := env.bucket.Check(checkCtx)
+	checkCancel()
+	if err != nil {
+		return offsite.Result{}, err
+	}
+	archiveDir, uploadsDir := src.archiveDir, src.uploadsDir
+	if archiveDir == "" && src.backupPVC != "" {
+		dir, err := resolveVolumeDir(ctx, cfg.K8s.Namespace, src.backupPVC, archiveVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		archiveDir = dir
+	}
+	// An s3:// uploads store is off the host already; only a local one, on
+	// the uploads PVC, needs the copy.
+	if uploadsDir == "" && src.uploadsPVC != "" && isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+		dir, err := resolveVolumeDir(ctx, platform.DefaultControlNamespace, src.uploadsPVC, uploadsVolume, false, log)
+		if err != nil {
+			return offsite.Result{}, err
+		}
+		uploadsDir = dir
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		return offsite.Result{}, fmt.Errorf("open database: %w", err)
+	}
+	defer drv.Close()
+	s := &offsite.Syncer{
+		Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key,
+		ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Log: log,
+	}
+	if src.registry != "" {
+		s.Images = newRegistryImages(src.registry)
+		s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL)
+	}
+	return s.Run(ctx)
+}
+
+// volumeKind names a PVC the off-site copy reads or restores, for messages,
+// with the flag that bypasses finding it through the cluster.
+type volumeKind struct{ what, dirFlag, empty string }
+
+var (
+	archiveVolume = volumeKind{"archive volume", "-archive-dir", "no world has been archived"}
+	uploadsVolume = volumeKind{"uploads volume", "-uploads-dir", "no modpack has been uploaded"}
+)
+
+// resolveVolumeDir finds the host directory behind a PVC: a local-path volume
+// is a directory on this node. A PVC still waiting for its first consumer
+// holds nothing yet: without bind that is "" (nothing to copy), with bind it
+// is bound first, for a fetch to write into.
+func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind, bind bool, log io.Writer) (string, error) {
+	if ns == "" {
+		ns = platform.DefaultMinecraftNamespace
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		return "", fmt.Errorf("reach the cluster to find the %s (or pass %s): %w", kind.what, kind.dirFlag, err)
+	}
+	var pvc corev1.PersistentVolumeClaim
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+		return "", fmt.Errorf("%s %s/%s: %w", kind.what, ns, pvcName, err)
+	}
+	if pvc.Spec.VolumeName == "" {
+		if !bind {
+			fmt.Fprintf(log, "felis offsite: %s %s/%s is not bound yet; %s\n", kind.what, ns, pvcName, kind.empty)
+			return "", nil
+		}
+		if err := bindVolume(ctx, cl, ns, pvcName, kind, log); err != nil {
+			return "", err
+		}
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+			return "", err
+		}
+	}
+	var pv corev1.PersistentVolume
+	if err := cl.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil {
+		return "", fmt.Errorf("%s %s: %w", kind.what, pvc.Spec.VolumeName, err)
+	}
+	var dir string
+	switch {
+	case pv.Spec.Local != nil:
+		dir = pv.Spec.Local.Path
+	case pv.Spec.HostPath != nil:
+		dir = pv.Spec.HostPath.Path
+	default:
+		return "", fmt.Errorf("%s %s is not a directory on a node (local or hostPath); pass %s with where it is mounted on this host", kind.what, pv.Name, kind.dirFlag)
+	}
+	if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
+		return "", fmt.Errorf("%s %s is %s on its node, which is not a directory here; run this on the node that holds it, or pass %s", kind.what, pv.Name, dir, kind.dirFlag)
+	}
+	return dir, nil
+}
+
+// bindVolume runs a pod that mounts the PVC and exits, which is what makes a
+// WaitForFirstConsumer volume (k3s local-path) get provisioned. The pod uses
+// the control plane's own image, which every install already has.
+func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind volumeKind, log io.Writer) error {
+	var api appsv1.Deployment
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: platform.DefaultControlNamespace, Name: "felis-api"}, &api); err != nil {
+		return fmt.Errorf("find the felis image to bind the %s with: %w", kind.what, err)
+	}
+	if len(api.Spec.Template.Spec.Containers) == 0 {
+		return errors.New("felis-api has no container to take the image from")
+	}
+	image := api.Spec.Template.Spec.Containers[0].Image
+	pod := platform.VolumeBinderPod(ns, pvcName, image)
+	if err := cl.Create(ctx, pod); err != nil {
+		return fmt.Errorf("start a pod to bind the %s: %w", kind.what, err)
+	}
+	fmt.Fprintf(log, "felis offsite: binding the %s %s/%s (pod %s)\n", kind.what, ns, pvcName, pod.Name)
+	defer func() {
+		_ = cl.Delete(context.Background(), pod, client.PropagationPolicy(metav1.DeletePropagationBackground))
+	}()
+	deadline := time.Now().Add(3 * time.Minute)
+	for time.Now().Before(deadline) {
+		var pvc corev1.PersistentVolumeClaim
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err == nil && pvc.Spec.VolumeName != "" && pvc.Status.Phase == corev1.ClaimBound {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+	return fmt.Errorf("the %s %s/%s did not bind within 3 minutes; see kubectl -n %s describe pod %s", kind.what, ns, pvcName, ns, pod.Name)
+}
+
+func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if !cfg.Offsite.Enabled() {
+		fmt.Fprintln(stdout, "off-site copy: not configured. World archives, database bundles, user images and uploaded modpacks exist on this machine only.")
+		fmt.Fprintln(stdout, "See docs/troubleshooting.md §16, \"Keep a copy somewhere else\".")
+		return 1
+	}
+	o := cfg.Offsite
+	fmt.Fprintf(stdout, "bucket:       %s at %s", o.Bucket, o.Endpoint)
+	if o.Prefix != "" {
+		fmt.Fprintf(stdout, ", prefix %s", o.Prefix)
+	}
+	fmt.Fprintln(stdout)
+	st, err := offsite.ReadStatus(*statusFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite status: %v\n", err)
+		return 1
+	}
+	if st == nil {
+		fmt.Fprintln(stdout, "last sync:    never (sudo systemctl start felis-offsite.service)")
+		return 1
+	}
+	now := time.Now()
+	fmt.Fprintf(stdout, "key id:       %s\n", st.KeyID)
+	fmt.Fprintf(stdout, "last attempt: %s (%s ago)\n", st.LastAttempt.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastAttempt)))
+	if st.LastSuccess.IsZero() {
+		fmt.Fprintln(stdout, "last success: never")
+	} else {
+		fmt.Fprintf(stdout, "last success: %s (%s ago)\n", st.LastSuccess.Local().Format(time.DateTime), dbbackup.Age(now.Sub(st.LastSuccess)))
+	}
+	if st.LastError != "" {
+		fmt.Fprintf(stdout, "last error:   %s\n", st.LastError)
+	}
+	r := st.Result
+	fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
+		r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
+	if r.ImageIndex != "" {
+		fmt.Fprintf(stdout, "images:       %d in %d repositories (%s), registry index %s\n",
+			r.Images, r.ImageRepos, offsite.HumanBytes(r.RemoteImageBytes), r.ImageIndex)
+	} else {
+		fmt.Fprintln(stdout, "images:       not copied (no in-cluster registry, or no sync has reached it yet)")
+	}
+	if r.UploadIndex != "" {
+		fmt.Fprintf(stdout, "uploads:      %d submission contexts (%s), uploads index %s\n",
+			r.Uploads, offsite.HumanBytes(r.RemoteUploadBytes), r.UploadIndex)
+	} else {
+		fmt.Fprintln(stdout, "uploads:      not copied (an s3:// uploads store, or no sync has reached the volume yet)")
+	}
+	fmt.Fprintf(stdout, "waiting:      %d world archives not yet copied\n", r.WorldsPending)
+	for _, m := range r.WorldsMissing {
+		fmt.Fprintf(stdout, "missing:      %s is recorded but not on the volume\n", m)
+	}
+	for _, m := range r.ImagesIncomplete {
+		fmt.Fprintf(stdout, "not whole:    %s\n", m)
+	}
+	if st.LastSuccess.IsZero() || now.Sub(st.LastSuccess) > offsite.StaleAfter {
+		fmt.Fprintf(stdout, "\nThe last successful sync is older than %s: journalctl -u felis-offsite -n 50\n", dbbackup.Age(offsite.StaleAfter))
+		return 1
+	}
+	return 0
+}
+
+func orNone(s string) string {
+	if s == "" {
+		return "none"
+	}
+	return s
+}
+
+func offsiteList(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	_, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	return printOffsiteList(env, stdout, stderr)
+}
+
+func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	bundles, err := offsite.ListDB(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	worlds, err := env.bucket.List(ctx, "worlds/")
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "database bundles (%d, newest first):\n", len(bundles))
+	for _, b := range bundles {
+		fmt.Fprintf(stdout, "  %s  %s\n", b.Key, offsite.HumanBytes(b.Size))
+	}
+	var total int64
+	for _, w := range worlds {
+		total += w.Size
+	}
+	fmt.Fprintf(stdout, "world archives: %d (%s)\n", len(worlds), offsite.HumanBytes(total))
+	versions, err := offsite.ImageIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "registry index versions (%d, newest first; restore one with fetch-images -at):\n", len(versions))
+	for i := len(versions) - 1; i >= 0; i-- {
+		x, err := offsite.LoadImageIndex(ctx, env.bucket, env.key, versions[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", versions[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d images in %d repositories\n", versions[i], x.Images(), len(x.Repositories))
+	}
+	uploads, err := offsite.UploadIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "uploads index versions (%d, newest first; restore one with fetch-uploads -at):\n", len(uploads))
+	for i := len(uploads) - 1; i >= 0; i-- {
+		x, err := offsite.LoadUploadIndex(ctx, env.bucket, env.key, uploads[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", uploads[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d submission contexts (%s)\n", uploads[i], len(x.Contexts), offsite.HumanBytes(x.Bytes()))
+	}
+	return 0
+}
+
+func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	endpoint := fs.String("endpoint", "", "bucket endpoint, when there is no felis.toml")
+	bucket := fs.String("bucket", "", "bucket name, when there is no felis.toml")
+	region := fs.String("region", "", "bucket region, when there is no felis.toml")
+	prefix := fs.String("prefix", "", "key prefix, when there is no felis.toml")
+	dir := fs.String("dir", dbbackup.DefaultDir, "directory to write the bundle to")
+	arg, ok := parseWithArg(fs, args)
+	if !ok {
+		return 2
+	}
+	if arg == "" {
+		fmt.Fprint(stderr, offsiteUsage)
+		return 2
+	}
+	if err := loadEnvFile(*envFile); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: read %s: %v\n", *envFile, err)
+		return 1
+	}
+	var oc config.OffsiteConfig
+	if *bucket != "" {
+		oc = config.OffsiteConfig{
+			Endpoint: *endpoint, Bucket: *bucket, Region: *region, Prefix: *prefix,
+			AccessKeyRef: config.DefaultOffsiteAccessKeyEnv, SecretKeyRef: config.DefaultOffsiteSecretKeyEnv,
+			KeyRef: config.DefaultOffsiteKeyEnv,
+		}
+	} else {
+		cfg, err := config.Load(*cfgPath)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v (on a host with no install yet, pass -endpoint and -bucket)\n", err)
+			return 1
+		}
+		oc = cfg.Offsite
+	}
+	env, err := resolveOffsite(oc)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	name := arg
+	if name == "latest" {
+		bundles, err := offsite.ListDB(ctx, env.bucket)
+		if err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+			return 1
+		}
+		if len(bundles) == 0 {
+			fmt.Fprintln(stderr, "felis offsite fetch-db: the bucket holds no database bundle")
+			return 1
+		}
+		name = bundles[0].Key
+	}
+	if _, _, ok := dbbackup.ParseBundleName(name); !ok {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %q is not a bundle name (felis-db-<stamp>-<label>.tar); see `felis offsite list`\n", name)
+		return 2
+	}
+	if err := os.MkdirAll(*dir, 0o700); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	dst := filepath.Join(*dir, name)
+	if err := offsite.FetchObject(ctx, env.bucket, env.key, offsite.DBKey(name), dst, 0o600); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
+		return 1
+	}
+	if _, err := dbbackup.Verify(dst); err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-db: fetched %s but it does not verify: %v\n", dst, err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-db: wrote %s (verified)\n", dst)
+	return 0
+}
+
+func offsiteFetchWorlds(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC, binding it if needed)")
+	backupPVC := fs.String("backup-pvc", "felis-backups", "the world archive PVC, in the [k8s] namespace")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	dir := *archiveDir
+	if dir == "" {
+		if dir, err = resolveVolumeDir(ctx, cfg.K8s.Namespace, *backupPVC, archiveVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+			return 1
+		}
+	}
+	drv, err := openStore(ctx, cfg.Database.URL, false)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: open database: %v\n", err)
+		return 1
+	}
+	defer drv.Close()
+	res, err := offsite.FetchWorlds(ctx, env.bucket, offsite.PGCatalog{DB: drv.DB()}, env.key, dir, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-worlds: %d recorded archives, %d fetched into %s, %d with no copy in the bucket\n",
+		res.Present, len(res.Fetched), dir, len(res.Missing))
+	for _, m := range res.Missing {
+		fmt.Fprintf(stdout, "  no off-site copy: %s\n", m)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_images.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/registrygate"
+	"felis.lolicon.best/internal/registryprune"
+)
+
+// registryImages is the platform registry as the off-site copy sees it: read
+// anonymously through the gate (the catalog, its manifest index, manifests and
+// blobs) and, for a restore, written as the platform principal. Both go through
+// the node's loopback hostPort, the way the installer pushes.
+type registryImages struct {
+	host   string
+	index  *registryprune.Client
+	source *imagepush.Source
+	pusher *imagepush.Pusher
+}
+
+func newRegistryImages(endpoint string) *registryImages {
+	return &registryImages{
+		host:   endpoint,
+		index:  &registryprune.Client{Endpoint: "http://" + endpoint},
+		source: &imagepush.Source{Scheme: "http"},
+	}
+}
+
+func (r *registryImages) Repositories(ctx context.Context) ([]string, error) {
+	return r.index.Repositories(ctx)
+}
+
+func (r *registryImages) Revisions(ctx context.Context, repo string) ([]string, map[string]string, error) {
+	idx, err := r.index.Index(ctx, repo)
+	if err != nil {
+		return nil, nil, err
+	}
+	digests := make([]string, 0, len(idx.Revisions))
+	for _, rev := range idx.Revisions {
+		digests = append(digests, rev.Digest)
+	}
+	return digests, idx.Tags, nil
+}
+
+func (r *registryImages) Manifest(ctx context.Context, repo, digest string) ([]byte, string, error) {
+	body, mt, err := r.source.Manifest(ctx, r.host, repo, digest)
+	return body, mt, registryGone(err)
+}
+
+func (r *registryImages) Blob(ctx context.Context, repo, digest string) (io.ReadCloser, error) {
+	rc, err := r.source.Blob(ctx, r.host, repo, digest)
+	return rc, registryGone(err)
+}
+
+func (r *registryImages) PutBlob(ctx context.Context, repo, digest string, size int64, open func() (io.ReadCloser, error)) error {
+	return r.pusher.UploadBlob(ctx, r.host, repo, digest, size, open)
+}
+
+func (r *registryImages) PutManifest(ctx context.Context, repo, reference, mediaType string, body []byte) error {
+	_, err := r.pusher.PutManifest(ctx, r.host, repo, reference, mediaType, body)
+	return err
+}
+
+// imagePins lists, per repository of the registry refs spell as host, the
+// digests the MinecraftServers' specs and the image whitelist pin: what a
+// restored database and its servers will ask the registry for.
+func imagePins(db *sql.DB, host string) func(ctx context.Context) (map[string][]string, error) {
+	return func(ctx context.Context) (map[string][]string, error) {
+		cl, err := buildSystemServerClient()
+		if err != nil {
+			return nil, fmt.Errorf("reach the cluster: %w", err)
+		}
+		var servers v1alpha1.MinecraftServerList
+		if err := cl.List(ctx, &servers); err != nil {
+			return nil, fmt.Errorf("list MinecraftServers: %w", err)
+		}
+		refs := make([]string, 0, len(servers.Items))
+		for _, s := range servers.Items {
+			refs = append(refs, s.Spec.Image)
+		}
+		rows, err := db.QueryContext(ctx, `SELECT image_ref FROM image_whitelist`)
+		if err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var ref string
+			if err := rows.Scan(&ref); err != nil {
+				return nil, fmt.Errorf("read the image whitelist: %w", err)
+			}
+			refs = append(refs, ref)
+		}
+		if err := rows.Err(); err != nil {
+			return nil, fmt.Errorf("read the image whitelist: %w", err)
+		}
+		pins := map[string][]string{}
+		for _, ref := range refs {
+			repo, _, digest, ok := registryprune.ParseRef(ref, host)
+			if ok && digest != "" && !slices.Contains(pins[repo], digest) {
+				pins[repo] = append(pins[repo], digest)
+			}
+		}
+		return pins, nil
+	}
+}
+
+// registryGone marks a 404 as a manifest or blob the registry no longer holds.
+func registryGone(err error) error {
+	var se *imagepush.StatusError
+	if errors.As(err, &se) && se.Code == http.StatusNotFound {
+		return fmt.Errorf("%w: %v", offsite.ErrImageGone, err)
+	}
+	return err
+}
+
+// offsiteRegistryEndpoint is where the host reaches the registry whose images
+// the off-site copy covers: flag when given ("off" for none), otherwise the
+// loopback hostPort of the in-cluster registry [registry] url names. A
+// registry outside the cluster is not this install's to copy.
+func offsiteRegistryEndpoint(flag string, reg config.RegistryConfig) string {
+	switch flag {
+	case "off":
+		return ""
+	case "":
+	default:
+		return flag
+	}
+	host, _, _ := strings.Cut(reg.URL, "/")
+	name, _, _ := strings.Cut(host, ":")
+	if strings.HasSuffix(name, ".svc") || strings.HasSuffix(name, ".svc.cluster.local") {
+		return loopbackEndpoint(host)
+	}
+	return ""
+}
+
+// registryWriteCredential is the credential a host-side push uses:
+// FELIS_REGISTRY_USERNAME/PASSWORD when set, otherwise the platform principal
+// with REGISTRY_PLATFORM_TOKEN from the installer's secrets file.
+func registryWriteCredential(secrets string) (string, string, error) {
+	if err := loadEnvFile(secrets); err != nil {
+		return "", "", fmt.Errorf("read %s: %w", secrets, err)
+	}
+	user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if pass == "" {
+		user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
+	}
+	if pass == "" {
+		return "", "", errors.New("no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
+	}
+	return user, pass, nil
+}
+
+func offsiteFetchImages(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	registry := fs.String("registry", "", "host[:port] of the registry to push into (default: the loopback hostPort of the in-cluster registry)")
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	at := fs.String("at", "", "registry index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	endpoint := offsiteRegistryEndpoint(*registry, cfg.Registry)
+	if endpoint == "" {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: [registry] url %q is not the in-cluster registry; pass -registry host:port\n", cfg.Registry.URL)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 2
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseImageIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-images: restoring registry index %s (%d repositories, %d images) into %s\n",
+		stamp, len(idx.Repositories), idx.Images(), endpoint)
+	target := newRegistryImages(endpoint)
+	target.pusher = &imagepush.Pusher{Scheme: "http", Username: user, Password: pass}
+	res, err := offsite.FetchImages(ctx, env.bucket, env.key, idx, target, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-images: %d of %d repositories restored, %d images, %d tags, %d blobs pushed (%s)\n",
+		res.Repositories, len(idx.Repositories), res.Manifests, res.Tags, res.BlobsPushed, offsite.HumanBytes(res.BytesPushed))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v (a second run pushes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_test.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+)
+
+func TestLoadOffsiteEnvFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "offsite.env")
+	body := `# written by bootstrap
+FELIS_OFFSITE_ACCESS_KEY=AKIA123
+export FELIS_OFFSITE_SECRET_KEY="se=cret"
+FELIS_OFFSITE_KEY='k'
+
+not a line
+`
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "from-the-shell")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "")
+	t.Setenv("FELIS_OFFSITE_KEY", "")
+	if err := loadEnvFile(path); err != nil {
+		t.Fatal(err)
+	}
+	for k, want := range map[string]string{
+		"FELIS_OFFSITE_ACCESS_KEY": "from-the-shell", // the environment wins
+		"FELIS_OFFSITE_SECRET_KEY": "se=cret",
+		"FELIS_OFFSITE_KEY":        "k",
+	} {
+		if got := os.Getenv(k); got != want {
+			t.Errorf("%s = %q, want %q", k, got, want)
+		}
+	}
+	if err := loadEnvFile(filepath.Join(t.TempDir(), "absent")); err != nil {
+		t.Errorf("a missing env file is not an error: %v", err)
+	}
+}
+
+func TestResolveOffsiteNamesTheMissingVariable(t *testing.T) {
+	c := config.OffsiteConfig{
+		Endpoint: "https://s3.example", Bucket: "b",
+		AccessKeyRef: "T_AK", SecretKeyRef: "T_SK", KeyRef: "T_KEY",
+	}
+	t.Setenv("T_AK", "ak")
+	t.Setenv("T_SK", "sk")
+	t.Setenv("T_KEY", "")
+	if _, err := resolveOffsite(c); err == nil || !strings.Contains(err.Error(), "T_KEY") {
+		t.Fatalf("err = %v, want it to name T_KEY", err)
+	}
+	t.Setenv("T_KEY", "not base64 at all")
+	if _, err := resolveOffsite(c); err == nil {
+		t.Fatal("a malformed key was accepted")
+	}
+	key, _ := offsite.NewKey()
+	t.Setenv("T_KEY", key)
+	env, err := resolveOffsite(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(env.key) != offsite.KeySize {
+		t.Fatalf("key is %d bytes", len(env.key))
+	}
+	if _, err := resolveOffsite(config.OffsiteConfig{}); err == nil {
+		t.Fatal("an unconfigured [offsite] resolved")
+	}
+}
+
+func TestOffsiteKeygen(t *testing.T) {
+	var out, errb bytes.Buffer
+	if code := cmdOffsite([]string{"keygen"}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	if _, err := offsite.ParseKey(strings.TrimSpace(out.String())); err != nil {
+		t.Fatalf("keygen printed %q: %v", out.String(), err)
+	}
+}
+
+func TestOffsiteFetchDBRejectsOddNames(t *testing.T) {
+	key, _ := offsite.NewKey()
+	t.Setenv("FELIS_OFFSITE_ACCESS_KEY", "ak")
+	t.Setenv("FELIS_OFFSITE_SECRET_KEY", "sk")
+	t.Setenv("FELIS_OFFSITE_KEY", key)
+	var out, errb bytes.Buffer
+	code := cmdOffsite([]string{"fetch-db", "-env-file", "", "-endpoint", "http://127.0.0.1:1", "-bucket", "b",
+		"-dir", t.TempDir(), "../../etc/shadow"}, &out, &errb)
+	if code != 2 || !strings.Contains(errb.String(), "not a bundle name") {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+}
+
+func TestOffsiteRegistryEndpoint(t *testing.T) {
+	for _, tc := range []struct{ flag, url, want string }{
+		{"", "registry.felis.svc:5000", "127.0.0.1:5000"},
+		{"", "registry.felis.svc.cluster.local:5001", "127.0.0.1:5001"},
+		{"", "ghcr.io/acme", ""},
+		{"", "", ""},
+		{"off", "registry.felis.svc:5000", ""},
+		{"10.0.0.5:5000", "ghcr.io/acme", "10.0.0.5:5000"},
+	} {
+		if got := offsiteRegistryEndpoint(tc.flag, config.RegistryConfig{URL: tc.url}); got != tc.want {
+			t.Errorf("offsiteRegistryEndpoint(%q, %q) = %q, want %q", tc.flag, tc.url, got, tc.want)
+		}
+	}
+}
+
+func TestRegistryGoneMarksNotFound(t *testing.T) {
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 404}); !errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("404 = %v, want ErrImageGone", err)
+	}
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 503}); errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("503 = %v, want it kept an ordinary failure", err)
+	}
+}

cmd/felis/offsite_uploads.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+)
+
+func offsiteFetchUploads(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC, binding it if needed)")
+	uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, "the submission uploads PVC, in the control-plane namespace")
+	at := fs.String("at", "", "uploads index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseUploadIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+		return 1
+	}
+	dir := *uploadsDir
+	if dir == "" {
+		if !isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: [registry] user_uploads_context %q is not the uploads volume; pass -uploads-dir to restore into a directory anyway\n", cfg.Registry.UserUploadsContext)
+			return 2
+		}
+		if dir, err = resolveVolumeDir(ctx, platform.DefaultControlNamespace, *uploadsPVC, uploadsVolume, true, stderr); err != nil {
+			fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
+			return 1
+		}
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: restoring uploads index %s (%d submission contexts, %s) into %s\n",
+		stamp, len(idx.Contexts), offsite.HumanBytes(idx.Bytes()), dir)
+	res, err := offsite.FetchUploads(ctx, env.bucket, env.key, idx, dir, platform.ControlPlaneUID, platform.ControlPlaneUID, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-uploads: %d written (%s), %d already in place, %d failed\n",
+		res.Written, offsite.HumanBytes(res.Bytes), res.Present, len(res.Failures))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v (a second run writes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/operator.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"log/slog"
+	"net/http"
 	"os"
+	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	felismetrics "felis.lolicon.best/internal/metrics"
@@ -75,16 +79,19 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	fmt.Fprintf(stderr, "felis operator: watching namespace %q\n", *namespace)
 
-	// Register the two probe endpoints. controller-runtime only mounts /healthz and
+	// /healthz fails while a reconcile pass has been stuck past its limit, so the
-	// /readyz once at least one check is registered, so a bare listener would 404.
+	// liveness probe restarts an operator whose workers are wedged (a Pod whose
-	// The checks are the canonical always-pass ping: the probes' contract is "the
+	// process answers but no server starts or stops). /readyz waits for the
-	// manager process is up and serving", and a dependency hiccup (e.g. an API blip)
+	// informer caches: until they sync the operator acts on nothing, and one that
-	// must not restart the operator.
+	// never syncs (lost RBAC, an unreachable API) never reports Available.
-	if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
+	// A dependency hiccup fails neither: the caches ride through API blips, and
+	// each pass is bounded well inside the stuck limit.
+	watch := &operator.ReconcileWatch{}
+	if err := mgr.AddHealthzCheck("reconcile", watch.Check); err != nil {
 		fmt.Fprintf(stderr, "felis operator: register healthz check: %v\n", err)
 		return 1
 	}
-	if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil {
+	if err := mgr.AddReadyzCheck("informers", cacheSynced(mgr.GetCache())); err != nil {
 		fmt.Fprintf(stderr, "felis operator: register readyz check: %v\n", err)
 		return 1
 	}
@@ -98,6 +105,7 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis operator: register metrics: %v\n", err)
 		return 1
 	}
+	felismetrics.SetBuildInfo("operator", resolvedVersion())
 
 	r := &operator.Reconciler{
 		Client: mgr.GetClient(),
@@ -107,6 +115,14 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		// injects into user servers. The Deployment passes it as FELIS_IMAGE (see
 		// platform.OperatorDeployment); absent, that injection is simply skipped.
 		FelisImage: os.Getenv("FELIS_IMAGE"),
+		// Uncached: the maintenance-lock check lists Jobs only when a server is
+		// about to start, which does not justify a namespace-wide Job informer.
+		Jobs: mgr.GetAPIReader(),
+		// Uncached too: RCON Secrets are read by name, so the Role grants
+		// secrets:get without the list/watch an informer would need.
+		Secrets:  mgr.GetAPIReader(),
+		Recorder: mgr.GetEventRecorderFor("felis-operator"),
+		Watch:    watch,
 	}
 	if err := r.SetupWithManager(mgr); err != nil {
 		fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
@@ -128,3 +144,18 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 	}
 	return 0
 }
+
+// cacheSynced is a readyz check that passes once every informer the manager
+// started has synced. It waits at most a second, well inside the probe timeout.
+func cacheSynced(c interface {
+	WaitForCacheSync(ctx context.Context) bool
+}) healthz.Checker {
+	return func(req *http.Request) error {
+		ctx, cancel := context.WithTimeout(req.Context(), time.Second)
+		defer cancel()
+		if !c.WaitForCacheSync(ctx) {
+			return errors.New("informer caches not synced")
+		}
+		return nil
+	}
+}

cmd/felis/operator_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"context"
+	"net/http/httptest"
+	"testing"
+)
+
+type fakeCache bool
+
+func (f fakeCache) WaitForCacheSync(ctx context.Context) bool {
+	if !f {
+		<-ctx.Done()
+	}
+	return bool(f)
+}
+
+// TestCacheSynced: the operator reports ready only once its informers synced,
+// and a check against caches that never sync returns within its own deadline.
+func TestCacheSynced(t *testing.T) {
+	req := httptest.NewRequest("GET", "/readyz", nil)
+	if err := cacheSynced(fakeCache(true))(req); err != nil {
+		t.Errorf("synced: %v", err)
+	}
+	if err := cacheSynced(fakeCache(false))(req); err == nil {
+		t.Error("unsynced caches reported ready")
+	}
+}

cmd/felis/pinimages.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// defaultRegistryURL is the [registry] url every install uses; deploy/bootstrap.sh
+// spells the same value as REGISTRY_URL.
+const defaultRegistryURL = "registry.felis.svc:5000"
+
+// cmdPinImages pins every user server whose spec.image still names a mutable tag
+// in the platform registry to the digest that tag names now (internal/imagepin).
+// felis-api pins on create, so this covers the servers created before it did.
+//
+// deploy/bootstrap.sh runs it before it rebuilds the game images and pushes them
+// over the same tags: run after the push, it would pin those servers to the new
+// build, which is exactly the silent Minecraft upgrade pinning exists to stop.
+// It reaches the registry through the node's loopback hostPort, the same way the
+// installer pushes.
+func cmdPinImages(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("pin-images", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	namespace := fs.String("namespace", platform.DefaultMinecraftNamespace, "namespace the MinecraftServers live in")
+	registry := fs.String("registry", defaultRegistryURL, "registry host[:port] the image refs spell")
+	endpoint := fs.String("endpoint", "", "host[:port] to reach the registry at (default: 127.0.0.1 on the registry's port, its hostPort on this node)")
+	system := fs.String("system", "", "pin this system server ("+naming.SystemLoginServer+" or "+naming.SystemLobbyServer+") to the build its tag names now, instead of the user servers")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if *system != "" && *system != naming.SystemLoginServer && *system != naming.SystemLobbyServer {
+		fmt.Fprintf(stderr, "felis pin-images: --system takes %s or %s, not %q\n", naming.SystemLoginServer, naming.SystemLobbyServer, *system)
+		return 2
+	}
+	if *endpoint == "" {
+		*endpoint = loopbackEndpoint(*registry)
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	if *system != "" {
+		return reportSystemPin(ctx, cl, *namespace, *system, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint}, stdout, stderr)
+	}
+	outcomes, err := pinUserServerImages(ctx, cl, *namespace, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint})
+	if meta.IsNoMatchError(err) {
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %v\n", err)
+		return 1
+	}
+	if len(outcomes) == 0 {
+		fmt.Fprintln(stdout, "felis pin-images: every user server already runs a pinned image")
+		return 0
+	}
+	fmt.Fprintln(stdout, "felis pin-images: pinning user servers to the build their tag names now:")
+	exit := 0
+	for _, o := range outcomes {
+		if o.err != nil {
+			fmt.Fprintf(stdout, "  - %s: ERROR %v\n", o.name, o.err)
+			exit = 1
+			continue
+		}
+		fmt.Fprintf(stdout, "  - %s: %s\n", o.name, strings.Join(o.changes, ", "))
+	}
+	return exit
+}
+
+// reportSystemPin runs pinSystemServerImage for `felis pin-images --system` and
+// prints what it did. Only a failed pin exits non-zero: the installer falls back
+// to restarting the pod on its tag then.
+func reportSystemPin(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver, stdout, stderr io.Writer) int {
+	o, err := pinSystemServerImage(ctx, cl, namespace, name, r)
+	switch {
+	case meta.IsNoMatchError(err):
+		fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
+		return 0
+	case err == nil && o.err != nil:
+		err = o.err
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis pin-images: %s: %v\n", name, err)
+		return 1
+	}
+	switch {
+	case o.updated:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s; the operator rolls it onto that build\n", name, strings.Join(o.changes, ", "))
+	default:
+		fmt.Fprintf(stdout, "felis pin-images: %s: %s\n", name, o.skipped)
+	}
+	return 0
+}
+
+// pinSystemServerImage fixes a system server to the build its image tag names now,
+// replacing the digest of an earlier build. The installer runs it after pushing a
+// rebuilt login or lobby image, and the operator rolls the StatefulSet onto the new
+// ref, so the build a system server runs is written in its spec and moves only when
+// a build did. An image outside the platform registry, or one naming no tag to
+// follow, is the admin's choice and is left alone; so is a server whose image an
+// admin retargets while this runs.
+func pinSystemServerImage(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver) (systemServerOutcome, error) {
+	var ms v1alpha1.MinecraftServer
+	if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &ms); err != nil {
+		if apierrors.IsNotFound(err) {
+			return systemServerOutcome{name: name, skipped: "not present yet; sudo felis setup creates it"}, nil
+		}
+		return systemServerOutcome{}, err
+	}
+	if ms.Labels[v1alpha1.LabelSystemRole] != name {
+		return systemServerOutcome{name: name, err: fmt.Errorf(
+			"MinecraftServer %s/%s is not marked as the Felis %q system server; left alone", namespace, name, name)}, nil
+	}
+	tagged := withoutDigest(ms.Spec.Image)
+	if !r.Covers(tagged) || !strings.Contains(tagged[strings.LastIndex(tagged, "/")+1:], ":") {
+		return systemServerOutcome{name: name, available: true, skipped: "runs " + ms.Spec.Image +
+			", which names no platform registry tag to follow; left alone"}, nil
+	}
+	pinned, err := r.Pin(ctx, tagged)
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("resolve %s: %w", tagged, err)}, nil
+	}
+	changed, err := patchOnConflictRetry(ctx, cl, &ms, func() bool {
+		if withoutDigest(ms.Spec.Image) != tagged || ms.Spec.Image == pinned {
+			return false
+		}
+		ms.Spec.Image = pinned
+		return true
+	})
+	if err != nil {
+		return systemServerOutcome{name: name, err: fmt.Errorf("patch %s: %w", name, err)}, nil
+	}
+	if !changed {
+		return systemServerOutcome{name: name, available: true, skipped: "already runs " + ms.Spec.Image}, nil
+	}
+	return systemServerOutcome{name: name, available: true, updated: true,
+		changes: []string{"spec.image pinned to " + pinned}}, nil
+}
+
+// withoutDigest drops the @sha256:… of a pinned ref, leaving the tag it came from.
+func withoutDigest(ref string) string {
+	if i := strings.Index(ref, "@"); i >= 0 {
+		return ref[:i]
+	}
+	return ref
+}
+
+// loopbackEndpoint is the registry's port on 127.0.0.1: the registry Deployment
+// binds it as a hostPort, and containerd's mirror and the installer's pushes use
+// the same address.
+func loopbackEndpoint(registry string) string {
+	if i := strings.LastIndex(registry, ":"); i >= 0 {
+		return "127.0.0.1" + registry[i:]
+	}
+	return "127.0.0.1"
+}
+
+// pinUserServerImages patches spec.image of every user server whose image the
+// resolver covers and is not yet pinned. System servers are the installer's to move:
+// it re-pins them with --system when it rolls them onto a new build
+// (restart_existing_system_servers).
+// A server that is already pinned, or runs an image from elsewhere, produces no
+// outcome, so a pinned fleet reports nothing. A running server restarts once as
+// the operator rolls its StatefulSet onto the pinned ref, which is the build it
+// already runs.
+func pinUserServerImages(ctx context.Context, cl client.Client, namespace string, r imagepin.Resolver) ([]systemServerOutcome, error) {
+	var list v1alpha1.MinecraftServerList
+	if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
+		return nil, fmt.Errorf("list servers: %w", err)
+	}
+	var out []systemServerOutcome
+	for i := range list.Items {
+		ms := &list.Items[i]
+		if ms.Labels[v1alpha1.LabelSystemRole] != "" || imagepin.Pinned(ms.Spec.Image) || !r.Covers(ms.Spec.Image) {
+			continue
+		}
+		pinned, err := r.Pin(ctx, ms.Spec.Image)
+		if errors.Is(err, imagepin.ErrNotFound) {
+			err = fmt.Errorf("%s is not in the registry, so there is no build to pin it to; left unpinned: %w", ms.Spec.Image, err)
+		}
+		if err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: err})
+			continue
+		}
+		patch := client.MergeFrom(ms.DeepCopy())
+		ms.Spec.Image = pinned
+		if err := cl.Patch(ctx, ms, patch); err != nil {
+			out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("patch %s: %w", ms.Name, err)})
+			continue
+		}
+		out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
+			changes: []string{"spec.image pinned to " + pinned}})
+	}
+	return out, nil
+}

cmd/felis/pinimages_test.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/imagepin"
+	"felis.lolicon.best/internal/naming"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+const pinTestDigest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
+
+// TestPinUserServerImages pins exactly the user servers still on a platform tag,
+// reports a tag the registry lost as an error without touching that server, and
+// has nothing left to do on a second pass.
+func TestPinUserServerImages(t *testing.T) {
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v2/felis/paper/manifests/demo" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Docker-Content-Digest", pinTestDigest)
+	}))
+	defer reg.Close()
+	res := imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
+
+	paper := defaultRegistryURL + "/felis/paper:demo"
+	mk := func(name, image, role string) *v1alpha1.MinecraftServer {
+		ms := &v1alpha1.MinecraftServer{}
+		ms.Name, ms.Namespace = name, "minecraft"
+		ms.Spec.Image = image
+		if role != "" {
+			ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
+		}
+		return ms
+	}
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
+		mk("legacy", paper, ""),
+		mk("pinned", paper+"@sha256:"+strings.Repeat("3", 64), ""),
+		mk("external", "docker.io/itzg/minecraft-server:java21", ""),
+		mk("gone", defaultRegistryURL+"/felis/paper:old", ""),
+		mk(naming.SystemLobbyServer, defaultRegistryURL+"/felis/felis-lobby:demo", naming.SystemLobbyServer),
+	).Build()
+	ctx := context.Background()
+
+	outcomes, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil {
+		t.Fatalf("pinUserServerImages: %v", err)
+	}
+	byName := map[string]systemServerOutcome{}
+	for _, o := range outcomes {
+		byName[o.name] = o
+	}
+	if len(outcomes) != 2 || byName["legacy"].err != nil || byName["gone"].err == nil {
+		t.Fatalf("outcomes = %+v, want legacy pinned and gone reported", outcomes)
+	}
+
+	want := map[string]string{
+		"legacy":                 paper + "@" + pinTestDigest,
+		"pinned":                 paper + "@sha256:" + strings.Repeat("3", 64),
+		"external":               "docker.io/itzg/minecraft-server:java21",
+		"gone":                   defaultRegistryURL + "/felis/paper:old",
+		naming.SystemLobbyServer: defaultRegistryURL + "/felis/felis-lobby:demo",
+	}
+	for name, image := range want {
+		var ms v1alpha1.MinecraftServer
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		if ms.Spec.Image != image {
+			t.Errorf("%s image = %q, want %q", name, ms.Spec.Image, image)
+		}
+	}
+
+	again, err := pinUserServerImages(ctx, cl, "minecraft", res)
+	if err != nil || len(again) != 1 || again[0].name != "gone" {
+		t.Fatalf("second pass = %+v, %v; want only the unresolvable server again", again, err)
+	}
+}
+
+// A fresh install has no CRD yet; the command must read that as nothing to pin.
+func TestPinUserServerImagesNoCRD(t *testing.T) {
+	cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
+		List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error {
+			return &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "felis.lolicon.best", Kind: "MinecraftServer"}}
+		},
+	}).Build()
+	_, err := pinUserServerImages(context.Background(), cl, "minecraft", imagepin.Resolver{Registry: defaultRegistryURL})
+	if !meta.IsNoMatchError(err) {
+		t.Fatalf("err = %v, want a NoMatch error the command can recognise", err)
+	}
+}
+
+func TestLoopbackEndpoint(t *testing.T) {
+	for in, want := range map[string]string{
+		"registry.felis.svc:5000": "127.0.0.1:5000",
+		"registry.example":        "127.0.0.1",
+	} {
+		if got := loopbackEndpoint(in); got != want {
+			t.Errorf("loopbackEndpoint(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

cmd/felis/reaper.go

@@ -16,10 +16,8 @@ import (
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/backup"
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/mail"
 	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/reaper"
-	"felis.lolicon.best/internal/store"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -33,11 +31,17 @@ import (
 // cadence, and RunOnce is idempotent and restart-safe, so a missed or retried
 // run simply converges. Only the tarLocal archive backend is wired in this
 // build; the snapshot backends (§19) are a later integration.
+//
+// --retention-only runs the archive-store half alone (reaper.RunRetention):
+// the CronJob an install without a worlds root gets, so backups past their
+// expiry still leave the store there. It builds no Kubernetes client and reads
+// no world.
 func cmdReaper(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("reaper", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
 	worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>, else the stock local-path <root>/<pv-name>_<ns>_<pvc-name>)")
+	retentionOnly := fs.Bool("retention-only", false, "only expire, read back and sweep the archive store: no server is evaluated and no world is read or deleted, so neither the worlds root nor the Kubernetes API is needed")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -56,14 +60,17 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 
 	ctx := ctrl.SetupSignalHandler()
 
+	var cl client.Client
+	if !*retentionOnly {
 		scheme := runtime.NewScheme()
 		utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 		utilruntime.Must(v1alpha1.AddToScheme(scheme))
-	cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
+		cl, err = client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
 		if err != nil {
 			fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
 			return 1
 		}
+	}
 
 	archiver, err := buildArchiver(ctx, cfg, *worldsRoot, cl)
 	if err != nil {
@@ -71,7 +78,7 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	drv, err := openStore(ctx, cfg.Database.URL, false)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
 		return 1
@@ -81,9 +88,13 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 	r := &reaper.Reaper{
 		Cfg:      rcfg,
 		Store:    reaper.NewPGStore(drv.DB()),
-		Cluster:  reaper.NewK8sCluster(cl, cfg.K8s.Namespace),
 		Archiver: archiver,
 	}
+	if *retentionOnly {
+		fmt.Fprintln(stderr, "felis reaper: retention only — no worlds root is configured, so idle worlds are neither archived nor released")
+		return reportReaperRun(r.RunRetention(ctx), stdout, stderr)
+	}
+	r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace)
 
 	// Pre-reap warnings go out by email when [smtp] is configured (the same
 	// relay and password_ref convention felis-api uses); without it the channel
@@ -114,13 +125,7 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 				}
 				return email, nil
 			},
-			notifier: &mail.SMTP{
+			notifier: smtpRelay(cfg.SMTP, password),
-				Host:     cfg.SMTP.Host,
-				Port:     cfg.SMTP.Port,
-				From:     cfg.SMTP.From,
-				Username: cfg.SMTP.Username,
-				Password: password,
-			},
 		}
 	} else {
 		fmt.Fprintln(stderr, "felis reaper: [smtp] not configured — pre-reap warnings are logged and NOT marked sent")
@@ -131,10 +136,34 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis reaper: %v\n", err)
 		return 1
 	}
-	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d warned=%d skipped=%d evicted=%d expired=%d\n",
+	return reportReaperRun(sum, stdout, stderr)
-		sum.Evaluated, sum.WorldsReaped, sum.Warned, sum.Skipped, sum.EvictedEarly, sum.BackupsExpired)
+}
+
+// reportReaperRun prints the run's tally and turns a run that left work undone
+// into exit 1, so the Job fails and the watchdog's job-failed check (and the
+// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
+// is kept, and without this nobody would learn that it is never reaped.
+func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
+	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d awaiting_stop=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
+		sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.AwaitingStop, sum.Warned, sum.Skipped, sum.StoreFull,
+		sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
+		sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
+	if !sum.Failed() {
 		return 0
 	}
+	if sum.Skipped > 0 || sum.ExpireFailed > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
+			sum.Skipped, sum.StoreFull, sum.ExpireFailed)
+	}
+	if sum.Corrupt > 0 {
+		fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)
+	}
+	if sum.VerifyFailed > 0 || sum.SweepFailed {
+		fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",
+			sum.VerifyFailed, !sum.SweepFailed)
+	}
+	return 1
+}
 
 // mailWarner delivers a pre-reap notice to the owner's verified email — the
 // only channel this build can reach. Unowned owners and owners who never proved
@@ -169,8 +198,9 @@ func (w *mailWarner) Warn(ctx context.Context, ownerID, server, remaining string
 }
 
 // reaperConfig derives the reaper's retention windows from felis.toml. The 15d
-// idle deadline is fixed by §18; only the warning offsets, retention, and the
+// idle deadline is fixed by §18; only the warning offsets, retention, the
-// store soft-cap are configurable (§24).
+// store soft-cap and the on-demand backup bounds are configurable (§24). The
+// backup Job and felis-api read the manual_* bounds through it too.
 func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 	rc := reaper.DefaultConfig()
 	if v := cfg.Archive.Retention; v != "" {
@@ -198,19 +228,46 @@ func reaperConfig(cfg *config.Config) (reaper.Config, error) {
 		}
 		rc.MaxLocalBytes = b
 	}
+	if v := cfg.Archive.ManualRetention; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d <= 0 {
+			return rc, fmt.Errorf("[archive] manual_retention %q: want a positive span such as 30d", v)
+		}
+		rc.ManualRetention = d
+	}
+	switch n := cfg.Archive.ManualKeep; {
+	case n < 0:
+		return rc, fmt.Errorf("[archive] manual_keep %d: want 1 or more", n)
+	case n > 0:
+		rc.ManualKeep = n
+	}
+	if v := cfg.Archive.ManualCooldown; v != "" {
+		d, err := parseSpanDuration(v)
+		if err != nil || d < 0 {
+			return rc, fmt.Errorf("[archive] manual_cooldown %q: want a span such as 10m (0s for none)", v)
+		}
+		rc.ManualCooldown = d
+	}
+	rc.RequireOffsite = cfg.Offsite.Enabled()
 	return rc, nil
 }
 
 // buildArchiver constructs the WorldArchiver. Only tarLocal is implemented in
 // this build; the resolver maps each world PVC to its directory under worldsRoot
-// (resolveWorldDir).
+// (resolveWorldDir). A nil cl is the retention-only run, which never archives a
+// world, so its archiver resolves none.
 func buildArchiver(ctx context.Context, cfg *config.Config, worldsRoot string, cl client.Client) (backup.WorldArchiver, error) {
 	switch cfg.Archive.Store {
 	case "tarLocal":
-		return &backup.TarLocal{
+		t := &backup.TarLocal{BackupRoot: cfg.Archive.LocalPath}
-			BackupRoot: cfg.Archive.LocalPath,
+		if cl != nil {
-			Resolve:    resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot),
+			t.Resolve = resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot)
-		}, nil
+		} else {
+			t.Resolve = func(pvc string) (string, error) {
+				return "", fmt.Errorf("resolve world PVC %s: this run has no worlds root (retention only)", pvc)
+			}
+		}
+		return t, nil
 	default:
 		return nil, fmt.Errorf("[archive] store %q is not implemented in this build (only tarLocal)", cfg.Archive.Store)
 	}
@@ -253,27 +310,9 @@ func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoo
 	}
 }
 
-// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
+// parseSpanDuration parses the human spans used in felis.toml's [archive] table
-// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
+// (config.ParseSpan).
-func parseSpanDuration(s string) (time.Duration, error) {
+func parseSpanDuration(s string) (time.Duration, error) { return config.ParseSpan(s) }
-	s = strings.TrimSpace(s)
-	switch {
-	case strings.HasSuffix(s, "mo"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
-		if err != nil {
-			return 0, err
-		}
-		return time.Duration(n) * 30 * 24 * time.Hour, nil
-	case strings.HasSuffix(s, "d"):
-		n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
-		if err != nil {
-			return 0, err
-		}
-		return time.Duration(n) * 24 * time.Hour, nil
-	default:
-		return time.ParseDuration(s)
-	}
-}
 
 // parseByteSize parses a Kubernetes-style quantity ("200Gi", "10G") into bytes.
 // An empty string means unlimited (0).

cmd/felis/reaper_test.go

@@ -1,22 +1,90 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/reaper"
 )
 
+// TestReportReaperRunFailsTheJob: a run that could not process a server, or
+// could not remove an expired backup, exits 1 so the Job shows as failed.
+func TestReportReaperRunFailsTheJob(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		sum  reaper.Summary
+		want int
+	}{
+		{"clean", reaper.Summary{Evaluated: 3, WorldsReaped: 1, AwaitingOffsite: 1}, 0},
+		{"waiting for a stop", reaper.Summary{Evaluated: 3, AwaitingStop: 1}, 0},
+		{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
+		{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
+		{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},
+		{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},
+		{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},
+		{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},
+		{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},
+	} {
+		var out, errb bytes.Buffer
+		if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {
+			t.Errorf("%s: exit %d, want %d", tc.name, got, tc.want)
+		}
+		if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") {
+			t.Errorf("%s: summary line = %q", tc.name, out.String())
+		}
+		if (tc.want == 1) != (errb.Len() > 0) {
+			t.Errorf("%s: stderr = %q", tc.name, errb.String())
+		}
+	}
+}
+
 // TestResolveWorldDir pins the two world layouts the reaper must find, and the
 // fail-closed miss. The stock local-path arm is derived from the live PVC's
 // volumeName — a name-based guess (glob) could tar a stale deleted PV's bytes and
 // then delete the current world, which is why it is read from the API instead.
+// TestReaperConfigManualKeys: the on-demand backup keys default to 30 days,
+// five per server and a ten-minute cooldown, accept overrides, and refuse
+// values that would keep nothing or throttle backwards.
+func TestReaperConfigManualKeys(t *testing.T) {
+	rc, err := reaperConfig(&config.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 30*reaper.Day || rc.ManualKeep != 5 || rc.ManualCooldown != 10*time.Minute {
+		t.Fatalf("defaults = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
+		ManualRetention: "7d", ManualKeep: 2, ManualCooldown: "0s"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rc.ManualRetention != 7*reaper.Day || rc.ManualKeep != 2 || rc.ManualCooldown != 0 {
+		t.Fatalf("overrides = %v / %d / %v", rc.ManualRetention, rc.ManualKeep, rc.ManualCooldown)
+	}
+	for _, bad := range []config.ArchiveConfig{
+		{ManualRetention: "0d"},
+		{ManualRetention: "soon"},
+		{ManualKeep: -1},
+		{ManualCooldown: "-5m"},
+		{ManualCooldown: "often"},
+	} {
+		if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
+			t.Errorf("%+v was accepted", bad)
+		}
+	}
+}
+
 func TestResolveWorldDir(t *testing.T) {
 	ctx := context.Background()
 	root := t.TempDir()

cmd/felis/registrygate.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/registrygate"
+)
+
+// cmdRegistryGate is the sidecar entrypoint in the registry pod: it owns the
+// registry port (and the loopback hostPort containerd pulls through), lets reads
+// through anonymously, and forwards writes to the loopback-only registry:2 only
+// for an authenticated principal allowed to write that repository. See
+// internal/registrygate for the policy.
+//
+// Tokens are files under --auth-dir, one per principal (platform, build, prune),
+// mounted from the registry-auth Secret. A missing file disables that principal:
+// writes fail closed while every pull keeps working, which is the right way round
+// for a registry the running workloads depend on.
+//
+// --maint-listen is the GC sidecar's read-only handshake (registrygate.MaintHandler).
+// It has no authentication, so it must name a loopback address; --maint-dir keeps
+// an open window across a gate restart.
+func cmdRegistryGate(args []string, _, stderr io.Writer) int {
+	fs := flag.NewFlagSet("registry-gate", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	listen := fs.String("listen", ":5000", "address the gate serves the registry API on")
+	upstream := fs.String("upstream", "http://127.0.0.1:5001", "the loopback registry the gate forwards to")
+	authDir := fs.String("auth-dir", "/etc/felis-registry-auth", "directory holding one token file per principal")
+	maintListen := fs.String("maint-listen", "", "loopback address for the GC sidecar's read-only handshake (empty disables it)")
+	maintDir := fs.String("maint-dir", "", "directory that keeps an open read-only window across a gate restart")
+	quiet := fs.Duration("maint-quiet", registrygate.DefaultQuiet, "how long writes must be idle before a read-only window is granted")
+	dataDir := fs.String("data-dir", "", "the registry's storage root, mounted read-only, for the manifest index (empty disables it)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *maintListen != "" && !loopbackAddr(*maintListen) {
+		fmt.Fprintf(stderr, "felis registry-gate: --maint-listen %q must be a loopback address: the handshake has no authentication\n", *maintListen)
+		return 2
+	}
+	target, err := url.Parse(*upstream)
+	if err != nil || target.Scheme == "" || target.Host == "" {
+		fmt.Fprintf(stderr, "felis registry-gate: bad --upstream %q\n", *upstream)
+		return 2
+	}
+	log := slog.New(slog.NewTextHandler(stderr, nil))
+	tokens := map[string]string{}
+	for _, p := range registrygate.Principals {
+		b, err := os.ReadFile(filepath.Join(*authDir, p))
+		tok := strings.TrimSpace(string(b))
+		if err != nil || tok == "" {
+			log.Warn("registry principal disabled: no token", "principal", p, "dir", *authDir)
+			continue
+		}
+		tokens[p] = tok
+	}
+
+	gate := registrygate.New(target, tokens, log)
+	gate.SetQuiet(*quiet)
+	gate.DataDir = *dataDir
+	if *maintDir != "" {
+		if err := gate.SetMaintenanceState(registrygate.MaintStatePath(*maintDir)); err != nil {
+			// A corrupt file must not keep the registry from serving pulls.
+			log.Warn("ignoring the saved read-only window", "err", err)
+		}
+	}
+	srv := &http.Server{
+		Addr:              *listen,
+		Handler:           gate,
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	var maint *http.Server
+	if *maintListen != "" {
+		maint = &http.Server{Addr: *maintListen, Handler: gate.MaintHandler(), ReadHeaderTimeout: 10 * time.Second}
+		go func() {
+			if err := maint.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				log.Error("maintenance listener stopped; garbage collection cannot get a read-only window", "err", err)
+			}
+		}()
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	go func() {
+		<-ctx.Done()
+		shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(shutdown)
+		if maint != nil {
+			_ = maint.Shutdown(shutdown)
+		}
+	}()
+	log.Info("registry gate listening", "addr", *listen, "upstream", target.String(), "principals", len(tokens))
+	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintf(stderr, "felis registry-gate: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+// loopbackAddr reports whether a host:port listen address binds loopback only.
+func loopbackAddr(addr string) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return false
+	}
+	if host == "localhost" {
+		return true
+	}
+	ip := net.ParseIP(host)
+	return ip != nil && ip.IsLoopback()
+}
+
+// cmdPushImage is the build Job's publish step. It runs after Kaniko built the
+// image into a tarball (--no-push) and Trivy passed that tarball, and it is the
+// only container of the build pod that holds the registry credential — the one
+// executing the untrusted Dockerfile never sees it.
+func cmdPushImage(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("push-image", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path")
+	ref := fs.String("ref", "", "host/repository:tag to publish it as")
+	scheme := fs.String("scheme", "http", "registry scheme: http for the in-cluster registry, https otherwise")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *tarPath == "" || *ref == "" {
+		fmt.Fprintln(stderr, "felis push-image: --tar and --ref are required")
+		return 2
+	}
+	if *scheme != "http" && *scheme != "https" {
+		fmt.Fprintf(stderr, "felis push-image: bad --scheme %q\n", *scheme)
+		return 2
+	}
+	user := os.Getenv("FELIS_REGISTRY_USERNAME")
+	pass := os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if user == "" || pass == "" {
+		fmt.Fprintln(stderr, "felis push-image: FELIS_REGISTRY_USERNAME/FELIS_REGISTRY_PASSWORD are empty — the registry refuses anonymous writes")
+		return 2
+	}
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	p := &imagepush.Pusher{Scheme: *scheme, Username: user, Password: pass, Log: stderr}
+	digest, err := p.Push(ctx, *tarPath, *ref)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis push-image: %v\n", err)
+		return 1
+	}
+	fmt.Fprintln(stdout, digest)
+	return 0
+}

cmd/felis/restore_test.go

@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup
 		BackupRoot: backupRoot,
 		Resolve:    func(string) (string, error) { return srcDir, nil },
 	}
-	got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0")
+	got, err := ar.Archive(context.Background(), "survival", "world-survival-0")
 	if err != nil {
 		t.Fatalf("Archive: %v", err)
 	}
-	return string(got), backupRoot
+	return string(got.Ref), backupRoot
 }
 
 // The restore subcommand must extract the archived world into the target world

cmd/felis/rotatetoken.go

@@ -0,0 +1,309 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/platform"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// rotate-token replaces one internal caller's token (naming.CallerTokens): a new
+// value goes into the installer's record, the control-namespace Secret and the
+// replica the caller's pods mount, felis-api rolls so it accepts only the new
+// value, and then the caller restarts so it presents it. Between the api's
+// rollout and the caller's restart the caller is turned away with 401; for the
+// login gate and the proxy that is the few seconds of a pod or unit restart.
+
+const (
+	defaultSecretsEnvPath = "/etc/felis/secrets.env"
+	defaultLinkPropsPath  = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
+	velocityUnit          = "felis-velocity"
+)
+
+// installerTokenKeys names each caller's token in the installer's secrets.env
+// (deploy/bootstrap.sh load_or_make_secrets). A re-run of the installer applies
+// these values to the Secrets, so a rotation that skipped the file would be
+// undone by the next upgrade.
+var installerTokenKeys = map[string]string{
+	"velocity": "SERVICE_TOKEN",
+	"limbo":    "LIMBO_TOKEN",
+	"build":    "BUILD_TOKEN",
+	"ops":      "OPS_TOKEN",
+}
+
+type tokenRotator struct {
+	cl          client.Client
+	controlNS   string
+	minecraftNS string
+	buildNS     string
+	// secretsEnv and linkProps are the installer's record and the proxy's
+	// felis-link.properties; a missing file is reported and skipped.
+	secretsEnv string
+	linkProps  string
+	newToken   func() (string, error)
+	// rollAPI restarts felis-api and waits for the rollout.
+	rollAPI func(ctx context.Context) error
+	// restartUnit restarts a systemd unit on this host.
+	restartUnit func(ctx context.Context, unit string) error
+	out         io.Writer
+}
+
+func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("rotate-token", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
+	secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
+	linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
+	fs.Usage = func() {
+		fmt.Fprintf(stderr, "Usage: felis rotate-token [flags] <%s>\n\n", strings.Join(callerNames(), "|"))
+		fmt.Fprintln(stderr, "Replaces one internal caller's token: the Secrets, felis-api, then the caller itself.")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if fs.NArg() != 1 {
+		fs.Usage()
+		return 2
+	}
+	if _, ok := callerToken(fs.Arg(0)); !ok {
+		fmt.Fprintf(stderr, "felis rotate-token: unknown caller %q (one of %s)\n", fs.Arg(0), strings.Join(callerNames(), ", "))
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+fs.Arg(0)+")")
+		return 1
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	cl, err := buildSystemServerClient()
+	if err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	buildNS := cfg.Registry.BuildNamespace
+	if buildNS == "" {
+		buildNS = platform.DefaultBuildNamespace
+	}
+	r := tokenRotator{
+		cl:          cl,
+		controlNS:   platform.DefaultControlNamespace,
+		minecraftNS: cfg.K8s.Namespace,
+		buildNS:     buildNS,
+		secretsEnv:  *secretsEnv,
+		linkProps:   *linkProps,
+		newToken:    randomToken,
+		rollAPI: func(ctx context.Context) error {
+			if err := kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "restart", "deployment/felis-api"); err != nil {
+				return err
+			}
+			return kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "status", "deployment/felis-api", "--timeout=180s")
+		},
+		restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
+		out:         stdout,
+	}
+	if err := r.rotate(context.Background(), fs.Arg(0)); err != nil {
+		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+func callerNames() []string {
+	names := make([]string, 0, len(naming.CallerTokens))
+	for _, ct := range naming.CallerTokens {
+		names = append(names, ct.Caller)
+	}
+	return names
+}
+
+func callerToken(name string) (naming.CallerToken, bool) {
+	for _, ct := range naming.CallerTokens {
+		if ct.Caller == name {
+			return ct, true
+		}
+	}
+	return naming.CallerToken{}, false
+}
+
+// randomToken is 32 random bytes in hex, the shape the installer generates.
+func randomToken() (string, error) {
+	b := make([]byte, 32)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(b), nil
+}
+
+func (r tokenRotator) rotate(ctx context.Context, caller string) error {
+	ct, ok := callerToken(caller)
+	if !ok {
+		return fmt.Errorf("unknown caller %q", caller)
+	}
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a token: %w", err)
+	}
+
+	// The installer's record first: from here on, whatever fails, a re-run of the
+	// installer puts the new value everywhere.
+	switch err := setKeyValueLine(r.secretsEnv, installerTokenKeys[ct.Caller], "=", tok); {
+	case errors.Is(err, fs.ErrNotExist):
+		fmt.Fprintf(r.out, "  - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
+	case err != nil:
+		return fmt.Errorf("record the new token in %s: %w", r.secretsEnv, err)
+	default:
+		fmt.Fprintf(r.out, "  - %s: %s updated\n", r.secretsEnv, installerTokenKeys[ct.Caller])
+	}
+
+	namespaces := []string{r.controlNS}
+	replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
+	if replica != "" && replica != r.controlNS {
+		namespaces = append(namespaces, replica)
+	}
+	for _, ns := range namespaces {
+		if err := writeTokenSecret(ctx, r.cl, ns, ct.Secret, tok); err != nil {
+			return fmt.Errorf("write Secret %s/%s: %w", ns, ct.Secret, err)
+		}
+		fmt.Fprintf(r.out, "  - Secret %s/%s: updated\n", ns, ct.Secret)
+	}
+
+	hostProxy := false
+	if ct.Caller == "velocity" {
+		switch err := setKeyValueLine(r.linkProps, "service-token", "=", tok); {
+		case errors.Is(err, fs.ErrNotExist):
+			fmt.Fprintf(r.out, "  - %s: not found; set service-token in your proxy's felis-link.properties to the value in Secret %s/%s and restart it\n",
+				r.linkProps, r.controlNS, ct.Secret)
+		case err != nil:
+			return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
+		default:
+			hostProxy = true
+			fmt.Fprintf(r.out, "  - %s: service-token updated\n", r.linkProps)
+		}
+	}
+
+	if err := r.rollAPI(ctx); err != nil {
+		return fmt.Errorf("roll felis-api: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - felis-api: rolled out, accepting only the new token")
+
+	switch ct.Caller {
+	case "velocity":
+		if hostProxy {
+			if err := r.restartUnit(ctx, velocityUnit); err != nil {
+				return fmt.Errorf("restart %s: %w", velocityUnit, err)
+			}
+			fmt.Fprintf(r.out, "  - %s: restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit)
+		}
+	case "limbo":
+		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS),
+			client.MatchingLabels{v1alpha1.LabelServer: naming.SystemLoginServer}); err != nil {
+			return fmt.Errorf("restart the login gate: %w", err)
+		}
+		fmt.Fprintln(r.out, "  - login gate: pod restarted to read the new token")
+	case "build":
+		fmt.Fprintln(r.out, "  - builds: the next build Job reads the new token; one fetching its context right now fails and can be submitted again")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now reads the new token on its next run")
+	}
+	return nil
+}
+
+// writeTokenSecret sets the token in a Secret, creating it when absent.
+func writeTokenSecret(ctx context.Context, cl client.Client, namespace, name, token string) error {
+	var sec corev1.Secret
+	err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &sec)
+	if apierrors.IsNotFound(err) {
+		return cl.Create(ctx, &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
+		})
+	}
+	if err != nil {
+		return err
+	}
+	if sec.Data == nil {
+		sec.Data = map[string][]byte{}
+	}
+	sec.Data[naming.ServiceTokenSecretKey] = []byte(token)
+	return cl.Update(ctx, &sec)
+}
+
+// setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file
+// (secrets.env, a .properties file), appending one when the key is absent. The
+// file is replaced atomically and keeps its mode and owner: felis-link.properties
+// is root:felis-velocity 0640, and the proxy must still be able to read it.
+func setKeyValueLine(path, key, sep, value string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
+	found := false
+	for i, ln := range lines {
+		k, _, ok := strings.Cut(ln, sep)
+		if ok && strings.TrimSpace(k) == key {
+			lines[i] = key + sep + value
+			found = true
+		}
+	}
+	if !found {
+		lines = append(lines, key+sep+value)
+	}
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	if err := tmp.Chmod(info.Mode().Perm()); err != nil {
+		tmp.Close()
+		return err
+	}
+	if st, ok := info.Sys().(*syscall.Stat_t); ok {
+		if err := tmp.Chown(int(st.Uid), int(st.Gid)); err != nil {
+			tmp.Close()
+			return err
+		}
+	}
+	if _, err := tmp.WriteString(strings.Join(lines, "\n") + "\n"); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
+}

cmd/felis/rotatetoken_test.go

@@ -0,0 +1,283 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+type rotationRig struct {
+	r      tokenRotator
+	cl     client.Client
+	out    *bytes.Buffer
+	events []string
+	dir    string
+}
+
+func tokenSecret(ns, name, val string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name},
+		Data:       map[string][]byte{"token": []byte(val)},
+	}
+}
+
+func serverPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{"felis.lolicon.best/server": server}}}
+}
+
+func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
+	t.Helper()
+	rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir()}
+	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
+	rig.r = tokenRotator{
+		cl:          rig.cl,
+		controlNS:   "felis",
+		minecraftNS: "minecraft",
+		buildNS:     "felis-build",
+		secretsEnv:  filepath.Join(rig.dir, "secrets.env"),
+		linkProps:   filepath.Join(rig.dir, "felis-link.properties"),
+		newToken:    func() (string, error) { return "NEWTOKEN", nil },
+		rollAPI: func(context.Context) error {
+			rig.events = append(rig.events, "roll-api")
+			return nil
+		},
+		restartUnit: func(_ context.Context, unit string) error {
+			rig.events = append(rig.events, "restart "+unit)
+			return nil
+		},
+		out: rig.out,
+	}
+	return rig
+}
+
+func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
+	t.Helper()
+	var s corev1.Secret
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+		return "<missing>"
+	}
+	return string(s.Data["token"])
+}
+
+func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(body), mode); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(path, mode); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRotateLimboToken(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-service-token", "proxy"),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600)
+
+	// felis-api must roll only after both copies hold the new value, and the login
+	// pod must still be there then: restarting it earlier would have it present
+	// the new token to an api that does not know it yet.
+	rig.r.rollAPI = func(context.Context) error {
+		rig.events = append(rig.events, "roll-api")
+		if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the control Secret held %q", got)
+		}
+		if got := rig.secret(t, "minecraft", "felis-limbo-token"); got != "NEWTOKEN" {
+			t.Errorf("api rolled while the minecraft replica held %q", got)
+		}
+		var pod corev1.Pod
+		if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+			t.Errorf("the login pod was restarted before the api rolled")
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), "limbo"); err != nil {
+		t.Fatal(err)
+	}
+
+	raw, _ := os.ReadFile(rig.r.secretsEnv)
+	if string(raw) != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
+		t.Errorf("secrets.env = %q", raw)
+	}
+	if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 {
+		t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm())
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" {
+		t.Errorf("the proxy's token changed to %q", got)
+	}
+	var pods corev1.PodList
+	if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
+		t.Fatal(err)
+	}
+	if len(pods.Items) != 1 || pods.Items[0].Name != "survival-0" {
+		t.Errorf("pods left = %v, want only survival-0 (the login pod restarted, user servers untouched)", pods.Items)
+	}
+	if strings.Join(rig.events, ",") != "roll-api" {
+		t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new token was printed: %s", rig.out.String())
+	}
+}
+
+func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n", 0o640)
+
+	if err := rig.r.rotate(context.Background(), "velocity"); err != nil {
+		t.Fatal(err)
+	}
+	raw, _ := os.ReadFile(rig.r.linkProps)
+	if string(raw) != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
+		t.Errorf("felis-link.properties = %q", raw)
+	}
+	if info, _ := os.Stat(rig.r.linkProps); info.Mode().Perm() != 0o640 {
+		t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm())
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	// The proxy's token has no replica: it must not appear in a workload namespace.
+	if got := rig.secret(t, "minecraft", "felis-service-token"); got != "<missing>" {
+		t.Errorf("rotation copied the proxy token into minecraft (%q)", got)
+	}
+	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+		t.Errorf("events = %v, want the api roll then the proxy restart", rig.events)
+	}
+}
+
+// An external proxy has no felis-link.properties here: the Secret still rotates,
+// nothing is restarted on this host, and the output says where the value is
+// without printing it.
+func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	if err := rig.r.rotate(context.Background(), "velocity"); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if strings.Join(rig.events, ",") != "roll-api" {
+		t.Errorf("events = %v, want no proxy restart", rig.events)
+	}
+	out := rig.out.String()
+	if !strings.Contains(out, "felis/felis-service-token") || strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("output should point at the Secret without the value: %s", out)
+	}
+}
+
+func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old"))
+	// An install from before per-caller tokens has no BUILD_TOKEN line yet.
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600)
+	if err := rig.r.rotate(context.Background(), "build"); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("build replica = %q, want NEWTOKEN (created when absent)", got)
+	}
+	if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	raw, _ := os.ReadFile(rig.r.secretsEnv)
+	if string(raw) != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", raw)
+	}
+}
+
+// A failed api rollout stops the rotation before the caller restarts: the login
+// gate keeps running on the old value the old api pods still accept.
+func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) {
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		serverPod("minecraft", "login-0", "login"),
+	)
+	rig.r.rollAPI = func(context.Context) error { return errors.New("rollout timed out") }
+	err := rig.r.rotate(context.Background(), "limbo")
+	if err == nil || !strings.Contains(err.Error(), "rollout timed out") {
+		t.Fatalf("err = %v, want the rollout failure", err)
+	}
+	var pod corev1.Pod
+	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
+		t.Error("the login pod was restarted although the api never rolled")
+	}
+}
+
+func TestRotateRefusesAnUnknownCaller(t *testing.T) {
+	rig := newRotationRig(t)
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600)
+	if err := rig.r.rotate(context.Background(), "admin"); err == nil {
+		t.Fatal("rotated a token for an unknown caller")
+	}
+	if raw, _ := os.ReadFile(rig.r.secretsEnv); string(raw) != "SERVICE_TOKEN=proxy\n" {
+		t.Errorf("secrets.env = %q, want it untouched", raw)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want nothing touched", rig.events)
+	}
+}
+
+// The installer and rotate-token must agree on where each caller's token lives:
+// rotate-token writes installerTokenKeys into secrets.env, and the installer
+// applies those same keys to the Secrets on its next run. A key the installer
+// does not read would be silently reverted by the next upgrade.
+func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	for caller, key := range installerTokenKeys {
+		ct, ok := callerToken(caller)
+		if !ok {
+			t.Fatalf("installerTokenKeys names unknown caller %q", caller)
+		}
+		if !strings.Contains(script, key+`="${`+key+`:-$(openssl rand -hex 32)}"`) {
+			t.Errorf("bootstrap.sh does not generate %s", key)
+		}
+		if !strings.Contains(script, key+"=${"+key+"}\n") {
+			t.Errorf("bootstrap.sh does not persist %s to secrets.env", key)
+		}
+		apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`)
+		if !apply.MatchString(script) {
+			t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key)
+		}
+	}
+	// The replicas the installer applies straight into the workload namespaces, so an
+	// upgrade has them before the new operator and build Jobs reference them.
+	for _, line := range []string{
+		`apply_literal_secret "$MINECRAFT_NS" felis-limbo-token token "$LIMBO_TOKEN"`,
+		`apply_literal_secret "$BUILD_NS" felis-build-token token "$BUILD_TOKEN"`,
+		`kube -n "$MINECRAFT_NS" delete secret felis-service-token --ignore-not-found`,
+		`kube -n "$BUILD_NS" delete secret felis-service-token --ignore-not-found`,
+	} {
+		if !strings.Contains(script, line) {
+			t.Errorf("bootstrap.sh lacks %s", line)
+		}
+	}
+	for _, ns := range []string{"MINECRAFT_NS", "BUILD_NS"} {
+		if strings.Contains(script, `apply_literal_secret "$`+ns+`" felis-service-token`) {
+			t.Errorf("bootstrap.sh still copies the proxy's token into %s", ns)
+		}
+	}
+	if len(installerTokenKeys) != len(callerNames()) {
+		t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames()))
+	}
+}

cmd/felis/run.go

@@ -11,7 +11,9 @@ Usage:
   felis <command> [flags]
 
 Commands:
-  migrate up        Apply embedded database migrations under an advisory lock
+  migrate up        Apply embedded database migrations under an advisory lock (snapshots the database first)
+  db                Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check)
+  offsite           Copy world archives, database bundles, user images and uploads to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|fetch-images|fetch-uploads|keygen)
   operator          Run the MinecraftServer controller-manager
   api               Run the felis-api HTTP server
   nano              Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)
@@ -19,11 +21,18 @@ Commands:
   restore           Extract a world archive into a world volume (internal Job entrypoint)
   backup            Archive a world into the backup store and record it (internal Job entrypoint)
   files             List/read/write one file in a stopped server's world (internal Job entrypoint)
+  egress-gate       Hold a build pod until its egress NetworkPolicy is enforced (internal Job entrypoint)
   fetch-context     Fetch and extract a submission's build context (internal Job entrypoint)
+  scan-gate         Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
+  push-image        Push a scanned image tarball to the registry (internal Job entrypoint)
+  mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
+  registry-gate     Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
   manifests         Render the control-plane RBAC + NetworkPolicy install bundle as YAML
   apply             Create a MinecraftServer CRD (direct K8s write; use -f server.json)
   setup             Run host bootstrap + first-run setup console (TUI; requires root/sudo)
   converge          Fill in fields a newer desired spec added to already-installed system servers
+  rotate-token      Replace one internal caller's token and restart what holds it (velocity|limbo|build|ops; requires root/sudo)
+  watchdog          Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
   version           Print the build stamp of this binary
   update            Report which platform components have updates available
   breakGlass        Open the local break-glass emergency console (TUI; requires root/sudo)
@@ -42,6 +51,8 @@ Run "felis <command> -h" for command-specific flags.
 // subcommand, and listing them would make the table disagree with the command list.
 var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
 	"migrate":            cmdMigrate,
+	"db":                 cmdDB,
+	"offsite":            cmdOffsite,
 	"operator":           cmdOperator,
 	"api":                cmdAPI,
 	"nano":               cmdNano,
@@ -49,16 +60,25 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
 	"restore":            cmdRestore,
 	"backup":             cmdBackup,
 	"files":              cmdFiles,
+	"egress-gate":        cmdEgressGate,
 	"fetch-context":      cmdFetchContext,
+	"scan-gate":          cmdScanGate,
+	"push-image":         cmdPushImage,
+	"mirror-build-tools": cmdMirrorBuildTools,
+	"registry-gate":      cmdRegistryGate,
 	"manifests":          cmdManifests,
 	"apply":              cmdApply,
 	"setup":              cmdSetup,
 	"converge":           cmdConverge,
+	"rotate-token":       cmdRotateToken,
 	"breakGlass":         cmdBreakGlass,
 	"bootstrap-assets":   cmdBootstrapAssets,
 	"init-forwarding":    cmdInitForwarding,
+	"init-volume":        cmdInitVolume,
+	"pin-images":         cmdPinImages,
 	"version":            cmdVersion,
 	"update":             cmdUpdate,
+	"watchdog":           cmdWatchdog,
 }
 
 // run dispatches a subcommand. It is separate from main so the router is

cmd/felis/run_test.go

@@ -38,9 +38,13 @@ func TestRunUnknownCommand(t *testing.T) {
 }
 
 // undocumentedCommands are routable on purpose but kept out of the usage text: they
-// are called by deploy/bootstrap.sh, not by a human at a prompt. Listing them here is
+// are called by deploy/bootstrap.sh or the operator's initContainers, not by a human
-// what makes their absence from usage a deliberate decision rather than an oversight.
+// at a prompt. Listing them here is what makes their absence from usage a deliberate
-var undocumentedCommands = map[string]bool{"bootstrap-assets": true, "init-forwarding": true}
+// decision rather than an oversight.
+var undocumentedCommands = map[string]bool{
+	"bootstrap-assets": true, "init-forwarding": true, "init-volume": true,
+	"pin-images": true,
+}
 
 // The usage text and the dispatch table must describe the same set of commands.
 //

cmd/felis/scangate.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"strings"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// maxScanDocument bounds each document scan-gate reads: a modpack report lists a
+// few thousand packages, far below this. Tests shrink it.
+var maxScanDocument int64 = 64 << 20
+
+// cmdScanGate is the build pod's verdict step, after trivy wrote its full JSON
+// report and trivy convert wrote the CycloneDX SBOM. It applies the scan policy
+// to the report, prints the verdict and every blocking finding, then appends the
+// verdict, the report and the SBOM to its log as the envelope felis-api keeps on
+// the build (build.WriteScanEnvelope). It exits 1 when a finding blocks, which
+// fails the pod before the push step runs, and 2 when the report cannot be read,
+// so a scan that produced nothing usable never admits an image.
+func cmdScanGate(args []string, stdout, stderr io.Writer) int {
+	fset := flag.NewFlagSet("scan-gate", flag.ContinueOnError)
+	fset.SetOutput(stderr)
+	reportPath := fset.String("report", "", "trivy JSON report (required)")
+	sbomPath := fset.String("sbom", "", "CycloneDX SBOM to keep with the report")
+	failOn := fset.String("fail-on", strings.Join(build.DefaultScanFailOn, ","), "comma-separated severities that block the image")
+	failUnfixed := fset.Bool("fail-unfixed", false, "block on vulnerabilities that have no fixed release too")
+	accept := fset.String("accept", "", "comma-separated vulnerability ids and secret rule ids that never block")
+	termLog := fset.String("termination-log", "/dev/termination-log", "where the one-line verdict goes for the pod status")
+	if err := fset.Parse(args); err != nil {
+		return 2
+	}
+	// A stray argument is a policy the gate would otherwise drop without a word
+	// (a comma split out of --fail-on, say).
+	if fset.NArg() > 0 {
+		fmt.Fprintf(stderr, "felis scan-gate: unexpected argument %q\n", fset.Arg(0))
+		return 2
+	}
+	sevs, err := build.ParseSeverities(*failOn)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --fail-on: %v\n", err)
+		return 2
+	}
+	accepted, err := build.ParseScanAccept(*accept)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis scan-gate: --accept: %v\n", err)
+		return 2
+	}
+	if *reportPath == "" {
+		fmt.Fprintln(stderr, "felis scan-gate: --report is required")
+		return 2
+	}
+	fail := func(msg string) int {
+		fmt.Fprintln(stderr, "felis scan-gate: "+msg)
+		writeTerminationLog(*termLog, msg)
+		return 2
+	}
+	report, err := readScanDocument(*reportPath)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	policy := build.ScanPolicy{FailOn: sevs, FailUnfixed: *failUnfixed, Accept: accepted}
+	summary, err := build.Summarize(report, policy)
+	if err != nil {
+		return fail("the scan report is unreadable: " + err.Error())
+	}
+	env := build.ScanEnvelope{Summary: summary, Report: report}
+	if *sbomPath != "" {
+		switch sbom, err := readScanDocument(*sbomPath); {
+		case err == nil:
+			env.SBOM = sbom
+		case errors.Is(err, fs.ErrNotExist):
+			fmt.Fprintf(stdout, "felis scan-gate: no SBOM at %s; keeping the report alone\n", *sbomPath)
+		default:
+			return fail("the SBOM is unreadable: " + err.Error())
+		}
+	}
+
+	printScanVerdict(stdout, summary)
+	written, err := build.WriteScanEnvelope(stdout, env)
+	if err != nil {
+		return fail("could not write the scan envelope: " + err.Error())
+	}
+	for _, doc := range written.Summary.Omitted {
+		fmt.Fprintf(stderr, "felis scan-gate: the %s is too large to keep with the build and was left out\n", doc)
+	}
+	if summary.Blocked {
+		writeTerminationLog(*termLog, summary.Reason())
+		return 1
+	}
+	writeTerminationLog(*termLog, "the scan passed")
+	return 0
+}
+
+// printScanVerdict writes the human half of scan-gate's log.
+func printScanVerdict(w io.Writer, s build.ScanSummary) {
+	var counts []string
+	for _, sev := range build.Severities {
+		counts = append(counts, fmt.Sprintf("%s %d", sev, s.Counts[sev]))
+	}
+	fmt.Fprintf(w, "felis scan-gate: %d packages; findings: %s\n", s.Packages, strings.Join(counts, ", "))
+	unfixed := "vulnerabilities with no fixed release do not block"
+	if s.Policy.FailUnfixed {
+		unfixed = "vulnerabilities with no fixed release block too"
+	}
+	fmt.Fprintf(w, "felis scan-gate: blocking on %s (%s)\n", strings.Join(s.Policy.FailOn, ", "), unfixed)
+	if len(s.Policy.Accept) > 0 {
+		matched := 0
+		for _, f := range s.Findings {
+			if f.Accepted {
+				matched++
+			}
+		}
+		fmt.Fprintf(w, "felis scan-gate: accepted ids, never blocking: %s; listed findings under them: %d\n", strings.Join(s.Policy.Accept, ", "), matched)
+	}
+	if !s.Blocked {
+		fmt.Fprintln(w, "felis scan-gate: nothing blocks this image")
+		return
+	}
+	fmt.Fprintln(w, "felis scan-gate: "+build.Printable(s.Reason()))
+	for _, f := range s.Findings {
+		if !f.Blocking {
+			break
+		}
+		fix := f.Fixed
+		if fix == "" {
+			fix = "no fix"
+		}
+		if f.Kind == build.FindingSecret {
+			fmt.Fprintf(w, "  %s %s secret in %s: %s\n", build.Printable(f.ID), f.Severity, build.Printable(f.Target), build.Printable(f.Title))
+			continue
+		}
+		fmt.Fprintf(w, "  %s %s %s %s -> %s (%s)\n", build.Printable(f.ID), f.Severity,
+			build.Printable(f.Package), build.Printable(f.Installed), build.Printable(fix), build.Printable(f.Target))
+	}
+}
+
+func readScanDocument(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	b, err := io.ReadAll(io.LimitReader(f, maxScanDocument+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(b)) > maxScanDocument {
+		return nil, fmt.Errorf("%s exceeds %d bytes", path, maxScanDocument)
+	}
+	return b, nil
+}
+
+// writeTerminationLog leaves msg where the kubelet copies it into the container
+// status. It is best effort: the log carries the same verdict.
+func writeTerminationLog(path, msg string) {
+	if path == "" {
+		return
+	}
+	_ = os.WriteFile(path, []byte(build.Printable(msg)), 0o644)
+}

cmd/felis/scangate_test.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/build"
+)
+
+// scanReport has one fixed CRITICAL, one unfixed HIGH and one fixed MEDIUM; the
+// CRITICAL's package name carries a line break and a forged envelope frame.
+const scanReport = `{"SchemaVersion":2,"Results":[{"Target":"data/mods/core.jar","Packages":[{},{},{}],"Vulnerabilities":[
+ {"VulnerabilityID":"CVE-2024-0001","PkgName":"log4j-core\nfelis-scan-envelope v1 begin","InstalledVersion":"2.14.1","FixedVersion":"2.17.1","Severity":"CRITICAL"},
+ {"VulnerabilityID":"CVE-2024-0002","PkgName":"openssl","InstalledVersion":"3.0.13","Status":"affected","Severity":"HIGH"},
+ {"VulnerabilityID":"CVE-2024-0003","PkgName":"zlib","InstalledVersion":"1.3","FixedVersion":"1.3.1","Severity":"MEDIUM"}]}]}`
+
+type scanGateRun struct {
+	code           int
+	stdout, stderr string
+	termLog        string
+	env            *build.ScanEnvelope
+}
+
+func runScanGate(t *testing.T, report, sbom string, extra ...string) scanGateRun {
+	t.Helper()
+	dir := t.TempDir()
+	reportPath := filepath.Join(dir, "trivy.json")
+	if report != "" {
+		if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	sbomPath := filepath.Join(dir, "sbom.cdx.json")
+	if sbom != "" {
+		if err := os.WriteFile(sbomPath, []byte(sbom), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	termPath := filepath.Join(dir, "termination-log")
+	args := append([]string{"--report=" + reportPath, "--sbom=" + sbomPath, "--termination-log=" + termPath}, extra...)
+	var stdout, stderr bytes.Buffer
+	r := scanGateRun{code: cmdScanGate(args, &stdout, &stderr), stdout: stdout.String(), stderr: stderr.String()}
+	if b, err := os.ReadFile(termPath); err == nil {
+		r.termLog = string(b)
+	}
+	if env, err := build.ReadScanEnvelope(strings.NewReader(r.stdout)); err == nil {
+		r.env = env
+	}
+	return r
+}
+
+func TestScanGateBlocksAndHandsOverTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`)
+	if r.code != 1 {
+		t.Fatalf("exit %d, want 1; stderr %s", r.code, r.stderr)
+	}
+	if r.termLog != "the scan blocked the image: 1 CRITICAL (CVE-2024-0001)" {
+		t.Errorf("termination log = %q", r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: 3 packages; findings: CRITICAL 1, HIGH 1, MEDIUM 1, LOW 0, UNKNOWN 0\n",
+		"felis scan-gate: blocking on CRITICAL (vulnerabilities with no fixed release do not block)\n",
+		"felis scan-gate: the scan blocked the image: 1 CRITICAL (CVE-2024-0001)\n",
+		"  CVE-2024-0001 CRITICAL log4j-core?felis-scan-envelope v1 begin 2.14.1 -> 2.17.1 (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	if strings.Contains(r.stdout, "  CVE-2024-0002") {
+		t.Errorf("an unfixed HIGH was listed as blocking:\n%s", r.stdout)
+	}
+	if strings.Count(r.stdout, "\nfelis-scan-envelope v1 begin\n") != 1 {
+		t.Errorf("stdout must hold exactly one frame start on a line of its own:\n%s", r.stdout)
+	}
+	if r.env == nil {
+		t.Fatal("no envelope in stdout")
+	}
+	if !r.env.Summary.Blocked || string(r.env.SBOM) != `{"bomFormat":"CycloneDX"}` || !strings.Contains(string(r.env.Report), "CVE-2024-0003") {
+		t.Errorf("envelope = blocked %t, sbom %s, report %d bytes", r.env.Summary.Blocked, r.env.SBOM, len(r.env.Report))
+	}
+}
+
+func TestScanGatePolicyFlags(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=high", "--fail-unfixed")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 HIGH (CVE-2024-0002)" {
+		t.Errorf("HIGH + unfixed: exit %d, termination log %q", r.code, r.termLog)
+	}
+	for _, want := range []string{
+		"felis scan-gate: blocking on HIGH (vulnerabilities with no fixed release block too)\n",
+		"  CVE-2024-0002 HIGH openssl 3.0.13 -> no fix (data/mods/core.jar)\n",
+	} {
+		if !strings.Contains(r.stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
+		}
+	}
+	r = runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`, "--fail-on=LOW")
+	if r.code != 0 || r.termLog != "the scan passed" || !strings.Contains(r.stdout, "felis scan-gate: nothing blocks this image\n") {
+		t.Errorf("LOW only: exit %d, termination log %q, stdout:\n%s", r.code, r.termLog, r.stdout)
+	}
+	if r.env == nil || r.env.Summary.Blocked || r.env.SBOM == nil {
+		t.Errorf("a passing scan must still hand over its report and SBOM: %+v", r.env)
+	}
+}
+
+func TestScanGateAcceptedIDsNeverBlock(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=CRITICAL,MEDIUM", "--accept=CVE-2024-0001, CVE-2024-0002,CVE-2099-0001")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 MEDIUM (CVE-2024-0003)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "felis scan-gate: accepted ids, never blocking: CVE-2024-0001, CVE-2024-0002, CVE-2099-0001; listed findings under them: 2\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+	if r.env == nil || strings.Join(r.env.Summary.Policy.Accept, ",") != "CVE-2024-0001,CVE-2024-0002,CVE-2099-0001" {
+		t.Fatalf("envelope = %+v", r.env)
+	}
+	for _, f := range r.env.Summary.Findings {
+		if f.ID == "CVE-2024-0001" && (f.Blocking || !f.Accepted) {
+			t.Errorf("accepted finding = %+v", f)
+		}
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001")
+	if r.code != 0 || r.termLog != "the scan passed" {
+		t.Errorf("only an accepted CRITICAL: exit %d, termination log %q", r.code, r.termLog)
+	}
+}
+
+func TestScanGateWithoutAnSBOMKeepsTheReport(t *testing.T) {
+	r := runScanGate(t, scanReport, "", "--fail-on=LOW")
+	if r.code != 0 || !strings.Contains(r.stdout, "felis scan-gate: no SBOM at ") {
+		t.Errorf("exit %d, stdout:\n%s", r.code, r.stdout)
+	}
+	if r.env == nil || r.env.SBOM != nil || r.env.Report == nil {
+		t.Errorf("envelope = %+v", r.env)
+	}
+}
+
+func TestScanGateListsABlockingSecret(t *testing.T) {
+	r := runScanGate(t, `{"SchemaVersion":2,"Results":[{"Target":"config/keys.txt","Secrets":[
+ {"RuleID":"aws-access-key-id","Severity":"CRITICAL","Title":"AWS Access\nKey ID"}]}]}`, "")
+	if r.code != 1 || r.termLog != "the scan blocked the image: 1 CRITICAL (aws-access-key-id)" {
+		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
+	}
+	if !strings.Contains(r.stdout, "  aws-access-key-id CRITICAL secret in config/keys.txt: AWS Access?Key ID\n") {
+		t.Errorf("stdout:\n%s", r.stdout)
+	}
+}
+
+func TestScanGateFailsClosed(t *testing.T) {
+	r := runScanGate(t, "", "")
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the scan report is unreadable: open ") || r.env != nil {
+		t.Errorf("missing report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	r = runScanGate(t, `{"SchemaVersion":1}`, "")
+	if r.code != 2 || r.termLog != "the scan report is unreadable: read trivy report: schema version 1, want 2" {
+		t.Errorf("old schema: exit %d, termination log %q", r.code, r.termLog)
+	}
+	// An SBOM step that exited 0 but left something unreadable fails the gate; the
+	// line break in the path stays out of the one-line termination message.
+	sbomDir := filepath.Join(t.TempDir(), "sb\nom")
+	if err := os.Mkdir(sbomDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	r = runScanGate(t, scanReport, "", "--sbom="+sbomDir)
+	if r.code != 2 || !strings.HasPrefix(r.termLog, "the SBOM is unreadable: read ") ||
+		!strings.HasSuffix(r.termLog, "/sb?om: is a directory") || r.env != nil {
+		t.Errorf("unreadable SBOM: exit %d, termination log %q", r.code, r.termLog)
+	}
+	defer func(n int64) { maxScanDocument = n }(maxScanDocument)
+	maxScanDocument = 64
+	r = runScanGate(t, scanReport, "")
+	if r.code != 2 || !strings.HasSuffix(r.termLog, "/trivy.json exceeds 64 bytes") || r.env != nil {
+		t.Errorf("oversized report: exit %d, termination log %q", r.code, r.termLog)
+	}
+	maxScanDocument = 64 << 20
+	r = runScanGate(t, scanReport, "", "--fail-on=SEVERE")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --fail-on: unknown severity "SEVERE"`) {
+		t.Errorf("bad severity: exit %d, stderr %q", r.code, r.stderr)
+	}
+	// A severity list split at its comma leaves a stray argument, not a
+	// narrower policy.
+	r = runScanGate(t, scanReport, "", "--fail-on=LOW", "CRITICAL")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: unexpected argument "CRITICAL"`) || r.env != nil {
+		t.Errorf("stray argument: exit %d, stderr %q", r.code, r.stderr)
+	}
+	r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001 CVE-2024-0003")
+	if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --accept: "CVE-2024-0001 CVE-2024-0003" is not a vulnerability id or secret rule id`) {
+		t.Errorf("bad accept list: exit %d, stderr %q", r.code, r.stderr)
+	}
+	var stdout, stderr bytes.Buffer
+	if code := cmdScanGate(nil, &stdout, &stderr); code != 2 || !strings.Contains(stderr.String(), "--report is required") {
+		t.Errorf("no report flag: exit %d, stderr %q", code, stderr.String())
+	}
+}

cmd/felis/setup.go

@@ -13,7 +13,6 @@ import (
 	"felis.lolicon.best/internal/api"
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/store"
 )
@@ -216,18 +215,18 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 			"Re-run `sudo felis setup` on the control-plane host once the cluster is reachable", err)
 	}
 	// The login limbo authenticates to the felis-api INTERNAL face, so it needs the
-	// internal base URL, the root domain (to link players at the console), and the
+	// internal base URL, the root domain (to link players at the console), and its
-	// service token. The first two are plain env baked into the pod here; the token
+	// own token (felis-limbo-token). The first two are plain env baked into the pod
-	// is a Secret the operator injects by reference — but a secretKeyRef is
+	// here; the token is a Secret the operator injects by reference — but a
-	// namespace-local, so first replicate the token Secret from the control namespace
+	// secretKeyRef is namespace-local, so first replicate the token Secret from the
-	// into the minecraft namespace where the login pod runs. The control namespace is
+	// control namespace into the minecraft namespace where the login pod runs. The control namespace is
 	// the platform default (there is no felis.toml override for it); a deployment that
 	// renamed it must replicate the Secret by hand.
 	controlNS := platform.DefaultControlNamespace
 	apiBaseURL := platform.InternalAPIBaseURL(controlNS)
 	// These Secrets must land in the minecraft namespace before the pods that
-	// mount them are created: the service token (login authenticates to felis-api
+	// mount them are created: the login gate's token (login authenticates to
-	// with it), the Velocity forwarding secret (every backend verifies the proxy's
+	// felis-api with it), the Velocity forwarding secret (every backend verifies the proxy's
 	// signed handshake with it — without it the login gate would derive an OFFLINE
 	// UUID and the Owner would bind the wrong Minecraft identity), and felis-config
 	// (the on-demand BACKUP Job runs in the minecraft namespace and mounts it to
@@ -239,31 +238,7 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
 	if buildNS == "" {
 		buildNS = platform.DefaultBuildNamespace
 	}
-	secretOutcomes := []systemServerOutcome{
+	secretOutcomes := provisionSecretReplicas(ctx, cl, controlNS, cfg.K8s.Namespace, buildNS)
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false),
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
-		// refresh=true: felis-config is the rendered config, not a credential. The
-		// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
-		// must update it when the control plane's render has moved on (a stale copy
-		// e.g. keeps an old database URL after a credential rotation).
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			"felis-config", "felis.toml", "config", "minecraft ns", true),
-		// The reaper's pre-reap warning emails authenticate with the same relay
-		// password felis-api uses; the reaper pod runs in the minecraft namespace,
-		// where a secretKeyRef resolves only against a local mirror. Skipped while
-		// the relay is not configured yet — the "configure email" screen refreshes
-		// both mirrors when it applies.
-		ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
-			"felis-smtp", "password", "smtp", "minecraft ns", false),
-		// The build namespace needs the same token: the build Job's fetch
-		// initContainer reads the submission context from the internal face. Best
-		// effort — a deployment that only installs the control plane simply never
-		// builds a user submission.
-		ensureSecretReplica(ctx, cl, controlNS, buildNS,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "felis-build ns", false),
-	}
 	outcomes := ensureSystemServers(ctx, cl, cfg.K8s.Namespace, cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, apiBaseURL, cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
 	outcomes = append(secretOutcomes, outcomes...)
 	fmt.Fprintln(out, "\nfelis setup: login/lobby system servers (always-on, reaper-exempt):")
@@ -345,7 +320,9 @@ func openConfiguredSetup(ctx context.Context, cfgPath string) (*configuredSetup,
 	if err != nil {
 		return nil, &setupOpenError{stage: "load config", err: err}
 	}
-	drv, err := store.Open(ctx, cfg.Database.URL)
+	// Pending migrations are the preflight's to apply; a newer schema is a rolled-back
+	// binary, and nothing this console writes would match it.
+	drv, err := openStore(ctx, cfg.Database.URL, true)
 	if err != nil {
 		return nil, &setupOpenError{stage: "open database", err: err}
 	}

cmd/felis/systemservers.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -392,7 +393,7 @@ var derivedSystemEnv = map[string]bool{
 // operator every run.
 func refreshDerivedEnv(ctx context.Context, cl client.Client, existing, desired *v1alpha1.MinecraftServer) (bool, error) {
 	want := derivedEnvWanted(desired)
-
+	changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
 		changed := false
 		for i, e := range existing.Spec.Env {
 			if v, ok := want[e.Name]; ok && v != e.Value {
@@ -400,13 +401,40 @@ func refreshDerivedEnv(ctx context.Context, cl client.Client, existing, desired
 				changed = true
 			}
 		}
-	if !changed {
+		return changed
-		return false, nil
+	})
-	}
+	if err != nil {
-	if err := cl.Update(ctx, existing); err != nil {
 		return false, fmt.Errorf("refresh %s env: %w", existing.Name, err)
 	}
-	return true, nil
+	return changed, nil
+}
+
+// patchOnConflictRetry applies mutate to obj and sends only the difference, as a
+// merge patch that carries the resourceVersion obj was read at. The operator writes
+// status and felis-api patches spec.idle on these same objects, so a write can land
+// between setup's read and its patch: the apiserver then answers 409, and this
+// re-reads obj and runs mutate again on the fresh copy, up to retry.DefaultRetry's
+// five attempts. The pinned resourceVersion is what keeps a list field such as
+// spec.env safe — a merge patch replaces a list whole, and without the lock an
+// entry added concurrently would be dropped. mutate reports whether it changed
+// anything; nothing is sent when it did not. obj holds the stored object after.
+func patchOnConflictRetry(ctx context.Context, cl client.Client, obj client.Object, mutate func() bool) (bool, error) {
+	key := client.ObjectKeyFromObject(obj)
+	changed, reread := false, false
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		if reread {
+			if err := cl.Get(ctx, key, obj); err != nil {
+				return err
+			}
+		}
+		reread = true
+		base := obj.DeepCopyObject().(client.Object)
+		if changed = mutate(); !changed {
+			return nil
+		}
+		return cl.Patch(ctx, obj, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}))
+	})
+	return changed, err
 }
 
 // derivedEnvWanted maps the derived env keys of desired onto their values.
@@ -469,6 +497,8 @@ func convergeSystemServers(ctx context.Context, cl client.Client, namespace, log
 		}
 
 		var changes []string
+		changed, err := patchOnConflictRetry(ctx, cl, &existing, func() bool {
+			changes = nil
 			if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
 				existing.Spec.Rcon = desired.Spec.Rcon
 				changes = append(changes, "spec.rcon")
@@ -478,13 +508,14 @@ func convergeSystemServers(ctx context.Context, cl client.Client, namespace, log
 				changes = append(changes, "spec.startup.healthHTTPPort")
 			}
 			changes = append(changes, convergeDerivedEnv(&existing, desired)...)
-
+			return len(changes) > 0
-		if len(changes) == 0 {
+		})
-			outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
+		if err != nil {
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
 			continue
 		}
-		if err := cl.Update(ctx, &existing); err != nil {
+		if !changed {
-			outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
+			outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
 			continue
 		}
 		outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, updated: true, changes: changes})
@@ -588,15 +619,51 @@ func phaseOrPending(p v1alpha1.Phase) string {
 	return string(p)
 }
 
+// provisionSecretReplicas copies the Secrets workload pods mount from the control
+// namespace into the namespaces those pods run in. The proxy's felis-service-token
+// is not among them: it lives in the control namespace and on the host, and a copy
+// anywhere else would open every game route to whoever reads that namespace.
+func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, minecraftNS, buildNS string) []systemServerOutcome {
+	return []systemServerOutcome{
+		// refresh=true for both caller tokens: the control namespace holds the
+		// current value and `felis rotate-token` replaces it there, so a replica
+		// that differs is stale and the login gate would be turned away with it.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.LimboTokenSecretName, naming.ServiceTokenSecretKey, "limbo-token", "minecraft ns", true),
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
+		// refresh=true: felis-config is the rendered config, not a credential. The
+		// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
+		// must update it when the control plane's render has moved on (a stale copy
+		// e.g. keeps an old database URL after a credential rotation).
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-config", "felis.toml", "config", "minecraft ns", true),
+		// The reaper's pre-reap warning emails authenticate with the same relay
+		// password felis-api uses; the reaper pod runs in the minecraft namespace,
+		// where a secretKeyRef resolves only against a local mirror. Skipped while
+		// the relay is not configured yet — the "configure email" screen refreshes
+		// both mirrors when it applies.
+		ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
+			"felis-smtp", "password", "smtp", "minecraft ns", false),
+		// The build namespace needs the build token: the build Job's fetch
+		// initContainer reads the submission context from the internal face, and
+		// that is all this token opens. Best effort — a deployment that only
+		// installs the control plane simply never builds a user submission.
+		ensureSecretReplica(ctx, cl, controlNS, buildNS,
+			naming.BuildTokenSecretName, naming.ServiceTokenSecretKey, "build-token", "felis-build ns", true),
+	}
+}
+
 // ensureSecretReplica copies one Secret from the control namespace into a workload
 // namespace (minecraft — or the build namespace, whose fetch initContainer reads the
-// context from the felis-api internal face with the same token) so a pod can mount it
+// context from the felis-api internal face with the build token) so a pod can mount it
 // via secretKeyRef. A secretKeyRef is namespace-local, but those workloads do not run
 // beside the control plane — so without this replica the secretKeyRef would dangle and
 // wedge the pod in CreateContainerConfigError.
 //
-// Three Secrets need it, for different reasons: the service token (the login limbo and
+// Several Secrets need it, for different reasons: the caller tokens of the login
-// the build Pod's context fetch — both authenticate to the felis-api internal face),
+// limbo and the build Pod's context fetch (both authenticate to the felis-api
+// internal face, each with its own token),
 // the Velocity modern-forwarding secret (every backend — it is how a backend knows
 // a login really came from the proxy, and so that the player's UUID is Mojang-verified
 // rather than offline-derived), and the SMTP relay password (the reaper's pre-reap
@@ -609,12 +676,14 @@ func phaseOrPending(p v1alpha1.Phase) string {
 // copies only Type and Data — never labels/annotations/ownerRefs — so the replica
 // carries no accidental GC owner or managed-by lineage.
 //
-// refreshExisting switches the felis-config mirror to refresh-in-place: that Secret is
+// refreshExisting switches a replica to refresh-in-place from the control namespace.
-// a rendered config, never a hand-rotated credential, and the workload Jobs that mount
+// The felis-config mirror uses it because that Secret is a rendered config and the
-// it (backup/restore/fileedit) plus the reaper silently misbehave on a stale copy —
+// workload Jobs that mount it (backup/restore/fileedit) plus the reaper silently
-// e.g. after a database credential rotation the control plane moves on while every
+// misbehave on a stale copy — e.g. after a database credential rotation the control
-// backup Job keeps failing auth. Credential Secrets keep the never-overwrite rule so a
+// plane moves on while every backup Job keeps failing auth. The caller tokens use it
-// rotated value survives; to rotate those, delete the replica and re-run setup.
+// because `felis rotate-token` replaces them in the control namespace, which makes a
+// differing replica stale by definition. The forwarding and SMTP Secrets keep the
+// never-overwrite rule.
 func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label, where string, refreshExisting bool) systemServerOutcome {
 	name := label + " (" + where + ")"
 	validate := func(secret *corev1.Secret, location, skipped string) systemServerOutcome {
@@ -639,16 +708,22 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 		if out := validate(&src, controlNamespace, ""); !out.available {
 			return out
 		}
+		changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
 			if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
-			return validate(existing, minecraftNamespace, "already current")
+				return false
 			}
 			if existing.Data == nil {
 				existing.Data = map[string][]byte{}
 			}
 			existing.Data[secretKey] = src.Data[secretKey]
-		if err := cl.Update(ctx, existing); err != nil {
+			return true
+		})
+		if err != nil {
 			return systemServerOutcome{name: name, err: err}
 		}
+		if !changed {
+			return validate(existing, minecraftNamespace, "already current")
+		}
 		return systemServerOutcome{name: name, updated: true, available: true}
 	}
 	if controlNamespace == minecraftNamespace {
@@ -712,7 +787,7 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
 
 func requiredProvisioningError(outcomes []systemServerOutcome) error {
 	required := map[string]struct{}{
-		"service-token (minecraft ns)":     {},
+		"limbo-token (minecraft ns)":       {},
 		"forwarding-secret (minecraft ns)": {},
 		naming.SystemLoginServer:           {},
 	}

cmd/felis/systemservers_test.go

@@ -142,21 +142,21 @@ func TestLoginSystemServerEnv(t *testing.T) {
 // namespace (create-if-absent), so the operator's secretKeyRef on the backend pod
 // resolves. It must not overwrite an existing replica, and must degrade gracefully
 // when the source is missing or the namespaces coincide. Exercised here with the
-// service token; setup runs it a second time for the Velocity forwarding secret.
+// Velocity forwarding secret, which setup replicates in this never-overwrite mode.
 func TestEnsureSecretReplica(t *testing.T) {
 	scheme := newSystemServerScheme(t)
 	ctx := context.Background()
 
 	srcSecret := func() *corev1.Secret {
 		return &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "felis"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "felis"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("s3cr3t")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("s3cr3t")},
 		}
 	}
 	replicate := func(cl client.Client, controlNS, mcNS string) systemServerOutcome {
 		return ensureSecretReplica(ctx, cl, controlNS, mcNS,
-			naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false)
+			naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false)
 	}
 
 	t.Run("replicates when absent", func(t *testing.T) {
@@ -166,19 +166,19 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want created", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "s3cr3t" {
+		if string(replica.Data[naming.ForwardingSecretKey]) != "s3cr3t" {
-			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ServiceTokenSecretKey])
+			t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ForwardingSecretKey])
 		}
 	})
 
 	t.Run("does not overwrite existing replica", func(t *testing.T) {
 		existing := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte("rotated")},
+			Data:       map[string][]byte{naming.ForwardingSecretKey: []byte("rotated")},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), existing).Build()
 		out := replicate(cl, "felis", "minecraft")
@@ -186,11 +186,11 @@ func TestEnsureSecretReplica(t *testing.T) {
 			t.Fatalf("outcome = %+v, want skipped (not clobbered)", out)
 		}
 		var replica corev1.Secret
-		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
 			t.Fatalf("get replica: %v", err)
 		}
-		if string(replica.Data[naming.ServiceTokenSecretKey]) != "rotated" {
+		if string(replica.Data[naming.ForwardingSecretKey]) != "rotated" {
-			t.Error("existing replica was overwritten — a rotated token must survive")
+			t.Error("existing replica was overwritten — a hand-set value must survive")
 		}
 	})
 
@@ -204,22 +204,22 @@ func TestEnsureSecretReplica(t *testing.T) {
 
 	t.Run("rejects a source with an empty required key", func(t *testing.T) {
 		bad := srcSecret()
-		bad.Data[naming.ServiceTokenSecretKey] = nil
+		bad.Data[naming.ForwardingSecretKey] = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
 
 	t.Run("rejects an existing replica with an empty required key", func(t *testing.T) {
 		bad := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
+			ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
 			Data:       map[string][]byte{},
 		}
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), bad).Build()
 		out := replicate(cl, "felis", "minecraft")
-		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable existing replica", out)
 		}
 	})
@@ -245,7 +245,7 @@ func TestEnsureSecretReplica(t *testing.T) {
 		bad.Data = nil
 		cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
 		out := replicate(cl, "felis", "felis")
-		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
+		if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
 			t.Fatalf("outcome = %+v, want unavailable required key", out)
 		}
 	})
@@ -334,7 +334,7 @@ func TestEnsureSecretReplicaRefresh(t *testing.T) {
 
 func TestRequiredProvisioningError(t *testing.T) {
 	ready := []systemServerOutcome{
-		{name: "service-token (minecraft ns)", available: true},
+		{name: "limbo-token (minecraft ns)", available: true},
 		{name: "forwarding-secret (minecraft ns)", available: true},
 		{name: naming.SystemLoginServer, available: true},
 		{name: naming.SystemLobbyServer, skipped: "image not configured"},
@@ -349,6 +349,14 @@ func TestRequiredProvisioningError(t *testing.T) {
 		t.Fatalf("missing forwarding secret = %v, want named error", err)
 	}
 
+	// The login gate cannot reach felis-api without its token, so setup must not
+	// report success while that replica is missing.
+	noToken := append([]systemServerOutcome(nil), ready...)
+	noToken[0] = systemServerOutcome{name: "limbo-token (minecraft ns)", skipped: "source missing"}
+	if err := requiredProvisioningError(noToken); err == nil || !strings.Contains(err.Error(), "limbo-token") {
+		t.Fatalf("missing limbo token = %v, want named error", err)
+	}
+
 	failed := append([]systemServerOutcome(nil), ready...)
 	failed[3] = systemServerOutcome{name: naming.SystemLobbyServer, err: context.DeadlineExceeded}
 	if err := requiredProvisioningError(failed); err == nil || !strings.Contains(err.Error(), naming.SystemLobbyServer) {
@@ -662,3 +670,62 @@ func TestEnsureSystemServersRefreshesDerivedEnv(t *testing.T) {
 		}
 	})
 }
+
+// Setup's replicas carry each workload its own caller token and nothing more: the
+// login gate gets felis-limbo-token in the minecraft namespace, the build Jobs get
+// felis-build-token in the build namespace, a replica left stale by a rotation is
+// brought up to date, and the proxy's felis-service-token is copied nowhere.
+func TestProvisionSecretReplicasCarryCallerTokens(t *testing.T) {
+	scheme := newSystemServerScheme(t)
+	ctx := context.Background()
+	secret := func(ns, name, key, val string) *corev1.Secret {
+		return &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
+			Type:       corev1.SecretTypeOpaque,
+			Data:       map[string][]byte{key: []byte(val)},
+		}
+	}
+	cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		secret("felis", "felis-service-token", "token", "proxy-tok"),
+		secret("felis", "felis-limbo-token", "token", "limbo-new"),
+		secret("felis", "felis-build-token", "token", "build-tok"),
+		secret("felis", "felis-ops-token", "token", "ops-tok"),
+		secret("felis", naming.ForwardingSecretName, naming.ForwardingSecretKey, "fwd"),
+		// What a rotation leaves behind before setup runs again.
+		secret("minecraft", "felis-limbo-token", "token", "limbo-old"),
+	).Build()
+
+	outcomes := provisionSecretReplicas(ctx, cl, "felis", "minecraft", "felis-build")
+	for _, o := range outcomes {
+		if o.err != nil {
+			t.Fatalf("%s: %v", o.name, o.err)
+		}
+	}
+
+	read := func(ns, name string) (string, bool) {
+		var s corev1.Secret
+		if err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
+			return "", false
+		}
+		return string(s.Data["token"]), true
+	}
+	if got, _ := read("minecraft", "felis-limbo-token"); got != "limbo-new" {
+		t.Errorf("minecraft/felis-limbo-token = %q, want the rotated limbo-new", got)
+	}
+	if got, _ := read("felis-build", "felis-build-token"); got != "build-tok" {
+		t.Errorf("felis-build/felis-build-token = %q, want build-tok", got)
+	}
+	for _, ns := range []string{"minecraft", "felis-build"} {
+		for _, name := range []string{"felis-service-token", "felis-ops-token"} {
+			if _, ok := read(ns, name); ok {
+				t.Errorf("%s/%s was replicated; only the control namespace holds it", ns, name)
+			}
+		}
+	}
+	if _, ok := read("minecraft", "felis-build-token"); ok {
+		t.Error("the build token was copied into the minecraft namespace")
+	}
+	if _, ok := read("felis-build", "felis-limbo-token"); ok {
+		t.Error("the limbo token was copied into the build namespace")
+	}
+}

cmd/felis/tui_edge.go

@@ -437,6 +437,10 @@ func (m *edgeModel) errorView() string {
 	if m.lastErr != nil {
 		b.WriteString(tuiHint.Render(m.lastErr.Error()) + "\n")
 	}
+	// Nothing done above is rolled back, and nothing needs to be: every step finds what an
+	// earlier attempt created (the tunnel, its DNS route, the Access app and policy) and
+	// carries on from it.
+	b.WriteString("\n" + tuiHint.Render("Retrying is safe: it reuses the tunnel, DNS record and Access app created so far instead of making duplicates.") + "\n")
 	b.WriteString("\n" + tuiAction("enter", "retry", "esc", "edit"))
 	return b.String()
 }

cmd/felis/tui_edge_apply.go

@@ -32,7 +32,9 @@ func applyCloudflareEdge(ctx context.Context, result *cfsetup.Result, panelHost,
 	if adminHost == "" {
 		return fmt.Errorf("admin hostname is required")
 	}
-	if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud); err != nil {
+	// cloudflared is the only way in once the NodePort is fenced, so the
+	// visitor address it writes can key the sign-in rate limit.
+	if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud, "CF-Connecting-IP"); err != nil {
 		return err
 	}
 	if err := applyFelisConfigSecret(ctx); err != nil {
@@ -78,7 +80,8 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
 	if adminHost == "" {
 		return fmt.Errorf("admin hostname is required")
 	}
-	if err := writeConnectionConfig(panelHost, adminHost, ""); err != nil {
+	// Caddy, nginx and Traefik all append the peer they saw to X-Forwarded-For.
+	if err := writeConnectionConfig(panelHost, adminHost, "", "X-Forwarded-For"); err != nil {
 		return err
 	}
 	if err := applyFelisConfigSecret(ctx); err != nil {
@@ -93,16 +96,17 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
 // writeConnectionConfig stamps the chosen hostnames (and optional Access audience)
 // into both the host and pod config files. An empty aud clears any prior
 // Cloudflare audience, which is correct when switching to a non-Access front.
-func writeConnectionConfig(panelHost, adminHost, aud string) error {
+// clientIPHeader is the header that front writes the visitor address into.
+func writeConnectionConfig(panelHost, adminHost, aud, clientIPHeader string) error {
 	for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} {
-		if err := updateAuthConfig(path, panelHost, adminHost, aud); err != nil {
+		if err := updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func updateAuthConfig(path, panelHost, adminHost, aud string) error {
+func updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader string) error {
 	cfg, err := config.Load(path)
 	if err != nil {
 		return err
@@ -112,6 +116,7 @@ func updateAuthConfig(path, panelHost, adminHost, aud string) error {
 	}
 	cfg.Auth.AdminHostname = adminHost
 	cfg.Auth.AccessJWTAud = aud
+	cfg.Auth.ClientIPHeader = clientIPHeader
 	return writeConfig(path, cfg)
 }
 

cmd/felis/tui_migration.go

@@ -3,8 +3,10 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"time"
 
+	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/store"
 )
 
@@ -12,7 +14,7 @@ import (
 // applied count. Used by the preflight stage to self-heal a freshly bootstrapped
 // (or upgraded) database.
 func applyMigrations(dbURL string) (int, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
 	drv, err := store.Open(ctx, dbURL)
 	if err != nil {
@@ -23,20 +25,17 @@ func applyMigrations(dbURL string) (int, error) {
 	if err != nil {
 		return 0, err
 	}
+	// Same guard as `felis migrate up`: never roll a populated database forward
+	// without a snapshot to roll back to.
+	if _, err := preMigrateBackup(ctx, drv, migrations, dbURL, dbbackup.DefaultDir, io.Discard); err != nil {
+		return 0, fmt.Errorf("pre-migration backup: %w", err)
+	}
 	if _, err := store.Up(ctx, drv, migrations); err != nil {
 		return 0, err
 	}
-	applied, err := drv.AppliedVersions(ctx)
+	s, err := store.ReadSchema(ctx, drv)
 	if err != nil {
 		return 0, err
 	}
-	return len(applied), nil
+	return s.Applied, nil
-}
-
-func totalMigrations() (int, error) {
-	migrations, err := store.LoadMigrations()
-	if err != nil {
-		return 0, err
-	}
-	return len(migrations), nil
 }

cmd/felis/tui_postgres.go

@@ -72,20 +72,14 @@ func parseDBURL(url string) (dbCfg, error) {
 	}, nil
 }
 
-func countMigrations(dbURL string) (int, error) {
+// readSchema lines the database's recorded migrations up with this build's.
+func readSchema(dbURL string) (store.Schema, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	drv, err := store.Open(ctx, dbURL)
 	if err != nil {
-		return 0, err
+		return store.Schema{}, err
 	}
 	defer drv.Close()
-	if err := drv.EnsureVersionTable(ctx); err != nil {
+	return store.ReadSchema(ctx, drv)
-		return 0, err
-	}
-	applied, err := drv.AppliedVersions(ctx)
-	if err != nil {
-		return 0, err
-	}
-	return len(applied), nil
 }

cmd/felis/tui_preflight.go

@@ -46,6 +46,7 @@ type pfDBMsg struct{ err error }
 type pfMigCheckMsg struct {
 	applied int
 	total   int
+	pending bool
 	err     error
 }
 
@@ -84,7 +85,7 @@ func (m *preflightModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			return m, nil
 		}
 		m.applied, m.total = msg.applied, msg.total
-		if msg.applied < msg.total {
+		if msg.pending {
 			m.state = pfApplyMig
 			return m, m.applyMigrations()
 		}
@@ -204,12 +205,14 @@ func (m *preflightModel) checkDB() tea.Cmd {
 
 func (m *preflightModel) checkMigrations() tea.Cmd {
 	return func() tea.Msg {
-		applied, err := countMigrations(m.dbURL)
+		// The sets, not their sizes: a database a newer release migrated can hold as
-		if err != nil {
+		// many rows as this build has migrations, and must stop here rather than be
-			return pfMigCheckMsg{err: err}
+		// "healed" by an older binary.
+		s, err := readSchema(m.dbURL)
+		if err == nil {
+			err = s.Newer()
 		}
-		total, err := totalMigrations()
+		return pfMigCheckMsg{applied: s.Applied, total: s.Total, pending: len(s.Pending) > 0, err: err}
-		return pfMigCheckMsg{applied: applied, total: total, err: err}
 	}
 }
 

cmd/felis/tui_smtp.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"felis.lolicon.best/internal/config"
-	"felis.lolicon.best/internal/mail"
 	"felis.lolicon.best/internal/platform"
 
 	"github.com/charmbracelet/bubbles/spinner"
@@ -311,24 +310,22 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
 	if err != nil {
 		return fmt.Errorf("port %q is not a number", in.port)
 	}
-	relay := &mail.SMTP{Host: in.host, Port: port, From: in.from, Username: in.username, Password: in.password}
+	var prev config.SMTPConfig
-	if err := relay.Ping(ctx); err != nil {
+	if cur, err := config.Load(hostSetupConfigPath); err == nil {
+		prev = cur.SMTP
+	}
+	// Ping under the posture felis api will send with, so a relay without
+	// STARTTLS is turned down here rather than at a player's first code.
+	if err := smtpRelay(setupSMTPConfig(in, port, prev), in.password).Ping(ctx); err != nil {
 		return err
 	}
 
-	smtpCfg := config.SMTPConfig{
-		Host:        in.host,
-		Port:        port,
-		From:        in.from,
-		Username:    in.username,
-		PasswordRef: platform.SMTPPasswordEnv,
-	}
 	for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} {
 		cfg, err := config.Load(path)
 		if err != nil {
 			return err
 		}
-		cfg.SMTP = smtpCfg
+		cfg.SMTP = setupSMTPConfig(in, port, cfg.SMTP)
 		if err := writeConfig(path, cfg); err != nil {
 			return err
 		}
@@ -351,6 +348,22 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
 	return kubectl(ctx, "-n", "felis", "rollout", "status", "deployment/felis-api", "--timeout=180s")
 }
 
+// setupSMTPConfig is the [smtp] block this screen writes: the relay it just
+// proved, plus the keys only an operator sets by hand (require_tls,
+// max_per_hour), carried over from the block it replaces so reconfiguring the
+// relay does not quietly reset them.
+func setupSMTPConfig(in smtpInputs, port int, prev config.SMTPConfig) config.SMTPConfig {
+	return config.SMTPConfig{
+		Host:        in.host,
+		Port:        port,
+		From:        in.from,
+		Username:    in.username,
+		PasswordRef: platform.SMTPPasswordEnv,
+		MaxPerHour:  prev.MaxPerHour,
+		RequireTLS:  prev.RequireTLS,
+	}
+}
+
 // smtpSecretManifest renders the felis-smtp Secret for the given namespace, the
 // one the receiving Deployment/CronJob resolves its secretKeyRef against (felis
 // for felis-api, the workload namespace for the reaper's mirror). The namespace

cmd/felis/tui_smtp_test.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"reflect"
 	"strings"
 	"testing"
 
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
 	"sigs.k8s.io/yaml"
 )
 
@@ -35,3 +38,39 @@ func TestSMTPSecretManifestCarriesTargetNamespace(t *testing.T) {
 		}
 	}
 }
+
+// TestSMTPRelayCarriesTLSPosture: the relay felis api, the reaper and the
+// watchdog send through refuses plaintext for a remote host and allows it for
+// one on this host, as [smtp] says.
+func TestSMTPRelayCarriesTLSPosture(t *testing.T) {
+	remote := smtpRelay(config.SMTPConfig{Host: "smtp.example.net", Port: 587, From: "[email protected]", Username: "felis"}, "pw")
+	want := &mail.SMTP{Host: "smtp.example.net", Port: 587, From: "[email protected]", Username: "felis", Password: "pw", RequireTLS: true}
+	if !reflect.DeepEqual(remote, want) {
+		t.Errorf("remote relay = %+v, want %+v", remote, want)
+	}
+	if local := smtpRelay(config.SMTPConfig{Host: "127.0.0.1", Port: 25, From: "[email protected]"}, ""); local.RequireTLS {
+		t.Error("a relay on this host must not require TLS by default")
+	}
+}
+
+// TestSetupSMTPConfigKeepsHandSetKeys: re-running the email screen replaces the
+// relay but keeps require_tls and max_per_hour, which only an operator sets.
+func TestSetupSMTPConfigKeepsHandSetKeys(t *testing.T) {
+	off := false
+	prev := config.SMTPConfig{Host: "old.example.net", Port: 25, From: "[email protected]", MaxPerHour: 500, RequireTLS: &off}
+	in := smtpInputs{host: "smtp.example.net", from: "[email protected]", username: "felis"}
+	got := setupSMTPConfig(in, 465, prev)
+	if got.Host != "smtp.example.net" || got.Port != 465 || got.From != "[email protected]" ||
+		got.Username != "felis" || got.PasswordRef != "FELIS_SMTP_PASSWORD" {
+		t.Errorf("relay fields = %+v", got)
+	}
+	if got.MaxPerHour != 500 {
+		t.Errorf("max_per_hour = %d, want 500", got.MaxPerHour)
+	}
+	if got.RequireTLS == nil || *got.RequireTLS {
+		t.Errorf("require_tls = %v, want the operator's false", got.RequireTLS)
+	}
+	if fresh := setupSMTPConfig(in, 587, config.SMTPConfig{}); fresh.RequireTLS != nil || fresh.MaxPerHour != 0 {
+		t.Errorf("first setup = %+v, want require_tls and max_per_hour unset", fresh)
+	}
+}

cmd/felis/update.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -9,6 +11,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/jackc/pgx/v5"
+
+	"felis.lolicon.best/internal/config"
 	"felis.lolicon.best/internal/updater"
 	"felis.lolicon.best/internal/updates"
 )
@@ -34,6 +39,8 @@ const updateTimeout = 60 * time.Second
 type updateTarget struct {
 	// selector is the flag name without dashes.
 	selector string
+	// help is the flag's usage line.
+	help string
 	// component is the updates planner's name for this piece, or "" when the planner
 	// deliberately does not track it (Minecraft, which is pinned).
 	component string
@@ -41,19 +48,28 @@ type updateTarget struct {
 	note string
 	// command is the exact, already-tested way to apply it.
 	command string
+	// installer marks a command that re-runs the installer, which the trailer explains.
+	installer bool
 }
 
-// installerRerun is the tested apply path for every planner-backed selector: re-run the
+// installerRerun is the tested apply path for every selector Felis installs: re-run the
 // installer. It is idempotent, and it is the only path that fetches a newer version --
 // `felis setup` skips its host-bootstrap phase on a completed install (all four install
 // markers already exist), so there it opens the config console and moves no component,
 // and even on the bootstrap path it re-images felis-api from the binary setup is already
 // running (FELIS_BOOTSTRAP_BINARY), which looks like an update and changes nothing.
 //
-// The URL is the same one-liner both READMEs hand out. While the repo is private it
+// The URL is the one-liner both READMEs hand out, read at a tag rather than main: the
+// script's release channel installs the newest release's binary, and main can carry
+// installer changes that binary was never tested with. installerRef picks the tag and
+// renderApplyGuidance substitutes it for {ref}. While the repo is private the URL
 // answers 404 (raw.githubusercontent.com hides private repos), which is why the trailer
 // below points at the README's token'd form for that case.
-const installerRerun = "curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash"
+const installerRerun = "curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/{ref}/deploy/bootstrap.sh | sudo bash"
+
+// installerRerunDeps is the same re-run with FELIS_UPGRADE_DEPS=1, which lets it move an
+// installed k3s and cloudflared to the versions the release pins.
+const installerRerunDeps = "curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/{ref}/deploy/bootstrap.sh | sudo FELIS_UPGRADE_DEPS=1 bash"
 
 // updateTargets is the selector table. panel and plugins both resolve to felis-api
 // because they are not separately versioned: the panel is compiled into the felis
@@ -62,24 +78,62 @@ const installerRerun = "curl -fsSL https://raw.githubusercontent.com/FelisMC/Fel
 var updateTargets = []updateTarget{
 	{
 		selector:  "panel",
+		help:      "select the panel + control plane (felis-api)",
 		component: "felis-api",
 		note:      "the panel is embedded in the felis binary (//go:embed), so updating it means rebuilding the felis image and rolling felis-api",
 		command:   installerRerun,
+		installer: true,
 	},
 	{
 		selector:  "velocity",
+		help:      "select the Velocity proxy",
 		component: "velocity",
-		note:      "re-runs install_velocity: newest BUILD of the pinned minor (FELIS_VELOCITY_VERSION), atomic jar install, then restarts felis-velocity",
+		note:      "re-runs install_velocity: the build the release pins in deploy/game-stack.lock (FELIS_VELOCITY_VERSION=<minor> takes that minor's newest build instead), sha256-checked, atomic jar install, then restarts felis-velocity only if the jar or its config changed",
 		command:   installerRerun,
+		installer: true,
 	},
 	{
 		selector:  "plugins",
+		help:      "select the Felis plugin jars (velocity/paper/limbo)",
 		component: "felis-api",
 		note:      "felis-velocity.jar is a host-file swap, but felis-paper.jar and felis-limbo.jar are baked into the lobby/limbo images and need a rebuild + re-mirror into the in-cluster registry (the installer re-run does both)",
 		command:   installerRerun,
+		installer: true,
+	},
+	{
+		selector:  "k3s",
+		help:      "select k3s",
+		component: "k3s",
+		note:      "FELIS_UPGRADE_DEPS=1 moves k3s to the version the Felis release pins, which can trail the newest upstream; it moves one minor version at a time and refuses a larger jump. Running game servers keep running while k3s restarts",
+		command:   installerRerunDeps,
+		installer: true,
+	},
+	{
+		selector:  "cloudflared",
+		help:      "select cloudflared",
+		component: "cloudflared",
+		note:      "FELIS_UPGRADE_DEPS=1 swaps the binary for the sha256-pinned build the Felis release names and restarts cloudflared-felis; the panel's tunnel drops for a few seconds",
+		command:   installerRerunDeps,
+		installer: true,
+	},
+	{
+		selector:  "jre",
+		help:      "select the Temurin JRE Velocity runs on",
+		component: "jre",
+		note:      "the installer installs the Temurin build the Felis release pins (sha256-checked) and restarts felis-velocity when it changed; a newer upstream build reaches the host with a release that pins it",
+		command:   installerRerun,
+		installer: true,
+	},
+	{
+		selector:  "postgres",
+		help:      "select PostgreSQL",
+		component: "postgresql",
+		note:      "PostgreSQL comes from the distribution's packages, so a minor release is a package update followed by a restart (a few seconds without the API). A new major needs pg_upgrade first: docs/operations.md §4",
+		command:   "sudo dnf upgrade 'postgresql*' || sudo apt-get install --only-upgrade 'postgresql*'; sudo systemctl restart postgresql",
 	},
 	{
 		selector:  "mc",
+		help:      "select Minecraft (pinned; reported only)",
 		component: "", // never tracked: see the pin note below
 		note:      "Minecraft is pinned by policy (\"能不动的就别动\") and Felis never proposes a version change for it. A server's version is a property of that server's image — change it on the server, not through a platform update",
 		command:   "",
@@ -89,26 +143,29 @@ var updateTargets = []updateTarget{
 // cmdUpdate reports what can be updated and what is already current.
 //
 // Bare `felis update` prints the status of every tracked component. Selector flags
-// (--panel/--velocity/--mc/--plugins/--all) narrow that report to the components
+// (--panel/--velocity/--plugins/--k3s/--cloudflared/--jre/--postgres/--mc/--all) narrow that report to the components
 // they name AND print how to apply each one. --force additionally prints the apply
 // instruction for a selected component that is already up to date, for the
 // reinstall/repair case.
 //
 // It never applies anything and never mutates the node, so unlike setup/breakGlass
-// it needs no root. The versions it reads come from this host: k3s and cloudflared
+// it needs no root. The versions it reads come from this host: k3s, cloudflared and
-// answer `--version`, Velocity's version is read out of the installed jar's
+// PostgreSQL answer `--version`, Velocity's version is read out of the installed jar's
-// manifest, and felis-api's is this binary's own build stamp — the same value
+// manifest, the JRE's out of its release file, and felis-api's is this binary's own
-// `felis version` prints, which is what the user asked to be the source of truth.
+// build stamp — the same value `felis version` prints, which is what the user asked
+// to be the source of truth.
 func cmdUpdate(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("update", flag.ContinueOnError)
 	fs.SetOutput(stderr)
-	panel := fs.Bool("panel", false, "select the panel + control plane (felis-api)")
+	flags := map[string]*bool{}
-	velocity := fs.Bool("velocity", false, "select the Velocity proxy")
+	for _, t := range updateTargets {
-	mc := fs.Bool("mc", false, "select Minecraft (pinned; reported only)")
+		flags[t.selector] = fs.Bool(t.selector, false, t.help)
-	plugins := fs.Bool("plugins", false, "select the Felis plugin jars (velocity/paper/limbo)")
+	}
 	all := fs.Bool("all", false, "select every component above")
 	force := fs.Bool("force", false, "print the apply command for a selected component even when it is already up to date")
 	velocityJar := fs.String("velocity-jar", updater.DefaultVelocityJarPath, "path to the installed Velocity jar to read the current version from")
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml, read for the maintenance window the panel stores")
+	record := fs.Bool("record", false, "also store this check for the panel's Updates page (felis-update-check.timer runs it daily)")
 	if err := fs.Parse(args); err != nil {
 		return 2
 	}
@@ -118,8 +175,8 @@ func cmdUpdate(args []string, stdout, stderr io.Writer) int {
 	}
 
 	selected := map[string]bool{}
-	for sel, on := range map[string]bool{"panel": *panel, "velocity": *velocity, "mc": *mc, "plugins": *plugins} {
+	for sel, on := range flags {
-		if on || *all {
+		if *on || *all {
 			selected[sel] = true
 		}
 	}
@@ -127,9 +184,10 @@ func cmdUpdate(args []string, stdout, stderr io.Writer) int {
 	ctx, cancel := context.WithTimeout(context.Background(), updateTimeout)
 	defer cancel()
 
+	src := updater.NewRoutingSource(updater.Topology())
 	rn := &updater.Runner{
 		Gatherer: updater.NewHostGatherer(resolvedVersion(), *velocityJar),
-		Source:   updater.NewRoutingSource(updater.Topology()),
+		Source:   src,
 		// Notifier and Applier stay nil on purpose: a human typing this command IS the
 		// notification, and nothing here applies. The zero Window below means every
 		// Scheduled component degrades to a notify, so the report can never claim an
@@ -141,13 +199,104 @@ func cmdUpdate(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	now := time.Now()
+	win, winErr := readUpdateWindow(ctx, *cfgPath)
+	fmt.Fprint(stdout, renderWindowLine(win, winErr, now))
 	fmt.Fprint(stdout, renderUpdateReport(res, selected))
+	fmt.Fprint(stdout, renderNotes(src.Notes(), selected))
 	if len(selected) > 0 {
+		if winErr == nil && !win.Start.IsZero() && !win.Contains(now) {
+			fmt.Fprint(stdout, "Warning: this is outside the maintenance window; the commands below take effect as soon as you run them.\n")
+		}
 		fmt.Fprint(stdout, renderApplyGuidance(res, selected, *force))
 	}
+	if *record {
+		// A fresh context: the discovery pass may have spent most of updateTimeout.
+		rctx, rcancel := context.WithTimeout(context.Background(), updateWindowTimeout)
+		defer rcancel()
+		if err := recordUpdateStatus(rctx, *cfgPath, buildStatusReport(res, src.Notes(), resolvedVersion(), now)); err != nil {
+			fmt.Fprintf(stderr, "felis update: record the check for the panel: %v\n", err)
+			return 1
+		}
+		fmt.Fprint(stdout, "Recorded this check for the panel's Updates page.\n")
+	}
 	return 0
 }
 
+// buildStatusReport turns one run into the record the panel shows: every planned
+// component in plan order, then each component whose installed version could not
+// be read, by name. A component the feed could not answer for is StateUnknown with
+// the reason, never StateCurrent: the panel must not call a component current when
+// nobody could check.
+func buildStatusReport(res updater.Result, notes map[string]string, felis string, now time.Time) updates.StatusReport {
+	selectorOf := map[string]string{}
+	for _, t := range updateTargets {
+		if t.component != "" && selectorOf[t.component] == "" {
+			selectorOf[t.component] = t.selector
+		}
+	}
+	rep := updates.StatusReport{CheckedAt: now.UTC(), Felis: felis, Components: []updates.ComponentStatus{}}
+	for _, a := range res.RunResult.Plan {
+		cs := updates.ComponentStatus{
+			Name:     a.Component,
+			Current:  a.Current.String(),
+			Selector: selectorOf[a.Component],
+			Note:     notes[a.Component],
+		}
+		switch {
+		case a.Kind == updates.ActionPinned:
+			cs.State = updates.StatePinned
+		case a.Kind == updates.ActionNotify || a.Kind == updates.ActionApply:
+			cs.State = updates.StateAvailable
+			cs.Latest = a.Latest.String()
+		case a.LatestKnown:
+			cs.State = updates.StateCurrent
+		default:
+			cs.State = updates.StateUnknown
+			if err := res.RunResult.SourceErrors[a.Component]; err != nil {
+				cs.Error = err.Error()
+			}
+		}
+		rep.Components = append(rep.Components, cs)
+	}
+	names := make([]string, 0, len(res.GatherErrors))
+	for name := range res.GatherErrors {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		rep.Components = append(rep.Components, updates.ComponentStatus{
+			Name:     name,
+			State:    updates.StateUnreadable,
+			Selector: selectorOf[name],
+			Note:     notes[name],
+			Error:    res.GatherErrors[name].Error(),
+		})
+	}
+	return rep
+}
+
+// recordUpdateStatus upserts rep into platform_settings[updates.StatusKey], the
+// row the API serves to the panel's Updates page.
+func recordUpdateStatus(ctx context.Context, cfgPath string, rep updates.StatusReport) error {
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return err
+	}
+	v, err := json.Marshal(rep)
+	if err != nil {
+		return err
+	}
+	conn, err := pgx.Connect(ctx, cfg.Database.URL)
+	if err != nil {
+		return err
+	}
+	defer conn.Close(context.Background())
+	_, err = conn.Exec(ctx, `INSERT INTO platform_settings (key, value) VALUES ($1, $2::jsonb)
+		ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`, updates.StatusKey, string(v))
+	return err
+}
+
 // renderUpdateReport renders the component status table. With no selectors it shows
 // every tracked component; with selectors it shows only the components those
 // selectors name, so `felis update --velocity` is a focused answer rather than the
@@ -196,6 +345,23 @@ func renderUpdateReport(res updater.Result, selected map[string]bool) string {
 	return b.String()
 }
 
+// renderNotes prints what the release lookups learned beyond the versions (today: a
+// PostgreSQL major past its end of life), for the components the selectors show.
+func renderNotes(notes map[string]string, selected map[string]bool) string {
+	names := make([]string, 0, len(notes))
+	for name := range notes {
+		if len(selected) == 0 || selectedCovers(selected, name) {
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	var b strings.Builder
+	for _, name := range names {
+		fmt.Fprintf(&b, "%-13s note: %s\n", name, notes[name])
+	}
+	return b.String()
+}
+
 // writeErrs appends one explanatory line per failed component, in a stable order so
 // the output does not shuffle between runs, honouring the active selector filter.
 func writeErrs(b *strings.Builder, label string, errs map[string]error, selected map[string]bool) {
@@ -231,7 +397,7 @@ func renderApplyGuidance(res updater.Result, selected map[string]bool, force boo
 	}
 
 	var b strings.Builder
-	var offeredCommand bool
+	var offeredInstaller bool
 	for _, t := range updateTargets {
 		if !selected[t.selector] {
 			continue
@@ -258,8 +424,8 @@ func renderApplyGuidance(res updater.Result, selected map[string]bool, force boo
 			// component, and reinstalling the current release is a valid repair action.
 			fmt.Fprintf(&b, "  note: cannot tell whether %s is current — its latest version could not be discovered (see above); this reinstalls it either way\n", t.component)
 		}
-		fmt.Fprintf(&b, "  run: %s\n", t.command)
+		fmt.Fprintf(&b, "  run: %s\n", strings.ReplaceAll(t.command, "{ref}", installerRef(byComponent)))
-		offeredCommand = true
+		offeredInstaller = offeredInstaller || t.installer
 	}
 	// Only explain the command when one was actually offered; a --mc-only run has
 	// nothing to run and the trailer would be a non-sequitur.
@@ -267,13 +433,95 @@ func renderApplyGuidance(res updater.Result, selected map[string]bool, force boo
 	// One trailer serves every selector now: setup is not an apply path at all on a
 	// completed install (shouldRunHostBootstrapBeforeConfig only enters the host
 	// bootstrap while an install marker is missing), so the installer re-run is the one
-	// worked path for all three components and there is no per-component exception left
+	// worked path for every component Felis installs and there is no per-component exception left
 	// to scope. Two caveats stay because following the advice without them bites real
 	// hosts: the channel is not persisted anywhere (a bare re-run on a main host quietly
 	// moves it onto releases), and the private repo's one-liner needs the read token
 	// back in the environment before it can resolve anything.
-	if offeredCommand {
+	if offeredInstaller {
-		b.WriteString("\nRe-running the installer applies everything above: it fetches the newest version on\nthe channel in effect and re-applies the bundle (release is the default). The channel\nis not persisted, so pass FELIS_VERSION_BOOTSTRAP=dev if this host tracks main. While\nthis repo is private, the one-liner above 404s without a token; the README's install\nsection has the token'd form that works. felis setup is not this path: on a completed\ninstall it opens the config console and installs nothing newer. Restart game servers\nafterwards.\n")
+		b.WriteString("\nRe-running the installer applies each installer command above: it fetches the newest version on\nthe channel in effect and re-applies the bundle (release is the default). The channel\nis not persisted, so pass FELIS_VERSION_BOOTSTRAP=dev if this host tracks main. While\nthis repo is private, the one-liner above 404s without a token; the README's install\nsection has the token'd form that works. felis setup is not this path: on a completed\ninstall it opens the config console and installs nothing newer. Restart game servers\nafterwards.\n")
 	}
 	return b.String()
 }
+
+// installerRef is the git ref the installer re-run reads bootstrap.sh from: the newest
+// stable felis release when the feed answered, which is the release that script then
+// installs; else the release this host runs; main only when neither is a release tag.
+func installerRef(byComponent map[string]updates.Action) string {
+	a, ok := byComponent["felis-api"]
+	if !ok {
+		return "main"
+	}
+	if a.LatestKnown && isReleaseTag(a.Latest) {
+		return a.Latest.String()
+	}
+	if isReleaseTag(a.Current) {
+		return a.Current.String()
+	}
+	return "main"
+}
+
+// isReleaseTag reports whether v was read from a stable vX.Y.Z tag, the only refs
+// release.yml publishes a binary for. A source build stamps v0.0.0+g<commit>, which
+// names no tag, so build metadata disqualifies a version too.
+func isReleaseTag(v updates.Version) bool {
+	s := v.String()
+	if !strings.HasPrefix(s, "v") || v.IsPrerelease() || strings.Contains(s, "+") {
+		return false
+	}
+	_, err := updates.Parse(s)
+	return err == nil
+}
+
+// updateWindowTimeout bounds the maintenance-window read, so an unreachable
+// database costs the report a line and never the report itself.
+const updateWindowTimeout = 3 * time.Second
+
+// readUpdateWindow reads the maintenance window the panel stores
+// (platform_settings "update_window"). Felis applies nothing on its own: this
+// command is the window's consumer, showing it and warning before an apply
+// outside it. A missing row is an unset window.
+func readUpdateWindow(ctx context.Context, cfgPath string) (updates.Window, error) {
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return updates.Window{}, err
+	}
+	ctx, cancel := context.WithTimeout(ctx, updateWindowTimeout)
+	defer cancel()
+	conn, err := pgx.Connect(ctx, cfg.Database.URL)
+	if err != nil {
+		return updates.Window{}, err
+	}
+	defer conn.Close(context.Background())
+	var raw []byte
+	err = conn.QueryRow(ctx, `SELECT value FROM platform_settings WHERE key = 'update_window'`).Scan(&raw)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return updates.Window{}, nil
+	}
+	if err != nil {
+		return updates.Window{}, err
+	}
+	var w updates.Window
+	if err := json.Unmarshal(raw, &w); err != nil {
+		return updates.Window{}, fmt.Errorf("stored window: %w", err)
+	}
+	return w, nil
+}
+
+// renderWindowLine is the report's first line: where now sits against the
+// maintenance window.
+func renderWindowLine(w updates.Window, err error, now time.Time) string {
+	const layout = "2006-01-02 15:04 MST"
+	switch {
+	case err != nil:
+		return fmt.Sprintf("Maintenance window: unknown (%v).\n", err)
+	case w.Start.IsZero() || w.End.IsZero():
+		return "Maintenance window: not set; apply whenever suits you.\n"
+	case w.Contains(now):
+		return fmt.Sprintf("Maintenance window: open now, until %s.\n", w.End.Local().Format(layout))
+	case now.Before(w.Start):
+		return fmt.Sprintf("Maintenance window: opens %s, until %s. Felis applies nothing on its own; run the apply commands inside it.\n", w.Start.Local().Format(layout), w.End.Local().Format(layout))
+	default:
+		return fmt.Sprintf("Maintenance window: ended %s; set a new one in the panel before applying.\n", w.End.Local().Format(layout))
+	}
+}

cmd/felis/update_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	"felis.lolicon.best/internal/updater"
 	"felis.lolicon.best/internal/updates"
@@ -167,3 +169,165 @@ func TestApplyGuidancePointsEveryComponentAtTheInstaller(t *testing.T) {
 		t.Fatalf("--mc offers no command; the trailer is a non-sequitur:\n%s", mc)
 	}
 }
+
+// The re-run reads bootstrap.sh at the tag whose binary it installs. main can carry
+// installer changes no release was tested with.
+func TestApplyGuidanceReadsTheInstallerAtTheReleaseTag(t *testing.T) {
+	v := func(s string) updates.Version {
+		t.Helper()
+		out, err := updates.Parse(s)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return out
+	}
+	cases := []struct {
+		name string
+		api  []updates.Action
+		want string
+	}{
+		{"latest known", []updates.Action{{Component: "felis-api", Kind: updates.ActionNotify, Current: v("v1.3.0"), Latest: v("v1.4.0"), LatestKnown: true}}, "/FelisMC/Felis/v1.4.0/deploy/bootstrap.sh"},
+		{"latest unknown", []updates.Action{{Component: "felis-api", Kind: updates.ActionNone, Current: v("v1.3.0")}}, "/FelisMC/Felis/v1.3.0/deploy/bootstrap.sh"},
+		{"prerelease latest", []updates.Action{{Component: "felis-api", Kind: updates.ActionNone, Current: v("v1.3.0"), Latest: v("v1.4.0-rc.1"), LatestKnown: true}}, "/FelisMC/Felis/v1.3.0/deploy/bootstrap.sh"},
+		{"nothing known", nil, "/FelisMC/Felis/main/deploy/bootstrap.sh"},
+	}
+	for _, c := range cases {
+		out := renderApplyGuidance(planResult(c.api), map[string]bool{"velocity": true, "panel": true}, true)
+		if !strings.Contains(out, c.want) {
+			t.Errorf("%s: want %q in:\n%s", c.name, c.want, out)
+		}
+		if strings.Contains(out, "{ref}") {
+			t.Errorf("%s: placeholder left in:\n%s", c.name, out)
+		}
+	}
+}
+
+// Every selector in the table is a flag: the FlagSet is built from the table.
+func TestUpdateSelectorsAreFlags(t *testing.T) {
+	var out, errb strings.Builder
+	if code := cmdUpdate([]string{"-h"}, &out, &errb); code != 2 {
+		t.Fatalf("-h exit = %d, want 2", code)
+	}
+	for _, target := range updateTargets {
+		if !strings.Contains(errb.String(), "-"+target.selector+"\n") {
+			t.Errorf("usage has no -%s flag:\n%s", target.selector, errb.String())
+		}
+	}
+}
+
+// k3s and cloudflared move only when the re-run is told to; PostgreSQL is the package
+// manager's, so its guidance carries no installer trailer.
+func TestApplyGuidanceForHostDependencies(t *testing.T) {
+	notify := func(c string) updater.Result {
+		return planResult([]updates.Action{{Component: c, Kind: updates.ActionNotify, LatestKnown: true}})
+	}
+	for _, sel := range []string{"k3s", "cloudflared"} {
+		out := renderApplyGuidance(notify(sel), map[string]bool{sel: true}, false)
+		if !strings.Contains(out, "sudo FELIS_UPGRADE_DEPS=1 bash") || !strings.Contains(out, "Re-running the installer") {
+			t.Errorf("--%s guidance must re-run the installer with FELIS_UPGRADE_DEPS=1:\n%s", sel, out)
+		}
+	}
+	jre := renderApplyGuidance(notify("jre"), map[string]bool{"jre": true}, false)
+	if !strings.Contains(jre, "| sudo bash") || strings.Contains(jre, "FELIS_UPGRADE_DEPS") {
+		t.Errorf("--jre guidance is the plain installer re-run:\n%s", jre)
+	}
+	pg := renderApplyGuidance(notify("postgresql"), map[string]bool{"postgres": true}, false)
+	if !strings.Contains(pg, "apt-get install --only-upgrade") || strings.Contains(pg, "Re-running the installer") {
+		t.Errorf("--postgres guidance is the package manager, without the installer trailer:\n%s", pg)
+	}
+}
+
+func TestRenderNotesHonoursSelectors(t *testing.T) {
+	notes := map[string]string{"postgresql": "PostgreSQL 13 reached end of life on 2025-11-13"}
+	if out := renderNotes(notes, nil); !strings.Contains(out, "postgresql") || !strings.Contains(out, "note: PostgreSQL 13 reached end of life") {
+		t.Errorf("unfiltered notes = %q", out)
+	}
+	if out := renderNotes(notes, map[string]bool{"postgres": true}); !strings.Contains(out, "end of life") {
+		t.Errorf("--postgres must show its note, got %q", out)
+	}
+	if out := renderNotes(notes, map[string]bool{"velocity": true}); out != "" {
+		t.Errorf("--velocity must not show the postgresql note, got %q", out)
+	}
+}
+
+// A source build's v0.0.0+g<commit> names no tag, so the installer one-liner has to
+// fall back to main instead of a 404ing ref.
+func TestInstallerRefNamesATag(t *testing.T) {
+	v := func(s string) updates.Version {
+		t.Helper()
+		x, err := updates.Parse(s)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return x
+	}
+	cases := []struct {
+		name string
+		api  updates.Action
+		want string
+	}{
+		{"newest release", updates.Action{Current: v("v1.2.0"), Latest: v("v1.3.0"), LatestKnown: true}, "v1.3.0"},
+		{"feed down, host on a release", updates.Action{Current: v("v1.2.0")}, "v1.2.0"},
+		{"source build", updates.Action{Current: v("v0.0.0+gunknown")}, "main"},
+		{"source build with commit", updates.Action{Current: v("v0.0.0+g1a2b3c4")}, "main"},
+		{"prerelease", updates.Action{Current: v("v1.3.0-rc.1")}, "main"},
+	}
+	for _, c := range cases {
+		if got := installerRef(map[string]updates.Action{"felis-api": c.api}); got != c.want {
+			t.Errorf("%s: installerRef = %q, want %q", c.name, got, c.want)
+		}
+	}
+	if got := installerRef(nil); got != "main" {
+		t.Errorf("no felis-api row: installerRef = %q, want main", got)
+	}
+}
+
+func mustVersion(t *testing.T, s string) updates.Version {
+	t.Helper()
+	v, err := updates.Parse(s)
+	if err != nil {
+		t.Fatalf("parse %q: %v", s, err)
+	}
+	return v
+}
+
+// The record the panel shows keeps every component with a state that cannot be
+// mistaken: a feed failure is "unknown" with its reason, an unreadable install is
+// listed after the plan, and each row carries the selector that prints its apply.
+func TestBuildStatusReport(t *testing.T) {
+	res := planResult([]updates.Action{
+		{Component: "felis-api", Current: mustVersion(t, "v0.4.0"), Latest: mustVersion(t, "v0.5.0"), LatestKnown: true, Kind: updates.ActionNotify},
+		{Component: "velocity", Current: mustVersion(t, "3.4.0"), Latest: mustVersion(t, "3.4.0"), LatestKnown: true, Kind: updates.ActionNone},
+		{Component: "k3s", Current: mustVersion(t, "v1.36.2+k3s1"), Kind: updates.ActionNone},
+		{Component: "cloudflared", Current: mustVersion(t, "2026.6.1"), Latest: mustVersion(t, "2026.9.0"), LatestKnown: true, Kind: updates.ActionApply},
+		{Component: "mc-lobby", Current: mustVersion(t, "1.21.4"), Kind: updates.ActionPinned},
+	})
+	res.RunResult.SourceErrors["k3s"] = errors.New("github: HTTP 403")
+	res.GatherErrors["postgresql"] = errors.New("psql: not found")
+	res.GatherErrors["jre"] = errors.New("release file missing")
+	notes := map[string]string{"postgresql": "PostgreSQL 13 is past its end of life", "velocity": "pinned minor 3.4"}
+	now := time.Date(2026, 9, 25, 3, 4, 5, 0, time.FixedZone("CST", 8*3600))
+
+	b, err := json.Marshal(buildStatusReport(res, notes, "v0.4.0", now))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := `{"checked_at":"2026-09-24T19:04:05Z","felis":"v0.4.0","components":[` +
+		`{"name":"felis-api","current":"v0.4.0","latest":"v0.5.0","state":"available","selector":"panel"},` +
+		`{"name":"velocity","current":"3.4.0","state":"current","selector":"velocity","note":"pinned minor 3.4"},` +
+		`{"name":"k3s","current":"v1.36.2+k3s1","state":"unknown","selector":"k3s","error":"github: HTTP 403"},` +
+		`{"name":"cloudflared","current":"2026.6.1","latest":"2026.9.0","state":"available","selector":"cloudflared"},` +
+		`{"name":"mc-lobby","current":"1.21.4","state":"pinned"},` +
+		`{"name":"jre","state":"unreadable","selector":"jre","error":"release file missing"},` +
+		`{"name":"postgresql","state":"unreadable","selector":"postgres","note":"PostgreSQL 13 is past its end of life","error":"psql: not found"}]}`
+	if string(b) != want {
+		t.Errorf("status report =\n%s\nwant\n%s", b, want)
+	}
+
+	// Nothing tracked still records an empty list, so the panel can tell "checked,
+	// nothing to show" from a report that never arrived.
+	b, _ = json.Marshal(buildStatusReport(planResult(nil), nil, "v0.4.0", now))
+	if !strings.Contains(string(b), `"components":[]`) {
+		t.Errorf("an empty check = %s, want an empty components list", b)
+	}
+}

cmd/felis/update_window_test.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/updates"
+)
+
+func TestRenderWindowLinePlacesNowAgainstTheWindow(t *testing.T) {
+	prev := time.Local
+	time.Local = time.FixedZone("CST", 8*3600)
+	t.Cleanup(func() { time.Local = prev })
+
+	w := updates.Window{
+		Start: time.Date(2026, 9, 26, 18, 0, 0, 0, time.UTC),
+		End:   time.Date(2026, 9, 26, 20, 0, 0, 0, time.UTC),
+	}
+	cases := []struct {
+		name string
+		w    updates.Window
+		err  error
+		now  time.Time
+		want string
+	}{
+		{"unreadable", updates.Window{}, errors.New("connection refused"), w.Start, "Maintenance window: unknown (connection refused).\n"},
+		{"unset", updates.Window{}, nil, w.Start, "Maintenance window: not set; apply whenever suits you.\n"},
+		{"half set", updates.Window{Start: w.Start}, nil, w.Start, "Maintenance window: not set; apply whenever suits you.\n"},
+		{"at the opening instant", w, nil, w.Start, "Maintenance window: open now, until 2026-09-27 04:00 CST.\n"},
+		{"before", w, nil, w.Start.Add(-time.Minute), "Maintenance window: opens 2026-09-27 02:00 CST, until 2026-09-27 04:00 CST. Felis applies nothing on its own; run the apply commands inside it.\n"},
+		{"at the closing instant", w, nil, w.End, "Maintenance window: ended 2026-09-27 04:00 CST; set a new one in the panel before applying.\n"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := renderWindowLine(tc.w, tc.err, tc.now); got != tc.want {
+				t.Fatalf("got  %q\nwant %q", got, tc.want)
+			}
+		})
+	}
+}

cmd/felis/watchdog.go

@@ -0,0 +1,268 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/store"
+	"felis.lolicon.best/internal/watchdog"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// proxyFor is how long the game proxy may refuse connections before it is
+// mailed: a restart takes seconds.
+const proxyFor = 3 * time.Minute
+
+// cmdWatchdog runs one pass of the platform watchdog (internal/watchdog): it
+// checks the cluster, PostgreSQL, the game proxy, the database backups and the
+// host, prints every finding, and mails the platform owners what came due.
+// deploy/bootstrap.sh runs it every two minutes from felis-watchdog.timer.
+func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
+	statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")
+	quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
+	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
+	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/postgresql,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
+	proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)
+	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")
+	offsiteStatus := fs.String("offsite-status", offsite.DefaultStatusFile, "the record `felis offsite sync` leaves, checked when [offsite] is configured")
+	toolsStatus := fs.String("build-tools-status", defaultBuildToolsStatus, "the record `felis mirror-build-tools` leaves, checked when builds scan against the registry's DB copy")
+	dryRun := fs.Bool("dry-run", false, "print every finding and the mail that is due; send nothing and keep the state as it was")
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	cfg, err := config.Load(*cfgPath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis watchdog: %v\n", err)
+		return 1
+	}
+	state, err := watchdog.LoadState(*statePath)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis watchdog: %v\n", err)
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+	now := time.Now()
+
+	var report watchdog.Report
+	add := func(f *watchdog.Finding) {
+		if f != nil {
+			report.Findings = append(report.Findings, *f)
+		}
+	}
+
+	// The cluster: one unreachable API server stands in for every check behind it.
+	minecraftNS := cfg.K8s.Namespace
+	if minecraftNS == "" {
+		minecraftNS = platform.DefaultMinecraftNamespace
+	}
+	cl, err := buildSystemServerClient()
+	var found []watchdog.Finding
+	if err == nil {
+		found, err = watchdog.Cluster{Client: cl, ControlNamespace: *controlNS, MinecraftNamespace: minecraftNS}.Check(ctx, now)
+	}
+	if err != nil {
+		f := watchdog.KubeAPIDown(err)
+		add(&f)
+		report.Unknown = append(report.Unknown, watchdog.ClusterPrefixes...)
+	} else {
+		report.Findings = append(report.Findings, found...)
+		if cfg.SMTP.Host != "" {
+			refreshSMTPPassword(ctx, cl, *controlNS, state, stderr)
+		}
+	}
+
+	if recipients, err := ownerEmails(ctx, cfg.Database.URL); err != nil {
+		f := watchdog.PostgresDown(err)
+		add(&f)
+	} else {
+		state.Recipients = recipients
+	}
+
+	if *proxyAddr != "" {
+		add(proxyFinding(ctx, *proxyAddr))
+	}
+	if *backupDir != "" {
+		add(watchdog.BackupFinding(*backupDir, now))
+	}
+	if cfg.Offsite.Enabled() {
+		add(watchdog.OffsiteFinding(*offsiteStatus, now))
+	}
+	if usesMirroredScanDB(cfg) {
+		add(watchdog.ScanDBFinding(*toolsStatus, now))
+	}
+	report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(*diskPaths))...)
+	add(watchdog.MemoryFinding("/proc/meminfo"))
+
+	if len(report.Findings) == 0 {
+		fmt.Fprintln(stdout, "felis watchdog: every check passed")
+	}
+	for _, f := range report.Findings {
+		fmt.Fprintf(stdout, "felis watchdog: [%s] %s: %s\n", f.Severity, f.Key, f.SummaryEN)
+	}
+
+	plan := state.Observe(report, now)
+	host, _ := os.Hostname()
+	subject, body := plan.Message(host, now)
+	if *dryRun {
+		if plan.Empty() {
+			fmt.Fprintln(stdout, "felis watchdog: nothing is due to be mailed")
+		} else {
+			fmt.Fprintf(stdout, "felis watchdog: due to be mailed to %s:\nSubject: %s\n\n%s", strings.Join(state.Recipients, ", "), subject, strings.ReplaceAll(body, "\r\n", "\n"))
+		}
+		return 0
+	}
+
+	save := func() int {
+		if err := watchdog.SaveState(*statePath, state); err != nil {
+			fmt.Fprintf(stderr, "felis watchdog: save state: %v\n", err)
+			return 1
+		}
+		return 0
+	}
+	if plan.Empty() {
+		return save()
+	}
+	if until := watchdog.QuietUntil(*quietPath); now.Before(until) {
+		fmt.Fprintf(stdout, "felis watchdog: quiet until %s (installer running); holding this mail: %s\n", until.UTC().Format(time.RFC3339), subject)
+		return save()
+	}
+	switch {
+	case cfg.SMTP.Host == "":
+		fmt.Fprintf(stdout, "felis watchdog: no [smtp] relay configured, so this is logged only: %s\n", subject)
+	case len(state.Recipients) == 0:
+		fmt.Fprintf(stdout, "felis watchdog: no owner account has a verified email, so this is logged only: %s\n", subject)
+	default:
+		if err := sendAlert(ctx, cfg, state, subject, body); err != nil {
+			// Not committed: the same alerts come due again next run.
+			fmt.Fprintf(stderr, "felis watchdog: mail %q: %v\n", subject, err)
+			save()
+			return 1
+		}
+		fmt.Fprintf(stdout, "felis watchdog: mailed %s: %s\n", strings.Join(state.Recipients, ", "), subject)
+	}
+	state.Commit(plan, now)
+	return save()
+}
+
+// usesMirroredScanDB reports whether build scans read the vulnerability DB copy
+// felis mirror-build-tools keeps in the platform registry: the default, or an
+// explicit trivy_db_repository under the registry's mirror/.
+func usesMirroredScanDB(cfg *config.Config) bool {
+	if cfg.Registry.URL == "" {
+		return false
+	}
+	repo := cfg.Registry.TrivyDBRepository
+	return repo == "" || strings.HasPrefix(repo, cfg.Registry.URL+"/mirror/")
+}
+
+// refreshSMTPPassword caches the relay password from the felis-smtp Secret, or
+// forgets it when the Secret is gone (a relay without AUTH). An env var named by
+// [smtp] password_ref, when set, wins at send time instead.
+func refreshSMTPPassword(ctx context.Context, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) {
+	var sec corev1.Secret
+	err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: platform.SMTPSecretName}, &sec)
+	switch {
+	case apierrors.IsNotFound(err):
+		state.SMTPPassword = ""
+	case err != nil:
+		fmt.Fprintf(stderr, "felis watchdog: read %s/%s (keeping the cached relay password): %v\n", ns, platform.SMTPSecretName, err)
+	default:
+		state.SMTPPassword = string(sec.Data[platform.SMTPSecretPasswordKey])
+	}
+}
+
+// ownerEmails pings PostgreSQL and returns the verified addresses of the
+// enabled owner accounts, the people who can act on an alert.
+func ownerEmails(ctx context.Context, url string) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+	drv, err := store.Open(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	defer drv.Close()
+	rows, err := drv.DB().QueryContext(ctx,
+		`SELECT email FROM users
+		 WHERE role = 'owner' AND email_verified AND COALESCE(email, '') <> ''
+		   AND NOT disabled AND deleted_at IS NULL
+		 ORDER BY email`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []string
+	for rows.Next() {
+		var email sql.NullString
+		if err := rows.Scan(&email); err != nil {
+			return nil, err
+		}
+		out = append(out, email.String)
+	}
+	return out, rows.Err()
+}
+
+// proxyFinding dials the game proxy; players reach every server through it.
+func proxyFinding(ctx context.Context, addr string) *watchdog.Finding {
+	d := net.Dialer{Timeout: 5 * time.Second}
+	conn, err := d.DialContext(ctx, "tcp", addr)
+	if err == nil {
+		conn.Close()
+		return nil
+	}
+	return &watchdog.Finding{
+		Key: "proxy", Severity: watchdog.Critical, For: proxyFor,
+		Summary:   fmt.Sprintf("游戏代理 %s 无法连接：玩家进不了任何服务器", addr),
+		SummaryEN: fmt.Sprintf("the game proxy at %s refuses connections: players cannot reach any server", addr),
+		Hint:      fmt.Sprintf("systemctl status felis-velocity; journalctl -u felis-velocity -n 200 (%v)", err),
+	}
+}
+
+// sendAlert mails subject/body to every recipient; it fails only when no
+// recipient got it.
+func sendAlert(ctx context.Context, cfg *config.Config, state *watchdog.State, subject, body string) error {
+	password := state.SMTPPassword
+	if ref := cfg.SMTP.PasswordRef; ref != "" && os.Getenv(ref) != "" {
+		password = os.Getenv(ref)
+	}
+	relay := smtpRelay(cfg.SMTP, password)
+	var errs []error
+	for _, to := range state.Recipients {
+		if err := relay.SendNotice(ctx, to, subject, body); err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", to, err))
+		}
+	}
+	if len(errs) == len(state.Recipients) {
+		return errors.Join(errs...)
+	}
+	return nil
+}
+
+func splitList(s string) []string {
+	var out []string
+	for _, p := range strings.Split(s, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}

cmd/felis/watchdog_test.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"context"
+	"net"
+	"strings"
+	"testing"
+)
+
+// TestProxyFinding: a listening proxy is healthy; a closed port is the critical
+// "players cannot reach any server" finding.
+func TestProxyFinding(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := ln.Addr().String()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			c.Close()
+		}
+	}()
+	if f := proxyFinding(context.Background(), addr); f != nil {
+		t.Fatalf("listening proxy reported: %+v", f)
+	}
+	ln.Close()
+	f := proxyFinding(context.Background(), addr)
+	if f == nil || f.Key != "proxy" || !strings.Contains(f.SummaryEN, addr) {
+		t.Fatalf("closed proxy = %+v, want the proxy finding", f)
+	}
+}
+
+func TestSplitList(t *testing.T) {
+	got := splitList(" /, /var/lib/felis ,,")
+	if strings.Join(got, "|") != "/|/var/lib/felis" {
+		t.Fatalf("splitList = %q", got)
+	}
+}

deploy/alerts/felis-alerts.yaml

@@ -1,10 +1,21 @@
 # Felis alert rules — plain Prometheus format (also the promtool-tested source
 # for felis-prometheusrule.yaml). See docs/troubleshooting.md §14 for scraping
-# and loading instructions.
+# and loading instructions. These are for a deployment that brings its own
+# Prometheus; every install already runs `felis watchdog` on the host, which
+# checks the same conditions without one and mails the owners (§14).
 #
-# felis_* series come from two processes:
+# felis_* series come from these processes:
-#   - felis-operator pod :8080/metrics      → felis_servers_total, felis_start_duration_seconds
+#   - felis-operator-metrics Service :8080  → felis_servers_total, felis_server_phase,
-#   - felis-api internal :8081/metrics      → felis_image_build_failures_total
+#                                             felis_start_duration_seconds,
+#                                             felis_build_info{component="operator"},
+#                                             controller_runtime_*, workqueue_*
+#   - felis-api-internal Service :8081      → felis_image_build_failures_total,
+#                                             felis_mail_total, felis_rate_limited_total,
+#                                             felis_auth_otp_lockouts_total,
+#                                             felis_auth_failures_total,
+#                                             felis_audit_write_failures_total,
+#                                             felis_build_info{component="api"}
+#   - node-exporter textfile collector      → felis_db_backup_* (felis-db-backup.timer)
 # node_* / kube_* series come from node-exporter / kube-state-metrics.
 groups:
   - name: felis.rules
@@ -32,6 +43,116 @@ groups:
             observed when readiness is first reached). A start that never completes
             records nothing — cross-check desiredState=Running servers with no ready
             phase (troubleshooting §1).
+  - name: felis.platform.rules
+    rules:
+      - alert: FelisOperatorDown
+        expr: absent(felis_build_info{component="operator"})
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "felis-operator is down or not scraped"
+          description: >-
+            No felis_build_info{component="operator"} series for 10 minutes. Without
+            the operator no server starts, stops or recovers. Check
+            `kubectl -n felis get deploy felis-operator` and its log; if the pod is
+            healthy, the felis-operator-metrics Service is not being scraped
+            (troubleshooting §14).
+      - alert: FelisAPIDown
+        expr: absent(felis_build_info{component="api"})
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "felis-api is down or not scraped"
+          description: >-
+            No felis_build_info{component="api"} series for 10 minutes. The panel,
+            sign-in and the proxy's player lookups all go through felis-api. Check
+            `kubectl -n felis get deploy felis-api` and its log; if the pod is
+            healthy, the felis-api-internal Service is not being scraped
+            (troubleshooting §14).
+      - alert: FelisLoginGateDown
+        expr: felis_server_phase{role="login",desired="Running",phase!="Running"} == 1
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "the login gate {{ $labels.server }} is {{ $labels.phase }}"
+          description: >-
+            Every player connection passes through the login server first, so no one
+            can join. The MinecraftServer's conditions carry the reason:
+            `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+            (troubleshooting §1, §2).
+      - alert: FelisSystemServerDown
+        expr: felis_server_phase{role!="",role!="login",desired="Running",phase!="Running"} == 1
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "system server {{ $labels.server }} ({{ $labels.role }}) is {{ $labels.phase }}"
+          description: >-
+            Players who sign in are sent to the lobby; while it is down they stay at
+            the gate. `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+            shows the reason (troubleshooting §1, §2).
+      - alert: FelisServerFailed
+        expr: felis_server_phase{role="",phase="Failed"} == 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "server {{ $labels.server }} is Failed"
+          description: >-
+            The operator gave up on this server (a crash loop, an image that will not
+            pull, a world volume that will not mount). Its conditions carry the
+            reason: `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+            (troubleshooting §2).
+      - alert: FelisReconcileErrors
+        expr: sum(increase(controller_runtime_reconcile_errors_total{controller="minecraftserver"}[15m])) > 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "the operator failed over 10 reconciles in 15 minutes"
+          description: >-
+            Server changes are being retried instead of applied. The felis-operator
+            log names each failing server and its error.
+      - alert: FelisReconcileStuck
+        expr: max(workqueue_longest_running_processor_seconds{name="minecraftserver"}) > 300
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "an operator reconcile has been running for over 5 minutes"
+          description: >-
+            Each reconcile is bounded at 3 minutes, so this one is ignoring its
+            deadline and holding a worker. The liveness probe restarts the operator
+            once a pass passes 10 minutes; the log from before the restart shows
+            where it hung.
+  - name: felis.jobs.rules
+    rules:
+      - alert: FelisWorldJobFailed
+        expr: kube_job_failed{namespace="minecraft",condition="true"} == 1
+        labels:
+          severity: warning
+        annotations:
+          summary: "Job {{ $labels.job_name }} failed"
+          description: >-
+            A world backup, restore or reaper run failed; after a failed backup that
+            world's newest archive is older than planned.
+            `kubectl -n minecraft logs job/{{ $labels.job_name }}` has the error
+            (troubleshooting §10).
+      - alert: FelisReaperStale
+        expr: time() - kube_cronjob_status_last_successful_time{namespace="minecraft",cronjob="felis-reaper"} > 26 * 3600
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "the world reaper has not succeeded in over 26h"
+          description: >-
+            felis-reaper runs daily; idle worlds are neither backed up nor reclaimed
+            while it fails. `kubectl -n minecraft get jobs --sort-by=.metadata.creationTimestamp`
+            lists its runs, and the newest one's log
+            shows why (troubleshooting §10).
   - name: felis.node.rules
     rules:
       - alert: FelisNodeDiskSpaceLow
@@ -67,3 +188,97 @@ groups:
           description: >-
             PostgreSQL, the control plane, the registry and game servers share one
             node; sustained memory pressure risks OOM kills.
+  - name: felis.backup.rules
+    rules:
+      - alert: FelisDBBackupStale
+        expr: time() - max(felis_db_backup_last_success_timestamp_seconds) > 26 * 3600
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "no control-plane database backup in over 26h"
+          description: >-
+            felis-db-backup.timer runs daily; the newest bundle is more than a day
+            old. Read `journalctl -u felis-db-backup` on the host, then take one now
+            with `sudo felis db backup` (troubleshooting §16).
+      - alert: FelisDBBackupMetricMissing
+        expr: absent(felis_db_backup_last_success_timestamp_seconds)
+        for: 2h
+        labels:
+          severity: warning
+        annotations:
+          summary: "database backup freshness is not being scraped"
+          description: >-
+            No felis_db_backup_last_success_timestamp_seconds series, so
+            FelisDBBackupStale cannot fire. Point node-exporter's
+            --collector.textfile.directory at the directory of
+            FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
+            (troubleshooting §16).
+  - name: felis.auth.rules
+    rules:
+      - alert: FelisMailBudgetExhausted
+        expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "the install-wide mail budget refused mail"
+          description: >-
+            felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
+            spent, and every sign-in code is refused with 429 mail_rate_limited until
+            it refills. Check felis_rate_limited_total for a flood before raising the
+            budget (troubleshooting §17).
+      - alert: FelisMailDeliveryFailing
+        expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "the SMTP relay refused mail in the last 15m"
+          description: >-
+            felis_mail_total{result="failed"} increased: sign-in codes are not being
+            delivered (502 mail_undeliverable). The relay's reason is in the
+            felis-api log (troubleshooting §17).
+      - alert: FelisSignInFlood
+        expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "sign-in doors refusing over 10 requests a minute"
+          description: >-
+            The per-address sign-in limit has been refusing callers for 10 minutes.
+            A script is hammering the auth doors; if real users report rate_limited
+            at once instead, [auth] client_ip_header is missing and everyone shares
+            the proxy's address (troubleshooting §17).
+      - alert: FelisOTPAccountLocked
+        expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "an account's email-code sign-in locked after 10 wrong codes"
+          description: >-
+            Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}).
+            The audit log names the account (action auth.otp.locked); the owner was
+            mailed. Unless they fumbled codes, someone is guessing at it
+            (troubleshooting §17).
+      - alert: FelisSignInFailures
+        expr: sum(increase(felis_auth_failures_total[15m])) > 30
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "over 30 refused sign-ins in 15 minutes"
+          description: >-
+            Wrong codes, unknown addresses or bad passkey assertions well above people
+            mistyping: someone is guessing or enumerating. `sum by (door, reason)
+            (increase(felis_auth_failures_total[15m]))` shows where; the audit rows
+            (action auth.<door>.failed) carry each caller's client_ip (troubleshooting §17).
+      - alert: FelisAuditWriteFailing
+        expr: increase(felis_audit_write_failures_total[15m]) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "felis-api failed to write audit rows"
+          description: >-
+            The actions went through but their audit rows were lost. The felis-api log
+            names each lost row (`audit: lost ...`); the usual cause is PostgreSQL
+            being unreachable or out of disk.

deploy/alerts/felis-alerts_test.yml

@@ -105,3 +105,405 @@ tests:
               description: >-
                 PostgreSQL, the control plane, the registry and game servers share one
                 node; sustained memory pressure risks OOM kills.
+  - name: database backup freshness
+    interval: 1m
+    input_series:
+      # The newest bundle was taken at t=0 and none since.
+      - series: 'felis_db_backup_last_success_timestamp_seconds{instance="node1",job="node-exporter",label="daily"}'
+        values: '0x1630'
+    alert_rule_test:
+      - eval_time: 25h
+        alertname: FelisDBBackupStale
+        exp_alerts: []
+      - eval_time: 27h
+        alertname: FelisDBBackupStale
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+            exp_annotations:
+              summary: "no control-plane database backup in over 26h"
+              description: >-
+                felis-db-backup.timer runs daily; the newest bundle is more than a day
+                old. Read `journalctl -u felis-db-backup` on the host, then take one now
+                with `sudo felis db backup` (troubleshooting §16).
+      - eval_time: 27h
+        alertname: FelisDBBackupMetricMissing
+        exp_alerts: []
+  - name: database backup freshness not scraped
+    interval: 1m
+    input_series:
+      - series: 'up{job="node-exporter"}'
+        values: '1x200'
+    alert_rule_test:
+      - eval_time: 1h
+        alertname: FelisDBBackupMetricMissing
+        exp_alerts: []
+      - eval_time: 3h
+        alertname: FelisDBBackupMetricMissing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "database backup freshness is not being scraped"
+              description: >-
+                No felis_db_backup_last_success_timestamp_seconds series, so
+                FelisDBBackupStale cannot fire. Point node-exporter's
+                --collector.textfile.directory at the directory of
+                FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
+                (troubleshooting §16).
+  - name: sign-in mail budget and relay
+    interval: 1m
+    input_series:
+      # Created at zero on start; the budget refuses one mail at t=3m.
+      - series: 'felis_mail_total{kind="otp",result="throttled",job="felis-api"}'
+        values: '0 0 0 1x30'
+      - series: 'felis_mail_total{kind="otp",result="failed",job="felis-api"}'
+        values: '0x33'
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: FelisMailBudgetExhausted
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: FelisMailBudgetExhausted
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "the install-wide mail budget refused mail"
+              description: >-
+                felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
+                spent, and every sign-in code is refused with 429 mail_rate_limited until
+                it refills. Check felis_rate_limited_total for a flood before raising the
+                budget (troubleshooting §17).
+      - eval_time: 5m
+        alertname: FelisMailDeliveryFailing
+        exp_alerts: []
+  - name: sign-in flood
+    interval: 1m
+    input_series:
+      # 30 refusals a minute from t=0; a lone refused script at 2/min stays quiet.
+      - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}'
+        values: '0+30x40'
+    alert_rule_test:
+      - eval_time: 10m
+        alertname: FelisSignInFlood
+        exp_alerts: []
+      - eval_time: 20m
+        alertname: FelisSignInFlood
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "sign-in doors refusing over 10 requests a minute"
+              description: >-
+                The per-address sign-in limit has been refusing callers for 10 minutes.
+                A script is hammering the auth doors; if real users report rate_limited
+                at once instead, [auth] client_ip_header is missing and everyone shares
+                the proxy's address (troubleshooting §17).
+  - name: sign-in trickle stays quiet
+    interval: 1m
+    input_series:
+      - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}'
+        values: '0+2x40'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: FelisSignInFlood
+        exp_alerts: []
+  - name: account email-code lock
+    interval: 1m
+    input_series:
+      - series: 'felis_auth_otp_lockouts_total{purpose="login_email",job="felis-api"}'
+        values: '0 0 1x90'
+      - series: 'felis_auth_otp_lockouts_total{purpose="op_login",job="felis-api"}'
+        values: '0x92'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: FelisOTPAccountLocked
+        exp_alerts: []
+      - eval_time: 10m
+        alertname: FelisOTPAccountLocked
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              purpose: login_email
+            exp_annotations:
+              summary: "an account's email-code sign-in locked after 10 wrong codes"
+              description: >-
+                Someone entered 10 wrong codes for one account within 24h (login_email).
+                The audit log names the account (action auth.otp.locked); the owner was
+                mailed. Unless they fumbled codes, someone is guessing at it
+                (troubleshooting §17).
+      - eval_time: 90m
+        alertname: FelisOTPAccountLocked
+        exp_alerts: []
+  - name: sign-in failure rate
+    interval: 1m
+    input_series:
+      # Two doors failing at 3/min between them from t=0.
+      - series: 'felis_auth_failures_total{door="login_email",reason="bad_code",job="felis-api"}'
+        values: '0+2x40'
+      - series: 'felis_auth_failures_total{door="op_login",reason="no_account",job="felis-api"}'
+        values: '0+1x40'
+    alert_rule_test:
+      - eval_time: 8m
+        alertname: FelisSignInFailures
+        exp_alerts: []
+      - eval_time: 25m
+        alertname: FelisSignInFailures
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "over 30 refused sign-ins in 15 minutes"
+              description: >-
+                Wrong codes, unknown addresses or bad passkey assertions well above people
+                mistyping: someone is guessing or enumerating. `sum by (door, reason)
+                (increase(felis_auth_failures_total[15m]))` shows where; the audit rows
+                (action auth.<door>.failed) carry each caller's client_ip (troubleshooting §17).
+  - name: people mistyping stays quiet
+    interval: 1m
+    input_series:
+      - series: 'felis_auth_failures_total{door="login_email",reason="bad_code",job="felis-api"}'
+        values: '0 0 1 1 2 2 3 3 4 4 5x30'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: FelisSignInFailures
+        exp_alerts: []
+  - name: audit rows lost
+    interval: 1m
+    input_series:
+      - series: 'felis_audit_write_failures_total{job="felis-api",instance="api-0"}'
+        values: '0 0 0 2x20'
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: FelisAuditWriteFailing
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: FelisAuditWriteFailing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              job: felis-api
+              instance: api-0
+            exp_annotations:
+              summary: "felis-api failed to write audit rows"
+              description: >-
+                The actions went through but their audit rows were lost. The felis-api log
+                names each lost row (`audit: lost ...`); the usual cause is PostgreSQL
+                being unreachable or out of disk.
+  - name: operator and api presence
+    interval: 1m
+    input_series:
+      - series: 'felis_build_info{component="operator",version="v1",job="felis-operator",instance="op-0"}'
+        values: '1x30'
+      # felis-api stops being scraped after 5m; the series goes stale 5m later.
+      - series: 'felis_build_info{component="api",version="v1",job="felis-api",instance="api-0"}'
+        values: '1x5'
+    alert_rule_test:
+      - eval_time: 25m
+        alertname: FelisOperatorDown
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: FelisAPIDown
+        exp_alerts: []
+      - eval_time: 25m
+        alertname: FelisAPIDown
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              component: api
+            exp_annotations:
+              summary: "felis-api is down or not scraped"
+              description: >-
+                No felis_build_info{component="api"} series for 10 minutes. The panel,
+                sign-in and the proxy's player lookups all go through felis-api. Check
+                `kubectl -n felis get deploy felis-api` and its log; if the pod is
+                healthy, the felis-api-internal Service is not being scraped
+                (troubleshooting §14).
+  - name: operator never scraped
+    interval: 1m
+    input_series:
+      - series: 'felis_build_info{component="api",version="v1",job="felis-api",instance="api-0"}'
+        values: '1x30'
+    alert_rule_test:
+      - eval_time: 5m
+        alertname: FelisOperatorDown
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: FelisOperatorDown
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              component: operator
+            exp_annotations:
+              summary: "felis-operator is down or not scraped"
+              description: >-
+                No felis_build_info{component="operator"} series for 10 minutes. Without
+                the operator no server starts, stops or recovers. Check
+                `kubectl -n felis get deploy felis-operator` and its log; if the pod is
+                healthy, the felis-operator-metrics Service is not being scraped
+                (troubleshooting §14).
+  - name: system and user servers down
+    interval: 1m
+    input_series:
+      - series: 'felis_server_phase{server="login",role="login",phase="Starting",desired="Running"}'
+        values: '1x20'
+      - series: 'felis_server_phase{server="lobby",role="lobby",phase="Failed",desired="Running"}'
+        values: '1x20'
+      # Stopped on purpose: not an outage.
+      - series: 'felis_server_phase{server="lobby2",role="lobby",phase="Stopped",desired="Stopped"}'
+        values: '1x20'
+      # A user server carries no role label (the operator publishes role="").
+      - series: 'felis_server_phase{server="survival",phase="Failed",desired="Running"}'
+        values: '1x20'
+      - series: 'felis_server_phase{server="creative",phase="Running",desired="Running"}'
+        values: '1x20'
+    alert_rule_test:
+      - eval_time: 9m
+        alertname: FelisLoginGateDown
+        exp_alerts: []
+      - eval_time: 11m
+        alertname: FelisLoginGateDown
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              server: login
+              role: login
+              phase: Starting
+              desired: Running
+            exp_annotations:
+              summary: "the login gate login is Starting"
+              description: >-
+                Every player connection passes through the login server first, so no one
+                can join. The MinecraftServer's conditions carry the reason:
+                `kubectl -n minecraft describe minecraftserver login`
+                (troubleshooting §1, §2).
+      - eval_time: 11m
+        alertname: FelisSystemServerDown
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              server: lobby
+              role: lobby
+              phase: Failed
+              desired: Running
+            exp_annotations:
+              summary: "system server lobby (lobby) is Failed"
+              description: >-
+                Players who sign in are sent to the lobby; while it is down they stay at
+                the gate. `kubectl -n minecraft describe minecraftserver lobby`
+                shows the reason (troubleshooting §1, §2).
+      - eval_time: 3m
+        alertname: FelisServerFailed
+        exp_alerts: []
+      - eval_time: 6m
+        alertname: FelisServerFailed
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              server: survival
+              phase: Failed
+              desired: Running
+            exp_annotations:
+              summary: "server survival is Failed"
+              description: >-
+                The operator gave up on this server (a crash loop, an image that will not
+                pull, a world volume that will not mount). Its conditions carry the
+                reason: `kubectl -n minecraft describe minecraftserver survival`
+                (troubleshooting §2).
+  - name: operator reconcile errors and a stuck pass
+    interval: 1m
+    input_series:
+      # Two failed reconciles a minute from 6m on.
+      - series: 'controller_runtime_reconcile_errors_total{controller="minecraftserver",job="felis-operator"}'
+        values: '0x5 0+2x20'
+      # One pass that started at 5m and never returns.
+      - series: 'workqueue_longest_running_processor_seconds{name="minecraftserver",controller="minecraftserver",job="felis-operator"}'
+        values: '0x5 60+60x20'
+    alert_rule_test:
+      - eval_time: 5m
+        alertname: FelisReconcileErrors
+        exp_alerts: []
+      - eval_time: 25m
+        alertname: FelisReconcileErrors
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              summary: "the operator failed over 10 reconciles in 15 minutes"
+              description: >-
+                Server changes are being retried instead of applied. The felis-operator
+                log names each failing server and its error.
+      - eval_time: 12m
+        alertname: FelisReconcileStuck
+        exp_alerts: []
+      - eval_time: 20m
+        alertname: FelisReconcileStuck
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+            exp_annotations:
+              summary: "an operator reconcile has been running for over 5 minutes"
+              description: >-
+                Each reconcile is bounded at 3 minutes, so this one is ignoring its
+                deadline and holding a worker. The liveness probe restarts the operator
+                once a pass passes 10 minutes; the log from before the restart shows
+                where it hung.
+  - name: world job failures and a late reaper
+    interval: 1m
+    input_series:
+      - series: 'kube_job_failed{namespace="minecraft",job_name="backup-survival-abc",condition="true"}'
+        values: '0x2 1x10'
+      - series: 'kube_job_failed{namespace="minecraft",job_name="backup-survival-abc",condition="false"}'
+        values: '1x2 0x10'
+      # A build Job in another namespace is FelisImageBuildFailures' business.
+      - series: 'kube_job_failed{namespace="felis-build",job_name="build-x",condition="true"}'
+        values: '1x12'
+      # Evaluation starts at the epoch, so "over a day ago" is a negative timestamp.
+      - series: 'kube_cronjob_status_last_successful_time{namespace="minecraft",cronjob="felis-reaper"}'
+        values: '-100000x30'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: FelisWorldJobFailed
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: FelisWorldJobFailed
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              namespace: minecraft
+              job_name: backup-survival-abc
+              condition: "true"
+            exp_annotations:
+              summary: "Job backup-survival-abc failed"
+              description: >-
+                A world backup, restore or reaper run failed; after a failed backup that
+                world's newest archive is older than planned.
+                `kubectl -n minecraft logs job/backup-survival-abc` has the error
+                (troubleshooting §10).
+      - eval_time: 5m
+        alertname: FelisReaperStale
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: FelisReaperStale
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              namespace: minecraft
+              cronjob: felis-reaper
+            exp_annotations:
+              summary: "the world reaper has not succeeded in over 26h"
+              description: >-
+                felis-reaper runs daily; idle worlds are neither backed up nor reclaimed
+                while it fails. `kubectl -n minecraft get jobs --sort-by=.metadata.creationTimestamp`
+                lists its runs, and the newest one's log
+                shows why (troubleshooting §10).
+  - name: a reaper that ran yesterday stays quiet
+    interval: 1m
+    input_series:
+      - series: 'kube_cronjob_status_last_successful_time{namespace="minecraft",cronjob="felis-reaper"}'
+        values: '-50000x30'
+    alert_rule_test:
+      - eval_time: 25m
+        alertname: FelisReaperStale
+        exp_alerts: []

deploy/alerts/felis-prometheusrule.yaml

@@ -37,6 +37,116 @@ spec:
               observed when readiness is first reached). A start that never completes
               records nothing — cross-check desiredState=Running servers with no ready
               phase (troubleshooting §1).
+    - name: felis.platform.rules
+      rules:
+        - alert: FelisOperatorDown
+          expr: absent(felis_build_info{component="operator"})
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "felis-operator is down or not scraped"
+            description: >-
+              No felis_build_info{component="operator"} series for 10 minutes. Without
+              the operator no server starts, stops or recovers. Check
+              `kubectl -n felis get deploy felis-operator` and its log; if the pod is
+              healthy, the felis-operator-metrics Service is not being scraped
+              (troubleshooting §14).
+        - alert: FelisAPIDown
+          expr: absent(felis_build_info{component="api"})
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "felis-api is down or not scraped"
+            description: >-
+              No felis_build_info{component="api"} series for 10 minutes. The panel,
+              sign-in and the proxy's player lookups all go through felis-api. Check
+              `kubectl -n felis get deploy felis-api` and its log; if the pod is
+              healthy, the felis-api-internal Service is not being scraped
+              (troubleshooting §14).
+        - alert: FelisLoginGateDown
+          expr: felis_server_phase{role="login",desired="Running",phase!="Running"} == 1
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "the login gate {{ $labels.server }} is {{ $labels.phase }}"
+            description: >-
+              Every player connection passes through the login server first, so no one
+              can join. The MinecraftServer's conditions carry the reason:
+              `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+              (troubleshooting §1, §2).
+        - alert: FelisSystemServerDown
+          expr: felis_server_phase{role!="",role!="login",desired="Running",phase!="Running"} == 1
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "system server {{ $labels.server }} ({{ $labels.role }}) is {{ $labels.phase }}"
+            description: >-
+              Players who sign in are sent to the lobby; while it is down they stay at
+              the gate. `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+              shows the reason (troubleshooting §1, §2).
+        - alert: FelisServerFailed
+          expr: felis_server_phase{role="",phase="Failed"} == 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "server {{ $labels.server }} is Failed"
+            description: >-
+              The operator gave up on this server (a crash loop, an image that will not
+              pull, a world volume that will not mount). Its conditions carry the
+              reason: `kubectl -n minecraft describe minecraftserver {{ $labels.server }}`
+              (troubleshooting §2).
+        - alert: FelisReconcileErrors
+          expr: sum(increase(controller_runtime_reconcile_errors_total{controller="minecraftserver"}[15m])) > 10
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "the operator failed over 10 reconciles in 15 minutes"
+            description: >-
+              Server changes are being retried instead of applied. The felis-operator
+              log names each failing server and its error.
+        - alert: FelisReconcileStuck
+          expr: max(workqueue_longest_running_processor_seconds{name="minecraftserver"}) > 300
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "an operator reconcile has been running for over 5 minutes"
+            description: >-
+              Each reconcile is bounded at 3 minutes, so this one is ignoring its
+              deadline and holding a worker. The liveness probe restarts the operator
+              once a pass passes 10 minutes; the log from before the restart shows
+              where it hung.
+    - name: felis.jobs.rules
+      rules:
+        - alert: FelisWorldJobFailed
+          expr: kube_job_failed{namespace="minecraft",condition="true"} == 1
+          labels:
+            severity: warning
+          annotations:
+            summary: "Job {{ $labels.job_name }} failed"
+            description: >-
+              A world backup, restore or reaper run failed; after a failed backup that
+              world's newest archive is older than planned.
+              `kubectl -n minecraft logs job/{{ $labels.job_name }}` has the error
+              (troubleshooting §10).
+        - alert: FelisReaperStale
+          expr: time() - kube_cronjob_status_last_successful_time{namespace="minecraft",cronjob="felis-reaper"} > 26 * 3600
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "the world reaper has not succeeded in over 26h"
+            description: >-
+              felis-reaper runs daily; idle worlds are neither backed up nor reclaimed
+              while it fails. `kubectl -n minecraft get jobs --sort-by=.metadata.creationTimestamp`
+              lists its runs, and the newest one's log
+              shows why (troubleshooting §10).
     - name: felis.node.rules
       rules:
         - alert: FelisNodeDiskSpaceLow
@@ -72,3 +182,97 @@ spec:
             description: >-
               PostgreSQL, the control plane, the registry and game servers share one
               node; sustained memory pressure risks OOM kills.
+    - name: felis.backup.rules
+      rules:
+        - alert: FelisDBBackupStale
+          expr: time() - max(felis_db_backup_last_success_timestamp_seconds) > 26 * 3600
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "no control-plane database backup in over 26h"
+            description: >-
+              felis-db-backup.timer runs daily; the newest bundle is more than a day
+              old. Read `journalctl -u felis-db-backup` on the host, then take one now
+              with `sudo felis db backup` (troubleshooting §16).
+        - alert: FelisDBBackupMetricMissing
+          expr: absent(felis_db_backup_last_success_timestamp_seconds)
+          for: 2h
+          labels:
+            severity: warning
+          annotations:
+            summary: "database backup freshness is not being scraped"
+            description: >-
+              No felis_db_backup_last_success_timestamp_seconds series, so
+              FelisDBBackupStale cannot fire. Point node-exporter's
+              --collector.textfile.directory at the directory of
+              FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
+              (troubleshooting §16).
+    - name: felis.auth.rules
+      rules:
+        - alert: FelisMailBudgetExhausted
+          expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "the install-wide mail budget refused mail"
+            description: >-
+              felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
+              spent, and every sign-in code is refused with 429 mail_rate_limited until
+              it refills. Check felis_rate_limited_total for a flood before raising the
+              budget (troubleshooting §17).
+        - alert: FelisMailDeliveryFailing
+          expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "the SMTP relay refused mail in the last 15m"
+            description: >-
+              felis_mail_total{result="failed"} increased: sign-in codes are not being
+              delivered (502 mail_undeliverable). The relay's reason is in the
+              felis-api log (troubleshooting §17).
+        - alert: FelisSignInFlood
+          expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "sign-in doors refusing over 10 requests a minute"
+            description: >-
+              The per-address sign-in limit has been refusing callers for 10 minutes.
+              A script is hammering the auth doors; if real users report rate_limited
+              at once instead, [auth] client_ip_header is missing and everyone shares
+              the proxy's address (troubleshooting §17).
+        - alert: FelisOTPAccountLocked
+          expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "an account's email-code sign-in locked after 10 wrong codes"
+            description: >-
+              Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}).
+              The audit log names the account (action auth.otp.locked); the owner was
+              mailed. Unless they fumbled codes, someone is guessing at it
+              (troubleshooting §17).
+        - alert: FelisSignInFailures
+          expr: sum(increase(felis_auth_failures_total[15m])) > 30
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "over 30 refused sign-ins in 15 minutes"
+            description: >-
+              Wrong codes, unknown addresses or bad passkey assertions well above people
+              mistyping: someone is guessing or enumerating. `sum by (door, reason)
+              (increase(felis_auth_failures_total[15m]))` shows where; the audit rows
+              (action auth.<door>.failed) carry each caller's client_ip (troubleshooting §17).
+        - alert: FelisAuditWriteFailing
+          expr: increase(felis_audit_write_failures_total[15m]) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "felis-api failed to write audit rows"
+            description: >-
+              The actions went through but their audit rows were lost. The felis-api log
+              names each lost row (`audit: lost ...`); the usual cause is PostgreSQL
+              being unreachable or out of disk.

deploy/crd/felis.lolicon.best_minecraftservers.yaml

@@ -101,6 +101,8 @@ spec:
                       EmptySecondsBeforeStop is how long the server may sit empty before the
                       operator scales it down.
                     format: int32
+                    maximum: 604800
+                    minimum: 0
                     type: integer
                 type: object
               image:
@@ -123,14 +125,12 @@ spec:
               lifecycle:
                 description: Lifecycle tunes graceful shutdown (spec §7).
                 properties:
-                  preStopSaveAndStop:
-                    description: PreStopSaveAndStop enables the operator-injected
-                      RCON save+stop preStop.
-                    type: boolean
                   terminationGracePeriodSeconds:
                     description: TerminationGracePeriodSeconds is the pod grace period
                       (default 300).
                     format: int64
+                    maximum: 3600
+                    minimum: 0
                     type: integer
                 type: object
               motd:
@@ -163,6 +163,8 @@ spec:
                   port:
                     description: Port is the RCON TCP port (default 25575).
                     format: int32
+                    maximum: 65535
+                    minimum: 0
                     type: integer
                   secretRef:
                     description: SecretRef points at the Secret holding the RCON password.
@@ -176,6 +178,10 @@ spec:
                     - name
                     type: object
                 type: object
+                x-kubernetes-validations:
+                - message: the allow-rcon NetworkPolicy admits only port 25575; leave
+                    port unset
+                  rule: '!has(self.port) || self.port == 0 || self.port == 25575'
               reaperExempt:
                 description: ReaperExempt opts this server out of the world reaper
                   entirely (spec §18).
@@ -254,16 +260,22 @@ spec:
                       (notably LOOHP/Limbo) where the felis-limbo plugin reports true
                       readiness only after the first server tick.
                     format: int32
+                    maximum: 65535
+                    minimum: 0
                     type: integer
                   readinessTimeoutSeconds:
                     description: ReadinessTimeoutSeconds is the budget for the first
                       successful RCON probe.
                     format: int32
+                    maximum: 86400
+                    minimum: 0
                     type: integer
                   timeoutSeconds:
                     description: TimeoutSeconds is the overall budget before the server
                       is marked Failed.
                     format: int32
+                    maximum: 86400
+                    minimum: 0
                     type: integer
                 type: object
               storage:
@@ -289,6 +301,13 @@ spec:
           status:
             description: MinecraftServerStatus is the observed state (spec §4 status.*).
             properties:
+              autoRestarts:
+                description: |-
+                  AutoRestarts counts how often the operator recreated the pod of a start
+                  that timed out (at most 3, with a doubling backoff); reaching Ready or
+                  stopping resets it.
+                format: int32
+                type: integer
               conditions:
                 description: Conditions are the standard metav1 conditions (Ready,
                   RconReached, ...).
@@ -365,6 +384,11 @@ spec:
                     description: Mode is "direct" or "fallback".
                     type: string
                 type: object
+              lastAutoRestartAt:
+                description: LastAutoRestartAt is when the operator last recreated
+                  the pod.
+                format: date-time
+                type: string
               liveMotd:
                 description: LiveMotd is the MOTD currently advertised for the active
                   phase.

deploy/e2e_check.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# Checks a host that deploy/bootstrap.sh just installed (or re-ran on). The e2e workflow
+# runs it on a fresh GitHub runner after each of its two installer runs:
+#
+#   sudo bash deploy/e2e_check.sh install   # after the first run
+#   sudo bash deploy/e2e_check.sh rerun     # after the same commit ran again
+#   sudo bash deploy/e2e_check.sh release   # after the newest release installed
+#   sudo bash deploy/e2e_check.sh upgrade   # after this commit ran over a release
+#
+# It asks what an operator's first minutes ask: the binary runs, the control plane is
+# rolled out and ready, the panel answers on its NodePort, the proxy answers a Minecraft
+# status ping, and the host timers are there. A rerun must also leave the proxy running
+# (it restarts only when what it runs changed) and keep every earlier answer.
+set -euo pipefail
+
+phase="${1:?usage: e2e_check.sh install|rerun|release|upgrade}"
+export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+KUBECTL=(/usr/local/bin/k3s kubectl)
+PID_FILE=/var/tmp/felis-e2e-velocity.pid
+fails=0
+
+pass() { printf 'PASS %s\n' "$*"; }
+fail() { printf 'FAIL %s\n' "$*"; fails=$((fails + 1)); }
+check() { # label command...
+  local label="$1"
+  shift
+  if "$@"; then pass "$label"; else fail "$label"; fi
+}
+
+check "felis version runs" sh -c '/usr/local/bin/felis version | grep -q "^felis "'
+
+for d in felis-api felis-operator registry; do
+  check "deployment ${d} is rolled out" "${KUBECTL[@]}" -n felis rollout status "deploy/${d}" --timeout=180s
+done
+
+node_ip="$(ip -4 route get 1.1.1.1 | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}')"
+check "the panel serves its page on the NodePort" \
+  sh -c "curl -skf --retry 10 --retry-delay 3 --retry-all-errors https://${node_ip}:30443/ | grep -qi '<html'"
+# Readiness lives on the internal face only; a Service ClusterIP routes from the node.
+internal="$("${KUBECTL[@]}" -n felis get svc felis-api-internal -o jsonpath='{.spec.clusterIP}:{.spec.ports[0].port}')"
+check "felis-api is ready (database and cluster reachable)" \
+  curl -sf --retry 10 --retry-delay 3 --retry-all-errors -o /dev/null "http://${internal}/readyz"
+
+for unit in k3s postgresql felis-velocity; do
+  check "${unit} is active" systemctl is-active --quiet "$unit"
+done
+# A release may predate a timer; what this commit installs has them all.
+if [ "$phase" != release ]; then
+  for timer in felis-db-backup.timer felis-watchdog.timer felis-update-check.timer; do
+    check "${timer} is scheduled" systemctl is-enabled --quiet "$timer"
+  done
+fi
+
+# A status ping is the proxy's own answer (ping passthrough is off), so it proves the JRE,
+# Velocity and its config without a login gate or a Mojang account.
+ping_proxy() {
+  python3 - <<'EOF'
+import json, socket, struct, sys
+
+def varint(n):
+    out = b""
+    n &= 0xFFFFFFFF
+    while True:
+        b, n = n & 0x7F, n >> 7
+        if n:
+            out += bytes([b | 0x80])
+        else:
+            return out + bytes([b])
+
+def read_varint(s):
+    n = 0
+    for i in range(5):
+        b = s.recv(1)
+        if not b:
+            raise EOFError("connection closed")
+        n |= (b[0] & 0x7F) << (7 * i)
+        if not b[0] & 0x80:
+            return n
+    raise ValueError("varint too long")
+
+host, port = "127.0.0.1", 25565
+s = socket.create_connection((host, port), timeout=10)
+hs = varint(0) + varint(767) + varint(len(host)) + host.encode() + struct.pack(">H", port) + varint(1)
+s.sendall(varint(len(hs)) + hs + varint(1) + varint(0))
+read_varint(s)
+read_varint(s)
+size = read_varint(s)
+data = b""
+while len(data) < size:
+    chunk = s.recv(size - len(data))
+    if not chunk:
+        raise EOFError("short status response")
+    data += chunk
+print(json.loads(data)["version"]["name"])
+EOF
+}
+# The installer returns once the unit is started; the JVM binds the port a few
+# seconds later.
+for _ in $(seq 30); do
+  if version="$(ping_proxy 2>&1)"; then
+    break
+  fi
+  sleep 2
+done
+if version="$(ping_proxy 2>&1)"; then
+  pass "the proxy answers a status ping (${version})"
+else
+  fail "the proxy answers a status ping: ${version}"
+fi
+
+pid="$(systemctl show -p MainPID --value felis-velocity)"
+case "$phase" in
+  install | release) printf '%s\n' "$pid" > "$PID_FILE" ;;
+  rerun)
+    if [ "$pid" = "$(cat "$PID_FILE" 2>/dev/null)" ]; then
+      pass "the rerun left the proxy running (pid ${pid})"
+    else
+      fail "the rerun restarted the proxy (pid $(cat "$PID_FILE" 2>/dev/null || echo '?') -> ${pid}) though nothing it runs changed"
+    fi
+    ;;
+  upgrade) ;; # a new release may well change what the proxy runs
+  *) fail "unknown phase ${phase}" ;;
+esac
+
+if [ "$fails" -eq 0 ]; then
+  echo "ALL PASS (${phase})"
+else
+  echo "${fails} FAILED (${phase})"
+fi
+exit "$fails"

deploy/game-stack.lock

@@ -0,0 +1,23 @@
+# The upstream builds this release installs. deploy/bootstrap.sh reads this file (the
+# default FELIS_GAME_STACK=pinned), downloads exactly these artifacts and refuses any whose
+# sha256 differs, so every host installing one release gets the same login gate, lobby,
+# plain-Paper image and proxy, and a rerun rebuilds nothing that did not change.
+#
+# MC_VERSION is the protocol the whole stack speaks: Limbo speaks exactly one, and Paper
+# follows it so a client that passes the login gate can also reach the lobby.
+#
+# Refresh with deploy/update-game-stack-lock.sh, which resolves upstream's newest builds and
+# hashes them. Plain KEY=value lines only; bootstrap reads it without evaluating it.
+MC_VERSION=26.3
+LIMBO_VERSION=2026.0.3-ALPHA
+LIMBO_JAR_URL=https://ci.loohpjames.com/job/Limbo/76/artifact/target/Limbo-2026.0.3-ALPHA-26.3.jar
+LIMBO_JAR_SHA256=a2de91fcaa2255aed8a111b8786f370213423c7798eee778c00d016d83aa65d2
+LIMBO_SCHEM_URL=https://ci.loohpjames.com/job/Limbo/76/artifact/spawn.schem
+LIMBO_SCHEM_SHA256=70c85dae2db157971ef513e318820c5a5e10a96b813b70e21f8af2b632cbcbc7
+PAPER_JAR_URL=https://fill-data.papermc.io/v1/objects/49399919246cbf443efc8507447dc948eb7477c41be560b0e87e2a455aff824a/paper-26.3-40.jar
+PAPER_JAR_SHA256=49399919246cbf443efc8507447dc948eb7477c41be560b0e87e2a455aff824a
+LUCKPERMS_JAR_URL=https://download.luckperms.net/1672/bukkit/loader/LuckPerms-Bukkit-5.5.85.jar
+LUCKPERMS_JAR_SHA256=dc637ce18f48d3b75a7ffd1784b85be16a627359090dfe5adf15dab6d145dc7d
+VELOCITY_VERSION=3.5.1
+VELOCITY_JAR_URL=https://fill-data.papermc.io/v1/objects/b4e3164df5377346854dc6cb9e6a78022b1946ff69e89676313f5f6f1c6f0fb3/velocity-3.5.1-615.jar
+VELOCITY_JAR_SHA256=b4e3164df5377346854dc6cb9e6a78022b1946ff69e89676313f5f6f1c6f0fb3

deploy/limbo/Dockerfile

@@ -10,10 +10,11 @@
 # There is no bundled server.properties — Limbo writes a default on first run.
 # So the runtime is assembled from those two URLs (not a zip) via --build-arg:
 #
+#   . <(grep '^LIMBO_' deploy/game-stack.lock)
 #   docker build -f deploy/limbo/Dockerfile \
-#     --build-arg LIMBO_JAR_URL=https://ci.loohpjames.com/job/Limbo/<n>/artifact/target/Limbo-<ver>.jar \
+#     --build-arg LIMBO_JAR_URL="$LIMBO_JAR_URL" --build-arg LIMBO_JAR_SHA256="$LIMBO_JAR_SHA256" \
-#     --build-arg LIMBO_SCHEM_URL=https://ci.loohpjames.com/job/Limbo/<n>/artifact/spawn.schem \
+#     --build-arg LIMBO_SCHEM_URL="$LIMBO_SCHEM_URL" --build-arg LIMBO_SCHEM_SHA256="$LIMBO_SCHEM_SHA256" \
-#     --build-arg LIMBO_VERSION=<maven-api-version> \
+#     --build-arg LIMBO_VERSION="$LIMBO_VERSION" \
 #     -t felis-limbo:demo .
 #
 # Note LIMBO_VERSION (the maven API version the plugin compiles against, e.g.
@@ -28,43 +29,59 @@
 #      overridable via FELIS_HEALTH_PORT) and returns 200 only after the first tick.
 
 # ---- build the felis-limbo plugin jar ----
-# gradle:*-jdk21 — an official Gradle image on JDK 21. JDK 21 is required because
+# The same pinned Gradle image as the lobby build (see deploy/lobby/Dockerfile). Its JDK
-# current LOOHP/Limbo releases ship Java 21 API classes (class-file major 65); a
+# must be >= 21 because current LOOHP/Limbo releases ship Java 21 API classes
-# JDK 17 fails to read them with "wrong version 65.0, should be 61.0". The image
+# (class-file major 65); a JDK 17 fails to read them with "wrong version 65.0, should be
-# also provides the `gradle` binary (this tree vendors no Gradle wrapper).
+# 61.0". build.gradle still targets release 17 bytecode so the plugin loads on Java 17+.
-# build.gradle still targets release 17 bytecode so the plugin loads on Java 17+.
+FROM gradle:9.8.0-jdk25@sha256:2b2fc1b1dfc3604a2acc916839f36eb5ee48fd7f232427fc5faca224c73bcb01 AS plugin
-FROM gradle:8.14-jdk21 AS plugin
 WORKDIR /src
 # Copy what the limbo module needs: its own tree plus the shared link core it
 # srcDir-includes (../shared/src/main/java → /src/plugins/shared/src/main/java), so
 # the account-link client + config loader compile straight into the jar.
 COPY plugins/limbo/ ./plugins/limbo/
 COPY plugins/shared/ ./plugins/shared/
-ARG LIMBO_VERSION=+
+# The Limbo API release to compile against: deploy/game-stack.lock's LIMBO_VERSION, which
-RUN cd plugins/limbo \
+# bootstrap passes. Required — the API is checked against the checksum
-    && (test -x ./gradlew && ./gradlew --no-daemon -PlimboVersion="$LIMBO_VERSION" build \
+# plugins/limbo/gradle/verification-metadata.xml holds for that release.
-        || gradle --no-daemon -PlimboVersion="$LIMBO_VERSION" build) \
+ARG LIMBO_VERSION
+RUN if [ -z "${LIMBO_VERSION:-}" ]; then \
+        echo "LIMBO_VERSION is required (deploy/game-stack.lock)" >&2; exit 1; \
+    fi \
+    && cd plugins/limbo \
+    && gradle --no-daemon -PlimboVersion="$LIMBO_VERSION" build \
     && cp build/libs/*.jar /felis-limbo.jar
 
 # ---- assemble the runtime ----
 # 21-jre: the Limbo jar is Java 21 bytecode (class-file major 65), so a Java 17
 # JRE cannot run it (UnsupportedClassVersionError). A 21 JRE also runs the
 # plugin's release-17 bytecode fine.
-FROM eclipse-temurin:21-jre
+FROM eclipse-temurin:21-jre@sha256:49e21e16e3c86eb7816a44a67549910ed090fbeb40c29c525d58bf5e02e91b0f
 ARG LIMBO_JAR_URL
+ARG LIMBO_JAR_SHA256
 ARG LIMBO_SCHEM_URL
+ARG LIMBO_SCHEM_SHA256
 WORKDIR /limbo
 # Pull the two loose LOOHP/Limbo CI artifacts: the server jar (required, saved as
-# Limbo.jar) and the default spawn schematic (optional). Fail loudly if the jar
+# Limbo.jar) and the default spawn schematic (optional). Each is checked against the
-# URL was not supplied.
+# digest deploy/game-stack.lock names (bootstrap.sh passes it): the login gate is the
+# first thing every player's connection reaches, and Limbo's CI publishes no digest of
+# its own.
 RUN set -eu; \
     if [ -z "${LIMBO_JAR_URL:-}" ]; then \
       echo "ERROR: --build-arg LIMBO_JAR_URL=<Limbo server jar> is required" >&2; exit 1; \
     fi; \
+    if [ -z "${LIMBO_JAR_SHA256:-}" ]; then \
+      echo "ERROR: --build-arg LIMBO_JAR_SHA256=<Limbo jar sha256> is required" >&2; exit 1; \
+    fi; \
+    if [ -n "${LIMBO_SCHEM_URL:-}" ] && [ -z "${LIMBO_SCHEM_SHA256:-}" ]; then \
+      echo "ERROR: --build-arg LIMBO_SCHEM_SHA256=<spawn.schem sha256> is required with LIMBO_SCHEM_URL" >&2; exit 1; \
+    fi; \
     apt-get update && apt-get install -y --no-install-recommends curl ca-certificates; \
     curl -fSL "$LIMBO_JAR_URL" -o /limbo/Limbo.jar; \
+    echo "$LIMBO_JAR_SHA256  /limbo/Limbo.jar" | sha256sum -c; \
     if [ -n "${LIMBO_SCHEM_URL:-}" ]; then \
       curl -fSL "$LIMBO_SCHEM_URL" -o /limbo/spawn.schem; \
+      echo "$LIMBO_SCHEM_SHA256  /limbo/spawn.schem" | sha256sum -c; \
     fi; \
     apt-get purge -y curl && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*; \
     mkdir -p /limbo/plugins
@@ -78,6 +95,13 @@ COPY deploy/limbo/entrypoint.sh /usr/local/bin/felis-entrypoint.sh
 # The operator mounts the world PVC at /data. Runtime state lives there; /limbo
 # remains the immutable image seed copied into the volume by the entrypoint.
 WORKDIR /data
+# Run as the game uid (naming.GameUID in the Go tree). The operator pins the same uid in
+# the pod securityContext whatever USER an image declares; declaring it here as well
+# keeps a plain `docker run` of this image off root, and chowning the empty /data seed
+# lets that run write its world. The jar seed above stays root-owned and read-only to
+# the server.
+RUN chown 1000:1000 /data
+USER 1000:1000
 
 ENV FELIS_HEALTH_PORT=8080
 # FELIS_GAME_PORT is the port the entrypoint pins Limbo to; it MUST equal the operator's

deploy/limbo/README.md

@@ -132,10 +132,13 @@ set them by hand:
   `spec.env` by `felis setup` (`cmd/felis` derives the internal API URL from the
   control namespace — the platform default `felis`; a renamed control namespace must
   be reflected by hand — and the root domain from `felis.toml`).
-- `FELIS_SERVICE_TOKEN` is a **secret**, so it is never written into the CRD. `felis
+- `FELIS_SERVICE_TOKEN` is a **secret**, so it is never written into the CRD. The
-  setup` replicates the `felis-service-token` Secret from the control namespace into
+  login gate has its own internal-API token, `felis-limbo-token`, which may only mint
-  the minecraft namespace, and the operator injects it into the `login` pod (only)
+  link codes, poll link status and check the blacklist. The installer applies it into
-  via a `secretKeyRef`, keyed off the reserved `login` name. Until the token is
+  the minecraft namespace (and `felis setup` refreshes that replica from the control
+  namespace), and the operator injects it into the `login` pod (only) as
+  `FELIS_SERVICE_TOKEN` via a `secretKeyRef`, keyed off the reserved `login` name.
+  `sudo felis rotate-token limbo` replaces it and restarts the pod. Until the token is
   present the plugin fail-safes to readiness-only, so the gate is never broken — it
   simply does not authenticate yet.
 - **Service:** the login pod dials `FELIS_API_BASE_URL`, which resolves to the
@@ -143,11 +146,14 @@ set them by hand:
   pod's internal port 8081. That Service is deliberately separate from the external
   NodePort `felis-api` (443) so the no-Zero-Trust internal face is never published on
   a node's external IP.
-- **NetworkPolicy:** none is required today — neither the minecraft-namespace egress
+- **NetworkPolicy:** the minecraft namespace is egress-locked
-  nor the control-namespace ingress is policy-locked, so the login pod's call to the
+  (`felis-server-egress`: DNS plus the public internet, every private range
-  API internal port is reachable. If a future deployment adds a minecraft egress lock
+  excluded), so the internal API is unreachable from a game server by default.
-  or a control-namespace ingress fence, it must also open the login-pod →
+  `felis-login-to-internal-api` opens exactly the login pod → felis-api (8081) path,
-  felis-api-internal (8081) path.
+  selecting on the reserved `login` name AND the setup-owned
+  `felis.lolicon.best/system-role=login` label the operator copies onto the pod — the
+  same pair that decides who receives `FELIS_SERVICE_TOKEN`, so a user server cannot
+  match it by picking a name.
 
 The Velocity gate/lobby wiring is printed by `felis setup` and enforces the
 invariant: fresh connections hit `login` first, and only an authenticated release

deploy/lobby/Dockerfile

@@ -5,13 +5,13 @@
 # the POST-auth /menu hub: it is reached only when the login gate transfers an
 # authenticated player onward, and it must never be a fallback target.
 #
-# Build (deploy/bootstrap.sh does this for you; the PAPER_JAR_URL comes from PaperMC's
+# Build (deploy/bootstrap.sh does this for you, with the URLs and digests
-# Fill v3 API — api.papermc.io v2 has returned HTTP 410 since 2026-07-01):
+# deploy/game-stack.lock names):
+#   . <(grep -E '^(PAPER|LUCKPERMS)_' deploy/game-stack.lock)
 #   docker build -f deploy/lobby/Dockerfile \
-#     --build-arg PAPER_JAR_URL=https://fill-data.papermc.io/v1/objects/<sha>/paper-26.2-<build>.jar \
+#     --build-arg PAPER_JAR_URL="$PAPER_JAR_URL" --build-arg PAPER_JAR_SHA256="$PAPER_JAR_SHA256" \
-#     --build-arg PAPER_JAR_SHA256=<that same sha — the objects/ path segment> \
+#     --build-arg LUCKPERMS_JAR_URL="$LUCKPERMS_JAR_URL" \
-#     --build-arg LUCKPERMS_JAR_URL="$(curl -fsSL https://metadata.luckperms.net/data/all \
+#     --build-arg LUCKPERMS_JAR_SHA256="$LUCKPERMS_JAR_SHA256" \
-#       | grep -o 'https://download.luckperms.net/[^"]*/bukkit/loader/[^"]*\.jar')" \
 #     -t felis-lobby:demo .
 #   docker save felis-lobby:demo | sudo k3s ctr images import -
 #   # felis.toml → [velocity] lobby_image = "felis-lobby:demo"
@@ -25,23 +25,26 @@
 # Secret) and refuses to start without it: a lobby that cannot verify the proxy's signed
 # handshake would trust an offline, forgeable UUID.
 
-# ---- build the felis-paper plugin jar (Paper API is Java 21) ----
+# ---- build the felis-paper plugin jar (Paper API is Java 25) ----
-# gradle:8.14-jdk21 — an official Gradle image on JDK 21 (this tree vendors no Gradle
+# The official Gradle image on JDK 25, the Gradle version plugins/*/gradle/wrapper pins.
-# wrapper, and a bare JDK image ships no `gradle`). JDK 21 matches the Paper API.
+# The limbo Dockerfile and bootstrap's velocity build use this same image, digest and all
-FROM gradle:8.14-jdk21 AS plugin
+# (bootstrap_asset_test.go holds the three together). The image's own `gradle` runs the
+# build rather than the module's wrapper, which would download the same distribution
+# again on every image build. The build checks every dependency against
+# plugins/paper/gradle/verification-metadata.xml and fails on a mismatch.
+FROM gradle:9.8.0-jdk25@sha256:2b2fc1b1dfc3604a2acc916839f36eb5ee48fd7f232427fc5faca224c73bcb01 AS plugin
 WORKDIR /src
 COPY plugins/paper/ ./plugins/paper/
 COPY plugins/shared/ ./plugins/shared/
 RUN cd plugins/paper \
-    && (test -x ./gradlew && ./gradlew --no-daemon build \
+    && gradle --no-daemon build \
-        || gradle --no-daemon build) \
     && cp build/libs/*.jar /felis-paper.jar
 
 # ---- assemble the runtime ----
 # 25-jre, not 21: Paper 26.2 declares `java.version.minimum = 25` (PaperMC Fill v3,
 # GET /v3/projects/paper/versions/26.2) and refuses to boot on anything older. A 25 JRE
 # also runs the plugin's Java-21 bytecode, so only the runtime moves.
-FROM eclipse-temurin:25-jre
+FROM eclipse-temurin:25-jre@sha256:bb036ed6cfdc57e3da7c22634d15f1b840d2caf76183861c80e81ca4b5104abb
 ARG PAPER_JAR_URL
 # Required alongside the URL: Fill's URLs are content-addressed, but nothing enforces
 # that shape at build time. Checking the digest after the download turns a truncated or
@@ -51,10 +54,12 @@ ARG PAPER_JAR_SHA256
 # (internal/api/handlers_access.go) issues `lp user ...` over RCON, so a lobby built
 # without it answers every grant with "Unknown command" — a failure the operator only
 # discovers in production, because the server itself starts and runs perfectly well.
-# Failing the build is the cheap place to notice. Resolved by URL rather than pinned
+# Failing the build is the cheap place to notice. Passed in rather than pinned here for
-# here for the same reason PAPER_JAR_URL is: bootstrap.sh asks upstream for the current
+# the same reason PAPER_JAR_URL is: deploy/game-stack.lock names the build, so this file
-# build, so this file does not go stale on every LuckPerms release.
+# does not change on every LuckPerms release. The digest is required like Paper's; the
+# jar runs inside the lobby with the server's full permissions.
 ARG LUCKPERMS_JAR_URL
+ARG LUCKPERMS_JAR_SHA256
 WORKDIR /paper
 RUN set -eu; \
     if [ -z "${PAPER_JAR_URL:-}" ]; then \
@@ -66,11 +71,15 @@ RUN set -eu; \
     if [ -z "${LUCKPERMS_JAR_URL:-}" ]; then \
       echo "ERROR: --build-arg LUCKPERMS_JAR_URL=<luckperms bukkit jar> is required" >&2; exit 1; \
     fi; \
+    if [ -z "${LUCKPERMS_JAR_SHA256:-}" ]; then \
+      echo "ERROR: --build-arg LUCKPERMS_JAR_SHA256=<luckperms jar sha256> is required" >&2; exit 1; \
+    fi; \
     apt-get update && apt-get install -y --no-install-recommends curl ca-certificates; \
     mkdir -p /paper/plugins; \
     curl -fSL "$PAPER_JAR_URL" -o /paper/paper.jar; \
     echo "$PAPER_JAR_SHA256  /paper/paper.jar" | sha256sum -c; \
     curl -fSL "$LUCKPERMS_JAR_URL" -o /paper/plugins/LuckPerms.jar; \
+    echo "$LUCKPERMS_JAR_SHA256  /paper/plugins/LuckPerms.jar" | sha256sum -c; \
     apt-get purge -y curl && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*; \
     echo "eula=true" > /paper/eula.txt
 COPY --from=plugin /felis-paper.jar /paper/plugins/felis-paper.jar
@@ -82,6 +91,13 @@ COPY deploy/lobby/entrypoint.sh /usr/local/bin/felis-entrypoint.sh
 # The operator mounts the world PVC at /data. Runtime state lives there; /paper
 # remains the immutable image seed copied into the volume by the entrypoint.
 WORKDIR /data
+# Run as the game uid (naming.GameUID in the Go tree). The operator pins the same uid in
+# the pod securityContext whatever USER an image declares; declaring it here as well
+# keeps a plain `docker run` of this image off root, and chowning the empty /data seed
+# lets that run write its world. The jar seed above stays root-owned and read-only to
+# the server.
+RUN chown 1000:1000 /data
+USER 1000:1000
 
 # FELIS_GAME_PORT is the port the entrypoint pins Paper to; it MUST equal the operator's
 # GamePort (internal/operator/builders.go). Default 25565 — override only in lockstep

deploy/paper/Dockerfile

@@ -5,7 +5,7 @@
 # internal/store/migrations/0019_recommended_paper.sql). It is NOT a system server: it
 # carries no felis-paper /menu plugin, no LuckPerms, and no forwarding-secret gate.
 #
-# It writes NO Velocity forwarding config itself. The operator injects a root
+# It writes NO Velocity forwarding config itself. The operator injects a
 # `felis init-forwarding` initContainer into every USER server (internal/operator/
 # builders.go: buildStatefulSet) that writes config/paper-global.yml + server.properties
 # online-mode=false onto the /data PVC before this container starts. That external step is
@@ -14,11 +14,11 @@
 # image a user brings is made joinable the same way. If the initContainer is absent (no
 # FELIS_IMAGE configured) Paper boots as a standalone online server: degraded, not broken.
 #
-# Build (deploy/bootstrap.sh does this for you; PAPER_JAR_URL comes from PaperMC's Fill v3
+# Build (deploy/bootstrap.sh does this for you, with the SAME Paper build the lobby uses —
-# API — the SAME url the lobby build resolves, so this reuses it and adds no new dependency):
+# deploy/game-stack.lock names it — so this adds no new dependency):
+#   . <(grep '^PAPER_' deploy/game-stack.lock)
 #   docker build -f deploy/paper/Dockerfile \
-#     --build-arg PAPER_JAR_URL=https://fill-data.papermc.io/v1/objects/<sha>/paper-<ver>-<build>.jar \
+#     --build-arg PAPER_JAR_URL="$PAPER_JAR_URL" --build-arg PAPER_JAR_SHA256="$PAPER_JAR_SHA256" \
-#     --build-arg PAPER_JAR_SHA256=<that same sha — the objects/ path segment> \
 #     -t felis-paper:demo .
 #   docker save felis-paper:demo | sudo k3s ctr images import -
 #   # felis.toml → recommended via 0019_recommended_paper.sql (no [velocity] key points here)
@@ -29,7 +29,7 @@
 
 # 25-jre, not 21: Paper 26.2 declares java.version.minimum=25 (PaperMC Fill v3) and refuses
 # to boot on anything older.
-FROM eclipse-temurin:25-jre
+FROM eclipse-temurin:25-jre@sha256:bb036ed6cfdc57e3da7c22634d15f1b840d2caf76183861c80e81ca4b5104abb
 ARG PAPER_JAR_URL
 # Required alongside the URL: Fill's URLs are content-addressed, but nothing enforces
 # that shape at build time. Checking the digest after the download turns a truncated or
@@ -54,6 +54,13 @@ COPY deploy/paper/entrypoint.sh /usr/local/bin/felis-entrypoint.sh
 # on the PVC. /paper stays the immutable image seed: the jar is never copied onto the
 # volume, so the panel file editor (which sees only /data) cannot tamper with it.
 WORKDIR /data
+# Run as the game uid (naming.GameUID in the Go tree). The operator pins the same uid in
+# the pod securityContext whatever USER an image declares; declaring it here as well
+# keeps a plain `docker run` of this image off root, and chowning the empty /data seed
+# lets that run write its world. The jar seed above stays root-owned and read-only to
+# the server.
+RUN chown 1000:1000 /data
+USER 1000:1000
 
 # FELIS_GAME_PORT is the port the entrypoint pins Paper to; it MUST equal the operator's
 # GamePort (internal/operator/builders.go). Default 25565 — override only in lockstep with

deploy/paper/entrypoint.sh

@@ -3,7 +3,7 @@
 #
 # A plain Paper backend for a user's OWN world — NOT a system server. Unlike deploy/limbo
 # and deploy/lobby it writes no Velocity forwarding config and has no secret gate: the
-# operator injects a root `felis init-forwarding` initContainer that writes
+# operator injects a `felis init-forwarding` initContainer that writes
 # config/paper-global.yml + server.properties online-mode=false onto /data BEFORE this
 # container starts, so forwarding is configured externally and this stays a drop-in Paper
 # image. With no initContainer (no FELIS_IMAGE) Paper just boots standalone-online —

deploy/uninstall.sh

@@ -0,0 +1,428 @@
+#!/usr/bin/env bash
+# Removes what deploy/bootstrap.sh installed on this host.
+#
+#   sudo bash deploy/uninstall.sh                 # remove Felis, keep the data
+#   sudo bash deploy/uninstall.sh --purge         # remove the data too
+#   curl -fsSL <raw-url>/deploy/uninstall.sh | sudo bash -s -- --yes
+#
+# Options:
+#   --purge       also drop the felis database and role, and delete /etc/felis and
+#                 /var/lib/felis (the database bundles, and anything an earlier keep-data
+#                 run set aside). Asks for the word "purge" unless --yes is given.
+#   --keep-k3s    leave k3s installed and remove only Felis's namespaces and CRD.
+#   --remove-k3s  run k3s's own uninstaller even when other workloads live in the cluster.
+#   --no-backup   skip the final database bundle keep-data mode takes first.
+#   --yes         do not ask.
+#
+# Keep-data mode (the default) first takes a database bundle (`felis db backup -label
+# manual`) and stops if that fails. It leaves PostgreSQL's felis database, /etc/felis (the
+# secrets, felis.toml, offsite.env) and /var/lib/felis in place. The world, archive,
+# registry and upload volumes live under k3s's storage directory, which k3s's uninstaller
+# deletes, so they are moved to /var/lib/felis/retained/k3s-storage-<UTC stamp> first; with
+# --keep-k3s their PersistentVolumes are switched to Retain before the namespaces go.
+# docs/operations.md walks through reinstalling on top of what is left.
+#
+# Either mode leaves packages alone (Docker, PostgreSQL, git and the rest), and the swap
+# file a low-memory host got: other software may use them. docs/operations.md lists the
+# package commands for a bare host.
+set -Eeuo pipefail
+
+STATE_DIR="${STATE_DIR:-/etc/felis}"
+DATA_DIR="${DATA_DIR:-/var/lib/felis}"
+RETAIN_DIR="${DATA_DIR}/retained"
+HOST_BIN="${HOST_BIN:-/usr/local/bin/felis}"
+OPT_DIR="${OPT_DIR:-/opt/felis}"
+UNIT_DIR="${UNIT_DIR:-/etc/systemd/system}"
+K3S_BIN_DIR="${K3S_BIN_DIR:-/usr/local/bin}"
+K3S_STORAGE="${K3S_STORAGE:-/var/lib/rancher/k3s/storage}"
+K3S_REGISTRIES="${K3S_REGISTRIES:-/etc/rancher/k3s/registries.yaml}"
+export KUBECONFIG="${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}"
+CLOUDFLARED_BIN="${CLOUDFLARED_BIN:-/usr/local/bin/cloudflared}"
+VELOCITY_USER="felis-velocity"
+DB_NAME="felis"
+DB_USER="felis"
+POD_CIDR="10.42.0.0/16"
+SERVICE_CIDR="10.43.0.0/16"
+FELIS_PANEL_NODEPORT="${FELIS_PANEL_NODEPORT:-30443}"
+CONFIRM_TTY="${CONFIRM_TTY:-/dev/tty}"
+FELIS_NAMESPACES=(felis minecraft felis-build)
+FELIS_CRD="minecraftservers.felis.lolicon.best"
+# Every unit the installer and `felis setup` write. Timers first, so none fires into a
+# service that is already gone.
+FELIS_UNITS=(
+  felis-db-backup.timer felis-watchdog.timer felis-offsite.timer felis-build-tools.timer felis-update-check.timer
+  felis-db-backup.service felis-watchdog.service felis-offsite.service felis-build-tools.service felis-update-check.service
+  felis-velocity.service felis-nano.service cloudflared-felis.service
+  felis-postgres-firewall.service
+)
+
+PURGE=0
+K3S_MODE=auto
+BACKUP=1
+ASSUME_YES=0
+TUNNEL_CONFIG=""
+
+log()  { printf '\033[1;36m[felis]\033[0m %s\n' "$*"; }
+ok()   { printf '\033[1;32m[ ok ]\033[0m %s\n' "$*"; }
+warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
+die()  { printf '\033[1;31m[fail]\033[0m %s\n' "$*" >&2; exit 1; }
+
+parse_args() {
+  while [ $# -gt 0 ]; do
+    case "$1" in
+      --purge) PURGE=1 ;;
+      --keep-k3s) K3S_MODE=keep ;;
+      --remove-k3s) K3S_MODE=remove ;;
+      --no-backup) BACKUP=0 ;;
+      --yes|-y) ASSUME_YES=1 ;;
+      -h|--help) printf 'usage: uninstall.sh [--purge] [--keep-k3s|--remove-k3s] [--no-backup] [--yes]\n'; exit 0 ;;
+      *) die "unknown option: $1 (see --help)" ;;
+    esac
+    shift
+  done
+}
+
+kube() { "${K3S_BIN_DIR}/k3s" kubectl "$@"; }
+k3s_present() { [ -x "${K3S_BIN_DIR}/k3s" ]; }
+
+# foreign_namespaces prints the namespaces that are neither k3s's own nor Felis's, one per
+# line. A cluster with none of them exists for Felis alone, and removing k3s takes nothing
+# else with it.
+foreign_namespaces() {
+  kube get namespaces -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' \
+    | awk '$0 != "" && $0 != "default" && $0 !~ /^kube-/ && $0 != "felis" && $0 != "minecraft" && $0 != "felis-build"'
+}
+
+# decide_k3s turns K3S_MODE=auto into keep or remove.
+decide_k3s() {
+  k3s_present || { K3S_MODE=absent; return 0; }
+  [ "$K3S_MODE" = auto ] || return 0
+  local others
+  if ! others="$(foreign_namespaces)"; then
+    die "k3s does not answer, so this cannot tell whether it runs anything besides Felis; start it (systemctl start k3s) or pass --keep-k3s or --remove-k3s"
+  fi
+  if [ -z "$others" ]; then
+    K3S_MODE=remove
+  else
+    K3S_MODE=keep
+    log "k3s also runs namespaces Felis did not create ($(printf '%s' "$others" | tr '\n' ' ')); leaving k3s installed"
+  fi
+}
+
+confirm() {
+  [ "$ASSUME_YES" = 1 ] && return 0
+  local want="yes" answer=""
+  [ "$PURGE" = 1 ] && want="purge"
+  # A piped script has no stdin to read from; the terminal is asked directly.
+  { exec 3<"$CONFIRM_TTY" 4>>"$CONFIRM_TTY"; } 2>/dev/null || die "no terminal to confirm on; re-run with --yes"
+  printf 'Type "%s" to continue: ' "$want" >&4
+  read -r answer <&3 || true
+  exec 3<&- 4>&-
+  [ "$answer" = "$want" ] || die "not confirmed; nothing was changed"
+}
+
+print_plan() {
+  log "this will remove from $(uname -n):"
+  log "  the felis-* systemd units, cloudflared-felis.service, the ${VELOCITY_USER} user,"
+  log "  ${OPT_DIR}, ${HOST_BIN}, the felis_postgres and felis_edge nftables tables and the firewalld openings"
+  case "$K3S_MODE" in
+    remove) log "  k3s, with everything in it (${K3S_BIN_DIR}/k3s-uninstall.sh)" ;;
+    keep) log "  Felis's namespaces (${FELIS_NAMESPACES[*]}) and the ${FELIS_CRD} CRD; k3s stays" ;;
+    absent) ;;
+  esac
+  if [ "$PURGE" = 1 ]; then
+    log "  PURGE: the ${DB_NAME} database and role, ${STATE_DIR} (secrets), ${DATA_DIR} (database bundles"
+    log "  and anything set aside before), every world and archive, the Felis images and Docker's build cache"
+  else
+    [ "$BACKUP" = 1 ] && log "  after a final database bundle into ${DATA_DIR}/db-backups"
+    log "  kept: the ${DB_NAME} database, ${STATE_DIR}, ${DATA_DIR}; the volumes move to ${RETAIN_DIR}/"
+  fi
+}
+
+final_backup() {
+  [ "$PURGE" = 0 ] && [ "$BACKUP" = 1 ] || return 0
+  [ -x "$HOST_BIN" ] && [ -r "${STATE_DIR}/felis.host.toml" ] || {
+    warn "no ${HOST_BIN} or ${STATE_DIR}/felis.host.toml; skipping the final database bundle"
+    return 0
+  }
+  log "taking a final database bundle"
+  "$HOST_BIN" db backup -config "${STATE_DIR}/felis.host.toml" -label manual \
+    || die "the final database bundle failed, so nothing was removed. Fix the database (sudo felis db check), or pass --no-backup to go on without one"
+  ok "database bundle written to ${DATA_DIR}/db-backups"
+}
+
+# game_port reads the proxy's port from velocity.toml before /opt/felis goes.
+game_port() {
+  local toml="${OPT_DIR}/velocity/velocity.toml" port=""
+  [ -r "$toml" ] && port="$(sed -n 's/^bind *= *"[^"]*:\([0-9][0-9]*\)".*/\1/p' "$toml" | head -n 1)"
+  printf '%s\n' "${FELIS_GAME_PORT:-${port:-25565}}"
+}
+
+# tunnel_config reads the cloudflared config felis setup pointed its unit at (by default
+# /etc/felis/cloudflared.yml) before the unit goes.
+tunnel_config() {
+  local unit="${UNIT_DIR}/cloudflared-felis.service" path=""
+  [ -r "$unit" ] && path="$(sed -n 's/^ExecStart=.* --config \([^ ]*\) tunnel run$/\1/p' "$unit" | head -n 1)"
+  printf '%s\n' "${path:-${STATE_DIR}/cloudflared.yml}"
+}
+
+# nano_port reads felis-nano's port from its unit before the unit goes.
+nano_port() {
+  local unit="${UNIT_DIR}/felis-nano.service"
+  [ -r "$unit" ] || return 0
+  sed -n 's/^ExecStart=.* -listen [^ ]*:\([0-9][0-9]*\).*$/\1/p' "$unit" | head -n 1
+}
+
+remove_units() {
+  local unit removed=0
+  for unit in "${FELIS_UNITS[@]}"; do
+    [ -f "${UNIT_DIR}/${unit}" ] || continue
+    systemctl disable --now "$unit" >/dev/null 2>&1 || systemctl stop "$unit" >/dev/null 2>&1 || true
+    rm -f "${UNIT_DIR}/${unit}"
+    removed=$((removed + 1))
+  done
+  systemctl daemon-reload
+  systemctl reset-failed >/dev/null 2>&1 || true
+  ok "${removed} systemd unit(s) removed"
+}
+
+remove_nft_tables() {
+  command -v nft >/dev/null 2>&1 || return 0
+  local t
+  for t in felis_postgres felis_edge; do
+    if nft list table inet "$t" >/dev/null 2>&1; then
+      nft delete table inet "$t"
+      ok "nftables table inet ${t} removed"
+    fi
+  done
+}
+
+# remove_firewalld_rules takes back what configure_k3s_firewall, configure_velocity_firewall
+# and the nano setup opened. The k3s ones stay when k3s does.
+remove_firewalld_rules() { # game-port nano-port
+  command -v firewall-cmd >/dev/null 2>&1 || return 0
+  systemctl is-active --quiet firewalld || return 0
+  local game="$1" nano="$2" port rule changed=0
+  local ports=("${game}/tcp" "${FELIS_PANEL_NODEPORT}/tcp")
+  [ -n "$nano" ] && ports+=("${nano}/tcp")
+  [ "$K3S_MODE" = remove ] && ports+=("6443/tcp")
+  for port in "${ports[@]}"; do
+    if firewall-cmd --permanent --query-port="$port" >/dev/null 2>&1; then
+      firewall-cmd --permanent --remove-port="$port" >/dev/null
+      changed=1
+    fi
+  done
+  if [ -n "$nano" ]; then
+    while IFS= read -r rule; do
+      case "$rule" in
+        *"port=\"${nano}\""*) firewall-cmd --permanent --remove-rich-rule="$rule" >/dev/null; changed=1 ;;
+      esac
+    done < <(firewall-cmd --permanent --list-rich-rules 2>/dev/null)
+  fi
+  if [ "$K3S_MODE" = remove ]; then
+    local cidr
+    for cidr in "$POD_CIDR" "$SERVICE_CIDR"; do
+      if firewall-cmd --permanent --zone=trusted --query-source="$cidr" >/dev/null 2>&1; then
+        firewall-cmd --permanent --zone=trusted --remove-source="$cidr" >/dev/null
+        changed=1
+      fi
+    done
+  fi
+  if [ "$changed" = 1 ]; then
+    firewall-cmd --reload >/dev/null
+    ok "firewalld openings removed"
+  fi
+}
+
+# retain_volumes_in_cluster keeps every volume Felis's claims are bound to when the
+# namespaces go: local-path deletes a Delete-policy volume's directory with its claim.
+retain_volumes_in_cluster() {
+  local pv
+  while IFS= read -r pv; do
+    [ -n "$pv" ] || continue
+    kube patch pv "$pv" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' >/dev/null
+  done < <(kube get pv -o 'jsonpath={range .items[*]}{.metadata.name} {.spec.claimRef.namespace}{"\n"}{end}' \
+    | awk '$2 == "felis" || $2 == "minecraft" || $2 == "felis-build" { print $1 }')
+  ok "Felis's volumes set to Retain; their directories stay under ${K3S_STORAGE}"
+}
+
+remove_from_cluster() {
+  [ "$PURGE" = 1 ] || retain_volumes_in_cluster
+  log "deleting Felis's namespaces and CRD"
+  # The operator is part of what goes, so nothing would clear a MinecraftServer finalizer
+  # and the minecraft namespace would stay Terminating. Drop them first.
+  local s
+  while IFS= read -r s; do
+    [ -n "$s" ] || continue
+    kube -n minecraft patch minecraftserver "$s" --type=merge -p '{"metadata":{"finalizers":null}}' >/dev/null 2>&1 || true
+  done < <(kube -n minecraft get minecraftservers -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
+  kube delete namespace "${FELIS_NAMESPACES[@]}" --ignore-not-found --wait=true --timeout=300s >/dev/null \
+    || warn "a namespace is still terminating; check: k3s kubectl get namespaces"
+  kube delete crd "$FELIS_CRD" --ignore-not-found >/dev/null || true
+  if [ "$PURGE" = 1 ]; then
+    local pv
+    while IFS= read -r pv; do
+      [ -n "$pv" ] && kube delete pv "$pv" --ignore-not-found >/dev/null
+    done < <(kube get pv -o 'jsonpath={range .items[*]}{.metadata.name} {.spec.claimRef.namespace}{"\n"}{end}' \
+      | awk '$2 == "felis" || $2 == "minecraft" || $2 == "felis-build" { print $1 }')
+  fi
+  if [ -f "$K3S_REGISTRIES" ] && grep -q 'registry\.felis\.svc' "$K3S_REGISTRIES"; then
+    rm -f "$K3S_REGISTRIES"
+    systemctl restart k3s
+  fi
+  ok "Felis removed from the cluster; k3s stays"
+}
+
+remove_k3s() {
+  local stamp
+  if [ "$PURGE" = 0 ] && [ -d "$K3S_STORAGE" ]; then
+    # k3s-killall.sh stops every pod and unmounts their volumes, so nothing is writing a
+    # world while it moves.
+    "${K3S_BIN_DIR}/k3s-killall.sh" >/dev/null 2>&1 || systemctl stop k3s
+    stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+    install -d -m 0700 "$RETAIN_DIR"
+    mv "$K3S_STORAGE" "${RETAIN_DIR}/k3s-storage-${stamp}"
+    ok "volumes moved to ${RETAIN_DIR}/k3s-storage-${stamp}"
+  fi
+  if [ -x "${K3S_BIN_DIR}/k3s-uninstall.sh" ]; then
+    log "running k3s-uninstall.sh"
+    "${K3S_BIN_DIR}/k3s-uninstall.sh" >/dev/null 2>&1 || warn "k3s-uninstall.sh reported an error; check /var/lib/rancher and /etc/rancher"
+    ok "k3s removed"
+  else
+    warn "k3s is at ${K3S_BIN_DIR}/k3s but ${K3S_BIN_DIR}/k3s-uninstall.sh is missing; remove k3s by hand"
+  fi
+}
+
+# remove_cloudflared_binary deletes the binary the installer put in /usr/local/bin, unless
+# a unit other than Felis's still runs it.
+remove_cloudflared_binary() {
+  [ -x "$CLOUDFLARED_BIN" ] || return 0
+  local others
+  others="$(grep -ls "$CLOUDFLARED_BIN" "${UNIT_DIR}"/*.service /lib/systemd/system/*.service /usr/lib/systemd/system/*.service 2>/dev/null || true)"
+  if [ -n "$others" ]; then
+    log "leaving ${CLOUDFLARED_BIN}: $(printf '%s' "$others" | tr '\n' ' ')uses it"
+    return 0
+  fi
+  rm -f "$CLOUDFLARED_BIN"
+  ok "${CLOUDFLARED_BIN} removed"
+}
+
+remove_host_files() {
+  if id "$VELOCITY_USER" >/dev/null 2>&1; then
+    userdel "$VELOCITY_USER" >/dev/null 2>&1 || warn "could not remove the ${VELOCITY_USER} user"
+  fi
+  rm -rf "$OPT_DIR"
+  rm -f "$HOST_BIN" "${HOST_BIN}.new" "${HOST_BIN}.prev"
+  remove_cloudflared_binary
+  if [ "$PURGE" = 0 ]; then
+    # What describes the removed install goes; what a reinstall reuses stays. Without
+    # bootstrap.done the next run takes the first-install path.
+    rm -f "${STATE_DIR}/bootstrap.done" "${STATE_DIR}/system-server-images" \
+      "${STATE_DIR}/velocity.fingerprint" "${STATE_DIR}/previous-felis-image"
+  fi
+  ok "${OPT_DIR} and ${HOST_BIN} removed"
+}
+
+as_postgres() { (cd / && runuser -u postgres -- "$@"); }
+
+# remove_hba_block drops the block write_pg_hba_block maintains, and nothing else.
+remove_hba_block() { # file
+  local tmp
+  tmp="$(mktemp)"
+  awk '
+    $0 == "# BEGIN FELIS MANAGED HBA" { skip = 1; next }
+    $0 == "# END FELIS MANAGED HBA" { skip = 0; blank = 1; next }
+    blank && $0 == "" { blank = 0; next }
+    { blank = 0 }
+    !skip { print }
+  ' "$1" > "$tmp"
+  cat "$tmp" > "$1"
+  rm -f "$tmp"
+}
+
+purge_database() {
+  [ "$PURGE" = 1 ] || return 0
+  if ! systemctl is-active --quiet postgresql 2>/dev/null; then
+    warn "PostgreSQL is not running; the ${DB_NAME} database and role are left in it"
+    return 0
+  fi
+  local hba
+  hba="$(as_postgres psql -tAc 'SHOW hba_file;' 2>/dev/null || true)"
+  as_postgres psql -v ON_ERROR_STOP=1 -q <<SQL
+SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${DB_NAME}' AND pid <> pg_backend_pid();
+DROP DATABASE IF EXISTS ${DB_NAME};
+DROP ROLE IF EXISTS ${DB_USER};
+ALTER SYSTEM RESET listen_addresses;
+SQL
+  [ -n "$hba" ] && [ -f "$hba" ] && remove_hba_block "$hba"
+  systemctl restart postgresql
+  ok "database and role '${DB_NAME}' dropped; PostgreSQL listens on its default address again"
+}
+
+purge_images() {
+  [ "$PURGE" = 1 ] || return 0
+  command -v docker >/dev/null 2>&1 || return 0
+  # The installer stops Docker after its builds; start it just long enough to clean up.
+  local was_active=1 refs
+  systemctl is-active --quiet docker || { was_active=0; systemctl start docker >/dev/null 2>&1 || return 0; }
+  refs="$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -E '^(registry\.felis\.svc:5000/felis/|felis/)' || true)"
+  if [ -n "$refs" ]; then
+    # shellcheck disable=SC2086 # one ref per word
+    docker image rm -f $refs >/dev/null 2>&1 || true
+  fi
+  docker builder prune -af >/dev/null 2>&1 || true
+  [ "$was_active" = 1 ] || systemctl stop docker docker.socket >/dev/null 2>&1 || true
+  ok "Felis images and Docker's build cache removed"
+}
+
+purge_state() {
+  [ "$PURGE" = 1 ] || return 0
+  # felis setup's tunnel credentials sit beside cloudflared's login (cert.pem, which stays:
+  # it is the Cloudflare account's, not Felis's). The tunnel itself lives on in the account
+  # until it is deleted there (docs/operations.md).
+  local cred=""
+  [ -r "$TUNNEL_CONFIG" ] \
+    && cred="$(sed -n 's/^credentials-file: *"\{0,1\}\([^"]*\)"\{0,1\} *$/\1/p' "$TUNNEL_CONFIG" | head -n 1)"
+  if [ -n "$cred" ]; then rm -f "$cred"; fi
+  rm -f "$TUNNEL_CONFIG"
+  rm -rf "$STATE_DIR" "$DATA_DIR"
+  ok "${STATE_DIR} and ${DATA_DIR} removed"
+}
+
+main() {
+  parse_args "$@"
+  [ "$(id -u)" = 0 ] || die "run as root: sudo bash $0"
+  [ -e "$STATE_DIR" ] || [ -e "$HOST_BIN" ] || [ -e "$OPT_DIR" ] \
+    || die "no Felis install here (${STATE_DIR}, ${HOST_BIN} and ${OPT_DIR} are all absent)"
+  decide_k3s
+  print_plan
+  confirm
+  final_backup
+
+  local game nano
+  game="$(game_port)"
+  nano="$(nano_port)"
+  TUNNEL_CONFIG="$(tunnel_config)"
+  remove_units
+  case "$K3S_MODE" in
+    remove) remove_k3s ;;
+    keep) remove_from_cluster ;;
+  esac
+  remove_nft_tables
+  remove_firewalld_rules "$game" "$nano"
+  remove_host_files
+  purge_database
+  purge_images
+  purge_state
+
+  if [ "$PURGE" = 1 ]; then
+    ok "Felis is gone from this host"
+  else
+    ok "Felis is removed; the data stays in the ${DB_NAME} database, ${STATE_DIR} and ${DATA_DIR}"
+    log "reinstalling reuses it: see docs/operations.md, \"Reinstall on top of kept data\""
+  fi
+}
+
+if [ "${FELIS_UNINSTALL_SOURCED:-0}" != 1 ]; then
+  main "$@"
+fi

deploy/uninstall_test.sh

@@ -0,0 +1,221 @@
+#!/bin/sh
+# Checks for deploy/uninstall.sh. Run it as: sh deploy/uninstall_test.sh
+#
+# The script is sourced with FELIS_UNINSTALL_SOURCED=1, every host path pointed into a
+# scratch directory, and the commands that would change the host (systemctl, k3s, nft,
+# firewall-cmd, runuser, docker, userdel) replaced by stubs that log their arguments.
+set -u
+
+US="${1:-$(dirname "$0")/uninstall.sh}"
+[ -f "$US" ] || { echo "no such script: $US"; exit 1; }
+fails=0
+
+expect() { # label needle haystack
+  case "$3" in
+    *"$2"*) echo "PASS $1" ;;
+    *) echo "FAIL $1: expected <$2> in:"; echo "$3"; fails=$((fails + 1)) ;;
+  esac
+}
+refute() { # label needle haystack
+  case "$3" in
+    *"$2"*) echo "FAIL $1: did not expect <$2> in:"; echo "$3"; fails=$((fails + 1)) ;;
+    *) echo "PASS $1" ;;
+  esac
+}
+
+root="$(mktemp -d)"
+trap 'rm -rf "$root"' EXIT
+
+# fresh_host lays out what an install leaves: units, state, /opt/felis, the host binary, a
+# k3s with its uninstaller and a volume, a tunnel config and its credentials.
+fresh_host() {
+  rm -rf "$root/h"
+  mkdir -p "$root/h/units" "$root/h/etc" "$root/h/data/db-backups" "$root/h/opt/velocity" \
+    "$root/h/bin" "$root/h/storage/pvc-1_minecraft_world-a-0" "$root/h/cf"
+  for u in felis-db-backup.timer felis-db-backup.service felis-velocity.service felis-postgres-firewall.service; do
+    printf '[Unit]\n' > "$root/h/units/$u"
+  done
+  printf '[Service]\nExecStart=/usr/local/bin/cloudflared --config %s tunnel run\n' "$root/h/etc/cloudflared.yml" \
+    > "$root/h/units/cloudflared-felis.service"
+  printf 'tunnel: abc\ncredentials-file: %s\n' "$root/h/cf/abc.json" > "$root/h/etc/cloudflared.yml"
+  printf '{}\n' > "$root/h/cf/abc.json"
+  printf 'bind = "0.0.0.0:25577"\n' > "$root/h/opt/velocity/velocity.toml"
+  for f in secrets.env felis.host.toml bootstrap.done system-server-images velocity.fingerprint; do
+    printf 'x\n' > "$root/h/etc/$f"
+  done
+  printf 'world\n' > "$root/h/storage/pvc-1_minecraft_world-a-0/level.dat"
+  for b in felis k3s k3s-killall.sh k3s-uninstall.sh; do
+    printf '#!/bin/sh\necho "RUN %s $*" >> "%s"\n' "$b" "$root/calls" > "$root/h/bin/$b"
+    chmod +x "$root/h/bin/$b"
+  done
+  cat > "$root/h/hba.conf" <<'EOF'
+# BEGIN FELIS MANAGED HBA
+# Felis rules must precede distro defaults such as 127.0.0.1 ident.
+host felis felis 127.0.0.1/32 scram-sha-256
+# END FELIS MANAGED HBA
+
+local all all peer
+host all all 127.0.0.1/32 ident
+EOF
+  : > "$root/calls"
+}
+
+# run_uninstall <namespaces> <args...>: runs main with the stubs; <namespaces> is what
+# `kubectl get namespaces` answers, or "down" for a k3s that does not answer.
+run_uninstall() {
+  ns="$1"; shift
+  NS="$ns" ROOT="$root" STATE_DIR="$root/h/etc" DATA_DIR="$root/h/data" HOST_BIN="$root/h/bin/felis" \
+  OPT_DIR="$root/h/opt" UNIT_DIR="$root/h/units" K3S_BIN_DIR="$root/h/bin" \
+  K3S_STORAGE="$root/h/storage" K3S_REGISTRIES="$root/h/registries.yaml" \
+  CLOUDFLARED_BIN="$root/h/no-cloudflared" FELIS_UNINSTALL_SOURCED=1 bash -c '
+    set -Eeuo pipefail
+    . "$0"
+    calls="$ROOT/calls"
+    id() { if [ "${1:-}" = -u ]; then echo 0; else echo "ID $*" >> "$calls"; fi; }
+    systemctl() {
+      echo "SYSTEMCTL $*" >> "$calls"
+      case "$*" in "is-active --quiet firewalld") return 1 ;; esac
+      return 0
+    }
+    kube() {
+      echo "KUBE $*" >> "$calls"
+      case "$*" in
+        "get namespaces"*) [ "$NS" = down ] && return 1; printf "%s\n" $NS ;;
+        "get pv"*) printf "pvc-1 minecraft\npvc-9 other\n" ;;
+      esac
+    }
+    nft() { echo "NFT $*" >> "$calls"; return 1; }
+    runuser() {
+      shift 3
+      case "$*" in
+        *"SHOW hba_file"*) echo "$ROOT/h/hba.conf" ;;
+        *) echo "PSQL $* $(cat)" >> "$calls" ;;
+      esac
+    }
+    docker() { echo "DOCKER $*" >> "$calls"; }
+    userdel() { echo "USERDEL $*" >> "$calls"; }
+    uname() { echo testhost; }
+    main "$@"' "$US" "$@" 2>&1
+}
+
+# --- keep-data, a cluster that runs only Felis -------------------------------------------
+fresh_host
+out="$(run_uninstall "default kube-system felis minecraft felis-build" --yes)"
+calls="$(cat "$root/calls")"
+expect "keep-data takes a final bundle first" "RUN felis db backup -config $root/h/etc/felis.host.toml -label manual" "$calls"
+expect "a Felis-only cluster is removed with k3s's uninstaller" "RUN k3s-uninstall.sh" "$calls"
+expect "the pods are stopped before the volumes move" "RUN k3s-killall.sh" "$calls"
+kept="$(ls "$root/h/data/retained" 2>/dev/null)"
+expect "the volumes are set aside before k3s deletes them" "k3s-storage-" "$kept"
+[ -f "$root/h/data/retained/$kept/pvc-1_minecraft_world-a-0/level.dat" ] \
+  && echo "PASS a world survives the uninstall" \
+  || { echo "FAIL the world did not survive: $(ls -R "$root/h/data")"; fails=$((fails + 1)); }
+[ -f "$root/h/etc/secrets.env" ] && [ -f "$root/h/etc/felis.host.toml" ] \
+  && echo "PASS the secrets and felis.toml stay" \
+  || { echo "FAIL keep-data removed the secrets"; fails=$((fails + 1)); }
+[ ! -e "$root/h/etc/bootstrap.done" ] && [ ! -e "$root/h/etc/velocity.fingerprint" ] \
+  && echo "PASS the markers of the removed install go, so a reinstall starts fresh" \
+  || { echo "FAIL bootstrap.done or the proxy fingerprint was left"; fails=$((fails + 1)); }
+refute "keep-data leaves the database alone" "DROP DATABASE" "$calls"
+[ ! -e "$root/h/opt" ] && [ ! -e "$root/h/bin/felis" ] \
+  && echo "PASS /opt/felis and the host binary are removed" \
+  || { echo "FAIL /opt/felis or the host binary is still there"; fails=$((fails + 1)); }
+[ -z "$(ls "$root/h/units")" ] && echo "PASS every Felis unit file is removed" \
+  || { echo "FAIL units left: $(ls "$root/h/units")"; fails=$((fails + 1)); }
+expect "the timers are disabled" "SYSTEMCTL disable --now felis-db-backup.timer" "$calls"
+expect "the velocity user is removed" "USERDEL felis-velocity" "$calls"
+expect "the run ends pointing at the reinstall steps" "Reinstall on top of kept data" "$out"
+
+# --- a failed final bundle stops everything ------------------------------------------------
+fresh_host
+printf '#!/bin/sh\necho "RUN felis $*" >> "%s"\nexit 1\n' "$root/calls" > "$root/h/bin/felis"
+out="$(run_uninstall "default felis minecraft" --yes)"
+expect "a failed bundle is fatal" "the final database bundle failed, so nothing was removed" "$out"
+[ -d "$root/h/opt" ] && [ -f "$root/h/units/felis-velocity.service" ] \
+  && echo "PASS nothing is removed when the bundle fails" \
+  || { echo "FAIL the uninstall went on after the bundle failed"; fails=$((fails + 1)); }
+run_uninstall "default felis minecraft" --yes --no-backup >/dev/null
+[ ! -d "$root/h/opt" ] && echo "PASS --no-backup goes on without one" \
+  || { echo "FAIL --no-backup did not remove anything"; fails=$((fails + 1)); }
+
+# --- a shared cluster --------------------------------------------------------------------
+fresh_host
+out="$(run_uninstall "default kube-system felis minecraft felis-build shop" --yes)"
+calls="$(cat "$root/calls")"
+refute "a cluster that runs something else keeps k3s" "RUN k3s-uninstall.sh" "$calls"
+expect "the reason names the other namespace" "shop" "$out"
+expect "Felis's volumes are retained before their claims go" 'KUBE patch pv pvc-1 -p {"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' "$calls"
+refute "a volume of another namespace is not touched" "patch pv pvc-9" "$calls"
+expect "Felis's namespaces are deleted" "KUBE delete namespace felis minecraft felis-build" "$calls"
+expect "the CRD is deleted" "KUBE delete crd minecraftservers.felis.lolicon.best" "$calls"
+
+out="$(fresh_host; run_uninstall down --yes)"
+expect "a k3s that does not answer stops the run" "k3s does not answer" "$out"
+fresh_host
+run_uninstall down --yes --keep-k3s >/dev/null
+calls="$(cat "$root/calls")"
+refute "--keep-k3s never runs k3s's uninstaller" "RUN k3s-uninstall.sh" "$calls"
+
+# --- purge -------------------------------------------------------------------------------
+fresh_host
+out="$(run_uninstall "default felis minecraft" --purge --yes)"
+calls="$(cat "$root/calls")"
+refute "purge takes no bundle" "db backup" "$calls"
+expect "purge drops the database" "DROP DATABASE IF EXISTS felis;" "$calls"
+expect "purge drops the role" "DROP ROLE IF EXISTS felis;" "$calls"
+expect "purge puts listen_addresses back" "ALTER SYSTEM RESET listen_addresses;" "$calls"
+[ ! -e "$root/h/etc" ] && [ ! -e "$root/h/data" ] && echo "PASS purge removes /etc/felis and /var/lib/felis" \
+  || { echo "FAIL purge left state behind"; fails=$((fails + 1)); }
+[ ! -e "$root/h/cf/abc.json" ] && echo "PASS purge removes the tunnel's credentials file" \
+  || { echo "FAIL the tunnel credentials are still there"; fails=$((fails + 1)); }
+[ ! -e "$root/h/data/retained" ] && echo "PASS purge does not set the volumes aside" \
+  || { echo "FAIL purge kept the volumes"; fails=$((fails + 1)); }
+hba="$(cat "$root/h/hba.conf")"
+refute "the Felis block leaves pg_hba.conf" "FELIS MANAGED" "$hba"
+expect "the distro's own rules stay" "host all all 127.0.0.1/32 ident" "$hba"
+case "$hba" in
+  "local all all peer"*) echo "PASS no blank line is left where the block was" ;;
+  *) echo "FAIL pg_hba.conf starts with: $(printf '%s' "$hba" | head -n 1)"; fails=$((fails + 1)) ;;
+esac
+expect "purge cleans Docker's build cache" "DOCKER builder prune -af" "$calls"
+
+# --- the pieces read before they are removed ---------------------------------------------
+fresh_host
+lib() { STATE_DIR="$root/h/etc" OPT_DIR="$root/h/opt" UNIT_DIR="$root/h/units" FELIS_UNINSTALL_SOURCED=1 \
+  bash -c '. "$0"; '"$1" "$US"; }
+expect "the game port comes from velocity.toml" "25577" "$(lib game_port)"
+expect "FELIS_GAME_PORT overrides it" "25599" "$(FELIS_GAME_PORT=25599 lib game_port)"
+rm "$root/h/opt/velocity/velocity.toml"
+expect "without velocity.toml the port is the default" "25565" "$(lib game_port)"
+printf '[Service]\nExecStart=/usr/local/bin/felis nano -listen 10.0.0.5:8082 -config x\n' > "$root/h/units/felis-nano.service"
+expect "the nano port comes from its unit" "8082" "$(lib nano_port)"
+expect "the tunnel config comes from the cloudflared unit" "$root/h/etc/cloudflared.yml" "$(lib tunnel_config)"
+rm "$root/h/units/cloudflared-felis.service"
+expect "without the unit the tunnel config is the default path" "$root/h/etc/cloudflared.yml" "$(lib tunnel_config)"
+
+out="$(FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; parse_args --bogus' "$US" 2>&1)"
+expect "an unknown option is refused" "unknown option: --bogus" "$out"
+printf 'nope\n' > "$root/tty"
+out="$(CONFIRM_TTY="$root/tty" FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; confirm; echo WENT ON' "$US" 2>&1)"
+expect "any answer but the word stops it" "not confirmed; nothing was changed" "$out"
+printf 'yes\n' > "$root/tty"
+out="$(CONFIRM_TTY="$root/tty" FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; confirm; echo WENT ON' "$US" 2>&1)"
+expect "yes goes on" "WENT ON" "$out"
+out="$(CONFIRM_TTY="$root/tty" FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; PURGE=1; confirm; echo WENT ON' "$US" 2>&1)"
+expect "a purge wants the word purge" "not confirmed" "$out"
+out="$(CONFIRM_TTY="$root/no-tty/x" FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; confirm; echo WENT ON' "$US" 2>&1)"
+expect "with no terminal it asks for --yes" "no terminal to confirm on; re-run with --yes" "$out"
+
+# Every unit the installer writes is one the uninstaller removes: a timer left behind
+# keeps firing a felis binary that is gone.
+units="$(awk '/^FELIS_UNITS=\(/ { f = 1; next } f && /^\)/ { f = 0 } f' "$US")"
+for u in $(sed -n 's|^[A-Z_]*="/etc/systemd/system/\([^"]*\)"$|\1|p' "$(dirname "$US")/bootstrap.sh"); do
+  expect "the uninstaller removes $u" " $u" " $(printf '%s' "$units" | tr '\n' ' ')"
+done
+
+if [ "$fails" -eq 0 ]; then
+  echo "ALL PASS"
+else
+  echo "$fails FAILED"
+fi
+exit "$fails"

deploy/update-game-stack-lock.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# Refreshes deploy/game-stack.lock to upstream's newest builds, hashed.
+#
+#   bash deploy/update-game-stack-lock.sh            # rewrite the lock in place
+#   bash deploy/update-game-stack-lock.sh --check    # exit 1 if upstream moved on
+#
+# It runs the same resolver bootstrap.sh uses for FELIS_GAME_STACK=latest (the functions are
+# lifted out of bootstrap.sh, so the two cannot drift), then pins Velocity's newest build of
+# VELOCITY_LATEST_MINOR. Limbo and LuckPerms publish no digest, so their jars are downloaded
+# and hashed here; Paper and Velocity come from Fill's content-addressed URLs.
+#
+# Review the diff before committing: MC_VERSION moves the login gate's protocol, and the
+# lobby, plain-Paper image and every client follow it. The plugins compile against these
+# same builds (paper-api, the Limbo API, velocity-api) with their checksums pinned in
+# plugins/*/gradle/verification-metadata.xml, so a moved build also moves those; go test .
+# names what is left to bring along.
+set -Eeuo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BS="${here}/bootstrap.sh"
+LOCK="${here}/game-stack.lock"
+check=0
+case "${1:-}" in
+  --check) check=1 ;;
+  "") ;;
+  *) printf 'usage: %s [--check]\n' "$0" >&2; exit 2 ;;
+esac
+
+log()  { printf '[lock] %s\n' "$*" >&2; }
+ok()   { printf '[ ok ] %s\n' "$*" >&2; }
+warn() { :; }
+die()  { printf '[fail] %s\n' "$*" >&2; exit 1; }
+
+lift() { # function-name
+  local body
+  body="$(awk -v f="$1" '$0 ~ "^" f "\\(\\) \\{" {on=1} on {print} on && /^}/ {exit}' "$BS")"
+  [ -n "$body" ] || die "bootstrap.sh no longer defines $1"
+  eval "$body"
+}
+for fn in meta_get papermc_latest_jar luckperms_latest_jar url_sha256 resolve_latest_game_jars; do
+  lift "$fn"
+done
+eval "$(grep '^VELOCITY_LATEST_MINOR=' "$BS")"
+[ -n "${VELOCITY_LATEST_MINOR:-}" ] || die "bootstrap.sh no longer sets VELOCITY_LATEST_MINOR"
+
+resolve_latest_game_jars
+log "resolving the newest Velocity ${VELOCITY_LATEST_MINOR} build"
+velocity="$(papermc_latest_jar velocity "$VELOCITY_LATEST_MINOR")" \
+  || die "no Velocity build for ${VELOCITY_LATEST_MINOR}"
+# shellcheck disable=SC2034 # read back through ${!key} below
+VELOCITY_VERSION="$VELOCITY_LATEST_MINOR"
+# shellcheck disable=SC2034
+VELOCITY_JAR_URL="${velocity% *}"
+# shellcheck disable=SC2034
+VELOCITY_JAR_SHA256="${velocity##* }"
+
+tmp="$(mktemp)"
+trap 'rm -f "$tmp"' EXIT
+# The comment header is kept as it is; only the KEY=value lines are regenerated.
+sed -n '/^#/p;/^#/!q' "$LOCK" > "$tmp"
+for key in MC_VERSION LIMBO_VERSION LIMBO_JAR_URL LIMBO_JAR_SHA256 LIMBO_SCHEM_URL LIMBO_SCHEM_SHA256 \
+  PAPER_JAR_URL PAPER_JAR_SHA256 LUCKPERMS_JAR_URL LUCKPERMS_JAR_SHA256 \
+  VELOCITY_VERSION VELOCITY_JAR_URL VELOCITY_JAR_SHA256; do
+  printf '%s=%s\n' "$key" "${!key}" >> "$tmp"
+done
+
+if cmp -s "$tmp" "$LOCK"; then
+  ok "game-stack.lock already pins upstream's newest builds"
+  exit 0
+fi
+diff -u "$LOCK" "$tmp" >&2 || true
+if [ "$check" = 1 ]; then
+  die "upstream has newer builds than game-stack.lock"
+fi
+cp "$tmp" "$LOCK"
+ok "game-stack.lock updated; move plugins/paper's paper-api pin to the new Paper build and regenerate the plugins' gradle/verification-metadata.xml (plugins/README.md \"Dependency verification\"), run go test . and the bootstrap tests, then commit"

docs/deferred-seams.md

@@ -28,7 +28,10 @@ A grep across `*.md` and `*.go` returns both sets; only the Go ones are seams.
 
 - `internal/updates/seams.go:32` — `Notifier`. `internal/mail` sends OTP over SMTP,
   but nothing adapts it to this interface and no in-game channel exists. `felis
-  update` passes nil deliberately: a human typing the command is the notification.
+  update` passes nil deliberately. The notification is the panel instead:
+  `felis-update-check.timer` runs `felis update --record` daily on the host, which
+  stores the report under `platform_settings.update_report`, and **Admin → Updates →
+  Component versions** shows it with the command that applies each update.
 - `internal/updates/seams.go:43` — `Applier`. Nothing applies an update anywhere. A
   nil applier is not silent — `Run` records `errNoApplier` against every planned
   apply, so a mis-scheduled apply is loud rather than lost.
@@ -37,11 +40,10 @@ A grep across `*.md` and `*.go` returns both sets; only the Go ones are seams.
   control-plane Deployment image, and the Velocity jar inspection. Both are answered
   on the host path (see "Built" below), so this gap is specific to a caller that has
   a cluster client instead of the node.
-- `internal/api/handlers_updates.go:21,28` — the maintenance window persists and the
+- `internal/api/handlers_updates.go` — the maintenance window is advisory: no
-  API serves it, but the in-cluster CronJob that would hand a real window to a runner
+  in-cluster runner applies updates. `felis update` reads the stored window, prints
-  does not exist. `felis update` runs with a zero window, under which every
+  where now sits against it and warns before an apply outside it; the runner itself
-  `Scheduled` component degrades to a notify, so no path can currently claim an
+  still runs with a zero window, so no path can claim an apply is under way.
-  apply is under way.
 - `internal/submit/blobstore.go` — CLOSED 2026-09-22. The uploads PVC still cannot
   cross namespaces, so the transport went through the API instead of a mount: the
   derived context ref is now the internal-face URL
@@ -52,17 +54,10 @@ A grep across `*.md` and `*.go` returns both sets; only the Go ones are seams.
   Secret-replica mechanism the login gate uses (bootstrap + `felis setup`), and the
   build egress lock allows exactly the control namespace on the internal port.
   Uniform for local and s3:// stores — neither hands the sandboxed build Pod a
-  filesystem view or object-store credentials. Kaniko/Trivy images are
+  filesystem view or object-store credentials. Kaniko, Trivy and Trivy's two DBs
-  external-only by default; `[registry] kaniko_image / trivy_image /
+  come from the registry's `mirror/` copies, which the installer and
-  build_cpu_limit / build_mem_limit` override them for mirrored or air-gapped
+  felis-build-tools.timer keep current (`felis mirror-build-tools`,
-  installs. Trivy's vulnerability DB is the same story, and now has its own knob:
+  docs/troubleshooting.md §8e); the `[registry]` keys override them.
-  `[registry] trivy_db_repository` points `--db-repository` at an internal mirror
-  (recipe in docs/troubleshooting.md §8e); `trivy_java_db_repository` does the
-  same for the Java DB, which Trivy fetches so soon as the scanned image contains
-  a jar — i.e. for every real modpack build. Left unset on an egress-locked box
-  the scan step fails closed — Kaniko pushes, Trivy exits on the DB download —
-  which is the correct fail direction but leaves the build unfinished, so the
-  mirrors are part of a production build install.
 
 ## Built; only its I/O is unverifiable from this repo
 
@@ -95,10 +90,19 @@ worth revisiting.
   moved inside `ClaimServer` (advisory lock + re-check + UPDATE in one transaction),
   red-then-green in the pgint suite, which is exactly the real-Postgres harness this
   line was waiting for.
-- `internal/api/api.go:671` — `cooldownLimiter` is process-local, so across N api
+- `internal/api/api.go:773` — `cooldownLimiter` is process-local, so across N api
   replicas a caller could draw up to N OTP codes per window. The intra-replica burst
   is closed; cross-replica bounding needs a shared store, out of scope for a
-  single-replica install.
+  single-replica install. Revisit before the api Deployment runs more than one
+  replica.
+- `internal/submit/submit.go:524` — the per-user upload storage budget reads the
+  stored bytes, then writes. On one replica the API's per-user upload reservation
+  serializes it; across replicas a burst can overshoot by one blob per interleaved
+  upload, each still under the single-blob cap. The pending-submission cap no
+  longer has this shape: `CreateSubmission` counts and inserts under a
+  per-submitter advisory lock (pgint `TestSubmitPendingCapHoldsUnderConcurrency`).
+  Revisit with the cooldown above, before scaling api replicas: a reservation row
+  per upload in the same kind of transaction closes it.
 - `internal/submit/submit.go:436` and `internal/submit/submit_test.go:351` — a
   post-CAS `Approve`
   failure leaves a row indistinguishable from the benign case, so `Approve` returns a
@@ -114,12 +118,11 @@ worth revisiting.
 
 ## Wired since the marker was written
 
-- `internal/api/handlers_email_otp.go:54,223,227` and `internal/api/api.go:84` —
+- `internal/api/handlers_email_otp.go` and `internal/api/api.go` — SMTP shipped on
-  SMTP shipped on
+  2026-07-20 (`internal/mail`, wired in `cmd/felis/api.go`). An install with no
-  2026-07-20 (`internal/mail`, wired at `cmd/felis/api.go:264`). The nil-`Mailer`
+  `[smtp]` section leaves the `Mailer` nil, and every door that mails a code answers
-  branch that logs the code server-side is a runtime fallback for an install with no
+  503 `mail_unavailable`; codes are never logged. The reading "Felis cannot send
-  `[smtp]` section, not an unbuilt feature. The comments are accurate; the reading
+  mail" is stale.
-  "Felis cannot send mail" is not.
 - `internal/config/config.go:117` — was stale. It described the upload transport as a
   deferred integration after both backends had shipped (`LocalContextStore`,
   `S3ContextStore`, selected in `cmd/felis/api.go` by the shape of the configured
@@ -132,8 +135,8 @@ worth revisiting.
 
 ## Recorded outside the code
 
-- `deploy/limbo/README.md:139` — no NetworkPolicy locks the minecraft-namespace
+- The minecraft-namespace egress is locked (`felis-server-egress`, DNS plus the
-  egress or the control-namespace ingress today, which is why the login pod reaches
+  public internet with every private range and the node's own global addresses
-  `felis-api-internal:8081`. This is a conditional obligation rather than a seam: if
+  excluded) and `felis-login-to-internal-api` opens the one platform path a game pod
-  a future deployment adds either lock, it must also open that path. Spec v4.1 §21
+  needs — login → felis-api:8081. Any new in-cluster service a game server must call
-  asks for those policies; `cmd/felis/manifests.go` renders the game-port one.
+  needs its own allow policy next to that one (`internal/platform/netpol.go`).

docs/operations.md

@@ -0,0 +1,350 @@
+# Felis Operations Guide
+
+What a Felis host needs, how big it should be, how to take Felis off it again, and where
+the disaster-recovery procedures live. Fault-finding is in
+[troubleshooting.md](troubleshooting.md); this document refers to its sections as §N.
+
+Evidence tags follow troubleshooting.md: **[VM-VERIFIED]** was run on a real host,
+**[CI]** runs end to end on every push to main (`.github/workflows/e2e.yml`),
+**[GO-TESTED]** / **[SH-TESTED]** is covered by `go test` or the shell tests under
+`deploy/`, **[CODE-ONLY]** is what the code does and has not been run end to end.
+
+## 1. Supported hosts
+
+`deploy/bootstrap.sh` provisions a single node. It needs systemd, root, and one of the
+package managers below; everything else (Docker, k3s, PostgreSQL, the JRE, cloudflared)
+it installs.
+
+| OS family | Package manager | Architectures | Status |
+|---|---|---|---|
+| CentOS Stream 9 (firewalld active, PostgreSQL 13) | dnf | aarch64 | **[VM-VERIFIED]** install, same-version rerun, upgrade, uninstall and reinstall |
+| Ubuntu 24.04 LTS | apt | x86_64 | **[CI]** fresh install, same-commit rerun, and upgrade from the newest release to the pushed commit |
+| RHEL / Rocky / Alma 9, Fedora | dnf | x86_64, aarch64 | [CODE-ONLY] same code path as CentOS Stream |
+| Debian 12, other Ubuntu releases | apt | x86_64, aarch64 | [CODE-ONLY] |
+| openSUSE Leap / Tumbleweed | zypper | x86_64, aarch64 | [CODE-ONLY] |
+| Arch Linux | pacman | x86_64, aarch64 | [CODE-ONLY] |
+
+Pinned component versions (a fresh install gets exactly these; an installed k3s or
+cloudflared is left as it is, see §4):
+
+| Component | Version | Where it is pinned |
+|---|---|---|
+| k3s | v1.36.4+k3s1 | `FELIS_K3S_VERSION` in `bootstrap.sh` |
+| cloudflared | 2026.9.1 | `FELIS_CLOUDFLARED_VERSION`, sha256 per architecture |
+| Temurin JRE (Velocity) | 25, patch build pinned | `FELIS_JRE_VERSION`, sha256 per architecture |
+| Go (nano builds) | 1.26.8 | `GO_PINNED_VERSION`, sha256 per architecture |
+| Minecraft / Limbo / Paper / Velocity / LuckPerms | `deploy/game-stack.lock` | §15b |
+| PostgreSQL | the distribution's package | 13 and 18 are exercised by the `pgint` CI job |
+
+32-bit hosts are not supported: there is no k3s, JRE or Go build the installer will fetch
+for them.
+
+One node is the whole supported shape. A world volume is a ReadWriteOnce claim on the
+node's local-path storage, so a game server's pod is pinned to the node that first
+scheduled it and cannot move when that node fails; the operator and felis-api each run
+as a single replica without leader election, so an upgrade or a node restart pauses
+wakes and stops until their pod is back. Joining k3s agents to the cluster is untested
+and gains no failover. A multi-node shape would need, at least, storage that can follow a
+pod to another node and leader election in felis-operator (controller-runtime's
+`LeaderElection`) so a second replica can stand by.
+
+### While felis-api restarts
+
+An installer rerun that changes felis-api, a node restart or a crashed pod takes the API
+away until its new pod is ready: about 12 s on the reference VM (`kubectl rollout
+restart` to Available). Its Deployment keeps one replica with the Recreate strategy, so
+the old pod is gone before the new one starts. Two pods at once would be wrong for
+felis-api: the uploads volume is ReadWriteOnce, a chunked upload is serialized inside the
+process, and the build reconciler, restore settler, registry pruner, upload reapers and
+audit retention run in-process without leader election, so each would run twice. During
+the window:
+
+- Players already on a server stay there; game servers keep running.
+- A player leaving the login gate or joining a server by its address is admitted when
+  felis-api confirmed their link within the last 10 minutes; anyone else is told login
+  verification is temporarily unavailable.
+- The login gate retries a new login for up to 60 s and tells the player it is retrying,
+  so a restart shorter than that only delays the login.
+- Wakes, stops, `/link` and the panel wait for the API.
+- A Velocity restart in the window routes on
+  `/opt/felis/velocity/plugins/felis-link/last-servers.json`, the last server list the
+  API answered with, until a refresh succeeds (every 15 s).
+
+## 2. Sizing
+
+### What the platform itself uses
+
+Measured on the verification host (4 vCPU, 5.5 GB RAM, 6 GB swap, CentOS Stream 9
+aarch64) with the control plane, the login and lobby system servers and one idle Paper
+server running **[VM-VERIFIED]**:
+
+| Process | Resident memory |
+|---|---|
+| k3s (server, kubelet, containerd) | ~1.1 GB |
+| Velocity (`-Xms512M -Xmx1G`, heap pre-touched) | ~0.73 GB |
+| lobby (Paper, pod limit 1 GiB) | ~0.7–0.85 GB |
+| login (Limbo, pod limit 512 MiB) | ~0.16 GB |
+| felis-api, felis-operator, registry gate | ~50 MB each |
+| PostgreSQL | ~30 MB plus page cache |
+| **Total in use** | **~3.4 GB** |
+
+Every game server adds the memory its owner gave it: the pod's limit equals its request,
+and the JVM heap is derived from it (§1a). Quotas cap it per user (panel → 管理 → 配额).
+
+The installer's own peak is the image builds (Docker plus a Gradle container); it stops
+Docker afterwards so that memory goes back to the servers. On a host under 2 GB of RAM
+without swap it adds a 2 GiB `/swapfile`.
+
+### Recommendations
+
+| Concurrent players | Game servers running | CPU | RAM | `FELIS_VELOCITY_XMX` |
+|---|---|---|---|---|
+| up to 20 | 1–2 small | 2 vCPU | 4 GB + 2 GB swap | 1G (default) |
+| up to 100 | 3–5 | 4 vCPU | 8–16 GB | 1G |
+| up to 300 | 5–10 | 8 vCPU | 16–32 GB | 2G |
+| 300+ | more | 8+ vCPU | 32 GB+ | 3G–4G |
+
+The player-count rows are planning figures, not measurements: a Minecraft server's cost
+depends mostly on what its players do (view distance, redstone, mods). Size RAM as the
+platform's ~3.5 GB plus the sum of the servers you expect to run at once, then add a
+quarter for the page cache and PostgreSQL. Velocity itself needs little per player; raise
+its heap when `journalctl -u felis-velocity` shows long GC pauses or `OutOfMemoryError`.
+
+`FELIS_VELOCITY_XMX` (default `1G`, at least `256M`, written `<n>M` or `<n>G`) is read on
+every installer run. The initial heap stays at 512M, or equals the maximum when that is
+lower. Changing it rewrites the unit, and the rerun restarts the proxy, which disconnects
+everyone online; do it in a quiet hour **[VM-VERIFIED]**:
+
+```
+curl -fsSL <raw-url>/deploy/bootstrap.sh | sudo FELIS_VELOCITY_XMX=2G bash
+```
+
+### Disk
+
+| What | Where | Size |
+|---|---|---|
+| Worlds | one volume per server under `/var/lib/rancher/k3s/storage` | what the world grows to |
+| World archives | the `felis-backups` volume (`FELIS_BACKUP_STORAGE`, default 10Gi requested) | about one compressed world per backup kept |
+| In-cluster registry | the `registry` volume (default 10Gi requested) | 2–3 GB for the stock images; grows with custom builds, pruned daily (§9) |
+| k3s's containerd images | `/var/lib/rancher/k3s/agent/containerd` | 6–9 GB |
+| Docker's images and build cache | `/var/lib/containerd` (Docker's containerd store) | 5–10 GB after repeated upgrades |
+| Toolchains and sources | `/opt/felis` | ~2.5 GB |
+| Database bundles | `/var/lib/felis/db-backups` | a few MB each, 14 daily kept |
+
+k3s's local-path volumes do not enforce the requested sizes (§9), so every volume shares
+the root filesystem. Give the host at least **40 GB**, and 60 GB or more once worlds and
+custom images accumulate. The watchdog mails the owners when a watched filesystem passes
+its threshold, and §13b covers a full disk. `docker builder prune -af` (with Docker
+started) reclaims the build cache when space is short; the next upgrade rebuilds it.
+
+## 3. Uninstall
+
+`deploy/uninstall.sh` takes off what the installer put on. It prints what it will remove
+and asks before it starts (`--yes` skips the question) **[SH-TESTED]
+[VM-VERIFIED]**:
+
+```
+curl -fsSL <raw-url>/deploy/uninstall.sh | sudo bash -s -- --yes     # keep the data
+curl -fsSL <raw-url>/deploy/uninstall.sh | sudo bash -s -- --purge   # remove the data too
+```
+
+With a private repository, fetch it the way the README fetches `bootstrap.sh`.
+
+Both modes remove the `felis-*` systemd units and `cloudflared-felis.service`, the
+Velocity user, `/opt/felis`, `/usr/local/bin/felis`, the installer's cloudflared binary
+(unless another unit runs it), the `felis_postgres` and `felis_edge` nftables tables and
+the firewalld ports the installer opened. k3s goes with k3s's own `k3s-uninstall.sh` when
+the cluster holds nothing but Felis's namespaces; when it runs anything else only
+`felis`, `minecraft`, `felis-build` and the MinecraftServer CRD are deleted.
+`--keep-k3s` and `--remove-k3s` override that choice.
+
+| | keep data (default) | `--purge` |
+|---|---|---|
+| Final database bundle | taken first (`felis db backup -label manual`); a failure stops the uninstall before anything is removed. `--no-backup` skips it | none |
+| `felis` database and role | kept | dropped; `listen_addresses` and `pg_hba.conf` go back to how they were |
+| `/etc/felis` (secrets, `felis.toml`, `offsite.env`, tunnel config) | kept; `bootstrap.done` and the per-run records go | deleted, with the tunnel's credentials file |
+| `/var/lib/felis` (database bundles) | kept | deleted |
+| Worlds, archives, registry, uploads | moved to `/var/lib/felis/retained/k3s-storage-<stamp>/` (with `--keep-k3s`: their volumes switch to `Retain` and stay in place) | deleted |
+| Felis images, Docker build cache | kept | deleted |
+
+Neither mode removes packages (Docker, PostgreSQL, git, nftables) or the swap file: other
+software may use them. On a host that should end up bare:
+
+```
+sudo swapoff /swapfile && sudo rm /swapfile && sudo sed -i '\|^/swapfile |d' /etc/fstab
+sudo dnf remove docker-ce docker-ce-cli containerd.io postgresql-server   # or apt/zypper/pacman
+```
+
+The Cloudflare side outlives the host. After an uninstall that is final, delete the
+tunnel (Zero Trust → Networks → Tunnels, or `cloudflared tunnel delete <name>`), its
+DNS records for the panel hostnames, and the Access application.
+
+### Reinstall on top of kept data
+
+A keep-data uninstall leaves everything a reinstall needs. The installer reuses
+`/etc/felis/secrets.env`, so the database password and the forwarding and session
+secrets are unchanged, and it migrates the kept database instead of creating one
+**[VM-VERIFIED]**.
+
+Each step below was run on the reference VM after a keep-data uninstall, and the
+restored worlds matched their kept `level.dat` checksums **[VM-VERIFIED]**. `kept` names
+the directory the uninstall moved the volumes to:
+
+```
+kept="$(ls -d /var/lib/felis/retained/k3s-storage-* | tail -n 1)"
+store=/var/lib/rancher/k3s/storage
+```
+
+1. Install as usual (`curl ... | sudo bash`). Name the same root domain if it was not
+   the `<ip>.nip.io` default: `felis.host.toml` is kept, and the installer reads the
+   domain from it.
+2. Run `sudo felis setup`. It recreates the login and lobby servers; the Owner already
+   exists, so it opens on the status screen and you can quit there.
+3. Put the image registry and the uploads back. They hold every custom server image
+   and uploaded file; without the registry, a restored server fails to pull its image.
+
+   ```
+   sudo k3s kubectl -n felis scale deploy/registry deploy/felis-api --replicas=0
+   sudo k3s kubectl -n felis wait --for=delete pod -l app.kubernetes.io/component=registry --timeout=120s
+   sudo k3s kubectl -n felis wait --for=delete pod -l app.kubernetes.io/component=api --timeout=120s
+   sudo rsync -a --delete "$kept"/pvc-*_felis_registry/ "$(ls -d $store/pvc-*_felis_registry)"/
+   sudo rsync -a --delete "$kept"/pvc-*_felis_felis-uploads/ "$(ls -d $store/pvc-*_felis_felis-uploads)"/
+   sudo k3s kubectl -n felis scale deploy/registry deploy/felis-api --replicas=1
+   ```
+
+   Then run the installer once more. It pushes this release's images over the older
+   copies the kept registry carried.
+4. Bring the game servers back. The final bundle holds every MinecraftServer as it was;
+   the selector skips login and lobby, which step 2 created for this release:
+
+   ```
+   b="$(ls -t /var/lib/felis/db-backups/felis-db-*-manual.tar | head -n 1)"
+   tar -xOf "$b" k8s/minecraftservers.json \
+     | sudo k3s kubectl apply -l '!felis.lolicon.best/system-role' -f -
+   ```
+
+5. Put each world back. A server's volume exists once it has started once, so start it
+   from the panel, stop it again, and copy the kept world over the new one:
+
+   ```
+   s=<server>
+   sudo rsync -a --delete "$kept"/pvc-*_minecraft_world-$s-0/ "$(ls -d $store/pvc-*_minecraft_world-$s-0)"/
+   ```
+
+   Then start it. The lobby works the same way: stop it with
+   `sudo k3s kubectl -n minecraft patch minecraftserver lobby --type=merge -p '{"spec":{"desiredState":"Stopped"}}'`,
+   copy `world-lobby-0`, and patch it back to `Running`.
+6. Bring the archives back so the panel's restore points work again. The archive volume
+   appears with the first backup, so back up any server from the panel first, then:
+
+   ```
+   sudo rsync -a "$kept"/pvc-*_minecraft_felis-backups/ "$(ls -d $store/pvc-*_minecraft_felis-backups)"/
+   ```
+
+   With an off-site bucket configured, `sudo felis offsite fetch-worlds` fetches them
+   instead (troubleshooting §16).
+7. Delete `/var/lib/felis/retained/` once every server is back.
+
+## 4. Upgrading the pieces around Felis
+
+A rerun of the installer upgrades Felis itself (§15). The components it installs keep
+the version they were installed with unless noted:
+
+| Component | How a rerun treats it | Upgrade |
+|---|---|---|
+| Velocity, Limbo, Paper, LuckPerms | follow `deploy/game-stack.lock` | rerun after a release that moves the lock (§15b) |
+| Temurin JRE | moves to the pinned patch build | rerun |
+| k3s | left alone | rerun with `FELIS_UPGRADE_DEPS=1`: moves to the pinned release through that tag's install script, one minor version at a time (a bigger jump stops before anything changes and names the release to go through), never backwards |
+| cloudflared | left alone | rerun with `FELIS_UPGRADE_DEPS=1`: swaps `/usr/local/bin/cloudflared` for the pinned, sha256-checked release and restarts `cloudflared-felis`; a cloudflared the distribution installed stays with its package manager |
+| PostgreSQL | the distribution's package | the package manager for a minor release; a major version needs `pg_upgrade` first (below) |
+| Docker, git, nftables | distribution packages | the package manager |
+
+```sh
+curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh \
+  | sudo FELIS_UPGRADE_DEPS=1 bash
+```
+
+`sudo felis update` reports Felis, Velocity, k3s, cloudflared, the JRE and PostgreSQL
+against their newest releases; `--k3s`, `--cloudflared`, `--jre` and `--postgres` narrow
+it to one. PostgreSQL is compared within its major, since a minor release is a package
+update, and a major past its end of life gets a note naming the current one.
+
+The installer also sets up `felis-update-check.timer`, which runs `felis update --record`
+once a day around 05:30 (and at boot after a missed run). `--record` stores the result
+in `platform_settings`, and the panel's **Admin → Updates → Component versions** card
+shows it: each component's installed and newest version, and for the ones with a newer
+release the `sudo felis update --<component>` line that prints how to apply it. Felis
+applies nothing on its own; the installer re-run above is the apply path. The card turns
+red when the newest record is older than 26 hours, meaning the timer stopped:
+
+```sh
+systemctl list-timers felis-update-check.timer
+journalctl -u felis-update-check -n 50 --no-pager
+sudo felis update --record   # record a fresh check now
+```
+
+### PostgreSQL major versions [CODE-ONLY]
+
+The installer takes the major the distribution ships (13 on EL9) and never moves it. To
+go to a newer one, stop the writers, keep a dump, then use the distribution's upgrade
+path:
+
+```sh
+sudo k3s kubectl -n felis scale deploy/felis-api deploy/felis-operator --replicas=0
+sudo -u postgres pg_dumpall > /root/felis-pg-$(date +%F).sql
+# EL9: sudo systemctl stop postgresql; sudo dnf module switch-to postgresql:16
+#      sudo dnf install postgresql-upgrade; sudo postgresql-setup --upgrade
+# Debian/Ubuntu: sudo pg_upgradecluster <old-major> main
+sudo systemctl start postgresql
+sudo k3s kubectl -n felis scale deploy/felis-api deploy/felis-operator --replicas=1
+```
+
+### The MinecraftServer CRD [VM-VERIFIED]
+
+Every rerun applies the CRD embedded in the `felis` binary (`felis bootstrap-assets crd`).
+It serves and stores the single version `v1alpha1`, and the apiserver refuses values the
+operator cannot act on:
+
+| Field | Accepted |
+|---|---|
+| `spec.rcon.port` | unset, `0` or `25575`: the allow-rcon NetworkPolicy opens only 25575, so any other port leaves the server unprobeable |
+| `spec.startup.timeoutSeconds`, `readinessTimeoutSeconds` | 0 – 86400 |
+| `spec.startup.healthHTTPPort` | 0 – 65535 |
+| `spec.lifecycle.terminationGracePeriodSeconds` | 0 – 3600 |
+| `spec.idle.emptySecondsBeforeStop` | 0 – 604800 (the panel caps it at 86400) |
+
+`0` means the operator's default throughout. An object stored before these rules keeps an
+out-of-range value until someone edits that field (CRD validation ratcheting). The operator
+reads a negative value as its default and an oversized one as written, so fix such a
+value by hand: `kubectl -n minecraft edit minecraftserver <name>`.
+
+**Moving to `v1beta1` (planned, not built).** The first breaking change to the spec ships as a new
+version, in this order, each step one release:
+
+1. The CRD serves `v1alpha1` and `v1beta1`, storage stays `v1alpha1`. While the two
+   schemas carry the same fields, `conversion.strategy: None` suffices; a renamed or
+   reshaped field needs a conversion webhook, which felis-operator would serve.
+2. Storage moves to `v1beta1`. The installer rewrites every object so etcd holds the new
+   version (`kubectl get minecraftservers -A -o json | kubectl replace -f -`), then sets
+   `status.storedVersions` of the CRD to `["v1beta1"]`.
+3. A later release stops serving `v1alpha1`. Felis itself reads through one Go type at a
+   time, so the operator and felis-api switch in the release that moves storage.
+
+## 5. Disaster recovery
+
+The procedures are in §16: what a database bundle holds, restoring one on the same host,
+rolling back an upgrade, and rebuilding on a new host from the off-site copy. For a
+production install:
+
+- **Configure the off-site copy** (`FELIS_OFFSITE_*`, §16 "Keep a copy somewhere
+  else"). Without it the world archives sit on the same disk as the worlds, and the
+  database bundles on the same disk as the database; losing the disk loses both. The
+  installer ends with `NO OFF-SITE COPY` until it is set.
+- **Keep the off-site encryption key off the host**, in a password manager. The bucket
+  holds only sealed objects.
+- **Keep one database bundle off the host** as well when there is no bucket. It contains
+  `secrets.env`, which a rebuild needs to read the rest.
+- **Rehearse the rebuild** once on a spare VM: §16 "Rebuild on a new host", steps 1–5,
+  then log in and restore one world. `felis offsite status` and `felis db check` exit
+  non-zero when the copy or the newest bundle is stale; wire them into your monitoring,
+  or rely on the watchdog's mail.

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/descope/virtualwebauthn v1.0.5
 	github.com/go-logr/logr v1.4.2
 	github.com/go-webauthn/webauthn v0.17.4
-	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgx/v5 v5.9.2
 	github.com/minio/minio-go/v7 v7.2.1
@@ -20,11 +19,13 @@ require (
 	k8s.io/api v0.31.3
 	k8s.io/apimachinery v0.31.3
 	k8s.io/client-go v0.31.0
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340
 	sigs.k8s.io/controller-runtime v0.19.3
 	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -49,6 +50,7 @@ require (
 	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/go-webauthn/x v0.2.6 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
@@ -108,7 +110,6 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.31.0 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect

go.sum

@@ -2,6 +2,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk
 github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=

internal/api/account_notice.go

@@ -0,0 +1,125 @@
+package api
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/metrics"
+)
+
+// Account change notices tell the owner of an account, at the verified address,
+// that a way into it was just added, removed or moved: a passkey registered or
+// removed, the email replaced (that notice goes to the OLD address, which is the
+// one the owner still reads if someone else made the change). They carry the time
+// and the source address and say what to do if the change was not theirs.
+// Best effort, like the lock notice: the change already happened.
+
+// notifyAccountChange mails one notice to the given address.
+func (a *API) notifyAccountChange(r *http.Request, to, subject, body string) {
+	if to == "" {
+		return
+	}
+	sender, ok := a.Mailer.(noticeSender)
+	if !ok {
+		log.Printf("auth: no notice mailer; account change notice %q was not sent (request_id=%s)",
+			subject, requestIDFromContext(r.Context()))
+		return
+	}
+	if ok, _ := a.mailGate().take(mailGateKey); !ok {
+		metrics.MailTotal.WithLabelValues("notice", "throttled").Inc()
+		log.Printf("auth: mail budget spent; account change notice %q was not sent (request_id=%s)",
+			subject, requestIDFromContext(r.Context()))
+		return
+	}
+	if err := sender.SendNotice(r.Context(), to, subject, body); err != nil {
+		metrics.MailTotal.WithLabelValues("notice", "failed").Inc()
+		log.Printf("auth: account change notice failed (request_id=%s): %v", requestIDFromContext(r.Context()), err)
+		return
+	}
+	metrics.MailTotal.WithLabelValues("notice", "sent").Inc()
+}
+
+// verifiedEmail is where a notice about p's account goes: the address it proved,
+// or nothing.
+func verifiedEmail(p *Principal) string {
+	if !p.EmailVerified {
+		return ""
+	}
+	return p.Email
+}
+
+func (a *API) notifyPasskeyAdded(r *http.Request, p *Principal) {
+	subject, body := accountChangeNotice(
+		"已添加 Passkey", "passkey added",
+		"你的 Felis 账户刚刚添加了一个 Passkey。", "A passkey was just added to your Felis account.",
+		"删除这个 Passkey", "remove that passkey",
+		a.now(), a.noticeIP(r))
+	a.notifyAccountChange(r, verifiedEmail(p), subject, body)
+}
+
+func (a *API) notifyPasskeyRemoved(r *http.Request, p *Principal) {
+	subject, body := accountChangeNotice(
+		"已删除 Passkey", "passkey removed",
+		"你的 Felis 账户刚刚删除了一个 Passkey，其它设备上的登录已全部退出。",
+		"A passkey was just removed from your Felis account, and every other device was signed out.",
+		"检查剩下的 Passkey", "check the passkeys that remain",
+		a.now(), a.noticeIP(r))
+	a.notifyAccountChange(r, verifiedEmail(p), subject, body)
+}
+
+// notifyEmailChanged tells the previous verified address where the account's
+// mail now goes, masked so the notice does not hand the new address to whoever
+// reads the old mailbox.
+func (a *API) notifyEmailChanged(r *http.Request, oldEmail, newEmail string) {
+	masked := maskEmail(newEmail)
+	subject, body := accountChangeNotice(
+		"邮箱已更换", "email changed",
+		"你的 Felis 账户的邮箱刚刚更换为 "+masked+"，这个地址以后不会再收到登录验证码。",
+		"The email on your Felis account was just changed to "+masked+". This address will no longer receive sign-in codes.",
+		"把邮箱改回来", "change the email back",
+		a.now(), a.noticeIP(r))
+	a.notifyAccountChange(r, oldEmail, subject, body)
+}
+
+func (a *API) noticeIP(r *http.Request) string {
+	if ip := a.clientIP(r); ip.IsValid() {
+		return ip.String()
+	}
+	return ""
+}
+
+// accountChangeNotice renders a bilingual notice. zhUndo/enUndo name the step
+// that reverses the change, for the "if this wasn't you" line.
+func accountChangeNotice(zhTitle, enTitle, zhWhat, enWhat, zhUndo, enUndo string, at time.Time, ip string) (subject, body string) {
+	when := at.UTC().Format("2006-01-02 15:04 MST")
+	zhIP, enIP := ip, ip
+	if ip == "" {
+		zhIP, enIP = "未知", "unknown"
+	}
+	subject = "Felis " + zhTitle + " · " + enTitle
+	body = fmt.Sprintf(`%s
+时间：%s
+来源 IP：%s
+如果不是你本人操作，请立即登录 Felis，在账户页%s并退出其它设备，然后联系服务器管理员。
+
+%s
+Time: %s
+From IP: %s
+If this wasn't you, sign in to Felis now, %s and sign out other devices on the Account page, then contact the server operator.
+`, zhWhat, when, zhIP, zhUndo, enWhat, when, enIP, enUndo)
+	return subject, body
+}
+
+// maskEmail keeps the first character of the local part and the domain:
+// [email protected] → a***@example.com.
+func maskEmail(email string) string {
+	at := strings.LastIndexByte(email, '@')
+	if at <= 0 {
+		return "***"
+	}
+	first := []rune(email[:at])[0]
+	return string(first) + "***" + email[at:]
+}

internal/api/api.go

@@ -1,9 +1,10 @@
 // Package api implements felis-api: one binary serving two faces (spec §7).
 //
 // The internal face (velocity / backend callbacks) authenticates with a static
-// service token and is never wrapped in Zero Trust. The external face (people /
+// per-caller service token and is never wrapped in Zero Trust. The external face
-// panel) authenticates with a Cloudflare Access JWT; admin-tier operations
+// (people / panel) authenticates the local session cookie; admin-tier operations
-// additionally require the admin Access path (spec §14, graded by operation).
+// additionally require a staff session on the operator console host (spec §14,
+// graded by operation).
 //
 // Handlers depend on the Repo and Cluster interfaces, so the request routing,
 // dual-face auth, input validation and authorization are all unit-tested with
@@ -14,7 +15,11 @@ package api
 
 import (
 	"context"
+	"fmt"
+	"log"
+	"log/slog"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
@@ -33,6 +38,11 @@ type API struct {
 	// still exercised even before the subsystem is wired in.
 	Builder ImageBuilder
 
+	// Images pins a whitelisted image ref to the digest it names when a server is
+	// created or its image is changed (internal/imagepin), so a later push over
+	// the same tag never reaches an existing world. Nil stores refs as given.
+	Images ImagePinner
+
 	// Console is the synchronous RCON write channel (spec §8 写=RCON). It is
 	// wired in production (cmd/felis); a nil Console makes the command route report
 	// 503 rather than panic, so the ownership boundary is still exercised in tests.
@@ -68,6 +78,12 @@ type API struct {
 	// endpoints only answer 202). Optional: nil → that route reports 503.
 	JobStatus JobStatusReader
 
+	// RestoreChains finds and settles the safety snapshots that run in front of a
+	// restore (restorechain.go). A restore takes a snapshot first only when this
+	// is set, the Backuper can chain a restore and the Restorer is wired, since
+	// something has to start the restore once the snapshot is done.
+	RestoreChains RestoreChains
+
 	// Files is the server file editor (list / read / write a file in a stopped
 	// server's world volume — the "one wrong line in server.properties" repair).
 	// Like Restorer and Backuper it is optional: when nil the file routes report
@@ -85,10 +101,10 @@ type API struct {
 	Submissions SubmissionService
 
 	// Mailer delivers player email one-time codes (spec §B2 onboarding). It is
-	// optional: when nil the email-OTP start route mints and persists the code but
+	// optional: when nil (no [smtp] relay) every door that mails a code answers 503
-	// logs it server-side instead of mailing it (a KNOWN-LIMITATION — the demo has no
+	// mail_unavailable before minting one, auth options stops offering email_otp,
-	// SMTP), so the verify flow is still exercised end-to-end. Production wires a real
+	// and a verified email stops counting as a reauth factor. The code is never
-	// sender. The code is never returned to the client on either path.
+	// returned to the client or logged.
 	Mailer OTPMailer
 
 	// Passkey verifies WebAuthn credential-creation ceremonies (spec §14 / Phase 6
@@ -123,6 +139,14 @@ type API struct {
 	// on the wake lever). Zero disables throttling.
 	WakeCooldown time.Duration
 
+	// BackupCooldown spaces out an owner's on-demand backups of one server, and
+	// BackupStoreCap refuses them once the present backups reach [archive]
+	// max_local_bytes (data-durability-9): each archive lands on the node disk
+	// the worlds and the database share. Admins and the break-glass console are
+	// exempt. Zero disables each lever.
+	BackupCooldown time.Duration
+	BackupStoreCap int64
+
 	// SubmitCreateCooldown / SubmitUploadCooldown throttle the user-modpack
 	// submission lane per user: create bounds how quickly review-queue rows can
 	// appear, upload bounds how often a user may stream a (up to 1 GiB) build
@@ -158,6 +182,20 @@ type API struct {
 	// Consumed by handleHasJoined (handlers_hasjoined.go).
 	AuthSources []AuthSource
 
+	// AuthDoorLimit bounds how often one client address may call the public
+	// pre-session auth doors (ratelimit.go). MailLimit bounds all mail the API
+	// sends, install-wide. Zero values disable them; cmd/felis wires both.
+	AuthDoorLimit RateLimit
+	MailLimit     RateLimit
+	// ClientIPHeader names the header the install's edge writes the client
+	// address into (CF-Connecting-IP behind the Cloudflare tunnel,
+	// X-Forwarded-For behind an operator proxy). Empty means the TCP peer.
+	ClientIPHeader string
+
+	// AccessLog receives one line per API request (observe.go). Nil logs logfmt
+	// to stderr.
+	AccessLog *slog.Logger
+
 	// Now is the clock, injectable for tests. Defaults to time.Now.
 	Now func() time.Time
 
@@ -172,6 +210,31 @@ type API struct {
 
 	streamCapOnce sync.Once
 	streamCap     *streamLimiter
+
+	authDoorOnce    sync.Once
+	authDoorBuckets *bucketSet
+	mailOnce        sync.Once
+	mailBuckets     *bucketSet
+
+	drainInit  sync.Once
+	drainClose sync.Once
+	drain      chan struct{}
+}
+
+// streamsClosing is closed once CloseStreams runs.
+func (a *API) streamsClosing() <-chan struct{} {
+	a.drainInit.Do(func() { a.drain = make(chan struct{}) })
+	return a.drain
+}
+
+// CloseStreams ends every log stream this API is relaying, now and from now on.
+// http.Server.Shutdown waits for handlers to return and cancels nothing, so a
+// console left open would hold the process until the pod's grace period ran out;
+// register this with RegisterOnShutdown. The EventSource on the other end
+// reconnects, and resumes from its Last-Event-ID on the next instance.
+func (a *API) CloseStreams() {
+	a.streamsClosing()
+	a.drainClose.Do(func() { close(a.drain) })
 }
 
 // panelURL returns the public player-console origin ("https://console.<root>"),
@@ -241,7 +304,7 @@ func (a *API) streamGate() *streamLimiter {
 }
 
 // streamKey identifies the principal a stream slot is charged to. It prefers the
-// stable user id and falls back to the email so a JWT principal without a user id is
+// stable user id and falls back to the email so a principal without a user id is
 // still bucketed by identity; an empty key (no authenticated identity, which the
 // external face's auth guard already precludes) shares one bucket, which is safe
 // because it is more restrictive, never less.
@@ -286,6 +349,15 @@ type apiRoute struct {
 	// whose EmailVerified is false is restricted to these routes only.
 	SetupAllowed bool
 
+	// AuthDoor marks a public pre-session auth door: it is rate limited per
+	// client address (throttleAuthDoor). The op-login status poll is left off,
+	// since the browser calls it every few seconds while it waits.
+	AuthDoor bool
+
+	// Callers lists the machines an internal-face route serves; every
+	// authenticated internal route names at least one, and external routes none.
+	Callers []Caller
+
 	h http.HandlerFunc
 }
 
@@ -293,6 +365,14 @@ type apiRoute struct {
 // service-token auth, never Zero Trust. It carries both health probes and the
 // metrics scrape.
 func (a *API) internalAPIRoutes() []apiRoute {
+	// Who may call what (Caller). The proxy drives the game-facing routes; the
+	// login gate only checks a joining player's bar and link and mints their bind
+	// code; the build Job only reads the context of the submission it builds; the
+	// on-node console only asks for a break-glass backup.
+	proxy := []Caller{CallerVelocity}
+	gate := []Caller{CallerVelocity, CallerLimbo}
+	build := []Caller{CallerBuild}
+	ops := []Caller{CallerOps}
 	return []apiRoute{
 		{Method: "GET", Pattern: "/healthz", Public: true, h: a.handleHealthz},
 		{Method: "GET", Pattern: "/readyz", Public: true, h: a.handleReadyz},
@@ -300,47 +380,47 @@ func (a *API) internalAPIRoutes() []apiRoute {
 		// no token, internal-only so it is never exposed off-cluster.
 		{Method: "GET", Pattern: "/metrics", Public: true, h: a.handleMetrics},
 
-		{Method: "GET", Pattern: "/api/v1/servers", h: a.handleListServers},
+		{Method: "GET", Pattern: "/api/v1/servers", Callers: proxy, h: a.handleListServers},
 		// The build Pod's context-fetch initContainer streams a submission's stored
 		// modpack through this route (build namespace cannot mount the uploads PVC).
-		{Method: "GET", Pattern: "/api/v1/internal/submissions/{id}/context", h: a.handleInternalSubmissionContext},
+		{Method: "GET", Pattern: "/api/v1/internal/submissions/{id}/context", Callers: build, h: a.handleInternalSubmissionContext},
-		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/ready", h: a.handleReady},
+		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/ready", Callers: proxy, h: a.handleReady},
-		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/join-event", h: a.handleJoinEvent},
+		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/join-event", Callers: proxy, h: a.handleJoinEvent},
 		// Domain-autostart (spec §9.1, §14): velocity drives the wake lever and polls
 		// status with its service token, identifying the joining player by online-mode
 		// UUID. These live on the internal face because velocity holds no web Principal;
 		// the external face keeps its own Principal-gated wake/status for the panel.
-		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/wake", h: a.handleInternalWake},
+		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/wake", Callers: proxy, h: a.handleInternalWake},
-		{Method: "GET", Pattern: "/api/v1/internal/servers/{name}/status", h: a.handleStatus},
+		{Method: "GET", Pattern: "/api/v1/internal/servers/{name}/status", Callers: proxy, h: a.handleStatus},
 		// Lobby `/menu` (spec §12): the felis-paper lobby is a pure UI face holding no
 		// token, so velocity drives these on its behalf — claim by online-mode UUID
 		// (the lobby's `Claim & Start`, separate from the autostartPolicy-gated wake)
 		// and the menu projection that adds the ownership-derived `claimable` the §11
 		// list/status views never carry.
-		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/claim", h: a.handleInternalClaim},
+		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/claim", Callers: proxy, h: a.handleInternalClaim},
-		{Method: "GET", Pattern: "/api/v1/internal/servers/{name}/menu", h: a.handleInternalMenuStatus},
+		{Method: "GET", Pattern: "/api/v1/internal/servers/{name}/menu", Callers: proxy, h: a.handleInternalMenuStatus},
 		// Account linking (spec §10): the in-game /link side mints a one-time code for a
 		// verified UUID. Internal-only — the code is born from an online-mode UUID the
 		// web never holds (account_link_codes has no user_id column).
-		{Method: "POST", Pattern: "/api/v1/internal/account/link/code", h: a.handleCreateLinkCode},
+		{Method: "POST", Pattern: "/api/v1/internal/account/link/code", Callers: gate, h: a.handleCreateLinkCode},
 		// QR scan-to-login completion poll (spec §B3 player game-login). After the player
 		// scans the QR-encoded code and the web verify writes the durable link, velocity
 		// polls this for the UUID it minted against and admits on {linked:true}. Read-only
 		// and keyed by the verified UUID (not the scanned code), so it consumes nothing
 		// and is safe to poll repeatedly.
-		{Method: "GET", Pattern: "/api/v1/internal/account/link/status/{mc_uuid}", h: a.handleLinkStatus},
+		{Method: "GET", Pattern: "/api/v1/internal/account/link/status/{mc_uuid}", Callers: gate, h: a.handleLinkStatus},
 		// Account migration (spec §B3 inherit), in-game side: /felis migrate puts the
 		// account linked to the running player's verified UUID into migrate mode. Internal
 		// only — the initiator is proven by online-mode auth, and the sensitive proof
 		// (step-up) still happens web-side before anything transfers.
-		{Method: "POST", Pattern: "/api/v1/internal/account/migrate/start", h: a.handleMigrateStart},
+		{Method: "POST", Pattern: "/api/v1/internal/account/migrate/start", Callers: proxy, h: a.handleMigrateStart},
 		// Username-collision reclaim (spec §B3): velocity records a Mojang-priority
 		// reclaim (bar the squatter UUID + stash its data for 30 days) and gates the
 		// limbo login by checking whether a connecting UUID was barred. Internal-only —
 		// velocity holds a service token, and the bar is keyed by UUID so the genuine
 		// Mojang player (same name, different UUID) always passes.
-		{Method: "POST", Pattern: "/api/v1/internal/player/reclaim", h: a.handleReclaimUsername},
+		{Method: "POST", Pattern: "/api/v1/internal/player/reclaim", Callers: proxy, h: a.handleReclaimUsername},
-		{Method: "GET", Pattern: "/api/v1/internal/player/blacklist/{mc_uuid}", h: a.handleCheckBlacklist},
+		{Method: "GET", Pattern: "/api/v1/internal/player/blacklist/{mc_uuid}", Callers: gate, h: a.handleCheckBlacklist},
 		// Felis-nano multi-source session verifier, behind player game-login. Velocity is
 		// pointed here with -Dmojang.sessionserver and issues the request itself; it speaks
 		// the vanilla sessionserver protocol and carries no token, so this is Public. It
@@ -353,19 +433,20 @@ func (a *API) internalAPIRoutes() []apiRoute {
 		// /felis web op approve. Internal face carries the pending queue and the
 		// approve action (service-token auth, no Principal); the public face carries
 		// the start/status/finish the staff member's browser drives.
-		{Method: "GET", Pattern: "/api/v1/internal/op-login/pending", h: a.handleOpLoginPending},
+		{Method: "GET", Pattern: "/api/v1/internal/op-login/pending", Callers: proxy, h: a.handleOpLoginPending},
-		{Method: "POST", Pattern: "/api/v1/internal/op-login/{id}/approve", h: a.handleOpLoginApprove},
+		{Method: "GET", Pattern: "/api/v1/internal/op-login/{id}", Callers: proxy, h: a.handleOpLoginShow},
+		{Method: "POST", Pattern: "/api/v1/internal/op-login/{id}/approve", Callers: proxy, h: a.handleOpLoginApprove},
 
 		// Break-glass backup (spec §B4 "Sync"): the on-node console POSTs here to
 		// snapshot a stopped world while the API is alive. Service-token auth (no
 		// Principal); the shared enqueueBackup tail enforces the RWO stopped-gate.
-		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/backup", h: a.handleInternalBackup},
+		{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/backup", Callers: ops, h: a.handleInternalBackup},
 	}
 }
 
 // externalAPIRoutes is the external face's served route table (spec §7, §14):
-// Cloudflare Access-JWT auth on every /api/v1 route; the Admin entries are
+// session auth on every non-public /api/v1 route; the Admin entries are
-// additionally gated on the admin Zero-Trust path. It exposes liveness only —
+// additionally gated on the operator console host. It exposes liveness only —
 // readiness is an internal concern.
 func (a *API) externalAPIRoutes() []apiRoute {
 	return []apiRoute{
@@ -381,21 +462,21 @@ func (a *API) externalAPIRoutes() []apiRoute {
 		// counter-slice to the anti-enumeration doors — the ONE sanctioned place existence
 		// is disclosed — but it never reveals staffness (methods computed with no role
 		// branch, so a staff and a player address in the same state are indistinguishable).
-		{Method: "POST", Pattern: "/api/v1/auth/options", Public: true, h: a.handleAuthOptions},
+		{Method: "POST", Pattern: "/api/v1/auth/options", Public: true, AuthDoor: true, h: a.handleAuthOptions},
-		{Method: "POST", Pattern: "/api/v1/auth/setup/redeem", Public: true, h: a.handleSetupRedeem},
+		{Method: "POST", Pattern: "/api/v1/auth/setup/redeem", Public: true, AuthDoor: true, h: a.handleSetupRedeem},
 		{Method: "GET", Pattern: "/api/v1/auth/setup/status", SetupAllowed: true, h: a.handleSetupStatus},
-		{Method: "POST", Pattern: "/api/v1/auth/passkey/login/begin", Public: true, h: a.handlePasskeyLoginBegin},
+		{Method: "POST", Pattern: "/api/v1/auth/passkey/login/begin", Public: true, AuthDoor: true, h: a.handlePasskeyLoginBegin},
-		{Method: "POST", Pattern: "/api/v1/auth/passkey/login/finish", Public: true, h: a.handlePasskeyLoginFinish},
+		{Method: "POST", Pattern: "/api/v1/auth/passkey/login/finish", Public: true, AuthDoor: true, h: a.handlePasskeyLoginFinish},
 		// Discoverable ("usernameless") passkey login (task #40): the from-zero sibling of the
 		// email-first pair above — no identifier typed, the account is resolved from the
 		// userHandle inside the signed assertion (handlers_passkey_discoverable.go).
-		{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/begin", Public: true, h: a.handlePasskeyLoginDiscoverableBegin},
+		{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/begin", Public: true, AuthDoor: true, h: a.handlePasskeyLoginDiscoverableBegin},
-		{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/finish", Public: true, h: a.handlePasskeyLoginDiscoverableFinish},
+		{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/finish", Public: true, AuthDoor: true, h: a.handlePasskeyLoginDiscoverableFinish},
-		{Method: "POST", Pattern: "/api/v1/auth/email/start", Public: true, h: a.handleLoginEmailStart},
+		{Method: "POST", Pattern: "/api/v1/auth/email/start", Public: true, AuthDoor: true, h: a.handleLoginEmailStart},
-		{Method: "POST", Pattern: "/api/v1/auth/email/verify", Public: true, h: a.handleLoginEmailVerify},
+		{Method: "POST", Pattern: "/api/v1/auth/email/verify", Public: true, AuthDoor: true, h: a.handleLoginEmailVerify},
-		{Method: "POST", Pattern: "/api/v1/auth/op-login/start", Public: true, h: a.handleOpLoginStart},
+		{Method: "POST", Pattern: "/api/v1/auth/op-login/start", Public: true, AuthDoor: true, h: a.handleOpLoginStart},
 		{Method: "GET", Pattern: "/api/v1/auth/op-login/status/{id}", Public: true, h: a.handleOpLoginStatus},
-		{Method: "POST", Pattern: "/api/v1/auth/op-login/finish", Public: true, h: a.handleOpLoginFinish},
+		{Method: "POST", Pattern: "/api/v1/auth/op-login/finish", Public: true, AuthDoor: true, h: a.handleOpLoginFinish},
 		// Player-console bootstrap (console-tier access model): the account-less
 		// player's door into console.<root_domain>. Public — like login there is no prior
 		// principal — and session-minting, but the artifact it consumes is a one-time
@@ -403,7 +484,7 @@ func (a *API) externalAPIRoutes() []apiRoute {
 		// possession already proves a Minecraft identity. A code whose UUID belongs to
 		// staff is refused (403) so this never yields an admin session; op.console stays
 		// behind Zero Trust (handlers_onboard.go).
-		{Method: "POST", Pattern: "/api/v1/auth/bind", Public: true, h: a.handleBindRedeem},
+		{Method: "POST", Pattern: "/api/v1/auth/bind", Public: true, AuthDoor: true, h: a.handleBindRedeem},
 
 		// App-auth tier: operations on your own servers (spec §14).
 		{Method: "POST", Pattern: "/api/v1/servers/{name}/wake", h: a.handleWake},
@@ -494,6 +575,22 @@ func (a *API) externalAPIRoutes() []apiRoute {
 		{Method: "POST", Pattern: "/api/v1/account/passkey/register/finish", SetupAllowed: true, h: a.handlePasskeyRegisterFinish},
 		{Method: "GET", Pattern: "/api/v1/account/passkey/credentials", SetupAllowed: true, h: a.handlePasskeyList},
 		{Method: "DELETE", Pattern: "/api/v1/account/passkey/credentials/{id}", SetupAllowed: true, h: a.handlePasskeyDelete},
+		// Reauth (reauth.go): the fresh proof that passkey enrollment and removal and an
+		// email change require once the account has a factor. Status says whether one
+		// is needed and how to give it; the pairs below take a passkey assertion or an
+		// email code and mark the caller's session. SetupAllowed like the routes they
+		// unlock.
+		{Method: "GET", Pattern: "/api/v1/account/reauth", SetupAllowed: true, h: a.handleReauthStatus},
+		{Method: "POST", Pattern: "/api/v1/account/reauth/passkey/begin", SetupAllowed: true, h: a.handleReauthPasskeyBegin},
+		{Method: "POST", Pattern: "/api/v1/account/reauth/passkey/finish", SetupAllowed: true, h: a.handleReauthPasskeyFinish},
+		{Method: "POST", Pattern: "/api/v1/account/reauth/email/start", SetupAllowed: true, h: a.handleReauthEmailStart},
+		{Method: "POST", Pattern: "/api/v1/account/reauth/email/verify", SetupAllowed: true, h: a.handleReauthEmailVerify},
+		// The caller's own sessions (handlers_account_sessions.go): list every signed-in
+		// device and sign out one or all the others. App-tier and scoped to the caller
+		// inside the handler, like the passkey routes above.
+		{Method: "GET", Pattern: "/api/v1/account/sessions", h: a.handleListMySessions},
+		{Method: "DELETE", Pattern: "/api/v1/account/sessions/{hash}", h: a.handleRevokeMySession},
+		{Method: "POST", Pattern: "/api/v1/account/sessions/revoke-others", h: a.handleRevokeMyOtherSessions},
 		// Account migration (spec §B3 inherit), web side. App-tier, principal-scoped: the
 		// SOURCE drives status → step-up confirm (passkey forced when enrolled, else
 		// email-OTP) → issue-code+name-target; the TARGET drives redeem as itself. Not
@@ -513,11 +610,19 @@ func (a *API) externalAPIRoutes() []apiRoute {
 		// session is the correct gate (the admin verdict lives below, behind adminOnly).
 		{Method: "POST", Pattern: "/api/v1/me/submissions", h: a.handleCreateSubmission},
 		{Method: "GET", Pattern: "/api/v1/me/submissions", h: a.handleMySubmissions},
+		{Method: "GET", Pattern: "/api/v1/me/submissions/limits", h: a.handleSubmissionLimits},
 		// The blob upload for a submission the caller owns: the request body is the
 		// raw gzip build context, streamed to the derived, id-namespaced location.
 		// App-tier and owner-scoped (the id must belong to the principal), exactly
 		// like the create/list routes above.
 		{Method: "POST", Pattern: "/api/v1/me/submissions/{id}/context", h: a.handleUploadSubmissionContext},
+		// The chunked form of that upload, for a context larger than one request
+		// carries through the edge (Cloudflare refuses bodies over 100 MB): GET
+		// reports the staged length (the resume point), PUT ?offset= appends one
+		// part, POST .../complete stores the staged whole. Same owner scoping.
+		{Method: "GET", Pattern: "/api/v1/me/submissions/{id}/context/upload", h: a.handleContextUploadStatus},
+		{Method: "PUT", Pattern: "/api/v1/me/submissions/{id}/context/upload", h: a.handleContextUploadPart},
+		{Method: "POST", Pattern: "/api/v1/me/submissions/{id}/context/upload/complete", h: a.handleContextUploadComplete},
 		// Withdraw the caller's OWN pending submission: the row and its uploaded
 		// context are deleted, freeing the pending slot and storage budget. Same
 		// owner-scoping as the upload route — a reviewed submission is frozen (409)
@@ -539,9 +644,13 @@ func (a *API) externalAPIRoutes() []apiRoute {
 		// is build-time RCE against the cluster, so submission requires the admin
 		// Zero-Trust path, not merely an authenticated session.
 		{Method: "POST", Pattern: "/api/v1/images/build", Admin: true, h: a.handleBuildImage},
+		{Method: "GET", Pattern: "/api/v1/images/build", Admin: true, h: a.handleListBuilds},
 		{Method: "GET", Pattern: "/api/v1/images/build/{id}", Admin: true, h: a.handleGetBuild},
 		{Method: "GET", Pattern: "/api/v1/images/build/{id}/logs", Admin: true, h: a.handleBuildLogs},
 		{Method: "POST", Pattern: "/api/v1/images/build/{id}/cancel", Admin: true, h: a.handleCancelBuild},
+		{Method: "GET", Pattern: "/api/v1/images/build/{id}/scan", Admin: true, h: a.handleBuildScan},
+		{Method: "GET", Pattern: "/api/v1/images/build/{id}/scan/report", Admin: true, h: a.handleBuildScanReport},
+		{Method: "GET", Pattern: "/api/v1/images/build/{id}/sbom", Admin: true, h: a.handleBuildSBOM},
 		{Method: "GET", Pattern: "/api/v1/images", Admin: true, h: a.handleListImages},
 		{Method: "POST", Pattern: "/api/v1/images", Admin: true, h: a.handleAddImage},
 		{Method: "DELETE", Pattern: "/api/v1/images", Admin: true, h: a.handleRemoveImage},
@@ -561,10 +670,15 @@ func (a *API) externalAPIRoutes() []apiRoute {
 		{Method: "GET", Pattern: "/api/v1/submissions/{id}/context", Admin: true, h: a.handleAdminSubmissionContext},
 		// Auto-update maintenance window (spec §B; decision core internal/updates).
 		// Admin-tier: it governs whether Felis may apply an update to itself, so setting
-		// it requires the admin Zero-Trust path, not a mere session. API+persistence
+		// it requires the admin Zero-Trust path, not a mere session. Advisory: `felis
-		// only — the runner/executors that consume the window are still INTEGRATION-ONLY.
+		// update` on the host reads it and warns before an apply outside it.
 		{Method: "GET", Pattern: "/api/v1/updates/window", Admin: true, h: a.handleGetUpdateWindow},
 		{Method: "PUT", Pattern: "/api/v1/updates/window", Admin: true, h: a.handleSetUpdateWindow},
+		// The newest version check felis-update-check.timer recorded on the host.
+		{Method: "GET", Pattern: "/api/v1/updates/report", Admin: true, h: a.handleGetUpdateReport},
+		// Control-plane database backup freshness, as the host's felis-db-backup.timer
+		// last recorded it. Admin-tier: it names the host backup directory.
+		{Method: "GET", Pattern: "/api/v1/platform/db-backup", Admin: true, h: a.handleGetDBBackup},
 
 		// User admin (spec §7, owner-only). Every route gates on the admin Zero-Trust
 		// path AND the owner role: listing, mutating, disabling, or deleting users is
@@ -589,14 +703,14 @@ func (a *API) externalAPIRoutes() []apiRoute {
 // InternalHandler builds the internal-face http.Handler: service-token auth, no
 // Zero Trust (spec §14 red line). /healthz and /readyz are unauthenticated.
 func (a *API) InternalHandler() http.Handler {
-	return a.buildFace(a.internalAPIRoutes(), a.requireInternal)
+	return a.buildFace("internal", a.internalAPIRoutes(), a.requireInternal)
 }
 
-// ExternalHandler builds the external-face http.Handler: Access-JWT auth on every
+// ExternalHandler builds the external-face http.Handler: session auth on every
-// /api/v1 route, with admin-tier routes additionally gated by the admin Access
+// non-public /api/v1 route, with admin-tier routes additionally gated on the
-// path inside their handlers.
+// operator console host inside their handlers.
 func (a *API) ExternalHandler() http.Handler {
-	return a.buildFace(a.externalAPIRoutes(), a.requireExternal)
+	return a.buildFace("external", a.externalAPIRoutes(), a.requireExternal)
 }
 
 // buildFace assembles one face from its route table. Public routes are mounted
@@ -605,16 +719,26 @@ func (a *API) ExternalHandler() http.Handler {
 // adminOnly, and Owner routes in ownerOnly. Because both faces are built from the
 // same table the OpenAPI parity test reads, the served surface and the documented
 // surface cannot drift apart without failing the build.
-func (a *API) buildFace(routes []apiRoute, guard func(http.Handler) http.Handler) http.Handler {
+func (a *API) buildFace(face string, routes []apiRoute, guard func(http.Handler) http.Handler) http.Handler {
 	mux := http.NewServeMux()
 	auth := http.NewServeMux()
 	for _, rt := range routes {
 		pattern := rt.Method + " " + rt.Pattern
 		if rt.Public {
-			mux.HandleFunc(pattern, rt.h)
+			h := rt.h
+			if rt.AuthDoor {
+				h = a.throttleAuthDoor(h)
+			}
+			mux.HandleFunc(pattern, tagRoute(rt.Pattern, h))
 			continue
 		}
 		h := rt.h
+		if (face == "internal") != (len(rt.Callers) > 0) {
+			panic(fmt.Sprintf("%s route %s %s: internal routes list their callers, external ones none", face, rt.Method, rt.Pattern))
+		}
+		if len(rt.Callers) > 0 {
+			h = callersOnly(rt.Callers, h)
+		}
 		if rt.Owner {
 			h = a.ownerOnly(rt.h)
 		}
@@ -627,22 +751,61 @@ func (a *API) buildFace(routes []apiRoute, guard func(http.Handler) http.Handler
 		if !rt.SetupAllowed {
 			h = a.requireOnboarded(h)
 		}
-		auth.HandleFunc(pattern, h)
+		auth.HandleFunc(pattern, tagRoute(rt.Pattern, h))
 	}
 	guarded := guard(auth)
 	mux.Handle("/api/v1/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if _, pattern := auth.Handler(r); pattern == "" {
+		_, pattern := auth.Handler(r)
-			http.NotFound(w, r)
+		if pattern == "" {
+			writeNoRoute(w, r, mux, auth)
 			return
 		}
+		// Named before the guard runs, so a refused request is counted under
+		// the route it asked for.
+		noteRoute(r, pattern)
 		guarded.ServeHTTP(w, r)
 	}))
-	return a.baseChain(mux)
+	top := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if _, pattern := mux.Handler(r); pattern == "" {
+			writeNoRoute(w, r, mux, auth)
+			return
+		}
+		mux.ServeHTTP(w, r)
+	})
+	return a.baseChain(face, top)
 }
 
-// baseChain wraps a handler in the cross-cutting middleware shared by both faces.
+// baseChain wraps a handler in the cross-cutting middleware shared by both faces:
-func (a *API) baseChain(h http.Handler) http.Handler {
+// the request id, then the access log and metrics (which see the final status of
-	return withRequestID(withRecover(h))
+// everything inside), the response security headers, the cross-site write fence,
+// the request-body read deadline, and panic recovery.
+func (a *API) baseChain(face string, h http.Handler) http.Handler {
+	return withRequestID(a.observe(face, withSecurityHeaders(rejectCrossSiteWrites(withBodyDeadline(withRecover(h))))))
+}
+
+// writeNoRoute answers a request no route took, in the API's error envelope: 405
+// with an Allow header when the path exists under other methods, 404 otherwise.
+// ServeMux's own answers are plain text and turn a wrong method on a guarded
+// route into a 404, since the guarded routes sit behind one catch-all.
+func writeNoRoute(w http.ResponseWriter, r *http.Request, muxes ...*http.ServeMux) {
+	var allow []string
+	for _, m := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete} {
+		probe := r.Clone(r.Context())
+		probe.Method = m
+		for _, mux := range muxes {
+			if _, p := mux.Handler(probe); p != "" && p != "/api/v1/" {
+				allow = append(allow, m)
+				break
+			}
+		}
+	}
+	if len(allow) > 0 {
+		w.Header().Set("Allow", strings.Join(allow, ", "))
+		writeError(w, r, newError(http.StatusMethodNotAllowed, "method_not_allowed",
+			"%s is not allowed here; use %s", r.Method, strings.Join(allow, ", ")))
+		return
+	}
+	writeError(w, r, newError(http.StatusNotFound, "not_found", "no such endpoint"))
 }
 
 // requireOnboarded fences an authenticated route behind the setup-lockdown: a
@@ -664,7 +827,15 @@ func (a *API) requireOnboarded(h http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		p := principalFromContext(r.Context())
 		if p != nil && p.ViaSession && !p.EmailVerified {
-			creds, _ := a.Repo.PasskeyCredentialsForUser(r.Context(), p.UserID)
+			creds, err := a.Repo.PasskeyCredentialsForUser(r.Context(), p.UserID)
+			if err != nil {
+				// A store outage reads as retry-later; setup_required would send
+				// the caller off to enroll a passkey they may already have.
+				log.Printf("api: %s %s: onboarding check (request_id=%s): %v",
+					r.Method, r.URL.Path, requestIDFromContext(r.Context()), err)
+				writeError(w, r, errAuthUnavailable)
+				return
+			}
 			if len(creds) == 0 {
 				writeError(w, r, newError(http.StatusForbidden, "setup_required",
 					"passkey enrollment is required before this action is available"))
@@ -694,6 +865,8 @@ type ctxKey int
 const (
 	ctxKeyRequestID ctxKey = iota
 	ctxKeyPrincipal
+	ctxKeyReqInfo
+	ctxKeyCaller
 )
 
 func requestIDFromContext(ctx context.Context) string {
@@ -711,6 +884,21 @@ func principalFromContext(ctx context.Context) *Principal {
 	return nil
 }
 
+// callerFromContext returns the internal-face caller, or "" off that face.
+func callerFromContext(ctx context.Context) Caller {
+	c, _ := ctx.Value(ctxKeyCaller).(Caller)
+	return c
+}
+
+// internalSource is the audit Source for an action taken on the internal face,
+// naming the caller whose token asked for it ("internal:velocity").
+func internalSource(r *http.Request) string {
+	if c := callerFromContext(r.Context()); c != "" {
+		return "internal:" + string(c)
+	}
+	return "internal"
+}
+
 // ---- per-key cooldown (wake + OTP) ----
 
 // cooldownLimiter is an in-memory per-key cooldown. It backs two throttles with
@@ -724,10 +912,35 @@ func principalFromContext(ctx context.Context) *Principal {
 // reserve/release pair closes the intra-replica concurrent burst (the bug fixed in
 // #35); cross-replica bounding would need a shared store (out of scope for the
 // single-replica demo).
+//
+// Entries older than the longest window the limiter has been asked about can
+// no longer block anything, so checks sweep them out (at most once per
+// bucketSweepEvery). Without that, every distinct address typed into a public
+// door, whose neutral branch keeps its reservation, stayed in the map for the
+// life of the process.
 type cooldownLimiter struct {
 	mu        sync.Mutex
 	now       func() time.Time
 	last      map[string]time.Time
+	maxWindow time.Duration
+	swept     time.Time
+}
+
+// noteWindow widens the retention to window and sweeps stale entries when due.
+// The caller holds mu.
+func (c *cooldownLimiter) noteWindow(window time.Duration, now time.Time) {
+	if window > c.maxWindow {
+		c.maxWindow = window
+	}
+	if c.maxWindow <= 0 || now.Sub(c.swept) < bucketSweepEvery {
+		return
+	}
+	c.swept = now
+	for k, t := range c.last {
+		if now.Sub(t) >= c.maxWindow {
+			delete(c.last, k)
+		}
+	}
 }
 
 // allowed reports whether name may wake now WITHOUT recording the attempt. A
@@ -742,7 +955,9 @@ func (c *cooldownLimiter) allowed(name string, window time.Duration) bool {
 	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	if last, ok := c.last[name]; ok && c.now().Sub(last) < window {
+	now := c.now()
+	c.noteWindow(window, now)
+	if last, ok := c.last[name]; ok && now.Sub(last) < window {
 		return false
 	}
 	return true
@@ -761,7 +976,8 @@ func (c *cooldownLimiter) record(name string) {
 }
 
 // reserve atomically checks name's cooldown AND, if the window is open, records it
-// in the same critical section, returning the reservation time and true. Unlike
+// in the same critical section, returning the reservation time and true; inside the
+// window it returns the standing reservation's time and false. Unlike
 // allowed→record there is no gap between the check and the commit, so a burst of
 // truly concurrent callers yields exactly one winner. Use it where the throttle is
 // the SOLE defense and each admitted call has a non-idempotent side effect (an OTP
@@ -775,10 +991,11 @@ func (c *cooldownLimiter) reserve(name string, window time.Duration) (time.Time,
 	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	if last, ok := c.last[name]; ok && c.now().Sub(last) < window {
-		return time.Time{}, false
-	}
 	t := c.now()
+	c.noteWindow(window, t)
+	if last, ok := c.last[name]; ok && t.Sub(last) < window {
+		return last, false
+	}
 	c.last[name] = t
 	return t, true
 }

internal/api/audit.go

@@ -0,0 +1,185 @@
+package api
+
+import (
+	"cmp"
+	"context"
+	"encoding/json"
+	"errors"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"felis.lolicon.best/internal/metrics"
+)
+
+// Audit rows (audit_logs, spec §6).
+//
+// Every row an HTTP request writes carries the acting account's id
+// (actor_user_id), the caller's address (the one the sign-in limit keys on) and
+// user agent; actor is display text. A failed write never fails the operation
+// it records, which already happened, but it is logged and counted
+// (felis_audit_write_failures_total, FelisAuditWriteFailing): a silent drop is
+// how a database blip erases the trail.
+
+const (
+	// auditWriteTimeout bounds one audit insert. The write outlives the caller's
+	// request context, so a client that hangs up right after the action cannot
+	// cancel its own audit row.
+	auditWriteTimeout = 5 * time.Second
+	// auditUserAgentMax bounds the stored user agent; the header is the caller's
+	// to write.
+	auditUserAgentMax = 256
+	// anonymousActor names a caller no account was resolved for.
+	anonymousActor = "anonymous"
+)
+
+// auditActor is the display name for a principal: an email only when something
+// vouches for it (a session whose address was verified, or an ExternalAuth other
+// than SessionAuth that resolved the principal itself), else the username. A player can set their address to anyone's before verifying it,
+// so an unverified email would let them sign rows as that person.
+func auditActor(p *Principal) string {
+	switch {
+	case p == nil:
+		return anonymousActor
+	case p.Email != "" && (p.EmailVerified || !p.ViaSession):
+		return p.Email
+	case p.Username != "":
+		return p.Username
+	case p.UserID != "":
+		return p.UserID
+	}
+	return anonymousActor
+}
+
+// audit records an action by the signed-in caller. target is the object acted
+// on (a server name, a user or credential id) and lands in server_name.
+func (a *API) audit(r *http.Request, action, target string) {
+	p := principalFromContext(r.Context())
+	e := AuditEntry{Actor: auditActor(p), Action: action, ServerName: target}
+	if p != nil {
+		e.ActorUserID = p.UserID
+	}
+	a.auditEntry(r, e)
+}
+
+// auditImageChange records a confirmed image change as server.patch with the
+// image it replaced and the one it set, so the audit log alone can say which
+// build a world ran before it was moved.
+func (a *API) auditImageChange(r *http.Request, server, from, to string) {
+	p := principalFromContext(r.Context())
+	e := AuditEntry{Actor: auditActor(p), Action: "server.patch", ServerName: server}
+	if p != nil {
+		e.ActorUserID = p.UserID
+	}
+	e.Payload = auditPayload(map[string]any{"image_from": from, "image_to": to})
+	a.auditEntry(r, e)
+}
+
+// auditAccount records an action a pre-session door took for the account it
+// resolved (u nil: none was). The username is the actor: the door has not yet
+// proven anything about the address.
+func (a *API) auditAccount(r *http.Request, u *StaffUser, action, target string) {
+	e := AuditEntry{Actor: anonymousActor, Action: action, ServerName: target}
+	if u != nil {
+		e.Actor, e.ActorUserID = u.Username, u.ID
+	}
+	a.auditEntry(r, e)
+}
+
+// auditEntry fills the request detail into e and writes it. Source defaults to
+// external; internal callers set it and the component actor themselves.
+func (a *API) auditEntry(r *http.Request, e AuditEntry) {
+	if e.Source == "" {
+		e.Source = "external"
+	}
+	e.RequestID = requestIDFromContext(r.Context())
+	if e.Source == "external" {
+		if ip := a.clientIP(r); ip.IsValid() {
+			e.ClientIP = ip.String()
+		}
+		e.UserAgent = truncateUTF8(r.UserAgent(), auditUserAgentMax)
+	}
+	a.writeAudit(r.Context(), e)
+}
+
+// writeAudit inserts e, logging and counting a failure.
+func (a *API) writeAudit(ctx context.Context, e AuditEntry) {
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), auditWriteTimeout)
+	defer cancel()
+	if err := a.Repo.Audit(ctx, e); err != nil {
+		metrics.AuditWriteFailuresTotal.Inc()
+		log.Printf("audit: lost %s by %s (user %q, request_id=%s): %v",
+			e.Action, e.Actor, e.ActorUserID, e.RequestID, err)
+	}
+}
+
+// authFailure records one refused sign-in attempt: felis_auth_failures_total
+// by door and reason, and an auth.<door>.failed row naming the account when
+// the door resolved one (u nil: the signed-in caller if any, else anonymous).
+// The doors keep their answers uniform so a prober learns nothing; the reason
+// is for the operator.
+func (a *API) authFailure(r *http.Request, door, reason string, u *StaffUser) {
+	metrics.AuthFailuresTotal.WithLabelValues(door, reason).Inc()
+	e := AuditEntry{Action: "auth." + door + ".failed", Payload: auditPayload(map[string]any{"reason": reason})}
+	switch p := principalFromContext(r.Context()); {
+	case u != nil:
+		e.Actor, e.ActorUserID = cmp.Or(u.Username, u.ID), u.ID
+	case p != nil:
+		e.Actor, e.ActorUserID = auditActor(p), p.UserID
+	default:
+		e.Actor = anonymousActor
+	}
+	a.auditEntry(r, e)
+}
+
+// passkeyCloneRejected records an assertion refused for a regressed signature
+// counter. It keeps its own action so a cloned authenticator stands out from
+// ordinary failures, and counts as a failure of its door. u nil: a signed-in
+// step-up, attributed to the caller.
+func (a *API) passkeyCloneRejected(r *http.Request, door string, u *StaffUser, credentialID string) {
+	metrics.AuthFailuresTotal.WithLabelValues(door, "clone_rejected").Inc()
+	if u == nil {
+		a.audit(r, "auth.passkey_clone_rejected", credentialID)
+		return
+	}
+	a.auditAccount(r, u, "auth.passkey_clone_rejected", credentialID)
+}
+
+// isOTPRefusal reports whether err is a refused code (wrong, spent, or the
+// account's budget locked), as opposed to a fault.
+func isOTPRefusal(err error) bool {
+	return errors.Is(err, ErrOTPInvalid) || errors.Is(err, ErrOTPLocked) || errors.Is(err, ErrOTPAccountLocked)
+}
+
+// otpFailureReason names a refused code for authFailure.
+func otpFailureReason(err error) string {
+	switch {
+	case errors.Is(err, ErrOTPAccountLocked):
+		return "account_locked"
+	case errors.Is(err, ErrOTPLocked):
+		return "code_locked"
+	}
+	return "bad_code"
+}
+
+// auditPayload marshals a small detail map for AuditEntry.Payload.
+func auditPayload(v map[string]any) []byte {
+	b, _ := json.Marshal(v)
+	return b
+}
+
+// truncateUTF8 makes s valid UTF-8 (a header may carry any byte, a text
+// column refuses invalid sequences) and cuts it to at most n bytes on a rune
+// boundary.
+func truncateUTF8(s string, n int) string {
+	s = strings.ToValidUTF8(s, "\uFFFD")
+	if len(s) <= n {
+		return s
+	}
+	for n > 0 && !utf8.RuneStart(s[n]) {
+		n--
+	}
+	return s[:n]
+}

internal/api/audit_test.go

@@ -0,0 +1,177 @@
+package api
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+
+	"felis.lolicon.best/internal/metrics"
+)
+
+// Audit attribution: a row names the acting account by id and never by an
+// address the caller merely asserted; refused sign-ins, throttling and logouts
+// leave rows; a failed write is counted instead of vanishing.
+
+func TestAuditActorIgnoresUnverifiedEmail(t *testing.T) {
+	cases := []struct {
+		name string
+		p    *Principal
+		want string
+	}{
+		{"verified session email", &Principal{UserID: "u1", Username: "alice", Email: "[email protected]", EmailVerified: true, ViaSession: true}, "[email protected]"},
+		{"unverified session email", &Principal{UserID: "u2", Username: "mallory", Email: "[email protected]", ViaSession: true}, "mallory"},
+		{"access jwt email", &Principal{UserID: "sub", Email: "[email protected]"}, "[email protected]"},
+		{"no email", &Principal{UserID: "u3", Username: "bob", ViaSession: true}, "bob"},
+		{"id only", &Principal{UserID: "u4", ViaSession: true}, "u4"},
+		{"nobody", nil, anonymousActor},
+	}
+	for _, c := range cases {
+		if got := auditActor(c.p); got != c.want {
+			t.Errorf("%s: auditActor = %q, want %q", c.name, got, c.want)
+		}
+	}
+}
+
+// A player who sets their address to the owner's still signs every row as
+// themselves, by username and by id.
+func TestAuditCannotBeSignedWithAnotherPersonsEmail(t *testing.T) {
+	repo := newFakeRepo()
+	repo.settings[LocalAuthEnabledKey] = []byte("true")
+	repo.staff["owner"] = &StaffUser{ID: "u1", Username: "owner", Email: "[email protected]", Role: "owner", EmailVerified: true}
+	repo.staff["mallory"] = &StaffUser{ID: "u2", Username: "mallory", Email: "[email protected]", Role: "user", EmailVerified: true}
+	repo.sessions[hashCookie("tok")] = &fakeSession{userID: "u2", expiresAt: frozenNow.Add(time.Hour), reauthAt: frozenNow}
+	api := newTestAPI(repo, newFakeCluster())
+	api.External = SessionAuth{Repo: repo, RootDomain: testRoot, Now: api.now}
+	api.ClientIPHeader = "CF-Connecting-IP"
+	eh := api.ExternalHandler()
+	hdr := map[string]string{
+		"Content-Type": "application/json", "Cookie": sessionCookieName + "=tok",
+		"CF-Connecting-IP": "203.0.113.5", "User-Agent": "probe/1.0",
+	}
+	for _, email := range []string{"[email protected]", "[email protected]"} {
+		if w := do(eh, "POST", "/api/v1/account/email", `{"email":"`+email+`"}`, hdr); w.Code != http.StatusOK {
+			t.Fatalf("set email = %d (%s)", w.Code, w.Body.String())
+		}
+	}
+	// The first write ran while the address was still verified; the second
+	// ran with the owner's address set and unverified.
+	last := repo.audits[len(repo.audits)-1]
+	if last.Actor != "mallory" || last.ActorUserID != "u2" {
+		t.Fatalf("audit after spoofing = actor %q user %q, want mallory/u2", last.Actor, last.ActorUserID)
+	}
+	if last.ClientIP != "203.0.113.5" || last.UserAgent != "probe/1.0" || last.RequestID == "" {
+		t.Fatalf("audit request detail = %+v", last)
+	}
+}
+
+func TestSignInFailuresAreAuditedAndCounted(t *testing.T) {
+	api, repo, mailer := seedLoginEmailAPI(t)
+	eh := api.ExternalHandler()
+	noAccount := metrics.AuthFailuresTotal.WithLabelValues("login_email", "no_account")
+	badCode := metrics.AuthFailuresTotal.WithLabelValues("login_email", "bad_code")
+	n0, b0 := testutil.ToFloat64(noAccount), testutil.ToFloat64(badCode)
+
+	do(eh, "POST", "/api/v1/auth/email/verify", `{"email":"[email protected]","code":"123456"}`, jsonHeader)
+	if w := do(eh, "POST", "/api/v1/auth/email/start", `{"email":"[email protected]"}`, jsonHeader); w.Code != http.StatusAccepted {
+		t.Fatalf("start = %d", w.Code)
+	}
+	wrong := "000000"
+	if mailer.code == wrong {
+		wrong = "111111"
+	}
+	do(eh, "POST", "/api/v1/auth/email/verify", `{"email":"[email protected]","code":"`+wrong+`"}`, jsonHeader)
+
+	if got := testutil.ToFloat64(noAccount) - n0; got != 1 {
+		t.Errorf("no_account failures counted %v, want 1", got)
+	}
+	if got := testutil.ToFloat64(badCode) - b0; got != 1 {
+		t.Errorf("bad_code failures counted %v, want 1", got)
+	}
+	var failed []AuditEntry
+	for _, e := range repo.audits {
+		if e.Action == "auth.login_email.failed" {
+			failed = append(failed, e)
+		}
+	}
+	if len(failed) != 2 {
+		t.Fatalf("failure audits = %+v, want 2", failed)
+	}
+	if failed[0].Actor != anonymousActor || failed[0].ActorUserID != "" || !strings.Contains(string(failed[0].Payload), "no_account") {
+		t.Errorf("unknown-address failure = %+v", failed[0])
+	}
+	if failed[1].Actor != "player" || failed[1].ActorUserID != "u1" || !strings.Contains(string(failed[1].Payload), "bad_code") {
+		t.Errorf("wrong-code failure = %+v", failed[1])
+	}
+}
+
+func TestThrottleAuditsOncePerEpisode(t *testing.T) {
+	api, repo, _ := seedLoginEmailAPI(t)
+	api.AuthDoorLimit = RateLimit{Burst: 1, PerMinute: 1}
+	clock := time.Unix(1_700_000_000, 0)
+	api.Now = func() time.Time { return clock }
+	eh := api.ExternalHandler()
+	options := func() int {
+		return do(eh, "POST", "/api/v1/auth/options", `{"email":"[email protected]"}`, jsonHeader).Code
+	}
+	throttled := func() int {
+		n := 0
+		for _, e := range repo.audits {
+			if e.Action == "auth.rate_limited" {
+				n++
+			}
+		}
+		return n
+	}
+	options()
+	for i := 0; i < 5; i++ {
+		if c := options(); c != http.StatusTooManyRequests {
+			t.Fatalf("call %d = %d, want 429", i+2, c)
+		}
+	}
+	if n := throttled(); n != 1 {
+		t.Fatalf("5 refusals left %d audit rows, want 1", n)
+	}
+	clock = clock.Add(time.Minute)
+	options()
+	options()
+	if n := throttled(); n != 2 {
+		t.Fatalf("a second episode left %d rows in total, want 2", n)
+	}
+}
+
+func TestAuditWriteFailureIsCountedNotFatal(t *testing.T) {
+	api, repo, _ := seedLoginEmailAPI(t)
+	repo.failAudit = errors.New("db down")
+	before := testutil.ToFloat64(metrics.AuditWriteFailuresTotal)
+	if w := do(api.ExternalHandler(), "POST", "/api/v1/auth/email/start", `{"email":"[email protected]"}`, jsonHeader); w.Code != http.StatusAccepted {
+		t.Fatalf("start with the audit store down = %d, want 202", w.Code)
+	}
+	if got := testutil.ToFloat64(metrics.AuditWriteFailuresTotal) - before; got != 1 {
+		t.Fatalf("audit write failures counted %v, want 1", got)
+	}
+}
+
+func TestLogoutAuditsTheLiveSession(t *testing.T) {
+	api, repo, _ := seedLoginEmailAPI(t)
+	repo.sessions[hashCookie("tok")] = &fakeSession{userID: "u1", expiresAt: time.Unix(1_700_000_000, 0).Add(time.Hour)}
+	eh := api.ExternalHandler()
+	cookie := map[string]string{"Content-Type": "application/json", "Cookie": sessionCookieName + "=tok"}
+	do(eh, "POST", "/api/v1/auth/logout", "", cookie)
+	do(eh, "POST", "/api/v1/auth/logout", "", cookie) // already revoked: no second row
+	if len(repo.audits) != 1 || repo.audits[0].Action != "auth.logout" || repo.audits[0].ActorUserID != "u1" || repo.audits[0].Actor != "player" {
+		t.Fatalf("logout audits = %+v, want one auth.logout by player/u1", repo.audits)
+	}
+}
+
+func TestTruncateUTF8KeepsRunesWhole(t *testing.T) {
+	if got := truncateUTF8("ab\xffc", 10); got != "ab�c" {
+		t.Errorf("invalid byte = %q", got)
+	}
+	if got := truncateUTF8("猫猫", 4); got != "猫" {
+		t.Errorf("cut mid-rune = %q, want 猫", got)
+	}
+}

internal/api/auth.go

@@ -5,26 +5,26 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
-
+	"time"
-	"github.com/golang-jwt/jwt/v5"
 )
 
 // Principal is the authenticated external-face caller (spec §7, §14). The
 // internal face (service token) never produces a Principal — it is a trusted
 // machine caller, not a person.
 type Principal struct {
-	// UserID is the stable web identity (SSO subject → users.id).
+	// UserID is the account's users.id.
 	UserID string
-	// Email is the audited actor identity (spec §14: audit actor = Access email).
+	// Username is the account's login name.
+	Username string
+	// Email is the account's address. Only EmailVerified vouches for it: a player
+	// can set any address before verifying it (auditActor).
 	Email string
 	// Role is "owner", "admin", or "user" (mirrors users.role).
 	Role string
-	// ViaAdminAccess is true only when the request arrived through an admin-graded
+	// ViaAdminAccess is true only when a staff session arrived on the operator
-	// path: the admin.* Zero-Trust hostname (Cloudflare Access, the remote face) OR
+	// console host (hostIsAdminConsole). Admin-tier operations require it in
-	// a local session presented on the op.console host (SessionAuth, the
+	// addition to a staff role (spec §14: ZT is graded by operation). A staff
-	// passwordless face). Admin-tier operations require it in addition to
+	// session arriving on the player console (console.*) never sets it.
-	// a staff role (spec §14: ZT is graded by operation). A staff session
-	// arriving on the player console (console.*) never sets it.
 	ViaAdminAccess bool
 	// EmailVerified mirrors users.email_verified. The lockdown middleware gates
 	// setup-incomplete accounts (EmailVerified=false, e.g. a freshly bootstrapped
@@ -32,12 +32,13 @@ type Principal struct {
 	// routes only, so an intercepted setup URL cannot yield full admin access
 	// before the email-OTP verification step completes.
 	EmailVerified bool
-	// ViaSession is true when the principal was authenticated via a local session
+	// ViaSession is true for every principal SessionAuth resolves from a session
-	// cookie (SessionAuth), not a Cloudflare-Access JWT. The setup-lockdown gate
+	// cookie. The session-scoped gates (setup lockdown, reauth, the device list)
-	// only applies to session-authenticated principals — a JWT caller already
+	// key on it.
-	// passed Zero Trust at the edge, so the local-email-verification gate is not
-	// the right boundary for them.
 	ViaSession bool
+	// ReauthAt is when the holder of the session last proved a factor of the
+	// account; zero for a session that never did.
+	ReauthAt time.Time
 }
 
 // staffRole reports whether a stored user role carries staff standing: admin,
@@ -49,8 +50,8 @@ func staffRole(role string) bool {
 }
 
 // IsAdmin reports whether the principal may perform admin-tier operations.
-// Both the role claim and the admin Access path are required: a staff
+// Both the staff role and arrival on the operator console host are required: a
-// session arriving on panel.* must not bypass the Zero-Trust boundary.
+// staff session arriving on the player console must not reach admin routes.
 // An owner implicitly passes this check (the owner role is a superset of admin).
 func (p *Principal) IsAdmin() bool {
 	return p != nil && staffRole(p.Role) && p.ViaAdminAccess
@@ -59,104 +60,88 @@ func (p *Principal) IsAdmin() bool {
 // IsOwner reports whether the principal holds the platform-level owner role
 // — the single identity that may manage users, quotas, and sessions. Only the
 // first staff account minted by break-glass carries this role; every subsequent
-// Operator is a plain admin. Like IsAdmin, it requires the admin Access path.
+// Operator is a plain admin. Like IsAdmin, it requires the operator console host.
 func (p *Principal) IsOwner() bool {
 	return p != nil && p.Role == "owner" && p.ViaAdminAccess
 }
 
-// InternalAuth authenticates the internal face (velocity / backend callbacks):
+// Caller names the machine behind an internal-face token. Each caller holds a
-// a static service token presented as a Bearer credential. The internal face
+// token of its own and each internal route lists the callers it serves
-// is never wrapped in Zero Trust (spec §1.8, §14 red line).
+// (apiRoute.Callers), so a token copied out of one namespace opens only what
+// that caller needs: the build Job's token reads a submission's context and
+// nothing else, and only the proxy and the login gate can mint link codes.
+type Caller string
+
+const (
+	// CallerVelocity is the proxy's felis-link plugin (felis-service-token,
+	// written into felis-link.properties on the host).
+	CallerVelocity Caller = "velocity"
+	// CallerLimbo is the login gate's felis-limbo plugin (felis-limbo-token,
+	// injected into the login pod only).
+	CallerLimbo Caller = "limbo"
+	// CallerBuild is the build Job's context-fetch initContainer
+	// (felis-build-token in the build namespace).
+	CallerBuild Caller = "build"
+	// CallerOps is the on-node console, `felis backup-now` (felis-ops-token,
+	// control namespace only).
+	CallerOps Caller = "ops"
+)
+
+// InternalAuth authenticates the internal face: a per-caller static token
+// presented as a Bearer credential, answered with the caller it belongs to. The
+// internal face is never wrapped in Zero Trust (spec §1.8, §14 red line).
 type InternalAuth interface {
-	Authenticate(r *http.Request) error
+	Authenticate(r *http.Request) (Caller, error)
 }
 
 // ExternalAuth authenticates the external face (people / panel) and returns the
-// resolved Principal. Production verifies a Cloudflare Access JWT and checks its
+// resolved Principal. Production is SessionAuth: the local session cookie the
-// audience; the verification key source (JWKS) is injected so the audience and
+// sign-in doors mint. Cloudflare Access, when an install sits behind it, is
-// expiry logic stay unit-testable.
+// enforced at the edge only; felis-api does not read the Access JWT, so the
+// identity and role always come from the users table.
 type ExternalAuth interface {
 	Authenticate(r *http.Request) (*Principal, error)
 }
 
-// BearerTokenAuth is the production InternalAuth: a constant-time comparison
+// CallerTokens is the production InternalAuth: each caller's token, compared in
-// against the configured service token. A zero token fails closed so a
+// constant time. A caller with no token cannot authenticate, so a missing
-// misconfiguration can never silently disable internal-face auth.
+// Secret fails closed for that caller alone.
-type BearerTokenAuth struct {
+type CallerTokens map[Caller]string
-	Token string
+
+// NewCallerTokens refuses a set that would make the caller ambiguous: two
+// callers sharing a value, which is also what an install whose callers all
+// still hold the one old service token would look like.
+func NewCallerTokens(tokens map[Caller]string) (CallerTokens, error) {
+	seen := map[string]Caller{}
+	for caller, tok := range tokens {
+		if tok == "" {
+			continue
+		}
+		if other, dup := seen[tok]; dup {
+			return nil, fmt.Errorf("the %s and %s tokens are the same value; each caller needs its own", other, caller)
+		}
+		seen[tok] = caller
+	}
+	return CallerTokens(tokens), nil
 }
 
-// Authenticate checks the Authorization: Bearer header against the token.
+// Authenticate matches the Authorization: Bearer header against every caller's
-func (b BearerTokenAuth) Authenticate(r *http.Request) error {
+// token, comparing each so the time taken does not say which one matched.
-	if b.Token == "" {
+func (c CallerTokens) Authenticate(r *http.Request) (Caller, error) {
-		return fmt.Errorf("internal auth not configured")
-	}
 	got := bearerToken(r)
 	if got == "" {
-		return fmt.Errorf("missing bearer token")
+		return "", fmt.Errorf("missing bearer token")
 	}
-	if subtle.ConstantTimeCompare([]byte(got), []byte(b.Token)) != 1 {
+	var match Caller
-		return fmt.Errorf("invalid service token")
+	for caller, tok := range c {
+		if tok != "" && subtle.ConstantTimeCompare([]byte(got), []byte(tok)) == 1 {
+			match = caller
 		}
-	return nil
 	}
-
+	if match == "" {
-// AccessVerifier is the production ExternalAuth: it parses a Cloudflare Access
+		return "", fmt.Errorf("invalid service token")
-// JWT, verifies the signature with the injected key function, and enforces the
-// configured audience (spec §7 "验 aud"). AdminAudience, when set, marks a token
-// minted for the admin.* application so admin-tier routes can require it.
-type AccessVerifier struct {
-	// Audience is the required `aud` claim for any external request.
-	Audience string
-	// AdminAudience, if non-empty and present in the token's aud set, flags the
-	// principal as having passed the admin Zero-Trust path.
-	AdminAudience string
-	// Keyfunc resolves the signing key (production: a JWKS-backed keyfunc).
-	Keyfunc jwt.Keyfunc
 	}
-
+	return match, nil
-// accessClaims are the subset of Access JWT claims we consume.
-type accessClaims struct {
-	Email string `json:"email"`
-	Role  string `json:"felis_role"`
-	jwt.RegisteredClaims
-}
-
-// Authenticate verifies the Access JWT and maps it onto a Principal.
-func (v AccessVerifier) Authenticate(r *http.Request) (*Principal, error) {
-	if v.Keyfunc == nil {
-		return nil, fmt.Errorf("external auth not configured")
-	}
-	raw := accessToken(r)
-	if raw == "" {
-		return nil, fmt.Errorf("missing access token")
-	}
-
-	var claims accessClaims
-	parser := jwt.NewParser(jwt.WithExpirationRequired())
-	if _, err := parser.ParseWithClaims(raw, &claims, v.Keyfunc); err != nil {
-		return nil, fmt.Errorf("invalid access token: %w", err)
-	}
-
-	// Audience check: the configured app aud must be present. We do not delegate
-	// to jwt.WithAudience so we can additionally detect the admin audience.
-	if !audienceContains(claims.Audience, v.Audience) {
-		return nil, fmt.Errorf("token audience does not include %q", v.Audience)
-	}
-	if claims.Subject == "" {
-		return nil, fmt.Errorf("token missing subject")
-	}
-
-	role := claims.Role
-	if role == "" {
-		role = "user"
-	}
-	return &Principal{
-		UserID:         claims.Subject,
-		Email:          claims.Email,
-		Role:           role,
-		ViaAdminAccess: v.AdminAudience != "" && audienceContains(claims.Audience, v.AdminAudience),
-	}, nil
 }
 
 // bearerToken extracts a Bearer credential from the Authorization header.
@@ -168,22 +153,3 @@ func bearerToken(r *http.Request) string {
 	}
 	return ""
 }
-
-// accessToken prefers the Cloudflare Access assertion header, falling back to a
-// Bearer credential so the same verifier works behind a proxy or directly.
-func accessToken(r *http.Request) string {
-	if h := r.Header.Get("Cf-Access-Jwt-Assertion"); h != "" {
-		return h
-	}
-	return bearerToken(r)
-}
-
-// audienceContains reports whether want appears in the aud claim set.
-func audienceContains(aud jwt.ClaimStrings, want string) bool {
-	for _, a := range aud {
-		if a == want {
-			return true
-		}
-	}
-	return false
-}

internal/api/backuper.go

@@ -23,3 +23,13 @@ import "context"
 type Backuper interface {
 	Backup(ctx context.Context, serverName, formerOwner string) error
 }
+
+// RestoreSnapshotter is the Backuper's safety-snapshot lever: it enqueues a backup
+// of the world as it is now, labelled with the restore to run once that backup
+// has succeeded (backupID, backupRef). RestoreChains settles it: it starts the
+// restore after a successful snapshot and gives the restore up after a failed
+// one, so a restore never overwrites a world that has no copy. Until it is
+// settled the snapshot holds the world volume as a restore (internal/maintenance).
+type RestoreSnapshotter interface {
+	BackupThenRestore(ctx context.Context, serverName, formerOwner, backupID, backupRef string) error
+}