FelisMC/Felis
2
0 Fork 0
Code Issues Pull Requests 6 Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: FelisMC/Felis:main
FelisMC/Felis:main FelisMC/Felis:dependabot/github_actions/anchore/sbom-action-0.24.2 FelisMC/Felis:dependabot/github_actions/docker/setup-buildx-action-4.4.1 FelisMC/Felis:dependabot/github_actions/actions/download-artifact-8.0.1 FelisMC/Felis:dependabot/github_actions/actions/setup-java-6.0.1 FelisMC/Felis:dependabot/github_actions/actions/setup-go-7.0.0 FelisMC/Felis:dependabot/docker/golang-1.27 FelisMC/Felis:audit-fixes-20260922 FelisMC/Felis:chore/issue-sweep
FelisMC/Felis:v0.2.0-rc2 FelisMC/Felis:v0.2.0 FelisMC/Felis:v0.2.0-rc1 FelisMC/Felis:v0.1.0 FelisMC/Felis:v0.1.0-rc1
..
pull from: FelisMC/Felis:dependabot/github_actions/actions/setup-go-7.0.0
FelisMC/Felis:main FelisMC/Felis:dependabot/github_actions/anchore/sbom-action-0.24.2 FelisMC/Felis:dependabot/github_actions/docker/setup-buildx-action-4.4.1 FelisMC/Felis:dependabot/github_actions/actions/download-artifact-8.0.1 FelisMC/Felis:dependabot/github_actions/actions/setup-java-6.0.1 FelisMC/Felis:dependabot/github_actions/actions/setup-go-7.0.0 FelisMC/Felis:dependabot/docker/golang-1.27 FelisMC/Felis:audit-fixes-20260922 FelisMC/Felis:chore/issue-sweep
FelisMC/Felis:v0.2.0-rc2 FelisMC/Felis:v0.2.0 FelisMC/Felis:v0.2.0-rc1 FelisMC/Felis:v0.1.0 FelisMC/Felis:v0.1.0-rc1
1 Commits
Author SHA1 Message Date
dependabot[bot] a252f04b0f chore(deps): bump actions/setup-go from 5.6.0 to 7.0.0 
Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.6.0 to 7.0.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/40f1582b2485089dde7abd97c1529aa768e1baff...b7ad1dad31e06c5925ef5d2fc7ad053ef454303e)

---
updated-dependencies:
- dependency-name: actions/setup-go
  dependency-version: 7.0.0
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>
 2026-09-24 16:45:30 +00:00
864 changed files with 11803 additions and 157683 deletions
Expand all files Collapse all files

No files matched your search

.github/workflows/ci.yml
+11 -74
View File
@@ -9,23 +9,19 @@
# The push trigger is limited to main rather than every branch, for the reason release.yml is
# not repeated here: a branch with an open PR would otherwise run the whole suite twice per
# push, once for refs/heads/<branch> and once for refs/pull/N/merge. Those are different
# concurrency groups, so neither cancels the other, and both would occupy runners for the
# same commit. A branch with no PR open yet is the one case that loses coverage, and opening the
# concurrency groups, so neither cancels the other, and this repository is private and billed
# for both. A branch with no PR open yet is the one case that loses coverage, and opening the
# PR is what asks for the answer.
name: ci
#
# release.yml calls this workflow (workflow_call) before it builds anything, so a tag passes
# exactly these gates and there is one list of them.
#
# workflow_dispatch reruns the suite on a commit whose push produced no run, such as one
# pushed while Actions was unavailable.
on:
push:
branches: [main]
pull_request:
workflow_call:
workflow_dispatch:
permissions:
contents: read
@@ -40,7 +36,7 @@ jobs:
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
@@ -67,7 +63,7 @@ jobs:
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
@@ -76,9 +72,6 @@ jobs:
# The business stores' SQL against a real PostgreSQL (internal/pgint): the unit suites run
# on fakes, and PGRepo drifted from them three times while those stayed green. 13 is the
# oldest server a supported distribution installs (EL9), 18 the newest (Arch).
# `felis db backup` and `restore` run there too, with the tools inside the service
# container, as production runs them inside felis-postgres: the runner's own client is one
# major version, and pg_dump refuses a newer server.
pgint:
runs-on: ubuntu-latest
strategy:
@@ -102,14 +95,13 @@ jobs:
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
- run: go test -race -tags pgint -count=1 ./internal/pgint/
env:
FELIS_TEST_PG_URL: postgres://felis:pgint@localhost:5432/felis_pgint?sslmode=disable
FELIS_TEST_PG_EXEC: docker exec -i ${{ job.services.postgres.id }}
shell:
runs-on: ubuntu-latest
@@ -142,35 +134,7 @@ jobs:
tar -xJf shellcheck.tar.xz
./shellcheck-v0.11.0/shellcheck -S warning $(git ls-files '*.sh')
# An exit status is 8 bits, so `exit "$fails"` reads 256 failures as a pass. The suites
# exit 1 on any failure; this keeps the next one from carrying its count out.
- name: No script exits with its failure count
run: |
if git grep -nE 'exit +"?\$\{?[a-z_]*fail[a-z_]*\}?"?[[:space:]]*$' -- '*.sh'; then exit 1; fi
- run: sh deploy/bootstrap_test.sh
- run: sh deploy/uninstall_test.sh
- run: bash deploy/e2e_release_test.sh
- run: bash deploy/e2e_upstream_test.sh
# The shipped alert rules (deploy/alerts): promtool parses them and runs their unit tests,
# which pin when each alert fires and that it stays quiet before. internal/metrics'
# alerts_test.go pins what promtool cannot see from there: the PrometheusRule twin and the
# metric names the rules read.
alerts:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# A pinned release, like shellcheck's, so a new Prometheus cannot change what fails.
- name: promtool
run: |
curl -fsSL -o prometheus.tar.gz \
https://github.com/prometheus/prometheus/releases/download/v3.15.0/prometheus-3.15.0.linux-amd64.tar.gz
echo "2a542df32eac02ee17b9d844fb2aa1de00dafa5476579ba8a3ba862e9d572ea0 prometheus.tar.gz" | sha256sum -c
tar -xzf prometheus.tar.gz --strip-components=1 prometheus-3.15.0.linux-amd64/promtool
./promtool check rules deploy/alerts/felis-alerts.yaml
./promtool test rules deploy/alerts/felis-alerts_test.yml
panel:
runs-on: ubuntu-latest
@@ -200,53 +164,26 @@ jobs:
- run: npm test
working-directory: panel
# `tsc -b`: tsconfig.json is a solution file (files: [] plus references), so a plain
# `tsc --noEmit` checked nothing and passed with type errors in the tree.
- run: npm run typecheck
working-directory: panel
# Rules of hooks and effect dependency lists, with --max-warnings 0.
- run: npm run lint
working-directory: panel
# openapi.gen.ts is generated from docs/openapi.yaml and checked in, so the
# compile-time parity in src/lib/types.parity.ts needs no generator in the build;
# a schema edit that was not regenerated fails here.
- name: openapi.gen.ts matches docs/openapi.yaml
working-directory: panel
run: |
npm run gen:api
git diff --exit-code -- src/lib/openapi.gen.ts
# Browser smoke over the mock-mode dev server, in the runner's installed Chrome
# (playwright.config.ts sets channel: chrome, so nothing is downloaded).
- run: npm run test:e2e
working-directory: panel
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: failure()
with:
name: panel-playwright-report
path: |
panel/playwright-report
panel/test-results
retention-days: 7
plugins:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# The other jobs never touch the Java layer: the plugin jars were only ever
# compiled by bootstrap on a live host, and the test mains under plugins/*/test
# were run by hand. JDK 25 is what the plugin build image runs (paper-api 26.x
# needs it); each module's wrapper brings the Gradle the image pins.
# compiled by bootstrap on a live host, and the three test mains under
# plugins/*/test were run by hand. JDK 21 plus the Gradle major the plugin
# Dockerfiles pin (8.14) is that same toolchain, in CI.
- uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
with:
distribution: temurin
java-version: '25'
java-version: '21'
- uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
with:
gradle-version: '8.14'
- run: bash plugins/test.sh
.github/workflows/e2e.yml
-343
View File
@@ -1,343 +0,0 @@
# deploy/bootstrap.sh end to end on a fresh Ubuntu 24.04 x86_64 runner: the paths every
# host goes through, run for real instead of by hand on a VM.
#
# artifacts this commit's release assets, built by deploy/build-release-artifacts.sh the
# way release.yml builds a tag's (amd64 only, the runners' architecture)
# install a full install from those assets, the way a release installs, on a host with
# ufw enabled, then the same assets again (a rerun must converge without
# restarting what did not change, importing or uploading an image again, or
# touching Docker)
# readme the README's one-line install as a new host runs it today: this commit's
# installer on its default channel, which installs the newest published
# release's binary, images and plugin from that release's assets; skipped until
# a release exists
# upgrade the newest published release through its own installer and its own assets,
# rows of every kind seeded into its database, then this commit's assets on
# top of it; skipped until a release exists
# source a full install built on the host from this checkout (FELIS_SKIP_FETCH):
# the fallback a release without assets, or an unsupported one, takes. It builds
# everything with Docker, so it runs by hand and weekly only
#
# This runs on pushes that touch what gets installed, by hand, and weekly (a moving
# upstream: apt mirrors, k3s's install script and release assets, Adoptium).
# deploy/e2e_check.sh holds the assertions, deploy/e2e_seed.sh the upgrade's seed and its check.
#
# An installer that stops on an upstream download refused or dropped (a GitHub 403, a 5xx, a
# timeout) ends its job green, with a warning on the run: each install step hands a failed
# run's log to deploy/e2e_upstream.sh, which sets E2E_UPSTREAM_SKIP for the job's later
# steps. Every other installer failure, a 404 included, fails the job.
name: e2e
on:
push:
branches: [main]
paths:
- 'deploy/**'
- 'cmd/**'
- 'internal/**'
- 'panel/**'
- 'plugins/**'
- 'Dockerfile'
- 'go.mod'
- 'go.sum'
- '.github/workflows/e2e.yml'
workflow_dispatch:
schedule:
- cron: '23 4 * * 1'
permissions:
contents: read
# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` carries the
# installer's status to deploy/e2e_upstream.sh; the default shell reports tee's status.
defaults:
run:
shell: bash
concurrency:
group: e2e-${{ github.ref }}
cancel-in-progress: true
jobs:
artifacts:
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
fetch-depth: 0 # the newest tag is the version's base, as on the dev channel
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Stamped the way bootstrap stamps a build of main: "<newest tag>+g<sha>".
- name: Build the release assets
run: |
base="$(git describe --tags --abbrev=0 --match 'v*' 2>/dev/null || echo v0.0.0)"
FELIS_RELEASE_ARCHES=amd64 deploy/build-release-artifacts.sh "${base}+g$(git rev-parse --short HEAD)" dist
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: e2e-assets
path: dist/
if-no-files-found: error
retention-days: 1
install:
needs: artifacts
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# The images, k3s and the JRE need more room than a stock runner leaves free.
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: e2e-assets
path: dist
# The runner has Docker preinstalled, so its absence proves nothing: the log shows
# whether bootstrap reached for it. From FELIS_ARTIFACT_DIR every image and the plugin
# come out of the assets, so it must not have. (`! grep` is spelled `if grep ... exit 1`
# below because bash -e ignores a failing `!` command.)
# ufw, enabled on many Ubuntu hosts, drops every inbound packet no rule admits, the
# pods' traffic to the API server among them; the runner ships it disabled. The
# runner's own traffic is outbound, which ufw lets through.
- name: Enable ufw
run: sudo ufw --force enable
- name: Install
run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log || bash deploy/e2e_upstream.sh install.log $?
- name: Check the install
if: env.E2E_UPSTREAM_SKIP != '1'
run: |
sudo bash deploy/e2e_check.sh install
for tag in felis-k3s-pods felis-k3s-services felis-panel felis-proxy; do
sudo ufw status | grep -qE "# ${tag} *$"
done
if grep -E 'docker already installed|installing docker|docker running' install.log; then exit 1; fi
for role in felis limbo lobby paper; do
grep -q "felis/${role}:[^ ]* is the release's" install.log
done
grep -q "docker.io/library/registry@sha256:[0-9a-f]* is the release's" install.log
grep -q "docker.io/library/postgres@sha256:[0-9a-f]* is the release's" install.log
grep -q "felis-velocity.jar is the release's" install.log
- name: Rerun the same assets
if: env.E2E_UPSTREAM_SKIP != '1'
run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log || bash deploy/e2e_upstream.sh rerun.log $?
# containerd and the registry already hold every image under the digest the listing
# names, so nothing is imported or uploaded twice.
- name: Check the rerun
if: env.E2E_UPSTREAM_SKIP != '1'
run: |
sudo bash deploy/e2e_check.sh rerun
grep -q 'felis-velocity unchanged; left running' rerun.log
if grep -E 'docker already installed|installing docker|docker running' rerun.log; then exit 1; fi
if grep -E 'importing felis-image-|mirroring .* into the internal registry' rerun.log; then exit 1; fi
grep -q 'is already in the internal registry' rerun.log
- name: Diagnostics
if: failure()
run: |
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
k() { sudo -E /usr/local/bin/k3s kubectl "$@"; }
k get pods -A -o wide || true
k get events -A --sort-by=.lastTimestamp | tail -n 60 || true
for p in $(k get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" {print $1 "/" $2}'); do
k -n "${p%%/*}" describe pod "${p#*/}" | tail -n 40 || true
k -n "${p%%/*}" logs "${p#*/}" --all-containers --tail=60 || true
done
k -n felis logs deploy/felis-postgres --tail=60 || true
sudo journalctl -u k3s -u felis-velocity --no-pager -n 120 || true
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: e2e-install-logs
path: '*.log'
if-no-files-found: ignore
# The README's command on a fresh host: this commit's installer, as main serves it, on its
# default channel with nothing pinned. It resolves the newest release and installs that
# release's binary, images and plugin from its assets, each checked against its
# SHA256SUMS. The workflow's token is the only thing added: it lifts GitHub's limit of 60 API
# calls an hour for an address without one. FELIS_INSTALL_MODE picks the mode the setup
# console would ask for.
readme:
runs-on: ubuntu-24.04
timeout-minutes: 120
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- name: Find the newest release
id: release
env:
GH_TOKEN: ${{ github.token }}
run: bash deploy/e2e_release.sh find
- name: Install as the README does
if: steps.release.outputs.tag != ''
env:
TOKEN: ${{ github.token }}
run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log || bash deploy/e2e_upstream.sh readme.log $?
# The binary is the release's, so the phase is `release`: its database backup and
# timers are the release's to have or lack.
- name: Check the install
if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
env:
TAG: ${{ steps.release.outputs.tag }}
BINARY: ${{ steps.release.outputs.binary }}
SUMS: ${{ steps.release.outputs.sums }}
run: |
sudo bash deploy/e2e_check.sh release
sudo /usr/local/bin/felis version | grep -qx "felis ${TAG}"
bash deploy/e2e_release.sh check-readme readme.log
- name: Diagnostics
if: failure()
run: |
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: e2e-readme-logs
path: '*.log'
if-no-files-found: ignore
upgrade:
needs: artifacts
runs-on: ubuntu-24.04
timeout-minutes: 120
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
fetch-depth: 0
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: e2e-assets
path: dist
- name: Find the newest release
id: release
env:
GH_TOKEN: ${{ github.token }}
run: bash deploy/e2e_release.sh find
# The release's own installer on its default channel, fetching the release's own
# binary: what a host that installed that release is running today. FELIS_REF would
# build the tag from source instead, a path no host takes by default. FELIS_RELEASE pins
# the tag found above; an installer older than FELIS_RELEASE ignores it and resolves
# the newest release, the same tag.
- name: Install the newest release
if: steps.release.outputs.tag != ''
env:
TAG: ${{ steps.release.outputs.tag }}
TOKEN: ${{ github.token }}
run: |
git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh
sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log || bash deploy/e2e_upstream.sh release.log $?
- name: Check the release install
if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
env:
TAG: ${{ steps.release.outputs.tag }}
BINARY: ${{ steps.release.outputs.binary }}
run: |
sudo bash deploy/e2e_check.sh release
bash deploy/e2e_release.sh check-own release.log
# A fresh install's database holds only what its migrations wrote: the seed gives the
# upgrade's move and its pending migrations existing rows to carry.
- name: Seed the release's database
if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
run: sudo bash deploy/e2e_seed.sh seed
- name: Upgrade to this commit
if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log || bash deploy/e2e_upstream.sh upgrade.log $?
# Both checks run and report: the seeded rows go last, after the restore drill has
# also put them back from a bundle.
- name: Check the upgrade
if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
run: |
rc=0
sudo bash deploy/e2e_check.sh upgrade || rc=1
sudo bash deploy/e2e_seed.sh check || rc=1
exit "$rc"
- name: Diagnostics
if: failure()
run: |
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
# The release ran the database on the host; the upgrade moves it into k3s.
sudo journalctl -u k3s -u felis-velocity -u postgresql --no-pager -n 120 || true
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: e2e-upgrade-logs
path: '*.log'
if-no-files-found: ignore
source:
if: github.event_name != 'push'
runs-on: ubuntu-24.04
timeout-minutes: 90
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# FELIS_SKIP_FETCH builds whatever sits in /opt/felis/src; the .git directory is what
# stamps the build. Root owns it so git, run as root by the installer, does not refuse
# it as dubious.
- name: Stage this commit as the installer's source
run: |
sudo mkdir -p /opt/felis
sudo cp -a "$GITHUB_WORKSPACE" /opt/felis/src
sudo chown -R root:root /opt/felis/src
- name: Install
run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log || bash deploy/e2e_upstream.sh source.log $?
- name: Check the install
if: env.E2E_UPSTREAM_SKIP != '1'
run: sudo bash deploy/e2e_check.sh install
- name: Diagnostics
if: failure()
run: |
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: e2e-source-logs
path: '*.log'
if-no-files-found: ignore
.github/workflows/release.yml
+63 -45
View File
@@ -2,15 +2,14 @@
#
# Two things depend on this job. /repos/{repo}/releases/latest must ANSWER — that endpoint is
# what `felis update` polls (internal/updater/github.go) and what deploy/bootstrap.sh's default
# "release" channel resolves its ref from. And the assets below are what that channel then
# INSTALLS: bootstrap downloads the felis binary, the image tars, their listing and the Velocity
# plugin instead of building anything on the target host, so these are the shipped artifact,
# not a convenience. A host installing a release needs neither Docker, Gradle, Go nor Docker Hub.
# "release" channel resolves its ref from. And the binaries below are what that channel then
# INSTALLS: bootstrap downloads felis-linux-<arch> instead of compiling on the target host, so
# these are the shipped artifact, not a convenience.
#
# deploy/build-release-artifacts.sh builds every asset in one run and documents each name; the
# names are a contract with deploy/bootstrap.sh. They are plain literals on both sides: a YAML
# workflow and a go:embed'ed shell script have no honest way to share a constant, and the
# failure mode is benign — bootstrap warns and falls back to building on the host.
# The asset NAME is a contract with deploy/bootstrap.sh (download_release_binary builds
# "felis-linux-${arch}"). It is deliberately a plain literal on both sides: a GitHub Actions
# YAML and a go:embed'ed shell script have no honest way to share a constant, and the failure
# mode is benign — bootstrap warns and falls back to a source build of the same tag.
#
# The binary is built through the repo Dockerfile rather than a plain `go build`.
# internal/panel/static holds a tracked PLACEHOLDER index.html so the //go:embed
@@ -19,12 +18,13 @@
# build first, and is the same recipe bootstrap uses, so there is one way to build felis
# rather than two that can drift.
#
# SHA256SUMS is a contract with bootstrap too: it refuses any asset whose hash is not listed
# there, BEFORE it runs or imports it.
# SHA256SUMS is a contract with bootstrap too: download_release_binary refuses a binary whose
# hash is not listed there, BEFORE it runs it. A release without the file installs by source
# build instead.
#
# The write token never meets the test suite: `gates` (ci.yml) and `build` run the tests,
# Gradle and the Docker builds (each of which executes third-party code) with a read-only
# token, and `build` hands the assets over as a workflow artifact; `publish` holds contents:write and runs only
# Gradle and the Docker build (each of which executes third-party code) with a read-only
# token, and `build` hands the binaries over as a workflow artifact; `publish` holds contents:write and runs only
# pinned actions and gh. Every action is pinned to a commit SHA (the tag in the trailing
# comment is for humans); .github/dependabot.yml proposes the bumps.
name: release
@@ -39,10 +39,9 @@ permissions:
jobs:
# A tag that ships red is worse than a tag that fails to ship. These are ci.yml's gates,
# called rather than copied: Go (race, vet, staticcheck), govulncheck, the PostgreSQL
# contract suite, shellcheck and the bootstrap tests, promtool on the alert rules, the
# panel, and the Java layer the binary EMBEDS (bootstrap_asset.go ships the plugin
# sources, so a tag whose plugins do not compile turns every install of that release into
# a failed bootstrap).
# contract suite, shellcheck and the bootstrap tests, the panel, and the Java layer the
# binary EMBEDS (bootstrap_asset.go ships the plugin sources, so a tag whose plugins do
# not compile turns every install of that release into a failed bootstrap).
gates:
uses: ./.github/workflows/ci.yml
@@ -53,47 +52,69 @@ jobs:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# Both architectures, because bootstrap's default release channel DOWNLOADS these
# rather than building on the target host — an arm64 host with no asset falls back to
# a slow on-host build. The felis binary and the plugin jars cross-compile on the
# runner (their build stages are pinned to $BUILDPLATFORM); the game images' runtime
# stages run apt-get for the target platform, which for arm64 takes QEMU.
- uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
# rather than compiling on the target host — an arm64 host with no asset silently
# falls back to a slow source build. Neither stage is emulated: the Dockerfile pins
# both build stages to $BUILDPLATFORM and the Go stage cross-compiles via TARGETARCH,
# so the second architecture costs about a minute.
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
# Two architectures of five images plus BuildKit's cache outgrow the runner's free disk
# with its preinstalled SDKs in place; none of them is used here.
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- name: Build the stamped binaries
run: |
docker buildx build --platform linux/amd64,linux/arm64 \
--build-arg FELIS_VERSION="${GITHUB_REF_NAME}" \
--output type=local,dest=out .
mv out/linux_amd64/usr/local/bin/felis ./felis-linux-amd64
mv out/linux_arm64/usr/local/bin/felis ./felis-linux-arm64
chmod +x ./felis-linux-amd64 ./felis-linux-arm64
# The script checks what the old inline steps did: the host binary reports exactly
# "felis <tag>" (unstamped, `felis update` refuses to compare), and the other one is an
# ELF of its architecture, read with file(1) — binfmt would run a misnamed binary happily.
- name: Build the release assets
run: deploy/build-release-artifacts.sh "${GITHUB_REF_NAME}" dist
# The stamp is the whole point and it fails silently: an unstamped binary reports
# "dev", which `felis update` refuses to compare, disabling update reporting for
# every install built from it. Assert it end to end instead of trusting the ARG
# reached the linker.
- name: Verify the version stamp
run: |
got="$(./felis-linux-amd64 version | head -1)"
echo "reported: ${got}"
[ "$got" = "felis ${GITHUB_REF_NAME}" ] \
|| { echo "expected 'felis ${GITHUB_REF_NAME}' — the -X main.version stamp did not reach the binary"; exit 1; }
# The arm64 binary is checked by ELF machine type, NOT by running it. Runners have
# binfmt/QEMU registered, so `./felis-linux-arm64 version` would happily succeed on
# an amd64 binary misnamed arm64 — which is exactly the failure the Dockerfile's
# ${TARGETARCH:-$(go env GOARCH)} fallback produces if buildx did not take. Both
# binaries come out of one RUN with one -ldflags string, so the stamp is checked once.
file ./felis-linux-arm64
file ./felis-linux-arm64 | grep -q 'ARM aarch64' \
|| { echo "felis-linux-arm64 is not an arm64 ELF — TARGETARCH did not reach the go build"; exit 1; }
- name: Checksum the binaries
run: sha256sum felis-linux-amd64 felis-linux-arm64 | tee SHA256SUMS
# A CycloneDX SBOM per binary: the Go modules (and versions) linked into it, read
# from the build info the linker embeds. Written after SHA256SUMS, so beside it in the
# release but outside it: that lists what bootstrap installs. Straight into dist/,
# because the action does not create a missing output directory.
# from the build info the linker embeds.
- uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
with:
file: dist/felis-linux-amd64
file: felis-linux-amd64
format: cyclonedx-json
output-file: dist/felis-linux-amd64.cdx.json
output-file: felis-linux-amd64.cdx.json
upload-artifact: false
upload-release-assets: false
- uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
with:
file: dist/felis-linux-arm64
file: felis-linux-arm64
format: cyclonedx-json
output-file: dist/felis-linux-arm64.cdx.json
output-file: felis-linux-arm64.cdx.json
upload-artifact: false
upload-release-assets: false
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: release-assets
path: dist/
path: |
felis-linux-amd64
felis-linux-arm64
felis-linux-amd64.cdx.json
felis-linux-arm64.cdx.json
SHA256SUMS
if-no-files-found: error
retention-days: 7
@@ -113,7 +134,7 @@ jobs:
- run: sha256sum -c SHA256SUMS
# Signed SLSA provenance: which workflow run, commit and repository produced each
# asset. Check one with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`.
# binary. Check one with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`.
# GitHub only stores attestations for private repositories on Enterprise Cloud, and a
# failure here would block the release, so a private repository skips the step and
# relies on SHA256SUMS alone.
@@ -122,9 +143,8 @@ jobs:
uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0
with:
subject-path: |
felis-linux-*
felis-image-*.tar
felis-velocity.jar
felis-linux-amd64
felis-linux-arm64
# --verify-tag refuses to invent a release for a tag that is not pushed.
#
@@ -144,9 +164,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
run: |
# Every file the build handed over: SHA256SUMS names the installable ones, and the
# SBOMs ride along.
assets="$(ls)"
assets="felis-linux-amd64 felis-linux-arm64 felis-linux-amd64.cdx.json felis-linux-arm64.cdx.json SHA256SUMS"
flags=""
case "$GITHUB_REF_NAME" in *-*) flags="--prerelease" ;; esac
if ! gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
.gitignore
-4
View File
@@ -51,7 +51,3 @@ AGENTS.md
docs/Felis-Spec-V4.1.md
panel/DESIGN.md
panel/DESIGN-WEB-3SIDES.md
# Audit ledgers and readiness reviews stay on the maintainer's disk.
/AUDIT-*.md
/READINESS-*.md
AUDIT-2026-09-22.md
+1013
View File
File diff suppressed because it is too large. Load diff
CONTRIBUTING.md
+6 -35
View File
@@ -23,28 +23,6 @@ The codebase is intentionally split by responsibility. Prefer changing the
smallest owning module instead of adding broad abstractions or rebuilding nearby
code.
## Interface Copy
Interface copy must use a formal documentation style: objective, concise, and
precise. State the current condition, its cause or impact, and the available
action. Describe confirmed facts only; avoid conversational phrasing,
personification, and unsupported duration estimates. Chinese instructions should
use explicit verbs such as “执行”, “选择”, “查看”, and “配置”, with operation names
matching the actual UI labels. English and Chinese copy must retain the same
meaning and degree of formality. Update existing translation entries rather than
adding duplicate messages or components.
## Panel Visual Style
Reuse `PageHeader`, the `Card` family, and the shared form and button components.
Card titles, borders, corner radii, spacing, and save footers follow their shared
definitions; avoid redefining these styles in individual pages. Use white card
backgrounds in the light theme, restrained semantic colors, and 16px functional
icons. Use `CardFooter` for save actions and `Button` for interactive controls.
Tables, consoles, compact statistics, and status messages may retain spacing and
colors suited to their content. Preserve the shared page margins and bottom
spacing.
## Local Development
You can do most day-to-day development on macOS or Linux without a full cluster.
@@ -100,15 +78,8 @@ FELIS_TEST_PG_URL='postgres://felis:***@127.0.0.1:5432/felis_pgint?sslmode=disab
```
Run it after touching anything under `internal/api/pgrepo.go`, `internal/submit`,
`internal/build` or `internal/dbbackup` that speaks SQL: the fakes encode the
contract, and this suite exists to catch the drift between the fakes and the real
queries. The `felis db backup` and `restore` tests also run `pg_dump`, `pg_restore`
and `psql`, which must be the server's major version. For a server in a container,
run them in it, as production does in felis-postgres:
```bash
FELIS_TEST_PG_EXEC='docker exec -i <container>' FELIS_TEST_PG_URL=... go test -tags pgint ./internal/pgint/
```
or `internal/build` that speaks SQL: the fakes encode the contract, and this
suite exists to catch the drift between the fakes and the real queries.
Build the CLI:
@@ -234,13 +205,13 @@ export FELIS_IMAGE=felis:dev
export FELIS_ROOT_DOMAIN=<node-ip>.nip.io
```
By default the installer builds the newest **published GitHub release**. Building the
development tip needs an opt-in, and installing from a private fork additionally needs a
token for the release lookup and the clone:
By default the installer builds the newest **published GitHub release**. While this
repository is private that lookup — and the clone itself — needs a token, and building
the development tip needs an opt-in:
```bash
export FELIS_GITHUB_TOKEN=<token with read access to the repo>
export FELIS_VERSION_BOOTSTRAP=dev # build main instead of the newest release
export FELIS_GITHUB_TOKEN=<token> # private forks only: read access to the fork
```
`dev` is also the escape hatch before the first `vX.Y.Z` tag exists: with no published
README.md
+47 -92
View File
@@ -1,25 +1,11 @@
<div align="center">
<h1 align="center">
<img src="docs/assets/felis-logo.png" alt="Felis logo" width="270"><br>
Felis
</h1>
<p align="center">
基于 Kubernetes 的 Minecraft 服务器托管平台<br>
单条命令完成部署，自动管理服务器生命周期、备份与安全
<br><br>
<a href="README.md">简体中文</a> | <a href="README_EN.md">English</a>
<br>
<a href="https://felismc.com/">官网</a> | <a href="https://docs.felismc.com/">文档</a>
</p>
</div>
# Felis
> [!CAUTION]
> **当前仅提供开发快照，已撤下公开 Release。仅供开发者在隔离的测试环境中验证，不建议普通用户部署。**
> **此项目仍处于早期开发阶段，您不该在任何生产环境使用该项目。若产生任何问题，贵用户的使用行为与 FelisMC 团队无任何民事刑事法律关系。**<br>
> **THIS PROJECT IS STILL WIP, YOU SHOULD DO NOT USE THIS PROJECT IN ANY PRODUCTION USAGE. WE ARE NOT RESPOND FOR ANY LEGAL OR HUMANLY PROBLEM.**
一款 Kubernetes 驱动的 Minecraft 服务器托管平台，一行命令部署，自动管理生命周期与安全。
A Kubernetes-driven Minecraft server hosting platform — one command to deploy, automatic lifecycle, backup, and security.
<details>
<summary>目录</summary>
[简体中文](README.md) | [English](README_EN.md)
目录
- [特性](#特性)
- [使用方式](#使用方式)
@@ -27,79 +13,48 @@
- [开源协议](#开源协议)
- [致谢](#致谢)
</details>
## 特性
* **按需启停**：玩家连接代理时自动启动目标服务器，启动期间玩家进入等待队列，服务器就绪后自动传送；服务器空闲后自动停止，释放内存。
* **Web 控制面板**：在浏览器中查看服务器状态、在线玩家与资源用量。
* 控制台（RCON）、白名单、封禁、OP 与 LuckPerms 权限管理
* 文件管理：新建、删除、重命名、分片上传、下载，以及停服状态下解压 zip，可用于导入世界
* 计划任务：按星期与时区定时执行命令、重启、停止、启动或备份，执行前在游戏内向玩家发送提醒
* **备份与恢复**：默认启用，归档 PVC 及其路径由安装器生成。
* 手动备份：将服务器的完整数据卷（`/data`，含世界、配置、插件与模组）归档至集群内的归档存储，可回滚至任一备份点
* 每日恢复点：当天有玩家进入过的服务器在停止后自动生成恢复点，默认保留 7 个，保存期限 90 天；恢复点单独轮换，不影响手动备份
* 下载与导出：支持下载单个备份（附 sha256 校验）、删除单个备份及导出完整世界
* 异地副本（可选）：备份在主机上加密后同步至 S3 兼容存储（AWS S3、Cloudflare R2、Backblaze B2、MinIO 等）
* 控制面数据库：存放账号、服务器归属、配额与存档索引的数据库每日自动备份，每次升级迁移前额外创建快照，故障时可通过 `felis db restore` 整库原子回滚；面板「维护与备份」页显示最近一次备份的时效（参见 [故障排查 §16](docs/troubleshooting.md)）
* **运维诊断**
* `sudo felis status`：汇总显示节点、控制面、游戏代理、各服务器、备份及未解决的告警
* `sudo felis doctor`：执行全部健康检查，按模块列出问题及排查方向，执行过程中不发送邮件
* `sudo felis support-bundle`：生成已脱敏的诊断包，供提交问题时附带（参见 [故障排查 §0](docs/troubleshooting.md)）
* 看门狗：每 2 分钟执行一次巡检，异常持续时向平台所有者发送邮件告警，支持外部心跳监测
* **世界回收（可选）**：超过 15 天无人游玩的世界在备份后删除，以释放磁盘空间。安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认为 `/var/lib/rancher/k3s/storage`）即启用每日回收；未设置时不删除任何世界。过期备份的每日清理与此设置无关，始终执行。
* **多核心支持**：兼容 Paper、Fabric、Forge 与 NeoForge，统一经由 Velocity 代理接入。
* **模组包投稿**：玩家可上传模组包，经服主审批后自动构建并通过 Trivy 安全扫描；构建产物加入镜像白名单后，可直接选作服务器镜像。
* **安全**
* Passkey 登录：支持指纹、面容识别及硬件密钥等无密码认证方式
* 零信任访问：面板流量经 Cloudflare Access 保护，集群内部 API 不对公网开放
* **多机部署（实验性，默认关闭）**：由一台主控节点统一下发指令，其余节点仅运行游戏服务器，已停止的服务器可迁移至其他节点。该功能目前仅位于 main 分支，尚未完成三机验收（参见 [多机部署](docs/distributed.md)）。
- **即开即玩**：玩家尝试连接时自动唤醒服务器，空闲后自动休眠，像游戏主机一样省资源。
- **Web 控制面板**：浏览器中查看服务器状态、在线玩家与资源用量，管理备份与恢复。
- **备份与恢复**：一键把整服数据（世界、配置、插件/模组，即整个 /data 卷）打包进集群内的归档库，支持从任意备份点回滚；默认安装就已启用（归档 PVC 与路径由安装器一并生成）。
- **控制面数据库备份**：账号、服务器归属、配额与存档索引所在的数据库每天自动备份，每次升级迁移前先快照，出错可用 `felis db restore` 整库原子回滚；面板「维护与备份」页显示备份是否新鲜（见 [故障排查 §16](docs/troubleshooting.md)）。
- **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。
- **多核心支持**：兼容 Paper、Fabric、Forge、NeoForge，经由 Velocity 代理统一入口。
- **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建；构建产物进入镜像白名单，可直接选用为服务器镜像完成部署。
- **Passkey 登录**：支持指纹、面容、硬件密钥等无密码认证方式。
- **零信任安全**：面板流量由 Cloudflare Access 保护，集群内 API 不暴露到公网。
## 使用方式
开发测试需在已准备好的 Linux 测试主机上显式选择 `dev` 通道（跟随 `main`，从源码构建）：
在准备好的 Linux 主机上执行：
```bash
curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo FELIS_VERSION_BOOTSTRAP=dev bash
curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
```
脚本将安装 K3s，在 K3s 中部署 PostgreSQL 与控制平面，随后启动设置向导。设置完成后，通过浏览器访问所配置的域名即可进入控制面板。
脚本将自动安装 K3s、部署控制平面并启动设置向导。完成后浏览器访问已配置的域名进入控制面板即可使用。
* **设置向导**：先完成面板访问方式与存储配置，再由主机管理员创建首位 Owner，终端会给出一次性网页设置链接（30 分钟内有效）。用浏览器打开链接、登记邮箱并创建通行密钥，即可进入面板，无需启动 Minecraft。角色关联可稍后在“账户”页选择认证源、输入角色名或 UUID 并确认；普通玩家继续通过游戏绑定码验证身份。未完成网页登录或链接过期时，再次执行 `sudo felis setup` 会提供新链接；已设置登录凭证的账号不会被重置。登录服或大厅故障不会阻止面板初始化。“账户”页会解释世界树（Yggdrasil）认证、显示进服地址与登录服/大厅所需的 Java 版客户端版本；安装器会把实际构建版本写入 `[velocity].game_version`，自定义镜像需自行填写，未配置时不会猜测版本。标准世界树接口可直接查询角色；非标准 `hasJoined` 地址可通过 `[[auth_source]].api_url` 指定认证站 API 根地址，游戏绑定码仍可作为替代方式。安装器仅在交互式终端中自动启动向导；输出重定向至日志或经由 cloud-init 安装时，请在安装结束后执行 `sudo felis setup`。设置 `FELIS_NO_SETUP=1` 时，安装器在输出摘要后直接结束。
> **本仓库当前为私有**，上面这条会返回 404。请改用带凭据的形式；安装器自身也需要同一个 token
> 去解析并下载 release，所以用 `sudo -E` 把它带进去：
>
> ```bash
> export FELIS_GITHUB_TOKEN=<对本仓库有读权限的 token>
> printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \
> | curl -fsSL --config - -H "Accept: application/vnd.github.raw" \
> https://api.github.com/repos/FelisMC/Felis/contents/deploy/bootstrap.sh \
> | sudo -E bash
> ```
>
> token 经 stdin 交给 `curl --config -`，不放在命令行上：argv 在 `/proc` 下对本机任意用户可读，
> 而这正是安装器内部 `github_api` 采用同一写法的原因。
* **认证源管理**：Owner 可在左侧“平台 → 认证源”添加、编辑、排序、停用第三方 Yggdrasil 认证站，并测试认证接口。保存后立即用于下一次登录和角色查询，无需重启；面板配置优先于安装配置。已保存的永久标识不能改名或删除，以保留玩家 UUID 与账号绑定；不再使用的源可停用。Mojang 始终优先验证。Nano 继续使用 TOML 配置。
* **支持的系统**：CentOS Stream 9（aarch64）已在实机上验证；Ubuntu 24.04（x86_64）在每次推送时由 CI 验证开发构建的全新安装与重复安装（参见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
* **安装前检查**：安装器在修改主机之前检查内存、磁盘、端口、网段冲突、已有的 Kubernetes 及外网连通性。发现问题时一次性列出全部问题并退出，主机保持原状（检查项参见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
* **升级**：重新执行安装命令即可将 felis-api 升级至新版本；`felis setup` 仅使用本机已安装的二进制，无法用于升级。重新执行时沿用已安装的根域名，发布通道需重新指定：跟随 main 分支的主机须同时设置 `export FELIS_VERSION_BOOTSTRAP=dev`。早期版本安装在宿主机上的 PostgreSQL 会在重新执行时整库迁入 K3s，宿主机上的原实例停用并保留，以便回退（参见 [运维手册 §4](docs/operations.md#4-upgrading-the-pieces-around-felis)）。
<details>
<summary>安装来源与受限网络环境下的安装</summary>
<br>
以下说明发布机制；当前暂无公开 Release 附件，开发测试使用源码构建。
安装发布版时，二进制文件、全部镜像及 Velocity 插件均取自该版本由 CI 预构建的 release 附件，逐一校验 `SHA256SUMS` 后导入。主机无需安装 Docker、Gradle 或 Go，也无需访问 Docker Hub。若某个附件缺失或校验失败，仅该镜像回退为本机构建，并输出提示（参见 [故障排查 §15c](docs/troubleshooting.md)）。
也可将附件预先复制到主机，再通过 `FELIS_ARTIFACT_DIR=<绝对路径>` 安装，此时 Felis 自身的二进制、镜像与插件均从该目录读取。k3s 及其镜像、JRE、cloudflared、Velocity 与 Via 插件仍从 GitHub 和 PaperMC 下载；RHEL、Fedora、openSUSE Leap 等启用 SELinux 的主机还需从 rpm.rancher.io 安装 k3s-selinux；系统软件包来自发行版软件源。
因此，出站网络受限的主机须放行上述地址的 HTTPS 访问，或设置 `https_proxy`。preflight 会在修改主机之前逐一探测这些地址。目前暂不支持完全离线安装（地址清单参见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
</details>
重跑这条命令也是把 felis-api 升到新版本的方式（`felis setup` 做不到，它用的是本机已有的二进制）。
重跑会沿用已安装的根域名，但**不会**沿用通道：若本机跟随 main，需一并 `export FELIS_VERSION_BOOTSTRAP=dev`。
## 从源码构建
本项目基于 Go 与 Node.js 开发：
本项目基于 Go 和 Node.js 开发：
```bash
# 后端（Go 1.26+）
@@ -116,22 +71,22 @@ docker build -t felis:custom .
## 开源协议
本项目采用 [AGPL-3.0-only](LICENSE) 许可证。
本项目遵循 [AGPL-3.0-only](LICENSE) 开源协议。
### 协议注意事项
1. **衍生作品须采用 AGPL**：分发本项目副本或基于本项目的衍生软件时，须以 AGPL-3.0 开源，并保留原作者的版权声明与许可声明。
2. **网络服务同样须提供源码**（AGPL 第 13 条）：通过网络向他人提供经修改的 Felis 服务时，即使未分发任何二进制文件，也须向这些用户提供修改后的完整源码。这是 AGPL 与 GPL 唯一的实质区别；Felis 作为通过网络访问的托管平台，几乎所有部署场景都适用此条款。
3. **免责声明**：本项目按"原样"提供，作者不承担因使用本项目而产生的任何法律责任。
1. **衍生作品同样是 AGPL**：分发本项目的副本或基于本项目衍生的软件时，必须以 AGPL-3.0 开源，并保留原作者的版权声明和许可声明。
2. **通过网络提供服务同样要开源**（AGPL 第 13 条）：如果你把修改过的 Felis 架起来给别人用，即使从不分发任何二进制，也必须向这些用户提供你那份修改后的完整源码。这是 AGPL 相对 GPL 的唯一实质区别，而 Felis 正是一个跑在网络上的托管平台，所以这一条基本总会触发。
3. **免责声明**：本项目按"原样"提供，作者不承担任何因使用本项目而产生的法律责任。
## 致谢
* [Kubernetes](https://kubernetes.io/)：容器编排引擎
* [K3s](https://k3s.io/)：轻量级 Kubernetes 发行版
* [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/)：零信任安全基础设施
* [PostgreSQL](https://www.postgresql.org/)：数据持久化
* [React](https://react.dev/)：前端用户界面框架
* [Vite](https://vitejs.dev/)：前端构建工具
* [TailwindCSS](https://tailwindcss.com/)：CSS 框架
* [Bubble Tea](https://github.com/charmbracelet/bubbletea)：TUI 框架
* [Minecraft](https://www.minecraft.net/)：本项目服务的游戏
- [Kubernetes](https://kubernetes.io/)：底层容器编排引擎
- [K3s](https://k3s.io/)：轻量级 Kubernetes 发行版
- [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/)：零信任安全基础设施
- [PostgreSQL](https://www.postgresql.org/)：数据持久化
- [React](https://react.dev/)：前端用户界面框架
- [Vite](https://vitejs.dev/)：前端构建工具
- [TailwindCSS](https://tailwindcss.com/)：CSS 框架
- [Bubble Tea](https://github.com/charmbracelet/bubbletea)：TUI 框架
- [Minecraft](https://www.minecraft.net/)：让这一切值得做
README_EN.md
+49 -87
View File
@@ -1,24 +1,11 @@
<div align="center">
<h1 align="center">
<img src="docs/assets/felis-logo.png" alt="Felis logo" width="270"><br>
Felis
</h1>
<p align="center">
A Kubernetes-driven Minecraft server hosting platform<br>
One command to deploy, with automatic lifecycle, backup, and security
<br><br>
<a href="README.md">简体中文</a> | <a href="README_EN.md">English</a>
<br>
<a href="https://felismc.com/">Website</a> | <a href="https://docs.felismc.com/en/">Documentation</a>
</p>
</div>
# Felis
> [!CAUTION]
> **Only development snapshots are available; public releases have been withdrawn. These builds are for developers testing in isolated environments and are not recommended for general deployment.**
> **THIS PROJECT IS STILL WIP, YOU SHOULD DO NOT USE THIS PROJECT IN ANY PRODUCTION USAGE. WE ARE NOT RESPOND FOR ANY LEGAL OR HUMANLY PROBLEM.**
A Kubernetes-driven Minecraft server hosting platform — one command to deploy, automatic lifecycle, backup, and security.
一款 Kubernetes 驱动的 Minecraft 服务器托管平台，一行命令部署，自动管理生命周期与安全。
<details>
<summary>Table of Contents</summary>
[简体中文](README.md) | [English](README_EN.md)
Table of Contents
- [Features](#features)
- [Getting Started](#getting-started)
@@ -26,73 +13,48 @@
- [License](#license)
- [Acknowledgements](#acknowledgements)
</details>
## Features
* **On-demand Start and Stop**: A server starts when a player connects to the proxy. The player waits in a queue during start-up and is transferred once the server is ready. Idle servers stop automatically to free memory.
* **Web Dashboard**: Monitor server status, online players, and resource usage from your browser.
* Console (RCON), whitelist, bans, OPs and LuckPerms permissions
* File manager: create, delete, rename, chunked upload, download, and unzip while the server is stopped; also used to import worlds
* Schedules: run commands, restart, stop, start or back up by weekday and time zone, with an in-game warning to players beforehand
* **Backup & Restore**: Enabled by default; the installer renders the archive PVC and its path.
* Manual backups: archive a server's entire data volume (`/data`, including worlds, configuration, plugins and mods) to the cluster's archive store, with rollback to any backup point
* Daily restore points: a server played that day gets a restore point once it stops; by default 7 are kept for up to 90 days, rotated separately from manual backups
* Download and export: download a single backup (with sha256 verification), delete a single backup, or export a whole world
* Off-site copy (optional): backups are encrypted on the host and synced to S3-compatible storage (AWS S3, Cloudflare R2, Backblaze B2, MinIO and others)
* Control-plane database: the database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migration; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows the age of the latest backup (see [troubleshooting §16](docs/troubleshooting.md))
* **Diagnostics**
* `sudo felis status`: a summary of the node, control plane, game proxy, each server, backups and open alerts
* `sudo felis doctor`: runs all health checks and lists problems by area with troubleshooting pointers; sends no email
* `sudo felis support-bundle`: generates a redacted diagnostics archive to attach to support requests (see [troubleshooting §0](docs/troubleshooting.md))
* Watchdog: runs a check every 2 minutes and emails the platform owners when a problem persists; supports an external heartbeat monitor
* **World Reaper** (optional): Worlds idle for more than 15 days are backed up and then removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is deleted. Expired backups are cleaned up daily regardless of this setting.
* **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, accessed through a single Velocity proxy.
* **Modpack Submission**: Players can upload modpacks. After admin approval, each modpack is built automatically and scanned with Trivy; the result is added to the image whitelist and can be selected as a server image.
* **Security**
* Passkey login: passwordless authentication via fingerprint, face recognition, or hardware security keys
* Zero-trust access: panel traffic is protected by Cloudflare Access, and the internal API is not exposed to the internet
* **Multi-node Deployment** (experimental, off by default): a single controller node issues all commands, the other nodes run game servers only, and a stopped server can be migrated to another node. Currently available only on the main branch; three-node acceptance testing is not yet complete (see [distributed mode](docs/distributed.md), in Chinese).
- **Wake on Join**: Servers start automatically when a player connects, and stop when idle — like hibernate for your server.
- **Web Dashboard**: Monitor server status, online players, and resource usage from your browser, with backup and restore management.
- **Backup & Restore**: One-click snapshots of a server's whole data volume (worlds, config, plugins/mods — the entire /data volume) into the cluster's archive store, with rollback from any backup point — enabled by default (the installer renders the archive PVC and its path).
- **Control-plane database backups**: The database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migrates it; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows whether the newest backup is fresh (see [troubleshooting §16](docs/troubleshooting.md)).
- **World Reaper** (opt in): Worlds idle for more than 15 days are automatically backed up and removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is ever deleted.
- **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, federated behind a Velocity proxy.
- **Modpack Submission**: Players submit custom modpacks; admin approval triggers an automatic build, and the result is whitelisted as a server image you can select to deploy.
- **Passkey Login**: Passwordless authentication via fingerprint, face recognition, or hardware security keys.
- **Zero Trust Security**: Panel traffic protected by Cloudflare Access; the internal API is never exposed to the internet.
## Getting Started
For development testing on a prepared Linux test host, explicitly select the `dev` channel, which follows `main` and builds from source:
On a prepared Linux host, run:
```bash
curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo FELIS_VERSION_BOOTSTRAP=dev bash
curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
```
The script installs K3s, deploys PostgreSQL and the control plane inside it, and launches a setup wizard. When setup completes, open the configured domain in a browser to reach the control panel.
The script installs K3s, deploys the control plane, and launches a setup wizard. Once done, open your browser at the configured domain.
* **Setup wizard**: Configure panel access and storage first. The host administrator then initializes the first Owner and receives a one-time browser setup link (valid for 30 minutes). Open it, record an email, and create a passkey to enter the panel; Minecraft is not required. Later, link a game role from Account by selecting an authentication source, entering a role name or UUID, and confirming it. Players retain the in-game bind-code flow. Rerun `sudo felis setup` if login setup is unfinished or the link expires; accounts with an established login factor are never reset. Login/lobby failures do not block panel initialization. Account explains Yggdrasil authentication and shows the join address and Java client version for the login/lobby servers. Bootstrap records the built protocol in `[velocity].game_version`; set it yourself for custom images, otherwise the panel reports it as unknown. Standard Yggdrasil endpoints support role lookup directly; sources with a nonstandard `hasJoined` path can set `[[auth_source]].api_url` to their API root, with game-code linking still available as a fallback. The installer launches the wizard automatically only on an interactive terminal; when output is redirected to a log or the install runs under cloud-init, run `sudo felis setup` after it finishes. Setting `FELIS_NO_SETUP=1` makes the installer end at its summary.
> **This repository is currently private**, so the command above returns 404. Use the
> credentialed form instead; the installer itself needs the same token to resolve and
> download the release, so pass it through with `sudo -E`:
>
> ```bash
> export FELIS_GITHUB_TOKEN=<a token with read access to this repository>
> printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \
> | curl -fsSL --config - -H "Accept: application/vnd.github.raw" \
> https://api.github.com/repos/FelisMC/Felis/contents/deploy/bootstrap.sh \
> | sudo -E bash
> ```
>
> The token reaches `curl --config -` over stdin instead of the command line: argv is
> readable by any local user via `/proc`, and that is exactly why the installer's
> internal `github_api` uses the same form.
* **Supported hosts**: CentOS Stream 9 (aarch64) is verified on physical hardware; CI tests fresh installation and repeat installation of development builds on Ubuntu 24.04 (x86_64) on every push (see [operations §1](docs/operations.md#1-supported-hosts)).
* **Preflight checks**: Before modifying the host, the installer checks memory, disk, ports, network range conflicts, existing Kubernetes installations and outbound connectivity. If any check fails, it lists all problems and exits, leaving the host unchanged (see [operations §1](docs/operations.md#1-supported-hosts) for the checks).
* **Upgrading**: Rerun the install command to upgrade felis-api to a newer version; `felis setup` only uses the binary already installed on the host and cannot upgrade it. A rerun keeps the installed root domain, and the release channel must be specified again: hosts that follow the main branch must also set `export FELIS_VERSION_BOOTSTRAP=dev`. A PostgreSQL instance installed on the host by an earlier release is migrated into K3s during the rerun; the original instance on the host is stopped and retained for rollback (see [operations §4](docs/operations.md#4-upgrading-the-pieces-around-felis)).
<details>
<summary>Installation sources and restricted networks</summary>
<br>
The following describes the release mechanism. No public release assets are currently available; development testing uses source builds.
A release installation takes the binary, all images and the Velocity plugin from the release assets prebuilt in CI, verifying each against `SHA256SUMS` before import. The host requires no Docker, Gradle or Go, and no access to Docker Hub. If an asset is missing or fails verification, only that image falls back to a local build, and the installer prints a notice (see [troubleshooting §15c](docs/troubleshooting.md)).
The assets can also be copied to the host in advance and installed with `FELIS_ARTIFACT_DIR=<absolute path>`; the Felis binary, images and plugin are then read from that directory. k3s and its images, the JRE, cloudflared, Velocity and the Via plugins are still downloaded from GitHub and PaperMC; hosts with SELinux enabled, such as RHEL, Fedora and openSUSE Leap, additionally install k3s-selinux from rpm.rancher.io; system packages come from the distribution's repositories.
A host with restricted outbound access must therefore allow HTTPS to these addresses or set `https_proxy`. Preflight probes each address before changing the host. Fully offline installation is not yet supported (see [operations §1](docs/operations.md#1-supported-hosts) for the address list).
</details>
Rerunning this command is also how you upgrade felis-api to a newer version (`felis setup`
cannot — it uses the binary already installed on the host). The rerun keeps the installed
root domain but **not** the channel: if this host follows main, also
`export FELIS_VERSION_BOOTSTRAP=dev`.
## Build from Source
@@ -113,22 +75,22 @@ docker build -t felis:custom .
## License
This project is licensed under [AGPL-3.0-only](LICENSE).
The source code is released under [AGPL-3.0-only](LICENSE).
### License Notes
1. **Derivative works must use AGPL**: Any distribution of this project or of software derived from it must be released under AGPL-3.0 and must include the original copyright notice and license statement.
2. **Network services must also provide source** (AGPL section 13): anyone who offers a modified Felis to others over a network must provide those users with the complete source of the modified version, even without distributing any binary. This is the only substantive difference between AGPL and GPL; as Felis is a hosting platform accessed over a network, this clause applies to virtually every deployment.
1. **Derivative works are AGPL too**: Any distribution of this project or of software derived from it must be released under AGPL-3.0 and must include the original copyright notice and license statement.
2. **Running it as a network service also triggers the source obligation** (AGPL section 13): if you host a modified Felis for other people to use, you must offer those users the complete source of your modified version — even if you never distribute a binary. This is the one substantive difference between AGPL and GPL, and since Felis is a hosting platform reached over a network, it will essentially always apply.
3. **Disclaimer**: This project is provided "as is", without warranty of any kind.
## Acknowledgements
* [Kubernetes](https://kubernetes.io/): Container orchestration engine
* [K3s](https://k3s.io/): Lightweight Kubernetes distribution
* [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/): Zero trust security infrastructure
* [PostgreSQL](https://www.postgresql.org/): Data persistence
* [React](https://react.dev/): User interface framework
* [Vite](https://vitejs.dev/): Frontend build tool
* [TailwindCSS](https://tailwindcss.com/): CSS framework
* [Bubble Tea](https://github.com/charmbracelet/bubbletea): TUI framework
* [Minecraft](https://www.minecraft.net/): The game this project serves
- [Kubernetes](https://kubernetes.io/): Container orchestration engine
- [K3s](https://k3s.io/): Lightweight Kubernetes distribution
- [Cloudflare Zero Trust](https://www.cloudflare.com/zero-trust/): Zero trust security infrastructure
- [PostgreSQL](https://www.postgresql.org/): Data persistence
- [React](https://react.dev/): User interface framework
- [Vite](https://vitejs.dev/): Frontend build tool
- [TailwindCSS](https://tailwindcss.com/): CSS framework
- [Bubble Tea](https://github.com/charmbracelet/bubbletea): TUI framework
- [Minecraft](https://www.minecraft.net/): What makes this all worthwhile
bootstrap_asset.go
+5 -7
View File
@@ -22,18 +22,16 @@ var bootstrapAssets embed.FS
// developer's working tree carries gradle output (plugins/*/build, plugins/*/bin,
// and for the modded loaders a decompiled Minecraft under build/) which would
// otherwise be baked into every felis binary. Keep them explicit — add a source
// directory here, never a parent. Each module's gradle/verification-metadata.xml rides
// along, since Gradle builds unverified without it; the wrapper stays out, because the
// image builds run the pinned build image's own gradle.
// directory here, never a parent.
//
//go:embed deploy/game-stack.lock
//go:embed deploy/limbo/Dockerfile deploy/limbo/entrypoint.sh
//go:embed deploy/lobby/Dockerfile deploy/lobby/entrypoint.sh
//go:embed deploy/paper/Dockerfile deploy/paper/entrypoint.sh
//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src plugins/limbo/gradle/verification-metadata.xml
//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src plugins/paper/gradle/verification-metadata.xml
//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src plugins/velocity/gradle/verification-metadata.xml
//go:embed plugins/shared/src plugins/shared/build-progress.gradle
//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src
//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src
//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src
//go:embed plugins/shared/src
var gameStackAssets embed.FS
// GameStackTar streams the embedded game-stack sources as a tar, rooted so that
bootstrap_asset_test.go
+11 -201
View File
@@ -1,7 +1,6 @@
package felis
import (
"encoding/xml"
"io/fs"
"os"
"regexp"
@@ -147,9 +146,6 @@ func TestBootstrapPinsViaBlockConnectionsOff(t *testing.T) {
// inputs from the script and from each Dockerfile's own COPY lines instead of restating
// them here; a fourth image inherits the check for free.
func TestGameStackTarCarriesEveryBuildInput(t *testing.T) {
// The plugin builds invoke this shared init script after COPYing the directory;
// a directory entry alone would pass the COPY check even if the script was omitted.
requireEmbedded(t, "plugins/shared/build-progress.gradle")
// Matches the path only when GAME_STACK_DIR is followed by one, which skips the
// build-context arguments (`"$GAME_STACK_DIR"`, `"${GAME_STACK_DIR}:/src:z"`) and
// the glob for gradle's output, none of which are inputs this tar has to carry.
@@ -214,7 +210,17 @@ func requireEmbedded(t *testing.T, path string) {
// with a strict KEY=value parser that dies on anything unexpected, so a malformed lock is a
// failed install on every host. Check the shipped copy the same way here.
func TestGameStackLockIsComplete(t *testing.T) {
lock := gameStackLock(t)
lock := map[string]string{}
for line := range strings.SplitSeq(readGameStackFile(t, "deploy/game-stack.lock"), "\n") {
if line == "" || strings.HasPrefix(line, "#") {
continue
}
k, v, ok := strings.Cut(line, "=")
if !ok {
t.Fatalf("not a KEY=value line: %q", line)
}
lock[k] = v
}
m := regexp.MustCompile(`GAME_STACK_LOCK_KEYS="([^"]*)"`).FindStringSubmatch(BootstrapScript())
if m == nil {
t.Fatal("bootstrap.sh no longer declares GAME_STACK_LOCK_KEYS")
@@ -307,202 +313,6 @@ func TestDockerfileBaseImagesArePinnedByDigest(t *testing.T) {
}
}
// The plugin jars are built in four places the installer controls — the lobby and limbo
// image builds, bootstrap's Velocity build and the release build of the same jar — and
// through each module's wrapper by a developer or CI. A tag alone is whatever it points
// at on build day, and two Gradle versions are two chances for a build to pass in one
// place and break in the other, so all of them run one image, pinned by digest, whose
// Gradle is the wrappers' Gradle.
func TestPluginBuildsRunOnePinnedGradle(t *testing.T) {
sources := map[string]string{
"deploy/lobby/Dockerfile": readGameStackFile(t, "deploy/lobby/Dockerfile"),
"deploy/limbo/Dockerfile": readGameStackFile(t, "deploy/limbo/Dockerfile"),
"deploy/bootstrap.sh": BootstrapScript(),
// The release build compiles felis-velocity.jar for hosts that install prebuilt.
"deploy/build-release-artifacts.sh": readRepoFile(t, "deploy/build-release-artifacts.sh"),
}
anyRef := regexp.MustCompile(`gradle:[\w.-]+(@sha256:\w+)?`)
pinned := regexp.MustCompile(`^gradle:(\d+\.\d+(?:\.\d+)?)-jdk\d+@sha256:[0-9a-f]{64}$`)
images := map[string]bool{}
gradle := ""
for name, body := range sources {
refs := anyRef.FindAllString(body, -1)
if len(refs) == 0 {
t.Errorf("%s names no gradle image", name)
}
for _, ref := range refs {
m := pinned.FindStringSubmatch(ref)
if m == nil {
t.Errorf("%s: %s is not a gradle image pinned by digest", name, ref)
continue
}
images[ref] = true
gradle = m[1]
}
}
if len(images) != 1 {
t.Fatalf("the plugin builds use %d different gradle images, want one: %v", len(images), images)
}
// bootstrap names the image once and has to spend it where it builds the jar.
if !strings.Contains(BootstrapScript(), `"$PLUGIN_BUILD_IMAGE" gradle --no-daemon clean build`) {
t.Error("build_velocity_plugin does not build in $PLUGIN_BUILD_IMAGE")
}
for _, module := range []string{"velocity", "paper", "limbo"} {
props := wrapperProperties(t, module)
if want := "gradle-" + gradle + "-bin.zip"; !strings.HasSuffix(props["distributionUrl"], "/"+want) {
t.Errorf("plugins/%s wrapper runs %s; the image builds run Gradle %s", module, props["distributionUrl"], gradle)
}
}
// The mods are no part of the install, but a wrapper without a checksum runs whatever
// the download handed it.
for _, module := range []string{"velocity", "paper", "limbo", "fabric", "forge", "neoforge"} {
if sum := wrapperProperties(t, module)["distributionSha256Sum"]; !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(sum) {
t.Errorf("plugins/%s wrapper pins no distribution sha256 (got %q)", module, sum)
}
}
}
// Each plugin compiles against the API of the exact build the install runs, and Gradle
// checks those bytes against the module's verification file. Nothing but this test ties
// the three to deploy/game-stack.lock: a lock refresh that leaves them behind builds the
// lobby against yesterday's API, or fails every image build on a checksum the file does
// not have.
func TestPluginApisAreTheLockedBuilds(t *testing.T) {
lock := gameStackLock(t)
// Paper: paper-<mc>-<build>.jar runs; paper-api <mc>.build.<build>-<channel> compiles.
jar := regexp.MustCompile(`/paper-([^/]+)-(\d+)\.jar$`).FindStringSubmatch(lock["PAPER_JAR_URL"])
if jar == nil {
t.Fatalf("PAPER_JAR_URL %s does not name paper-<mc>-<build>.jar", lock["PAPER_JAR_URL"])
}
dep := regexp.MustCompile(`compileOnly 'io\.papermc\.paper:paper-api:([^']+)'`).
FindStringSubmatch(readGameStackFile(t, "plugins/paper/build.gradle"))
if dep == nil {
t.Fatal("plugins/paper/build.gradle declares no paper-api dependency")
}
if !regexp.MustCompile(`^` + regexp.QuoteMeta(jar[1]+".build."+jar[2]) + `(-[a-z]+)?$`).MatchString(dep[1]) {
t.Errorf("paper-api %s is not the API of the locked server paper-%s-%s.jar", dep[1], jar[1], jar[2])
}
requireVerified(t, "paper", "io.papermc.paper", "paper-api", dep[1])
// Limbo: the lock's release, passed to the image build, which refuses to guess one.
limbo := lock["LIMBO_VERSION"]
if !strings.Contains(BootstrapScript(), `--build-arg LIMBO_VERSION="$LIMBO_VERSION"`) {
t.Error("bootstrap.sh does not pass the locked LIMBO_VERSION to the limbo image build")
}
dockerfile := readGameStackFile(t, "deploy/limbo/Dockerfile")
if !regexp.MustCompile(`(?m)^ARG LIMBO_VERSION$`).MatchString(dockerfile) ||
!strings.Contains(dockerfile, `if [ -z "${LIMBO_VERSION:-}" ]`) {
t.Error("deploy/limbo/Dockerfile does not require LIMBO_VERSION; a build without it would " +
"compile against a version nobody chose")
}
// LOOHP publishes the jar the login gate runs as the Limbo API artifact itself, so the
// checksum Gradle holds for it is the lock's: compiled-against and running are one file.
if got := requireVerified(t, "limbo", "com.loohp", "Limbo", limbo)["Limbo-"+limbo+".jar"]; got != lock["LIMBO_JAR_SHA256"] {
t.Errorf("verification-metadata.xml holds %q for Limbo-%s.jar; the login gate runs %s", got, limbo, lock["LIMBO_JAR_SHA256"])
}
// Velocity: the API default is the proxy the install runs.
api := regexp.MustCompile(`findProperty\('velocityApi'\) \?: '([^']+)'`).
FindStringSubmatch(readGameStackFile(t, "plugins/velocity/build.gradle"))
if api == nil {
t.Fatal("plugins/velocity/build.gradle has no velocityApi default")
}
if api[1] != lock["VELOCITY_VERSION"] {
t.Errorf("velocity-api defaults to %s; the install runs Velocity %s", api[1], lock["VELOCITY_VERSION"])
}
requireVerified(t, "velocity", "com.velocitypowered", "velocity-api", api[1])
}
// requireVerified asserts the module's shipped verification file checks metadata and
// pins group:name:version, and returns that component's artifact sha256s by file name.
func requireVerified(t *testing.T, module, group, name, version string) map[string]string {
t.Helper()
path := "plugins/" + module + "/gradle/verification-metadata.xml"
var doc struct {
VerifyMetadata bool `xml:"configuration>verify-metadata"`
Components []struct {
Group string `xml:"group,attr"`
Name string `xml:"name,attr"`
Version string `xml:"version,attr"`
Artifacts []struct {
Name string `xml:"name,attr"`
SHA256 []struct {
Value string `xml:"value,attr"`
} `xml:"sha256"`
} `xml:"artifact"`
} `xml:"components>component"`
}
if err := xml.Unmarshal([]byte(readGameStackFile(t, path)), &doc); err != nil {
t.Fatalf("%s: %v", path, err)
}
if !doc.VerifyMetadata {
t.Errorf("%s does not verify metadata; a swapped pom could redirect the graph", path)
}
for _, c := range doc.Components {
if c.Group != group || c.Name != name || c.Version != version {
continue
}
sums := map[string]string{}
for _, a := range c.Artifacts {
if len(a.SHA256) > 0 {
sums[a.Name] = a.SHA256[0].Value
}
}
if sums[name+"-"+version+".jar"] == "" {
t.Errorf("%s pins %s:%s:%s but no sha256 for its jar", path, group, name, version)
}
return sums
}
t.Errorf("%s has no checksum for %s:%s:%s; the build would refuse it", path, group, name, version)
return nil
}
// wrapperProperties reads a module's gradle-wrapper.properties off disk: the wrappers are
// for developers and CI, and nothing embeds them.
func wrapperProperties(t *testing.T, module string) map[string]string {
t.Helper()
b, err := os.ReadFile("plugins/" + module + "/gradle/wrapper/gradle-wrapper.properties")
if err != nil {
t.Fatal(err)
}
props := map[string]string{}
for line := range strings.SplitSeq(string(b), "\n") {
if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok && !strings.HasPrefix(k, "#") {
props[k] = strings.ReplaceAll(v, `\:`, ":")
}
}
return props
}
// gameStackLock parses the shipped deploy/game-stack.lock the way bootstrap.sh does.
func gameStackLock(t *testing.T) map[string]string {
t.Helper()
lock := map[string]string{}
for line := range strings.SplitSeq(readGameStackFile(t, "deploy/game-stack.lock"), "\n") {
if line == "" || strings.HasPrefix(line, "#") {
continue
}
k, v, ok := strings.Cut(line, "=")
if !ok {
t.Fatalf("not a KEY=value line: %q", line)
}
lock[k] = v
}
return lock
}
// readRepoFile reads a file of the checkout that the binary does not embed.
func readRepoFile(t *testing.T, name string) string {
t.Helper()
b, err := os.ReadFile(name)
if err != nil {
t.Fatal(err)
}
return string(b)
}
func readGameStackFile(t *testing.T, name string) string {
t.Helper()
b, err := gameStackAssets.ReadFile(name)
cmd/felis/api.go
+72 -462
View File
@@ -8,10 +8,8 @@ import (
"log/slog"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
@@ -20,30 +18,24 @@ import (
"felis.lolicon.best/internal/backupjob"
"felis.lolicon.best/internal/build"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/distributed"
"felis.lolicon.best/internal/fileedit"
"felis.lolicon.best/internal/imagepin"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/metrics"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/nodecontrol"
"felis.lolicon.best/internal/panel"
"felis.lolicon.best/internal/passkey"
"felis.lolicon.best/internal/placement"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/reaper"
"felis.lolicon.best/internal/registryprune"
"felis.lolicon.best/internal/restore"
"felis.lolicon.best/internal/retention"
"felis.lolicon.best/internal/store"
"felis.lolicon.best/internal/submit"
"felis.lolicon.best/internal/worldexport"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -53,22 +45,6 @@ import (
// the code marks Identity (UUIDs trusted verbatim); config can never add another.
const mojangSessionServer = "https://sessionserver.mojang.com/session/minecraft/hasJoined"
// passkeyRelyingParty is the one WebAuthn relying party both web faces share: its id
// is the player console host, derived from server.root_domain the way the panel
// handler derives it when auth.panel_hostname is unset, and its origins are that host
// plus the operator host. An empty id means the install names no panel host at all.
func passkeyRelyingParty(cfg *config.Config) (string, []string) {
rpID := defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname)
if rpID == "" {
return "", nil
}
origins := []string{"https://" + rpID, fmt.Sprintf("https://%s:%d", rpID, setupPanelNodePort())}
if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != rpID {
origins = append(origins, "https://"+admin, fmt.Sprintf("https://%s:%d", admin, setupPanelNodePort()))
}
return rpID, origins
}
// authSourcesFromConfig builds the multiplexer's priority list from the configured
// [[auth_source]] entries: Mojang leads as the code-owned identity anchor (正版优先, the ONLY
// Identity source — config can only append namespace-rewritten third-party sources, never a
@@ -78,14 +54,16 @@ func authSourcesFromConfig(configured []config.AuthSourceConfig) []api.AuthSourc
sources := make([]api.AuthSource, 0, len(configured)+1)
sources = append(sources, api.AuthSource{Tag: "mojang", URL: mojangSessionServer, Identity: true})
for _, s := range configured {
sources = append(sources, api.AuthSource{Tag: s.Tag, Prefix: s.Prefix, URL: s.URL, APIURL: s.APIURL})
sources = append(sources, api.AuthSource{Tag: s.Tag, Prefix: s.Prefix, URL: s.URL})
}
return sources
}
// cmdAPI runs felis-api: two listeners, two middleware chains (spec §7). The
// internal face authenticates per-caller service tokens; the external face
// authenticates the local session cookie.
// internal face (service token) is fully wired. The external face is wired but
// fails closed until an Access JWKS key function is configured — the verifier's
// audience logic is unit-tested (internal/api), the JWKS source is a deployment
// integration point.
func cmdAPI(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("api", flag.ContinueOnError)
fs.SetOutput(stderr)
@@ -108,19 +86,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
return 1
}
// Load already refused a malformed [audit] retention.
auditRetention, _ := cfg.Audit.RetentionPeriod()
if auditRetention == 0 {
fmt.Fprintln(stdout, "felis api: audit rows are kept forever ([audit] retention = \"forever\")")
} else {
fmt.Fprintf(stdout, "felis api: audit rows older than %d days are deleted ([audit] retention; export them first with felis db audit-export)\n", int(auditRetention/(24*time.Hour)))
}
ctx := ctrl.SetupSignalHandler()
// Before anything serves: an api on a schema it was not built for answers with
// errors, or writes rows the other version cannot read.
drv, err := openPodStore(ctx, cfg.Database.URL, "api", stderr)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
fmt.Fprintf(stderr, "felis api: open database: %v\n", err)
return 1
@@ -149,22 +117,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
metrics.SetBuildInfo("api", resolvedVersion())
internalAuth, err := internalCallerTokens(os.Getenv)
if err != nil {
fmt.Fprintf(stderr, "felis api: internal face tokens: %v\n", err)
return 1
}
for _, ct := range naming.CallerTokens {
if internalAuth[api.Caller(ct.Caller)] == "" {
fmt.Fprintf(stderr, "felis api: warning: %s unset — the internal face turns the %s caller away\n", ct.APIEnv, ct.Caller)
}
token := os.Getenv("FELIS_SERVICE_TOKEN")
if token == "" {
fmt.Fprintln(stderr, "felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers")
}
// Email one-time codes go through the [smtp] relay when one is configured; the
// password is read from the env var password_ref names (default SMTPPasswordEnv,
// injected from the felis-smtp Secret). No [smtp] host ⇒ mailer stays nil and
// every door that mails a code answers 503 mail_unavailable: a code that is
// not mailed is never written anywhere else either.
// deliverOTP logs each code server-side (the pre-SMTP bootstrap posture).
var mailer api.OTPMailer
if cfg.SMTP.Host != "" {
passRef := cfg.SMTP.PasswordRef
@@ -175,12 +136,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
if cfg.SMTP.Username != "" && password == "" {
fmt.Fprintf(stderr, "felis api: warning: [smtp] username is set but credentials env %s is empty — OTP sends will fail AUTH\n", passRef)
}
mailer = smtpRelay(cfg.SMTP, password)
if !cfg.SMTP.TLSRequired() {
fmt.Fprintf(stderr, "felis api: warning: [smtp] %s may be sent codes without TLS (require_tls off or a relay on this host)\n", cfg.SMTP.Host)
mailer = &mail.SMTP{
Host: cfg.SMTP.Host,
Port: cfg.SMTP.Port,
From: cfg.SMTP.From,
Username: cfg.SMTP.Username,
Password: password,
}
} else {
fmt.Fprintln(stderr, "felis api: [smtp] not configured — email sign-in and verification are off (503 mail_unavailable); sign in with a passkey, or run felis setup to add a relay")
fmt.Fprintln(stderr, "felis api: [smtp] not configured — email one-time codes are logged, not mailed")
}
// Build subsystem (spec §16): the weak-SA build Job runs in the configured
@@ -193,10 +157,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
buildCfg.FelisImage = os.Getenv("FELIS_IMAGE")
buildJobs := build.NewK8sJobs(cl, buildCfg)
builder := &build.Builder{
Store: build.NewPGStore(drv.DB()),
Jobs: buildJobs,
Config: buildCfg,
Outcomes: build.NewK8sOutcomes(clientset, buildCfg),
Store: build.NewPGStore(drv.DB()),
Jobs: buildJobs,
Config: buildCfg,
}
go probeBuildUserNamespaces(ctx, buildJobs, buildCfg, stderr)
@@ -250,9 +213,6 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
ContextBaseURL: internalAPIBaseURL(),
Blobs: blobs,
}
if blobs != nil {
submissions.Parts = &submit.PartStore{Dir: uploadPartsDir(contextBase)}
}
if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
if n, err := parseByteSize(v); err != nil || n <= 0 {
fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
@@ -260,11 +220,6 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
submissions.MaxStoredBytesTotal = n
}
}
if n, err := contextMaxBytes(cfg); err != nil {
fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 512Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
} else {
submissions.MaxContextBytes = n
}
// Restore subsystem (spec §7): the weak-SA restore Job mounts the target
// world PVC + the backup PVC and runs `felis restore`. It needs deployment-
@@ -276,23 +231,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// store at load, so by this point cfg.Archive.Store is guaranteed tarLocal.)
var restorer api.Restorer
felisImage, backupPVC := os.Getenv("FELIS_IMAGE"), os.Getenv("FELIS_BACKUP_PVC")
worldResolver := placement.Resolve(cl, cfg.K8s.Namespace)
distribution, err := distributionManager(cl, cfg, felisImage)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if distribution != nil {
worldResolver = distribution.Resolve
}
if felisImage != "" && backupPVC != "" {
rcfg := restoreConfig(cfg, felisImage, backupPVC)
rcfg.ResolveWorld = worldResolver
var jobs restore.Jobs = restore.NewK8sJobs(cl)
if distribution != nil {
jobs = distribution
}
restorer = &restore.Restorer{Jobs: jobs, Config: rcfg}
restorer = &restore.Restorer{Jobs: restore.NewK8sJobs(cl), Config: rcfg}
} else {
fmt.Fprintln(stderr, "felis api: restore executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — restore endpoint returns 503")
}
@@ -305,30 +246,11 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// endpoint honestly returns 503.
var backuper api.Backuper
if felisImage != "" && backupPVC != "" {
bcfg := backupConfig(cfg, felisImage, backupPVC)
bcfg.ResolveWorld = worldResolver
var jobs backupjob.Jobs = backupjob.NewK8sJobs(cl)
if distribution != nil {
jobs = distribution
}
backuper = &backupjob.Backuper{Jobs: jobs, Config: bcfg}
backuper = &backupjob.Backuper{Jobs: backupjob.NewK8sJobs(cl), Config: backupConfig(cfg, felisImage, backupPVC)}
} else {
fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503")
}
// World export: a weak-SA Job mounts the world PVC, or the backup PVC, read-only
// and PUTs the archive to the internal face, which streams it on to the owner's
// browser (internal/worldexport). A backup export mounts the backup PVC, so it
// is wired under the restore gate; otherwise the export routes return 503.
var exporter api.Exporter
if felisImage != "" && backupPVC != "" {
ecfg := exportConfig(cfg, felisImage, backupPVC)
ecfg.ResolveWorld = worldResolver
exporter = worldexport.New(clientset, ecfg)
} else {
fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503")
}
// Server file editor: a weak-SA Job mounts ONLY the target world PVC and runs
// `felis files`, printing its result for felis-api to read back through
// pods/log (see internal/fileedit). It needs FELIS_IMAGE but — unlike restore
@@ -337,27 +259,11 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// endpoints honestly return 503. It takes the typed clientset rather than the
// controller-runtime client because the log subresource lives only on the typed
// CoreV1 client, and one client covers its Job create, Pod list, and log read.
//
// Uploads additionally stage their bytes on this pod's disk until the Job
// fetches them from the internal face; whatever a previous process staged is
// orphaned (the index is in memory), so the stage starts empty.
var files api.FileEditor
var fileStage *fileedit.Stage
var fileBrowser *fileedit.Browser
if felisImage != "" {
fcfg := fileEditConfig(cfg, felisImage)
fcfg.ResolveWorld = worldResolver
fileBrowser = &fileedit.Browser{BaseURL: internalAPIBaseURL()}
runner := fileedit.NewK8sRunner(clientset)
runner.Browser = fileBrowser
files = &fileedit.Editor{
Runner: runner,
Config: fcfg,
}
fileStage = &fileedit.Stage{Dir: fileStagingDir()}
if err := fileStage.Sweep(); err != nil {
fmt.Fprintf(stderr, "felis api: %v — file uploads return 503\n", err)
fileStage = nil
Runner: fileedit.NewK8sRunner(clientset),
Config: fileEditConfig(cfg, felisImage),
}
} else {
fmt.Fprintln(stderr, "felis api: file editor disabled (needs FELIS_IMAGE) — file endpoints return 503")
@@ -368,9 +274,6 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// the same store the auth handlers write to, so a login and the next request
// agree on what local auth knows.
repo := api.NewPGRepo(drv.DB())
if err := api.RegisterStorePool(drv.DB()); err != nil {
fmt.Fprintf(stderr, "felis api: store pool metrics unavailable: %v\n", err)
}
// The owner's on-demand backup levers come from [archive], the same keys the
// backup Job and the reaper read. A malformed key leaves the defaults in
@@ -381,27 +284,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
rcfg = reaper.DefaultConfig()
}
serverCache, serversSynced, err := startServerCache(ctx, restCfg, scheme, cfg.K8s.Namespace, stderr)
if err != nil {
fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err)
return 1
}
cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced).WithDistributed(distribution != nil, os.Getenv("FELIS_CONTROLLER_NODE"))
if distribution != nil {
distribution.Record = func(ctx context.Context, b distributed.Backup) error {
keep, retention, ok := backupPolicy(b.Reason, rcfg)
if !ok {
return fmt.Errorf("unknown backup reason %s", b.Reason)
}
st := reaper.NewPGStore(drv.DB())
if err := st.InsertBackup(ctx, reaper.BackupRecord{ID: "bk-" + b.ID, ServerName: b.Server, FormerOwner: b.Owner, BackupRef: b.Receipt.Ref, SizeBytes: b.Receipt.Size, SHA256: b.Receipt.SHA256, Reason: b.Reason, ExpiresAt: time.Now().Add(retention)}); err != nil {
return err
}
pruneBackups(ctx, st, distribution.Archive, b.Server, b.Owner, b.Reason, keep, b.Protect, stdout, stderr)
return nil
}
}
jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace)
cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace)
a := &api.API{
Repo: repo,
Cluster: cluster,
@@ -409,32 +292,26 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
Logs: api.NewK8sLogStreamer(clientset, cfg.K8s.Namespace),
// Build-log stream (spec §16) is scoped to the BUILD namespace — the same
// value the Builder renders Jobs into — so it follows where build Pods run.
BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
Internal: internalAuth,
Builder: builder,
Images: imagePinner(cfg.Registry.URL),
Restorer: restorer,
Backuper: backuper,
JobStatus: jobStatus,
// A restore starts with a safety snapshot; settleRestoreChains starts the
// restore behind each one.
RestoreChains: jobStatus,
Files: files,
FileStage: fileStage,
FileBrowser: fileBrowser,
// The file Job fetches an upload from here; it runs in the minecraft
// namespace, where the internal face is reachable like it is for the login
// gate.
InternalBaseURL: internalAPIBaseURL(),
Exporter: exporter,
Submissions: submissions,
Mailer: mailer,
Schedules: repo,
// The external face authenticates the local session cookie the sign-in doors
// mint, live once `felis breakGlass` flips local_auth_enabled on. Cloudflare
// Access, when the install sits behind it, is enforced at the edge only.
BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
Internal: api.BearerTokenAuth{Token: token},
Builder: builder,
Images: imagePinner(cfg.Registry.URL),
Restorer: restorer,
Backuper: backuper,
JobStatus: api.NewK8sJobStatus(cl, cfg.K8s.Namespace),
Files: files,
Submissions: submissions,
Mailer: mailer,
// The external face is fronted by SessionAuth: it prefers a local session
// cookie (minted by the passwordless doors) and otherwise delegates to the
// Cloudflare-Access JWT verifier, so both auth models coexist on one face. The
// delegate's Keyfunc is intentionally nil — the JWT path fails closed until a
// JWKS-backed key function is wired (deployment integration point) — while the
// local session path is live the moment `felis breakGlass` flips
// local_auth_enabled on.
External: api.SessionAuth{
Repo: repo,
Delegate: api.AccessVerifier{Audience: cfg.Auth.AccessJWTAud},
RootDomain: cfg.Server.RootDomain,
AdminHostname: cfg.Auth.AdminHostname,
},
@@ -463,6 +340,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(),
MailLimit: mailLimit(cfg.SMTP.MaxPerHour),
}
fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)")
if a.ClientIPHeader != "" {
fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader)
} else {
@@ -476,8 +354,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// [[auth_source]] is configured, so an empty list has to mean a Mojang-only relay, the
// same as under `felis nano`. A nil list would 204 every login, premium ones included.
a.AuthSources = authSourcesFromConfig(cfg.AuthSources)
a.AuthSourceSettings = &api.AuthSourceSettings{Repo: repo, Defaults: a.AuthSources}
fmt.Fprintf(stderr, "felis api: hasJoined multiplexer active — Mojang + %d default third-party source(s); panel settings take precedence\n", len(cfg.AuthSources))
fmt.Fprintf(stderr, "felis api: hasJoined multiplexer active — Mojang + %d third-party source(s)\n", len(cfg.AuthSources))
// Passkey (WebAuthn) verifier (spec §14, Phase 6). One relying party spans BOTH
// web faces: the RP id is the panel hostname (console.<root>), and because that is
@@ -485,15 +362,22 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// enrolled once asserts on either face — one binding, usable on the player console
// AND the operator console. Both hosts are therefore listed as permitted origins,
// while the RP id stays the panel host so the credential's scope is ONE relying
// party, not two. Without a panel host (neither auth.panel_hostname nor
// server.root_domain) a.Passkey stays nil and the passkey routes honestly return
// 503 (the authenticated enrollment boundary is still enforced by the handlers).
if rpID, origins := passkeyRelyingParty(cfg); rpID == "" {
fmt.Fprintln(stderr, "felis api: passkey verifier disabled (no panel host: set server.root_domain or auth.panel_hostname) — passkey endpoints return 503")
} else if pv, err := passkey.New(rpID, "Felis", origins); err != nil {
fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
// party, not two. Wired only when auth.panel_hostname is configured; otherwise
// a.Passkey stays nil and the passkey routes honestly return 503 (the authenticated
// enrollment boundary is still enforced by the handlers).
if cfg.Auth.PanelHostname != "" {
origins := []string{"https://" + cfg.Auth.PanelHostname}
if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != cfg.Auth.PanelHostname {
origins = append(origins, "https://"+admin)
}
pv, err := passkey.New(cfg.Auth.PanelHostname, "Felis", origins)
if err != nil {
fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
} else {
a.Passkey = pv
}
} else {
a.Passkey = pv
fmt.Fprintln(stderr, "felis api: passkey verifier disabled (auth.panel_hostname unset) — passkey endpoints return 503")
}
// Derive the console hostnames when felis.toml leaves them unset, exactly as the
@@ -502,7 +386,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain,
defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname),
defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname),
cfg.Velocity.GamePort, cfg.Velocity.GameVersion, resolvedVersion(), distribution != nil)
resolvedVersion())
internalSrv := newAPIServer(*internalAddr, a.InternalHandler())
externalSrv := newAPIServer(cfg.Server.Listen, externalHandler)
@@ -524,47 +408,21 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// and advance any whose Job has reached a terminal phase. GET on a build also
// reconciles it, but this loop converges builds nobody is polling.
go reconcileBuilds(ctx, builder, stderr)
go settleRestoreChains(ctx, a, stderr)
go runSchedules(ctx, a, stderr)
// A daily restore point of every world played since its last one, taken
// once the server stops ([archive] scheduled_every; 0s turns it off).
if backuper != nil && rcfg.ScheduledEvery > 0 {
go scheduleBackups(ctx, &api.BackupScheduler{API: a, Store: repo, Jobs: jobStatus, Every: rcfg.ScheduledEvery}, stderr)
} else {
fmt.Fprintln(stderr, "felis api: scheduled backups off (needs the backup executor and [archive] scheduled_every above 0s)")
}
if pruner := registryPruner(cfg, builder.Store, cluster, stderr); pruner != nil {
go pruner.Loop(ctx, registryPruneInterval)
}
go reapRejectedContexts(ctx, submissions, stderr)
if fileStage != nil {
go expireFileSessions(ctx, fileStage, fileSessionSweep, stderr)
}
if exporter != nil {
go expireExports(ctx, a, exportSweep)
}
go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
if distribution != nil {
a.Distribution = distribution
go reconcileDistribution(ctx, distribution, stderr)
}
if socket := os.Getenv("FELIS_NODE_CONTROL_SOCKET"); socket != "" {
a.NodeControl = nodecontrol.NewClient(socket)
cluster.NodeMaintenanceGuard = api.NodeMaintenanceGuard(a.NodeControl)
}
servers := []*http.Server{internalSrv, externalSrv}
if httpsSrv != nil {
servers = append(servers, httpsSrv)
}
for _, srv := range servers {
srv.RegisterOnShutdown(a.CloseStreams)
}
select {
case <-ctx.Done():
shutdownServers(servers, apiShutdownGrace, stderr)
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = internalSrv.Shutdown(shutdownCtx)
_ = externalSrv.Shutdown(shutdownCtx)
if httpsSrv != nil {
_ = httpsSrv.Shutdown(shutdownCtx)
}
return 0
case err := <-errc:
if err != nil && err != http.ErrServerClosed {
@@ -584,30 +442,8 @@ const (
// apiIdleTimeout caps how long a kept-alive connection may sit idle between
// requests before the server closes it, bounding idle-connection exhaustion.
apiIdleTimeout = 120 * time.Second
// apiShutdownGrace is how long the listeners drain after SIGTERM. The pod gets
// the Kubernetes default of 30s before SIGKILL; this leaves the rest for the
// process to exit.
apiShutdownGrace = 20 * time.Second
)
// shutdownServers drains every listener at once under one deadline: in turn, a
// slow first listener would spend the time the others needed. Log streams end
// through RegisterOnShutdown (API.CloseStreams); what is still running when the
// deadline passes is cut off with the process.
func shutdownServers(servers []*http.Server, grace time.Duration, stderr io.Writer) {
ctx, cancel := context.WithTimeout(context.Background(), grace)
defer cancel()
var wg sync.WaitGroup
for _, srv := range servers {
wg.Go(func() {
if err := srv.Shutdown(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: shutdown %s: %v\n", srv.Addr, err)
}
})
}
wg.Wait()
}
// newAPIServer builds an http.Server with hardened header/idle timeouts (gosec
// G112) shared by all three felis-api listeners (internal, external, https).
// WriteTimeout and ReadTimeout are deliberately LEFT UNSET: the external and https
@@ -645,9 +481,6 @@ func buildConfig(cfg *config.Config) build.Config {
MaxConcurrent: cfg.Registry.MaxConcurrentBuilds,
TrivyDBRepository: cfg.Registry.TrivyDBRepository,
TrivyJavaDBRepository: cfg.Registry.TrivyJavaDBRepository,
ScanFailOn: cfg.Registry.ScanFailOn,
ScanFailUnfixed: cfg.Registry.ScanFailUnfixed,
ScanAccept: cfg.Registry.ScanAccept,
// The submit lane's derived context URLs live here; the fetch step's
// service token goes nowhere else.
ContextOrigin: internalAPIBaseURL(),
@@ -761,17 +594,6 @@ func backupConfig(cfg *config.Config, image, backupPVC string) backupjob.Config
}
}
// exportConfig builds the world export executor's config. BackupRoot mirrors
// restoreConfig: the stored refs are absolute paths under [archive] local_path.
func exportConfig(cfg *config.Config, image, backupPVC string) worldexport.Config {
return worldexport.Config{
Namespace: cfg.K8s.Namespace,
Image: image,
BackupPVC: backupPVC,
BackupRoot: cfg.Archive.LocalPath,
}
}
// fileEditConfig builds the file editor's config from felis.toml plus the
// deployment-supplied image. It is the shortest of the three: the editor mounts
// only the world PVC, so it needs no archive coordinates at all, and everything
@@ -802,112 +624,10 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
}
}
// settleRestoreChains starts the restore behind each safety snapshot that has
// finished (and gives up the one behind a snapshot that failed). The world stays
// locked in between, so the interval is how long a finished snapshot keeps the
// server down before its restore begins.
func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
t := time.NewTicker(5 * time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := a.SettleRestoreChains(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: restore chains: %v\n", err)
}
}
}
}
// runSchedules runs the servers' scheduled tasks (api.API.RunSchedules). The
// interval is how late a task may start, and how often a restart or backup in
// progress checks whether it can take its next step.
func runSchedules(ctx context.Context, a *api.API, stderr io.Writer) {
t := time.NewTicker(15 * time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := a.RunSchedules(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: scheduled tasks: %v\n", err)
}
}
}
}
// scheduleBackups starts the scheduled backups (api.BackupScheduler). Each tick
// starts at most one, so the interval also spaces the worlds that stopped at
// the same time: a world that stops waits at most this long for its point to
// start once the Jobs ahead of it are done.
func scheduleBackups(ctx context.Context, s *api.BackupScheduler, stderr io.Writer) {
t := time.NewTicker(2 * time.Minute)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := s.Tick(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: scheduled backups: %v\n", err)
}
}
}
}
// fileSessionSweep is how often expireFileSessions looks for idle upload
// sessions: small beside fileedit.SessionIdle, so an abandoned one gives its
// room back within minutes of going stale.
const fileSessionSweep = 10 * time.Minute
// expireFileSessions drops the file manager's upload sessions left untouched
// for fileedit.SessionIdle. Each reserved room on the staging disk for its whole
// file when it began, so one abandoned would otherwise hold that room until
// felis-api restarts.
func expireFileSessions(ctx context.Context, s *fileedit.Stage, every time.Duration, stderr io.Writer) {
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if n := s.Expire(); n > 0 {
fmt.Fprintf(stderr, "felis api: dropped %d upload session(s) left idle for %s\n", n, fileedit.SessionIdle)
}
}
}
}
// exportSweep is how often expireExports runs: an export whose Job never
// connected is stopped within a minute of going stale.
const exportSweep = time.Minute
// expireExports runs the export sweep (api.API.ExpireExports) on a ticker. The
// export routes sweep as they are called, and an owner who closed the tab calls
// none; a Job whose Pod never got going would then keep the server from
// starting until the Job's deadline.
func expireExports(ctx context.Context, a interface{ ExpireExports() }, every time.Duration) {
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
a.ExpireExports()
}
}
}
// reapRejectedContexts deletes, once an hour, the uploaded contexts of
// submissions rejected more than submit.RejectedContextRetention ago, and the
// chunked uploads left untouched for submit.StalePartRetention. Without it a
// rejected modpack or an abandoned upload keeps its bytes on the uploads store
// (and against its submitter's budget) until an admin deletes the row.
// submissions rejected more than submit.RejectedContextRetention ago. Without it a
// rejected modpack keeps its bytes on the uploads store (and against its
// submitter's budget) until an admin deletes the row.
func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
t := time.NewTicker(time.Hour)
defer t.Stop()
@@ -919,13 +639,6 @@ func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writ
if n > 0 {
fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
}
n, err = m.ReapStaleParts(submit.StalePartRetention)
if err != nil {
fmt.Fprintf(stderr, "felis api: reap abandoned uploads: %v\n", err)
}
if n > 0 {
fmt.Fprintf(stderr, "felis api: deleted %d abandoned chunked upload(s)\n", n)
}
select {
case <-ctx.Done():
return
@@ -934,11 +647,6 @@ func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writ
}
}
// retentionInterval spaces the runs that delete spent sign-in rows and audit rows
// past [audit] retention. The rows are spent for weeks before they go, so a few
// runs a day keep the tables flat.
const retentionInterval = 6 * time.Hour
// registryPruneInterval spaces the registry pruner's runs. The registry-gc
// sidecar sweeps once a day, so pruning more often only changes which sweep frees
// a layer.
@@ -1039,101 +747,3 @@ func imagePinner(registry string) api.ImagePinner {
}
return imagepin.Resolver{Registry: registry}
}
// internalCallerTokens reads each internal caller's token from the env var the
// Deployment feeds it from (naming.CallerTokens). Two callers sharing a value
// would make the caller ambiguous, so that refuses to start.
func internalCallerTokens(getenv func(string) string) (api.CallerTokens, error) {
tokens := map[api.Caller]string{}
for _, ct := range naming.CallerTokens {
tokens[api.Caller(ct.Caller)] = strings.TrimSpace(getenv(ct.APIEnv))
}
return api.NewCallerTokens(tokens)
}
// smtpRelay is the relay [smtp] names, with the resolved password and the TLS
// posture config.SMTPConfig.TLSRequired picks. felis api, the reaper and the
// watchdog all send through it, so none can drift to a weaker posture.
func smtpRelay(c config.SMTPConfig, password string) *mail.SMTP {
return &mail.SMTP{
Host: c.Host,
Port: c.Port,
From: c.From,
Username: c.Username,
Password: password,
RequireTLS: c.TLSRequired(),
}
}
// startServerCache starts the informer that serves the api's fleet-wide
// MinecraftServer reads (api.K8sCluster.WithServerCache): one watch on the
// namespace instead of a full List per velocity pull, fleet page and wake. It
// caches MinecraftServers only — ReaderFailOnMissingInformer turns any other read
// through it into an error rather than a new informer the api's Role cannot back —
// indexes spec.subdomain for GetBySubdomain, and drops managedFields to keep the
// copy small. It returns without waiting: the reads block until the first list
// lands and /readyz reports not-ready until then.
func startServerCache(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, stderr io.Writer) (cache.Cache, func() bool, error) {
c, err := cache.New(cfg, cache.Options{
Scheme: scheme,
DefaultNamespaces: map[string]cache.Config{namespace: {}},
DefaultTransform: cache.TransformStripManagedFields(),
ReaderFailOnMissingInformer: true,
})
if err != nil {
return nil, nil, err
}
if err := c.IndexField(ctx, &v1alpha1.MinecraftServer{}, api.SubdomainIndex, api.SubdomainOf); err != nil {
return nil, nil, fmt.Errorf("index %s: %w", api.SubdomainIndex, err)
}
inf, err := c.GetInformer(ctx, &v1alpha1.MinecraftServer{})
if err != nil {
return nil, nil, err
}
go func() {
if err := c.Start(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: MinecraftServer cache stopped: %v\n", err)
}
}()
return c, inf.HasSynced, nil
}
// uploadPartsDir is where chunked uploads are staged: beside a local store's
// contexts, so the room check and the budget see one disk and a staged upload
// survives an API restart; for an s3:// store, on the uploads volume the
// platform mounts either way, or the pod's /tmp when run by hand without it.
func uploadPartsDir(contextBase string) string {
if isLocalUploadsPath(contextBase) {
return filepath.Join(strings.TrimPrefix(contextBase, "file://"), ".parts")
}
if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
return filepath.Join(platform.UploadsLocalPath, ".parts")
}
return filepath.Join(os.TempDir(), "felis-upload-parts")
}
// fileStagingDir is where file uploads wait for their Job: on the uploads
// volume, whose capacity is its own, or the pod's /tmp when run by hand without
// it — /tmp is the node's disk, which a burst of uploads should not fill.
func fileStagingDir() string {
if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
return filepath.Join(platform.UploadsLocalPath, ".file-staging")
}
return filepath.Join(os.TempDir(), "felis-file-staging")
}
// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
// package's own default (1 GiB). The Cloudflare edge refuses a single request
// body over 100 MB, which the panel's chunked upload stays under, so the edge
// does not lower the cap.
func contextMaxBytes(cfg *config.Config) (int64, error) {
v := cfg.Registry.ContextMaxBytes
if v == "" {
return 0, nil
}
n, err := parseByteSize(v)
if err != nil || n <= 0 {
return 0, fmt.Errorf("not a positive size: %q", v)
}
return n, nil
}
cmd/felis/api_test.go
+2 -173
View File
@@ -1,71 +1,17 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"slices"
"sync"
"sync/atomic"
"testing"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/build"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/fileedit"
)
// The passkey relying party follows the panel host the SPA is served on: an install
// that names only its root domain still gets passkeys, on console.<root>, with the
// operator host as the second origin; only an install with no panel host goes without.
func TestPasskeyRelyingParty(t *testing.T) {
t.Setenv("FELIS_PANEL_NODEPORT", "")
for _, tc := range []struct {
name string
root, panel, admin string
wantRP string
wantOrigins []string
}{
{"root domain only", "example.net", "", "", "console.example.net",
[]string{"https://console.example.net", "https://op.console.example.net"}},
{"configured hosts", "example.net", " play.example.net ", "ops.example.net", "play.example.net",
[]string{"https://play.example.net", "https://ops.example.net"}},
{"operator host equal to the panel host", "example.net", "console.example.net", "console.example.net",
"console.example.net", []string{"https://console.example.net"}},
{"panel host without a root domain", "", "console.example.org", "", "console.example.org",
[]string{"https://console.example.org"}},
{"no host at all", "", "", "", "", nil},
} {
t.Run(tc.name, func(t *testing.T) {
cfg := &config.Config{}
cfg.Server.RootDomain, cfg.Auth.PanelHostname, cfg.Auth.AdminHostname = tc.root, tc.panel, tc.admin
var wantOrigins []string
for _, origin := range tc.wantOrigins {
wantOrigins = append(wantOrigins, origin, fmt.Sprintf("%s:%d", origin, defaultPanelNodePort))
}
rp, origins := passkeyRelyingParty(cfg)
if rp != tc.wantRP || !slices.Equal(origins, wantOrigins) {
t.Fatalf("relying party = %q %q, want %q %q", rp, origins, tc.wantRP, wantOrigins)
}
})
}
}
func TestPasskeyRelyingPartyIncludesConfiguredNodePort(t *testing.T) {
t.Setenv("FELIS_PANEL_NODEPORT", "30445")
cfg := &config.Config{}
cfg.Server.RootDomain = "example.com"
_, origins := passkeyRelyingParty(cfg)
if !slices.Contains(origins, "https://op.console.example.com:30445") {
t.Fatalf("configured NodePort origin missing: %v", origins)
}
}
// TestAuthSourcesFromConfig pins the one place the hasJoined identity anchor is decided:
// Mojang is prepended in code, first, and is the only source whose UUIDs are trusted as-is.
// The empty case matters on its own — both `felis api` and `felis nano` call this with a
@@ -79,7 +25,7 @@ func TestAuthSourcesFromConfig(t *testing.T) {
{"no configured sources", nil},
{"configured sources", []config.AuthSourceConfig{
{Tag: "littleskin", Prefix: "LS", URL: "https://littleskin.example/hasJoined"},
{Tag: "guild", Prefix: "GD", URL: "https://guild.example/hasJoined", APIURL: "https://guild.example/api"},
{Tag: "guild", Prefix: "GD", URL: "https://guild.example/hasJoined"},
}},
} {
t.Run(tc.name, func(t *testing.T) {
@@ -95,7 +41,7 @@ func TestAuthSourcesFromConfig(t *testing.T) {
if s.Identity {
t.Errorf("configured source %q is marked Identity; only Mojang may be", c.Tag)
}
if s.Tag != c.Tag || s.Prefix != c.Prefix || s.URL != c.URL || s.APIURL != c.APIURL {
if s.Tag != c.Tag || s.Prefix != c.Prefix || s.URL != c.URL {
t.Errorf("source %d = %+v, want %+v in config order", i+1, s, c)
}
}
@@ -151,50 +97,6 @@ func TestNewAPIServerSetsHardenedTimeouts(t *testing.T) {
}
}
// Every listener drains at once, and the shutdown hook (API.CloseStreams in
// cmdAPI) runs on each: two listeners each holding a request that ends when
// the hook fires take one hook's worth of time, well inside the deadline.
func TestShutdownServersDrainsListenersTogether(t *testing.T) {
release := make(chan struct{})
var hooks int32
started := make(chan struct{}, 2)
var servers []*http.Server
for range 2 {
srv := newAPIServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
started <- struct{}{}
<-release
}))
srv.RegisterOnShutdown(func() {
if atomic.AddInt32(&hooks, 1) == 1 {
time.AfterFunc(100*time.Millisecond, func() { close(release) })
}
})
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
srv.Addr = l.Addr().String()
go func() { _ = srv.Serve(l) }()
go func() {
if resp, err := http.Get("http://" + srv.Addr); err == nil {
resp.Body.Close()
}
}()
servers = append(servers, srv)
}
<-started
<-started
begun := time.Now()
shutdownServers(servers, 5*time.Second, io.Discard)
if took := time.Since(begun); took > 2*time.Second {
t.Fatalf("shutdown took %v", took)
}
if got := atomic.LoadInt32(&hooks); got != 2 {
t.Fatalf("shutdown hook ran %d times, want once per listener", got)
}
}
type fakeRefStore struct {
images []build.Image
builds []build.Build
@@ -253,76 +155,3 @@ func TestInUseImageRefsCoversEverySource(t *testing.T) {
t.Fatal("a failing whitelist read produced a reference list")
}
}
// TestExpireFileSessions runs the loop against a stage whose clock the test
// holds: the session idle past fileedit.SessionIdle goes, the one touched since
// stays, and the drop is said once.
func TestExpireFileSessions(t *testing.T) {
var mu sync.Mutex
now := time.Date(2026, 9, 28, 10, 0, 0, 0, time.UTC)
advance := func(d time.Duration) { mu.Lock(); now = now.Add(d); mu.Unlock() }
st := &fileedit.Stage{Dir: t.TempDir(), MinFree: 1e-9, Now: func() time.Time {
mu.Lock()
defer mu.Unlock()
return now
}}
idle, err := st.Begin("u1", "survival", "a.jar", 3)
if err != nil {
t.Fatal(err)
}
advance(fileedit.SessionIdle - time.Minute)
fresh, err := st.Begin("u1", "survival", "b.jar", 3)
if err != nil {
t.Fatal(err)
}
advance(2 * time.Minute)
ctx, cancel := context.WithCancel(context.Background())
var out bytes.Buffer
done := make(chan struct{})
go func() { expireFileSessions(ctx, st, time.Millisecond, &out); close(done) }()
for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(time.Millisecond) {
if _, err := st.Status("u1", "survival", idle.ID); errors.Is(err, fileedit.ErrNotStaged) {
break
}
if time.Now().After(deadline) {
cancel()
t.Fatal("the idle session was never dropped")
}
}
// A few more ticks with nothing idle, which must stay quiet.
time.Sleep(20 * time.Millisecond)
cancel()
<-done
if _, err := st.Status("u1", "survival", fresh.ID); err != nil {
t.Fatalf("the session touched since went too: %v", err)
}
if got := out.String(); got != "felis api: dropped 1 upload session(s) left idle for 6h0m0s\n" {
t.Fatalf("said %q", got)
}
}
type sweepCount struct{ n atomic.Int32 }
func (s *sweepCount) ExpireExports() { s.n.Add(1) }
// TestExpireExports: the loop sweeps on each tick, and returns once felis-api
// shuts down.
func TestExpireExports(t *testing.T) {
var s sweepCount
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { expireExports(ctx, &s, time.Millisecond); close(done) }()
for deadline := time.Now().Add(5 * time.Second); s.n.Load() < 3; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
cancel()
t.Fatalf("swept %d times in 5s at a 1ms tick", s.n.Load())
}
}
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the loop outlived its context")
}
}
cmd/felis/api_tokens_test.go
-38
View File
@@ -1,38 +0,0 @@
package main
import (
"net/http/httptest"
"strings"
"testing"
"felis.lolicon.best/internal/api"
)
// felis-api maps each env var onto the caller the Deployment feeds it for, so the
// build Job's token (FELIS_BUILD_TOKEN) authenticates as build and only as build.
func TestInternalCallerTokensReadEachCallersEnv(t *testing.T) {
env := map[string]string{
"FELIS_SERVICE_TOKEN": "v-tok",
"FELIS_LIMBO_TOKEN": "l-tok",
"FELIS_BUILD_TOKEN": " b-tok\n",
"FELIS_OPS_TOKEN": "o-tok",
}
auth, err := internalCallerTokens(func(k string) string { return env[k] })
if err != nil {
t.Fatal(err)
}
for tok, want := range map[string]api.Caller{"v-tok": api.CallerVelocity, "l-tok": api.CallerLimbo, "b-tok": api.CallerBuild, "o-tok": api.CallerOps} {
r := httptest.NewRequest("GET", "/", nil)
r.Header.Set("Authorization", "Bearer "+tok)
if got, err := auth.Authenticate(r); err != nil || got != want {
t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want)
}
}
// An install where the build namespace still holds a copy of the proxy's token
// would let that copy act as the proxy; the api refuses to start on it.
env["FELIS_BUILD_TOKEN"] = "v-tok"
if _, err := internalCallerTokens(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "same value") {
t.Fatalf("shared token: err = %v, want a same-value refusal", err)
}
}
cmd/felis/apply.go
+20 -16
View File
@@ -8,9 +8,7 @@ import (
"fmt"
"io"
"os"
"os/signal"
"strings"
"syscall"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/naming"
@@ -18,6 +16,10 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -109,15 +111,20 @@ func cmdApply(args []string, stdout, stderr io.Writer) int {
}
// ------- K8s client (one context, one client) -------
// The operator runs this on the node, where the kubeconfig is k3s's own file and
// neither $KUBECONFIG nor ~/.kube is set. buildSystemServerClient falls back to that
// file and names what it tried; ctrl.GetConfigOrDie exited 1 there without a word,
// because controller-runtime's logger is never set up in a CLI command.
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
cl, err := buildSystemServerClient()
// SetupSignalHandler must be called exactly once per process —
// controller-runtime panics on a second call. We create ctx and the
// K8s client here and thread both through every downstream call so no
// callee ever needs to call SetupSignalHandler again.
ctx := ctrl.SetupSignalHandler()
scheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(v1alpha1.AddToScheme(scheme))
cfg := ctrl.GetConfigOrDie()
cl, err := client.New(cfg, client.Options{Scheme: scheme})
if err != nil {
fmt.Fprintf(stderr, "felis apply: %v\n", err)
fmt.Fprintf(stderr, "felis apply: build client: %v\n", err)
return 1
}
@@ -160,10 +167,6 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
if err := naming.ValidateServerName(req.Subdomain); err != nil {
return nil, fmt.Errorf("invalid subdomain: %w", err)
}
displayName, err := naming.CleanDisplayName(req.DisplayName)
if err != nil {
return nil, fmt.Errorf("invalid displayName: %w", err)
}
if strings.TrimSpace(req.Image) == "" {
return nil, fmt.Errorf("image is required")
}
@@ -233,7 +236,7 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
},
Spec: v1alpha1.MinecraftServerSpec{
Subdomain: req.Subdomain,
DisplayName: displayName,
DisplayName: req.DisplayName,
Image: req.Image,
JavaMemory: deriveApplyJavaHeap(memLim),
DesiredState: v1alpha1.DesiredStopped,
@@ -261,7 +264,8 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
// request if any CRD already carries the given spec.subdomain. metadata.name
// uniqueness is enforced by K8s on Create, but spec.subdomain must be checked
// here because two CRDs with different names could otherwise share a subdomain.
// It reuses the caller's context and K8s client.
// It reuses the caller's context and K8s client — it never calls
// SetupSignalHandler or builds its own client.
func checkSubdomainUnique(ctx context.Context, cl client.Client, namespace, subdomain string) error {
var list v1alpha1.MinecraftServerList
if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
cmd/felis/apply_test.go
+5 -41
View File
@@ -1,10 +1,7 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
@@ -140,12 +137,11 @@ func TestDeriveApplyJavaHeap(t *testing.T) {
func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
req := applyRequest{
Name: "test-server",
Subdomain: "test-server",
DisplayName: " Test Server ",
Image: "registry.felis.svc/paper:1.21",
Memory: "4Gi",
Storage: "20Gi",
Name: "test-server",
Subdomain: "test-server",
Image: "registry.felis.svc/paper:1.21",
Memory: "4Gi",
Storage: "20Gi",
}
ms, err := buildMinecraftServerFromApplyRequest(req, "minecraft")
if err != nil {
@@ -160,9 +156,6 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
if ms.Spec.Subdomain != "test-server" {
t.Errorf("Subdomain = %q", ms.Spec.Subdomain)
}
if ms.Spec.DisplayName != "Test Server" {
t.Errorf("DisplayName = %q, want it trimmed to Test Server", ms.Spec.DisplayName)
}
if ms.Spec.Image != "registry.felis.svc/paper:1.21" {
t.Errorf("Image = %q", ms.Spec.Image)
}
@@ -287,11 +280,6 @@ func TestBuildMinecraftServerFromApplyRequest_Errors(t *testing.T) {
applyRequest{Name: ok, Subdomain: "", Image: "x", Memory: "1Gi", Storage: "1Gi"},
"invalid subdomain",
},
{
"display name with a tab",
applyRequest{Name: ok, Subdomain: ok, DisplayName: "a" + string(rune(0x09)) + "b", Image: "x", Memory: "1Gi", Storage: "1Gi"},
"invalid displayName",
},
{
"empty image",
applyRequest{Name: ok, Subdomain: ok, Image: "", Memory: "1Gi", Storage: "1Gi"},
@@ -382,27 +370,3 @@ func resList(specs ...string) corev1.ResourceList {
}
return rl
}
// TestApplyReportsAMissingKubeconfig pins the node-side failure: with no kubeconfig to
// find, apply says which ones it tried and exits 1. It used to call
// ctrl.GetConfigOrDie, which ended the process with exit 1 and nothing printed.
func TestApplyReportsAMissingKubeconfig(t *testing.T) {
if _, err := os.Stat(hostBootstrapKubeconfigPath); err == nil {
t.Skipf("%s exists on this machine", hostBootstrapKubeconfigPath)
}
dir := t.TempDir()
t.Setenv("KUBECONFIG", filepath.Join(dir, "missing"))
t.Setenv("KUBERNETES_SERVICE_HOST", "")
form := filepath.Join(dir, "server.json")
if err := os.WriteFile(form, []byte(`{"name":"alpha","subdomain":"alpha","image":"registry.felis.svc:5000/felis/paper:demo","memory":"1Gi","storage":"1Gi"}`), 0o600); err != nil {
t.Fatal(err)
}
var out, errw bytes.Buffer
if code := cmdApply([]string{"-f", form}, &out, &errw); code != 1 {
t.Fatalf("exit = %d, want 1; stderr %q", code, errw.String())
}
want := "felis apply: no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and " + hostBootstrapKubeconfigPath + "): stat " + hostBootstrapKubeconfigPath + ": no such file or directory\n"
if errw.String() != want || out.Len() != 0 {
t.Fatalf("stdout %q, stderr %q, want stderr %q", out.String(), errw.String(), want)
}
}
cmd/felis/archive_serve.go
-58
View File
@@ -1,58 +0,0 @@
package main
import (
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"time"
"felis.lolicon.best/internal/archivetransfer"
)
func cmdArchiveServe(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("archive-serve", flag.ContinueOnError)
fs.SetOutput(stderr)
root := fs.String("root", "/backups", "archive PVC mount (must match archive.local_path)")
addr := fs.String("listen", ":8090", "private archive listener")
limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum compressed archive bytes")
if err := fs.Parse(args); err != nil {
return 2
}
key := os.Getenv(archivetransfer.KeyEnv)
if len(key) < 32 || *limit <= 0 {
fmt.Fprintln(stderr, "archive-serve: signing key and positive size limit required")
return 2
}
if err := os.MkdirAll(*root, 0750); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
limitHeapToCgroup()
transport := &archivetransfer.Server{Root: *root, Key: key, Limit: *limit}
done := make(chan struct{})
defer close(done)
go func() {
ticker := time.NewTicker(time.Hour)
defer ticker.Stop()
for {
if err := transport.Sweep(time.Now()); err != nil {
fmt.Fprintln(stderr, "archive journal cleanup:", err)
}
select {
case <-done:
return
case <-ticker.C:
}
}
}()
srv := &http.Server{Addr: *addr, Handler: transport, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Hour, WriteTimeout: 2 * time.Hour, MaxHeaderBytes: 16 << 10}
fmt.Fprintln(stdout, "archive transport listening", *addr)
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
fmt.Fprintln(stderr, err)
return 1
}
return 0
}
cmd/felis/backup.go
+15 -61
View File
@@ -7,14 +7,13 @@ import (
"flag"
"fmt"
"io"
"strings"
"time"
"felis.lolicon.best/internal/backup"
"felis.lolicon.best/internal/backupjob"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/reaper"
"felis.lolicon.best/internal/store"
ctrl "sigs.k8s.io/controller-runtime"
)
@@ -37,8 +36,6 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
server := fs.String("server", "", "server name whose world is being backed up")
formerOwner := fs.String("former-owner", "", "owner recorded on the backup row (empty for an unowned server)")
worldsRoot := fs.String("worlds-root", "/world", "mount path of the world PVC being archived")
reason := fs.String("reason", reasonManual, "world_backups reason: manual, pre_restore for the safety snapshot in front of a restore, or scheduled for felis-api's daily restore point")
protect := fs.String("protect", "", "backup id the prune must keep (the one a chained restore extracts)")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -46,10 +43,6 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
fmt.Fprintln(stderr, "felis backup: --server is required")
return 2
}
if _, _, ok := backupPolicy(*reason, reaper.DefaultConfig()); !ok {
fmt.Fprintf(stderr, "felis backup: unknown --reason %q (manual, %s or %s)\n", *reason, backupjob.ReasonPreRestore, backupjob.ReasonScheduled)
return 2
}
cfg, err := config.Load(*cfgPath)
if err != nil {
@@ -60,14 +53,13 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "felis backup: archive store %q is not implemented in this build (only tarLocal)\n", cfg.Archive.Store)
return 1
}
// The [archive] parse the reaper uses: it holds each reason's keep and
// retention.
// The [archive] parse the reaper uses; an on-demand backup takes its
// manual_retention and manual_keep.
rcfg, err := reaperConfig(cfg)
if err != nil {
fmt.Fprintf(stderr, "felis backup: %v\n", err)
return 1
}
keep, retention, _ := backupPolicy(*reason, rcfg)
// The world PVC is mounted directly at worldsRoot; the resolver returns it for
// any target, exactly as in cmdRestore. This is the same TarLocal the reaper
@@ -88,18 +80,13 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
return 1
}
a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
if err != nil {
fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
return 1
}
ref, size := a.Ref, a.Size
if len(a.Skipped) > 0 {
fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",
len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))
}
drv, err := openPodStore(ctx, cfg.Database.URL, "backup", stderr)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
fmt.Fprintf(stderr, "felis backup: open database: %v\n", err)
return 1
@@ -112,11 +99,8 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
FormerOwner: *formerOwner,
BackupRef: string(ref),
SizeBytes: size,
Reason: *reason,
ExpiresAt: time.Now().Add(retention),
SHA256: a.SHA256,
SkippedEntries: len(a.Skipped),
Reason: "manual",
ExpiresAt: time.Now().Add(rcfg.ManualRetention),
}
st := reaper.NewPGStore(drv.DB())
if err := st.InsertBackup(ctx, rec); err != nil {
@@ -132,46 +116,16 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
}
fmt.Fprintf(stdout, "felis backup: server=%s archived %d bytes to %s (backup %s)\n", *server, size, ref, rec.ID)
pruneBackups(ctx, st, archiver, *server, *formerOwner, *reason, keep, *protect, stdout, stderr)
pruneManualBackups(ctx, st, archiver, *server, rcfg.ManualKeep, stdout, stderr)
return 0
}
const (
reasonManual = "manual"
// preRestoreKeep is how many safety snapshots a server keeps: enough to walk
// back a couple of restores in a row, without every restore adding a world's
// worth of bytes for the full manual retention.
preRestoreKeep = 3
)
// backupPolicy is how many backups of one reason a server keeps and how long
// each lives: an owner's own backups and the safety snapshots in front of a
// restore by [archive] manual_keep / manual_retention (the snapshots capped at
// preRestoreKeep), felis-api's daily restore points by scheduled_keep /
// scheduled_retention, so neither kind crowds out the other. ok is false for a
// reason this command does not record.
func backupPolicy(reason string, rcfg reaper.Config) (keep int, retention time.Duration, ok bool) {
switch reason {
case reasonManual:
return rcfg.ManualKeep, rcfg.ManualRetention, true
case backupjob.ReasonPreRestore:
return preRestoreKeep, rcfg.ManualRetention, true
case backupjob.ReasonScheduled:
return rcfg.ScheduledKeep, rcfg.ScheduledRetention, true
}
return 0, 0, false
}
// pruneBackups keeps the newest keep backups of this reason that owner holds of
// server and removes the rest, oldest first, so repeated backups of one world
// cannot fill the shared archive store and a new owner's backups never remove a
// previous owner's. protect is never removed: it is the backup a chained restore
// is about to extract. The new backup is already recorded; a removal that fails
// is reported and retried after the next backup.
func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver interface {
Delete(context.Context, backup.ArchiveRef) error
}, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) {
excess, err := st.ExcessBackups(ctx, server, owner, reason, keep, protect)
// pruneManualBackups keeps server's newest keep on-demand backups and removes
// the rest, oldest first, so repeated backups of one world cannot fill the
// shared archive store. The new backup is already recorded; a removal that
// fails is reported and retried after the next backup.
func pruneManualBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server string, keep int, stdout, stderr io.Writer) {
excess, err := st.ExcessManualBackups(ctx, server, keep)
if err != nil {
fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)
return
@@ -185,7 +139,7 @@ func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver interface {
fmt.Fprintf(stderr, "felis backup: record the removal of %s: %v\n", b.ID, err)
continue
}
fmt.Fprintf(stdout, "felis backup: removed older %s backup %s of %s (keeping the newest %d)\n", reason, b.ID, server, keep)
fmt.Fprintf(stdout, "felis backup: removed older backup %s of %s (keeping the newest %d)\n", b.ID, server, keep)
}
}
cmd/felis/backup_test.go
-48
View File
@@ -1,48 +0,0 @@
package main
import (
"bytes"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/reaper"
)
// The reason decides which backups the new one's prune may remove, so an
// unknown one is refused before anything is archived.
func TestBackupSubcommandRejectsUnknownReason(t *testing.T) {
var stderr bytes.Buffer
if code := cmdBackup([]string{"--server", "survival", "--reason", "inactive_15d"}, &bytes.Buffer{}, &stderr); code != 2 {
t.Fatalf("exit = %d, want 2 (%s)", code, stderr.String())
}
if !strings.Contains(stderr.String(), "unknown --reason") {
t.Fatalf("stderr = %q", stderr.String())
}
}
// Each reason is pruned and expired by its own [archive] keys: a daily
// restore point must never count against, or take the lifetime of, the
// backups an owner asked for.
func TestBackupPolicyPerReason(t *testing.T) {
rcfg := reaper.DefaultConfig()
rcfg.ManualKeep, rcfg.ManualRetention = 5, 30*reaper.Day
rcfg.ScheduledKeep, rcfg.ScheduledRetention = 7, 90*reaper.Day
for _, tc := range []struct {
reason string
keep int
retention time.Duration
}{
{"manual", 5, 30 * reaper.Day},
{"pre_restore", preRestoreKeep, 30 * reaper.Day},
{"scheduled", 7, 90 * reaper.Day},
} {
keep, retention, ok := backupPolicy(tc.reason, rcfg)
if !ok || keep != tc.keep || retention != tc.retention {
t.Errorf("backupPolicy(%q) = (%d, %v, %v); want (%d, %v, true)", tc.reason, keep, retention, ok, tc.keep, tc.retention)
}
}
if _, _, ok := backupPolicy("inactive_15d", rcfg); ok {
t.Error("backupPolicy accepted inactive_15d; the reaper records those itself")
}
}
cmd/felis/backupnow.go
+7 -459
View File
@@ -4,28 +4,15 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/signal"
"sort"
"strings"
"syscall"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/operator"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -33,11 +20,9 @@ import (
// backupnow is the break-glass "back up a world now" op (§B4 "Sync"). Unlike halt —
// which writes the CRD directly — a backup needs felis-api's deployment coordinates
// (FELIS_IMAGE / FELIS_BACKUP_PVC) to render the one-shot backup Job, so the console
// cannot do it in-process. It POSTs the felis-api INTERNAL face (ops-token auth)
// cannot do it in-process. It POSTs the felis-api INTERNAL face (service-token auth)
// while the API is alive, and the API renders the Job and audits the action. This file
// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue. The
// `felis backup-now` command (cmdBackupNow, below) takes the same route for every
// user server in turn.
// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue.
// backupNowOutcome is the durable result of a backup request, re-printed after the TUI
// alt-screen tears down.
@@ -48,7 +33,7 @@ type backupNowOutcome struct {
// resolveInternalAPI reads the two things the on-node console needs to reach the
// felis-api internal face: the felis-api-internal Service ClusterIP (the host's
// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the ops
// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the service
// token. Both live in the control namespace.
func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace string) (baseURL, token string, err error) {
var svc corev1.Service
@@ -60,14 +45,13 @@ func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace
return "", "", fmt.Errorf("%s Service has no ClusterIP yet", platform.APIInternalServiceName)
}
// The console's own token, which the api serves on the backup route alone.
var sec corev1.Secret
if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.OpsTokenSecretName}, &sec); err != nil {
return "", "", fmt.Errorf("get %s Secret (re-run the installer to create it): %w", naming.OpsTokenSecretName, err)
if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.ServiceTokenSecretName}, &sec); err != nil {
return "", "", fmt.Errorf("get %s Secret: %w", naming.ServiceTokenSecretName, err)
}
token = string(sec.Data[naming.ServiceTokenSecretKey])
if token == "" {
return "", "", fmt.Errorf("secret %s has no %s key", naming.OpsTokenSecretName, naming.ServiceTokenSecretKey)
return "", "", fmt.Errorf("secret %s has no %s key", naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey)
}
return fmt.Sprintf("http://%s:%d", ip, platform.APIInternalPort), token, nil
@@ -90,7 +74,7 @@ func requestBackup(ctx context.Context, hc *http.Client, baseURL, token, name, o
resp, err := hc.Do(req)
if err != nil {
return backupNowOutcome{}, fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
return backupNowOutcome{}, fmt.Errorf("felis-api unreachable (a backup needs it alive): %w", err)
}
defer resp.Body.Close()
@@ -160,439 +144,3 @@ func backupPickable(servers []haltableServer) []haltableServer {
}
return out
}
// cmdBackupNow is `felis backup-now`: the world of every user server (or of the
// named ones) archived now, one at a time, through the internal backup route the
// console's Sync uses. A world lives only in its volume and the off-site copy holds
// only its archives, so this is the lever in front of a planned move to another
// host, a disk swap or anything else that could lose a volume.
func cmdBackupNow(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("backup-now", flag.ContinueOnError)
fs.SetOutput(stderr)
cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
yes := fs.Bool("yes", false, "back up; without it the plan is printed and nothing changes")
stop := fs.Bool("stop", false, "stop the running servers first: their players are disconnected and the servers stay stopped")
fs.Usage = func() {
fmt.Fprintln(stderr, "Usage: felis backup-now [-yes] [-stop] [server ...]")
fmt.Fprintln(stderr)
fmt.Fprintln(stderr, "Archives the world of every user server, or of the named ones, one at a time, and waits for each archive.")
fmt.Fprintln(stderr, "A running server is skipped unless -stop is given. Without -yes it prints what it would do.")
fs.PrintDefaults()
}
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis backup-now: refused — it reads the cluster's ops token, so it must run as root (try: sudo felis backup-now)")
return 1
}
cfg, err := config.Load(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
return 1
}
cl, err := buildSystemServerClient()
if err != nil {
fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
return 1
}
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
ns := cfg.K8s.Namespace
osUser := accountableOSUser()
jobs := api.NewK8sJobStatus(cl, ns)
var baseURL, token string
var repo ownerStore
var drv *store.PostgresDriver
defer func() {
if drv != nil {
_ = drv.Close()
}
}()
hc := &http.Client{Timeout: 10 * time.Second}
r := backupNowRun{
out: stdout,
errw: stderr,
ns: ns,
list: func(ctx context.Context) ([]backupNowWorld, error) { return listBackupNowWorlds(ctx, cl, ns) },
stopped: func(ctx context.Context, name string) (bool, error) {
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &ms); err != nil {
return false, err
}
return backupNowStopped(ctx, cl, &ms)
},
halt: func(ctx context.Context, name string) error {
// The database is opened only once a server is to be stopped: the halt
// is audited like the console's, and a run with nothing running needs
// no more than the API.
if repo == nil {
d, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
return fmt.Errorf("open the database for the audit log: %w", err)
}
drv, repo = d, api.NewPGRepo(d.DB())
}
out, err := performHalt(ctx, cl, repo, ns, name, osUser)
if err != nil {
return err
}
if out.auditErr != nil {
fmt.Fprintf(stderr, "felis backup-now: the audit row for stopping %s was not written: %v\n", name, out.auditErr)
}
return nil
},
request: func(ctx context.Context, name string) error {
if baseURL == "" {
var err error
if baseURL, token, err = resolveInternalAPI(ctx, cl, platform.DefaultControlNamespace); err != nil {
return fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
}
}
_, err := requestBackup(ctx, hc, baseURL, token, name, osUser)
return err
},
jobs: jobs.LatestJobs,
now: time.Now,
sleep: func(ctx context.Context, d time.Duration) { sleepCtx(ctx, d) },
}
return r.run(ctx, fs.Args(), *yes, *stop)
}
// sleepCtx waits d or until ctx ends.
func sleepCtx(ctx context.Context, d time.Duration) {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
case <-t.C:
}
}
// backupNowWorld is one user server as backup-now sees it.
type backupNowWorld struct {
name string
phase string // the observed phase, or the desired state before the operator reconciled it
stopped bool // the backup route's stopped gate admits it
hasWorld bool // its world volume exists
}
// listBackupNowWorlds lists the user servers of namespace in the API server's
// order (by name), each with what the backup route checks. System servers are
// left out: they have no row in the servers table, so the route refuses them
// (backupPickable).
func listBackupNowWorlds(ctx context.Context, cl client.Client, namespace string) ([]backupNowWorld, error) {
var list v1alpha1.MinecraftServerList
if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
return nil, err
}
var out []backupNowWorld
for i := range list.Items {
ms := &list.Items[i]
if isSystemServer(ms.Name) {
continue
}
stopped, err := backupNowStopped(ctx, cl, ms)
if err != nil {
return nil, err
}
var pvc corev1.PersistentVolumeClaim
err = cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: naming.WorldPVCName(ms.Name)}, &pvc)
if err != nil && !apierrors.IsNotFound(err) {
return nil, fmt.Errorf("look up the world volume of %s: %w", ms.Name, err)
}
phase := string(ms.Status.Phase)
if phase == "" {
phase = string(ms.Spec.DesiredState)
}
out = append(out, backupNowWorld{name: ms.Name, phase: phase, stopped: stopped, hasWorld: err == nil})
}
return out, nil
}
// backupNowStopped is the backup route's stopped gate (api.enqueueBackup and
// K8sCluster.AcquireMaintenance together): desired Stopped, not ready, phase
// Stopped and no game pod left.
func backupNowStopped(ctx context.Context, cl client.Client, ms *v1alpha1.MinecraftServer) (bool, error) {
if ms.Spec.DesiredState != v1alpha1.DesiredStopped || ms.Status.Ready || ms.Status.Phase != v1alpha1.PhaseStopped {
return false, nil
}
var pods corev1.PodList
if err := cl.List(ctx, &pods, client.InNamespace(ms.Namespace), client.MatchingLabels{
v1alpha1.LabelServer: ms.Name, v1alpha1.LabelComponent: operator.ComponentValue,
}); err != nil {
return false, fmt.Errorf("look up the pod of %s: %w", ms.Name, err)
}
return len(pods.Items) == 0, nil
}
// errBackupAPIUnreachable is a backup request that never reached felis-api. It
// ends a backup-now run: every later world would fail the same way, and stopping
// servers for backups that cannot be taken only takes them away from players.
var errBackupAPIUnreachable = errors.New("felis-api unreachable (a backup needs it alive)")
// Polling of backup-now. A graceful stop saves the world first; the backup Job's
// own deadline (backupjob, 30 minutes) ends a Job that hangs, so its wait needs no
// cap of its own.
const (
backupNowPoll = 2 * time.Second
backupNowStopWait = 10 * time.Minute
backupNowJobAppear = time.Minute
)
// backupNowRun is backup-now over seams, so the plan and the run are tested
// without a cluster or felis-api.
type backupNowRun struct {
out, errw io.Writer
ns string // where the servers and their Jobs live, for the kubectl hints
list func(ctx context.Context) ([]backupNowWorld, error)
stopped func(ctx context.Context, name string) (bool, error)
halt func(ctx context.Context, name string) error
request func(ctx context.Context, name string) error
jobs func(ctx context.Context, name string) ([]api.AsyncJob, error)
now func() time.Time
sleep func(ctx context.Context, d time.Duration)
}
// pickBackupNowWorlds narrows worlds to names, in the order given, or keeps them
// all when names is empty.
func pickBackupNowWorlds(worlds []backupNowWorld, names []string) ([]backupNowWorld, error) {
if len(names) == 0 {
return worlds, nil
}
byName := make(map[string]backupNowWorld, len(worlds))
for _, w := range worlds {
byName[w.name] = w
}
var out []backupNowWorld
seen := map[string]bool{}
for _, n := range names {
if seen[n] {
continue
}
seen[n] = true
w, ok := byName[n]
switch {
case ok:
out = append(out, w)
case isSystemServer(n):
return nil, fmt.Errorf("%s is a system server: its world is rebuilt by felis setup and has no backups", n)
default:
return nil, fmt.Errorf("no server named %q", n)
}
}
return out, nil
}
// backupNowAction is what the plan does with a world.
func backupNowAction(w backupNowWorld, stop bool) string {
switch {
case w.stopped && !w.hasWorld:
return "skip: no world volume (never started, nothing to save)"
case w.stopped:
return "back up"
case stop:
return "stop, then back up"
default:
return "skip: running (stop it first, or pass -stop)"
}
}
func (r *backupNowRun) run(ctx context.Context, names []string, yes, stop bool) int {
all, err := r.list(ctx)
if err != nil {
fmt.Fprintf(r.errw, "felis backup-now: list the servers: %v\n", err)
return 1
}
worlds, err := pickBackupNowWorlds(all, names)
if err != nil {
fmt.Fprintf(r.errw, "felis backup-now: %v\n", err)
return 2
}
if len(worlds) == 0 {
fmt.Fprintln(r.out, "felis backup-now: there are no user servers")
return 0
}
// The stopped worlds go first, so a felis-api that cannot take a backup is
// found before any server is stopped for one.
sort.SliceStable(worlds, func(i, j int) bool { return worlds[i].stopped && !worlds[j].stopped })
width := 0
for _, w := range worlds {
width = max(width, len(w.name))
}
fmt.Fprintf(r.out, "felis backup-now: %d server(s), backed up one at a time:\n", len(worlds))
stopping, work := false, 0
for _, w := range worlds {
fmt.Fprintf(r.out, " %-*s %-8s %s\n", width, w.name, w.phase, backupNowAction(w, stop))
stopping = stopping || (!w.stopped && stop)
if (w.stopped && w.hasWorld) || (!w.stopped && stop) {
work++
}
}
if work > 0 {
fmt.Fprintln(r.out, "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.")
}
if stopping {
fmt.Fprintln(r.out, "Stopping disconnects the players on those servers, and they stay stopped afterwards.")
}
if !yes {
if work == 0 {
fmt.Fprintln(r.out, "Nothing to back up.")
} else {
fmt.Fprintln(r.out, "Nothing changed. Run again with -yes to back them up.")
}
return 0
}
var done, failed, running, empty int
var leftStopped []string
for _, w := range worlds {
if err := ctx.Err(); err != nil {
break
}
switch {
case w.stopped && !w.hasWorld:
empty++
continue
case !w.stopped && !stop:
running++
continue
}
if !w.stopped {
halted, err := r.stopWorld(ctx, w.name)
if halted {
leftStopped = append(leftStopped, w.name)
}
if err != nil {
fmt.Fprintf(r.out, " %s: failed: %v\n", w.name, err)
failed++
continue
}
}
err := r.backUp(ctx, w.name)
if errors.Is(err, errBackupAPIUnreachable) {
fmt.Fprintf(r.out, " %s: failed: %v\n", w.name, err)
fmt.Fprintln(r.out, "Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).")
failed++
break
}
if err != nil {
fmt.Fprintf(r.out, " %s: failed: %v\n", w.name, err)
failed++
continue
}
done++
}
interrupted := ctx.Err() != nil
if interrupted {
fmt.Fprintln(r.out, "Interrupted: a backup Job already started runs to its end.")
}
parts := []string{fmt.Sprintf("%d backed up", done)}
if failed > 0 {
parts = append(parts, fmt.Sprintf("%d failed", failed))
}
if running > 0 {
parts = append(parts, fmt.Sprintf("%d skipped (running)", running))
}
if empty > 0 {
parts = append(parts, fmt.Sprintf("%d without a world", empty))
}
fmt.Fprintf(r.out, "%s.\n", strings.Join(parts, ", "))
if len(leftStopped) > 0 {
fmt.Fprintf(r.out, "Left stopped: %s. Start them from the panel when you are done.\n", strings.Join(leftStopped, ", "))
}
if done > 0 {
fmt.Fprintln(r.out, "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.")
}
if failed > 0 || running > 0 || interrupted {
return 1
}
return 0
}
// stopWorld stops one server and waits until the backup route would admit it.
// halted is whether the stop was asked for: the server then stays stopped, even
// when it takes longer than the wait.
func (r *backupNowRun) stopWorld(ctx context.Context, name string) (halted bool, err error) {
fmt.Fprintf(r.out, " %s: stopping\n", name)
if err := r.halt(ctx, name); err != nil {
return false, err
}
start := r.now()
for {
ok, err := r.stopped(ctx, name)
if err != nil {
return true, err
}
if ok {
fmt.Fprintf(r.out, " %s: stopped after %s\n", name, r.now().Sub(start).Round(time.Second))
return true, nil
}
if r.now().Sub(start) >= backupNowStopWait {
return true, fmt.Errorf("did not stop within %s (kubectl -n %s describe minecraftserver %s)", backupNowStopWait, r.ns, name)
}
if err := ctx.Err(); err != nil {
return true, err
}
r.sleep(ctx, backupNowPoll)
}
}
// backUp requests one world's backup and waits for its Job to finish. The Job is
// the one of this server that was not there before the request.
func (r *backupNowRun) backUp(ctx context.Context, name string) error {
before, err := r.jobs(ctx, name)
if err != nil {
return fmt.Errorf("list its backup Jobs: %w", err)
}
known := make(map[string]bool, len(before))
for _, j := range before {
known[j.Name] = true
}
if err := r.request(ctx, name); err != nil {
return err
}
fmt.Fprintf(r.out, " %s: backing up\n", name)
start := r.now()
seen := ""
for {
jobs, err := r.jobs(ctx, name)
if err != nil {
return fmt.Errorf("list its backup Jobs: %w", err)
}
var job *api.AsyncJob
for i := range jobs {
if jobs[i].Kind == "backup" && (jobs[i].Name == seen || seen == "" && !known[jobs[i].Name]) {
job = &jobs[i]
break
}
}
switch {
case job == nil && seen != "":
return fmt.Errorf("its backup Job %s was deleted before it finished", seen)
case job == nil && r.now().Sub(start) >= backupNowJobAppear:
return fmt.Errorf("felis-api accepted the backup, but no backup Job appeared within %s (kubectl -n %s get jobs)", backupNowJobAppear, r.ns)
case job != nil && job.State == "succeeded":
fmt.Fprintf(r.out, " %s: archived in %s\n", name, r.now().Sub(start).Round(time.Second))
return nil
case job != nil && job.State == "failed":
msg := job.Message
if msg == "" {
msg = "the backup Job failed"
}
return fmt.Errorf("%s (kubectl -n %s logs job/%s)", msg, r.ns, job.Name)
case job != nil:
seen = job.Name
}
if err := ctx.Err(); err != nil {
return err
}
r.sleep(ctx, backupNowPoll)
}
}
cmd/felis/backupnow_cmd_test.go
-563
View File
@@ -1,563 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/naming"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// bnRig drives backupNowRun against scripted servers and Jobs on a fake clock that
// moves only when the run sleeps.
type bnRig struct {
out, errw bytes.Buffer
clock time.Time
events []string
worlds []backupNowWorld
// stopAfter is how many polls a halted server takes to stop; -1 never does.
stopAfter map[string]int
polls map[string]int
reqErr map[string]error
// states is what the server's new backup Job reports on each poll after the
// request, the last one repeating; "" is no Job.
states map[string][]string
messages map[string]string
jobPolls map[string]int
// stopErr fails a server's stop polls; jobsFail fails the nth (1-based) Job
// list of a server.
stopErr map[string]error
jobsFail map[string]int
jobCalls map[string]int
// ctx is the run's context, and onAct runs after each halt and request.
ctx context.Context
onAct func(event string)
}
func newBNRig(worlds ...backupNowWorld) *bnRig {
return &bnRig{
clock: time.Unix(1_800_000_000, 0),
worlds: worlds,
stopAfter: map[string]int{},
polls: map[string]int{},
reqErr: map[string]error{},
states: map[string][]string{},
messages: map[string]string{},
jobPolls: map[string]int{},
stopErr: map[string]error{},
jobsFail: map[string]int{},
jobCalls: map[string]int{},
ctx: context.Background(),
onAct: func(string) {},
}
}
func (g *bnRig) run(names []string, yes, stop bool) int {
r := backupNowRun{
out: &g.out, errw: &g.errw, ns: "minecraft",
list: func(context.Context) ([]backupNowWorld, error) {
return append([]backupNowWorld(nil), g.worlds...), nil
},
stopped: func(_ context.Context, name string) (bool, error) {
g.polls[name]++
if err := g.stopErr[name]; err != nil {
return false, err
}
n := g.stopAfter[name]
return n >= 0 && g.polls[name] > n, nil
},
halt: func(_ context.Context, name string) error {
g.events = append(g.events, "halt "+name)
g.onAct("halt " + name)
return nil
},
request: func(_ context.Context, name string) error {
g.events = append(g.events, "request "+name)
g.onAct("request " + name)
if err := g.reqErr[name]; err != nil {
return err
}
g.jobPolls[name] = 0
return nil
},
jobs: func(_ context.Context, name string) ([]api.AsyncJob, error) {
// Every server has an older finished backup, and a restore Job that
// shows up with the new backup: neither is the Job to wait for.
g.jobCalls[name]++
if g.jobCalls[name] == g.jobsFail[name] {
return nil, errors.New("the apiserver is gone")
}
out := []api.AsyncJob{{Name: "backup-" + name + "-old", Kind: "backup", State: "succeeded"}}
n, requested := g.jobPolls[name]
if !requested {
return out, nil
}
g.jobPolls[name] = n + 1
states := g.states[name]
if len(states) == 0 {
states = []string{"succeeded"}
}
state := states[min(n, len(states)-1)]
if state == "" {
return out, nil
}
return append([]api.AsyncJob{
{Name: "restore-" + name + "-x", Kind: "restore", State: "succeeded"},
{Name: "backup-" + name + "-new", Kind: "backup", State: state, Message: g.messages[name]},
}, out...), nil
},
now: func() time.Time { return g.clock },
sleep: func(_ context.Context, d time.Duration) { g.clock = g.clock.Add(d) },
}
return r.run(g.ctx, names, yes, stop)
}
func stoppedWorld(name string) backupNowWorld {
return backupNowWorld{name: name, phase: "Stopped", stopped: true, hasWorld: true}
}
func runningWorld(name string) backupNowWorld {
return backupNowWorld{name: name, phase: "Running", hasWorld: true}
}
func emptyWorld(name string) backupNowWorld {
return backupNowWorld{name: name, phase: "Stopped", stopped: true}
}
const (
bnManualKeep = "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.\n"
bnStopping = "Stopping disconnects the players on those servers, and they stay stopped afterwards.\n"
bnOffsite = "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.\n"
)
func TestBackupNowPlanChangesNothing(t *testing.T) {
for _, tc := range []struct {
stop bool
want string
}{
{false, "felis backup-now: 4 server(s), backed up one at a time:\n" +
" alpha Stopped back up\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
" bravo Running skip: running (stop it first, or pass -stop)\n" +
" delta Starting skip: running (stop it first, or pass -stop)\n" +
bnManualKeep +
"Nothing changed. Run again with -yes to back them up.\n"},
{true, "felis backup-now: 4 server(s), backed up one at a time:\n" +
" alpha Stopped back up\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
" bravo Running stop, then back up\n" +
" delta Starting stop, then back up\n" +
bnManualKeep + bnStopping +
"Nothing changed. Run again with -yes to back them up.\n"},
} {
t.Run(fmt.Sprintf("stop=%v", tc.stop), func(t *testing.T) {
delta := runningWorld("delta")
delta.phase = "Starting"
g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), delta)
if code := g.run(nil, false, tc.stop); code != 0 {
t.Fatalf("exit = %d, want 0; stderr %q", code, g.errw.String())
}
if g.out.String() != tc.want {
t.Fatalf("plan =\n%s\nwant\n%s", g.out.String(), tc.want)
}
if len(g.events) != 0 || len(g.polls) != 0 {
t.Fatalf("the plan acted: events %v, polls %v", g.events, g.polls)
}
})
}
}
// A plan that saves nothing says so, without the manual_keep warning; a running
// server counts as something to save once -stop is given.
func TestBackupNowPlanWithNothingToSave(t *testing.T) {
for _, tc := range []struct {
stop bool
want string
}{
{false, "felis backup-now: 2 server(s), backed up one at a time:\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
" bravo Running skip: running (stop it first, or pass -stop)\n" +
"Nothing to back up.\n"},
{true, "felis backup-now: 2 server(s), backed up one at a time:\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
" bravo Running stop, then back up\n" +
bnManualKeep + bnStopping +
"Nothing changed. Run again with -yes to back them up.\n"},
} {
g := newBNRig(runningWorld("bravo"), emptyWorld("charlie"))
if code := g.run(nil, false, tc.stop); code != 0 {
t.Fatalf("stop=%v: exit = %d, want 0", tc.stop, code)
}
if g.out.String() != tc.want {
t.Fatalf("stop=%v: plan =\n%s\nwant\n%s", tc.stop, g.out.String(), tc.want)
}
}
}
func TestBackupNowBacksUpEachWorldInTurn(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), stoppedWorld("delta"))
g.states["alpha"] = []string{"", "running", "succeeded"}
g.states["delta"] = []string{"running", "failed"}
g.messages["delta"] = "felis backup: not enough free disk for the archive"
g.stopAfter["bravo"] = 2
g.states["bravo"] = []string{"running", "running", "running", "succeeded"}
code := g.run(nil, true, true)
if code != 1 {
t.Fatalf("exit = %d, want 1 (delta failed)", code)
}
if want := []string{"request alpha", "request delta", "halt bravo", "request bravo"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
_, run, _ := strings.Cut(g.out.String(), bnStopping+"")
want := " alpha: backing up\n" +
" alpha: archived in 4s\n" +
" delta: backing up\n" +
" delta: failed: felis backup: not enough free disk for the archive (kubectl -n minecraft logs job/backup-delta-new)\n" +
" bravo: stopping\n" +
" bravo: stopped after 4s\n" +
" bravo: backing up\n" +
" bravo: archived in 6s\n" +
"2 backed up, 1 failed, 1 without a world.\n" +
"Left stopped: bravo. Start them from the panel when you are done.\n" +
bnOffsite
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
}
func TestBackupNowSkipsRunningServersWithoutStop(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"))
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1 (bravo was not backed up)", code)
}
if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
if len(g.polls) != 0 {
t.Fatalf("polled a server it did not stop: %v", g.polls)
}
_, run, _ := strings.Cut(g.out.String(), "Nothing changed")
if run != "" {
t.Fatalf("-yes printed the plan's closing line")
}
_, run, _ = strings.Cut(g.out.String(), bnManualKeep)
want := " alpha: backing up\n alpha: archived in 0s\n1 backed up, 1 skipped (running).\n" + bnOffsite
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
}
func TestBackupNowExitsCleanWhenEverythingIsSaved(t *testing.T) {
// -stop with nothing running stops nothing and says nothing about stopping.
g := newBNRig(stoppedWorld("alpha"), emptyWorld("charlie"))
if code := g.run(nil, true, true); code != 0 {
t.Fatalf("exit = %d, want 0; output\n%s", code, g.out.String())
}
_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
if want := " alpha: backing up\n alpha: archived in 0s\n1 backed up, 1 without a world.\n" + bnOffsite; run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
g = newBNRig(emptyWorld("charlie"))
if code := g.run(nil, true, false); code != 0 {
t.Fatalf("exit = %d, want 0 for a server with nothing to save", code)
}
// Nothing to archive: no manual_keep warning, and no off-site hint.
if want := "felis backup-now: 1 server(s), backed up one at a time:\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
"0 backed up, 1 without a world.\n"; g.out.String() != want {
t.Fatalf("output =\n%s\nwant\n%s", g.out.String(), want)
}
if len(g.events) != 0 {
t.Fatalf("events = %v, want none", g.events)
}
}
func TestBackupNowStopsAtAnUnreachableAPI(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), runningWorld("charlie"))
g.reqErr["alpha"] = fmt.Errorf("%w: dial tcp 10.43.0.9:8081: connect: connection refused", errBackupAPIUnreachable)
if code := g.run(nil, true, true); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v: nothing after the API proved unreachable, and no server stopped", g.events, want)
}
_, run, _ := strings.Cut(g.out.String(), bnStopping)
want := " alpha: failed: felis-api unreachable (a backup needs it alive): dial tcp 10.43.0.9:8081: connect: connection refused\n" +
"Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).\n" +
"0 backed up, 1 failed.\n"
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
// Any other refusal is that world's alone: the run goes on.
g = newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
g.reqErr["alpha"] = errors.New("felis-api: the world is being restored")
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if want := []string{"request alpha", "request bravo"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
}
func TestBackupNowGivesUpOnAServerThatDoesNotStop(t *testing.T) {
g := newBNRig(runningWorld("bravo"), runningWorld("echo"))
g.stopAfter["bravo"] = -1
if code := g.run(nil, true, true); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if want := []string{"halt bravo", "halt echo", "request echo"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
// One poll at the start and one per 2s sleep up to the 10-minute mark.
if g.polls["bravo"] != 301 {
t.Fatalf("bravo polled %d times, want 301", g.polls["bravo"])
}
_, run, _ := strings.Cut(g.out.String(), bnStopping)
want := " bravo: stopping\n" +
" bravo: failed: did not stop within 10m0s (kubectl -n minecraft describe minecraftserver bravo)\n" +
" echo: stopping\n" +
" echo: stopped after 0s\n" +
" echo: backing up\n" +
" echo: archived in 0s\n" +
"1 backed up, 1 failed.\n" +
"Left stopped: bravo, echo. Start them from the panel when you are done.\n" +
bnOffsite
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
}
func TestBackupNowReportsAJobThatNeverRuns(t *testing.T) {
for _, tc := range []struct {
name string
states []string
polls int
want string
}{
{"never appears", []string{""}, 31,
"felis-api accepted the backup, but no backup Job appeared within 1m0s (kubectl -n minecraft get jobs)"},
{"deleted while running", []string{"running", "running", ""}, 3,
"its backup Job backup-alpha-new was deleted before it finished"},
{"fails without a message", []string{"failed"}, 1,
"the backup Job failed (kubectl -n minecraft logs job/backup-alpha-new)"},
} {
t.Run(tc.name, func(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"))
g.states["alpha"] = tc.states
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if !strings.Contains(g.out.String(), " alpha: failed: "+tc.want+"\n") {
t.Fatalf("output =\n%s\nwant the line %q", g.out.String(), tc.want)
}
if g.jobPolls["alpha"] != tc.polls {
t.Fatalf("polled the Jobs %d times after the request, want %d", g.jobPolls["alpha"], tc.polls)
}
})
}
}
func TestBackupNowNamedServers(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), stoppedWorld("delta"))
if code := g.run([]string{"delta", "alpha", "delta"}, true, false); code != 0 {
t.Fatalf("exit = %d, want 0", code)
}
if want := []string{"request delta", "request alpha"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
if !strings.HasPrefix(g.out.String(), "felis backup-now: 2 server(s), backed up one at a time:\n delta Stopped back up\n alpha Stopped back up\n") {
t.Fatalf("plan =\n%s", g.out.String())
}
for _, tc := range []struct{ name, want string }{
{"login", "felis backup-now: login is a system server: its world is rebuilt by felis setup and has no backups\n"},
{"lobby", "felis backup-now: lobby is a system server: its world is rebuilt by felis setup and has no backups\n"},
{"nope", "felis backup-now: no server named \"nope\"\n"},
} {
g := newBNRig(stoppedWorld("alpha"))
if code := g.run([]string{"alpha", tc.name}, true, false); code != 2 {
t.Fatalf("%s: exit = %d, want 2", tc.name, code)
}
if g.errw.String() != tc.want {
t.Fatalf("%s: stderr = %q, want %q", tc.name, g.errw.String(), tc.want)
}
if len(g.events) != 0 || g.out.Len() != 0 {
t.Fatalf("%s: acted on a bad name: events %v, output %q", tc.name, g.events, g.out.String())
}
}
g = newBNRig()
if code := g.run(nil, true, false); code != 0 || g.out.String() != "felis backup-now: there are no user servers\n" {
t.Fatalf("empty fleet: exit %d, output %q", code, g.out.String())
}
}
// The world list mirrors the backup route's own gate, so the plan says exactly what
// the route would refuse.
func TestListBackupNowWorlds(t *testing.T) {
withStatus := func(ms *v1alpha1.MinecraftServer, ready bool) *v1alpha1.MinecraftServer {
ms.Status.Ready = ready
return ms
}
pvc := func(server, ns string) *corev1.PersistentVolumeClaim {
return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: naming.WorldPVCName(server), Namespace: ns}}
}
pod := func(name, ns string, labels map[string]string) *corev1.Pod {
return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}}
}
gameLabels := func(server string) map[string]string {
return map[string]string{v1alpha1.LabelServer: server, v1alpha1.LabelComponent: "server"}
}
other := mcServer("zulu", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped)
other.Namespace = "elsewhere"
hotel := mcServer("hotel", v1alpha1.DesiredStopped, "")
objs := []client.Object{
mcServer("alpha", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("alpha", haltNS),
// A backup Job's pod carries the server label with its own component, and
// a game pod of the same name in another namespace is somebody else's.
pod("backup-alpha-1-x", haltNS, map[string]string{v1alpha1.LabelServer: "alpha", "app.kubernetes.io/component": "world-backup"}),
pod("alpha-0", "elsewhere", gameLabels("alpha")),
withStatus(mcServer("bravo", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning), true), pvc("bravo", haltNS), pod("bravo-0", haltNS, gameLabels("bravo")),
mcServer("charlie", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped),
mcServer("delta", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("delta", haltNS), pod("delta-0", haltNS, gameLabels("delta")),
mcServer("echo", v1alpha1.DesiredStopped, v1alpha1.PhaseStopping), pvc("echo", haltNS),
mcServer("foxtrot", v1alpha1.DesiredRunning, v1alpha1.PhaseStopped), pvc("foxtrot", haltNS),
withStatus(mcServer("golf", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), true), pvc("golf", haltNS),
hotel, pvc("hotel", haltNS),
mcServer("login", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("login", haltNS),
mcServer("lobby", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("lobby", haltNS),
other, pvc("zulu", "elsewhere"),
}
got, err := listBackupNowWorlds(context.Background(), haltClient(t, objs...), haltNS)
if err != nil {
t.Fatal(err)
}
want := []backupNowWorld{
{name: "alpha", phase: "Stopped", stopped: true, hasWorld: true},
{name: "bravo", phase: "Running", hasWorld: true},
{name: "charlie", phase: "Stopped", stopped: true},
{name: "delta", phase: "Stopped", hasWorld: true}, // its pod is still going
{name: "echo", phase: "Stopping", hasWorld: true}, // still stopping
{name: "foxtrot", phase: "Stopped", hasWorld: true}, // asked to start
{name: "golf", phase: "Stopped", hasWorld: true}, // still reports ready
{name: "hotel", phase: "Stopped", hasWorld: true}, // not reconciled yet: the desired state shows
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("worlds =\n%+v\nwant\n%+v", got, want)
}
}
func TestBackupNowStopsWhenInterrupted(t *testing.T) {
t.Run("between worlds", func(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
g.ctx = ctx
g.onAct = func(e string) {
if e == "request alpha" {
cancel()
}
}
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
want := " alpha: backing up\n alpha: archived in 0s\n" +
"Interrupted: a backup Job already started runs to its end.\n" +
"1 backed up.\n" + bnOffsite
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
})
t.Run("while a Job runs", func(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"))
g.states["alpha"] = []string{"running"}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
g.ctx = ctx
g.onAct = func(string) { cancel() }
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if g.jobPolls["alpha"] != 1 {
t.Fatalf("polled the Job %d times after the interrupt, want 1", g.jobPolls["alpha"])
}
if !strings.Contains(g.out.String(), " alpha: failed: context canceled\nInterrupted: a backup Job already started runs to its end.\n0 backed up, 1 failed.\n") {
t.Fatalf("output =\n%s", g.out.String())
}
})
t.Run("while a server stops", func(t *testing.T) {
g := newBNRig(runningWorld("bravo"))
g.stopAfter["bravo"] = -1
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
g.ctx = ctx
g.onAct = func(string) { cancel() }
if code := g.run(nil, true, true); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if g.polls["bravo"] != 1 {
t.Fatalf("polled bravo %d times after the interrupt, want 1", g.polls["bravo"])
}
_, run, _ := strings.Cut(g.out.String(), bnStopping)
want := " bravo: stopping\n bravo: failed: context canceled\n" +
"Interrupted: a backup Job already started runs to its end.\n" +
"0 backed up, 1 failed.\n" +
"Left stopped: bravo. Start them from the panel when you are done.\n"
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
})
}
func TestBackupNowReportsClusterErrors(t *testing.T) {
g := newBNRig(runningWorld("bravo"))
g.stopErr["bravo"] = errors.New("the apiserver is gone")
if code := g.run(nil, true, true); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
_, run, _ := strings.Cut(g.out.String(), bnStopping)
// The stop was asked for, so the server stays stopped whatever the poll said.
want := " bravo: stopping\n bravo: failed: the apiserver is gone\n0 backed up, 1 failed.\n" +
"Left stopped: bravo. Start them from the panel when you are done.\n"
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
for _, tc := range []struct {
call int
events []string
}{
{1, nil}, // before the request: nothing is asked for
{2, []string{"request alpha"}}, // the first poll after it
} {
g := newBNRig(stoppedWorld("alpha"))
g.jobsFail["alpha"] = tc.call
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("call %d: exit = %d, want 1", tc.call, code)
}
if !reflect.DeepEqual(g.events, tc.events) {
t.Fatalf("call %d: events = %v, want %v", tc.call, g.events, tc.events)
}
if !strings.Contains(g.out.String(), " alpha: failed: list its backup Jobs: the apiserver is gone\n") {
t.Fatalf("call %d: output =\n%s", tc.call, g.out.String())
}
}
}
cmd/felis/backupnow_test.go
+3 -13
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -12,6 +11,7 @@ import (
"testing"
"time"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
corev1 "k8s.io/api/core/v1"
@@ -28,15 +28,9 @@ func internalAPIObjs(clusterIP, token string) []client.Object {
ObjectMeta: metav1.ObjectMeta{Name: platform.APIInternalServiceName, Namespace: bgControlNS},
Spec: corev1.ServiceSpec{ClusterIP: clusterIP},
},
// The console presents the ops token; the proxy's felis-service-token sits
// beside it and must not be the one picked up.
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "felis-ops-token", Namespace: bgControlNS},
Data: map[string][]byte{"token": []byte(token)},
},
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "felis-service-token", Namespace: bgControlNS},
Data: map[string][]byte{"token": []byte("proxy-" + token)},
ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: bgControlNS},
Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
},
}
}
@@ -153,10 +147,6 @@ func TestRequestBackup(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "unreachable") {
t.Fatalf("err = %v, want an 'unreachable' transport error", err)
}
// backup-now ends its run on this error alone.
if !errors.Is(err, errBackupAPIUnreachable) {
t.Fatalf("err = %v, want it to wrap errBackupAPIUnreachable", err)
}
})
}
cmd/felis/bootstrap_assets.go
+2 -9
View File
@@ -9,17 +9,10 @@ import (
func cmdBootstrapAssets(args []string, stdout, stderr io.Writer) int {
if len(args) != 1 {
fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack|config-keys")
fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack")
return 2
}
switch args[0] {
case "config-keys":
// Optional keys the installer may emit; older binaries reject this verb.
_, err := fmt.Fprintln(stdout, "velocity.game_version")
if err != nil {
fmt.Fprintf(stderr, "felis bootstrap-assets: write: %v\n", err)
return 1
}
case "crd":
crd, err := felis.MinecraftServerCRD()
if err != nil {
@@ -39,7 +32,7 @@ func cmdBootstrapAssets(args []string, stdout, stderr io.Writer) int {
return 1
}
default:
fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack|config-keys")
fmt.Fprintln(stderr, "felis bootstrap-assets: usage: felis bootstrap-assets crd|game-stack")
return 2
}
return 0
cmd/felis/breakglass.go
+121 -96
View File
@@ -11,14 +11,12 @@ import (
"flag"
"fmt"
"io"
"net/url"
"os"
"strings"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
tea "github.com/charmbracelet/bubbletea"
@@ -35,15 +33,13 @@ import (
//
// Root is necessary but NOT sufficient for accountability: root is machine
// authority, not a human identity, so the console additionally captures WHO is
// breaking the glass. When a staff account already exists the operator names one
// and types the one-time code the console mails to its verified address
// (breakglass_otp.go); that account is then the accountable actor. When no code can
// be sent or proven, the typed OVERRIDE proceeds as the OS user and the audit row
// says why. When no staff account exists yet it bootstraps the first Owner and
// attributes the act to the OS user. The audit row records which of these
// happened. This attribution is best-effort, not tamper-proof — whoever runs this
// is root and can edit Postgres directly — but it produces an honest trail for an
// honest operator, which is the point.
// breaking the glass. When a staff account already exists it asks the operator to
// authenticate as an existing admin (the verified identity is the accountable
// actor); when none exists yet it bootstraps the first Owner from the typed
// credential and attributes the act to the OS user. The audit row records the
// difference. This attribution is best-effort, not tamper-proof — whoever runs
// this is root and can edit Postgres directly — but it produces an honest trail
// for an honest operator, which is the point.
//
// When a staff account already exists the console opens on a thin top-level menu
// (menuModel) so that operations are peers, not tails of one wizard. Two account
@@ -66,7 +62,7 @@ import (
// suspension for the interactive `cloudflared tunnel login` browser consent.
// breakGlassOverrideToken is the literal an operator must type to proceed when no
// admin could be verified by a mailed code. Requiring an explicit, deliberate word (not a
// admin credential could be verified. Requiring an explicit, deliberate word (not a
// bare Enter) keeps the unverified root override from happening by reflex.
const breakGlassOverrideToken = "OVERRIDE"
@@ -94,9 +90,11 @@ type ownerStore interface {
// role=owner identity (migration 0011 adds that role); the two are the only
// staff roles.
InsertOperator(ctx context.Context, id, username, email string) error
// CompleteOwnerSetup creates or resumes the first panel login atomically.
CompleteOwnerSetup(ctx context.Context, newUserID string, now time.Time,
tokenHash string, tokenExpiresAt time.Time) (userID, username string, err error)
// CompleteOwnerSetup atomically consumes the in-game link code, creates or
// promotes the bound Owner, enables local auth, and stores the one-time setup
// token. A failure rolls all four writes back so setup is always retryable.
CompleteOwnerSetup(ctx context.Context, newUserID, code string, now time.Time,
tokenHash string, tokenExpiresAt time.Time) (userID, mcUUID, authSource string, err error)
SetSetting(ctx context.Context, key string, value []byte) error
// Audit records the break-glass accountability row.
Audit(ctx context.Context, e api.AuditEntry) error
@@ -144,7 +142,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
repo := api.NewPGRepo(drv.DB())
// Decide bootstrap (no admin yet → typed credential mints the first Owner) vs
// recovery (an admin exists → the operator proves one with a mailed code) BEFORE the
// recovery (an admin exists → the operator must authenticate as one) BEFORE the
// alt-screen TUI takes over, so a database fault surfaces as a plain error.
adminExists, err := repo.AdminExists(ctx)
if err != nil {
@@ -152,12 +150,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
return 1
}
// Recovery mails its code through [smtp]; the relay is opened only if a code is
// asked for.
host, _ := os.Hostname()
recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, hostSMTPPasswordPath, platform.DefaultControlNamespace), host: host}
res, err := runBreakGlassTUI(ctx, repo, cfg.Database, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists, recovery)
res, err := runBreakGlassTUI(ctx, repo, cfg.Database.URL, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists)
if err != nil {
fmt.Fprintf(stderr, "felis breakGlass: %v\n", err)
return 1
@@ -181,9 +174,6 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stdout, "\nfelis breakGlass: Owner account %q provisioned; local session sign-in is ENABLED.\n", res.username)
}
fmt.Fprintf(stdout, "Recorded as %q (mode: %s, os user: %s).\n", res.accountable, res.mode, res.osUser)
if res.mode == "root_override" {
fmt.Fprintln(stdout, "No admin was proven by an email code; the audit row records this run as an unverified root override and why.")
}
if res.setupTokenURL != "" {
fmt.Fprintf(stdout, "One-time setup URL (opens a lockdown session to verify email / enroll passkey):\n\n %s\n\n", res.setupTokenURL)
}
@@ -268,6 +258,29 @@ func newOwnerID() string {
return "usr-" + hex.EncodeToString(b[:])
}
// authenticateAdmin resolves a typed admin username for recovery-mode attribution.
// Password verification is gone (passwordless design); Phase 3 replaces this with
// email-OTP recovery. For now it confirms the named admin exists.
func authenticateAdmin(ctx context.Context, s ownerStore, username string) (matched string, ok bool, err error) {
username = strings.TrimSpace(username)
if username == "" {
return "", false, nil
}
u, err := s.UserByUsername(ctx, username)
if errors.Is(err, api.ErrNotFound) {
return "", false, nil
}
if err != nil {
return "", false, err
}
// Staff means admin OR owner: recovery attribution must accept the Owner (the
// primary break-glass identity), not just plain admins.
if u.Role != "admin" && u.Role != "owner" {
return "", false, nil
}
return u.Username, true, nil
}
// provisionOwner mints or resets the single Owner account direct-to-Postgres,
// passwordless. The account is role=owner with no password — the Owner completes
// passwordless login setup via the web setup-token flow after `felis setup`.
@@ -358,16 +371,12 @@ type breakGlassOp struct {
ownerUsername string
ownerEmail string
attemptedAdmin string // recovery / override: the admin username the operator typed
verifiedBy string // recovery: how the admin was proven (verifiedByEmailOTP)
codeSentTo string // recovery: the address the proving code went to
otpSkipped string // root_override: why no code proved an admin (otpSkip*)
otpSkipDetail string // root_override: what failed, when something did
}
// breakGlassOutcome is what performBreakGlass reports back to the TUI.
type breakGlassOutcome struct {
setupTokenURL string // non-empty when setup minted a one-time first-login URL
ownerUsername string // panel Owner created or resumed by setup
ownerIdentity string // verified Minecraft UUID for the setup Owner-bind path
auditErr error // non-nil if the accountability row could not be written
}
@@ -407,12 +416,25 @@ func newSetupToken() (raw, hash string, err error) {
return raw, hex.EncodeToString(sum[:]), nil
}
// performSetupOwner establishes panel access under the caller's host-root
// authority. Minecraft identity can be linked later from the authenticated panel.
func performSetupOwner(ctx context.Context, s ownerStore, panelURL, osUser string) (breakGlassOutcome, error) {
base, err := url.Parse(strings.TrimRight(panelURL, "/"))
if err != nil || base.Scheme != "https" || base.Host == "" {
return breakGlassOutcome{}, errors.New("a configured HTTPS operator console is required")
// performSetupMCBind is the `felis setup` Owner-establishment path: the operator
// binds their Minecraft account via a one-time link code the login gate handed
// them in-game, the bound user is promoted to role='owner' (passwordless Owner),
// local auth is enabled, and a one-time setup URL is minted for the first web
// login where the Owner verifies email / enrolls a passkey. adminHostname is the
// operator-console host the URL points at (op.console.<root>): the Owner is staff,
// so first-run onboarding belongs on the operator face, not the player panel. The
// passkey verifier's RP id is the panel host, but its permitted origins now include
// op.console (cmd/felis/api.go), so enrollment on op.console is a valid ceremony —
// one binding that works on both faces. osUser is recorded as the accountable actor.
//
// Local auth is as load-bearing here as it is in break-glass, and for a sharper
// reason: an MC-bound Owner has no password AND no email, so the setup token is
// their ONLY door. CompleteOwnerSetup therefore commits the identity bind, auth
// toggle, and token together; any failed write leaves the link code retryable.
func performSetupMCBind(ctx context.Context, s ownerStore, code, adminHostname, osUser string) (breakGlassOutcome, error) {
code = strings.TrimSpace(strings.ToUpper(code))
if code == "" {
return breakGlassOutcome{}, errors.New("link code is required")
}
newID := newOwnerID()
if newID == "" {
@@ -423,21 +445,48 @@ func performSetupOwner(ctx context.Context, s ownerStore, panelURL, osUser strin
return breakGlassOutcome{}, err
}
now := time.Now()
userID, username, err := s.CompleteOwnerSetup(ctx, newID, now, hash, now.Add(setupTokenTTL))
out := breakGlassOutcome{ownerUsername: username}
_, mcUUID, authSource, err := s.CompleteOwnerSetup(
ctx, newID, code, now, hash, now.Add(setupTokenTTL))
if err != nil {
return out, err
return breakGlassOutcome{}, fmt.Errorf("complete owner setup: %w", err)
}
base.Path = "/setup"
base.RawQuery = url.Values{"token": {raw}}.Encode()
out.setupTokenURL = base.String()
blob, _ := json.Marshal(map[string]string{"os_user": osUser, "user_id": userID, "username": username})
out.auditErr = s.Audit(ctx, api.AuditEntry{
Actor: osUser, Source: "setup", Action: "setup.owner_login", Payload: blob,
})
// The load-bearing writes committed together above. Accountability remains
// best-effort: an unhappy audit sink never costs the operator their install.
out := breakGlassOutcome{
ownerIdentity: mcUUID,
auditErr: auditSetupMCBind(ctx, s, osUser, mcUUID, authSource),
}
host := strings.TrimSpace(adminHostname)
if host == "" {
host = "op.console.localhost"
}
out.setupTokenURL = "https://" + host + "/setup?token=" + raw
return out, nil
}
// auditSetupMCBind records who claimed the Owner seat at setup. It carries the
// Minecraft identity rather than a username because that IS the evidence: the
// login gate only issues a link code to a player it authenticated, so mc_uuid +
// auth_source say which account was verified and by whom. Actor is the OS user who
// ran `felis setup` — honest attribution, not proof (root can edit the row).
func auditSetupMCBind(ctx context.Context, s ownerStore, osUser, mcUUID, authSource string) error {
blob, err := json.Marshal(map[string]any{
"mode": "setup",
"os_user": osUser,
"mc_uuid": mcUUID,
"auth_source": authSource,
})
if err != nil {
return err
}
return s.Audit(ctx, api.AuditEntry{
Actor: osUser,
Source: "setup",
Action: "setup.owner_bind",
Payload: blob,
})
}
// auditBreakGlass writes the break-glass accountability row. The actor is the
// resolved human identity (a verified admin in recovery, the OS user otherwise);
// the payload carries the full who/what/how so an after-the-fact reader can tell a
@@ -445,7 +494,16 @@ func performSetupOwner(ctx context.Context, s ownerStore, panelURL, osUser strin
// does not fail the recovery if this write fails — and intentionally honest: it
// records attribution, it does not prove it (a malicious root can edit the row).
func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
blob, err := json.Marshal(breakGlassPayload(op, "owner"))
payload := map[string]any{
"mode": op.mode,
"owner": op.ownerUsername,
"os_user": op.osUser,
"verified": op.mode == "recovery",
}
if op.attemptedAdmin != "" {
payload["admin_account"] = op.attemptedAdmin
}
blob, err := json.Marshal(payload)
if err != nil {
return err
}
@@ -457,33 +515,6 @@ func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
})
}
// breakGlassPayload is the who/how both account audits carry, with the account the
// run wrote under subjectKey. verified is true only for a run a mailed code proved;
// such a run names the address the code went to, and an override names why no code
// proved anyone.
func breakGlassPayload(op breakGlassOp, subjectKey string) map[string]any {
payload := map[string]any{
"mode": op.mode,
subjectKey: op.ownerUsername,
"os_user": op.osUser,
"verified": op.verifiedBy != "",
}
if op.attemptedAdmin != "" {
payload["admin_account"] = op.attemptedAdmin
}
if op.verifiedBy != "" {
payload["verified_by"] = op.verifiedBy
payload["code_sent_to"] = op.codeSentTo
}
if op.otpSkipped != "" {
payload["otp_skipped"] = op.otpSkipped
if op.otpSkipDetail != "" {
payload["otp_skip_detail"] = op.otpSkipDetail
}
}
return payload
}
// performAddOperator mints a NEW Operator account and records a best-effort
// accountability row. It mirrors performBreakGlass — passwordless — with two
// deliberate differences. (1) It provisions insert-only (provisionOperator), so it
@@ -507,7 +538,16 @@ func performAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) (bre
// break_glass.operator_create action, naming the new account under an "operator" key
// rather than "owner".
func auditAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) error {
blob, err := json.Marshal(breakGlassPayload(op, "operator"))
payload := map[string]any{
"mode": op.mode,
"operator": op.ownerUsername,
"os_user": op.osUser,
"verified": op.mode == "recovery",
}
if op.attemptedAdmin != "" {
payload["admin_account"] = op.attemptedAdmin
}
blob, err := json.Marshal(payload)
if err != nil {
return err
}
@@ -586,31 +626,16 @@ const (
cloudflareAPITokenDocsURL = "https://developers.cloudflare.com/fundamentals/api/how-to/account-owned-token-template/"
)
func runBreakGlassTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, recovery recoveryConfig) (breakGlassResult, error) {
return runConsoleTUI(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass, recovery)
func runBreakGlassTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass)
}
// runSetupTUI configures deployment before issuing the first panel login link.
func runSetupTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
return runConsoleRoot(newConsoleRoot(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup, recoveryConfig{}))
func runSetupTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup)
}
// newConsoleRoot is the console's root model as the host runs it: the summary
// reads this host's alert route.
func newConsoleRoot(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) *rootModel {
rm := newRootModel(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
rm.recovery = recovery
rm.alertRoute = func(ctx context.Context) alertRoute {
return hostAlertRoute(ctx, hostSetupConfigPath, db.URL, defaultHeartbeatFile)
}
return rm
}
func runConsoleTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) (breakGlassResult, error) {
return runConsoleRoot(newConsoleRoot(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode, recovery))
}
func runConsoleRoot(rm *rootModel) (breakGlassResult, error) {
func runConsoleTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) (breakGlassResult, error) {
rm := newRootModel(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
final, err := tea.NewProgram(rm, tea.WithAltScreen()).Run()
if err != nil {
return breakGlassResult{}, err
cmd/felis/breakglass_otp.go
-259
View File
@@ -1,259 +0,0 @@
package main
import (
"context"
"crypto/rand"
"crypto/subtle"
"errors"
"fmt"
"math/big"
"os"
"strings"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/platform"
)
// Recovery mode proves who is breaking the glass (#13). Naming a staff account is
// where it starts: the console then mails a one-time code to that account's verified
// address, and only that code, typed within recoveryCodeTTL, makes the run a
// recovery attributed to the account. Every other ending — no such account, no
// verified address, no relay, a send that fails, a wrong or late code, or the
// operator giving up on the mail — leads to the typed OVERRIDE, which the audit row
// records as an unverified root_override together with the reason (otp_skipped).
// The code goes through the same [smtp] relay as the panel's login codes, so with
// that relay down recovery still works, as an override that says why.
const (
recoveryCodeTTL = 10 * time.Minute
recoveryCodeAttempts = 5
)
// The reasons a run fell back to the override, recorded as otp_skipped.
const (
otpSkipUnknownAdmin = "unknown_admin"
otpSkipNoVerifiedEmail = "no_verified_email"
otpSkipNoRelay = "no_relay"
otpSkipSendFailed = "send_failed"
otpSkipCodeExpired = "code_expired"
otpSkipCodeRejected = "code_rejected"
otpSkipByOperator = "operator_skipped"
)
// verifiedByEmailOTP is the audit's verified_by for a recovery the mailed code proved.
const verifiedByEmailOTP = "email_otp"
// recoveryMailer is the one relay call a recovery code needs; *mail.SMTP has it.
type recoveryMailer interface {
SendNotice(ctx context.Context, email, subject, body string) error
}
// recoveryConfig is what the console needs to mail a recovery code. open resolves
// the relay only when a code is about to go out, so a console used to halt a server
// never touches [smtp] or the cluster; its error says why no relay is available.
// host names this machine in the mail. The zero value has no relay.
type recoveryConfig struct {
open func(ctx context.Context) (recoveryMailer, error)
host string
now func() time.Time
}
func (r recoveryConfig) clock() time.Time {
if r.now != nil {
return r.now()
}
return time.Now()
}
// recoveryCode is one mailed code: its value, when it stops working, and how many
// wrong codes were typed against it.
type recoveryCode struct {
value string
expires time.Time
failures int
}
func newRecoveryCode(now time.Time) (*recoveryCode, error) {
n, err := rand.Int(rand.Reader, big.NewInt(1_000_000))
if err != nil {
return nil, fmt.Errorf("generate recovery code: %w", err)
}
return &recoveryCode{value: fmt.Sprintf("%06d", n.Int64()), expires: now.Add(recoveryCodeTTL)}, nil
}
type codeVerdict int
const (
codeAccepted codeVerdict = iota
codeWrong
codeExpired
codeExhausted
)
// check compares a typed code in constant time. Each wrong code counts; the one
// that reaches recoveryCodeAttempts exhausts the code, which then accepts nothing,
// and neither does an expired one.
func (c *recoveryCode) check(typed string, now time.Time) codeVerdict {
if c.failures >= recoveryCodeAttempts {
return codeExhausted
}
if !now.Before(c.expires) {
return codeExpired
}
if subtle.ConstantTimeCompare([]byte(strings.TrimSpace(typed)), []byte(c.value)) == 1 {
return codeAccepted
}
c.failures++
if c.failures >= recoveryCodeAttempts {
return codeExhausted
}
return codeWrong
}
func (c *recoveryCode) attemptsLeft() int { return recoveryCodeAttempts - c.failures }
// recoveryStart is where naming an admin led: a code on its way to that admin, or
// the reason the run has to fall back to the override.
type recoveryStart struct {
admin *api.StaffUser // the named staff account; nil when none matched
code *recoveryCode // set when the code went out
skip string // otpSkip* when it did not
detail string // what failed, for the override screen and the audit row
}
// resolveAdmin loads the staff account (admin or owner) a typed username names, or
// nil when there is none.
func resolveAdmin(ctx context.Context, s ownerStore, username string) (*api.StaffUser, error) {
username = strings.TrimSpace(username)
if username == "" {
return nil, nil
}
u, err := s.UserByUsername(ctx, username)
if errors.Is(err, api.ErrNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
// Staff means admin or owner: the Owner is the primary break-glass identity.
if u.Role != "admin" && u.Role != "owner" {
return nil, nil
}
return u, nil
}
// beginRecovery resolves the named admin and mails it a recovery code. Only a
// datastore or entropy fault is an error; every other way the code cannot go out is
// a recoveryStart with skip set.
func beginRecovery(ctx context.Context, s ownerStore, rc recoveryConfig, username, osUser string, op bgOperation) (recoveryStart, error) {
admin, err := resolveAdmin(ctx, s, username)
if err != nil {
return recoveryStart{}, err
}
if admin == nil {
return recoveryStart{skip: otpSkipUnknownAdmin}, nil
}
st := recoveryStart{admin: admin}
// An address nobody ever proved vouches for nobody.
email := strings.TrimSpace(admin.Email)
if email == "" || !admin.EmailVerified {
st.skip = otpSkipNoVerifiedEmail
return st, nil
}
if rc.open == nil {
st.skip, st.detail = otpSkipNoRelay, "this console has no mail relay"
return st, nil
}
relay, err := rc.open(ctx)
if err != nil {
st.skip, st.detail = otpSkipNoRelay, err.Error()
return st, nil
}
code, err := newRecoveryCode(rc.clock())
if err != nil {
return recoveryStart{}, err
}
sendCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
subject, body := recoveryMail(code.value, rc.host, osUser, admin.Username, op)
if err := relay.SendNotice(sendCtx, email, subject, body); err != nil {
st.skip, st.detail = otpSkipSendFailed, err.Error()
return st, nil
}
st.code = code
return st, nil
}
// recoveryMail words the code mail. It says where, by whom and for what the console
// was opened, so an admin who did not ask for it learns that root on that machine is
// in someone else's hands.
func recoveryMail(code, host, osUser, admin string, op bgOperation) (subject, body string) {
what, whatZH := "reset the Owner account", "重置 Owner 账号"
if op == bgAddOperator {
what, whatZH = "add an Operator account", "添加 Operator 账号"
}
if host == "" {
host = "the Felis host"
}
minutes := int(recoveryCodeTTL / time.Minute)
subject = "Felis break-glass recovery code / 紧急恢复验证码"
body = fmt.Sprintf(`Someone with root on %[1]s (OS user %[2]s) opened felis breakGlass and named your staff account %[3]q to %[4]s.
Recovery code: %[6]s
It works for %[7]d minutes.
If this was not you, root on that machine is in someone else's hands: change its credentials and read the audit log for break_glass entries.
有人在 %[1]s 上以 root 身份（系统用户 %[2]s）打开了 felis breakGlass，指名你的管理员账号 %[3]q 来%[5]s。
恢复验证码：%[6]s
%[7]d 分钟内有效。
如果不是你本人，这台机器的 root 已落入他人之手：请更换它的凭据，并查看审计日志中的 break_glass 记录。
`, host, osUser, admin, what, whatZH, code, minutes)
return subject, body
}
// maskEmail keeps the first character of the local part and the domain, enough for
// the operator to recognise the address without putting it on screen whole.
func maskEmail(email string) string {
at := strings.LastIndex(email, "@")
if at <= 0 {
return "***"
}
return email[:1] + strings.Repeat("*", max(at-1, 3)) + email[at:]
}
// hostRecoveryMailer opens the [smtp] relay from the host the way the watchdog does:
// the password is the env var password_ref names when that is set, else the host
// copy at passwordPath, else the felis-smtp Secret, whose absence means a relay
// without AUTH. The cluster is reached only when the host copy is missing, so a
// break-glass on a host whose k3s is down still gets its code.
func hostRecoveryMailer(c config.SMTPConfig, passwordPath, controlNS string) func(context.Context) (recoveryMailer, error) {
return func(ctx context.Context) (recoveryMailer, error) {
if strings.TrimSpace(c.Host) == "" {
return nil, errors.New("[smtp] is not configured in felis.toml")
}
if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" {
return smtpRelay(c, os.Getenv(ref)), nil
}
if password, ok, err := readHostCredential(passwordPath); err != nil {
return nil, fmt.Errorf("read the relay password: %w", err)
} else if ok {
return smtpRelay(c, password), nil
}
cl, err := buildSystemServerClient()
if err != nil {
return nil, fmt.Errorf("reach the cluster for the relay password: %w", err)
}
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
password, err := smtpSecretPassword(ctx, cl, controlNS)
if err != nil {
return nil, fmt.Errorf("read the relay password from %s/%s: %w", controlNS, platform.SMTPSecretName, err)
}
return smtpRelay(c, password), nil
}
}
cmd/felis/breakglass_otp_test.go
-498
View File
@@ -1,498 +0,0 @@
package main
import (
"context"
"errors"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/platform"
tea "github.com/charmbracelet/bubbletea"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)
// These tests cover the recovery proof (#13): the mailed code's rules, where
// naming an admin leads, what the mail says, how the console walks from a name to
// a proven (or overridden) run, and what the audit row then records. No mail
// leaves the process: the relay is a fake that keeps what it was handed.
type sentMail struct{ to, subject, body string }
type fakeRelay struct {
sent []sentMail
err error
}
func (r *fakeRelay) SendNotice(_ context.Context, to, subject, body string) error {
if r.err != nil {
return r.err
}
r.sent = append(r.sent, sentMail{to, subject, body})
return nil
}
// sentCode pulls the code out of the one mail the relay carried.
func (r *fakeRelay) sentCode(t *testing.T) string {
t.Helper()
if len(r.sent) != 1 {
t.Fatalf("relay carried %d mails, want 1", len(r.sent))
}
_, after, ok := strings.Cut(r.sent[0].body, "Recovery code: ")
if !ok || len(after) < 6 {
t.Fatalf("mail carries no recovery code:\n%s", r.sent[0].body)
}
return after[:6]
}
func relayConfig(r *fakeRelay, now func() time.Time) recoveryConfig {
return recoveryConfig{
open: func(context.Context) (recoveryMailer, error) { return r, nil },
host: "felis-host-1",
now: now,
}
}
func verifiedAdmin(username, email string) *api.StaffUser {
return &api.StaffUser{ID: "usr-" + username, Username: username, Role: "owner", Email: email, EmailVerified: true}
}
func TestRecoveryCodeRules(t *testing.T) {
t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
t.Run("a fresh code is six digits and lives for the TTL", func(t *testing.T) {
seen := map[string]bool{}
for i := 0; i < 20; i++ {
c, err := newRecoveryCode(t0)
if err != nil {
t.Fatal(err)
}
if !isRecoveryCodeShape(c.value) {
t.Fatalf("code %q is not six digits", c.value)
}
if !c.expires.Equal(t0.Add(recoveryCodeTTL)) {
t.Fatalf("expires = %v, want %v", c.expires, t0.Add(recoveryCodeTTL))
}
seen[c.value] = true
}
if len(seen) < 2 {
t.Error("twenty codes were all the same")
}
})
t.Run("the right code is accepted, surrounding space ignored", func(t *testing.T) {
c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
if v := c.check(" 042917 ", t0); v != codeAccepted {
t.Errorf("check = %v, want accepted", v)
}
})
t.Run("wrong codes count down and the last one exhausts it", func(t *testing.T) {
c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
for i := 1; i < recoveryCodeAttempts; i++ {
if v := c.check("000000", t0); v != codeWrong {
t.Fatalf("wrong code %d: check = %v, want wrong", i, v)
}
if left := c.attemptsLeft(); left != recoveryCodeAttempts-i {
t.Fatalf("after %d wrong codes attemptsLeft = %d, want %d", i, left, recoveryCodeAttempts-i)
}
}
if v := c.check("000000", t0); v != codeExhausted {
t.Fatalf("wrong code %d: check = %v, want exhausted", recoveryCodeAttempts, v)
}
if v := c.check("042917", t0); v != codeExhausted {
t.Errorf("the right code after exhaustion: check = %v, want exhausted", v)
}
})
t.Run("an expired code accepts nothing", func(t *testing.T) {
c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
if v := c.check("042917", t0.Add(recoveryCodeTTL-time.Second)); v != codeAccepted {
t.Fatalf("a second before expiry: check = %v, want accepted", v)
}
c = &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
if v := c.check("042917", t0.Add(recoveryCodeTTL)); v != codeExpired {
t.Errorf("at expiry: check = %v, want expired", v)
}
})
}
func TestBeginRecovery(t *testing.T) {
ctx := context.Background()
t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
clock := func() time.Time { return t0 }
t.Run("mails a code to the named admin's verified address", func(t *testing.T) {
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
r := &fakeRelay{}
st, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgAddOperator)
if err != nil {
t.Fatal(err)
}
if st.code == nil || st.skip != "" {
t.Fatalf("start = %+v, want a code and no skip", st)
}
if st.admin == nil || st.admin.Username != "root" {
t.Fatalf("start.admin = %+v, want root", st.admin)
}
if got := r.sentCode(t); got != st.code.value {
t.Errorf("mailed code %q, want the code the console checks (%q)", got, st.code.value)
}
m := r.sent[0]
if m.to != "[email protected]" {
t.Errorf("mail went to %q, want [email protected]", m.to)
}
for _, want := range []string{"felis-host-1", "OS user alice", `"root"`, "add an Operator account", "添加 Operator 账号", "10 minutes"} {
if !strings.Contains(m.body, want) {
t.Errorf("mail body lacks %q:\n%s", want, m.body)
}
}
if !st.code.expires.Equal(t0.Add(recoveryCodeTTL)) {
t.Errorf("code expires %v, want %v", st.code.expires, t0.Add(recoveryCodeTTL))
}
})
t.Run("the Owner reset is named as such", func(t *testing.T) {
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
r := &fakeRelay{}
if _, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgProvisionOwner); err != nil {
t.Fatal(err)
}
if body := r.sent[0].body; !strings.Contains(body, "reset the Owner account") || !strings.Contains(body, "重置 Owner 账号") {
t.Errorf("mail body does not name the Owner reset:\n%s", body)
}
})
// Each way the code cannot go out ends in a skip reason and no mail.
unverified := verifiedAdmin("root", "[email protected]")
unverified.EmailVerified = false
noEmail := verifiedAdmin("root", "")
cases := []struct {
name string
user *api.StaffUser
rc func(r *fakeRelay) recoveryConfig
skip string
detail string
wantAdmin bool
relayError error
}{
{name: "unknown admin", user: nil, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipUnknownAdmin},
{name: "unverified address", user: unverified, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
{name: "no address", user: noEmail, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
{name: "no relay wired", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig { return recoveryConfig{} }, skip: otpSkipNoRelay, detail: "this console has no mail relay", wantAdmin: true},
{name: "relay cannot open", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig {
return recoveryConfig{open: func(context.Context) (recoveryMailer, error) {
return nil, errors.New("[smtp] is not configured in felis.toml")
}}
}, skip: otpSkipNoRelay, detail: "[smtp] is not configured in felis.toml", wantAdmin: true},
{name: "send fails", user: verifiedAdmin("root", "[email protected]"), rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) },
skip: otpSkipSendFailed, detail: "554 relay refused", wantAdmin: true, relayError: errors.New("554 relay refused")},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
f := &fakeOwnerStore{users: map[string]*api.StaffUser{}}
if tc.user != nil {
f.users["root"] = tc.user
}
r := &fakeRelay{err: tc.relayError}
st, err := beginRecovery(ctx, f, tc.rc(r), "root", "alice", bgProvisionOwner)
if err != nil {
t.Fatal(err)
}
if st.code != nil || st.skip != tc.skip || st.detail != tc.detail {
t.Errorf("start = {code:%v skip:%q detail:%q}, want no code, skip %q, detail %q", st.code, st.skip, st.detail, tc.skip, tc.detail)
}
if (st.admin != nil) != tc.wantAdmin {
t.Errorf("start.admin = %+v, want present=%v", st.admin, tc.wantAdmin)
}
if len(r.sent) != 0 {
t.Errorf("relay carried %d mails, want none", len(r.sent))
}
})
}
t.Run("a datastore fault is an error", func(t *testing.T) {
f := &fakeOwnerStore{userErr: errors.New("db down")}
if _, err := beginRecovery(ctx, f, relayConfig(&fakeRelay{}, clock), "root", "alice", bgProvisionOwner); err == nil {
t.Fatal("want the store fault")
}
})
}
func TestMaskEmail(t *testing.T) {
for in, want := range map[string]string{
"[email protected]": "a****@example.com",
"[email protected]": "a***@example.com",
"@example.com": "***",
"nonsense": "***",
} {
if got := maskEmail(in); got != want {
t.Errorf("maskEmail(%q) = %q, want %q", in, got, want)
}
}
}
func TestHostRecoveryMailer(t *testing.T) {
ctx := context.Background()
t.Run("no [smtp] host is no relay", func(t *testing.T) {
_, err := hostRecoveryMailer(config.SMTPConfig{}, hostSMTPPasswordPath, "felis")(ctx)
if err == nil || !strings.Contains(err.Error(), "[smtp]") {
t.Fatalf("err = %v, want it to name [smtp]", err)
}
})
t.Run("the password_ref env var supplies the password", func(t *testing.T) {
t.Setenv("FELIS_TEST_RELAY_PW", "from-env")
off := false
c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_RELAY_PW", RequireTLS: &off}
got, err := hostRecoveryMailer(c, hostSMTPPasswordPath, "felis")(ctx)
if err != nil {
t.Fatal(err)
}
relay, ok := got.(*mail.SMTP)
if !ok {
t.Fatalf("relay is %T, want *mail.SMTP", got)
}
if relay.Password != "from-env" || relay.Host != "mail.example.com" || relay.Port != 2525 || relay.From != "[email protected]" || relay.Username != "felis" || relay.RequireTLS {
t.Errorf("relay = %+v, want the [smtp] fields with the env password and TLS as configured", *relay)
}
})
}
func TestSMTPSecretPassword(t *testing.T) {
ctx := context.Background()
scheme := haltScheme(t)
t.Run("reads the password key", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
Data: map[string][]byte{platform.SMTPSecretPasswordKey: []byte("s3cret")},
}).Build()
if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "s3cret" {
t.Errorf("smtpSecretPassword = (%q, %v), want (s3cret, nil)", pw, err)
}
})
t.Run("a missing Secret is a relay without AUTH", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(scheme).Build()
if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "" {
t.Errorf("smtpSecretPassword = (%q, %v), want (\"\", nil)", pw, err)
}
})
t.Run("any other read failure is an error", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
return apierrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, platform.SMTPSecretName, errors.New("rbac"))
},
}).Build()
if _, err := smtpSecretPassword(ctx, cl, "felis"); err == nil {
t.Fatal("want the read failure")
}
})
}
// recoveryModel is an Owner-reset console for admin "root" (verified address
// [email protected]), run by OS user alice, with the fake relay behind it.
func recoveryModel(t *testing.T, f *fakeOwnerStore, r *fakeRelay, now func() time.Time) *ownerModel {
t.Helper()
if f.users == nil {
f.users = map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}
}
return newOwnerModel(context.Background(), f, "alice", true).withRecovery(relayConfig(r, now))
}
// nameAdmin submits the admin-name form and feeds the result of the send back in.
func nameAdmin(t *testing.T, m *ownerModel, name string) *ownerModel {
t.Helper()
m.authUser = name
_, cmd := m.onFormComplete()
if m.step != owWorking {
t.Fatalf("after naming the admin step = %v, want owWorking", m.step)
}
msg := findMsg[owAuthMsg](t, cmd)
next, _ := m.Update(msg)
return next.(*ownerModel)
}
// findMsg runs a (possibly batched) command and returns the first T it produces.
func findMsg[T any](t *testing.T, cmd tea.Cmd) T {
t.Helper()
var zero T
if cmd == nil {
t.Fatalf("no command, want one producing %T", zero)
}
switch msg := cmd().(type) {
case T:
return msg
case tea.BatchMsg:
for _, c := range msg {
if c == nil {
continue
}
if got, ok := c().(T); ok {
return got
}
}
}
t.Fatalf("command produced no %T", zero)
return zero
}
// typeCode submits the code form with typed.
func typeCode(t *testing.T, m *ownerModel, typed string) {
t.Helper()
if m.step != owCode {
t.Fatalf("step = %v, want owCode", m.step)
}
m.codeInput = typed
m.onFormComplete()
}
// provisionAudit finishes the run as an Owner reset and returns its audit payload.
func provisionAudit(t *testing.T, m *ownerModel, f *fakeOwnerStore) (api.AuditEntry, map[string]any) {
t.Helper()
if m.step != owProvision {
t.Fatalf("step = %v, want owProvision", m.step)
}
m.username = "owner"
msg := m.provisionCmd()().(owProvisionMsg)
if msg.err != nil {
t.Fatalf("provision: %v", msg.err)
}
return auditOf(t, f)
}
func TestRecoveryConsoleFlow(t *testing.T) {
t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
fixed := func() time.Time { return t0 }
t.Run("the mailed code proves the admin and the audit says so", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
typeCode(t, m, r.sentCode(t))
if m.mode != "recovery" || m.accountable != "root" {
t.Fatalf("mode=%q accountable=%q, want recovery attributed to root", m.mode, m.accountable)
}
e, payload := provisionAudit(t, m, f)
if e.Actor != "root" || e.Action != "break_glass.recovery" {
t.Errorf("audit = %+v, want actor=root action=break_glass.recovery", e)
}
if payload["verified"] != true || payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" || payload["os_user"] != "alice" {
t.Errorf("payload = %v, want verified by email_otp to [email protected], os_user alice", payload)
}
})
t.Run("a wrong code asks again, and the last wrong one leads to the override", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
wrong := "000000"
if r.sentCode(t) == wrong {
wrong = "111111"
}
for i := 1; i < recoveryCodeAttempts; i++ {
typeCode(t, m, wrong)
if m.step != owCode || !strings.Contains(m.codeNote, "wrong") {
t.Fatalf("wrong code %d: step=%v note=%q, want the code form again with a note", i, m.step, m.codeNote)
}
}
typeCode(t, m, wrong)
if m.step != owOverride || m.skip != otpSkipCodeRejected {
t.Fatalf("after %d wrong codes step=%v skip=%q, want the override for code_rejected", recoveryCodeAttempts, m.step, m.skip)
}
m.onFormComplete() // OVERRIDE typed
e, payload := provisionAudit(t, m, f)
if e.Actor != "alice" || e.Action != "break_glass.root_override" {
t.Errorf("audit = %+v, want actor=alice action=break_glass.root_override", e)
}
if payload["verified"] != false || payload["otp_skipped"] != otpSkipCodeRejected || payload["admin_account"] != "root" {
t.Errorf("payload = %v, want unverified, otp_skipped=code_rejected, admin_account=root", payload)
}
})
t.Run("a late code leads to the override", func(t *testing.T) {
now := t0
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, func() time.Time { return now }), "root")
now = t0.Add(recoveryCodeTTL)
typeCode(t, m, r.sentCode(t))
if m.step != owOverride || m.skip != otpSkipCodeExpired {
t.Fatalf("step=%v skip=%q, want the override for code_expired", m.step, m.skip)
}
})
t.Run("OVERRIDE at the code prompt goes on unverified, saying the operator skipped", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
typeCode(t, m, breakGlassOverrideToken)
if m.mode != "root_override" || m.accountable != "alice" {
t.Fatalf("mode=%q accountable=%q, want root_override as alice", m.mode, m.accountable)
}
_, payload := provisionAudit(t, m, f)
if payload["verified"] != false || payload["otp_skipped"] != otpSkipByOperator {
t.Errorf("payload = %v, want unverified, otp_skipped=operator_skipped", payload)
}
})
t.Run("a relay failure leads to the override naming it", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{err: errors.New("dial tcp 10.0.0.9:587: connect: connection refused")}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
if m.step != owOverride || m.skip != otpSkipSendFailed {
t.Fatalf("step=%v skip=%q, want the override for send_failed", m.step, m.skip)
}
if reason := m.overrideReason(); !strings.Contains(reason, "connection refused") {
t.Errorf("override reason %q does not name the failure", reason)
}
m.onFormComplete()
_, payload := provisionAudit(t, m, f)
if payload["otp_skipped"] != otpSkipSendFailed || !strings.Contains(payload["otp_skip_detail"].(string), "connection refused") {
t.Errorf("payload = %v, want otp_skipped=send_failed with the relay's error", payload)
}
})
t.Run("esc at the code prompt starts over and forgets the code", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
code := r.sentCode(t)
next, _ := m.Update(key(tea.KeyEsc))
m = next.(*ownerModel)
if m.step != owAuth || m.code != nil || m.admin != nil {
t.Fatalf("after esc step=%v code=%v admin=%v, want owAuth with the attempt forgotten", m.step, m.code, m.admin)
}
// The old code cannot be replayed: the next name mails a new one.
m = nameAdmin(t, m, "root")
if len(r.sent) != 2 {
t.Fatalf("relay carried %d mails, want a second one for the new attempt", len(r.sent))
}
_, fresh, _ := strings.Cut(r.sent[1].body, "Recovery code: ")
if m.code.value != fresh[:6] {
t.Errorf("the console checks %q, want the newly mailed %q (old one was %q)", m.code.value, fresh[:6], code)
}
})
}
func TestRootHandsRecoveryToAccountOperations(t *testing.T) {
for _, op := range []bgOperation{bgProvisionOwner, bgAddOperator} {
m := newTestRoot(true, consoleModeBreakGlass, "")
m.recovery = recoveryConfig{host: "felis-host-1"}
m = drive(t, m, menuChoiceMsg{op: op})
om, ok := m.screen.(*ownerModel)
if !ok {
t.Fatalf("op %v: screen = %T, want *ownerModel", op, m.screen)
}
if om.recovery.host != "felis-host-1" {
t.Errorf("op %v: the account screen has no relay config; its codes could never go out", op)
}
}
}
cmd/felis/breakglass_test.go
+203 -133
View File
@@ -24,14 +24,16 @@ type fakeOwnerStore struct {
settings map[string][]byte
audits []api.AuditEntry
tokens []setupTokenCall
setupIDs []string
redeems []redeemCall
users map[string]*api.StaffUser // keyed by username
admins bool // AdminExists answer
ownerSeat string // OwnerUsername answer: the occupied seat, "" when none
setupUserID string
setupUsername string
onboarded bool
// CompleteOwnerSetup's success result. redeemUserID defaults to the fresh id
// the caller passes (the unlinked-UUID case) when left empty.
redeemUserID string
redeemMCUUID string
redeemAuthSource string
upsertErr error
insertErr error
@@ -57,6 +59,12 @@ type setupTokenCall struct {
expiresAt time.Time
}
// redeemCall records the inputs CompleteOwnerSetup was called with.
type redeemCall struct {
newUserID string
code string
}
func (f *fakeOwnerStore) AdminExists(_ context.Context) (bool, error) {
if f.adminErr != nil {
return false, f.adminErr
@@ -110,31 +118,30 @@ func (f *fakeOwnerStore) InsertOperator(_ context.Context, id, username, email s
return nil
}
// CompleteOwnerSetup models the transaction without any game link code.
func (f *fakeOwnerStore) CompleteOwnerSetup(_ context.Context, newUserID string, _ time.Time,
tokenHash string, expiresAt time.Time) (string, string, error) {
for _, err := range []error{f.redeemErr, f.setErr, f.createTokenErr} {
if err != nil {
return "", "", err
}
// CompleteOwnerSetup models the real all-or-nothing transaction: injected failures
// record none of the redeem, auth-toggle, or setup-token writes.
func (f *fakeOwnerStore) CompleteOwnerSetup(_ context.Context, newUserID, code string, _ time.Time,
tokenHash string, expiresAt time.Time) (string, string, string, error) {
if f.redeemErr != nil {
return "", "", "", f.redeemErr
}
userID, username := f.setupUserID, f.setupUsername
if f.setErr != nil {
return "", "", "", f.setErr
}
if f.createTokenErr != nil {
return "", "", "", f.createTokenErr
}
f.redeems = append(f.redeems, redeemCall{newUserID, code})
userID := f.redeemUserID
if userID == "" {
userID = newUserID
userID = newUserID // unlinked UUID → the fresh id becomes the Owner
}
if username == "" {
username = "owner"
}
if f.onboarded {
return userID, username, api.ErrConflict
}
f.setupIDs = append(f.setupIDs, newUserID)
if f.settings == nil {
f.settings = map[string][]byte{}
}
f.settings[api.LocalAuthEnabledKey] = []byte("true")
f.tokens = append(f.tokens, setupTokenCall{tokenHash, userID, expiresAt})
return userID, username, nil
return userID, f.redeemMCUUID, f.redeemAuthSource, nil
}
func (f *fakeOwnerStore) SetSetting(_ context.Context, key string, value []byte) error {
@@ -262,57 +269,71 @@ func TestEnableLocalAuth(t *testing.T) {
}
}
func TestResolveAdmin(t *testing.T) {
func TestAuthenticateAdmin(t *testing.T) {
ctx := context.Background()
// resolveAdmin only finds the staff account a typed name points at; proving the
// operator holds it is the mailed code's job (beginRecovery).
// Password verification is gone (passwordless design): authenticateAdmin now only
// resolves the named admin so recovery can attribute the audit to a real identity.
// The security boundary is the break-glass root gate, not a typed secret.
t.Run("resolves an existing admin", func(t *testing.T) {
t.Run("resolves an existing admin for attribution", func(t *testing.T) {
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": mkAdmin("root")}}
u, err := resolveAdmin(ctx, f, " root ")
matched, ok, err := authenticateAdmin(ctx, f, "root")
if err != nil {
t.Fatalf("resolveAdmin: %v", err)
t.Fatalf("authenticateAdmin: %v", err)
}
if u == nil || u.Username != "root" {
t.Fatalf("resolveAdmin = %+v, want the root admin", u)
if !ok {
t.Fatal("ok = false, want true for an existing admin")
}
if matched != "root" {
t.Errorf("matched = %q, want root", matched)
}
})
t.Run("a non-admin role is no staff account", func(t *testing.T) {
t.Run("a non-admin role can never attribute a break-glass", func(t *testing.T) {
player := mkAdmin("alice")
player.Role = "user" // a player row is not staff
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"alice": player}}
if u, err := resolveAdmin(ctx, f, "alice"); u != nil || err != nil {
t.Errorf("resolveAdmin(player) = (%+v, %v), want (nil, nil)", u, err)
_, ok, err := authenticateAdmin(ctx, f, "alice")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if ok {
t.Error("ok = true, want false for a non-admin role")
}
})
t.Run("the owner role counts as staff", func(t *testing.T) {
t.Run("the owner role attributes like an admin", func(t *testing.T) {
owner := mkAdmin("root")
owner.Role = "owner" // the platform owner is staff too (migration 0011)
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": owner}}
if u, err := resolveAdmin(ctx, f, "root"); err != nil || u == nil {
t.Fatalf("resolveAdmin(owner) = (%+v, %v), want the owner", u, err)
matched, ok, err := authenticateAdmin(ctx, f, "root")
if err != nil || !ok || matched != "root" {
t.Fatalf("authenticateAdmin(owner) = (%q, %v, %v), want (root, true, nil)", matched, ok, err)
}
})
t.Run("an unknown user is nil, not an error", func(t *testing.T) {
if u, err := resolveAdmin(ctx, &fakeOwnerStore{}, "nobody"); u != nil || err != nil {
t.Errorf("resolveAdmin(unknown) = (%+v, %v), want (nil, nil)", u, err)
t.Run("an unknown user is a non-match, not an error", func(t *testing.T) {
f := &fakeOwnerStore{}
_, ok, err := authenticateAdmin(ctx, f, "nobody")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if ok {
t.Error("ok = true, want false for an unknown user")
}
})
t.Run("an empty username makes no store call", func(t *testing.T) {
t.Run("an empty username is a non-match with no store call", func(t *testing.T) {
f := &fakeOwnerStore{userErr: errors.New("must not be called")}
if u, err := resolveAdmin(ctx, f, " "); u != nil || err != nil {
t.Errorf("empty username: (%+v, %v), want (nil, nil)", u, err)
if _, ok, err := authenticateAdmin(ctx, f, ""); ok || err != nil {
t.Errorf("empty username: ok=%v err=%v, want false,nil", ok, err)
}
})
t.Run("a datastore fault is surfaced", func(t *testing.T) {
f := &fakeOwnerStore{userErr: errors.New("db down")}
if _, err := resolveAdmin(ctx, f, "root"); err == nil {
if _, _, err := authenticateAdmin(ctx, f, "root"); err == nil {
t.Fatal("want error when the store fails")
}
})
@@ -380,8 +401,6 @@ func TestPerformBreakGlass(t *testing.T) {
osUser: "alice",
ownerUsername: "owner",
attemptedAdmin: "root",
verifiedBy: verifiedByEmailOTP,
codeSentTo: "[email protected]",
}
out, err := performBreakGlass(ctx, f, op)
if err != nil {
@@ -406,23 +425,6 @@ func TestPerformBreakGlass(t *testing.T) {
if payload["admin_account"] != "root" {
t.Errorf("payload.admin_account = %v, want root", payload["admin_account"])
}
if payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" {
t.Errorf("payload = %v, want verified_by=email_otp [email protected]", payload)
}
if _, present := payload["otp_skipped"]; present {
t.Error("a proven recovery carries no otp_skipped")
}
})
t.Run("recovery without a proof is recorded unverified", func(t *testing.T) {
f := &fakeOwnerStore{}
op := breakGlassOp{mode: "recovery", accountable: "root", osUser: "alice", ownerUsername: "owner", attemptedAdmin: "root"}
if _, err := performBreakGlass(ctx, f, op); err != nil {
t.Fatalf("performBreakGlass: %v", err)
}
if _, payload := auditOf(t, f); payload["verified"] != false {
t.Errorf("payload.verified = %v, want false: only a mailed code verifies", payload["verified"])
}
})
t.Run("root override records an unverified row attributed to the OS user", func(t *testing.T) {
@@ -433,8 +435,6 @@ func TestPerformBreakGlass(t *testing.T) {
osUser: "alice",
ownerUsername: "owner",
attemptedAdmin: "typo-admin",
otpSkipped: otpSkipSendFailed,
otpSkipDetail: "dial tcp: connection refused",
}
if _, err := performBreakGlass(ctx, f, op); err != nil {
t.Fatalf("performBreakGlass: %v", err)
@@ -450,13 +450,6 @@ func TestPerformBreakGlass(t *testing.T) {
if payload["admin_account"] != "typo-admin" {
t.Errorf("payload.admin_account = %v, want typo-admin", payload["admin_account"])
}
// Why no code proved anyone is part of the record.
if payload["otp_skipped"] != otpSkipSendFailed || payload["otp_skip_detail"] != "dial tcp: connection refused" {
t.Errorf("payload = %v, want otp_skipped=send_failed with its detail", payload)
}
if _, present := payload["verified_by"]; present {
t.Error("an override carries no verified_by")
}
})
t.Run("an audit failure does not fail the recovery", func(t *testing.T) {
@@ -623,8 +616,6 @@ func TestPerformAddOperator(t *testing.T) {
ownerUsername: "ops-jordan",
ownerEmail: "[email protected]",
attemptedAdmin: "root",
verifiedBy: verifiedByEmailOTP,
codeSentTo: "[email protected]",
}
out, err := performAddOperator(ctx, f, op)
if err != nil {
@@ -651,14 +642,14 @@ func TestPerformAddOperator(t *testing.T) {
if _, present := payload["owner"]; present {
t.Error("payload.owner present, want the new account under the operator key")
}
if payload["verified"] != true || payload["admin_account"] != "root" || payload["verified_by"] != verifiedByEmailOTP {
t.Errorf("payload = %v, want verified=true admin_account=root verified_by=email_otp", payload)
if payload["verified"] != true || payload["admin_account"] != "root" {
t.Errorf("payload = %v, want verified=true admin_account=root", payload)
}
})
t.Run("root override records an unverified operator row", func(t *testing.T) {
f := &fakeOwnerStore{}
op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin", otpSkipped: otpSkipUnknownAdmin}
op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin"}
if _, err := performAddOperator(ctx, f, op); err != nil {
t.Fatalf("performAddOperator: %v", err)
}
@@ -666,8 +657,8 @@ func TestPerformAddOperator(t *testing.T) {
t.Fatalf("want 1 insert, got %d", len(f.inserts))
}
_, payload := auditOf(t, f)
if payload["verified"] != false || payload["otp_skipped"] != otpSkipUnknownAdmin {
t.Errorf("payload = %v, want verified=false otp_skipped=unknown_admin", payload)
if payload["verified"] != false {
t.Errorf("payload.verified = %v, want false for root_override", payload["verified"])
}
})
@@ -703,70 +694,149 @@ func TestPerformAddOperator(t *testing.T) {
})
}
func TestPerformSetupOwner(t *testing.T) {
func TestPerformSetupMCBind(t *testing.T) {
ctx := context.Background()
f := &fakeOwnerStore{setupUserID: "usr-owner-1"}
out, err := performSetupOwner(ctx, f, "https://op.console.example.com:30443", "deploybot")
if err != nil {
t.Fatal(err)
}
if out.ownerUsername != "owner" {
t.Fatalf("username = %q", out.ownerUsername)
}
const prefix = "https://op.console.example.com:30443/setup?token="
if !strings.HasPrefix(out.setupTokenURL, prefix) {
t.Fatalf("URL = %q", out.setupTokenURL)
}
if len(f.tokens) != 1 || f.tokens[0].userID != "usr-owner-1" {
t.Fatalf("tokens = %+v", f.tokens)
}
raw := strings.TrimPrefix(out.setupTokenURL, prefix)
sum := sha256.Sum256([]byte(raw))
if f.tokens[0].tokenHash != hex.EncodeToString(sum[:]) {
t.Fatal("stored token does not match URL")
}
if !f.tokens[0].expiresAt.After(time.Now()) {
t.Fatal("token already expired")
}
if string(f.settings[api.LocalAuthEnabledKey]) != "true" {
t.Fatal("local auth stayed disabled")
}
e, payload := auditOf(t, f)
if e.Actor != "deploybot" || e.Action != "setup.owner_login" || payload["user_id"] != "usr-owner-1" {
t.Fatalf("audit = %+v, %v", e, payload)
}
t.Run("failed writes leave no partially initialized login", func(t *testing.T) {
for _, store := range []*fakeOwnerStore{
{redeemErr: errors.New("database unavailable")},
{setErr: errors.New("settings write failed")},
{createTokenErr: errors.New("token write failed")},
} {
if _, err := performSetupOwner(ctx, store, "https://op.console.example.com", "root"); err == nil {
t.Fatal("want failure")
}
if len(store.tokens) != 0 || len(store.setupIDs) != 0 || len(store.settings) != 0 {
t.Fatal("partial setup")
}
t.Run("binds the owner and mints a setup URL whose token hash is what is stored", func(t *testing.T) {
f := &fakeOwnerStore{redeemUserID: "usr-owner-1", redeemMCUUID: "mc-uuid-1", redeemAuthSource: "mojang"}
out, err := performSetupMCBind(ctx, f, " abc-123 ", "console.example.com", "deploybot")
if err != nil {
t.Fatalf("performSetupMCBind: %v", err)
}
// The Owner this mints has no password and no email, so the setup token is the
// only door — and handleSetupRedeem is gated on local_auth_enabled. A bind that
// leaves the toggle off hands back a URL that answers 403.
if _, ok := f.settings[api.LocalAuthEnabledKey]; !ok {
t.Error("local auth was not enabled — the setup URL would 403 local_auth_disabled")
}
// The bind is attributed by Minecraft identity, because that is what the login
// gate verified; a username would be the one thing nobody checked.
e, payload := auditOf(t, f)
if e.Actor != "deploybot" || e.Source != "setup" || e.Action != "setup.owner_bind" {
t.Errorf("audit = %+v, want actor=deploybot source=setup action=setup.owner_bind", e)
}
if payload["mc_uuid"] != "mc-uuid-1" || payload["auth_source"] != "mojang" {
t.Errorf("audit payload = %v, want the redeemed mc_uuid + auth_source", payload)
}
if out.ownerIdentity != "mc-uuid-1" {
t.Errorf("owner identity = %q, want the verified Minecraft UUID", out.ownerIdentity)
}
const prefix = "https://console.example.com/setup?token="
if !strings.HasPrefix(out.setupTokenURL, prefix) {
t.Fatalf("setup URL = %q, want prefix %q", out.setupTokenURL, prefix)
}
// The link code is trimmed and upper-cased before redemption.
if len(f.redeems) != 1 {
t.Fatalf("want 1 redeem, got %d", len(f.redeems))
}
if f.redeems[0].code != "ABC-123" {
t.Errorf("redeemed code = %q, want ABC-123 (trimmed + upper-cased)", f.redeems[0].code)
}
if !strings.HasPrefix(f.redeems[0].newUserID, "usr-") {
t.Errorf("redeem newUserID = %q, want usr- prefix", f.redeems[0].newUserID)
}
// Exactly one token minted, for the redeemed user, and only its hash stored —
// the stored hash must be sha-256 of the raw token carried in the URL.
if len(f.tokens) != 1 {
t.Fatalf("want 1 setup token, got %d", len(f.tokens))
}
tok := f.tokens[0]
if tok.userID != "usr-owner-1" {
t.Errorf("token userID = %q, want usr-owner-1 (the redeemed owner)", tok.userID)
}
raw := strings.TrimPrefix(out.setupTokenURL, prefix)
sum := sha256.Sum256([]byte(raw))
if tok.tokenHash != hex.EncodeToString(sum[:]) {
t.Error("stored token hash is not sha-256 of the raw token in the URL")
}
if tok.tokenHash == raw || tok.tokenHash == "" {
t.Error("the raw token (or nothing) was stored instead of its hash")
}
// The token is short-lived and in the future.
if !tok.expiresAt.After(time.Now()) {
t.Errorf("token expiresAt = %v, want a future time", tok.expiresAt)
}
})
t.Run("settled account is not reset", func(t *testing.T) {
store := &fakeOwnerStore{setupUserID: "existing", setupUsername: "alice", onboarded: true}
out, err := performSetupOwner(ctx, store, "https://op.console.example.com", "root")
if !errors.Is(err, api.ErrConflict) || out.ownerUsername != "alice" || len(store.tokens) != 0 {
t.Fatalf("out = %+v err = %v", out, err)
t.Run("an empty link code mints nothing", func(t *testing.T) {
f := &fakeOwnerStore{}
if _, err := performSetupMCBind(ctx, f, " ", "console.example.com", "root"); err == nil {
t.Fatal("want error for an empty link code")
}
if len(f.redeems) != 0 || len(f.tokens) != 0 {
t.Errorf("want no redeem/token on an empty code, got redeems=%d tokens=%d", len(f.redeems), len(f.tokens))
}
if _, ok := f.settings[api.LocalAuthEnabledKey]; ok {
t.Error("local auth was enabled without an owner — the gate must not open on a failed bind")
}
})
t.Run("audit failure still returns the login link", func(t *testing.T) {
out, err := performSetupOwner(ctx, &fakeOwnerStore{auditErr: errors.New("audit down")}, "https://op.console.example.com", "root")
if err != nil || out.auditErr == nil || out.setupTokenURL == "" {
t.Fatalf("out = %+v err = %v", out, err)
t.Run("a link-code redemption failure mints no token", func(t *testing.T) {
f := &fakeOwnerStore{redeemErr: errors.New("code expired")}
if _, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root"); err == nil {
t.Fatal("want error when the link code cannot be redeemed")
}
if len(f.tokens) != 0 {
t.Errorf("want no token minted on a redeem failure, got %d", len(f.tokens))
}
if _, ok := f.settings[api.LocalAuthEnabledKey]; ok {
t.Error("local auth was enabled without an owner — the gate must not open on a failed redeem")
}
})
t.Run("missing HTTPS origin cannot create an account", func(t *testing.T) {
store := &fakeOwnerStore{}
if _, err := performSetupOwner(ctx, store, "", "root"); err == nil || len(store.tokens) != 0 {
t.Fatal("invalid origin accepted")
t.Run("a local-auth failure fails the bind rather than minting an unredeemable URL", func(t *testing.T) {
f := &fakeOwnerStore{redeemUserID: "usr-owner-1", setErr: errors.New("db down")}
if _, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root"); err == nil {
t.Fatal("want error when local auth cannot be enabled")
}
if len(f.redeems) != 0 || len(f.tokens) != 0 {
t.Errorf("atomic setup was partially recorded: redeems=%d tokens=%d", len(f.redeems), len(f.tokens))
}
})
t.Run("a token-store failure rolls the bind back", func(t *testing.T) {
f := &fakeOwnerStore{redeemUserID: "usr-owner-1", createTokenErr: errors.New("db down")}
if _, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root"); err == nil {
t.Fatal("want error when the setup token cannot be stored")
}
if len(f.redeems) != 0 {
t.Errorf("link code was consumed despite token failure, got %d redeems", len(f.redeems))
}
if len(f.tokens) != 0 {
t.Errorf("want no recorded token when the store fails, got %d", len(f.tokens))
}
if _, ok := f.settings[api.LocalAuthEnabledKey]; ok {
t.Error("local auth stayed enabled despite transaction rollback")
}
})
t.Run("an audit failure does not cost the operator their install", func(t *testing.T) {
f := &fakeOwnerStore{redeemUserID: "usr-owner-1", auditErr: errors.New("audit sink down")}
out, err := performSetupMCBind(ctx, f, "abc-123", "console.example.com", "root")
if err != nil {
t.Fatalf("an audit failure must not fail the bind: %v", err)
}
if out.auditErr == nil {
t.Error("the audit failure was swallowed instead of surfaced on the outcome")
}
if out.setupTokenURL == "" {
t.Error("no setup URL minted despite a recoverable audit failure")
}
if _, ok := f.settings[api.LocalAuthEnabledKey]; !ok {
t.Error("local auth was not enabled despite a recoverable audit failure")
}
})
t.Run("defaults to the op.console host when adminHostname is empty", func(t *testing.T) {
f := &fakeOwnerStore{redeemUserID: "usr-owner-1"}
out, err := performSetupMCBind(ctx, f, "abc-123", " ", "root")
if err != nil {
t.Fatalf("performSetupMCBind: %v", err)
}
// The Owner is staff, so onboarding lands on the operator console, not the
// player panel — the empty-host fallback must reflect that.
if !strings.HasPrefix(out.setupTokenURL, "https://op.console.localhost/setup?token=") {
t.Errorf("setup URL = %q, want the op.console.localhost default host", out.setupTokenURL)
}
})
}
cmd/felis/conflict_test.go
-358
View File
@@ -1,358 +0,0 @@
package main
import (
"context"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/imagepin"
"felis.lolicon.best/internal/naming"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)
// racingClient lands one concurrent write (race) on the stored object just before
// setup's first write of it goes out, the way the operator's status update or
// felis-api's idle patch can, and counts setup's writes.
func racingClient(t *testing.T, race func(ctx context.Context, c client.WithWatch), objs ...client.Object) (client.Client, *int) {
t.Helper()
writes := 0
before := func(ctx context.Context, c client.WithWatch) {
writes++
if writes == 1 {
race(ctx, c)
}
}
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
WithStatusSubresource(&v1alpha1.MinecraftServer{}).
WithInterceptorFuncs(interceptor.Funcs{
Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
before(ctx, c)
return c.Update(ctx, obj, opts...)
},
Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
before(ctx, c)
return c.Patch(ctx, obj, patch, opts...)
},
}).Build()
return cl, &writes
}
func staleLoginGate(t *testing.T) *v1alpha1.MinecraftServer {
t.Helper()
ms, err := loginSystemServer("felis-limbo:demo", "minecraft",
"http://old.internal:8081", "203.0.113.10.nip.io", "console.203.0.113.10.nip.io")
if err != nil {
t.Fatal(err)
}
return ms
}
func getLogin(t *testing.T, cl client.Client) *v1alpha1.MinecraftServer {
t.Helper()
var ms v1alpha1.MinecraftServer
if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
t.Fatal(err)
}
return &ms
}
func envMap(ms *v1alpha1.MinecraftServer) map[string]string {
m := map[string]string{}
for _, e := range ms.Spec.Env {
m[e.Name] = e.Value
}
return m
}
// A hand edit that lands while setup refreshes the console hostnames costs setup a
// re-read and a second write; both the edit and the refresh survive.
func TestRefreshDerivedEnvRetriesAConcurrentEdit(t *testing.T) {
ctx := context.Background()
cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
ms := getLogin(t, c)
ms.Spec.Env = append(ms.Spec.Env, v1alpha1.EnvVar{Name: "HAND_TUNED", Value: "1"})
if err := c.Update(ctx, ms); err != nil {
t.Fatal(err)
}
}, staleLoginGate(t))
desired, err := loginSystemServer("felis-limbo:demo", "minecraft",
"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
if err != nil {
t.Fatal(err)
}
refreshed, err := refreshDerivedEnv(ctx, cl, getLogin(t, cl), desired)
if err != nil || !refreshed {
t.Fatalf("refreshed=%v err=%v, want a refresh after the retry", refreshed, err)
}
env := envMap(getLogin(t, cl))
if env[envPanelHostname] != "console.mc.example.net" || env[envRootDomain] != "mc.example.net" {
t.Errorf("env = %v, want the new hostnames", env)
}
if env["HAND_TUNED"] != "1" {
t.Errorf("env = %v: the concurrent hand edit was dropped", env)
}
if *writes != 2 {
t.Errorf("writes = %d, want 2 (one conflict, one retry)", *writes)
}
}
// converge reports each fill once even when a status write forced a retry.
func TestConvergeRetriesAConcurrentStatusWrite(t *testing.T) {
ctx := context.Background()
lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
if err != nil {
t.Fatal(err)
}
lobby.Spec.Rcon = v1alpha1.RconSpec{}
cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
var ms v1alpha1.MinecraftServer
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
t.Fatal(err)
}
ms.Status.Phase = v1alpha1.PhaseRunning
if err := c.Status().Update(ctx, &ms); err != nil {
t.Fatal(err)
}
}, lobby)
var got systemServerOutcome
for _, o := range convergeSystemServers(ctx, cl, "minecraft", "", "reg/lobby:1",
"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
if o.name == naming.SystemLobbyServer {
got = o
}
}
if got.err != nil || !got.updated || !slices.Equal(got.changes, []string{"spec.rcon"}) {
t.Fatalf("lobby outcome = %+v, want spec.rcon filled once", got)
}
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
t.Fatal(err)
}
if !ms.Spec.Rcon.Enabled || ms.Status.Phase != v1alpha1.PhaseRunning {
t.Errorf("rcon.enabled=%v phase=%q, want the fill and the status write both kept", ms.Spec.Rcon.Enabled, ms.Status.Phase)
}
if *writes != 2 {
t.Errorf("writes = %d, want 2", *writes)
}
}
// A replica refresh racing another writer keeps that writer's key.
func TestSecretReplicaRefreshRetriesAConcurrentWrite(t *testing.T) {
secret := func(ns, body string) *corev1.Secret {
return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
Data: map[string][]byte{"felis.toml": []byte(body)}}
}
cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
var s corev1.Secret
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
t.Fatal(err)
}
s.Data["extra"] = []byte("x")
if err := c.Update(ctx, &s); err != nil {
t.Fatal(err)
}
}, secret("felis", "current"), secret("minecraft", "stale"))
out := ensureSecretReplica(context.Background(), cl, "felis", "minecraft",
"felis-config", "felis.toml", "config", "minecraft ns", true)
if out.err != nil || !out.updated {
t.Fatalf("outcome = %+v, want refreshed", out)
}
var s corev1.Secret
if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
t.Fatal(err)
}
if string(s.Data["felis.toml"]) != "current" || string(s.Data["extra"]) != "x" {
t.Errorf("replica data = %q, want felis.toml=current and extra=x", s.Data)
}
if *writes != 2 {
t.Errorf("writes = %d, want 2", *writes)
}
}
const (
sysOldDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
sysNewDigest = "sha256:4444444444444444444444444444444444444444444444444444444444444444"
)
func systemPinRegistry(t *testing.T) imagepin.Resolver {
t.Helper()
reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v2/felis/limbo/manifests/demo", "/v2/felis/lobby/manifests/demo":
w.Header().Set("Docker-Content-Digest", sysNewDigest)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(reg.Close)
return imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
}
func systemServer(name, image string) *v1alpha1.MinecraftServer {
ms := &v1alpha1.MinecraftServer{}
ms.Name, ms.Namespace = name, "minecraft"
ms.Labels = map[string]string{v1alpha1.LabelSystemRole: name}
ms.Spec.Image = image
return ms
}
// The installer's --system pin moves a system server onto the build its tag names
// now, from the bare tag or from an earlier digest, and is a no-op the second time.
func TestPinSystemServerImage(t *testing.T) {
ctx := context.Background()
res := systemPinRegistry(t)
limbo := defaultRegistryURL + "/felis/limbo:demo"
lobby := defaultRegistryURL + "/felis/lobby:demo"
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
systemServer(naming.SystemLoginServer, limbo),
systemServer(naming.SystemLobbyServer, lobby+"@"+sysOldDigest),
).Build()
image := func(name string) string {
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
t.Fatal(err)
}
return ms.Spec.Image
}
for name, want := range map[string]string{
naming.SystemLoginServer: limbo + "@" + sysNewDigest,
naming.SystemLobbyServer: lobby + "@" + sysNewDigest,
} {
o, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
if err != nil || o.err != nil || !o.updated {
t.Fatalf("%s: outcome=%+v err=%v, want pinned", name, o, err)
}
if got := image(name); got != want {
t.Errorf("%s image = %q, want %q", name, got, want)
}
again, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
if err != nil || again.updated || again.skipped != "already runs "+want {
t.Errorf("%s second pass = %+v, %v; want already runs %s", name, again, err, want)
}
}
}
func TestPinSystemServerImageLeavesOthersAlone(t *testing.T) {
ctx := context.Background()
res := systemPinRegistry(t)
pin := func(t *testing.T, ms *v1alpha1.MinecraftServer) (systemServerOutcome, string) {
t.Helper()
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(ms).Build()
o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
if err != nil {
t.Fatal(err)
}
var got v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKeyFromObject(ms), &got); err != nil {
t.Fatal(err)
}
return o, got.Spec.Image
}
t.Run("external image", func(t *testing.T) {
o, img := pin(t, systemServer(naming.SystemLoginServer, "docker.io/example/limbo:1.2"))
if o.err != nil || o.updated || img != "docker.io/example/limbo:1.2" ||
o.skipped != "runs docker.io/example/limbo:1.2, which names no platform registry tag to follow; left alone" {
t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
}
})
t.Run("digest without a tag", func(t *testing.T) {
ref := defaultRegistryURL + "/felis/limbo@" + sysOldDigest
o, img := pin(t, systemServer(naming.SystemLoginServer, ref))
if o.err != nil || o.updated || img != ref {
t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
}
})
t.Run("not a system server", func(t *testing.T) {
ms := systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo")
ms.Labels = nil
o, img := pin(t, ms)
if o.err == nil || img != defaultRegistryURL+"/felis/limbo:demo" {
t.Fatalf("outcome=%+v image=%q, want refused", o, img)
}
})
t.Run("tag the registry lost", func(t *testing.T) {
o, img := pin(t, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:gone"))
if o.err == nil || img != defaultRegistryURL+"/felis/limbo:gone" {
t.Fatalf("outcome=%+v image=%q, want an error the installer falls back on", o, img)
}
})
t.Run("absent", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).Build()
o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
if err != nil || o.err != nil || o.skipped != "not present yet; sudo felis setup creates it" {
t.Fatalf("outcome=%+v err=%v, want a skip", o, err)
}
})
t.Run("retargeted while pinning", func(t *testing.T) {
cl, _ := racingClient(t, func(ctx context.Context, c client.WithWatch) {
ms := getLogin(t, c)
ms.Spec.Image = "docker.io/example/limbo:1.2"
if err := c.Update(ctx, ms); err != nil {
t.Fatal(err)
}
}, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo"))
o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
if err != nil || o.err != nil || o.updated {
t.Fatalf("outcome=%+v err=%v, want nothing written", o, err)
}
if img := getLogin(t, cl).Spec.Image; img != "docker.io/example/limbo:1.2" {
t.Errorf("image = %q, want the admin's retarget kept", img)
}
})
}
// --system names one of the two system servers; anything else is a usage error
// before any cluster is touched.
func TestPinImagesSystemFlagTakesOnlySystemServers(t *testing.T) {
var stdout, stderr strings.Builder
if code := cmdPinImages([]string{"--system", "survival"}, &stdout, &stderr); code != 2 {
t.Fatalf("exit = %d, want 2", code)
}
if got := stderr.String(); got != "felis pin-images: --system takes login or lobby, not \"survival\"\n" {
t.Errorf("stderr = %q", got)
}
}
// An idle setting the panel saves while converge fills the default is kept.
func TestConvergeIdleKeepsAConcurrentPanelEdit(t *testing.T) {
ctx := context.Background()
srv := &v1alpha1.MinecraftServer{}
srv.Name, srv.Namespace = "survival", "minecraft"
cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
var ms v1alpha1.MinecraftServer
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
t.Fatal(err)
}
ms.Spec.Idle = v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}
if err := c.Update(ctx, &ms); err != nil {
t.Fatal(err)
}
}, srv)
if out := convergeUserServerIdle(ctx, cl, "minecraft"); len(out) != 0 {
t.Fatalf("outcomes = %+v, want none: the server has a setting by the time converge writes", out)
}
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
t.Fatal(err)
}
if want := (v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}); ms.Spec.Idle != want {
t.Errorf("idle = %+v, want the panel's %+v", ms.Spec.Idle, want)
}
if *writes != 1 {
t.Errorf("writes = %d, want 1 (the conflicted attempt; the retry sends nothing)", *writes)
}
}
cmd/felis/context_max_test.go
-57
View File
@@ -1,57 +0,0 @@
package main
import (
"os"
"path/filepath"
"testing"
"felis.lolicon.best/internal/config"
)
func TestContextMaxBytes(t *testing.T) {
cases := []struct {
name string
reg config.RegistryConfig
auth config.AuthConfig
want int64
wantErr bool
}{
{name: "direct install keeps the package default", want: 0},
// The panel uploads in parts under the edge's 100 MB body limit, so the
// Cloudflare edge keeps the full default.
{name: "the Cloudflare edge keeps the package default", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 0},
{name: "CF-Connecting-IP keeps the package default", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 0},
{name: "explicit value behind the edge", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, err := contextMaxBytes(&config.Config{Registry: tc.reg, Auth: tc.auth})
if (err != nil) != tc.wantErr {
t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
}
if got != tc.want {
t.Fatalf("contextMaxBytes = %d, want %d", got, tc.want)
}
})
}
}
func TestUploadPartsDir(t *testing.T) {
if got := uploadPartsDir("/var/lib/felis/uploads"); got != "/var/lib/felis/uploads/.parts" {
t.Errorf("local store: parts dir = %q, want beside the contexts", got)
}
if got := uploadPartsDir("file:///srv/uploads"); got != "/srv/uploads/.parts" {
t.Errorf("file:// store: parts dir = %q, want /srv/uploads/.parts", got)
}
// This machine has no /var/lib/felis/uploads mount, so an s3:// store falls
// back to the temp dir.
if _, err := os.Stat("/var/lib/felis/uploads"); err == nil {
t.Skip("/var/lib/felis/uploads exists here")
}
if got := uploadPartsDir("s3://bucket/uploads"); got != filepath.Join(os.TempDir(), "felis-upload-parts") {
t.Errorf("s3 store without the uploads mount: parts dir = %q", got)
}
}
cmd/felis/converge.go
+4 -66
View File
@@ -11,14 +11,12 @@ import (
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// cmdConverge is the explicit convergence pass over already-installed system
// servers (#1), plus the idle-stop default for user servers that predate it, and
// with -user-rcon their RCON block (#3). Provisioning is create-if-absent, so a field the desired spec
// servers (#1), plus the idle-stop default for user servers that predate it. Provisioning is create-if-absent, so a field the desired spec
// gained after an install (spec.rcon, spec.startup.healthHTTPPort, a derived env
// key) never reaches the existing CR — and nothing says so. This command fills
// exactly those zero-value fields; see convergeSystemServers for the full contract
@@ -31,7 +29,6 @@ func cmdConverge(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("converge", flag.ContinueOnError)
fs.SetOutput(stderr)
cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
userRcon := fs.Bool("user-rcon", false, "also turn RCON on for user servers created before it was the default")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
@@ -62,7 +59,6 @@ func cmdConverge(args []string, stdout, stderr io.Writer) int {
defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
outcomes = append(outcomes, convergeUserServerIdle(context.Background(), cl, cfg.K8s.Namespace)...)
outcomes = append(outcomes, convergeUserServerRcon(context.Background(), cl, cfg.K8s.Namespace, *userRcon)...)
fmt.Fprintln(stdout, "felis converge: filling fields an installed server predates (operator-set values are never overwritten):")
exit := 0
@@ -96,72 +92,14 @@ func convergeUserServerIdle(ctx context.Context, cl client.Client, namespace str
if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
continue
}
// Re-checked on the copy each attempt reads: an idle setting the panel saved
// meanwhile is the user's, and the default must not land over it.
changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
if ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
return false
}
ms.Spec.Idle = v1alpha1.DefaultIdle()
return true
})
if err != nil {
patch := client.MergeFrom(ms.DeepCopy())
ms.Spec.Idle = v1alpha1.DefaultIdle()
if err := cl.Patch(ctx, ms, patch); err != nil {
out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
continue
}
if !changed {
continue
}
out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
changes: []string{fmt.Sprintf("spec.idle (stop after %ds empty)", v1alpha1.DefaultEmptySecondsBeforeStop)}})
}
return out
}
// convergeUserServerRcon handles user servers created before RCON was part of every
// new server (694e3cb): spec.rcon entirely unset. Such a server has a dead console,
// reports nobody online, and never idles out, because all three ride RCON.
//
// Only with fill does it turn RCON on, with the same block CreateServer writes
// today; without it each such server gets a line saying so. The fill is opt-in
// because the operator gates readiness on the RCON probe: a server whose image does
// not open the listener RCON_PASSWORD asks for would sit in Starting until it is
// marked Failed. Felis's own paper and lobby images open it; an image a user brought
// may not, and only the operator running this can tell. A server that already
// carries any RCON setting (on or off) is left alone and produces no line.
func convergeUserServerRcon(ctx context.Context, cl client.Client, namespace string, fill bool) []systemServerOutcome {
var list v1alpha1.MinecraftServerList
if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
}
var out []systemServerOutcome
for i := range list.Items {
ms := &list.Items[i]
if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
continue
}
if !fill {
out = append(out, systemServerOutcome{name: ms.Name, available: true,
skipped: "no RCON (console, online count and idle stop are off); once its image serves RCON, sudo felis converge -user-rcon turns it on"})
continue
}
changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
if ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
return false
}
ms.Spec.Rcon = v1alpha1.RconSpec{
Enabled: true,
SecretRef: v1alpha1.SecretKeyRef{Name: naming.RconSecretName(ms.Name), Key: naming.RconSecretKey},
}
return true
})
if err != nil {
out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
continue
}
if changed {
out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true, changes: []string{"spec.rcon"}})
}
}
return out
}
cmd/felis/converge_test.go
-64
View File
@@ -229,67 +229,3 @@ func TestConvergeUserServerIdle(t *testing.T) {
t.Fatalf("second pass = %+v, want nothing to do", again)
}
}
// TestConvergeUserServerRcon reports a user server with no RCON block at all and
// fills it only when asked, with the block CreateServer writes. RCON turned off on
// purpose, a server with its own secret, and a system server stay as they are and
// produce no line.
func TestConvergeUserServerRcon(t *testing.T) {
scheme := newSystemServerScheme(t)
ctx := context.Background()
mk := func(name string, rcon v1alpha1.RconSpec, role string) *v1alpha1.MinecraftServer {
ms := &v1alpha1.MinecraftServer{}
ms.Name, ms.Namespace = name, "minecraft"
ms.Spec.Rcon = rcon
if role != "" {
ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
}
return ms
}
own := v1alpha1.RconSpec{Enabled: true, Port: 25580, SecretRef: v1alpha1.SecretKeyRef{Name: "own", Key: "pw"}}
off := v1alpha1.RconSpec{SecretRef: v1alpha1.SecretKeyRef{Name: "rcon-off", Key: naming.RconSecretKey}}
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
mk("demo", v1alpha1.RconSpec{}, ""),
mk("off", off, ""),
mk("own", own, ""),
mk(naming.SystemLobbyServer, v1alpha1.RconSpec{}, naming.SystemLobbyServer),
).Build()
get := func(name string) v1alpha1.RconSpec {
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
t.Fatalf("get %s: %v", name, err)
}
return ms.Spec.Rcon
}
report := convergeUserServerRcon(ctx, cl, "minecraft", false)
if len(report) != 1 || report[0].name != "demo" || report[0].updated || report[0].err != nil ||
!strings.Contains(report[0].skipped, "-user-rcon") {
t.Fatalf("report = %+v, want one skipped line for demo naming -user-rcon", report)
}
if got := get("demo"); got != (v1alpha1.RconSpec{}) {
t.Fatalf("the report-only pass wrote demo's rcon: %+v", got)
}
filled := convergeUserServerRcon(ctx, cl, "minecraft", true)
if len(filled) != 1 || filled[0].name != "demo" || !filled[0].updated || filled[0].err != nil {
t.Fatalf("fill = %+v, want exactly one update for demo", filled)
}
want := map[string]v1alpha1.RconSpec{
"demo": {Enabled: true, SecretRef: v1alpha1.SecretKeyRef{
Name: naming.RconSecretName("demo"), Key: naming.RconSecretKey}},
"off": off,
"own": own,
naming.SystemLobbyServer: {},
}
for name, rcon := range want {
if got := get(name); got != rcon {
t.Errorf("%s rcon = %+v, want %+v", name, got, rcon)
}
}
for _, fill := range []bool{false, true} {
if again := convergeUserServerRcon(ctx, cl, "minecraft", fill); len(again) != 0 {
t.Fatalf("second pass (fill=%v) = %+v, want nothing to do", fill, again)
}
}
}
cmd/felis/db.go
+17 -184
View File
@@ -7,7 +7,6 @@ import (
"flag"
"fmt"
"io"
neturl "net/url"
"os"
"os/exec"
"path/filepath"
@@ -16,8 +15,6 @@ import (
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/retention"
)
const dbUsage = `usage:
@@ -27,7 +24,6 @@ const dbUsage = `usage:
felis db verify [-dir dir] <bundle>
felis db list [-dir dir]
felis db check [-dir dir] [-max-age 26h]
felis db audit-export [-config path] [-since date] [-until date] [-out file]
`
// defaultKeep is how many bundles of a label a backup leaves behind. Manual
@@ -36,7 +32,6 @@ var defaultKeep = map[string]int{
dbbackup.LabelDaily: 14,
dbbackup.LabelPreMigrate: 10,
dbbackup.LabelPreRestore: 5,
dbbackup.LabelOffsite: 1,
}
// cmdDB implements `felis db`: logical backups of the control-plane database
@@ -63,8 +58,6 @@ func cmdDB(args []string, stdout, stderr io.Writer) int {
return dbList(fs, dir, rest, stdout, stderr)
case "check":
return dbCheck(fs, dir, rest, stdout, stderr)
case "audit-export":
return dbAuditExport(fs, rest, stdout, stderr)
case "-h", "--help", "help":
fmt.Fprint(stdout, dbUsage)
return 0
@@ -95,49 +88,17 @@ func parseWithArg(fs *flag.FlagSet, args []string) (string, bool) {
}
func dbDatabaseURL(path string) (string, error) {
db, err := dbDatabase(path)
return db.URL, err
}
func dbDatabase(path string) (config.DatabaseConfig, error) {
cfg, err := config.Load(path)
if err != nil {
return config.DatabaseConfig{}, err
return "", err
}
return cfg.Database, nil
}
// dbTools places pg_dump, pg_restore and psql. The installer's database runs in
// k3s and the host has no PostgreSQL client, so when the host config names the
// [database] deployment the tools run in its postgres container over the
// container's socket, as the URL's role on the URL's database. Otherwise they
// come from PATH and connect with the URL.
func dbTools(db config.DatabaseConfig) (dbbackup.Tools, error) {
if db.Deployment == "" {
return dbbackup.Tools{}, nil
}
ns, name, _ := strings.Cut(db.Deployment, "/")
u, err := neturl.Parse(db.URL)
if err != nil || u.User == nil || u.User.Username() == "" || strings.TrimPrefix(u.Path, "/") == "" {
return dbbackup.Tools{}, errors.New("[database] url must name the role and the database to run the tools in the database's pod")
}
return dbbackup.Tools{
Exec: []string{"k3s", "kubectl", "exec", "-i", "-n", ns, "deploy/" + name, "-c", platform.PostgresContainer, "--"},
Conn: fmt.Sprintf("host=%s port=%d dbname=%s user=%s connect_timeout=15",
platform.PostgresSocketDir, platform.PostgresPort,
libpqQuote(strings.TrimPrefix(u.Path, "/")), libpqQuote(u.User.Username())),
}, nil
}
// libpqQuote renders v as a single-quoted libpq connection-string value.
func libpqQuote(v string) string {
return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(v) + "'"
return cfg.Database.URL, nil
}
func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never")
keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, offsite 1, manual all)")
keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, manual all)")
stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`)
noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle")
metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)")
@@ -148,12 +109,7 @@ func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
fmt.Fprint(stderr, dbUsage)
return 2
}
db, err := dbDatabase(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db backup: %v\n", err)
return 1
}
tools, err := dbTools(db)
url, err := dbDatabaseURL(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db backup: %v\n", err)
return 1
@@ -162,7 +118,7 @@ func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
*keep = defaultKeep[*label]
}
o := dbbackup.BackupOptions{
DatabaseURL: db.URL, Tools: tools, Dir: *dir, Label: *label, Keep: *keep,
DatabaseURL: url, Dir: *dir, Label: *label, Keep: *keep,
StateDir: *stateDir, Version: resolvedVersion(), Log: stderr,
MetricsFile: *metrics, Record: true,
}
@@ -172,14 +128,6 @@ func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
path, err := dbbackup.Backup(ctx, o)
if errors.Is(err, dbbackup.ErrServersMissing) {
// The bundle is on disk and holds the database; the exit status fails
// the timer's run so the gap shows in systemctl and the journal, and
// the panel and the watchdog read it from the record and the manifest.
fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
fmt.Fprintf(stderr, "felis db backup: %v\n a restore from %s brings back the database but no servers; check `k3s kubectl get minecraftservers -A`, then run `felis db backup` again\n", err, filepath.Base(path))
return 1
}
if err != nil {
fmt.Fprintf(stderr, "felis db backup: %v\n", err)
return 1
@@ -220,17 +168,12 @@ func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.W
return 1
}
if !*yes {
fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d, holding %s).\n",
filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion, m.Counts.String())
fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d).\n",
filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion)
fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.")
return 2
}
db, err := dbDatabase(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db restore: %v\n", err)
return 1
}
tools, err := dbTools(db)
url, err := dbDatabaseURL(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db restore: %v\n", err)
return 1
@@ -238,7 +181,7 @@ func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.W
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
defer cancel()
_, safety, err := dbbackup.Restore(ctx, dbbackup.RestoreOptions{
DatabaseURL: db.URL, Tools: tools, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
DatabaseURL: url, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
Safety: dbbackup.BackupOptions{Keep: defaultKeep[dbbackup.LabelPreRestore], StateDir: *stateDir,
Version: resolvedVersion(), ExportServers: exportMinecraftServers},
Log: stderr,
@@ -277,9 +220,8 @@ func dbVerify(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
fmt.Fprintf(stderr, "felis db verify: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "%s: ok\n taken %s (%s)\n felis %s\n schema %d\n holds %s\n %s\n",
filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion,
m.Counts.String(), orUnknown(m.PGDumpVersion))
fmt.Fprintf(stdout, "%s: ok\n taken %s (%s)\n felis %s\n schema %d\n %s\n",
filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion, orUnknown(m.PGDumpVersion))
for _, f := range m.Files {
if f.Link != "" {
fmt.Fprintf(stdout, " %-40s -> %s\n", f.Name, f.Link)
@@ -320,88 +262,6 @@ func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writ
return 0
}
// dbAuditExport writes audit rows to a file (or stdout) as JSON lines, so an
// install can keep them past [audit] retention, after which felis-api deletes them.
func dbAuditExport(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
sinceFlag := fs.String("since", "", "first day (or RFC 3339 instant) to export, inclusive; empty starts at the oldest row")
untilFlag := fs.String("until", "", "day (or RFC 3339 instant) to stop before, exclusive; empty runs to the newest row")
out := fs.String("out", "", "file to write (created 0600, never overwritten); empty writes to stdout")
if err := fs.Parse(args); err != nil {
return 2
}
if fs.NArg() > 0 {
fmt.Fprint(stderr, dbUsage)
return 2
}
since, err := parseExportBound(*sinceFlag)
if err != nil {
fmt.Fprintf(stderr, "felis db audit-export: -since: %v\n", err)
return 2
}
until, err := parseExportBound(*untilFlag)
if err != nil {
fmt.Fprintf(stderr, "felis db audit-export: -until: %v\n", err)
return 2
}
if !since.IsZero() && !until.IsZero() && !until.After(since) {
fmt.Fprintf(stderr, "felis db audit-export: -until %s is not after -since %s\n", *untilFlag, *sinceFlag)
return 2
}
url, err := dbDatabaseURL(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
return 1
}
w := stdout
var f *os.File
if *out != "" {
if f, err = os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600); err != nil {
fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
return 1
}
w = f
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
n, err := exportAudit(ctx, url, since, until, w)
if f != nil {
if cerr := f.Close(); err == nil {
err = cerr
}
}
if err != nil {
fmt.Fprintf(stderr, "felis db audit-export: %v (%d rows written)\n", err, n)
return 1
}
fmt.Fprintf(stderr, "felis db audit-export: %d audit rows written\n", n)
return 0
}
func exportAudit(ctx context.Context, url string, since, until time.Time, w io.Writer) (int, error) {
drv, err := openStore(ctx, url, false)
if err != nil {
return 0, fmt.Errorf("open database: %w", err)
}
defer drv.Close()
return retention.ExportAudit(ctx, drv.DB(), since, until, w)
}
// parseExportBound reads a -since/-until value: a day (midnight UTC) or an
// RFC 3339 instant; empty is an open bound.
func parseExportBound(v string) (time.Time, error) {
if v == "" {
return time.Time{}, nil
}
if t, err := time.Parse(time.DateOnly, v); err == nil {
return t, nil
}
if t, err := time.Parse(time.RFC3339, v); err == nil {
return t.UTC(), nil
}
return time.Time{}, fmt.Errorf("%q is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)", v)
}
func humanBytes(n int64) string {
const unit = 1024
if n < unit {
@@ -415,11 +275,10 @@ func humanBytes(n int64) string {
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}
// dbCheck is the freshness probe: exit 1 when the newest daily bundle is
// missing or older than -max-age, for a monitor or the break-glass console to
// act on.
// dbCheck is the freshness probe: exit 1 when the newest bundle is missing or
// older than -max-age, for a monitor or the break-glass console to act on.
func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest daily bundle")
maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest bundle")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -428,40 +287,14 @@ func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wri
fmt.Fprintf(stderr, "felis db check: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "felis db check: ok, newest daily backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
fmt.Fprintf(stdout, "felis db check: ok, newest backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
return 0
}
// serverExportTries and serverExportRetry are how long a backup waits out a
// cluster that is briefly away (an apiserver restart) before its bundle goes
// without the MinecraftServer objects.
const serverExportTries = 3
var serverExportRetry = 10 * time.Second
// exportMinecraftServers is dbbackup's ExportServers on the host: the
// MinecraftServer objects through k3s kubectl, tried serverExportTries times.
func exportMinecraftServers(ctx context.Context) ([]byte, error) {
for try := 1; ; try++ {
out, err := getMinecraftServers(ctx)
if err == nil {
return out, nil
}
if try == serverExportTries {
return nil, fmt.Errorf("%w (tried %d times)", err, try)
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("%w (tried %d times)", err, try)
case <-time.After(serverExportRetry):
}
}
}
// getMinecraftServers reads every MinecraftServer through the host's k3s
// exportMinecraftServers reads every MinecraftServer through the host's k3s
// kubectl and strips what the API server owns, so the result can be fed back
// with `kubectl apply -f` on a rebuilt cluster.
func getMinecraftServers(ctx context.Context) ([]byte, error) {
func exportMinecraftServers(ctx context.Context) ([]byte, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning
cmd/felis/db_test.go
+6 -422
View File
@@ -1,20 +1,14 @@
package main
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"flag"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/store"
)
@@ -30,63 +24,13 @@ func TestDBUsage(t *testing.T) {
}
}
func TestDBVerifySaysWhatTheBundleHolds(t *testing.T) {
dir := newPodRig(t)
cfg := podConfig(t, dir)
bundles := filepath.Join(dir, "bundles")
var out, errBuf bytes.Buffer
if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
t.Fatalf("backup: exit %d: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
out.Reset()
if code := run([]string{"db", "verify", "-dir", bundles, filepath.Base(bundle)}, &out, &errBuf); code != 0 {
t.Fatalf("verify: exit %d: %s", code, errBuf.String())
}
for _, want := range []string{filepath.Base(bundle) + ": ok", "schema 3", "holds 4 accounts, 2 servers", "pg_dump (PostgreSQL) 18.6"} {
if !strings.Contains(out.String(), want) {
t.Errorf("verify output lacks %q:\n%s", want, out.String())
}
}
}
// TestDBRestoreNeedsYes: without -yes a restore describes the bundle and stops
// before anything reaches the database, even with -force and
// -no-safety-backup, which would otherwise let the replay run at once.
func TestDBRestoreNeedsYes(t *testing.T) {
dir := newPodRig(t)
cfg := podConfig(t, dir)
bundles := filepath.Join(dir, "bundles")
// A bundle that does not exist fails verification (1) before -yes matters;
// the -yes gate itself is exercised against a real bundle in internal/dbbackup
// and on the VM. Here: the refusal path never reaches the config or database.
var out, errBuf bytes.Buffer
if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
t.Fatalf("backup: exit %d: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
podRuns(t, dir)
out.Reset()
errBuf.Reset()
code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-force", "-no-safety-backup", filepath.Base(bundle)}, &out, &errBuf)
if code != 2 {
t.Fatalf("exit %d, want 2; stderr %q", code, errBuf.String())
}
if want := filepath.Base(bundle) + " (manual, taken "; !strings.Contains(errBuf.String(), want) || !strings.Contains(errBuf.String(), "schema 3, holding 4 accounts, 2 servers).") {
t.Errorf("stderr %q does not describe the bundle", errBuf.String())
}
if !strings.Contains(errBuf.String(), "re-run with -yes") {
t.Errorf("stderr %q does not say how to go on", errBuf.String())
}
if argv, err := os.ReadFile(filepath.Join(dir, "k3s.args")); err == nil {
t.Errorf("a restore without -yes ran in the database pod:\n%s", argv)
}
// A bundle that does not verify is refused before -yes is weighed.
errBuf.Reset()
if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "missing.tar"}, &out, &errBuf); code != 1 {
t.Errorf("missing bundle: exit %d, want 1; stderr %q", code, errBuf.String())
}
if _, err := os.Stat(filepath.Join(dir, "k3s.args")); err == nil {
t.Error("a missing bundle reached the database pod")
if code := run([]string{"db", "restore", "-dir", t.TempDir(), "missing.tar"}, &out, &errBuf); code != 1 {
t.Fatalf("exit %d, stderr %q", code, errBuf.String())
}
}
@@ -196,7 +140,7 @@ func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
{"up to date", map[int]struct{}{1: {}, 2: {}}, false},
{"pending on a populated database", map[int]struct{}{1: {}}, true},
} {
path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, config.DatabaseConfig{URL: badURL}, t.TempDir(), io.Discard)
path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, badURL, t.TempDir(), io.Discard)
if attempted := err != nil; attempted != tc.attempt {
t.Errorf("%s: attempted = %v (err %v), want %v", tc.name, attempted, err, tc.attempt)
}
@@ -205,363 +149,3 @@ func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
}
}
}
// audit-export takes a day or an RFC 3339 instant for each bound, and refuses a
// malformed or inverted window before it opens the config or the database.
func TestAuditExportBounds(t *testing.T) {
for _, tc := range []struct{ in, want string }{
{"", "0001-01-01T00:00:00Z"},
{"2026-01-31", "2026-01-31T00:00:00Z"},
{"2026-01-31T12:30:00+08:00", "2026-01-31T04:30:00Z"},
} {
got, err := parseExportBound(tc.in)
if err != nil || got.Format(time.RFC3339) != tc.want {
t.Errorf("parseExportBound(%q) = %v, %v; want %s", tc.in, got, err, tc.want)
}
}
if _, err := parseExportBound("31/01/2026"); err == nil || err.Error() != `"31/01/2026" is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)` {
t.Errorf("parseExportBound(31/01/2026) err = %v", err)
}
for _, tc := range []struct {
args []string
wantErr string
}{
{[]string{"db", "audit-export", "-since", "yesterday"}, `felis db audit-export: -since: "yesterday" is neither`},
{[]string{"db", "audit-export", "-until", "2026-13-01"}, `felis db audit-export: -until: "2026-13-01" is neither`},
{[]string{"db", "audit-export", "-since", "2026-02-01", "-until", "2026-02-01"}, "felis db audit-export: -until 2026-02-01 is not after -since 2026-02-01"},
{[]string{"db", "audit-export", "extra"}, "felis db audit-export [-config path]"},
} {
var out, errBuf bytes.Buffer
code := run(append(tc.args, "-config", "/nonexistent/felis.toml"), &out, &errBuf)
if code != 2 || !strings.Contains(errBuf.String(), tc.wantErr) {
t.Errorf("%v: exit %d, stderr %q; want 2 and %q", tc.args, code, errBuf.String(), tc.wantErr)
}
}
}
// podK3s stands in for `k3s kubectl exec ... --`: it logs its argv and runs the
// command after -- from the "container" directory, which is the only place the
// PostgreSQL tools exist, as on an installed host. `kubectl get` lists one
// MinecraftServer, logged to k3s.get, and refuses its first N calls while
// servers_fail holds N.
const podK3s = `#!/bin/sh
if [ "$1" = kubectl ] && [ "$2" = get ]; then
printf '%s\n' "$*" >> "$FAKE_DIR/k3s.get"
n=$(/usr/bin/wc -l < "$FAKE_DIR/k3s.get")
if [ -f "$FAKE_DIR/servers_fail" ] && [ "$n" -le "$(/bin/cat "$FAKE_DIR/servers_fail")" ]; then
echo "The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?" >&2
exit 1
fi
echo '{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer","metadata":{"name":"lobby","namespace":"felis-servers","uid":"u-1"},"spec":{"type":"PAPER"},"status":{"phase":"Running"}}]}'
exit 0
fi
printf '%s\n' "$*" >> "$FAKE_DIR/k3s.args"
while [ $# -gt 0 ] && [ "$1" != "--" ]; do shift; done
shift
tool=$1; shift
exec /usr/bin/env -i FAKE_DIR="$FAKE_DIR" PATH=/usr/bin:/bin "$FAKE_DIR/container/$tool" "$@"
`
var podTools = map[string]string{
"pg_dump": `#!/bin/sh
case "$1" in --version) echo "pg_dump (PostgreSQL) 18.6"; exit 0 ;; esac
printf 'PGDMP-fake-archive'
`,
"pg_restore": `#!/bin/sh
cat > /dev/null
`,
"psql": `#!/bin/sh
for a in "$@"; do case "$a" in *"FROM users"*) echo "4|2"; exit 0 ;; esac; done
for a in "$@"; do [ "$a" = "-c" ] && { echo 3; exit 0; }; done
cat > /dev/null
`,
}
const podPassword = "pw-must-stay-on-the-host"
// podDB is the host config's [database] on an installed host.
var podDB = config.DatabaseConfig{
URL: "postgres://felis:" + podPassword + "@127.0.0.1:15432/felis?sslmode=disable",
Deployment: "felis/felis-postgres",
}
const podExecPrefix = "kubectl exec -i -n felis deploy/felis-postgres -c postgres -- "
// newPodRig puts the fake k3s on PATH, alone, and returns the directory its
// k3s.args log lands in.
func newPodRig(t *testing.T) string {
t.Helper()
dir := t.TempDir()
bin := filepath.Join(dir, "bin")
container := filepath.Join(dir, "container")
for _, d := range []string{bin, container} {
if err := os.Mkdir(d, 0o755); err != nil {
t.Fatal(err)
}
}
writeTestFile(t, filepath.Join(bin, "k3s"), podK3s, 0o755)
for name, body := range podTools {
writeTestFile(t, filepath.Join(container, name), body, 0o755)
}
t.Setenv("PATH", bin)
t.Setenv("FAKE_DIR", dir)
return dir
}
// podRuns returns what the fake k3s ran since the last call, failing on any
// run outside the database container or with the password on its command
// line (visible to every local user in ps).
func podRuns(t *testing.T, dir string) []string {
t.Helper()
log := filepath.Join(dir, "k3s.args")
argv, err := os.ReadFile(log)
if err != nil {
t.Fatalf("nothing ran through k3s: %v", err)
}
os.Remove(log)
var runs []string
for _, line := range strings.Split(strings.TrimSpace(string(argv)), "\n") {
if !strings.HasPrefix(line, podExecPrefix) {
t.Errorf("k3s ran %q, want everything under %q", line, podExecPrefix)
}
if strings.Contains(line, podPassword) {
t.Errorf("the password crossed into the pod on a command line: %q", line)
}
runs = append(runs, strings.TrimPrefix(line, podExecPrefix))
}
return runs
}
// podConfig writes an installed host's felis.toml, [database] pointing at the
// pod, into dir.
func podConfig(t *testing.T, dir string) string {
t.Helper()
toml := strings.Replace(installerTOML("example.com", "127.0.0.1"),
`url = "postgres://felis:[email protected]:5432/felis?sslmode=disable"`,
`url = "`+podDB.URL+`"
deployment = "`+podDB.Deployment+`"`, 1)
cfg := filepath.Join(dir, "felis.toml")
writeTestFile(t, cfg, toml, 0o600)
return cfg
}
func ranIn(runs []string, prefix string) bool {
for _, r := range runs {
if strings.HasPrefix(r, prefix) {
return true
}
}
return false
}
// TestDBBackupAndRestoreRunTheToolsInTheDatabasePod: on an installed host the
// database is a k3s Deployment and no PostgreSQL client exists outside it, so
// `felis db backup` and `restore` must reach the tools through kubectl exec,
// over the pod's socket, and without putting the role's password on a command
// line.
func TestDBBackupAndRestoreRunTheToolsInTheDatabasePod(t *testing.T) {
dir := newPodRig(t)
cfg := podConfig(t, dir)
var out, errBuf bytes.Buffer
bundles := filepath.Join(dir, "bundles")
if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
t.Fatalf("backup: exit %d: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
if _, err := dbbackupVerify(bundle); err != nil {
t.Fatalf("the bundle does not verify: %v", err)
}
runs := podRuns(t, dir)
if !ranIn(runs, "pg_dump --format=custom --no-password --dbname=host=/var/run/postgresql port=5432 dbname='felis' user='felis'") {
t.Errorf("pg_dump did not dump over the pod's socket as felis on felis: %q", runs)
}
out.Reset()
errBuf.Reset()
if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "-force", "-no-safety-backup", bundle}, &out, &errBuf); code != 0 {
t.Fatalf("restore: exit %d: %s", code, errBuf.String())
}
runs = podRuns(t, dir)
if !ranIn(runs, "pg_restore --no-owner --no-privileges --file=-") || !ranIn(runs, "psql -X -q -w -v ON_ERROR_STOP=1 -d host=/var/run/postgresql") {
t.Errorf("the replay did not run in the pod: %q", runs)
}
}
// TestPreMigrateBackupRunsInTheDatabasePod: the snapshot in front of an upgrade
// is the one taken most often, by bootstrap on every rerun.
func TestPreMigrateBackupRunsInTheDatabasePod(t *testing.T) {
dir := newPodRig(t)
ms := []store.Migration{{Version: 1}, {Version: 2}}
// The snapshot also bundles /etc/felis, which a test machine may lack; the
// dump runs first either way.
_, err := preMigrateBackup(context.Background(), appliedDriver{done: map[int]struct{}{1: {}}}, ms, podDB, filepath.Join(dir, "bundles"), io.Discard)
if err != nil && !strings.Contains(err.Error(), "read host state") {
t.Fatalf("snapshot: %v", err)
}
if runs := podRuns(t, dir); !ranIn(runs, "pg_dump --format=custom") {
t.Errorf("pg_dump did not run in the pod: %q", runs)
}
}
func TestDBToolsNeedTheRoleAndDatabase(t *testing.T) {
if tools, err := dbTools(config.DatabaseConfig{URL: "postgres://felis:pw@db:5432/felis"}); err != nil || len(tools.Exec) != 0 {
t.Errorf("no deployment: tools %+v err %v, want the PATH tools", tools, err)
}
for _, u := range []string{"postgres://db:5432/felis", "postgres://felis:pw@db:5432/"} {
if _, err := dbTools(config.DatabaseConfig{URL: u, Deployment: "felis/felis-postgres"}); err == nil {
t.Errorf("%s: no error, want a refusal (the pod connection needs the role and the database)", u)
}
}
tools, err := dbTools(config.DatabaseConfig{URL: `postgres://o%27brien@db/my%20db`, Deployment: "felis/felis-postgres"})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(tools.Conn, `dbname='my db' user='o\'brien'`) {
t.Errorf("Conn = %q, want the values quoted for libpq", tools.Conn)
}
}
var dbbackupVerify = dbbackup.Verify
func noServerExportWait(t *testing.T) {
t.Helper()
old := serverExportRetry
serverExportRetry = 0
t.Cleanup(func() { serverExportRetry = old })
}
// serverGets counts the `kubectl get` calls the fake k3s answered or refused.
func serverGets(dir string) int {
b, _ := os.ReadFile(filepath.Join(dir, "k3s.get"))
return strings.Count(string(b), "\n")
}
// bundleServers returns the bundle's k8s/minecraftservers.json, or nil.
func bundleServers(t *testing.T, bundle string) []byte {
t.Helper()
f, err := os.Open(bundle)
if err != nil {
t.Fatal(err)
}
defer f.Close()
tr := tar.NewReader(f)
for {
h, err := tr.Next()
if err == io.EOF {
return nil
}
if err != nil {
t.Fatal(err)
}
if h.Name == "k8s/minecraftservers.json" {
data, err := io.ReadAll(tr)
if err != nil {
t.Fatal(err)
}
return data
}
}
}
// TestDBBackupWithoutServersFails: when the cluster stays away the daily
// bundle is still written, and `felis db backup` exits 1, so the timer's run
// shows failed, saying a restore from the bundle brings back no servers.
func TestDBBackupWithoutServersFails(t *testing.T) {
noServerExportWait(t)
dir := newPodRig(t)
cfg := podConfig(t, dir)
writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
var out, errBuf bytes.Buffer
code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", "", "-label", "daily"}, &out, &errBuf)
if code != 1 {
t.Fatalf("exit %d, want 1: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
m, err := dbbackupVerify(bundle)
if err != nil {
t.Fatalf("the database must still be bundled: %v", err)
}
if !strings.Contains(m.ServersError, "6443 was refused") || !strings.Contains(m.ServersError, "(tried 3 times)") || bundleServers(t, bundle) != nil {
t.Errorf("manifest servers error = %q", m.ServersError)
}
if n := serverGets(dir); n != serverExportTries {
t.Errorf("export tried %d times, want %d", n, serverExportTries)
}
if msg := errBuf.String(); !strings.Contains(msg, "a restore from "+filepath.Base(bundle)+" brings back the database but no servers") {
t.Errorf("stderr = %q", msg)
}
}
// TestDBBackupRetriesTheServerExport: a cluster back on the last try costs the
// bundle nothing, and what it holds is ready for kubectl apply.
func TestDBBackupRetriesTheServerExport(t *testing.T) {
noServerExportWait(t)
dir := newPodRig(t)
cfg := podConfig(t, dir)
writeTestFile(t, filepath.Join(dir, "servers_fail"), "2", 0o600)
var out, errBuf bytes.Buffer
if code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", ""}, &out, &errBuf); code != 0 {
t.Fatalf("exit %d: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
servers := string(bundleServers(t, bundle))
if !strings.Contains(servers, `"name": "lobby"`) || strings.Contains(servers, "status") || strings.Contains(servers, "u-1") {
t.Errorf("k8s/minecraftservers.json = %s", servers)
}
if n := serverGets(dir); n != 3 {
t.Errorf("export tried %d times, want 3", n)
}
}
// TestPreMigrateBackupExportsServers: the snapshot every upgrade takes, often
// the newest bundle, carries the MinecraftServer objects too; a cluster that
// is away does not hold back the migration, whose rollback needs the database
// alone.
func TestPreMigrateBackupExportsServers(t *testing.T) {
noServerExportWait(t)
dir := newPodRig(t)
old := preMigrateStateDir
preMigrateStateDir = ""
t.Cleanup(func() { preMigrateStateDir = old })
ms := []store.Migration{{Version: 1}, {Version: 2}}
bundles := filepath.Join(dir, "bundles")
pending := appliedDriver{done: map[int]struct{}{1: {}}}
path, err := preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
if err != nil {
t.Fatalf("snapshot: %v", err)
}
if !strings.Contains(string(bundleServers(t, path)), `"name": "lobby"`) {
t.Errorf("%s holds no MinecraftServer objects", path)
}
writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
path, err = preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
if err != nil || path == "" {
t.Fatalf("snapshot with the cluster away = %q, %v; want the bundle and no error", path, err)
}
if m, err := dbbackupVerify(path); err != nil || m.ServersError == "" || bundleServers(t, path) != nil {
t.Errorf("snapshot with the cluster away: %+v, %v", m, err)
}
}
// TestServerExportStopsWaitingWithTheContext: a backup whose time is up stops
// waiting for the cluster between tries.
func TestServerExportStopsWaitingWithTheContext(t *testing.T) {
dir := newPodRig(t)
writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
// Long enough for the first try to run to its refusal: starting the fake
// k3s on a busy machine can take a few hundred ms. Still far below the
// 10s retry wait, so waiting it out would fail the check below.
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
start := time.Now()
_, err := exportMinecraftServers(ctx)
if err == nil || !strings.Contains(err.Error(), "(tried 1 times)") || serverGets(dir) != 1 {
t.Fatalf("err = %v after %d tries, want the first failure alone", err, serverGets(dir))
}
if took := time.Since(start); took > 5*time.Second {
t.Errorf("took %s, want the context's deadline, not the %s retry wait", took, serverExportRetry)
}
}
cmd/felis/distributed.go
-44
View File
@@ -1,44 +0,0 @@
package main
import (
"context"
"fmt"
"io"
"os"
"time"
"felis.lolicon.best/internal/archivetransfer"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/distributed"
"felis.lolicon.best/internal/placement"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func distributionManager(cl client.Client, cfg *config.Config, image string) (*distributed.Manager, error) {
if os.Getenv("FELIS_DISTRIBUTED") != "true" {
return nil, nil
}
controller, url, key := os.Getenv("FELIS_CONTROLLER_NODE"), os.Getenv("FELIS_ARCHIVE_URL"), os.Getenv(archivetransfer.KeyEnv)
if controller == "" || url == "" || len(key) < 32 || image == "" || cfg.Archive.Store != "tarLocal" {
return nil, fmt.Errorf("distributed mode requires controller identity, archive service/key, Felis image and tarLocal")
}
return &distributed.Manager{Client: cl, Namespace: cfg.K8s.Namespace, Image: image, Controller: controller, Archive: archivetransfer.Client{URL: url, Root: cfg.Archive.LocalPath, Key: key}, Resolve: placement.Resolve(cl, cfg.K8s.Namespace, controller)}, nil
}
func reconcileDistribution(ctx context.Context, m *distributed.Manager, stderr io.Writer) {
ticker := time.NewTicker(3 * time.Second)
defer ticker.Stop()
for {
if err := m.SettleBackups(ctx); err != nil {
fmt.Fprintln(stderr, "distributed backup:", err)
}
if err := m.ReconcileMigrations(ctx); err != nil {
fmt.Fprintln(stderr, "migration:", err)
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
cmd/felis/doctor.go
-387
View File
@@ -1,387 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/watchdog"
)
// systemdUnitDir is where the installer writes its units.
const systemdUnitDir = "/etc/systemd/system"
// hostCommand runs a host tool (systemctl, journalctl, k3s) and returns its
// stdout. A tool that exits non-zero still returns what it printed:
// `systemctl is-active` prints "inactive" and exits 3.
func hostCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
return exec.CommandContext(ctx, name, args...).Output()
}
// hostServices are the long-running units a full install depends on, checked
// when their unit file is present: k3s runs the cluster, felis-velocity is the
// game proxy, felis-nano the single-binary host that runs without k3s.
var hostServices = []string{"k3s.service", "felis-velocity.service", "felis-nano.service"}
// cmdDoctor runs every check felis watchdog runs, with the settings
// felis-watchdog.service gives it, plus what only the host shows (systemd
// units that failed or stopped, timers that no longer fire, alerts that reach
// no one), and prints them grouped by area with where to look next. It mails
// nothing, pings no heartbeat and leaves the watchdog's state alone: it is
// safe to run at any time, as often as wanted.
func cmdDoctor(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("doctor", flag.ContinueOnError)
fs.SetOutput(stderr)
unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis doctor: run as root (sudo felis doctor): the checks read root-only state under /etc/felis and /var/lib/felis")
return 1
}
host, _ := os.Hostname()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
return runDoctor(ctx, doctorEnv{unitDir: *unitDir, run: hostCommand, now: time.Now(), host: host}, stdout)
}
// doctorEnv is what one doctor run reads the host through.
type doctorEnv struct {
unitDir string
run func(ctx context.Context, name string, args ...string) ([]byte, error)
now time.Time
host string
}
// doctorAreas are the report's headings in order, by the area findingArea
// puts a finding under.
var doctorAreas = []struct{ key, title string }{
{key: "config", title: "configuration"},
{key: "cluster", title: "Kubernetes cluster"},
{key: "postgres", title: "PostgreSQL"},
{key: "proxy", title: "game proxy"},
{key: "db-backup", title: "database backups"},
{key: "offsite", title: "off-site copy"},
{key: "scan-db", title: "build scan database"},
{key: "disk", title: "disk space"},
{key: "memory", title: "memory"},
{key: "k3s-certs", title: "k3s certificates"},
{key: "host-address", title: "node address"},
{key: "clock", title: "clock"},
{key: "systemd", title: "systemd units and timers"},
{key: "alerts", title: "alerting"},
}
func runDoctor(ctx context.Context, env doctorEnv, stdout io.Writer) int {
unit := filepath.Join(env.unitDir, "felis-watchdog.service")
w, found, unitErr := watchdogUnitFlags(unit)
var report watchdog.Report
var notes []string
fmt.Fprintf(stdout, "felis doctor on %s at %s\n", env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
switch {
case unitErr != nil:
report.Findings = append(report.Findings, watchdog.Finding{
Key: "watchdog/unit", Severity: watchdog.Critical,
SummaryEN: fmt.Sprintf("cannot read the watchdog's settings: %v", unitErr),
Hint: "rerun the installer (deploy/bootstrap.sh) to rewrite felis-watchdog.service",
})
fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
case !found:
report.Findings = append(report.Findings, watchdog.Finding{
Key: "watchdog/unit", Severity: watchdog.Critical,
SummaryEN: fmt.Sprintf("%s is not installed: nothing checks this host or mails anyone when it breaks", unit),
Hint: "rerun the installer (deploy/bootstrap.sh), which installs felis-watchdog.timer",
})
fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
default:
fmt.Fprintf(stdout, "checks run as %s runs them (config %s)\n", unit, w.cfgPath)
}
fmt.Fprintln(stdout)
skip := map[string]string{}
cfg, cfgErr := config.Load(w.cfgPath)
if cfgErr != nil {
report.Findings = append(report.Findings, watchdog.Finding{
Key: "config", Severity: watchdog.Critical,
SummaryEN: cfgErr.Error(),
Hint: "the installer writes it (deploy/bootstrap.sh); felis watchdog fails on every run until it loads",
})
// The host's units are read without it; whether alerts reach anyone
// is not known without its relay.
for _, a := range doctorAreas {
if a.key != "config" && a.key != "systemd" {
skip[a.key] = "the configuration did not load"
}
}
} else {
_, owners, ownersErr := watchdogProbes(ctx, w, cfg, env.now, &report)
report.Findings = append(report.Findings, alertReachFindings(cfg, owners, ownersErr)...)
if w.proxyAddr == "" {
skip["proxy"] = "no -proxy-addr"
}
if w.backupDir == "" {
skip["db-backup"] = "no -backup-dir"
}
if !cfg.Offsite.Enabled() {
skip["offsite"] = "not configured"
}
if !usesMirroredScanDB(cfg) {
skip["scan-db"] = "builds do not scan against the registry's copy"
}
if len(splitList(w.certDirs)) == 0 {
skip["k3s-certs"] = "no -k3s-cert-dirs"
}
if w.nodeIP == "" {
skip["host-address"] = "no -node-ip"
}
}
report.Findings = append(report.Findings, unitFindings(ctx, env)...)
switch url, err := readHeartbeatURL(w.heartbeatFile); {
case err != nil:
report.Findings = append(report.Findings, watchdog.Finding{
Key: "watchdog/heartbeat", Severity: watchdog.Warning,
SummaryEN: fmt.Sprintf("the heartbeat URL is unusable, so no run pings it: %v", err),
Hint: "rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL set (docs/troubleshooting.md §14)",
})
case url == "":
notes = append(notes, "no heartbeat URL is set: a host that goes down entirely, or a watchdog that stops running, alerts no one. "+
"Rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL (docs/troubleshooting.md §14)")
}
if until := watchdog.QuietUntil(w.quietPath); env.now.Before(until) {
notes = append(notes, fmt.Sprintf("the watchdog mails nothing until %s (%s): the installer holds it while it restarts things on purpose, "+
"and a marker an installer killed mid-run left behind holds it until then",
until.UTC().Format("2006-01-02 15:04 UTC"), w.quietPath))
}
return printDoctorReport(stdout, report.Findings, skip, notes)
}
// printDoctorReport prints each area's findings under its heading, an area
// with none as fine or, when skip says why, as not checked, then the notes
// and the count. It returns the exit status: 1 when anything was found.
func printDoctorReport(stdout io.Writer, findings []watchdog.Finding, skip map[string]string, notes []string) int {
byArea := map[string][]watchdog.Finding{}
for _, f := range findings {
a := findingArea(f.Key)
byArea[a] = append(byArea[a], f)
}
var critical, warning int
for _, a := range doctorAreas {
fs := byArea[a.key]
switch {
case len(fs) > 0:
case skip[a.key] != "":
fmt.Fprintf(stdout, "- %s: not checked, %s\n", a.title, skip[a.key])
continue
default:
fmt.Fprintf(stdout, "✓ %s\n", a.title)
continue
}
mark := "!"
if slices.ContainsFunc(fs, func(f watchdog.Finding) bool { return f.Severity == watchdog.Critical }) {
mark = "✗"
}
fmt.Fprintf(stdout, "%s %s\n", mark, a.title)
for _, f := range fs {
if f.Severity == watchdog.Critical {
critical++
} else {
warning++
}
fmt.Fprintf(stdout, " %-8s %s: %s\n", f.Severity, f.Key, f.SummaryEN)
if f.Hint != "" {
fmt.Fprintf(stdout, " → %s\n", f.Hint)
}
}
}
for _, n := range notes {
fmt.Fprintf(stdout, "\nnote: %s\n", n)
}
fmt.Fprintln(stdout)
if critical+warning == 0 {
fmt.Fprintln(stdout, "no problems found")
return 0
}
fmt.Fprintf(stdout, "%d problem(s): %d critical, %d warning(s)\n", critical+warning, critical, warning)
return 1
}
// findingArea is the report heading a finding key goes under.
func findingArea(key string) string {
head, _, _ := strings.Cut(key, "/")
switch head {
case "kube-api", "deployment", "system-server", "server-failed", "job-failed", "reaper-stale", "node":
return "cluster"
case "db-backup", "db-backup-servers":
return "db-backup"
case "unit", "timer":
return "systemd"
case "watchdog":
return "alerts"
}
return head
}
// alertReachFindings is why the watchdog's alerts would reach no one, which
// its own runs only log: no relay, or no owner with a verified address.
func alertReachFindings(cfg *config.Config, owners []string, ownersErr error) []watchdog.Finding {
var out []watchdog.Finding
if cfg.SMTP.Host == "" {
out = append(out, watchdog.Finding{
Key: "alerts/relay", Severity: watchdog.Warning,
SummaryEN: "no [smtp] relay is configured: the watchdog logs its alerts to the journal and mails no one",
Hint: "sudo felis setup, step SMTP",
})
}
if ownersErr == nil && len(owners) == 0 {
out = append(out, watchdog.Finding{
Key: "alerts/recipients", Severity: watchdog.Warning,
SummaryEN: "no owner account has a verified email: the watchdog's alerts reach no one",
Hint: "an owner verifies an address in the panel's account settings",
})
}
return out
}
// unitFindings reports the installer's systemd units that failed, the
// long-running ones that are not running, and timers that no longer fire.
func unitFindings(ctx context.Context, env doctorEnv) []watchdog.Finding {
var out []watchdog.Finding
seen := map[string]bool{}
failed, err := env.run(ctx, "systemctl", "list-units", "--all", "--plain", "--no-legend", "--no-pager", "--state=failed", "felis-*", "k3s.service")
if err != nil && len(failed) == 0 {
return []watchdog.Finding{{
Key: "unit/systemctl", Severity: watchdog.Warning,
SummaryEN: fmt.Sprintf("systemctl list-units failed, so no unit was checked: %v", err),
}}
}
for _, line := range strings.Split(string(failed), "\n") {
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
name := fields[0]
seen[name] = true
out = append(out, watchdog.Finding{
Key: "unit/" + name, Severity: watchdog.Critical,
SummaryEN: name + " failed",
Hint: fmt.Sprintf("journalctl -u %s -n 100 --no-pager; once fixed, sudo systemctl reset-failed %s (a timer's job clears on its next good run)", name, name),
})
}
for _, name := range hostServices {
if seen[name] {
continue
}
if _, err := os.Stat(filepath.Join(env.unitDir, name)); err != nil {
continue
}
if state := unitActiveState(ctx, env, name); state != "active" {
out = append(out, watchdog.Finding{
Key: "unit/" + name, Severity: watchdog.Critical,
SummaryEN: fmt.Sprintf("%s is %s", name, state),
Hint: fmt.Sprintf("sudo systemctl start %s; journalctl -u %s -n 100 --no-pager", name, name),
})
}
}
timers, _ := filepath.Glob(filepath.Join(env.unitDir, "felis-*.timer"))
for _, path := range timers {
name := filepath.Base(path)
if state := unitActiveState(ctx, env, name); state != "active" {
out = append(out, watchdog.Finding{
Key: "timer/" + name, Severity: watchdog.Warning,
SummaryEN: fmt.Sprintf("%s is %s: the job it starts no longer runs", name, state),
Hint: fmt.Sprintf("sudo systemctl enable --now %s", name),
})
}
}
return out
}
// unitActiveState is what `systemctl is-active` says of unit.
func unitActiveState(ctx context.Context, env doctorEnv, unit string) string {
out, err := env.run(ctx, "systemctl", "is-active", unit)
if state := strings.TrimSpace(string(out)); state != "" {
return state
}
return fmt.Sprintf("in an unknown state (systemctl is-active: %v)", err)
}
// watchdogUnitFlags reads the flags felis-watchdog.service runs felis
// watchdog with. found is false when there is no such unit; w is then the
// watchdog's defaults.
func watchdogUnitFlags(path string) (w watchdogFlags, found bool, err error) {
fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
fs.SetOutput(io.Discard)
w.register(fs)
raw, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return w, false, nil
}
if err != nil {
return w, false, err
}
for _, line := range strings.Split(string(raw), "\n") {
cmd, ok := strings.CutPrefix(strings.TrimSpace(line), "ExecStart=")
if !ok {
continue
}
fields := execArgs(cmd)
i := slices.Index(fields, "watchdog")
if i < 0 {
continue
}
if err := fs.Parse(fields[i+1:]); err != nil {
return w, true, fmt.Errorf("%s: %w", path, err)
}
return w, true, nil
}
return w, true, fmt.Errorf("%s runs no `felis watchdog`", path)
}
// execArgs splits an ExecStart= command line into its words. A word may be
// quoted with " or ', as systemd allows, which is how an empty value is
// written.
func execArgs(s string) []string {
var out []string
var cur strings.Builder
inWord := false
var quote rune
for _, r := range s {
switch {
case quote != 0:
if r == quote {
quote = 0
} else {
cur.WriteRune(r)
}
case r == '"' || r == '\'':
quote, inWord = r, true
case r == ' ' || r == '\t':
if inWord {
out = append(out, cur.String())
cur.Reset()
inWord = false
}
default:
cur.WriteRune(r)
inWord = true
}
}
if inWord {
out = append(out, cur.String())
}
return out
}
cmd/felis/doctor_test.go
-423
View File
@@ -1,423 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/watchdog"
)
// bootstrapWatchdogExecStart is the ExecStart= line deploy/bootstrap.sh writes
// into felis-watchdog.service, with its variables filled in as an install
// fills them.
func bootstrapWatchdogExecStart(t *testing.T, vars map[string]string) string {
t.Helper()
raw, err := os.ReadFile("../../deploy/bootstrap.sh")
if err != nil {
t.Fatal(err)
}
var line string
for _, l := range strings.Split(string(raw), "\n") {
if strings.HasPrefix(l, "ExecStart=${HOST_BIN} watchdog -config") {
line = l
}
}
if line == "" {
t.Fatal("deploy/bootstrap.sh writes no `ExecStart=${HOST_BIN} watchdog -config` line")
}
line = strings.ReplaceAll(line, "${NODE_IP:+ -node-ip ${NODE_IP}}", " -node-ip "+vars["NODE_IP"])
line = regexp.MustCompile(`\$\{([A-Za-z_]+)\}`).ReplaceAllStringFunc(line, func(m string) string {
v, ok := vars[m[2:len(m)-1]]
if !ok {
t.Fatalf("bootstrap's watchdog ExecStart= uses %s, which this test does not fill in", m)
}
return v
})
return line
}
// felis doctor reads the watchdog's settings from the unit the installer
// writes, so it checks the paths the timer's runs check.
func TestWatchdogUnitFlagsReadsTheInstallersUnit(t *testing.T) {
dir := t.TempDir()
exec := bootstrapWatchdogExecStart(t, map[string]string{
"HOST_BIN": "/usr/local/bin/felis", "STATE_DIR": "/srv/felis-etc", "WATCHDOG_STATE": "/srv/watchdog/state.json",
"WATCHDOG_QUIET_FILE": "/srv/quiet-until", "FELIS_DB_BACKUP_DIR": "/srv/db-backups", "FELIS_GAME_PORT": "25577",
"disks": "/,/srv/data", "NODE_IP": "10.0.0.5",
})
unit := filepath.Join(dir, "felis-watchdog.service")
writeTestFile(t, unit, "[Unit]\nDescription=Felis watchdog\n\n[Service]\nType=oneshot\n"+exec+"\nTimeoutStartSec=3min\n", 0o644)
w, found, err := watchdogUnitFlags(unit)
if err != nil || !found {
t.Fatalf("found %v, err %v", found, err)
}
got := []string{w.cfgPath, w.statePath, w.quietPath, w.backupDir, w.proxyAddr, w.diskPaths, w.nodeIP, w.heartbeatFile, w.controlNS}
want := []string{"/srv/felis-etc/felis.host.toml", "/srv/watchdog/state.json", "/srv/quiet-until", "/srv/db-backups", "127.0.0.1:25577", "/,/srv/data", "10.0.0.5", defaultHeartbeatFile, "felis"}
if strings.Join(got, "|") != strings.Join(want, "|") {
t.Errorf("read\n %q\nwant\n %q", got, want)
}
w, found, err = watchdogUnitFlags(filepath.Join(dir, "missing.service"))
if err != nil || found || w.cfgPath != "/etc/felis/felis.toml" || w.backupDir != "/var/lib/felis/db-backups" {
t.Errorf("no unit: found %v, err %v, config %q, backups %q; want the watchdog's defaults", found, err, w.cfgPath, w.backupDir)
}
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis version\n", 0o644)
if _, found, err := watchdogUnitFlags(unit); !found || err == nil || !strings.Contains(err.Error(), "runs no `felis watchdog`") {
t.Errorf("a unit that runs something else: found %v, err %v", found, err)
}
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
if _, _, err := watchdogUnitFlags(unit); err == nil {
t.Error("a flag this binary does not know was accepted")
}
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -backup-dir \"\"\t-proxy-addr '127.0.0.1:1' -disk-paths \"/a b\"\n", 0o644)
if w, _, err := watchdogUnitFlags(unit); err != nil || w.backupDir != "" || w.proxyAddr != "127.0.0.1:1" || w.diskPaths != "/a b" {
t.Errorf("quoted words: backups %q, proxy %q, disks %q, err %v", w.backupDir, w.proxyAddr, w.diskPaths, err)
}
}
func TestFindingArea(t *testing.T) {
for key, want := range map[string]string{
"kube-api": "cluster",
"deployment/felis-api": "cluster",
"system-server/lobby": "cluster",
"server-failed/survival": "cluster",
"job-failed/reaper-123": "cluster",
"reaper-stale": "cluster",
"node/felis-1/NotReady": "cluster",
"postgres": "postgres",
"proxy": "proxy",
"db-backup": "db-backup",
"db-backup-servers": "db-backup",
"offsite": "offsite",
"scan-db": "scan-db",
"disk//var/lib/felis": "disk",
"memory": "memory",
"k3s-certs": "k3s-certs",
"host-address": "host-address",
"clock": "clock",
"config": "config",
"unit/felis-offsite.service": "systemd",
"timer/felis-db-backup.timer": "systemd",
"watchdog/unit": "alerts",
"watchdog/heartbeat": "alerts",
"alerts/relay": "alerts",
"alerts/recipients": "alerts",
"something-a-later-release-reported": "something-a-later-release-reported",
} {
if got := findingArea(key); got != want {
t.Errorf("findingArea(%q) = %q, want %q", key, got, want)
}
}
}
func TestAlertReachFindings(t *testing.T) {
relay := &config.Config{SMTP: config.SMTPConfig{Host: "smtp.example.com"}}
keys := func(fs []watchdog.Finding) string {
var k []string
for _, f := range fs {
k = append(k, f.Key)
}
return strings.Join(k, ",")
}
for _, tc := range []struct {
what string
cfg *config.Config
owners []string
ownersErr error
want string
}{
{"a relay and an owner", relay, []string{"[email protected]"}, nil, ""},
{"no relay", &config.Config{}, []string{"[email protected]"}, nil, "alerts/relay"},
{"no owner with an address", relay, nil, nil, "alerts/recipients"},
{"PostgreSQL down: its own finding says so", relay, nil, errors.New("refused"), ""},
{"neither", &config.Config{}, nil, nil, "alerts/relay,alerts/recipients"},
} {
if got := keys(alertReachFindings(tc.cfg, tc.owners, tc.ownersErr)); got != tc.want {
t.Errorf("%s: %q, want %q", tc.what, got, tc.want)
}
}
}
// fakeSystemctl answers list-units with failed and is-active from states;
// a unit missing from states is "inactive", as systemctl says, and one whose
// state is "" gets no answer.
func fakeSystemctl(failed string, states map[string]string) func(ctx context.Context, name string, args ...string) ([]byte, error) {
return func(_ context.Context, name string, args ...string) ([]byte, error) {
if name != "systemctl" || len(args) == 0 {
return nil, errors.New("unexpected command " + name)
}
switch args[0] {
case "list-units":
return []byte(failed), nil
case "is-active":
if s, ok := states[args[1]]; ok && s == "" {
return nil, errors.New("signal: killed")
} else if ok {
return []byte(s + "\n"), nil
}
return []byte("inactive\n"), errors.New("exit status 3")
}
return nil, errors.New("unexpected systemctl " + args[0])
}
}
func TestUnitFindings(t *testing.T) {
dir := t.TempDir()
for _, f := range []string{"k3s.service", "felis-velocity.service", "felis-db-backup.timer", "felis-offsite.timer", "felis-offsite.service"} {
writeTestFile(t, filepath.Join(dir, f), "[Unit]\n", 0o644)
}
env := doctorEnv{unitDir: dir, run: fakeSystemctl(
"felis-offsite.service loaded failed failed Felis off-site copy\nfelis-velocity.service loaded failed failed Velocity\n",
map[string]string{"k3s.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "inactive"},
)}
var got []string
for _, f := range unitFindings(context.Background(), env) {
got = append(got, string(f.Severity)+" "+f.Key+": "+f.SummaryEN)
}
want := []string{
"critical unit/felis-offsite.service: felis-offsite.service failed",
"critical unit/felis-velocity.service: felis-velocity.service failed",
"warning timer/felis-offsite.timer: felis-offsite.timer is inactive: the job it starts no longer runs",
}
if strings.Join(got, "\n") != strings.Join(want, "\n") {
t.Errorf("findings:\n%s\nwant (felis-nano.service has no unit file here, and a failed unit is reported once):\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
env.run = fakeSystemctl("", map[string]string{"k3s.service": "activating", "felis-velocity.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "active"})
got = nil
for _, f := range unitFindings(context.Background(), env) {
got = append(got, f.Key+": "+f.SummaryEN)
}
if strings.Join(got, "\n") != "unit/k3s.service: k3s.service is activating" {
t.Errorf("findings %q, want only k3s.service, which is not active", got)
}
env.run = fakeSystemctl("", map[string]string{"k3s.service": "active", "felis-velocity.service": "active", "felis-db-backup.timer": "", "felis-offsite.timer": "active"})
got = nil
for _, f := range unitFindings(context.Background(), env) {
got = append(got, f.Key+": "+f.SummaryEN)
}
if want := "timer/felis-db-backup.timer: felis-db-backup.timer is in an unknown state (systemctl is-active: signal: killed): the job it starts no longer runs"; strings.Join(got, "\n") != want {
t.Errorf("findings %q, want %q", got, want)
}
env.run = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("no systemctl") }
if fs := unitFindings(context.Background(), env); len(fs) != 1 || fs[0].Key != "unit/systemctl" || fs[0].Severity != watchdog.Warning {
t.Errorf("without systemctl: %+v, want the one unit/systemctl warning", fs)
}
}
// quoteArgs writes args as an ExecStart= line does, each word in quotes so an
// empty one survives.
func quoteArgs(args []string) string {
q := make([]string, len(args))
for i, a := range args {
q[i] = `"` + a + `"`
}
return strings.Join(q, " ")
}
// doctorHost is a host with the installer's watchdog unit, whose API server
// and PostgreSQL are down.
func doctorHost(t *testing.T, cfg string, extraFlags string) (env doctorEnv, h *watchdogHost) {
t.Helper()
h = newWatchdogHost(t, cfg, nil)
unitDir := t.TempDir()
writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"),
"[Service]\nType=oneshot\nExecStart=/usr/local/bin/felis watchdog "+quoteArgs(h.args)+
// Off this machine's disk, whose free space is not the test's.
` -disk-paths "/nonexistent-felis-doctor-test"`+extraFlags+"\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "felis-offsite.timer"), "[Unit]\n", 0o644)
return doctorEnv{
unitDir: unitDir, now: time.Now(), host: "felis-test",
run: fakeSystemctl("felis-db-backup.service loaded failed failed Felis database backup\n",
map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"}),
}, h
}
func TestDoctorReportsByArea(t *testing.T) {
env, _ := doctorHost(t, testWatchdogConfig, "")
var out bytes.Buffer
code := runDoctor(context.Background(), env, &out)
got := out.String()
if code != 1 {
t.Errorf("exit %d, want 1 with problems found", code)
}
for _, want := range []string{
"felis doctor on felis-test at ",
"checks run as " + filepath.Join(env.unitDir, "felis-watchdog.service") + " runs them (config ",
"✓ configuration\n",
"✗ Kubernetes cluster\n critical kube-api: ",
"✗ PostgreSQL\n critical postgres: ",
"- game proxy: not checked, no -proxy-addr\n",
"- database backups: not checked, no -backup-dir\n",
"- off-site copy: not checked, not configured\n",
"- build scan database: not checked, builds do not scan against the registry's copy\n",
"- k3s certificates: not checked, no -k3s-cert-dirs\n",
"- node address: not checked, no -node-ip\n",
"✗ systemd units and timers\n critical unit/felis-db-backup.service: felis-db-backup.service failed\n" +
" → journalctl -u felis-db-backup.service -n 100 --no-pager; once fixed, sudo systemctl reset-failed felis-db-backup.service",
"! alerting\n warning alerts/relay: no [smtp] relay is configured",
"\n4 problem(s): 3 critical, 1 warning(s)\n",
} {
if !strings.Contains(got, want) {
t.Errorf("report lacks %q:\n%s", want, got)
}
}
if strings.Contains(got, "alerts/recipients") {
t.Errorf("reported no recipients although PostgreSQL, which names them, is down:\n%s", got)
}
if strings.Contains(got, "note: no heartbeat URL") {
t.Errorf("the host has a heartbeat URL:\n%s", got)
}
}
// A doctor run is only a look: whatever is due to be mailed stays due, no
// heartbeat is pinged, and the watchdog's state is left as it was.
func TestDoctorMailsPingsAndSavesNothing(t *testing.T) {
cfg := testWatchdogConfig + "[smtp]\nhost = \"smtp.example.com\"\nport = 587\nfrom = \"[email protected]\"\n"
env, h := doctorHost(t, cfg, "")
if err := watchdog.SaveState(h.statePath, duePostgres("cached-pw")); err != nil {
t.Fatal(err)
}
before, err := os.ReadFile(h.statePath)
if err != nil {
t.Fatal(err)
}
watchdogSender = h.rec.sender
defer func() { watchdogSender = smtpSender }()
var out bytes.Buffer
runDoctor(context.Background(), env, &out)
if !strings.Contains(out.String(), "critical postgres: ") {
t.Fatalf("the due PostgreSQL alert was not seen:\n%s", out.String())
}
if len(h.rec.sent) != 0 || len(h.rec.relays) != 0 {
t.Errorf("mailed %v", h.rec.sent)
}
if n := len(h.pings.pings); n != 0 {
t.Errorf("pinged the heartbeat %d times", n)
}
after, err := os.ReadFile(h.statePath)
if err != nil || !bytes.Equal(before, after) {
t.Errorf("the watchdog state changed (err %v)", err)
}
if _, err := os.Stat(h.fallbackPath); !errors.Is(err, os.ErrNotExist) {
t.Errorf("a fallback state was written: %v", err)
}
}
func TestDoctorNotes(t *testing.T) {
env, h := doctorHost(t, testWatchdogConfig, "")
if err := os.Remove(filepath.Join(h.dir, "watchdog-heartbeat-url")); err != nil {
t.Fatal(err)
}
until := env.now.Add(10 * time.Minute).Unix()
writeTestFile(t, filepath.Join(h.dir, "quiet"), strconv.FormatInt(until, 10)+"\n", 0o644)
var out bytes.Buffer
runDoctor(context.Background(), env, &out)
for _, want := range []string{
"\nnote: no heartbeat URL is set: ",
"\nnote: the watchdog mails nothing until " + time.Unix(until, 0).UTC().Format("2006-01-02 15:04 UTC") + " (" + filepath.Join(h.dir, "quiet") + ")",
} {
if !strings.Contains(out.String(), want) {
t.Errorf("report lacks %q:\n%s", want, out.String())
}
}
writeTestFile(t, filepath.Join(h.dir, "watchdog-heartbeat-url"), "not a url\n", 0o600)
out.Reset()
runDoctor(context.Background(), env, &out)
if !strings.Contains(out.String(), "warning watchdog/heartbeat: the heartbeat URL is unusable") {
t.Errorf("an unusable heartbeat URL is not reported:\n%s", out.String())
}
}
// Without the watchdog's unit the doctor still checks, with the watchdog's
// defaults, and says the host has no watchdog; a configuration that does not
// load leaves the checks that need it unchecked and the host's own checks on.
func TestDoctorWithoutUnitOrConfig(t *testing.T) {
env, _ := doctorHost(t, testWatchdogConfig, "")
if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
t.Fatal(err)
}
writeTestFile(t, filepath.Join(env.unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+filepath.Join(t.TempDir(), "absent.toml")+"\n", 0o644)
env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"})
var out bytes.Buffer
if code := runDoctor(context.Background(), env, &out); code != 1 {
t.Errorf("exit %d, want 1", code)
}
for _, want := range []string{
"✗ configuration\n critical config: ",
"- Kubernetes cluster: not checked, the configuration did not load\n",
"- PostgreSQL: not checked, the configuration did not load\n",
"- disk space: not checked, the configuration did not load\n",
"✓ systemd units and timers\n",
"- alerting: not checked, the configuration did not load\n",
} {
if !strings.Contains(out.String(), want) {
t.Errorf("report lacks %q:\n%s", want, out.String())
}
}
if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
t.Fatal(err)
}
out.Reset()
runDoctor(context.Background(), env, &out)
if !strings.Contains(out.String(), "critical watchdog/unit: "+filepath.Join(env.unitDir, "felis-watchdog.service")+" is not installed") ||
!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
t.Errorf("a host without the watchdog's unit:\n%s", out.String())
}
unit := filepath.Join(env.unitDir, "felis-watchdog.service")
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
out.Reset()
runDoctor(context.Background(), env, &out)
if !strings.Contains(out.String(), "critical watchdog/unit: cannot read the watchdog's settings: "+unit+": flag provided but not defined: -no-such-flag\n") ||
!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
t.Errorf("a watchdog unit this binary cannot read:\n%s", out.String())
}
}
func TestPrintDoctorReport(t *testing.T) {
var out bytes.Buffer
if code := printDoctorReport(&out, nil, map[string]string{"proxy": "no -proxy-addr"}, nil); code != 0 {
t.Errorf("exit %d with nothing found, want 0", code)
}
want := "✓ configuration\n✓ Kubernetes cluster\n✓ PostgreSQL\n- game proxy: not checked, no -proxy-addr\n✓ database backups\n✓ off-site copy\n" +
"✓ build scan database\n✓ disk space\n✓ memory\n✓ k3s certificates\n✓ node address\n✓ clock\n✓ systemd units and timers\n✓ alerting\n" +
"\nno problems found\n"
if out.String() != want {
t.Errorf("report:\n%s\nwant:\n%s", out.String(), want)
}
out.Reset()
code := printDoctorReport(&out, []watchdog.Finding{
{Key: "unit/systemctl", Severity: watchdog.Warning, SummaryEN: "systemctl list-units failed"},
{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "PostgreSQL is down", Hint: "kubectl -n felis get pods"},
}, map[string]string{"postgres": "a skip loses to what was found"}, []string{"a note"})
if code != 1 {
t.Errorf("exit %d with problems, want 1", code)
}
for _, want := range []string{
"✗ PostgreSQL\n critical postgres: PostgreSQL is down\n → kubectl -n felis get pods\n✓ game proxy\n",
"! systemd units and timers\n warning unit/systemctl: systemctl list-units failed\n✓ alerting\n",
"✓ alerting\n\nnote: a note\n\n2 problem(s): 1 critical, 1 warning(s)\n",
} {
if !strings.Contains(out.String(), want) {
t.Errorf("report lacks %q:\n%s", want, out.String())
}
}
}
cmd/felis/domain.go
-1359
View File
File diff suppressed because it is too large. Load diff
cmd/felis/domain_test.go
-892
View File
@@ -1,892 +0,0 @@
package main
import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"math/big"
"net"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)
// installerTOML is felis.toml as deploy/bootstrap.sh write_felis_toml renders it,
// comments included: the domain move has to leave all of it but three values alone.
func installerTOML(root, dbHost string) string {
return `# Generated by deploy/bootstrap.sh; rerun the installer to regenerate. Hand edits are
# overwritten, except [smtp], [[auth_source]], [offsite], and the operator-owned
# [registry] / [archive] overrides, which carry forward.
[server]
listen = "0.0.0.0:8080"
root_domain = "` + root + `"
[database]
url = "postgres://felis:pw@` + dbHost + `:5432/felis?sslmode=disable"
[k8s]
namespace = "minecraft"
egress_mode = "nodeport"
[velocity]
# The two always-on system servers that felis setup provisions.
login_image = "felis/limbo:1"
lobby_image = "felis/lobby:1"
game_port = 25565
[registry]
url = "registry.felis.svc:5000"
build_namespace = "felis-build"
[archive]
store = "tarLocal"
local_path = "/var/lib/felis/archives"
[auth]
admin_hostname = "op.console.` + root + `"
panel_hostname = "console.` + root + `"
access_jwt_aud = "aud123"
# Third-party Yggdrasil sources federated by the hasJoined multiplexer.
[[auth_source]]
tag = "littleskin"
prefix = "LS"
url = "https://littleskin.cn/api/yggdrasil/sessionserver/session/minecraft/hasJoined"
`
}
const linkPropsBody = `# Generated by deploy/bootstrap.sh — do not edit by hand; rerun the installer.
api-base-url=http://10.43.0.9:8081
service-token=TOKEN-NOT-TO-TOUCH
root-domain=old.example
panel-hostname=console.old.example
admin-hostname=op.console.old.example
login-server=login
lobby-server=lobby
`
var oldNames = domainNames{root: "old.example", panel: "console.old.example", admin: "op.console.old.example"}
var newNames = domainNames{root: "new.example", panel: "console.new.example", admin: "op.console.new.example"}
// domainRig models the host: the files, the cluster, and a felis-api, proxy and
// operator that pick up config the way the real ones do — the api serves what it
// read at its last restart, the proxy runs since its last restart, and the login
// pod carries the env its MinecraftServer had when it was last rolled.
type domainRig struct {
h domainHost
cl client.Client
out *bytes.Buffer
dir string
events []string
served domainNames
servedCert *x509.Certificate
proxySince time.Time
proxyLoaded bool
unresolved map[string]bool
// The fake operator: the CR env the login pod was last rolled to, and how
// many looks at the pod since the CR moved on.
rolledTo string
pending int
}
func (rig *domainRig) path(name string) string { return filepath.Join(rig.dir, name) }
func newDomainRig(t *testing.T) *domainRig {
t.Helper()
rig := &domainRig{out: &bytes.Buffer{}, dir: t.TempDir(), proxyLoaded: true, unresolved: map[string]bool{}}
writeTestFile(t, rig.path("felis.host.toml"), installerTOML("old.example", "127.0.0.1"), 0o600)
writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
if err := os.Symlink(rig.path("felis.host.toml"), rig.path("felis.toml")); err != nil {
t.Fatal(err)
}
certPEM, keyPEM, err := issuePanelCert(oldNames, []net.IP{net.ParseIP("10.211.55.6")}, time.Now())
if err != nil {
t.Fatal(err)
}
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
// The proxy started before its config was last written, which is how it
// stands after an install.
rig.proxySince = time.Now().Add(-time.Hour)
pod := []byte(installerTOML("old.example", "10.211.55.6"))
login, err := loginSystemServer("felis/limbo:1", "minecraft", platform.InternalAPIBaseURL("felis"), oldNames.root, oldNames.panel)
if err != nil {
t.Fatal(err)
}
lobby, err := lobbySystemServer("felis/lobby:1", "minecraft")
if err != nil {
t.Fatal(err)
}
loginPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"},
Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "minecraft", Env: podEnv(login.Spec.Env)}}},
Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}},
}
rig.rolledTo = envKey(login.Spec.Env)
rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if key.Name == naming.SystemLoginServer+"-0" {
rig.operatorTick(t, c)
}
return c.Get(ctx, key, obj, opts...)
},
}).WithObjects(
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.ConfigSecretName},
Data: map[string][]byte{platform.ConfigSecretKey: pod}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: platform.ConfigSecretName},
Data: map[string][]byte{platform.ConfigSecretKey: pod}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.APITLSSecretName},
Type: corev1.SecretTypeTLS, Data: map[string][]byte{corev1.TLSCertKey: certPEM, corev1.TLSPrivateKeyKey: keyPEM}},
login, lobby, loginPod,
).Build()
rig.served = oldNames
rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
rig.h = domainHost{
paths: domainPaths{
hostTOML: rig.path("felis.host.toml"), podTOML: rig.path("felis.pod.toml"), defaultTOML: rig.path("felis.toml"),
cert: rig.path("panel-tls.crt"), key: rig.path("panel-tls.key"),
linkProps: rig.path("felis-link.properties"), tunnelConfig: rig.path("cloudflared.yml"),
},
cl: rig.cl,
controlNS: "felis",
rollAPI: func(ctx context.Context) error {
rig.events = append(rig.events, "roll-api")
rig.restartAPI(t)
return nil
},
restartUnit: func(_ context.Context, unit string) error {
rig.events = append(rig.events, "restart "+unit)
rig.proxySince = time.Now().Add(time.Second)
return nil
},
unitState: func(context.Context, string) (unitStatus, error) {
return unitStatus{loaded: rig.proxyLoaded, active: rig.proxyLoaded, since: rig.proxySince}, nil
},
liveAPI: func(context.Context, string) (liveAPIView, error) {
return liveAPIView{names: rig.served, cert: rig.servedCert}, nil
},
lookupHost: func(_ context.Context, host string) ([]string, error) {
if rig.unresolved[host] {
return nil, errors.New("no such host")
}
return []string{"10.211.55.6"}, nil
},
passkeys: func(context.Context) (int, int, error) { return 3, 2, nil },
now: time.Now,
out: rig.out,
loginWait: 50 * time.Millisecond,
pollEvery: time.Millisecond,
}
return rig
}
func podEnv(env []v1alpha1.EnvVar) []corev1.EnvVar {
out := make([]corev1.EnvVar, len(env))
for i, e := range env {
out[i] = corev1.EnvVar{Name: e.Name, Value: e.Value}
}
return out
}
// restartAPI makes the fake felis-api load the config and certificate its
// Secrets hold now.
func (rig *domainRig) restartAPI(t *testing.T) {
t.Helper()
var cfg, tlsSec corev1.Secret
ctx := context.Background()
if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.ConfigSecretName}, &cfg); err != nil {
t.Fatal(err)
}
if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.APITLSSecretName}, &tlsSec); err != nil {
t.Fatal(err)
}
names, err := tomlDomainNames(cfg.Data[platform.ConfigSecretKey])
if err != nil {
t.Fatal(err)
}
rig.served = names
rig.servedCert, err = x509.ParseCertificate(mustCertDER(tlsSec.Data[corev1.TLSCertKey]))
if err != nil {
t.Fatal(err)
}
}
// rollLoginPod is the operator restarting the login pod onto its CR's env.
// operatorTick is the operator as the login pod is watched: once the CR's env
// changes it takes operatorLag looks at the pod before the restarted pod
// carries the new env, the way a real rollout lags the CR.
func (rig *domainRig) operatorTick(t *testing.T, c client.Client) {
t.Helper()
ctx := context.Background()
var ms v1alpha1.MinecraftServer
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
t.Fatal(err)
}
if envKey(ms.Spec.Env) == rig.rolledTo {
return
}
if rig.pending++; rig.pending < operatorLag {
return
}
var pod corev1.Pod
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"}, &pod); err != nil {
t.Fatal(err)
}
pod.Spec.Containers[0].Env = podEnv(ms.Spec.Env)
if err := c.Update(ctx, &pod); err != nil {
t.Fatal(err)
}
rig.rolledTo, rig.pending = envKey(ms.Spec.Env), 0
}
const operatorLag = 3
func envKey(env []v1alpha1.EnvVar) string {
var b strings.Builder
for _, e := range env {
b.WriteString(e.Name + "=" + e.Value + "\n")
}
return b.String()
}
func (rig *domainRig) read(t *testing.T, name string) string {
t.Helper()
b, err := os.ReadFile(rig.path(name))
if err != nil {
t.Fatal(err)
}
return string(b)
}
func (rig *domainRig) secret(t *testing.T, ns, name string) map[string][]byte {
t.Helper()
var s corev1.Secret
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
t.Fatal(err)
}
return s.Data
}
func (rig *domainRig) crEnv(t *testing.T, name string) map[string]string {
t.Helper()
var ms v1alpha1.MinecraftServer
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
t.Fatal(err)
}
env := map[string]string{}
for _, e := range ms.Spec.Env {
env[e.Name] = e.Value
}
return env
}
// snapshot is every byte `set` may touch, for proving a refused or dry run
// touched none of it.
func (rig *domainRig) snapshot(t *testing.T) string {
t.Helper()
var b strings.Builder
entries, _ := os.ReadDir(rig.dir)
for _, e := range entries {
b.WriteString(e.Name() + "\n" + rig.read(t, e.Name()) + "\n")
}
var lines []string
for _, s := range []struct{ ns, name string }{{"felis", platform.ConfigSecretName}, {"minecraft", platform.ConfigSecretName}, {"felis", platform.APITLSSecretName}} {
for k, v := range rig.secret(t, s.ns, s.name) {
lines = append(lines, s.ns+"/"+s.name+"/"+k+"\n"+string(v))
}
}
for k, v := range rig.crEnv(t, naming.SystemLoginServer) {
lines = append(lines, "env "+k+"="+v)
}
sort.Strings(lines)
b.WriteString(strings.Join(lines, "\n"))
return b.String()
}
func TestNormalizeRootDomain(t *testing.T) {
for in, want := range map[string]string{
"Example.COM.": "example.com",
" mc.example.org ": "mc.example.org",
"10.211.55.6.nip.io": "10.211.55.6.nip.io",
"xn--bcher-kva.example": "xn--bcher-kva.example",
} {
got, err := normalizeRootDomain(in)
if err != nil || got != want {
t.Errorf("normalizeRootDomain(%q) = %q, %v; want %q", in, got, err, want)
}
}
for _, in := range []string{"", "https://example.com", "example.com:443", "example.com/x", "10.0.0.1", "::1",
"localhost", "a_b.example", "-a.example", "a-.example", strings.Repeat("a", 64) + ".example",
strings.Repeat("abcdefghi.", 25) + "example"} {
if got, err := normalizeRootDomain(in); err == nil {
t.Errorf("normalizeRootDomain(%q) = %q, want an error", in, got)
}
}
}
func TestPlanDomainChangeMovesDefaultsAndKeepsHandSetNames(t *testing.T) {
p := planDomainChange(oldNames, "new.example")
if p.to != newNames || p.customPanel || p.customAdmin {
t.Fatalf("defaults: %+v", p)
}
p = planDomainChange(domainNames{root: "old.example", panel: "play.corp.net", admin: "op.console.old.example"}, "new.example")
if p.to.panel != "play.corp.net" || !p.customPanel || p.to.admin != "op.console.new.example" || p.customAdmin {
t.Fatalf("hand-set panel: %+v", p)
}
p = planDomainChange(domainNames{root: "old.example", panel: "console.old.example", admin: "admin.corp.net"}, "new.example")
if p.to.admin != "admin.corp.net" || !p.customAdmin || p.to.panel != "console.new.example" {
t.Fatalf("hand-set admin: %+v", p)
}
}
func TestEditTOMLStringsChangesOnlyTheDomainLines(t *testing.T) {
orig := installerTOML("old.example", "127.0.0.1")
out, err := editTOMLStrings([]byte(orig), domainTOMLEdits(newNames))
if err != nil {
t.Fatal(err)
}
a, b := strings.Split(orig, "\n"), strings.Split(string(out), "\n")
if len(a) != len(b) {
t.Fatalf("line count %d → %d:\n%s", len(a), len(b), out)
}
changed := map[string]string{}
for i := range a {
if a[i] != b[i] {
changed[a[i]] = b[i]
}
}
want := map[string]string{
`root_domain = "old.example"`: `root_domain = "new.example"`,
`admin_hostname = "op.console.old.example"`: `admin_hostname = "op.console.new.example"`,
`panel_hostname = "console.old.example"`: `panel_hostname = "console.new.example"`,
}
if len(changed) != len(want) {
t.Fatalf("changed lines %v, want %v", changed, want)
}
for k, v := range want {
if changed[k] != v {
t.Errorf("%q → %q, want %q", k, changed[k], v)
}
}
}
func TestEditTOMLStringsAddsMissingKeysInTheirTable(t *testing.T) {
in := "[server]\nroot_domain = \"old.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\n\n[smtp]\nhost = \"relay\"\n"
out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames))
if err != nil {
t.Fatal(err)
}
want := "[server]\nroot_domain = \"new.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\npanel_hostname = \"console.new.example\"\nadmin_hostname = \"op.console.new.example\"\n\n[smtp]\nhost = \"relay\"\n"
if string(out) != want {
t.Fatalf("got:\n%s\nwant:\n%s", out, want)
}
out, err = editTOMLStrings([]byte("[server]\nroot_domain = \"old.example\"\n\n[[auth_source]]\ntag = \"ls\"\n"), domainTOMLEdits(newNames))
if err != nil {
t.Fatal(err)
}
got, err := tomlDomainNames(out)
if err != nil || got != newNames || !strings.Contains(string(out), "[[auth_source]]\ntag = \"ls\"\n") {
t.Fatalf("no [auth] table: %v %+v\n%s", err, got, out)
}
}
func TestEditTOMLStringsRefusesWhatItCannotEditExactly(t *testing.T) {
for name, in := range map[string]string{
"multi-line value": "[server]\nroot_domain = \"\"\"\nold.example\"\"\"\n[auth]\n",
// The key's line sits inside another value; the real key is absent.
"key inside a string": "[server]\nmotd = \"\"\"\nroot_domain = \"old.example\"\n\"\"\"\n[auth]\n",
"quoted header": "[server]\nroot_domain = \"old.example\"\n[\"auth\"]\npanel_hostname = \"console.old.example\"\n",
"dotted key": "server.root_domain = \"old.example\"\n",
"inline table": "server = { root_domain = \"old.example\" }\n",
} {
if out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames)); err == nil {
t.Errorf("%s: edited instead of refusing:\n%s", name, out)
}
}
}
// caSignedCert is an operator's certificate from their own CA; it names
// localhost too, so only the issuer tells it apart from the installer's.
func caSignedCert(t *testing.T, hosts ...string) (certPEM, keyPEM []byte) {
t.Helper()
caKey, _ := rsa.GenerateKey(rand.Reader, 2048)
ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Corp CA"}, IsCA: true,
BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
caCert, _ := x509.ParseCertificate(caDER)
key, _ := rsa.GenerateKey(rand.Reader, 2048)
leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: hosts[0]},
DNSNames: append(hosts, "localhost"), IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
der, err := x509.CreateCertificate(rand.Reader, leaf, caCert, &key.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
pk, _ := x509.MarshalPKCS8PrivateKey(key)
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pk})
}
func TestFelisIssuedCert(t *testing.T) {
mine, _, err := issuePanelCert(oldNames, nil, time.Now())
if err != nil {
t.Fatal(err)
}
c, _ := x509.ParseCertificate(mustCertDER(mine))
if !felisIssuedCert(c) {
t.Error("the installer's kind of certificate is not recognised as Felis-issued")
}
theirs, _ := caSignedCert(t, "console.old.example")
c, _ = x509.ParseCertificate(mustCertDER(theirs))
if felisIssuedCert(c) {
t.Error("a CA-signed certificate is taken for Felis-issued")
}
// Self-signed but without one of the installer's localhost names: someone
// else's.
key, _ := rsa.GenerateKey(rand.Reader, 2048)
for name, self := range map[string]*x509.Certificate{
"no localhost": {DNSNames: []string{"console.old.example"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}},
"no 127.0.0.1": {DNSNames: []string{"console.old.example", "localhost"}},
} {
self.SerialNumber, self.Subject = big.NewInt(3), pkix.Name{CommonName: "x"}
self.NotBefore, self.NotAfter = time.Now().Add(-time.Hour), time.Now().Add(time.Hour)
der, _ := x509.CreateCertificate(rand.Reader, self, self, &key.PublicKey, key)
c, _ = x509.ParseCertificate(der)
if felisIssuedCert(c) {
t.Errorf("%s: a self-signed certificate is taken for Felis-issued", name)
}
}
}
func TestIssuePanelCertIsTheInstallersShape(t *testing.T) {
now := time.Now()
certPEM, keyPEM, err := issuePanelCert(newNames, []net.IP{net.ParseIP("10.211.55.6"), net.IPv4(127, 0, 0, 1)}, now)
if err != nil {
t.Fatal(err)
}
if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil {
t.Fatalf("key does not match the certificate: %v", err)
}
if b, _ := pem.Decode(keyPEM); b == nil || b.Type != "PRIVATE KEY" {
t.Fatalf("key is not PKCS#8 PEM like openssl writes")
}
c, _ := x509.ParseCertificate(mustCertDER(certPEM))
if !certCovers(c, newNames.panel, newNames.admin, "localhost") || !felisIssuedCert(c) {
t.Fatalf("names %v", c.DNSNames)
}
if len(c.IPAddresses) != 2 || !c.IPAddresses[1].Equal(net.ParseIP("10.211.55.6")) {
t.Fatalf("addresses %v, want 127.0.0.1 and the node address once each", c.IPAddresses)
}
if c.Subject.CommonName != newNames.admin || c.NotAfter.Sub(now) < 824*24*time.Hour || c.NotAfter.Sub(now) > 826*24*time.Hour {
t.Fatalf("CN %q, valid until %s", c.Subject.CommonName, c.NotAfter)
}
if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth || c.IsCA {
t.Fatalf("usage %v, CA %v", c.ExtKeyUsage, c.IsCA)
}
}
func TestDomainSetMovesEverySurface(t *testing.T) {
rig := newDomainRig(t)
hostBefore := rig.read(t, "felis.host.toml")
code, err := rig.h.set(context.Background(), "New.Example", true)
if err != nil || code != 0 {
t.Fatalf("set = %d, %v\n%s", code, err, rig.out)
}
// The configs: the three values moved, everything else — comments, the
// database host of each copy, the Access audience — is as it was.
host := rig.read(t, "felis.host.toml")
if want := strings.NewReplacer("old.example", "new.example").Replace(hostBefore); host != want {
t.Fatalf("host toml:\n%s", host)
}
pod := rig.read(t, "felis.pod.toml")
if got, _ := tomlDomainNames([]byte(pod)); got != newNames || !strings.Contains(pod, "@10.211.55.6:5432") {
t.Fatalf("pod toml:\n%s", pod)
}
if link, err := os.Readlink(rig.path("felis.toml")); err != nil || link != rig.path("felis.host.toml") {
t.Fatalf("felis.toml is no longer the link to the host copy: %q %v", link, err)
}
if st, _ := os.Stat(rig.path("felis.host.toml")); st.Mode().Perm() != 0o600 {
t.Fatalf("host toml mode %v", st.Mode().Perm())
}
// The certificate: reissued for the new names, the node address kept, the old
// pair beside it.
c, err := readCertFile(rig.path("panel-tls.crt"))
if err != nil || !certCovers(c, newNames.panel, newNames.admin) || !felisIssuedCert(c) {
t.Fatalf("certificate: %v %v", err, c.DNSNames)
}
if !c.IPAddresses[len(c.IPAddresses)-1].Equal(net.ParseIP("10.211.55.6")) {
t.Fatalf("addresses %v", c.IPAddresses)
}
if _, err := tls.LoadX509KeyPair(rig.path("panel-tls.crt"), rig.path("panel-tls.key")); err != nil {
t.Fatalf("new pair: %v", err)
}
if st, _ := os.Stat(rig.path("panel-tls.key")); st.Mode().Perm() != 0o600 {
t.Fatalf("key mode %v", st.Mode().Perm())
}
backups, _ := filepath.Glob(rig.path("panel-tls.*.pre-domain-*"))
if len(backups) != 2 {
t.Fatalf("old pair kept as %v", backups)
}
for _, b := range backups {
if st, _ := os.Stat(b); strings.Contains(b, ".key.") && st.Mode().Perm() != 0o600 {
t.Fatalf("kept key %s has mode %v", b, st.Mode().Perm())
}
old, _ := os.ReadFile(b)
if strings.Contains(b, ".crt.") {
oc, _ := x509.ParseCertificate(mustCertDER(old))
if oc == nil || !certCovers(oc, oldNames.panel) {
t.Fatalf("kept certificate is not the old one")
}
}
}
// The Secrets carry the files.
for _, ns := range []string{"felis", "minecraft"} {
if got := rig.secret(t, ns, platform.ConfigSecretName)[platform.ConfigSecretKey]; string(got) != pod {
t.Fatalf("%s/felis-config is not felis.pod.toml", ns)
}
}
tlsData := rig.secret(t, "felis", platform.APITLSSecretName)
if string(tlsData[corev1.TLSCertKey]) != rig.read(t, "panel-tls.crt") || string(tlsData[corev1.TLSPrivateKeyKey]) != rig.read(t, "panel-tls.key") {
t.Fatal("felis-api-tls does not hold the new pair")
}
// The login gate's env moved and nothing else did.
env := rig.crEnv(t, naming.SystemLoginServer)
if env[envRootDomain] != newNames.root || env[envPanelHostname] != newNames.panel || env[envAPIBaseURL] != platform.InternalAPIBaseURL("felis") {
t.Fatalf("login env %v", env)
}
// The proxy's file: the three keys moved, its token and mode did not.
props := rig.read(t, "felis-link.properties")
if want := strings.NewReplacer("old.example", "new.example").Replace(linkPropsBody); props != want {
t.Fatalf("felis-link.properties:\n%s", props)
}
if st, _ := os.Stat(rig.path("felis-link.properties")); st.Mode().Perm() != 0o640 {
t.Fatalf("felis-link.properties mode %v", st.Mode().Perm())
}
if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
t.Fatalf("events %v", rig.events)
}
if !strings.Contains(rig.out.String(), "Every surface is on new.example.") {
t.Fatalf("the closing check did not pass:\n%s", rig.out)
}
}
func TestDomainSetWithoutYesChangesNothing(t *testing.T) {
rig := newDomainRig(t)
before := rig.snapshot(t)
code, err := rig.h.set(context.Background(), "new.example", false)
if err != nil || code != 0 {
t.Fatalf("set = %d, %v", code, err)
}
if rig.snapshot(t) != before || len(rig.events) != 0 {
t.Fatalf("a dry run changed something (events %v)", rig.events)
}
out := rig.out.String()
for _, want := range []string{
"console.old.example → console.new.example",
"3 passkey(s) of 2 user(s) are bound to console.old.example",
"does not cover op.console.new.example",
"No [smtp] relay is configured",
"sudo felis domain set -yes new.example",
} {
if !strings.Contains(out, want) {
t.Errorf("plan lacks %q:\n%s", want, out)
}
}
}
func TestDomainSetKeepsAHandSetPanelHostname(t *testing.T) {
rig := newDomainRig(t)
for _, f := range []string{"felis.host.toml", "felis.pod.toml"} {
writeTestFile(t, rig.path(f), strings.Replace(rig.read(t, f), `panel_hostname = "console.old.example"`, `panel_hostname = "play.corp.net"`, 1), 0o600)
}
code, err := rig.h.set(context.Background(), "new.example", true)
if err != nil {
t.Fatalf("set: %v\n%s", err, rig.out)
}
got, _ := tomlDomainNames([]byte(rig.read(t, "felis.host.toml")))
if got != (domainNames{root: "new.example", panel: "play.corp.net", admin: "op.console.new.example"}) {
t.Fatalf("names %+v", got)
}
c, _ := readCertFile(rig.path("panel-tls.crt"))
if !certCovers(c, "play.corp.net", "op.console.new.example") {
t.Fatalf("certificate names %v", c.DNSNames)
}
if env := rig.crEnv(t, naming.SystemLoginServer); env[envPanelHostname] != "play.corp.net" {
t.Fatalf("login env %v", env)
}
if code != 0 || !strings.Contains(rig.out.String(), "play.corp.net (set by hand, kept") {
t.Fatalf("code %d:\n%s", code, rig.out)
}
}
func TestDomainSetRefusesAnOperatorCertificateForOtherNames(t *testing.T) {
rig := newDomainRig(t)
certPEM, keyPEM := caSignedCert(t, "console.old.example", "op.console.old.example")
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
before := rig.snapshot(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "not issued by Felis") {
t.Fatalf("err = %v", err)
}
if rig.snapshot(t) != before || len(rig.events) != 0 {
t.Fatal("a refused move changed something")
}
// The operator's certificate for the new names is kept as it is.
certPEM, keyPEM = caSignedCert(t, "console.new.example", "op.console.new.example")
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatalf("set: %v", err)
}
if rig.read(t, "panel-tls.crt") != string(certPEM) {
t.Fatal("the operator's certificate was replaced")
}
}
func TestDomainSetRefusesAConfigItCannotEdit(t *testing.T) {
rig := newDomainRig(t)
writeTestFile(t, rig.path("felis.pod.toml"), strings.Replace(rig.read(t, "felis.pod.toml"), "[auth]", "[\"auth\"]", 1), 0o600)
before := rig.snapshot(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "by hand") {
t.Fatalf("err = %v", err)
}
if rig.snapshot(t) != before || len(rig.events) != 0 {
t.Fatal("a refused move changed something")
}
}
func TestDomainSetAgainOnlyConvergesWhatIsBehind(t *testing.T) {
rig := newDomainRig(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
rig.events, rig.out = nil, &bytes.Buffer{}
rig.h.out = rig.out
before := rig.snapshot(t)
code, err := rig.h.set(context.Background(), "new.example", true)
if err != nil || code != 0 {
t.Fatalf("second set = %d, %v\n%s", code, err, rig.out)
}
if len(rig.events) != 0 || rig.snapshot(t) != before {
t.Fatalf("a converged install was touched again: %v\n%s", rig.events, rig.out)
}
// A proxy that was not restarted after the move is restarted by a re-run, and
// an api still on the old config is rolled.
rig.proxySince = time.Now().Add(-time.Hour)
rig.served = oldNames
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
t.Fatalf("events %v", rig.events)
}
// An api on the new names that still presents the old certificate is rolled.
rig.events = nil
oldCert, _, _ := issuePanelCert(oldNames, nil, time.Now())
rig.servedCert, _ = x509.ParseCertificate(mustCertDER(oldCert))
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
if strings.Join(rig.events, ",") != "roll-api" {
t.Fatalf("events %v", rig.events)
}
}
func TestDomainSetRefusesALoginServerItDoesNotOwn(t *testing.T) {
rig := newDomainRig(t)
var ms v1alpha1.MinecraftServer
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
t.Fatal(err)
}
delete(ms.Labels, v1alpha1.LabelSystemRole)
if err := rig.cl.Update(context.Background(), &ms); err != nil {
t.Fatal(err)
}
if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "system role") {
t.Fatalf("err = %v", err)
}
if env := rig.crEnv(t, naming.SystemLoginServer); env[envRootDomain] != oldNames.root {
t.Fatalf("a server not marked as the login gate was changed: %v", env)
}
}
func TestDomainCheckNamesTheSurfaceThatIsBehind(t *testing.T) {
cases := []struct {
surface string
breakIt func(t *testing.T, rig *domainRig)
}{
{"felis.pod.toml", func(t *testing.T, rig *domainRig) {
writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
}},
{"Secret minecraft/felis-config", func(t *testing.T, rig *domainRig) {
rig.putSecret(t, "minecraft", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
}},
{"Secret felis/felis-config", func(t *testing.T, rig *domainRig) {
rig.putSecret(t, "felis", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
}},
{"panel certificate", func(t *testing.T, rig *domainRig) {
// The Secret follows the file, so only the certificate's names are wrong.
certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
}},
{"Secret felis/felis-api-tls", func(t *testing.T, rig *domainRig) {
certPEM, _, _ := issuePanelCert(newNames, nil, time.Now())
rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
}},
{"felis-api", func(t *testing.T, rig *domainRig) { rig.served = oldNames }},
{"felis-api", func(t *testing.T, rig *domainRig) {
certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
}},
{"proxy", func(t *testing.T, rig *domainRig) {
writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
rig.proxySince = time.Now().Add(time.Hour)
}},
{"proxy", func(t *testing.T, rig *domainRig) { rig.proxySince = time.Now().Add(-time.Hour) }},
{"login gate", func(t *testing.T, rig *domainRig) {
var ms v1alpha1.MinecraftServer
_ = rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms)
for i := range ms.Spec.Env {
if ms.Spec.Env[i].Name == envRootDomain {
ms.Spec.Env[i].Value = "old.example"
}
}
if err := rig.cl.Update(context.Background(), &ms); err != nil {
t.Fatal(err)
}
}},
{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envRootDomain, "old.example") }},
{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envPanelHostname, "console.old.example") }},
{"Cloudflare tunnel", func(t *testing.T, rig *domainRig) {
writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n service: https://127.0.0.1:30443\n- hostname: op.console.old.example\n service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
}},
}
for _, tc := range cases {
rig := newDomainRig(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
rig.out.Reset()
tc.breakIt(t, rig)
if code := rig.h.check(context.Background()); code != 1 {
t.Errorf("%s behind: check = %d\n%s", tc.surface, code, rig.out)
continue
}
var failed []string
for _, ln := range strings.Split(rig.out.String(), "\n") {
if strings.HasPrefix(ln, " FAIL ") {
failed = append(failed, ln)
}
}
if len(failed) != 1 || !strings.Contains(failed[0], tc.surface) {
t.Errorf("%s behind: FAIL lines %q", tc.surface, failed)
}
}
}
func TestDomainCheckPassesAConvergedInstallAndWarnsOnDNS(t *testing.T) {
rig := newDomainRig(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n service: https://127.0.0.1:30443\n- hostname: op.console.new.example\n service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
rig.unresolved["op.console.new.example"] = true
rig.unresolved[dnsProbeLabel+".new.example"] = true
rig.out.Reset()
if code := rig.h.check(context.Background()); code != 0 {
t.Fatalf("check = %d\n%s", code, rig.out)
}
out := rig.out.String()
for _, want := range []string{" ok Cloudflare tunnel: routes both names", " warn DNS: op.console.new.example, *.new.example do not resolve from this host\n"} {
if !strings.Contains(out, want) {
t.Errorf("check lacks %q:\n%s", want, out)
}
}
if strings.Contains(out, "TOKEN-NOT-TO-TOUCH") {
t.Fatal("check printed the proxy's service token")
}
// A zone with only the wildcard: the admin name alone is missing, and why is said.
delete(rig.unresolved, dnsProbeLabel+".new.example")
rig.out.Reset()
rig.h.check(context.Background())
if want := " warn DNS: op.console.new.example does not resolve from this host: the *.new.example wildcard does not cover op.console.new.example, which needs its own record\n"; !strings.Contains(rig.out.String(), want) {
t.Errorf("check lacks %q:\n%s", want, rig.out)
}
}
func (rig *domainRig) setPodEnv(t *testing.T, name, value string) {
t.Helper()
var pod corev1.Pod
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
t.Fatal(err)
}
for i, e := range pod.Spec.Containers[0].Env {
if e.Name == name {
pod.Spec.Containers[0].Env[i].Value = value
}
}
if err := rig.cl.Update(context.Background(), &pod); err != nil {
t.Fatal(err)
}
}
func (rig *domainRig) putSecret(t *testing.T, ns, name, key, val string) {
t.Helper()
var s corev1.Secret
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
t.Fatal(err)
}
s.Data[key] = []byte(val)
if err := rig.cl.Update(context.Background(), &s); err != nil {
t.Fatal(err)
}
}
func TestParseUnitShow(t *testing.T) {
st := parseUnitShow("LoadState=loaded\nActiveState=active\nActiveEnterTimestamp=@1790000000\n")
if !st.loaded || !st.active || !st.since.Equal(time.Unix(1790000000, 0)) {
t.Fatalf("%+v", st)
}
st = parseUnitShow("LoadState=not-found\nActiveState=inactive\nActiveEnterTimestamp=\n")
if st.loaded || st.active || !st.since.IsZero() {
t.Fatalf("%+v", st)
}
}
cmd/felis/egressgate.go
+11 -35
View File
@@ -17,29 +17,23 @@ var (
egressPollInterval = 200 * time.Millisecond
)
// cmdEgressGate is the first initContainer of every build pod and the last of
// every game server pod. A pod's NetworkPolicy is programmed asynchronously after
// the pod starts (live on k3s: a build-labelled pod reached the internet and the
// Kubernetes API for its first ~0.7 s, a server-labelled one felis-api's internal
// face on its first request), so the gate dials a destination the policy denies
// until it stops answering, and only then lets the pod's next container, the
// untrusted Dockerfile or server image, start.
// cmdEgressGate is the first initContainer of every build pod. The pod's
// NetworkPolicy is programmed asynchronously after the pod starts (live on k3s:
// a build-labelled pod reached the internet and the Kubernetes API for its first
// ~0.7 s), so the gate dials a destination the policy denies until it stops
// answering, and only then lets the pod's next container, eventually the
// untrusted Dockerfile, start.
//
// The default probe is the Kubernetes API Service, which the kubelet names in
// every pod's environment and neither policy admits. A probe that still answers
// after --wait means the policy is not enforced at all (a CNI without
// every pod's environment and the build policy never admits. A probe that still
// answers after --wait means the policy is not enforced at all (a CNI without
// NetworkPolicy support, or k3s run with --disable-network-policy), and the
// build fails closed. A server passes --fail-open: an operator's
// --server-egress-allow-cidr may cover the node the API Service leads to, so a
// probe that keeps answering does not prove the fence is missing, and by then
// the policy has had --wait to land.
// build fails closed.
func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError)
fs.SetOutput(stderr)
probe := fs.String("probe", "", "host:port the pod's NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
positive := fs.String("positive-probe", "", "allowed host:port that must remain reachable during denial checks")
wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the gate gives up")
failOpen := fs.Bool("fail-open", false, "when --wait runs out, warn and let the pod go on instead of refusing it")
probe := fs.String("probe", "", "host:port the build NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the build is refused")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -54,18 +48,6 @@ func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
start := time.Now()
for {
if *positive != "" {
allowed, err := net.DialTimeout("tcp", *positive, egressDialTimeout)
if err != nil {
if time.Since(start) >= *wait {
fmt.Fprintln(stderr, "felis egress-gate: positive probe unavailable; refusing to start", err)
return 1
}
time.Sleep(egressPollInterval)
continue
}
allowed.Close()
}
conn, err := net.DialTimeout("tcp", *probe, egressDialTimeout)
if err != nil {
fmt.Fprintf(stdout, "felis egress-gate: %s is unreachable after %s (%v); the egress lock is in effect\n",
@@ -74,12 +56,6 @@ func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
}
_ = conn.Close()
if time.Since(start) >= *wait {
if *failOpen {
fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s; starting anyway. Either this namespace's "+
"NetworkPolicy is not enforced (a CNI without NetworkPolicy support, or k3s started with "+
"--disable-network-policy), or an allowed CIDR admits the address behind it\n", *probe, *wait)
return 0
}
fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s: the build namespace's NetworkPolicy is not enforced "+
"(a CNI without NetworkPolicy support, or k3s started with --disable-network-policy); refusing to run the build\n",
*probe, *wait)
cmd/felis/egressgate_test.go
-48
View File
@@ -15,20 +15,6 @@ func shrinkEgressGate(t *testing.T) {
t.Cleanup(func() { egressDialTimeout, egressPollInterval = dial, poll })
}
func TestEgressGateRequiresPositiveReachability(t *testing.T) {
shrinkEgressGate(t)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
closed := ln.Addr().String()
ln.Close()
var out, errb bytes.Buffer
if code := cmdEgressGate([]string{"--positive-probe", closed, "--probe", closed, "--wait", "20ms"}, &out, &errb); code != 1 {
t.Fatal("unavailable probes allowed startup", code)
}
}
// The gate holds while the probe answers and lets the pod go on once the policy
// lands, which the test plays by closing the listener.
func TestEgressGateWaitsForTheLock(t *testing.T) {
@@ -89,40 +75,6 @@ func TestEgressGateRefusesAnOpenNetwork(t *testing.T) {
}
}
// A server's gate waits out --wait all the same, then lets the pod start with a
// warning in its log.
func TestEgressGateFailOpenWaitsThenWarns(t *testing.T) {
shrinkEgressGate(t)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
_ = c.Close()
}
}()
var out, errb bytes.Buffer
start := time.Now()
if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "150ms", "--fail-open"}, &out, &errb); code != 0 {
t.Fatalf("exit %d, want 0: %s", code, errb.String())
}
if waited := time.Since(start); waited < 150*time.Millisecond {
t.Errorf("gave up after %s, before --wait ran out", waited)
}
if !strings.Contains(errb.String(), ln.Addr().String()+" still answers after 150ms; starting anyway") {
t.Errorf("stderr = %q", errb.String())
}
if out.Len() != 0 {
t.Errorf("stdout = %q, want nothing: the lock was never seen", out.String())
}
}
func TestEgressGateDefaultsToTheKubernetesService(t *testing.T) {
shrinkEgressGate(t)
t.Setenv("KUBERNETES_SERVICE_HOST", "")
cmd/felis/export.go
-307
View File
@@ -1,307 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"hash"
"io"
"io/fs"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"felis.lolicon.best/internal/backup"
"felis.lolicon.best/internal/fileedit"
"felis.lolicon.best/internal/worldexport"
)
// cmdExport is the in-Pod entrypoint the export Job runs. internal/worldexport
// renders a Pod whose command is `/usr/local/bin/felis export`. It archives the
// mounted world, re-streams one archive from the mounted backup store, or sends
// one file or folder of the world, PUTs it to felis-api's internal face, and
// exits once felis-api says the owner's browser got all of it. It is NOT a
// user-facing command and is never invoked by hand.
//
// Like cmdRestore it holds no database credentials and never calls config.Load:
// felis-api made every decision (who may download what, that the server is
// stopped, which archive) before the Job existed. Its input is the flags below
// plus the one-time upload token in the environment, which opens this one
// export and nothing else.
//
// Whatever leaves goes through the same guards as the file editor
// (fileedit.Guard): the proxy forwarding secret, which every server on the
// install shares, never leaves, and server.properties leaves with its RCON
// password redacted. A backup is stored with both, since a restore must bring
// the world back whole, so it is filtered on the way out rather than handed
// over as stored.
//
// Exit status: 0 once felis-api answers 204 (the download completed), 1 when
// the export could not be read or handed over, a backup failed its digest
// check, or felis-api refused it (the browser never came or left early), 2 on
// bad flags. The last stderr line reaches the export's status and the jobs list.
func cmdExport(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("export", flag.ContinueOnError)
fs.SetOutput(stderr)
mode := fs.String("mode", "", "what to export: world, backup or files")
server := fs.String("server", "", "server name being exported (for logging)")
target := fs.String("target-url", "", "felis-api URL to PUT the export to")
ref := fs.String("ref", "", "backup only: absolute path to the archive on the backup mount")
backupRoot := fs.String("backup-root", "/backups", "backup only: mount path of the backup PVC (the ref must resolve under it)")
sum := fs.String("sha256", "", "backup only: the sha256 recorded when the archive was written; a mismatch fails the export before its end is sent")
worldsRoot := fs.String("worlds-root", "/world", "world and files: mount path of the world PVC")
path := fs.String("path", "", "files only: the file or folder to send, relative to the world root")
rawArchive := fs.Bool("archive-raw", false, "internal archive transfer: preserve the complete world")
dir := fs.Bool("dir", false, "files only: the path is a folder, sent as a zip")
if err := fs.Parse(args); err != nil {
return 2
}
token := os.Getenv(worldexport.TokenEnv)
if *target == "" || token == "" {
fmt.Fprintf(stderr, "felis export: --target-url and %s are required\n", worldexport.TokenEnv)
return 2
}
limitHeapToCgroup()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
var err error
switch *mode {
case worldexport.ModeBackup:
if *ref == "" {
fmt.Fprintln(stderr, "felis export: --ref is required for a backup")
return 2
}
err = exportBackup(ctx, *target, token, *ref, *backupRoot, *sum, stdout)
case worldexport.ModeWorld:
if *rawArchive {
err = streamExport(ctx, *target, token, archiveType, -1, func(w io.Writer) error { _, _, err := backup.WriteTarGz(ctx, w, *worldsRoot, nil); return err })
} else {
err = exportWorld(ctx, *target, token, *worldsRoot, stdout)
}
case worldexport.ModeFiles:
if *path == "" {
fmt.Fprintln(stderr, "felis export: --path is required for files")
return 2
}
err = exportFiles(ctx, *target, token, *worldsRoot, *path, *dir, stdout)
default:
fmt.Fprintf(stderr, "felis export: --mode must be %s, %s or %s\n", worldexport.ModeWorld, worldexport.ModeBackup, worldexport.ModeFiles)
return 2
}
if err != nil {
fmt.Fprintf(stderr, "felis export: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "felis export: server=%s mode=%s downloaded\n", *server, *mode)
return 0
}
// archiveType is the content type of a world or backup export.
const archiveType = "application/gzip"
// errBackupDigest fails a backup export whose stored archive no longer hashes
// to what was recorded when it was written.
var errBackupDigest = errors.New("the backup archive does not match the sha256 recorded when it was written")
// exportBackup re-streams one stored archive through the export guards
// (backup.FilterTarGz with archiveFilter). Its length changes on the way, so it
// goes chunked. With want set, the stored bytes are hashed as they are read,
// and FilterTarGz reads them to their end before it closes its own archive: a
// mismatch aborts the upload while what felis-api has passed on still lacks
// its end, so the browser never keeps a complete-looking corrupt file.
func exportBackup(ctx context.Context, target, token, ref, root, want string, stdout io.Writer) error {
// Defense in depth, as in cmdRestore: the ref comes from felis-api, but this
// process opens it, so it confirms the ref stays on the backup mount.
if !refWithinRoot(ref, root) {
return fmt.Errorf("ref %q is not under backup root %q", ref, root)
}
f, err := os.Open(ref)
if err != nil {
return err
}
defer f.Close()
var src io.Reader = f
if want != "" {
src = &digestReader{r: f, sum: sha256.New(), want: want}
}
var withheld []string
err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
var err error
withheld, err = backup.FilterTarGz(ctx, w, src, archiveFilter)
return err
})
if errors.Is(err, errBackupDigest) {
return errBackupDigest // the jobs list shows it as it is, not wrapped as a read error
}
reportWithheld(stdout, len(withheld))
return err
}
// exportWorld archives the world straight into the request body: nothing is
// staged, so a world bigger than the Pod's memory or any scratch disk exports
// the same.
func exportWorld(ctx context.Context, target, token, root string, stdout io.Writer) error {
r, err := os.OpenRoot(root)
if err != nil {
return err
}
guard := fileedit.NewGuard(r)
r.Close()
var skipped, withheld []string
err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
var err error
skipped, withheld, err = backup.WriteTarGz(ctx, w, root, worldFilter(guard))
return err
})
if len(skipped) > 0 {
fmt.Fprintf(stdout, "felis export: left out %d entries a tar cannot hold (symbolic links, devices, sockets)\n", len(skipped))
}
reportWithheld(stdout, len(withheld))
return err
}
// exportFiles sends one file or folder of the world (fileedit.OpenDownload): a
// file with its exact length, a folder as a zip made as it streams. dir is
// what the owner saw at path when they asked.
func exportFiles(ctx context.Context, target, token, root, path string, dir bool, stdout io.Writer) error {
d, err := fileedit.OpenDownload(root, path, dir)
if err != nil {
return err
}
defer d.Close()
err = streamExport(ctx, target, token, d.ContentType, d.Size, func(w io.Writer) error { return d.WriteTo(ctx, w) })
if d.Skipped > 0 {
fmt.Fprintf(stdout, "felis export: left out %d entries a zip does not carry (symbolic links, devices, sockets)\n", d.Skipped)
}
reportWithheld(stdout, d.Withheld)
return err
}
func reportWithheld(stdout io.Writer, n int) {
if n > 0 {
fmt.Fprintf(stdout, "felis export: left out %d files that hold platform secrets\n", n)
}
}
// worldFilter guards a live world by file identity, so a link to a guarded
// file under another name is caught as well.
func worldFilter(g fileedit.Guard) backup.Filter {
return func(_ string, info fs.FileInfo) (bool, func([]byte) []byte) {
return guardAction(g.Rule(info))
}
}
// archiveFilter guards a stored archive, which has only names.
func archiveFilter(name string, _ fs.FileInfo) (bool, func([]byte) []byte) {
return guardAction(fileedit.ArchiveRule(name))
}
func guardAction(withhold, redact bool) (bool, func([]byte) []byte) {
if redact {
return withhold, fileedit.RedactProps
}
return withhold, nil
}
// digestReader passes r through, hashing it, and turns r's EOF into
// errBackupDigest when the bytes do not hash to want.
type digestReader struct {
r io.Reader
sum hash.Hash
want string
}
func (d *digestReader) Read(p []byte) (int, error) {
n, err := d.r.Read(p)
d.sum.Write(p[:n])
if err == io.EOF && !strings.EqualFold(hex.EncodeToString(d.sum.Sum(nil)), d.want) {
return n, errBackupDigest
}
return n, err
}
// streamExport runs write straight into the body of the PUT, hashing it as it
// goes. Once write has finished, the SHA-256 of all it wrote rides the
// request's trailer (worldexport.DigestTrailer), and felis-api holds back the
// last bytes from the browser until what it received hashes the same. An error
// from write aborts the chunked body before the trailer, and felis-api then
// cuts the browser's download off rather than end it; that error is the one
// reported, since the PUT's own error only wraps it. When the PUT ends first,
// write is stopped.
func streamExport(ctx context.Context, target, token, contentType string, size int64, write func(io.Writer) error) error {
pr, pw := io.Pipe()
trailer := http.Header{worldexport.DigestTrailer: nil}
werr := make(chan error, 1)
go func() {
sum := sha256.New()
err := write(io.MultiWriter(pw, sum))
if err == nil {
// Set before the body ends: the transport reads the trailer once it
// has read the body to its end.
trailer.Set(worldexport.DigestTrailer, "sha-256=:"+base64.StdEncoding.EncodeToString(sum.Sum(nil))+":")
}
pw.CloseWithError(err)
werr <- err
}()
err := putExport(ctx, target, token, contentType, pr, size, trailer)
pr.CloseWithError(io.ErrClosedPipe)
if w := <-werr; w != nil && !errors.Is(w, io.ErrClosedPipe) {
return w
}
return err
}
// putExport PUTs the export to felis-api. There is no retry: the token opens
// the export once, so a second attempt could only be refused. Redirects are
// refused because the request carries the token and the internal face never
// redirects. felis-api answers only after the whole download, which the Job's
// activeDeadlineSeconds bounds, so the header timeout is a backstop for a
// wedged endpoint and not the real limit.
//
// The body always goes chunked, which is what lets it end with a trailer; a
// size the Job knows (-1 when it does not) goes as worldexport.LengthHeader in
// place of Content-Length.
func putExport(ctx context.Context, target, token, contentType string, body io.Reader, size int64, trailer http.Header) error {
req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body)
if err != nil {
return err
}
req.ContentLength = -1
req.Trailer = trailer
if size >= 0 {
req.Header.Set(worldexport.LengthHeader, strconv.FormatInt(size, 10))
}
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", contentType)
client := &http.Client{
Transport: &http.Transport{ResponseHeaderTimeout: 10 * time.Minute},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
return nil
}
var e struct {
Error struct {
Message string `json:"message"`
} `json:"error"`
}
if json.NewDecoder(io.LimitReader(resp.Body, 4<<10)).Decode(&e) == nil && e.Error.Message != "" {
return fmt.Errorf("felis-api answered %s: %s", resp.Status, e.Error.Message)
}
return fmt.Errorf("felis-api answered %s", resp.Status)
}
cmd/felis/export_test.go
-532
View File
@@ -1,532 +0,0 @@
package main
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"slices"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/worldexport"
)
// exportReceiver stands in for felis-api's internal upload route: it records
// the PUT it gets (or the error reading it ended on) and answers with reply.
type exportReceiver struct {
srv *httptest.Server
hits atomic.Int32
req *http.Request
body []byte
readErr error
served chan struct{} // one send per request, once it is answered
}
func receiveExport(t *testing.T, reply func(w http.ResponseWriter)) *exportReceiver {
t.Helper()
rcv := &exportReceiver{served: make(chan struct{}, 1)}
rcv.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rcv.hits.Add(1)
rcv.req = r
rcv.body, rcv.readErr = io.ReadAll(r.Body)
reply(w)
rcv.served <- struct{}{}
}))
t.Cleanup(rcv.srv.Close)
return rcv
}
func noContent(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) }
// sentWhole fails unless the upload rcv got ended with the Content-Digest
// trailer of its own bytes, and declared length as its size (-1: none).
func sentWhole(t *testing.T, rcv *exportReceiver, length int64) {
t.Helper()
sum := sha256.Sum256(rcv.body)
want := "sha-256=:" + base64.StdEncoding.EncodeToString(sum[:]) + ":"
wantLength := ""
if length >= 0 {
wantLength = strconv.FormatInt(length, 10)
}
r := rcv.req
if rcv.readErr != nil || r.Trailer.Get(worldexport.DigestTrailer) != want || r.Header.Get(worldexport.LengthHeader) != wantLength ||
r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
t.Fatalf("upload read %v, trailer %v, %s %q, length %d, encoding %v; want trailer %q and %s %q, chunked",
rcv.readErr, r.Trailer, worldexport.LengthHeader, r.Header.Get(worldexport.LengthHeader), r.ContentLength, r.TransferEncoding,
want, worldexport.LengthHeader, wantLength)
}
}
func tarEntries(t *testing.T, archive []byte) map[string]string {
t.Helper()
gz, err := gzip.NewReader(bytes.NewReader(archive))
if err != nil {
t.Fatalf("not gzip: %v", err)
}
out := map[string]string{}
tr := tar.NewReader(gz)
for {
h, err := tr.Next()
if err == io.EOF {
return out
}
if err != nil {
t.Fatalf("tar: %v", err)
}
b, _ := io.ReadAll(tr)
out[h.Name] = string(b)
}
}
// writeTree writes name → body under root, making the folders on the way.
func writeTree(t *testing.T, root string, files map[string]string) {
t.Helper()
for name, body := range files {
p := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
}
// The two secrets a world holds, and what server.properties reads as once
// redacted.
const (
secretProps = "motd=hi\nrcon.password=hunter2\n"
redactedProps = "motd=hi\nrcon.password=<redacted by felis>\n"
forwardingKey = "secret: aVeryRealForwardingKey\n"
)
// secretWorld is a world holding both secrets, with a hard link to the
// forwarding secret under a name nothing would guard by.
func secretWorld(t *testing.T) string {
t.Helper()
root := t.TempDir()
writeTree(t, root, map[string]string{
"server.properties": secretProps,
"config/paper-global.yml": forwardingKey,
"world/region/r.0.0.mca": "chunks",
})
if err := os.MkdirAll(filepath.Join(root, "plugins"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(root, "plugins/copy.yml")); err != nil {
t.Fatal(err)
}
return root
}
func TestCmdExportWorld(t *testing.T) {
root := secretWorld(t)
if err := os.Symlink("server.properties", filepath.Join(root, "props-link")); err != nil {
t.Fatal(err)
}
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
code := cmdExport([]string{"--mode", "world", "--server", "survival", "--target-url", rcv.srv.URL + "/api/v1/internal/exports/ab",
"--worlds-root", root}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
r := rcv.req
if r.Method != http.MethodPut || r.URL.Path != "/api/v1/internal/exports/ab" || r.Header.Get("Authorization") != "Bearer tok" ||
r.Header.Get("Content-Type") != "application/gzip" || r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
}
want := map[string]string{
"server.properties": redactedProps, "config/": "", "plugins/": "",
"world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks",
}
if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
t.Fatalf("archive holds %v\nwant %v", got, want)
}
sentWhole(t, rcv, -1)
want2 := "felis export: left out 1 entries a tar cannot hold (symbolic links, devices, sockets)\n" +
"felis export: left out 2 files that hold platform secrets\n" +
"felis export: server=survival mode=world downloaded\n"
if stdout.String() != want2 {
t.Errorf("stdout = %q, want %q", stdout.String(), want2)
}
}
// A world root that cannot be opened fails before anything reaches felis-api.
func TestCmdExportWorldUnreadable(t *testing.T) {
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
if code != 1 || rcv.hits.Load() != 0 {
t.Fatalf("exit %d with %d requests, want 1 and none", code, rcv.hits.Load())
}
}
// An export that fails part-way must never reach felis-api as a complete body:
// the chunked upload is cut off, so felis-api aborts the browser's download,
// and the failure itself is what the Job reports.
func TestStreamExportWriteErrorAbortsTheUpload(t *testing.T) {
broken := errors.New("disk read failed")
rcv := receiveExport(t, noContent)
err := streamExport(context.Background(), rcv.srv.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
if _, err := w.Write(bytes.Repeat([]byte("x"), 100_000)); err != nil {
return err
}
return broken
})
if err != broken {
t.Fatalf("err = %v, want the write's own error, unwrapped", err)
}
select {
case <-rcv.served:
if rcv.readErr == nil {
t.Fatalf("felis-api read a complete %d-byte body from a failed export", len(rcv.body))
}
case <-time.After(5 * time.Second):
t.Fatal("the request never reached felis-api")
}
}
// When felis-api refuses first, its reason is reported, not the closed pipe
// that then stops the writer.
func TestStreamExportRefusalStopsTheWriter(t *testing.T) {
refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer refusing.Close()
stopped := make(chan error, 1)
err := streamExport(context.Background(), refusing.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
for {
if _, err := w.Write(make([]byte, 32<<10)); err != nil {
stopped <- err
return err
}
}
})
if err == nil || err.Error() != "felis-api answered 404 Not Found" {
t.Fatalf("err = %v, want felis-api's answer", err)
}
if werr := <-stopped; !errors.Is(werr, io.ErrClosedPipe) {
t.Fatalf("the writer stopped on %v, want the closed pipe", werr)
}
// A PUT that never starts leaves no transport to close the body: the writer
// is still stopped, and the export fails rather than hangs.
done := make(chan error, 1)
go func() {
done <- streamExport(context.Background(), "http://[::1", "tok", "application/gzip", -1, func(w io.Writer) error {
_, err := w.Write([]byte("x"))
return err
})
}()
select {
case err := <-done:
if err == nil || !strings.Contains(err.Error(), "missing ']'") {
t.Fatalf("err = %v, want the bad URL", err)
}
case <-time.After(5 * time.Second):
t.Fatal("an export whose PUT never started hung")
}
}
// storedBackup writes, at path, a gzip+tar like one the backup store holds:
// the world whole, both secrets included, and a region file that does not
// compress. It returns the archive's sha256.
func storedBackup(t *testing.T, path string) string {
t.Helper()
region := make([]byte, 64<<10)
if _, err := rand.Read(region); err != nil {
t.Fatal(err)
}
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for _, e := range []struct{ name, body string }{
{"server.properties", secretProps},
{"config/paper-global.yml", forwardingKey},
{"world/level.dat", "level"},
{"world/region/r.0.0.mca", string(region)},
} {
if err := tw.WriteHeader(&tar.Header{Name: e.name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(e.body))}); err != nil {
t.Fatal(err)
}
if _, err := io.WriteString(tw, e.body); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(buf.Bytes())
return hex.EncodeToString(sum[:])
}
func TestCmdExportBackup(t *testing.T) {
root := t.TempDir()
ref := filepath.Join(root, "survival-1.tar.gz")
sum := storedBackup(t, ref)
stored, err := os.ReadFile(ref)
if err != nil {
t.Fatal(err)
}
region := tarEntries(t, stored)["world/region/r.0.0.mca"]
args := func(url, ref string) []string {
return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
}
for name, extra := range map[string][]string{
"no digest recorded": nil,
"recorded digest matches": {"--sha256", sum},
} {
t.Run(name+": re-streamed through the guards", func(t *testing.T) {
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(append(args(rcv.srv.URL, ref), extra...), &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
r := rcv.req
if r.ContentLength != -1 || r.Header.Get("Content-Type") != "application/gzip" || r.Header.Get("Authorization") != "Bearer tok" {
t.Fatalf("length %d, headers %v", r.ContentLength, r.Header)
}
want := map[string]string{"server.properties": redactedProps, "world/level.dat": "level", "world/region/r.0.0.mca": region}
if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
t.Fatalf("archive holds %d entries, want exactly the redacted properties, level.dat and the region file", len(got))
}
sentWhole(t, rcv, -1)
if want := "felis export: left out 1 files that hold platform secrets\nfelis export: server=survival mode=backup downloaded\n"; stdout.String() != want {
t.Errorf("stdout = %q, want %q", stdout.String(), want)
}
})
}
// The stored bytes are checked as they stream, and the archive the Job sends
// is only closed once they are all read: a mismatch cuts the upload off
// short of its end, so felis-api never passes on a complete-looking copy.
t.Run("a digest mismatch cuts the upload off before its end", func(t *testing.T) {
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(append(args(rcv.srv.URL, ref), "--sha256", strings.Repeat("ab", 32)), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if want := "felis export: the backup archive does not match the sha256 recorded when it was written\n"; stderr.String() != want {
t.Fatalf("stderr = %q, want %q", stderr.String(), want)
}
select {
case <-rcv.served:
case <-time.After(5 * time.Second):
t.Fatal("the upload never reached felis-api")
}
if rcv.readErr == nil {
t.Fatalf("felis-api read a complete %d-byte body", len(rcv.body))
}
zr, err := gzip.NewReader(bytes.NewReader(rcv.body))
if err == nil {
_, err = io.ReadAll(zr)
}
if err == nil {
t.Fatal("what felis-api got is a complete archive")
}
})
t.Run("a ref outside the backup root is refused before any request", func(t *testing.T) {
outside := filepath.Join(t.TempDir(), "secret.tar.gz")
if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil {
t.Fatal(err)
}
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(rcv.srv.URL, outside), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if n := rcv.hits.Load(); n != 0 {
t.Fatalf("felis-api got %d requests", n)
}
})
t.Run("a refusal exits 1 with felis-api's reason", func(t *testing.T) {
rcv := receiveExport(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusGone)
io.WriteString(w, `{"error":{"code":"export_expired","message":"nobody opened the download"}}`)
})
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if got := stderr.String(); got != "felis export: felis-api answered 410 Gone: nobody opened the download\n" {
t.Fatalf("stderr = %q", got)
}
})
// The request carries the token and the internal face never redirects, so a
// redirect is refused rather than followed with the token attached. A 302 or
// 303 is the one net/http would follow on its own (as a GET, and to the same
// host with the Authorization header still on it).
for _, status := range []int{http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
t.Run("a redirect is not followed: "+strconv.Itoa(status), func(t *testing.T) {
elsewhere := receiveExport(t, noContent)
redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.srv.URL, status))
defer redirecting.Close()
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(redirecting.URL, ref), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if n := elsewhere.hits.Load(); n != 0 {
t.Fatalf("the redirect target got %d requests", n)
}
if got, want := stderr.String(), "felis export: felis-api answered "+strconv.Itoa(status)+" "+http.StatusText(status)+"\n"; got != want {
t.Fatalf("stderr = %q, want %q", got, want)
}
})
}
}
func TestCmdExportFiles(t *testing.T) {
root := secretWorld(t)
writeTree(t, root, map[string]string{"plugins/Essentials/config.yml": "x: 1"})
if err := os.Symlink("config.yml", filepath.Join(root, "plugins/Essentials/link.yml")); err != nil {
t.Fatal(err)
}
export := func(t *testing.T, path string, dir bool) (*exportReceiver, int, string, string) {
t.Helper()
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
args := []string{"--mode", "files", "--server", "survival", "--target-url", rcv.srv.URL, "--worlds-root", root, "--path", path}
if dir {
args = append(args, "--dir")
}
var stdout, stderr bytes.Buffer
code := cmdExport(args, &stdout, &stderr)
return rcv, code, stdout.String(), stderr.String()
}
for path, want := range map[string]string{
"world/region/r.0.0.mca": "chunks",
"server.properties": redactedProps,
} {
t.Run("a file goes with its exact length: "+path, func(t *testing.T) {
rcv, code, stdout, stderr := export(t, path, false)
if code != 0 || stdout != "felis export: server=survival mode=files downloaded\n" {
t.Fatalf("exit %d, stdout %q, stderr %q", code, stdout, stderr)
}
if string(rcv.body) != want || rcv.req.Header.Get("Content-Type") != "application/octet-stream" {
t.Fatalf("body %q, type %q; want %q", rcv.body, rcv.req.Header.Get("Content-Type"), want)
}
sentWhole(t, rcv, int64(len(want)))
})
}
t.Run("a folder goes as a zip, guarded", func(t *testing.T) {
rcv, code, stdout, stderr := export(t, "plugins", true)
if code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr)
}
if rcv.req.ContentLength != -1 || rcv.req.Header.Get("Content-Type") != "application/zip" {
t.Fatalf("length %d, type %q", rcv.req.ContentLength, rcv.req.Header.Get("Content-Type"))
}
zr, err := zip.NewReader(bytes.NewReader(rcv.body), int64(len(rcv.body)))
if err != nil {
t.Fatal(err)
}
var names []string
for _, f := range zr.File {
names = append(names, f.Name)
}
if want := []string{"plugins/", "plugins/Essentials/", "plugins/Essentials/config.yml"}; !slices.Equal(names, want) {
t.Fatalf("zip holds %v, want %v", names, want)
}
want := "felis export: left out 1 entries a zip does not carry (symbolic links, devices, sockets)\n" +
"felis export: left out 1 files that hold platform secrets\n" +
"felis export: server=survival mode=files downloaded\n"
if stdout != want {
t.Errorf("stdout = %q, want %q", stdout, want)
}
})
for _, c := range []struct {
path string
dir bool
want string
}{
{"config/paper-global.yml", false, "forwarding secret"},
{"plugins/copy.yml", false, "forwarding secret"},
{"plugins", false, "is a folder now"},
{"server.properties", true, "is not a folder now"},
} {
t.Run("refused before any request: "+c.path, func(t *testing.T) {
rcv, code, _, stderr := export(t, c.path, c.dir)
if code != 1 || rcv.hits.Load() != 0 || !strings.Contains(stderr, c.want) {
t.Fatalf("exit %d, %d requests, stderr %q; want 1, none, and %q", code, rcv.hits.Load(), stderr, c.want)
}
})
}
}
func TestCmdExportUsage(t *testing.T) {
for name, tc := range map[string]struct {
token string
args []string
}{
"no token": {"", []string{"--mode", "world", "--target-url", "http://api/x"}},
"no target": {"tok", []string{"--mode", "world"}},
"unknown mode": {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
"files without path": {"tok", []string{"--mode", "files", "--target-url", "http://api/x"}},
} {
t.Run(name, func(t *testing.T) {
t.Setenv(worldexport.TokenEnv, tc.token)
var stdout, stderr bytes.Buffer
if code := cmdExport(tc.args, &stdout, &stderr); code != 2 {
t.Fatalf("exit %d, want 2; stderr %q", code, stderr.String())
}
})
}
}
// TestCmdExportWiring: the Job's `felis export` reaches cmdExport, and
// felis-api's executor mounts the backup store at the path the archives were
// written under, since a backup's ref is an absolute path there.
func TestCmdExportWiring(t *testing.T) {
t.Setenv(worldexport.TokenEnv, "")
var stdout, stderr bytes.Buffer
if code := run([]string{"export"}, &stdout, &stderr); code != 2 ||
stderr.String() != "felis export: --target-url and "+worldexport.TokenEnv+" are required\n" {
t.Fatalf("felis export = %d, stderr %q", code, stderr.String())
}
cfg := &config.Config{}
cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups"
want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"}
if got := exportConfig(cfg, "felis:1", "felis-backups"); !reflect.DeepEqual(got, want) {
t.Fatalf("exportConfig = %+v, want %+v", got, want)
}
}
cmd/felis/files.go
+19 -124
View File
@@ -1,15 +1,11 @@
package main
import (
"context"
"encoding/base64"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"felis.lolicon.best/internal/fileedit"
)
@@ -23,55 +19,32 @@ import (
// Like cmdRestore it deliberately holds NO database credentials and never calls
// config.Load: felis-api made the authorization decision (the caller owns this
// server, and the server is stopped so the RWO world volume is free); this process
// is the unprivileged hands that touch bytes. Its entire input is the flags
// below plus, for a write, the content variables and, for an upload, one token.
// Every isolation guarantee lives in the Pod spec (internal/fileedit/jobspec.go),
// and the path-containment guarantee lives in fileedit.Execute, which resolves
// every path through os.Root and therefore cannot be walked out of the world
// mount.
// is the unprivileged hands that touch bytes. Its entire input is the three flags
// below plus, for a write, one environment variable. Every isolation guarantee
// lives in the Pod spec (internal/fileedit/jobspec.go), and the path-containment
// guarantee lives in fileedit.Execute, which resolves the path through os.Root and
// therefore cannot be walked out of the world mount.
//
// Exit status carries a specific meaning that felis-api depends on: a CALLER-fault
// outcome — a path that escapes the root, a file that is missing or too large — is
// a SUCCESSFUL run that prints a Result carrying an error code, so the API can map
// it to a precise 4xx. A non-zero exit means the operation could not be attempted
// at all (the world mount is unreadable, an upload's bytes could not be fetched
// intact, the result unprintable), which the API reports as a 500.
// at all (the world mount is unreadable, the result unprintable), which the API
// reports as a 500.
func cmdFiles(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("files", flag.ContinueOnError)
fs.SetOutput(stderr)
op := fs.String("op", "", "operation: list, read, write, mkdir, delete, rename, upload or unzip")
browseURL := fs.String("browse-url", "", "internal command channel for a read-only file browser")
op := fs.String("op", "", "operation: list, read, or write")
path := fs.String("path", "", "path to operate on, relative to the world root (empty = the root itself)")
worldsRoot := fs.String("worlds-root", "/data", "mount path of the world PVC; every path resolves under it")
expect := fs.String("expect-sha256", "", "write only: refuse unless the file's current SHA-256 (hex) is this")
createOnly := fs.Bool("create-only", false, "write only: refuse a path that already exists")
to := fs.String("to", "", "rename only: the destination path")
sourceURL := fs.String("source-url", "", "upload only: felis-api URL to fetch the bytes from")
size := fs.Int64("size", -1, "upload only: the byte count the fetched file must have")
sum := fs.String("sha256", "", "write and upload: the SHA-256 (hex) the content or the fetched file must have")
overwrite := fs.Bool("overwrite", false, "upload and unzip: replace files already there")
if err := fs.Parse(args); err != nil {
return 2
}
if *browseURL != "" {
limitHeapToCgroup()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := fileedit.Browse(ctx, *worldsRoot, *browseURL, os.Getenv(fileedit.BrowserTokenEnv)); err != nil {
fmt.Fprintf(stderr, "felis files: %v\n", err)
return 1
}
return 0
}
if *op == "" {
fmt.Fprintln(stderr, "felis files: --op is required")
fmt.Fprintln(stderr, "felis files: --op is required (list, read, or write)")
return 2
}
limitHeapToCgroup()
req := fileedit.Request{
Op: *op, Path: *path, To: *to, Expect: *expect, CreateOnly: *createOnly, Overwrite: *overwrite,
}
// New content arrives base64-encoded in the environment rather than in argv:
// a process's arguments are world-readable on the node (/proc/<pid>/cmdline),
@@ -79,46 +52,22 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
// secrets — an RCON password in server.properties is the obvious case. The
// encoding is what lets arbitrary bytes (CRLF endings, a BOM, a NUL) survive a
// channel that must be a valid string.
switch *op {
case fileedit.OpWrite:
// The content's SHA-256 comes with it, so bytes that changed on the way
// to this Job are refused rather than written (Request.ContentSHA256).
if *sum == "" {
fmt.Fprintln(stderr, "felis files: a write needs --sha256")
var content []byte
if *op == fileedit.OpWrite {
raw, ok := os.LookupEnv(fileedit.ContentEnv)
if !ok {
fmt.Fprintf(stderr, "felis files: a write needs %s in the environment\n", fileedit.ContentEnv)
return 2
}
content, err := fileedit.ContentFromEnv(os.LookupEnv)
decoded, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
fmt.Fprintf(stderr, "felis files: %v\n", err)
fmt.Fprintf(stderr, "felis files: %s is not valid base64: %v\n", fileedit.ContentEnv, err)
return 2
}
req.Content, req.ContentSHA256 = content, *sum
case fileedit.OpUpload:
token := os.Getenv(fileedit.UploadTokenEnv)
if *sourceURL == "" || token == "" {
fmt.Fprintf(stderr, "felis files: an upload needs --source-url and %s\n", fileedit.UploadTokenEnv)
return 2
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
req.Upload = &fileedit.Upload{
Size: *size, SHA256: *sum,
Open: func() (io.ReadCloser, error) { return fetchUpload(ctx, *sourceURL, token) },
Landed: func() {
if err := reportLanded(ctx, *sourceURL, token); err != nil {
// The file is in place; felis-api drops its copy when it
// has sat idle long enough, and the panel cancels it too.
fmt.Fprintf(stderr, "felis files: tell felis-api the upload landed: %v\n", err)
}
},
}
content = decoded
}
// An upload or an unzip (the only ops that report progress) can run long
// enough that felis-api does not wait on its Job, and the panel shows how far
// it has got from the latest of these lines (fileedit.K8sRunner.Ops).
req.Progress = fileedit.ThrottledProgress(stdout, time.Second, time.Now)
res, err := fileedit.Execute(*worldsRoot, req)
res, err := fileedit.Execute(*worldsRoot, *op, *path, content)
if err != nil {
// The operation could not be attempted — infrastructure, not caller fault.
fmt.Fprintf(stderr, "felis files: %v\n", err)
@@ -133,57 +82,3 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
}
return 0
}
// fetchUpload opens the staged upload on felis-api's internal face. There is no
// retry: the token opens the upload once (fileedit.Stage), so a second attempt
// could only be refused, and the caller retries the failed Job whole (a file
// sent in parts stays staged until its Job reports it landed, so that retry
// does not send it again). Redirects are refused because the request carries the
// token and the internal face never redirects; the header timeout catches a
// wedged endpoint, and the Job's activeDeadlineSeconds bounds the body.
func fetchUpload(ctx context.Context, url, token string) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+token)
client := &http.Client{
Transport: &http.Transport{ResponseHeaderTimeout: 30 * time.Second},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("GET returned %s", resp.Status)
}
return resp.Body, nil
}
// reportLanded tells felis-api the upload's file is in place (DELETE on the URL
// it was fetched from, with the same token), so it deletes the copy it staged.
// One try: the file has landed whatever the answer, and a copy nobody deletes
// is dropped once it has sat idle for fileedit.SessionIdle.
func reportLanded(ctx context.Context, url, token string) error {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+token)
client := &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
resp, err := client.Do(req)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
return fmt.Errorf("DELETE returned %s", resp.Status)
}
return nil
}
cmd/felis/files_test.go
-358
View File
@@ -1,358 +0,0 @@
package main
import (
"archive/zip"
"bytes"
"cmp"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"felis.lolicon.best/internal/fileedit"
)
// filesResult is the Result a `felis files` run printed on its marked line,
// the last it prints.
func filesResult(t *testing.T, stdout string) fileedit.Result {
t.Helper()
lines := strings.Split(strings.TrimSpace(stdout), "\n")
line, ok := strings.CutPrefix(lines[len(lines)-1], fileedit.ResultPrefix)
if !ok {
t.Fatalf("stdout has no result line: %q", stdout)
}
var res fileedit.Result
if err := json.Unmarshal([]byte(line), &res); err != nil {
t.Fatalf("result line %q: %v", line, err)
}
return res
}
// stagedSource is felis-api's internal face for one staged upload. It serves
// body to a GET carrying Bearer token and 404 to any other, and answers the
// DELETE that reports the file landed with landedCode (204 when unset),
// redirecting to landedTo when that is a redirect. reports counts those
// DELETEs, each with the token and at the path the bytes came from.
type stagedSource struct {
*httptest.Server
reports, strays atomic.Int32
landedCode int
landedTo string
}
func stagedUpload(t *testing.T, token string, body []byte) *stagedSource {
t.Helper()
s := &stagedSource{}
s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer "+token || r.URL.Path != "/u" {
s.strays.Add(1)
http.Error(w, "no such upload", http.StatusNotFound)
return
}
switch r.Method {
case http.MethodGet:
w.Write(body)
case http.MethodDelete:
s.reports.Add(1)
if s.landedTo != "" {
w.Header().Set("Location", s.landedTo)
}
w.WriteHeader(cmp.Or(s.landedCode, http.StatusNoContent))
default:
s.strays.Add(1)
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}))
t.Cleanup(s.Close)
return s
}
func uploadArgs(root, sourceURL string, body []byte) []string {
sum := sha256.Sum256(body)
return []string{
"--op", "upload", "--path", "plugins/a.jar", "--worlds-root", root,
"--source-url", sourceURL, "--size", "4", "--sha256", hex.EncodeToString(sum[:]),
}
}
// uploadRoot is a world with the plugins folder an upload lands in.
func uploadRoot(t *testing.T) string {
t.Helper()
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "plugins"), 0o755); err != nil {
t.Fatal(err)
}
return root
}
func TestCmdFilesUpload(t *testing.T) {
body := []byte("PK\x03\x04")
t.Run("fetches the staged bytes with its token and lands them", func(t *testing.T) {
root := uploadRoot(t)
srv := stagedUpload(t, "tok", body)
t.Setenv(fileedit.UploadTokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix+`{"done":4,"total":4}`+"\n") {
t.Fatalf("stdout %q does not start with the progress to the last byte", stdout.String())
}
if res := filesResult(t, stdout.String()); res.Code != "" {
t.Fatalf("result = %+v", res)
}
got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar"))
if err != nil || !bytes.Equal(got, body) {
t.Fatalf("landed %q, %v", got, err)
}
if n, strays := srv.reports.Load(), srv.strays.Load(); n != 1 || strays != 0 || stderr.Len() != 0 {
t.Fatalf("reported landed %d times, %d stray requests, stderr %q; want once", n, strays, stderr.String())
}
})
t.Run("a file already there is a result, and nothing is reported landed", func(t *testing.T) {
root := uploadRoot(t)
if err := os.WriteFile(filepath.Join(root, "plugins", "a.jar"), []byte("old!"), 0o644); err != nil {
t.Fatal(err)
}
srv := stagedUpload(t, "tok", body)
t.Setenv(fileedit.UploadTokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeExists || srv.reports.Load() != 0 {
t.Fatalf("result = %+v, reported landed %d times", res, srv.reports.Load())
}
})
// The file is in place whatever felis-api answers, so the Job still succeeds
// and says why the staged copy may linger. A redirect is not followed, since
// the request carries the token.
for name, tc := range map[string]struct {
code int
stderr string
}{
"refused": {http.StatusNotFound, "felis files: tell felis-api the upload landed: DELETE returned 404 Not Found\n"},
"redirected": {http.StatusFound, "felis files: tell felis-api the upload landed: DELETE returned 302 Found\n"},
} {
t.Run("a landed report "+name+" still lands the file", func(t *testing.T) {
var elsewhere atomic.Int32
away := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { elsewhere.Add(1) }))
defer away.Close()
root := uploadRoot(t)
srv := stagedUpload(t, "tok", body)
srv.landedCode, srv.landedTo = tc.code, away.URL+"/u"
t.Setenv(fileedit.UploadTokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != "" || stderr.String() != tc.stderr || elsewhere.Load() != 0 {
t.Fatalf("result = %+v, stderr %q, redirect followed %d times", res, stderr.String(), elsewhere.Load())
}
if got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar")); err != nil || !bytes.Equal(got, body) {
t.Fatalf("landed %q, %v", got, err)
}
})
}
// A refused fetch is the Job failing, never a Result: the API answers it with a
// 500 the caller retries whole.
t.Run("a refused fetch exits 1 and lands nothing", func(t *testing.T) {
root := uploadRoot(t)
srv := stagedUpload(t, "tok", body)
t.Setenv(fileedit.UploadTokenEnv, "wrong")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1; stdout %q", code, stdout.String())
}
if !strings.Contains(stderr.String(), "404") {
t.Fatalf("stderr %q does not name the status", stderr.String())
}
if srv.reports.Load() != 0 {
t.Fatal("a refused fetch was reported landed")
}
if _, err := os.Lstat(filepath.Join(root, "plugins", "a.jar")); !os.IsNotExist(err) {
t.Fatalf("a refused fetch left a file: %v", err)
}
})
// The request carries the token, and the internal face never redirects, so a
// redirect is refused rather than followed with the token attached.
t.Run("a redirect is not followed", func(t *testing.T) {
root := uploadRoot(t)
var hits atomic.Int32
elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Write(body)
}))
defer elsewhere.Close()
redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.URL+"/u", http.StatusFound))
defer redirecting.Close()
t.Setenv(fileedit.UploadTokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, redirecting.URL+"/u", body), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if n := hits.Load(); n != 0 {
t.Fatalf("the redirect target was fetched %d times", n)
}
})
for name, tc := range map[string]struct {
token string
drop string
}{
"no token": {"", ""},
"no source URL": {"tok", "--source-url"},
} {
t.Run(name+" exits 2", func(t *testing.T) {
srv := stagedUpload(t, "tok", body)
t.Setenv(fileedit.UploadTokenEnv, tc.token)
args := uploadArgs(uploadRoot(t), srv.URL+"/u", body)
if tc.drop != "" {
for i, a := range args {
if a == tc.drop {
args = append(args[:i:i], args[i+2:]...)
break
}
}
}
var stdout, stderr bytes.Buffer
if code := cmdFiles(args, &stdout, &stderr); code != 2 {
t.Fatalf("exit %d, want 2", code)
}
})
}
}
func TestCmdFilesWrite(t *testing.T) {
root := t.TempDir()
content := []byte("[]\r\n")
sum := sha256.Sum256(content)
args := []string{"--op", "write", "--path", "ops.json", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}
t.Run("reassembles the content parts", func(t *testing.T) {
t.Setenv(fileedit.ContentPartsEnv, "1")
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
var stdout, stderr bytes.Buffer
if code := cmdFiles(args, &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != "" {
t.Fatalf("result = %+v", res)
}
if got, err := os.ReadFile(filepath.Join(root, "ops.json")); err != nil || !bytes.Equal(got, content) {
t.Fatalf("wrote %q, %v", got, err)
}
})
// Writing what did arrive of an incomplete spec would truncate the file.
t.Run("an incomplete content spec exits 2 and writes nothing", func(t *testing.T) {
t.Setenv(fileedit.ContentPartsEnv, "2")
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("x")))
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 2 {
t.Fatalf("exit %d, want 2", code)
}
if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
t.Fatalf("an incomplete spec wrote a file: %v", err)
}
})
// Without the content's SHA-256 the Job could not tell bytes changed on the
// way from the bytes felis-api sent.
t.Run("a write without its SHA-256 exits 2 and writes nothing", func(t *testing.T) {
t.Setenv(fileedit.ContentPartsEnv, "1")
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root}, &stdout, &stderr); code != 2 {
t.Fatalf("exit %d, want 2", code)
}
if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
t.Fatalf("a write without its SHA-256 wrote a file: %v", err)
}
})
t.Run("content that changed on the way is a result and writes nothing", func(t *testing.T) {
t.Setenv(fileedit.ContentPartsEnv, "1")
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("[]\n")))
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeDigestMismatch {
t.Fatalf("result = %+v, want %s", res, fileedit.CodeDigestMismatch)
}
if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
t.Fatalf("changed content wrote a file: %v", err)
}
})
}
// A caller-fault outcome is a successful run carrying a code, so felis-api can
// answer the precise 4xx instead of a 500.
func TestCmdFilesCallerFaultIsAResult(t *testing.T) {
root := t.TempDir()
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "mkdir", "--path", "../out", "--worlds-root", root}, &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeBadPath {
t.Fatalf("result = %+v, want code %s", res, fileedit.CodeBadPath)
}
stdout.Reset()
if code := cmdFiles([]string{"--worlds-root", root}, &stdout, &stderr); code != 2 {
t.Fatalf("no --op: exit %d, want 2", code)
}
}
// TestCmdFilesUnzip checks an unzip extracts next to the archive and reports its
// progress before its result, the same way an upload does.
func TestCmdFilesUnzip(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "maps"), 0o755); err != nil {
t.Fatal(err)
}
var zb bytes.Buffer
zw := zip.NewWriter(&zb)
for name, body := range map[string]string{"world/level.dat": "level", "world/region/r.0.0.mca": "region!"} {
w, err := zw.Create(name)
if err != nil {
t.Fatal(err)
}
io.WriteString(w, body)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "maps", "a.zip"), zb.Bytes(), 0o644); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "unzip", "--path", "maps/a.zip", "--worlds-root", root}, &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != "" || res.Files != 2 || res.Bytes != 12 {
t.Fatalf("result = %+v", res)
}
if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix) ||
!strings.Contains(stdout.String(), fileedit.ProgressPrefix+`{"done":12,"total":12}`+"\n") {
t.Fatalf("stdout %q does not report the progress to the last byte", stdout.String())
}
got, err := os.ReadFile(filepath.Join(root, "maps", "world", "region", "r.0.0.mca"))
if err != nil || string(got) != "region!" {
t.Fatalf("extracted %q, %v", got, err)
}
}
cmd/felis/hostcreds.go
-84
View File
@@ -1,84 +0,0 @@
package main
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// Host copies of the credentials `felis setup` takes at the keyboard: the [smtp]
// relay password and the uploads bucket's keys. The cluster reads them from the
// felis-smtp and felis-uploads-s3 Secrets, and a Secret lives in k3s's datastore,
// which a reinstall (uninstall.sh keeps /etc/felis) or a host rebuilt from a
// database bundle's state/ starts empty. Each file holds the bare value, mode
// 0600, directly in /etc/felis beside secrets.env: every installer run applies
// the Secrets from these files, and every database bundle, so the off-site copy
// too, carries them.
const (
hostSMTPPasswordPath = "/etc/felis/smtp-password"
hostUploadsS3AccessKeyPath = "/etc/felis/uploads-s3-access-key"
hostUploadsS3SecretKeyPath = "/etc/felis/uploads-s3-secret-key"
)
// writeHostCredential replaces the file at path with value, mode 0600, through a
// temporary file in the same directory, so a crash leaves the old value or the
// new one and never a partial one.
func writeHostCredential(path, value string) error {
tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
if err != nil {
return err
}
tmpPath := tmp.Name()
defer os.Remove(tmpPath)
// CreateTemp already makes the file 0600; the Chmod states it rather than
// leaning on that.
if err := tmp.Chmod(0o600); err != nil {
_ = tmp.Close()
return err
}
if _, err := tmp.WriteString(value); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmpPath, path)
}
// readHostCredential returns the value in path; ok is false when there is no
// such file. An empty file is a value: the relay password of a relay without AUTH.
func readHostCredential(path string) (value string, ok bool, err error) {
b, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return "", false, nil
}
if err != nil {
return "", false, err
}
return string(b), true, nil
}
// relayPassword is the [smtp] relay password as the host holds it: the copy
// `felis setup` keeps at path, else, on an install from before that copy, the
// felis-smtp Secret in ns, whose absence means a relay without AUTH. cl is only
// used when the file is missing; a nil cl then reports errClusterUnreachable.
func relayPassword(ctx context.Context, path string, cl client.Client, ns string) (string, error) {
if pw, ok, err := readHostCredential(path); err != nil || ok {
return pw, err
}
if cl == nil {
return "", errClusterUnreachable
}
return smtpSecretPassword(ctx, cl, ns)
}
var errClusterUnreachable = errors.New("the cluster did not answer")
cmd/felis/hostcreds_test.go
-198
View File
@@ -1,198 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/watchdog"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
func TestHostCredentialFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "smtp-password")
if _, ok, err := readHostCredential(path); ok || err != nil {
t.Fatalf("a missing file read as ok=%v err=%v; want not there", ok, err)
}
// A copy an operator put there by hand, readable by everyone, is tightened.
if err := os.WriteFile(path, []byte("by hand"), 0o644); err != nil {
t.Fatal(err)
}
for _, v := range []string{"first secret", "a \"quoted\" $second\nsecret", ""} {
if err := writeHostCredential(path, v); err != nil {
t.Fatal(err)
}
got, ok, err := readHostCredential(path)
if err != nil || !ok || got != v {
t.Fatalf("read back (%q, %v, %v); want (%q, true, nil)", got, ok, err, v)
}
fi, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if mode := fi.Mode().Perm(); mode != 0o600 {
t.Fatalf("mode %v; want 0600", mode)
}
}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
if len(entries) != 1 {
t.Fatalf("the directory holds %d entries; want the credential alone, no leftover temporary file", len(entries))
}
if _, _, err := readHostCredential(dir); err == nil {
t.Fatal("an unreadable credential read as fine")
}
}
func smtpSecretClient(t *testing.T, password string) client.Client {
t.Helper()
return fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
Data: map[string][]byte{platform.SMTPSecretPasswordKey: []byte(password)},
}).Build()
}
func TestRelayPassword(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
host := filepath.Join(dir, "smtp-password")
cl := smtpSecretClient(t, "from-secret")
if pw, err := relayPassword(ctx, host, cl, "felis"); err != nil || pw != "from-secret" {
t.Fatalf("without a host copy = (%q, %v); want the Secret's", pw, err)
}
if _, err := relayPassword(ctx, host, nil, "felis"); !errors.Is(err, errClusterUnreachable) {
t.Fatalf("without a host copy or a cluster err = %v; want errClusterUnreachable", err)
}
if err := writeHostCredential(host, "from-host"); err != nil {
t.Fatal(err)
}
for _, c := range []client.Client{cl, nil} {
if pw, err := relayPassword(ctx, host, c, "felis"); err != nil || pw != "from-host" {
t.Fatalf("with a host copy (cluster %v) = (%q, %v); want the host copy", c != nil, pw, err)
}
}
if _, err := relayPassword(ctx, dir, cl, "felis"); err == nil {
t.Fatal("an unreadable host copy fell through to the Secret")
}
}
// The watchdog mails the most while the cluster is down: the host copy must
// reach it then, and without one the password the last good run cached stays.
func TestRefreshSMTPPassword(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
host := filepath.Join(dir, "smtp-password")
var stderr bytes.Buffer
state := &watchdog.State{SMTPPassword: "cached"}
refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
if state.SMTPPassword != "cached" || stderr.Len() != 0 {
t.Fatalf("cluster down, no host copy: password %q, stderr %q; want the cached one kept quietly", state.SMTPPassword, stderr.String())
}
refreshSMTPPassword(ctx, host, smtpSecretClient(t, "from-secret"), "felis", state, &stderr)
if state.SMTPPassword != "from-secret" {
t.Fatalf("cluster up, no host copy: password %q; want the Secret's", state.SMTPPassword)
}
if err := writeHostCredential(host, "from-host"); err != nil {
t.Fatal(err)
}
refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
if state.SMTPPassword != "from-host" {
t.Fatalf("cluster down, host copy: password %q; want the host copy", state.SMTPPassword)
}
refreshSMTPPassword(ctx, dir, nil, "felis", state, &stderr)
if state.SMTPPassword != "from-host" || !strings.Contains(stderr.String(), "keeping the cached one") {
t.Fatalf("unreadable host copy: password %q, stderr %q; want the cached one kept and the failure said", state.SMTPPassword, stderr.String())
}
}
func TestHostRecoveryMailerHostCopy(t *testing.T) {
ctx := context.Background()
// No kubeconfig anywhere: reaching for the cluster fails, so a pass proves
// the host copy was enough.
t.Setenv("KUBECONFIG", filepath.Join(t.TempDir(), "no-kubeconfig"))
dir := t.TempDir()
host := filepath.Join(dir, "smtp-password")
if err := writeHostCredential(host, "from-host"); err != nil {
t.Fatal(err)
}
off := false
c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_UNSET_RELAY_PW", RequireTLS: &off}
got, err := hostRecoveryMailer(c, host, "felis")(ctx)
if err != nil {
t.Fatalf("with the host copy: %v", err)
}
if relay, ok := got.(*mail.SMTP); !ok || relay.Password != "from-host" {
t.Fatalf("relay = %#v; want the host copy's password", got)
}
if _, err := hostRecoveryMailer(c, dir, "felis")(ctx); err == nil || !strings.Contains(err.Error(), "read the relay password") {
t.Fatalf("unreadable host copy: err = %v; want it named", err)
}
if _, err := hostRecoveryMailer(c, filepath.Join(dir, "none"), "felis")(ctx); err == nil || !strings.Contains(err.Error(), "reach the cluster") {
t.Fatalf("no host copy and no cluster: err = %v; want the cluster named", err)
}
}
// A whole watchdog run with the API server and PostgreSQL both down still
// takes the relay password from the host copy, so the outage mail can
// authenticate even when no earlier run cached it.
func TestWatchdogReadsHostCopyWhileClusterDown(t *testing.T) {
dir := t.TempDir()
t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
cfgPath := filepath.Join(dir, "felis.toml")
if err := os.WriteFile(cfgPath, []byte(`[database]
url = "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
[server]
root_domain = "example.com"
[archive]
store = "tarLocal"
[k8s]
egress_mode = "nodeport"
[smtp]
host = "127.0.0.1"
port = 1
from = "[email protected]"
username = "felis"
password_ref = "FELIS_TEST_UNSET_RELAY_PW"
`), 0o600); err != nil {
t.Fatal(err)
}
pwPath := filepath.Join(dir, "smtp-password")
if err := writeHostCredential(pwPath, "from-host"); err != nil {
t.Fatal(err)
}
statePath := filepath.Join(dir, "state.json")
var stdout, stderr bytes.Buffer
cmdWatchdog([]string{
"-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"),
"-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", pwPath, "-heartbeat-file", filepath.Join(dir, "no-heartbeat"),
}, &stdout, &stderr)
if !strings.Contains(stdout.String(), "kube-api") {
t.Fatalf("the run found the API server up; the test needs it down (stdout %s)", stdout.String())
}
state, err := watchdog.LoadState(statePath)
if err != nil {
t.Fatalf("load state: %v (stderr %s)", err, stderr.String())
}
if state.SMTPPassword != "from-host" {
t.Fatalf("cached relay password %q; want the host copy (stdout %s, stderr %s)", state.SMTPPassword, stdout.String(), stderr.String())
}
}
cmd/felis/imagebundle.go
-109
View File
@@ -1,109 +0,0 @@
package main
import (
"bufio"
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"strings"
"syscall"
"felis.lolicon.best/internal/imagepush"
)
// cmdImageBundle writes a release's image bundle: one OCI layout tar holding every
// image an install runs, for one platform, plus its listing (one "role name
// manifest-digest config-digest" line per image). deploy/build-release-artifacts.sh
// runs it in CI; deploy/bootstrap.sh imports the tar into k3s's containerd and
// pushes it into the platform registry with push-image --image.
//
// --layout role=name=path an image buildx wrote with --output type=oci
// --pull role=ref a digest-pinned public image, named repository@digest
//
// The tar and the listing are written beside their final paths and renamed into
// place, so a failed run leaves neither behind.
func cmdImageBundle(args []string, _, stderr io.Writer) int {
fs := flag.NewFlagSet("image-bundle", flag.ContinueOnError)
fs.SetOutput(stderr)
platform := fs.String("platform", "", "os/arch the bundle is for, e.g. linux/arm64")
out := fs.String("out", "", "path of the bundle tar to write")
list := fs.String("list", "", "path of the listing to write")
var images []imagepush.BundleImage
fs.Func("layout", "role=name=path of an OCI layout tar (repeatable)", func(v string) error {
role, rest, ok := strings.Cut(v, "=")
name, path, ok2 := strings.Cut(rest, "=")
if !ok || !ok2 || role == "" || name == "" || path == "" {
return fmt.Errorf("want role=name=path, got %q", v)
}
images = append(images, imagepush.BundleImage{Role: role, Name: name, Layout: path})
return nil
})
fs.Func("pull", "role=ref of a digest-pinned public image (repeatable)", func(v string) error {
role, ref, ok := strings.Cut(v, "=")
if !ok || role == "" || !strings.Contains(ref, "@sha256:") {
return fmt.Errorf("want role=ref with ref pinned by digest, got %q", v)
}
images = append(images, imagepush.BundleImage{Role: role, Name: imagepush.PinnedName(ref), Source: ref})
return nil
})
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if *platform == "" || *out == "" || *list == "" || len(images) == 0 {
fmt.Fprintln(stderr, "felis image-bundle: --platform, --out, --list and at least one --layout or --pull are required")
return 2
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := writeImageBundle(ctx, &imagepush.Source{Platform: *platform}, images, *out, *list); err != nil {
fmt.Fprintf(stderr, "felis image-bundle: %v\n", err)
return 1
}
return 0
}
func writeImageBundle(ctx context.Context, s *imagepush.Source, images []imagepush.BundleImage, out, list string) (err error) {
tmpOut, tmpList := out+".tmp", list+".tmp"
defer func() {
if err != nil {
os.Remove(tmpOut)
os.Remove(tmpList)
}
}()
f, err := os.Create(tmpOut)
if err != nil {
return err
}
w := bufio.NewWriterSize(f, 1<<20)
entries, err := imagepush.WriteBundle(ctx, s, images, w)
if err == nil {
err = w.Flush()
}
if err == nil {
err = f.Sync()
}
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
return err
}
var b strings.Builder
for _, e := range entries {
fmt.Fprintf(&b, "%s %s %s %s\n", e.Role, e.Name, e.Digest, e.Config)
}
if err := os.WriteFile(tmpList, []byte(b.String()), 0o644); err != nil {
return err
}
if err := os.Rename(tmpOut, out); err != nil {
return err
}
return os.Rename(tmpList, list)
}
cmd/felis/imagebundle_test.go
-131
View File
@@ -1,131 +0,0 @@
package main
import (
"archive/tar"
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
)
// writeTestLayout writes an OCI layout tar holding one linux/arch image with one
// layer, the way buildx --output type=oci leaves a single-platform build, and
// returns its path with the manifest and config digests.
func writeTestLayout(t *testing.T, dir, arch, layer string) (path, manifestDigest, configDigest string) {
t.Helper()
blobs := map[string][]byte{}
add := func(b []byte) (string, int) {
sum := sha256.Sum256(b)
d := "sha256:" + hex.EncodeToString(sum[:])
blobs[d] = b
return d, len(b)
}
cfgDigest, cfgSize := add([]byte(`{"architecture":"` + arch + `","os":"linux","rootfs":{"type":"layers"}}`))
layerDigest, layerSize := add([]byte(layer))
manifest := fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",`+
`"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":%q,"size":%d},`+
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":%q,"size":%d}]}`,
cfgDigest, cfgSize, layerDigest, layerSize)
mDigest, mSize := add([]byte(manifest))
index, _ := json.Marshal(map[string]any{
"schemaVersion": 2,
"manifests": []map[string]any{{
"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": mDigest, "size": mSize,
}},
})
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
put := func(name string, b []byte) {
tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(b)), Typeflag: tar.TypeReg})
tw.Write(b)
}
put("oci-layout", []byte(`{"imageLayoutVersion":"1.0.0"}`))
put("index.json", index)
for d, b := range blobs {
put("blobs/sha256/"+strings.TrimPrefix(d, "sha256:"), b)
}
tw.Close()
path = filepath.Join(dir, arch+"-"+layer+".tar")
if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil {
t.Fatal(err)
}
return path, mDigest, cfgDigest
}
func TestImageBundleWritesTheListingTheInstallerReads(t *testing.T) {
dir := t.TempDir()
limbo, limboDigest, limboConfig := writeTestLayout(t, dir, "arm64", "limbo")
lobby, lobbyDigest, lobbyConfig := writeTestLayout(t, dir, "arm64", "lobby")
out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
var stderr bytes.Buffer
code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo,
"--layout", "lobby=registry.felis.svc:5000/felis/lobby:demo=" + lobby,
}, nil, &stderr)
if code != 0 {
t.Fatalf("image-bundle = %d: %s", code, stderr.String())
}
// deploy/bootstrap.sh reads this with `read -r role name digest config`.
got, _ := os.ReadFile(list)
want := "limbo registry.felis.svc:5000/felis/limbo:demo " + limboDigest + " " + limboConfig + "\n" +
"lobby registry.felis.svc:5000/felis/lobby:demo " + lobbyDigest + " " + lobbyConfig + "\n"
if string(got) != want {
t.Errorf("listing:\n%s\nwant:\n%s", got, want)
}
if fi, err := os.Stat(out); err != nil || fi.Size() == 0 {
t.Errorf("bundle: %v", err)
}
if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
t.Errorf("left behind %v", leftovers)
}
}
func TestImageBundleLeavesNothingWhenAnImageIsRefused(t *testing.T) {
dir := t.TempDir()
amd, _, _ := writeTestLayout(t, dir, "amd64", "limbo")
out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
// The pair an earlier run wrote stays as it was: a listing beside a bundle it
// does not describe would have the installer look for images that are not there.
os.WriteFile(out, []byte("old bundle"), 0o644)
os.WriteFile(list, []byte("old listing\n"), 0o644)
var stderr bytes.Buffer
code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + amd}, nil, &stderr)
if code != 1 || !strings.Contains(stderr.String(), "is a linux/amd64 image") {
t.Fatalf("image-bundle = %d: %s", code, stderr.String())
}
if got, _ := os.ReadFile(out); string(got) != "old bundle" {
t.Errorf("bundle was replaced with %d bytes", len(got))
}
if got, _ := os.ReadFile(list); string(got) != "old listing\n" {
t.Errorf("listing was replaced with %q", got)
}
if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
t.Errorf("left behind %v", leftovers)
}
}
func TestPushImageReadsABundleByImageName(t *testing.T) {
dir := t.TempDir()
limbo, _, _ := writeTestLayout(t, dir, "arm64", "limbo")
out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
if code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo}, nil, &bytes.Buffer{}); code != 0 {
t.Fatal("image-bundle failed")
}
t.Setenv("FELIS_REGISTRY_USERNAME", "platform")
t.Setenv("FELIS_REGISTRY_PASSWORD", "x")
// The name is looked up in the bundle's index before the registry is contacted,
// so a name the bundle lacks fails here with what it does hold.
var stderr bytes.Buffer
code := cmdPushImage([]string{"--tar", out, "--image", "registry.felis.svc:5000/felis/lobby:demo",
"--ref", "127.0.0.1:1/felis/lobby:demo"}, &bytes.Buffer{}, &stderr)
if code != 1 || !strings.Contains(stderr.String(), "holds no image named registry.felis.svc:5000/felis/lobby:demo (it holds: registry.felis.svc:5000/felis/limbo:demo") {
t.Fatalf("push-image = %d: %s", code, stderr.String())
}
}
cmd/felis/main.go
-24
View File
@@ -6,32 +6,8 @@ package main
import (
"os"
"path/filepath"
)
func main() {
ensureHostBinDirOnPath()
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
}
// hostBinDir is where deploy/bootstrap.sh installs felis, k3s and cloudflared.
const hostBinDir = "/usr/local/bin"
// ensureHostBinDirOnPath appends hostBinDir to PATH when it is missing, so the
// k3s and cloudflared this binary execs are found beside it. sudo's secure_path
// on EL leaves /usr/local/bin out: `sudo /usr/local/bin/felis db backup` would
// otherwise run with no k3s to reach the database's pod through. Appended, so a
// PATH that names another k3s first keeps it.
func ensureHostBinDirOnPath() {
path := os.Getenv("PATH")
for _, dir := range filepath.SplitList(path) {
if dir == hostBinDir {
return
}
}
if path == "" {
os.Setenv("PATH", hostBinDir)
return
}
os.Setenv("PATH", path+string(os.PathListSeparator)+hostBinDir)
}
cmd/felis/main_test.go
-33
View File
@@ -1,33 +0,0 @@
package main
import (
"os"
"regexp"
"testing"
)
func TestEnsureHostBinDirOnPath(t *testing.T) {
for _, c := range []struct{ in, want string }{
{"/usr/sbin:/usr/bin", "/usr/sbin:/usr/bin:/usr/local/bin"},
{"/usr/local/bin:/usr/bin", "/usr/local/bin:/usr/bin"},
{"/opt/k3s:/usr/bin:/usr/local/bin", "/opt/k3s:/usr/bin:/usr/local/bin"},
{"", "/usr/local/bin"},
} {
t.Setenv("PATH", c.in)
ensureHostBinDirOnPath()
if got := os.Getenv("PATH"); got != c.want {
t.Errorf("PATH %q became %q, want %q", c.in, got, c.want)
}
}
}
// run is what the tests drive, so the PATH fix must sit in main, before it.
func TestMainFixesPathBeforeRunning(t *testing.T) {
b, err := os.ReadFile("main.go")
if err != nil {
t.Fatal(err)
}
if !regexp.MustCompile(`func main\(\) \{\n\tensureHostBinDirOnPath\(\)\n\tos\.Exit\(run\(`).Match(b) {
t.Fatal("main does not call ensureHostBinDirOnPath before run")
}
}
cmd/felis/manifests.go
+6 -59
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"net"
"path/filepath"
"strings"
"felis.lolicon.best/internal/platform"
@@ -36,21 +35,9 @@ func (m *multiFlag) Set(v string) error {
// --velocity-cidr records the proxy host addresses allowed by the game NetworkPolicy.
// Kubernetes permits resident-node traffic regardless, but remote proxy deployments
// need an explicit CIDR, so the renderer refuses to guess.
//
// --only postgres renders just the control-plane database (platform.PostgresObjects),
// which the installer brings up before migrations, before it has anything else
// to render the full bundle with; it needs neither --felis-image nor
// --velocity-cidr.
func cmdManifests(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("manifests", flag.ContinueOnError)
fs.SetOutput(stderr)
distributed := fs.Bool("distributed", false, "enable approved workers and archive transport")
controller := fs.String("controller-node", "", "protected controller identity for A")
probe := fs.String("egress-probe", "", "reachable controller host:port denied to game Pods")
var registryNodes multiFlag
fs.Var(&registryNodes, "registry-node-cidr", "exact node pull source for the registry (repeatable)")
socket := fs.String("node-control-socket", "", "host node-control Unix socket (optional, API only)")
nodeControlNode := fs.String("node-control-node", "", "controller hostname hosting the socket")
controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace the control plane (api/operator/reaper) runs in")
minecraftNS := fs.String("minecraft-namespace", platform.DefaultMinecraftNamespace, "namespace MinecraftServer workloads run in")
buildNS := fs.String("build-namespace", platform.DefaultBuildNamespace, "namespace image-build Jobs run in")
@@ -59,11 +46,9 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
panelNodePort := fs.Int("panel-node-port", int(platform.DefaultPanelNodePort), "NodePort that exposes the built-in HTTPS panel/API origin")
felisImage := fs.String("felis-image", "", "container image the felis-api/operator Deployments run, also passed through as FELIS_IMAGE (REQUIRED)")
registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry 2.8.3, pinned by digest)")
postgresImage := fs.String("postgres-image", "", "control-plane database image (default: PostgreSQL 18.6, pinned by digest)")
only := fs.String("only", "", `render one part of the bundle instead of all of it; "postgres" is the control-plane database`)
backupPVC := fs.String("backup-pvc", "felis-backups", "name of the world-archive PVC this bundle renders in the Minecraft namespace and advertises to the backup/restore executors via FELIS_BACKUP_PVC (default: felis-backups; pass an empty value to render none, leaving backup/restore answering 503)")
worldsHostPath := fs.String("worlds-host-path", "", "node directory the reaper reads worlds from: each world PVC resolves as <path>/<pvc>, or as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (k3s storage root: /var/lib/rancher/k3s/storage); enables the reaper CronJob (requires --archive-local-path and a non-empty --backup-pvc)")
archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path. With the backup PVC alone it renders the retention-only CronJob, which deletes backups past their expiry and never touches a world")
archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
registryStorage := fs.String("registry-storage", "", "capacity the registry PVC requests (default 10Gi; k3s local-path does not enforce it)")
uploadsStorage := fs.String("uploads-storage", "", "capacity the uploads PVC requests (default 5Gi; k3s local-path does not enforce it)")
backupStorage := fs.String("backup-storage", "", "capacity the world-archive PVC requests (default 10Gi; k3s local-path does not enforce it)")
@@ -79,19 +64,6 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
if err := fs.Parse(args); err != nil {
return 2
}
switch *only {
case "":
case "postgres":
return renderManifests(stdout, stderr, platform.PostgresObjects(platform.Params{
ControllerNode: *controller,
ControlNamespace: *controlNS,
MinecraftNamespace: *minecraftNS,
PostgresImage: *postgresImage,
}))
default:
fmt.Fprintf(stderr, "felis manifests: --only %q: the one part that renders alone is \"postgres\"\n", *only)
return 2
}
// Keep proxy placement explicit. This matters for remote proxies and documents
// the expected source even when Velocity runs on the resident node.
@@ -124,7 +96,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
// The node pin exists only for the reaper's hostPath: naming a node without the
// worlds root would be silently dropped (no CronJob renders), so fail loud like
// the storage-trio check below.
if *reaperNode != "" && *worldsHostPath == "" && !*distributed {
if *reaperNode != "" && *worldsHostPath == "" {
fmt.Fprintln(stderr, "felis manifests: --reaper-node requires --worlds-host-path "+
"(it pins the reaper CronJob, which renders only with the retention storage trio)")
return 2
@@ -165,32 +137,12 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
"writes under /var/lib/rancher/k3s/storage). Any other provisioner needs its volumes exposed as "+
"<path>/<pvc>, or each candidate's archive fails and the world is preserved;\n"+
" - %s.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, pin)
} else if !*distributed {
switch {
case *archiveLocalPath != "" && *backupPVC == "":
fmt.Fprintln(stderr, "felis manifests: --archive-local-path names where the backup PVC is mounted, "+
"but --backup-pvc is empty (no archive store renders); drop one or the other")
return 2
case *archiveLocalPath != "":
fmt.Fprintln(stderr, "felis manifests: note: rendering the reaper CronJob retention-only: backups past "+
"their expiry are deleted daily, idle worlds are never archived or deleted "+
"(pass --worlds-host-path to reap them too)")
case *backupPVC != "":
fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered: backups are never expired, so "+
"the archive store only grows until the disk fills (pass --archive-local-path, equal to felis.toml "+
"[archive] local_path, for the retention-only CronJob, and --worlds-host-path as well to reap idle worlds)")
default:
fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered (backups are disabled)")
}
} else {
fmt.Fprintln(stderr, "felis manifests: note: retention reaper CronJob not rendered "+
"(pass --worlds-host-path and --archive-local-path — the archive PVC defaults to felis-backups — to enable it)")
}
if *socket != "" && (!filepath.IsAbs(*socket) || *nodeControlNode == "") {
fmt.Fprintln(stderr, "node-control requires an absolute socket and its controller hostname")
return 2
}
params := platform.Params{
NodeControlSocket: *socket, NodeControlNode: *nodeControlNode,
Distributed: *distributed, ControllerNode: *controller, EgressProbe: *probe, RegistryNodeCIDRs: registryNodes,
ControlNamespace: *controlNS,
MinecraftNamespace: *minecraftNS,
BuildNamespace: *buildNS,
@@ -199,7 +151,6 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
PanelNodePort: int32(*panelNodePort),
FelisImage: *felisImage,
RegistryImage: *registryImage,
PostgresImage: *postgresImage,
BackupPVC: *backupPVC,
WorldsHostPath: *worldsHostPath,
ReaperNode: *reaperNode,
@@ -218,11 +169,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "felis manifests: %v\n", err)
return 2
}
return renderManifests(stdout, stderr, platform.Objects(params))
}
func renderManifests(stdout, stderr io.Writer, objs []platform.Object) int {
out, err := platform.RenderObjects(objs)
out, err := platform.RenderYAML(params)
if err != nil {
fmt.Fprintf(stderr, "felis manifests: render: %v\n", err)
return 1
cmd/felis/manifests_test.go
+4 -73
View File
@@ -90,13 +90,12 @@ func TestManifestsRendersBundle(t *testing.T) {
t.Error("rendered bundle must not contain ClusterRole/ClusterRoleBinding")
}
// Without the retention flags, the reaper CronJob is not rendered and the
// generator says so on stderr, naming what that leaves: backups that never
// expire.
// generator says so on stderr.
if strings.Contains(text, "kind: CronJob") {
t.Error("no reaper CronJob must render without --archive-local-path")
t.Error("no reaper CronJob must render without --worlds-host-path")
}
if !strings.Contains(errBuf.String(), "not rendered") || !strings.Contains(errBuf.String(), "never expired") {
t.Errorf("expected a 'reaper not rendered, backups never expired' notice on stderr, got %q", errBuf.String())
if !strings.Contains(errBuf.String(), "not rendered") {
t.Errorf("expected a 'reaper not rendered' notice on stderr, got %q", errBuf.String())
}
}
@@ -145,40 +144,6 @@ func TestManifestsBackupPVCOptOut(t *testing.T) {
}
}
// TestManifestsRendersRetentionOnly: the archive store without a worlds root
// still gets the daily CronJob, retention-only, so backups past their expiry
// leave the store on an install that never reaps a world; the operator is told
// which of the two it got. An archive path with the store switched off is a
// mistake and fails loud.
func TestManifestsRendersRetentionOnly(t *testing.T) {
var out, errBuf bytes.Buffer
code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
"--archive-local-path", "/var/lib/felis/archives"}, &out, &errBuf)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
}
text := out.String()
for _, want := range []string{"kind: CronJob", "name: felis-reaper", "--retention-only", "claimName: felis-backups"} {
if !strings.Contains(text, want) {
t.Errorf("retention-only bundle missing %q", want)
}
}
if strings.Contains(text, "kind: PersistentVolume\n") || strings.Contains(text, "--worlds-root") {
t.Error("a retention-only bundle must not reach for a worlds root")
}
if !strings.Contains(errBuf.String(), "retention-only") {
t.Errorf("stderr must say the CronJob is retention-only, got %q", errBuf.String())
}
out.Reset()
errBuf.Reset()
code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
"--archive-local-path", "/var/lib/felis/archives", "--backup-pvc="}, &out, &errBuf)
if code != 2 || out.Len() != 0 || !strings.Contains(errBuf.String(), "--backup-pvc is empty") {
t.Errorf("archive path without an archive store: exit=%d out=%d bytes stderr=%q, want a fail-loud 2", code, out.Len(), errBuf.String())
}
}
// TestManifestsRendersReaper proves the happy path with the full retention trio:
// a batch/v1 CronJob is emitted, named felis-reaper, mounting the backup PVC at the
// supplied archive path.
@@ -278,37 +243,3 @@ func TestManifestsStorageSizes(t *testing.T) {
t.Errorf("--registry-storage lots: exit %d, stderr %q; want a refusal naming the flag", code, errBuf.String())
}
}
// TestManifestsOnlyPostgres: the installer renders the database before it has
// an image or a proxy address for the rest of the bundle, so --only postgres
// must render without them, and render the database and nothing else (a stray
// Deployment in that apply would start without its identities).
func TestManifestsOnlyPostgres(t *testing.T) {
var out, errBuf bytes.Buffer
code := run([]string{"manifests", "--only", "postgres", "--control-namespace", "ctl", "--postgres-image", "example/pg:18@sha256:abc"}, &out, &errBuf)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
}
var kinds []string
for _, doc := range strings.Split(out.String(), "\n---\n") {
for _, line := range strings.Split(doc, "\n") {
if strings.HasPrefix(line, "kind: ") {
kinds = append(kinds, strings.TrimPrefix(line, "kind: "))
}
}
}
if got, want := strings.Join(kinds, ","), "Namespace,ConfigMap,NetworkPolicy,Deployment,Service"; got != want {
t.Errorf("rendered kinds %s, want %s", got, want)
}
for _, want := range []string{"name: felis-postgres", "namespace: ctl", "image: example/pg:18@sha256:abc"} {
if !strings.Contains(out.String(), want) {
t.Errorf("rendered database missing %q", want)
}
}
out.Reset()
errBuf.Reset()
if code := run([]string{"manifests", "--only", "registry"}, &out, &errBuf); code != 2 || out.Len() != 0 {
t.Errorf("--only registry: exit %d with %d bytes of YAML, want 2 and none", code, out.Len())
}
}
cmd/felis/memlimit.go
-48
View File
@@ -1,48 +0,0 @@
package main
import (
"os"
"runtime/debug"
"strconv"
"strings"
)
// cgroupMemoryFiles are where a container reads the memory it is allowed:
// cgroup v2 first, then v1.
var cgroupMemoryFiles = []string{"/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"}
// limitHeapToCgroup sets the Go heap's soft limit from the container's memory
// limit, so the collector works harder as a Job nears it and the kernel does not
// kill the Job first. An extraction or a folder zipped for download keeps a few
// hundred bytes per entry for as long as it runs; without the limit the heap
// grows to twice that before a collection, and a 256 MiB Job was killed at
// 400,000 entries whose live heap was 115 MB. GOMEMLIMIT set by hand wins.
func limitHeapToCgroup() {
if os.Getenv("GOMEMLIMIT") != "" {
return
}
for _, f := range cgroupMemoryFiles {
b, err := os.ReadFile(f)
if err != nil {
continue
}
if n, ok := softMemoryLimit(string(b)); ok {
debug.SetMemoryLimit(n)
}
return
}
}
// softMemoryLimit answers three fifths of the limit a cgroup memory file holds,
// or false for "max" (no limit) and anything unreadable. The rest is left for
// what the kernel charges the container beyond the Go heap: the page cache of
// the files it reads and writes, and the inodes it creates. Under a 256 MiB
// limit, 400,000 extracted entries peaked at 184 MB resident with the heap held
// to 150 MiB.
func softMemoryLimit(content string) (int64, bool) {
n, err := strconv.ParseInt(strings.TrimSpace(content), 10, 64)
if err != nil || n <= 0 {
return 0, false
}
return n / 5 * 3, true
}
cmd/felis/memlimit_test.go
-58
View File
@@ -1,58 +0,0 @@
package main
import (
"math"
"os"
"path/filepath"
"runtime/debug"
"testing"
)
func TestSoftMemoryLimit(t *testing.T) {
for _, c := range []struct {
in string
want int64
ok bool
}{
{"268435456\n", 161061273, true}, // 256 MiB, as memory.max holds it
{"max\n", 0, false},
{"0\n", 0, false},
{"-1", 0, false},
{"", 0, false},
} {
got, ok := softMemoryLimit(c.in)
if got != c.want || ok != c.ok {
t.Errorf("softMemoryLimit(%q) = %d %v, want %d %v", c.in, got, ok, c.want, c.ok)
}
}
}
// TestLimitHeapToCgroup checks the limit comes from the first cgroup file there
// is, and that GOMEMLIMIT set by hand leaves the heap alone.
func TestLimitHeapToCgroup(t *testing.T) {
prevFiles, prevLimit := cgroupMemoryFiles, debug.SetMemoryLimit(-1)
t.Cleanup(func() { cgroupMemoryFiles = prevFiles; debug.SetMemoryLimit(prevLimit) })
dir := t.TempDir()
v1, v1b := filepath.Join(dir, "v1"), filepath.Join(dir, "v1b")
if err := os.WriteFile(v1, []byte("268435456\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(v1b, []byte("536870912\n"), 0o600); err != nil {
t.Fatal(err)
}
cgroupMemoryFiles = []string{filepath.Join(dir, "missing"), v1, v1b}
t.Setenv("GOMEMLIMIT", "")
debug.SetMemoryLimit(math.MaxInt64)
limitHeapToCgroup()
if got := debug.SetMemoryLimit(-1); got != 161061273 {
t.Fatalf("limit = %d, want three fifths of 256 MiB", got)
}
t.Setenv("GOMEMLIMIT", "1GiB")
debug.SetMemoryLimit(math.MaxInt64)
limitHeapToCgroup()
if got := debug.SetMemoryLimit(-1); got != math.MaxInt64 {
t.Fatalf("limit = %d with GOMEMLIMIT set, want it left alone", got)
}
}
cmd/felis/migrate.go
+7 -75
View File
@@ -2,11 +2,9 @@ package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
@@ -20,8 +18,7 @@ import (
// database that already holds a schema and has migrations pending is bundled
// first (internal/dbbackup, label pre-migrate). A failed snapshot stops the
// upgrade; -no-backup is the explicit way past it, e.g. for an external
// database (no [database] deployment, so the host's own pg_dump runs) whose
// server is newer than that pg_dump.
// database whose server is newer than the host's pg_dump.
func cmdMigrate(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("migrate", flag.ContinueOnError)
fs.SetOutput(stderr)
@@ -61,7 +58,7 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
}
if !*noBackup {
path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database, *backupDir, stderr)
path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database.URL, *backupDir, stderr)
if err != nil {
fmt.Fprintf(stderr, "felis migrate: pre-migration backup failed, nothing applied: %v\n", err)
fmt.Fprintln(stderr, " fix the backup, or re-run with -no-backup to migrate without one")
@@ -85,14 +82,10 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
return 0
}
// preMigrateStateDir is the host state a pre-migrate bundle carries; tests
// point it at a directory of their own.
var preMigrateStateDir = dbbackup.DefaultStateDir
// preMigrateBackup bundles the database when it already carries a schema and
// some of migrations are not applied yet, and returns the bundle's path ("" when
// there was nothing to protect: a fresh database, or nothing pending).
func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, db config.DatabaseConfig, dir string, log io.Writer) (string, error) {
func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, dbURL, dir string, log io.Writer) (string, error) {
if err := drv.EnsureVersionTable(ctx); err != nil {
return "", fmt.Errorf("ensure version table: %w", err)
}
@@ -103,22 +96,11 @@ func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.
if len(done) == 0 || !hasPending(done, migrations) {
return "", nil
}
tools, err := dbTools(db)
if err != nil {
return "", err
}
path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelPreMigrate,
Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: preMigrateStateDir,
Version: resolvedVersion(), ExportServers: exportMinecraftServers, Log: log, Record: true,
return dbbackup.Backup(ctx, dbbackup.BackupOptions{
DatabaseURL: dbURL, Dir: dir, Label: dbbackup.LabelPreMigrate,
Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir,
Version: resolvedVersion(), Log: log, Record: true,
})
if errors.Is(err, dbbackup.ErrServersMissing) {
// Rolling the migration back needs the database alone. Backup logged
// the gap, and the panel and the watchdog show it while this is the
// newest bundle.
return path, nil
}
return path, err
}
func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
@@ -129,53 +111,3 @@ func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
}
return false
}
// openStore opens the business database for a command that reads and writes its
// tables, and refuses one whose schema this build was not written against: a newer
// Felis migrated it (a rolled-back binary), or, unless allowPending, migrations this
// build embeds have not run yet (a binary swapped in ahead of `felis migrate up`).
func openStore(ctx context.Context, url string, allowPending bool) (*store.PostgresDriver, error) {
drv, err := store.Open(ctx, url)
if err != nil {
return nil, err
}
return checkSchema(ctx, drv, allowPending)
}
// podDBWindow and podDBInterval bound how long a pod that has just started retries its
// first database dial while the network policy has yet to admit it
// (store.OpenRetrying). A minute is far past the sync lag and far inside every Job's
// deadline. Vars so a test can shrink them.
var (
podDBWindow = time.Minute
podDBInterval = time.Second
)
// openPodStore is openStore for felis-api and the reaper and backup Jobs, whose first
// dial comes milliseconds after their pod starts.
func openPodStore(ctx context.Context, url, prog string, stderr io.Writer) (*store.PostgresDriver, error) {
drv, err := store.OpenRetrying(ctx, url, podDBWindow, podDBInterval, func(err error) {
fmt.Fprintf(stderr, "felis %s: %v; retrying (a pod that has just started waits for the network policy to admit it)\n", prog, err)
})
if err != nil {
return nil, err
}
return checkSchema(ctx, drv, false)
}
// checkSchema closes drv and fails when its schema is not the one this build was
// written against (see openStore).
func checkSchema(ctx context.Context, drv *store.PostgresDriver, allowPending bool) (*store.PostgresDriver, error) {
s, err := store.ReadSchema(ctx, drv)
if err == nil {
err = s.Err()
if allowPending {
err = s.Newer()
}
}
if err != nil {
drv.Close()
return nil, err
}
return drv, nil
}
cmd/felis/migrate_test.go
-48
View File
@@ -1,48 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"net"
"strings"
"syscall"
"testing"
"time"
)
// openPodStore retries a refused first dial for podDBWindow, saying so on stderr
// under the calling command's name each time.
func TestOpenPodStoreRetriesARefusedDial(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
addr := ln.Addr().String()
ln.Close()
window, interval := podDBWindow, podDBInterval
podDBWindow, podDBInterval = 200*time.Millisecond, 20*time.Millisecond
t.Cleanup(func() { podDBWindow, podDBInterval = window, interval })
var stderr bytes.Buffer
start := time.Now()
_, err = openPodStore(context.Background(), fmt.Sprintf("postgres://felis@%s/felis?sslmode=disable", addr), "reaper", &stderr)
if !errors.Is(err, syscall.ECONNREFUSED) {
t.Fatalf("err = %v, want a refused dial", err)
}
if elapsed := time.Since(start); elapsed < podDBWindow {
t.Fatalf("gave up after %s, inside the %s window", elapsed, podDBWindow)
}
lines := strings.Split(strings.TrimSuffix(stderr.String(), "\n"), "\n")
if len(lines) < 2 || len(lines) > 11 {
t.Fatalf("%d retry lines in a 200ms window at 20ms:\n%s", len(lines), stderr.String())
}
for _, l := range lines {
if !strings.HasPrefix(l, "felis reaper: failed to connect to `user=felis database=felis`: ") ||
!strings.HasSuffix(l, "; retrying (a pod that has just started waits for the network policy to admit it)") {
t.Fatalf("retry line %q", l)
}
}
}
cmd/felis/mirrortools.go
+11 -3
View File
@@ -14,6 +14,7 @@ import (
"felis.lolicon.best/internal/build"
"felis.lolicon.best/internal/imagepush"
"felis.lolicon.best/internal/registrygate"
)
// defaultBuildToolsStatus is where mirror-build-tools records its last run; the
@@ -48,9 +49,16 @@ func cmdMirrorBuildTools(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
return 2
}
user, pass, err := registryWriteCredential(*secrets)
if err != nil {
fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
if err := loadEnvFile(*secrets); err != nil {
fmt.Fprintf(stderr, "felis mirror-build-tools: read %s: %v\n", *secrets, err)
return 1
}
user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
if pass == "" {
user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
}
if pass == "" {
fmt.Fprintln(stderr, "felis mirror-build-tools: no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
return 2
}
cmd/felis/node_approve.go
-466
View File
@@ -1,466 +0,0 @@
package main
import (
"bufio"
"context"
"fmt"
"io"
"net"
"os/exec"
"strconv"
"strings"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/archivetransfer"
"felis.lolicon.best/internal/operator"
"felis.lolicon.best/internal/placement"
"felis.lolicon.best/internal/platform"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func approveNode(ctx context.Context, cl client.Client, cs kubernetes.Interface, ns, controlNS, name, image, remote string, stdout io.Writer) error {
var n corev1.Node
if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil {
return err
}
_, controlPlane := n.Labels["node-role.kubernetes.io/control-plane"]
if !placement.Online(&n) || controlPlane || n.Labels[placement.LabelRole] == placement.RoleController {
return fmt.Errorf("candidate must be an online agent")
}
var nodes corev1.NodeList
if err := cl.List(ctx, &nodes); err != nil {
return err
}
var controller *corev1.Node
var peers []string
var denied []string
for i := range nodes.Items {
node := &nodes.Items[i]
if node.Labels[placement.LabelRole] == placement.RoleController {
if controller != nil {
return fmt.Errorf("multiple controllers found")
}
controller = node
}
for _, addr := range node.Status.Addresses {
if addr.Type == corev1.NodeInternalIP || addr.Type == corev1.NodeExternalIP {
cidr, err := exactCIDR(addr.Address)
if err != nil {
return err
}
peers = append(peers, cidr)
for _, p := range []string{"6443", "10250", "5000", "30443"} {
denied = append(denied, net.JoinHostPort(addr.Address, p))
}
}
}
}
if controller == nil || controller.Name == n.Name || controller.Status.NodeInfo.Architecture != n.Status.NodeInfo.Architecture {
return fmt.Errorf("candidate architecture must match the sole controller")
}
if n.Status.NodeInfo.KubeletVersion != controller.Status.NodeInfo.KubeletVersion {
return fmt.Errorf("candidate k3s version must match A")
}
var apiSvc, registrySvc, archiveSvc, kubernetesSvc corev1.Service
for _, svc := range []struct {
obj *corev1.Service
ns, name string
}{{&kubernetesSvc, "default", "kubernetes"}, {&apiSvc, controlNS, platform.SAAPI}, {&registrySvc, controlNS, "registry"}, {&archiveSvc, ns, platform.ArchiveName}} {
if err := cl.Get(ctx, types.NamespacedName{Namespace: svc.ns, Name: svc.name}, svc.obj); err != nil {
return err
}
}
if len(apiSvc.Spec.Ports) == 0 || apiSvc.Spec.Ports[0].NodePort == 0 {
return fmt.Errorf("controller API NodePort is absent")
}
// A's exact interface addresses let the probe observe DNAT/SNAT before the production policy is adjusted.
var aCIDRs []string
if addrs, err := net.InterfaceAddrs(); err == nil {
for _, a := range addrs {
ip, _, err := net.ParseCIDR(a.String())
if err == nil && !ip.IsLoopback() {
cidr, _ := exactCIDR(ip.String())
aCIDRs = append(aCIDRs, cidr)
}
}
}
if len(aCIDRs) == 0 {
return fmt.Errorf("cannot determine A's exact interface addresses")
}
onController := false
for _, addr := range controller.Status.Addresses {
cidr, err := exactCIDR(addr.Address)
if err != nil {
continue
}
for _, local := range aCIDRs {
if cidr == local {
onController = true
}
}
}
if !onController || !placement.Online(controller) {
return fmt.Errorf("run admission on the online controller A")
}
// Quarantine first. Approval is the final write, after all checks have succeeded.
before := n.DeepCopy()
if n.Labels == nil {
n.Labels = map[string]string{}
}
delete(n.Labels, placement.LabelApproved)
found := false
for _, t := range n.Spec.Taints {
if t.Key == "felis.lolicon.best/unapproved" {
found = true
}
}
if !found {
n.Spec.Taints = append(n.Spec.Taints, corev1.Taint{Key: "felis.lolicon.best/unapproved", Value: "true", Effect: corev1.TaintEffectNoSchedule})
}
if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil {
return err
}
if err := denyNodeAddresses(ctx, cl, ns, nodes.Items); err != nil {
return err
}
// The host's Service dial may be masqueraded to its bridge gateway. Permit exact host addresses only.
pullSources := append([]string{}, peers...)
if ip, network, err := net.ParseCIDR(n.Spec.PodCIDR); err == nil && ip.To4() != nil {
v := append(net.IP(nil), network.IP.To4()...)
pullSources = append(pullSources, v.String()+"/32")
v[3]++
pullSources = append(pullSources, v.String()+"/32")
}
var registryNP networkingv1.NetworkPolicy
if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNS, Name: "felis-registry-ingress"}, &registryNP); err != nil {
return err
}
if len(registryNP.Spec.Ingress) == 0 {
return fmt.Errorf("registry ingress policy is not configured")
}
prev := registryNP.DeepCopy()
for _, cidr := range pullSources {
registryNP.Spec.Ingress[0].From = append(registryNP.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
}
if err := cl.Patch(ctx, &registryNP, client.MergeFrom(prev)); err != nil {
return err
}
ctrlIP := ""
for _, a := range controller.Status.Addresses {
if a.Type == corev1.NodeInternalIP {
ctrlIP = a.Address
break
}
}
if ctrlIP == "" {
return fmt.Errorf("controller has no address")
}
script := "set -euo pipefail\numask 077\n" +
"systemctl is-active --quiet k3s-agent\n! systemctl is-active --quiet k3s\ntest ! -f /etc/rancher/k3s/k3s.yaml\n" +
"ip -d link show flannel-wg | grep -q wireguard\n" +
"/usr/local/bin/felis node firewall --peers " + shellQuote(strings.Join(peers, ",")) + " --controller-ip " + shellQuote(ctrlIP) + " --api-service-ip " + shellQuote(kubernetesSvc.Spec.ClusterIP) + " --node-port " + strconv.Itoa(int(apiSvc.Spec.Ports[0].NodePort)) + " --namespace " + shellQuote(ns) + " --control-namespace " + shellQuote(controlNS) + "\n" +
"kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig\n" +
"/usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" get node " + shellQuote(name) + " -o name >/dev/null\n" +
"proof=$(mktemp); trap 'rm -f \"$proof\"' EXIT\n" +
"if /usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" label node " + shellQuote(name) + " felis.node-restriction.kubernetes.io/probe=controller --overwrite 2>\"$proof\"; then echo 'NodeRestriction failed' >&2;exit 1;fi\ngrep -qi forbidden \"$proof\"\n" +
"image=" + shellQuote(image) + "\nif [ -n \"$(/usr/local/bin/k3s crictl images -q \"$image\")\" ]; then /usr/local/bin/k3s crictl rmi \"$image\" >/dev/null; fi\ntest -z \"$(/usr/local/bin/k3s crictl images -q \"$image\")\"\n/usr/local/bin/k3s crictl pull \"$image\" >/dev/null\n"
cmd := exec.CommandContext(ctx, "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=10", "--", remote, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi")
cmd.Stdin = strings.NewReader(script)
cmd.Stdout = stdout
cmd.Stderr = stdout
if err := cmd.Run(); err != nil {
return fmt.Errorf("worker host, NodeRestriction or uncached registry pull check failed: %w", err)
}
id := "felis-probe-" + archivetransfer.ID()[:12]
var objects []client.Object
defer func() {
cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
for i := len(objects) - 1; i >= 0; i-- {
cl.Delete(cleanup, objects[i])
}
}()
create := func(o client.Object) error {
if err := cl.Create(ctx, o); err != nil {
return err
}
objects = append(objects, o)
return nil
}
var endpoints []string
var echoPods []*corev1.Pod
for i := range nodes.Items {
node := &nodes.Items[i]
if !placement.Online(node) || (node.Name != name && node.Name != controller.Name && node.Labels[placement.LabelApproved] != "true") {
continue
}
echoName := fmt.Sprintf("%s-%d", id, i)
p := probePod(echoName, ns, node.Name, image, []string{"--listen", ":25565"}, true)
p.Labels["felis.lolicon.best/probe-echo"] = id
if err := create(p); err != nil {
return err
}
echoPods = append(echoPods, p)
svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: echoName, Namespace: ns}, Spec: corev1.ServiceSpec{Selector: map[string]string{"felis.lolicon.best/probe-name": echoName}, Ports: []corev1.ServicePort{{Port: 25565, TargetPort: intstr.FromInt32(25565)}}}}
if err := create(svc); err != nil {
return err
}
endpoints = append(endpoints, net.JoinHostPort(svc.Spec.ClusterIP, "25565"))
}
tcp := corev1.ProtocolTCP
port := intstr.FromInt32(25565)
policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id, Namespace: ns}, Spec: networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-echo": id}}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}}}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &port}}}}}}
for _, cidr := range aCIDRs {
policy.Spec.Ingress[0].From = append(policy.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
}
if err := create(policy); err != nil {
return err
}
for _, p := range echoPods {
if err := waitProbePod(ctx, cl, p); err != nil {
return err
}
}
open := append([]string{}, endpoints...)
open = append(open, net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), net.JoinHostPort(registrySvc.Spec.ClusterIP, "5000"), net.JoinHostPort(archiveSvc.Spec.ClusterIP, "8090"))
// Positive probes need temporary, narrowly scoped ingress grants where the
// production fence admits only the controller or archive-transfer jobs.
for _, svc := range []*corev1.Service{&apiSvc, &registrySvc, &archiveSvc} {
if len(svc.Spec.Selector) == 0 || len(svc.Spec.Ports) == 0 {
return fmt.Errorf("probe Service %s has no backend", svc.Name)
}
targetPort := svc.Spec.Ports[0].TargetPort
if targetPort.IntVal == 0 && targetPort.StrVal == "" {
targetPort = intstr.FromInt32(svc.Spec.Ports[0].Port)
}
allow := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id + "-" + svc.Name, Namespace: svc.Namespace}, Spec: networkingv1.NetworkPolicySpec{
PodSelector: metav1.LabelSelector{MatchLabels: svc.Spec.Selector}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ns}},
PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}},
}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &targetPort}}}},
}}
if err := create(allow); err != nil {
return err
}
}
positive := probeJob(id+"-positive", ns, name, image, probeArgs("--open", open), false)
positive.Spec.Template.Labels["felis.lolicon.best/probe-source"] = id
if err := create(positive); err != nil {
return err
}
if err := waitProbeJob(ctx, cl, positive); err != nil {
return err
}
denied = append(denied, open...)
denied = append(denied, "169.254.169.254:80", "169.254.170.2:80")
negative := probeJob(id+"-negative", ns, name, image, probeArgs("--closed", denied), true)
negative.Spec.Template.Spec.InitContainers = []corev1.Container{{Name: "egress-gate", Image: image, Command: []string{"/usr/local/bin/felis", "egress-gate", "--probe", net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), "--wait", "2m"}, SecurityContext: negative.Spec.Template.Spec.Containers[0].SecurityContext}}
if err := create(negative); err != nil {
return err
}
if err := waitProbeJob(ctx, cl, negative); err != nil {
return err
}
// Dial from Velocity's host namespace and record the address actually observed inside each backend.
for _, endpoint := range endpoints {
c, err := net.DialTimeout("tcp", endpoint, 5*time.Second)
if err != nil {
return fmt.Errorf("velocity host cannot dial backend Service %s: %w", endpoint, err)
}
c.Close()
}
var observed []string
time.Sleep(500 * time.Millisecond)
for _, p := range echoPods {
foundA := false
stream, err := cs.CoreV1().Pods(ns).GetLogs(p.Name, &corev1.PodLogOptions{}).Stream(ctx)
if err != nil {
return err
}
scan := bufio.NewScanner(stream)
for scan.Scan() {
line := scan.Text()
if strings.HasPrefix(line, "felis-probe-source ") {
host, _, err := net.SplitHostPort(strings.TrimPrefix(line, "felis-probe-source "))
if err == nil {
cidr, _ := exactCIDR(host)
for _, a := range aCIDRs {
if cidr == a {
observed = append(observed, cidr)
foundA = true
}
}
}
}
}
if !foundA {
stream.Close()
return fmt.Errorf("backend %s did not observe A as an exact source", p.Name)
}
err = scan.Err()
stream.Close()
if err != nil {
return err
}
}
if len(observed) < len(echoPods) {
return fmt.Errorf("backend observed a source outside A's exact interfaces")
}
var game networkingv1.NetworkPolicy
if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-allow-game-from-velocity"}, &game); err != nil {
return err
}
if len(game.Spec.Ingress) == 0 {
return fmt.Errorf("velocity ingress policy is not configured")
}
prevGame := game.DeepCopy()
for _, cidr := range observed {
exists := false
for _, peer := range game.Spec.Ingress[0].From {
if peer.IPBlock != nil && peer.IPBlock.CIDR == cidr {
exists = true
break
}
}
if exists {
continue
}
game.Spec.Ingress[0].From = append(game.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
}
if err := cl.Patch(ctx, &game, client.MergeFrom(prevGame)); err != nil {
return err
}
// Read fresh resourceVersion so concurrent node health writes cannot be overwritten.
if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil {
return err
}
if !placement.Online(&n) {
return fmt.Errorf("worker became offline during validation")
}
before = n.DeepCopy()
n.Labels[placement.LabelIdentity] = name
n.Labels[placement.LabelRole] = placement.RoleWorker
n.Labels[placement.LabelApproved] = "true"
taints := n.Spec.Taints[:0]
for _, t := range n.Spec.Taints {
if t.Key != "felis.lolicon.best/unapproved" {
taints = append(taints, t)
}
}
n.Spec.Taints = taints
if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil {
return err
}
fmt.Fprintln(stdout, "approved worker", name, "Velocity sources", strings.Join(observed, ","))
return nil
}
func probeArgs(flag string, addresses []string) []string {
var args []string
for _, a := range addresses {
args = append(args, flag, a)
}
return args
}
func probePod(name, ns, node, image string, args []string, game bool) *corev1.Pod {
no, yes := false, true
uid := int64(1000)
labels := map[string]string{"felis.lolicon.best/probe-name": name}
if game {
labels[v1alpha1.LabelManagedBy] = operator.ManagedByValue
labels[v1alpha1.LabelComponent] = operator.ComponentValue
labels[v1alpha1.LabelServer] = name
}
return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, Spec: corev1.PodSpec{NodeName: node, RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &no, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &yes, RunAsUser: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "probe", Image: image, ImagePullPolicy: corev1.PullAlways, Command: []string{"/usr/local/bin/felis", "node-probe"}, Args: args, Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi"), corev1.ResourceCPU: resource.MustParse("200m")}}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &no, ReadOnlyRootFilesystem: &yes, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}}}}}
}
func probeJob(name, ns, node, image string, args []string, game bool) *batchv1.Job {
p := probePod(name, ns, node, image, args, game)
zero := int32(0)
deadline := int64(600)
return &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, Spec: batchv1.JobSpec{BackoffLimit: &zero, ActiveDeadlineSeconds: &deadline, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: p.Labels}, Spec: p.Spec}}}
}
func waitProbePod(ctx context.Context, cl client.Client, p *corev1.Pod) error {
t := time.NewTicker(time.Second)
defer t.Stop()
for {
if err := cl.Get(ctx, client.ObjectKeyFromObject(p), p); err != nil {
return err
}
for _, c := range p.Status.Conditions {
if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue {
return nil
}
}
if p.Status.Phase == corev1.PodFailed {
return fmt.Errorf("probe Pod %s failed", p.Name)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
}
}
}
func waitProbeJob(ctx context.Context, cl client.Client, j *batchv1.Job) error {
t := time.NewTicker(time.Second)
defer t.Stop()
for {
if err := cl.Get(ctx, client.ObjectKeyFromObject(j), j); err != nil {
return err
}
for _, c := range j.Status.Conditions {
if c.Status != corev1.ConditionTrue {
continue
}
if c.Type == batchv1.JobComplete {
return nil
}
if c.Type == batchv1.JobFailed {
return fmt.Errorf("admission probe Job %s failed; inspect its Pod log", j.Name)
}
}
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
}
}
}
func denyNodeAddresses(ctx context.Context, cl client.Client, ns string, nodes []corev1.Node) error {
var np networkingv1.NetworkPolicy
if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-server-egress"}, &np); err != nil {
return err
}
before := np.DeepCopy()
for _, n := range nodes {
for _, a := range n.Status.Addresses {
if a.Type != corev1.NodeInternalIP && a.Type != corev1.NodeExternalIP {
continue
}
cidr, err := exactCIDR(a.Address)
if err != nil {
return err
}
for i := range np.Spec.Egress {
for k := range np.Spec.Egress[i].To {
block := np.Spec.Egress[i].To[k].IPBlock
if block != nil && ((strings.Contains(cidr, ":") && block.CIDR == "::/0") || (!strings.Contains(cidr, ":") && block.CIDR == "0.0.0.0/0")) {
block.Except = append(block.Except, cidr)
}
}
}
}
}
return cl.Patch(ctx, &np, client.MergeFrom(before))
}
cmd/felis/node_control.go
-581
View File
@@ -1,581 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"os/signal"
"path/filepath"
"runtime"
"slices"
"strings"
"syscall"
"time"
felis "felis.lolicon.best"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/nodecontrol"
"felis.lolicon.best/internal/placement"
"felis.lolicon.best/internal/platform"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
)
func cmdNodeControl(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("node-control", flag.ContinueOnError)
fs.SetOutput(stderr)
socket := fs.String("socket", nodecontrol.Socket, "local API-only Unix socket")
dir := fs.String("state", "/var/lib/felis/node-control", "root-owned persistent task state")
kubeconfig := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "controller kubeconfig")
config := fs.String("config", "/etc/felis/felis.host.toml", "host configuration")
ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "control namespace")
if err := fs.Parse(args); err != nil {
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "node-control requires root")
return 1
}
cfg, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
cs, err := kubernetes.NewForConfig(cfg)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
exe, err := os.Executable()
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
execute := nodeExecutor{cs: cs, binary: exe, kubeconfig: *kubeconfig, config: *config, namespace: *ns, controlNamespace: *controlNS, stateDir: *dir}
socketDir := filepath.Dir(*socket)
if err = os.MkdirAll(socketDir, 0750); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if err = os.Chown(socketDir, 0, 65532); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if err = os.Chmod(socketDir, 0750); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if existing, err := os.Lstat(*socket); err == nil {
if existing.Mode()&os.ModeSocket == 0 {
fmt.Fprintln(stderr, "refusing to replace non-socket path")
return 1
}
conn, err := net.DialTimeout("unix", *socket, time.Second)
if err == nil {
conn.Close()
fmt.Fprintln(stderr, "node-control is already running")
return 1
}
if err = os.Remove(*socket); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
}
listener, err := net.Listen("unix", *socket)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
defer listener.Close()
if err = os.Chown(*socket, 0, 65532); err == nil {
err = os.Chmod(*socket, 0660)
}
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
// /run is recreated at boot; label both the directory and the new socket inode.
if os.Getenv("FELIS_NODE_CONTROL_SELINUX") == "1" {
if err := exec.Command("chcon", "-R", "-t", "felis_node_control_socket_t", socketDir).Run(); err != nil {
fmt.Fprintln(stderr, "node-control socket labeling failed:", err)
return 1
}
}
manager, err := nodecontrol.Open(ctx, *dir, execute.run)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
server := &http.Server{Handler: manager.Handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second}
go func() { <-ctx.Done(); server.Close() }()
fmt.Fprintln(stdout, "node-control listening on", *socket)
err = server.Serve(listener)
cancel()
manager.Wait()
if err != nil && !errors.Is(err, http.ErrServerClosed) {
fmt.Fprintln(stderr, err)
return 1
}
return 0
}
type nodeExecutor struct {
cs kubernetes.Interface
binary, kubeconfig, config, namespace, controlNamespace, stateDir string
}
func (e nodeExecutor) run(ctx context.Context, r nodecontrol.Request, stage func(string) error, out io.Writer) (resultErr error) {
// Host-wide changes and cache-removing admission probes are serialized by the manager.
if r.Action == "approve" {
node, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
if err != nil {
return err
}
if node.Labels[placement.LabelRole] != placement.RoleWorker {
return errors.New("only worker nodes can be approved")
}
patch := []byte(`{"spec":{"unschedulable":true}}`)
if !node.Spec.Unschedulable {
patch = []byte(`{"spec":{"unschedulable":true},"metadata":{"annotations":{"felis.lolicon.best/node-control-cordon":"true"}}}`)
}
if _, err = e.cs.CoreV1().Nodes().Patch(ctx, r.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
return err
}
}
if err := e.requireStopped(ctx, r); err != nil {
return err
}
nodes, err := e.cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
return err
}
controller, peers, err := nodeController(nodes.Items, r.ExternalIP, r.Peers)
if err != nil {
return err
}
deployment, err := e.cs.AppsV1().Deployments(e.controlNamespace).Get(ctx, platform.SAAPI, metav1.GetOptions{})
if err != nil {
return err
}
if len(deployment.Spec.Template.Spec.Containers) == 0 {
return errors.New("the Felis API image is unavailable")
}
image := deployment.Spec.Template.Spec.Containers[0].Image
if r.Action == "enable" {
if err := stage("pause_operator"); err != nil {
return err
}
scale, err := e.cs.AppsV1().Deployments(e.controlNamespace).GetScale(ctx, platform.SAOperator, metav1.GetOptions{})
if err != nil {
return err
}
replicas := scale.Spec.Replicas
if replicas < 1 {
replicas = 1
}
scale.Spec.Replicas = 0
if _, err = e.cs.AppsV1().Deployments(e.controlNamespace).UpdateScale(ctx, platform.SAOperator, scale, metav1.UpdateOptions{}); err != nil {
return err
}
defer func() {
cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
latest, err := e.cs.AppsV1().Deployments(e.controlNamespace).GetScale(cleanup, platform.SAOperator, metav1.GetOptions{})
if err == nil {
latest.Spec.Replicas = replicas
_, err = e.cs.AppsV1().Deployments(e.controlNamespace).UpdateScale(cleanup, platform.SAOperator, latest, metav1.UpdateOptions{})
}
if err != nil {
fmt.Fprintln(out, "Operator restoration failed:", err)
resultErr = fmt.Errorf("operator restoration failed: %w", err)
}
}()
// Drain the old reconciler before the second stopped-state check.
for {
pods, err := e.cs.CoreV1().Pods(e.controlNamespace).List(ctx, metav1.ListOptions{LabelSelector: platform.LabelComponent + "=" + platform.ComponentOperator})
if err != nil {
return err
}
if len(pods.Items) == 0 {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Second):
}
}
if err = e.requireStopped(ctx, r); err != nil {
return err
}
return e.enable(ctx, r, controller, peers, stage, out)
}
if controller.Labels[placement.LabelRole] != placement.RoleController {
return errors.New("distributed mode is not configured on the controller")
}
if r.Name == controller.Name {
return errors.New("worker name must differ from the controller")
}
// SSH trust and keys are configured on A; no password, key or bootstrap token crosses the API.
if err := stage("ssh_check"); err != nil {
return err
}
arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[runtime.GOARCH]
if arch == "" {
return errors.New("unsupported host architecture")
}
check := "set -eu\n[ \"$(uname -s)\" = Linux ] || { echo 'worker requires Linux' >&2; exit 1; }\n[ \"$(uname -m)\" = " + shellQuote(arch) + " ] || { echo 'worker architecture differs from controller' >&2; exit 1; }\n"
if r.Action == "join" {
check += "[ ! -e /etc/rancher/k3s/k3s.yaml ] && [ ! -e /var/lib/rancher/k3s/agent ] || { echo 'k3s is already installed; enrollment will not overwrite an existing node' >&2; exit 1; }\n"
check += "ip -o address show | awk '{print $4}' | cut -d/ -f1 | grep -Fx -- " + shellQuote(r.ExternalIP) + " >/dev/null || { echo 'fixed node IP is not assigned to the worker' >&2; exit 1; }\n"
}
if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi", strings.NewReader(check), out); err != nil {
return fmt.Errorf("worker preflight or SSH connection failed: %w", err)
}
if r.Action == "join" {
for _, n := range nodes.Items {
if n.Name == r.Name {
return errors.New("node already exists; use approval instead of reinstalling it")
}
}
if err = e.join(ctx, r, controller, peers, stage, out); err != nil {
return err
}
}
if err = stage("approval"); err != nil {
return err
}
cmd := exec.CommandContext(ctx, e.binary, "node", "approve", "--name", r.Name, "--ssh-target", r.SSHTarget, "--image", image, "--kubeconfig", e.kubeconfig, "--namespace", e.namespace, "--control-namespace", e.controlNamespace)
cmd.Stdout = out
cmd.Stderr = out
if err = cmd.Run(); err != nil {
return fmt.Errorf("node approval failed; node remains quarantined: %w", err)
}
// Admission succeeded; release only the task's scheduling quarantine.
admitted, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
if err != nil {
return err
}
if admitted.Annotations["felis.lolicon.best/node-control-cordon"] == "true" {
if _, err = e.cs.CoreV1().Nodes().Patch(ctx, r.Name, types.MergePatchType, []byte(`{"spec":{"unschedulable":false},"metadata":{"annotations":{"felis.lolicon.best/node-control-cordon":null}}}`), metav1.PatchOptions{}); err != nil {
return fmt.Errorf("node approved but scheduling remains disabled: %w", err)
}
}
return stage("complete")
}
func (e nodeExecutor) requireStopped(ctx context.Context, r nodecontrol.Request) error {
// Even pre-existing worker names may contain worlds. Never run installer/admission on an occupied worker.
ss, err := e.cs.AppsV1().StatefulSets(e.namespace).List(ctx, metav1.ListOptions{})
if err != nil {
return err
}
for _, s := range ss.Items {
if r.Action != "enable" && s.Spec.Template.Spec.NodeSelector[placement.LabelIdentity] != r.Name {
continue
}
if s.Spec.Replicas != nil && *s.Spec.Replicas > 0 {
return fmt.Errorf("StatefulSet %s still requests %d replicas; stop the server before changing nodes", s.Name, *s.Spec.Replicas)
}
}
pods, err := e.cs.CoreV1().Pods(e.namespace).List(ctx, metav1.ListOptions{})
if err != nil {
return err
}
for _, p := range pods.Items {
if r.Action != "enable" && p.Spec.NodeName != r.Name && p.Spec.NodeSelector[placement.LabelIdentity] != r.Name {
continue
}
if p.Status.Phase != corev1.PodSucceeded && p.Status.Phase != corev1.PodFailed {
return fmt.Errorf("pod %s has not exited (phase %s); wait for game and maintenance workloads to stop", p.Name, p.Status.Phase)
}
}
// Prevent a pending wake intent racing admission or an installation.
raw, err := e.cs.CoreV1().RESTClient().Get().AbsPath("/apis/" + v1alpha1.GroupVersion.Group + "/" + v1alpha1.GroupVersion.Version + "/namespaces/" + e.namespace + "/minecraftservers").DoRaw(ctx)
if err != nil {
return err
}
var servers v1alpha1.MinecraftServerList
if err = json.Unmarshal(raw, &servers); err != nil {
return err
}
for _, s := range servers.Items {
if r.Action != "enable" && s.Spec.NodeName != r.Name && s.Status.NodeName != r.Name {
continue
}
if s.Spec.DesiredState != "" && string(s.Spec.DesiredState) != "Stopped" {
return fmt.Errorf("server %s must have stopped intent", s.Name)
}
}
return nil
}
func nodeController(nodes []corev1.Node, ip string, extra []string) (*corev1.Node, []string, error) {
var controller *corev1.Node
peers := append([]string{}, extra...)
for i := range nodes {
n := &nodes[i]
_, controlPlane := n.Labels["node-role.kubernetes.io/control-plane"]
if controlPlane || n.Labels[placement.LabelRole] == placement.RoleController {
if controller != nil {
return nil, nil, errors.New("exactly one controller is required")
}
controller = n
}
for _, a := range n.Status.Addresses {
if a.Type == corev1.NodeExternalIP || a.Type == corev1.NodeInternalIP {
cidr, err := exactCIDR(a.Address)
if err != nil {
return nil, nil, err
}
peers = append(peers, cidr)
}
}
}
if controller == nil || !placement.Online(controller) {
return nil, nil, errors.New("the sole controller must be online")
}
if ip != "" {
cidr, err := exactCIDR(ip)
if err != nil {
return nil, nil, err
}
peers = append(peers, cidr)
}
slices.Sort(peers)
peers = slices.Compact(peers)
return controller, peers, nil
}
func controllerIP(n *corev1.Node) string {
for _, kind := range []corev1.NodeAddressType{corev1.NodeExternalIP, corev1.NodeInternalIP} {
for _, a := range n.Status.Addresses {
if a.Type == kind {
return a.Address
}
}
}
return ""
}
func (e nodeExecutor) ssh(ctx context.Context, target, command string, in io.Reader, out io.Writer) error {
cmd := exec.CommandContext(ctx, "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=10", "--", target, command)
cmd.Stdin = in
cmd.Stdout = out
cmd.Stderr = out
return cmd.Run()
}
func (e nodeExecutor) join(ctx context.Context, r nodecontrol.Request, controller *corev1.Node, peers []string, stage func(string) error, out io.Writer) error {
registry, err := e.cs.CoreV1().Services(e.controlNamespace).Get(ctx, "registry", metav1.GetOptions{})
if err != nil {
return err
}
if err = stage("firewall"); err != nil {
return err
}
// All peers must be updated before the new agent is admitted. Existing worker SSH targets must be configured by stable node name.
list, err := e.cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
return err
}
for _, n := range list.Items {
if n.Name == controller.Name || n.Name == r.Name {
continue
}
if n.Labels[placement.LabelRole] != placement.RoleWorker {
return fmt.Errorf("node %s has an unknown role", n.Name)
}
script := shellQuote(e.binary) + " node firewall --peers " + shellQuote(strings.Join(peers, ",")) + " --controller-ip " + shellQuote(controllerIP(controller))
script += " --namespace " + shellQuote(e.namespace) + " --control-namespace " + shellQuote(e.controlNamespace)
if err = e.ssh(ctx, n.Name, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n "+script+"; fi", nil, out); err != nil {
return fmt.Errorf("update peer firewall on %s: %w", n.Name, err)
}
}
cmd := exec.CommandContext(ctx, e.binary, "node", "firewall", "--controller", "--controller-ip", controllerIP(controller), "--peers", strings.Join(peers, ","), "--namespace", e.namespace, "--control-namespace", e.controlNamespace)
cmd.Stdout = out
cmd.Stderr = out
if err = cmd.Run(); err != nil {
return err
}
if err = stage("bootstrap_token"); err != nil {
return err
}
temp, err := os.MkdirTemp("", "felis-node-join-")
if err != nil {
return err
}
defer os.RemoveAll(temp)
tokenPath := filepath.Join(temp, "bootstrap")
tokenCmd := exec.CommandContext(ctx, e.binary, "node", "token", "--name", r.Name, "--ttl", "1h", "--out", tokenPath)
tokenCmd.Stdout = out
tokenCmd.Stderr = out
if err = tokenCmd.Run(); err != nil {
return err
}
// Revoke even on failure. The worker exchanges bootstrap credentials for its node identity.
token, err := os.ReadFile(tokenPath)
if err != nil {
return err
}
defer func() {
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
exec.CommandContext(cleanup, "k3s", "token", "delete", bootstrapTokenID(string(token))).Run()
}()
remoteDir := "/var/tmp/felis-node-" + filepath.Base(temp)
if err = stage("transfer"); err != nil {
return err
}
script := "set -eu; umask 077; mkdir " + shellQuote(remoteDir) + "; cat > " + shellQuote(remoteDir+"/bootstrap")
rootCommand := "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi"
defer func() {
cleanup, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
e.ssh(cleanup, r.SSHTarget, rootCommand, strings.NewReader("rm -rf -- "+shellQuote(remoteDir)), io.Discard)
}()
// stdin carries the token; it is never in command arguments, task records or logs.
if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n sh -c "+shellQuote(script)+"; fi", strings.NewReader(string(token)), out); err != nil {
return err
}
binary, err := os.Open(e.binary)
if err != nil {
return err
}
defer binary.Close()
script = "set -eu; cat > " + shellQuote(remoteDir+"/felis") + "; chmod 0700 " + shellQuote(remoteDir+"/felis")
if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n sh -c "+shellQuote(script)+"; fi", binary, out); err != nil {
return err
}
if err = stage("install_worker"); err != nil {
return err
}
script = "set -eu\nexport FELIS_K3S_VERSION=" + shellQuote(controller.Status.NodeInfo.KubeletVersion) + "\n" + shellQuote(remoteDir+"/felis") + " node join --name " + shellQuote(r.Name) + " --server " + shellQuote("https://"+net.JoinHostPort(controllerIP(controller), "6443")) + " --external-ip " + shellQuote(r.ExternalIP) + " --token-file " + shellQuote(remoteDir+"/bootstrap") + " --registry-ip " + shellQuote(registry.Spec.ClusterIP) + " --peers " + shellQuote(strings.Join(peers, ",")) + "\n"
if err = e.ssh(ctx, r.SSHTarget, rootCommand, strings.NewReader(script), out); err != nil {
return fmt.Errorf("worker installation failed: %w", err)
}
if err = stage("wait_ready"); err != nil {
return err
}
deadline := time.NewTimer(5 * time.Minute)
defer deadline.Stop()
ticker := time.NewTicker(3 * time.Second)
defer ticker.Stop()
for {
n, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
if err == nil && placement.Online(n) {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-deadline.C:
return errors.New("worker did not become Ready within five minutes")
case <-ticker.C:
}
}
}
func (e nodeExecutor) enable(ctx context.Context, r nodecontrol.Request, controller *corev1.Node, peers []string, stage func(string) error, out io.Writer) error {
if controllerIP(controller) != r.ExternalIP {
return errors.New("controller IP must match the registered fixed node address")
}
if len(peers) == 0 {
return errors.New("peer addresses are required")
}
if err := stage("database_backup"); err != nil {
return err
}
backup := exec.CommandContext(ctx, e.binary, "db", "backup", "-config", e.config, "-dir", "/var/lib/felis/db-backups", "-label", "pre-migrate")
backup.Stdout = out
backup.Stderr = out
if err := backup.Run(); err != nil {
return err
}
if err := stage("cluster_backup"); err != nil {
return err
}
if err := e.backupCluster(ctx, out); err != nil {
return err
}
if err := stage("configure_controller"); err != nil {
return err
}
patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s","%s":"controller"}}}`, placement.LabelIdentity, controller.Name, placement.LabelRole)
if _, err := e.cs.CoreV1().Nodes().Patch(ctx, controller.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil {
return err
}
for _, name := range []string{platform.SAAPI, platform.SAOperator, platform.PostgresName, "registry"} {
body := fmt.Sprintf(`{"spec":{"template":{"spec":{"nodeSelector":{"%s":"%s"}}}}}`, placement.LabelIdentity, controller.Name)
if _, err := e.cs.AppsV1().Deployments(e.controlNamespace).Patch(ctx, name, types.MergePatchType, []byte(body), metav1.PatchOptions{}); err != nil {
return err
}
}
if err := stage("install_controller"); err != nil {
return err
}
cmd := exec.CommandContext(ctx, "bash", "-s")
cmd.Stdin = strings.NewReader(felis.BootstrapScript())
cmd.Stdout = out
cmd.Stderr = out
cmd.Env = append(os.Environ(), "FELIS_DISTRIBUTED=1", "FELIS_NODE_EXTERNAL_IP="+r.ExternalIP, "FELIS_PEER_CIDRS="+strings.Join(peers, ","), "FELIS_BOOTSTRAP_FROM_TUI=1", "FELIS_BOOTSTRAP_BINARY="+e.binary, "FELIS_NO_SETUP=1", "FELIS_NODE_CONTROL_TASK=1", "FELIS_INSTALL_MODE=full")
if err := cmd.Run(); err != nil {
return fmt.Errorf("controller installation failed; retain the database backup: %w", err)
}
return stage("complete")
}
func bootstrapTokenID(token string) string {
token = strings.TrimSpace(token)
if _, short, ok := strings.Cut(token, "::"); ok {
token = short
}
id, _, _ := strings.Cut(token, ".")
return id
}
func (e nodeExecutor) backupCluster(ctx context.Context, out io.Writer) (err error) {
dir := filepath.Join(filepath.Dir(e.stateDir), "cluster-backups")
if err = os.MkdirAll(dir, 0700); err != nil {
return err
}
path := filepath.Join(dir, "pre-distributed-"+time.Now().UTC().Format("20060102T150405.000000000Z")+".tar")
stop := exec.CommandContext(ctx, "systemctl", "stop", "k3s")
stop.Stdout = out
stop.Stderr = out
if err = stop.Run(); err != nil {
return err
}
defer func() {
restart, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
cmd := exec.CommandContext(restart, "systemctl", "start", "k3s")
cmd.Stdout = out
cmd.Stderr = out
if restartErr := cmd.Run(); restartErr != nil {
err = fmt.Errorf("k3s restart failed after snapshot: %w", restartErr)
}
}()
cmd := exec.CommandContext(ctx, "tar", "-cf", path, "-C", "/var/lib/rancher/k3s/server", "db", "token", "tls")
cmd.Stdout = out
cmd.Stderr = out
if err = cmd.Run(); err != nil {
return fmt.Errorf("cluster snapshot failed: %w", err)
}
if err = os.Chmod(path, 0600); err != nil {
return err
}
fmt.Fprintln(out, "Cluster snapshot:", path)
return nil
}
cmd/felis/node_control_test.go
-40
View File
@@ -1,40 +0,0 @@
package main
import (
"context"
"strings"
"testing"
"felis.lolicon.best/internal/nodecontrol"
"felis.lolicon.best/internal/platform"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
func TestNodeControlTopologyAndBootstrapID(t *testing.T) {
node := corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "controller", Labels: map[string]string{"node-role.kubernetes.io/control-plane": "true"}}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.0.2.1"}}}}
controller, peers, err := nodeController([]corev1.Node{node}, "192.0.2.2", []string{"192.0.2.1/32"})
if err != nil || controller.Name != "controller" || strings.Join(peers, ",") != "192.0.2.1/32,192.0.2.2/32" {
t.Fatal(controller, peers, err)
}
duplicate := node
duplicate.Name = "second"
if _, _, err = nodeController([]corev1.Node{node, duplicate}, "", nil); err == nil {
t.Fatal("multiple controllers accepted")
}
for _, token := range []string{"abcdef.0123456789abcdef", "K10hash::abcdef.0123456789abcdef\n"} {
if bootstrapTokenID(token) != "abcdef" {
t.Fatal("token secret passed to revocation")
}
}
}
func TestNodeControlRejectsActiveWorkloadsBeforeHostCommands(t *testing.T) {
replicas := int32(1)
set := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"}, Spec: appsv1.StatefulSetSpec{Replicas: &replicas}}
executor := nodeExecutor{cs: fake.NewSimpleClientset(set), namespace: platform.DefaultMinecraftNamespace}
if err := executor.requireStopped(context.Background(), nodecontrol.Request{Action: "enable"}); err == nil {
t.Fatal("host changes permitted with active worlds")
}
}
cmd/felis/node_firewall.go
-187
View File
@@ -1,187 +0,0 @@
package main
import (
"flag"
"fmt"
"io"
"net"
"os"
"os/exec"
"strconv"
"strings"
)
func nodeFirewall(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("node firewall", flag.ContinueOnError)
fs.SetOutput(stderr)
peers := fs.String("peers", "", "comma-separated exact node IP CIDRs")
controller := fs.String("controller-ip", "", "controller address (optionally :6443)")
main := fs.Bool("controller", false, "A hosts the public API NodePort")
pod := fs.String("pod-cidr", "10.42.0.0/16", "cluster Pod CIDR")
dryRun := fs.Bool("dry-run", false, "print firewall scripts without installing")
controlNS := fs.String("control-namespace", "felis", "controller namespace")
minecraftNS := fs.String("namespace", "minecraft", "game namespace")
apiIP := fs.String("api-service-ip", "10.43.0.1", "Kubernetes API Service IP")
port := fs.Int("node-port", 30443, "API NodePort to block on workers before DNAT")
if err := fs.Parse(args); err != nil {
return 2
}
if host, _, err := net.SplitHostPort(*controller); err == nil {
*controller = host
}
if net.ParseIP(*apiIP) == nil || net.ParseIP(*controller) == nil || *port < 30000 || *port > 32767 {
fmt.Fprintln(stderr, "node firewall: controller IP and NodePort required")
return 2
}
if _, _, err := net.ParseCIDR(*pod); err != nil {
fmt.Fprintln(stderr, err)
return 2
}
var v4, v6 []string
for _, p := range strings.Split(*peers, ",") {
ip, n, err := net.ParseCIDR(p)
if err != nil {
fmt.Fprintln(stderr, "node firewall: invalid peer CIDR")
return 2
}
ones, bits := n.Mask.Size()
if ones != bits {
fmt.Fprintln(stderr, "node firewall: only exact peer addresses accepted")
return 2
}
if ip.To4() != nil {
v4 = append(v4, n.String())
} else {
v6 = append(v6, n.String())
}
}
script := "#!/bin/bash\nset -euo pipefail\n"
for _, f := range []struct {
bin string
peers []string
metadata string
}{{"iptables", v4, "169.254.0.0/16"}, {"ip6tables", v6, "fe80::/10"}} {
if f.bin == "ip6tables" && len(f.peers) == 0 {
continue
}
b := f.bin + " -w 10"
script += b + " -N FELIS-HOST 2>/dev/null || true\n" + b + " -F FELIS-HOST\n"
script += b + " -N FELIS-FORWARD 2>/dev/null || true\n" + b + " -F FELIS-FORWARD\n"
script += b + " -t raw -N FELIS-NODEPORT 2>/dev/null || true\n" + b + " -t raw -F FELIS-NODEPORT\n"
for _, c := range []struct{ table, parent, chain string }{{"filter", "INPUT", "FELIS-HOST"}, {"filter", "FORWARD", "FELIS-FORWARD"}, {"raw", "PREROUTING", "FELIS-NODEPORT"}} {
script += "while " + b + " -t " + c.table + " -D " + c.parent + " -j " + c.chain + " 2>/dev/null; do :; done\n" + b + " -t " + c.table + " -I " + c.parent + " 1 -j " + c.chain + "\n"
}
script += b + " -A FELIS-HOST -i lo -j RETURN\n"
if *main {
script += b + " -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -A FELIS-HOST -j FELIS-CONTROL\n"
script += b + " -t raw -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -t raw -A FELIS-NODEPORT -j FELIS-CONTROL\n"
}
script += b + " -A FELIS-HOST -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n" + b + " -A FELIS-FORWARD -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n"
family := 4
if f.bin == "ip6tables" {
family = 6
}
podIP, _, _ := net.ParseCIDR(*pod)
podFamily := 6
if podIP.To4() != nil {
podFamily = 4
}
if family == podFamily {
script += b + " -A FELIS-HOST -s " + *pod + " -j DROP\n"
// raw precedes DNAT and kube-router's filter ACCEPT rules. Block new
// host connections while preserving established Velocity/RCON replies.
script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --syn -j DROP\n"
script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p udp -j DROP\n"
if (net.ParseIP(*apiIP).To4() != nil) == (family == 4) {
script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + *apiIP + " -p tcp --dport 443 --syn -j DROP\n"
}
script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n"
script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + f.metadata + " -j DROP\n"
script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + f.metadata + " -j DROP\n"
for _, p := range f.peers {
script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + p + " -j DROP\n"
script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p tcp --syn -j DROP\n"
script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p udp -j DROP\n"
}
}
if !*main {
script += b + " -t raw -A FELIS-NODEPORT -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n"
}
for _, p := range f.peers {
script += b + " -A FELIS-HOST -s " + p + " -p udp --dport 51820:51821 -j RETURN\n"
}
script += b + " -A FELIS-HOST -p udp --dport 51820:51821 -j DROP\n"
ctrlIP := net.ParseIP(*controller)
ctrlFamily := 6
if ctrlIP.To4() != nil {
ctrlFamily = 4
}
if *main {
for _, p := range f.peers {
script += b + " -A FELIS-HOST -s " + p + " -p tcp --dport 6443 -j RETURN\n"
}
}
if ctrlFamily == family {
script += b + " -A FELIS-HOST -s " + ctrlIP.String() + " -p tcp --dport 10250 -j RETURN\n"
}
script += b + " -A FELIS-HOST -p tcp -m multiport --dports 6443,6444,10250,10255,2379,2380,5000,5001,15432 -j DROP\n"
}
if *dryRun {
fmt.Fprint(stdout, script)
if *main {
fmt.Fprint(stdout, controlFirewallScript(*controlNS, *minecraftNS))
}
return 0
}
if err := os.MkdirAll("/etc/felis", 0700); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if err := os.WriteFile("/etc/felis/node-firewall.sh", []byte(script), 0700); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
unit := "[Unit]\nDescription=Felis host and NodePort isolation\nAfter=network-online.target firewalld.service ufw.service\nBefore=k3s.service k3s-agent.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/node-firewall.sh\nRemainAfterExit=yes\n[Install]\nWantedBy=multi-user.target\n"
if err := os.WriteFile("/etc/systemd/system/felis-node-firewall.service", []byte(unit), 0644); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if *main {
refresh := controlFirewallScript(*controlNS, *minecraftNS)
if err := os.WriteFile("/etc/felis/control-firewall.sh", []byte(refresh), 0700); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
svc := "[Unit]\nDescription=Refresh exact controller Pod access to the apiserver\nAfter=k3s.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/control-firewall.sh\n"
timer := "[Unit]\nDescription=Track trusted controller Pods after rescheduling\n[Timer]\nOnBootSec=5s\nOnUnitActiveSec=5s\n[Install]\nWantedBy=timers.target\n"
if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.service", []byte(svc), 0644); err != nil {
return 1
}
if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.timer", []byte(timer), 0644); err != nil {
return 1
}
}
for _, cmd := range []*exec.Cmd{exec.Command("systemctl", "daemon-reload"), exec.Command("systemctl", "enable", "felis-node-firewall.service"), exec.Command("bash", "/etc/felis/node-firewall.sh")} {
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
}
if *main {
cmd := exec.Command("systemctl", "enable", "--now", "felis-control-firewall.timer")
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
}
return 0
}
func controlFirewallScript(controlNS, minecraftNS string) string {
return "#!/bin/bash\nset -euo pipefail\niptables -w 10 -F FELIS-CONTROL\niptables -w 10 -t raw -F FELIS-CONTROL\n/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{\" \"}{.spec.serviceAccountName}{\" \"}{.status.podIP}{\"\\n\"}{end}' | while read -r ns sa ip; do\ncase \"$ns/$sa\" in " + shellQuote(controlNS+"/felis-api") + "|" + shellQuote(controlNS+"/felis-operator") + "|" + shellQuote(minecraftNS+"/felis-reaper") + "|kube-system/*) ;; *) continue;; esac\n[[ $ip =~ ^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$ ]] || continue\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p tcp --dport 6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p tcp -m multiport --dports 443,6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\ndone\n"
}
cmd/felis/node_firewall_test.go
-39
View File
@@ -1,39 +0,0 @@
package main
import (
"bytes"
"os/exec"
"strings"
"testing"
)
func TestDistributedFirewallPathsAndSyntax(t *testing.T) {
for _, controller := range []bool{false, true} {
args := []string{"--dry-run", "--peers", "192.0.2.1/32,192.0.2.2/32", "--controller-ip", "192.0.2.1"}
if controller {
args = append(args, "--controller")
}
var out, err bytes.Buffer
if code := nodeFirewall(args, &out, &err); code != 0 {
t.Fatal(code, err.String())
}
script := out.String()
for _, must := range []string{"-I INPUT 1 -j FELIS-HOST", "-I FORWARD 1 -j FELIS-FORWARD", "-t raw -I PREROUTING 1 -j FELIS-NODEPORT", "-A FELIS-HOST -s 10.42.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --dport 30443 -j DROP", "-d 169.254.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --syn -j DROP", "-d 10.43.0.1 -p tcp --dport 443 --syn -j DROP"} {
if !strings.Contains(script, must) {
t.Fatal("missing protection", must)
}
}
if !controller && strings.Contains(script, " -p tcp --dport 6443 -j RETURN") {
t.Fatal("worker exposed apiserver")
}
check := exec.Command("bash", "-n")
check.Stdin = strings.NewReader(script)
if result, e := check.CombinedOutput(); e != nil {
t.Fatalf("invalid firewall script: %s %v", result, e)
}
}
var out, err bytes.Buffer
if code := nodeFirewall([]string{"--dry-run", "--peers", "10.0.0.0/8", "--controller-ip", "10.0.0.1"}, &out, &err); code != 2 {
t.Fatal("broad node range accepted")
}
}
cmd/felis/node_probe.go
-56
View File
@@ -1,56 +0,0 @@
package main
import (
"flag"
"fmt"
"io"
"net"
"time"
)
// A trusted probe runs with the same server labels and security context before node admission.
func cmdNodeProbe(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("node-probe", flag.ContinueOnError)
fs.SetOutput(stderr)
listen := fs.String("listen", "", "listen and print observed source addresses (admission only)")
var open, closed multiFlag
fs.Var(&open, "open", "required reachable host:port")
fs.Var(&closed, "closed", "required blocked host:port")
if err := fs.Parse(args); err != nil {
return 2
}
if *listen != "" {
l, err := net.Listen("tcp", *listen)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
defer l.Close()
for {
c, err := l.Accept()
if err != nil {
return 1
}
fmt.Fprintln(stdout, "felis-probe-source", c.RemoteAddr().String())
c.Write([]byte("felis-probe\n"))
c.Close()
}
}
time.Sleep(3 * time.Second)
for _, check := range []struct {
addresses []string
wantOpen bool
}{{open, true}, {closed, false}} {
for _, addr := range check.addresses {
c, err := net.DialTimeout("tcp", addr, 2*time.Second)
if c != nil {
c.Close()
}
if (err == nil) != check.wantOpen {
fmt.Fprintf(stderr, "node-probe: %s open=%t, expected %t\n", addr, err == nil, check.wantOpen)
return 1
}
}
}
return 0
}
cmd/felis/nodes.go
-157
View File
@@ -1,157 +0,0 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net"
"os"
"os/exec"
"strings"
"time"
felis "felis.lolicon.best"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/distributed"
"felis.lolicon.best/internal/platform"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func cmdNode(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprintln(stderr, "felis node: list | token | join | approve | firewall")
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis node requires root/sudo")
return 1
}
if args[0] == "firewall" {
return nodeFirewall(args[1:], stdout, stderr)
}
fs := flag.NewFlagSet("node "+args[0], flag.ContinueOnError)
fs.SetOutput(stderr)
name := fs.String("name", "", "stable worker node name")
kubeconfig := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local administrator kubeconfig")
ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "controller namespace")
image := fs.String("image", "", "Felis image to re-pull and use for admission probes")
remote := fs.String("ssh-target", "", "SSH target of the trusted worker (normal SSH host verification applies)")
out := fs.String("out", "", "token output file; never printed to stdout")
ttl := fs.Duration("ttl", 10*time.Minute, "bootstrap token lifetime (maximum 1h)")
server := fs.String("server", "", "A's https://address:6443 endpoint for join")
tokenFile := fs.String("token-file", "", "CA-pinned bootstrap token file for join")
registry := fs.String("registry-ip", "", "registry ClusterIP for join")
peers := fs.String("peers", "", "comma-separated exact peer CIDRs for host firewall")
external := fs.String("external-ip", "", "this worker's fixed external IP")
if err := fs.Parse(args[1:]); err != nil {
return 2
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancel()
switch args[0] {
case "token":
if *name == "" || *out == "" || *ttl <= 0 || *ttl > time.Hour {
fmt.Fprintln(stderr, "node token: name, output file and lifetime <=1h required")
return 2
}
cmd := exec.CommandContext(ctx, "k3s", "token", "create", "--ttl", ttl.String(), "--description", "Felis worker "+*name)
cmd.Stderr = stderr
raw, err := cmd.Output()
if err != nil {
fmt.Fprintln(stderr, "node token: creation failed:", err)
return 1
}
f, err := os.OpenFile(*out, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
_, err = f.Write(raw)
if err == nil {
err = f.Sync()
}
f.Close()
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
fmt.Fprintln(stdout, "limited bootstrap token written to", *out)
return 0
case "join":
exe, err := os.Executable()
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
cmd := exec.CommandContext(ctx, "bash", "-s")
cmd.Stdin = strings.NewReader(felis.BootstrapScript())
cmd.Stdout = stdout
cmd.Stderr = stderr
cmd.Env = append(os.Environ(), "FELIS_INSTALL_MODE=worker", "FELIS_BOOTSTRAP_FROM_TUI=1", "FELIS_BOOTSTRAP_BINARY="+exe, "FELIS_NODE_NAME="+*name, "FELIS_SERVER_URL="+*server, "FELIS_BOOTSTRAP_TOKEN_FILE="+*tokenFile, "FELIS_REGISTRY_CLUSTER_IP="+*registry, "FELIS_PEER_CIDRS="+*peers, "FELIS_NODE_EXTERNAL_IP="+*external)
if err = cmd.Run(); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
return 0
case "list", "approve":
default:
fmt.Fprintln(stderr, "unknown node operation")
return 2
}
cfg, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
scheme := runtime.NewScheme()
clientgoscheme.AddToScheme(scheme)
v1alpha1.AddToScheme(scheme)
cl, err := client.New(cfg, client.Options{Scheme: scheme})
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
m := &distributed.Manager{Client: cl, Namespace: *ns}
if args[0] == "list" {
nodes, err := m.Nodes(ctx)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
json.NewEncoder(stdout).Encode(nodes)
return 0
}
if *name == "" || *image == "" || *remote == "" || strings.HasPrefix(*remote, "-") {
fmt.Fprintln(stderr, "node approve requires name, image and SSH target")
return 2
}
cs, err := kubernetes.NewForConfig(cfg)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if err = approveNode(ctx, cl, cs, *ns, *controlNS, *name, *image, *remote, stdout); err != nil {
fmt.Fprintln(stderr, "node remains quarantined:", err)
return 1
}
return 0
}
func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" }
func exactCIDR(ip string) (string, error) {
parsed := net.ParseIP(ip)
if parsed == nil {
return "", fmt.Errorf("invalid node address %q", ip)
}
if parsed.To4() != nil {
return parsed.String() + "/32", nil
}
return parsed.String() + "/128", nil
}
cmd/felis/offsite.go
+67 -457
View File
@@ -16,6 +16,7 @@ import (
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,34 +25,18 @@ import (
)
const offsiteUsage = `usage:
felis offsite sync [-config path] [-archive-dir dir] [-db-dir dir] [-registry host:port|off]
[-uploads-dir dir] [-status-file path]
felis offsite sync [-config path] [-archive-dir dir] [-db-dir dir] [-status-file path]
felis offsite status [-config path] [-status-file path]
felis offsite list [-config path]
felis offsite fetch-db [-config path | -endpoint url -bucket name [-region r] [-prefix p]]
[-dir dir] latest|<bundle>
felis offsite fetch-worlds [-config path] [-archive-dir dir]
felis offsite fetch-images [-config path] [-registry host:port] [-at version]
felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
felis offsite check-key [-config path]
felis offsite take-over [-config path] [-status-file path] [-yes]
felis offsite keygen
Every verb but keygen reads the bucket credentials and the encryption key from
the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY,
FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
-env-file (default /etc/felis/offsite.env).
check-key tells whether the key is the one the bucket's objects are sealed
with, writing nothing; it exits 3 when they are sealed with another key, and
sync then refuses to write or prune anything in the bucket.
take-over names the host that writes the bucket, writing nothing; it exits 4
when that is another host and this one never wrote it, and 5 when another
host took the bucket over from this one. A host built from another host's
backup (a rehearsal, or a rebuild) copies nothing into that host's bucket
until -yes makes it the writer; the host it replaces then stops copying and
says so.
`
// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
@@ -59,10 +44,9 @@ says so.
const defaultOffsiteEnvFile = "/etc/felis/offsite.env"
// cmdOffsite implements `felis offsite`: the off-site copy of the world
// archives, the database bundles, the registry's user images and the
// submission uploads (internal/offsite). felis-offsite.timer runs `sync`
// hourly on the host; the fetch verbs are the way back after the node is lost
// (docs/troubleshooting.md §16).
// archives and the database bundles (internal/offsite). felis-offsite.timer
// runs `sync` hourly on the host; the fetch verbs are the way back after the
// node is lost (docs/troubleshooting.md §16).
func cmdOffsite(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprint(stderr, offsiteUsage)
@@ -83,14 +67,6 @@ func cmdOffsite(args []string, stdout, stderr io.Writer) int {
return offsiteFetchDB(fs, rest, stdout, stderr)
case "fetch-worlds":
return offsiteFetchWorlds(fs, rest, stdout, stderr)
case "fetch-images":
return offsiteFetchImages(fs, rest, stdout, stderr)
case "fetch-uploads":
return offsiteFetchUploads(fs, rest, stdout, stderr)
case "check-key":
return offsiteCheckKey(fs, rest, stdout, stderr)
case "take-over":
return offsiteTakeOver(fs, rest, stdout, stderr)
case "keygen":
k, err := offsite.NewKey()
if err != nil {
@@ -211,10 +187,6 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory bundled into the database bundle taken after archives are copied ("" for none)`)
registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC through the cluster)")
uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, `the submission uploads PVC, in the control-plane namespace ("" copies no uploads)`)
statusFile := fs.String("status-file", offsite.DefaultStatusFile, "where the result of this run is recorded for the watchdog and `status`")
if err := fs.Parse(args); err != nil {
return 2
@@ -224,38 +196,29 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
return 1
}
st, lease := startRun(env.cfg, env.key, *statusFile, time.Now())
res, err := runOffsiteSync(cfg, env, offsiteSources{
archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir, stateDir: *stateDir,
registry: offsiteRegistryEndpoint(*registry, cfg.Registry),
uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
}, &lease, stderr)
recordRun(&st, res, err, lease)
st := offsite.Status{
LastAttempt: time.Now().UTC(), Endpoint: env.cfg.Endpoint, Bucket: env.cfg.Bucket,
Prefix: env.cfg.Prefix, KeyID: offsite.KeyID(env.key),
}
if prev, _ := offsite.ReadStatus(*statusFile); prev != nil {
st.LastSuccess = prev.LastSuccess
}
res, err := runOffsiteSync(cfg, env, *archiveDir, *backupPVC, *dbDir, stderr)
st.Result = res
if err != nil {
st.LastError = err.Error()
} else {
st.LastSuccess = st.LastAttempt
}
if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
}
return reportRun(res, err, stdout, stderr)
}
// reportRun prints one pass's outcome and its exit code. A run stopped before
// it copied anything (a bucket that did not answer, another key's objects,
// another host writing the bucket) prints no counts: its zeros would read as
// an empty bucket.
func reportRun(res offsite.Result, err error, stdout, stderr io.Writer) int {
if err == nil || len(res.Errors) > 0 {
fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n",
res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
res.UploadsUploaded, res.UploadObjectsPruned,
res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes),
res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes))
}
fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; bucket holds %d worlds (%s) and %d bundles\n",
res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
res.DBUploaded, res.DBPruned, res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB)
for _, m := range res.WorldsMissing {
fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
}
for _, m := range res.ImagesIncomplete {
fmt.Fprintf(stderr, "felis offsite sync: registry image not whole: %s\n", m)
}
if err != nil {
fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
return 1
@@ -263,62 +226,8 @@ func reportRun(res offsite.Result, err error, stdout, stderr io.Writer) int {
return 0
}
// startRun begins a pass: its status record, in this release's format and
// carrying the last success over, and this host's lease, read from the record
// the last pass left.
func startRun(cfg config.OffsiteConfig, key []byte, statusFile string, now time.Time) (offsite.Status, offsite.Lease) {
st := offsite.Status{
LastAttempt: now.UTC(), Endpoint: cfg.Endpoint, Bucket: cfg.Bucket,
Prefix: cfg.Prefix, KeyID: offsite.KeyID(key), Format: offsite.StatusFormat,
}
if prev, _ := offsite.ReadStatus(statusFile); prev != nil {
st.LastSuccess = prev.LastSuccess
}
return st, offsite.HostLease(statusFile)
}
// recordRun puts one pass's outcome into its status record. A host that
// inherited the bucket from an older release keeps that until it has an id.
func recordRun(st *offsite.Status, res offsite.Result, err error, lease offsite.Lease) {
st.Result = res
if err != nil {
st.LastError = err.Error()
st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch)
var we *offsite.WriterError
if errors.As(err, &we) {
st.Standby = errors.Is(err, offsite.ErrStandby)
st.Displaced = errors.Is(err, offsite.ErrDisplaced)
st.Writer = we.Writer
}
} else {
st.LastSuccess = st.LastAttempt
}
if lease.Inherited {
id, _ := lease.ID()
st.Inherited = id == ""
}
}
// offsiteSources is where one sync pass reads from: the world archive volume
// (archiveDir, or the backupPVC's directory), the bundle directory (with the
// host state the pass bundles, stateDir), the registry's loopback endpoint and
// the uploads volume (uploadsDir, or the uploadsPVC's directory). An empty
// source is skipped.
type offsiteSources struct {
archiveDir, backupPVC string
dbDir, stateDir string
registry string
uploadsDir, uploadsPVC string
}
// offsiteRunLimit backstops one sync pass. Each upload has its own deadline,
// scaled to its size (internal/offsite), so a pass over a big archive may run
// for hours; the timer starts no second pass while one runs, and the unit's
// TimeoutStartSec sits above this.
const offsiteRunLimit = 23 * time.Hour
func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lease *offsite.Lease, log io.Writer) (offsite.Result, error) {
ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit)
func runOffsiteSync(cfg *config.Config, env *offsiteEnv, archiveDir, backupPVC, dbDir string, log io.Writer) (offsite.Result, error) {
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Minute)
defer cancel()
checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
err := env.bucket.Check(checkCtx)
@@ -326,117 +235,47 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lea
if err != nil {
return offsite.Result{}, err
}
archiveDir, uploadsDir := src.archiveDir, src.uploadsDir
if archiveDir == "" && src.backupPVC != "" {
dir, err := resolveVolumeDir(ctx, cfg.K8s.Namespace, src.backupPVC, archiveVolume, false, log)
if archiveDir == "" && backupPVC != "" {
dir, err := resolveArchiveDir(ctx, cfg.K8s.Namespace, backupPVC, false, log)
if err != nil {
return offsite.Result{}, err
}
archiveDir = dir
}
// An s3:// uploads store is off the host already; only a local one, on
// the uploads PVC, needs the copy.
if uploadsDir == "" && src.uploadsPVC != "" && isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
dir, err := resolveVolumeDir(ctx, platform.DefaultControlNamespace, src.uploadsPVC, uploadsVolume, false, log)
if err != nil {
return offsite.Result{}, err
}
uploadsDir = dir
}
drv, err := openStore(ctx, cfg.Database.URL, false)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
return offsite.Result{}, fmt.Errorf("open database: %w", err)
}
defer drv.Close()
s := offsiteSyncer(cfg, env, src, archiveDir, uploadsDir, lease, log)
s.Catalog = offsite.PGCatalog{DB: drv.DB()}
if src.registry != "" {
s.Images = newRegistryImages(src.registry)
s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL)
s := &offsite.Syncer{
Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key,
ArchiveDir: archiveDir, DBDir: dbDir, DBKeep: env.cfg.DBKeep, Log: log,
}
return s.Run(ctx)
}
// offsiteSyncer is the pass runOffsiteSync runs over the resolved archive and
// uploads directories, before its catalog and registry are attached. It
// snapshots the database into the bundle directory after copying archives, and
// sweeps world objects no backup records once they outlive every retention in
// [archive]; a retention that does not parse sweeps none.
func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, archiveDir, uploadsDir string, lease *offsite.Lease, log io.Writer) *offsite.Syncer {
s := &offsite.Syncer{
Bucket: env.bucket, Key: env.key,
ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log,
}
if src.dbDir != "" {
s.Snapshot = offsiteSnapshot(cfg.Database, src.dbDir, src.stateDir, log)
}
if rc, err := reaperConfig(cfg); err != nil {
fmt.Fprintf(log, "felis offsite: world objects no backup records are kept: %v\n", err)
} else {
s.OrphanAfter = max(rc.Retention, rc.ManualRetention, rc.ScheduledRetention)
}
return s
}
// offsiteSnapshot takes the bundle a pass sends after copying world archives
// (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite,
// with the newest one kept in dir. It is not recorded for the panel, whose
// backup card watches felis-db-backup.timer: snapshots come only when archives
// are copied, and would hide a daily timer that stopped. It requires the
// MinecraftServer objects: it becomes the newest bundle in the bucket, which a
// lost host restores from, and a pass that cannot take a whole one fails and
// tries again next hour.
func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error {
return func(ctx context.Context) error {
tools, err := dbTools(db)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
defer cancel()
path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite,
Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(),
ExportServers: exportMinecraftServers, RequireServers: true, Log: log,
})
if err == nil {
fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path))
}
return err
}
}
// volumeKind names a PVC the off-site copy reads or restores, for messages,
// with the flag that bypasses finding it through the cluster.
type volumeKind struct{ what, dirFlag, empty string }
var (
archiveVolume = volumeKind{"archive volume", "-archive-dir", "no world has been archived"}
uploadsVolume = volumeKind{"uploads volume", "-uploads-dir", "no modpack has been uploaded"}
)
// resolveVolumeDir finds the host directory behind a PVC: a local-path volume
// is a directory on this node. A PVC still waiting for its first consumer
// holds nothing yet: without bind that is "" (nothing to copy), with bind it
// is bound first, for a fetch to write into.
func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind, bind bool, log io.Writer) (string, error) {
// resolveArchiveDir finds the host directory behind the world archive PVC: a
// local-path volume is a directory on this node. A PVC still waiting for its
// first consumer holds nothing yet: without bind that is "" (no archives),
// with bind it is bound first, for fetch-worlds to write into.
func resolveArchiveDir(ctx context.Context, ns, pvcName string, bind bool, log io.Writer) (string, error) {
if ns == "" {
ns = platform.DefaultMinecraftNamespace
}
cl, err := buildSystemServerClient()
if err != nil {
return "", fmt.Errorf("reach the cluster to find the %s (or pass %s): %w", kind.what, kind.dirFlag, err)
return "", fmt.Errorf("reach the cluster to find the archive volume (or pass -archive-dir): %w", err)
}
var pvc corev1.PersistentVolumeClaim
if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
return "", fmt.Errorf("%s %s/%s: %w", kind.what, ns, pvcName, err)
return "", fmt.Errorf("archive volume %s/%s: %w", ns, pvcName, err)
}
if pvc.Spec.VolumeName == "" {
if !bind {
fmt.Fprintf(log, "felis offsite: %s %s/%s is not bound yet; %s\n", kind.what, ns, pvcName, kind.empty)
fmt.Fprintf(log, "felis offsite: archive volume %s/%s is not bound yet; no world has been archived\n", ns, pvcName)
return "", nil
}
if err := bindVolume(ctx, cl, ns, pvcName, kind, log); err != nil {
if err := bindVolume(ctx, cl, ns, pvcName, log); err != nil {
return "", err
}
if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
@@ -445,7 +284,7 @@ func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind,
}
var pv corev1.PersistentVolume
if err := cl.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil {
return "", fmt.Errorf("%s %s: %w", kind.what, pvc.Spec.VolumeName, err)
return "", fmt.Errorf("archive volume %s: %w", pvc.Spec.VolumeName, err)
}
var dir string
switch {
@@ -454,10 +293,10 @@ func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind,
case pv.Spec.HostPath != nil:
dir = pv.Spec.HostPath.Path
default:
return "", fmt.Errorf("%s %s is not a directory on a node (local or hostPath); pass %s with where it is mounted on this host", kind.what, pv.Name, kind.dirFlag)
return "", fmt.Errorf("archive volume %s is not a directory on a node (local or hostPath); pass -archive-dir with where it is mounted on this host", pv.Name)
}
if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
return "", fmt.Errorf("%s %s is %s on its node, which is not a directory here; run this on the node that holds it, or pass %s", kind.what, pv.Name, dir, kind.dirFlag)
return "", fmt.Errorf("archive volume %s is %s on its node, which is not a directory here; run this on the node that holds it, or pass -archive-dir", pv.Name, dir)
}
return dir, nil
}
@@ -465,10 +304,10 @@ func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind,
// bindVolume runs a pod that mounts the PVC and exits, which is what makes a
// WaitForFirstConsumer volume (k3s local-path) get provisioned. The pod uses
// the control plane's own image, which every install already has.
func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind volumeKind, log io.Writer) error {
func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, log io.Writer) error {
var api appsv1.Deployment
if err := cl.Get(ctx, types.NamespacedName{Namespace: platform.DefaultControlNamespace, Name: "felis-api"}, &api); err != nil {
return fmt.Errorf("find the felis image to bind the %s with: %w", kind.what, err)
return fmt.Errorf("find the felis image to bind the archive volume with: %w", err)
}
if len(api.Spec.Template.Spec.Containers) == 0 {
return errors.New("felis-api has no container to take the image from")
@@ -476,9 +315,9 @@ func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind
image := api.Spec.Template.Spec.Containers[0].Image
pod := platform.VolumeBinderPod(ns, pvcName, image)
if err := cl.Create(ctx, pod); err != nil {
return fmt.Errorf("start a pod to bind the %s: %w", kind.what, err)
return fmt.Errorf("start a pod to bind the archive volume: %w", err)
}
fmt.Fprintf(log, "felis offsite: binding the %s %s/%s (pod %s)\n", kind.what, ns, pvcName, pod.Name)
fmt.Fprintf(log, "felis offsite: binding the archive volume %s/%s (pod %s)\n", ns, pvcName, pod.Name)
defer func() {
_ = cl.Delete(context.Background(), pod, client.PropagationPolicy(metav1.DeletePropagationBackground))
}()
@@ -494,7 +333,7 @@ func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind
case <-time.After(2 * time.Second):
}
}
return fmt.Errorf("the %s %s/%s did not bind within 3 minutes; see kubectl -n %s describe pod %s", kind.what, ns, pvcName, ns, pod.Name)
return fmt.Errorf("the archive volume %s/%s did not bind within 3 minutes; see kubectl -n %s describe pod %s", ns, pvcName, ns, pod.Name)
}
func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
@@ -509,7 +348,7 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
return 1
}
if !cfg.Offsite.Enabled() {
fmt.Fprintln(stdout, "off-site copy: not configured. World archives, database bundles, user images and uploaded modpacks exist on this machine only.")
fmt.Fprintln(stdout, "off-site copy: not configured. World archives and database bundles exist on this machine only.")
fmt.Fprintln(stdout, "See docs/troubleshooting.md §16, \"Keep a copy somewhere else\".")
return 1
}
@@ -539,49 +378,13 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
if st.LastError != "" {
fmt.Fprintf(stdout, "last error: %s\n", st.LastError)
}
if st.KeyMismatch {
fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile)
return 1
}
if st.Displaced && st.Writer != nil {
fmt.Fprintf(stdout, "\nThe last run was refused: %s took the bucket over (it last wrote it at %s), and this host copies nothing there any more. If that host is a rehearsal machine, take the bucket back: sudo felis offsite take-over -yes\n",
st.Writer, st.Writer.At.Local().Format(time.DateTime))
return 1
}
if st.Standby {
switch w := st.StandsBy(now); {
case w != nil:
fmt.Fprintf(stdout, "\nThis host stands by: %s writes the bucket (last at %s). This host was built from its backup, copies nothing into the bucket and, while that host keeps writing, mails no watchdog alert.", w, w.At.Local().Format(time.DateTime))
case st.Writer != nil:
fmt.Fprintf(stdout, "\nThis host copies nothing into the bucket: %s wrote it, last at %s, and this host was built from its backup.", st.Writer, st.Writer.At.Local().Format(time.DateTime))
default:
fmt.Fprint(stdout, "\nThis host copies nothing into the bucket: it holds copies this host did not write, and names no host writing it.")
}
fmt.Fprintln(stdout, " Once this host replaces that one for good: sudo felis offsite take-over -yes")
return 1
}
r := st.Result
fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
if r.ImageIndex != "" {
fmt.Fprintf(stdout, "images: %d in %d repositories (%s), registry index %s\n",
r.Images, r.ImageRepos, offsite.HumanBytes(r.RemoteImageBytes), r.ImageIndex)
} else {
fmt.Fprintln(stdout, "images: not copied (no in-cluster registry, or no sync has reached it yet)")
}
if r.UploadIndex != "" {
fmt.Fprintf(stdout, "uploads: %d submission contexts (%s), uploads index %s\n",
r.Uploads, offsite.HumanBytes(r.RemoteUploadBytes), r.UploadIndex)
} else {
fmt.Fprintln(stdout, "uploads: not copied (an s3:// uploads store, or no sync has reached the volume yet)")
}
fmt.Fprintf(stdout, "waiting: %d world archives not yet copied\n", r.WorldsPending)
for _, m := range r.WorldsMissing {
fmt.Fprintf(stdout, "missing: %s is recorded but not on the volume\n", m)
}
for _, m := range r.ImagesIncomplete {
fmt.Fprintf(stdout, "not whole: %s\n", m)
}
if st.LastSuccess.IsZero() || now.Sub(st.LastSuccess) > offsite.StaleAfter {
fmt.Fprintf(stdout, "\nThe last successful sync is older than %s: journalctl -u felis-offsite -n 50\n", dbbackup.Age(offsite.StaleAfter))
return 1
@@ -623,201 +426,18 @@ func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
return 1
}
printDBBundles(ctx, env.bucket, env.key, bundles, stdout)
fmt.Fprintf(stdout, "database bundles (%d, newest first):\n", len(bundles))
for _, b := range bundles {
fmt.Fprintf(stdout, " %s %s\n", b.Key, offsite.HumanBytes(b.Size))
}
var total int64
for _, w := range worlds {
total += w.Size
}
fmt.Fprintf(stdout, "world archives: %d (%s)\n", len(worlds), offsite.HumanBytes(total))
versions, err := offsite.ImageIndexes(ctx, env.bucket)
if err != nil {
fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "registry index versions (%d, newest first; restore one with fetch-images -at):\n", len(versions))
for i := len(versions) - 1; i >= 0; i-- {
x, err := offsite.LoadImageIndex(ctx, env.bucket, env.key, versions[i])
if err != nil {
fmt.Fprintf(stdout, " %s unreadable: %v\n", versions[i], err)
continue
}
fmt.Fprintf(stdout, " %s %d images in %d repositories\n", versions[i], x.Images(), len(x.Repositories))
}
uploads, err := offsite.UploadIndexes(ctx, env.bucket)
if err != nil {
fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "uploads index versions (%d, newest first; restore one with fetch-uploads -at):\n", len(uploads))
for i := len(uploads) - 1; i >= 0; i-- {
x, err := offsite.LoadUploadIndex(ctx, env.bucket, env.key, uploads[i])
if err != nil {
fmt.Fprintf(stdout, " %s unreadable: %v\n", uploads[i], err)
continue
}
fmt.Fprintf(stdout, " %s %d submission contexts (%s)\n", uploads[i], len(x.Contexts), offsite.HumanBytes(x.Bytes()))
}
return 0
}
// printDBBundles lists the database bundles with what each one's database
// held, read off the front of each, so a restore can pick one by its contents.
func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles []offsite.Object, stdout io.Writer) {
fmt.Fprintf(stdout, "database bundles (%d, newest first; restore one with fetch-db):\n", len(bundles))
for _, o := range bundles {
m, err := offsite.PeekDB(ctx, b, key, o.Key)
if err != nil {
fmt.Fprintf(stdout, " %s %s unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err)
continue
}
gap := ""
if m.ServersError != "" {
gap = ", no MinecraftServer objects"
}
fmt.Fprintf(stdout, " %s %s %s%s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String(), gap)
}
}
func offsiteCheckKey(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
if err := fs.Parse(args); err != nil {
return 2
}
_, env, err := loadOffsite(*cfgPath, *envFile)
if err != nil {
fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if err := env.bucket.Check(ctx); err != nil {
fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
return 1
}
return checkKey(ctx, env.bucket, env.key, stdout, stderr)
}
// checkKey is check-key once the bucket is open: 0 when the key fits, 3 when
// the bucket's objects are sealed with another one, 1 when it cannot tell.
func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr io.Writer) int {
fit, err := offsite.CheckKey(ctx, b, key)
if err != nil {
fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
if errors.Is(err, offsite.ErrKeyMismatch) {
return 3
}
return 1
}
id := offsite.KeyID(key)
switch fit {
case offsite.KeyRecorded:
fmt.Fprintf(stdout, "felis offsite check-key: the bucket records key id %s, this key's\n", id)
case offsite.KeyOpens:
fmt.Fprintf(stdout, "felis offsite check-key: the bucket's newest objects open with this key (key id %s); the next sync records it\n", id)
case offsite.KeyUnused:
fmt.Fprintf(stdout, "felis offsite check-key: the bucket holds no sealed object yet; the first sync records key id %s\n", id)
}
return 0
}
func offsiteTakeOver(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes; this host's id is kept next to it")
yes := fs.Bool("yes", false, "make this host the one that writes the bucket")
if err := fs.Parse(args); err != nil {
return 2
}
_, env, err := loadOffsite(*cfgPath, *envFile)
if err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if err := env.bucket.Check(ctx); err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
return 1
}
return takeOver(ctx, env.bucket, env.key, offsite.HostLease(*statusFile), *statusFile, *yes, time.Now(), stdout, stderr)
}
// takeOver is take-over once the bucket is open: without yes it says which
// host writes the bucket, 0 for this one (or none yet), 4 for another and 5
// for one that took the bucket over from this host; with
// yes it records this host as the writer. A key the bucket's objects refuse
// is 3, as in check-key: taking over a bucket this host cannot copy into
// would only stop the host that can.
func takeOver(ctx context.Context, b offsite.Bucket, key []byte, lease offsite.Lease, statusFile string, yes bool, now time.Time, stdout, stderr io.Writer) int {
fit, err := offsite.CheckKey(ctx, b, key)
if err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
if errors.Is(err, offsite.ErrKeyMismatch) {
return 3
}
return 1
}
role, w, err := lease.Plan(ctx, b, fit == offsite.KeyUnused)
if err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
return 1
}
switch role {
case offsite.RoleWrites:
id, _ := lease.ID()
fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket; nothing to take over\n", id)
return 0
case offsite.RoleClaims:
fmt.Fprintln(stdout, "felis offsite take-over: the bucket names no host writing it; this host's next sync records itself")
return 0
}
who := "another host"
if w != nil {
who = w.String()
fmt.Fprintf(stdout, "felis offsite take-over: %s writes the bucket, last at %s (%s ago)\n", w, w.At.Local().Format(time.DateTime), dbbackup.Age(now.Sub(w.At)))
} else {
fmt.Fprintln(stdout, "felis offsite take-over: the bucket holds copies this host did not write, and names no host writing it")
}
if !yes {
if role == offsite.RoleDisplaced {
fmt.Fprintf(stdout, "It took the bucket over from this host: this host copies nothing there any more, and its watchdog mails the owners about it. If that host is a rehearsal machine, take the bucket back:\n sudo felis offsite take-over -yes\n")
return 5
}
fmt.Fprintf(stdout, "This host was built from its backup and copies nothing into the bucket.\n")
fmt.Fprintf(stdout, "Taking it over makes this host the one that copies into the bucket and prunes it; %s stops at its next copy and mails its owners. Do it once that host is gone for good, or is a rehearsal machine you are done with:\n sudo felis offsite take-over -yes\n", who)
return 4
}
if _, err := lease.TakeOver(ctx, b, now); err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
return 1
}
// The refusal the last sync recorded is over: the watchdog mails again
// from now on, and status shows the next run's outcome.
if st, err := offsite.ReadStatus(statusFile); err == nil && st != nil && (st.Standby || st.Displaced) {
st.Standby, st.Displaced, st.Writer, st.LastError, st.Inherited = false, false, nil, "", false
if err := offsite.WriteStatus(statusFile, *st); err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: record status: %v\n", err)
}
}
id, _ := lease.ID()
fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket now; %s stops at its next copy.\nStart the first copy: sudo systemctl start felis-offsite.service\n", id, who)
return 0
}
// keyHint explains an object the key cannot open when the bucket records
// another key's id, "" otherwise.
func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string {
if !errors.Is(err, offsite.ErrAuth) {
return ""
}
id, ierr := offsite.BucketKeyID(ctx, b)
if ierr != nil || id == "" || id == offsite.KeyID(key) {
return ""
}
return fmt.Sprintf("\n the bucket records key id %s, and this key is %s: set FELIS_OFFSITE_KEY to the key the bucket was written with", id, offsite.KeyID(key))
}
func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
@@ -860,47 +480,37 @@ func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) i
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
return fetchDB(ctx, env.bucket, env.key, arg, *dir, time.Now(), stdout, stderr)
}
// fetchDB is fetch-db once the bucket is open: arg is a bundle name or latest.
func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string, now time.Time, stdout, stderr io.Writer) int {
name := arg
if name == "latest" {
var err error
if name, _, err = offsite.ChooseDB(ctx, b, key); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
bundles, err := offsite.ListDB(ctx, env.bucket)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
return 1
}
if len(bundles) == 0 {
fmt.Fprintln(stderr, "felis offsite fetch-db: the bucket holds no database bundle")
return 1
}
name = bundles[0].Key
}
if _, _, ok := dbbackup.ParseBundleName(name); !ok {
fmt.Fprintf(stderr, "felis offsite fetch-db: %q is not a bundle name (felis-db-<stamp>-<label>.tar); see `felis offsite list`\n", name)
return 2
}
if err := os.MkdirAll(dir, 0o700); err != nil {
if err := os.MkdirAll(*dir, 0o700); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
return 1
}
dst := filepath.Join(dir, name)
if err := offsite.FetchObject(ctx, b, key, offsite.DBKey(name), dst, 0o600); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
dst := filepath.Join(*dir, name)
if err := offsite.FetchObject(ctx, env.bucket, env.key, offsite.DBKey(name), dst, 0o600); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
return 1
}
m, err := dbbackup.Verify(dst)
if err != nil {
if _, err := dbbackup.Verify(dst); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: fetched %s but it does not verify: %v\n", dst, err)
return 1
}
fmt.Fprintf(stdout, "felis offsite fetch-db: wrote %s (verified)\n", dst)
fmt.Fprintf(stdout, " taken %s (%s, %s ago)\n felis %s, schema %d\n holds %s\n",
m.CreatedAt.Format(time.RFC3339), m.Label, dbbackup.Age(now.Sub(m.CreatedAt)),
orUnknown(m.FelisVersion), m.SchemaVersion, m.Counts.String())
if m.Counts.Fresh() {
fmt.Fprintln(stdout, " This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.")
}
if m.ServersError != "" {
fmt.Fprintf(stdout, " This bundle lacks the MinecraftServer objects (%s): `felis db restore` brings back the database, and the servers come from k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap.\n", m.ServersError)
}
return 0
}
@@ -921,12 +531,12 @@ func offsiteFetchWorlds(fs *flag.FlagSet, args []string, stdout, stderr io.Write
defer cancel()
dir := *archiveDir
if dir == "" {
if dir, err = resolveVolumeDir(ctx, cfg.K8s.Namespace, *backupPVC, archiveVolume, true, stderr); err != nil {
if dir, err = resolveArchiveDir(ctx, cfg.K8s.Namespace, *backupPVC, true, stderr); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
return 1
}
}
drv, err := openStore(ctx, cfg.Database.URL, false)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-worlds: open database: %v\n", err)
return 1
cmd/felis/offsite_images.go
-213
View File
@@ -1,213 +0,0 @@
package main
import (
"context"
"database/sql"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"slices"
"strings"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/imagepush"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/registrygate"
"felis.lolicon.best/internal/registryprune"
)
// registryImages is the platform registry as the off-site copy sees it: read
// anonymously through the gate (the catalog, its manifest index, manifests and
// blobs) and, for a restore, written as the platform principal. Both go through
// the node's loopback hostPort, the way the installer pushes.
type registryImages struct {
host string
index *registryprune.Client
source *imagepush.Source
pusher *imagepush.Pusher
}
func newRegistryImages(endpoint string) *registryImages {
return &registryImages{
host: endpoint,
index: &registryprune.Client{Endpoint: "http://" + endpoint},
source: &imagepush.Source{Scheme: "http"},
}
}
func (r *registryImages) Repositories(ctx context.Context) ([]string, error) {
return r.index.Repositories(ctx)
}
func (r *registryImages) Revisions(ctx context.Context, repo string) ([]string, map[string]string, error) {
idx, err := r.index.Index(ctx, repo)
if err != nil {
return nil, nil, err
}
digests := make([]string, 0, len(idx.Revisions))
for _, rev := range idx.Revisions {
digests = append(digests, rev.Digest)
}
return digests, idx.Tags, nil
}
func (r *registryImages) Manifest(ctx context.Context, repo, digest string) ([]byte, string, error) {
body, mt, err := r.source.Manifest(ctx, r.host, repo, digest)
return body, mt, registryGone(err)
}
func (r *registryImages) Blob(ctx context.Context, repo, digest string) (io.ReadCloser, error) {
rc, err := r.source.Blob(ctx, r.host, repo, digest)
return rc, registryGone(err)
}
func (r *registryImages) PutBlob(ctx context.Context, repo, digest string, size int64, open func() (io.ReadCloser, error)) error {
return r.pusher.UploadBlob(ctx, r.host, repo, digest, size, open)
}
func (r *registryImages) PutManifest(ctx context.Context, repo, reference, mediaType string, body []byte) error {
_, err := r.pusher.PutManifest(ctx, r.host, repo, reference, mediaType, body)
return err
}
// imagePins lists, per repository of the registry refs spell as host, the
// digests the MinecraftServers' specs and the image whitelist pin: what a
// restored database and its servers will ask the registry for.
func imagePins(db *sql.DB, host string) func(ctx context.Context) (map[string][]string, error) {
return func(ctx context.Context) (map[string][]string, error) {
cl, err := buildSystemServerClient()
if err != nil {
return nil, fmt.Errorf("reach the cluster: %w", err)
}
var servers v1alpha1.MinecraftServerList
if err := cl.List(ctx, &servers); err != nil {
return nil, fmt.Errorf("list MinecraftServers: %w", err)
}
refs := make([]string, 0, len(servers.Items))
for _, s := range servers.Items {
refs = append(refs, s.Spec.Image)
}
rows, err := db.QueryContext(ctx, `SELECT image_ref FROM image_whitelist`)
if err != nil {
return nil, fmt.Errorf("read the image whitelist: %w", err)
}
defer rows.Close()
for rows.Next() {
var ref string
if err := rows.Scan(&ref); err != nil {
return nil, fmt.Errorf("read the image whitelist: %w", err)
}
refs = append(refs, ref)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read the image whitelist: %w", err)
}
pins := map[string][]string{}
for _, ref := range refs {
repo, _, digest, ok := registryprune.ParseRef(ref, host)
if ok && digest != "" && !slices.Contains(pins[repo], digest) {
pins[repo] = append(pins[repo], digest)
}
}
return pins, nil
}
}
// registryGone marks a 404 as a manifest or blob the registry no longer holds.
func registryGone(err error) error {
var se *imagepush.StatusError
if errors.As(err, &se) && se.Code == http.StatusNotFound {
return fmt.Errorf("%w: %v", offsite.ErrImageGone, err)
}
return err
}
// offsiteRegistryEndpoint is where the host reaches the registry whose images
// the off-site copy covers: flag when given ("off" for none), otherwise the
// loopback hostPort of the in-cluster registry [registry] url names. A
// registry outside the cluster is not this install's to copy.
func offsiteRegistryEndpoint(flag string, reg config.RegistryConfig) string {
switch flag {
case "off":
return ""
case "":
default:
return flag
}
host, _, _ := strings.Cut(reg.URL, "/")
name, _, _ := strings.Cut(host, ":")
if strings.HasSuffix(name, ".svc") || strings.HasSuffix(name, ".svc.cluster.local") {
return loopbackEndpoint(host)
}
return ""
}
// registryWriteCredential is the credential a host-side push uses:
// FELIS_REGISTRY_USERNAME/PASSWORD when set, otherwise the platform principal
// with REGISTRY_PLATFORM_TOKEN from the installer's secrets file.
func registryWriteCredential(secrets string) (string, string, error) {
if err := loadEnvFile(secrets); err != nil {
return "", "", fmt.Errorf("read %s: %w", secrets, err)
}
user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
if pass == "" {
user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
}
if pass == "" {
return "", "", errors.New("no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
}
return user, pass, nil
}
func offsiteFetchImages(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
registry := fs.String("registry", "", "host[:port] of the registry to push into (default: the loopback hostPort of the in-cluster registry)")
secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
at := fs.String("at", "", "registry index version to restore (default: the newest; `felis offsite list` shows them)")
if err := fs.Parse(args); err != nil {
return 2
}
cfg, env, err := loadOffsite(*cfgPath, *envFile)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
return 1
}
endpoint := offsiteRegistryEndpoint(*registry, cfg.Registry)
if endpoint == "" {
fmt.Fprintf(stderr, "felis offsite fetch-images: [registry] url %q is not the in-cluster registry; pass -registry host:port\n", cfg.Registry.URL)
return 2
}
user, pass, err := registryWriteCredential(*secrets)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
return 2
}
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
defer cancel()
stamp, idx, err := offsite.ChooseImageIndex(ctx, env.bucket, env.key, *at)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "felis offsite fetch-images: restoring registry index %s (%d repositories, %d images) into %s\n",
stamp, len(idx.Repositories), idx.Images(), endpoint)
target := newRegistryImages(endpoint)
target.pusher = &imagepush.Pusher{Scheme: "http", Username: user, Password: pass}
res, err := offsite.FetchImages(ctx, env.bucket, env.key, idx, target, stderr)
fmt.Fprintf(stdout, "felis offsite fetch-images: %d of %d repositories restored, %d images, %d tags, %d blobs pushed (%s)\n",
res.Repositories, len(idx.Repositories), res.Manifests, res.Tags, res.BlobsPushed, offsite.HumanBytes(res.BytesPushed))
for _, f := range res.Failures {
fmt.Fprintf(stderr, "felis offsite fetch-images: %s\n", f)
}
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-images: %v (a second run pushes only what is still missing)\n", err)
return 1
}
return 0
}
cmd/felis/offsite_test.go
-679
View File
@@ -1,25 +1,13 @@
package main
import (
"archive/tar"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/imagepush"
"felis.lolicon.best/internal/offsite"
)
@@ -106,670 +94,3 @@ func TestOffsiteFetchDBRejectsOddNames(t *testing.T) {
t.Fatalf("exit %d: %s", code, errb.String())
}
}
func TestOffsiteRegistryEndpoint(t *testing.T) {
for _, tc := range []struct{ flag, url, want string }{
{"", "registry.felis.svc:5000", "127.0.0.1:5000"},
{"", "registry.felis.svc.cluster.local:5001", "127.0.0.1:5001"},
{"", "ghcr.io/acme", ""},
{"", "", ""},
{"off", "registry.felis.svc:5000", ""},
{"10.0.0.5:5000", "ghcr.io/acme", "10.0.0.5:5000"},
} {
if got := offsiteRegistryEndpoint(tc.flag, config.RegistryConfig{URL: tc.url}); got != tc.want {
t.Errorf("offsiteRegistryEndpoint(%q, %q) = %q, want %q", tc.flag, tc.url, got, tc.want)
}
}
}
func TestRegistryGoneMarksNotFound(t *testing.T) {
if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 404}); !errors.Is(err, offsite.ErrImageGone) {
t.Fatalf("404 = %v, want ErrImageGone", err)
}
if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 503}); errors.Is(err, offsite.ErrImageGone) {
t.Fatalf("503 = %v, want it kept an ordinary failure", err)
}
}
// mapBucket is an in-memory offsite.Bucket.
type mapBucket map[string][]byte
func (b mapBucket) Put(_ context.Context, key string, r io.Reader, _ int64) error {
data, err := io.ReadAll(r)
b[key] = data
return err
}
func (b mapBucket) Get(_ context.Context, key string) (io.ReadCloser, error) {
data, ok := b[key]
if !ok {
return nil, offsite.ErrNotFound
}
return io.NopCloser(bytes.NewReader(data)), nil
}
func (b mapBucket) List(_ context.Context, prefix string) ([]offsite.Object, error) {
var out []offsite.Object
for k, v := range b {
if strings.HasPrefix(k, prefix) {
out = append(out, offsite.Object{Key: k, Size: int64(len(v))})
}
}
return out, nil
}
func (b mapBucket) Remove(_ context.Context, key string) error {
delete(b, key)
return nil
}
var fetchT0 = time.Date(2026, 9, 20, 3, 30, 0, 0, time.UTC)
// putBundle seals a bundle that verifies, taken daysAgo days before fetchT0,
// into b and returns its name.
func putBundle(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts) string {
t.Helper()
return putBundleWith(t, b, key, daysAgo, counts, "")
}
// putBundleWith is putBundle for a bundle whose server export failed with
// serversError, when that is not empty.
func putBundleWith(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts, serversError string) string {
t.Helper()
created := fetchT0.AddDate(0, 0, -daysAgo)
dump := []byte("PGDMP " + created.String())
sum := sha256.Sum256(dump)
manifest, err := json.Marshal(dbbackup.Manifest{
Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, ServersError: serversError,
Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}},
})
if err != nil {
t.Fatal(err)
}
var plain bytes.Buffer
tw := tar.NewWriter(&plain)
for _, f := range []struct {
name string
data []byte
}{{"MANIFEST.json", manifest}, {"db.dump", dump}} {
if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o600, Size: int64(len(f.data)), Typeflag: tar.TypeReg}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write(f.data); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
var sealed bytes.Buffer
if err := offsite.Encrypt(&sealed, &plain, key); err != nil {
t.Fatal(err)
}
name := dbbackup.BundleName(created, dbbackup.LabelDaily)
b[offsite.DBKey(name)] = sealed.Bytes()
return name
}
func TestOffsiteFetchDB(t *testing.T) {
rawKey, _ := offsite.NewKey()
key, _ := offsite.ParseKey(rawKey)
now := fetchT0.Add(2 * time.Hour)
fetch := func(b mapBucket, arg string) (dir string, code int, stdout, stderr string) {
dir = t.TempDir()
var out, errb bytes.Buffer
code = fetchDB(context.Background(), b, key, arg, dir, now, &out, &errb)
return dir, code, out.String(), errb.String()
}
fetched := func(t *testing.T, dir string) []string {
t.Helper()
var names []string
entries, _ := os.ReadDir(dir)
for _, e := range entries {
names = append(names, e.Name())
}
return names
}
t.Run("latest skips a rebuilt host's empty bundle", func(t *testing.T) {
b := mapBucket{}
full := putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
dir, code, out, errb := fetch(b, "latest")
if code != 1 || !strings.Contains(errb, empty) || !strings.Contains(errb, full+" (5 accounts, 3 servers)") {
t.Fatalf("exit %d, stdout %q, stderr %q; want a refusal naming %s", code, out, errb, full)
}
if got := fetched(t, dir); len(got) != 0 {
t.Errorf("a refused fetch wrote %v", got)
}
})
t.Run("latest takes the newest bundle and says what it holds", func(t *testing.T) {
b := mapBucket{}
putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 2})
newest := putBundle(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3})
dir, code, out, errb := fetch(b, "latest")
if code != 0 {
t.Fatalf("exit %d: %s", code, errb)
}
if got := fetched(t, dir); !slices.Equal(got, []string{newest}) {
t.Errorf("wrote %v, want %s", got, newest)
}
for _, want := range []string{
"wrote " + filepath.Join(dir, newest) + " (verified)",
"taken 2026-09-19T03:30:00Z (daily, 26h0m ago)",
"felis v1.2.3, schema 21",
"holds 5 accounts, 3 servers",
} {
if !strings.Contains(out, want) {
t.Errorf("stdout lacks %q:\n%s", want, out)
}
}
if strings.Contains(out, "new install") {
t.Errorf("a bundle with servers flagged as a new install's:\n%s", out)
}
})
t.Run("an empty bundle named outright is fetched with a warning", func(t *testing.T) {
b := mapBucket{}
putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
empty := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 1})
dir, code, out, errb := fetch(b, empty)
if code != 0 || !slices.Equal(fetched(t, dir), []string{empty}) {
t.Fatalf("exit %d, wrote %v: %s", code, fetched(t, dir), errb)
}
if !strings.Contains(out, "holds 1 account, 0 servers") || !strings.Contains(out, "like a new install's") {
t.Errorf("stdout = %s", out)
}
})
t.Run("a bundle without the servers says where they come from", func(t *testing.T) {
b := mapBucket{}
gapped := putBundleWith(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused (tried 3 times)")
_, code, out, errb := fetch(b, gapped)
if code != 0 || !strings.Contains(out, "This bundle lacks the MinecraftServer objects (connection refused (tried 3 times))") ||
!strings.Contains(out, "k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap") {
t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
}
whole := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
if _, _, out, _ := fetch(b, whole); strings.Contains(out, "lacks the MinecraftServer objects") {
t.Errorf("a whole bundle flagged:\n%s", out)
}
})
t.Run("a bundle from before counts says so", func(t *testing.T) {
b := mapBucket{}
old := putBundle(t, b, key, 0, nil)
_, code, out, errb := fetch(b, "latest")
if code != 0 || !strings.Contains(out, old) || !strings.Contains(out, "holds not recorded") || strings.Contains(out, "new install") {
t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
}
})
}
func TestOffsiteCheckKey(t *testing.T) {
newKey := func() []byte {
raw, _ := offsite.NewKey()
k, _ := offsite.ParseKey(raw)
return k
}
key, other := newKey(), newKey()
marked := func(k []byte) mapBucket { return mapBucket{"felis-key-id": []byte(offsite.KeyID(k) + "\n")} }
unmarked := func(k []byte) mapBucket {
b := mapBucket{}
putBundle(t, b, k, 0, nil)
return b
}
for _, tc := range []struct {
what string
bucket mapBucket
code int
says []string
}{
{"the recorded key", marked(key), 0, []string{"records key id " + offsite.KeyID(key)}},
{"an unmarked bucket the key opens", unmarked(key), 0, []string{"open with this key", "the next sync records it"}},
{"an empty bucket", mapBucket{}, 0, []string{"no sealed object yet", "records key id " + offsite.KeyID(key)}},
{"another recorded key", marked(other), 3, []string{offsite.KeyID(other), offsite.KeyID(key), "FELIS_OFFSITE_KEY"}},
{"an unmarked bucket under another key", unmarked(other), 3, []string{"opens none of db/felis-db-"}},
{"a marker Felis did not write", mapBucket{"felis-key-id": []byte("hello")}, 1, []string{"not a key id"}},
} {
t.Run(tc.what, func(t *testing.T) {
before := len(tc.bucket)
var out, errb bytes.Buffer
code := checkKey(context.Background(), tc.bucket, key, &out, &errb)
if code != tc.code {
t.Fatalf("exit %d, want %d; stdout %q, stderr %q", code, tc.code, out.String(), errb.String())
}
said := out.String() + errb.String()
for _, s := range tc.says {
if !strings.Contains(said, s) {
t.Errorf("output lacks %q: %s", s, said)
}
}
if len(tc.bucket) != before {
t.Errorf("check-key wrote to the bucket: %d objects, had %d", len(tc.bucket), before)
}
})
}
// fetch-db names both ids when the bucket records another key.
b := marked(other)
name := putBundle(t, b, other, 0, &dbbackup.Counts{Users: 5, Servers: 3})
for _, arg := range []string{"latest", name} {
var out, errb bytes.Buffer
if code := fetchDB(context.Background(), b, key, arg, t.TempDir(), fetchT0, &out, &errb); code != 1 ||
!strings.Contains(errb.String(), "the bucket records key id "+offsite.KeyID(other)+", and this key is "+offsite.KeyID(key)) {
t.Errorf("fetch-db %s under another key: exit %d, stderr %q", arg, code, errb.String())
}
}
// A bundle the bucket lacks is not the key's fault.
var missOut, missErr bytes.Buffer
if code := fetchDB(context.Background(), b, key, "felis-db-20200101T000000Z-daily.tar", t.TempDir(), fetchT0, &missOut, &missErr); code != 1 || strings.Contains(missErr.String(), "records key id") {
t.Errorf("missing bundle: exit %d, stderr %q", code, missErr.String())
}
// A bundle damaged under the recorded key gets no such hint.
b = marked(key)
name = putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
b[offsite.DBKey(name)][60] ^= 1
var out, errb bytes.Buffer
if code := fetchDB(context.Background(), b, key, name, t.TempDir(), fetchT0, &out, &errb); code != 1 || strings.Contains(errb.String(), "records key id") {
t.Errorf("damaged bundle: exit %d, stderr %q", code, errb.String())
}
}
func TestRecordRun(t *testing.T) {
t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}
for _, tc := range []struct {
err error
mismatch, standby, displaced, success bool
}{
{nil, false, false, false, true},
{errors.New("list worlds/ in the bucket: connection reset"), false, false, false, false},
{fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false, false, false},
{&offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, false, true, false, false},
{&offsite.WriterError{Kind: offsite.ErrDisplaced, Writer: w}, false, false, true, false},
} {
st := offsite.Status{LastAttempt: t0}
recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err, offsite.Lease{})
if st.KeyMismatch != tc.mismatch || st.Standby != tc.standby || st.Displaced != tc.displaced ||
(st.Writer != nil) != (tc.standby || tc.displaced) || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 {
t.Errorf("err %v: status %+v", tc.err, st)
}
}
// A host that copied before writers were recorded keeps that claim over
// failed runs until it has an id.
l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Inherited: true}
st := offsite.Status{}
recordRun(&st, offsite.Result{}, errors.New("cannot reach bucket"), l)
if !st.Inherited {
t.Error("a failed run on a host with no id dropped the older release's claim")
}
writeTestFile(t, l.IDFile, "aaaaaaaaaaaaaaaa\n", 0o600)
st = offsite.Status{Inherited: true}
recordRun(&st, offsite.Result{}, nil, l)
if st.Inherited {
t.Error("a host with an id still carries the older release's claim")
}
}
// A refused run has copied nothing and listed nothing: its zero counts would
// tell the journal the bucket is empty. A pass that ran and failed a step
// shows what it did get to.
func TestReportRun(t *testing.T) {
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)}
for _, tc := range []struct {
name string
res offsite.Result
err error
code int
counts bool
}{
{"a pass", offsite.Result{DBUploaded: 1, RemoteDB: 3}, nil, 0, true},
{"a pass with a failed step", offsite.Result{RemoteDB: 3, Errors: []string{"copy db/x: timeout"}}, errors.New("1 of this run's steps failed; first: copy db/x: timeout"), 1, true},
{"standing by", offsite.Result{}, &offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, 1, false},
{"another key", offsite.Result{}, fmt.Errorf("%w: the bucket records key id 1111111111111111", offsite.ErrKeyMismatch), 1, false},
{"no bucket", offsite.Result{}, errors.New("bucket: access denied"), 1, false},
} {
t.Run(tc.name, func(t *testing.T) {
var out, errOut bytes.Buffer
code := reportRun(tc.res, tc.err, &out, &errOut)
if code != tc.code {
t.Errorf("exit %d, want %d", code, tc.code)
}
if got := strings.Contains(out.String(), "bucket holds 0 worlds (0 B), 3 bundles"); got != tc.counts {
t.Errorf("counts shown = %v, want %v: %q", got, tc.counts, out.String())
}
if !tc.counts && out.Len() > 0 {
t.Errorf("a refused run printed %q", out.String())
}
if tc.err != nil && !strings.Contains(errOut.String(), "felis offsite sync: "+tc.err.Error()) {
t.Errorf("stderr %q lacks the error", errOut.String())
}
})
}
}
func TestOffsiteTakeOver(t *testing.T) {
newKey := func() []byte {
raw, _ := offsite.NewKey()
k, _ := offsite.ParseKey(raw)
return k
}
key, other := newKey(), newKey()
const mine, theirs = "aaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbb"
t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
sealed := func(k []byte, writer string) mapBucket {
b := mapBucket{}
putBundle(t, b, k, 0, nil)
b["felis-key-id"] = []byte(offsite.KeyID(k) + "\n")
if writer != "" {
raw, _ := json.Marshal(offsite.Writer{HostID: writer, Host: "prod-1", At: t0.Add(-20 * time.Minute)})
b["felis-writer"] = raw
}
return b
}
lease := func(id string) offsite.Lease {
l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Host: "spare-1"}
if id != "" {
writeTestFile(t, l.IDFile, id+"\n", 0o600)
}
return l
}
run := func(b mapBucket, l offsite.Lease, statusFile string, yes bool) (int, string, string) {
t.Helper()
var out, errb bytes.Buffer
code := takeOver(context.Background(), b, key, l, statusFile, yes, t0, &out, &errb)
return code, out.String(), errb.String()
}
for _, tc := range []struct {
what string
bucket mapBucket
id string
code int
says []string
}{
{"this host writes the bucket", sealed(key, mine), mine, 0, []string{"this host (id " + mine + ") writes the bucket; nothing to take over"}},
{"an empty bucket", mapBucket{}, "", 0, []string{"names no host writing it; this host's next sync records itself"}},
{"a host built from the writer's backup", sealed(key, theirs), "", 4, []string{"host prod-1 (id " + theirs + ") writes the bucket, last at", "(20m ago)", "built from its backup", "sudo felis offsite take-over -yes"}},
{"another host's copies, no writer named", sealed(key, ""), "", 4, []string{"holds copies this host did not write, and names no host writing it", "take-over -yes"}},
{"a host another one took over from", sealed(key, theirs), mine, 5, []string{"took the bucket over from this host"}},
{"a key the bucket refuses", sealed(other, theirs), "", 3, []string{"sealed with another key"}},
} {
t.Run(tc.what, func(t *testing.T) {
before := string(tc.bucket["felis-writer"])
l := lease(tc.id)
code, out, errb := run(tc.bucket, l, filepath.Join(t.TempDir(), "status.json"), false)
if code != tc.code {
t.Fatalf("exit %d, want %d\n%s%s", code, tc.code, out, errb)
}
for _, s := range tc.says {
if !strings.Contains(out+errb, s) {
t.Errorf("output lacks %q:\n%s%s", s, out, errb)
}
}
if string(tc.bucket["felis-writer"]) != before {
t.Error("take-over without -yes wrote the bucket's writer")
}
if tc.id == "" {
if _, err := os.Stat(l.IDFile); !errors.Is(err, os.ErrNotExist) {
t.Errorf("take-over without -yes made this host an id: %v", err)
}
}
})
}
// -yes on a standby host: the bucket names it, and the refusal the last
// sync recorded is cleared, so the watchdog mails again at once.
b := sealed(key, theirs)
l := lease("")
statusFile := filepath.Join(t.TempDir(), "status.json")
lastSuccess := t0.Add(-48 * time.Hour)
if err := offsite.WriteStatus(statusFile, offsite.Status{
LastAttempt: t0.Add(-time.Hour), LastSuccess: lastSuccess, LastError: "offsite: another host writes this bucket", Format: offsite.StatusFormat,
Standby: true, Writer: &offsite.Writer{HostID: theirs, Host: "prod-1", At: t0}, Inherited: true,
}); err != nil {
t.Fatal(err)
}
code, out, errb := run(b, l, statusFile, true)
id, _ := l.ID()
if code != 0 || id == "" || !strings.Contains(out, "this host (id "+id+") writes the bucket now; host prod-1 (id "+theirs+") stops at its next copy") || !strings.Contains(out, "systemctl start felis-offsite.service") {
t.Fatalf("take-over -yes: exit %d, id %q\n%s%s", code, id, out, errb)
}
if w, err := offsite.BucketWriter(context.Background(), b); err != nil || w.HostID != id || w.Host != "spare-1" || !w.At.Equal(t0) {
t.Errorf("writer after take-over -yes = %+v, %v", w, err)
}
st, err := offsite.ReadStatus(statusFile)
if err != nil || st.Standby || st.Writer != nil || st.LastError != "" || st.Inherited || !st.LastSuccess.Equal(lastSuccess) {
t.Errorf("status after take-over -yes = %+v, %v; want the refusal cleared and the last success kept", st, err)
}
// -yes with a key the bucket refuses writes nothing.
b = sealed(other, theirs)
before := string(b["felis-writer"])
if code, _, _ := run(b, lease(""), filepath.Join(t.TempDir(), "status.json"), true); code != 3 || string(b["felis-writer"]) != before {
t.Errorf("take-over -yes under another key: exit %d, writer %s", code, b["felis-writer"])
}
}
func TestOffsiteStatusSaysWhoWritesTheBucket(t *testing.T) {
dir := t.TempDir()
cfg := filepath.Join(dir, "felis.toml")
writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
statusFile := filepath.Join(dir, "status.json")
now := time.Now()
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-30 * time.Minute)}
stale := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-offsite.WriterLive - time.Hour)}
for _, tc := range []struct {
what string
st offsite.Status
says []string
not string
}{
{"standing by for a live writer", offsite.Status{Standby: true, Writer: w}, []string{"This host stands by: host prod-1 (id bbbbbbbbbbbbbbbb) writes the bucket", "mails no watchdog alert", "take-over -yes"}, "wrote it, last at"},
{"standing by for a writer gone quiet", offsite.Status{Standby: true, Writer: stale}, []string{"copies nothing into the bucket: host prod-1 (id bbbbbbbbbbbbbbbb) wrote it, last at", "take-over -yes"}, "mails no watchdog alert"},
{"standing by, no writer named", offsite.Status{Standby: true}, []string{"names no host writing it", "take-over -yes"}, "stands by:"},
{"displaced", offsite.Status{Displaced: true, Writer: w}, []string{"host prod-1 (id bbbbbbbbbbbbbbbb) took the bucket over", "rehearsal machine", "take-over -yes"}, "stands by"},
} {
t.Run(tc.what, func(t *testing.T) {
tc.st.LastAttempt, tc.st.LastSuccess, tc.st.LastError = now.Add(-time.Minute), now.Add(-time.Hour), "offsite: another host writes this bucket"
if err := offsite.WriteStatus(statusFile, tc.st); err != nil {
t.Fatal(err)
}
var out, errb bytes.Buffer
code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
if code != 1 || strings.Contains(out.String(), "bucket holds:") || strings.Contains(out.String(), tc.not) {
t.Errorf("exit %d\n%s%s", code, out.String(), errb.String())
}
for _, s := range tc.says {
if !strings.Contains(out.String(), s) {
t.Errorf("output lacks %q:\n%s", s, out.String())
}
}
})
}
}
func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) {
dir := t.TempDir()
cfg := filepath.Join(dir, "felis.toml")
writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
statusFile := filepath.Join(dir, "status.json")
st := offsite.Status{LastAttempt: time.Now().Add(-time.Minute), LastSuccess: time.Now().Add(-time.Hour), KeyID: "0123456789abcdef"}
status := func() (int, string) {
t.Helper()
if err := offsite.WriteStatus(statusFile, st); err != nil {
t.Fatal(err)
}
var out, errb bytes.Buffer
code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
return code, out.String() + errb.String()
}
if code, out := status(); code != 0 || strings.Contains(out, "refused") {
t.Fatalf("a recent success: exit %d\n%s", code, out)
}
st.LastError, st.KeyMismatch = "offsite: the bucket's objects are sealed with another key", true
code, out := status()
if code != 1 || !strings.Contains(out, "The last run was refused") || !strings.Contains(out, "key id 0123456789abcdef") || strings.Contains(out, "bucket holds:") {
t.Fatalf("a refused run: exit %d\n%s", code, out)
}
}
func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
rawKey, _ := offsite.NewKey()
key, _ := offsite.ParseKey(rawKey)
otherRaw, _ := offsite.NewKey()
other, _ := offsite.ParseKey(otherRaw)
b := mapBucket{}
gapped := putBundleWith(t, b, key, 4, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused")
old := putBundle(t, b, key, 3, nil)
full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3})
sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3})
empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
bundles, err := offsite.ListDB(context.Background(), b)
if err != nil {
t.Fatal(err)
}
var out bytes.Buffer
printDBBundles(context.Background(), b, key, bundles, &out)
lines := strings.Split(strings.TrimSpace(out.String()), "\n")
want := []struct{ name, holds string }{
{empty, "0 accounts, 0 servers"},
{sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"},
{full, "5 accounts, 3 servers"},
{old, "not recorded"},
{gapped, "5 accounts, 3 servers, no MinecraftServer objects"},
}
if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (5, newest first") {
t.Fatalf("output:\n%s", out.String())
}
for i, w := range want {
if l := lines[i+1]; !strings.HasPrefix(l, " "+w.name+" ") || !strings.Contains(l, w.holds) || strings.Contains(l, "no MinecraftServer") != (w.name == gapped) {
t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds)
}
}
}
// TestRestoredHostKeepsStandingBy walks the status file across runs: a host
// restored from the writer's backup stands by on its first run and on every
// run after it, a host an older release left copying claims the bucket once,
// and a failed first run after the upgrade keeps that claim.
func TestRestoredHostKeepsStandingBy(t *testing.T) {
rawKey, _ := offsite.NewKey()
key, _ := offsite.ParseKey(rawKey)
cfg := config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-backups"}
t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
standby := &offsite.WriterError{Kind: offsite.ErrStandby, Writer: &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}}
pass := func(statusFile string, at time.Time, err error) offsite.Lease {
t.Helper()
st, lease := startRun(cfg, key, statusFile, at)
recordRun(&st, offsite.Result{}, err, lease)
if werr := offsite.WriteStatus(statusFile, st); werr != nil {
t.Fatal(werr)
}
return lease
}
restored := filepath.Join(t.TempDir(), "status.json")
for i := range 3 {
if l := pass(restored, t0.Add(time.Duration(i)*time.Hour), standby); l.Inherited {
t.Fatalf("run %d of a restored host claims the bucket", i+1)
}
}
if st, _ := offsite.ReadStatus(restored); !st.Standby || st.Format != offsite.StatusFormat || st.KeyID != offsite.KeyID(key) || st.Bucket != "felis-backups" {
t.Errorf("restored host's status = %+v", st)
}
upgraded := filepath.Join(t.TempDir(), "status.json")
if err := offsite.WriteStatus(upgraded, offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-time.Hour)}); err != nil {
t.Fatal(err)
}
if l := pass(upgraded, t0, errors.New("cannot reach bucket")); !l.Inherited {
t.Fatal("the first run after the upgrade does not claim the bucket")
}
l := pass(upgraded, t0.Add(time.Hour), nil)
if !l.Inherited {
t.Fatal("a failed first run after the upgrade lost the claim")
}
if st, _ := offsite.ReadStatus(upgraded); !st.LastSuccess.Equal(t0.Add(time.Hour)) {
t.Errorf("upgraded host's status = %+v", st)
}
}
// TestOffsiteSyncerSnapshotsAndSweeps: the pass `offsite sync` runs takes its
// snapshot the way `felis db backup` does, in the database pod, into the
// bundle directory, keeping the newest one there; and it sweeps unrecorded
// world objects only past the longest retention [archive] gives any backup.
func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) {
dir := newPodRig(t)
bundles := filepath.Join(dir, "bundles")
env := &offsiteEnv{cfg: config.OffsiteConfig{DBKeep: 5}}
var log bytes.Buffer
cfg := &config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "120d"}}
s := offsiteSyncer(cfg, env, offsiteSources{dbDir: bundles}, "/archives", "/uploads", nil, &log)
if s.DBDir != bundles || s.DBKeep != 5 || s.ArchiveDir != "/archives" || s.UploadsDir != "/uploads" {
t.Fatalf("syncer = %+v", s)
}
if s.OrphanAfter != 120*24*time.Hour {
t.Errorf("OrphanAfter = %s, want the 120d retention", s.OrphanAfter)
}
if s.Snapshot == nil {
t.Fatal("the pass takes no snapshot after copying archives")
}
for i := 0; i < 2; i++ {
if err := s.Snapshot(context.Background()); err != nil {
t.Fatalf("snapshot %d: %v", i, err)
}
}
got, err := dbbackup.List(bundles)
if err != nil || len(got) != 1 || got[0].Label != dbbackup.LabelOffsite {
t.Fatalf("bundle directory = %+v, %v; want the newest offsite bundle alone", got, err)
}
if _, err := dbbackupVerify(got[0].Path); err != nil {
t.Fatalf("the snapshot does not verify: %v", err)
}
// The MinecraftServer objects are exported alongside, as in the daily bundle.
argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args"))
if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") {
t.Errorf("k3s ran %q, want pg_dump in the pod", ran)
}
if !strings.Contains(string(bundleServers(t, got[0].Path)), `"name": "lobby"`) {
t.Errorf("the snapshot holds no MinecraftServer objects")
}
if !strings.Contains(log.String(), "took database bundle "+got[0].Name) {
t.Errorf("the snapshot is not logged:\n%s", log.String())
}
// It becomes the newest bundle in the bucket, so with the cluster away it
// fails, leaves no bundle, and the next pass tries again.
noServerExportWait(t)
writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
if err := s.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "export the MinecraftServer objects") {
t.Errorf("snapshot with the cluster away: %v, want a failure", err)
}
if again, _ := dbbackup.List(bundles); len(again) != 1 || again[0].Name != got[0].Name {
t.Errorf("bundle directory after the failed snapshot = %+v, want %s alone", again, got[0].Name)
}
for _, c := range []struct {
archive config.ArchiveConfig
want time.Duration
}{
{config.ArchiveConfig{}, 90 * 24 * time.Hour},
{config.ArchiveConfig{ScheduledRetention: "200d"}, 200 * 24 * time.Hour},
{config.ArchiveConfig{ManualRetention: "150d", Retention: "30d", ScheduledRetention: "60d"}, 150 * 24 * time.Hour},
} {
s := offsiteSyncer(&config.Config{Database: podDB, Archive: c.archive}, env, offsiteSources{dbDir: bundles}, "", "", nil, io.Discard)
if s.OrphanAfter != c.want {
t.Errorf("%+v: OrphanAfter = %s, want %s", c.archive, s.OrphanAfter, c.want)
}
}
log.Reset()
s = offsiteSyncer(&config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "soon"}}, env, offsiteSources{}, "", "", nil, &log)
if s.OrphanAfter != 0 || !strings.Contains(log.String(), "world objects no backup records are kept") {
t.Errorf("a retention that does not parse: OrphanAfter %s, log %q; want no sweep, said", s.OrphanAfter, log.String())
}
if s.Snapshot != nil {
t.Error("a pass that copies no bundles takes a snapshot")
}
}
cmd/felis/offsite_uploads.go
-59
View File
@@ -1,59 +0,0 @@
package main
import (
"context"
"flag"
"fmt"
"io"
"time"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/platform"
)
func offsiteFetchUploads(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC, binding it if needed)")
uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, "the submission uploads PVC, in the control-plane namespace")
at := fs.String("at", "", "uploads index version to restore (default: the newest; `felis offsite list` shows them)")
if err := fs.Parse(args); err != nil {
return 2
}
cfg, env, err := loadOffsite(*cfgPath, *envFile)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
defer cancel()
stamp, idx, err := offsite.ChooseUploadIndex(ctx, env.bucket, env.key, *at)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
return 1
}
dir := *uploadsDir
if dir == "" {
if !isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: [registry] user_uploads_context %q is not the uploads volume; pass -uploads-dir to restore into a directory anyway\n", cfg.Registry.UserUploadsContext)
return 2
}
if dir, err = resolveVolumeDir(ctx, platform.DefaultControlNamespace, *uploadsPVC, uploadsVolume, true, stderr); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
return 1
}
}
fmt.Fprintf(stdout, "felis offsite fetch-uploads: restoring uploads index %s (%d submission contexts, %s) into %s\n",
stamp, len(idx.Contexts), offsite.HumanBytes(idx.Bytes()), dir)
res, err := offsite.FetchUploads(ctx, env.bucket, env.key, idx, dir, platform.ControlPlaneUID, platform.ControlPlaneUID, stderr)
fmt.Fprintf(stdout, "felis offsite fetch-uploads: %d written (%s), %d already in place, %d failed\n",
res.Written, offsite.HumanBytes(res.Bytes), res.Present, len(res.Failures))
for _, f := range res.Failures {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %s\n", f)
}
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v (a second run writes only what is still missing)\n", err)
return 1
}
return 0
}
cmd/felis/operator.go
+3 -16
View File
@@ -114,24 +114,11 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
// The operator's own image, for the forwarding-config initContainer it
// injects into user servers. The Deployment passes it as FELIS_IMAGE (see
// platform.OperatorDeployment); absent, that injection is simply skipped.
FelisImage: os.Getenv("FELIS_IMAGE"),
Nodes: nil,
EgressProbe: os.Getenv("FELIS_EGRESS_PROBE"),
ControllerNode: os.Getenv("FELIS_CONTROLLER_NODE"),
FelisImage: os.Getenv("FELIS_IMAGE"),
// Uncached: the maintenance-lock check lists Jobs only when a server is
// about to start, which does not justify a namespace-wide Job informer.
Jobs: mgr.GetAPIReader(),
// Uncached too: RCON Secrets are read by name, so the Role grants
// secrets:get without the list/watch an informer would need.
Secrets: mgr.GetAPIReader(),
// And pods: pod-0 is read by name, so the Role grants pods:get and no
// namespace-wide pod informer runs.
Pods: mgr.GetAPIReader(),
Recorder: mgr.GetEventRecorderFor("felis-operator"),
Watch: watch,
}
if os.Getenv("FELIS_DISTRIBUTED") == "true" {
r.Nodes = mgr.GetAPIReader()
Jobs: mgr.GetAPIReader(),
Watch: watch,
}
if err := r.SetupWithManager(mgr); err != nil {
fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
cmd/felis/pinimages.go
+2 -91
View File
@@ -11,9 +11,7 @@ import (
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/imagepin"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -37,17 +35,12 @@ func cmdPinImages(args []string, stdout, stderr io.Writer) int {
namespace := fs.String("namespace", platform.DefaultMinecraftNamespace, "namespace the MinecraftServers live in")
registry := fs.String("registry", defaultRegistryURL, "registry host[:port] the image refs spell")
endpoint := fs.String("endpoint", "", "host[:port] to reach the registry at (default: 127.0.0.1 on the registry's port, its hostPort on this node)")
system := fs.String("system", "", "pin this system server ("+naming.SystemLoginServer+" or "+naming.SystemLobbyServer+") to the build its tag names now, instead of the user servers")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if *system != "" && *system != naming.SystemLoginServer && *system != naming.SystemLobbyServer {
fmt.Fprintf(stderr, "felis pin-images: --system takes %s or %s, not %q\n", naming.SystemLoginServer, naming.SystemLobbyServer, *system)
return 2
}
if *endpoint == "" {
*endpoint = loopbackEndpoint(*registry)
}
@@ -58,9 +51,6 @@ func cmdPinImages(args []string, stdout, stderr io.Writer) int {
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
if *system != "" {
return reportSystemPin(ctx, cl, *namespace, *system, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint}, stdout, stderr)
}
outcomes, err := pinUserServerImages(ctx, cl, *namespace, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint})
if meta.IsNoMatchError(err) {
fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
@@ -87,84 +77,6 @@ func cmdPinImages(args []string, stdout, stderr io.Writer) int {
return exit
}
// reportSystemPin runs pinSystemServerImage for `felis pin-images --system` and
// prints what it did. Only a failed pin exits non-zero: the installer falls back
// to restarting the pod on its tag then.
func reportSystemPin(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver, stdout, stderr io.Writer) int {
o, err := pinSystemServerImage(ctx, cl, namespace, name, r)
switch {
case meta.IsNoMatchError(err):
fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
return 0
case err == nil && o.err != nil:
err = o.err
}
if err != nil {
fmt.Fprintf(stderr, "felis pin-images: %s: %v\n", name, err)
return 1
}
switch {
case o.updated:
fmt.Fprintf(stdout, "felis pin-images: %s: %s; the operator rolls it onto that build\n", name, strings.Join(o.changes, ", "))
default:
fmt.Fprintf(stdout, "felis pin-images: %s: %s\n", name, o.skipped)
}
return 0
}
// pinSystemServerImage fixes a system server to the build its image tag names now,
// replacing the digest of an earlier build. The installer runs it after pushing a
// rebuilt login or lobby image, and the operator rolls the StatefulSet onto the new
// ref, so the build a system server runs is written in its spec and moves only when
// a build did. An image outside the platform registry, or one naming no tag to
// follow, is the admin's choice and is left alone; so is a server whose image an
// admin retargets while this runs.
func pinSystemServerImage(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver) (systemServerOutcome, error) {
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &ms); err != nil {
if apierrors.IsNotFound(err) {
return systemServerOutcome{name: name, skipped: "not present yet; sudo felis setup creates it"}, nil
}
return systemServerOutcome{}, err
}
if ms.Labels[v1alpha1.LabelSystemRole] != name {
return systemServerOutcome{name: name, err: fmt.Errorf(
"MinecraftServer %s/%s is not marked as the Felis %q system server; left alone", namespace, name, name)}, nil
}
tagged := withoutDigest(ms.Spec.Image)
if !r.Covers(tagged) || !strings.Contains(tagged[strings.LastIndex(tagged, "/")+1:], ":") {
return systemServerOutcome{name: name, available: true, skipped: "runs " + ms.Spec.Image +
", which names no platform registry tag to follow; left alone"}, nil
}
pinned, err := r.Pin(ctx, tagged)
if err != nil {
return systemServerOutcome{name: name, err: fmt.Errorf("resolve %s: %w", tagged, err)}, nil
}
changed, err := patchOnConflictRetry(ctx, cl, &ms, func() bool {
if withoutDigest(ms.Spec.Image) != tagged || ms.Spec.Image == pinned {
return false
}
ms.Spec.Image = pinned
return true
})
if err != nil {
return systemServerOutcome{name: name, err: fmt.Errorf("patch %s: %w", name, err)}, nil
}
if !changed {
return systemServerOutcome{name: name, available: true, skipped: "already runs " + ms.Spec.Image}, nil
}
return systemServerOutcome{name: name, available: true, updated: true,
changes: []string{"spec.image pinned to " + pinned}}, nil
}
// withoutDigest drops the @sha256:… of a pinned ref, leaving the tag it came from.
func withoutDigest(ref string) string {
if i := strings.Index(ref, "@"); i >= 0 {
return ref[:i]
}
return ref
}
// loopbackEndpoint is the registry's port on 127.0.0.1: the registry Deployment
// binds it as a hostPort, and containerd's mirror and the installer's pushes use
// the same address.
@@ -176,9 +88,8 @@ func loopbackEndpoint(registry string) string {
}
// pinUserServerImages patches spec.image of every user server whose image the
// resolver covers and is not yet pinned. System servers are the installer's to move:
// it re-pins them with --system when it rolls them onto a new build
// (restart_existing_system_servers).
// resolver covers and is not yet pinned. System servers are left on their tags:
// the installer rebuilds and restarts them on purpose (restart_existing_system_servers).
// A server that is already pinned, or runs an image from elsewhere, produces no
// outcome, so a pinned fleet reports nothing. A running server restarts once as
// the operator rolls its StatefulSet onto the pinned ref, which is the build it
cmd/felis/reaper.go
+51 -89
View File
@@ -16,9 +16,10 @@ import (
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/backup"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/distributed"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/reaper"
"felis.lolicon.best/internal/store"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -32,17 +33,11 @@ import (
// cadence, and RunOnce is idempotent and restart-safe, so a missed or retried
// run simply converges. Only the tarLocal archive backend is wired in this
// build; the snapshot backends (§19) are a later integration.
//
// --retention-only runs the archive-store half alone (reaper.RunRetention):
// the CronJob an install without a worlds root gets, so backups past their
// expiry still leave the store there. It builds no Kubernetes client and reads
// no world.
func cmdReaper(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("reaper", flag.ContinueOnError)
fs.SetOutput(stderr)
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>, else the stock local-path <root>/<pv-name>_<ns>_<pvc-name>)")
retentionOnly := fs.Bool("retention-only", false, "only expire, read back and sweep the archive store: no server is evaluated and no world is read or deleted, so neither the worlds root nor the Kubernetes API is needed")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -61,16 +56,13 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
ctx := ctrl.SetupSignalHandler()
var cl client.Client
if !*retentionOnly {
scheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(v1alpha1.AddToScheme(scheme))
cl, err = client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
if err != nil {
fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
return 1
}
scheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(v1alpha1.AddToScheme(scheme))
cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
if err != nil {
fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
return 1
}
archiver, err := buildArchiver(ctx, cfg, *worldsRoot, cl)
@@ -79,20 +71,7 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
return 1
}
if os.Getenv("FELIS_DISTRIBUTED") == "true" && !*retentionOnly {
m, err := distributionManager(cl, cfg, os.Getenv("FELIS_IMAGE"))
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
local, ok := archiver.(*backup.TarLocal)
if !ok {
fmt.Fprintln(stderr, "remote reaper requires tarLocal")
return 1
}
archiver = &distributed.RemoteArchiver{TarLocal: local, Manager: m}
}
drv, err := openPodStore(ctx, cfg.Database.URL, "reaper", stderr)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
return 1
@@ -102,13 +81,9 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
r := &reaper.Reaper{
Cfg: rcfg,
Store: reaper.NewPGStore(drv.DB()),
Cluster: reaper.NewK8sCluster(cl, cfg.K8s.Namespace),
Archiver: archiver,
}
if *retentionOnly {
fmt.Fprintln(stderr, "felis reaper: retention only — no worlds root is configured, so idle worlds are neither archived nor released")
return reportReaperRun(r.RunRetention(ctx), stdout, stderr)
}
r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace).WithDistributed(os.Getenv("FELIS_DISTRIBUTED") == "true")
// Pre-reap warnings go out by email when [smtp] is configured (the same
// relay and password_ref convention felis-api uses); without it the channel
@@ -139,7 +114,13 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
}
return email, nil
},
notifier: smtpRelay(cfg.SMTP, password),
notifier: &mail.SMTP{
Host: cfg.SMTP.Host,
Port: cfg.SMTP.Port,
From: cfg.SMTP.From,
Username: cfg.SMTP.Username,
Password: password,
},
}
} else {
fmt.Fprintln(stderr, "felis reaper: [smtp] not configured — pre-reap warnings are logged and NOT marked sent")
@@ -158,24 +139,14 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
// is kept, and without this nobody would learn that it is never reaped.
func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d released=%d deleted=%d awaiting_offsite=%d awaiting_stop=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
sum.Evaluated, sum.WorldsReaped, sum.Released, sum.ServersDeleted, sum.AwaitingOffsite, sum.AwaitingStop, sum.Warned, sum.Skipped, sum.StoreFull,
sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d\n",
sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.Warned, sum.Skipped, sum.StoreFull,
sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed)
if !sum.Failed() {
return 0
}
if sum.Skipped > 0 || sum.ExpireFailed > 0 {
fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
sum.Skipped, sum.StoreFull, sum.ExpireFailed)
}
if sum.Corrupt > 0 {
fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)
}
if sum.VerifyFailed > 0 || sum.SweepFailed {
fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",
sum.VerifyFailed, !sum.SweepFailed)
}
fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
sum.Skipped, sum.StoreFull, sum.ExpireFailed)
return 1
}
@@ -213,9 +184,8 @@ func (w *mailWarner) Warn(ctx context.Context, ownerID, server, remaining string
// reaperConfig derives the reaper's retention windows from felis.toml. The 15d
// idle deadline is fixed by §18; only the warning offsets, retention, the
// store soft-cap, the on-demand backup bounds and the scheduled restore points
// are configurable (§24). The backup Job and felis-api read the manual_* and
// scheduled_* keys through it too.
// store soft-cap and the on-demand backup bounds are configurable (§24). The
// backup Job and felis-api read the manual_* bounds through it too.
func reaperConfig(cfg *config.Config) (reaper.Config, error) {
rc := reaper.DefaultConfig()
if v := cfg.Archive.Retention; v != "" {
@@ -263,46 +233,20 @@ func reaperConfig(cfg *config.Config) (reaper.Config, error) {
}
rc.ManualCooldown = d
}
if v := cfg.Archive.ScheduledEvery; v != "" {
d, err := parseSpanDuration(v)
if err != nil || d < 0 {
return rc, fmt.Errorf("[archive] scheduled_every %q: want a span such as 1d (0s for none)", v)
}
rc.ScheduledEvery = d
}
switch n := cfg.Archive.ScheduledKeep; {
case n < 0:
return rc, fmt.Errorf("[archive] scheduled_keep %d: want 1 or more", n)
case n > 0:
rc.ScheduledKeep = n
}
if v := cfg.Archive.ScheduledRetention; v != "" {
d, err := parseSpanDuration(v)
if err != nil || d <= 0 {
return rc, fmt.Errorf("[archive] scheduled_retention %q: want a positive span such as 90d", v)
}
rc.ScheduledRetention = d
}
rc.RequireOffsite = cfg.Offsite.Enabled()
return rc, nil
}
// buildArchiver constructs the WorldArchiver. Only tarLocal is implemented in
// this build; the resolver maps each world PVC to its directory under worldsRoot
// (resolveWorldDir). A nil cl is the retention-only run, which never archives a
// world, so its archiver resolves none.
// (resolveWorldDir).
func buildArchiver(ctx context.Context, cfg *config.Config, worldsRoot string, cl client.Client) (backup.WorldArchiver, error) {
switch cfg.Archive.Store {
case "tarLocal":
t := &backup.TarLocal{BackupRoot: cfg.Archive.LocalPath}
if cl != nil {
t.Resolve = resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot)
} else {
t.Resolve = func(pvc string) (string, error) {
return "", fmt.Errorf("resolve world PVC %s: this run has no worlds root (retention only)", pvc)
}
}
return t, nil
return &backup.TarLocal{
BackupRoot: cfg.Archive.LocalPath,
Resolve: resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot),
}, nil
default:
return nil, fmt.Errorf("[archive] store %q is not implemented in this build (only tarLocal)", cfg.Archive.Store)
}
@@ -345,9 +289,27 @@ func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoo
}
}
// parseSpanDuration parses the human spans used in felis.toml's [archive] table
// (config.ParseSpan).
func parseSpanDuration(s string) (time.Duration, error) { return config.ParseSpan(s) }
// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
func parseSpanDuration(s string) (time.Duration, error) {
s = strings.TrimSpace(s)
switch {
case strings.HasSuffix(s, "mo"):
n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
if err != nil {
return 0, err
}
return time.Duration(n) * 30 * 24 * time.Hour, nil
case strings.HasSuffix(s, "d"):
n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
if err != nil {
return 0, err
}
return time.Duration(n) * 24 * time.Hour, nil
default:
return time.ParseDuration(s)
}
}
// parseByteSize parses a Kubernetes-style quantity ("200Gi", "10G") into bytes.
// An empty string means unlimited (0).
cmd/felis/reaper_test.go
+1 -45
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -28,22 +27,15 @@ func TestReportReaperRunFailsTheJob(t *testing.T) {
want int
}{
{"clean", reaper.Summary{Evaluated: 3, WorldsReaped: 1, AwaitingOffsite: 1}, 0},
{"retirements", reaper.Summary{Evaluated: 5, WorldsReaped: 3, Released: 2, ServersDeleted: 1}, 0},
{"waiting for a stop", reaper.Summary{Evaluated: 3, AwaitingStop: 1}, 0},
{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},
{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},
{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},
{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},
{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},
} {
var out, errb bytes.Buffer
if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {
t.Errorf("%s: exit %d, want %d", tc.name, got, tc.want)
}
if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") ||
!strings.Contains(out.String(), fmt.Sprintf(" released=%d deleted=%d ", tc.sum.Released, tc.sum.ServersDeleted)) {
if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") {
t.Errorf("%s: summary line = %q", tc.name, out.String())
}
if (tc.want == 1) != (errb.Len() > 0) {
@@ -88,42 +80,6 @@ func TestReaperConfigManualKeys(t *testing.T) {
}
}
// TestReaperConfigScheduledKeys: the scheduled restore points default to one a
// day, seven per server and the reaper's 90 days, accept overrides ("0s" turns
// them off), and refuse values that would keep nothing or run backwards.
func TestReaperConfigScheduledKeys(t *testing.T) {
rc, err := reaperConfig(&config.Config{})
if err != nil {
t.Fatal(err)
}
if rc.ScheduledEvery != reaper.Day || rc.ScheduledKeep != 7 || rc.ScheduledRetention != 90*reaper.Day {
t.Fatalf("defaults = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
}
rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
ScheduledEvery: "12h", ScheduledKeep: 3, ScheduledRetention: "14d"}})
if err != nil {
t.Fatal(err)
}
if rc.ScheduledEvery != 12*time.Hour || rc.ScheduledKeep != 3 || rc.ScheduledRetention != 14*reaper.Day {
t.Fatalf("overrides = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
}
rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{ScheduledEvery: "0s"}})
if err != nil || rc.ScheduledEvery != 0 {
t.Fatalf("scheduled_every 0s = %v, %v; want off", rc.ScheduledEvery, err)
}
for _, bad := range []config.ArchiveConfig{
{ScheduledEvery: "-1h"},
{ScheduledEvery: "daily"},
{ScheduledKeep: -1},
{ScheduledRetention: "0d"},
{ScheduledRetention: "forever"},
} {
if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
t.Errorf("%+v was accepted", bad)
}
}
}
func TestResolveWorldDir(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
cmd/felis/registrygate.go
+2 -9
View File
@@ -131,8 +131,7 @@ func loopbackAddr(addr string) bool {
func cmdPushImage(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("push-image", flag.ContinueOnError)
fs.SetOutput(stderr)
tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path, or with --image an OCI layout tar")
image := fs.String("image", "", "push the image this name (io.containerd.image.name) marks in the OCI layout tar --tar, e.g. a release's image bundle")
tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path")
ref := fs.String("ref", "", "host/repository:tag to publish it as")
scheme := fs.String("scheme", "http", "registry scheme: http for the in-cluster registry, https otherwise")
if err := fs.Parse(args); err != nil {
@@ -155,13 +154,7 @@ func cmdPushImage(args []string, stdout, stderr io.Writer) int {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
p := &imagepush.Pusher{Scheme: *scheme, Username: user, Password: pass, Log: stderr}
var digest string
var err error
if *image != "" {
digest, err = p.PushLayout(ctx, *tarPath, *image, *ref)
} else {
digest, err = p.Push(ctx, *tarPath, *ref)
}
digest, err := p.Push(ctx, *tarPath, *ref)
if err != nil {
fmt.Fprintf(stderr, "felis push-image: %v\n", err)
return 1
cmd/felis/restore.go
+2 -30
View File
@@ -1,18 +1,14 @@
package main
import (
"context"
"flag"
"fmt"
"io"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"felis.lolicon.best/internal/archivetransfer"
"felis.lolicon.best/internal/backup"
ctrl "sigs.k8s.io/controller-runtime"
)
// cmdRestore is the in-Pod entrypoint the restore Job runs. internal/restore
@@ -30,9 +26,6 @@ import (
func cmdRestore(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("restore", flag.ContinueOnError)
fs.SetOutput(stderr)
source := fs.String("source-url", "", "one-use archive download URL")
sum := fs.String("sha256", "", "required digest for remote archive")
limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum download size")
server := fs.String("server", "", "server name being restored (for logging)")
ref := fs.String("ref", "", "absolute path to the archive on the backup mount")
store := fs.String("archive-store", "tarLocal", "archive backend (only tarLocal is implemented)")
@@ -42,22 +35,6 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int {
return 2
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if *source != "" {
dir, err := os.MkdirTemp("/tmp", "felis-restore-")
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
defer os.RemoveAll(dir)
*backupRoot = dir
*ref = filepath.Join(dir, "world.tar.gz")
if err := archivetransfer.Fetch(ctx, *source, os.Getenv(archivetransfer.TokenEnv), *ref, *sum, *limit); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
}
if *ref == "" {
fmt.Fprintln(stderr, "felis restore: --ref is required")
return 2
@@ -85,16 +62,11 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int {
},
}
ctx := ctrl.SetupSignalHandler()
if err := archiver.Restore(ctx, backup.ArchiveRef(*ref), *server); err != nil {
fmt.Fprintf(stderr, "felis restore: %v\n", err)
return 1
}
if *source != "" {
if err := backup.VerifyRestored(ctx, *ref, *worldsRoot); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
}
fmt.Fprintf(stdout, "felis restore: server=%s restored from %s into %s\n", *server, *ref, *worldsRoot)
return 0
}
cmd/felis/restore_test.go
+2 -2
View File
@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup
BackupRoot: backupRoot,
Resolve: func(string) (string, error) { return srcDir, nil },
}
got, err := ar.Archive(context.Background(), "survival", "world-survival-0")
got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0")
if err != nil {
t.Fatalf("Archive: %v", err)
}
return string(got.Ref), backupRoot
return string(got), backupRoot
}
// The restore subcommand must extract the archived world into the target world
cmd/felis/rotatetoken.go
-854
View File
@@ -1,854 +0,0 @@
package main
import (
"context"
"crypto/hmac"
"crypto/pbkdf2"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"flag"
"fmt"
"io"
"io/fs"
neturl "net/url"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/maintenance"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/operator"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/registrygate"
"github.com/BurntSushi/toml"
"github.com/jackc/pgx/v5"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// rotate-token replaces one credential the installer generated: an internal
// caller's token (naming.CallerTokens), the registry's write tokens, the
// Velocity forwarding secret or the database password. The new value goes into
// the installer's record first, so that whatever fails later a re-run of the
// installer puts it everywhere; then into the Secrets and files that carry it;
// then whatever read the old value at start restarts. Without -yes it prints
// what it would change and what that interrupts, and changes nothing.
const (
defaultSecretsEnvPath = "/etc/felis/secrets.env"
defaultLinkPropsPath = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
defaultForwardingPath = "/opt/felis/velocity/forwarding.secret"
velocityUnit = "felis-velocity"
apiDeployment = "felis-api"
registryDeployment = "registry"
kindRegistry = "registry"
kindForwarding = "forwarding"
kindDB = "db"
// proxyReloadWait is how long the host proxy gets, once felis-api has rolled,
// to show it re-read its token: it reads the file again on its next call to
// felis-api, and it calls every 15 seconds.
proxyReloadWait = 60 * time.Second
// apiRestartNote is in every plan: felis-api runs as one replica replaced in
// place, so its restart is a short outage of everything that talks to it.
apiRestartNote = "felis-api restarts (a single replica): the panel, sign-in and the proxy's calls are unavailable for the few seconds that takes"
)
// installerTokenKeys names each caller's token in the installer's secrets.env
// (deploy/bootstrap.sh load_or_make_secrets). A re-run of the installer applies
// these values to the Secrets, so a rotation that skipped the file would be
// undone by the next upgrade.
var installerTokenKeys = map[string]string{
"velocity": "SERVICE_TOKEN",
"limbo": "LIMBO_TOKEN",
"build": "BUILD_TOKEN",
"ops": "OPS_TOKEN",
}
// installerRegistryKeys names the registry principals' tokens in secrets.env,
// in registrygate.Principals order.
var installerRegistryKeys = map[string]string{
registrygate.PrincipalPlatform: "REGISTRY_PLATFORM_TOKEN",
registrygate.PrincipalBuild: "REGISTRY_BUILD_TOKEN",
registrygate.PrincipalPrune: "REGISTRY_PRUNE_TOKEN",
}
// installerForwardingKey and installerDBKey name the forwarding secret and the
// database password in secrets.env.
const (
installerForwardingKey = "FORWARDING_SECRET"
installerDBKey = "DB_PASSWORD"
)
type tokenRotator struct {
cl client.Client
controlNS string
minecraftNS string
buildNS string
// secretsEnv, linkProps and forwardingFile are the installer's record and the
// host proxy's felis-link.properties and forwarding.secret; a missing file is
// reported and skipped.
secretsEnv string
linkProps string
forwardingFile string
// hostTOML, podTOML and defaultTOML are the config copies that carry the
// database URL (defaultTOML only when it is a file of its own).
hostTOML string
podTOML string
defaultTOML string
newToken func() (string, error)
// rollout restarts a control-namespace Deployment and waits for it.
rollout func(ctx context.Context, deployment string) error
// restartUnit restarts a systemd unit on this host.
restartUnit func(ctx context.Context, unit string) error
// proxyLog is what the host proxy has logged since a moment, as far as it
// can be read.
proxyLog func(ctx context.Context, since time.Time) string
// alterRole stores a password verifier for a role of the database that runs
// as deployment ("namespace/name"); verifyDB connects with a URL.
alterRole func(ctx context.Context, deployment, role, verifier string) error
verifyDB func(ctx context.Context, url string) error
now func() time.Time
// reloadWait bounds the wait for the proxy to re-read its token, polled
// every pollEvery.
reloadWait time.Duration
pollEvery time.Duration
out io.Writer
}
func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("rotate-token", flag.ContinueOnError)
fs.SetOutput(stderr)
cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
forwarding := fs.String("forwarding-secret", defaultForwardingPath, "the host proxy's forwarding secret file (forwarding only)")
yes := fs.Bool("yes", false, "rotate; without it the plan is printed and nothing changes")
fs.Usage = func() {
fmt.Fprintf(stderr, "Usage: felis rotate-token [-yes] [flags] <%s>\n\n", strings.Join(rotationKinds(), "|"))
fmt.Fprintln(stderr, "Replaces one generated credential: the installer's record, the Secrets and files that carry it, then what reads it.")
fmt.Fprintln(stderr, "Without -yes it prints what would change and what that interrupts.")
fs.PrintDefaults()
}
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if fs.NArg() != 1 {
fs.Usage()
return 2
}
kind := fs.Arg(0)
if !knownRotation(kind) {
fmt.Fprintf(stderr, "felis rotate-token: unknown credential %q (one of %s)\n", kind, strings.Join(rotationKinds(), ", "))
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+kind+")")
return 1
}
cfg, err := config.Load(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
return 1
}
cl, err := buildSystemServerClient()
if err != nil {
fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
return 1
}
buildNS := cfg.Registry.BuildNamespace
if buildNS == "" {
buildNS = platform.DefaultBuildNamespace
}
controlNS := platform.DefaultControlNamespace
r := tokenRotator{
cl: cl,
controlNS: controlNS,
minecraftNS: cfg.K8s.Namespace,
buildNS: buildNS,
secretsEnv: *secretsEnv,
linkProps: *linkProps,
forwardingFile: *forwarding,
hostTOML: hostSetupConfigPath,
podTOML: podSetupConfigPath,
defaultTOML: defaultSetupConfigPath,
newToken: randomToken,
rollout: func(ctx context.Context, deployment string) error {
if err := kubectl(ctx, "-n", controlNS, "rollout", "restart", "deployment/"+deployment); err != nil {
return err
}
return kubectl(ctx, "-n", controlNS, "rollout", "status", "deployment/"+deployment, "--timeout=180s")
},
restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
proxyLog: journalSince,
alterRole: alterRoleInPod,
verifyDB: func(ctx context.Context, url string) error {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
conn, err := pgx.Connect(ctx, url)
if err != nil {
return err
}
return conn.Close(ctx)
},
now: time.Now,
reloadWait: proxyReloadWait,
pollEvery: 3 * time.Second,
out: stdout,
}
if err := r.rotate(context.Background(), kind, *yes); err != nil {
fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
return 1
}
return 0
}
func callerNames() []string {
names := make([]string, 0, len(naming.CallerTokens))
for _, ct := range naming.CallerTokens {
names = append(names, ct.Caller)
}
return names
}
func callerToken(name string) (naming.CallerToken, bool) {
for _, ct := range naming.CallerTokens {
if ct.Caller == name {
return ct, true
}
}
return naming.CallerToken{}, false
}
// rotationKinds is every credential rotate-token replaces: the callers' tokens
// and the installer's other generated secrets.
func rotationKinds() []string {
return append(callerNames(), kindRegistry, kindForwarding, kindDB)
}
func knownRotation(kind string) bool {
for _, k := range rotationKinds() {
if k == kind {
return true
}
}
return false
}
// randomToken is 32 random bytes in hex, the shape the installer generates.
func randomToken() (string, error) {
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
// tokenFingerprint is how the proxy names the token it reloaded in its log
// (plugins/shared FileToken.fingerprint): the first twelve hex digits of its
// SHA-256, enough to tell tokens apart and useless for finding one.
func tokenFingerprint(token string) string {
sum := sha256.Sum256([]byte(token))
return hex.EncodeToString(sum[:])[:12]
}
func (r tokenRotator) rotate(ctx context.Context, kind string, apply bool) error {
switch kind {
case kindRegistry:
return r.rotateRegistry(ctx, apply)
case kindForwarding:
return r.rotateForwarding(ctx, apply)
case kindDB:
return r.rotateDB(ctx, apply)
}
ct, ok := callerToken(kind)
if !ok {
return fmt.Errorf("unknown credential %q (one of %s)", kind, strings.Join(rotationKinds(), ", "))
}
return r.rotateCaller(ctx, ct, apply)
}
// confirm ends the plan: without apply it says nothing changed and how to go
// ahead, and reports false.
func (r tokenRotator) confirm(kind string, apply bool) bool {
if !apply {
fmt.Fprintf(r.out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes %s\n", kind)
return false
}
fmt.Fprintln(r.out, "\nRotating:")
return true
}
// record writes new values into the installer's secrets.env. It comes first in
// every rotation: from then on, whatever fails, a re-run of the installer puts
// the new values everywhere.
func (r tokenRotator) record(kv ...[2]string) error {
keys := make([]string, len(kv))
for i, p := range kv {
keys[i] = p[0]
}
switch err := setKeyValueLines(r.secretsEnv, "=", kv); {
case errors.Is(err, fs.ErrNotExist):
fmt.Fprintf(r.out, " - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
case err != nil:
return fmt.Errorf("record the new value in %s: %w", r.secretsEnv, err)
default:
fmt.Fprintf(r.out, " - %s: %s updated\n", r.secretsEnv, strings.Join(keys, ", "))
}
return nil
}
func (r tokenRotator) putSecret(ctx context.Context, ns, name string, data map[string][]byte) error {
if _, err := putSecretKeys(ctx, r.cl, ns, name, corev1.SecretTypeOpaque, data); err != nil {
return fmt.Errorf("write Secret %s/%s: %w", ns, name, err)
}
fmt.Fprintf(r.out, " - Secret %s/%s: updated\n", ns, name)
return nil
}
func (r tokenRotator) rollAPI(ctx context.Context) error {
if err := r.rollout(ctx, apiDeployment); err != nil {
return fmt.Errorf("roll felis-api: %w", err)
}
fmt.Fprintln(r.out, " - felis-api: rolled out on the new value")
return nil
}
// withMinecraft is the control namespace plus the minecraft namespace when that
// is a different one: where the Secrets game pods and Jobs mount are mirrored.
func (r tokenRotator) withMinecraft() []string {
if r.minecraftNS == "" || r.minecraftNS == r.controlNS {
return []string{r.controlNS}
}
return []string{r.controlNS, r.minecraftNS}
}
func (r tokenRotator) rotateCaller(ctx context.Context, ct naming.CallerToken, apply bool) error {
namespaces := []string{r.controlNS}
replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
if replica != "" && replica != r.controlNS {
namespaces = append(namespaces, replica)
}
key := installerTokenKeys[ct.Caller]
fmt.Fprintf(r.out, "felis rotate-token %s: a new internal token for the %s caller\n", ct.Caller, ct.Caller)
secrets := make([]string, len(namespaces))
for i, ns := range namespaces {
secrets[i] = "Secret " + ns + "/" + ct.Secret
}
writes := append([]string{r.secretsEnv + " (" + key + ")"}, secrets...)
hostProxy := ct.Caller == "velocity" && fileExists(r.linkProps)
if hostProxy {
writes = append(writes, r.linkProps+" (service-token)")
}
fmt.Fprintf(r.out, " - writes %s\n", strings.Join(writes, ", "))
fmt.Fprintf(r.out, " - %s\n", apiRestartNote)
switch ct.Caller {
case "velocity":
if hostProxy {
fmt.Fprintf(r.out, " - the proxy re-reads its token from the file and keeps its players; if it has not within %s of felis-api's restart, it is restarted, which disconnects everyone online\n", r.reloadWait)
} else {
fmt.Fprintf(r.out, " - no proxy on this host (%s): set service-token in your proxy's felis-link.properties to the value in Secret %s/%s afterwards; it re-reads the file without a restart\n",
r.linkProps, r.controlNS, ct.Secret)
}
case "limbo":
fmt.Fprintln(r.out, " - the login gate's pod restarts: a player signing in at that moment reconnects")
case "build":
fmt.Fprintln(r.out, " - a build fetching its context at that moment fails and can be submitted again")
case "ops":
fmt.Fprintln(r.out, " - felis backup-now presents the new token on its next run")
}
if !r.confirm(ct.Caller, apply) {
return nil
}
tok, err := r.newToken()
if err != nil {
return fmt.Errorf("generate a token: %w", err)
}
if err := r.record([2]string{key, tok}); err != nil {
return err
}
for _, ns := range namespaces {
if err := r.putSecret(ctx, ns, ct.Secret, map[string][]byte{naming.ServiceTokenSecretKey: []byte(tok)}); err != nil {
return err
}
}
// The proxy's file changes before felis-api rolls, so the file never falls
// behind the Secret; the proxy's log is read from just before the write,
// since it may pick the new token up before the rollout ends.
since := r.now()
if hostProxy {
if err := setProxyToken(r.linkProps, tok); err != nil {
return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
}
fmt.Fprintf(r.out, " - %s: service-token updated\n", r.linkProps)
}
if err := r.rollAPI(ctx); err != nil {
return err
}
switch ct.Caller {
case "velocity":
if !hostProxy {
break
}
if r.proxyReloaded(ctx, since, tokenFingerprint(tok)) {
fmt.Fprintf(r.out, " - %s: took the new token from its properties; players stayed connected\n", velocityUnit)
break
}
if err := r.restartUnit(ctx, velocityUnit); err != nil {
return fmt.Errorf("restart %s: %w", velocityUnit, err)
}
fmt.Fprintf(r.out, " - %s: had not taken the new token within %s, restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit, r.reloadWait)
case "limbo":
// Only the operator's server pods carry its managed-by label; a backup or
// restore Job's pod carries the server label too, and app.kubernetes.io ones.
if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), client.MatchingLabels{
v1alpha1.LabelServer: naming.SystemLoginServer,
v1alpha1.LabelManagedBy: operator.ManagedByValue,
}); err != nil {
return fmt.Errorf("restart the login gate: %w", err)
}
fmt.Fprintln(r.out, " - login gate: pod restarted to read the new token")
case "build":
fmt.Fprintln(r.out, " - builds: the next build Job reads the new token")
case "ops":
fmt.Fprintln(r.out, " - felis backup-now reads the new token on its next run")
}
return nil
}
// proxyReloaded waits up to reloadWait for the host proxy to log that it
// reloaded the token with this fingerprint (plugins/shared FileToken).
func (r tokenRotator) proxyReloaded(ctx context.Context, since time.Time, fingerprint string) bool {
want := "(fingerprint " + fingerprint + ")"
deadline := r.now().Add(r.reloadWait)
for {
if strings.Contains(r.proxyLog(ctx, since), want) {
return true
}
if !r.now().Before(deadline) {
return false
}
select {
case <-ctx.Done():
return false
case <-time.After(r.pollEvery):
}
}
}
// journalSince is the proxy unit's journal from the second since falls in. A
// journal that cannot be read reads as one without the line, which ends in the
// restart a rotation made before the proxy could reload.
func journalSince(ctx context.Context, since time.Time) string {
out, _ := exec.CommandContext(ctx, "journalctl", "-u", velocityUnit, "--since", "@"+strconv.FormatInt(since.Unix(), 10),
"-o", "cat", "--no-pager", "-q").Output()
return string(out)
}
// setProxyToken writes the proxy's token into felis-link.properties and puts
// the file's modification time back. The proxy re-reads its token by itself,
// while `felis domain check` and `felis domain set` read a file newer than the
// proxy's start as config it has not loaded (the installer's
// install_if_changed keeps the time for the same reason).
func setProxyToken(path, token string) error {
info, err := os.Stat(path)
if err != nil {
return err
}
if err := setKeyValueLine(path, "service-token", "=", token); err != nil {
return err
}
return os.Chtimes(path, time.Time{}, info.ModTime())
}
func (r tokenRotator) rotateRegistry(ctx context.Context, apply bool) error {
keys := make([]string, len(registrygate.Principals))
for i, p := range registrygate.Principals {
keys[i] = installerRegistryKeys[p]
}
fmt.Fprintf(r.out, "felis rotate-token %s: new write tokens for the image registry's principals (%s)\n", kindRegistry, strings.Join(registrygate.Principals, ", "))
fmt.Fprintf(r.out, " - writes %s (%s), Secret %s/%s, Secret %s/%s\n", r.secretsEnv, strings.Join(keys, ", "),
r.controlNS, naming.RegistryAuthSecretName, r.buildNS, naming.RegistryPushSecretName)
fmt.Fprintln(r.out, " - the registry restarts to load them: a build pushing its image at that moment fails and can be submitted again, and an image pull in that moment retries")
fmt.Fprintf(r.out, " - %s (it presents the prune token)\n", apiRestartNote)
if !r.confirm(kindRegistry, apply) {
return nil
}
tokens := map[string][]byte{}
kv := make([][2]string, 0, len(registrygate.Principals))
for _, p := range registrygate.Principals {
tok, err := r.newToken()
if err != nil {
return fmt.Errorf("generate a token: %w", err)
}
tokens[p] = []byte(tok)
kv = append(kv, [2]string{installerRegistryKeys[p], tok})
}
if err := r.record(kv...); err != nil {
return err
}
if err := r.putSecret(ctx, r.controlNS, naming.RegistryAuthSecretName, tokens); err != nil {
return err
}
if err := r.putSecret(ctx, r.buildNS, naming.RegistryPushSecretName, map[string][]byte{
naming.RegistryPushUsernameKey: []byte(registrygate.PrincipalBuild),
naming.RegistryPushPasswordKey: tokens[registrygate.PrincipalBuild],
}); err != nil {
return err
}
if err := r.rollout(ctx, registryDeployment); err != nil {
return fmt.Errorf("restart the registry: %w", err)
}
fmt.Fprintln(r.out, " - registry: restarted on the new tokens")
return r.rollAPI(ctx)
}
func (r tokenRotator) rotateForwarding(ctx context.Context, apply bool) error {
restart, held, err := r.gamePods(ctx)
if err != nil {
return err
}
hostProxy := fileExists(r.forwardingFile)
fmt.Fprintf(r.out, "felis rotate-token %s: a new Velocity forwarding secret, the key a server checks each player's identity with\n", kindForwarding)
secrets := []string{}
for _, ns := range r.withMinecraft() {
secrets = append(secrets, "Secret "+ns+"/"+naming.ForwardingSecretName)
}
writes := append([]string{r.secretsEnv + " (" + installerForwardingKey + ")"}, secrets...)
if hostProxy {
writes = append(writes, r.forwardingFile)
}
fmt.Fprintf(r.out, " - writes %s\n", strings.Join(writes, ", "))
fmt.Fprintf(r.out, " - every running server restarts to read it (%d now), saving its world on the way down, and the proxy restarts: everyone online is disconnected and can rejoin once their server is back\n", len(restart))
if len(held) > 0 {
fmt.Fprintf(r.out, " - left running, because a backup, restore or file write holds its world: %s. Players cannot join it until it restarts: stop and start it from the panel once that finishes\n", strings.Join(held, ", "))
}
if !hostProxy {
fmt.Fprintf(r.out, " - no proxy on this host (%s): put the value in Secret %s/%s into your proxy's forwarding secret file and restart it\n",
r.forwardingFile, r.controlNS, naming.ForwardingSecretName)
}
if !r.confirm(kindForwarding, apply) {
return nil
}
tok, err := r.newToken()
if err != nil {
return fmt.Errorf("generate a secret: %w", err)
}
if err := r.record([2]string{installerForwardingKey, tok}); err != nil {
return err
}
if hostProxy {
info, err := os.Stat(r.forwardingFile)
if err != nil {
return err
}
if err := replaceFileKeepingMode(r.forwardingFile, info, []byte(tok)); err != nil {
return fmt.Errorf("write %s: %w", r.forwardingFile, err)
}
fmt.Fprintf(r.out, " - %s: updated\n", r.forwardingFile)
}
for _, ns := range r.withMinecraft() {
if err := r.putSecret(ctx, ns, naming.ForwardingSecretName, map[string][]byte{naming.ForwardingSecretKey: []byte(tok)}); err != nil {
return err
}
}
// The servers go first: their pods read the Secret as they are recreated,
// and a player who rejoins through the restarted proxy meets a server on the
// new secret, or one still starting.
for i := range restart {
p := &restart[i]
if err := r.cl.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) {
return fmt.Errorf("restart %s: %w", p.Labels[v1alpha1.LabelServer], err)
}
}
fmt.Fprintf(r.out, " - servers: %d restarting on the new secret\n", len(restart))
if hostProxy {
if err := r.restartUnit(ctx, velocityUnit); err != nil {
return fmt.Errorf("restart %s: %w", velocityUnit, err)
}
fmt.Fprintf(r.out, " - %s: restarted on the new secret\n", velocityUnit)
}
return nil
}
// gamePods lists the running game server pods: those a rotation may restart,
// and the servers left alone because a backup, restore or file write holds
// their world (internal/maintenance), which a restart in the middle of would
// break.
func (r tokenRotator) gamePods(ctx context.Context) ([]corev1.Pod, []string, error) {
var pods corev1.PodList
// The operator's managed-by label is on its server pods alone (see rotateCaller).
if err := r.cl.List(ctx, &pods, client.InNamespace(r.minecraftNS), client.MatchingLabels{v1alpha1.LabelManagedBy: operator.ManagedByValue}); err != nil {
return nil, nil, fmt.Errorf("list the servers' pods: %w", err)
}
var jobs batchv1.JobList
if err := r.cl.List(ctx, &jobs, client.InNamespace(r.minecraftNS)); err != nil {
return nil, nil, fmt.Errorf("list the maintenance Jobs: %w", err)
}
var restart []corev1.Pod
var held []string
for _, p := range pods.Items {
if p.DeletionTimestamp != nil {
continue
}
server := p.Labels[v1alpha1.LabelServer]
var ms v1alpha1.MinecraftServer
if err := r.cl.Get(ctx, client.ObjectKey{Namespace: r.minecraftNS, Name: server}, &ms); client.IgnoreNotFound(err) != nil {
return nil, nil, fmt.Errorf("read server %s: %w", server, err)
}
if kind, ok := maintenance.Holder(server, ms.Annotations, jobs.Items, r.now()); ok {
held = append(held, server+" ("+kind+")")
continue
}
restart = append(restart, p)
}
return restart, held, nil
}
func (r tokenRotator) rotateDB(ctx context.Context, apply bool) error {
cfg, err := config.Load(r.hostTOML)
if err != nil {
return err
}
if cfg.Database.Deployment == "" {
return fmt.Errorf("[database] deployment is unset in %s, so the database is not the installer's felis-postgres: change the role's password where it runs, then in [database] url of each config copy", r.hostTOML)
}
u, err := neturl.Parse(cfg.Database.URL)
if err != nil || u.User == nil || u.User.Username() == "" {
return fmt.Errorf("[database] url in %s names no role", r.hostTOML)
}
role := u.User.Username()
targets, err := tomlTargetsOf(r.hostTOML, r.podTOML, r.defaultTOML)
if err != nil {
return err
}
paths := make([]string, len(targets))
for i, t := range targets {
paths[i] = t.path
}
secrets := []string{}
for _, ns := range r.withMinecraft() {
secrets = append(secrets, ns+"/"+platform.ConfigSecretName)
}
fmt.Fprintf(r.out, "felis rotate-token %s: a new password for the database role %q\n", kindDB, role)
fmt.Fprintf(r.out, " - writes %s (%s), the role in %s, [database] url in %s, Secret %s\n",
r.secretsEnv, installerDBKey, cfg.Database.Deployment, strings.Join(paths, " and "), strings.Join(secrets, " and "))
fmt.Fprintf(r.out, " - %s\n", apiRestartNote)
fmt.Fprintln(r.out, " - a backup, restore or file Job that connects in the seconds between the password change and felis-api's restart fails and can be run again; the host's timers read the new config on their next run")
if !r.confirm(kindDB, apply) {
return nil
}
password, err := r.newToken()
if err != nil {
return fmt.Errorf("generate a password: %w", err)
}
// Every config copy is edited in memory first, so one this cannot edit stops
// the rotation before the role's password changes.
edited := make([][]byte, len(targets))
var hostURL string
var podConfig []byte
for i, t := range targets {
raw, err := os.ReadFile(t.real)
if err != nil {
return err
}
var doc struct {
Database struct {
URL string `toml:"url"`
} `toml:"database"`
}
if _, err := toml.Decode(string(raw), &doc); err != nil {
return fmt.Errorf("%s: %w", t.path, err)
}
next, err := withPassword(doc.Database.URL, password)
if err != nil {
return fmt.Errorf("%s: %w", t.path, err)
}
if edited[i], err = editTOMLStrings(raw, []tomlStringEdit{{"database", "url", next}}); err != nil {
return fmt.Errorf("%s: %w; set the password in its [database] url by hand", t.path, err)
}
switch t.path {
case r.hostTOML:
hostURL = next
case r.podTOML:
podConfig = edited[i]
}
}
if podConfig == nil {
return fmt.Errorf("%s resolves to the same file as %s; the pods reach the database at another address and need a copy of their own", r.podTOML, r.hostTOML)
}
if err := r.record([2]string{installerDBKey, password}); err != nil {
return err
}
salt := make([]byte, 16)
if _, err := rand.Read(salt); err != nil {
return err
}
verifier, err := scramVerifier(password, salt, scramIterations)
if err != nil {
return err
}
if err := r.alterRole(ctx, cfg.Database.Deployment, role, verifier); err != nil {
return fmt.Errorf("set the role's password: %w", err)
}
if err := r.verifyDB(ctx, hostURL); err != nil {
return fmt.Errorf("the database does not accept the new password (%v); the config copies still hold the old one: run the installer again (sudo bash deploy/bootstrap.sh), which sets the password in %s everywhere", err, r.secretsEnv)
}
fmt.Fprintf(r.out, " - role %s: password changed, and the database accepts it\n", role)
for i, t := range targets {
info, err := os.Stat(t.real)
if err != nil {
return err
}
if err := replaceFileKeepingMode(t.real, info, edited[i]); err != nil {
return fmt.Errorf("write %s: %w", t.path, err)
}
fmt.Fprintf(r.out, " - %s: [database] url updated\n", t.path)
}
for _, ns := range r.withMinecraft() {
if err := r.putSecret(ctx, ns, platform.ConfigSecretName, map[string][]byte{platform.ConfigSecretKey: podConfig}); err != nil {
return err
}
}
return r.rollAPI(ctx)
}
// withPassword is a database URL with its password replaced.
func withPassword(raw, password string) (string, error) {
u, err := neturl.Parse(raw)
if err != nil || u.User == nil || u.User.Username() == "" {
return "", errors.New("its [database] url names no role")
}
u.User = neturl.UserPassword(u.User.Username(), password)
return u.String(), nil
}
// scramIterations is PostgreSQL's default scram_iterations.
const scramIterations = 4096
// scramVerifier is the SCRAM-SHA-256 verifier PostgreSQL stores for a password
// (RFC 5802 and RFC 7677, in the form libpq's PQencryptPasswordConn makes).
// ALTER ROLE stores a verifier as it is given, so the password itself never
// reaches the server, where a failing statement is logged with its text.
func scramVerifier(password string, salt []byte, iterations int) (string, error) {
salted, err := pbkdf2.Key(sha256.New, password, salt, iterations, sha256.Size)
if err != nil {
return "", err
}
mac := func(msg string) []byte {
h := hmac.New(sha256.New, salted)
h.Write([]byte(msg))
return h.Sum(nil)
}
stored := sha256.Sum256(mac("Client Key"))
b64 := base64.StdEncoding.EncodeToString
return fmt.Sprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, b64(salt), b64(stored[:]), b64(mac("Server Key"))), nil
}
// alterRoleInPod runs ALTER ROLE as the superuser inside the database's
// container, over its socket, with the statement on stdin.
func alterRoleInPod(ctx context.Context, deployment, role, verifier string) error {
ns, name, _ := strings.Cut(deployment, "/")
return kubectlWithInput(ctx, []byte(alterRoleSQL(role, verifier)), "-n", ns, "exec", "-i", "deploy/"+name, "-c", platform.PostgresContainer, "--",
"psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres")
}
// alterRoleSQL sets role's password to a SCRAM verifier, which holds no quote.
func alterRoleSQL(role, verifier string) string {
return "ALTER ROLE \"" + strings.ReplaceAll(role, `"`, `""`) + "\" WITH PASSWORD '" + verifier + "';\n"
}
// setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file
// (secrets.env, a .properties file), appending one when the key is absent. The
// file is replaced atomically and keeps its mode and owner: felis-link.properties
// is root:felis-velocity 0640, and the proxy must still be able to read it.
func setKeyValueLine(path, key, sep, value string) error {
return setKeyValueLines(path, sep, [][2]string{{key, value}})
}
// setKeyValueLines is setKeyValueLine for several keys in one rewrite.
func setKeyValueLines(path, sep string, kv [][2]string) error {
info, err := os.Stat(path)
if err != nil {
return err
}
raw, err := os.ReadFile(path)
if err != nil {
return err
}
lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
for _, p := range kv {
key, value := p[0], p[1]
found := false
for i, ln := range lines {
k, _, ok := strings.Cut(ln, sep)
if ok && strings.TrimSpace(k) == key {
lines[i] = key + sep + value
found = true
}
}
if !found {
lines = append(lines, key+sep+value)
}
}
return replaceFileKeepingMode(path, info, []byte(strings.Join(lines, "\n")+"\n"))
}
// replaceFileKeepingMode atomically replaces path with data, keeping the mode and
// owner info describes: these files are read by other users (the proxy's) and
// some hold credentials, so a rewrite must not widen or narrow who can read them.
func replaceFileKeepingMode(path string, info os.FileInfo, data []byte) error {
tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if err := tmp.Chmod(info.Mode().Perm()); err != nil {
tmp.Close()
return err
}
if st, ok := info.Sys().(*syscall.Stat_t); ok {
if err := tmp.Chown(int(st.Uid), int(st.Gid)); err != nil {
tmp.Close()
return err
}
}
if _, err := tmp.Write(data); err != nil {
tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}
cmd/felis/rotatetoken_test.go
-883
View File
@@ -1,883 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
type rotationRig struct {
r tokenRotator
cl client.Client
out *bytes.Buffer
events []string
dir string
clock time.Time
}
func tokenSecret(ns, name, val string) *corev1.Secret {
return keySecret(ns, name, "token", val)
}
func keySecret(ns, name, key, val string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name},
Data: map[string][]byte{key: []byte(val)},
}
}
// serverPod is a game server's pod as the operator labels it.
func serverPod(ns, name, server string) *corev1.Pod {
return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
Labels: map[string]string{
"felis.lolicon.best/server": server,
"felis.lolicon.best/managed-by": "felis-operator",
"felis.lolicon.best/component": "server",
}}}
}
// backupPod is a backup Job's pod as internal/backupjob labels it: it carries
// the server's label too, and no rotation may take it for the server's own.
func backupPod(ns, name, server string) *corev1.Pod {
return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
Labels: map[string]string{
"felis.lolicon.best/server": server,
"app.kubernetes.io/managed-by": "felis-backup",
"app.kubernetes.io/component": "world-backup",
}}}
}
func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
t.Helper()
rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir(), clock: time.Unix(1_800_000_000, 0)}
rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
rig.r = tokenRotator{
cl: rig.cl,
controlNS: "felis",
minecraftNS: "minecraft",
buildNS: "felis-build",
secretsEnv: filepath.Join(rig.dir, "secrets.env"),
linkProps: filepath.Join(rig.dir, "felis-link.properties"),
forwardingFile: filepath.Join(rig.dir, "forwarding.secret"),
hostTOML: filepath.Join(rig.dir, "felis.host.toml"),
podTOML: filepath.Join(rig.dir, "felis.pod.toml"),
defaultTOML: filepath.Join(rig.dir, "felis.toml"),
newToken: func() (string, error) { return "NEWTOKEN", nil },
rollout: func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
return nil
},
restartUnit: func(_ context.Context, unit string) error {
rig.events = append(rig.events, "restart "+unit)
return nil
},
proxyLog: func(context.Context, time.Time) string { return "" },
alterRole: func(context.Context, string, string, string) error {
rig.events = append(rig.events, "alter-role")
return nil
},
verifyDB: func(context.Context, string) error {
rig.events = append(rig.events, "verify-db")
return nil
},
// Every look at the clock moves it a second on, so a wait measured with it
// ends after a known number of looks.
now: func() time.Time {
rig.clock = rig.clock.Add(time.Second)
return rig.clock
},
reloadWait: 5 * time.Second,
out: rig.out,
}
return rig
}
func (rig *rotationRig) secretKey(t *testing.T, ns, name, key string) string {
t.Helper()
var s corev1.Secret
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
return "<missing>"
}
return string(s.Data[key])
}
func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
t.Helper()
return rig.secretKey(t, ns, name, "token")
}
func (rig *rotationRig) pods(t *testing.T) string {
t.Helper()
var pods corev1.PodList
if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
t.Fatal(err)
}
names := make([]string, len(pods.Items))
for i, p := range pods.Items {
names[i] = p.Name
}
return strings.Join(names, ",")
}
func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
t.Helper()
if err := os.WriteFile(path, []byte(body), mode); err != nil {
t.Fatal(err)
}
if err := os.Chmod(path, mode); err != nil {
t.Fatal(err)
}
}
func readTestFile(t *testing.T, path string) string {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
func TestRotateLimboToken(t *testing.T) {
rig := newRotationRig(t,
tokenSecret("felis", "felis-limbo-token", "old"),
tokenSecret("minecraft", "felis-limbo-token", "old"),
tokenSecret("felis", "felis-service-token", "proxy"),
serverPod("minecraft", "login-0", "login"),
serverPod("minecraft", "survival-0", "survival"),
backupPod("minecraft", "login-backup-x", "login"),
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600)
// felis-api must roll only after both copies hold the new value, and the login
// pod must still be there then: restarting it earlier would have it present
// the new token to an api that does not know it yet.
rig.r.rollout = func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" {
t.Errorf("api rolled while the control Secret held %q", got)
}
if got := rig.secret(t, "minecraft", "felis-limbo-token"); got != "NEWTOKEN" {
t.Errorf("api rolled while the minecraft replica held %q", got)
}
var pod corev1.Pod
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
t.Errorf("the login pod was restarted before the api rolled")
}
return nil
}
if err := rig.r.rotate(context.Background(), "limbo", true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
t.Errorf("secrets.env = %q", got)
}
if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 {
t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm())
}
if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" {
t.Errorf("the proxy's token changed to %q", got)
}
// The login pod restarted; a user server and the backup Job's pod, which
// carries the login server's label too, are untouched.
if got := rig.pods(t); got != "login-backup-x,survival-0" {
t.Errorf("pods left = %s, want login-backup-x,survival-0", got)
}
if strings.Join(rig.events, ",") != "roll felis-api" {
t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events)
}
if strings.Contains(rig.out.String(), "NEWTOKEN") {
t.Errorf("the new token was printed: %s", rig.out.String())
}
}
const testLinkProps = "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n"
func reloadLine(token string) string {
return "[12:00:00 INFO] [felis-link]: Felis: service-token reloaded from /x/felis-link.properties (fingerprint " + tokenFingerprint(token) + ")\n"
}
// The host proxy re-reads its token: the rotation waits for it to say so and
// leaves it running.
func TestRotateVelocityTokenReloadsTheHostProxy(t *testing.T) {
rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
if err := os.Chtimes(rig.r.linkProps, written, written); err != nil {
t.Fatal(err)
}
// The proxy logs its reload on its next call to felis-api, which may be
// while the api rolls: the log must be read from before that.
var reloadedAt time.Time
rig.r.rollout = func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
if got := readTestFile(t, rig.r.linkProps); !strings.Contains(got, "service-token=NEWTOKEN\n") {
t.Errorf("api rolled before the proxy's file held the new token: %q", got)
}
reloadedAt = rig.clock
return nil
}
looks := 0
rig.r.proxyLog = func(_ context.Context, since time.Time) string {
looks++
log := reloadLine("old") // an earlier rotation's
if looks >= 3 && !since.After(reloadedAt) {
log += reloadLine("NEWTOKEN")
}
return log
}
if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.linkProps); got != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
t.Errorf("felis-link.properties = %q", got)
}
info, _ := os.Stat(rig.r.linkProps)
if info.Mode().Perm() != 0o640 {
t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm())
}
if !info.ModTime().Equal(written) {
t.Errorf("properties mtime = %v, want it kept at %v (felis domain check would read the proxy as stale)", info.ModTime(), written)
}
if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
t.Errorf("control Secret = %q, want NEWTOKEN", got)
}
// The proxy's token has no replica: it must not appear in a workload namespace.
if got := rig.secret(t, "minecraft", "felis-service-token"); got != "<missing>" {
t.Errorf("rotation copied the proxy token into minecraft (%q)", got)
}
if strings.Join(rig.events, ",") != "roll felis-api" {
t.Errorf("events = %v, want the api roll and no proxy restart", rig.events)
}
if looks != 3 {
t.Errorf("the log was read %d times, want 3 (until the line appeared)", looks)
}
if out := rig.out.String(); !strings.Contains(out, "felis-velocity: took the new token from its properties; players stayed connected") {
t.Errorf("output does not say the players stayed: %s", out)
}
}
// A proxy that never logs the new fingerprint (an older plugin, a stuck proxy)
// is restarted once the wait is over.
func TestRotateVelocityTokenRestartsAProxyThatDoesNotReload(t *testing.T) {
rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
looks := 0
rig.r.proxyLog = func(context.Context, time.Time) string {
looks++
return reloadLine("old")
}
if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
t.Fatal(err)
}
if strings.Join(rig.events, ",") != "roll felis-api,restart felis-velocity" {
t.Errorf("events = %v, want the api roll then the proxy restart", rig.events)
}
// reloadWait is 5s and each look moves the clock a second.
if looks != 5 {
t.Errorf("the log was read %d times, want 5", looks)
}
if out := rig.out.String(); !strings.Contains(out, "felis-velocity: had not taken the new token within 5s, restarted") {
t.Errorf("output does not explain the restart: %s", out)
}
}
// The proxy and the Java plugin name a token by the same fingerprint
// (plugins/shared LinkConfigLoaderTest fileTokenFollowsTheFile).
func TestTokenFingerprintMatchesThePlugin(t *testing.T) {
if got := tokenFingerprint("new-token"); got != "348e9df2a42b" {
t.Errorf("tokenFingerprint(new-token) = %q, want 348e9df2a42b", got)
}
}
// An external proxy has no felis-link.properties here: the Secret still rotates,
// nothing is restarted on this host, and the output says where the value is
// without printing it.
func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) {
rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
t.Fatal(err)
}
if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
t.Errorf("control Secret = %q, want NEWTOKEN", got)
}
if strings.Join(rig.events, ",") != "roll felis-api" {
t.Errorf("events = %v, want no proxy restart", rig.events)
}
out := rig.out.String()
if !strings.Contains(out, "felis/felis-service-token") || strings.Contains(out, "NEWTOKEN") {
t.Errorf("output should point at the Secret without the value: %s", out)
}
}
func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old"))
// An install from before per-caller tokens has no BUILD_TOKEN line yet.
writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600)
if err := rig.r.rotate(context.Background(), "build", true); err != nil {
t.Fatal(err)
}
if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" {
t.Errorf("build replica = %q, want NEWTOKEN (created when absent)", got)
}
if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" {
t.Errorf("control Secret = %q, want NEWTOKEN", got)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
t.Errorf("secrets.env = %q", got)
}
}
// A failed api rollout stops the rotation before the caller restarts: the login
// gate keeps running on the old value the old api pods still accept.
func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) {
rig := newRotationRig(t,
tokenSecret("felis", "felis-limbo-token", "old"),
serverPod("minecraft", "login-0", "login"),
)
rig.r.rollout = func(context.Context, string) error { return errors.New("rollout timed out") }
err := rig.r.rotate(context.Background(), "limbo", true)
if err == nil || !strings.Contains(err.Error(), "rollout timed out") {
t.Fatalf("err = %v, want the rollout failure", err)
}
if got := rig.pods(t); got != "login-0" {
t.Errorf("pods left = %s: the login pod was restarted although the api never rolled", got)
}
}
func TestRotateRefusesAnUnknownCredential(t *testing.T) {
rig := newRotationRig(t)
writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600)
if err := rig.r.rotate(context.Background(), "admin", true); err == nil {
t.Fatal("rotated an unknown credential")
}
if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\n" {
t.Errorf("secrets.env = %q, want it untouched", got)
}
if len(rig.events) != 0 {
t.Errorf("events = %v, want nothing touched", rig.events)
}
}
const (
testHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
testPodTOML = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
)
// installedRig is a host as the installer leaves it, with every credential in
// place, for the plan test.
func installedRig(t *testing.T) *rotationRig {
t.Helper()
rig := newRotationRig(t,
tokenSecret("felis", "felis-service-token", "old"),
tokenSecret("felis", "felis-limbo-token", "old"),
tokenSecret("minecraft", "felis-limbo-token", "old"),
tokenSecret("felis", "felis-build-token", "old"),
tokenSecret("felis-build", "felis-build-token", "old"),
tokenSecret("felis", "felis-ops-token", "old"),
keySecret("felis", "felis-registry-auth", "platform", "old"),
keySecret("felis-build", "felis-registry-push", "password", "old"),
keySecret("felis", "felis-forwarding-secret", "secret", "old"),
keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
keySecret("felis", "felis-config", "felis.toml", testPodTOML),
keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
serverPod("minecraft", "login-0", "login"),
serverPod("minecraft", "survival-0", "survival"),
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=old\n", 0o600)
writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
return rig
}
// Without -yes every rotation prints its plan and changes nothing.
func TestRotatePlanChangesNothing(t *testing.T) {
for _, kind := range rotationKinds() {
t.Run(kind, func(t *testing.T) {
rig := installedRig(t)
files := map[string]string{}
for _, p := range []string{rig.r.secretsEnv, rig.r.linkProps, rig.r.forwardingFile, rig.r.hostTOML, rig.r.podTOML} {
files[p] = readTestFile(t, p)
}
rig.r.newToken = func() (string, error) {
t.Error("a plan generated a token")
return "NEWTOKEN", nil
}
if err := rig.r.rotate(context.Background(), kind, false); err != nil {
t.Fatal(err)
}
for p, before := range files {
if got := readTestFile(t, p); got != before {
t.Errorf("%s changed to %q", filepath.Base(p), got)
}
}
for _, s := range [][3]string{
{"felis", "felis-service-token", "token"}, {"felis", "felis-limbo-token", "token"},
{"felis-build", "felis-build-token", "token"}, {"felis", "felis-ops-token", "token"},
{"felis", "felis-registry-auth", "platform"}, {"felis-build", "felis-registry-push", "password"},
{"minecraft", "felis-forwarding-secret", "secret"},
} {
if got := rig.secretKey(t, s[0], s[1], s[2]); got != "old" {
t.Errorf("Secret %s/%s changed to %q", s[0], s[1], got)
}
}
if got := rig.secretKey(t, "minecraft", "felis-config", "felis.toml"); got != testPodTOML {
t.Errorf("felis-config changed to %q", got)
}
if len(rig.events) != 0 {
t.Errorf("events = %v, want none", rig.events)
}
if got := rig.pods(t); got != "login-0,survival-0" {
t.Errorf("pods left = %s, want all", got)
}
out := rig.out.String()
if !strings.HasSuffix(out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes "+kind+"\n") {
t.Errorf("the plan does not end with how to go ahead: %s", out)
}
// Each plan names the interruption it causes.
want := apiRestartNote
if kind == kindForwarding {
want = "everyone online is disconnected"
}
if !strings.Contains(out, want) {
t.Errorf("the plan does not say %q: %s", want, out)
}
})
}
}
func TestRotateRegistryTokens(t *testing.T) {
rig := newRotationRig(t,
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: "felis-registry-auth"},
Data: map[string][]byte{"platform": []byte("a"), "build": []byte("b"), "prune": []byte("c")}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis-build", Name: "felis-registry-push"},
Data: map[string][]byte{"username": []byte("build"), "password": []byte("b")}},
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=a\nREGISTRY_BUILD_TOKEN=b\nREGISTRY_PRUNE_TOKEN=c\n", 0o600)
n := 0
rig.r.newToken = func() (string, error) {
n++
return fmt.Sprintf("NEWTOKEN%d", n), nil
}
// The registry reads its tokens as it starts: it restarts after the Secret
// holds them, and felis-api (the prune token) after that.
rig.r.rollout = func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
if got := rig.secretKey(t, "felis", "felis-registry-auth", "prune"); got != "NEWTOKEN3" {
t.Errorf("%s rolled while the prune token was %q", deployment, got)
}
return nil
}
if err := rig.r.rotate(context.Background(), kindRegistry, true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=NEWTOKEN1\nREGISTRY_BUILD_TOKEN=NEWTOKEN2\nREGISTRY_PRUNE_TOKEN=NEWTOKEN3\n" {
t.Errorf("secrets.env = %q", got)
}
for key, want := range map[string]string{"platform": "NEWTOKEN1", "build": "NEWTOKEN2", "prune": "NEWTOKEN3"} {
if got := rig.secretKey(t, "felis", "felis-registry-auth", key); got != want {
t.Errorf("felis-registry-auth %s = %q, want %s", key, got, want)
}
}
if got := rig.secretKey(t, "felis-build", "felis-registry-push", "password"); got != "NEWTOKEN2" {
t.Errorf("the build Jobs' push password = %q, want the build token", got)
}
if got := rig.secretKey(t, "felis-build", "felis-registry-push", "username"); got != "build" {
t.Errorf("the build Jobs' push username = %q, want build", got)
}
if strings.Join(rig.events, ",") != "roll registry,roll felis-api" {
t.Errorf("events = %v, want the registry then felis-api", rig.events)
}
if strings.Contains(rig.out.String(), "NEWTOKEN") {
t.Errorf("a new token was printed: %s", rig.out.String())
}
}
func TestRotateForwardingSecret(t *testing.T) {
lock := time.Unix(1_800_000_000, 0)
rig := newRotationRig(t,
keySecret("felis", "felis-forwarding-secret", "secret", "old"),
keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
serverPod("minecraft", "survival-0", "survival"),
serverPod("minecraft", "lobby-0", "lobby"),
// creative is being backed up: a running Job holds its world.
serverPod("minecraft", "creative-0", "creative"),
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "creative-backup",
Labels: map[string]string{maintenance.LabelServer: "creative", maintenance.LabelManagedBy: "felis-backup"}}},
// survival's last backup is over; its finished pod stays until the Job's TTL.
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival-backup",
Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: "felis-backup"}},
Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}},
backupPod("minecraft", "survival-backup-x", "survival"),
// A pod already on its way out is neither counted nor deleted again.
func() *corev1.Pod {
p := serverPod("minecraft", "lobby-old", "lobby")
p.DeletionTimestamp = &metav1.Time{Time: lock}
p.Finalizers = []string{"felis.lolicon.best/test"}
return p
}(),
// skyblock's restore has just been admitted, its Job not created yet.
serverPod("minecraft", "skyblock-0", "skyblock"),
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "skyblock",
Annotations: map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lock)}}},
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival"}},
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nFORWARDING_SECRET=old\n", 0o600)
writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
// The proxy restarts after the servers were told to: a player who rejoins
// must not meet a server still on the old secret.
rig.r.restartUnit = func(_ context.Context, unit string) error {
rig.events = append(rig.events, "restart "+unit)
if got := rig.pods(t); strings.Contains(got, "survival-0") {
t.Errorf("the proxy restarted before the servers (pods %s)", got)
}
if got := readTestFile(t, rig.r.forwardingFile); got != "NEWTOKEN" {
t.Errorf("the proxy restarted on forwarding.secret %q", got)
}
return nil
}
if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nFORWARDING_SECRET=NEWTOKEN\n" {
t.Errorf("secrets.env = %q", got)
}
if info, _ := os.Stat(rig.r.forwardingFile); info.Mode().Perm() != 0o640 {
t.Errorf("forwarding.secret mode = %v, want 0640", info.Mode().Perm())
}
for _, ns := range []string{"felis", "minecraft"} {
if got := rig.secretKey(t, ns, "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
t.Errorf("Secret %s/felis-forwarding-secret = %q, want NEWTOKEN", ns, got)
}
}
if got := rig.pods(t); got != "creative-0,lobby-old,skyblock-0,survival-backup-x" {
t.Errorf("pods left = %s, want the held servers, the terminating pod and the backup's pod", got)
}
if strings.Join(rig.events, ",") != "restart felis-velocity" {
t.Errorf("events = %v, want only the proxy restart (felis-api does not hold this secret)", rig.events)
}
out := rig.out.String()
for _, want := range []string{
"every running server restarts to read it (2 now)",
"left running, because a backup, restore or file write holds its world: creative (backup), skyblock (restore).",
" - servers: 2 restarting on the new secret\n",
} {
if !strings.Contains(out, want) {
t.Errorf("output lacks %q: %s", want, out)
}
}
if strings.Contains(out, "NEWTOKEN") {
t.Errorf("the new secret was printed: %s", out)
}
}
func TestRotateForwardingSecretForAnExternalProxy(t *testing.T) {
rig := newRotationRig(t, keySecret("felis", "felis-forwarding-secret", "secret", "old"))
if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
t.Fatal(err)
}
if got := rig.secretKey(t, "felis", "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
t.Errorf("control Secret = %q, want NEWTOKEN", got)
}
if len(rig.events) != 0 {
t.Errorf("events = %v, want no proxy restart on this host", rig.events)
}
if out := rig.out.String(); !strings.Contains(out, "put the value in Secret felis/felis-forwarding-secret into your proxy's forwarding secret file") {
t.Errorf("output does not say where the value is: %s", out)
}
}
// dbRig is an installed host for the database rotation: felis.toml links to the
// host copy, as the installer makes it.
func dbRig(t *testing.T) *rotationRig {
t.Helper()
rig := newRotationRig(t,
keySecret("felis", "felis-config", "felis.toml", testPodTOML),
keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n", 0o600)
writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
if err := os.Symlink("felis.host.toml", rig.r.defaultTOML); err != nil {
t.Fatal(err)
}
return rig
}
const (
wantHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
wantPodTOML = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
)
func TestRotateDatabasePassword(t *testing.T) {
rig := dbRig(t)
rig.r.alterRole = func(_ context.Context, deployment, role, verifier string) error {
rig.events = append(rig.events, "alter-role")
if deployment != "felis/felis-postgres" || role != "felis" {
t.Errorf("alterRole(%q, %q), want felis/felis-postgres and felis", deployment, role)
}
if !strings.Contains(readTestFile(t, rig.r.secretsEnv), "DB_PASSWORD=NEWTOKEN\n") {
t.Error("the role changed before secrets.env recorded the new password")
}
// The verifier is the new password's, under the salt it carries.
m := regexp.MustCompile(`^SCRAM-SHA-256\$4096:([^$]+)\$`).FindStringSubmatch(verifier)
if m == nil {
t.Fatalf("verifier %q is not a SCRAM-SHA-256 verifier", verifier)
}
salt, err := base64.StdEncoding.DecodeString(m[1])
if err != nil || len(salt) != 16 {
t.Fatalf("verifier salt %q: %v", m[1], err)
}
if want, _ := scramVerifier("NEWTOKEN", salt, 4096); verifier != want {
t.Errorf("verifier = %q, want %q", verifier, want)
}
return nil
}
// The configs change only once the database accepts the new password.
rig.r.verifyDB = func(_ context.Context, url string) error {
rig.events = append(rig.events, "verify-db")
if url != "postgres://felis:[email protected]:30432/felis?sslmode=disable" {
t.Errorf("verified with %q, want the host URL with the new password", url)
}
if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
t.Errorf("the host config changed before the database accepted the password: %q", got)
}
return nil
}
rig.r.rollout = func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != wantPodTOML {
t.Errorf("felis-api rolled on felis-config %q", got)
}
return nil
}
if err := rig.r.rotate(context.Background(), kindDB, true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=NEWTOKEN\nSERVICE_TOKEN=s\n" {
t.Errorf("secrets.env = %q", got)
}
if got := readTestFile(t, rig.r.hostTOML); got != wantHostTOML {
t.Errorf("host config = %q", got)
}
if got := readTestFile(t, rig.r.podTOML); got != wantPodTOML {
t.Errorf("pod config = %q", got)
}
if info, err := os.Lstat(rig.r.defaultTOML); err != nil || info.Mode()&os.ModeSymlink == 0 {
t.Errorf("felis.toml is no longer the link to the host copy (%v)", err)
}
for _, ns := range []string{"felis", "minecraft"} {
if got := rig.secretKey(t, ns, "felis-config", "felis.toml"); got != wantPodTOML {
t.Errorf("Secret %s/felis-config = %q", ns, got)
}
}
if strings.Join(rig.events, ",") != "alter-role,verify-db,roll felis-api" {
t.Errorf("events = %v", rig.events)
}
if strings.Contains(rig.out.String(), "NEWTOKEN") {
t.Errorf("the new password was printed: %s", rig.out.String())
}
}
// A password the database does not accept leaves the configs on the old one
// and felis-api running: the installer's record, already new, is the way back.
func TestRotateDatabasePasswordStopsWhenTheDatabaseRefuses(t *testing.T) {
rig := dbRig(t)
rig.r.verifyDB = func(context.Context, string) error {
rig.events = append(rig.events, "verify-db")
return errors.New("password authentication failed")
}
err := rig.r.rotate(context.Background(), kindDB, true)
if err == nil || !strings.Contains(err.Error(), "password authentication failed") || !strings.Contains(err.Error(), "run the installer again") {
t.Fatalf("err = %v, want the refusal and the way back", err)
}
if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
t.Errorf("host config = %q, want it unchanged", got)
}
if got := readTestFile(t, rig.r.podTOML); got != testPodTOML {
t.Errorf("pod config = %q, want it unchanged", got)
}
if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != testPodTOML {
t.Errorf("felis-config = %q, want it unchanged", got)
}
if strings.Join(rig.events, ",") != "alter-role,verify-db" {
t.Errorf("events = %v, want no api roll", rig.events)
}
}
// Everything that can refuse does so before the installer's record or the
// role change; what the host config alone shows is refused by the plan too.
func TestRotateDatabasePasswordRefusesEarly(t *testing.T) {
for _, tc := range []struct {
name string
apply bool
setup func(t *testing.T, rig *rotationRig)
want string
}{
{"a database the installer does not run", false, func(t *testing.T, rig *rotationRig) {
writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "deployment = \"felis/felis-postgres\"\n", "", 1), 0o600)
}, "[database] deployment is unset"},
{"a database url without a role", false, func(t *testing.T, rig *rotationRig) {
writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "felis:oldpw@", "", 1), 0o600)
}, "names no role"},
{"a config copy the line editor cannot edit", true, func(t *testing.T, rig *rotationRig) {
writeTestFile(t, rig.r.podTOML, "[database]\nurl = \"\"\"\npostgres://felis:[email protected]:5432/felis\"\"\"\n", 0o600)
}, "set the password in its [database] url by hand"},
{"a pod copy that is the host copy", true, func(t *testing.T, rig *rotationRig) {
if err := os.Remove(rig.r.podTOML); err != nil {
t.Fatal(err)
}
if err := os.Symlink("felis.host.toml", rig.r.podTOML); err != nil {
t.Fatal(err)
}
}, "resolves to the same file as"},
} {
t.Run(tc.name, func(t *testing.T) {
rig := dbRig(t)
tc.setup(t, rig)
err := rig.r.rotate(context.Background(), kindDB, tc.apply)
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("err = %v, want %q", err, tc.want)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n" {
t.Errorf("secrets.env = %q, want it untouched", got)
}
if len(rig.events) != 0 {
t.Errorf("events = %v, want nothing done", rig.events)
}
})
}
}
// The RFC 7677 example (user "user", password "pencil"), with the stored and
// server keys computed by openssl's PBKDF2 and HMAC rather than this code.
func TestScramVerifier(t *testing.T) {
salt, _ := base64.StdEncoding.DecodeString("W22ZaJ0SNY7soEsUEjb6gQ==")
got, err := scramVerifier("pencil", salt, 4096)
if err != nil {
t.Fatal(err)
}
if want := "SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==$WG5d8oPm3OtcPnkdi4Uo7BkeZkBFzpcXkuLmtbsT4qY=:wfPLwcE6nTWhTAmQ7tl2KeoiWGPlZqQxSrmfPwDl2dU="; got != want {
t.Errorf("scramVerifier = %q\nwant %q", got, want)
}
}
func TestAlterRoleSQL(t *testing.T) {
if got := alterRoleSQL(`fe"lis`, "SCRAM-SHA-256$4096:c2FsdA==$a:b"); got != "ALTER ROLE \"fe\"\"lis\" WITH PASSWORD 'SCRAM-SHA-256$4096:c2FsdA==$a:b';\n" {
t.Errorf("alterRoleSQL = %q", got)
}
}
// installerSecretKeys is every secrets.env key a rotation writes.
func installerSecretKeys() map[string]bool {
keys := map[string]bool{installerForwardingKey: true, installerDBKey: true}
for _, k := range installerTokenKeys {
keys[k] = true
}
for _, k := range installerRegistryKeys {
keys[k] = true
}
return keys
}
// The installer and rotate-token must agree on where each caller's token lives:
// rotate-token writes installerTokenKeys into secrets.env, and the installer
// applies those same keys to the Secrets on its next run. A key the installer
// does not read would be silently reverted by the next upgrade.
func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
if err != nil {
t.Fatal(err)
}
script := string(raw)
for caller, key := range installerTokenKeys {
ct, ok := callerToken(caller)
if !ok {
t.Fatalf("installerTokenKeys names unknown caller %q", caller)
}
apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`)
if !apply.MatchString(script) {
t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key)
}
}
// The replicas the installer applies straight into the workload namespaces, so an
// upgrade has them before the new operator and build Jobs reference them.
for _, line := range []string{
`apply_literal_secret "$MINECRAFT_NS" felis-limbo-token token "$LIMBO_TOKEN"`,
`apply_literal_secret "$BUILD_NS" felis-build-token token "$BUILD_TOKEN"`,
`kube -n "$MINECRAFT_NS" delete secret felis-service-token --ignore-not-found`,
`kube -n "$BUILD_NS" delete secret felis-service-token --ignore-not-found`,
} {
if !strings.Contains(script, line) {
t.Errorf("bootstrap.sh lacks %s", line)
}
}
for _, ns := range []string{"MINECRAFT_NS", "BUILD_NS"} {
if strings.Contains(script, `apply_literal_secret "$`+ns+`" felis-service-token`) {
t.Errorf("bootstrap.sh still copies the proxy's token into %s", ns)
}
}
if len(installerTokenKeys) != len(callerNames()) {
t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames()))
}
}
// Every value the installer keeps in secrets.env has a rotation that rewrites
// that very key, and every key a rotation writes is one the installer
// generates, keeps and so re-applies on its next run.
func TestInstallerSecretsAreAllRotatable(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
if err != nil {
t.Fatal(err)
}
script := string(raw)
m := regexp.MustCompile(`(?s)write_file_atomic "\$SECRETS_ENV" 0600 <<EOF\n(.*?)\nEOF\n`).FindStringSubmatch(script)
if m == nil {
t.Fatal("bootstrap.sh: no secrets.env heredoc found")
}
persisted := map[string]bool{}
for _, line := range strings.Split(m[1], "\n") {
key, value, _ := strings.Cut(line, "=")
if value != "${"+key+"}" {
t.Errorf("secrets.env line %q is not KEY=${KEY}", line)
}
persisted[key] = true
}
rotated := installerSecretKeys()
for key := range persisted {
if !rotated[key] {
t.Errorf("secrets.env keeps %s, which no rotation replaces", key)
}
}
for key := range rotated {
if !persisted[key] {
t.Errorf("a rotation writes %s, which the installer does not keep in secrets.env", key)
}
if !regexp.MustCompile(`\n ` + key + `="\$\{` + key + `:-\$\(openssl rand -hex [0-9]+\)\}"\n`).MatchString(script) {
t.Errorf("bootstrap.sh does not generate %s when secrets.env lacks it", key)
}
}
}
cmd/felis/run.go
+4 -31
View File
@@ -13,39 +13,26 @@ Usage:
Commands:
migrate up Apply embedded database migrations under an advisory lock (snapshots the database first)
db Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check)
offsite Copy world archives, database bundles, user images and uploads to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|fetch-images|fetch-uploads|keygen)
offsite Copy world archives and database bundles to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|keygen)
operator Run the MinecraftServer controller-manager
api Run the felis-api HTTP server
nano Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)
reaper Run the world reaper / backup batch
restore Extract a world archive into a world volume (internal Job entrypoint)
backup Archive a world into the backup store and record it (internal Job entrypoint)
backup-now Archive every user server's world now, one at a time (or the named ones; -stop stops running ones first; prints the plan, -yes applies; requires root/sudo)
files List, read, write, mkdir, delete, rename, upload or unzip one path in a stopped server's world (internal Job entrypoint)
export Archive a stopped server's world, or read one of its backups, and hand it to felis-api for download (internal Job entrypoint)
egress-gate Hold a build or game server pod until its egress NetworkPolicy is enforced (internal init container entrypoint)
files List/read/write one file in a stopped server's world (internal Job entrypoint)
egress-gate Hold a build pod until its egress NetworkPolicy is enforced (internal Job entrypoint)
fetch-context Fetch and extract a submission's build context (internal Job entrypoint)
scan-gate Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
push-image Push a scanned image tarball to the registry (internal Job entrypoint)
mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
server-migrate Move a stopped world between approved nodes (start|status|retry; requires root)
node Join and approve trusted daemon nodes (list|token|join|approve|firewall; requires root)
node-control Run the host node-task service on an API-only Unix socket (requires root)
node-probe Verify reachability and observed sources (internal admission probe)
archive-serve Serve scoped one-use archive transfers on the controller (internal entrypoint)
registry-gate Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
manifests Render the control-plane RBAC + NetworkPolicy install bundle as YAML
apply Create a MinecraftServer CRD (direct K8s write; use -f server.json)
setup Run host bootstrap + first-run setup console (TUI; requires root/sudo)
converge Fill in fields a newer desired spec added to already-installed system servers
rotate-token Replace a generated credential and restart what reads it (velocity|limbo|build|ops|registry|forwarding|db; prints the plan, -yes applies; requires root/sudo)
domain Move the install to a new root domain on every surface that carries it, or check each one (set|check; requires root/sudo)
status Print the platform at a glance: node, control plane, proxy, servers, backups, host, open alerts (requires root/sudo)
doctor Run every health check once, grouped by area, with where to look next; mails nothing (requires root/sudo)
support-bundle Collect status, doctor, logs and cluster state into one redacted tar.gz to share when asking for help (requires root/sudo)
watchdog Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
version Print the build stamp of this binary
update Check platform versions; --apply installs a reviewed target
update Report which platform components have updates available
breakGlass Open the local break-glass emergency console (TUI; requires root/sudo)
Run "felis <command> -h" for command-specific flags.
@@ -70,38 +57,24 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
"reaper": cmdReaper,
"restore": cmdRestore,
"backup": cmdBackup,
"backup-now": cmdBackupNow,
"files": cmdFiles,
"export": cmdExport,
"egress-gate": cmdEgressGate,
"fetch-context": cmdFetchContext,
"scan-gate": cmdScanGate,
"push-image": cmdPushImage,
"mirror-build-tools": cmdMirrorBuildTools,
"registry-gate": cmdRegistryGate,
"archive-serve": cmdArchiveServe,
"node": cmdNode,
"node-control": cmdNodeControl,
"server-migrate": cmdServerMigrate,
"node-probe": cmdNodeProbe,
"manifests": cmdManifests,
"apply": cmdApply,
"setup": cmdSetup,
"converge": cmdConverge,
"rotate-token": cmdRotateToken,
"domain": cmdDomain,
"breakGlass": cmdBreakGlass,
"bootstrap-assets": cmdBootstrapAssets,
"init-forwarding": cmdInitForwarding,
"init-volume": cmdInitVolume,
"pin-images": cmdPinImages,
"image-bundle": cmdImageBundle,
"version": cmdVersion,
"update": cmdUpdate,
"watchdog": cmdWatchdog,
"status": cmdStatus,
"doctor": cmdDoctor,
"support-bundle": cmdSupportBundle,
}
// run dispatches a subcommand. It is separate from main so the router is
cmd/felis/run_test.go
+1 -20
View File
@@ -5,27 +5,8 @@ import (
"os"
"strings"
"testing"
"github.com/BurntSushi/toml"
"felis.lolicon.best/internal/config"
)
func TestBootstrapConfigKeys(t *testing.T) {
var out, errBuf bytes.Buffer
if code := run([]string{"bootstrap-assets", "config-keys"}, &out, &errBuf); code != 0 {
t.Fatalf("exit code = %d: %s", code, errBuf.String())
}
var cfg config.Config
meta, err := toml.Decode(strings.TrimSpace(out.String())+` = "26.3"`, &cfg)
if err != nil || len(meta.Undecoded()) != 0 {
t.Fatalf("advertised config key is unsupported: %v, undecoded: %v", err, meta.Undecoded())
}
if cfg.Velocity.GameVersion != "26.3" {
t.Fatalf("advertised key did not set the game version: %q", cfg.Velocity.GameVersion)
}
}
func TestRunNoArgsPrintsUsage(t *testing.T) {
var out, errBuf bytes.Buffer
if code := run(nil, &out, &errBuf); code != 2 {
@@ -62,7 +43,7 @@ func TestRunUnknownCommand(t *testing.T) {
// decision rather than an oversight.
var undocumentedCommands = map[string]bool{
"bootstrap-assets": true, "init-forwarding": true, "init-volume": true,
"pin-images": true, "image-bundle": true,
"pin-images": true,
}
// The usage text and the dispatch table must describe the same set of commands.
cmd/felis/scangate.go
-166
View File
@@ -1,166 +0,0 @@
package main
import (
"errors"
"flag"
"fmt"
"io"
"io/fs"
"os"
"strings"
"felis.lolicon.best/internal/build"
)
// maxScanDocument bounds each document scan-gate reads: a modpack report lists a
// few thousand packages, far below this. Tests shrink it.
var maxScanDocument int64 = 64 << 20
// cmdScanGate is the build pod's verdict step, after trivy wrote its full JSON
// report and trivy convert wrote the CycloneDX SBOM. It applies the scan policy
// to the report, prints the verdict and every blocking finding, then appends the
// verdict, the report and the SBOM to its log as the envelope felis-api keeps on
// the build (build.WriteScanEnvelope). It exits 1 when a finding blocks, which
// fails the pod before the push step runs, and 2 when the report cannot be read,
// so a scan that produced nothing usable never admits an image.
func cmdScanGate(args []string, stdout, stderr io.Writer) int {
fset := flag.NewFlagSet("scan-gate", flag.ContinueOnError)
fset.SetOutput(stderr)
reportPath := fset.String("report", "", "trivy JSON report (required)")
sbomPath := fset.String("sbom", "", "CycloneDX SBOM to keep with the report")
failOn := fset.String("fail-on", strings.Join(build.DefaultScanFailOn, ","), "comma-separated severities that block the image")
failUnfixed := fset.Bool("fail-unfixed", false, "block on vulnerabilities that have no fixed release too")
accept := fset.String("accept", "", "comma-separated vulnerability ids and secret rule ids that never block")
termLog := fset.String("termination-log", "/dev/termination-log", "where the one-line verdict goes for the pod status")
if err := fset.Parse(args); err != nil {
return 2
}
// A stray argument is a policy the gate would otherwise drop without a word
// (a comma split out of --fail-on, say).
if fset.NArg() > 0 {
fmt.Fprintf(stderr, "felis scan-gate: unexpected argument %q\n", fset.Arg(0))
return 2
}
sevs, err := build.ParseSeverities(*failOn)
if err != nil {
fmt.Fprintf(stderr, "felis scan-gate: --fail-on: %v\n", err)
return 2
}
accepted, err := build.ParseScanAccept(*accept)
if err != nil {
fmt.Fprintf(stderr, "felis scan-gate: --accept: %v\n", err)
return 2
}
if *reportPath == "" {
fmt.Fprintln(stderr, "felis scan-gate: --report is required")
return 2
}
fail := func(msg string) int {
fmt.Fprintln(stderr, "felis scan-gate: "+msg)
writeTerminationLog(*termLog, msg)
return 2
}
report, err := readScanDocument(*reportPath)
if err != nil {
return fail("the scan report is unreadable: " + err.Error())
}
policy := build.ScanPolicy{FailOn: sevs, FailUnfixed: *failUnfixed, Accept: accepted}
summary, err := build.Summarize(report, policy)
if err != nil {
return fail("the scan report is unreadable: " + err.Error())
}
env := build.ScanEnvelope{Summary: summary, Report: report}
if *sbomPath != "" {
switch sbom, err := readScanDocument(*sbomPath); {
case err == nil:
env.SBOM = sbom
case errors.Is(err, fs.ErrNotExist):
fmt.Fprintf(stdout, "felis scan-gate: no SBOM at %s; keeping the report alone\n", *sbomPath)
default:
return fail("the SBOM is unreadable: " + err.Error())
}
}
printScanVerdict(stdout, summary)
written, err := build.WriteScanEnvelope(stdout, env)
if err != nil {
return fail("could not write the scan envelope: " + err.Error())
}
for _, doc := range written.Summary.Omitted {
fmt.Fprintf(stderr, "felis scan-gate: the %s is too large to keep with the build and was left out\n", doc)
}
if summary.Blocked {
writeTerminationLog(*termLog, summary.Reason())
return 1
}
writeTerminationLog(*termLog, "the scan passed")
return 0
}
// printScanVerdict writes the human half of scan-gate's log.
func printScanVerdict(w io.Writer, s build.ScanSummary) {
var counts []string
for _, sev := range build.Severities {
counts = append(counts, fmt.Sprintf("%s %d", sev, s.Counts[sev]))
}
fmt.Fprintf(w, "felis scan-gate: %d packages; findings: %s\n", s.Packages, strings.Join(counts, ", "))
unfixed := "vulnerabilities with no fixed release do not block"
if s.Policy.FailUnfixed {
unfixed = "vulnerabilities with no fixed release block too"
}
fmt.Fprintf(w, "felis scan-gate: blocking on %s (%s)\n", strings.Join(s.Policy.FailOn, ", "), unfixed)
if len(s.Policy.Accept) > 0 {
matched := 0
for _, f := range s.Findings {
if f.Accepted {
matched++
}
}
fmt.Fprintf(w, "felis scan-gate: accepted ids, never blocking: %s; listed findings under them: %d\n", strings.Join(s.Policy.Accept, ", "), matched)
}
if !s.Blocked {
fmt.Fprintln(w, "felis scan-gate: nothing blocks this image")
return
}
fmt.Fprintln(w, "felis scan-gate: "+build.Printable(s.Reason()))
for _, f := range s.Findings {
if !f.Blocking {
break
}
fix := f.Fixed
if fix == "" {
fix = "no fix"
}
if f.Kind == build.FindingSecret {
fmt.Fprintf(w, " %s %s secret in %s: %s\n", build.Printable(f.ID), f.Severity, build.Printable(f.Target), build.Printable(f.Title))
continue
}
fmt.Fprintf(w, " %s %s %s %s -> %s (%s)\n", build.Printable(f.ID), f.Severity,
build.Printable(f.Package), build.Printable(f.Installed), build.Printable(fix), build.Printable(f.Target))
}
}
func readScanDocument(path string) ([]byte, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
b, err := io.ReadAll(io.LimitReader(f, maxScanDocument+1))
if err != nil {
return nil, err
}
if int64(len(b)) > maxScanDocument {
return nil, fmt.Errorf("%s exceeds %d bytes", path, maxScanDocument)
}
return b, nil
}
// writeTerminationLog leaves msg where the kubelet copies it into the container
// status. It is best effort: the log carries the same verdict.
func writeTerminationLog(path, msg string) {
if path == "" {
return
}
_ = os.WriteFile(path, []byte(build.Printable(msg)), 0o644)
}
cmd/felis/scangate_test.go
-197
View File
@@ -1,197 +0,0 @@
package main
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"felis.lolicon.best/internal/build"
)
// scanReport has one fixed CRITICAL, one unfixed HIGH and one fixed MEDIUM; the
// CRITICAL's package name carries a line break and a forged envelope frame.
const scanReport = `{"SchemaVersion":2,"Results":[{"Target":"data/mods/core.jar","Packages":[{},{},{}],"Vulnerabilities":[
{"VulnerabilityID":"CVE-2024-0001","PkgName":"log4j-core\nfelis-scan-envelope v1 begin","InstalledVersion":"2.14.1","FixedVersion":"2.17.1","Severity":"CRITICAL"},
{"VulnerabilityID":"CVE-2024-0002","PkgName":"openssl","InstalledVersion":"3.0.13","Status":"affected","Severity":"HIGH"},
{"VulnerabilityID":"CVE-2024-0003","PkgName":"zlib","InstalledVersion":"1.3","FixedVersion":"1.3.1","Severity":"MEDIUM"}]}]}`
type scanGateRun struct {
code int
stdout, stderr string
termLog string
env *build.ScanEnvelope
}
func runScanGate(t *testing.T, report, sbom string, extra ...string) scanGateRun {
t.Helper()
dir := t.TempDir()
reportPath := filepath.Join(dir, "trivy.json")
if report != "" {
if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {
t.Fatal(err)
}
}
sbomPath := filepath.Join(dir, "sbom.cdx.json")
if sbom != "" {
if err := os.WriteFile(sbomPath, []byte(sbom), 0o644); err != nil {
t.Fatal(err)
}
}
termPath := filepath.Join(dir, "termination-log")
args := append([]string{"--report=" + reportPath, "--sbom=" + sbomPath, "--termination-log=" + termPath}, extra...)
var stdout, stderr bytes.Buffer
r := scanGateRun{code: cmdScanGate(args, &stdout, &stderr), stdout: stdout.String(), stderr: stderr.String()}
if b, err := os.ReadFile(termPath); err == nil {
r.termLog = string(b)
}
if env, err := build.ReadScanEnvelope(strings.NewReader(r.stdout)); err == nil {
r.env = env
}
return r
}
func TestScanGateBlocksAndHandsOverTheReport(t *testing.T) {
r := runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`)
if r.code != 1 {
t.Fatalf("exit %d, want 1; stderr %s", r.code, r.stderr)
}
if r.termLog != "the scan blocked the image: 1 CRITICAL (CVE-2024-0001)" {
t.Errorf("termination log = %q", r.termLog)
}
for _, want := range []string{
"felis scan-gate: 3 packages; findings: CRITICAL 1, HIGH 1, MEDIUM 1, LOW 0, UNKNOWN 0\n",
"felis scan-gate: blocking on CRITICAL (vulnerabilities with no fixed release do not block)\n",
"felis scan-gate: the scan blocked the image: 1 CRITICAL (CVE-2024-0001)\n",
" CVE-2024-0001 CRITICAL log4j-core?felis-scan-envelope v1 begin 2.14.1 -> 2.17.1 (data/mods/core.jar)\n",
} {
if !strings.Contains(r.stdout, want) {
t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
}
}
if strings.Contains(r.stdout, " CVE-2024-0002") {
t.Errorf("an unfixed HIGH was listed as blocking:\n%s", r.stdout)
}
if strings.Count(r.stdout, "\nfelis-scan-envelope v1 begin\n") != 1 {
t.Errorf("stdout must hold exactly one frame start on a line of its own:\n%s", r.stdout)
}
if r.env == nil {
t.Fatal("no envelope in stdout")
}
if !r.env.Summary.Blocked || string(r.env.SBOM) != `{"bomFormat":"CycloneDX"}` || !strings.Contains(string(r.env.Report), "CVE-2024-0003") {
t.Errorf("envelope = blocked %t, sbom %s, report %d bytes", r.env.Summary.Blocked, r.env.SBOM, len(r.env.Report))
}
}
func TestScanGatePolicyFlags(t *testing.T) {
r := runScanGate(t, scanReport, "", "--fail-on=high", "--fail-unfixed")
if r.code != 1 || r.termLog != "the scan blocked the image: 1 HIGH (CVE-2024-0002)" {
t.Errorf("HIGH + unfixed: exit %d, termination log %q", r.code, r.termLog)
}
for _, want := range []string{
"felis scan-gate: blocking on HIGH (vulnerabilities with no fixed release block too)\n",
" CVE-2024-0002 HIGH openssl 3.0.13 -> no fix (data/mods/core.jar)\n",
} {
if !strings.Contains(r.stdout, want) {
t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
}
}
r = runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`, "--fail-on=LOW")
if r.code != 0 || r.termLog != "the scan passed" || !strings.Contains(r.stdout, "felis scan-gate: nothing blocks this image\n") {
t.Errorf("LOW only: exit %d, termination log %q, stdout:\n%s", r.code, r.termLog, r.stdout)
}
if r.env == nil || r.env.Summary.Blocked || r.env.SBOM == nil {
t.Errorf("a passing scan must still hand over its report and SBOM: %+v", r.env)
}
}
func TestScanGateAcceptedIDsNeverBlock(t *testing.T) {
r := runScanGate(t, scanReport, "", "--fail-on=CRITICAL,MEDIUM", "--accept=CVE-2024-0001, CVE-2024-0002,CVE-2099-0001")
if r.code != 1 || r.termLog != "the scan blocked the image: 1 MEDIUM (CVE-2024-0003)" {
t.Errorf("exit %d, termination log %q", r.code, r.termLog)
}
if !strings.Contains(r.stdout, "felis scan-gate: accepted ids, never blocking: CVE-2024-0001, CVE-2024-0002, CVE-2099-0001; listed findings under them: 2\n") {
t.Errorf("stdout:\n%s", r.stdout)
}
if r.env == nil || strings.Join(r.env.Summary.Policy.Accept, ",") != "CVE-2024-0001,CVE-2024-0002,CVE-2099-0001" {
t.Fatalf("envelope = %+v", r.env)
}
for _, f := range r.env.Summary.Findings {
if f.ID == "CVE-2024-0001" && (f.Blocking || !f.Accepted) {
t.Errorf("accepted finding = %+v", f)
}
}
r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001")
if r.code != 0 || r.termLog != "the scan passed" {
t.Errorf("only an accepted CRITICAL: exit %d, termination log %q", r.code, r.termLog)
}
}
func TestScanGateWithoutAnSBOMKeepsTheReport(t *testing.T) {
r := runScanGate(t, scanReport, "", "--fail-on=LOW")
if r.code != 0 || !strings.Contains(r.stdout, "felis scan-gate: no SBOM at ") {
t.Errorf("exit %d, stdout:\n%s", r.code, r.stdout)
}
if r.env == nil || r.env.SBOM != nil || r.env.Report == nil {
t.Errorf("envelope = %+v", r.env)
}
}
func TestScanGateListsABlockingSecret(t *testing.T) {
r := runScanGate(t, `{"SchemaVersion":2,"Results":[{"Target":"config/keys.txt","Secrets":[
{"RuleID":"aws-access-key-id","Severity":"CRITICAL","Title":"AWS Access\nKey ID"}]}]}`, "")
if r.code != 1 || r.termLog != "the scan blocked the image: 1 CRITICAL (aws-access-key-id)" {
t.Errorf("exit %d, termination log %q", r.code, r.termLog)
}
if !strings.Contains(r.stdout, " aws-access-key-id CRITICAL secret in config/keys.txt: AWS Access?Key ID\n") {
t.Errorf("stdout:\n%s", r.stdout)
}
}
func TestScanGateFailsClosed(t *testing.T) {
r := runScanGate(t, "", "")
if r.code != 2 || !strings.HasPrefix(r.termLog, "the scan report is unreadable: open ") || r.env != nil {
t.Errorf("missing report: exit %d, termination log %q", r.code, r.termLog)
}
r = runScanGate(t, `{"SchemaVersion":1}`, "")
if r.code != 2 || r.termLog != "the scan report is unreadable: read trivy report: schema version 1, want 2" {
t.Errorf("old schema: exit %d, termination log %q", r.code, r.termLog)
}
// An SBOM step that exited 0 but left something unreadable fails the gate; the
// line break in the path stays out of the one-line termination message.
sbomDir := filepath.Join(t.TempDir(), "sb\nom")
if err := os.Mkdir(sbomDir, 0o755); err != nil {
t.Fatal(err)
}
r = runScanGate(t, scanReport, "", "--sbom="+sbomDir)
if r.code != 2 || !strings.HasPrefix(r.termLog, "the SBOM is unreadable: read ") ||
!strings.HasSuffix(r.termLog, "/sb?om: is a directory") || r.env != nil {
t.Errorf("unreadable SBOM: exit %d, termination log %q", r.code, r.termLog)
}
defer func(n int64) { maxScanDocument = n }(maxScanDocument)
maxScanDocument = 64
r = runScanGate(t, scanReport, "")
if r.code != 2 || !strings.HasSuffix(r.termLog, "/trivy.json exceeds 64 bytes") || r.env != nil {
t.Errorf("oversized report: exit %d, termination log %q", r.code, r.termLog)
}
maxScanDocument = 64 << 20
r = runScanGate(t, scanReport, "", "--fail-on=SEVERE")
if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --fail-on: unknown severity "SEVERE"`) {
t.Errorf("bad severity: exit %d, stderr %q", r.code, r.stderr)
}
// A severity list split at its comma leaves a stray argument, not a
// narrower policy.
r = runScanGate(t, scanReport, "", "--fail-on=LOW", "CRITICAL")
if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: unexpected argument "CRITICAL"`) || r.env != nil {
t.Errorf("stray argument: exit %d, stderr %q", r.code, r.stderr)
}
r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001 CVE-2024-0003")
if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --accept: "CVE-2024-0001 CVE-2024-0003" is not a vulnerability id or secret rule id`) {
t.Errorf("bad accept list: exit %d, stderr %q", r.code, r.stderr)
}
var stdout, stderr bytes.Buffer
if code := cmdScanGate(nil, &stdout, &stderr); code != 2 || !strings.Contains(stderr.String(), "--report is required") {
t.Errorf("no report flag: exit %d, stderr %q", code, stderr.String())
}
}
cmd/felis/server_migrate.go
-114
View File
@@ -1,114 +0,0 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"flag"
"fmt"
"io"
"os"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/distributed"
"felis.lolicon.best/internal/placement"
"felis.lolicon.best/internal/platform"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func cmdServerMigrate(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprintln(stderr, "server-migrate: start | status | retry (separate from database migrate)")
return 2
}
fs := flag.NewFlagSet("server-migrate "+args[0], flag.ContinueOnError)
fs.SetOutput(stderr)
name := fs.String("name", "", "stopped user server")
target := fs.String("target-node", "", "approved target worker")
id := fs.String("id", "", "operation id (required for retry)")
kube := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local kubeconfig")
ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
cfgPath := fs.String("config", "/var/lib/felis/felis.host.toml", "host config used to record the current owner on the safety backup")
if err := fs.Parse(args[1:]); err != nil {
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "server-migrate requires root/sudo")
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cfg, err := clientcmd.BuildConfigFromFlags("", *kube)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
scheme := runtime.NewScheme()
clientgoscheme.AddToScheme(scheme)
v1alpha1.AddToScheme(scheme)
cl, err := client.New(cfg, client.Options{Scheme: scheme})
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
m := &distributed.Manager{Client: cl, Namespace: *ns, Resolve: placement.Resolve(cl, *ns)}
var nodes corev1.NodeList
if err = cl.List(ctx, &nodes); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
for _, n := range nodes.Items {
if n.Labels[placement.LabelRole] == placement.RoleController {
m.Controller = n.Name
}
}
if m.Controller == "" {
fmt.Fprintln(stderr, "distributed controller identity is not configured")
return 1
}
m.Resolve = placement.Resolve(cl, *ns, m.Controller)
var op distributed.Operation
switch args[0] {
case "start":
host, cfgErr := config.Load(*cfgPath)
if cfgErr != nil {
fmt.Fprintln(stderr, cfgErr)
return 1
}
drv, dbErr := openPodStore(ctx, host.Database.URL, "migration", stderr)
if dbErr != nil {
fmt.Fprintln(stderr, dbErr)
return 1
}
defer drv.Close()
var owner sql.NullString
if err := drv.DB().QueryRowContext(ctx, "SELECT owner_id FROM servers WHERE name=$1 AND deleted_at IS NULL AND retire_requested_at IS NULL", *name).Scan(&owner); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
op, err = m.BeginMigration(ctx, *name, *target, owner.String)
case "status":
op, err = m.Migration(ctx, *name, *id)
case "retry":
if *id == "" {
fmt.Fprintln(stderr, "retry requires --id")
return 2
}
op, err = m.RetryMigration(ctx, *name, *id)
default:
fmt.Fprintln(stderr, "unknown migration operation")
return 2
}
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
json.NewEncoder(stdout).Encode(op)
return 0
}
cmd/felis/setup.go
+86 -36
View File
@@ -11,7 +11,9 @@ import (
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
)
@@ -99,7 +101,19 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
}
defer setup.drv.Close()
res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
// The wizard's first screen asks the operator to join the server and run /link:
// the Owner IS the Minecraft account, so the login gate must be UP before we ask
// for a link code. This used to run after the wizard, which is why setup asked
// for a code from a server that had never been started. On a re-run the Owner
// already exists, so provisioning stays best-effort and never blocks the
// operator from reaching the status screen.
if err := provisionSystemServers(ctx, setup.cfg, stdout, !setup.adminExists); err != nil {
fmt.Fprintf(stderr, "felis setup: %v\n", err)
fmt.Fprintln(stderr, "The Owner is bound by joining the login gate in-game, so setup cannot continue without it.")
return 1
}
res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database.URL, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
if err != nil {
fmt.Fprintf(stderr, "felis setup: %v\n", err)
return 1
@@ -108,50 +122,34 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
if panelURL == "" {
panelURL = localPanelURL(setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname)
}
// Game services are provisioned independently. A failed login/lobby must not
// stop the host administrator from opening the panel to diagnose it.
if err := provisionSystemServers(ctx, setup.cfg, stdout); err != nil {
fmt.Fprintf(stdout, "felis setup: Minecraft services need attention: %v\n", err)
}
reportSetupResult(stdout, res, bootstrapped, setup.adminExists, panelURL)
return 0
}
// reportSetupResult preserves the browser handoff after the TUI closes.
func reportSetupResult(stdout io.Writer, res breakGlassResult, bootstrapped, adminExisted bool, panelURL string) {
if !adminExisted && !res.provisioned {
defer fmt.Fprintln(stdout, "\nOwner login is unfinished. Run sudo felis setup again to get a panel setup link; Minecraft is not required.")
}
if !res.provisioned && !res.connectConfigured {
if bootstrapped {
fmt.Fprintln(stdout, "felis setup: host bootstrap completed; panel/connection setup unfinished.")
fmt.Fprintln(stdout, "felis setup: host bootstrap completed; Owner/connection setup skipped.")
if panelURL != "" {
fmt.Fprintf(stdout, "Panel: %s\n", panelURL)
fmt.Fprintln(stdout, "The local HTTPS certificate is self-signed; your browser may ask for confirmation on first visit.")
}
return
return 0
}
// A re-run lands on the status screen, which changes nothing by design —
// reporting that as "cancelled" reads as a failure the operator did not cause.
msg := "felis setup: cancelled — no changes made."
switch {
case res.alreadySetUp:
if res.alreadySetUp {
msg = "felis setup: already set up — nothing to change."
}
fmt.Fprintln(stdout, msg)
if panelURL != "" {
fmt.Fprintf(stdout, "Panel: %s\n", panelURL)
}
return
return 0
}
if res.provisioned {
fmt.Fprintf(stdout, "\nfelis setup: Owner account %q provisioned (passwordless).\n", res.username)
fmt.Fprintf(stdout, "Recorded as %q (mode: %s, os user: %s).\n", res.accountable, res.mode, res.osUser)
if res.setupTokenURL != "" {
fmt.Fprintf(stdout, "Open this URL to record your email and create a passkey (Minecraft is optional):\n\n %s\n\n", res.setupTokenURL)
fmt.Fprintf(stdout, "Open this URL to complete passwordless login setup (verify email / enroll passkey):\n\n %s\n\n", res.setupTokenURL)
}
if res.auditWarning != "" {
fmt.Fprintf(stdout, "WARNING: the accountability audit row was NOT written: %s\n", res.auditWarning)
@@ -179,15 +177,28 @@ func reportSetupResult(stdout io.Writer, res breakGlassResult, bootstrapped, adm
fmt.Fprintln(stdout, "Felis config, Kubernetes Secret and API rollout were updated.")
}
}
return 0
}
// provisionSystemServers ensures the login limbo and lobby system services exist,
// then prints the login-first Velocity wiring. deploy/bootstrap.sh writes this
// configuration for its host proxy; operators only need to mirror it when they
// deliberately run Velocity elsewhere.
func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writer) error {
//
// required is set on a first run, where the next screen asks the operator to join
// the server and run /link. There a gate that never comes up is not a degraded
// install, it is an impossible one — so every soft landing below becomes a hard
// error and we block until the gate reports Ready. On a re-run the Owner already
// exists and nothing downstream needs the gate, so unconfigured images or an
// unreachable cluster degrade to printed guidance exactly as before.
func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writer, required bool) error {
// fail is the one place the two modes diverge: fatal on a first run, guidance
// on a re-run.
fail := func(format string, args ...any) error {
if required {
return fmt.Errorf(format, args...)
}
fmt.Fprintf(out, "\nfelis setup: "+format+"\n", args...)
return nil
}
@@ -195,26 +206,30 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
return fail("login/lobby system servers NOT provisioned — set [velocity] login_image " +
"and lobby_image in felis.toml (build them from deploy/limbo and deploy/lobby), then re-run `sudo felis setup`")
}
if required && cfg.Velocity.LoginImage == "" {
return errors.New("the Owner binds by joining the login gate, but [velocity] login_image is not set in felis.toml " +
"(build it from deploy/limbo), then re-run `sudo felis setup`")
}
cl, err := buildSystemServerClient()
if err != nil {
return fail("could not reach the cluster to provision the login/lobby system servers: %v\n"+
"Re-run `sudo felis setup` on the control-plane host once the cluster is reachable", err)
}
// The login limbo authenticates to the felis-api INTERNAL face, so it needs the
// internal base URL, the root domain (to link players at the console), and its
// own token (felis-limbo-token). The first two are plain env baked into the pod
// here; the token is a Secret the operator injects by reference — but a
// secretKeyRef is namespace-local, so first replicate the token Secret from the
// control namespace into the minecraft namespace where the login pod runs. The control namespace is
// internal base URL, the root domain (to link players at the console), and the
// service token. The first two are plain env baked into the pod here; the token
// is a Secret the operator injects by reference — but a secretKeyRef is
// namespace-local, so first replicate the token Secret from the control namespace
// into the minecraft namespace where the login pod runs. The control namespace is
// the platform default (there is no felis.toml override for it); a deployment that
// renamed it must replicate the Secret by hand.
controlNS := platform.DefaultControlNamespace
apiBaseURL := platform.InternalAPIBaseURL(controlNS)
// These Secrets must land in the minecraft namespace before the pods that
// mount them are created: the login gate's token (login authenticates to
// felis-api with it), the Velocity forwarding secret (every backend verifies the proxy's
// mount them are created: the service token (login authenticates to felis-api
// with it), the Velocity forwarding secret (every backend verifies the proxy's
// signed handshake with it — without it the login gate would derive an OFFLINE
// UUID and players would bind the wrong Minecraft identity), and felis-config
// UUID and the Owner would bind the wrong Minecraft identity), and felis-config
// (the on-demand BACKUP Job runs in the minecraft namespace and mounts it to
// self-record its world_backups row; without the replica the Job's volume
// mount fails and every backup request strands in the cluster).
@@ -224,7 +239,31 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
if buildNS == "" {
buildNS = platform.DefaultBuildNamespace
}
secretOutcomes := provisionSecretReplicas(ctx, cl, controlNS, cfg.K8s.Namespace, buildNS)
secretOutcomes := []systemServerOutcome{
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false),
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
// refresh=true: felis-config is the rendered config, not a credential. The
// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
// must update it when the control plane's render has moved on (a stale copy
// e.g. keeps an old database URL after a credential rotation).
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
"felis-config", "felis.toml", "config", "minecraft ns", true),
// The reaper's pre-reap warning emails authenticate with the same relay
// password felis-api uses; the reaper pod runs in the minecraft namespace,
// where a secretKeyRef resolves only against a local mirror. Skipped while
// the relay is not configured yet — the "configure email" screen refreshes
// both mirrors when it applies.
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
"felis-smtp", "password", "smtp", "minecraft ns", false),
// The build namespace needs the same token: the build Job's fetch
// initContainer reads the submission context from the internal face. Best
// effort — a deployment that only installs the control plane simply never
// builds a user submission.
ensureSecretReplica(ctx, cl, controlNS, buildNS,
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "felis-build ns", false),
}
outcomes := ensureSystemServers(ctx, cl, cfg.K8s.Namespace, cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, apiBaseURL, cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
outcomes = append(secretOutcomes, outcomes...)
fmt.Fprintln(out, "\nfelis setup: login/lobby system servers (always-on, reaper-exempt):")
@@ -240,6 +279,19 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
fmt.Fprintf(out, " - %s: skipped (%s)\n", o.name, o.skipped)
}
}
if required {
if err := requiredProvisioningError(outcomes); err != nil {
return fmt.Errorf("required Minecraft provisioning failed: %w", err)
}
fmt.Fprintln(out, "\nfelis setup: waiting for the login gate to accept players…")
err := awaitLoginGateReady(ctx, cl, cfg.K8s.Namespace, loginGateReadyTimeout, loginGatePollInterval, func(p v1alpha1.Phase) {
fmt.Fprintf(out, " login: %s\n", phaseOrPending(p))
})
if err != nil {
return err
}
fmt.Fprintln(out, " login: Ready")
}
printVelocityWiringGuidance(out, cfg.Server.RootDomain)
return nil
}
@@ -293,9 +345,7 @@ func openConfiguredSetup(ctx context.Context, cfgPath string) (*configuredSetup,
if err != nil {
return nil, &setupOpenError{stage: "load config", err: err}
}
// Pending migrations are the preflight's to apply; a newer schema is a rolled-back
// binary, and nothing this console writes would match it.
drv, err := openStore(ctx, cfg.Database.URL, true)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
return nil, &setupOpenError{stage: "open database", err: err}
}
cmd/felis/setup_panel.go
+3 -18
View File
@@ -33,6 +33,9 @@ func setupPanelNodePort() int {
}
func localPanelURL(rootDomain, adminHostname string) string {
if ip := rootDomainEmbeddedIP(rootDomain); ip != "" {
return fmt.Sprintf("https://%s:%d", ip, setupPanelNodePort())
}
host := defaultAdminHostname(rootDomain, adminHostname)
if host == "" {
return ""
@@ -54,24 +57,6 @@ func rootDomainEmbeddedIP(rootDomain string) string {
return ""
}
// setupGameAddress is where players join Minecraft: the
// IP a nip.io or sslip.io root domain spells out (nothing to resolve), otherwise the
// root domain, with the port when it is not Minecraft's default. The proxy lands
// every fresh connection on the login server whatever name it was dialled by.
func setupGameAddress(rootDomain string, gamePort int) string {
host := rootDomainEmbeddedIP(rootDomain)
if host == "" {
host = strings.TrimSpace(strings.TrimSuffix(rootDomain, "."))
}
if host == "" {
return ""
}
if gamePort == 0 || gamePort == 25565 {
return host
}
return net.JoinHostPort(host, strconv.Itoa(gamePort))
}
func localPanelOrigin() string {
return fmt.Sprintf("https://127.0.0.1:%d", setupPanelNodePort())
}
cmd/felis/status.go
-438
View File
@@ -1,438 +0,0 @@
package main
import (
"bufio"
"context"
"errors"
"flag"
"fmt"
"io"
"net"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"text/tabwriter"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
"felis.lolicon.best/internal/watchdog"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// cmdStatus prints the platform at a glance: the release and the node, the
// control plane's workloads, the game proxy, every server with its players
// and newest world backup, the database backups and the off-site copy, the
// host's disks and memory, and what the watchdog has open. It changes
// nothing, and a part that is down (the cluster, PostgreSQL) reads as such
// while the rest still prints. felis doctor says what is wrong and where to
// look.
func cmdStatus(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("status", flag.ContinueOnError)
fs.SetOutput(stderr)
unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis status: run as root (sudo felis status): it reads root-only state under /etc/felis and /var/lib/felis")
return 1
}
env, err := hostStatusEnv(*unitDir)
if err != nil {
fmt.Fprintf(stderr, "felis status: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
printStatus(ctx, env, stdout)
return 0
}
// statusEnv is what one status report reads the host through.
type statusEnv struct {
cfg *config.Config
// w is how felis-watchdog.service runs the watchdog: where the backups,
// the off-site record and the proxy are.
w watchdogFlags
cl client.Client // nil while the API server is unreachable
clErr error
backups func(ctx context.Context) (map[string]time.Time, error)
run func(ctx context.Context, name string, args ...string) ([]byte, error)
unitDir string
meminfo string
host string
now time.Time
}
// hostStatusEnv reads this host: the watchdog's settings, the configuration
// they name, and the cluster.
func hostStatusEnv(unitDir string) (statusEnv, error) {
w, _, err := watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
if err != nil {
return statusEnv{}, err
}
cfg, err := config.Load(w.cfgPath)
if err != nil {
return statusEnv{}, err
}
cl, clErr := buildSystemServerClient()
if clErr != nil {
cl = nil
}
host, _ := os.Hostname()
return statusEnv{
cfg: cfg, w: w, cl: cl, clErr: clErr,
backups: func(ctx context.Context) (map[string]time.Time, error) {
return newestWorldBackups(ctx, cfg.Database.URL)
},
run: hostCommand, unitDir: unitDir, meminfo: "/proc/meminfo", host: host, now: time.Now(),
}, nil
}
func printStatus(ctx context.Context, env statusEnv, out io.Writer) {
fmt.Fprintf(out, "felis %s on %s at %s\n", resolvedVersion(), env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
if env.cl == nil {
fmt.Fprintf(out, "cluster: unreachable (%v)\n", env.clErr)
} else {
statusCluster(ctx, env, out)
}
statusProxy(ctx, env, out)
statusServers(ctx, env, out)
statusBackups(env, out)
statusHost(env, out)
statusWatchdog(ctx, env, out)
}
func statusCluster(ctx context.Context, env statusEnv, out io.Writer) {
var nodes corev1.NodeList
if err := env.cl.List(ctx, &nodes); err != nil {
fmt.Fprintf(out, "cluster: unreachable (%v)\n", err)
return
}
for _, n := range nodes.Items {
ready := "NotReady"
for _, c := range n.Status.Conditions {
if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue {
ready = "Ready"
}
}
info := n.Status.NodeInfo
fmt.Fprintf(out, "node: %s %s, k3s %s, %s, kernel %s\n", n.Name, ready, info.KubeletVersion, info.OSImage, info.KernelVersion)
}
ns := env.w.controlNS
var deps appsv1.DeploymentList
var pods corev1.PodList
err := env.cl.List(ctx, &deps, client.InNamespace(ns))
if err == nil {
err = env.cl.List(ctx, &pods, client.InNamespace(ns))
}
fmt.Fprintf(out, "\ncontrol plane (namespace %s):\n", ns)
if err != nil {
fmt.Fprintf(out, " cannot list it: %v\n", err)
return
}
sort.Slice(deps.Items, func(i, j int) bool { return deps.Items[i].Name < deps.Items[j].Name })
tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
for _, d := range deps.Items {
want := int32(1)
if d.Spec.Replicas != nil {
want = *d.Spec.Replicas
}
image := "-"
if cs := d.Spec.Template.Spec.Containers; len(cs) > 0 {
image = shortImage(cs[0].Image)
}
fmt.Fprintf(tw, " %s\t%d/%d ready\t%s\trestarts %d\n", d.Name, d.Status.ReadyReplicas, want, image, podRestarts(d.Spec.Selector, pods.Items))
}
tw.Flush()
}
// podRestarts adds up the container restarts of the pods selector picks (the
// API server refuses a Deployment whose selector is empty).
func podRestarts(selector *metav1.LabelSelector, pods []corev1.Pod) int32 {
sel, err := metav1.LabelSelectorAsSelector(selector)
if err != nil {
return 0
}
var n int32
for _, p := range pods {
if !sel.Matches(labels.Set(p.Labels)) {
continue
}
for _, cs := range p.Status.ContainerStatuses {
n += cs.RestartCount
}
}
return n
}
// shortImage is an image reference without its registry and repository path,
// and with its digest cut to 12 characters.
func shortImage(ref string) string {
if i := strings.LastIndex(ref, "/"); i >= 0 {
ref = ref[i+1:]
}
if name, digest, ok := strings.Cut(ref, "@sha256:"); ok && len(digest) > 12 {
ref = name + "@" + digest[:12]
}
return ref
}
func statusProxy(ctx context.Context, env statusEnv, out io.Writer) {
var parts []string
if _, err := os.Stat(filepath.Join(env.unitDir, "felis-velocity.service")); err == nil {
parts = append(parts, "felis-velocity "+unitActiveState(ctx, doctorEnv{run: env.run}, "felis-velocity.service"))
}
if env.w.proxyAddr != "" {
d := net.Dialer{Timeout: 3 * time.Second}
if conn, err := d.DialContext(ctx, "tcp", env.w.proxyAddr); err != nil {
parts = append(parts, fmt.Sprintf("%s refuses connections (%v)", env.w.proxyAddr, err))
} else {
conn.Close()
parts = append(parts, env.w.proxyAddr+" accepts connections")
}
}
if len(parts) == 0 {
parts = append(parts, "not on this host")
}
fmt.Fprintf(out, "\nproxy: %s\n", strings.Join(parts, ", "))
}
func statusServers(ctx context.Context, env statusEnv, out io.Writer) {
ns := env.cfg.K8s.Namespace
if ns == "" {
ns = platform.DefaultMinecraftNamespace
}
if env.cl == nil {
fmt.Fprintf(out, "\nservers (namespace %s): unknown while the cluster is unreachable\n", ns)
return
}
var list v1alpha1.MinecraftServerList
if err := env.cl.List(ctx, &list, client.InNamespace(ns)); err != nil {
fmt.Fprintf(out, "\nservers (namespace %s): cannot list them: %v\n", ns, err)
return
}
newest, backupErr := env.backups(ctx)
running, online := 0, int32(0)
for _, ms := range list.Items {
if ms.Status.Phase == v1alpha1.PhaseRunning {
running++
online += ms.Status.Players.Online
}
}
fmt.Fprintf(out, "\nservers (namespace %s): %d, %d running, %d players online\n", ns, len(list.Items), running, online)
if len(list.Items) == 0 {
return
}
sort.Slice(list.Items, func(i, j int) bool { return list.Items[i].Name < list.Items[j].Name })
tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
fmt.Fprintln(tw, " NAME\tROLE\tDESIRED\tPHASE\tPLAYERS\tNEWEST WORLD BACKUP")
for _, ms := range list.Items {
role := ms.Labels[v1alpha1.LabelSystemRole]
if role == "" {
role = "-"
}
phase := string(ms.Status.Phase)
if phase == "" {
phase = "-"
}
players := "-"
if ms.Status.Phase == v1alpha1.PhaseRunning {
players = fmt.Sprintf("%d/%d", ms.Status.Players.Online, ms.Status.Players.Max)
}
backup := "none"
switch at, ok := newest[ms.Name]; {
case backupErr != nil:
backup = "?"
case ok:
backup = dbbackup.Age(env.now.Sub(at)) + " ago"
}
fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%s\t%s\n", ms.Name, role, orDash(string(ms.Spec.DesiredState)), phase, players, backup)
}
tw.Flush()
if backupErr != nil {
fmt.Fprintf(out, " world backups unknown: %v\n", backupErr)
}
}
func orDash(s string) string {
if s == "" {
return "-"
}
return s
}
// newestWorldBackups is when each server's newest world backup that a restore
// can use was taken.
func newestWorldBackups(ctx context.Context, url string) (map[string]time.Time, error) {
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
drv, err := store.Open(ctx, url)
if err != nil {
return nil, err
}
defer drv.Close()
rows, err := drv.DB().QueryContext(ctx, `SELECT server_name, max(created_at) FROM world_backups
WHERE status = 'present' AND corrupt_at IS NULL GROUP BY server_name`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]time.Time{}
for rows.Next() {
var name string
var at time.Time
if err := rows.Scan(&name, &at); err != nil {
return nil, err
}
out[name] = at
}
return out, rows.Err()
}
func statusBackups(env statusEnv, out io.Writer) {
fmt.Fprintln(out, "\nbackups:")
switch bundles, err := dbbackup.List(env.w.backupDir); {
case env.w.backupDir == "":
fmt.Fprintln(out, " database: not checked (felis-watchdog.service names no -backup-dir)")
case err != nil:
fmt.Fprintf(out, " database: cannot read %s: %v\n", env.w.backupDir, err)
case len(bundles) == 0:
fmt.Fprintf(out, " database: none in %s\n", env.w.backupDir)
default:
fmt.Fprintf(out, " database: newest %s, %s ago; %d bundles in %s\n",
bundles[0].Name, dbbackup.Age(env.now.Sub(bundles[0].Created)), len(bundles), env.w.backupDir)
}
if !env.cfg.Offsite.Enabled() {
fmt.Fprintln(out, " off-site: not configured, every backup is on this machine only")
return
}
switch st, err := offsite.ReadStatus(env.w.offsiteStatus); {
case err != nil:
fmt.Fprintf(out, " off-site: %v\n", err)
case st == nil:
fmt.Fprintln(out, " off-site: never synced")
case st.LastSuccess.IsZero():
fmt.Fprintf(out, " off-site: never succeeded; last attempt %s ago: %s\n", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
default:
line := fmt.Sprintf(" off-site: last good sync %s ago to %s", dbbackup.Age(env.now.Sub(st.LastSuccess)), st.Bucket)
if st.LastAttempt.After(st.LastSuccess) { // a failed run records no success
line += fmt.Sprintf("; the last attempt, %s ago, failed: %s", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
}
fmt.Fprintln(out, line)
}
}
func statusHost(env statusEnv, out io.Writer) {
fmt.Fprintln(out, "\nhost:")
seen := map[uint64]bool{}
for _, p := range splitList(env.w.diskPaths) {
var st syscall.Stat_t
if err := syscall.Stat(p, &st); err != nil {
continue
}
dev := uint64(st.Dev) // int32 on darwin
if seen[dev] {
continue
}
seen[dev] = true
var fs syscall.Statfs_t
if err := syscall.Statfs(p, &fs); err != nil || fs.Blocks == 0 {
continue
}
bsize := uint64(fs.Bsize) // uint32 on darwin
total, avail := uint64(fs.Blocks)*bsize, uint64(fs.Bavail)*bsize
fmt.Fprintf(out, " disk %s: %s free of %s (%.0f%% free)\n", p, offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)), float64(fs.Bavail)/float64(fs.Blocks)*100)
}
if total, avail, ok := readMeminfo(env.meminfo); ok {
fmt.Fprintf(out, " memory: %s available of %s\n", offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)))
}
}
// readMeminfo reads MemTotal and MemAvailable, in bytes, from a /proc/meminfo
// style file.
func readMeminfo(path string) (total, avail uint64, ok bool) {
f, err := os.Open(path)
if err != nil {
return 0, 0, false
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
fields := strings.Fields(sc.Text())
if len(fields) < 2 {
continue
}
v, _ := strconv.ParseUint(fields[1], 10, 64) // the kernel writes numbers
switch fields[0] {
case "MemTotal:":
total = v * 1024
case "MemAvailable:":
avail = v * 1024
}
}
return total, avail, total > 0
}
func statusWatchdog(ctx context.Context, env statusEnv, out io.Writer) {
fmt.Fprintln(out, "\nwatchdog:")
if _, err := os.Stat(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
fmt.Fprintln(out, " not installed: nothing checks this host")
return
}
line := " timer " + unitActiveState(ctx, doctorEnv{run: env.run}, "felis-watchdog.timer")
show, _ := env.run(ctx, "systemctl", "show", "--timestamp=unix", "-p", "Result", "-p", "ExecMainExitTimestamp", "felis-watchdog.service")
props := map[string]string{}
for _, ln := range strings.Split(string(show), "\n") {
if k, v, ok := strings.Cut(strings.TrimSpace(ln), "="); ok {
props[k] = v
}
}
// ExecMainExitTimestamp is empty until the service has run.
if sec, _ := strconv.ParseInt(strings.TrimPrefix(props["ExecMainExitTimestamp"], "@"), 10, 64); sec > 0 {
line += fmt.Sprintf(", last run %s ago (%s)", dbbackup.Age(env.now.Sub(time.Unix(sec, 0))), orDash(props["Result"]))
}
fmt.Fprintln(out, line)
state, err := watchdog.LoadState(watchdog.NewestState(env.w.statePath, env.w.fallbackState))
if err != nil {
fmt.Fprintf(out, " alerts: unknown (%v)\n", err)
return
}
var open []string
for key, a := range state.Alerts {
if !a.ClearedAt.IsZero() {
continue
}
if a.Notified.IsZero() {
open = append(open, fmt.Sprintf("%s (%s, seen %s ago, not mailed yet)", key, a.Severity, dbbackup.Age(env.now.Sub(a.FirstSeen))))
} else {
open = append(open, fmt.Sprintf("%s (%s, mailed %s ago)", key, a.Severity, dbbackup.Age(env.now.Sub(a.Notified))))
}
}
if len(open) == 0 {
fmt.Fprintln(out, " alerts: none open")
return
}
sort.Strings(open)
fmt.Fprintf(out, " alerts: %d open (sudo felis doctor says where to look)\n", len(open))
for _, o := range open {
fmt.Fprintf(out, " %s\n", o)
}
}
cmd/felis/status_test.go
-490
View File
@@ -1,490 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/watchdog"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)
// statusCluster is a one-node install: felis-api with a restarted pod, three
// servers (one of them the lobby), and a pod of another app whose restarts
// are not felis-api's.
func statusClusterObjects() []client.Object {
replicas := int32(1)
apiLabels := map[string]string{"app": "felis-api"}
return []client.Object{
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{Name: "felis-1"},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}},
NodeInfo: corev1.NodeSystemInfo{KubeletVersion: "v1.36.4+k3s1", OSImage: "CentOS Stream 9", KernelVersion: "5.14.0-630.el9.aarch64"},
},
},
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{MatchLabels: apiLabels},
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{
Name: "api", Image: "registry.felis.svc:5000/felis/felis-api:v1.4.0@sha256:0123456789abcdef0123456789abcdef",
}}}},
},
Status: appsv1.DeploymentStatus{ReadyReplicas: 1},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "felis-api-7d9", Namespace: "felis", Labels: apiLabels},
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 2}}},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "other-1", Namespace: "felis", Labels: map[string]string{"app": "other"}},
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "x", RestartCount: 5}}},
},
&v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseRunning, Players: v1alpha1.PlayersStatus{Online: 2, Max: 20}},
},
&v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: "creative", Namespace: "minecraft"},
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped},
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped},
},
&v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: "lobby", Namespace: "minecraft", Labels: map[string]string{v1alpha1.LabelSystemRole: "lobby"}},
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStarting},
},
// Another namespace's server is not this install's.
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "elsewhere", Namespace: "other"}},
}
}
// statusTestEnv is a host with the cluster above, a database backup nine hours
// old, an off-site copy last good two hours ago, and one alert open.
func statusTestEnv(t *testing.T) statusEnv {
t.Helper()
now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
dir := t.TempDir()
var w watchdogFlags
w.controlNS = "felis"
w.backupDir = filepath.Join(dir, "db-backups")
w.offsiteStatus = filepath.Join(dir, "offsite-status.json")
w.statePath = filepath.Join(dir, "state.json")
w.fallbackState = filepath.Join(dir, "fallback.json")
w.diskPaths = "/nonexistent-felis-status-test"
unitDir := filepath.Join(dir, "systemd")
for _, d := range []string{w.backupDir, unitDir} {
if err := os.MkdirAll(d, 0o700); err != nil {
t.Fatal(err)
}
}
writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-9*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-33*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
if err := offsite.WriteStatus(w.offsiteStatus, offsite.Status{
LastAttempt: now.Add(-2 * time.Hour), LastSuccess: now.Add(-2 * time.Hour), Bucket: "felis-dr", Endpoint: "https://s3.example.com",
}); err != nil {
t.Fatal(err)
}
state := &watchdog.State{Alerts: map[string]*watchdog.Alert{
"db-backup-servers": {Finding: watchdog.Finding{Key: "db-backup-servers", Severity: watchdog.Warning}, FirstSeen: now.Add(-3 * time.Hour), Notified: now.Add(-90 * time.Minute)},
"proxy": {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: now.Add(-time.Hour), Notified: now.Add(-time.Hour), ClearedAt: now.Add(-10 * time.Minute)},
"disk//var": {Finding: watchdog.Finding{Key: "disk//var", Severity: watchdog.Critical}, FirstSeen: now.Add(-4 * time.Minute)},
}}
if err := watchdog.SaveState(w.statePath, state); err != nil {
t.Fatal(err)
}
meminfo := filepath.Join(dir, "meminfo")
writeTestFile(t, meminfo, "MemTotal: 8000000 kB\nMemFree: 100000 kB\nMemAvailable: 2000000 kB\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.timer"), "[Unit]\n", 0o644)
cfg := &config.Config{Offsite: config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-dr"}}
return statusEnv{
cfg: cfg, w: w, cl: fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).Build(),
backups: func(context.Context) (map[string]time.Time, error) {
return map[string]time.Time{"survival": now.Add(-3 * time.Hour)}, nil
},
run: func(_ context.Context, name string, args ...string) ([]byte, error) {
switch strings.Join(append([]string{name}, args...), " ") {
case "systemctl is-active felis-watchdog.timer":
return []byte("active\n"), nil
case "systemctl show --timestamp=unix -p Result -p ExecMainExitTimestamp felis-watchdog.service":
return []byte("Result=success\nExecMainExitTimestamp=@" + strconv.FormatInt(now.Add(-70*time.Second).Unix(), 10) + "\n"), nil
}
return nil, errors.New("unexpected")
},
unitDir: unitDir, meminfo: meminfo, host: "felis-test", now: now,
}
}
// lineFields is the words of the first line of out that starts with prefix
// once trimmed.
func lineFields(out, prefix string) []string {
for _, l := range strings.Split(out, "\n") {
if strings.HasPrefix(strings.TrimSpace(l), prefix) {
return strings.Fields(l)
}
}
return nil
}
func TestStatusReport(t *testing.T) {
env := statusTestEnv(t)
var out bytes.Buffer
printStatus(context.Background(), env, &out)
got := out.String()
for _, want := range []string{
"on felis-test at 2026-09-27 12:00 UTC\n",
"node: felis-1 Ready, k3s v1.36.4+k3s1, CentOS Stream 9, kernel 5.14.0-630.el9.aarch64\n",
"\ncontrol plane (namespace felis):\n",
"\nproxy: not on this host\n",
"\nservers (namespace minecraft): 3, 1 running, 2 players online\n",
" database: newest " + dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily) + ", 9h0m ago; 2 bundles in " + env.w.backupDir + "\n",
" off-site: last good sync 2h0m ago to felis-dr\n",
" memory: 1.9 GiB available of 7.6 GiB\n",
" timer active, last run 1m ago (success)\n",
" alerts: 2 open (sudo felis doctor says where to look)\n" +
" db-backup-servers (warning, mailed 1h30m ago)\n" +
" disk//var (critical, seen 4m ago, not mailed yet)\n",
} {
if !strings.Contains(got, want) {
t.Errorf("status lacks %q:\n%s", want, got)
}
}
for prefix, want := range map[string]string{
"felis-api": "felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
"NAME": "NAME ROLE DESIRED PHASE PLAYERS NEWEST WORLD BACKUP",
"creative": "creative - Stopped Stopped - none",
"lobby": "lobby lobby Running Starting - none",
"survival": "survival - Running Running 2/20 3h0m ago",
} {
if f := strings.Join(lineFields(got, prefix), " "); f != want {
t.Errorf("row %q = %q, want %q\n%s", prefix, f, want, got)
}
}
if strings.Contains(got, "elsewhere") || strings.Contains(got, "other-1") {
t.Errorf("status shows what is not this install's:\n%s", got)
}
}
// Whatever is down reads as down, and the rest of the report still prints.
func TestStatusWithPartsDown(t *testing.T) {
env := statusTestEnv(t)
env.cl, env.clErr = nil, errors.New("connection refused")
env.cfg = &config.Config{}
var out bytes.Buffer
printStatus(context.Background(), env, &out)
for _, want := range []string{
"cluster: unreachable (connection refused)\n",
"\nservers (namespace minecraft): unknown while the cluster is unreachable\n",
" off-site: not configured, every backup is on this machine only\n",
" timer active, last run",
} {
if !strings.Contains(out.String(), want) {
t.Errorf("status lacks %q:\n%s", want, out.String())
}
}
env = statusTestEnv(t)
env.backups = func(context.Context) (map[string]time.Time, error) { return nil, errors.New("postgres is down") }
out.Reset()
printStatus(context.Background(), env, &out)
if f := strings.Join(lineFields(out.String(), "survival"), " "); f != "survival - Running Running 2/20 ?" {
t.Errorf("survival with PostgreSQL down = %q", f)
}
if !strings.Contains(out.String(), " world backups unknown: postgres is down\n") {
t.Errorf("status does not say why the backups are unknown:\n%s", out.String())
}
}
func TestShortImage(t *testing.T) {
for in, want := range map[string]string{
"registry.felis.svc:5000/felis/felis-api:v1.4.0": "felis-api:v1.4.0",
"docker.io/library/postgres:17@sha256:0123456789abcdef0123": "postgres:17@0123456789ab",
"felis-operator@sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba98765432": "felis-operator@fedcba987654",
"busybox": "busybox",
} {
if got := shortImage(in); got != want {
t.Errorf("shortImage(%q) = %q, want %q", in, got, want)
}
}
}
// A node that is not ready, a Deployment without replicas or containers, and
// lists the API server refuses each read as such.
func TestStatusClusterEdges(t *testing.T) {
env := statusTestEnv(t)
env.cfg = &config.Config{K8s: config.K8sConfig{Namespace: "games"}}
env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}, Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionFalse}, {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionTrue}}}},
&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-bare", Namespace: "felis"}},
&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "felis"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 4}}}},
).Build()
var out bytes.Buffer
printStatus(context.Background(), env, &out)
got := out.String()
for _, want := range []string{
"node: felis-1 NotReady, k3s , , kernel \n",
"\nservers (namespace games): 0, 0 running, 0 players online\n\nbackups:\n",
} {
if !strings.Contains(got, want) {
t.Errorf("status lacks %q:\n%s", want, got)
}
}
if f := strings.Join(lineFields(got, "felis-bare"), " "); f != "felis-bare 0/1 ready - restarts 0" {
t.Errorf("row felis-bare = %q, want its one replica wanted, no image, and no pod of its own:\n%s", f, got)
}
failing := func(what client.ObjectList) {
env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).
WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
if fmt.Sprintf("%T", list) == fmt.Sprintf("%T", what) {
return errors.New("forbidden")
}
return c.List(ctx, list, opts...)
}}).Build()
out.Reset()
printStatus(context.Background(), env, &out)
}
for _, tc := range []struct {
list client.ObjectList
want string
}{
{&corev1.NodeList{}, "cluster: unreachable (forbidden)\n\nproxy:"},
{&appsv1.DeploymentList{}, "\ncontrol plane (namespace felis):\n cannot list it: forbidden\n\nproxy:"},
{&corev1.PodList{}, "\ncontrol plane (namespace felis):\n cannot list it: forbidden\n\nproxy:"},
{&v1alpha1.MinecraftServerList{}, "\nservers (namespace games): cannot list them: forbidden\n\nbackups:"},
} {
failing(tc.list)
if !strings.Contains(out.String(), tc.want) {
t.Errorf("listing %T refused: status lacks %q:\n%s", tc.list, tc.want, out.String())
}
}
}
func TestStatusProxy(t *testing.T) {
env := statusTestEnv(t)
writeTestFile(t, filepath.Join(env.unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active"})
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
env.w.proxyAddr = ln.Addr().String()
var out bytes.Buffer
statusProxy(context.Background(), env, &out)
if want := "\nproxy: felis-velocity active, " + env.w.proxyAddr + " accepts connections\n"; out.String() != want {
t.Errorf("proxy listening: %q, want %q", out.String(), want)
}
ln.Close()
out.Reset()
statusProxy(context.Background(), env, &out)
if want := "\nproxy: felis-velocity active, " + env.w.proxyAddr + " refuses connections (dial tcp " + env.w.proxyAddr + ": "; !strings.HasPrefix(out.String(), want) {
t.Errorf("proxy gone: %q, want it to start %q", out.String(), want)
}
}
func TestStatusBackups(t *testing.T) {
env := statusTestEnv(t)
status := func() string {
var out bytes.Buffer
statusBackups(env, &out)
return out.String()
}
for _, tc := range []struct {
what, dir, want string
}{
{"no -backup-dir", "", " database: not checked (felis-watchdog.service names no -backup-dir)\n"},
{"an empty directory", t.TempDir(), " database: none in %s\n"},
{"a file where the directory should be", filepath.Join(env.w.backupDir, dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily)), " database: cannot read %s: "},
} {
env.w.backupDir = tc.dir
want := tc.want
if strings.Contains(want, "%s") {
want = fmt.Sprintf(want, tc.dir)
}
if got := status(); !strings.Contains(got, want) {
t.Errorf("%s: %q, want %q", tc.what, got, want)
}
}
for _, tc := range []struct {
what string
status *offsite.Status
raw string
want string
}{
{"never synced", nil, "", " off-site: never synced\n"},
{"never succeeded", &offsite.Status{LastAttempt: env.now.Add(-time.Hour), LastError: "403 Forbidden"}, "", " off-site: never succeeded; last attempt 1h0m ago: 403 Forbidden\n"},
{"the last attempt failed", &offsite.Status{LastAttempt: env.now.Add(-30 * time.Minute), LastSuccess: env.now.Add(-26 * time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
" off-site: last good sync 26h0m ago to felis-dr; the last attempt, 30m ago, failed: timeout\n"},
{"an error an earlier attempt left", &offsite.Status{LastAttempt: env.now.Add(-2 * time.Hour), LastSuccess: env.now.Add(-time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
" off-site: last good sync 1h0m ago to felis-dr\n"},
{"an unreadable record", nil, "{", " off-site: unexpected end of JSON input\n"},
} {
os.Remove(env.w.offsiteStatus)
switch {
case tc.status != nil:
if err := offsite.WriteStatus(env.w.offsiteStatus, *tc.status); err != nil {
t.Fatal(err)
}
case tc.raw != "":
writeTestFile(t, env.w.offsiteStatus, tc.raw, 0o600)
}
if got := status(); !strings.HasSuffix(got, tc.want) {
t.Errorf("%s: %q, want it to end %q", tc.what, got, tc.want)
}
}
}
func TestStatusHost(t *testing.T) {
env := statusTestEnv(t)
dir := t.TempDir()
env.w.diskPaths = "/nonexistent-felis-status-test," + dir + "," + dir
env.meminfo = filepath.Join(dir, "meminfo")
var out bytes.Buffer
statusHost(env, &out)
lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
if len(lines) != 3 || lines[0] != "" || lines[1] != "host:" || !strings.HasPrefix(lines[2], " disk "+dir+": ") || !strings.HasSuffix(lines[2], "% free)") {
t.Errorf("host: %q, want one line for %s (a path on a disk already shown, and one that is not there, print none) and no memory line", lines, dir)
}
writeTestFile(t, env.meminfo, "MemTotal: 4096 kB\ngarbage\nMemAvailable: 1024 kB\n", 0o644)
if total, avail, ok := readMeminfo(env.meminfo); !ok || total != 4096*1024 || avail != 1024*1024 {
t.Errorf("readMeminfo = %d, %d, %v", total, avail, ok)
}
writeTestFile(t, env.meminfo, "MemAvailable: 1024 kB\n", 0o644)
if _, _, ok := readMeminfo(env.meminfo); ok {
t.Error("readMeminfo without MemTotal: ok")
}
}
func TestStatusWatchdog(t *testing.T) {
env := statusTestEnv(t)
status := func() string {
var out bytes.Buffer
statusWatchdog(context.Background(), env, &out)
return out.String()
}
run := env.run
env.run = func(ctx context.Context, name string, args ...string) ([]byte, error) {
if len(args) > 0 && args[0] == "show" {
return []byte("Result=success\nExecMainExitTimestamp=\n"), nil
}
return run(ctx, name, args...)
}
if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: map[string]*watchdog.Alert{
"proxy": {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: env.now.Add(-time.Hour), ClearedAt: env.now.Add(-time.Minute)},
}}); err != nil {
t.Fatal(err)
}
if got, want := status(), "\nwatchdog:\n timer active\n alerts: none open\n"; got != want {
t.Errorf("a watchdog that has not run and has nothing open: %q, want %q", got, want)
}
writeTestFile(t, env.w.statePath, "{", 0o600)
if got := status(); !strings.Contains(got, "\n alerts: unknown (") {
t.Errorf("an unreadable state: %q", got)
}
if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
t.Fatal(err)
}
if got, want := status(), "\nwatchdog:\n not installed: nothing checks this host\n"; got != want {
t.Errorf("no watchdog timer: %q, want %q", got, want)
}
}
// Rows print by name in whatever order the API server lists them, a server
// the operator has not reconciled yet reads as dashes, and open alerts print
// by key in whatever order the state's map yields them.
func TestStatusOrder(t *testing.T) {
env := statusTestEnv(t)
objs := append(statusClusterObjects(),
&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-operator", Namespace: "felis"}},
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "fresh", Namespace: "minecraft"}})
env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
// The fake lists by name; the other way round, then.
if err := c.List(ctx, list, opts...); err != nil {
return err
}
items, err := meta.ExtractList(list)
if err != nil {
return err
}
slices.Reverse(items)
return meta.SetList(list, items)
}}).Build()
var out bytes.Buffer
printStatus(context.Background(), env, &out)
var rows []string
for _, l := range strings.Split(out.String(), "\n") {
if f := strings.Fields(l); len(f) > 0 && (strings.HasPrefix(f[0], "felis-") || slices.Contains([]string{"creative", "fresh", "lobby", "survival"}, f[0])) {
rows = append(rows, strings.Join(f, " "))
}
}
want := []string{
"felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
"felis-operator 0/1 ready - restarts 0",
"creative - Stopped Stopped - none",
"fresh - - - - none",
"lobby lobby Running Starting - none",
"survival - Running Running 2/20 3h0m ago",
}
if !slices.Equal(rows, want) {
t.Errorf("rows %q, want %q:\n%s", rows, want, out.String())
}
alerts := map[string]*watchdog.Alert{}
for i, key := range []string{"a", "b", "c", "d"} {
alerts[key] = &watchdog.Alert{Finding: watchdog.Finding{Key: key, Severity: watchdog.Warning}, FirstSeen: env.now.Add(-time.Duration(i+1) * time.Minute)}
}
if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: alerts}); err != nil {
t.Fatal(err)
}
wantAlerts := " alerts: 4 open (sudo felis doctor says where to look)\n" +
" a (warning, seen 1m ago, not mailed yet)\n b (warning, seen 2m ago, not mailed yet)\n" +
" c (warning, seen 3m ago, not mailed yet)\n d (warning, seen 4m ago, not mailed yet)\n"
// A map starts its walk at random: fifty walks all in order by chance
// is out of the question.
for range 50 {
out.Reset()
statusWatchdog(context.Background(), env, &out)
if !strings.HasSuffix(out.String(), wantAlerts) {
t.Fatalf("alerts %q, want them to end %q", out.String(), wantAlerts)
}
}
}
// A selector the API server would have refused picks no pods.
func TestPodRestartsBadSelector(t *testing.T) {
pods := []corev1.Pod{{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 3}}}}}
if n := podRestarts(&metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "app", Operator: "Near"}}}, pods); n != 0 {
t.Errorf("podRestarts with a bad selector = %d, want 0", n)
}
if n := podRestarts(&metav1.LabelSelector{}, pods); n != 3 {
t.Errorf("podRestarts with a selector that picks all = %d, want 3", n)
}
}
cmd/felis/supportbundle.go
-862
View File
@@ -1,862 +0,0 @@
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"errors"
"flag"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"text/tabwriter"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/watchdog"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"
)
// supportBundleDir is where felis support-bundle writes unless told otherwise.
const supportBundleDir = "/var/lib/felis/support"
// redacted stands in for every value the bundle leaves out.
const redacted = "<redacted>"
// minScrubLen is the shortest known secret value the bundle searches for: a
// shorter one would blank ordinary words, and every secret the installer
// generates is far longer.
const minScrubLen = 8
// hostSecretSources are the files on a Felis host that hold secrets; the
// bundle reads them only to take each value out of what it collects.
var hostSecretSources = secretSources{
envFiles: []string{"/etc/felis/secrets.env", defaultOffsiteEnvFile},
valueFiles: []string{hostSMTPPasswordPath, hostUploadsS3AccessKeyPath, hostUploadsS3SecretKeyPath, defaultHeartbeatFile, "/opt/felis/velocity/forwarding.secret"},
propsFiles: []string{"/opt/felis/velocity/plugins/felis-link/felis-link.properties"},
tokenFiles: []string{"/var/lib/rancher/k3s/server/token", "/var/lib/rancher/k3s/server/agent-token"},
}
// cmdSupportBundle collects what someone helping with this host needs into
// one tar.gz: felis status and felis doctor, the logs of the control plane,
// the builds and the systemd units, the cluster's workloads and events, and a
// summary of the configuration. It never collects a Secret, a ConfigMap, the
// contents of a configuration file, the database or a world, and takes every
// value of the host's secret files out of what it does collect. Game server
// logs, which carry player names, addresses and chat, only with -server-logs.
func cmdSupportBundle(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("support-bundle", flag.ContinueOnError)
fs.SetOutput(stderr)
outDir := fs.String("o", supportBundleDir, "directory to write the bundle to (created mode 0700 when missing)")
logLines := fs.Int64("log-lines", 2000, "lines kept from the end of each pod log and each unit's journal")
since := fs.Duration("since", 48*time.Hour, "how far back each unit's journal is read")
serverLogs := fs.Bool("server-logs", false, "also collect the game servers' own logs, which carry player names, IP addresses and chat")
unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis support-bundle: run as root (sudo felis support-bundle): it reads root-only logs and state")
return 1
}
b := hostSupportBundle(*unitDir, *logLines, *since, *serverLogs)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
path, err := b.write(ctx, *outDir)
if err != nil {
fmt.Fprintf(stderr, "felis support-bundle: %v\n", err)
return 1
}
size := int64(0)
if st, err := os.Stat(path); err == nil {
size = st.Size()
}
fmt.Fprintf(stdout, "wrote %s (%s, mode 0600)\n", path, humanSize(size))
fmt.Fprintln(stdout, "MANIFEST.txt inside says what it holds and what was taken out. Read it through before you send it anywhere:")
fmt.Fprintln(stdout, "redaction finds this host's known secrets and the common ways a secret is logged, and a secret logged another way stays in.")
if !*serverLogs {
fmt.Fprintln(stdout, "Game server logs are left out; -server-logs adds them (player names, IP addresses, chat).")
}
return 0
}
func humanSize(n int64) string {
switch {
case n >= 1<<20:
return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
case n >= 1<<10:
return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10))
}
return fmt.Sprintf("%d B", n)
}
// shortDuration is d as Duration.String writes it, less the zero minutes and
// seconds after whole hours or minutes: 48h, and 90m as 1h30m.
func shortDuration(d time.Duration) string {
s := d.String()
if strings.HasSuffix(s, "m0s") {
s = strings.TrimSuffix(s, "0s")
}
if strings.HasSuffix(s, "h0m") {
s = strings.TrimSuffix(s, "0m")
}
return s
}
// supportBundle is one collection and what it reads the host through.
type supportBundle struct {
host string
now time.Time
unitDir string
// w is how felis-watchdog.service runs the watchdog, wErr why it could
// not be read (w then holds the defaults).
w watchdogFlags
wErr error
cfg *config.Config // nil when cfgErr
cfgErr error
cl client.Client // nil when clErr
clErr error
logs func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error)
run func(ctx context.Context, name string, args ...string) ([]byte, error)
backups func(ctx context.Context) (map[string]time.Time, error)
doctor func(ctx context.Context, out io.Writer)
secrets secretSources
logLines int64
since time.Duration
serverLogs bool
// Host files read whole, and the directory whose listing is kept.
meminfo, osRelease, procVersion, stateDir string
}
func hostSupportBundle(unitDir string, logLines int64, since time.Duration, serverLogs bool) *supportBundle {
host, _ := os.Hostname()
b := &supportBundle{
host: host, now: time.Now(), unitDir: unitDir, run: hostCommand, secrets: hostSecretSources,
logLines: logLines, since: since, serverLogs: serverLogs,
meminfo: "/proc/meminfo", osRelease: "/etc/os-release", procVersion: "/proc/version", stateDir: "/etc/felis",
}
var found bool
b.w, found, b.wErr = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
if b.wErr == nil && !found {
b.wErr = fmt.Errorf("%s is not installed; read the watchdog's defaults", filepath.Join(unitDir, "felis-watchdog.service"))
}
if b.cfg, b.cfgErr = config.Load(b.w.cfgPath); b.cfgErr == nil {
cfg := b.cfg
b.backups = func(ctx context.Context) (map[string]time.Time, error) {
return newestWorldBackups(ctx, cfg.Database.URL)
}
} else {
b.cfg = nil
}
if b.cl, b.clErr = buildSystemServerClient(); b.clErr != nil {
b.cl = nil
} else if rc, err := hostRESTConfig(); err != nil {
b.clErr = err
b.cl = nil
} else if cs, err := kubernetes.NewForConfig(rc); err != nil {
b.clErr = err
b.cl = nil
} else {
limit := int64(8 << 20)
b.logs = func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error) {
return cs.CoreV1().Pods(ns).GetLogs(pod, &corev1.PodLogOptions{
Container: container, Previous: previous, Timestamps: true, TailLines: &logLines, LimitBytes: &limit,
}).DoRaw(ctx)
}
}
b.doctor = func(ctx context.Context, out io.Writer) {
runDoctor(ctx, doctorEnv{unitDir: unitDir, run: hostCommand, now: b.now, host: host}, out)
}
return b
}
// bundleWriter streams scrubbed files into the archive and keeps what went
// wrong while collecting, for MANIFEST.txt.
type bundleWriter struct {
tw *tar.Writer
prefix string
now time.Time
scrub *scrubber
errs []string
}
// logText adds a log, or a report that quotes errors: known secrets and
// anything logged as one come out.
func (bw *bundleWriter) logText(name string, data []byte) error {
return bw.add(name, bw.scrub.text(data))
}
// plain adds a file whose secrets were already taken out by structure (the
// cluster's objects, the configuration summary) or that names none (the
// release, disk use, addresses): known secret values still come out, and
// nothing else is rewritten.
func (bw *bundleWriter) plain(name string, data []byte) error {
return bw.add(name, bw.scrub.values(data))
}
func (bw *bundleWriter) add(name string, data []byte) error {
hdr := &tar.Header{Name: path.Join(bw.prefix, name), Mode: 0o600, Size: int64(len(data)), ModTime: bw.now, Typeflag: tar.TypeReg}
if err := bw.tw.WriteHeader(hdr); err != nil {
return err
}
_, err := bw.tw.Write(data)
return err
}
func (bw *bundleWriter) failed(what string, err error) {
bw.errs = append(bw.errs, string(bw.scrub.text([]byte(fmt.Sprintf("%s: %v", what, err)))))
}
// write collects the bundle into dir and returns its path.
func (b *supportBundle) write(ctx context.Context, dir string) (string, error) {
if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
}
stamp := b.now.UTC().Format("20060102T150405Z")
base := fmt.Sprintf("felis-support-%s-%s", safeName(b.host), stamp)
final := filepath.Join(dir, base+".tar.gz")
f, err := os.CreateTemp(dir, "."+base+".*.partial") // mode 0600
if err != nil {
return "", err
}
keep := false
defer func() {
if !keep {
f.Close()
os.Remove(f.Name())
}
}()
gz := gzip.NewWriter(f)
bw := &bundleWriter{tw: tar.NewWriter(gz), prefix: base, now: b.now, scrub: b.scrubber()}
if err := b.collect(ctx, bw); err != nil {
return "", err
}
if err := bw.tw.Close(); err != nil {
return "", err
}
if err := gz.Close(); err != nil {
return "", err
}
if err := f.Sync(); err != nil {
return "", err
}
if err := f.Close(); err != nil {
return "", err
}
if err := os.Rename(f.Name(), final); err != nil {
return "", err
}
keep = true
return final, nil
}
// safeName keeps a host name usable in a file name.
func safeName(s string) string {
s = strings.Map(func(r rune) rune {
if r == '-' || r == '.' || r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
return r
}
return '_'
}, s)
if s == "" {
return "host"
}
return s
}
func (b *supportBundle) collect(ctx context.Context, bw *bundleWriter) error {
var buf bytes.Buffer
cmdVersion(nil, &buf, io.Discard)
for _, p := range []string{b.osRelease, b.procVersion} {
if raw, err := os.ReadFile(p); err == nil {
fmt.Fprintf(&buf, "\n# %s\n%s", p, raw)
}
}
if err := bw.plain("version.txt", buf.Bytes()); err != nil {
return err
}
buf.Reset()
switch {
case b.cfgErr != nil:
fmt.Fprintf(&buf, "felis status: the configuration did not load: %v\n", b.cfgErr)
default:
printStatus(ctx, statusEnv{
cfg: b.cfg, w: b.w, cl: b.cl, clErr: b.clErr, backups: b.backups, run: b.run,
unitDir: b.unitDir, meminfo: b.meminfo, host: b.host, now: b.now,
}, &buf)
}
if err := bw.logText("status.txt", buf.Bytes()); err != nil {
return err
}
buf.Reset()
b.doctor(ctx, &buf)
if err := bw.logText("doctor.txt", buf.Bytes()); err != nil {
return err
}
if b.cfg != nil {
if err := bw.plain("config.txt", configSummary(b.cfg, b.w.cfgPath)); err != nil {
return err
}
}
if err := b.collectHost(ctx, bw); err != nil {
return err
}
if err := b.collectJournal(ctx, bw); err != nil {
return err
}
if b.cl == nil {
bw.failed("cluster", b.clErr)
} else if err := b.collectCluster(ctx, bw); err != nil {
return err
}
return bw.add("MANIFEST.txt", b.manifest(bw))
}
func (b *supportBundle) collectHost(ctx context.Context, bw *bundleWriter) error {
commands := []struct {
name string
argv []string
}{
{"host/systemd-units.txt", []string{"systemctl", "list-units", "--all", "--no-pager", "--plain", "felis-*", "k3s.service"}},
{"host/systemd-timers.txt", []string{"systemctl", "list-timers", "--all", "--no-pager", "felis-*"}},
{"host/df.txt", []string{"df", "-h"}},
{"host/addresses.txt", []string{"ip", "-brief", "address"}},
}
for _, c := range commands {
out, err := b.run(ctx, c.argv[0], c.argv[1:]...)
if err != nil && len(out) == 0 {
bw.failed(strings.Join(c.argv, " "), err)
continue
}
if err := bw.plain(c.name, out); err != nil {
return err
}
}
if raw, err := os.ReadFile(b.meminfo); err == nil {
if err := bw.plain("host/meminfo.txt", raw); err != nil {
return err
}
}
listing, err := dirListing(b.stateDir)
if err != nil {
bw.failed("list "+b.stateDir, err)
return nil
}
return bw.plain("host/etc-felis.txt", listing)
}
// dirListing names every file under dir with its mode, size and time, and
// holds nothing of what is in them.
func dirListing(dir string) ([]byte, error) {
var buf bytes.Buffer
tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
fmt.Fprintf(tw, "# %s: names, modes, sizes and times only; no contents\n", dir)
err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
info, err := d.Info()
if err != nil {
return err
}
fmt.Fprintf(tw, "%s\t%d\t%s\t%s\n", info.Mode(), info.Size(), info.ModTime().UTC().Format(time.RFC3339), p)
return nil
})
tw.Flush()
return buf.Bytes(), err
}
func (b *supportBundle) collectJournal(ctx context.Context, bw *bundleWriter) error {
units, _ := filepath.Glob(filepath.Join(b.unitDir, "felis-*.service"))
if _, err := os.Stat(filepath.Join(b.unitDir, "k3s.service")); err == nil {
units = append(units, filepath.Join(b.unitDir, "k3s.service"))
}
since := "@" + strconv.FormatInt(b.now.Add(-b.since).Unix(), 10)
for _, u := range units {
unit := filepath.Base(u)
out, err := b.run(ctx, "journalctl", "-u", unit, "--since", since, "-n", strconv.FormatInt(b.logLines, 10), "--no-pager", "-o", "short-iso")
if err != nil && len(out) == 0 {
bw.failed("journalctl -u "+unit, err)
continue
}
if err := bw.logText("journal/"+strings.TrimSuffix(unit, ".service")+".log", out); err != nil {
return err
}
}
return nil
}
// bundleNamespaces are the namespaces whose objects and logs the bundle
// collects: the control plane, the builds and the game servers.
func (b *supportBundle) bundleNamespaces() (control, build, minecraft string) {
control, build, minecraft = b.w.controlNS, platform.DefaultBuildNamespace, platform.DefaultMinecraftNamespace
if b.cfg != nil {
if b.cfg.Registry.BuildNamespace != "" {
build = b.cfg.Registry.BuildNamespace
}
if b.cfg.K8s.Namespace != "" {
minecraft = b.cfg.K8s.Namespace
}
}
return control, build, minecraft
}
func (b *supportBundle) collectCluster(ctx context.Context, bw *bundleWriter) error {
control, build, minecraft := b.bundleNamespaces()
dump := func(name string, list client.ObjectList, opts ...client.ListOption) error {
if err := b.cl.List(ctx, list, opts...); err != nil {
bw.failed("list "+name, err)
return nil
}
redactList(list)
out, err := yaml.Marshal(list)
if err != nil {
bw.failed("encode "+name, err)
return nil
}
return bw.plain(name, out)
}
if err := dump("cluster/nodes.yaml", &corev1.NodeList{}); err != nil {
return err
}
if err := dump("cluster/persistentvolumes.yaml", &corev1.PersistentVolumeList{}); err != nil {
return err
}
if err := dump("cluster/minecraftservers.yaml", &v1alpha1.MinecraftServerList{}, client.InNamespace(minecraft)); err != nil {
return err
}
for _, ns := range []string{control, build, minecraft} {
for _, k := range []struct {
name string
list client.ObjectList
}{
{"pods", &corev1.PodList{}},
{"deployments", &appsv1.DeploymentList{}},
{"statefulsets", &appsv1.StatefulSetList{}},
{"jobs", &batchv1.JobList{}},
{"cronjobs", &batchv1.CronJobList{}},
{"services", &corev1.ServiceList{}},
{"persistentvolumeclaims", &corev1.PersistentVolumeClaimList{}},
{"networkpolicies", &networkingv1.NetworkPolicyList{}},
{"events", &corev1.EventList{}},
} {
if err := dump("cluster/"+ns+"/"+k.name+".yaml", k.list, client.InNamespace(ns)); err != nil {
return err
}
}
}
var all corev1.PodList
if err := b.cl.List(ctx, &all); err != nil {
bw.failed("list every pod", err)
} else if err := bw.plain("cluster/pods-all-namespaces.txt", podTable(all.Items, b.now)); err != nil {
return err
}
for _, ns := range []string{control, build, minecraft} {
var pods corev1.PodList
if err := b.cl.List(ctx, &pods, client.InNamespace(ns)); err != nil {
continue // already recorded by the dump above
}
for _, p := range pods.Items {
if err := b.collectPodLogs(ctx, bw, p, ns == minecraft && !b.serverLogs); err != nil {
return err
}
}
}
return nil
}
// collectPodLogs keeps the tail of each container's log, and of its previous
// run when it restarted. initOnly keeps only the init containers: a game
// server's own log carries player names, addresses and chat.
func (b *supportBundle) collectPodLogs(ctx context.Context, bw *bundleWriter, p corev1.Pod, initOnly bool) error {
type ctr struct {
name string
restarts int32
}
var ctrs []ctr
restarts := map[string]int32{}
for _, cs := range append(append([]corev1.ContainerStatus(nil), p.Status.InitContainerStatuses...), p.Status.ContainerStatuses...) {
restarts[cs.Name] = cs.RestartCount
}
for _, c := range p.Spec.InitContainers {
ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
}
if !initOnly {
for _, c := range p.Spec.Containers {
ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
}
}
for _, c := range ctrs {
for _, previous := range []bool{false, true} {
if previous && c.restarts == 0 {
continue
}
name := fmt.Sprintf("logs/%s/%s/%s.log", p.Namespace, p.Name, c.name)
if previous {
name = fmt.Sprintf("logs/%s/%s/%s.previous.log", p.Namespace, p.Name, c.name)
}
out, err := b.logs(ctx, p.Namespace, p.Name, c.name, previous)
if err != nil {
bw.failed(name, err)
continue
}
if err := bw.logText(name, out); err != nil {
return err
}
}
}
return nil
}
// podTable is every pod on the node, one line each.
func podTable(pods []corev1.Pod, now time.Time) []byte {
sort.Slice(pods, func(i, j int) bool {
if pods[i].Namespace != pods[j].Namespace {
return pods[i].Namespace < pods[j].Namespace
}
return pods[i].Name < pods[j].Name
})
var buf bytes.Buffer
tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
fmt.Fprintln(tw, "NAMESPACE\tNAME\tPHASE\tREADY\tRESTARTS\tAGE")
for _, p := range pods {
var ready, restarts int32
for _, cs := range p.Status.ContainerStatuses {
if cs.Ready {
ready++
}
restarts += cs.RestartCount
}
age := "-"
if !p.CreationTimestamp.IsZero() {
age = now.Sub(p.CreationTimestamp.Time).Round(time.Minute).String()
}
fmt.Fprintf(tw, "%s\t%s\t%s\t%d/%d\t%d\t%s\n", p.Namespace, p.Name, p.Status.Phase, ready, len(p.Spec.Containers), restarts, age)
}
tw.Flush()
return buf.Bytes()
}
// redactList takes out of every object what the bundle must not carry: the
// literal env values of pod specs and of MinecraftServers (valueFrom
// references stay, naming the Secret without its contents), the
// last-applied-configuration annotation that repeats them, and managedFields.
func redactList(list client.ObjectList) {
items, err := meta.ExtractList(list)
if err != nil {
return
}
for _, it := range items {
if acc, err := meta.Accessor(it); err == nil {
acc.SetManagedFields(nil)
if ann := acc.GetAnnotations(); ann != nil {
delete(ann, corev1.LastAppliedConfigAnnotation)
acc.SetAnnotations(ann)
}
}
switch o := it.(type) {
case *corev1.Pod:
redactPodSpec(&o.Spec)
case *appsv1.Deployment:
redactPodSpec(&o.Spec.Template.Spec)
case *appsv1.StatefulSet:
redactPodSpec(&o.Spec.Template.Spec)
case *batchv1.Job:
redactPodSpec(&o.Spec.Template.Spec)
case *batchv1.CronJob:
redactPodSpec(&o.Spec.JobTemplate.Spec.Template.Spec)
case *v1alpha1.MinecraftServer:
for i := range o.Spec.Env {
if o.Spec.Env[i].Value != "" {
o.Spec.Env[i].Value = redacted
}
}
}
}
}
func redactPodSpec(s *corev1.PodSpec) {
blank := func(cs []corev1.Container) {
for i := range cs {
for j := range cs[i].Env {
if cs[i].Env[j].Value != "" {
cs[i].Env[j].Value = redacted
}
}
}
}
blank(s.InitContainers)
blank(s.Containers)
for i := range s.EphemeralContainers {
for j := range s.EphemeralContainers[i].Env {
if s.EphemeralContainers[i].Env[j].Value != "" {
s.EphemeralContainers[i].Env[j].Value = redacted
}
}
}
}
// configSummary is felis.toml without a single credential: the hostnames,
// namespaces and which features are on. Fields are picked one by one, so a
// field added later stays out until someone decides it is safe.
func configSummary(c *config.Config, path string) []byte {
var buf bytes.Buffer
line := func(k string, v any) { fmt.Fprintf(&buf, "%-34s %v\n", k, v) }
fmt.Fprintf(&buf, "# a summary of %s; no password, key or token is in it\n", path)
line("server.root_domain", c.Server.RootDomain)
line("server.listen", c.Server.Listen)
db := "(unparsable)"
if u, err := url.Parse(c.Database.URL); err == nil {
db = u.Scheme + "://" + u.User.Username() + "@" + u.Host + u.Path
}
line("database.url (no password)", db)
line("database.deployment", c.Database.Deployment)
line("velocity.public_ip", c.Velocity.PublicIP)
line("velocity.game_port", c.Velocity.GamePort)
line("velocity.login_image", c.Velocity.LoginImage)
line("velocity.lobby_image", c.Velocity.LobbyImage)
line("auth.panel_hostname", c.Auth.PanelHostname)
line("auth.admin_hostname", c.Auth.AdminHostname)
line("auth.client_ip_header", c.Auth.ClientIPHeader)
line("k8s.namespace", c.K8s.Namespace)
line("k8s.egress_mode", c.K8s.EgressMode)
line("k8s.metallb_pool", c.K8s.MetalLBPool)
line("registry.url", c.Registry.URL)
line("registry.build_namespace", c.Registry.BuildNamespace)
line("registry.trivy_db_repository", c.Registry.TrivyDBRepository)
line("registry.build_user_namespaces", c.Registry.BuildUserNamespaces)
line("registry.build_runtime_class", c.Registry.BuildRuntimeClass)
line("registry.max_concurrent_builds", c.Registry.MaxConcurrentBuilds)
line("registry.user_uploads_context", c.Registry.UserUploadsContext)
line("archive.store", c.Archive.Store)
line("archive.local_path", c.Archive.LocalPath)
line("archive.retention", c.Archive.Retention)
line("archive.scheduled_every", c.Archive.ScheduledEvery)
line("archive.scheduled_keep", c.Archive.ScheduledKeep)
line("offsite (configured)", c.Offsite.Enabled())
if c.Offsite.Enabled() {
line("offsite.endpoint", c.Offsite.Endpoint)
line("offsite.bucket", c.Offsite.Bucket)
line("offsite.prefix", c.Offsite.Prefix)
}
line("smtp.host", c.SMTP.Host)
if c.SMTP.Host != "" {
line("smtp.port", c.SMTP.Port)
line("smtp.require_tls", c.SMTP.TLSRequired())
line("smtp.max_per_hour", c.SMTP.MaxPerHour)
}
for i, s := range c.AuthSources {
line(fmt.Sprintf("auth_source[%d]", i), s.Tag+" "+s.Prefix+" "+s.URL)
}
return buf.Bytes()
}
func (b *supportBundle) manifest(bw *bundleWriter) []byte {
control, build, minecraft := b.bundleNamespaces()
var buf bytes.Buffer
fmt.Fprintf(&buf, "Felis support bundle\nhost %s, collected %s, felis %s\n\n", b.host, b.now.UTC().Format(time.RFC3339), resolvedVersion())
fmt.Fprintf(&buf, `What it holds:
status.txt, doctor.txt felis status and felis doctor at collection time
version.txt the release, the OS and the kernel
config.txt a summary of felis.toml: hostnames, namespaces, which features are on
host/ systemd units and timers, disk use, memory, addresses, and the names,
modes, sizes and times of the files under %s (not what is in them)
journal/ up to %d lines per Felis unit and k3s, from the last %s
logs/ up to %d lines of each container of the pods in %s and %s, and of the
init containers of the game server pods in %s; the previous run too
where a container restarted
cluster/ nodes, volumes, the MinecraftServers, and the pods, workloads, services,
volume claims, network policies and events of %s, %s and %s
`, b.stateDir, b.logLines, shortDuration(b.since), b.logLines, control, build, minecraft, control, build, minecraft)
if b.serverLogs {
fmt.Fprintln(&buf, "\nThe game servers' own logs are in logs/ (-server-logs): they carry player names, IP addresses and chat.")
} else {
fmt.Fprintln(&buf, "\nThe game servers' own logs are left out (they carry player names, IP addresses and chat; -server-logs adds them).")
}
fmt.Fprint(&buf, `
Never collected: Kubernetes Secrets and ConfigMaps, what is in /etc/felis or any configuration
file, the database, worlds, uploads.
Taken out:
`)
if len(bw.scrub.sources) > 0 {
fmt.Fprintf(&buf, " - %d secret values, wherever they appear, read from:\n", len(bw.scrub.vals))
for _, s := range bw.scrub.sources {
fmt.Fprintf(&buf, " %s\n", s)
}
} else {
fmt.Fprintln(&buf, " - no secret file was found on this host to take values from")
}
fmt.Fprint(&buf, ` - passwords in URLs, private keys, Bearer and Basic credentials
- in logs and command output, whatever follows password=, secret=, token=, api_key=,
access_key=, private_key= or credentials= (and the same with a colon)
- every literal env value in pod specs and MinecraftServers (valueFrom references stay)
Read it through before you send it anywhere: redaction finds this host's known secrets and the
common ways a secret is logged, and a secret logged another way stays in.
`)
if b.wErr != nil {
fmt.Fprintf(&buf, "\nThe watchdog's settings: %v\n", b.wErr)
}
if len(bw.errs) > 0 {
fmt.Fprintln(&buf, "\nNot collected:")
for _, e := range bw.errs {
fmt.Fprintf(&buf, " - %s\n", e)
}
}
// Its own words name what is redacted, and would be redacted themselves;
// what it quotes was scrubbed as it was recorded.
return buf.Bytes()
}
// secretSources are files that hold secrets, by how each is laid out.
type secretSources struct {
envFiles []string // KEY=VALUE lines, every value a secret (a commented-out one too)
valueFiles []string // one secret, the whole file
propsFiles []string // key=value lines; the keys naming a token, secret, password or key hold one
tokenFiles []string // k3s join tokens: the whole token and its secret part after the last ':'
}
// scrubber takes secrets out of what the bundle collects.
type scrubber struct {
vals []string // longest first, so a secret that contains another goes whole
sources []string // the files vals came from
}
var (
pemPrivateKey = regexp.MustCompile(`(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?-----END [A-Z0-9 ]*PRIVATE KEY-----`)
urlUserinfo = regexp.MustCompile(`([A-Za-z][A-Za-z0-9+.-]*://[^/\s:@]*:)[^/\s@]+@`)
authScheme = regexp.MustCompile(`(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}`)
secretAssign = regexp.MustCompile(`(?i)((?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|credentials?)[A-Za-z0-9_.-]*"?[ \t]*[=:][ \t]*"?)([^\s"',;&]{4,})`)
secretPropKey = regexp.MustCompile(`(?i)token|secret|password|key`)
)
func (b *supportBundle) scrubber() *scrubber {
s := &scrubber{}
s.readSources(b.secrets)
if b.cfg != nil {
if u, err := url.Parse(b.cfg.Database.URL); err == nil {
if pw, ok := u.User.Password(); ok {
s.add(pw)
}
}
for _, ref := range []string{
b.cfg.Velocity.ServiceTokenRef, b.cfg.SMTP.PasswordRef,
b.cfg.Offsite.AccessKeyRef, b.cfg.Offsite.SecretKeyRef, b.cfg.Offsite.KeyRef,
b.cfg.Registry.S3.AccessKeyRef, b.cfg.Registry.S3.SecretKeyRef,
b.cfg.Archive.S3.AccessKeyRef, b.cfg.Archive.S3.SecretKeyRef,
} {
if ref != "" {
s.add(os.Getenv(ref))
}
}
}
if st, err := watchdog.LoadState(watchdog.NewestState(b.w.statePath, b.w.fallbackState)); err == nil {
s.add(st.SMTPPassword)
}
return s
}
func (s *scrubber) add(v string) bool {
v = strings.TrimSpace(v)
if len(v) < minScrubLen {
return false
}
for _, have := range s.vals {
if have == v {
return true
}
}
s.vals = append(s.vals, v)
sort.SliceStable(s.vals, func(i, j int) bool { return len(s.vals[i]) > len(s.vals[j]) })
return true
}
func (s *scrubber) readSources(src secretSources) {
read := func(p string, take func(content string) bool) {
raw, err := os.ReadFile(p)
if err != nil {
return
}
if take(string(raw)) {
s.sources = append(s.sources, p)
}
}
keyValues := func(content string, keep func(key string) bool) bool {
found := false
for _, line := range strings.Split(content, "\n") {
k, v, ok := strings.Cut(line, "=")
if !ok || !keep(strings.TrimSpace(k)) {
continue
}
v = strings.TrimSpace(v)
if len(v) >= 2 && (v[0] == '\'' || v[0] == '"') && v[len(v)-1] == v[0] {
v = v[1 : len(v)-1]
}
found = s.add(v) || found
}
return found
}
for _, p := range src.envFiles {
read(p, func(c string) bool { return keyValues(c, func(string) bool { return true }) })
}
for _, p := range src.propsFiles {
read(p, func(c string) bool { return keyValues(c, secretPropKey.MatchString) })
}
for _, p := range src.valueFiles {
read(p, func(c string) bool { return s.add(c) })
}
for _, p := range src.tokenFiles {
read(p, func(c string) bool {
c = strings.TrimSpace(c)
whole := s.add(c)
part := false
if i := strings.LastIndex(c, ":"); i >= 0 {
part = s.add(c[i+1:])
}
return whole || part
})
}
}
// values takes every known secret value out of data.
func (s *scrubber) values(data []byte) []byte {
t := pemPrivateKey.ReplaceAllString(string(data), "<redacted private key>")
for _, v := range s.vals {
t = strings.ReplaceAll(t, v, redacted)
}
t = urlUserinfo.ReplaceAllString(t, "${1}"+redacted+"@")
t = authScheme.ReplaceAllString(t, "${1} "+redacted)
return []byte(t)
}
// text is values, and whatever a log names as a secret as well.
func (s *scrubber) text(data []byte) []byte {
return secretAssign.ReplaceAll(s.values(data), []byte("${1}"+redacted))
}
cmd/felis/supportbundle_test.go
-433
View File
@@ -1,433 +0,0 @@
package main
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/watchdog"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
// The secrets a bundle host holds, each of which must be nowhere in the bundle.
var plantedSecrets = map[string]string{
"secrets.env SERVICE_TOKEN": "svc-token-planted-0a1b2c3d",
"secrets.env DB_PASSWORD": "db-pass-planted-4e5f6a7b",
"offsite.env FELIS_OFFSITE_KEY": "offsite-key-planted+8c9d/0e1f=",
"smtp-password": "smtp-pass-planted-2a3b",
"felis-link.properties token": "props-token-planted-4c5d",
"k3s token secret part": "k3s-secret-part-planted-6e7f",
"watchdog state relay password": "cached-relay-pw-planted-8a9b",
"pod env literal": "env-literal-planted-0c1d",
"deployment env literal": "deploy-env-planted-2e3f",
"MinecraftServer env literal": "cr-env-planted-4a5b",
"last-applied annotation": "annotation-planted-6c7d",
"logged password": "hunter2-planted-8e9f",
"logged bearer": "bearerplanted0a1b2c3d",
"URL password": "urlpass-planted-4e5f",
"token in an error": "errtoken-planted-0f1e",
"felis.host.toml DB password": "cfg-db-pass-planted-1c2d",
"service token by its env ref": "env-ref-token-planted-3e4f",
"init container env literal": "init-env-planted-5a6b",
"ephemeral container env": "ephemeral-env-planted-7c8d",
"statefulset env literal": "sts-env-planted-9e0f",
"job env literal": "job-env-planted-1a2b",
"cronjob env literal": "cronjob-env-planted-3c4d",
"commented-out offsite key": "old-offsite-key-planted-5e6f",
"doctor quoting a password": "doctor-pw-planted-7a8b",
"status quoting a password": "status-pw-planted-9c0d",
"journal quoting a password": "journal-pw-planted-1e2f",
}
// bundleHost is a Felis host for felis support-bundle: its secret files, its
// watchdog unit and state, a cluster with a control-plane pod that restarted
// and a game server, and logs and a journal that name secrets.
func bundleHost(t *testing.T) *supportBundle {
t.Helper()
p := plantedSecrets
dir := t.TempDir()
etc := filepath.Join(dir, "etc-felis")
for _, d := range []string{etc, filepath.Join(dir, "systemd"), filepath.Join(dir, "k3s")} {
if err := os.MkdirAll(d, 0o700); err != nil {
t.Fatal(err)
}
}
writeTestFile(t, filepath.Join(etc, "secrets.env"), "SERVICE_TOKEN="+p["secrets.env SERVICE_TOKEN"]+"\nDB_PASSWORD="+p["secrets.env DB_PASSWORD"]+"\nSHORT=abc\n", 0o600)
writeTestFile(t, filepath.Join(etc, "offsite.env"), "# before the rotation\n# FELIS_OFFSITE_KEY="+p["commented-out offsite key"]+"\nFELIS_OFFSITE_KEY='"+p["offsite.env FELIS_OFFSITE_KEY"]+"'\n", 0o600)
writeTestFile(t, filepath.Join(etc, "smtp-password"), p["smtp-password"]+"\n", 0o600)
writeTestFile(t, filepath.Join(etc, "felis-link.properties"), "api-base-url=http://10.43.0.10:8081\nservice-token="+p["felis-link.properties token"]+"\nroot-domain=games.example.org\n", 0o640)
writeTestFile(t, filepath.Join(dir, "k3s", "token"), "K10deadbeefcafe::server:"+p["k3s token secret part"]+"\n", 0o600)
cfgPath := filepath.Join(etc, "felis.host.toml")
writeTestFile(t, cfgPath, "[database]\nurl = \"postgres://felis:"+p["felis.host.toml DB password"]+"@127.0.0.1:1/felis?sslmode=disable&connect_timeout=1\"\n"+
"[server]\nroot_domain = \"games.example.org\"\n[archive]\nstore = \"tarLocal\"\n[k8s]\negress_mode = \"nodeport\"\n"+
"[velocity]\nservice_token_ref = \"FELIS_BUNDLE_TEST_SERVICE_TOKEN\"\n", 0o600)
t.Setenv("FELIS_BUNDLE_TEST_SERVICE_TOKEN", p["service token by its env ref"])
statePath := filepath.Join(dir, "state.json")
if err := watchdog.SaveState(statePath, &watchdog.State{SMTPPassword: p["watchdog state relay password"]}); err != nil {
t.Fatal(err)
}
unitDir := filepath.Join(dir, "systemd")
writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+cfgPath+" -state "+statePath+"\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "felis-offsite.service"), "[Unit]\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "k3s.service"), "[Unit]\n", 0o644)
meminfo := filepath.Join(dir, "meminfo")
writeTestFile(t, meminfo, "MemTotal: 8000000 kB\nMemAvailable: 2000000 kB\n", 0o644)
cfg, err := config.Load(cfgPath)
if err != nil {
t.Fatal(err)
}
var w watchdogFlags
w, _, err = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
if err != nil {
t.Fatal(err)
}
w.diskPaths = "/nonexistent-felis-bundle-test"
secretEnv := corev1.EnvVar{Name: "FELIS_SMTP_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "felis-smtp"}, Key: "password"}}}
replicas := int32(1)
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "felis-api-7d9", Namespace: "felis",
Annotations: map[string]string{corev1.LastAppliedConfigAnnotation: `{"env":"` + p["last-applied annotation"] + `"}`, "felis.lolicon.best/kept": "yes"},
ManagedFields: []metav1.ManagedFieldsEntry{{Manager: "kubectl-client-side-apply"}},
},
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{{Name: "wait-db", Image: "busybox", Env: []corev1.EnvVar{{Name: "FELIS_INIT_SETTING", Value: p["init container env literal"]}}}},
Containers: []corev1.Container{{Name: "api", Image: "felis-api:v1", Env: []corev1.EnvVar{
{Name: "FELIS_PLAIN_SETTING", Value: p["pod env literal"]}, secretEnv,
}}},
EphemeralContainers: []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "debug", Env: []corev1.EnvVar{{Name: "FELIS_DEBUG_SETTING", Value: p["ephemeral container env"]}}}}},
},
Status: corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 1, Ready: true}}},
},
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
Spec: appsv1.DeploymentSpec{Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "felis-api"}},
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "api", Env: []corev1.EnvVar{
{Name: "FELIS_DEPLOY_SETTING", Value: p["deployment env literal"]},
}}}}}},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "survival-0", Namespace: "minecraft"},
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{{Name: "prepare-data"}, {Name: "egress-gate"}},
Containers: []corev1.Container{{Name: "server"}},
},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
},
&v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
Spec: v1alpha1.MinecraftServerSpec{Env: []v1alpha1.EnvVar{{Name: "DISCORD_WEBHOOK", Value: p["MinecraftServer env literal"]}}},
},
&appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: "felis-postgres", Namespace: "felis"},
Spec: appsv1.StatefulSetSpec{Template: envTemplate("FELIS_STS_SETTING", p["statefulset env literal"])},
},
&batchv1.Job{
ObjectMeta: metav1.ObjectMeta{Name: "build-1", Namespace: "felis-build"},
Spec: batchv1.JobSpec{Template: envTemplate("FELIS_JOB_SETTING", p["job env literal"])},
},
&batchv1.CronJob{
ObjectMeta: metav1.ObjectMeta{Name: "felis-reaper", Namespace: "felis"},
Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: envTemplate("FELIS_CRON_SETTING", p["cronjob env literal"])}}},
},
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}},
).Build()
secretLog := fmt.Sprintf("started with SERVICE_TOKEN=%s\nlogin password=%s ok\nAuthorization: Bearer %s\ndial postgres://felis:%s@db:5432/felis\n",
p["secrets.env SERVICE_TOKEN"], p["logged password"], p["logged bearer"], p["URL password"])
return &supportBundle{
host: "felis-test", now: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC), unitDir: unitDir,
w: w, cfg: cfg, cl: cl,
logs: func(_ context.Context, ns, pod, container string, previous bool) ([]byte, error) {
if container == "wait-db" {
return nil, errors.New("container \"wait-db\" is waiting to start; token=" + p["token in an error"])
}
return []byte(fmt.Sprintf("log of %s/%s/%s previous=%v\n%s", ns, pod, container, previous, secretLog)), nil
},
run: func(_ context.Context, name string, args ...string) ([]byte, error) {
switch name {
case "journalctl":
return []byte("journal of " + args[1] + "\nFELIS_OFFSITE_KEY=" + p["offsite.env FELIS_OFFSITE_KEY"] + "\nrelay " + p["smtp-password"] + " refused\n" +
"relay login with the cached " + p["watchdog state relay password"] + "\ndatabase auth failed for " + p["felis.host.toml DB password"] +
"\nservice token " + p["service token by its env ref"] + " rejected\nnode joined with " + p["k3s token secret part"] + "\n" +
"old copies sealed with " + p["commented-out offsite key"] + "\nretrying with password=" + p["journal quoting a password"] + "\n"), nil
case "systemctl", "df", "ip":
return []byte(name + " output\n"), nil
}
return nil, errors.New("unexpected " + name)
},
backups: func(context.Context) (map[string]time.Time, error) {
return nil, errors.New("connect: password=" + p["status quoting a password"])
},
doctor: func(_ context.Context, out io.Writer) {
fmt.Fprintf(out, "doctor report; the k3s token K10deadbeefcafe::server:%s leaked here\nprobe said password=%s\n", p["k3s token secret part"], p["doctor quoting a password"])
},
secrets: secretSources{
envFiles: []string{filepath.Join(etc, "secrets.env"), filepath.Join(etc, "offsite.env"), filepath.Join(etc, "absent.env")},
valueFiles: []string{filepath.Join(etc, "smtp-password"), filepath.Join(etc, "uploads-s3-secret-key")},
propsFiles: []string{filepath.Join(etc, "felis-link.properties")},
tokenFiles: []string{filepath.Join(dir, "k3s", "token")},
},
logLines: 500, since: 48 * time.Hour,
meminfo: meminfo, osRelease: filepath.Join(dir, "os-release"), procVersion: filepath.Join(dir, "version"), stateDir: etc,
}
}
// envTemplate is a pod template whose one container sets name to a literal.
func envTemplate(name, value string) corev1.PodTemplateSpec {
return corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Env: []corev1.EnvVar{{Name: name, Value: value}}}}}}
}
// readBundle is every file in the bundle at path, by its name inside the
// bundle's directory, and each one's mode.
func readBundle(t *testing.T, path string) (map[string]string, map[string]int64) {
t.Helper()
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
t.Fatal(err)
}
tr := tar.NewReader(gz)
files, modes := map[string]string{}, map[string]int64{}
prefix := strings.TrimSuffix(filepath.Base(path), ".tar.gz") + "/"
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
name, ok := strings.CutPrefix(hdr.Name, prefix)
if !ok {
t.Errorf("%s is outside the bundle's directory %s", hdr.Name, prefix)
}
raw, err := io.ReadAll(tr)
if err != nil {
t.Fatal(err)
}
files[name], modes[name] = string(raw), hdr.Mode
}
return files, modes
}
func TestSupportBundle(t *testing.T) {
b := bundleHost(t)
out := filepath.Join(t.TempDir(), "support")
path, err := b.write(context.Background(), out)
if err != nil {
t.Fatal(err)
}
if want := filepath.Join(out, "felis-support-felis-test-20260927T120000Z.tar.gz"); path != want {
t.Errorf("path %s, want %s", path, want)
}
for p, want := range map[string]os.FileMode{out: 0o700 | os.ModeDir, path: 0o600} {
if st, err := os.Stat(p); err != nil || st.Mode() != want {
t.Errorf("%s: mode %v (err %v), want %v", p, st.Mode(), err, want)
}
}
if left, _ := filepath.Glob(filepath.Join(out, ".*partial")); len(left) != 0 {
t.Errorf("left behind %v", left)
}
files, modes := readBundle(t, path)
var names []string
for n := range files {
names = append(names, n)
if modes[n] != 0o600 {
t.Errorf("%s: mode %o in the archive, want 600", n, modes[n])
}
}
sort.Strings(names)
want := []string{
"MANIFEST.txt",
"cluster/felis-build/cronjobs.yaml", "cluster/felis-build/deployments.yaml", "cluster/felis-build/events.yaml", "cluster/felis-build/jobs.yaml",
"cluster/felis-build/networkpolicies.yaml", "cluster/felis-build/persistentvolumeclaims.yaml", "cluster/felis-build/pods.yaml",
"cluster/felis-build/services.yaml", "cluster/felis-build/statefulsets.yaml",
"cluster/felis/cronjobs.yaml", "cluster/felis/deployments.yaml", "cluster/felis/events.yaml", "cluster/felis/jobs.yaml",
"cluster/felis/networkpolicies.yaml", "cluster/felis/persistentvolumeclaims.yaml", "cluster/felis/pods.yaml",
"cluster/felis/services.yaml", "cluster/felis/statefulsets.yaml",
"cluster/minecraft/cronjobs.yaml", "cluster/minecraft/deployments.yaml", "cluster/minecraft/events.yaml", "cluster/minecraft/jobs.yaml",
"cluster/minecraft/networkpolicies.yaml", "cluster/minecraft/persistentvolumeclaims.yaml", "cluster/minecraft/pods.yaml",
"cluster/minecraft/services.yaml", "cluster/minecraft/statefulsets.yaml",
"cluster/minecraftservers.yaml", "cluster/nodes.yaml", "cluster/persistentvolumes.yaml", "cluster/pods-all-namespaces.txt",
"config.txt", "doctor.txt",
"host/addresses.txt", "host/df.txt", "host/etc-felis.txt", "host/meminfo.txt", "host/systemd-timers.txt", "host/systemd-units.txt",
"journal/felis-offsite.log", "journal/felis-watchdog.log", "journal/k3s.log",
"logs/felis/felis-api-7d9/api.log", "logs/felis/felis-api-7d9/api.previous.log",
"logs/minecraft/survival-0/egress-gate.log", "logs/minecraft/survival-0/prepare-data.log",
"status.txt", "version.txt",
}
if strings.Join(names, "\n") != strings.Join(want, "\n") {
t.Errorf("bundle holds\n %s\nwant\n %s", strings.Join(names, "\n "), strings.Join(want, "\n "))
}
for what, secret := range plantedSecrets {
for n, body := range files {
if strings.Contains(body, secret) {
t.Errorf("%s (%s) is in %s:\n%s", what, secret, n, body)
}
}
}
// What was taken out leaves what a reader needs around it.
for name, wants := range map[string][]string{
"logs/felis/felis-api-7d9/api.previous.log": {"log of felis/felis-api-7d9/api previous=true\n", "SERVICE_TOKEN=<redacted>\n", "login password=<redacted> ok\n", "Authorization: Bearer <redacted>\n", "postgres://felis:<redacted>@db:5432/felis\n"},
"journal/k3s.log": {"journal of k3s.service\n", "FELIS_OFFSITE_KEY=<redacted>\n", "relay <redacted> refused\n",
"relay login with the cached <redacted>\n", "database auth failed for <redacted>\n", "service token <redacted> rejected\n", "node joined with <redacted>\n"},
"cluster/felis/statefulsets.yaml": {"name: FELIS_STS_SETTING\n"},
"cluster/felis/cronjobs.yaml": {"name: FELIS_CRON_SETTING\n"},
"cluster/felis-build/jobs.yaml": {"name: FELIS_JOB_SETTING\n"},
"cluster/felis/pods.yaml": {"name: FELIS_PLAIN_SETTING\n value: <redacted>\n", "name: FELIS_SMTP_PASSWORD\n valueFrom:\n secretKeyRef:\n key: password\n name: felis-smtp\n", "felis.lolicon.best/kept: \"yes\"", "name: FELIS_INIT_SETTING\n", "name: FELIS_DEBUG_SETTING\n"},
"cluster/felis/deployments.yaml": {"name: FELIS_DEPLOY_SETTING\n value: <redacted>\n"},
"cluster/minecraftservers.yaml": {"name: DISCORD_WEBHOOK\n value: <redacted>\n"},
"config.txt": {fmt.Sprintf("%-34s %s\n", "server.root_domain", "games.example.org"), fmt.Sprintf("%-34s %s\n", "database.url (no password)", "postgres://[email protected]:1/felis")},
"doctor.txt": {"the k3s token <redacted> leaked here\nprobe said password=<redacted>\n"},
"status.txt": {" world backups unknown: connect: password=<redacted>\n"},
"host/etc-felis.txt": {"secrets.env\n", "smtp-password\n", "felis-link.properties\n"},
"MANIFEST.txt": {
"The game servers' own logs are left out",
" journal/ up to 500 lines per Felis unit and k3s, from the last 48h\n",
" - 11 secret values, wherever they appear, read from:\n",
"secrets.env\n", "offsite.env\n", "smtp-password\n", "felis-link.properties\n", "token\n",
"logs/felis/felis-api-7d9/wait-db.log: container \"wait-db\" is waiting to start; token=<redacted>\n",
// Its own words about what is redacted come through whole.
" - passwords in URLs, private keys, Bearer and Basic credentials\n",
"access_key=, private_key= or credentials= (and the same with a colon)\n",
"Read it through before you send it anywhere",
},
} {
for _, w := range wants {
if !strings.Contains(files[name], w) {
t.Errorf("%s lacks %q:\n%s", name, w, files[name])
}
}
}
for _, gone := range []string{"managedFields", "last-applied-configuration"} {
if strings.Contains(files["cluster/felis/pods.yaml"], gone) {
t.Errorf("pods.yaml keeps %s:\n%s", gone, files["cluster/felis/pods.yaml"])
}
}
if strings.Contains(files["MANIFEST.txt"], "absent.env") || strings.Contains(files["MANIFEST.txt"], "uploads-s3-secret-key") {
t.Errorf("MANIFEST names files this host does not have:\n%s", files["MANIFEST.txt"])
}
}
// -server-logs adds the game servers' own logs, and says so.
func TestSupportBundleServerLogs(t *testing.T) {
b := bundleHost(t)
b.serverLogs = true
path, err := b.write(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}
files, _ := readBundle(t, path)
if _, ok := files["logs/minecraft/survival-0/server.log"]; !ok {
t.Error("no server.log with -server-logs")
}
if !strings.Contains(files["MANIFEST.txt"], "The game servers' own logs are in logs/ (-server-logs)") {
t.Errorf("MANIFEST does not say server logs are in:\n%s", files["MANIFEST.txt"])
}
}
// With the cluster and the configuration gone the bundle still holds what the
// host shows, and MANIFEST says what it could not collect.
func TestSupportBundleWithTheClusterDown(t *testing.T) {
b := bundleHost(t)
b.cl, b.clErr = nil, errors.New("connection refused")
b.cfg, b.cfgErr = nil, errors.New("felis.host.toml: no such file")
b.wErr = errors.New("felis-watchdog.service is not installed; read the watchdog's defaults")
path, err := b.write(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}
files, _ := readBundle(t, path)
for _, n := range []string{"status.txt", "doctor.txt", "host/etc-felis.txt", "journal/k3s.log"} {
if _, ok := files[n]; !ok {
t.Errorf("no %s", n)
}
}
for n := range files {
if strings.HasPrefix(n, "cluster/") || strings.HasPrefix(n, "logs/") || n == "config.txt" {
t.Errorf("%s with no cluster and no configuration", n)
}
}
if !strings.Contains(files["status.txt"], "felis status: the configuration did not load: felis.host.toml: no such file") {
t.Errorf("status.txt:\n%s", files["status.txt"])
}
if !strings.Contains(files["MANIFEST.txt"], "\nThe watchdog's settings: felis-watchdog.service is not installed; read the watchdog's defaults\n") ||
!strings.Contains(files["MANIFEST.txt"], " - cluster: connection refused\n") {
t.Errorf("MANIFEST.txt:\n%s", files["MANIFEST.txt"])
}
}
func TestShortDuration(t *testing.T) {
for d, want := range map[time.Duration]string{
48 * time.Hour: "48h",
90 * time.Minute: "1h30m",
45 * time.Minute: "45m",
30 * time.Second: "30s",
time.Hour + 30*time.Second: "1h0m30s",
} {
if got := shortDuration(d); got != want {
t.Errorf("shortDuration(%v) = %q, want %q", d, got, want)
}
}
}
func TestScrubber(t *testing.T) {
s := &scrubber{}
for _, v := range []string{"known-secret-value", "known-secret-value-longer", "short", "known-secret-value"} {
s.add(v)
}
if got := strings.Join(s.vals, ","); got != "known-secret-value-longer,known-secret-value" {
t.Errorf("kept %q; want each value once, longest first, none under %d characters", s.vals, minScrubLen)
}
for in, want := range map[string]string{
"a known-secret-value-longer b": "a <redacted> b",
"a known-secret-value b": "a <redacted> b",
"password=abcd1234 next": "password=<redacted> next",
`{"token": "abcd1234"}`: `{"token": "<redacted>"}`,
"SMTP_PASSWORD: s3cr3t!x": "SMTP_PASSWORD: <redacted>",
"x-api-key=zzzz9999&q=1": "x-api-key=<redacted>&q=1",
"Authorization: Basic dXNlcjpwYXNzd29yZA==": "Authorization: Basic <redacted>",
"s3://AKIA:secretpart123@bucket/key": "s3://AKIA:<redacted>@bucket/key",
"https://user@host/path": "https://user@host/path",
"-----BEGIN EC PRIVATE KEY-----\nMHc\n-----END EC PRIVATE KEY-----": "<redacted private key>",
"tokens: 3": "tokens: 3",
"read the relay password (keeping the cached one)": "read the relay password (keeping the cached one)",
`secrets "felis-smtp" not found`: `secrets "felis-smtp" not found`,
} {
if got := string(s.text([]byte(in))); got != want {
t.Errorf("text(%q) = %q, want %q", in, got, want)
}
}
// Structured files keep what only looks like a secret.
if got := string(s.values([]byte("secretName: felis-forwarding-secret and known-secret-value"))); got != "secretName: felis-forwarding-secret and <redacted>" {
t.Errorf("values() = %q", got)
}
}
cmd/felis/systemservers.go
+133 -126
View File
@@ -4,18 +4,18 @@ import (
"bytes"
"context"
"fmt"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/naming"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -242,15 +242,6 @@ func buildSystemServerClient() (client.Client, error) {
if err := v1alpha1.AddToScheme(scheme); err != nil {
return nil, err
}
cfg, err := hostRESTConfig()
if err != nil {
return nil, err
}
return client.New(cfg, client.Options{Scheme: scheme})
}
// hostRESTConfig is the cluster connection buildSystemServerClient describes.
func hostRESTConfig() (*rest.Config, error) {
cfg, err := ctrl.GetConfig()
if err != nil {
cfg, err = clientcmd.BuildConfigFromFlags("", hostBootstrapKubeconfigPath)
@@ -258,7 +249,7 @@ func hostRESTConfig() (*rest.Config, error) {
return nil, fmt.Errorf("no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and %s): %w", hostBootstrapKubeconfigPath, err)
}
}
return cfg, nil
return client.New(cfg, client.Options{Scheme: scheme})
}
// systemServerOutcome records what ensureSystemServers did with one service so
@@ -401,48 +392,21 @@ var derivedSystemEnv = map[string]bool{
// operator every run.
func refreshDerivedEnv(ctx context.Context, cl client.Client, existing, desired *v1alpha1.MinecraftServer) (bool, error) {
want := derivedEnvWanted(desired)
changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
changed := false
for i, e := range existing.Spec.Env {
if v, ok := want[e.Name]; ok && v != e.Value {
existing.Spec.Env[i].Value = v
changed = true
}
changed := false
for i, e := range existing.Spec.Env {
if v, ok := want[e.Name]; ok && v != e.Value {
existing.Spec.Env[i].Value = v
changed = true
}
return changed
})
if err != nil {
}
if !changed {
return false, nil
}
if err := cl.Update(ctx, existing); err != nil {
return false, fmt.Errorf("refresh %s env: %w", existing.Name, err)
}
return changed, nil
}
// patchOnConflictRetry applies mutate to obj and sends only the difference, as a
// merge patch that carries the resourceVersion obj was read at. The operator writes
// status and felis-api patches spec.idle on these same objects, so a write can land
// between setup's read and its patch: the apiserver then answers 409, and this
// re-reads obj and runs mutate again on the fresh copy, up to retry.DefaultRetry's
// five attempts. The pinned resourceVersion is what keeps a list field such as
// spec.env safe — a merge patch replaces a list whole, and without the lock an
// entry added concurrently would be dropped. mutate reports whether it changed
// anything; nothing is sent when it did not. obj holds the stored object after.
func patchOnConflictRetry(ctx context.Context, cl client.Client, obj client.Object, mutate func() bool) (bool, error) {
key := client.ObjectKeyFromObject(obj)
changed, reread := false, false
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
if reread {
if err := cl.Get(ctx, key, obj); err != nil {
return err
}
}
reread = true
base := obj.DeepCopyObject().(client.Object)
if changed = mutate(); !changed {
return nil
}
return cl.Patch(ctx, obj, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}))
})
return changed, err
return true, nil
}
// derivedEnvWanted maps the derived env keys of desired onto their values.
@@ -505,25 +469,22 @@ func convergeSystemServers(ctx context.Context, cl client.Client, namespace, log
}
var changes []string
changed, err := patchOnConflictRetry(ctx, cl, &existing, func() bool {
changes = nil
if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
existing.Spec.Rcon = desired.Spec.Rcon
changes = append(changes, "spec.rcon")
}
if existing.Spec.Startup.HealthHTTPPort == 0 && desired.Spec.Startup.HealthHTTPPort != 0 {
existing.Spec.Startup.HealthHTTPPort = desired.Spec.Startup.HealthHTTPPort
changes = append(changes, "spec.startup.healthHTTPPort")
}
changes = append(changes, convergeDerivedEnv(&existing, desired)...)
return len(changes) > 0
})
if err != nil {
outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
existing.Spec.Rcon = desired.Spec.Rcon
changes = append(changes, "spec.rcon")
}
if existing.Spec.Startup.HealthHTTPPort == 0 && desired.Spec.Startup.HealthHTTPPort != 0 {
existing.Spec.Startup.HealthHTTPPort = desired.Spec.Startup.HealthHTTPPort
changes = append(changes, "spec.startup.healthHTTPPort")
}
changes = append(changes, convergeDerivedEnv(&existing, desired)...)
if len(changes) == 0 {
outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
continue
}
if !changed {
outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
if err := cl.Update(ctx, &existing); err != nil {
outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
continue
}
outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, updated: true, changes: changes})
@@ -558,51 +519,84 @@ func convergeDerivedEnv(existing, desired *v1alpha1.MinecraftServer) []string {
return changes
}
// provisionSecretReplicas copies the Secrets workload pods mount from the control
// namespace into the namespaces those pods run in. The proxy's felis-service-token
// is not among them: it lives in the control namespace and on the host, and a copy
// anywhere else would open every game route to whoever reads that namespace.
func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, minecraftNS, buildNS string) []systemServerOutcome {
return []systemServerOutcome{
// refresh=true for both caller tokens: the control namespace holds the
// current value and `felis rotate-token` replaces it there, so a replica
// that differs is stale and the login gate would be turned away with it.
ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
naming.LimboTokenSecretName, naming.ServiceTokenSecretKey, "limbo-token", "minecraft ns", true),
ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
// refresh=true: felis-config is the rendered config, not a credential. The
// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
// must update it when the control plane's render has moved on (a stale copy
// e.g. keeps an old database URL after a credential rotation).
ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
"felis-config", "felis.toml", "config", "minecraft ns", true),
// The reaper's pre-reap warning emails authenticate with the same relay
// password felis-api uses; the reaper pod runs in the minecraft namespace,
// where a secretKeyRef resolves only against a local mirror. Skipped while
// the relay is not configured yet — the "configure email" screen refreshes
// both mirrors when it applies.
ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
"felis-smtp", "password", "smtp", "minecraft ns", false),
// The build namespace needs the build token: the build Job's fetch
// initContainer reads the submission context from the internal face, and
// that is all this token opens. Best effort — a deployment that only
// installs the control plane simply never builds a user submission.
ensureSecretReplica(ctx, cl, controlNS, buildNS,
naming.BuildTokenSecretName, naming.ServiceTokenSecretKey, "build-token", "felis-build ns", true),
// The login gate is a hard prerequisite of the Owner bind, so setup waits for it
// rather than racing it. The ceiling covers a cold image pull on a fresh node;
// the poll is fast enough that a warm start feels immediate.
const (
loginGateReadyTimeout = 5 * time.Minute
loginGatePollInterval = 3 * time.Second
)
// awaitLoginGateReady blocks until the login system server reports status.ready.
//
// The Owner claims their seat by JOINING the game and running /link, so the gate
// being up is not a nicety — it is the precondition for the very next thing setup
// asks of the operator. progress is called on each phase change so the caller can
// show movement during a cold image pull; it may be nil.
func awaitLoginGateReady(ctx context.Context, cl client.Client, namespace string, timeout, poll time.Duration, progress func(v1alpha1.Phase)) error {
key := client.ObjectKey{Namespace: namespace, Name: naming.SystemLoginServer}
deadline := time.Now().Add(timeout)
last := v1alpha1.Phase("")
for {
var ms v1alpha1.MinecraftServer
switch err := cl.Get(ctx, key, &ms); {
case err == nil:
if ms.Status.Ready {
return nil
}
if ms.Status.Phase != last {
last = ms.Status.Phase
if progress != nil {
progress(last)
}
}
// The operator only marks Failed once its OWN startup deadline has already
// elapsed, so Failed is a settled verdict rather than a transient — sitting
// out the rest of our timeout on top of it would only hide the reason.
if ms.Status.Phase == v1alpha1.PhaseFailed {
return fmt.Errorf("the login gate failed to start: %s", readyConditionMessage(&ms))
}
case !apierrors.IsNotFound(err):
return err
}
if !time.Now().Before(deadline) {
return fmt.Errorf("timed out after %s waiting for the login gate to become ready (last phase: %s)", timeout, phaseOrPending(last))
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(poll):
}
}
}
// readyConditionMessage is the operator's own account of why the gate is not
// ready — far more useful to an operator than "phase: Failed".
func readyConditionMessage(ms *v1alpha1.MinecraftServer) string {
if c := meta.FindStatusCondition(ms.Status.Conditions, v1alpha1.ConditionReady); c != nil && c.Message != "" {
return c.Message
}
return "no Ready condition was reported"
}
// phaseOrPending names the empty phase, which means the operator has not
// reconciled the server yet (commonly: the operator itself is not running).
func phaseOrPending(p v1alpha1.Phase) string {
if p == "" {
return "not yet reconciled — is the felis operator running?"
}
return string(p)
}
// ensureSecretReplica copies one Secret from the control namespace into a workload
// namespace (minecraft — or the build namespace, whose fetch initContainer reads the
// context from the felis-api internal face with the build token) so a pod can mount it
// context from the felis-api internal face with the same token) so a pod can mount it
// via secretKeyRef. A secretKeyRef is namespace-local, but those workloads do not run
// beside the control plane — so without this replica the secretKeyRef would dangle and
// wedge the pod in CreateContainerConfigError.
//
// Several Secrets need it, for different reasons: the caller tokens of the login
// limbo and the build Pod's context fetch (both authenticate to the felis-api
// internal face, each with its own token),
// Three Secrets need it, for different reasons: the service token (the login limbo and
// the build Pod's context fetch — both authenticate to the felis-api internal face),
// the Velocity modern-forwarding secret (every backend — it is how a backend knows
// a login really came from the proxy, and so that the player's UUID is Mojang-verified
// rather than offline-derived), and the SMTP relay password (the reaper's pre-reap
@@ -615,14 +609,12 @@ func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, m
// copies only Type and Data — never labels/annotations/ownerRefs — so the replica
// carries no accidental GC owner or managed-by lineage.
//
// refreshExisting switches a replica to refresh-in-place from the control namespace.
// The felis-config mirror uses it because that Secret is a rendered config and the
// workload Jobs that mount it (backup/restore/fileedit) plus the reaper silently
// misbehave on a stale copy — e.g. after a database credential rotation the control
// plane moves on while every backup Job keeps failing auth. The caller tokens use it
// because `felis rotate-token` replaces them in the control namespace, which makes a
// differing replica stale by definition. The forwarding and SMTP Secrets keep the
// never-overwrite rule.
// refreshExisting switches the felis-config mirror to refresh-in-place: that Secret is
// a rendered config, never a hand-rotated credential, and the workload Jobs that mount
// it (backup/restore/fileedit) plus the reaper silently misbehave on a stale copy —
// e.g. after a database credential rotation the control plane moves on while every
// backup Job keeps failing auth. Credential Secrets keep the never-overwrite rule so a
// rotated value survives; to rotate those, delete the replica and re-run setup.
func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label, where string, refreshExisting bool) systemServerOutcome {
name := label + " (" + where + ")"
validate := func(secret *corev1.Secret, location, skipped string) systemServerOutcome {
@@ -647,22 +639,16 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
if out := validate(&src, controlNamespace, ""); !out.available {
return out
}
changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
return false
}
if existing.Data == nil {
existing.Data = map[string][]byte{}
}
existing.Data[secretKey] = src.Data[secretKey]
return true
})
if err != nil {
return systemServerOutcome{name: name, err: err}
}
if !changed {
if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
return validate(existing, minecraftNamespace, "already current")
}
if existing.Data == nil {
existing.Data = map[string][]byte{}
}
existing.Data[secretKey] = src.Data[secretKey]
if err := cl.Update(ctx, existing); err != nil {
return systemServerOutcome{name: name, err: err}
}
return systemServerOutcome{name: name, updated: true, available: true}
}
if controlNamespace == minecraftNamespace {
@@ -723,3 +709,24 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
}
return systemServerOutcome{name: name, created: true, available: true}
}
func requiredProvisioningError(outcomes []systemServerOutcome) error {
required := map[string]struct{}{
"service-token (minecraft ns)": {},
"forwarding-secret (minecraft ns)": {},
naming.SystemLoginServer: {},
}
for _, o := range outcomes {
if o.err != nil {
return fmt.Errorf("%s: %w", o.name, o.err)
}
if _, ok := required[o.name]; ok && !o.available {
reason := o.skipped
if reason == "" {
reason = "object was not created"
}
return fmt.Errorf("%s unavailable: %s", o.name, reason)
}
}
return nil
}
cmd/felis/systemservers_test.go
+102 -76
View File
@@ -4,6 +4,7 @@ import (
"context"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/naming"
@@ -141,21 +142,21 @@ func TestLoginSystemServerEnv(t *testing.T) {
// namespace (create-if-absent), so the operator's secretKeyRef on the backend pod
// resolves. It must not overwrite an existing replica, and must degrade gracefully
// when the source is missing or the namespaces coincide. Exercised here with the
// Velocity forwarding secret, which setup replicates in this never-overwrite mode.
// service token; setup runs it a second time for the Velocity forwarding secret.
func TestEnsureSecretReplica(t *testing.T) {
scheme := newSystemServerScheme(t)
ctx := context.Background()
srcSecret := func() *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "felis"},
ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "felis"},
Type: corev1.SecretTypeOpaque,
Data: map[string][]byte{naming.ForwardingSecretKey: []byte("s3cr3t")},
Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte("s3cr3t")},
}
}
replicate := func(cl client.Client, controlNS, mcNS string) systemServerOutcome {
return ensureSecretReplica(ctx, cl, controlNS, mcNS,
naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false)
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false)
}
t.Run("replicates when absent", func(t *testing.T) {
@@ -165,19 +166,19 @@ func TestEnsureSecretReplica(t *testing.T) {
t.Fatalf("outcome = %+v, want created", out)
}
var replica corev1.Secret
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
t.Fatalf("get replica: %v", err)
}
if string(replica.Data[naming.ForwardingSecretKey]) != "s3cr3t" {
t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ForwardingSecretKey])
if string(replica.Data[naming.ServiceTokenSecretKey]) != "s3cr3t" {
t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ServiceTokenSecretKey])
}
})
t.Run("does not overwrite existing replica", func(t *testing.T) {
existing := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
Type: corev1.SecretTypeOpaque,
Data: map[string][]byte{naming.ForwardingSecretKey: []byte("rotated")},
Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte("rotated")},
}
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), existing).Build()
out := replicate(cl, "felis", "minecraft")
@@ -185,11 +186,11 @@ func TestEnsureSecretReplica(t *testing.T) {
t.Fatalf("outcome = %+v, want skipped (not clobbered)", out)
}
var replica corev1.Secret
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
t.Fatalf("get replica: %v", err)
}
if string(replica.Data[naming.ForwardingSecretKey]) != "rotated" {
t.Error("existing replica was overwritten — a hand-set value must survive")
if string(replica.Data[naming.ServiceTokenSecretKey]) != "rotated" {
t.Error("existing replica was overwritten — a rotated token must survive")
}
})
@@ -203,22 +204,22 @@ func TestEnsureSecretReplica(t *testing.T) {
t.Run("rejects a source with an empty required key", func(t *testing.T) {
bad := srcSecret()
bad.Data[naming.ForwardingSecretKey] = nil
bad.Data[naming.ServiceTokenSecretKey] = nil
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
out := replicate(cl, "felis", "minecraft")
if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
t.Fatalf("outcome = %+v, want unavailable required key", out)
}
})
t.Run("rejects an existing replica with an empty required key", func(t *testing.T) {
bad := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
Data: map[string][]byte{},
}
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), bad).Build()
out := replicate(cl, "felis", "minecraft")
if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
t.Fatalf("outcome = %+v, want unavailable existing replica", out)
}
})
@@ -244,7 +245,7 @@ func TestEnsureSecretReplica(t *testing.T) {
bad.Data = nil
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
out := replicate(cl, "felis", "felis")
if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
t.Fatalf("outcome = %+v, want unavailable required key", out)
}
})
@@ -331,6 +332,90 @@ func TestEnsureSecretReplicaRefresh(t *testing.T) {
})
}
func TestRequiredProvisioningError(t *testing.T) {
ready := []systemServerOutcome{
{name: "service-token (minecraft ns)", available: true},
{name: "forwarding-secret (minecraft ns)", available: true},
{name: naming.SystemLoginServer, available: true},
{name: naming.SystemLobbyServer, skipped: "image not configured"},
}
if err := requiredProvisioningError(ready); err != nil {
t.Fatalf("ready outcomes: %v", err)
}
missing := append([]systemServerOutcome(nil), ready...)
missing[1] = systemServerOutcome{name: "forwarding-secret (minecraft ns)", skipped: "source missing"}
if err := requiredProvisioningError(missing); err == nil || !strings.Contains(err.Error(), "forwarding-secret") {
t.Fatalf("missing forwarding secret = %v, want named error", err)
}
failed := append([]systemServerOutcome(nil), ready...)
failed[3] = systemServerOutcome{name: naming.SystemLobbyServer, err: context.DeadlineExceeded}
if err := requiredProvisioningError(failed); err == nil || !strings.Contains(err.Error(), naming.SystemLobbyServer) {
t.Fatalf("lobby create failure = %v, want immediate named error", err)
}
}
// The Owner binds by joining the game, so setup blocks on the login gate rather
// than racing it. What matters is that each ending is distinguishable: Ready
// proceeds, Failed reports the operator's own reason instead of waiting out the
// clock, and a gate that never appears (no operator reconciling it) times out
// saying so rather than dropping the operator on a bind screen that cannot work.
func TestAwaitLoginGateReady(t *testing.T) {
scheme := newSystemServerScheme(t)
ctx := context.Background()
gate := func(mut func(*v1alpha1.MinecraftServer)) *v1alpha1.MinecraftServer {
ms := &v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: naming.SystemLoginServer, Namespace: "minecraft"},
}
mut(ms)
return ms
}
t.Run("returns once the gate is ready", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gate(func(ms *v1alpha1.MinecraftServer) {
ms.Status.Phase = v1alpha1.PhaseRunning
ms.Status.Ready = true
})).Build()
if err := awaitLoginGateReady(ctx, cl, "minecraft", time.Second, 10*time.Millisecond, nil); err != nil {
t.Fatalf("await: %v", err)
}
})
t.Run("fails fast on Failed, carrying the operator's reason", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gate(func(ms *v1alpha1.MinecraftServer) {
ms.Status.Phase = v1alpha1.PhaseFailed
ms.Status.Conditions = []metav1.Condition{{
Type: v1alpha1.ConditionReady,
Status: metav1.ConditionFalse,
Reason: "StartupTimeout",
Message: "pod never became ready: ImagePullBackOff",
LastTransitionTime: metav1.Now(),
}}
})).Build()
start := time.Now()
err := awaitLoginGateReady(ctx, cl, "minecraft", time.Minute, 10*time.Millisecond, nil)
if err == nil {
t.Fatal("await: nil error, want failure")
}
if !strings.Contains(err.Error(), "ImagePullBackOff") {
t.Errorf("error = %q, want the operator's Ready-condition message", err)
}
if time.Since(start) > 5*time.Second {
t.Error("await sat out the full timeout on a settled Failed verdict")
}
})
t.Run("times out when nothing ever reconciles the gate", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(scheme).Build()
err := awaitLoginGateReady(ctx, cl, "minecraft", 30*time.Millisecond, 10*time.Millisecond, nil)
if err == nil || !strings.Contains(err.Error(), "timed out") {
t.Fatalf("await = %v, want a timeout", err)
}
})
}
func newSystemServerScheme(t *testing.T) *runtime.Scheme {
t.Helper()
scheme := runtime.NewScheme()
@@ -577,62 +662,3 @@ func TestEnsureSystemServersRefreshesDerivedEnv(t *testing.T) {
}
})
}
// Setup's replicas carry each workload its own caller token and nothing more: the
// login gate gets felis-limbo-token in the minecraft namespace, the build Jobs get
// felis-build-token in the build namespace, a replica left stale by a rotation is
// brought up to date, and the proxy's felis-service-token is copied nowhere.
func TestProvisionSecretReplicasCarryCallerTokens(t *testing.T) {
scheme := newSystemServerScheme(t)
ctx := context.Background()
secret := func(ns, name, key, val string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
Type: corev1.SecretTypeOpaque,
Data: map[string][]byte{key: []byte(val)},
}
}
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
secret("felis", "felis-service-token", "token", "proxy-tok"),
secret("felis", "felis-limbo-token", "token", "limbo-new"),
secret("felis", "felis-build-token", "token", "build-tok"),
secret("felis", "felis-ops-token", "token", "ops-tok"),
secret("felis", naming.ForwardingSecretName, naming.ForwardingSecretKey, "fwd"),
// What a rotation leaves behind before setup runs again.
secret("minecraft", "felis-limbo-token", "token", "limbo-old"),
).Build()
outcomes := provisionSecretReplicas(ctx, cl, "felis", "minecraft", "felis-build")
for _, o := range outcomes {
if o.err != nil {
t.Fatalf("%s: %v", o.name, o.err)
}
}
read := func(ns, name string) (string, bool) {
var s corev1.Secret
if err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
return "", false
}
return string(s.Data["token"]), true
}
if got, _ := read("minecraft", "felis-limbo-token"); got != "limbo-new" {
t.Errorf("minecraft/felis-limbo-token = %q, want the rotated limbo-new", got)
}
if got, _ := read("felis-build", "felis-build-token"); got != "build-tok" {
t.Errorf("felis-build/felis-build-token = %q, want build-tok", got)
}
for _, ns := range []string{"minecraft", "felis-build"} {
for _, name := range []string{"felis-service-token", "felis-ops-token"} {
if _, ok := read(ns, name); ok {
t.Errorf("%s/%s was replicated; only the control namespace holds it", ns, name)
}
}
}
if _, ok := read("minecraft", "felis-build-token"); ok {
t.Error("the build token was copied into the minecraft namespace")
}
if _, ok := read("felis-build", "felis-limbo-token"); ok {
t.Error("the limbo token was copied into the build namespace")
}
}
cmd/felis/tui_alerts_test.go
-203
View File
@@ -1,203 +0,0 @@
package main
import (
"context"
"errors"
"path/filepath"
"strings"
"testing"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"felis.lolicon.best/internal/config"
)
// TestAlertRouteLines: the summary's alert rows say where the watchdog's
// alerts go, and flag every route that reaches no one.
func TestAlertRouteLines(t *testing.T) {
cases := []struct {
name string
r alertRoute
alerts string
alertsOK bool
beat string
beatOK bool
}{
{
name: "fresh install",
alerts: "only logged: no email relay (press e)",
beat: "none: no outside check (troubleshooting.md §14)",
},
{
name: "relay, no verified owner",
r: alertRoute{relay: "smtp.example.com:587", heartbeat: "https://hc-ping.com/..."},
alerts: "only logged: no verified Owner email (panel → Account)",
beat: "https://hc-ping.com/... every 2 minutes", beatOK: true,
},
{
name: "relay, owners unreadable",
r: alertRoute{relay: "smtp.example.com:587", lookupErr: errors.New("connection refused")},
alerts: "via smtp.example.com:587; could not read the Owner addresses",
beat: "none: no outside check (troubleshooting.md §14)",
},
{
name: "owners but no relay",
r: alertRoute{recipients: []string{"[email protected]"}},
alerts: "only logged: no email relay (press e)",
beat: "none: no outside check (troubleshooting.md §14)",
},
{
name: "mailed",
r: alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]", "[email protected]"}, heartbeatErr: errors.New("/etc/felis/watchdog-heartbeat-url: the heartbeat URL is not an http:// or https:// URL")},
alerts: "mailed to [email protected], [email protected] via smtp.example.com:587", alertsOK: true,
beat: "unreadable: /etc/felis/watchdog-heartbeat-url: the heartbeat URL is not an http:// or https:// URL",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if line, ok := c.r.alertsLine(); line != c.alerts || ok != c.alertsOK {
t.Errorf("alerts = %q, %v; want %q, %v", line, ok, c.alerts, c.alertsOK)
}
if line, ok := c.r.heartbeatLine(); line != c.beat || ok != c.beatOK {
t.Errorf("heartbeat = %q, %v; want %q, %v", line, ok, c.beat, c.beatOK)
}
})
}
}
// TestSummaryAlertRows: the rows show on the card, a route that needs action
// marked so it reads without colour, and stay off a summary with no route.
func TestSummaryAlertRows(t *testing.T) {
m := &summaryModel{ownerUsername: "owner", alerts: &alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]"}}}
v := m.View()
for _, want := range []string{"alerts mailed to [email protected] via smtp.example.com:587", "heartbeat ⚠ none: no outside check"} {
if !strings.Contains(v, want) {
t.Errorf("summary lacks %q:\n%s", want, v)
}
}
if strings.Contains(v, "⚠ mailed") {
t.Errorf("a route that reaches the owners is marked:\n%s", v)
}
m.alerts = &alertRoute{heartbeat: "https://hc-ping.com/..."}
v = m.View()
for _, want := range []string{"alerts ⚠ only logged: no email relay (press e)", "heartbeat https://hc-ping.com/... every 2 minutes"} {
if !strings.Contains(v, want) {
t.Errorf("summary lacks %q:\n%s", want, v)
}
}
m.alerts = nil
if v = m.View(); strings.Contains(v, "alerts") || strings.Contains(v, "heartbeat") {
t.Errorf("a summary with no route shows alert rows:\n%s", v)
}
}
// TestRootSummaryReadsAlertRoute: the summary and the re-run status screen
// read the route each time they open, so a relay configured with e shows at
// once.
func TestRootSummaryReadsAlertRoute(t *testing.T) {
m := newTestRoot(false, consoleModeSetup, "")
reads := 0
route := alertRoute{}
m.alertRoute = func(context.Context) alertRoute {
reads++
return route
}
m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk"})
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
sum, ok := m.screen.(*summaryModel)
if !ok || sum.alerts == nil || sum.alerts.relay != "" {
t.Fatalf("summary after storage: %T %+v", m.screen, sum)
}
route = alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]"}}
m = drive(t, m, smtpResultMsg{configured: true})
sum, ok = m.screen.(*summaryModel)
if !ok || sum.alerts == nil || sum.alerts.relay != "smtp.example.com:587" {
t.Fatalf("summary after configuring email: %T %+v", m.screen, sum)
}
if reads != 2 {
t.Errorf("route read %d times, want 2", reads)
}
status := newTestRoot(true, consoleModeSetup, "")
status.alertRoute = m.alertRoute
status.showStatus()
if sum, ok := status.screen.(*summaryModel); !ok || sum.alerts == nil || sum.alerts.relay != "smtp.example.com:587" {
t.Fatalf("status screen: %T %+v", status.screen, status.screen)
}
}
// TestHostAlertRoute reads the relay from the host config and the heartbeat
// file, shows the heartbeat by its host only, and reports a database it cannot
// reach.
func TestHostAlertRoute(t *testing.T) {
dir := t.TempDir()
cfg := filepath.Join(dir, "felis.host.toml")
writeTestFile(t, cfg, testWatchdogConfig+testWatchdogSMTP, 0o600)
beat := filepath.Join(dir, "watchdog-heartbeat-url")
writeTestFile(t, beat, "https://hc-ping.com/check-key\n", 0o600)
dbURL := "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
r := hostAlertRoute(context.Background(), cfg, dbURL, beat)
if r.relay != "smtp.config.example:2525" {
t.Errorf("relay = %q", r.relay)
}
if r.heartbeat != "https://hc-ping.com/..." || r.heartbeatErr != nil {
t.Errorf("heartbeat = %q, %v", r.heartbeat, r.heartbeatErr)
}
if r.lookupErr == nil {
t.Errorf("an unreachable database reads as %v", r.recipients)
}
noRelay := filepath.Join(dir, "no-relay.toml")
writeTestFile(t, noRelay, testWatchdogConfig, 0o600)
writeTestFile(t, beat, "hc-ping.com/check-key\n", 0o600)
r = hostAlertRoute(context.Background(), noRelay, dbURL, beat)
if r.relay != "" {
t.Errorf("no [smtp]: relay = %q", r.relay)
}
if r.heartbeatErr == nil || strings.Contains(r.heartbeatErr.Error(), "check-key") {
t.Errorf("a bad heartbeat file: %v", r.heartbeatErr)
}
r = hostAlertRoute(context.Background(), noRelay, dbURL, filepath.Join(dir, "none"))
if r.heartbeat != "" || r.heartbeatErr != nil {
t.Errorf("no heartbeat file: %q, %v", r.heartbeat, r.heartbeatErr)
}
}
// TestSummaryWithAlertsFitsTerminal: the two rows keep the summary inside the
// terminal, with the longest route lines.
func TestSummaryWithAlertsFitsTerminal(t *testing.T) {
for _, w := range []int{60, 80, 90} {
for _, h := range []int{24, 30, 45} {
m := newTestRoot(false, consoleModeSetup, "")
m.alertRoute = func(context.Context) alertRoute {
return alertRoute{relay: "smtp.example.com:587", lookupErr: errors.New("dial tcp 127.0.0.1:5432: connect: connection refused")}
}
m = drive(t, m, tea.WindowSizeMsg{Width: w, Height: h})
m = drive(t, m, preflightDoneMsg{})
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.example.com"})
m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"})
if _, ok := m.screen.(*summaryModel); !ok {
t.Fatalf("screen = %T, want the summary", m.screen)
}
if got := lipgloss.Height(m.View()); got > h {
t.Errorf("terminal %dx%d: summary with alert rows = %d rows (exceeds height)", w, h, got)
}
}
}
}
// TestConsoleRootReadsHostAlertRoute: the console the host runs gives its
// summary this host's route, read against its database.
func TestConsoleRootReadsHostAlertRoute(t *testing.T) {
db := config.DatabaseConfig{URL: "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"}
rm := newConsoleRoot(context.Background(), &fakeOwnerStore{}, db, "felis.example.com", "admin.felis.example.com", "panel.felis.example.com", "", "minecraft", "root", false, consoleModeSetup, recoveryConfig{})
if rm.alertRoute == nil {
t.Fatal("the console's summary reads no alert route")
}
if r := rm.alertRoute(context.Background()); r.lookupErr == nil {
t.Errorf("the route did not query the console's database: %+v", r)
}
}
cmd/felis/tui_mc_bind.go
+223
View File
@@ -0,0 +1,223 @@
package main
import (
"context"
"strings"
"github.com/charmbracelet/bubbles/spinner"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/huh"
)
// mcBindModel is the `felis setup` Owner-establishment screen: the operator
// joins the server, runs /link to get a one-time code, and types it here. The
// bound Minecraft account is promoted to the passwordless Owner, and a one-time
// setup URL is minted for the first web login. It replaces the old ownerModel
// bootstrap form in setup mode — no username/email/password is typed here, the
// MC identity is the root of trust.
type mcBindModel struct {
ctx context.Context
store ownerStore
adminHost string
osUser string
step mcBindStep
form *huh.Form
sp spinner.Model
working string
linkCode string
ownerIdentity string
setupTokenURL string
auditWarning string
width, height int
}
type mcBindStep int
const (
mcBindForm mcBindStep = iota
mcBindWorking
mcBindDone
)
type mcBindMsg struct {
outcome breakGlassOutcome
err error
}
func newMCBindModel(ctx context.Context, store ownerStore, adminHost, osUser string) *mcBindModel {
sp := spinner.New()
sp.Spinner = spinner.Dot
sp.Style = tuiLabel
m := &mcBindModel{
ctx: ctx,
store: store,
adminHost: adminHost,
osUser: osUser,
sp: sp,
step: mcBindForm,
}
m.form = m.buildForm()
return m
}
func (m *mcBindModel) buildForm() *huh.Form {
return m.sized(newFelisForm(huh.NewGroup(
huh.NewNote().
Title("Bind your Minecraft account").
Description("Join the server and run /link to get a one-time code,\nthen type it here. Your bound account becomes the\npasswordless Owner."),
huh.NewInput().
Title("Link code").
Placeholder("ABCD12").
Value(&m.linkCode).
Validate(requiredField("link code")),
)))
}
func (m *mcBindModel) sized(f *huh.Form) *huh.Form {
if m.width > 0 {
return f.WithWidth(m.width).WithHeight(m.height)
}
return f
}
func (m *mcBindModel) setSize(w, h int) {
m.width, m.height = w, h
if m.form != nil {
m.form = m.form.WithWidth(w).WithHeight(h)
}
}
func (m *mcBindModel) Init() tea.Cmd { return m.form.Init() }
func (m *mcBindModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case mcBindMsg:
if msg.err != nil {
return m, m.failCmd(msg.err)
}
m.step = mcBindDone
m.ownerIdentity = msg.outcome.ownerIdentity
m.setupTokenURL = msg.outcome.setupTokenURL
if msg.outcome.auditErr != nil {
m.auditWarning = msg.outcome.auditErr.Error()
}
return m, nil
case spinner.TickMsg:
if m.step == mcBindWorking {
var cmd tea.Cmd
m.sp, cmd = m.sp.Update(msg)
return m, cmd
}
return m, nil
case tea.KeyMsg:
switch m.step {
case mcBindDone:
switch msg.String() {
case "ctrl+c", "esc", "enter":
return m, m.resultCmd()
}
return m, nil
case mcBindWorking:
if msg.String() == "ctrl+c" {
return m, tea.Quit
}
return m, nil
default:
switch msg.String() {
case "ctrl+c", "esc":
return m, tea.Quit
}
}
}
if m.step == mcBindForm && m.form != nil {
form, cmd := m.form.Update(msg)
if f, ok := form.(*huh.Form); ok {
m.form = f
}
switch m.form.State {
case huh.StateCompleted:
return m.onFormComplete()
case huh.StateAborted:
return m, tea.Quit
}
return m, cmd
}
return m, nil
}
func (m *mcBindModel) onFormComplete() (tea.Model, tea.Cmd) {
m.step = mcBindWorking
m.working = "Binding Minecraft account…"
code := strings.TrimSpace(strings.ToUpper(m.linkCode))
return m, tea.Batch(m.sp.Tick, func() tea.Msg {
out, err := performSetupMCBind(m.ctx, m.store, code, m.adminHost, m.osUser)
return mcBindMsg{outcome: out, err: err}
})
}
func (m *mcBindModel) failCmd(err error) tea.Cmd {
return func() tea.Msg { return ownerResultMsg{err: err} }
}
func (m *mcBindModel) resultCmd() tea.Cmd {
return func() tea.Msg {
return ownerResultMsg{
username: m.ownerIdentity,
setupTokenURL: m.setupTokenURL,
mode: "setup",
accountable: m.osUser,
auditWarning: m.auditWarning,
}
}
}
func (m *mcBindModel) View() string {
switch m.step {
case mcBindWorking:
msg := m.working
if msg == "" {
msg = "Working…"
}
return " " + m.sp.View() + " " + tuiHint.Render(msg) + "\n"
case mcBindDone:
return m.doneView()
default:
if m.form == nil {
return ""
}
return m.form.View()
}
}
func (m *mcBindModel) doneView() string {
var b strings.Builder
b.WriteString(tuiSuccessBanner("Owner account is ready.") + "\n\n")
var box strings.Builder
if m.ownerIdentity != "" {
box.WriteString(tuiLabel.Render("minecraft ") + m.ownerIdentity + "\n")
}
if m.setupTokenURL != "" {
if box.Len() > 0 {
box.WriteString("\n")
}
box.WriteString(tuiLabel.Render("setup URL ") + "\n")
for _, line := range wrapDisplayURL(m.setupTokenURL, 70) {
box.WriteString(tuiPassword.Render(line) + "\n")
}
box.WriteString("\n")
box.WriteString(tuiWarn.Render("Open this URL to complete passwordless login setup.\nIt is shown only once."))
}
if m.auditWarning != "" {
box.WriteString("\n\n" + tuiWarn.Render("Audit warning: "+m.auditWarning))
}
b.WriteString(tuiCardStyle.Render(box.String()) + "\n\n")
b.WriteString(tuiAction("enter", "continue"))
return b.String()
}
cmd/felis/tui_menu_test.go
+16 -7
View File
@@ -211,14 +211,23 @@ func TestOwnerResultCmdCarriesIsOperator(t *testing.T) {
}
}
func TestSetupOwnerCarriesAuditWarning(t *testing.T) {
m := newSetupOwnerModel(context.Background(), &fakeOwnerStore{}, "https://op.console.example.com", "root")
_, cmd := m.Update(setupOwnerMsg{outcome: breakGlassOutcome{
ownerUsername: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken", auditErr: errors.New("audit insert failed"),
func TestMCBindCarriesAuditWarning(t *testing.T) {
m := newMCBindModel(context.Background(), &fakeOwnerStore{}, "console.example.com", "root")
next, _ := m.Update(mcBindMsg{outcome: breakGlassOutcome{
ownerIdentity: "mc-uuid-1",
setupTokenURL: "https://op.console.example.com/setup?token=t0ken",
auditErr: errors.New("audit insert failed"),
}})
res := cmd().(ownerResultMsg)
if res.username != "owner" || res.auditWarning != "audit insert failed" {
t.Fatalf("result = %+v", res)
bound := next.(*mcBindModel)
if !strings.Contains(bound.doneView(), "audit insert failed") {
t.Fatalf("done view did not surface audit warning:\n%s", bound.doneView())
}
res := bound.resultCmd()().(ownerResultMsg)
if res.username != "mc-uuid-1" {
t.Fatalf("result username = %q, want verified Minecraft UUID", res.username)
}
if res.auditWarning != "audit insert failed" {
t.Fatalf("result audit warning = %q", res.auditWarning)
}
}
cmd/felis/tui_migration.go
+13 -6
View File
@@ -6,7 +6,6 @@ import (
"io"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/store"
)
@@ -14,10 +13,10 @@ import (
// applyMigrations runs any pending schema migrations and returns the resulting
// applied count. Used by the preflight stage to self-heal a freshly bootstrapped
// (or upgraded) database.
func applyMigrations(db config.DatabaseConfig) (int, error) {
func applyMigrations(dbURL string) (int, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
drv, err := store.Open(ctx, db.URL)
drv, err := store.Open(ctx, dbURL)
if err != nil {
return 0, fmt.Errorf("open db: %w", err)
}
@@ -28,15 +27,23 @@ func applyMigrations(db config.DatabaseConfig) (int, error) {
}
// Same guard as `felis migrate up`: never roll a populated database forward
// without a snapshot to roll back to.
if _, err := preMigrateBackup(ctx, drv, migrations, db, dbbackup.DefaultDir, io.Discard); err != nil {
if _, err := preMigrateBackup(ctx, drv, migrations, dbURL, dbbackup.DefaultDir, io.Discard); err != nil {
return 0, fmt.Errorf("pre-migration backup: %w", err)
}
if _, err := store.Up(ctx, drv, migrations); err != nil {
return 0, err
}
s, err := store.ReadSchema(ctx, drv)
applied, err := drv.AppliedVersions(ctx)
if err != nil {
return 0, err
}
return s.Applied, nil
return len(applied), nil
}
func totalMigrations() (int, error) {
migrations, err := store.LoadMigrations()
if err != nil {
return 0, err
}
return len(migrations), nil
}
cmd/felis/tui_owner.go
+28 -147
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"strings"
"time"
"felis.lolicon.best/internal/api"
@@ -15,8 +14,9 @@ import (
)
type owAuthMsg struct {
start recoveryStart
err error
matched string
ok bool
err error
}
type owProvisionMsg struct {
@@ -29,15 +29,14 @@ type owStep int
const (
owAuth owStep = iota
owOverride
owCode // typing the recovery code mailed to the named admin
owProvision
owWorking
owDone
owError
)
// ownerModel collects the owner account. The input phases (admin name, recovery
// code, root override, owner details) are huh forms; the async phases (verifying,
// ownerModel collects the owner account. The input phases (admin auth, root
// override, owner details) are huh forms; the async phases (verifying,
// provisioning) show a spinner; the done phase shows the credential card. The
// outward contract is unchanged: it emits an ownerResultMsg when finished.
//
@@ -56,18 +55,6 @@ type ownerModel struct {
mode string // "bootstrap", "recovery", "root_override"
accountable string
attempt string
recovery recoveryConfig
// Recovery proof: the admin the typed name resolved to, the code mailed to it,
// and once settled either how it was proven or why the run fell back to the
// override.
admin *api.StaffUser
code *recoveryCode
codeNote string // "wrong code" line shown above a rebuilt code form
verifiedBy string
codeSentTo string
skip string // otpSkip*
skipDetail string
step owStep
form *huh.Form
@@ -79,7 +66,6 @@ type ownerModel struct {
// huh-bound form values
authUser string
codeInput string
overrideTok string
ownerUser string
ownerEmail string
@@ -138,12 +124,6 @@ func newOperatorModel(ctx context.Context, store ownerStore, osUser string) *own
return m
}
// withRecovery hands the model the relay its recovery codes go through.
func (m *ownerModel) withRecovery(r recoveryConfig) *ownerModel {
m.recovery = r
return m
}
func (m *ownerModel) Init() tea.Cmd { return m.form.Init() }
// subject is the human label for the account being provisioned, branching every
@@ -172,7 +152,7 @@ func (m *ownerModel) sized(f *huh.Form) *huh.Form {
}
func (m *ownerModel) isFormStep() bool {
return m.step == owAuth || m.step == owOverride || m.step == owCode || m.step == owProvision
return m.step == owAuth || m.step == owOverride || m.step == owProvision
}
func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
@@ -181,15 +161,16 @@ func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
if msg.err != nil {
return m, m.failCmd(msg.err)
}
m.admin = msg.start.admin
if msg.start.code != nil {
m.code = msg.start.code
m.codeNote = ""
m.step = owCode
m.form = m.sized(m.buildCodeForm())
if msg.ok {
m.mode = "recovery"
m.accountable = msg.matched
m.step = owProvision
m.form = m.sized(m.buildProvisionForm())
return m, m.form.Init()
}
return m.toOverride(msg.start.skip, msg.start.detail)
m.step = owOverride
m.form = m.sized(m.buildOverrideForm())
return m, m.form.Init()
case owProvisionMsg:
if msg.err != nil {
@@ -240,9 +221,7 @@ func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
case "ctrl+c":
return m, tea.Quit
case "esc":
if m.step == owOverride || m.step == owCode {
// Start over: a code mailed for the old attempt dies with it.
m.admin, m.code, m.skip, m.skipDetail = nil, nil, "", ""
if m.step == owOverride {
m.step = owAuth
m.form = m.sized(m.buildAuthForm())
return m, m.form.Init()
@@ -274,41 +253,18 @@ func (m *ownerModel) onFormComplete() (tea.Model, tea.Cmd) {
case owAuth:
m.attempt = strings.TrimSpace(m.authUser)
m.step = owWorking
m.working = "Sending a recovery code…"
user, rc, osUser, op := m.authUser, m.recovery, m.osUser, m.operation
m.working = "Verifying admin…"
user := m.authUser
return m, tea.Batch(m.sp.Tick, func() tea.Msg {
start, err := beginRecovery(m.ctx, m.store, rc, user, osUser, op)
return owAuthMsg{start: start, err: err}
matched, ok, err := authenticateAdmin(m.ctx, m.store, user)
return owAuthMsg{matched: matched, ok: ok, err: err}
})
case owCode:
typed := strings.TrimSpace(m.codeInput)
m.codeInput = ""
if typed == breakGlassOverrideToken {
m.skip, m.skipDetail = otpSkipByOperator, ""
m.code = nil
return m.proceedAsRoot()
}
switch m.code.check(typed, m.recovery.clock()) {
case codeAccepted:
m.mode = "recovery"
m.accountable = m.admin.Username
m.verifiedBy = verifiedByEmailOTP
m.codeSentTo = m.admin.Email
m.code = nil
m.step = owProvision
m.form = m.sized(m.buildProvisionForm())
return m, m.form.Init()
case codeWrong:
m.codeNote = fmt.Sprintf("That code is wrong. %d attempts left.", m.code.attemptsLeft())
m.form = m.sized(m.buildCodeForm())
return m, m.form.Init()
case codeExpired:
return m.toOverride(otpSkipCodeExpired, "")
default:
return m.toOverride(otpSkipCodeRejected, fmt.Sprintf("%d wrong codes", recoveryCodeAttempts))
}
case owOverride:
return m.proceedAsRoot()
m.mode = "root_override"
m.accountable = m.osUser
m.step = owProvision
m.form = m.sized(m.buildProvisionForm())
return m, m.form.Init()
case owProvision:
m.username = strings.TrimSpace(m.ownerUser)
m.step = owWorking
@@ -318,24 +274,6 @@ func (m *ownerModel) onFormComplete() (tea.Model, tea.Cmd) {
return m, nil
}
// toOverride records why no code proved an admin and asks for the typed OVERRIDE.
func (m *ownerModel) toOverride(skip, detail string) (tea.Model, tea.Cmd) {
m.skip, m.skipDetail = skip, detail
m.code = nil
m.step = owOverride
m.form = m.sized(m.buildOverrideForm())
return m, m.form.Init()
}
// proceedAsRoot is the typed OVERRIDE: the run goes on as the OS user, unverified.
func (m *ownerModel) proceedAsRoot() (tea.Model, tea.Cmd) {
m.mode = "root_override"
m.accountable = m.osUser
m.step = owProvision
m.form = m.sized(m.buildProvisionForm())
return m, m.form.Init()
}
func (m *ownerModel) provisionCmd() tea.Cmd {
op := breakGlassOp{
mode: m.mode,
@@ -344,10 +282,6 @@ func (m *ownerModel) provisionCmd() tea.Cmd {
ownerUsername: m.username,
ownerEmail: m.ownerEmail,
attemptedAdmin: m.attempt,
verifiedBy: m.verifiedBy,
codeSentTo: m.codeSentTo,
otpSkipped: m.skip,
otpSkipDetail: m.skipDetail,
}
// performAddOperator and performBreakGlass share a signature; the operation
// discriminator selects which one runs. The operator path is insert-only and
@@ -386,7 +320,7 @@ func (m *ownerModel) buildAuthForm() *huh.Form {
return m.sized(newFelisForm(huh.NewGroup(
huh.NewNote().
Title("Admin authentication").
Description("A staff account already exists. Name yours: a one-time code goes to its verified email address."),
Description("A staff account already exists. Identify yourself to continue."),
huh.NewInput().
Title("Admin username").
Value(&m.authUser).
@@ -394,64 +328,11 @@ func (m *ownerModel) buildAuthForm() *huh.Form {
)))
}
func (m *ownerModel) buildCodeForm() *huh.Form {
desc := fmt.Sprintf("A recovery code went to %s, the verified address of %q. It works for %d minutes.\n\n"+
"No mail? Type %s to go on as OS user %q with root authority; the audit log records that as an unverified override. Esc starts over.",
maskEmail(m.admin.Email), m.admin.Username, int(recoveryCodeTTL/time.Minute), breakGlassOverrideToken, m.osUser)
if m.codeNote != "" {
desc = m.codeNote + "\n\n" + desc
}
return m.sized(newFelisForm(huh.NewGroup(
huh.NewNote().Title("Email verification").Description(desc),
huh.NewInput().
Title("Recovery code").
Value(&m.codeInput).
Validate(func(s string) error {
s = strings.TrimSpace(s)
if s == breakGlassOverrideToken || isRecoveryCodeShape(s) {
return nil
}
return errors.New("enter the 6-digit code, or " + breakGlassOverrideToken)
}),
)))
}
func isRecoveryCodeShape(s string) bool {
if len(s) != 6 {
return false
}
for _, c := range s {
if c < '0' || c > '9' {
return false
}
}
return true
}
// overrideReason says why no code proved an admin, first line of the override form.
func (m *ownerModel) overrideReason() string {
switch m.skip {
case otpSkipUnknownAdmin:
return fmt.Sprintf("No staff account is named %q.", m.attempt)
case otpSkipNoVerifiedEmail:
return fmt.Sprintf("%q has no verified email address, so no recovery code can reach it.", m.admin.Username)
case otpSkipNoRelay:
return "No mail relay can send a recovery code: " + m.skipDetail + "."
case otpSkipSendFailed:
return "The recovery code could not be sent: " + m.skipDetail + "."
case otpSkipCodeExpired:
return "The recovery code expired."
case otpSkipCodeRejected:
return fmt.Sprintf("%d wrong codes; that code no longer works.", recoveryCodeAttempts)
}
return "No admin was verified."
}
func (m *ownerModel) buildOverrideForm() *huh.Form {
return m.sized(newFelisForm(huh.NewGroup(
huh.NewNote().
Title("Root override").
Description(m.overrideReason()+"\n\n"+fmt.Sprintf("Proceed as OS user %q with root authority by typing the confirmation token. The audit log records this run as an unverified root override and the reason above. Esc starts over.", m.osUser)),
Description(fmt.Sprintf("That credential did not match. Proceed as OS user %q with root authority by typing the confirmation token.", m.osUser)),
huh.NewInput().
Title("Type "+breakGlassOverrideToken+" to confirm").
Value(&m.overrideTok).
@@ -471,14 +352,14 @@ func (m *ownerModel) buildProvisionForm() *huh.Form {
desc := fmt.Sprintf("Create the first Owner — recorded as OS user %q.", m.osUser)
switch m.mode {
case "recovery":
desc = fmt.Sprintf("Verified as %q by an email code.", m.accountable)
desc = fmt.Sprintf("Authenticated as %q.", m.accountable)
case "root_override":
desc = "Root override — the Owner will be reset."
}
if m.operation == bgAddOperator {
switch m.mode {
case "recovery":
desc = fmt.Sprintf("Add an Operator — verified as %q by an email code.", m.accountable)
desc = fmt.Sprintf("Add an Operator — authenticated as %q.", m.accountable)
case "root_override":
desc = "Add an Operator (root override)."
}
cmd/felis/tui_postgres.go
+10 -4
View File
@@ -72,14 +72,20 @@ func parseDBURL(url string) (dbCfg, error) {
}, nil
}
// readSchema lines the database's recorded migrations up with this build's.
func readSchema(dbURL string) (store.Schema, error) {
func countMigrations(dbURL string) (int, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
drv, err := store.Open(ctx, dbURL)
if err != nil {
return store.Schema{}, err
return 0, err
}
defer drv.Close()
return store.ReadSchema(ctx, drv)
if err := drv.EnsureVersionTable(ctx); err != nil {
return 0, err
}
applied, err := drv.AppliedVersions(ctx)
if err != nil {
return 0, err
}
return len(applied), nil
}
cmd/felis/tui_preflight.go
+11 -16
View File
@@ -3,8 +3,6 @@ package main
import (
"fmt"
"felis.lolicon.best/internal/config"
"github.com/charmbracelet/bubbles/spinner"
tea "github.com/charmbracelet/bubbletea"
)
@@ -16,7 +14,7 @@ import (
// of the wizard can assume a healthy backend. It is non-interactive — it runs
// to completion and reports back to the root via preflightDoneMsg.
type preflightModel struct {
db config.DatabaseConfig
dbURL string
rootDomain string
adminHost string
@@ -48,7 +46,6 @@ type pfDBMsg struct{ err error }
type pfMigCheckMsg struct {
applied int
total int
pending bool
err error
}
@@ -59,11 +56,11 @@ type pfMigApplyMsg struct {
type pfPanelMsg struct{ err error }
func newPreflightModel(db config.DatabaseConfig, rootDomain, adminHostname string) *preflightModel {
func newPreflightModel(dbURL, rootDomain, adminHostname string) *preflightModel {
sp := spinner.New()
sp.Spinner = spinner.Dot
sp.Style = tuiLabel
return &preflightModel{db: db, rootDomain: rootDomain, adminHost: adminHostname, sp: sp, state: pfCheckDB}
return &preflightModel{dbURL: dbURL, rootDomain: rootDomain, adminHost: adminHostname, sp: sp, state: pfCheckDB}
}
func (m *preflightModel) Init() tea.Cmd {
@@ -87,7 +84,7 @@ func (m *preflightModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, nil
}
m.applied, m.total = msg.applied, msg.total
if msg.pending {
if msg.applied < msg.total {
m.state = pfApplyMig
return m, m.applyMigrations()
}
@@ -202,25 +199,23 @@ func (m *preflightModel) errText() string {
}
func (m *preflightModel) checkDB() tea.Cmd {
return func() tea.Msg { return pfDBMsg{err: checkPostgres(m.db.URL)} }
return func() tea.Msg { return pfDBMsg{err: checkPostgres(m.dbURL)} }
}
func (m *preflightModel) checkMigrations() tea.Cmd {
return func() tea.Msg {
// The sets, not their sizes: a database a newer release migrated can hold as
// many rows as this build has migrations, and must stop here rather than be
// "healed" by an older binary.
s, err := readSchema(m.db.URL)
if err == nil {
err = s.Newer()
applied, err := countMigrations(m.dbURL)
if err != nil {
return pfMigCheckMsg{err: err}
}
return pfMigCheckMsg{applied: s.Applied, total: s.Total, pending: len(s.Pending) > 0, err: err}
total, err := totalMigrations()
return pfMigCheckMsg{applied: applied, total: total, err: err}
}
}
func (m *preflightModel) applyMigrations() tea.Cmd {
return func() tea.Msg {
applied, err := applyMigrations(m.db)
applied, err := applyMigrations(m.dbURL)
return pfMigApplyMsg{applied: applied, err: err}
}
}
cmd/felis/tui_root.go
+36 -83
View File
@@ -5,7 +5,6 @@ import (
"strings"
"felis.lolicon.best/internal/cfsetup"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/platform"
tea "github.com/charmbracelet/bubbletea"
@@ -94,9 +93,9 @@ type wizardStage int
const (
stagePreflight wizardStage = iota
stageOwner
stageConnect
stageStorage
stageOwner
stageSummary
// stageMenu is the break-glass operation menu. It is appended last so the
// setup-flow rail indices (Preflight…Done) are unshifted; the rail is suppressed
@@ -109,7 +108,7 @@ const (
// and the post-install wizard owns cells 1–4. Defining it once keeps the two
// programs' breadcrumbs identical so the rail reads as a single continuous bar
// rather than restarting when the wizard takes over.
var setupRailSteps = []string{"Bootstrap", "Preflight", "Connection", "Storage", "Panel login", "Done"}
var setupRailSteps = []string{"Bootstrap", "Preflight", "Owner", "Connection", "Storage", "Done"}
type rootModel struct {
ctx context.Context
@@ -134,7 +133,7 @@ type rootModel struct {
err error
mode consoleMode
db config.DatabaseConfig
dbURL string
store ownerStore
osUser string
rootDomain string
@@ -143,17 +142,13 @@ type rootModel struct {
accessAud string
namespace string // minecraft workload namespace (cfg.K8s.Namespace); target of the halt op
adminExists bool
recovery recoveryConfig // how the account operations mail a recovery code
// alertRoute reads where the watchdog's alerts go for the summary; nil
// leaves those rows out.
alertRoute func(context.Context) alertRoute
}
func newRootModel(ctx context.Context, store ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) *rootModel {
func newRootModel(ctx context.Context, store ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) *rootModel {
rm := &rootModel{
ctx: ctx,
reviewing: -1,
db: db,
dbURL: dbURL,
store: store,
osUser: osUser,
rootDomain: rootDomain,
@@ -184,7 +179,7 @@ func newRootModel(ctx context.Context, store ownerStore, db config.DatabaseConfi
}
} else {
rm.stage = stagePreflight
rm.screen = newPreflightModel(db, rootDomain, adminHostname)
rm.screen = newPreflightModel(dbURL, rootDomain, adminHostname)
}
return rm
}
@@ -211,14 +206,14 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
case preflightDoneMsg:
if m.adminExists {
return m.startOwnerSetup()
// Re-run: setup already happened. Land on the status screen.
return m.showStatus()
}
m.stage = stageConnect
return m.adopt(newConnectChooserModel(m.rootDomain, m.adminHost, m.panelHost))
case setupReadyMsg:
m.result.username = msg.username
return m.showStatus()
m.stage = stageOwner
if m.mode == consoleModeSetup {
return m.adopt(newMCBindModel(m.ctx, m.store, defaultAdminHostname(m.rootDomain, m.adminHost), m.osUser))
}
return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, false))
case menuChoiceMsg:
// The break-glass menu picked an account operation; build its screen. Both reuse
@@ -226,7 +221,7 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.stage = stageOwner
switch msg.op {
case bgAddOperator:
return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser).withRecovery(m.recovery))
return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser))
case bgHaltServer:
return m.adopt(newHaltModel(m.ctx, m.store, m.namespace, m.osUser))
case bgSyncBackup:
@@ -234,7 +229,7 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
// Service + service token the peer dials live in the control namespace.
return m.adopt(newBackupModel(m.ctx, m.namespace, platform.DefaultControlNamespace, m.osUser))
default:
return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists).withRecovery(m.recovery))
return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists))
}
case haltResultMsg:
@@ -289,19 +284,14 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, tea.Quit
}
m.adminExists = true
if m.result.alreadySetUp {
return m.showStatus()
}
return m.showSummary()
m.stage = stageConnect
return m.adopt(newConnectChooserModel(m.rootDomain, m.adminHost, m.panelHost))
case connectResultMsg:
m.applyConnectResult(msg)
if m.reconfiguringConnect {
// Changing only the connection — storage is already set, so skip it.
m.reconfiguringConnect = false
if m.result.setupTokenURL != "" {
return m.startOwnerSetup()
}
return m.showSummary()
}
m.stage = stageStorage
@@ -310,9 +300,6 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
case storageResultMsg:
m.result.storageMethod = msg.method
m.result.storageDetail = msg.detail
if !m.adminExists {
return m.startOwnerSetup()
}
return m.showSummary()
case storageBackMsg:
@@ -423,7 +410,7 @@ func (m *rootModel) displayStage() int {
// reviewBody renders a read-only recap of an already-completed step. Steps in
// this wizard commit as you finish them (the Owner account and its one-time
// login link are created on submit), so review is deliberately look-only — there
// password are created on submit), so review is deliberately look-only — there
// is no re-editing a step you've passed.
func (m *rootModel) reviewBody(stage int) string {
var b strings.Builder
@@ -432,8 +419,11 @@ func (m *rootModel) reviewBody(stage int) string {
b.WriteString(tuiOK.Render("✓ Preflight") + "\n")
b.WriteString(tuiHint.Render("Control plane verified before configuration."))
case stageOwner:
b.WriteString(tuiOK.Render("✓ Panel login") + "\n")
b.WriteString(tuiHint.Render("Open the one-time setup link in the summary to create your passkey."))
b.WriteString(tuiOK.Render("✓ Owner account") + "\n")
if m.result.username != "" {
b.WriteString(tuiLabel.Render("username ") + m.result.username + "\n")
}
b.WriteString(tuiHint.Render("Created and recorded. The one-time setup URL was shown on the Owner step."))
case stageConnect:
b.WriteString(tuiOK.Render("✓ Connection") + "\n")
b.WriteString(tuiLabel.Render("method ") + connectMethodLabel(m.result.connectMethod) + "\n")
@@ -496,9 +486,10 @@ func (m *rootModel) View() string {
// adminExistsAtStart reports whether this run began with an Owner already
// present (a re-run). The rail only makes sense for the first-run linear wizard.
func (m *rootModel) adminExistsAtStart() bool {
// First-run creation flips adminExists, but must keep its rail. Resuming an
// unfinished browser login is still a re-run even though it renews a token.
return m.result.alreadySetUp || (m.adminExists && !m.result.provisioned)
// adminExists flips true once we provision the Owner mid-run; the rail should
// keep showing through the connect/summary stages of that same first run. So
// only suppress the rail when the Owner pre-existed AND we never provisioned.
return m.adminExists && !m.result.provisioned
}
func (m *rootModel) rail() string {
@@ -545,7 +536,7 @@ func (m *rootModel) applyConnectResult(msg connectResultMsg) {
m.result.connectConfigured = true
m.result.reverseProxyGuide = msg.guide
}
m.result.panelURL = panelURLFor(msg.method, m.rootDomain, m.result.adminHostname)
m.result.panelURL = panelURLFor(msg.method, msg.panelHostname, m.rootDomain, m.adminHost)
}
func (m *rootModel) showSummary() (tea.Model, tea.Cmd) {
@@ -558,38 +549,14 @@ func (m *rootModel) showSummary() (tea.Model, tea.Cmd) {
panelURL: m.result.panelURL,
ownerUsername: m.result.username,
setupTokenURL: m.result.setupTokenURL,
auditWarning: m.result.auditWarning,
accessLabel: connectMethodLabel(m.result.connectMethod),
storageLabel: m.result.storageDetail,
routedHosts: routed,
localHint: m.result.connectMethod == connectLocal,
alreadySetUp: m.result.alreadySetUp,
alerts: m.readAlertRoute(),
})
}
func (m *rootModel) startOwnerSetup() (tea.Model, tea.Cmd) {
m.stage = stageOwner
method := m.result.connectMethod
adminHost := defaultAdminHostname(m.rootDomain, m.adminHost)
if m.result.adminHostname != "" {
adminHost = m.result.adminHostname
}
if m.adminExists && !m.result.provisioned {
m.result.alreadySetUp = true
if m.accessAud != "" {
method = connectCloudflare
}
}
base := "https://" + adminHost
if method == connectLocal {
base = localPanelURL(m.rootDomain, adminHost)
}
model := newSetupOwnerModel(m.ctx, m.store, base, m.osUser)
model.probe = func() error { return checkPanelAccess(m.rootDomain, adminHost).err }
return m.adopt(model)
}
// showStatus is the re-run landing: prove the backend is up, then point the
// operator at the panel without forcing any reconfiguration.
func (m *rootModel) showStatus() (tea.Model, tea.Cmd) {
@@ -601,32 +568,18 @@ func (m *rootModel) showStatus() (tea.Model, tea.Cmd) {
method = connectCloudflare
accessLabel = connectMethodLabel(connectCloudflare)
}
m.result.panelURL = panelURLFor(method, m.rootDomain, m.result.adminHostname)
m.result.panelURL = panelURLFor(method, m.panelHost, m.rootDomain, m.adminHost)
return m.adopt(&summaryModel{
panelURL: m.result.panelURL,
ownerUsername: m.result.username,
setupTokenURL: m.result.setupTokenURL,
auditWarning: m.result.auditWarning,
accessLabel: accessLabel,
alreadySetUp: true,
localHint: m.accessAud == "" && rootDomainEmbeddedIP(m.rootDomain) != "",
alerts: m.readAlertRoute(),
panelURL: m.result.panelURL,
accessLabel: accessLabel,
alreadySetUp: true,
localHint: m.accessAud == "" && rootDomainEmbeddedIP(m.rootDomain) != "",
})
}
// readAlertRoute reads the alert route afresh, so the summary shows a relay the
// Owner just configured with e.
func (m *rootModel) readAlertRoute() *alertRoute {
if m.alertRoute == nil {
return nil
}
r := m.alertRoute(m.ctx)
return &r
}
func panelURLFor(method connectMethod, rootDomain, adminHostname string) string {
if method != connectLocal {
return "https://" + defaultAdminHostname(rootDomain, adminHostname)
func panelURLFor(method connectMethod, panelHostname, rootDomain, adminHostname string) string {
if method != connectLocal && panelHostname != "" {
return "https://" + panelHostname
}
return localPanelURL(rootDomain, adminHostname)
}
cmd/felis/tui_root_test.go
+132 -48
View File
@@ -5,8 +5,6 @@ import (
"strings"
"testing"
"felis.lolicon.best/internal/config"
tea "github.com/charmbracelet/bubbletea"
)
@@ -32,7 +30,7 @@ func newTestRoot(adminExists bool, mode consoleMode, accessAud string) *rootMode
return newRootModel(
context.Background(),
&fakeOwnerStore{},
config.DatabaseConfig{URL: "postgres://localhost/felis"},
"postgres://localhost/felis",
"felis.example.com",
"admin.felis.example.com",
"panel.felis.example.com",
@@ -46,38 +44,94 @@ func newTestRoot(adminExists bool, mode consoleMode, accessAud string) *rootMode
func TestRootSetupHappyPath(t *testing.T) {
m := newTestRoot(false, consoleModeSetup, "")
// First-run setup begins at preflight.
if m.stage != stagePreflight {
t.Fatalf("initial stage = %v, want stagePreflight", m.stage)
}
if _, ok := m.screen.(*preflightModel); !ok {
t.Fatalf("initial screen = %T, want *preflightModel", m.screen)
}
// Preflight done → MC-bind (setup mode establishes the Owner by binding a
// Minecraft account, not by typing a username/password). The stage label is
// still stageOwner; only the screen differs by mode.
m = drive(t, m, preflightDoneMsg{})
if m.stage != stageOwner {
t.Fatalf("after preflight, stage = %v, want stageOwner", m.stage)
}
if _, ok := m.screen.(*mcBindModel); !ok {
t.Fatalf("after preflight, screen = %T, want *mcBindModel", m.screen)
}
// Owner provisioned → Connection chooser.
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
if m.stage != stageConnect {
t.Fatalf("stage = %v, want Connection", m.stage)
t.Fatalf("after owner, stage = %v, want stageConnect", m.stage)
}
m = drive(t, m, connectResultMsg{method: connectReverseProxy, panelHostname: "panel.felis.example.com", adminHostname: "new-admin.felis.example.com", guide: "caddy…"})
if _, ok := m.screen.(*connectChooserModel); !ok {
t.Fatalf("after owner, screen = %T, want *connectChooserModel", m.screen)
}
if !m.result.provisioned || m.result.username != "owner" || m.result.setupTokenURL != "https://op.console.example.com/setup?token=t0ken" {
t.Fatalf("owner result not recorded: %+v", m.result)
}
// Reverse-proxy chosen → Storage chooser, with the connection recorded.
guide := "caddy config…"
m = drive(t, m, connectResultMsg{
method: connectReverseProxy,
panelHostname: "panel.felis.example.com",
adminHostname: "admin.felis.example.com",
guide: guide,
})
if m.stage != stageStorage {
t.Fatalf("stage = %v, want Storage", m.stage)
t.Fatalf("after connect, stage = %v, want stageStorage", m.stage)
}
m = drive(t, m, storageResultMsg{method: storageS3, detail: "s3://bucket"})
owner, ok := m.screen.(*setupOwnerModel)
if !ok || owner.panelURL != "https://new-admin.felis.example.com" {
t.Fatalf("owner setup = %#v", m.screen)
if _, ok := m.screen.(*storageChooserModel); !ok {
t.Fatalf("after connect, screen = %T, want *storageChooserModel", m.screen)
}
if m.result.provisioned {
t.Fatal("Owner must not be minted before configuration")
if !m.result.connectConfigured {
t.Fatalf("connectConfigured not set")
}
if m.result.connectMethod != connectReverseProxy {
t.Fatalf("connectMethod = %v, want connectReverseProxy", m.result.connectMethod)
}
if m.result.reverseProxyGuide != guide {
t.Fatalf("reverseProxyGuide = %q, want %q", m.result.reverseProxyGuide, guide)
}
// Storage chosen → Summary, with both the connection and storage recorded.
m = drive(t, m, storageResultMsg{method: storageS3, detail: "s3://bucket · minio:9000"})
if m.stage != stageSummary {
t.Fatalf("after storage, stage = %v, want stageSummary", m.stage)
}
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://new-admin.felis.example.com/setup?token=t0ken"})
sum, ok := m.screen.(*summaryModel)
if !ok || sum.setupTokenURL != m.result.setupTokenURL || sum.panelURL != "https://new-admin.felis.example.com" || sum.storageLabel != "s3://bucket" {
t.Fatalf("summary = %#v", m.screen)
if !ok {
t.Fatalf("after storage, screen = %T, want *summaryModel", m.screen)
}
if !strings.Contains(sum.View(), "Finish Owner login") || !strings.Contains(sum.View(), "Minecraft can be linked later") {
t.Fatal(sum.View())
if m.result.storageMethod != storageS3 || m.result.storageDetail == "" {
t.Fatalf("storage result not recorded: %+v", m.result)
}
if sum.storageLabel != m.result.storageDetail {
t.Fatalf("summary storageLabel = %q, want %q", sum.storageLabel, m.result.storageDetail)
}
if want := "https://panel.felis.example.com"; sum.panelURL != want {
t.Fatalf("summary panelURL = %q, want %q", sum.panelURL, want)
}
if want := "https://op.console.example.com/setup?token=t0ken"; sum.setupTokenURL != want {
t.Fatalf("summary setupTokenURL = %q, want %q", sum.setupTokenURL, want)
}
if sum.alreadySetUp {
t.Fatalf("first-run summary should not be marked alreadySetUp")
}
}
func TestRootSetupLocalSummary(t *testing.T) {
m := newTestRoot(false, consoleModeSetup, "")
m = drive(t, m, preflightDoneMsg{})
m = drive(t, m, ownerResultMsg{username: "owner"})
m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.felis.example.com"})
m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"})
m = drive(t, m, ownerResultMsg{username: "owner"})
sum, ok := m.screen.(*summaryModel)
if !ok {
@@ -99,9 +153,9 @@ func TestRootSetupLocalSummary(t *testing.T) {
func TestRootReconfigureConnectSkipsStorage(t *testing.T) {
m := newTestRoot(false, consoleModeSetup, "")
m = drive(t, m, preflightDoneMsg{})
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.felis.example.com"})
m = drive(t, m, storageResultMsg{method: storageS3, detail: "s3://bucket"})
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
if _, ok := m.screen.(*summaryModel); !ok {
t.Fatalf("after first run, screen = %T, want *summaryModel", m.screen)
}
@@ -118,10 +172,6 @@ func TestRootReconfigureConnectSkipsStorage(t *testing.T) {
// Completing it returns straight to the summary — NOT the storage chooser —
// with the original storage recap intact.
m = drive(t, m, connectResultMsg{method: connectReverseProxy, panelHostname: "panel.felis.example.com", guide: "caddy…"})
if _, ok := m.screen.(*setupOwnerModel); !ok {
t.Fatalf("pending link should refresh, screen = %T", m.screen)
}
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://admin.felis.example.com/setup?token=fresh"})
if m.stage != stageSummary {
t.Fatalf("after reconfigure connect, stage = %v, want stageSummary", m.stage)
}
@@ -143,9 +193,9 @@ func TestRootReconfigureConnectSkipsStorage(t *testing.T) {
func TestRootReconfigureStorageReEntersChooser(t *testing.T) {
m := newTestRoot(false, consoleModeSetup, "")
m = drive(t, m, preflightDoneMsg{})
m = drive(t, m, ownerResultMsg{username: "owner"})
m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.felis.example.com"})
m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"})
m = drive(t, m, ownerResultMsg{username: "owner"})
if _, ok := m.screen.(*summaryModel); !ok {
t.Fatalf("after first run, screen = %T, want *summaryModel", m.screen)
}
@@ -180,7 +230,6 @@ func TestRootReconfigureStorageReEntersChooser(t *testing.T) {
func TestRootReconfigureSMTP(t *testing.T) {
m := newTestRoot(true, consoleModeSetup, "")
m = drive(t, m, preflightDoneMsg{})
m = drive(t, m, setupReadyMsg{username: "owner"})
if _, ok := m.screen.(*summaryModel); !ok {
t.Fatalf("re-run after preflight, screen = %T, want *summaryModel", m.screen)
}
@@ -206,7 +255,6 @@ func TestRootReconfigureSMTP(t *testing.T) {
func TestRootReconfigureStorageKeepsStatusFraming(t *testing.T) {
m := newTestRoot(true, consoleModeSetup, "")
m = drive(t, m, preflightDoneMsg{})
m = drive(t, m, setupReadyMsg{username: "owner"})
if _, ok := m.screen.(*summaryModel); !ok {
t.Fatalf("re-run after preflight, screen = %T, want *summaryModel", m.screen)
}
@@ -230,27 +278,23 @@ func TestRootReconfigureStorageKeepsStatusFraming(t *testing.T) {
}
func TestRootRerunLandsOnStatus(t *testing.T) {
// adminExists at start of a setup run = re-run: preflight should skip straight
// to the "manage in panel" status screen, never touching owner/connect.
m := newTestRoot(true, consoleModeSetup, "")
m = drive(t, m, preflightDoneMsg{})
if _, ok := m.screen.(*setupOwnerModel); !ok {
t.Fatalf("screen = %T", m.screen)
if _, ok := m.screen.(*preflightModel); !ok {
t.Fatalf("re-run initial screen = %T, want *preflightModel", m.screen)
}
m = drive(t, m, setupReadyMsg{username: "owner"})
if m.stage != stageSummary || !m.result.alreadySetUp || m.result.provisioned {
t.Fatalf("result = %+v", m.result)
}
}
func TestRootRerunResumesUnfinishedLogin(t *testing.T) {
m := newTestRoot(true, consoleModeSetup, "")
m = drive(t, m, preflightDoneMsg{})
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://admin.felis.example.com:30443/setup?token=new"})
sum := m.screen.(*summaryModel)
if sum.setupTokenURL == "" || !sum.alreadySetUp {
t.Fatalf("summary = %+v", sum)
sum, ok := m.screen.(*summaryModel)
if !ok {
t.Fatalf("re-run after preflight, screen = %T, want *summaryModel", m.screen)
}
if strings.Contains(m.View(), "Bootstrap") {
t.Fatal("renewing a login link restarted the deployment rail")
if !sum.alreadySetUp {
t.Fatalf("re-run summary should be marked alreadySetUp")
}
if m.result.provisioned {
t.Fatalf("re-run must not provision an owner")
}
}
@@ -296,18 +340,58 @@ func key(t tea.KeyType) tea.KeyMsg { return tea.KeyMsg{Type: t} }
func TestRootRailReviewNavigation(t *testing.T) {
m := newTestRoot(false, consoleModeSetup, "")
m = drive(t, m, preflightDoneMsg{})
// On the Owner screen (text inputs) ← must NOT hijack the arrow: it stays
// with the field, so we remain on the live screen.
m = drive(t, m, key(tea.KeyLeft))
if m.reviewing != int(stagePreflight) || !strings.Contains(m.View(), "Control plane verified") {
t.Fatal("connection review should return to preflight")
if m.reviewing != -1 {
t.Fatalf("← on the owner (text-input) screen entered review (%d); arrows belong to the field", m.reviewing)
}
// Advance to the Connection chooser (a select — it yields ←/→).
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
if m.reviewing != -1 {
t.Fatalf("fresh chooser should start live, reviewing = %d", m.reviewing)
}
// ← walks back to Owner (read-only recap), then Preflight, then clamps.
m = drive(t, m, key(tea.KeyLeft))
if m.reviewing != int(stageOwner) {
t.Fatalf("first ← = stage %d, want stageOwner %d", m.reviewing, stageOwner)
}
if v := m.View(); !strings.Contains(v, "Owner account") || !strings.Contains(v, "username") {
t.Fatalf("owner review body missing recap, got:\n%s", v)
}
m = drive(t, m, key(tea.KeyLeft))
if m.reviewing != int(stagePreflight) {
t.Fatalf("second ← = stage %d, want stagePreflight %d", m.reviewing, stagePreflight)
}
m = drive(t, m, key(tea.KeyLeft))
if m.reviewing != int(stagePreflight) {
t.Fatalf("← past the first step should clamp, got %d", m.reviewing)
}
// → walks forward; stepping past the last completed step returns to live.
m = drive(t, m, key(tea.KeyRight))
if m.reviewing != int(stageOwner) {
t.Fatalf("→ = stage %d, want stageOwner %d", m.reviewing, stageOwner)
}
m = drive(t, m, key(tea.KeyRight))
if m.reviewing != -1 {
t.Fatal("right should return to live connection chooser")
t.Fatalf("→ past the last completed step should return live, reviewing = %d", m.reviewing)
}
if v := m.View(); !strings.Contains(v, "reach the panel") {
t.Fatalf("returning live should show the chooser, got:\n%s", v)
}
// esc is an immediate escape hatch back to the live screen.
m = drive(t, m, key(tea.KeyLeft))
if m.reviewing < 0 {
t.Fatalf("← should re-enter review")
}
m = drive(t, m, key(tea.KeyEsc))
if m.reviewing != -1 {
t.Fatal("escape should return to live screen")
t.Fatalf("esc should return to the live screen, reviewing = %d", m.reviewing)
}
}
@@ -325,8 +409,8 @@ func TestSetupRailSpansBootstrap(t *testing.T) {
m := newTestRoot(false, consoleModeSetup, "")
m = drive(t, m, tea.WindowSizeMsg{Width: 90, Height: 30})
m = drive(t, m, preflightDoneMsg{})
if _, ok := m.screen.(*connectChooserModel); !ok {
t.Fatalf("expected connection screen after preflight, got %T", m.screen)
if _, ok := m.screen.(*mcBindModel); !ok {
t.Fatalf("expected MC-bind screen after preflight, got %T", m.screen)
}
if v := m.View(); !strings.Contains(v, "✓ Bootstrap") {
t.Fatalf("wizard rail should carry Bootstrap as a completed step, got:\n%s", v)
cmd/felis/tui_setup_owner.go
-94
View File
@@ -1,94 +0,0 @@
package main
import (
"context"
"errors"
"felis.lolicon.best/internal/api"
"github.com/charmbracelet/bubbles/spinner"
tea "github.com/charmbracelet/bubbletea"
)
type setupOwnerMsg struct {
outcome breakGlassOutcome
err error
}
type setupReadyMsg struct{ username string }
// setupOwnerModel waits for the panel, then issues a host-authorized login link.
// It never needs a running Minecraft client or login server.
type setupOwnerModel struct {
ctx context.Context
store ownerStore
panelURL, osUser string
probe func() error
sp spinner.Model
err error
}
func newSetupOwnerModel(ctx context.Context, store ownerStore, panelURL, osUser string) *setupOwnerModel {
sp := spinner.New()
sp.Spinner, sp.Style = spinner.Dot, tuiLabel
return &setupOwnerModel{ctx: ctx, store: store, panelURL: panelURL, osUser: osUser, sp: sp}
}
func (m *setupOwnerModel) Init() tea.Cmd {
return tea.Batch(m.sp.Tick, func() tea.Msg {
if m.probe != nil {
if err := m.probe(); err != nil {
return setupOwnerMsg{err: err}
}
}
out, err := performSetupOwner(m.ctx, m.store, m.panelURL, m.osUser)
return setupOwnerMsg{outcome: out, err: err}
})
}
func (m *setupOwnerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case setupOwnerMsg:
if errors.Is(msg.err, api.ErrConflict) {
return m, func() tea.Msg { return setupReadyMsg{username: msg.outcome.ownerUsername} }
}
if msg.err != nil {
m.err = msg.err
return m, nil
}
return m, func() tea.Msg {
res := ownerResultMsg{username: msg.outcome.ownerUsername,
setupTokenURL: msg.outcome.setupTokenURL, mode: "setup", accountable: m.osUser}
if msg.outcome.auditErr != nil {
res.auditWarning = msg.outcome.auditErr.Error()
}
return res
}
case spinner.TickMsg:
if m.err == nil {
var cmd tea.Cmd
m.sp, cmd = m.sp.Update(msg)
return m, cmd
}
case tea.KeyMsg:
switch msg.String() {
case "ctrl+c", "esc":
return m, tea.Quit
case "r", "R", "enter":
if m.err != nil {
m.err = nil
return m, m.Init()
}
}
}
return m, nil
}
func (m *setupOwnerModel) View() string {
if m.err != nil {
return tuiWarn.Render("Panel login setup could not finish: "+m.err.Error()) + "\n\n" +
tuiHint.Render("Minecraft is not required. Fix the reported service, then retry.") + "\n\n" +
tuiAction("r/enter", "retry", "esc", "exit")
}
return " " + m.sp.View() + " " + tuiHint.Render("Preparing your first panel login…") + "\n"
}
cmd/felis/tui_setup_owner_test.go
-64
View File
@@ -1,64 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"strings"
"testing"
tea "github.com/charmbracelet/bubbletea"
)
func TestSetupOwnerWaitsForPanelBeforeMinting(t *testing.T) {
store := &fakeOwnerStore{}
m := newSetupOwnerModel(context.Background(), store, "https://op.console.example.com:30443", "root")
m.probe = func() error { return errors.New("panel not ready") }
batch := m.Init()().(tea.BatchMsg)
_, cmd := m.Update(batch[1]())
if cmd != nil || len(store.tokens) != 0 || !strings.Contains(m.View(), "panel not ready") {
t.Fatal("unready panel issued login")
}
m.probe = func() error { return nil }
_, cmd = m.Update(tea.KeyMsg{Type: tea.KeyEnter})
batch = cmd().(tea.BatchMsg)
_, cmd = m.Update(batch[1]())
res := cmd().(ownerResultMsg)
if res.setupTokenURL == "" || res.username != "owner" || len(store.tokens) != 1 {
t.Fatalf("result = %+v", res)
}
}
func TestReportSetupWithoutOwnerPointsAtBrowserSetup(t *testing.T) {
var b bytes.Buffer
reportSetupResult(&b, breakGlassResult{}, false, false, "https://op.console.example.com")
if !strings.Contains(b.String(), "sudo felis setup") || strings.Contains(b.String(), "join ") {
t.Fatalf("output = %s", b.String())
}
}
func TestLocalPanelSetupUsesPasskeyHostname(t *testing.T) {
t.Setenv("FELIS_PANEL_NODEPORT", "30445")
if got := localPanelURL("10.211.55.6.nip.io", ""); got != "https://op.console.10.211.55.6.nip.io:30445" {
t.Fatalf("local setup URL = %q; a bare IP cannot enroll the panel passkey", got)
}
}
func TestSetupGameAddress(t *testing.T) {
for _, tc := range []struct {
root string
port int
want string
}{
{"10.211.55.6.nip.io", 0, "10.211.55.6"},
{"10.211.55.6.nip.io", 25565, "10.211.55.6"},
{"10.211.55.6.sslip.io.", 25570, "10.211.55.6:25570"},
{"play.example.net", 0, "play.example.net"},
{"play.example.net", 25570, "play.example.net:25570"},
{"", 25570, ""},
} {
if got := setupGameAddress(tc.root, tc.port); got != tc.want {
t.Errorf("setupGameAddress(%q, %d) = %q, want %q", tc.root, tc.port, got, tc.want)
}
}
}
Loaded 100 of 864 files, more files were not shown because too many files have changed in this diff. Show more