FelisMC/Felis
2
0 Fork 0
Code Issues Pull Requests 6 Actions Packages Projects Releases Wiki Activity

Compare commits

base: FelisMC/Felis:v0.2.0-rc1
FelisMC/Felis:main FelisMC/Felis:dependabot/github_actions/anchore/sbom-action-0.24.2 FelisMC/Felis:dependabot/github_actions/docker/setup-buildx-action-4.4.1 FelisMC/Felis:dependabot/github_actions/actions/download-artifact-8.0.1 FelisMC/Felis:dependabot/github_actions/actions/setup-java-6.0.1 FelisMC/Felis:dependabot/github_actions/actions/setup-go-7.0.0 FelisMC/Felis:dependabot/docker/golang-1.27 FelisMC/Felis:audit-fixes-20260922 FelisMC/Felis:chore/issue-sweep
FelisMC/Felis:v0.2.0-rc2 FelisMC/Felis:v0.2.0 FelisMC/Felis:v0.2.0-rc1 FelisMC/Felis:v0.1.0 FelisMC/Felis:v0.1.0-rc1
..
compare: FelisMC/Felis:dependabot/github_actions/actions/setup-go-7.0.0
FelisMC/Felis:main FelisMC/Felis:dependabot/github_actions/anchore/sbom-action-0.24.2 FelisMC/Felis:dependabot/github_actions/docker/setup-buildx-action-4.4.1 FelisMC/Felis:dependabot/github_actions/actions/download-artifact-8.0.1 FelisMC/Felis:dependabot/github_actions/actions/setup-java-6.0.1 FelisMC/Felis:dependabot/github_actions/actions/setup-go-7.0.0 FelisMC/Felis:dependabot/docker/golang-1.27 FelisMC/Felis:audit-fixes-20260922 FelisMC/Felis:chore/issue-sweep
FelisMC/Felis:v0.2.0-rc2 FelisMC/Felis:v0.2.0 FelisMC/Felis:v0.2.0-rc1 FelisMC/Felis:v0.1.0 FelisMC/Felis:v0.1.0-rc1
1 Commits
v0.2.0-rc1 .. dependabot/github_actions/actions/setup-go-7.0.0
Author SHA1 Message Date
dependabot[bot] a252f04b0f chore(deps): bump actions/setup-go from 5.6.0 to 7.0.0 
Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.6.0 to 7.0.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/40f1582b2485089dde7abd97c1529aa768e1baff...b7ad1dad31e06c5925ef5d2fc7ad053ef454303e)

---
updated-dependencies:
- dependency-name: actions/setup-go
  dependency-version: 7.0.0
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>
 2026-09-24 16:45:30 +00:00
708 changed files with 8676 additions and 136976 deletions
Expand all files Collapse all files

No files matched your search

.github/workflows/ci.yml
+9 -71
View File
@@ -17,15 +17,11 @@ name: ci
#
# release.yml calls this workflow (workflow_call) before it builds anything, so a tag passes
# exactly these gates and there is one list of them.
#
# workflow_dispatch reruns the suite on a commit whose push produced no run, such as one
# pushed while Actions was unavailable.
on:
push:
branches: [main]
pull_request:
workflow_call:
workflow_dispatch:
permissions:
contents: read
@@ -40,7 +36,7 @@ jobs:
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
@@ -67,7 +63,7 @@ jobs:
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
@@ -76,9 +72,6 @@ jobs:
# The business stores' SQL against a real PostgreSQL (internal/pgint): the unit suites run
# on fakes, and PGRepo drifted from them three times while those stayed green. 13 is the
# oldest server a supported distribution installs (EL9), 18 the newest (Arch).
# `felis db backup` and `restore` run there too, with the tools inside the service
# container, as production runs them inside felis-postgres: the runner's own client is one
# major version, and pg_dump refuses a newer server.
pgint:
runs-on: ubuntu-latest
strategy:
@@ -102,14 +95,13 @@ jobs:
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
- run: go test -race -tags pgint -count=1 ./internal/pgint/
env:
FELIS_TEST_PG_URL: postgres://felis:pgint@localhost:5432/felis_pgint?sslmode=disable
FELIS_TEST_PG_EXEC: docker exec -i ${{ job.services.postgres.id }}
shell:
runs-on: ubuntu-latest
@@ -142,34 +134,7 @@ jobs:
tar -xJf shellcheck.tar.xz
./shellcheck-v0.11.0/shellcheck -S warning $(git ls-files '*.sh')
# An exit status is 8 bits, so `exit "$fails"` reads 256 failures as a pass. The suites
# exit 1 on any failure; this keeps the next one from carrying its count out.
- name: No script exits with its failure count
run: |
if git grep -nE 'exit +"?\$\{?[a-z_]*fail[a-z_]*\}?"?[[:space:]]*$' -- '*.sh'; then exit 1; fi
- run: sh deploy/bootstrap_test.sh
- run: sh deploy/uninstall_test.sh
- run: bash deploy/e2e_release_test.sh
# The shipped alert rules (deploy/alerts): promtool parses them and runs their unit tests,
# which pin when each alert fires and that it stays quiet before. internal/metrics'
# alerts_test.go pins what promtool cannot see from there: the PrometheusRule twin and the
# metric names the rules read.
alerts:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# A pinned release, like shellcheck's, so a new Prometheus cannot change what fails.
- name: promtool
run: |
curl -fsSL -o prometheus.tar.gz \
https://github.com/prometheus/prometheus/releases/download/v3.15.0/prometheus-3.15.0.linux-amd64.tar.gz
echo "2a542df32eac02ee17b9d844fb2aa1de00dafa5476579ba8a3ba862e9d572ea0 prometheus.tar.gz" | sha256sum -c
tar -xzf prometheus.tar.gz --strip-components=1 prometheus-3.15.0.linux-amd64/promtool
./promtool check rules deploy/alerts/felis-alerts.yaml
./promtool test rules deploy/alerts/felis-alerts_test.yml
panel:
runs-on: ubuntu-latest
@@ -199,53 +164,26 @@ jobs:
- run: npm test
working-directory: panel
# `tsc -b`: tsconfig.json is a solution file (files: [] plus references), so a plain
# `tsc --noEmit` checked nothing and passed with type errors in the tree.
- run: npm run typecheck
working-directory: panel
# Rules of hooks and effect dependency lists, with --max-warnings 0.
- run: npm run lint
working-directory: panel
# openapi.gen.ts is generated from docs/openapi.yaml and checked in, so the
# compile-time parity in src/lib/types.parity.ts needs no generator in the build;
# a schema edit that was not regenerated fails here.
- name: openapi.gen.ts matches docs/openapi.yaml
working-directory: panel
run: |
npm run gen:api
git diff --exit-code -- src/lib/openapi.gen.ts
# Browser smoke over the mock-mode dev server, in the runner's installed Chrome
# (playwright.config.ts sets channel: chrome, so nothing is downloaded).
- run: npm run test:e2e
working-directory: panel
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: failure()
with:
name: panel-playwright-report
path: |
panel/playwright-report
panel/test-results
retention-days: 7
plugins:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# The other jobs never touch the Java layer: the plugin jars were only ever
# compiled by bootstrap on a live host, and the test mains under plugins/*/test
# were run by hand. JDK 25 is what the plugin build image runs (paper-api 26.x
# needs it); each module's wrapper brings the Gradle the image pins.
# compiled by bootstrap on a live host, and the three test mains under
# plugins/*/test were run by hand. JDK 21 plus the Gradle major the plugin
# Dockerfiles pin (8.14) is that same toolchain, in CI.
- uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
with:
distribution: temurin
java-version: '25'
java-version: '21'
- uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
with:
gradle-version: '8.14'
- run: bash plugins/test.sh
.github/workflows/e2e.yml
-333
View File
@@ -1,333 +0,0 @@
# deploy/bootstrap.sh end to end on a fresh Ubuntu 24.04 x86_64 runner: the paths every
# host goes through, run for real instead of by hand on a VM.
#
# artifacts this commit's release assets, built by deploy/build-release-artifacts.sh the
# way release.yml builds a tag's (amd64 only, the runners' architecture)
# install a full install from those assets, the way a release installs, on a host with
# ufw enabled, then the same assets again (a rerun must converge without
# restarting what did not change, importing or uploading an image again, or
# touching Docker)
# readme the README's one-line install as a new host runs it today: this commit's
# installer on its default channel, which installs the newest published
# release's binary, images and plugin from that release's assets; skipped until
# a release exists
# upgrade the newest published release through its own installer and its own assets,
# rows of every kind seeded into its database, then this commit's assets on
# top of it; skipped until a release exists
# source a full install built on the host from this checkout (FELIS_SKIP_FETCH):
# the fallback a release without assets, or an unsupported one, takes. It builds
# everything with Docker, so it runs by hand and weekly only
#
# This runs on pushes that touch what gets installed, by hand, and weekly (a moving
# upstream: apt mirrors, k3s's install script and release assets, Adoptium).
# deploy/e2e_check.sh holds the assertions, deploy/e2e_seed.sh the upgrade's seed and its check.
name: e2e
on:
push:
branches: [main]
paths:
- 'deploy/**'
- 'cmd/**'
- 'internal/**'
- 'panel/**'
- 'plugins/**'
- 'Dockerfile'
- 'go.mod'
- 'go.sum'
- '.github/workflows/e2e.yml'
workflow_dispatch:
schedule:
- cron: '23 4 * * 1'
permissions:
contents: read
# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` fails the step
# when the installer fails; the default shell reports tee's status.
defaults:
run:
shell: bash
concurrency:
group: e2e-${{ github.ref }}
cancel-in-progress: true
jobs:
artifacts:
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
fetch-depth: 0 # the newest tag is the version's base, as on the dev channel
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Stamped the way bootstrap stamps a build of main: "<newest tag>+g<sha>".
- name: Build the release assets
run: |
base="$(git describe --tags --abbrev=0 --match 'v*' 2>/dev/null || echo v0.0.0)"
FELIS_RELEASE_ARCHES=amd64 deploy/build-release-artifacts.sh "${base}+g$(git rev-parse --short HEAD)" dist
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: e2e-assets
path: dist/
if-no-files-found: error
retention-days: 1
install:
needs: artifacts
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# The images, k3s and the JRE need more room than a stock runner leaves free.
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: e2e-assets
path: dist
# The runner has Docker preinstalled, so its absence proves nothing: the log shows
# whether bootstrap reached for it. From FELIS_ARTIFACT_DIR every image and the plugin
# come out of the assets, so it must not have. (`! grep` is spelled `if grep ... exit 1`
# below because bash -e ignores a failing `!` command.)
# ufw, enabled on many Ubuntu hosts, drops every inbound packet no rule admits, the
# pods' traffic to the API server among them; the runner ships it disabled. The
# runner's own traffic is outbound, which ufw lets through.
- name: Enable ufw
run: sudo ufw --force enable
- name: Install
run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log
- name: Check the install
run: |
sudo bash deploy/e2e_check.sh install
for tag in felis-k3s-pods felis-k3s-services felis-panel felis-proxy; do
sudo ufw status | grep -qE "# ${tag} *$"
done
if grep -E 'docker already installed|installing docker|docker running' install.log; then exit 1; fi
for role in felis limbo lobby paper; do
grep -q "felis/${role}:[^ ]* is the release's" install.log
done
grep -q "docker.io/library/registry@sha256:[0-9a-f]* is the release's" install.log
grep -q "docker.io/library/postgres@sha256:[0-9a-f]* is the release's" install.log
grep -q "felis-velocity.jar is the release's" install.log
- name: Rerun the same assets
run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log
# containerd and the registry already hold every image under the digest the listing
# names, so nothing is imported or uploaded twice.
- name: Check the rerun
run: |
sudo bash deploy/e2e_check.sh rerun
grep -q 'felis-velocity unchanged; left running' rerun.log
if grep -E 'docker already installed|installing docker|docker running' rerun.log; then exit 1; fi
if grep -E 'importing felis-image-|mirroring .* into the internal registry' rerun.log; then exit 1; fi
grep -q 'is already in the internal registry' rerun.log
- name: Diagnostics
if: failure()
run: |
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
k() { sudo -E /usr/local/bin/k3s kubectl "$@"; }
k get pods -A -o wide || true
k get events -A --sort-by=.lastTimestamp | tail -n 60 || true
for p in $(k get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" {print $1 "/" $2}'); do
k -n "${p%%/*}" describe pod "${p#*/}" | tail -n 40 || true
k -n "${p%%/*}" logs "${p#*/}" --all-containers --tail=60 || true
done
k -n felis logs deploy/felis-postgres --tail=60 || true
sudo journalctl -u k3s -u felis-velocity --no-pager -n 120 || true
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: e2e-install-logs
path: '*.log'
if-no-files-found: ignore
# The README's command on a fresh host: this commit's installer, as main serves it, on its
# default channel with nothing pinned. It resolves the newest release and installs that
# release's binary, images and plugin from its assets, each checked against its
# SHA256SUMS; the private repo's token is the only thing added. FELIS_INSTALL_MODE picks the
# mode the setup console would ask for.
readme:
runs-on: ubuntu-24.04
timeout-minutes: 120
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- name: Find the newest release
id: release
env:
GH_TOKEN: ${{ github.token }}
run: bash deploy/e2e_release.sh find
- name: Install as the README does
if: steps.release.outputs.tag != ''
env:
TOKEN: ${{ github.token }}
run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log
# The binary is the release's, so the phase is `release`: its database backup and
# timers are the release's to have or lack.
- name: Check the install
if: steps.release.outputs.tag != ''
env:
TAG: ${{ steps.release.outputs.tag }}
BINARY: ${{ steps.release.outputs.binary }}
SUMS: ${{ steps.release.outputs.sums }}
run: |
sudo bash deploy/e2e_check.sh release
sudo /usr/local/bin/felis version | grep -qx "felis ${TAG}"
bash deploy/e2e_release.sh check-readme readme.log
- name: Diagnostics
if: failure()
run: |
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: e2e-readme-logs
path: '*.log'
if-no-files-found: ignore
upgrade:
needs: artifacts
runs-on: ubuntu-24.04
timeout-minutes: 120
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
fetch-depth: 0
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: e2e-assets
path: dist
- name: Find the newest release
id: release
env:
GH_TOKEN: ${{ github.token }}
run: bash deploy/e2e_release.sh find
# The release's own installer on its default channel, fetching the release's own
# binary: what a host that installed that release is running today. FELIS_REF would
# build the tag from source instead, a path no host takes by default. FELIS_RELEASE pins
# the tag found above; an installer older than FELIS_RELEASE ignores it and resolves
# the newest release, the same tag.
- name: Install the newest release
if: steps.release.outputs.tag != ''
env:
TAG: ${{ steps.release.outputs.tag }}
TOKEN: ${{ github.token }}
run: |
git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh
sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log
- name: Check the release install
if: steps.release.outputs.tag != ''
env:
TAG: ${{ steps.release.outputs.tag }}
BINARY: ${{ steps.release.outputs.binary }}
run: |
sudo bash deploy/e2e_check.sh release
bash deploy/e2e_release.sh check-own release.log
# A fresh install's database holds only what its migrations wrote: the seed gives the
# upgrade's move and its pending migrations existing rows to carry.
- name: Seed the release's database
if: steps.release.outputs.tag != ''
run: sudo bash deploy/e2e_seed.sh seed
- name: Upgrade to this commit
if: steps.release.outputs.tag != ''
run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log
# Both checks run and report: the seeded rows go last, after the restore drill has
# also put them back from a bundle.
- name: Check the upgrade
if: steps.release.outputs.tag != ''
run: |
rc=0
sudo bash deploy/e2e_check.sh upgrade || rc=1
sudo bash deploy/e2e_seed.sh check || rc=1
exit "$rc"
- name: Diagnostics
if: failure()
run: |
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
sudo -E /usr/local/bin/k3s kubectl -n felis logs deploy/felis-postgres --tail=60 || true
# The release ran the database on the host; the upgrade moves it into k3s.
sudo journalctl -u k3s -u felis-velocity -u postgresql --no-pager -n 120 || true
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: e2e-upgrade-logs
path: '*.log'
if-no-files-found: ignore
source:
if: github.event_name != 'push'
runs-on: ubuntu-24.04
timeout-minutes: 90
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# FELIS_SKIP_FETCH builds whatever sits in /opt/felis/src; the .git directory is what
# stamps the build. Root owns it so git, run as root by the installer, does not refuse
# it as dubious.
- name: Stage this commit as the installer's source
run: |
sudo mkdir -p /opt/felis
sudo cp -a "$GITHUB_WORKSPACE" /opt/felis/src
sudo chown -R root:root /opt/felis/src
- name: Install
run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log
- name: Check the install
run: sudo bash deploy/e2e_check.sh install
- name: Diagnostics
if: failure()
run: |
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
sudo -E /usr/local/bin/k3s kubectl get pods -A -o wide || true
sudo journalctl -u k3s -u felis-velocity -u docker --no-pager -n 120 || true
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: e2e-source-logs
path: '*.log'
if-no-files-found: ignore
.github/workflows/release.yml
+62 -44
View File
@@ -2,15 +2,14 @@
#
# Two things depend on this job. /repos/{repo}/releases/latest must ANSWER — that endpoint is
# what `felis update` polls (internal/updater/github.go) and what deploy/bootstrap.sh's default
# "release" channel resolves its ref from. And the assets below are what that channel then
# INSTALLS: bootstrap downloads the felis binary, the image tars, their listing and the Velocity
# plugin instead of building anything on the target host, so these are the shipped artifact,
# not a convenience. A host installing a release needs neither Docker, Gradle, Go nor Docker Hub.
# "release" channel resolves its ref from. And the binaries below are what that channel then
# INSTALLS: bootstrap downloads felis-linux-<arch> instead of compiling on the target host, so
# these are the shipped artifact, not a convenience.
#
# deploy/build-release-artifacts.sh builds every asset in one run and documents each name; the
# names are a contract with deploy/bootstrap.sh. They are plain literals on both sides: a YAML
# workflow and a go:embed'ed shell script have no honest way to share a constant, and the
# failure mode is benign — bootstrap warns and falls back to building on the host.
# The asset NAME is a contract with deploy/bootstrap.sh (download_release_binary builds
# "felis-linux-${arch}"). It is deliberately a plain literal on both sides: a GitHub Actions
# YAML and a go:embed'ed shell script have no honest way to share a constant, and the failure
# mode is benign — bootstrap warns and falls back to a source build of the same tag.
#
# The binary is built through the repo Dockerfile rather than a plain `go build`.
# internal/panel/static holds a tracked PLACEHOLDER index.html so the //go:embed
@@ -19,12 +18,13 @@
# build first, and is the same recipe bootstrap uses, so there is one way to build felis
# rather than two that can drift.
#
# SHA256SUMS is a contract with bootstrap too: it refuses any asset whose hash is not listed
# there, BEFORE it runs or imports it.
# SHA256SUMS is a contract with bootstrap too: download_release_binary refuses a binary whose
# hash is not listed there, BEFORE it runs it. A release without the file installs by source
# build instead.
#
# The write token never meets the test suite: `gates` (ci.yml) and `build` run the tests,
# Gradle and the Docker builds (each of which executes third-party code) with a read-only
# token, and `build` hands the assets over as a workflow artifact; `publish` holds contents:write and runs only
# Gradle and the Docker build (each of which executes third-party code) with a read-only
# token, and `build` hands the binaries over as a workflow artifact; `publish` holds contents:write and runs only
# pinned actions and gh. Every action is pinned to a commit SHA (the tag in the trailing
# comment is for humans); .github/dependabot.yml proposes the bumps.
name: release
@@ -39,10 +39,9 @@ permissions:
jobs:
# A tag that ships red is worse than a tag that fails to ship. These are ci.yml's gates,
# called rather than copied: Go (race, vet, staticcheck), govulncheck, the PostgreSQL
# contract suite, shellcheck and the bootstrap tests, promtool on the alert rules, the
# panel, and the Java layer the binary EMBEDS (bootstrap_asset.go ships the plugin
# sources, so a tag whose plugins do not compile turns every install of that release into
# a failed bootstrap).
# contract suite, shellcheck and the bootstrap tests, the panel, and the Java layer the
# binary EMBEDS (bootstrap_asset.go ships the plugin sources, so a tag whose plugins do
# not compile turns every install of that release into a failed bootstrap).
gates:
uses: ./.github/workflows/ci.yml
@@ -53,47 +52,69 @@ jobs:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# Both architectures, because bootstrap's default release channel DOWNLOADS these
# rather than building on the target host — an arm64 host with no asset falls back to
# a slow on-host build. The felis binary and the plugin jars cross-compile on the
# runner (their build stages are pinned to $BUILDPLATFORM); the game images' runtime
# stages run apt-get for the target platform, which for arm64 takes QEMU.
- uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
# rather than compiling on the target host — an arm64 host with no asset silently
# falls back to a slow source build. Neither stage is emulated: the Dockerfile pins
# both build stages to $BUILDPLATFORM and the Go stage cross-compiles via TARGETARCH,
# so the second architecture costs about a minute.
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
# Two architectures of five images plus BuildKit's cache outgrow the runner's free disk
# with its preinstalled SDKs in place; none of them is used here.
- name: Free disk space
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- name: Build the stamped binaries
run: |
docker buildx build --platform linux/amd64,linux/arm64 \
--build-arg FELIS_VERSION="${GITHUB_REF_NAME}" \
--output type=local,dest=out .
mv out/linux_amd64/usr/local/bin/felis ./felis-linux-amd64
mv out/linux_arm64/usr/local/bin/felis ./felis-linux-arm64
chmod +x ./felis-linux-amd64 ./felis-linux-arm64
# The script checks what the old inline steps did: the host binary reports exactly
# "felis <tag>" (unstamped, `felis update` refuses to compare), and the other one is an
# ELF of its architecture, read with file(1) — binfmt would run a misnamed binary happily.
- name: Build the release assets
run: deploy/build-release-artifacts.sh "${GITHUB_REF_NAME}" dist
# The stamp is the whole point and it fails silently: an unstamped binary reports
# "dev", which `felis update` refuses to compare, disabling update reporting for
# every install built from it. Assert it end to end instead of trusting the ARG
# reached the linker.
- name: Verify the version stamp
run: |
got="$(./felis-linux-amd64 version | head -1)"
echo "reported: ${got}"
[ "$got" = "felis ${GITHUB_REF_NAME}" ] \
|| { echo "expected 'felis ${GITHUB_REF_NAME}' — the -X main.version stamp did not reach the binary"; exit 1; }
# The arm64 binary is checked by ELF machine type, NOT by running it. Runners have
# binfmt/QEMU registered, so `./felis-linux-arm64 version` would happily succeed on
# an amd64 binary misnamed arm64 — which is exactly the failure the Dockerfile's
# ${TARGETARCH:-$(go env GOARCH)} fallback produces if buildx did not take. Both
# binaries come out of one RUN with one -ldflags string, so the stamp is checked once.
file ./felis-linux-arm64
file ./felis-linux-arm64 | grep -q 'ARM aarch64' \
|| { echo "felis-linux-arm64 is not an arm64 ELF — TARGETARCH did not reach the go build"; exit 1; }
- name: Checksum the binaries
run: sha256sum felis-linux-amd64 felis-linux-arm64 | tee SHA256SUMS
# A CycloneDX SBOM per binary: the Go modules (and versions) linked into it, read
# from the build info the linker embeds.
- uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
with:
file: dist/felis-linux-amd64
file: felis-linux-amd64
format: cyclonedx-json
output-file: sbom/felis-linux-amd64.cdx.json
output-file: felis-linux-amd64.cdx.json
upload-artifact: false
upload-release-assets: false
- uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
with:
file: dist/felis-linux-arm64
file: felis-linux-arm64
format: cyclonedx-json
output-file: sbom/felis-linux-arm64.cdx.json
output-file: felis-linux-arm64.cdx.json
upload-artifact: false
upload-release-assets: false
# Outside SHA256SUMS, which lists what bootstrap installs; beside it in the release.
- run: mv sbom/*.cdx.json dist/
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: release-assets
path: dist/
path: |
felis-linux-amd64
felis-linux-arm64
felis-linux-amd64.cdx.json
felis-linux-arm64.cdx.json
SHA256SUMS
if-no-files-found: error
retention-days: 7
@@ -113,7 +134,7 @@ jobs:
- run: sha256sum -c SHA256SUMS
# Signed SLSA provenance: which workflow run, commit and repository produced each
# asset. Check one with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`.
# binary. Check one with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`.
# GitHub only stores attestations for private repositories on Enterprise Cloud, and a
# failure here would block the release, so a private repository skips the step and
# relies on SHA256SUMS alone.
@@ -122,9 +143,8 @@ jobs:
uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0
with:
subject-path: |
felis-linux-*
felis-image-*.tar
felis-velocity.jar
felis-linux-amd64
felis-linux-arm64
# --verify-tag refuses to invent a release for a tag that is not pushed.
#
@@ -144,9 +164,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
run: |
# Every file the build handed over: SHA256SUMS names the installable ones, and the
# SBOMs ride along.
assets="$(ls)"
assets="felis-linux-amd64 felis-linux-arm64 felis-linux-amd64.cdx.json felis-linux-arm64.cdx.json SHA256SUMS"
flags=""
case "$GITHUB_REF_NAME" in *-*) flags="--prerelease" ;; esac
if ! gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
CONTRIBUTING.md
+2 -9
View File
@@ -78,15 +78,8 @@ FELIS_TEST_PG_URL='postgres://felis:***@127.0.0.1:5432/felis_pgint?sslmode=disab
```
Run it after touching anything under `internal/api/pgrepo.go`, `internal/submit`,
`internal/build` or `internal/dbbackup` that speaks SQL: the fakes encode the
contract, and this suite exists to catch the drift between the fakes and the real
queries. The `felis db backup` and `restore` tests also run `pg_dump`, `pg_restore`
and `psql`, which must be the server's major version. For a server in a container,
run them in it, as production does in felis-postgres:
```bash
FELIS_TEST_PG_EXEC='docker exec -i <container>' FELIS_TEST_PG_URL=... go test -tags pgint ./internal/pgint/
```
or `internal/build` that speaks SQL: the fakes encode the contract, and this
suite exists to catch the drift between the fakes and the real queries.
Build the CLI:
README.md
+3 -11
View File
@@ -1,8 +1,5 @@
# Felis
**此项目仍处于早期开发阶段，您不该在任何生产环境使用该项目。若产生任何问题，贵用户的使用行为与 FelisMC 团队无任何民事刑事法律关系。**
**THIS PROJECT IS STILL WIP, YOU SHOULD DO NOT USE THIS PROJECT IN ANY PRODUCTION USAGE. WE ARE NOT RESPOND FOR ANY LEGAL OR HUMANLY PROBLEM.**
一款 Kubernetes 驱动的 Minecraft 服务器托管平台，一行命令部署，自动管理生命周期与安全。
A Kubernetes-driven Minecraft server hosting platform — one command to deploy, automatic lifecycle, backup, and security.
@@ -22,8 +19,7 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy,
- **Web 控制面板**：浏览器中查看服务器状态、在线玩家与资源用量，管理备份与恢复。
- **备份与恢复**：一键把整服数据（世界、配置、插件/模组，即整个 /data 卷）打包进集群内的归档库，支持从任意备份点回滚；默认安装就已启用（归档 PVC 与路径由安装器一并生成）。
- **控制面数据库备份**：账号、服务器归属、配额与存档索引所在的数据库每天自动备份，每次升级迁移前先快照，出错可用 `felis db restore` 整库原子回滚；面板「维护与备份」页显示备份是否新鲜（见 [故障排查 §16](docs/troubleshooting.md)）。
- **运维自检**：`sudo felis status` 一屏列出节点、控制面、游戏代理、每台服务器、备份与未解决的告警；`sudo felis doctor` 把健康检查全跑一遍，按区域给出问题和下一步去哪看，不发邮件；`sudo felis support-bundle` 打出一个脱敏的诊断包，求助时直接附上（见 [故障排查 §0](docs/troubleshooting.md)）。
- **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。过期备份无论是否开启都会每天清理。
- **智慧回收（可选开启）**：超过 15 天无人游玩的世界自动备份后删除，释放磁盘空间；安装时设置 `FELIS_WORLDS_HOST_PATH`（k3s 默认 `/var/lib/rancher/k3s/storage`）即启用每日回收，不设置则不删任何世界。
- **多核心支持**：兼容 Paper、Fabric、Forge、NeoForge，经由 Velocity 代理统一入口。
- **模组自助提交**：玩家自行上传模组包，服主审批通过后自动构建；构建产物进入镜像白名单，可直接选用为服务器镜像完成部署。
- **Passkey 登录**：支持指纹、面容、硬件密钥等无密码认证方式。
@@ -31,17 +27,13 @@ A Kubernetes-driven Minecraft server hosting platform — one command to deploy,
## 使用方式
在准备好的 Linux 主机上执行（已验证的发行版与架构见 [运维手册 §1](docs/operations.md#1-supported-hosts)：CentOS Stream 9 aarch64 实机验证，Ubuntu 24.04 x86_64 每次推送由 CI 跑全新安装、重跑、升级和下面这条命令本身）：
在准备好的 Linux 主机上执行：
```bash
curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
```
脚本将自动安装 K3s，在 K3s 内部署 PostgreSQL 与控制平面，并启动设置向导。完成后浏览器访问已配置的域名进入控制面板即可使用。
安装发布版时，二进制、全部镜像与 Velocity 插件都取自该版本在 CI 里预构建好的 release 附件，逐个核对 `SHA256SUMS` 后导入，主机上无需 Docker、Gradle 或 Go，也不从 Docker Hub 拉取；某个附件缺失或校验不符时，只有那一个镜像退回到本机构建，并给出提示（见 [故障排查 §15c](docs/troubleshooting.md)）。附件也可以先拷到本机，再用 `FELIS_ARTIFACT_DIR=<绝对路径>` 安装，Felis 自己的二进制、镜像和插件就都取自这个目录；k3s 及其镜像、JRE、cloudflared、Velocity 和 Via 插件照旧从 GitHub 与 PaperMC 下载，RHEL、Fedora、openSUSE Leap 这类开着 SELinux 的主机还要从 rpm.rancher.io 装 k3s-selinux，系统软件包来自发行版的源。所以出网受限的主机要放行这几处的 HTTPS（或设 `https_proxy`），preflight 会在改动主机之前逐个探测，完全断网的主机目前装不了（地址清单见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。旧版本装在宿主上的 PostgreSQL 会在重跑时整库迁进 K3s，宿主上的那份停用保留，供回退（见 [运维手册 §4](docs/operations.md#4-upgrading-the-pieces-around-felis)）。
动手之前，脚本先检查内存、磁盘、端口、网段冲突、已有的 Kubernetes 和外网连通，把所有问题一次列出并停下，主机上什么都没改（检查项见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。
脚本将自动安装 K3s、部署控制平面并启动设置向导。完成后浏览器访问已配置的域名进入控制面板即可使用。
> **本仓库当前为私有**，上面这条会返回 404。请改用带凭据的形式；安装器自身也需要同一个 token
> 去解析并下载 release，所以用 `sudo -E` 把它带进去：
README_EN.md
+2 -8
View File
@@ -19,7 +19,6 @@ Table of Contents
- **Web Dashboard**: Monitor server status, online players, and resource usage from your browser, with backup and restore management.
- **Backup & Restore**: One-click snapshots of a server's whole data volume (worlds, config, plugins/mods — the entire /data volume) into the cluster's archive store, with rollback from any backup point — enabled by default (the installer renders the archive PVC and its path).
- **Control-plane database backups**: The database holding accounts, server ownership, quotas and the archive index is backed up daily and snapshotted before every upgrade migrates it; `felis db restore` rolls it back atomically, and the panel's Maintenance & Backups page shows whether the newest backup is fresh (see [troubleshooting §16](docs/troubleshooting.md)).
- **Self-check for operators**: `sudo felis status` shows the node, the control plane, the game proxy, every server, the backups and the open alerts on one screen; `sudo felis doctor` runs every health check once and lists each problem by area with where to look next, mailing nothing; `sudo felis support-bundle` writes one redacted diagnostics archive to attach when asking for help (see [troubleshooting §0](docs/troubleshooting.md)).
- **World Reaper** (opt in): Worlds idle for more than 15 days are automatically backed up and removed to free disk space. Enable it by setting `FELIS_WORLDS_HOST_PATH` at install time (on k3s: `/var/lib/rancher/k3s/storage`); without it, no world is ever deleted.
- **Multi-core Support**: Compatible with Paper, Fabric, Forge, and NeoForge, federated behind a Velocity proxy.
- **Modpack Submission**: Players submit custom modpacks; admin approval triggers an automatic build, and the result is whitelisted as a server image you can select to deploy.
@@ -28,18 +27,13 @@ Table of Contents
## Getting Started
On a prepared Linux host, run (the verified distributions and architectures are listed in
[operations §1](docs/operations.md#1-supported-hosts): CentOS Stream 9 on aarch64 is verified
on a real host, and Ubuntu 24.04 on x86_64 gets a fresh install, rerun and upgrade in CI on
every push):
On a prepared Linux host, run:
```bash
curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap.sh | sudo bash
```
The script installs K3s, deploys PostgreSQL and the control plane inside it, and launches a setup wizard. Once done, open your browser at the configured domain. A PostgreSQL an earlier release installed on the host is moved into K3s on the next rerun, and the host copy is stopped and kept for a rollback (see [Operations §4](docs/operations.md#4-upgrading-the-pieces-around-felis)).
Before it changes anything, the script checks RAM, disk, ports, network-range clashes, any Kubernetes already there and outbound access, lists every problem at once and stops with the host untouched (the checks are in [operations §1](docs/operations.md#1-supported-hosts)).
The script installs K3s, deploys the control plane, and launches a setup wizard. Once done, open your browser at the configured domain.
> **This repository is currently private**, so the command above returns 404. Use the
> credentialed form instead; the installer itself needs the same token to resolve and
bootstrap_asset.go
+4 -6
View File
@@ -22,17 +22,15 @@ var bootstrapAssets embed.FS
// developer's working tree carries gradle output (plugins/*/build, plugins/*/bin,
// and for the modded loaders a decompiled Minecraft under build/) which would
// otherwise be baked into every felis binary. Keep them explicit — add a source
// directory here, never a parent. Each module's gradle/verification-metadata.xml rides
// along, since Gradle builds unverified without it; the wrapper stays out, because the
// image builds run the pinned build image's own gradle.
// directory here, never a parent.
//
//go:embed deploy/game-stack.lock
//go:embed deploy/limbo/Dockerfile deploy/limbo/entrypoint.sh
//go:embed deploy/lobby/Dockerfile deploy/lobby/entrypoint.sh
//go:embed deploy/paper/Dockerfile deploy/paper/entrypoint.sh
//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src plugins/limbo/gradle/verification-metadata.xml
//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src plugins/paper/gradle/verification-metadata.xml
//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src plugins/velocity/gradle/verification-metadata.xml
//go:embed plugins/limbo/build.gradle plugins/limbo/settings.gradle plugins/limbo/src
//go:embed plugins/paper/build.gradle plugins/paper/settings.gradle plugins/paper/src
//go:embed plugins/velocity/build.gradle plugins/velocity/settings.gradle plugins/velocity/src
//go:embed plugins/shared/src
var gameStackAssets embed.FS
bootstrap_asset_test.go
+11 -198
View File
@@ -1,7 +1,6 @@
package felis
import (
"encoding/xml"
"io/fs"
"os"
"regexp"
@@ -211,7 +210,17 @@ func requireEmbedded(t *testing.T, path string) {
// with a strict KEY=value parser that dies on anything unexpected, so a malformed lock is a
// failed install on every host. Check the shipped copy the same way here.
func TestGameStackLockIsComplete(t *testing.T) {
lock := gameStackLock(t)
lock := map[string]string{}
for line := range strings.SplitSeq(readGameStackFile(t, "deploy/game-stack.lock"), "\n") {
if line == "" || strings.HasPrefix(line, "#") {
continue
}
k, v, ok := strings.Cut(line, "=")
if !ok {
t.Fatalf("not a KEY=value line: %q", line)
}
lock[k] = v
}
m := regexp.MustCompile(`GAME_STACK_LOCK_KEYS="([^"]*)"`).FindStringSubmatch(BootstrapScript())
if m == nil {
t.Fatal("bootstrap.sh no longer declares GAME_STACK_LOCK_KEYS")
@@ -304,202 +313,6 @@ func TestDockerfileBaseImagesArePinnedByDigest(t *testing.T) {
}
}
// The plugin jars are built in four places the installer controls — the lobby and limbo
// image builds, bootstrap's Velocity build and the release build of the same jar — and
// through each module's wrapper by a developer or CI. A tag alone is whatever it points
// at on build day, and two Gradle versions are two chances for a build to pass in one
// place and break in the other, so all of them run one image, pinned by digest, whose
// Gradle is the wrappers' Gradle.
func TestPluginBuildsRunOnePinnedGradle(t *testing.T) {
sources := map[string]string{
"deploy/lobby/Dockerfile": readGameStackFile(t, "deploy/lobby/Dockerfile"),
"deploy/limbo/Dockerfile": readGameStackFile(t, "deploy/limbo/Dockerfile"),
"deploy/bootstrap.sh": BootstrapScript(),
// The release build compiles felis-velocity.jar for hosts that install prebuilt.
"deploy/build-release-artifacts.sh": readRepoFile(t, "deploy/build-release-artifacts.sh"),
}
anyRef := regexp.MustCompile(`gradle:[\w.-]+(@sha256:\w+)?`)
pinned := regexp.MustCompile(`^gradle:(\d+\.\d+(?:\.\d+)?)-jdk\d+@sha256:[0-9a-f]{64}$`)
images := map[string]bool{}
gradle := ""
for name, body := range sources {
refs := anyRef.FindAllString(body, -1)
if len(refs) == 0 {
t.Errorf("%s names no gradle image", name)
}
for _, ref := range refs {
m := pinned.FindStringSubmatch(ref)
if m == nil {
t.Errorf("%s: %s is not a gradle image pinned by digest", name, ref)
continue
}
images[ref] = true
gradle = m[1]
}
}
if len(images) != 1 {
t.Fatalf("the plugin builds use %d different gradle images, want one: %v", len(images), images)
}
// bootstrap names the image once and has to spend it where it builds the jar.
if !strings.Contains(BootstrapScript(), `"$PLUGIN_BUILD_IMAGE" gradle --no-daemon clean build`) {
t.Error("build_velocity_plugin does not build in $PLUGIN_BUILD_IMAGE")
}
for _, module := range []string{"velocity", "paper", "limbo"} {
props := wrapperProperties(t, module)
if want := "gradle-" + gradle + "-bin.zip"; !strings.HasSuffix(props["distributionUrl"], "/"+want) {
t.Errorf("plugins/%s wrapper runs %s; the image builds run Gradle %s", module, props["distributionUrl"], gradle)
}
}
// The mods are no part of the install, but a wrapper without a checksum runs whatever
// the download handed it.
for _, module := range []string{"velocity", "paper", "limbo", "fabric", "forge", "neoforge"} {
if sum := wrapperProperties(t, module)["distributionSha256Sum"]; !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(sum) {
t.Errorf("plugins/%s wrapper pins no distribution sha256 (got %q)", module, sum)
}
}
}
// Each plugin compiles against the API of the exact build the install runs, and Gradle
// checks those bytes against the module's verification file. Nothing but this test ties
// the three to deploy/game-stack.lock: a lock refresh that leaves them behind builds the
// lobby against yesterday's API, or fails every image build on a checksum the file does
// not have.
func TestPluginApisAreTheLockedBuilds(t *testing.T) {
lock := gameStackLock(t)
// Paper: paper-<mc>-<build>.jar runs; paper-api <mc>.build.<build>-<channel> compiles.
jar := regexp.MustCompile(`/paper-([^/]+)-(\d+)\.jar$`).FindStringSubmatch(lock["PAPER_JAR_URL"])
if jar == nil {
t.Fatalf("PAPER_JAR_URL %s does not name paper-<mc>-<build>.jar", lock["PAPER_JAR_URL"])
}
dep := regexp.MustCompile(`compileOnly 'io\.papermc\.paper:paper-api:([^']+)'`).
FindStringSubmatch(readGameStackFile(t, "plugins/paper/build.gradle"))
if dep == nil {
t.Fatal("plugins/paper/build.gradle declares no paper-api dependency")
}
if !regexp.MustCompile(`^` + regexp.QuoteMeta(jar[1]+".build."+jar[2]) + `(-[a-z]+)?$`).MatchString(dep[1]) {
t.Errorf("paper-api %s is not the API of the locked server paper-%s-%s.jar", dep[1], jar[1], jar[2])
}
requireVerified(t, "paper", "io.papermc.paper", "paper-api", dep[1])
// Limbo: the lock's release, passed to the image build, which refuses to guess one.
limbo := lock["LIMBO_VERSION"]
if !strings.Contains(BootstrapScript(), `--build-arg LIMBO_VERSION="$LIMBO_VERSION"`) {
t.Error("bootstrap.sh does not pass the locked LIMBO_VERSION to the limbo image build")
}
dockerfile := readGameStackFile(t, "deploy/limbo/Dockerfile")
if !regexp.MustCompile(`(?m)^ARG LIMBO_VERSION$`).MatchString(dockerfile) ||
!strings.Contains(dockerfile, `if [ -z "${LIMBO_VERSION:-}" ]`) {
t.Error("deploy/limbo/Dockerfile does not require LIMBO_VERSION; a build without it would " +
"compile against a version nobody chose")
}
// LOOHP publishes the jar the login gate runs as the Limbo API artifact itself, so the
// checksum Gradle holds for it is the lock's: compiled-against and running are one file.
if got := requireVerified(t, "limbo", "com.loohp", "Limbo", limbo)["Limbo-"+limbo+".jar"]; got != lock["LIMBO_JAR_SHA256"] {
t.Errorf("verification-metadata.xml holds %q for Limbo-%s.jar; the login gate runs %s", got, limbo, lock["LIMBO_JAR_SHA256"])
}
// Velocity: the API default is the proxy the install runs.
api := regexp.MustCompile(`findProperty\('velocityApi'\) \?: '([^']+)'`).
FindStringSubmatch(readGameStackFile(t, "plugins/velocity/build.gradle"))
if api == nil {
t.Fatal("plugins/velocity/build.gradle has no velocityApi default")
}
if api[1] != lock["VELOCITY_VERSION"] {
t.Errorf("velocity-api defaults to %s; the install runs Velocity %s", api[1], lock["VELOCITY_VERSION"])
}
requireVerified(t, "velocity", "com.velocitypowered", "velocity-api", api[1])
}
// requireVerified asserts the module's shipped verification file checks metadata and
// pins group:name:version, and returns that component's artifact sha256s by file name.
func requireVerified(t *testing.T, module, group, name, version string) map[string]string {
t.Helper()
path := "plugins/" + module + "/gradle/verification-metadata.xml"
var doc struct {
VerifyMetadata bool `xml:"configuration>verify-metadata"`
Components []struct {
Group string `xml:"group,attr"`
Name string `xml:"name,attr"`
Version string `xml:"version,attr"`
Artifacts []struct {
Name string `xml:"name,attr"`
SHA256 []struct {
Value string `xml:"value,attr"`
} `xml:"sha256"`
} `xml:"artifact"`
} `xml:"components>component"`
}
if err := xml.Unmarshal([]byte(readGameStackFile(t, path)), &doc); err != nil {
t.Fatalf("%s: %v", path, err)
}
if !doc.VerifyMetadata {
t.Errorf("%s does not verify metadata; a swapped pom could redirect the graph", path)
}
for _, c := range doc.Components {
if c.Group != group || c.Name != name || c.Version != version {
continue
}
sums := map[string]string{}
for _, a := range c.Artifacts {
if len(a.SHA256) > 0 {
sums[a.Name] = a.SHA256[0].Value
}
}
if sums[name+"-"+version+".jar"] == "" {
t.Errorf("%s pins %s:%s:%s but no sha256 for its jar", path, group, name, version)
}
return sums
}
t.Errorf("%s has no checksum for %s:%s:%s; the build would refuse it", path, group, name, version)
return nil
}
// wrapperProperties reads a module's gradle-wrapper.properties off disk: the wrappers are
// for developers and CI, and nothing embeds them.
func wrapperProperties(t *testing.T, module string) map[string]string {
t.Helper()
b, err := os.ReadFile("plugins/" + module + "/gradle/wrapper/gradle-wrapper.properties")
if err != nil {
t.Fatal(err)
}
props := map[string]string{}
for line := range strings.SplitSeq(string(b), "\n") {
if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok && !strings.HasPrefix(k, "#") {
props[k] = strings.ReplaceAll(v, `\:`, ":")
}
}
return props
}
// gameStackLock parses the shipped deploy/game-stack.lock the way bootstrap.sh does.
func gameStackLock(t *testing.T) map[string]string {
t.Helper()
lock := map[string]string{}
for line := range strings.SplitSeq(readGameStackFile(t, "deploy/game-stack.lock"), "\n") {
if line == "" || strings.HasPrefix(line, "#") {
continue
}
k, v, ok := strings.Cut(line, "=")
if !ok {
t.Fatalf("not a KEY=value line: %q", line)
}
lock[k] = v
}
return lock
}
// readRepoFile reads a file of the checkout that the binary does not embed.
func readRepoFile(t *testing.T, name string) string {
t.Helper()
b, err := os.ReadFile(name)
if err != nil {
t.Fatal(err)
}
return string(b)
}
func readGameStackFile(t *testing.T, name string) string {
t.Helper()
b, err := gameStackAssets.ReadFile(name)
cmd/felis/api.go
+66 -401
View File
@@ -8,10 +8,8 @@ import (
"log/slog"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
@@ -31,16 +29,13 @@ import (
"felis.lolicon.best/internal/reaper"
"felis.lolicon.best/internal/registryprune"
"felis.lolicon.best/internal/restore"
"felis.lolicon.best/internal/retention"
"felis.lolicon.best/internal/store"
"felis.lolicon.best/internal/submit"
"felis.lolicon.best/internal/worldexport"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -50,22 +45,6 @@ import (
// the code marks Identity (UUIDs trusted verbatim); config can never add another.
const mojangSessionServer = "https://sessionserver.mojang.com/session/minecraft/hasJoined"
// passkeyRelyingParty is the one WebAuthn relying party both web faces share: its id
// is the player console host, derived from server.root_domain the way the panel
// handler derives it when auth.panel_hostname is unset, and its origins are that host
// plus the operator host. An empty id means the install names no panel host at all.
func passkeyRelyingParty(cfg *config.Config) (string, []string) {
rpID := defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname)
if rpID == "" {
return "", nil
}
origins := []string{"https://" + rpID}
if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != rpID {
origins = append(origins, "https://"+admin)
}
return rpID, origins
}
// authSourcesFromConfig builds the multiplexer's priority list from the configured
// [[auth_source]] entries: Mojang leads as the code-owned identity anchor (正版优先, the ONLY
// Identity source — config can only append namespace-rewritten third-party sources, never a
@@ -81,8 +60,10 @@ func authSourcesFromConfig(configured []config.AuthSourceConfig) []api.AuthSourc
}
// cmdAPI runs felis-api: two listeners, two middleware chains (spec §7). The
// internal face authenticates per-caller service tokens; the external face
// authenticates the local session cookie.
// internal face (service token) is fully wired. The external face is wired but
// fails closed until an Access JWKS key function is configured — the verifier's
// audience logic is unit-tested (internal/api), the JWKS source is a deployment
// integration point.
func cmdAPI(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("api", flag.ContinueOnError)
fs.SetOutput(stderr)
@@ -105,19 +86,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
return 1
}
// Load already refused a malformed [audit] retention.
auditRetention, _ := cfg.Audit.RetentionPeriod()
if auditRetention == 0 {
fmt.Fprintln(stdout, "felis api: audit rows are kept forever ([audit] retention = \"forever\")")
} else {
fmt.Fprintf(stdout, "felis api: audit rows older than %d days are deleted ([audit] retention; export them first with felis db audit-export)\n", int(auditRetention/(24*time.Hour)))
}
ctx := ctrl.SetupSignalHandler()
// Before anything serves: an api on a schema it was not built for answers with
// errors, or writes rows the other version cannot read.
drv, err := openPodStore(ctx, cfg.Database.URL, "api", stderr)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
fmt.Fprintf(stderr, "felis api: open database: %v\n", err)
return 1
@@ -146,22 +117,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
metrics.SetBuildInfo("api", resolvedVersion())
internalAuth, err := internalCallerTokens(os.Getenv)
if err != nil {
fmt.Fprintf(stderr, "felis api: internal face tokens: %v\n", err)
return 1
}
for _, ct := range naming.CallerTokens {
if internalAuth[api.Caller(ct.Caller)] == "" {
fmt.Fprintf(stderr, "felis api: warning: %s unset — the internal face turns the %s caller away\n", ct.APIEnv, ct.Caller)
}
token := os.Getenv("FELIS_SERVICE_TOKEN")
if token == "" {
fmt.Fprintln(stderr, "felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers")
}
// Email one-time codes go through the [smtp] relay when one is configured; the
// password is read from the env var password_ref names (default SMTPPasswordEnv,
// injected from the felis-smtp Secret). No [smtp] host ⇒ mailer stays nil and
// every door that mails a code answers 503 mail_unavailable: a code that is
// not mailed is never written anywhere else either.
// deliverOTP logs each code server-side (the pre-SMTP bootstrap posture).
var mailer api.OTPMailer
if cfg.SMTP.Host != "" {
passRef := cfg.SMTP.PasswordRef
@@ -172,12 +136,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
if cfg.SMTP.Username != "" && password == "" {
fmt.Fprintf(stderr, "felis api: warning: [smtp] username is set but credentials env %s is empty — OTP sends will fail AUTH\n", passRef)
}
mailer = smtpRelay(cfg.SMTP, password)
if !cfg.SMTP.TLSRequired() {
fmt.Fprintf(stderr, "felis api: warning: [smtp] %s may be sent codes without TLS (require_tls off or a relay on this host)\n", cfg.SMTP.Host)
mailer = &mail.SMTP{
Host: cfg.SMTP.Host,
Port: cfg.SMTP.Port,
From: cfg.SMTP.From,
Username: cfg.SMTP.Username,
Password: password,
}
} else {
fmt.Fprintln(stderr, "felis api: [smtp] not configured — email sign-in and verification are off (503 mail_unavailable); sign in with a passkey, or run felis setup to add a relay")
fmt.Fprintln(stderr, "felis api: [smtp] not configured — email one-time codes are logged, not mailed")
}
// Build subsystem (spec §16): the weak-SA build Job runs in the configured
@@ -190,10 +157,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
buildCfg.FelisImage = os.Getenv("FELIS_IMAGE")
buildJobs := build.NewK8sJobs(cl, buildCfg)
builder := &build.Builder{
Store: build.NewPGStore(drv.DB()),
Jobs: buildJobs,
Config: buildCfg,
Outcomes: build.NewK8sOutcomes(clientset, buildCfg),
Store: build.NewPGStore(drv.DB()),
Jobs: buildJobs,
Config: buildCfg,
}
go probeBuildUserNamespaces(ctx, buildJobs, buildCfg, stderr)
@@ -247,9 +213,6 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
ContextBaseURL: internalAPIBaseURL(),
Blobs: blobs,
}
if blobs != nil {
submissions.Parts = &submit.PartStore{Dir: uploadPartsDir(contextBase)}
}
if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
if n, err := parseByteSize(v); err != nil || n <= 0 {
fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
@@ -257,11 +220,6 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
submissions.MaxStoredBytesTotal = n
}
}
if n, err := contextMaxBytes(cfg); err != nil {
fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 512Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
} else {
submissions.MaxContextBytes = n
}
// Restore subsystem (spec §7): the weak-SA restore Job mounts the target
// world PVC + the backup PVC and runs `felis restore`. It needs deployment-
@@ -293,17 +251,6 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503")
}
// World export: a weak-SA Job mounts the world PVC, or the backup PVC, read-only
// and PUTs the archive to the internal face, which streams it on to the owner's
// browser (internal/worldexport). A backup export mounts the backup PVC, so it
// is wired under the restore gate; otherwise the export routes return 503.
var exporter api.Exporter
if felisImage != "" && backupPVC != "" {
exporter = worldexport.New(clientset, exportConfig(cfg, felisImage, backupPVC))
} else {
fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503")
}
// Server file editor: a weak-SA Job mounts ONLY the target world PVC and runs
// `felis files`, printing its result for felis-api to read back through
// pods/log (see internal/fileedit). It needs FELIS_IMAGE but — unlike restore
@@ -312,22 +259,12 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// endpoints honestly return 503. It takes the typed clientset rather than the
// controller-runtime client because the log subresource lives only on the typed
// CoreV1 client, and one client covers its Job create, Pod list, and log read.
//
// Uploads additionally stage their bytes on this pod's disk until the Job
// fetches them from the internal face; whatever a previous process staged is
// orphaned (the index is in memory), so the stage starts empty.
var files api.FileEditor
var fileStage *fileedit.Stage
if felisImage != "" {
files = &fileedit.Editor{
Runner: fileedit.NewK8sRunner(clientset),
Config: fileEditConfig(cfg, felisImage),
}
fileStage = &fileedit.Stage{Dir: fileStagingDir()}
if err := fileStage.Sweep(); err != nil {
fmt.Fprintf(stderr, "felis api: %v — file uploads return 503\n", err)
fileStage = nil
}
} else {
fmt.Fprintln(stderr, "felis api: file editor disabled (needs FELIS_IMAGE) — file endpoints return 503")
}
@@ -337,9 +274,6 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// the same store the auth handlers write to, so a login and the next request
// agree on what local auth knows.
repo := api.NewPGRepo(drv.DB())
if err := api.RegisterStorePool(drv.DB()); err != nil {
fmt.Fprintf(stderr, "felis api: store pool metrics unavailable: %v\n", err)
}
// The owner's on-demand backup levers come from [archive], the same keys the
// backup Job and the reaper read. A malformed key leaves the defaults in
@@ -350,13 +284,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
rcfg = reaper.DefaultConfig()
}
serverCache, serversSynced, err := startServerCache(ctx, restCfg, scheme, cfg.K8s.Namespace, stderr)
if err != nil {
fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err)
return 1
}
cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced)
jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace)
cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace)
a := &api.API{
Repo: repo,
Cluster: cluster,
@@ -364,31 +292,26 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
Logs: api.NewK8sLogStreamer(clientset, cfg.K8s.Namespace),
// Build-log stream (spec §16) is scoped to the BUILD namespace — the same
// value the Builder renders Jobs into — so it follows where build Pods run.
BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
Internal: internalAuth,
Builder: builder,
Images: imagePinner(cfg.Registry.URL),
Restorer: restorer,
Backuper: backuper,
JobStatus: jobStatus,
// A restore starts with a safety snapshot; settleRestoreChains starts the
// restore behind each one.
RestoreChains: jobStatus,
Files: files,
FileStage: fileStage,
// The file Job fetches an upload from here; it runs in the minecraft
// namespace, where the internal face is reachable like it is for the login
// gate.
InternalBaseURL: internalAPIBaseURL(),
Exporter: exporter,
Submissions: submissions,
Mailer: mailer,
Schedules: repo,
// The external face authenticates the local session cookie the sign-in doors
// mint, live once `felis breakGlass` flips local_auth_enabled on. Cloudflare
// Access, when the install sits behind it, is enforced at the edge only.
BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
Internal: api.BearerTokenAuth{Token: token},
Builder: builder,
Images: imagePinner(cfg.Registry.URL),
Restorer: restorer,
Backuper: backuper,
JobStatus: api.NewK8sJobStatus(cl, cfg.K8s.Namespace),
Files: files,
Submissions: submissions,
Mailer: mailer,
// The external face is fronted by SessionAuth: it prefers a local session
// cookie (minted by the passwordless doors) and otherwise delegates to the
// Cloudflare-Access JWT verifier, so both auth models coexist on one face. The
// delegate's Keyfunc is intentionally nil — the JWT path fails closed until a
// JWKS-backed key function is wired (deployment integration point) — while the
// local session path is live the moment `felis breakGlass` flips
// local_auth_enabled on.
External: api.SessionAuth{
Repo: repo,
Delegate: api.AccessVerifier{Audience: cfg.Auth.AccessJWTAud},
RootDomain: cfg.Server.RootDomain,
AdminHostname: cfg.Auth.AdminHostname,
},
@@ -417,6 +340,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(),
MailLimit: mailLimit(cfg.SMTP.MaxPerHour),
}
fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)")
if a.ClientIPHeader != "" {
fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader)
} else {
@@ -438,15 +362,22 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// enrolled once asserts on either face — one binding, usable on the player console
// AND the operator console. Both hosts are therefore listed as permitted origins,
// while the RP id stays the panel host so the credential's scope is ONE relying
// party, not two. Without a panel host (neither auth.panel_hostname nor
// server.root_domain) a.Passkey stays nil and the passkey routes honestly return
// 503 (the authenticated enrollment boundary is still enforced by the handlers).
if rpID, origins := passkeyRelyingParty(cfg); rpID == "" {
fmt.Fprintln(stderr, "felis api: passkey verifier disabled (no panel host: set server.root_domain or auth.panel_hostname) — passkey endpoints return 503")
} else if pv, err := passkey.New(rpID, "Felis", origins); err != nil {
fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
// party, not two. Wired only when auth.panel_hostname is configured; otherwise
// a.Passkey stays nil and the passkey routes honestly return 503 (the authenticated
// enrollment boundary is still enforced by the handlers).
if cfg.Auth.PanelHostname != "" {
origins := []string{"https://" + cfg.Auth.PanelHostname}
if admin := defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname); admin != "" && admin != cfg.Auth.PanelHostname {
origins = append(origins, "https://"+admin)
}
pv, err := passkey.New(cfg.Auth.PanelHostname, "Felis", origins)
if err != nil {
fmt.Fprintf(stderr, "felis api: passkey verifier disabled: %v — passkey endpoints return 503\n", err)
} else {
a.Passkey = pv
}
} else {
a.Passkey = pv
fmt.Fprintln(stderr, "felis api: passkey verifier disabled (auth.panel_hostname unset) — passkey endpoints return 503")
}
// Derive the console hostnames when felis.toml leaves them unset, exactly as the
@@ -455,7 +386,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain,
defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname),
defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname),
cfg.Velocity.GamePort, resolvedVersion())
resolvedVersion())
internalSrv := newAPIServer(*internalAddr, a.InternalHandler())
externalSrv := newAPIServer(cfg.Server.Listen, externalHandler)
@@ -477,39 +408,21 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// and advance any whose Job has reached a terminal phase. GET on a build also
// reconciles it, but this loop converges builds nobody is polling.
go reconcileBuilds(ctx, builder, stderr)
go settleRestoreChains(ctx, a, stderr)
go runSchedules(ctx, a, stderr)
// A daily restore point of every world played since its last one, taken
// once the server stops ([archive] scheduled_every; 0s turns it off).
if backuper != nil && rcfg.ScheduledEvery > 0 {
go scheduleBackups(ctx, &api.BackupScheduler{API: a, Store: repo, Jobs: jobStatus, Every: rcfg.ScheduledEvery}, stderr)
} else {
fmt.Fprintln(stderr, "felis api: scheduled backups off (needs the backup executor and [archive] scheduled_every above 0s)")
}
if pruner := registryPruner(cfg, builder.Store, cluster, stderr); pruner != nil {
go pruner.Loop(ctx, registryPruneInterval)
}
go reapRejectedContexts(ctx, submissions, stderr)
if fileStage != nil {
go expireFileSessions(ctx, fileStage, fileSessionSweep, stderr)
}
if exporter != nil {
go expireExports(ctx, a, exportSweep)
}
go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
servers := []*http.Server{internalSrv, externalSrv}
if httpsSrv != nil {
servers = append(servers, httpsSrv)
}
for _, srv := range servers {
srv.RegisterOnShutdown(a.CloseStreams)
}
select {
case <-ctx.Done():
shutdownServers(servers, apiShutdownGrace, stderr)
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = internalSrv.Shutdown(shutdownCtx)
_ = externalSrv.Shutdown(shutdownCtx)
if httpsSrv != nil {
_ = httpsSrv.Shutdown(shutdownCtx)
}
return 0
case err := <-errc:
if err != nil && err != http.ErrServerClosed {
@@ -529,30 +442,8 @@ const (
// apiIdleTimeout caps how long a kept-alive connection may sit idle between
// requests before the server closes it, bounding idle-connection exhaustion.
apiIdleTimeout = 120 * time.Second
// apiShutdownGrace is how long the listeners drain after SIGTERM. The pod gets
// the Kubernetes default of 30s before SIGKILL; this leaves the rest for the
// process to exit.
apiShutdownGrace = 20 * time.Second
)
// shutdownServers drains every listener at once under one deadline: in turn, a
// slow first listener would spend the time the others needed. Log streams end
// through RegisterOnShutdown (API.CloseStreams); what is still running when the
// deadline passes is cut off with the process.
func shutdownServers(servers []*http.Server, grace time.Duration, stderr io.Writer) {
ctx, cancel := context.WithTimeout(context.Background(), grace)
defer cancel()
var wg sync.WaitGroup
for _, srv := range servers {
wg.Go(func() {
if err := srv.Shutdown(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: shutdown %s: %v\n", srv.Addr, err)
}
})
}
wg.Wait()
}
// newAPIServer builds an http.Server with hardened header/idle timeouts (gosec
// G112) shared by all three felis-api listeners (internal, external, https).
// WriteTimeout and ReadTimeout are deliberately LEFT UNSET: the external and https
@@ -590,9 +481,6 @@ func buildConfig(cfg *config.Config) build.Config {
MaxConcurrent: cfg.Registry.MaxConcurrentBuilds,
TrivyDBRepository: cfg.Registry.TrivyDBRepository,
TrivyJavaDBRepository: cfg.Registry.TrivyJavaDBRepository,
ScanFailOn: cfg.Registry.ScanFailOn,
ScanFailUnfixed: cfg.Registry.ScanFailUnfixed,
ScanAccept: cfg.Registry.ScanAccept,
// The submit lane's derived context URLs live here; the fetch step's
// service token goes nowhere else.
ContextOrigin: internalAPIBaseURL(),
@@ -706,17 +594,6 @@ func backupConfig(cfg *config.Config, image, backupPVC string) backupjob.Config
}
}
// exportConfig builds the world export executor's config. BackupRoot mirrors
// restoreConfig: the stored refs are absolute paths under [archive] local_path.
func exportConfig(cfg *config.Config, image, backupPVC string) worldexport.Config {
return worldexport.Config{
Namespace: cfg.K8s.Namespace,
Image: image,
BackupPVC: backupPVC,
BackupRoot: cfg.Archive.LocalPath,
}
}
// fileEditConfig builds the file editor's config from felis.toml plus the
// deployment-supplied image. It is the shortest of the three: the editor mounts
// only the world PVC, so it needs no archive coordinates at all, and everything
@@ -747,112 +624,10 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
}
}
// settleRestoreChains starts the restore behind each safety snapshot that has
// finished (and gives up the one behind a snapshot that failed). The world stays
// locked in between, so the interval is how long a finished snapshot keeps the
// server down before its restore begins.
func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
t := time.NewTicker(5 * time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := a.SettleRestoreChains(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: restore chains: %v\n", err)
}
}
}
}
// runSchedules runs the servers' scheduled tasks (api.API.RunSchedules). The
// interval is how late a task may start, and how often a restart or backup in
// progress checks whether it can take its next step.
func runSchedules(ctx context.Context, a *api.API, stderr io.Writer) {
t := time.NewTicker(15 * time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := a.RunSchedules(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: scheduled tasks: %v\n", err)
}
}
}
}
// scheduleBackups starts the scheduled backups (api.BackupScheduler). Each tick
// starts at most one, so the interval also spaces the worlds that stopped at
// the same time: a world that stops waits at most this long for its point to
// start once the Jobs ahead of it are done.
func scheduleBackups(ctx context.Context, s *api.BackupScheduler, stderr io.Writer) {
t := time.NewTicker(2 * time.Minute)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := s.Tick(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: scheduled backups: %v\n", err)
}
}
}
}
// fileSessionSweep is how often expireFileSessions looks for idle upload
// sessions: small beside fileedit.SessionIdle, so an abandoned one gives its
// room back within minutes of going stale.
const fileSessionSweep = 10 * time.Minute
// expireFileSessions drops the file manager's upload sessions left untouched
// for fileedit.SessionIdle. Each reserved room on the staging disk for its whole
// file when it began, so one abandoned would otherwise hold that room until
// felis-api restarts.
func expireFileSessions(ctx context.Context, s *fileedit.Stage, every time.Duration, stderr io.Writer) {
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if n := s.Expire(); n > 0 {
fmt.Fprintf(stderr, "felis api: dropped %d upload session(s) left idle for %s\n", n, fileedit.SessionIdle)
}
}
}
}
// exportSweep is how often expireExports runs: an export whose Job never
// connected is stopped within a minute of going stale.
const exportSweep = time.Minute
// expireExports runs the export sweep (api.API.ExpireExports) on a ticker. The
// export routes sweep as they are called, and an owner who closed the tab calls
// none; a Job whose Pod never got going would then keep the server from
// starting until the Job's deadline.
func expireExports(ctx context.Context, a interface{ ExpireExports() }, every time.Duration) {
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
a.ExpireExports()
}
}
}
// reapRejectedContexts deletes, once an hour, the uploaded contexts of
// submissions rejected more than submit.RejectedContextRetention ago, and the
// chunked uploads left untouched for submit.StalePartRetention. Without it a
// rejected modpack or an abandoned upload keeps its bytes on the uploads store
// (and against its submitter's budget) until an admin deletes the row.
// submissions rejected more than submit.RejectedContextRetention ago. Without it a
// rejected modpack keeps its bytes on the uploads store (and against its
// submitter's budget) until an admin deletes the row.
func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
t := time.NewTicker(time.Hour)
defer t.Stop()
@@ -864,13 +639,6 @@ func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writ
if n > 0 {
fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
}
n, err = m.ReapStaleParts(submit.StalePartRetention)
if err != nil {
fmt.Fprintf(stderr, "felis api: reap abandoned uploads: %v\n", err)
}
if n > 0 {
fmt.Fprintf(stderr, "felis api: deleted %d abandoned chunked upload(s)\n", n)
}
select {
case <-ctx.Done():
return
@@ -879,11 +647,6 @@ func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writ
}
}
// retentionInterval spaces the runs that delete spent sign-in rows and audit rows
// past [audit] retention. The rows are spent for weeks before they go, so a few
// runs a day keep the tables flat.
const retentionInterval = 6 * time.Hour
// registryPruneInterval spaces the registry pruner's runs. The registry-gc
// sidecar sweeps once a day, so pruning more often only changes which sweep frees
// a layer.
@@ -984,101 +747,3 @@ func imagePinner(registry string) api.ImagePinner {
}
return imagepin.Resolver{Registry: registry}
}
// internalCallerTokens reads each internal caller's token from the env var the
// Deployment feeds it from (naming.CallerTokens). Two callers sharing a value
// would make the caller ambiguous, so that refuses to start.
func internalCallerTokens(getenv func(string) string) (api.CallerTokens, error) {
tokens := map[api.Caller]string{}
for _, ct := range naming.CallerTokens {
tokens[api.Caller(ct.Caller)] = strings.TrimSpace(getenv(ct.APIEnv))
}
return api.NewCallerTokens(tokens)
}
// smtpRelay is the relay [smtp] names, with the resolved password and the TLS
// posture config.SMTPConfig.TLSRequired picks. felis api, the reaper and the
// watchdog all send through it, so none can drift to a weaker posture.
func smtpRelay(c config.SMTPConfig, password string) *mail.SMTP {
return &mail.SMTP{
Host: c.Host,
Port: c.Port,
From: c.From,
Username: c.Username,
Password: password,
RequireTLS: c.TLSRequired(),
}
}
// startServerCache starts the informer that serves the api's fleet-wide
// MinecraftServer reads (api.K8sCluster.WithServerCache): one watch on the
// namespace instead of a full List per velocity pull, fleet page and wake. It
// caches MinecraftServers only — ReaderFailOnMissingInformer turns any other read
// through it into an error rather than a new informer the api's Role cannot back —
// indexes spec.subdomain for GetBySubdomain, and drops managedFields to keep the
// copy small. It returns without waiting: the reads block until the first list
// lands and /readyz reports not-ready until then.
func startServerCache(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, stderr io.Writer) (cache.Cache, func() bool, error) {
c, err := cache.New(cfg, cache.Options{
Scheme: scheme,
DefaultNamespaces: map[string]cache.Config{namespace: {}},
DefaultTransform: cache.TransformStripManagedFields(),
ReaderFailOnMissingInformer: true,
})
if err != nil {
return nil, nil, err
}
if err := c.IndexField(ctx, &v1alpha1.MinecraftServer{}, api.SubdomainIndex, api.SubdomainOf); err != nil {
return nil, nil, fmt.Errorf("index %s: %w", api.SubdomainIndex, err)
}
inf, err := c.GetInformer(ctx, &v1alpha1.MinecraftServer{})
if err != nil {
return nil, nil, err
}
go func() {
if err := c.Start(ctx); err != nil {
fmt.Fprintf(stderr, "felis api: MinecraftServer cache stopped: %v\n", err)
}
}()
return c, inf.HasSynced, nil
}
// uploadPartsDir is where chunked uploads are staged: beside a local store's
// contexts, so the room check and the budget see one disk and a staged upload
// survives an API restart; for an s3:// store, on the uploads volume the
// platform mounts either way, or the pod's /tmp when run by hand without it.
func uploadPartsDir(contextBase string) string {
if isLocalUploadsPath(contextBase) {
return filepath.Join(strings.TrimPrefix(contextBase, "file://"), ".parts")
}
if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
return filepath.Join(platform.UploadsLocalPath, ".parts")
}
return filepath.Join(os.TempDir(), "felis-upload-parts")
}
// fileStagingDir is where file uploads wait for their Job: on the uploads
// volume, whose capacity is its own, or the pod's /tmp when run by hand without
// it — /tmp is the node's disk, which a burst of uploads should not fill.
func fileStagingDir() string {
if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
return filepath.Join(platform.UploadsLocalPath, ".file-staging")
}
return filepath.Join(os.TempDir(), "felis-file-staging")
}
// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
// package's own default (1 GiB). The Cloudflare edge refuses a single request
// body over 100 MB, which the panel's chunked upload stays under, so the edge
// does not lower the cap.
func contextMaxBytes(cfg *config.Config) (int64, error) {
v := cfg.Registry.ContextMaxBytes
if v == "" {
return 0, nil
}
n, err := parseByteSize(v)
if err != nil || n <= 0 {
return 0, fmt.Errorf("not a positive size: %q", v)
}
return n, nil
}
cmd/felis/api_test.go
-156
View File
@@ -1,56 +1,17 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"slices"
"sync"
"sync/atomic"
"testing"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/build"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/fileedit"
)
// The passkey relying party follows the panel host the SPA is served on: an install
// that names only its root domain still gets passkeys, on console.<root>, with the
// operator host as the second origin; only an install with no panel host goes without.
func TestPasskeyRelyingParty(t *testing.T) {
for _, tc := range []struct {
name string
root, panel, admin string
wantRP string
wantOrigins []string
}{
{"root domain only", "example.net", "", "", "console.example.net",
[]string{"https://console.example.net", "https://op.console.example.net"}},
{"configured hosts", "example.net", " play.example.net ", "ops.example.net", "play.example.net",
[]string{"https://play.example.net", "https://ops.example.net"}},
{"operator host equal to the panel host", "example.net", "console.example.net", "console.example.net",
"console.example.net", []string{"https://console.example.net"}},
{"panel host without a root domain", "", "console.example.org", "", "console.example.org",
[]string{"https://console.example.org"}},
{"no host at all", "", "", "", "", nil},
} {
t.Run(tc.name, func(t *testing.T) {
cfg := &config.Config{}
cfg.Server.RootDomain, cfg.Auth.PanelHostname, cfg.Auth.AdminHostname = tc.root, tc.panel, tc.admin
rp, origins := passkeyRelyingParty(cfg)
if rp != tc.wantRP || !slices.Equal(origins, tc.wantOrigins) {
t.Fatalf("relying party = %q %q, want %q %q", rp, origins, tc.wantRP, tc.wantOrigins)
}
})
}
}
// TestAuthSourcesFromConfig pins the one place the hasJoined identity anchor is decided:
// Mojang is prepended in code, first, and is the only source whose UUIDs are trusted as-is.
// The empty case matters on its own — both `felis api` and `felis nano` call this with a
@@ -136,50 +97,6 @@ func TestNewAPIServerSetsHardenedTimeouts(t *testing.T) {
}
}
// Every listener drains at once, and the shutdown hook (API.CloseStreams in
// cmdAPI) runs on each: two listeners each holding a request that ends when
// the hook fires take one hook's worth of time, well inside the deadline.
func TestShutdownServersDrainsListenersTogether(t *testing.T) {
release := make(chan struct{})
var hooks int32
started := make(chan struct{}, 2)
var servers []*http.Server
for range 2 {
srv := newAPIServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
started <- struct{}{}
<-release
}))
srv.RegisterOnShutdown(func() {
if atomic.AddInt32(&hooks, 1) == 1 {
time.AfterFunc(100*time.Millisecond, func() { close(release) })
}
})
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
srv.Addr = l.Addr().String()
go func() { _ = srv.Serve(l) }()
go func() {
if resp, err := http.Get("http://" + srv.Addr); err == nil {
resp.Body.Close()
}
}()
servers = append(servers, srv)
}
<-started
<-started
begun := time.Now()
shutdownServers(servers, 5*time.Second, io.Discard)
if took := time.Since(begun); took > 2*time.Second {
t.Fatalf("shutdown took %v", took)
}
if got := atomic.LoadInt32(&hooks); got != 2 {
t.Fatalf("shutdown hook ran %d times, want once per listener", got)
}
}
type fakeRefStore struct {
images []build.Image
builds []build.Build
@@ -238,76 +155,3 @@ func TestInUseImageRefsCoversEverySource(t *testing.T) {
t.Fatal("a failing whitelist read produced a reference list")
}
}
// TestExpireFileSessions runs the loop against a stage whose clock the test
// holds: the session idle past fileedit.SessionIdle goes, the one touched since
// stays, and the drop is said once.
func TestExpireFileSessions(t *testing.T) {
var mu sync.Mutex
now := time.Date(2026, 9, 28, 10, 0, 0, 0, time.UTC)
advance := func(d time.Duration) { mu.Lock(); now = now.Add(d); mu.Unlock() }
st := &fileedit.Stage{Dir: t.TempDir(), MinFree: 1e-9, Now: func() time.Time {
mu.Lock()
defer mu.Unlock()
return now
}}
idle, err := st.Begin("u1", "survival", "a.jar", 3)
if err != nil {
t.Fatal(err)
}
advance(fileedit.SessionIdle - time.Minute)
fresh, err := st.Begin("u1", "survival", "b.jar", 3)
if err != nil {
t.Fatal(err)
}
advance(2 * time.Minute)
ctx, cancel := context.WithCancel(context.Background())
var out bytes.Buffer
done := make(chan struct{})
go func() { expireFileSessions(ctx, st, time.Millisecond, &out); close(done) }()
for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(time.Millisecond) {
if _, err := st.Status("u1", "survival", idle.ID); errors.Is(err, fileedit.ErrNotStaged) {
break
}
if time.Now().After(deadline) {
cancel()
t.Fatal("the idle session was never dropped")
}
}
// A few more ticks with nothing idle, which must stay quiet.
time.Sleep(20 * time.Millisecond)
cancel()
<-done
if _, err := st.Status("u1", "survival", fresh.ID); err != nil {
t.Fatalf("the session touched since went too: %v", err)
}
if got := out.String(); got != "felis api: dropped 1 upload session(s) left idle for 6h0m0s\n" {
t.Fatalf("said %q", got)
}
}
type sweepCount struct{ n atomic.Int32 }
func (s *sweepCount) ExpireExports() { s.n.Add(1) }
// TestExpireExports: the loop sweeps on each tick, and returns once felis-api
// shuts down.
func TestExpireExports(t *testing.T) {
var s sweepCount
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { expireExports(ctx, &s, time.Millisecond); close(done) }()
for deadline := time.Now().Add(5 * time.Second); s.n.Load() < 3; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
cancel()
t.Fatalf("swept %d times in 5s at a 1ms tick", s.n.Load())
}
}
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the loop outlived its context")
}
}
cmd/felis/api_tokens_test.go
-38
View File
@@ -1,38 +0,0 @@
package main
import (
"net/http/httptest"
"strings"
"testing"
"felis.lolicon.best/internal/api"
)
// felis-api maps each env var onto the caller the Deployment feeds it for, so the
// build Job's token (FELIS_BUILD_TOKEN) authenticates as build and only as build.
func TestInternalCallerTokensReadEachCallersEnv(t *testing.T) {
env := map[string]string{
"FELIS_SERVICE_TOKEN": "v-tok",
"FELIS_LIMBO_TOKEN": "l-tok",
"FELIS_BUILD_TOKEN": " b-tok\n",
"FELIS_OPS_TOKEN": "o-tok",
}
auth, err := internalCallerTokens(func(k string) string { return env[k] })
if err != nil {
t.Fatal(err)
}
for tok, want := range map[string]api.Caller{"v-tok": api.CallerVelocity, "l-tok": api.CallerLimbo, "b-tok": api.CallerBuild, "o-tok": api.CallerOps} {
r := httptest.NewRequest("GET", "/", nil)
r.Header.Set("Authorization", "Bearer "+tok)
if got, err := auth.Authenticate(r); err != nil || got != want {
t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want)
}
}
// An install where the build namespace still holds a copy of the proxy's token
// would let that copy act as the proxy; the api refuses to start on it.
env["FELIS_BUILD_TOKEN"] = "v-tok"
if _, err := internalCallerTokens(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "same value") {
t.Fatalf("shared token: err = %v, want a same-value refusal", err)
}
}
cmd/felis/apply.go
+20 -16
View File
@@ -8,9 +8,7 @@ import (
"fmt"
"io"
"os"
"os/signal"
"strings"
"syscall"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/naming"
@@ -18,6 +16,10 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -109,15 +111,20 @@ func cmdApply(args []string, stdout, stderr io.Writer) int {
}
// ------- K8s client (one context, one client) -------
// The operator runs this on the node, where the kubeconfig is k3s's own file and
// neither $KUBECONFIG nor ~/.kube is set. buildSystemServerClient falls back to that
// file and names what it tried; ctrl.GetConfigOrDie exited 1 there without a word,
// because controller-runtime's logger is never set up in a CLI command.
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
cl, err := buildSystemServerClient()
// SetupSignalHandler must be called exactly once per process —
// controller-runtime panics on a second call. We create ctx and the
// K8s client here and thread both through every downstream call so no
// callee ever needs to call SetupSignalHandler again.
ctx := ctrl.SetupSignalHandler()
scheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(v1alpha1.AddToScheme(scheme))
cfg := ctrl.GetConfigOrDie()
cl, err := client.New(cfg, client.Options{Scheme: scheme})
if err != nil {
fmt.Fprintf(stderr, "felis apply: %v\n", err)
fmt.Fprintf(stderr, "felis apply: build client: %v\n", err)
return 1
}
@@ -160,10 +167,6 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
if err := naming.ValidateServerName(req.Subdomain); err != nil {
return nil, fmt.Errorf("invalid subdomain: %w", err)
}
displayName, err := naming.CleanDisplayName(req.DisplayName)
if err != nil {
return nil, fmt.Errorf("invalid displayName: %w", err)
}
if strings.TrimSpace(req.Image) == "" {
return nil, fmt.Errorf("image is required")
}
@@ -233,7 +236,7 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
},
Spec: v1alpha1.MinecraftServerSpec{
Subdomain: req.Subdomain,
DisplayName: displayName,
DisplayName: req.DisplayName,
Image: req.Image,
JavaMemory: deriveApplyJavaHeap(memLim),
DesiredState: v1alpha1.DesiredStopped,
@@ -261,7 +264,8 @@ func buildMinecraftServerFromApplyRequest(req applyRequest, namespace string) (*
// request if any CRD already carries the given spec.subdomain. metadata.name
// uniqueness is enforced by K8s on Create, but spec.subdomain must be checked
// here because two CRDs with different names could otherwise share a subdomain.
// It reuses the caller's context and K8s client.
// It reuses the caller's context and K8s client — it never calls
// SetupSignalHandler or builds its own client.
func checkSubdomainUnique(ctx context.Context, cl client.Client, namespace, subdomain string) error {
var list v1alpha1.MinecraftServerList
if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
cmd/felis/apply_test.go
+5 -41
View File
@@ -1,10 +1,7 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
@@ -140,12 +137,11 @@ func TestDeriveApplyJavaHeap(t *testing.T) {
func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
req := applyRequest{
Name: "test-server",
Subdomain: "test-server",
DisplayName: " Test Server ",
Image: "registry.felis.svc/paper:1.21",
Memory: "4Gi",
Storage: "20Gi",
Name: "test-server",
Subdomain: "test-server",
Image: "registry.felis.svc/paper:1.21",
Memory: "4Gi",
Storage: "20Gi",
}
ms, err := buildMinecraftServerFromApplyRequest(req, "minecraft")
if err != nil {
@@ -160,9 +156,6 @@ func TestBuildMinecraftServerFromApplyRequest_Valid(t *testing.T) {
if ms.Spec.Subdomain != "test-server" {
t.Errorf("Subdomain = %q", ms.Spec.Subdomain)
}
if ms.Spec.DisplayName != "Test Server" {
t.Errorf("DisplayName = %q, want it trimmed to Test Server", ms.Spec.DisplayName)
}
if ms.Spec.Image != "registry.felis.svc/paper:1.21" {
t.Errorf("Image = %q", ms.Spec.Image)
}
@@ -287,11 +280,6 @@ func TestBuildMinecraftServerFromApplyRequest_Errors(t *testing.T) {
applyRequest{Name: ok, Subdomain: "", Image: "x", Memory: "1Gi", Storage: "1Gi"},
"invalid subdomain",
},
{
"display name with a tab",
applyRequest{Name: ok, Subdomain: ok, DisplayName: "a" + string(rune(0x09)) + "b", Image: "x", Memory: "1Gi", Storage: "1Gi"},
"invalid displayName",
},
{
"empty image",
applyRequest{Name: ok, Subdomain: ok, Image: "", Memory: "1Gi", Storage: "1Gi"},
@@ -382,27 +370,3 @@ func resList(specs ...string) corev1.ResourceList {
}
return rl
}
// TestApplyReportsAMissingKubeconfig pins the node-side failure: with no kubeconfig to
// find, apply says which ones it tried and exits 1. It used to call
// ctrl.GetConfigOrDie, which ended the process with exit 1 and nothing printed.
func TestApplyReportsAMissingKubeconfig(t *testing.T) {
if _, err := os.Stat(hostBootstrapKubeconfigPath); err == nil {
t.Skipf("%s exists on this machine", hostBootstrapKubeconfigPath)
}
dir := t.TempDir()
t.Setenv("KUBECONFIG", filepath.Join(dir, "missing"))
t.Setenv("KUBERNETES_SERVICE_HOST", "")
form := filepath.Join(dir, "server.json")
if err := os.WriteFile(form, []byte(`{"name":"alpha","subdomain":"alpha","image":"registry.felis.svc:5000/felis/paper:demo","memory":"1Gi","storage":"1Gi"}`), 0o600); err != nil {
t.Fatal(err)
}
var out, errw bytes.Buffer
if code := cmdApply([]string{"-f", form}, &out, &errw); code != 1 {
t.Fatalf("exit = %d, want 1; stderr %q", code, errw.String())
}
want := "felis apply: no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and " + hostBootstrapKubeconfigPath + "): stat " + hostBootstrapKubeconfigPath + ": no such file or directory\n"
if errw.String() != want || out.Len() != 0 {
t.Fatalf("stdout %q, stderr %q, want stderr %q", out.String(), errw.String(), want)
}
}
cmd/felis/backup.go
+15 -59
View File
@@ -7,14 +7,13 @@ import (
"flag"
"fmt"
"io"
"strings"
"time"
"felis.lolicon.best/internal/backup"
"felis.lolicon.best/internal/backupjob"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/reaper"
"felis.lolicon.best/internal/store"
ctrl "sigs.k8s.io/controller-runtime"
)
@@ -37,8 +36,6 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
server := fs.String("server", "", "server name whose world is being backed up")
formerOwner := fs.String("former-owner", "", "owner recorded on the backup row (empty for an unowned server)")
worldsRoot := fs.String("worlds-root", "/world", "mount path of the world PVC being archived")
reason := fs.String("reason", reasonManual, "world_backups reason: manual, pre_restore for the safety snapshot in front of a restore, or scheduled for felis-api's daily restore point")
protect := fs.String("protect", "", "backup id the prune must keep (the one a chained restore extracts)")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -46,10 +43,6 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
fmt.Fprintln(stderr, "felis backup: --server is required")
return 2
}
if _, _, ok := backupPolicy(*reason, reaper.DefaultConfig()); !ok {
fmt.Fprintf(stderr, "felis backup: unknown --reason %q (manual, %s or %s)\n", *reason, backupjob.ReasonPreRestore, backupjob.ReasonScheduled)
return 2
}
cfg, err := config.Load(*cfgPath)
if err != nil {
@@ -60,14 +53,13 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "felis backup: archive store %q is not implemented in this build (only tarLocal)\n", cfg.Archive.Store)
return 1
}
// The [archive] parse the reaper uses: it holds each reason's keep and
// retention.
// The [archive] parse the reaper uses; an on-demand backup takes its
// manual_retention and manual_keep.
rcfg, err := reaperConfig(cfg)
if err != nil {
fmt.Fprintf(stderr, "felis backup: %v\n", err)
return 1
}
keep, retention, _ := backupPolicy(*reason, rcfg)
// The world PVC is mounted directly at worldsRoot; the resolver returns it for
// any target, exactly as in cmdRestore. This is the same TarLocal the reaper
@@ -88,18 +80,13 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
return 1
}
a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
if err != nil {
fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
return 1
}
ref, size := a.Ref, a.Size
if len(a.Skipped) > 0 {
fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",
len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))
}
drv, err := openPodStore(ctx, cfg.Database.URL, "backup", stderr)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
fmt.Fprintf(stderr, "felis backup: open database: %v\n", err)
return 1
@@ -112,11 +99,8 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
FormerOwner: *formerOwner,
BackupRef: string(ref),
SizeBytes: size,
Reason: *reason,
ExpiresAt: time.Now().Add(retention),
SHA256: a.SHA256,
SkippedEntries: len(a.Skipped),
Reason: "manual",
ExpiresAt: time.Now().Add(rcfg.ManualRetention),
}
st := reaper.NewPGStore(drv.DB())
if err := st.InsertBackup(ctx, rec); err != nil {
@@ -132,44 +116,16 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
}
fmt.Fprintf(stdout, "felis backup: server=%s archived %d bytes to %s (backup %s)\n", *server, size, ref, rec.ID)
pruneBackups(ctx, st, archiver, *server, *formerOwner, *reason, keep, *protect, stdout, stderr)
pruneManualBackups(ctx, st, archiver, *server, rcfg.ManualKeep, stdout, stderr)
return 0
}
const (
reasonManual = "manual"
// preRestoreKeep is how many safety snapshots a server keeps: enough to walk
// back a couple of restores in a row, without every restore adding a world's
// worth of bytes for the full manual retention.
preRestoreKeep = 3
)
// backupPolicy is how many backups of one reason a server keeps and how long
// each lives: an owner's own backups and the safety snapshots in front of a
// restore by [archive] manual_keep / manual_retention (the snapshots capped at
// preRestoreKeep), felis-api's daily restore points by scheduled_keep /
// scheduled_retention, so neither kind crowds out the other. ok is false for a
// reason this command does not record.
func backupPolicy(reason string, rcfg reaper.Config) (keep int, retention time.Duration, ok bool) {
switch reason {
case reasonManual:
return rcfg.ManualKeep, rcfg.ManualRetention, true
case backupjob.ReasonPreRestore:
return preRestoreKeep, rcfg.ManualRetention, true
case backupjob.ReasonScheduled:
return rcfg.ScheduledKeep, rcfg.ScheduledRetention, true
}
return 0, 0, false
}
// pruneBackups keeps the newest keep backups of this reason that owner holds of
// server and removes the rest, oldest first, so repeated backups of one world
// cannot fill the shared archive store and a new owner's backups never remove a
// previous owner's. protect is never removed: it is the backup a chained restore
// is about to extract. The new backup is already recorded; a removal that fails
// is reported and retried after the next backup.
func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) {
excess, err := st.ExcessBackups(ctx, server, owner, reason, keep, protect)
// pruneManualBackups keeps server's newest keep on-demand backups and removes
// the rest, oldest first, so repeated backups of one world cannot fill the
// shared archive store. The new backup is already recorded; a removal that
// fails is reported and retried after the next backup.
func pruneManualBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server string, keep int, stdout, stderr io.Writer) {
excess, err := st.ExcessManualBackups(ctx, server, keep)
if err != nil {
fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)
return
@@ -183,7 +139,7 @@ func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver backup.World
fmt.Fprintf(stderr, "felis backup: record the removal of %s: %v\n", b.ID, err)
continue
}
fmt.Fprintf(stdout, "felis backup: removed older %s backup %s of %s (keeping the newest %d)\n", reason, b.ID, server, keep)
fmt.Fprintf(stdout, "felis backup: removed older backup %s of %s (keeping the newest %d)\n", b.ID, server, keep)
}
}
cmd/felis/backup_test.go
-48
View File
@@ -1,48 +0,0 @@
package main
import (
"bytes"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/reaper"
)
// The reason decides which backups the new one's prune may remove, so an
// unknown one is refused before anything is archived.
func TestBackupSubcommandRejectsUnknownReason(t *testing.T) {
var stderr bytes.Buffer
if code := cmdBackup([]string{"--server", "survival", "--reason", "inactive_15d"}, &bytes.Buffer{}, &stderr); code != 2 {
t.Fatalf("exit = %d, want 2 (%s)", code, stderr.String())
}
if !strings.Contains(stderr.String(), "unknown --reason") {
t.Fatalf("stderr = %q", stderr.String())
}
}
// Each reason is pruned and expired by its own [archive] keys: a daily
// restore point must never count against, or take the lifetime of, the
// backups an owner asked for.
func TestBackupPolicyPerReason(t *testing.T) {
rcfg := reaper.DefaultConfig()
rcfg.ManualKeep, rcfg.ManualRetention = 5, 30*reaper.Day
rcfg.ScheduledKeep, rcfg.ScheduledRetention = 7, 90*reaper.Day
for _, tc := range []struct {
reason string
keep int
retention time.Duration
}{
{"manual", 5, 30 * reaper.Day},
{"pre_restore", preRestoreKeep, 30 * reaper.Day},
{"scheduled", 7, 90 * reaper.Day},
} {
keep, retention, ok := backupPolicy(tc.reason, rcfg)
if !ok || keep != tc.keep || retention != tc.retention {
t.Errorf("backupPolicy(%q) = (%d, %v, %v); want (%d, %v, true)", tc.reason, keep, retention, ok, tc.keep, tc.retention)
}
}
if _, _, ok := backupPolicy("inactive_15d", rcfg); ok {
t.Error("backupPolicy accepted inactive_15d; the reaper records those itself")
}
}
cmd/felis/backupnow.go
+7 -459
View File
@@ -4,28 +4,15 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/signal"
"sort"
"strings"
"syscall"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/operator"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -33,11 +20,9 @@ import (
// backupnow is the break-glass "back up a world now" op (§B4 "Sync"). Unlike halt —
// which writes the CRD directly — a backup needs felis-api's deployment coordinates
// (FELIS_IMAGE / FELIS_BACKUP_PVC) to render the one-shot backup Job, so the console
// cannot do it in-process. It POSTs the felis-api INTERNAL face (ops-token auth)
// cannot do it in-process. It POSTs the felis-api INTERNAL face (service-token auth)
// while the API is alive, and the API renders the Job and audits the action. This file
// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue. The
// `felis backup-now` command (cmdBackupNow, below) takes the same route for every
// user server in turn.
// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue.
// backupNowOutcome is the durable result of a backup request, re-printed after the TUI
// alt-screen tears down.
@@ -48,7 +33,7 @@ type backupNowOutcome struct {
// resolveInternalAPI reads the two things the on-node console needs to reach the
// felis-api internal face: the felis-api-internal Service ClusterIP (the host's
// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the ops
// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the service
// token. Both live in the control namespace.
func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace string) (baseURL, token string, err error) {
var svc corev1.Service
@@ -60,14 +45,13 @@ func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace
return "", "", fmt.Errorf("%s Service has no ClusterIP yet", platform.APIInternalServiceName)
}
// The console's own token, which the api serves on the backup route alone.
var sec corev1.Secret
if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.OpsTokenSecretName}, &sec); err != nil {
return "", "", fmt.Errorf("get %s Secret (re-run the installer to create it): %w", naming.OpsTokenSecretName, err)
if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.ServiceTokenSecretName}, &sec); err != nil {
return "", "", fmt.Errorf("get %s Secret: %w", naming.ServiceTokenSecretName, err)
}
token = string(sec.Data[naming.ServiceTokenSecretKey])
if token == "" {
return "", "", fmt.Errorf("secret %s has no %s key", naming.OpsTokenSecretName, naming.ServiceTokenSecretKey)
return "", "", fmt.Errorf("secret %s has no %s key", naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey)
}
return fmt.Sprintf("http://%s:%d", ip, platform.APIInternalPort), token, nil
@@ -90,7 +74,7 @@ func requestBackup(ctx context.Context, hc *http.Client, baseURL, token, name, o
resp, err := hc.Do(req)
if err != nil {
return backupNowOutcome{}, fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
return backupNowOutcome{}, fmt.Errorf("felis-api unreachable (a backup needs it alive): %w", err)
}
defer resp.Body.Close()
@@ -160,439 +144,3 @@ func backupPickable(servers []haltableServer) []haltableServer {
}
return out
}
// cmdBackupNow is `felis backup-now`: the world of every user server (or of the
// named ones) archived now, one at a time, through the internal backup route the
// console's Sync uses. A world lives only in its volume and the off-site copy holds
// only its archives, so this is the lever in front of a planned move to another
// host, a disk swap or anything else that could lose a volume.
func cmdBackupNow(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("backup-now", flag.ContinueOnError)
fs.SetOutput(stderr)
cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
yes := fs.Bool("yes", false, "back up; without it the plan is printed and nothing changes")
stop := fs.Bool("stop", false, "stop the running servers first: their players are disconnected and the servers stay stopped")
fs.Usage = func() {
fmt.Fprintln(stderr, "Usage: felis backup-now [-yes] [-stop] [server ...]")
fmt.Fprintln(stderr)
fmt.Fprintln(stderr, "Archives the world of every user server, or of the named ones, one at a time, and waits for each archive.")
fmt.Fprintln(stderr, "A running server is skipped unless -stop is given. Without -yes it prints what it would do.")
fs.PrintDefaults()
}
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis backup-now: refused — it reads the cluster's ops token, so it must run as root (try: sudo felis backup-now)")
return 1
}
cfg, err := config.Load(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
return 1
}
cl, err := buildSystemServerClient()
if err != nil {
fmt.Fprintf(stderr, "felis backup-now: %v\n", err)
return 1
}
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
ns := cfg.K8s.Namespace
osUser := accountableOSUser()
jobs := api.NewK8sJobStatus(cl, ns)
var baseURL, token string
var repo ownerStore
var drv *store.PostgresDriver
defer func() {
if drv != nil {
_ = drv.Close()
}
}()
hc := &http.Client{Timeout: 10 * time.Second}
r := backupNowRun{
out: stdout,
errw: stderr,
ns: ns,
list: func(ctx context.Context) ([]backupNowWorld, error) { return listBackupNowWorlds(ctx, cl, ns) },
stopped: func(ctx context.Context, name string) (bool, error) {
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &ms); err != nil {
return false, err
}
return backupNowStopped(ctx, cl, &ms)
},
halt: func(ctx context.Context, name string) error {
// The database is opened only once a server is to be stopped: the halt
// is audited like the console's, and a run with nothing running needs
// no more than the API.
if repo == nil {
d, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
return fmt.Errorf("open the database for the audit log: %w", err)
}
drv, repo = d, api.NewPGRepo(d.DB())
}
out, err := performHalt(ctx, cl, repo, ns, name, osUser)
if err != nil {
return err
}
if out.auditErr != nil {
fmt.Fprintf(stderr, "felis backup-now: the audit row for stopping %s was not written: %v\n", name, out.auditErr)
}
return nil
},
request: func(ctx context.Context, name string) error {
if baseURL == "" {
var err error
if baseURL, token, err = resolveInternalAPI(ctx, cl, platform.DefaultControlNamespace); err != nil {
return fmt.Errorf("%w: %w", errBackupAPIUnreachable, err)
}
}
_, err := requestBackup(ctx, hc, baseURL, token, name, osUser)
return err
},
jobs: jobs.LatestJobs,
now: time.Now,
sleep: func(ctx context.Context, d time.Duration) { sleepCtx(ctx, d) },
}
return r.run(ctx, fs.Args(), *yes, *stop)
}
// sleepCtx waits d or until ctx ends.
func sleepCtx(ctx context.Context, d time.Duration) {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
case <-t.C:
}
}
// backupNowWorld is one user server as backup-now sees it.
type backupNowWorld struct {
name string
phase string // the observed phase, or the desired state before the operator reconciled it
stopped bool // the backup route's stopped gate admits it
hasWorld bool // its world volume exists
}
// listBackupNowWorlds lists the user servers of namespace in the API server's
// order (by name), each with what the backup route checks. System servers are
// left out: they have no row in the servers table, so the route refuses them
// (backupPickable).
func listBackupNowWorlds(ctx context.Context, cl client.Client, namespace string) ([]backupNowWorld, error) {
var list v1alpha1.MinecraftServerList
if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
return nil, err
}
var out []backupNowWorld
for i := range list.Items {
ms := &list.Items[i]
if isSystemServer(ms.Name) {
continue
}
stopped, err := backupNowStopped(ctx, cl, ms)
if err != nil {
return nil, err
}
var pvc corev1.PersistentVolumeClaim
err = cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: naming.WorldPVCName(ms.Name)}, &pvc)
if err != nil && !apierrors.IsNotFound(err) {
return nil, fmt.Errorf("look up the world volume of %s: %w", ms.Name, err)
}
phase := string(ms.Status.Phase)
if phase == "" {
phase = string(ms.Spec.DesiredState)
}
out = append(out, backupNowWorld{name: ms.Name, phase: phase, stopped: stopped, hasWorld: err == nil})
}
return out, nil
}
// backupNowStopped is the backup route's stopped gate (api.enqueueBackup and
// K8sCluster.AcquireMaintenance together): desired Stopped, not ready, phase
// Stopped and no game pod left.
func backupNowStopped(ctx context.Context, cl client.Client, ms *v1alpha1.MinecraftServer) (bool, error) {
if ms.Spec.DesiredState != v1alpha1.DesiredStopped || ms.Status.Ready || ms.Status.Phase != v1alpha1.PhaseStopped {
return false, nil
}
var pods corev1.PodList
if err := cl.List(ctx, &pods, client.InNamespace(ms.Namespace), client.MatchingLabels{
v1alpha1.LabelServer: ms.Name, v1alpha1.LabelComponent: operator.ComponentValue,
}); err != nil {
return false, fmt.Errorf("look up the pod of %s: %w", ms.Name, err)
}
return len(pods.Items) == 0, nil
}
// errBackupAPIUnreachable is a backup request that never reached felis-api. It
// ends a backup-now run: every later world would fail the same way, and stopping
// servers for backups that cannot be taken only takes them away from players.
var errBackupAPIUnreachable = errors.New("felis-api unreachable (a backup needs it alive)")
// Polling of backup-now. A graceful stop saves the world first; the backup Job's
// own deadline (backupjob, 30 minutes) ends a Job that hangs, so its wait needs no
// cap of its own.
const (
backupNowPoll = 2 * time.Second
backupNowStopWait = 10 * time.Minute
backupNowJobAppear = time.Minute
)
// backupNowRun is backup-now over seams, so the plan and the run are tested
// without a cluster or felis-api.
type backupNowRun struct {
out, errw io.Writer
ns string // where the servers and their Jobs live, for the kubectl hints
list func(ctx context.Context) ([]backupNowWorld, error)
stopped func(ctx context.Context, name string) (bool, error)
halt func(ctx context.Context, name string) error
request func(ctx context.Context, name string) error
jobs func(ctx context.Context, name string) ([]api.AsyncJob, error)
now func() time.Time
sleep func(ctx context.Context, d time.Duration)
}
// pickBackupNowWorlds narrows worlds to names, in the order given, or keeps them
// all when names is empty.
func pickBackupNowWorlds(worlds []backupNowWorld, names []string) ([]backupNowWorld, error) {
if len(names) == 0 {
return worlds, nil
}
byName := make(map[string]backupNowWorld, len(worlds))
for _, w := range worlds {
byName[w.name] = w
}
var out []backupNowWorld
seen := map[string]bool{}
for _, n := range names {
if seen[n] {
continue
}
seen[n] = true
w, ok := byName[n]
switch {
case ok:
out = append(out, w)
case isSystemServer(n):
return nil, fmt.Errorf("%s is a system server: its world is rebuilt by felis setup and has no backups", n)
default:
return nil, fmt.Errorf("no server named %q", n)
}
}
return out, nil
}
// backupNowAction is what the plan does with a world.
func backupNowAction(w backupNowWorld, stop bool) string {
switch {
case w.stopped && !w.hasWorld:
return "skip: no world volume (never started, nothing to save)"
case w.stopped:
return "back up"
case stop:
return "stop, then back up"
default:
return "skip: running (stop it first, or pass -stop)"
}
}
func (r *backupNowRun) run(ctx context.Context, names []string, yes, stop bool) int {
all, err := r.list(ctx)
if err != nil {
fmt.Fprintf(r.errw, "felis backup-now: list the servers: %v\n", err)
return 1
}
worlds, err := pickBackupNowWorlds(all, names)
if err != nil {
fmt.Fprintf(r.errw, "felis backup-now: %v\n", err)
return 2
}
if len(worlds) == 0 {
fmt.Fprintln(r.out, "felis backup-now: there are no user servers")
return 0
}
// The stopped worlds go first, so a felis-api that cannot take a backup is
// found before any server is stopped for one.
sort.SliceStable(worlds, func(i, j int) bool { return worlds[i].stopped && !worlds[j].stopped })
width := 0
for _, w := range worlds {
width = max(width, len(w.name))
}
fmt.Fprintf(r.out, "felis backup-now: %d server(s), backed up one at a time:\n", len(worlds))
stopping, work := false, 0
for _, w := range worlds {
fmt.Fprintf(r.out, " %-*s %-8s %s\n", width, w.name, w.phase, backupNowAction(w, stop))
stopping = stopping || (!w.stopped && stop)
if (w.stopped && w.hasWorld) || (!w.stopped && stop) {
work++
}
}
if work > 0 {
fmt.Fprintln(r.out, "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.")
}
if stopping {
fmt.Fprintln(r.out, "Stopping disconnects the players on those servers, and they stay stopped afterwards.")
}
if !yes {
if work == 0 {
fmt.Fprintln(r.out, "Nothing to back up.")
} else {
fmt.Fprintln(r.out, "Nothing changed. Run again with -yes to back them up.")
}
return 0
}
var done, failed, running, empty int
var leftStopped []string
for _, w := range worlds {
if err := ctx.Err(); err != nil {
break
}
switch {
case w.stopped && !w.hasWorld:
empty++
continue
case !w.stopped && !stop:
running++
continue
}
if !w.stopped {
halted, err := r.stopWorld(ctx, w.name)
if halted {
leftStopped = append(leftStopped, w.name)
}
if err != nil {
fmt.Fprintf(r.out, " %s: failed: %v\n", w.name, err)
failed++
continue
}
}
err := r.backUp(ctx, w.name)
if errors.Is(err, errBackupAPIUnreachable) {
fmt.Fprintf(r.out, " %s: failed: %v\n", w.name, err)
fmt.Fprintln(r.out, "Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).")
failed++
break
}
if err != nil {
fmt.Fprintf(r.out, " %s: failed: %v\n", w.name, err)
failed++
continue
}
done++
}
interrupted := ctx.Err() != nil
if interrupted {
fmt.Fprintln(r.out, "Interrupted: a backup Job already started runs to its end.")
}
parts := []string{fmt.Sprintf("%d backed up", done)}
if failed > 0 {
parts = append(parts, fmt.Sprintf("%d failed", failed))
}
if running > 0 {
parts = append(parts, fmt.Sprintf("%d skipped (running)", running))
}
if empty > 0 {
parts = append(parts, fmt.Sprintf("%d without a world", empty))
}
fmt.Fprintf(r.out, "%s.\n", strings.Join(parts, ", "))
if len(leftStopped) > 0 {
fmt.Fprintf(r.out, "Left stopped: %s. Start them from the panel when you are done.\n", strings.Join(leftStopped, ", "))
}
if done > 0 {
fmt.Fprintln(r.out, "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.")
}
if failed > 0 || running > 0 || interrupted {
return 1
}
return 0
}
// stopWorld stops one server and waits until the backup route would admit it.
// halted is whether the stop was asked for: the server then stays stopped, even
// when it takes longer than the wait.
func (r *backupNowRun) stopWorld(ctx context.Context, name string) (halted bool, err error) {
fmt.Fprintf(r.out, " %s: stopping\n", name)
if err := r.halt(ctx, name); err != nil {
return false, err
}
start := r.now()
for {
ok, err := r.stopped(ctx, name)
if err != nil {
return true, err
}
if ok {
fmt.Fprintf(r.out, " %s: stopped after %s\n", name, r.now().Sub(start).Round(time.Second))
return true, nil
}
if r.now().Sub(start) >= backupNowStopWait {
return true, fmt.Errorf("did not stop within %s (kubectl -n %s describe minecraftserver %s)", backupNowStopWait, r.ns, name)
}
if err := ctx.Err(); err != nil {
return true, err
}
r.sleep(ctx, backupNowPoll)
}
}
// backUp requests one world's backup and waits for its Job to finish. The Job is
// the one of this server that was not there before the request.
func (r *backupNowRun) backUp(ctx context.Context, name string) error {
before, err := r.jobs(ctx, name)
if err != nil {
return fmt.Errorf("list its backup Jobs: %w", err)
}
known := make(map[string]bool, len(before))
for _, j := range before {
known[j.Name] = true
}
if err := r.request(ctx, name); err != nil {
return err
}
fmt.Fprintf(r.out, " %s: backing up\n", name)
start := r.now()
seen := ""
for {
jobs, err := r.jobs(ctx, name)
if err != nil {
return fmt.Errorf("list its backup Jobs: %w", err)
}
var job *api.AsyncJob
for i := range jobs {
if jobs[i].Kind == "backup" && (jobs[i].Name == seen || seen == "" && !known[jobs[i].Name]) {
job = &jobs[i]
break
}
}
switch {
case job == nil && seen != "":
return fmt.Errorf("its backup Job %s was deleted before it finished", seen)
case job == nil && r.now().Sub(start) >= backupNowJobAppear:
return fmt.Errorf("felis-api accepted the backup, but no backup Job appeared within %s (kubectl -n %s get jobs)", backupNowJobAppear, r.ns)
case job != nil && job.State == "succeeded":
fmt.Fprintf(r.out, " %s: archived in %s\n", name, r.now().Sub(start).Round(time.Second))
return nil
case job != nil && job.State == "failed":
msg := job.Message
if msg == "" {
msg = "the backup Job failed"
}
return fmt.Errorf("%s (kubectl -n %s logs job/%s)", msg, r.ns, job.Name)
case job != nil:
seen = job.Name
}
if err := ctx.Err(); err != nil {
return err
}
r.sleep(ctx, backupNowPoll)
}
}
cmd/felis/backupnow_cmd_test.go
-563
View File
@@ -1,563 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/naming"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// bnRig drives backupNowRun against scripted servers and Jobs on a fake clock that
// moves only when the run sleeps.
type bnRig struct {
out, errw bytes.Buffer
clock time.Time
events []string
worlds []backupNowWorld
// stopAfter is how many polls a halted server takes to stop; -1 never does.
stopAfter map[string]int
polls map[string]int
reqErr map[string]error
// states is what the server's new backup Job reports on each poll after the
// request, the last one repeating; "" is no Job.
states map[string][]string
messages map[string]string
jobPolls map[string]int
// stopErr fails a server's stop polls; jobsFail fails the nth (1-based) Job
// list of a server.
stopErr map[string]error
jobsFail map[string]int
jobCalls map[string]int
// ctx is the run's context, and onAct runs after each halt and request.
ctx context.Context
onAct func(event string)
}
func newBNRig(worlds ...backupNowWorld) *bnRig {
return &bnRig{
clock: time.Unix(1_800_000_000, 0),
worlds: worlds,
stopAfter: map[string]int{},
polls: map[string]int{},
reqErr: map[string]error{},
states: map[string][]string{},
messages: map[string]string{},
jobPolls: map[string]int{},
stopErr: map[string]error{},
jobsFail: map[string]int{},
jobCalls: map[string]int{},
ctx: context.Background(),
onAct: func(string) {},
}
}
func (g *bnRig) run(names []string, yes, stop bool) int {
r := backupNowRun{
out: &g.out, errw: &g.errw, ns: "minecraft",
list: func(context.Context) ([]backupNowWorld, error) {
return append([]backupNowWorld(nil), g.worlds...), nil
},
stopped: func(_ context.Context, name string) (bool, error) {
g.polls[name]++
if err := g.stopErr[name]; err != nil {
return false, err
}
n := g.stopAfter[name]
return n >= 0 && g.polls[name] > n, nil
},
halt: func(_ context.Context, name string) error {
g.events = append(g.events, "halt "+name)
g.onAct("halt " + name)
return nil
},
request: func(_ context.Context, name string) error {
g.events = append(g.events, "request "+name)
g.onAct("request " + name)
if err := g.reqErr[name]; err != nil {
return err
}
g.jobPolls[name] = 0
return nil
},
jobs: func(_ context.Context, name string) ([]api.AsyncJob, error) {
// Every server has an older finished backup, and a restore Job that
// shows up with the new backup: neither is the Job to wait for.
g.jobCalls[name]++
if g.jobCalls[name] == g.jobsFail[name] {
return nil, errors.New("the apiserver is gone")
}
out := []api.AsyncJob{{Name: "backup-" + name + "-old", Kind: "backup", State: "succeeded"}}
n, requested := g.jobPolls[name]
if !requested {
return out, nil
}
g.jobPolls[name] = n + 1
states := g.states[name]
if len(states) == 0 {
states = []string{"succeeded"}
}
state := states[min(n, len(states)-1)]
if state == "" {
return out, nil
}
return append([]api.AsyncJob{
{Name: "restore-" + name + "-x", Kind: "restore", State: "succeeded"},
{Name: "backup-" + name + "-new", Kind: "backup", State: state, Message: g.messages[name]},
}, out...), nil
},
now: func() time.Time { return g.clock },
sleep: func(_ context.Context, d time.Duration) { g.clock = g.clock.Add(d) },
}
return r.run(g.ctx, names, yes, stop)
}
func stoppedWorld(name string) backupNowWorld {
return backupNowWorld{name: name, phase: "Stopped", stopped: true, hasWorld: true}
}
func runningWorld(name string) backupNowWorld {
return backupNowWorld{name: name, phase: "Running", hasWorld: true}
}
func emptyWorld(name string) backupNowWorld {
return backupNowWorld{name: name, phase: "Stopped", stopped: true}
}
const (
bnManualKeep = "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.\n"
bnStopping = "Stopping disconnects the players on those servers, and they stay stopped afterwards.\n"
bnOffsite = "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.\n"
)
func TestBackupNowPlanChangesNothing(t *testing.T) {
for _, tc := range []struct {
stop bool
want string
}{
{false, "felis backup-now: 4 server(s), backed up one at a time:\n" +
" alpha Stopped back up\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
" bravo Running skip: running (stop it first, or pass -stop)\n" +
" delta Starting skip: running (stop it first, or pass -stop)\n" +
bnManualKeep +
"Nothing changed. Run again with -yes to back them up.\n"},
{true, "felis backup-now: 4 server(s), backed up one at a time:\n" +
" alpha Stopped back up\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
" bravo Running stop, then back up\n" +
" delta Starting stop, then back up\n" +
bnManualKeep + bnStopping +
"Nothing changed. Run again with -yes to back them up.\n"},
} {
t.Run(fmt.Sprintf("stop=%v", tc.stop), func(t *testing.T) {
delta := runningWorld("delta")
delta.phase = "Starting"
g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), delta)
if code := g.run(nil, false, tc.stop); code != 0 {
t.Fatalf("exit = %d, want 0; stderr %q", code, g.errw.String())
}
if g.out.String() != tc.want {
t.Fatalf("plan =\n%s\nwant\n%s", g.out.String(), tc.want)
}
if len(g.events) != 0 || len(g.polls) != 0 {
t.Fatalf("the plan acted: events %v, polls %v", g.events, g.polls)
}
})
}
}
// A plan that saves nothing says so, without the manual_keep warning; a running
// server counts as something to save once -stop is given.
func TestBackupNowPlanWithNothingToSave(t *testing.T) {
for _, tc := range []struct {
stop bool
want string
}{
{false, "felis backup-now: 2 server(s), backed up one at a time:\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
" bravo Running skip: running (stop it first, or pass -stop)\n" +
"Nothing to back up.\n"},
{true, "felis backup-now: 2 server(s), backed up one at a time:\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
" bravo Running stop, then back up\n" +
bnManualKeep + bnStopping +
"Nothing changed. Run again with -yes to back them up.\n"},
} {
g := newBNRig(runningWorld("bravo"), emptyWorld("charlie"))
if code := g.run(nil, false, tc.stop); code != 0 {
t.Fatalf("stop=%v: exit = %d, want 0", tc.stop, code)
}
if g.out.String() != tc.want {
t.Fatalf("stop=%v: plan =\n%s\nwant\n%s", tc.stop, g.out.String(), tc.want)
}
}
}
func TestBackupNowBacksUpEachWorldInTurn(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), stoppedWorld("delta"))
g.states["alpha"] = []string{"", "running", "succeeded"}
g.states["delta"] = []string{"running", "failed"}
g.messages["delta"] = "felis backup: not enough free disk for the archive"
g.stopAfter["bravo"] = 2
g.states["bravo"] = []string{"running", "running", "running", "succeeded"}
code := g.run(nil, true, true)
if code != 1 {
t.Fatalf("exit = %d, want 1 (delta failed)", code)
}
if want := []string{"request alpha", "request delta", "halt bravo", "request bravo"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
_, run, _ := strings.Cut(g.out.String(), bnStopping+"")
want := " alpha: backing up\n" +
" alpha: archived in 4s\n" +
" delta: backing up\n" +
" delta: failed: felis backup: not enough free disk for the archive (kubectl -n minecraft logs job/backup-delta-new)\n" +
" bravo: stopping\n" +
" bravo: stopped after 4s\n" +
" bravo: backing up\n" +
" bravo: archived in 6s\n" +
"2 backed up, 1 failed, 1 without a world.\n" +
"Left stopped: bravo. Start them from the panel when you are done.\n" +
bnOffsite
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
}
func TestBackupNowSkipsRunningServersWithoutStop(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"))
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1 (bravo was not backed up)", code)
}
if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
if len(g.polls) != 0 {
t.Fatalf("polled a server it did not stop: %v", g.polls)
}
_, run, _ := strings.Cut(g.out.String(), "Nothing changed")
if run != "" {
t.Fatalf("-yes printed the plan's closing line")
}
_, run, _ = strings.Cut(g.out.String(), bnManualKeep)
want := " alpha: backing up\n alpha: archived in 0s\n1 backed up, 1 skipped (running).\n" + bnOffsite
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
}
func TestBackupNowExitsCleanWhenEverythingIsSaved(t *testing.T) {
// -stop with nothing running stops nothing and says nothing about stopping.
g := newBNRig(stoppedWorld("alpha"), emptyWorld("charlie"))
if code := g.run(nil, true, true); code != 0 {
t.Fatalf("exit = %d, want 0; output\n%s", code, g.out.String())
}
_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
if want := " alpha: backing up\n alpha: archived in 0s\n1 backed up, 1 without a world.\n" + bnOffsite; run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
g = newBNRig(emptyWorld("charlie"))
if code := g.run(nil, true, false); code != 0 {
t.Fatalf("exit = %d, want 0 for a server with nothing to save", code)
}
// Nothing to archive: no manual_keep warning, and no off-site hint.
if want := "felis backup-now: 1 server(s), backed up one at a time:\n" +
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
"0 backed up, 1 without a world.\n"; g.out.String() != want {
t.Fatalf("output =\n%s\nwant\n%s", g.out.String(), want)
}
if len(g.events) != 0 {
t.Fatalf("events = %v, want none", g.events)
}
}
func TestBackupNowStopsAtAnUnreachableAPI(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), runningWorld("charlie"))
g.reqErr["alpha"] = fmt.Errorf("%w: dial tcp 10.43.0.9:8081: connect: connection refused", errBackupAPIUnreachable)
if code := g.run(nil, true, true); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v: nothing after the API proved unreachable, and no server stopped", g.events, want)
}
_, run, _ := strings.Cut(g.out.String(), bnStopping)
want := " alpha: failed: felis-api unreachable (a backup needs it alive): dial tcp 10.43.0.9:8081: connect: connection refused\n" +
"Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).\n" +
"0 backed up, 1 failed.\n"
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
// Any other refusal is that world's alone: the run goes on.
g = newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
g.reqErr["alpha"] = errors.New("felis-api: the world is being restored")
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if want := []string{"request alpha", "request bravo"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
}
func TestBackupNowGivesUpOnAServerThatDoesNotStop(t *testing.T) {
g := newBNRig(runningWorld("bravo"), runningWorld("echo"))
g.stopAfter["bravo"] = -1
if code := g.run(nil, true, true); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if want := []string{"halt bravo", "halt echo", "request echo"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
// One poll at the start and one per 2s sleep up to the 10-minute mark.
if g.polls["bravo"] != 301 {
t.Fatalf("bravo polled %d times, want 301", g.polls["bravo"])
}
_, run, _ := strings.Cut(g.out.String(), bnStopping)
want := " bravo: stopping\n" +
" bravo: failed: did not stop within 10m0s (kubectl -n minecraft describe minecraftserver bravo)\n" +
" echo: stopping\n" +
" echo: stopped after 0s\n" +
" echo: backing up\n" +
" echo: archived in 0s\n" +
"1 backed up, 1 failed.\n" +
"Left stopped: bravo, echo. Start them from the panel when you are done.\n" +
bnOffsite
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
}
func TestBackupNowReportsAJobThatNeverRuns(t *testing.T) {
for _, tc := range []struct {
name string
states []string
polls int
want string
}{
{"never appears", []string{""}, 31,
"felis-api accepted the backup, but no backup Job appeared within 1m0s (kubectl -n minecraft get jobs)"},
{"deleted while running", []string{"running", "running", ""}, 3,
"its backup Job backup-alpha-new was deleted before it finished"},
{"fails without a message", []string{"failed"}, 1,
"the backup Job failed (kubectl -n minecraft logs job/backup-alpha-new)"},
} {
t.Run(tc.name, func(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"))
g.states["alpha"] = tc.states
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if !strings.Contains(g.out.String(), " alpha: failed: "+tc.want+"\n") {
t.Fatalf("output =\n%s\nwant the line %q", g.out.String(), tc.want)
}
if g.jobPolls["alpha"] != tc.polls {
t.Fatalf("polled the Jobs %d times after the request, want %d", g.jobPolls["alpha"], tc.polls)
}
})
}
}
func TestBackupNowNamedServers(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), stoppedWorld("delta"))
if code := g.run([]string{"delta", "alpha", "delta"}, true, false); code != 0 {
t.Fatalf("exit = %d, want 0", code)
}
if want := []string{"request delta", "request alpha"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
if !strings.HasPrefix(g.out.String(), "felis backup-now: 2 server(s), backed up one at a time:\n delta Stopped back up\n alpha Stopped back up\n") {
t.Fatalf("plan =\n%s", g.out.String())
}
for _, tc := range []struct{ name, want string }{
{"login", "felis backup-now: login is a system server: its world is rebuilt by felis setup and has no backups\n"},
{"lobby", "felis backup-now: lobby is a system server: its world is rebuilt by felis setup and has no backups\n"},
{"nope", "felis backup-now: no server named \"nope\"\n"},
} {
g := newBNRig(stoppedWorld("alpha"))
if code := g.run([]string{"alpha", tc.name}, true, false); code != 2 {
t.Fatalf("%s: exit = %d, want 2", tc.name, code)
}
if g.errw.String() != tc.want {
t.Fatalf("%s: stderr = %q, want %q", tc.name, g.errw.String(), tc.want)
}
if len(g.events) != 0 || g.out.Len() != 0 {
t.Fatalf("%s: acted on a bad name: events %v, output %q", tc.name, g.events, g.out.String())
}
}
g = newBNRig()
if code := g.run(nil, true, false); code != 0 || g.out.String() != "felis backup-now: there are no user servers\n" {
t.Fatalf("empty fleet: exit %d, output %q", code, g.out.String())
}
}
// The world list mirrors the backup route's own gate, so the plan says exactly what
// the route would refuse.
func TestListBackupNowWorlds(t *testing.T) {
withStatus := func(ms *v1alpha1.MinecraftServer, ready bool) *v1alpha1.MinecraftServer {
ms.Status.Ready = ready
return ms
}
pvc := func(server, ns string) *corev1.PersistentVolumeClaim {
return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: naming.WorldPVCName(server), Namespace: ns}}
}
pod := func(name, ns string, labels map[string]string) *corev1.Pod {
return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}}
}
gameLabels := func(server string) map[string]string {
return map[string]string{v1alpha1.LabelServer: server, v1alpha1.LabelComponent: "server"}
}
other := mcServer("zulu", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped)
other.Namespace = "elsewhere"
hotel := mcServer("hotel", v1alpha1.DesiredStopped, "")
objs := []client.Object{
mcServer("alpha", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("alpha", haltNS),
// A backup Job's pod carries the server label with its own component, and
// a game pod of the same name in another namespace is somebody else's.
pod("backup-alpha-1-x", haltNS, map[string]string{v1alpha1.LabelServer: "alpha", "app.kubernetes.io/component": "world-backup"}),
pod("alpha-0", "elsewhere", gameLabels("alpha")),
withStatus(mcServer("bravo", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning), true), pvc("bravo", haltNS), pod("bravo-0", haltNS, gameLabels("bravo")),
mcServer("charlie", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped),
mcServer("delta", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("delta", haltNS), pod("delta-0", haltNS, gameLabels("delta")),
mcServer("echo", v1alpha1.DesiredStopped, v1alpha1.PhaseStopping), pvc("echo", haltNS),
mcServer("foxtrot", v1alpha1.DesiredRunning, v1alpha1.PhaseStopped), pvc("foxtrot", haltNS),
withStatus(mcServer("golf", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), true), pvc("golf", haltNS),
hotel, pvc("hotel", haltNS),
mcServer("login", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("login", haltNS),
mcServer("lobby", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("lobby", haltNS),
other, pvc("zulu", "elsewhere"),
}
got, err := listBackupNowWorlds(context.Background(), haltClient(t, objs...), haltNS)
if err != nil {
t.Fatal(err)
}
want := []backupNowWorld{
{name: "alpha", phase: "Stopped", stopped: true, hasWorld: true},
{name: "bravo", phase: "Running", hasWorld: true},
{name: "charlie", phase: "Stopped", stopped: true},
{name: "delta", phase: "Stopped", hasWorld: true}, // its pod is still going
{name: "echo", phase: "Stopping", hasWorld: true}, // still stopping
{name: "foxtrot", phase: "Stopped", hasWorld: true}, // asked to start
{name: "golf", phase: "Stopped", hasWorld: true}, // still reports ready
{name: "hotel", phase: "Stopped", hasWorld: true}, // not reconciled yet: the desired state shows
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("worlds =\n%+v\nwant\n%+v", got, want)
}
}
func TestBackupNowStopsWhenInterrupted(t *testing.T) {
t.Run("between worlds", func(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
g.ctx = ctx
g.onAct = func(e string) {
if e == "request alpha" {
cancel()
}
}
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
t.Fatalf("events = %v, want %v", g.events, want)
}
_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
want := " alpha: backing up\n alpha: archived in 0s\n" +
"Interrupted: a backup Job already started runs to its end.\n" +
"1 backed up.\n" + bnOffsite
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
})
t.Run("while a Job runs", func(t *testing.T) {
g := newBNRig(stoppedWorld("alpha"))
g.states["alpha"] = []string{"running"}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
g.ctx = ctx
g.onAct = func(string) { cancel() }
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if g.jobPolls["alpha"] != 1 {
t.Fatalf("polled the Job %d times after the interrupt, want 1", g.jobPolls["alpha"])
}
if !strings.Contains(g.out.String(), " alpha: failed: context canceled\nInterrupted: a backup Job already started runs to its end.\n0 backed up, 1 failed.\n") {
t.Fatalf("output =\n%s", g.out.String())
}
})
t.Run("while a server stops", func(t *testing.T) {
g := newBNRig(runningWorld("bravo"))
g.stopAfter["bravo"] = -1
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
g.ctx = ctx
g.onAct = func(string) { cancel() }
if code := g.run(nil, true, true); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
if g.polls["bravo"] != 1 {
t.Fatalf("polled bravo %d times after the interrupt, want 1", g.polls["bravo"])
}
_, run, _ := strings.Cut(g.out.String(), bnStopping)
want := " bravo: stopping\n bravo: failed: context canceled\n" +
"Interrupted: a backup Job already started runs to its end.\n" +
"0 backed up, 1 failed.\n" +
"Left stopped: bravo. Start them from the panel when you are done.\n"
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
})
}
func TestBackupNowReportsClusterErrors(t *testing.T) {
g := newBNRig(runningWorld("bravo"))
g.stopErr["bravo"] = errors.New("the apiserver is gone")
if code := g.run(nil, true, true); code != 1 {
t.Fatalf("exit = %d, want 1", code)
}
_, run, _ := strings.Cut(g.out.String(), bnStopping)
// The stop was asked for, so the server stays stopped whatever the poll said.
want := " bravo: stopping\n bravo: failed: the apiserver is gone\n0 backed up, 1 failed.\n" +
"Left stopped: bravo. Start them from the panel when you are done.\n"
if run != want {
t.Fatalf("run =\n%s\nwant\n%s", run, want)
}
for _, tc := range []struct {
call int
events []string
}{
{1, nil}, // before the request: nothing is asked for
{2, []string{"request alpha"}}, // the first poll after it
} {
g := newBNRig(stoppedWorld("alpha"))
g.jobsFail["alpha"] = tc.call
if code := g.run(nil, true, false); code != 1 {
t.Fatalf("call %d: exit = %d, want 1", tc.call, code)
}
if !reflect.DeepEqual(g.events, tc.events) {
t.Fatalf("call %d: events = %v, want %v", tc.call, g.events, tc.events)
}
if !strings.Contains(g.out.String(), " alpha: failed: list its backup Jobs: the apiserver is gone\n") {
t.Fatalf("call %d: output =\n%s", tc.call, g.out.String())
}
}
}
cmd/felis/backupnow_test.go
+3 -13
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -12,6 +11,7 @@ import (
"testing"
"time"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
corev1 "k8s.io/api/core/v1"
@@ -28,15 +28,9 @@ func internalAPIObjs(clusterIP, token string) []client.Object {
ObjectMeta: metav1.ObjectMeta{Name: platform.APIInternalServiceName, Namespace: bgControlNS},
Spec: corev1.ServiceSpec{ClusterIP: clusterIP},
},
// The console presents the ops token; the proxy's felis-service-token sits
// beside it and must not be the one picked up.
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "felis-ops-token", Namespace: bgControlNS},
Data: map[string][]byte{"token": []byte(token)},
},
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "felis-service-token", Namespace: bgControlNS},
Data: map[string][]byte{"token": []byte("proxy-" + token)},
ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: bgControlNS},
Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
},
}
}
@@ -153,10 +147,6 @@ func TestRequestBackup(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "unreachable") {
t.Fatalf("err = %v, want an 'unreachable' transport error", err)
}
// backup-now ends its run on this error alone.
if !errors.Is(err, errBackupAPIUnreachable) {
t.Fatalf("err = %v, want it to wrap errBackupAPIUnreachable", err)
}
})
}
cmd/felis/breakglass.go
+59 -73
View File
@@ -17,7 +17,6 @@ import (
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
tea "github.com/charmbracelet/bubbletea"
@@ -34,15 +33,13 @@ import (
//
// Root is necessary but NOT sufficient for accountability: root is machine
// authority, not a human identity, so the console additionally captures WHO is
// breaking the glass. When a staff account already exists the operator names one
// and types the one-time code the console mails to its verified address
// (breakglass_otp.go); that account is then the accountable actor. When no code can
// be sent or proven, the typed OVERRIDE proceeds as the OS user and the audit row
// says why. When no staff account exists yet it bootstraps the first Owner and
// attributes the act to the OS user. The audit row records which of these
// happened. This attribution is best-effort, not tamper-proof — whoever runs this
// is root and can edit Postgres directly — but it produces an honest trail for an
// honest operator, which is the point.
// breaking the glass. When a staff account already exists it asks the operator to
// authenticate as an existing admin (the verified identity is the accountable
// actor); when none exists yet it bootstraps the first Owner from the typed
// credential and attributes the act to the OS user. The audit row records the
// difference. This attribution is best-effort, not tamper-proof — whoever runs
// this is root and can edit Postgres directly — but it produces an honest trail
// for an honest operator, which is the point.
//
// When a staff account already exists the console opens on a thin top-level menu
// (menuModel) so that operations are peers, not tails of one wizard. Two account
@@ -65,7 +62,7 @@ import (
// suspension for the interactive `cloudflared tunnel login` browser consent.
// breakGlassOverrideToken is the literal an operator must type to proceed when no
// admin could be verified by a mailed code. Requiring an explicit, deliberate word (not a
// admin credential could be verified. Requiring an explicit, deliberate word (not a
// bare Enter) keeps the unverified root override from happening by reflex.
const breakGlassOverrideToken = "OVERRIDE"
@@ -145,7 +142,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
repo := api.NewPGRepo(drv.DB())
// Decide bootstrap (no admin yet → typed credential mints the first Owner) vs
// recovery (an admin exists → the operator proves one with a mailed code) BEFORE the
// recovery (an admin exists → the operator must authenticate as one) BEFORE the
// alt-screen TUI takes over, so a database fault surfaces as a plain error.
adminExists, err := repo.AdminExists(ctx)
if err != nil {
@@ -153,12 +150,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
return 1
}
// Recovery mails its code through [smtp]; the relay is opened only if a code is
// asked for.
host, _ := os.Hostname()
recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, hostSMTPPasswordPath, platform.DefaultControlNamespace), host: host}
res, err := runBreakGlassTUI(ctx, repo, cfg.Database, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists, recovery)
res, err := runBreakGlassTUI(ctx, repo, cfg.Database.URL, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists)
if err != nil {
fmt.Fprintf(stderr, "felis breakGlass: %v\n", err)
return 1
@@ -182,9 +174,6 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stdout, "\nfelis breakGlass: Owner account %q provisioned; local session sign-in is ENABLED.\n", res.username)
}
fmt.Fprintf(stdout, "Recorded as %q (mode: %s, os user: %s).\n", res.accountable, res.mode, res.osUser)
if res.mode == "root_override" {
fmt.Fprintln(stdout, "No admin was proven by an email code; the audit row records this run as an unverified root override and why.")
}
if res.setupTokenURL != "" {
fmt.Fprintf(stdout, "One-time setup URL (opens a lockdown session to verify email / enroll passkey):\n\n %s\n\n", res.setupTokenURL)
}
@@ -269,6 +258,29 @@ func newOwnerID() string {
return "usr-" + hex.EncodeToString(b[:])
}
// authenticateAdmin resolves a typed admin username for recovery-mode attribution.
// Password verification is gone (passwordless design); Phase 3 replaces this with
// email-OTP recovery. For now it confirms the named admin exists.
func authenticateAdmin(ctx context.Context, s ownerStore, username string) (matched string, ok bool, err error) {
username = strings.TrimSpace(username)
if username == "" {
return "", false, nil
}
u, err := s.UserByUsername(ctx, username)
if errors.Is(err, api.ErrNotFound) {
return "", false, nil
}
if err != nil {
return "", false, err
}
// Staff means admin OR owner: recovery attribution must accept the Owner (the
// primary break-glass identity), not just plain admins.
if u.Role != "admin" && u.Role != "owner" {
return "", false, nil
}
return u.Username, true, nil
}
// provisionOwner mints or resets the single Owner account direct-to-Postgres,
// passwordless. The account is role=owner with no password — the Owner completes
// passwordless login setup via the web setup-token flow after `felis setup`.
@@ -359,10 +371,6 @@ type breakGlassOp struct {
ownerUsername string
ownerEmail string
attemptedAdmin string // recovery / override: the admin username the operator typed
verifiedBy string // recovery: how the admin was proven (verifiedByEmailOTP)
codeSentTo string // recovery: the address the proving code went to
otpSkipped string // root_override: why no code proved an admin (otpSkip*)
otpSkipDetail string // root_override: what failed, when something did
}
// breakGlassOutcome is what performBreakGlass reports back to the TUI.
@@ -486,7 +494,16 @@ func auditSetupMCBind(ctx context.Context, s ownerStore, osUser, mcUUID, authSou
// does not fail the recovery if this write fails — and intentionally honest: it
// records attribution, it does not prove it (a malicious root can edit the row).
func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
blob, err := json.Marshal(breakGlassPayload(op, "owner"))
payload := map[string]any{
"mode": op.mode,
"owner": op.ownerUsername,
"os_user": op.osUser,
"verified": op.mode == "recovery",
}
if op.attemptedAdmin != "" {
payload["admin_account"] = op.attemptedAdmin
}
blob, err := json.Marshal(payload)
if err != nil {
return err
}
@@ -498,33 +515,6 @@ func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error {
})
}
// breakGlassPayload is the who/how both account audits carry, with the account the
// run wrote under subjectKey. verified is true only for a run a mailed code proved;
// such a run names the address the code went to, and an override names why no code
// proved anyone.
func breakGlassPayload(op breakGlassOp, subjectKey string) map[string]any {
payload := map[string]any{
"mode": op.mode,
subjectKey: op.ownerUsername,
"os_user": op.osUser,
"verified": op.verifiedBy != "",
}
if op.attemptedAdmin != "" {
payload["admin_account"] = op.attemptedAdmin
}
if op.verifiedBy != "" {
payload["verified_by"] = op.verifiedBy
payload["code_sent_to"] = op.codeSentTo
}
if op.otpSkipped != "" {
payload["otp_skipped"] = op.otpSkipped
if op.otpSkipDetail != "" {
payload["otp_skip_detail"] = op.otpSkipDetail
}
}
return payload
}
// performAddOperator mints a NEW Operator account and records a best-effort
// accountability row. It mirrors performBreakGlass — passwordless — with two
// deliberate differences. (1) It provisions insert-only (provisionOperator), so it
@@ -548,7 +538,16 @@ func performAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) (bre
// break_glass.operator_create action, naming the new account under an "operator" key
// rather than "owner".
func auditAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) error {
blob, err := json.Marshal(breakGlassPayload(op, "operator"))
payload := map[string]any{
"mode": op.mode,
"operator": op.ownerUsername,
"os_user": op.osUser,
"verified": op.mode == "recovery",
}
if op.attemptedAdmin != "" {
payload["admin_account"] = op.attemptedAdmin
}
blob, err := json.Marshal(payload)
if err != nil {
return err
}
@@ -627,29 +626,16 @@ const (
cloudflareAPITokenDocsURL = "https://developers.cloudflare.com/fundamentals/api/how-to/account-owned-token-template/"
)
func runBreakGlassTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, recovery recoveryConfig) (breakGlassResult, error) {
return runConsoleTUI(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass, recovery)
func runBreakGlassTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass)
}
// runSetupTUI never reaches recovery: setup with a staff account present lands on
// the status screen, so it has no relay to hand over.
func runSetupTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
return runConsoleTUI(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup, recoveryConfig{})
func runSetupTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup)
}
// newConsoleRoot is the console's root model as the host runs it: the summary
// reads this host's alert route.
func newConsoleRoot(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) *rootModel {
rm := newRootModel(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
rm.recovery = recovery
rm.alertRoute = func(ctx context.Context) alertRoute {
return hostAlertRoute(ctx, hostSetupConfigPath, db.URL, defaultHeartbeatFile)
}
return rm
}
func runConsoleTUI(ctx context.Context, s ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) (breakGlassResult, error) {
rm := newConsoleRoot(ctx, s, db, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode, recovery)
func runConsoleTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) (breakGlassResult, error) {
rm := newRootModel(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
final, err := tea.NewProgram(rm, tea.WithAltScreen()).Run()
if err != nil {
return breakGlassResult{}, err
cmd/felis/breakglass_otp.go
-259
View File
@@ -1,259 +0,0 @@
package main
import (
"context"
"crypto/rand"
"crypto/subtle"
"errors"
"fmt"
"math/big"
"os"
"strings"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/platform"
)
// Recovery mode proves who is breaking the glass (#13). Naming a staff account is
// where it starts: the console then mails a one-time code to that account's verified
// address, and only that code, typed within recoveryCodeTTL, makes the run a
// recovery attributed to the account. Every other ending — no such account, no
// verified address, no relay, a send that fails, a wrong or late code, or the
// operator giving up on the mail — leads to the typed OVERRIDE, which the audit row
// records as an unverified root_override together with the reason (otp_skipped).
// The code goes through the same [smtp] relay as the panel's login codes, so with
// that relay down recovery still works, as an override that says why.
const (
recoveryCodeTTL = 10 * time.Minute
recoveryCodeAttempts = 5
)
// The reasons a run fell back to the override, recorded as otp_skipped.
const (
otpSkipUnknownAdmin = "unknown_admin"
otpSkipNoVerifiedEmail = "no_verified_email"
otpSkipNoRelay = "no_relay"
otpSkipSendFailed = "send_failed"
otpSkipCodeExpired = "code_expired"
otpSkipCodeRejected = "code_rejected"
otpSkipByOperator = "operator_skipped"
)
// verifiedByEmailOTP is the audit's verified_by for a recovery the mailed code proved.
const verifiedByEmailOTP = "email_otp"
// recoveryMailer is the one relay call a recovery code needs; *mail.SMTP has it.
type recoveryMailer interface {
SendNotice(ctx context.Context, email, subject, body string) error
}
// recoveryConfig is what the console needs to mail a recovery code. open resolves
// the relay only when a code is about to go out, so a console used to halt a server
// never touches [smtp] or the cluster; its error says why no relay is available.
// host names this machine in the mail. The zero value has no relay.
type recoveryConfig struct {
open func(ctx context.Context) (recoveryMailer, error)
host string
now func() time.Time
}
func (r recoveryConfig) clock() time.Time {
if r.now != nil {
return r.now()
}
return time.Now()
}
// recoveryCode is one mailed code: its value, when it stops working, and how many
// wrong codes were typed against it.
type recoveryCode struct {
value string
expires time.Time
failures int
}
func newRecoveryCode(now time.Time) (*recoveryCode, error) {
n, err := rand.Int(rand.Reader, big.NewInt(1_000_000))
if err != nil {
return nil, fmt.Errorf("generate recovery code: %w", err)
}
return &recoveryCode{value: fmt.Sprintf("%06d", n.Int64()), expires: now.Add(recoveryCodeTTL)}, nil
}
type codeVerdict int
const (
codeAccepted codeVerdict = iota
codeWrong
codeExpired
codeExhausted
)
// check compares a typed code in constant time. Each wrong code counts; the one
// that reaches recoveryCodeAttempts exhausts the code, which then accepts nothing,
// and neither does an expired one.
func (c *recoveryCode) check(typed string, now time.Time) codeVerdict {
if c.failures >= recoveryCodeAttempts {
return codeExhausted
}
if !now.Before(c.expires) {
return codeExpired
}
if subtle.ConstantTimeCompare([]byte(strings.TrimSpace(typed)), []byte(c.value)) == 1 {
return codeAccepted
}
c.failures++
if c.failures >= recoveryCodeAttempts {
return codeExhausted
}
return codeWrong
}
func (c *recoveryCode) attemptsLeft() int { return recoveryCodeAttempts - c.failures }
// recoveryStart is where naming an admin led: a code on its way to that admin, or
// the reason the run has to fall back to the override.
type recoveryStart struct {
admin *api.StaffUser // the named staff account; nil when none matched
code *recoveryCode // set when the code went out
skip string // otpSkip* when it did not
detail string // what failed, for the override screen and the audit row
}
// resolveAdmin loads the staff account (admin or owner) a typed username names, or
// nil when there is none.
func resolveAdmin(ctx context.Context, s ownerStore, username string) (*api.StaffUser, error) {
username = strings.TrimSpace(username)
if username == "" {
return nil, nil
}
u, err := s.UserByUsername(ctx, username)
if errors.Is(err, api.ErrNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
// Staff means admin or owner: the Owner is the primary break-glass identity.
if u.Role != "admin" && u.Role != "owner" {
return nil, nil
}
return u, nil
}
// beginRecovery resolves the named admin and mails it a recovery code. Only a
// datastore or entropy fault is an error; every other way the code cannot go out is
// a recoveryStart with skip set.
func beginRecovery(ctx context.Context, s ownerStore, rc recoveryConfig, username, osUser string, op bgOperation) (recoveryStart, error) {
admin, err := resolveAdmin(ctx, s, username)
if err != nil {
return recoveryStart{}, err
}
if admin == nil {
return recoveryStart{skip: otpSkipUnknownAdmin}, nil
}
st := recoveryStart{admin: admin}
// An address nobody ever proved vouches for nobody.
email := strings.TrimSpace(admin.Email)
if email == "" || !admin.EmailVerified {
st.skip = otpSkipNoVerifiedEmail
return st, nil
}
if rc.open == nil {
st.skip, st.detail = otpSkipNoRelay, "this console has no mail relay"
return st, nil
}
relay, err := rc.open(ctx)
if err != nil {
st.skip, st.detail = otpSkipNoRelay, err.Error()
return st, nil
}
code, err := newRecoveryCode(rc.clock())
if err != nil {
return recoveryStart{}, err
}
sendCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
subject, body := recoveryMail(code.value, rc.host, osUser, admin.Username, op)
if err := relay.SendNotice(sendCtx, email, subject, body); err != nil {
st.skip, st.detail = otpSkipSendFailed, err.Error()
return st, nil
}
st.code = code
return st, nil
}
// recoveryMail words the code mail. It says where, by whom and for what the console
// was opened, so an admin who did not ask for it learns that root on that machine is
// in someone else's hands.
func recoveryMail(code, host, osUser, admin string, op bgOperation) (subject, body string) {
what, whatZH := "reset the Owner account", "重置 Owner 账号"
if op == bgAddOperator {
what, whatZH = "add an Operator account", "添加 Operator 账号"
}
if host == "" {
host = "the Felis host"
}
minutes := int(recoveryCodeTTL / time.Minute)
subject = "Felis break-glass recovery code / 紧急恢复验证码"
body = fmt.Sprintf(`Someone with root on %[1]s (OS user %[2]s) opened felis breakGlass and named your staff account %[3]q to %[4]s.
Recovery code: %[6]s
It works for %[7]d minutes.
If this was not you, root on that machine is in someone else's hands: change its credentials and read the audit log for break_glass entries.
有人在 %[1]s 上以 root 身份（系统用户 %[2]s）打开了 felis breakGlass，指名你的管理员账号 %[3]q 来%[5]s。
恢复验证码：%[6]s
%[7]d 分钟内有效。
如果不是你本人，这台机器的 root 已落入他人之手：请更换它的凭据，并查看审计日志中的 break_glass 记录。
`, host, osUser, admin, what, whatZH, code, minutes)
return subject, body
}
// maskEmail keeps the first character of the local part and the domain, enough for
// the operator to recognise the address without putting it on screen whole.
func maskEmail(email string) string {
at := strings.LastIndex(email, "@")
if at <= 0 {
return "***"
}
return email[:1] + strings.Repeat("*", max(at-1, 3)) + email[at:]
}
// hostRecoveryMailer opens the [smtp] relay from the host the way the watchdog does:
// the password is the env var password_ref names when that is set, else the host
// copy at passwordPath, else the felis-smtp Secret, whose absence means a relay
// without AUTH. The cluster is reached only when the host copy is missing, so a
// break-glass on a host whose k3s is down still gets its code.
func hostRecoveryMailer(c config.SMTPConfig, passwordPath, controlNS string) func(context.Context) (recoveryMailer, error) {
return func(ctx context.Context) (recoveryMailer, error) {
if strings.TrimSpace(c.Host) == "" {
return nil, errors.New("[smtp] is not configured in felis.toml")
}
if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" {
return smtpRelay(c, os.Getenv(ref)), nil
}
if password, ok, err := readHostCredential(passwordPath); err != nil {
return nil, fmt.Errorf("read the relay password: %w", err)
} else if ok {
return smtpRelay(c, password), nil
}
cl, err := buildSystemServerClient()
if err != nil {
return nil, fmt.Errorf("reach the cluster for the relay password: %w", err)
}
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
password, err := smtpSecretPassword(ctx, cl, controlNS)
if err != nil {
return nil, fmt.Errorf("read the relay password from %s/%s: %w", controlNS, platform.SMTPSecretName, err)
}
return smtpRelay(c, password), nil
}
}
cmd/felis/breakglass_otp_test.go
-498
View File
@@ -1,498 +0,0 @@
package main
import (
"context"
"errors"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/platform"
tea "github.com/charmbracelet/bubbletea"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)
// These tests cover the recovery proof (#13): the mailed code's rules, where
// naming an admin leads, what the mail says, how the console walks from a name to
// a proven (or overridden) run, and what the audit row then records. No mail
// leaves the process: the relay is a fake that keeps what it was handed.
type sentMail struct{ to, subject, body string }
type fakeRelay struct {
sent []sentMail
err error
}
func (r *fakeRelay) SendNotice(_ context.Context, to, subject, body string) error {
if r.err != nil {
return r.err
}
r.sent = append(r.sent, sentMail{to, subject, body})
return nil
}
// sentCode pulls the code out of the one mail the relay carried.
func (r *fakeRelay) sentCode(t *testing.T) string {
t.Helper()
if len(r.sent) != 1 {
t.Fatalf("relay carried %d mails, want 1", len(r.sent))
}
_, after, ok := strings.Cut(r.sent[0].body, "Recovery code: ")
if !ok || len(after) < 6 {
t.Fatalf("mail carries no recovery code:\n%s", r.sent[0].body)
}
return after[:6]
}
func relayConfig(r *fakeRelay, now func() time.Time) recoveryConfig {
return recoveryConfig{
open: func(context.Context) (recoveryMailer, error) { return r, nil },
host: "felis-host-1",
now: now,
}
}
func verifiedAdmin(username, email string) *api.StaffUser {
return &api.StaffUser{ID: "usr-" + username, Username: username, Role: "owner", Email: email, EmailVerified: true}
}
func TestRecoveryCodeRules(t *testing.T) {
t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
t.Run("a fresh code is six digits and lives for the TTL", func(t *testing.T) {
seen := map[string]bool{}
for i := 0; i < 20; i++ {
c, err := newRecoveryCode(t0)
if err != nil {
t.Fatal(err)
}
if !isRecoveryCodeShape(c.value) {
t.Fatalf("code %q is not six digits", c.value)
}
if !c.expires.Equal(t0.Add(recoveryCodeTTL)) {
t.Fatalf("expires = %v, want %v", c.expires, t0.Add(recoveryCodeTTL))
}
seen[c.value] = true
}
if len(seen) < 2 {
t.Error("twenty codes were all the same")
}
})
t.Run("the right code is accepted, surrounding space ignored", func(t *testing.T) {
c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
if v := c.check(" 042917 ", t0); v != codeAccepted {
t.Errorf("check = %v, want accepted", v)
}
})
t.Run("wrong codes count down and the last one exhausts it", func(t *testing.T) {
c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
for i := 1; i < recoveryCodeAttempts; i++ {
if v := c.check("000000", t0); v != codeWrong {
t.Fatalf("wrong code %d: check = %v, want wrong", i, v)
}
if left := c.attemptsLeft(); left != recoveryCodeAttempts-i {
t.Fatalf("after %d wrong codes attemptsLeft = %d, want %d", i, left, recoveryCodeAttempts-i)
}
}
if v := c.check("000000", t0); v != codeExhausted {
t.Fatalf("wrong code %d: check = %v, want exhausted", recoveryCodeAttempts, v)
}
if v := c.check("042917", t0); v != codeExhausted {
t.Errorf("the right code after exhaustion: check = %v, want exhausted", v)
}
})
t.Run("an expired code accepts nothing", func(t *testing.T) {
c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
if v := c.check("042917", t0.Add(recoveryCodeTTL-time.Second)); v != codeAccepted {
t.Fatalf("a second before expiry: check = %v, want accepted", v)
}
c = &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)}
if v := c.check("042917", t0.Add(recoveryCodeTTL)); v != codeExpired {
t.Errorf("at expiry: check = %v, want expired", v)
}
})
}
func TestBeginRecovery(t *testing.T) {
ctx := context.Background()
t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
clock := func() time.Time { return t0 }
t.Run("mails a code to the named admin's verified address", func(t *testing.T) {
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
r := &fakeRelay{}
st, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgAddOperator)
if err != nil {
t.Fatal(err)
}
if st.code == nil || st.skip != "" {
t.Fatalf("start = %+v, want a code and no skip", st)
}
if st.admin == nil || st.admin.Username != "root" {
t.Fatalf("start.admin = %+v, want root", st.admin)
}
if got := r.sentCode(t); got != st.code.value {
t.Errorf("mailed code %q, want the code the console checks (%q)", got, st.code.value)
}
m := r.sent[0]
if m.to != "[email protected]" {
t.Errorf("mail went to %q, want [email protected]", m.to)
}
for _, want := range []string{"felis-host-1", "OS user alice", `"root"`, "add an Operator account", "添加 Operator 账号", "10 minutes"} {
if !strings.Contains(m.body, want) {
t.Errorf("mail body lacks %q:\n%s", want, m.body)
}
}
if !st.code.expires.Equal(t0.Add(recoveryCodeTTL)) {
t.Errorf("code expires %v, want %v", st.code.expires, t0.Add(recoveryCodeTTL))
}
})
t.Run("the Owner reset is named as such", func(t *testing.T) {
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}}
r := &fakeRelay{}
if _, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgProvisionOwner); err != nil {
t.Fatal(err)
}
if body := r.sent[0].body; !strings.Contains(body, "reset the Owner account") || !strings.Contains(body, "重置 Owner 账号") {
t.Errorf("mail body does not name the Owner reset:\n%s", body)
}
})
// Each way the code cannot go out ends in a skip reason and no mail.
unverified := verifiedAdmin("root", "[email protected]")
unverified.EmailVerified = false
noEmail := verifiedAdmin("root", "")
cases := []struct {
name string
user *api.StaffUser
rc func(r *fakeRelay) recoveryConfig
skip string
detail string
wantAdmin bool
relayError error
}{
{name: "unknown admin", user: nil, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipUnknownAdmin},
{name: "unverified address", user: unverified, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
{name: "no address", user: noEmail, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true},
{name: "no relay wired", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig { return recoveryConfig{} }, skip: otpSkipNoRelay, detail: "this console has no mail relay", wantAdmin: true},
{name: "relay cannot open", user: verifiedAdmin("root", "[email protected]"), rc: func(*fakeRelay) recoveryConfig {
return recoveryConfig{open: func(context.Context) (recoveryMailer, error) {
return nil, errors.New("[smtp] is not configured in felis.toml")
}}
}, skip: otpSkipNoRelay, detail: "[smtp] is not configured in felis.toml", wantAdmin: true},
{name: "send fails", user: verifiedAdmin("root", "[email protected]"), rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) },
skip: otpSkipSendFailed, detail: "554 relay refused", wantAdmin: true, relayError: errors.New("554 relay refused")},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
f := &fakeOwnerStore{users: map[string]*api.StaffUser{}}
if tc.user != nil {
f.users["root"] = tc.user
}
r := &fakeRelay{err: tc.relayError}
st, err := beginRecovery(ctx, f, tc.rc(r), "root", "alice", bgProvisionOwner)
if err != nil {
t.Fatal(err)
}
if st.code != nil || st.skip != tc.skip || st.detail != tc.detail {
t.Errorf("start = {code:%v skip:%q detail:%q}, want no code, skip %q, detail %q", st.code, st.skip, st.detail, tc.skip, tc.detail)
}
if (st.admin != nil) != tc.wantAdmin {
t.Errorf("start.admin = %+v, want present=%v", st.admin, tc.wantAdmin)
}
if len(r.sent) != 0 {
t.Errorf("relay carried %d mails, want none", len(r.sent))
}
})
}
t.Run("a datastore fault is an error", func(t *testing.T) {
f := &fakeOwnerStore{userErr: errors.New("db down")}
if _, err := beginRecovery(ctx, f, relayConfig(&fakeRelay{}, clock), "root", "alice", bgProvisionOwner); err == nil {
t.Fatal("want the store fault")
}
})
}
func TestMaskEmail(t *testing.T) {
for in, want := range map[string]string{
"[email protected]": "a****@example.com",
"[email protected]": "a***@example.com",
"@example.com": "***",
"nonsense": "***",
} {
if got := maskEmail(in); got != want {
t.Errorf("maskEmail(%q) = %q, want %q", in, got, want)
}
}
}
func TestHostRecoveryMailer(t *testing.T) {
ctx := context.Background()
t.Run("no [smtp] host is no relay", func(t *testing.T) {
_, err := hostRecoveryMailer(config.SMTPConfig{}, hostSMTPPasswordPath, "felis")(ctx)
if err == nil || !strings.Contains(err.Error(), "[smtp]") {
t.Fatalf("err = %v, want it to name [smtp]", err)
}
})
t.Run("the password_ref env var supplies the password", func(t *testing.T) {
t.Setenv("FELIS_TEST_RELAY_PW", "from-env")
off := false
c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_RELAY_PW", RequireTLS: &off}
got, err := hostRecoveryMailer(c, hostSMTPPasswordPath, "felis")(ctx)
if err != nil {
t.Fatal(err)
}
relay, ok := got.(*mail.SMTP)
if !ok {
t.Fatalf("relay is %T, want *mail.SMTP", got)
}
if relay.Password != "from-env" || relay.Host != "mail.example.com" || relay.Port != 2525 || relay.From != "[email protected]" || relay.Username != "felis" || relay.RequireTLS {
t.Errorf("relay = %+v, want the [smtp] fields with the env password and TLS as configured", *relay)
}
})
}
func TestSMTPSecretPassword(t *testing.T) {
ctx := context.Background()
scheme := haltScheme(t)
t.Run("reads the password key", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
Data: map[string][]byte{platform.SMTPSecretPasswordKey: []byte("s3cret")},
}).Build()
if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "s3cret" {
t.Errorf("smtpSecretPassword = (%q, %v), want (s3cret, nil)", pw, err)
}
})
t.Run("a missing Secret is a relay without AUTH", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(scheme).Build()
if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "" {
t.Errorf("smtpSecretPassword = (%q, %v), want (\"\", nil)", pw, err)
}
})
t.Run("any other read failure is an error", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
return apierrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, platform.SMTPSecretName, errors.New("rbac"))
},
}).Build()
if _, err := smtpSecretPassword(ctx, cl, "felis"); err == nil {
t.Fatal("want the read failure")
}
})
}
// recoveryModel is an Owner-reset console for admin "root" (verified address
// [email protected]), run by OS user alice, with the fake relay behind it.
func recoveryModel(t *testing.T, f *fakeOwnerStore, r *fakeRelay, now func() time.Time) *ownerModel {
t.Helper()
if f.users == nil {
f.users = map[string]*api.StaffUser{"root": verifiedAdmin("root", "[email protected]")}
}
return newOwnerModel(context.Background(), f, "alice", true).withRecovery(relayConfig(r, now))
}
// nameAdmin submits the admin-name form and feeds the result of the send back in.
func nameAdmin(t *testing.T, m *ownerModel, name string) *ownerModel {
t.Helper()
m.authUser = name
_, cmd := m.onFormComplete()
if m.step != owWorking {
t.Fatalf("after naming the admin step = %v, want owWorking", m.step)
}
msg := findMsg[owAuthMsg](t, cmd)
next, _ := m.Update(msg)
return next.(*ownerModel)
}
// findMsg runs a (possibly batched) command and returns the first T it produces.
func findMsg[T any](t *testing.T, cmd tea.Cmd) T {
t.Helper()
var zero T
if cmd == nil {
t.Fatalf("no command, want one producing %T", zero)
}
switch msg := cmd().(type) {
case T:
return msg
case tea.BatchMsg:
for _, c := range msg {
if c == nil {
continue
}
if got, ok := c().(T); ok {
return got
}
}
}
t.Fatalf("command produced no %T", zero)
return zero
}
// typeCode submits the code form with typed.
func typeCode(t *testing.T, m *ownerModel, typed string) {
t.Helper()
if m.step != owCode {
t.Fatalf("step = %v, want owCode", m.step)
}
m.codeInput = typed
m.onFormComplete()
}
// provisionAudit finishes the run as an Owner reset and returns its audit payload.
func provisionAudit(t *testing.T, m *ownerModel, f *fakeOwnerStore) (api.AuditEntry, map[string]any) {
t.Helper()
if m.step != owProvision {
t.Fatalf("step = %v, want owProvision", m.step)
}
m.username = "owner"
msg := m.provisionCmd()().(owProvisionMsg)
if msg.err != nil {
t.Fatalf("provision: %v", msg.err)
}
return auditOf(t, f)
}
func TestRecoveryConsoleFlow(t *testing.T) {
t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC)
fixed := func() time.Time { return t0 }
t.Run("the mailed code proves the admin and the audit says so", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
typeCode(t, m, r.sentCode(t))
if m.mode != "recovery" || m.accountable != "root" {
t.Fatalf("mode=%q accountable=%q, want recovery attributed to root", m.mode, m.accountable)
}
e, payload := provisionAudit(t, m, f)
if e.Actor != "root" || e.Action != "break_glass.recovery" {
t.Errorf("audit = %+v, want actor=root action=break_glass.recovery", e)
}
if payload["verified"] != true || payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" || payload["os_user"] != "alice" {
t.Errorf("payload = %v, want verified by email_otp to [email protected], os_user alice", payload)
}
})
t.Run("a wrong code asks again, and the last wrong one leads to the override", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
wrong := "000000"
if r.sentCode(t) == wrong {
wrong = "111111"
}
for i := 1; i < recoveryCodeAttempts; i++ {
typeCode(t, m, wrong)
if m.step != owCode || !strings.Contains(m.codeNote, "wrong") {
t.Fatalf("wrong code %d: step=%v note=%q, want the code form again with a note", i, m.step, m.codeNote)
}
}
typeCode(t, m, wrong)
if m.step != owOverride || m.skip != otpSkipCodeRejected {
t.Fatalf("after %d wrong codes step=%v skip=%q, want the override for code_rejected", recoveryCodeAttempts, m.step, m.skip)
}
m.onFormComplete() // OVERRIDE typed
e, payload := provisionAudit(t, m, f)
if e.Actor != "alice" || e.Action != "break_glass.root_override" {
t.Errorf("audit = %+v, want actor=alice action=break_glass.root_override", e)
}
if payload["verified"] != false || payload["otp_skipped"] != otpSkipCodeRejected || payload["admin_account"] != "root" {
t.Errorf("payload = %v, want unverified, otp_skipped=code_rejected, admin_account=root", payload)
}
})
t.Run("a late code leads to the override", func(t *testing.T) {
now := t0
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, func() time.Time { return now }), "root")
now = t0.Add(recoveryCodeTTL)
typeCode(t, m, r.sentCode(t))
if m.step != owOverride || m.skip != otpSkipCodeExpired {
t.Fatalf("step=%v skip=%q, want the override for code_expired", m.step, m.skip)
}
})
t.Run("OVERRIDE at the code prompt goes on unverified, saying the operator skipped", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
typeCode(t, m, breakGlassOverrideToken)
if m.mode != "root_override" || m.accountable != "alice" {
t.Fatalf("mode=%q accountable=%q, want root_override as alice", m.mode, m.accountable)
}
_, payload := provisionAudit(t, m, f)
if payload["verified"] != false || payload["otp_skipped"] != otpSkipByOperator {
t.Errorf("payload = %v, want unverified, otp_skipped=operator_skipped", payload)
}
})
t.Run("a relay failure leads to the override naming it", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{err: errors.New("dial tcp 10.0.0.9:587: connect: connection refused")}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
if m.step != owOverride || m.skip != otpSkipSendFailed {
t.Fatalf("step=%v skip=%q, want the override for send_failed", m.step, m.skip)
}
if reason := m.overrideReason(); !strings.Contains(reason, "connection refused") {
t.Errorf("override reason %q does not name the failure", reason)
}
m.onFormComplete()
_, payload := provisionAudit(t, m, f)
if payload["otp_skipped"] != otpSkipSendFailed || !strings.Contains(payload["otp_skip_detail"].(string), "connection refused") {
t.Errorf("payload = %v, want otp_skipped=send_failed with the relay's error", payload)
}
})
t.Run("esc at the code prompt starts over and forgets the code", func(t *testing.T) {
f, r := &fakeOwnerStore{}, &fakeRelay{}
m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root")
code := r.sentCode(t)
next, _ := m.Update(key(tea.KeyEsc))
m = next.(*ownerModel)
if m.step != owAuth || m.code != nil || m.admin != nil {
t.Fatalf("after esc step=%v code=%v admin=%v, want owAuth with the attempt forgotten", m.step, m.code, m.admin)
}
// The old code cannot be replayed: the next name mails a new one.
m = nameAdmin(t, m, "root")
if len(r.sent) != 2 {
t.Fatalf("relay carried %d mails, want a second one for the new attempt", len(r.sent))
}
_, fresh, _ := strings.Cut(r.sent[1].body, "Recovery code: ")
if m.code.value != fresh[:6] {
t.Errorf("the console checks %q, want the newly mailed %q (old one was %q)", m.code.value, fresh[:6], code)
}
})
}
func TestRootHandsRecoveryToAccountOperations(t *testing.T) {
for _, op := range []bgOperation{bgProvisionOwner, bgAddOperator} {
m := newTestRoot(true, consoleModeBreakGlass, "")
m.recovery = recoveryConfig{host: "felis-host-1"}
m = drive(t, m, menuChoiceMsg{op: op})
om, ok := m.screen.(*ownerModel)
if !ok {
t.Fatalf("op %v: screen = %T, want *ownerModel", op, m.screen)
}
if om.recovery.host != "felis-host-1" {
t.Errorf("op %v: the account screen has no relay config; its codes could never go out", op)
}
}
}
cmd/felis/breakglass_test.go
+40 -56
View File
@@ -269,57 +269,71 @@ func TestEnableLocalAuth(t *testing.T) {
}
}
func TestResolveAdmin(t *testing.T) {
func TestAuthenticateAdmin(t *testing.T) {
ctx := context.Background()
// resolveAdmin only finds the staff account a typed name points at; proving the
// operator holds it is the mailed code's job (beginRecovery).
// Password verification is gone (passwordless design): authenticateAdmin now only
// resolves the named admin so recovery can attribute the audit to a real identity.
// The security boundary is the break-glass root gate, not a typed secret.
t.Run("resolves an existing admin", func(t *testing.T) {
t.Run("resolves an existing admin for attribution", func(t *testing.T) {
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": mkAdmin("root")}}
u, err := resolveAdmin(ctx, f, " root ")
matched, ok, err := authenticateAdmin(ctx, f, "root")
if err != nil {
t.Fatalf("resolveAdmin: %v", err)
t.Fatalf("authenticateAdmin: %v", err)
}
if u == nil || u.Username != "root" {
t.Fatalf("resolveAdmin = %+v, want the root admin", u)
if !ok {
t.Fatal("ok = false, want true for an existing admin")
}
if matched != "root" {
t.Errorf("matched = %q, want root", matched)
}
})
t.Run("a non-admin role is no staff account", func(t *testing.T) {
t.Run("a non-admin role can never attribute a break-glass", func(t *testing.T) {
player := mkAdmin("alice")
player.Role = "user" // a player row is not staff
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"alice": player}}
if u, err := resolveAdmin(ctx, f, "alice"); u != nil || err != nil {
t.Errorf("resolveAdmin(player) = (%+v, %v), want (nil, nil)", u, err)
_, ok, err := authenticateAdmin(ctx, f, "alice")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if ok {
t.Error("ok = true, want false for a non-admin role")
}
})
t.Run("the owner role counts as staff", func(t *testing.T) {
t.Run("the owner role attributes like an admin", func(t *testing.T) {
owner := mkAdmin("root")
owner.Role = "owner" // the platform owner is staff too (migration 0011)
f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": owner}}
if u, err := resolveAdmin(ctx, f, "root"); err != nil || u == nil {
t.Fatalf("resolveAdmin(owner) = (%+v, %v), want the owner", u, err)
matched, ok, err := authenticateAdmin(ctx, f, "root")
if err != nil || !ok || matched != "root" {
t.Fatalf("authenticateAdmin(owner) = (%q, %v, %v), want (root, true, nil)", matched, ok, err)
}
})
t.Run("an unknown user is nil, not an error", func(t *testing.T) {
if u, err := resolveAdmin(ctx, &fakeOwnerStore{}, "nobody"); u != nil || err != nil {
t.Errorf("resolveAdmin(unknown) = (%+v, %v), want (nil, nil)", u, err)
t.Run("an unknown user is a non-match, not an error", func(t *testing.T) {
f := &fakeOwnerStore{}
_, ok, err := authenticateAdmin(ctx, f, "nobody")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if ok {
t.Error("ok = true, want false for an unknown user")
}
})
t.Run("an empty username makes no store call", func(t *testing.T) {
t.Run("an empty username is a non-match with no store call", func(t *testing.T) {
f := &fakeOwnerStore{userErr: errors.New("must not be called")}
if u, err := resolveAdmin(ctx, f, " "); u != nil || err != nil {
t.Errorf("empty username: (%+v, %v), want (nil, nil)", u, err)
if _, ok, err := authenticateAdmin(ctx, f, ""); ok || err != nil {
t.Errorf("empty username: ok=%v err=%v, want false,nil", ok, err)
}
})
t.Run("a datastore fault is surfaced", func(t *testing.T) {
f := &fakeOwnerStore{userErr: errors.New("db down")}
if _, err := resolveAdmin(ctx, f, "root"); err == nil {
if _, _, err := authenticateAdmin(ctx, f, "root"); err == nil {
t.Fatal("want error when the store fails")
}
})
@@ -387,8 +401,6 @@ func TestPerformBreakGlass(t *testing.T) {
osUser: "alice",
ownerUsername: "owner",
attemptedAdmin: "root",
verifiedBy: verifiedByEmailOTP,
codeSentTo: "[email protected]",
}
out, err := performBreakGlass(ctx, f, op)
if err != nil {
@@ -413,23 +425,6 @@ func TestPerformBreakGlass(t *testing.T) {
if payload["admin_account"] != "root" {
t.Errorf("payload.admin_account = %v, want root", payload["admin_account"])
}
if payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "[email protected]" {
t.Errorf("payload = %v, want verified_by=email_otp [email protected]", payload)
}
if _, present := payload["otp_skipped"]; present {
t.Error("a proven recovery carries no otp_skipped")
}
})
t.Run("recovery without a proof is recorded unverified", func(t *testing.T) {
f := &fakeOwnerStore{}
op := breakGlassOp{mode: "recovery", accountable: "root", osUser: "alice", ownerUsername: "owner", attemptedAdmin: "root"}
if _, err := performBreakGlass(ctx, f, op); err != nil {
t.Fatalf("performBreakGlass: %v", err)
}
if _, payload := auditOf(t, f); payload["verified"] != false {
t.Errorf("payload.verified = %v, want false: only a mailed code verifies", payload["verified"])
}
})
t.Run("root override records an unverified row attributed to the OS user", func(t *testing.T) {
@@ -440,8 +435,6 @@ func TestPerformBreakGlass(t *testing.T) {
osUser: "alice",
ownerUsername: "owner",
attemptedAdmin: "typo-admin",
otpSkipped: otpSkipSendFailed,
otpSkipDetail: "dial tcp: connection refused",
}
if _, err := performBreakGlass(ctx, f, op); err != nil {
t.Fatalf("performBreakGlass: %v", err)
@@ -457,13 +450,6 @@ func TestPerformBreakGlass(t *testing.T) {
if payload["admin_account"] != "typo-admin" {
t.Errorf("payload.admin_account = %v, want typo-admin", payload["admin_account"])
}
// Why no code proved anyone is part of the record.
if payload["otp_skipped"] != otpSkipSendFailed || payload["otp_skip_detail"] != "dial tcp: connection refused" {
t.Errorf("payload = %v, want otp_skipped=send_failed with its detail", payload)
}
if _, present := payload["verified_by"]; present {
t.Error("an override carries no verified_by")
}
})
t.Run("an audit failure does not fail the recovery", func(t *testing.T) {
@@ -630,8 +616,6 @@ func TestPerformAddOperator(t *testing.T) {
ownerUsername: "ops-jordan",
ownerEmail: "[email protected]",
attemptedAdmin: "root",
verifiedBy: verifiedByEmailOTP,
codeSentTo: "[email protected]",
}
out, err := performAddOperator(ctx, f, op)
if err != nil {
@@ -658,14 +642,14 @@ func TestPerformAddOperator(t *testing.T) {
if _, present := payload["owner"]; present {
t.Error("payload.owner present, want the new account under the operator key")
}
if payload["verified"] != true || payload["admin_account"] != "root" || payload["verified_by"] != verifiedByEmailOTP {
t.Errorf("payload = %v, want verified=true admin_account=root verified_by=email_otp", payload)
if payload["verified"] != true || payload["admin_account"] != "root" {
t.Errorf("payload = %v, want verified=true admin_account=root", payload)
}
})
t.Run("root override records an unverified operator row", func(t *testing.T) {
f := &fakeOwnerStore{}
op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin", otpSkipped: otpSkipUnknownAdmin}
op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin"}
if _, err := performAddOperator(ctx, f, op); err != nil {
t.Fatalf("performAddOperator: %v", err)
}
@@ -673,8 +657,8 @@ func TestPerformAddOperator(t *testing.T) {
t.Fatalf("want 1 insert, got %d", len(f.inserts))
}
_, payload := auditOf(t, f)
if payload["verified"] != false || payload["otp_skipped"] != otpSkipUnknownAdmin {
t.Errorf("payload = %v, want verified=false otp_skipped=unknown_admin", payload)
if payload["verified"] != false {
t.Errorf("payload.verified = %v, want false for root_override", payload["verified"])
}
})
cmd/felis/conflict_test.go
-358
View File
@@ -1,358 +0,0 @@
package main
import (
"context"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/imagepin"
"felis.lolicon.best/internal/naming"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)
// racingClient lands one concurrent write (race) on the stored object just before
// setup's first write of it goes out, the way the operator's status update or
// felis-api's idle patch can, and counts setup's writes.
func racingClient(t *testing.T, race func(ctx context.Context, c client.WithWatch), objs ...client.Object) (client.Client, *int) {
t.Helper()
writes := 0
before := func(ctx context.Context, c client.WithWatch) {
writes++
if writes == 1 {
race(ctx, c)
}
}
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
WithStatusSubresource(&v1alpha1.MinecraftServer{}).
WithInterceptorFuncs(interceptor.Funcs{
Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
before(ctx, c)
return c.Update(ctx, obj, opts...)
},
Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
before(ctx, c)
return c.Patch(ctx, obj, patch, opts...)
},
}).Build()
return cl, &writes
}
func staleLoginGate(t *testing.T) *v1alpha1.MinecraftServer {
t.Helper()
ms, err := loginSystemServer("felis-limbo:demo", "minecraft",
"http://old.internal:8081", "203.0.113.10.nip.io", "console.203.0.113.10.nip.io")
if err != nil {
t.Fatal(err)
}
return ms
}
func getLogin(t *testing.T, cl client.Client) *v1alpha1.MinecraftServer {
t.Helper()
var ms v1alpha1.MinecraftServer
if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
t.Fatal(err)
}
return &ms
}
func envMap(ms *v1alpha1.MinecraftServer) map[string]string {
m := map[string]string{}
for _, e := range ms.Spec.Env {
m[e.Name] = e.Value
}
return m
}
// A hand edit that lands while setup refreshes the console hostnames costs setup a
// re-read and a second write; both the edit and the refresh survive.
func TestRefreshDerivedEnvRetriesAConcurrentEdit(t *testing.T) {
ctx := context.Background()
cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
ms := getLogin(t, c)
ms.Spec.Env = append(ms.Spec.Env, v1alpha1.EnvVar{Name: "HAND_TUNED", Value: "1"})
if err := c.Update(ctx, ms); err != nil {
t.Fatal(err)
}
}, staleLoginGate(t))
desired, err := loginSystemServer("felis-limbo:demo", "minecraft",
"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net")
if err != nil {
t.Fatal(err)
}
refreshed, err := refreshDerivedEnv(ctx, cl, getLogin(t, cl), desired)
if err != nil || !refreshed {
t.Fatalf("refreshed=%v err=%v, want a refresh after the retry", refreshed, err)
}
env := envMap(getLogin(t, cl))
if env[envPanelHostname] != "console.mc.example.net" || env[envRootDomain] != "mc.example.net" {
t.Errorf("env = %v, want the new hostnames", env)
}
if env["HAND_TUNED"] != "1" {
t.Errorf("env = %v: the concurrent hand edit was dropped", env)
}
if *writes != 2 {
t.Errorf("writes = %d, want 2 (one conflict, one retry)", *writes)
}
}
// converge reports each fill once even when a status write forced a retry.
func TestConvergeRetriesAConcurrentStatusWrite(t *testing.T) {
ctx := context.Background()
lobby, err := lobbySystemServer("reg/lobby:1", "minecraft")
if err != nil {
t.Fatal(err)
}
lobby.Spec.Rcon = v1alpha1.RconSpec{}
cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
var ms v1alpha1.MinecraftServer
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
t.Fatal(err)
}
ms.Status.Phase = v1alpha1.PhaseRunning
if err := c.Status().Update(ctx, &ms); err != nil {
t.Fatal(err)
}
}, lobby)
var got systemServerOutcome
for _, o := range convergeSystemServers(ctx, cl, "minecraft", "", "reg/lobby:1",
"http://felis-api-internal.felis.svc.cluster.local:8081", "mc.example.net", "console.mc.example.net") {
if o.name == naming.SystemLobbyServer {
got = o
}
}
if got.err != nil || !got.updated || !slices.Equal(got.changes, []string{"spec.rcon"}) {
t.Fatalf("lobby outcome = %+v, want spec.rcon filled once", got)
}
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLobbyServer}, &ms); err != nil {
t.Fatal(err)
}
if !ms.Spec.Rcon.Enabled || ms.Status.Phase != v1alpha1.PhaseRunning {
t.Errorf("rcon.enabled=%v phase=%q, want the fill and the status write both kept", ms.Spec.Rcon.Enabled, ms.Status.Phase)
}
if *writes != 2 {
t.Errorf("writes = %d, want 2", *writes)
}
}
// A replica refresh racing another writer keeps that writer's key.
func TestSecretReplicaRefreshRetriesAConcurrentWrite(t *testing.T) {
secret := func(ns, body string) *corev1.Secret {
return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "felis-config", Namespace: ns},
Data: map[string][]byte{"felis.toml": []byte(body)}}
}
cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
var s corev1.Secret
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
t.Fatal(err)
}
s.Data["extra"] = []byte("x")
if err := c.Update(ctx, &s); err != nil {
t.Fatal(err)
}
}, secret("felis", "current"), secret("minecraft", "stale"))
out := ensureSecretReplica(context.Background(), cl, "felis", "minecraft",
"felis-config", "felis.toml", "config", "minecraft ns", true)
if out.err != nil || !out.updated {
t.Fatalf("outcome = %+v, want refreshed", out)
}
var s corev1.Secret
if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "felis-config"}, &s); err != nil {
t.Fatal(err)
}
if string(s.Data["felis.toml"]) != "current" || string(s.Data["extra"]) != "x" {
t.Errorf("replica data = %q, want felis.toml=current and extra=x", s.Data)
}
if *writes != 2 {
t.Errorf("writes = %d, want 2", *writes)
}
}
const (
sysOldDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
sysNewDigest = "sha256:4444444444444444444444444444444444444444444444444444444444444444"
)
func systemPinRegistry(t *testing.T) imagepin.Resolver {
t.Helper()
reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v2/felis/limbo/manifests/demo", "/v2/felis/lobby/manifests/demo":
w.Header().Set("Docker-Content-Digest", sysNewDigest)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(reg.Close)
return imagepin.Resolver{Registry: defaultRegistryURL, Endpoint: strings.TrimPrefix(reg.URL, "http://")}
}
func systemServer(name, image string) *v1alpha1.MinecraftServer {
ms := &v1alpha1.MinecraftServer{}
ms.Name, ms.Namespace = name, "minecraft"
ms.Labels = map[string]string{v1alpha1.LabelSystemRole: name}
ms.Spec.Image = image
return ms
}
// The installer's --system pin moves a system server onto the build its tag names
// now, from the bare tag or from an earlier digest, and is a no-op the second time.
func TestPinSystemServerImage(t *testing.T) {
ctx := context.Background()
res := systemPinRegistry(t)
limbo := defaultRegistryURL + "/felis/limbo:demo"
lobby := defaultRegistryURL + "/felis/lobby:demo"
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
systemServer(naming.SystemLoginServer, limbo),
systemServer(naming.SystemLobbyServer, lobby+"@"+sysOldDigest),
).Build()
image := func(name string) string {
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
t.Fatal(err)
}
return ms.Spec.Image
}
for name, want := range map[string]string{
naming.SystemLoginServer: limbo + "@" + sysNewDigest,
naming.SystemLobbyServer: lobby + "@" + sysNewDigest,
} {
o, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
if err != nil || o.err != nil || !o.updated {
t.Fatalf("%s: outcome=%+v err=%v, want pinned", name, o, err)
}
if got := image(name); got != want {
t.Errorf("%s image = %q, want %q", name, got, want)
}
again, err := pinSystemServerImage(ctx, cl, "minecraft", name, res)
if err != nil || again.updated || again.skipped != "already runs "+want {
t.Errorf("%s second pass = %+v, %v; want already runs %s", name, again, err, want)
}
}
}
func TestPinSystemServerImageLeavesOthersAlone(t *testing.T) {
ctx := context.Background()
res := systemPinRegistry(t)
pin := func(t *testing.T, ms *v1alpha1.MinecraftServer) (systemServerOutcome, string) {
t.Helper()
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(ms).Build()
o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
if err != nil {
t.Fatal(err)
}
var got v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKeyFromObject(ms), &got); err != nil {
t.Fatal(err)
}
return o, got.Spec.Image
}
t.Run("external image", func(t *testing.T) {
o, img := pin(t, systemServer(naming.SystemLoginServer, "docker.io/example/limbo:1.2"))
if o.err != nil || o.updated || img != "docker.io/example/limbo:1.2" ||
o.skipped != "runs docker.io/example/limbo:1.2, which names no platform registry tag to follow; left alone" {
t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
}
})
t.Run("digest without a tag", func(t *testing.T) {
ref := defaultRegistryURL + "/felis/limbo@" + sysOldDigest
o, img := pin(t, systemServer(naming.SystemLoginServer, ref))
if o.err != nil || o.updated || img != ref {
t.Fatalf("outcome=%+v image=%q, want left alone", o, img)
}
})
t.Run("not a system server", func(t *testing.T) {
ms := systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo")
ms.Labels = nil
o, img := pin(t, ms)
if o.err == nil || img != defaultRegistryURL+"/felis/limbo:demo" {
t.Fatalf("outcome=%+v image=%q, want refused", o, img)
}
})
t.Run("tag the registry lost", func(t *testing.T) {
o, img := pin(t, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:gone"))
if o.err == nil || img != defaultRegistryURL+"/felis/limbo:gone" {
t.Fatalf("outcome=%+v image=%q, want an error the installer falls back on", o, img)
}
})
t.Run("absent", func(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).Build()
o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
if err != nil || o.err != nil || o.skipped != "not present yet; sudo felis setup creates it" {
t.Fatalf("outcome=%+v err=%v, want a skip", o, err)
}
})
t.Run("retargeted while pinning", func(t *testing.T) {
cl, _ := racingClient(t, func(ctx context.Context, c client.WithWatch) {
ms := getLogin(t, c)
ms.Spec.Image = "docker.io/example/limbo:1.2"
if err := c.Update(ctx, ms); err != nil {
t.Fatal(err)
}
}, systemServer(naming.SystemLoginServer, defaultRegistryURL+"/felis/limbo:demo"))
o, err := pinSystemServerImage(ctx, cl, "minecraft", naming.SystemLoginServer, res)
if err != nil || o.err != nil || o.updated {
t.Fatalf("outcome=%+v err=%v, want nothing written", o, err)
}
if img := getLogin(t, cl).Spec.Image; img != "docker.io/example/limbo:1.2" {
t.Errorf("image = %q, want the admin's retarget kept", img)
}
})
}
// --system names one of the two system servers; anything else is a usage error
// before any cluster is touched.
func TestPinImagesSystemFlagTakesOnlySystemServers(t *testing.T) {
var stdout, stderr strings.Builder
if code := cmdPinImages([]string{"--system", "survival"}, &stdout, &stderr); code != 2 {
t.Fatalf("exit = %d, want 2", code)
}
if got := stderr.String(); got != "felis pin-images: --system takes login or lobby, not \"survival\"\n" {
t.Errorf("stderr = %q", got)
}
}
// An idle setting the panel saves while converge fills the default is kept.
func TestConvergeIdleKeepsAConcurrentPanelEdit(t *testing.T) {
ctx := context.Background()
srv := &v1alpha1.MinecraftServer{}
srv.Name, srv.Namespace = "survival", "minecraft"
cl, writes := racingClient(t, func(ctx context.Context, c client.WithWatch) {
var ms v1alpha1.MinecraftServer
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
t.Fatal(err)
}
ms.Spec.Idle = v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}
if err := c.Update(ctx, &ms); err != nil {
t.Fatal(err)
}
}, srv)
if out := convergeUserServerIdle(ctx, cl, "minecraft"); len(out) != 0 {
t.Fatalf("outcomes = %+v, want none: the server has a setting by the time converge writes", out)
}
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
t.Fatal(err)
}
if want := (v1alpha1.IdleSpec{AutoStopEnabled: false, EmptySecondsBeforeStop: 1800}); ms.Spec.Idle != want {
t.Errorf("idle = %+v, want the panel's %+v", ms.Spec.Idle, want)
}
if *writes != 1 {
t.Errorf("writes = %d, want 1 (the conflicted attempt; the retry sends nothing)", *writes)
}
}
cmd/felis/context_max_test.go
-57
View File
@@ -1,57 +0,0 @@
package main
import (
"os"
"path/filepath"
"testing"
"felis.lolicon.best/internal/config"
)
func TestContextMaxBytes(t *testing.T) {
cases := []struct {
name string
reg config.RegistryConfig
auth config.AuthConfig
want int64
wantErr bool
}{
{name: "direct install keeps the package default", want: 0},
// The panel uploads in parts under the edge's 100 MB body limit, so the
// Cloudflare edge keeps the full default.
{name: "the Cloudflare edge keeps the package default", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 0},
{name: "CF-Connecting-IP keeps the package default", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 0},
{name: "explicit value behind the edge", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, err := contextMaxBytes(&config.Config{Registry: tc.reg, Auth: tc.auth})
if (err != nil) != tc.wantErr {
t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
}
if got != tc.want {
t.Fatalf("contextMaxBytes = %d, want %d", got, tc.want)
}
})
}
}
func TestUploadPartsDir(t *testing.T) {
if got := uploadPartsDir("/var/lib/felis/uploads"); got != "/var/lib/felis/uploads/.parts" {
t.Errorf("local store: parts dir = %q, want beside the contexts", got)
}
if got := uploadPartsDir("file:///srv/uploads"); got != "/srv/uploads/.parts" {
t.Errorf("file:// store: parts dir = %q, want /srv/uploads/.parts", got)
}
// This machine has no /var/lib/felis/uploads mount, so an s3:// store falls
// back to the temp dir.
if _, err := os.Stat("/var/lib/felis/uploads"); err == nil {
t.Skip("/var/lib/felis/uploads exists here")
}
if got := uploadPartsDir("s3://bucket/uploads"); got != filepath.Join(os.TempDir(), "felis-upload-parts") {
t.Errorf("s3 store without the uploads mount: parts dir = %q", got)
}
}
cmd/felis/converge.go
+4 -66
View File
@@ -11,14 +11,12 @@ import (
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// cmdConverge is the explicit convergence pass over already-installed system
// servers (#1), plus the idle-stop default for user servers that predate it, and
// with -user-rcon their RCON block (#3). Provisioning is create-if-absent, so a field the desired spec
// servers (#1), plus the idle-stop default for user servers that predate it. Provisioning is create-if-absent, so a field the desired spec
// gained after an install (spec.rcon, spec.startup.healthHTTPPort, a derived env
// key) never reaches the existing CR — and nothing says so. This command fills
// exactly those zero-value fields; see convergeSystemServers for the full contract
@@ -31,7 +29,6 @@ func cmdConverge(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("converge", flag.ContinueOnError)
fs.SetOutput(stderr)
cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
userRcon := fs.Bool("user-rcon", false, "also turn RCON on for user servers created before it was the default")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
@@ -62,7 +59,6 @@ func cmdConverge(args []string, stdout, stderr io.Writer) int {
defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
outcomes = append(outcomes, convergeUserServerIdle(context.Background(), cl, cfg.K8s.Namespace)...)
outcomes = append(outcomes, convergeUserServerRcon(context.Background(), cl, cfg.K8s.Namespace, *userRcon)...)
fmt.Fprintln(stdout, "felis converge: filling fields an installed server predates (operator-set values are never overwritten):")
exit := 0
@@ -96,72 +92,14 @@ func convergeUserServerIdle(ctx context.Context, cl client.Client, namespace str
if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
continue
}
// Re-checked on the copy each attempt reads: an idle setting the panel saved
// meanwhile is the user's, and the default must not land over it.
changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
if ms.Spec.Idle != (v1alpha1.IdleSpec{}) {
return false
}
ms.Spec.Idle = v1alpha1.DefaultIdle()
return true
})
if err != nil {
patch := client.MergeFrom(ms.DeepCopy())
ms.Spec.Idle = v1alpha1.DefaultIdle()
if err := cl.Patch(ctx, ms, patch); err != nil {
out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
continue
}
if !changed {
continue
}
out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true,
changes: []string{fmt.Sprintf("spec.idle (stop after %ds empty)", v1alpha1.DefaultEmptySecondsBeforeStop)}})
}
return out
}
// convergeUserServerRcon handles user servers created before RCON was part of every
// new server (694e3cb): spec.rcon entirely unset. Such a server has a dead console,
// reports nobody online, and never idles out, because all three ride RCON.
//
// Only with fill does it turn RCON on, with the same block CreateServer writes
// today; without it each such server gets a line saying so. The fill is opt-in
// because the operator gates readiness on the RCON probe: a server whose image does
// not open the listener RCON_PASSWORD asks for would sit in Starting until it is
// marked Failed. Felis's own paper and lobby images open it; an image a user brought
// may not, and only the operator running this can tell. A server that already
// carries any RCON setting (on or off) is left alone and produces no line.
func convergeUserServerRcon(ctx context.Context, cl client.Client, namespace string, fill bool) []systemServerOutcome {
var list v1alpha1.MinecraftServerList
if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
return []systemServerOutcome{{name: "user servers", err: fmt.Errorf("list servers: %w", err)}}
}
var out []systemServerOutcome
for i := range list.Items {
ms := &list.Items[i]
if ms.Labels[v1alpha1.LabelSystemRole] != "" || ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
continue
}
if !fill {
out = append(out, systemServerOutcome{name: ms.Name, available: true,
skipped: "no RCON (console, online count and idle stop are off); once its image serves RCON, sudo felis converge -user-rcon turns it on"})
continue
}
changed, err := patchOnConflictRetry(ctx, cl, ms, func() bool {
if ms.Spec.Rcon != (v1alpha1.RconSpec{}) {
return false
}
ms.Spec.Rcon = v1alpha1.RconSpec{
Enabled: true,
SecretRef: v1alpha1.SecretKeyRef{Name: naming.RconSecretName(ms.Name), Key: naming.RconSecretKey},
}
return true
})
if err != nil {
out = append(out, systemServerOutcome{name: ms.Name, err: fmt.Errorf("converge %s: %w", ms.Name, err)})
continue
}
if changed {
out = append(out, systemServerOutcome{name: ms.Name, available: true, updated: true, changes: []string{"spec.rcon"}})
}
}
return out
}
cmd/felis/converge_test.go
-64
View File
@@ -229,67 +229,3 @@ func TestConvergeUserServerIdle(t *testing.T) {
t.Fatalf("second pass = %+v, want nothing to do", again)
}
}
// TestConvergeUserServerRcon reports a user server with no RCON block at all and
// fills it only when asked, with the block CreateServer writes. RCON turned off on
// purpose, a server with its own secret, and a system server stay as they are and
// produce no line.
func TestConvergeUserServerRcon(t *testing.T) {
scheme := newSystemServerScheme(t)
ctx := context.Background()
mk := func(name string, rcon v1alpha1.RconSpec, role string) *v1alpha1.MinecraftServer {
ms := &v1alpha1.MinecraftServer{}
ms.Name, ms.Namespace = name, "minecraft"
ms.Spec.Rcon = rcon
if role != "" {
ms.Labels = map[string]string{v1alpha1.LabelSystemRole: role}
}
return ms
}
own := v1alpha1.RconSpec{Enabled: true, Port: 25580, SecretRef: v1alpha1.SecretKeyRef{Name: "own", Key: "pw"}}
off := v1alpha1.RconSpec{SecretRef: v1alpha1.SecretKeyRef{Name: "rcon-off", Key: naming.RconSecretKey}}
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
mk("demo", v1alpha1.RconSpec{}, ""),
mk("off", off, ""),
mk("own", own, ""),
mk(naming.SystemLobbyServer, v1alpha1.RconSpec{}, naming.SystemLobbyServer),
).Build()
get := func(name string) v1alpha1.RconSpec {
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
t.Fatalf("get %s: %v", name, err)
}
return ms.Spec.Rcon
}
report := convergeUserServerRcon(ctx, cl, "minecraft", false)
if len(report) != 1 || report[0].name != "demo" || report[0].updated || report[0].err != nil ||
!strings.Contains(report[0].skipped, "-user-rcon") {
t.Fatalf("report = %+v, want one skipped line for demo naming -user-rcon", report)
}
if got := get("demo"); got != (v1alpha1.RconSpec{}) {
t.Fatalf("the report-only pass wrote demo's rcon: %+v", got)
}
filled := convergeUserServerRcon(ctx, cl, "minecraft", true)
if len(filled) != 1 || filled[0].name != "demo" || !filled[0].updated || filled[0].err != nil {
t.Fatalf("fill = %+v, want exactly one update for demo", filled)
}
want := map[string]v1alpha1.RconSpec{
"demo": {Enabled: true, SecretRef: v1alpha1.SecretKeyRef{
Name: naming.RconSecretName("demo"), Key: naming.RconSecretKey}},
"off": off,
"own": own,
naming.SystemLobbyServer: {},
}
for name, rcon := range want {
if got := get(name); got != rcon {
t.Errorf("%s rcon = %+v, want %+v", name, got, rcon)
}
}
for _, fill := range []bool{false, true} {
if again := convergeUserServerRcon(ctx, cl, "minecraft", fill); len(again) != 0 {
t.Fatalf("second pass (fill=%v) = %+v, want nothing to do", fill, again)
}
}
}
cmd/felis/db.go
+17 -184
View File
@@ -7,7 +7,6 @@ import (
"flag"
"fmt"
"io"
neturl "net/url"
"os"
"os/exec"
"path/filepath"
@@ -16,8 +15,6 @@ import (
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/retention"
)
const dbUsage = `usage:
@@ -27,7 +24,6 @@ const dbUsage = `usage:
felis db verify [-dir dir] <bundle>
felis db list [-dir dir]
felis db check [-dir dir] [-max-age 26h]
felis db audit-export [-config path] [-since date] [-until date] [-out file]
`
// defaultKeep is how many bundles of a label a backup leaves behind. Manual
@@ -36,7 +32,6 @@ var defaultKeep = map[string]int{
dbbackup.LabelDaily: 14,
dbbackup.LabelPreMigrate: 10,
dbbackup.LabelPreRestore: 5,
dbbackup.LabelOffsite: 1,
}
// cmdDB implements `felis db`: logical backups of the control-plane database
@@ -63,8 +58,6 @@ func cmdDB(args []string, stdout, stderr io.Writer) int {
return dbList(fs, dir, rest, stdout, stderr)
case "check":
return dbCheck(fs, dir, rest, stdout, stderr)
case "audit-export":
return dbAuditExport(fs, rest, stdout, stderr)
case "-h", "--help", "help":
fmt.Fprint(stdout, dbUsage)
return 0
@@ -95,49 +88,17 @@ func parseWithArg(fs *flag.FlagSet, args []string) (string, bool) {
}
func dbDatabaseURL(path string) (string, error) {
db, err := dbDatabase(path)
return db.URL, err
}
func dbDatabase(path string) (config.DatabaseConfig, error) {
cfg, err := config.Load(path)
if err != nil {
return config.DatabaseConfig{}, err
return "", err
}
return cfg.Database, nil
}
// dbTools places pg_dump, pg_restore and psql. The installer's database runs in
// k3s and the host has no PostgreSQL client, so when the host config names the
// [database] deployment the tools run in its postgres container over the
// container's socket, as the URL's role on the URL's database. Otherwise they
// come from PATH and connect with the URL.
func dbTools(db config.DatabaseConfig) (dbbackup.Tools, error) {
if db.Deployment == "" {
return dbbackup.Tools{}, nil
}
ns, name, _ := strings.Cut(db.Deployment, "/")
u, err := neturl.Parse(db.URL)
if err != nil || u.User == nil || u.User.Username() == "" || strings.TrimPrefix(u.Path, "/") == "" {
return dbbackup.Tools{}, errors.New("[database] url must name the role and the database to run the tools in the database's pod")
}
return dbbackup.Tools{
Exec: []string{"k3s", "kubectl", "exec", "-i", "-n", ns, "deploy/" + name, "-c", platform.PostgresContainer, "--"},
Conn: fmt.Sprintf("host=%s port=%d dbname=%s user=%s connect_timeout=15",
platform.PostgresSocketDir, platform.PostgresPort,
libpqQuote(strings.TrimPrefix(u.Path, "/")), libpqQuote(u.User.Username())),
}, nil
}
// libpqQuote renders v as a single-quoted libpq connection-string value.
func libpqQuote(v string) string {
return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(v) + "'"
return cfg.Database.URL, nil
}
func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
label := fs.String("label", dbbackup.LabelManual, "bundle label; daily/pre-migrate/pre-restore bundles are pruned, manual ones never")
keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, offsite 1, manual all)")
keep := fs.Int("keep", -1, "bundles of this label to keep (default: daily 14, pre-migrate 10, pre-restore 5, manual all)")
stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory to bundle ("" for none)`)
noServers := fs.Bool("no-servers", false, "leave the MinecraftServer objects out of the bundle")
metrics := fs.String("metrics-file", "", "node-exporter textfile to rewrite on success (e.g. /var/lib/node_exporter/textfile_collector/felis_db_backup.prom)")
@@ -148,12 +109,7 @@ func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
fmt.Fprint(stderr, dbUsage)
return 2
}
db, err := dbDatabase(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db backup: %v\n", err)
return 1
}
tools, err := dbTools(db)
url, err := dbDatabaseURL(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db backup: %v\n", err)
return 1
@@ -162,7 +118,7 @@ func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
*keep = defaultKeep[*label]
}
o := dbbackup.BackupOptions{
DatabaseURL: db.URL, Tools: tools, Dir: *dir, Label: *label, Keep: *keep,
DatabaseURL: url, Dir: *dir, Label: *label, Keep: *keep,
StateDir: *stateDir, Version: resolvedVersion(), Log: stderr,
MetricsFile: *metrics, Record: true,
}
@@ -172,14 +128,6 @@ func dbBackup(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
path, err := dbbackup.Backup(ctx, o)
if errors.Is(err, dbbackup.ErrServersMissing) {
// The bundle is on disk and holds the database; the exit status fails
// the timer's run so the gap shows in systemctl and the journal, and
// the panel and the watchdog read it from the record and the manifest.
fmt.Fprintf(stdout, "felis db backup: wrote %s\n", path)
fmt.Fprintf(stderr, "felis db backup: %v\n a restore from %s brings back the database but no servers; check `k3s kubectl get minecraftservers -A`, then run `felis db backup` again\n", err, filepath.Base(path))
return 1
}
if err != nil {
fmt.Fprintf(stderr, "felis db backup: %v\n", err)
return 1
@@ -220,17 +168,12 @@ func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.W
return 1
}
if !*yes {
fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d, holding %s).\n",
filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion, m.Counts.String())
fmt.Fprintf(stderr, "felis db restore: this replaces every table in the felis database with %s (%s, taken %s, schema %d).\n",
filepath.Base(bundle), m.Label, m.CreatedAt.Format(time.RFC3339), m.SchemaVersion)
fmt.Fprintln(stderr, "Scale felis-api and felis-operator to 0 first, then re-run with -yes.")
return 2
}
db, err := dbDatabase(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db restore: %v\n", err)
return 1
}
tools, err := dbTools(db)
url, err := dbDatabaseURL(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db restore: %v\n", err)
return 1
@@ -238,7 +181,7 @@ func dbRestore(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.W
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
defer cancel()
_, safety, err := dbbackup.Restore(ctx, dbbackup.RestoreOptions{
DatabaseURL: db.URL, Tools: tools, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
DatabaseURL: url, Bundle: bundle, Dir: *dir, Force: *force, SkipSafetyBackup: *noSafety,
Safety: dbbackup.BackupOptions{Keep: defaultKeep[dbbackup.LabelPreRestore], StateDir: *stateDir,
Version: resolvedVersion(), ExportServers: exportMinecraftServers},
Log: stderr,
@@ -277,9 +220,8 @@ func dbVerify(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wr
fmt.Fprintf(stderr, "felis db verify: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "%s: ok\n taken %s (%s)\n felis %s\n schema %d\n holds %s\n %s\n",
filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion,
m.Counts.String(), orUnknown(m.PGDumpVersion))
fmt.Fprintf(stdout, "%s: ok\n taken %s (%s)\n felis %s\n schema %d\n %s\n",
filepath.Base(bundle), m.CreatedAt.Format(time.RFC3339), m.Label, orUnknown(m.FelisVersion), m.SchemaVersion, orUnknown(m.PGDumpVersion))
for _, f := range m.Files {
if f.Link != "" {
fmt.Fprintf(stdout, " %-40s -> %s\n", f.Name, f.Link)
@@ -320,88 +262,6 @@ func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writ
return 0
}
// dbAuditExport writes audit rows to a file (or stdout) as JSON lines, so an
// install can keep them past [audit] retention, after which felis-api deletes them.
func dbAuditExport(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
sinceFlag := fs.String("since", "", "first day (or RFC 3339 instant) to export, inclusive; empty starts at the oldest row")
untilFlag := fs.String("until", "", "day (or RFC 3339 instant) to stop before, exclusive; empty runs to the newest row")
out := fs.String("out", "", "file to write (created 0600, never overwritten); empty writes to stdout")
if err := fs.Parse(args); err != nil {
return 2
}
if fs.NArg() > 0 {
fmt.Fprint(stderr, dbUsage)
return 2
}
since, err := parseExportBound(*sinceFlag)
if err != nil {
fmt.Fprintf(stderr, "felis db audit-export: -since: %v\n", err)
return 2
}
until, err := parseExportBound(*untilFlag)
if err != nil {
fmt.Fprintf(stderr, "felis db audit-export: -until: %v\n", err)
return 2
}
if !since.IsZero() && !until.IsZero() && !until.After(since) {
fmt.Fprintf(stderr, "felis db audit-export: -until %s is not after -since %s\n", *untilFlag, *sinceFlag)
return 2
}
url, err := dbDatabaseURL(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
return 1
}
w := stdout
var f *os.File
if *out != "" {
if f, err = os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600); err != nil {
fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
return 1
}
w = f
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
n, err := exportAudit(ctx, url, since, until, w)
if f != nil {
if cerr := f.Close(); err == nil {
err = cerr
}
}
if err != nil {
fmt.Fprintf(stderr, "felis db audit-export: %v (%d rows written)\n", err, n)
return 1
}
fmt.Fprintf(stderr, "felis db audit-export: %d audit rows written\n", n)
return 0
}
func exportAudit(ctx context.Context, url string, since, until time.Time, w io.Writer) (int, error) {
drv, err := openStore(ctx, url, false)
if err != nil {
return 0, fmt.Errorf("open database: %w", err)
}
defer drv.Close()
return retention.ExportAudit(ctx, drv.DB(), since, until, w)
}
// parseExportBound reads a -since/-until value: a day (midnight UTC) or an
// RFC 3339 instant; empty is an open bound.
func parseExportBound(v string) (time.Time, error) {
if v == "" {
return time.Time{}, nil
}
if t, err := time.Parse(time.DateOnly, v); err == nil {
return t, nil
}
if t, err := time.Parse(time.RFC3339, v); err == nil {
return t.UTC(), nil
}
return time.Time{}, fmt.Errorf("%q is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)", v)
}
func humanBytes(n int64) string {
const unit = 1024
if n < unit {
@@ -415,11 +275,10 @@ func humanBytes(n int64) string {
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}
// dbCheck is the freshness probe: exit 1 when the newest daily bundle is
// missing or older than -max-age, for a monitor or the break-glass console to
// act on.
// dbCheck is the freshness probe: exit 1 when the newest bundle is missing or
// older than -max-age, for a monitor or the break-glass console to act on.
func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int {
maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest daily bundle")
maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest bundle")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -428,40 +287,14 @@ func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wri
fmt.Fprintf(stderr, "felis db check: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "felis db check: ok, newest daily backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
fmt.Fprintf(stdout, "felis db check: ok, newest backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created)))
return 0
}
// serverExportTries and serverExportRetry are how long a backup waits out a
// cluster that is briefly away (an apiserver restart) before its bundle goes
// without the MinecraftServer objects.
const serverExportTries = 3
var serverExportRetry = 10 * time.Second
// exportMinecraftServers is dbbackup's ExportServers on the host: the
// MinecraftServer objects through k3s kubectl, tried serverExportTries times.
func exportMinecraftServers(ctx context.Context) ([]byte, error) {
for try := 1; ; try++ {
out, err := getMinecraftServers(ctx)
if err == nil {
return out, nil
}
if try == serverExportTries {
return nil, fmt.Errorf("%w (tried %d times)", err, try)
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("%w (tried %d times)", err, try)
case <-time.After(serverExportRetry):
}
}
}
// getMinecraftServers reads every MinecraftServer through the host's k3s
// exportMinecraftServers reads every MinecraftServer through the host's k3s
// kubectl and strips what the API server owns, so the result can be fed back
// with `kubectl apply -f` on a rebuilt cluster.
func getMinecraftServers(ctx context.Context) ([]byte, error) {
func exportMinecraftServers(ctx context.Context) ([]byte, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
// Output, not the CombinedOutput kubectlOutput uses: a deprecation warning
cmd/felis/db_test.go
+6 -422
View File
@@ -1,20 +1,14 @@
package main
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"flag"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/store"
)
@@ -30,63 +24,13 @@ func TestDBUsage(t *testing.T) {
}
}
func TestDBVerifySaysWhatTheBundleHolds(t *testing.T) {
dir := newPodRig(t)
cfg := podConfig(t, dir)
bundles := filepath.Join(dir, "bundles")
var out, errBuf bytes.Buffer
if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
t.Fatalf("backup: exit %d: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
out.Reset()
if code := run([]string{"db", "verify", "-dir", bundles, filepath.Base(bundle)}, &out, &errBuf); code != 0 {
t.Fatalf("verify: exit %d: %s", code, errBuf.String())
}
for _, want := range []string{filepath.Base(bundle) + ": ok", "schema 3", "holds 4 accounts, 2 servers", "pg_dump (PostgreSQL) 18.6"} {
if !strings.Contains(out.String(), want) {
t.Errorf("verify output lacks %q:\n%s", want, out.String())
}
}
}
// TestDBRestoreNeedsYes: without -yes a restore describes the bundle and stops
// before anything reaches the database, even with -force and
// -no-safety-backup, which would otherwise let the replay run at once.
func TestDBRestoreNeedsYes(t *testing.T) {
dir := newPodRig(t)
cfg := podConfig(t, dir)
bundles := filepath.Join(dir, "bundles")
// A bundle that does not exist fails verification (1) before -yes matters;
// the -yes gate itself is exercised against a real bundle in internal/dbbackup
// and on the VM. Here: the refusal path never reaches the config or database.
var out, errBuf bytes.Buffer
if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
t.Fatalf("backup: exit %d: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
podRuns(t, dir)
out.Reset()
errBuf.Reset()
code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-force", "-no-safety-backup", filepath.Base(bundle)}, &out, &errBuf)
if code != 2 {
t.Fatalf("exit %d, want 2; stderr %q", code, errBuf.String())
}
if want := filepath.Base(bundle) + " (manual, taken "; !strings.Contains(errBuf.String(), want) || !strings.Contains(errBuf.String(), "schema 3, holding 4 accounts, 2 servers).") {
t.Errorf("stderr %q does not describe the bundle", errBuf.String())
}
if !strings.Contains(errBuf.String(), "re-run with -yes") {
t.Errorf("stderr %q does not say how to go on", errBuf.String())
}
if argv, err := os.ReadFile(filepath.Join(dir, "k3s.args")); err == nil {
t.Errorf("a restore without -yes ran in the database pod:\n%s", argv)
}
// A bundle that does not verify is refused before -yes is weighed.
errBuf.Reset()
if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "missing.tar"}, &out, &errBuf); code != 1 {
t.Errorf("missing bundle: exit %d, want 1; stderr %q", code, errBuf.String())
}
if _, err := os.Stat(filepath.Join(dir, "k3s.args")); err == nil {
t.Error("a missing bundle reached the database pod")
if code := run([]string{"db", "restore", "-dir", t.TempDir(), "missing.tar"}, &out, &errBuf); code != 1 {
t.Fatalf("exit %d, stderr %q", code, errBuf.String())
}
}
@@ -196,7 +140,7 @@ func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
{"up to date", map[int]struct{}{1: {}, 2: {}}, false},
{"pending on a populated database", map[int]struct{}{1: {}}, true},
} {
path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, config.DatabaseConfig{URL: badURL}, t.TempDir(), io.Discard)
path, err := preMigrateBackup(context.Background(), appliedDriver{done: tc.done}, ms, badURL, t.TempDir(), io.Discard)
if attempted := err != nil; attempted != tc.attempt {
t.Errorf("%s: attempted = %v (err %v), want %v", tc.name, attempted, err, tc.attempt)
}
@@ -205,363 +149,3 @@ func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
}
}
}
// audit-export takes a day or an RFC 3339 instant for each bound, and refuses a
// malformed or inverted window before it opens the config or the database.
func TestAuditExportBounds(t *testing.T) {
for _, tc := range []struct{ in, want string }{
{"", "0001-01-01T00:00:00Z"},
{"2026-01-31", "2026-01-31T00:00:00Z"},
{"2026-01-31T12:30:00+08:00", "2026-01-31T04:30:00Z"},
} {
got, err := parseExportBound(tc.in)
if err != nil || got.Format(time.RFC3339) != tc.want {
t.Errorf("parseExportBound(%q) = %v, %v; want %s", tc.in, got, err, tc.want)
}
}
if _, err := parseExportBound("31/01/2026"); err == nil || err.Error() != `"31/01/2026" is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)` {
t.Errorf("parseExportBound(31/01/2026) err = %v", err)
}
for _, tc := range []struct {
args []string
wantErr string
}{
{[]string{"db", "audit-export", "-since", "yesterday"}, `felis db audit-export: -since: "yesterday" is neither`},
{[]string{"db", "audit-export", "-until", "2026-13-01"}, `felis db audit-export: -until: "2026-13-01" is neither`},
{[]string{"db", "audit-export", "-since", "2026-02-01", "-until", "2026-02-01"}, "felis db audit-export: -until 2026-02-01 is not after -since 2026-02-01"},
{[]string{"db", "audit-export", "extra"}, "felis db audit-export [-config path]"},
} {
var out, errBuf bytes.Buffer
code := run(append(tc.args, "-config", "/nonexistent/felis.toml"), &out, &errBuf)
if code != 2 || !strings.Contains(errBuf.String(), tc.wantErr) {
t.Errorf("%v: exit %d, stderr %q; want 2 and %q", tc.args, code, errBuf.String(), tc.wantErr)
}
}
}
// podK3s stands in for `k3s kubectl exec ... --`: it logs its argv and runs the
// command after -- from the "container" directory, which is the only place the
// PostgreSQL tools exist, as on an installed host. `kubectl get` lists one
// MinecraftServer, logged to k3s.get, and refuses its first N calls while
// servers_fail holds N.
const podK3s = `#!/bin/sh
if [ "$1" = kubectl ] && [ "$2" = get ]; then
printf '%s\n' "$*" >> "$FAKE_DIR/k3s.get"
n=$(/usr/bin/wc -l < "$FAKE_DIR/k3s.get")
if [ -f "$FAKE_DIR/servers_fail" ] && [ "$n" -le "$(/bin/cat "$FAKE_DIR/servers_fail")" ]; then
echo "The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?" >&2
exit 1
fi
echo '{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"felis.lolicon.best/v1alpha1","kind":"MinecraftServer","metadata":{"name":"lobby","namespace":"felis-servers","uid":"u-1"},"spec":{"type":"PAPER"},"status":{"phase":"Running"}}]}'
exit 0
fi
printf '%s\n' "$*" >> "$FAKE_DIR/k3s.args"
while [ $# -gt 0 ] && [ "$1" != "--" ]; do shift; done
shift
tool=$1; shift
exec /usr/bin/env -i FAKE_DIR="$FAKE_DIR" PATH=/usr/bin:/bin "$FAKE_DIR/container/$tool" "$@"
`
var podTools = map[string]string{
"pg_dump": `#!/bin/sh
case "$1" in --version) echo "pg_dump (PostgreSQL) 18.6"; exit 0 ;; esac
printf 'PGDMP-fake-archive'
`,
"pg_restore": `#!/bin/sh
cat > /dev/null
`,
"psql": `#!/bin/sh
for a in "$@"; do case "$a" in *"FROM users"*) echo "4|2"; exit 0 ;; esac; done
for a in "$@"; do [ "$a" = "-c" ] && { echo 3; exit 0; }; done
cat > /dev/null
`,
}
const podPassword = "pw-must-stay-on-the-host"
// podDB is the host config's [database] on an installed host.
var podDB = config.DatabaseConfig{
URL: "postgres://felis:" + podPassword + "@127.0.0.1:15432/felis?sslmode=disable",
Deployment: "felis/felis-postgres",
}
const podExecPrefix = "kubectl exec -i -n felis deploy/felis-postgres -c postgres -- "
// newPodRig puts the fake k3s on PATH, alone, and returns the directory its
// k3s.args log lands in.
func newPodRig(t *testing.T) string {
t.Helper()
dir := t.TempDir()
bin := filepath.Join(dir, "bin")
container := filepath.Join(dir, "container")
for _, d := range []string{bin, container} {
if err := os.Mkdir(d, 0o755); err != nil {
t.Fatal(err)
}
}
writeTestFile(t, filepath.Join(bin, "k3s"), podK3s, 0o755)
for name, body := range podTools {
writeTestFile(t, filepath.Join(container, name), body, 0o755)
}
t.Setenv("PATH", bin)
t.Setenv("FAKE_DIR", dir)
return dir
}
// podRuns returns what the fake k3s ran since the last call, failing on any
// run outside the database container or with the password on its command
// line (visible to every local user in ps).
func podRuns(t *testing.T, dir string) []string {
t.Helper()
log := filepath.Join(dir, "k3s.args")
argv, err := os.ReadFile(log)
if err != nil {
t.Fatalf("nothing ran through k3s: %v", err)
}
os.Remove(log)
var runs []string
for _, line := range strings.Split(strings.TrimSpace(string(argv)), "\n") {
if !strings.HasPrefix(line, podExecPrefix) {
t.Errorf("k3s ran %q, want everything under %q", line, podExecPrefix)
}
if strings.Contains(line, podPassword) {
t.Errorf("the password crossed into the pod on a command line: %q", line)
}
runs = append(runs, strings.TrimPrefix(line, podExecPrefix))
}
return runs
}
// podConfig writes an installed host's felis.toml, [database] pointing at the
// pod, into dir.
func podConfig(t *testing.T, dir string) string {
t.Helper()
toml := strings.Replace(installerTOML("example.com", "127.0.0.1"),
`url = "postgres://felis:[email protected]:5432/felis?sslmode=disable"`,
`url = "`+podDB.URL+`"
deployment = "`+podDB.Deployment+`"`, 1)
cfg := filepath.Join(dir, "felis.toml")
writeTestFile(t, cfg, toml, 0o600)
return cfg
}
func ranIn(runs []string, prefix string) bool {
for _, r := range runs {
if strings.HasPrefix(r, prefix) {
return true
}
}
return false
}
// TestDBBackupAndRestoreRunTheToolsInTheDatabasePod: on an installed host the
// database is a k3s Deployment and no PostgreSQL client exists outside it, so
// `felis db backup` and `restore` must reach the tools through kubectl exec,
// over the pod's socket, and without putting the role's password on a command
// line.
func TestDBBackupAndRestoreRunTheToolsInTheDatabasePod(t *testing.T) {
dir := newPodRig(t)
cfg := podConfig(t, dir)
var out, errBuf bytes.Buffer
bundles := filepath.Join(dir, "bundles")
if code := run([]string{"db", "backup", "-config", cfg, "-dir", bundles, "-state-dir", "", "-no-servers"}, &out, &errBuf); code != 0 {
t.Fatalf("backup: exit %d: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
if _, err := dbbackupVerify(bundle); err != nil {
t.Fatalf("the bundle does not verify: %v", err)
}
runs := podRuns(t, dir)
if !ranIn(runs, "pg_dump --format=custom --no-password --dbname=host=/var/run/postgresql port=5432 dbname='felis' user='felis'") {
t.Errorf("pg_dump did not dump over the pod's socket as felis on felis: %q", runs)
}
out.Reset()
errBuf.Reset()
if code := run([]string{"db", "restore", "-config", cfg, "-dir", bundles, "-yes", "-force", "-no-safety-backup", bundle}, &out, &errBuf); code != 0 {
t.Fatalf("restore: exit %d: %s", code, errBuf.String())
}
runs = podRuns(t, dir)
if !ranIn(runs, "pg_restore --no-owner --no-privileges --file=-") || !ranIn(runs, "psql -X -q -w -v ON_ERROR_STOP=1 -d host=/var/run/postgresql") {
t.Errorf("the replay did not run in the pod: %q", runs)
}
}
// TestPreMigrateBackupRunsInTheDatabasePod: the snapshot in front of an upgrade
// is the one taken most often, by bootstrap on every rerun.
func TestPreMigrateBackupRunsInTheDatabasePod(t *testing.T) {
dir := newPodRig(t)
ms := []store.Migration{{Version: 1}, {Version: 2}}
// The snapshot also bundles /etc/felis, which a test machine may lack; the
// dump runs first either way.
_, err := preMigrateBackup(context.Background(), appliedDriver{done: map[int]struct{}{1: {}}}, ms, podDB, filepath.Join(dir, "bundles"), io.Discard)
if err != nil && !strings.Contains(err.Error(), "read host state") {
t.Fatalf("snapshot: %v", err)
}
if runs := podRuns(t, dir); !ranIn(runs, "pg_dump --format=custom") {
t.Errorf("pg_dump did not run in the pod: %q", runs)
}
}
func TestDBToolsNeedTheRoleAndDatabase(t *testing.T) {
if tools, err := dbTools(config.DatabaseConfig{URL: "postgres://felis:pw@db:5432/felis"}); err != nil || len(tools.Exec) != 0 {
t.Errorf("no deployment: tools %+v err %v, want the PATH tools", tools, err)
}
for _, u := range []string{"postgres://db:5432/felis", "postgres://felis:pw@db:5432/"} {
if _, err := dbTools(config.DatabaseConfig{URL: u, Deployment: "felis/felis-postgres"}); err == nil {
t.Errorf("%s: no error, want a refusal (the pod connection needs the role and the database)", u)
}
}
tools, err := dbTools(config.DatabaseConfig{URL: `postgres://o%27brien@db/my%20db`, Deployment: "felis/felis-postgres"})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(tools.Conn, `dbname='my db' user='o\'brien'`) {
t.Errorf("Conn = %q, want the values quoted for libpq", tools.Conn)
}
}
var dbbackupVerify = dbbackup.Verify
func noServerExportWait(t *testing.T) {
t.Helper()
old := serverExportRetry
serverExportRetry = 0
t.Cleanup(func() { serverExportRetry = old })
}
// serverGets counts the `kubectl get` calls the fake k3s answered or refused.
func serverGets(dir string) int {
b, _ := os.ReadFile(filepath.Join(dir, "k3s.get"))
return strings.Count(string(b), "\n")
}
// bundleServers returns the bundle's k8s/minecraftservers.json, or nil.
func bundleServers(t *testing.T, bundle string) []byte {
t.Helper()
f, err := os.Open(bundle)
if err != nil {
t.Fatal(err)
}
defer f.Close()
tr := tar.NewReader(f)
for {
h, err := tr.Next()
if err == io.EOF {
return nil
}
if err != nil {
t.Fatal(err)
}
if h.Name == "k8s/minecraftservers.json" {
data, err := io.ReadAll(tr)
if err != nil {
t.Fatal(err)
}
return data
}
}
}
// TestDBBackupWithoutServersFails: when the cluster stays away the daily
// bundle is still written, and `felis db backup` exits 1, so the timer's run
// shows failed, saying a restore from the bundle brings back no servers.
func TestDBBackupWithoutServersFails(t *testing.T) {
noServerExportWait(t)
dir := newPodRig(t)
cfg := podConfig(t, dir)
writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
var out, errBuf bytes.Buffer
code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", "", "-label", "daily"}, &out, &errBuf)
if code != 1 {
t.Fatalf("exit %d, want 1: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
m, err := dbbackupVerify(bundle)
if err != nil {
t.Fatalf("the database must still be bundled: %v", err)
}
if !strings.Contains(m.ServersError, "6443 was refused") || !strings.Contains(m.ServersError, "(tried 3 times)") || bundleServers(t, bundle) != nil {
t.Errorf("manifest servers error = %q", m.ServersError)
}
if n := serverGets(dir); n != serverExportTries {
t.Errorf("export tried %d times, want %d", n, serverExportTries)
}
if msg := errBuf.String(); !strings.Contains(msg, "a restore from "+filepath.Base(bundle)+" brings back the database but no servers") {
t.Errorf("stderr = %q", msg)
}
}
// TestDBBackupRetriesTheServerExport: a cluster back on the last try costs the
// bundle nothing, and what it holds is ready for kubectl apply.
func TestDBBackupRetriesTheServerExport(t *testing.T) {
noServerExportWait(t)
dir := newPodRig(t)
cfg := podConfig(t, dir)
writeTestFile(t, filepath.Join(dir, "servers_fail"), "2", 0o600)
var out, errBuf bytes.Buffer
if code := run([]string{"db", "backup", "-config", cfg, "-dir", filepath.Join(dir, "bundles"), "-state-dir", ""}, &out, &errBuf); code != 0 {
t.Fatalf("exit %d: %s", code, errBuf.String())
}
bundle := strings.TrimSpace(strings.TrimPrefix(out.String(), "felis db backup: wrote "))
servers := string(bundleServers(t, bundle))
if !strings.Contains(servers, `"name": "lobby"`) || strings.Contains(servers, "status") || strings.Contains(servers, "u-1") {
t.Errorf("k8s/minecraftservers.json = %s", servers)
}
if n := serverGets(dir); n != 3 {
t.Errorf("export tried %d times, want 3", n)
}
}
// TestPreMigrateBackupExportsServers: the snapshot every upgrade takes, often
// the newest bundle, carries the MinecraftServer objects too; a cluster that
// is away does not hold back the migration, whose rollback needs the database
// alone.
func TestPreMigrateBackupExportsServers(t *testing.T) {
noServerExportWait(t)
dir := newPodRig(t)
old := preMigrateStateDir
preMigrateStateDir = ""
t.Cleanup(func() { preMigrateStateDir = old })
ms := []store.Migration{{Version: 1}, {Version: 2}}
bundles := filepath.Join(dir, "bundles")
pending := appliedDriver{done: map[int]struct{}{1: {}}}
path, err := preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
if err != nil {
t.Fatalf("snapshot: %v", err)
}
if !strings.Contains(string(bundleServers(t, path)), `"name": "lobby"`) {
t.Errorf("%s holds no MinecraftServer objects", path)
}
writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
path, err = preMigrateBackup(context.Background(), pending, ms, podDB, bundles, io.Discard)
if err != nil || path == "" {
t.Fatalf("snapshot with the cluster away = %q, %v; want the bundle and no error", path, err)
}
if m, err := dbbackupVerify(path); err != nil || m.ServersError == "" || bundleServers(t, path) != nil {
t.Errorf("snapshot with the cluster away: %+v, %v", m, err)
}
}
// TestServerExportStopsWaitingWithTheContext: a backup whose time is up stops
// waiting for the cluster between tries.
func TestServerExportStopsWaitingWithTheContext(t *testing.T) {
dir := newPodRig(t)
writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
// Long enough for the first try to run to its refusal: starting the fake
// k3s on a busy machine can take a few hundred ms. Still far below the
// 10s retry wait, so waiting it out would fail the check below.
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
start := time.Now()
_, err := exportMinecraftServers(ctx)
if err == nil || !strings.Contains(err.Error(), "(tried 1 times)") || serverGets(dir) != 1 {
t.Fatalf("err = %v after %d tries, want the first failure alone", err, serverGets(dir))
}
if took := time.Since(start); took > 5*time.Second {
t.Errorf("took %s, want the context's deadline, not the %s retry wait", took, serverExportRetry)
}
}
cmd/felis/doctor.go
-387
View File
@@ -1,387 +0,0 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/watchdog"
)
// systemdUnitDir is where the installer writes its units.
const systemdUnitDir = "/etc/systemd/system"
// hostCommand runs a host tool (systemctl, journalctl, k3s) and returns its
// stdout. A tool that exits non-zero still returns what it printed:
// `systemctl is-active` prints "inactive" and exits 3.
func hostCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
return exec.CommandContext(ctx, name, args...).Output()
}
// hostServices are the long-running units a full install depends on, checked
// when their unit file is present: k3s runs the cluster, felis-velocity is the
// game proxy, felis-nano the single-binary host that runs without k3s.
var hostServices = []string{"k3s.service", "felis-velocity.service", "felis-nano.service"}
// cmdDoctor runs every check felis watchdog runs, with the settings
// felis-watchdog.service gives it, plus what only the host shows (systemd
// units that failed or stopped, timers that no longer fire, alerts that reach
// no one), and prints them grouped by area with where to look next. It mails
// nothing, pings no heartbeat and leaves the watchdog's state alone: it is
// safe to run at any time, as often as wanted.
func cmdDoctor(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("doctor", flag.ContinueOnError)
fs.SetOutput(stderr)
unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis doctor: run as root (sudo felis doctor): the checks read root-only state under /etc/felis and /var/lib/felis")
return 1
}
host, _ := os.Hostname()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
return runDoctor(ctx, doctorEnv{unitDir: *unitDir, run: hostCommand, now: time.Now(), host: host}, stdout)
}
// doctorEnv is what one doctor run reads the host through.
type doctorEnv struct {
unitDir string
run func(ctx context.Context, name string, args ...string) ([]byte, error)
now time.Time
host string
}
// doctorAreas are the report's headings in order, by the area findingArea
// puts a finding under.
var doctorAreas = []struct{ key, title string }{
{key: "config", title: "configuration"},
{key: "cluster", title: "Kubernetes cluster"},
{key: "postgres", title: "PostgreSQL"},
{key: "proxy", title: "game proxy"},
{key: "db-backup", title: "database backups"},
{key: "offsite", title: "off-site copy"},
{key: "scan-db", title: "build scan database"},
{key: "disk", title: "disk space"},
{key: "memory", title: "memory"},
{key: "k3s-certs", title: "k3s certificates"},
{key: "host-address", title: "node address"},
{key: "clock", title: "clock"},
{key: "systemd", title: "systemd units and timers"},
{key: "alerts", title: "alerting"},
}
func runDoctor(ctx context.Context, env doctorEnv, stdout io.Writer) int {
unit := filepath.Join(env.unitDir, "felis-watchdog.service")
w, found, unitErr := watchdogUnitFlags(unit)
var report watchdog.Report
var notes []string
fmt.Fprintf(stdout, "felis doctor on %s at %s\n", env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
switch {
case unitErr != nil:
report.Findings = append(report.Findings, watchdog.Finding{
Key: "watchdog/unit", Severity: watchdog.Critical,
SummaryEN: fmt.Sprintf("cannot read the watchdog's settings: %v", unitErr),
Hint: "rerun the installer (deploy/bootstrap.sh) to rewrite felis-watchdog.service",
})
fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
case !found:
report.Findings = append(report.Findings, watchdog.Finding{
Key: "watchdog/unit", Severity: watchdog.Critical,
SummaryEN: fmt.Sprintf("%s is not installed: nothing checks this host or mails anyone when it breaks", unit),
Hint: "rerun the installer (deploy/bootstrap.sh), which installs felis-watchdog.timer",
})
fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
default:
fmt.Fprintf(stdout, "checks run as %s runs them (config %s)\n", unit, w.cfgPath)
}
fmt.Fprintln(stdout)
skip := map[string]string{}
cfg, cfgErr := config.Load(w.cfgPath)
if cfgErr != nil {
report.Findings = append(report.Findings, watchdog.Finding{
Key: "config", Severity: watchdog.Critical,
SummaryEN: cfgErr.Error(),
Hint: "the installer writes it (deploy/bootstrap.sh); felis watchdog fails on every run until it loads",
})
// The host's units are read without it; whether alerts reach anyone
// is not known without its relay.
for _, a := range doctorAreas {
if a.key != "config" && a.key != "systemd" {
skip[a.key] = "the configuration did not load"
}
}
} else {
_, owners, ownersErr := watchdogProbes(ctx, w, cfg, env.now, &report)
report.Findings = append(report.Findings, alertReachFindings(cfg, owners, ownersErr)...)
if w.proxyAddr == "" {
skip["proxy"] = "no -proxy-addr"
}
if w.backupDir == "" {
skip["db-backup"] = "no -backup-dir"
}
if !cfg.Offsite.Enabled() {
skip["offsite"] = "not configured"
}
if !usesMirroredScanDB(cfg) {
skip["scan-db"] = "builds do not scan against the registry's copy"
}
if len(splitList(w.certDirs)) == 0 {
skip["k3s-certs"] = "no -k3s-cert-dirs"
}
if w.nodeIP == "" {
skip["host-address"] = "no -node-ip"
}
}
report.Findings = append(report.Findings, unitFindings(ctx, env)...)
switch url, err := readHeartbeatURL(w.heartbeatFile); {
case err != nil:
report.Findings = append(report.Findings, watchdog.Finding{
Key: "watchdog/heartbeat", Severity: watchdog.Warning,
SummaryEN: fmt.Sprintf("the heartbeat URL is unusable, so no run pings it: %v", err),
Hint: "rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL set (docs/troubleshooting.md §14)",
})
case url == "":
notes = append(notes, "no heartbeat URL is set: a host that goes down entirely, or a watchdog that stops running, alerts no one. "+
"Rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL (docs/troubleshooting.md §14)")
}
if until := watchdog.QuietUntil(w.quietPath); env.now.Before(until) {
notes = append(notes, fmt.Sprintf("the watchdog mails nothing until %s (%s): the installer holds it while it restarts things on purpose, "+
"and a marker an installer killed mid-run left behind holds it until then",
until.UTC().Format("2006-01-02 15:04 UTC"), w.quietPath))
}
return printDoctorReport(stdout, report.Findings, skip, notes)
}
// printDoctorReport prints each area's findings under its heading, an area
// with none as fine or, when skip says why, as not checked, then the notes
// and the count. It returns the exit status: 1 when anything was found.
func printDoctorReport(stdout io.Writer, findings []watchdog.Finding, skip map[string]string, notes []string) int {
byArea := map[string][]watchdog.Finding{}
for _, f := range findings {
a := findingArea(f.Key)
byArea[a] = append(byArea[a], f)
}
var critical, warning int
for _, a := range doctorAreas {
fs := byArea[a.key]
switch {
case len(fs) > 0:
case skip[a.key] != "":
fmt.Fprintf(stdout, "- %s: not checked, %s\n", a.title, skip[a.key])
continue
default:
fmt.Fprintf(stdout, "✓ %s\n", a.title)
continue
}
mark := "!"
if slices.ContainsFunc(fs, func(f watchdog.Finding) bool { return f.Severity == watchdog.Critical }) {
mark = "✗"
}
fmt.Fprintf(stdout, "%s %s\n", mark, a.title)
for _, f := range fs {
if f.Severity == watchdog.Critical {
critical++
} else {
warning++
}
fmt.Fprintf(stdout, " %-8s %s: %s\n", f.Severity, f.Key, f.SummaryEN)
if f.Hint != "" {
fmt.Fprintf(stdout, " → %s\n", f.Hint)
}
}
}
for _, n := range notes {
fmt.Fprintf(stdout, "\nnote: %s\n", n)
}
fmt.Fprintln(stdout)
if critical+warning == 0 {
fmt.Fprintln(stdout, "no problems found")
return 0
}
fmt.Fprintf(stdout, "%d problem(s): %d critical, %d warning(s)\n", critical+warning, critical, warning)
return 1
}
// findingArea is the report heading a finding key goes under.
func findingArea(key string) string {
head, _, _ := strings.Cut(key, "/")
switch head {
case "kube-api", "deployment", "system-server", "server-failed", "job-failed", "reaper-stale", "node":
return "cluster"
case "db-backup", "db-backup-servers":
return "db-backup"
case "unit", "timer":
return "systemd"
case "watchdog":
return "alerts"
}
return head
}
// alertReachFindings is why the watchdog's alerts would reach no one, which
// its own runs only log: no relay, or no owner with a verified address.
func alertReachFindings(cfg *config.Config, owners []string, ownersErr error) []watchdog.Finding {
var out []watchdog.Finding
if cfg.SMTP.Host == "" {
out = append(out, watchdog.Finding{
Key: "alerts/relay", Severity: watchdog.Warning,
SummaryEN: "no [smtp] relay is configured: the watchdog logs its alerts to the journal and mails no one",
Hint: "sudo felis setup, step SMTP",
})
}
if ownersErr == nil && len(owners) == 0 {
out = append(out, watchdog.Finding{
Key: "alerts/recipients", Severity: watchdog.Warning,
SummaryEN: "no owner account has a verified email: the watchdog's alerts reach no one",
Hint: "an owner verifies an address in the panel's account settings",
})
}
return out
}
// unitFindings reports the installer's systemd units that failed, the
// long-running ones that are not running, and timers that no longer fire.
func unitFindings(ctx context.Context, env doctorEnv) []watchdog.Finding {
var out []watchdog.Finding
seen := map[string]bool{}
failed, err := env.run(ctx, "systemctl", "list-units", "--all", "--plain", "--no-legend", "--no-pager", "--state=failed", "felis-*", "k3s.service")
if err != nil && len(failed) == 0 {
return []watchdog.Finding{{
Key: "unit/systemctl", Severity: watchdog.Warning,
SummaryEN: fmt.Sprintf("systemctl list-units failed, so no unit was checked: %v", err),
}}
}
for _, line := range strings.Split(string(failed), "\n") {
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
name := fields[0]
seen[name] = true
out = append(out, watchdog.Finding{
Key: "unit/" + name, Severity: watchdog.Critical,
SummaryEN: name + " failed",
Hint: fmt.Sprintf("journalctl -u %s -n 100 --no-pager; once fixed, sudo systemctl reset-failed %s (a timer's job clears on its next good run)", name, name),
})
}
for _, name := range hostServices {
if seen[name] {
continue
}
if _, err := os.Stat(filepath.Join(env.unitDir, name)); err != nil {
continue
}
if state := unitActiveState(ctx, env, name); state != "active" {
out = append(out, watchdog.Finding{
Key: "unit/" + name, Severity: watchdog.Critical,
SummaryEN: fmt.Sprintf("%s is %s", name, state),
Hint: fmt.Sprintf("sudo systemctl start %s; journalctl -u %s -n 100 --no-pager", name, name),
})
}
}
timers, _ := filepath.Glob(filepath.Join(env.unitDir, "felis-*.timer"))
for _, path := range timers {
name := filepath.Base(path)
if state := unitActiveState(ctx, env, name); state != "active" {
out = append(out, watchdog.Finding{
Key: "timer/" + name, Severity: watchdog.Warning,
SummaryEN: fmt.Sprintf("%s is %s: the job it starts no longer runs", name, state),
Hint: fmt.Sprintf("sudo systemctl enable --now %s", name),
})
}
}
return out
}
// unitActiveState is what `systemctl is-active` says of unit.
func unitActiveState(ctx context.Context, env doctorEnv, unit string) string {
out, err := env.run(ctx, "systemctl", "is-active", unit)
if state := strings.TrimSpace(string(out)); state != "" {
return state
}
return fmt.Sprintf("in an unknown state (systemctl is-active: %v)", err)
}
// watchdogUnitFlags reads the flags felis-watchdog.service runs felis
// watchdog with. found is false when there is no such unit; w is then the
// watchdog's defaults.
func watchdogUnitFlags(path string) (w watchdogFlags, found bool, err error) {
fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
fs.SetOutput(io.Discard)
w.register(fs)
raw, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return w, false, nil
}
if err != nil {
return w, false, err
}
for _, line := range strings.Split(string(raw), "\n") {
cmd, ok := strings.CutPrefix(strings.TrimSpace(line), "ExecStart=")
if !ok {
continue
}
fields := execArgs(cmd)
i := slices.Index(fields, "watchdog")
if i < 0 {
continue
}
if err := fs.Parse(fields[i+1:]); err != nil {
return w, true, fmt.Errorf("%s: %w", path, err)
}
return w, true, nil
}
return w, true, fmt.Errorf("%s runs no `felis watchdog`", path)
}
// execArgs splits an ExecStart= command line into its words. A word may be
// quoted with " or ', as systemd allows, which is how an empty value is
// written.
func execArgs(s string) []string {
var out []string
var cur strings.Builder
inWord := false
var quote rune
for _, r := range s {
switch {
case quote != 0:
if r == quote {
quote = 0
} else {
cur.WriteRune(r)
}
case r == '"' || r == '\'':
quote, inWord = r, true
case r == ' ' || r == '\t':
if inWord {
out = append(out, cur.String())
cur.Reset()
inWord = false
}
default:
cur.WriteRune(r)
inWord = true
}
}
if inWord {
out = append(out, cur.String())
}
return out
}
cmd/felis/doctor_test.go
-423
View File
@@ -1,423 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/watchdog"
)
// bootstrapWatchdogExecStart is the ExecStart= line deploy/bootstrap.sh writes
// into felis-watchdog.service, with its variables filled in as an install
// fills them.
func bootstrapWatchdogExecStart(t *testing.T, vars map[string]string) string {
t.Helper()
raw, err := os.ReadFile("../../deploy/bootstrap.sh")
if err != nil {
t.Fatal(err)
}
var line string
for _, l := range strings.Split(string(raw), "\n") {
if strings.HasPrefix(l, "ExecStart=${HOST_BIN} watchdog -config") {
line = l
}
}
if line == "" {
t.Fatal("deploy/bootstrap.sh writes no `ExecStart=${HOST_BIN} watchdog -config` line")
}
line = strings.ReplaceAll(line, "${NODE_IP:+ -node-ip ${NODE_IP}}", " -node-ip "+vars["NODE_IP"])
line = regexp.MustCompile(`\$\{([A-Za-z_]+)\}`).ReplaceAllStringFunc(line, func(m string) string {
v, ok := vars[m[2:len(m)-1]]
if !ok {
t.Fatalf("bootstrap's watchdog ExecStart= uses %s, which this test does not fill in", m)
}
return v
})
return line
}
// felis doctor reads the watchdog's settings from the unit the installer
// writes, so it checks the paths the timer's runs check.
func TestWatchdogUnitFlagsReadsTheInstallersUnit(t *testing.T) {
dir := t.TempDir()
exec := bootstrapWatchdogExecStart(t, map[string]string{
"HOST_BIN": "/usr/local/bin/felis", "STATE_DIR": "/srv/felis-etc", "WATCHDOG_STATE": "/srv/watchdog/state.json",
"WATCHDOG_QUIET_FILE": "/srv/quiet-until", "FELIS_DB_BACKUP_DIR": "/srv/db-backups", "FELIS_GAME_PORT": "25577",
"disks": "/,/srv/data", "NODE_IP": "10.0.0.5",
})
unit := filepath.Join(dir, "felis-watchdog.service")
writeTestFile(t, unit, "[Unit]\nDescription=Felis watchdog\n\n[Service]\nType=oneshot\n"+exec+"\nTimeoutStartSec=3min\n", 0o644)
w, found, err := watchdogUnitFlags(unit)
if err != nil || !found {
t.Fatalf("found %v, err %v", found, err)
}
got := []string{w.cfgPath, w.statePath, w.quietPath, w.backupDir, w.proxyAddr, w.diskPaths, w.nodeIP, w.heartbeatFile, w.controlNS}
want := []string{"/srv/felis-etc/felis.host.toml", "/srv/watchdog/state.json", "/srv/quiet-until", "/srv/db-backups", "127.0.0.1:25577", "/,/srv/data", "10.0.0.5", defaultHeartbeatFile, "felis"}
if strings.Join(got, "|") != strings.Join(want, "|") {
t.Errorf("read\n %q\nwant\n %q", got, want)
}
w, found, err = watchdogUnitFlags(filepath.Join(dir, "missing.service"))
if err != nil || found || w.cfgPath != "/etc/felis/felis.toml" || w.backupDir != "/var/lib/felis/db-backups" {
t.Errorf("no unit: found %v, err %v, config %q, backups %q; want the watchdog's defaults", found, err, w.cfgPath, w.backupDir)
}
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis version\n", 0o644)
if _, found, err := watchdogUnitFlags(unit); !found || err == nil || !strings.Contains(err.Error(), "runs no `felis watchdog`") {
t.Errorf("a unit that runs something else: found %v, err %v", found, err)
}
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
if _, _, err := watchdogUnitFlags(unit); err == nil {
t.Error("a flag this binary does not know was accepted")
}
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -backup-dir \"\"\t-proxy-addr '127.0.0.1:1' -disk-paths \"/a b\"\n", 0o644)
if w, _, err := watchdogUnitFlags(unit); err != nil || w.backupDir != "" || w.proxyAddr != "127.0.0.1:1" || w.diskPaths != "/a b" {
t.Errorf("quoted words: backups %q, proxy %q, disks %q, err %v", w.backupDir, w.proxyAddr, w.diskPaths, err)
}
}
func TestFindingArea(t *testing.T) {
for key, want := range map[string]string{
"kube-api": "cluster",
"deployment/felis-api": "cluster",
"system-server/lobby": "cluster",
"server-failed/survival": "cluster",
"job-failed/reaper-123": "cluster",
"reaper-stale": "cluster",
"node/felis-1/NotReady": "cluster",
"postgres": "postgres",
"proxy": "proxy",
"db-backup": "db-backup",
"db-backup-servers": "db-backup",
"offsite": "offsite",
"scan-db": "scan-db",
"disk//var/lib/felis": "disk",
"memory": "memory",
"k3s-certs": "k3s-certs",
"host-address": "host-address",
"clock": "clock",
"config": "config",
"unit/felis-offsite.service": "systemd",
"timer/felis-db-backup.timer": "systemd",
"watchdog/unit": "alerts",
"watchdog/heartbeat": "alerts",
"alerts/relay": "alerts",
"alerts/recipients": "alerts",
"something-a-later-release-reported": "something-a-later-release-reported",
} {
if got := findingArea(key); got != want {
t.Errorf("findingArea(%q) = %q, want %q", key, got, want)
}
}
}
func TestAlertReachFindings(t *testing.T) {
relay := &config.Config{SMTP: config.SMTPConfig{Host: "smtp.example.com"}}
keys := func(fs []watchdog.Finding) string {
var k []string
for _, f := range fs {
k = append(k, f.Key)
}
return strings.Join(k, ",")
}
for _, tc := range []struct {
what string
cfg *config.Config
owners []string
ownersErr error
want string
}{
{"a relay and an owner", relay, []string{"[email protected]"}, nil, ""},
{"no relay", &config.Config{}, []string{"[email protected]"}, nil, "alerts/relay"},
{"no owner with an address", relay, nil, nil, "alerts/recipients"},
{"PostgreSQL down: its own finding says so", relay, nil, errors.New("refused"), ""},
{"neither", &config.Config{}, nil, nil, "alerts/relay,alerts/recipients"},
} {
if got := keys(alertReachFindings(tc.cfg, tc.owners, tc.ownersErr)); got != tc.want {
t.Errorf("%s: %q, want %q", tc.what, got, tc.want)
}
}
}
// fakeSystemctl answers list-units with failed and is-active from states;
// a unit missing from states is "inactive", as systemctl says, and one whose
// state is "" gets no answer.
func fakeSystemctl(failed string, states map[string]string) func(ctx context.Context, name string, args ...string) ([]byte, error) {
return func(_ context.Context, name string, args ...string) ([]byte, error) {
if name != "systemctl" || len(args) == 0 {
return nil, errors.New("unexpected command " + name)
}
switch args[0] {
case "list-units":
return []byte(failed), nil
case "is-active":
if s, ok := states[args[1]]; ok && s == "" {
return nil, errors.New("signal: killed")
} else if ok {
return []byte(s + "\n"), nil
}
return []byte("inactive\n"), errors.New("exit status 3")
}
return nil, errors.New("unexpected systemctl " + args[0])
}
}
func TestUnitFindings(t *testing.T) {
dir := t.TempDir()
for _, f := range []string{"k3s.service", "felis-velocity.service", "felis-db-backup.timer", "felis-offsite.timer", "felis-offsite.service"} {
writeTestFile(t, filepath.Join(dir, f), "[Unit]\n", 0o644)
}
env := doctorEnv{unitDir: dir, run: fakeSystemctl(
"felis-offsite.service loaded failed failed Felis off-site copy\nfelis-velocity.service loaded failed failed Velocity\n",
map[string]string{"k3s.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "inactive"},
)}
var got []string
for _, f := range unitFindings(context.Background(), env) {
got = append(got, string(f.Severity)+" "+f.Key+": "+f.SummaryEN)
}
want := []string{
"critical unit/felis-offsite.service: felis-offsite.service failed",
"critical unit/felis-velocity.service: felis-velocity.service failed",
"warning timer/felis-offsite.timer: felis-offsite.timer is inactive: the job it starts no longer runs",
}
if strings.Join(got, "\n") != strings.Join(want, "\n") {
t.Errorf("findings:\n%s\nwant (felis-nano.service has no unit file here, and a failed unit is reported once):\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
env.run = fakeSystemctl("", map[string]string{"k3s.service": "activating", "felis-velocity.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "active"})
got = nil
for _, f := range unitFindings(context.Background(), env) {
got = append(got, f.Key+": "+f.SummaryEN)
}
if strings.Join(got, "\n") != "unit/k3s.service: k3s.service is activating" {
t.Errorf("findings %q, want only k3s.service, which is not active", got)
}
env.run = fakeSystemctl("", map[string]string{"k3s.service": "active", "felis-velocity.service": "active", "felis-db-backup.timer": "", "felis-offsite.timer": "active"})
got = nil
for _, f := range unitFindings(context.Background(), env) {
got = append(got, f.Key+": "+f.SummaryEN)
}
if want := "timer/felis-db-backup.timer: felis-db-backup.timer is in an unknown state (systemctl is-active: signal: killed): the job it starts no longer runs"; strings.Join(got, "\n") != want {
t.Errorf("findings %q, want %q", got, want)
}
env.run = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("no systemctl") }
if fs := unitFindings(context.Background(), env); len(fs) != 1 || fs[0].Key != "unit/systemctl" || fs[0].Severity != watchdog.Warning {
t.Errorf("without systemctl: %+v, want the one unit/systemctl warning", fs)
}
}
// quoteArgs writes args as an ExecStart= line does, each word in quotes so an
// empty one survives.
func quoteArgs(args []string) string {
q := make([]string, len(args))
for i, a := range args {
q[i] = `"` + a + `"`
}
return strings.Join(q, " ")
}
// doctorHost is a host with the installer's watchdog unit, whose API server
// and PostgreSQL are down.
func doctorHost(t *testing.T, cfg string, extraFlags string) (env doctorEnv, h *watchdogHost) {
t.Helper()
h = newWatchdogHost(t, cfg, nil)
unitDir := t.TempDir()
writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"),
"[Service]\nType=oneshot\nExecStart=/usr/local/bin/felis watchdog "+quoteArgs(h.args)+
// Off this machine's disk, whose free space is not the test's.
` -disk-paths "/nonexistent-felis-doctor-test"`+extraFlags+"\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "felis-offsite.timer"), "[Unit]\n", 0o644)
return doctorEnv{
unitDir: unitDir, now: time.Now(), host: "felis-test",
run: fakeSystemctl("felis-db-backup.service loaded failed failed Felis database backup\n",
map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"}),
}, h
}
func TestDoctorReportsByArea(t *testing.T) {
env, _ := doctorHost(t, testWatchdogConfig, "")
var out bytes.Buffer
code := runDoctor(context.Background(), env, &out)
got := out.String()
if code != 1 {
t.Errorf("exit %d, want 1 with problems found", code)
}
for _, want := range []string{
"felis doctor on felis-test at ",
"checks run as " + filepath.Join(env.unitDir, "felis-watchdog.service") + " runs them (config ",
"✓ configuration\n",
"✗ Kubernetes cluster\n critical kube-api: ",
"✗ PostgreSQL\n critical postgres: ",
"- game proxy: not checked, no -proxy-addr\n",
"- database backups: not checked, no -backup-dir\n",
"- off-site copy: not checked, not configured\n",
"- build scan database: not checked, builds do not scan against the registry's copy\n",
"- k3s certificates: not checked, no -k3s-cert-dirs\n",
"- node address: not checked, no -node-ip\n",
"✗ systemd units and timers\n critical unit/felis-db-backup.service: felis-db-backup.service failed\n" +
" → journalctl -u felis-db-backup.service -n 100 --no-pager; once fixed, sudo systemctl reset-failed felis-db-backup.service",
"! alerting\n warning alerts/relay: no [smtp] relay is configured",
"\n4 problem(s): 3 critical, 1 warning(s)\n",
} {
if !strings.Contains(got, want) {
t.Errorf("report lacks %q:\n%s", want, got)
}
}
if strings.Contains(got, "alerts/recipients") {
t.Errorf("reported no recipients although PostgreSQL, which names them, is down:\n%s", got)
}
if strings.Contains(got, "note: no heartbeat URL") {
t.Errorf("the host has a heartbeat URL:\n%s", got)
}
}
// A doctor run is only a look: whatever is due to be mailed stays due, no
// heartbeat is pinged, and the watchdog's state is left as it was.
func TestDoctorMailsPingsAndSavesNothing(t *testing.T) {
cfg := testWatchdogConfig + "[smtp]\nhost = \"smtp.example.com\"\nport = 587\nfrom = \"[email protected]\"\n"
env, h := doctorHost(t, cfg, "")
if err := watchdog.SaveState(h.statePath, duePostgres("cached-pw")); err != nil {
t.Fatal(err)
}
before, err := os.ReadFile(h.statePath)
if err != nil {
t.Fatal(err)
}
watchdogSender = h.rec.sender
defer func() { watchdogSender = smtpSender }()
var out bytes.Buffer
runDoctor(context.Background(), env, &out)
if !strings.Contains(out.String(), "critical postgres: ") {
t.Fatalf("the due PostgreSQL alert was not seen:\n%s", out.String())
}
if len(h.rec.sent) != 0 || len(h.rec.relays) != 0 {
t.Errorf("mailed %v", h.rec.sent)
}
if n := len(h.pings.pings); n != 0 {
t.Errorf("pinged the heartbeat %d times", n)
}
after, err := os.ReadFile(h.statePath)
if err != nil || !bytes.Equal(before, after) {
t.Errorf("the watchdog state changed (err %v)", err)
}
if _, err := os.Stat(h.fallbackPath); !errors.Is(err, os.ErrNotExist) {
t.Errorf("a fallback state was written: %v", err)
}
}
func TestDoctorNotes(t *testing.T) {
env, h := doctorHost(t, testWatchdogConfig, "")
if err := os.Remove(filepath.Join(h.dir, "watchdog-heartbeat-url")); err != nil {
t.Fatal(err)
}
until := env.now.Add(10 * time.Minute).Unix()
writeTestFile(t, filepath.Join(h.dir, "quiet"), strconv.FormatInt(until, 10)+"\n", 0o644)
var out bytes.Buffer
runDoctor(context.Background(), env, &out)
for _, want := range []string{
"\nnote: no heartbeat URL is set: ",
"\nnote: the watchdog mails nothing until " + time.Unix(until, 0).UTC().Format("2006-01-02 15:04 UTC") + " (" + filepath.Join(h.dir, "quiet") + ")",
} {
if !strings.Contains(out.String(), want) {
t.Errorf("report lacks %q:\n%s", want, out.String())
}
}
writeTestFile(t, filepath.Join(h.dir, "watchdog-heartbeat-url"), "not a url\n", 0o600)
out.Reset()
runDoctor(context.Background(), env, &out)
if !strings.Contains(out.String(), "warning watchdog/heartbeat: the heartbeat URL is unusable") {
t.Errorf("an unusable heartbeat URL is not reported:\n%s", out.String())
}
}
// Without the watchdog's unit the doctor still checks, with the watchdog's
// defaults, and says the host has no watchdog; a configuration that does not
// load leaves the checks that need it unchecked and the host's own checks on.
func TestDoctorWithoutUnitOrConfig(t *testing.T) {
env, _ := doctorHost(t, testWatchdogConfig, "")
if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
t.Fatal(err)
}
writeTestFile(t, filepath.Join(env.unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+filepath.Join(t.TempDir(), "absent.toml")+"\n", 0o644)
env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"})
var out bytes.Buffer
if code := runDoctor(context.Background(), env, &out); code != 1 {
t.Errorf("exit %d, want 1", code)
}
for _, want := range []string{
"✗ configuration\n critical config: ",
"- Kubernetes cluster: not checked, the configuration did not load\n",
"- PostgreSQL: not checked, the configuration did not load\n",
"- disk space: not checked, the configuration did not load\n",
"✓ systemd units and timers\n",
"- alerting: not checked, the configuration did not load\n",
} {
if !strings.Contains(out.String(), want) {
t.Errorf("report lacks %q:\n%s", want, out.String())
}
}
if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
t.Fatal(err)
}
out.Reset()
runDoctor(context.Background(), env, &out)
if !strings.Contains(out.String(), "critical watchdog/unit: "+filepath.Join(env.unitDir, "felis-watchdog.service")+" is not installed") ||
!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
t.Errorf("a host without the watchdog's unit:\n%s", out.String())
}
unit := filepath.Join(env.unitDir, "felis-watchdog.service")
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
out.Reset()
runDoctor(context.Background(), env, &out)
if !strings.Contains(out.String(), "critical watchdog/unit: cannot read the watchdog's settings: "+unit+": flag provided but not defined: -no-such-flag\n") ||
!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
t.Errorf("a watchdog unit this binary cannot read:\n%s", out.String())
}
}
func TestPrintDoctorReport(t *testing.T) {
var out bytes.Buffer
if code := printDoctorReport(&out, nil, map[string]string{"proxy": "no -proxy-addr"}, nil); code != 0 {
t.Errorf("exit %d with nothing found, want 0", code)
}
want := "✓ configuration\n✓ Kubernetes cluster\n✓ PostgreSQL\n- game proxy: not checked, no -proxy-addr\n✓ database backups\n✓ off-site copy\n" +
"✓ build scan database\n✓ disk space\n✓ memory\n✓ k3s certificates\n✓ node address\n✓ clock\n✓ systemd units and timers\n✓ alerting\n" +
"\nno problems found\n"
if out.String() != want {
t.Errorf("report:\n%s\nwant:\n%s", out.String(), want)
}
out.Reset()
code := printDoctorReport(&out, []watchdog.Finding{
{Key: "unit/systemctl", Severity: watchdog.Warning, SummaryEN: "systemctl list-units failed"},
{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "PostgreSQL is down", Hint: "kubectl -n felis get pods"},
}, map[string]string{"postgres": "a skip loses to what was found"}, []string{"a note"})
if code != 1 {
t.Errorf("exit %d with problems, want 1", code)
}
for _, want := range []string{
"✗ PostgreSQL\n critical postgres: PostgreSQL is down\n → kubectl -n felis get pods\n✓ game proxy\n",
"! systemd units and timers\n warning unit/systemctl: systemctl list-units failed\n✓ alerting\n",
"✓ alerting\n\nnote: a note\n\n2 problem(s): 1 critical, 1 warning(s)\n",
} {
if !strings.Contains(out.String(), want) {
t.Errorf("report lacks %q:\n%s", want, out.String())
}
}
}
cmd/felis/domain.go
-1359
View File
File diff suppressed because it is too large. Load diff
cmd/felis/domain_test.go
-892
View File
@@ -1,892 +0,0 @@
package main
import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"math/big"
"net"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)
// installerTOML is felis.toml as deploy/bootstrap.sh write_felis_toml renders it,
// comments included: the domain move has to leave all of it but three values alone.
func installerTOML(root, dbHost string) string {
return `# Generated by deploy/bootstrap.sh; rerun the installer to regenerate. Hand edits are
# overwritten, except [smtp], [[auth_source]], [offsite], and the operator-owned
# [registry] / [archive] overrides, which carry forward.
[server]
listen = "0.0.0.0:8080"
root_domain = "` + root + `"
[database]
url = "postgres://felis:pw@` + dbHost + `:5432/felis?sslmode=disable"
[k8s]
namespace = "minecraft"
egress_mode = "nodeport"
[velocity]
# The two always-on system servers that felis setup provisions.
login_image = "felis/limbo:1"
lobby_image = "felis/lobby:1"
game_port = 25565
[registry]
url = "registry.felis.svc:5000"
build_namespace = "felis-build"
[archive]
store = "tarLocal"
local_path = "/var/lib/felis/archives"
[auth]
admin_hostname = "op.console.` + root + `"
panel_hostname = "console.` + root + `"
access_jwt_aud = "aud123"
# Third-party Yggdrasil sources federated by the hasJoined multiplexer.
[[auth_source]]
tag = "littleskin"
prefix = "LS"
url = "https://littleskin.cn/api/yggdrasil/sessionserver/session/minecraft/hasJoined"
`
}
const linkPropsBody = `# Generated by deploy/bootstrap.sh — do not edit by hand; rerun the installer.
api-base-url=http://10.43.0.9:8081
service-token=TOKEN-NOT-TO-TOUCH
root-domain=old.example
panel-hostname=console.old.example
admin-hostname=op.console.old.example
login-server=login
lobby-server=lobby
`
var oldNames = domainNames{root: "old.example", panel: "console.old.example", admin: "op.console.old.example"}
var newNames = domainNames{root: "new.example", panel: "console.new.example", admin: "op.console.new.example"}
// domainRig models the host: the files, the cluster, and a felis-api, proxy and
// operator that pick up config the way the real ones do — the api serves what it
// read at its last restart, the proxy runs since its last restart, and the login
// pod carries the env its MinecraftServer had when it was last rolled.
type domainRig struct {
h domainHost
cl client.Client
out *bytes.Buffer
dir string
events []string
served domainNames
servedCert *x509.Certificate
proxySince time.Time
proxyLoaded bool
unresolved map[string]bool
// The fake operator: the CR env the login pod was last rolled to, and how
// many looks at the pod since the CR moved on.
rolledTo string
pending int
}
func (rig *domainRig) path(name string) string { return filepath.Join(rig.dir, name) }
func newDomainRig(t *testing.T) *domainRig {
t.Helper()
rig := &domainRig{out: &bytes.Buffer{}, dir: t.TempDir(), proxyLoaded: true, unresolved: map[string]bool{}}
writeTestFile(t, rig.path("felis.host.toml"), installerTOML("old.example", "127.0.0.1"), 0o600)
writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
if err := os.Symlink(rig.path("felis.host.toml"), rig.path("felis.toml")); err != nil {
t.Fatal(err)
}
certPEM, keyPEM, err := issuePanelCert(oldNames, []net.IP{net.ParseIP("10.211.55.6")}, time.Now())
if err != nil {
t.Fatal(err)
}
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
// The proxy started before its config was last written, which is how it
// stands after an install.
rig.proxySince = time.Now().Add(-time.Hour)
pod := []byte(installerTOML("old.example", "10.211.55.6"))
login, err := loginSystemServer("felis/limbo:1", "minecraft", platform.InternalAPIBaseURL("felis"), oldNames.root, oldNames.panel)
if err != nil {
t.Fatal(err)
}
lobby, err := lobbySystemServer("felis/lobby:1", "minecraft")
if err != nil {
t.Fatal(err)
}
loginPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"},
Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "minecraft", Env: podEnv(login.Spec.Env)}}},
Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}},
}
rig.rolledTo = envKey(login.Spec.Env)
rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if key.Name == naming.SystemLoginServer+"-0" {
rig.operatorTick(t, c)
}
return c.Get(ctx, key, obj, opts...)
},
}).WithObjects(
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.ConfigSecretName},
Data: map[string][]byte{platform.ConfigSecretKey: pod}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: platform.ConfigSecretName},
Data: map[string][]byte{platform.ConfigSecretKey: pod}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.APITLSSecretName},
Type: corev1.SecretTypeTLS, Data: map[string][]byte{corev1.TLSCertKey: certPEM, corev1.TLSPrivateKeyKey: keyPEM}},
login, lobby, loginPod,
).Build()
rig.served = oldNames
rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
rig.h = domainHost{
paths: domainPaths{
hostTOML: rig.path("felis.host.toml"), podTOML: rig.path("felis.pod.toml"), defaultTOML: rig.path("felis.toml"),
cert: rig.path("panel-tls.crt"), key: rig.path("panel-tls.key"),
linkProps: rig.path("felis-link.properties"), tunnelConfig: rig.path("cloudflared.yml"),
},
cl: rig.cl,
controlNS: "felis",
rollAPI: func(ctx context.Context) error {
rig.events = append(rig.events, "roll-api")
rig.restartAPI(t)
return nil
},
restartUnit: func(_ context.Context, unit string) error {
rig.events = append(rig.events, "restart "+unit)
rig.proxySince = time.Now().Add(time.Second)
return nil
},
unitState: func(context.Context, string) (unitStatus, error) {
return unitStatus{loaded: rig.proxyLoaded, active: rig.proxyLoaded, since: rig.proxySince}, nil
},
liveAPI: func(context.Context, string) (liveAPIView, error) {
return liveAPIView{names: rig.served, cert: rig.servedCert}, nil
},
lookupHost: func(_ context.Context, host string) ([]string, error) {
if rig.unresolved[host] {
return nil, errors.New("no such host")
}
return []string{"10.211.55.6"}, nil
},
passkeys: func(context.Context) (int, int, error) { return 3, 2, nil },
now: time.Now,
out: rig.out,
loginWait: 50 * time.Millisecond,
pollEvery: time.Millisecond,
}
return rig
}
func podEnv(env []v1alpha1.EnvVar) []corev1.EnvVar {
out := make([]corev1.EnvVar, len(env))
for i, e := range env {
out[i] = corev1.EnvVar{Name: e.Name, Value: e.Value}
}
return out
}
// restartAPI makes the fake felis-api load the config and certificate its
// Secrets hold now.
func (rig *domainRig) restartAPI(t *testing.T) {
t.Helper()
var cfg, tlsSec corev1.Secret
ctx := context.Background()
if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.ConfigSecretName}, &cfg); err != nil {
t.Fatal(err)
}
if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.APITLSSecretName}, &tlsSec); err != nil {
t.Fatal(err)
}
names, err := tomlDomainNames(cfg.Data[platform.ConfigSecretKey])
if err != nil {
t.Fatal(err)
}
rig.served = names
rig.servedCert, err = x509.ParseCertificate(mustCertDER(tlsSec.Data[corev1.TLSCertKey]))
if err != nil {
t.Fatal(err)
}
}
// rollLoginPod is the operator restarting the login pod onto its CR's env.
// operatorTick is the operator as the login pod is watched: once the CR's env
// changes it takes operatorLag looks at the pod before the restarted pod
// carries the new env, the way a real rollout lags the CR.
func (rig *domainRig) operatorTick(t *testing.T, c client.Client) {
t.Helper()
ctx := context.Background()
var ms v1alpha1.MinecraftServer
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
t.Fatal(err)
}
if envKey(ms.Spec.Env) == rig.rolledTo {
return
}
if rig.pending++; rig.pending < operatorLag {
return
}
var pod corev1.Pod
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"}, &pod); err != nil {
t.Fatal(err)
}
pod.Spec.Containers[0].Env = podEnv(ms.Spec.Env)
if err := c.Update(ctx, &pod); err != nil {
t.Fatal(err)
}
rig.rolledTo, rig.pending = envKey(ms.Spec.Env), 0
}
const operatorLag = 3
func envKey(env []v1alpha1.EnvVar) string {
var b strings.Builder
for _, e := range env {
b.WriteString(e.Name + "=" + e.Value + "\n")
}
return b.String()
}
func (rig *domainRig) read(t *testing.T, name string) string {
t.Helper()
b, err := os.ReadFile(rig.path(name))
if err != nil {
t.Fatal(err)
}
return string(b)
}
func (rig *domainRig) secret(t *testing.T, ns, name string) map[string][]byte {
t.Helper()
var s corev1.Secret
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
t.Fatal(err)
}
return s.Data
}
func (rig *domainRig) crEnv(t *testing.T, name string) map[string]string {
t.Helper()
var ms v1alpha1.MinecraftServer
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
t.Fatal(err)
}
env := map[string]string{}
for _, e := range ms.Spec.Env {
env[e.Name] = e.Value
}
return env
}
// snapshot is every byte `set` may touch, for proving a refused or dry run
// touched none of it.
func (rig *domainRig) snapshot(t *testing.T) string {
t.Helper()
var b strings.Builder
entries, _ := os.ReadDir(rig.dir)
for _, e := range entries {
b.WriteString(e.Name() + "\n" + rig.read(t, e.Name()) + "\n")
}
var lines []string
for _, s := range []struct{ ns, name string }{{"felis", platform.ConfigSecretName}, {"minecraft", platform.ConfigSecretName}, {"felis", platform.APITLSSecretName}} {
for k, v := range rig.secret(t, s.ns, s.name) {
lines = append(lines, s.ns+"/"+s.name+"/"+k+"\n"+string(v))
}
}
for k, v := range rig.crEnv(t, naming.SystemLoginServer) {
lines = append(lines, "env "+k+"="+v)
}
sort.Strings(lines)
b.WriteString(strings.Join(lines, "\n"))
return b.String()
}
func TestNormalizeRootDomain(t *testing.T) {
for in, want := range map[string]string{
"Example.COM.": "example.com",
" mc.example.org ": "mc.example.org",
"10.211.55.6.nip.io": "10.211.55.6.nip.io",
"xn--bcher-kva.example": "xn--bcher-kva.example",
} {
got, err := normalizeRootDomain(in)
if err != nil || got != want {
t.Errorf("normalizeRootDomain(%q) = %q, %v; want %q", in, got, err, want)
}
}
for _, in := range []string{"", "https://example.com", "example.com:443", "example.com/x", "10.0.0.1", "::1",
"localhost", "a_b.example", "-a.example", "a-.example", strings.Repeat("a", 64) + ".example",
strings.Repeat("abcdefghi.", 25) + "example"} {
if got, err := normalizeRootDomain(in); err == nil {
t.Errorf("normalizeRootDomain(%q) = %q, want an error", in, got)
}
}
}
func TestPlanDomainChangeMovesDefaultsAndKeepsHandSetNames(t *testing.T) {
p := planDomainChange(oldNames, "new.example")
if p.to != newNames || p.customPanel || p.customAdmin {
t.Fatalf("defaults: %+v", p)
}
p = planDomainChange(domainNames{root: "old.example", panel: "play.corp.net", admin: "op.console.old.example"}, "new.example")
if p.to.panel != "play.corp.net" || !p.customPanel || p.to.admin != "op.console.new.example" || p.customAdmin {
t.Fatalf("hand-set panel: %+v", p)
}
p = planDomainChange(domainNames{root: "old.example", panel: "console.old.example", admin: "admin.corp.net"}, "new.example")
if p.to.admin != "admin.corp.net" || !p.customAdmin || p.to.panel != "console.new.example" {
t.Fatalf("hand-set admin: %+v", p)
}
}
func TestEditTOMLStringsChangesOnlyTheDomainLines(t *testing.T) {
orig := installerTOML("old.example", "127.0.0.1")
out, err := editTOMLStrings([]byte(orig), domainTOMLEdits(newNames))
if err != nil {
t.Fatal(err)
}
a, b := strings.Split(orig, "\n"), strings.Split(string(out), "\n")
if len(a) != len(b) {
t.Fatalf("line count %d → %d:\n%s", len(a), len(b), out)
}
changed := map[string]string{}
for i := range a {
if a[i] != b[i] {
changed[a[i]] = b[i]
}
}
want := map[string]string{
`root_domain = "old.example"`: `root_domain = "new.example"`,
`admin_hostname = "op.console.old.example"`: `admin_hostname = "op.console.new.example"`,
`panel_hostname = "console.old.example"`: `panel_hostname = "console.new.example"`,
}
if len(changed) != len(want) {
t.Fatalf("changed lines %v, want %v", changed, want)
}
for k, v := range want {
if changed[k] != v {
t.Errorf("%q → %q, want %q", k, changed[k], v)
}
}
}
func TestEditTOMLStringsAddsMissingKeysInTheirTable(t *testing.T) {
in := "[server]\nroot_domain = \"old.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\n\n[smtp]\nhost = \"relay\"\n"
out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames))
if err != nil {
t.Fatal(err)
}
want := "[server]\nroot_domain = \"new.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\npanel_hostname = \"console.new.example\"\nadmin_hostname = \"op.console.new.example\"\n\n[smtp]\nhost = \"relay\"\n"
if string(out) != want {
t.Fatalf("got:\n%s\nwant:\n%s", out, want)
}
out, err = editTOMLStrings([]byte("[server]\nroot_domain = \"old.example\"\n\n[[auth_source]]\ntag = \"ls\"\n"), domainTOMLEdits(newNames))
if err != nil {
t.Fatal(err)
}
got, err := tomlDomainNames(out)
if err != nil || got != newNames || !strings.Contains(string(out), "[[auth_source]]\ntag = \"ls\"\n") {
t.Fatalf("no [auth] table: %v %+v\n%s", err, got, out)
}
}
func TestEditTOMLStringsRefusesWhatItCannotEditExactly(t *testing.T) {
for name, in := range map[string]string{
"multi-line value": "[server]\nroot_domain = \"\"\"\nold.example\"\"\"\n[auth]\n",
// The key's line sits inside another value; the real key is absent.
"key inside a string": "[server]\nmotd = \"\"\"\nroot_domain = \"old.example\"\n\"\"\"\n[auth]\n",
"quoted header": "[server]\nroot_domain = \"old.example\"\n[\"auth\"]\npanel_hostname = \"console.old.example\"\n",
"dotted key": "server.root_domain = \"old.example\"\n",
"inline table": "server = { root_domain = \"old.example\" }\n",
} {
if out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames)); err == nil {
t.Errorf("%s: edited instead of refusing:\n%s", name, out)
}
}
}
// caSignedCert is an operator's certificate from their own CA; it names
// localhost too, so only the issuer tells it apart from the installer's.
func caSignedCert(t *testing.T, hosts ...string) (certPEM, keyPEM []byte) {
t.Helper()
caKey, _ := rsa.GenerateKey(rand.Reader, 2048)
ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Corp CA"}, IsCA: true,
BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
caCert, _ := x509.ParseCertificate(caDER)
key, _ := rsa.GenerateKey(rand.Reader, 2048)
leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: hosts[0]},
DNSNames: append(hosts, "localhost"), IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
der, err := x509.CreateCertificate(rand.Reader, leaf, caCert, &key.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
pk, _ := x509.MarshalPKCS8PrivateKey(key)
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pk})
}
func TestFelisIssuedCert(t *testing.T) {
mine, _, err := issuePanelCert(oldNames, nil, time.Now())
if err != nil {
t.Fatal(err)
}
c, _ := x509.ParseCertificate(mustCertDER(mine))
if !felisIssuedCert(c) {
t.Error("the installer's kind of certificate is not recognised as Felis-issued")
}
theirs, _ := caSignedCert(t, "console.old.example")
c, _ = x509.ParseCertificate(mustCertDER(theirs))
if felisIssuedCert(c) {
t.Error("a CA-signed certificate is taken for Felis-issued")
}
// Self-signed but without one of the installer's localhost names: someone
// else's.
key, _ := rsa.GenerateKey(rand.Reader, 2048)
for name, self := range map[string]*x509.Certificate{
"no localhost": {DNSNames: []string{"console.old.example"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}},
"no 127.0.0.1": {DNSNames: []string{"console.old.example", "localhost"}},
} {
self.SerialNumber, self.Subject = big.NewInt(3), pkix.Name{CommonName: "x"}
self.NotBefore, self.NotAfter = time.Now().Add(-time.Hour), time.Now().Add(time.Hour)
der, _ := x509.CreateCertificate(rand.Reader, self, self, &key.PublicKey, key)
c, _ = x509.ParseCertificate(der)
if felisIssuedCert(c) {
t.Errorf("%s: a self-signed certificate is taken for Felis-issued", name)
}
}
}
func TestIssuePanelCertIsTheInstallersShape(t *testing.T) {
now := time.Now()
certPEM, keyPEM, err := issuePanelCert(newNames, []net.IP{net.ParseIP("10.211.55.6"), net.IPv4(127, 0, 0, 1)}, now)
if err != nil {
t.Fatal(err)
}
if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil {
t.Fatalf("key does not match the certificate: %v", err)
}
if b, _ := pem.Decode(keyPEM); b == nil || b.Type != "PRIVATE KEY" {
t.Fatalf("key is not PKCS#8 PEM like openssl writes")
}
c, _ := x509.ParseCertificate(mustCertDER(certPEM))
if !certCovers(c, newNames.panel, newNames.admin, "localhost") || !felisIssuedCert(c) {
t.Fatalf("names %v", c.DNSNames)
}
if len(c.IPAddresses) != 2 || !c.IPAddresses[1].Equal(net.ParseIP("10.211.55.6")) {
t.Fatalf("addresses %v, want 127.0.0.1 and the node address once each", c.IPAddresses)
}
if c.Subject.CommonName != newNames.admin || c.NotAfter.Sub(now) < 824*24*time.Hour || c.NotAfter.Sub(now) > 826*24*time.Hour {
t.Fatalf("CN %q, valid until %s", c.Subject.CommonName, c.NotAfter)
}
if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth || c.IsCA {
t.Fatalf("usage %v, CA %v", c.ExtKeyUsage, c.IsCA)
}
}
func TestDomainSetMovesEverySurface(t *testing.T) {
rig := newDomainRig(t)
hostBefore := rig.read(t, "felis.host.toml")
code, err := rig.h.set(context.Background(), "New.Example", true)
if err != nil || code != 0 {
t.Fatalf("set = %d, %v\n%s", code, err, rig.out)
}
// The configs: the three values moved, everything else — comments, the
// database host of each copy, the Access audience — is as it was.
host := rig.read(t, "felis.host.toml")
if want := strings.NewReplacer("old.example", "new.example").Replace(hostBefore); host != want {
t.Fatalf("host toml:\n%s", host)
}
pod := rig.read(t, "felis.pod.toml")
if got, _ := tomlDomainNames([]byte(pod)); got != newNames || !strings.Contains(pod, "@10.211.55.6:5432") {
t.Fatalf("pod toml:\n%s", pod)
}
if link, err := os.Readlink(rig.path("felis.toml")); err != nil || link != rig.path("felis.host.toml") {
t.Fatalf("felis.toml is no longer the link to the host copy: %q %v", link, err)
}
if st, _ := os.Stat(rig.path("felis.host.toml")); st.Mode().Perm() != 0o600 {
t.Fatalf("host toml mode %v", st.Mode().Perm())
}
// The certificate: reissued for the new names, the node address kept, the old
// pair beside it.
c, err := readCertFile(rig.path("panel-tls.crt"))
if err != nil || !certCovers(c, newNames.panel, newNames.admin) || !felisIssuedCert(c) {
t.Fatalf("certificate: %v %v", err, c.DNSNames)
}
if !c.IPAddresses[len(c.IPAddresses)-1].Equal(net.ParseIP("10.211.55.6")) {
t.Fatalf("addresses %v", c.IPAddresses)
}
if _, err := tls.LoadX509KeyPair(rig.path("panel-tls.crt"), rig.path("panel-tls.key")); err != nil {
t.Fatalf("new pair: %v", err)
}
if st, _ := os.Stat(rig.path("panel-tls.key")); st.Mode().Perm() != 0o600 {
t.Fatalf("key mode %v", st.Mode().Perm())
}
backups, _ := filepath.Glob(rig.path("panel-tls.*.pre-domain-*"))
if len(backups) != 2 {
t.Fatalf("old pair kept as %v", backups)
}
for _, b := range backups {
if st, _ := os.Stat(b); strings.Contains(b, ".key.") && st.Mode().Perm() != 0o600 {
t.Fatalf("kept key %s has mode %v", b, st.Mode().Perm())
}
old, _ := os.ReadFile(b)
if strings.Contains(b, ".crt.") {
oc, _ := x509.ParseCertificate(mustCertDER(old))
if oc == nil || !certCovers(oc, oldNames.panel) {
t.Fatalf("kept certificate is not the old one")
}
}
}
// The Secrets carry the files.
for _, ns := range []string{"felis", "minecraft"} {
if got := rig.secret(t, ns, platform.ConfigSecretName)[platform.ConfigSecretKey]; string(got) != pod {
t.Fatalf("%s/felis-config is not felis.pod.toml", ns)
}
}
tlsData := rig.secret(t, "felis", platform.APITLSSecretName)
if string(tlsData[corev1.TLSCertKey]) != rig.read(t, "panel-tls.crt") || string(tlsData[corev1.TLSPrivateKeyKey]) != rig.read(t, "panel-tls.key") {
t.Fatal("felis-api-tls does not hold the new pair")
}
// The login gate's env moved and nothing else did.
env := rig.crEnv(t, naming.SystemLoginServer)
if env[envRootDomain] != newNames.root || env[envPanelHostname] != newNames.panel || env[envAPIBaseURL] != platform.InternalAPIBaseURL("felis") {
t.Fatalf("login env %v", env)
}
// The proxy's file: the three keys moved, its token and mode did not.
props := rig.read(t, "felis-link.properties")
if want := strings.NewReplacer("old.example", "new.example").Replace(linkPropsBody); props != want {
t.Fatalf("felis-link.properties:\n%s", props)
}
if st, _ := os.Stat(rig.path("felis-link.properties")); st.Mode().Perm() != 0o640 {
t.Fatalf("felis-link.properties mode %v", st.Mode().Perm())
}
if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
t.Fatalf("events %v", rig.events)
}
if !strings.Contains(rig.out.String(), "Every surface is on new.example.") {
t.Fatalf("the closing check did not pass:\n%s", rig.out)
}
}
func TestDomainSetWithoutYesChangesNothing(t *testing.T) {
rig := newDomainRig(t)
before := rig.snapshot(t)
code, err := rig.h.set(context.Background(), "new.example", false)
if err != nil || code != 0 {
t.Fatalf("set = %d, %v", code, err)
}
if rig.snapshot(t) != before || len(rig.events) != 0 {
t.Fatalf("a dry run changed something (events %v)", rig.events)
}
out := rig.out.String()
for _, want := range []string{
"console.old.example → console.new.example",
"3 passkey(s) of 2 user(s) are bound to console.old.example",
"does not cover op.console.new.example",
"No [smtp] relay is configured",
"sudo felis domain set -yes new.example",
} {
if !strings.Contains(out, want) {
t.Errorf("plan lacks %q:\n%s", want, out)
}
}
}
func TestDomainSetKeepsAHandSetPanelHostname(t *testing.T) {
rig := newDomainRig(t)
for _, f := range []string{"felis.host.toml", "felis.pod.toml"} {
writeTestFile(t, rig.path(f), strings.Replace(rig.read(t, f), `panel_hostname = "console.old.example"`, `panel_hostname = "play.corp.net"`, 1), 0o600)
}
code, err := rig.h.set(context.Background(), "new.example", true)
if err != nil {
t.Fatalf("set: %v\n%s", err, rig.out)
}
got, _ := tomlDomainNames([]byte(rig.read(t, "felis.host.toml")))
if got != (domainNames{root: "new.example", panel: "play.corp.net", admin: "op.console.new.example"}) {
t.Fatalf("names %+v", got)
}
c, _ := readCertFile(rig.path("panel-tls.crt"))
if !certCovers(c, "play.corp.net", "op.console.new.example") {
t.Fatalf("certificate names %v", c.DNSNames)
}
if env := rig.crEnv(t, naming.SystemLoginServer); env[envPanelHostname] != "play.corp.net" {
t.Fatalf("login env %v", env)
}
if code != 0 || !strings.Contains(rig.out.String(), "play.corp.net (set by hand, kept") {
t.Fatalf("code %d:\n%s", code, rig.out)
}
}
func TestDomainSetRefusesAnOperatorCertificateForOtherNames(t *testing.T) {
rig := newDomainRig(t)
certPEM, keyPEM := caSignedCert(t, "console.old.example", "op.console.old.example")
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
before := rig.snapshot(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "not issued by Felis") {
t.Fatalf("err = %v", err)
}
if rig.snapshot(t) != before || len(rig.events) != 0 {
t.Fatal("a refused move changed something")
}
// The operator's certificate for the new names is kept as it is.
certPEM, keyPEM = caSignedCert(t, "console.new.example", "op.console.new.example")
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatalf("set: %v", err)
}
if rig.read(t, "panel-tls.crt") != string(certPEM) {
t.Fatal("the operator's certificate was replaced")
}
}
func TestDomainSetRefusesAConfigItCannotEdit(t *testing.T) {
rig := newDomainRig(t)
writeTestFile(t, rig.path("felis.pod.toml"), strings.Replace(rig.read(t, "felis.pod.toml"), "[auth]", "[\"auth\"]", 1), 0o600)
before := rig.snapshot(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "by hand") {
t.Fatalf("err = %v", err)
}
if rig.snapshot(t) != before || len(rig.events) != 0 {
t.Fatal("a refused move changed something")
}
}
func TestDomainSetAgainOnlyConvergesWhatIsBehind(t *testing.T) {
rig := newDomainRig(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
rig.events, rig.out = nil, &bytes.Buffer{}
rig.h.out = rig.out
before := rig.snapshot(t)
code, err := rig.h.set(context.Background(), "new.example", true)
if err != nil || code != 0 {
t.Fatalf("second set = %d, %v\n%s", code, err, rig.out)
}
if len(rig.events) != 0 || rig.snapshot(t) != before {
t.Fatalf("a converged install was touched again: %v\n%s", rig.events, rig.out)
}
// A proxy that was not restarted after the move is restarted by a re-run, and
// an api still on the old config is rolled.
rig.proxySince = time.Now().Add(-time.Hour)
rig.served = oldNames
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
t.Fatalf("events %v", rig.events)
}
// An api on the new names that still presents the old certificate is rolled.
rig.events = nil
oldCert, _, _ := issuePanelCert(oldNames, nil, time.Now())
rig.servedCert, _ = x509.ParseCertificate(mustCertDER(oldCert))
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
if strings.Join(rig.events, ",") != "roll-api" {
t.Fatalf("events %v", rig.events)
}
}
func TestDomainSetRefusesALoginServerItDoesNotOwn(t *testing.T) {
rig := newDomainRig(t)
var ms v1alpha1.MinecraftServer
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
t.Fatal(err)
}
delete(ms.Labels, v1alpha1.LabelSystemRole)
if err := rig.cl.Update(context.Background(), &ms); err != nil {
t.Fatal(err)
}
if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "system role") {
t.Fatalf("err = %v", err)
}
if env := rig.crEnv(t, naming.SystemLoginServer); env[envRootDomain] != oldNames.root {
t.Fatalf("a server not marked as the login gate was changed: %v", env)
}
}
func TestDomainCheckNamesTheSurfaceThatIsBehind(t *testing.T) {
cases := []struct {
surface string
breakIt func(t *testing.T, rig *domainRig)
}{
{"felis.pod.toml", func(t *testing.T, rig *domainRig) {
writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
}},
{"Secret minecraft/felis-config", func(t *testing.T, rig *domainRig) {
rig.putSecret(t, "minecraft", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
}},
{"Secret felis/felis-config", func(t *testing.T, rig *domainRig) {
rig.putSecret(t, "felis", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
}},
{"panel certificate", func(t *testing.T, rig *domainRig) {
// The Secret follows the file, so only the certificate's names are wrong.
certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
}},
{"Secret felis/felis-api-tls", func(t *testing.T, rig *domainRig) {
certPEM, _, _ := issuePanelCert(newNames, nil, time.Now())
rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
}},
{"felis-api", func(t *testing.T, rig *domainRig) { rig.served = oldNames }},
{"felis-api", func(t *testing.T, rig *domainRig) {
certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
}},
{"proxy", func(t *testing.T, rig *domainRig) {
writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
rig.proxySince = time.Now().Add(time.Hour)
}},
{"proxy", func(t *testing.T, rig *domainRig) { rig.proxySince = time.Now().Add(-time.Hour) }},
{"login gate", func(t *testing.T, rig *domainRig) {
var ms v1alpha1.MinecraftServer
_ = rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms)
for i := range ms.Spec.Env {
if ms.Spec.Env[i].Name == envRootDomain {
ms.Spec.Env[i].Value = "old.example"
}
}
if err := rig.cl.Update(context.Background(), &ms); err != nil {
t.Fatal(err)
}
}},
{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envRootDomain, "old.example") }},
{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envPanelHostname, "console.old.example") }},
{"Cloudflare tunnel", func(t *testing.T, rig *domainRig) {
writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n service: https://127.0.0.1:30443\n- hostname: op.console.old.example\n service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
}},
}
for _, tc := range cases {
rig := newDomainRig(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
rig.out.Reset()
tc.breakIt(t, rig)
if code := rig.h.check(context.Background()); code != 1 {
t.Errorf("%s behind: check = %d\n%s", tc.surface, code, rig.out)
continue
}
var failed []string
for _, ln := range strings.Split(rig.out.String(), "\n") {
if strings.HasPrefix(ln, " FAIL ") {
failed = append(failed, ln)
}
}
if len(failed) != 1 || !strings.Contains(failed[0], tc.surface) {
t.Errorf("%s behind: FAIL lines %q", tc.surface, failed)
}
}
}
func TestDomainCheckPassesAConvergedInstallAndWarnsOnDNS(t *testing.T) {
rig := newDomainRig(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n service: https://127.0.0.1:30443\n- hostname: op.console.new.example\n service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
rig.unresolved["op.console.new.example"] = true
rig.unresolved[dnsProbeLabel+".new.example"] = true
rig.out.Reset()
if code := rig.h.check(context.Background()); code != 0 {
t.Fatalf("check = %d\n%s", code, rig.out)
}
out := rig.out.String()
for _, want := range []string{" ok Cloudflare tunnel: routes both names", " warn DNS: op.console.new.example, *.new.example do not resolve from this host\n"} {
if !strings.Contains(out, want) {
t.Errorf("check lacks %q:\n%s", want, out)
}
}
if strings.Contains(out, "TOKEN-NOT-TO-TOUCH") {
t.Fatal("check printed the proxy's service token")
}
// A zone with only the wildcard: the admin name alone is missing, and why is said.
delete(rig.unresolved, dnsProbeLabel+".new.example")
rig.out.Reset()
rig.h.check(context.Background())
if want := " warn DNS: op.console.new.example does not resolve from this host: the *.new.example wildcard does not cover op.console.new.example, which needs its own record\n"; !strings.Contains(rig.out.String(), want) {
t.Errorf("check lacks %q:\n%s", want, rig.out)
}
}
func (rig *domainRig) setPodEnv(t *testing.T, name, value string) {
t.Helper()
var pod corev1.Pod
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
t.Fatal(err)
}
for i, e := range pod.Spec.Containers[0].Env {
if e.Name == name {
pod.Spec.Containers[0].Env[i].Value = value
}
}
if err := rig.cl.Update(context.Background(), &pod); err != nil {
t.Fatal(err)
}
}
func (rig *domainRig) putSecret(t *testing.T, ns, name, key, val string) {
t.Helper()
var s corev1.Secret
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
t.Fatal(err)
}
s.Data[key] = []byte(val)
if err := rig.cl.Update(context.Background(), &s); err != nil {
t.Fatal(err)
}
}
func TestParseUnitShow(t *testing.T) {
st := parseUnitShow("LoadState=loaded\nActiveState=active\nActiveEnterTimestamp=@1790000000\n")
if !st.loaded || !st.active || !st.since.Equal(time.Unix(1790000000, 0)) {
t.Fatalf("%+v", st)
}
st = parseUnitShow("LoadState=not-found\nActiveState=inactive\nActiveEnterTimestamp=\n")
if st.loaded || st.active || !st.since.IsZero() {
t.Fatalf("%+v", st)
}
}
cmd/felis/egressgate.go
+11 -22
View File
@@ -17,28 +17,23 @@ var (
egressPollInterval = 200 * time.Millisecond
)
// cmdEgressGate is the first initContainer of every build pod and the last of
// every game server pod. A pod's NetworkPolicy is programmed asynchronously after
// the pod starts (live on k3s: a build-labelled pod reached the internet and the
// Kubernetes API for its first ~0.7 s, a server-labelled one felis-api's internal
// face on its first request), so the gate dials a destination the policy denies
// until it stops answering, and only then lets the pod's next container, the
// untrusted Dockerfile or server image, start.
// cmdEgressGate is the first initContainer of every build pod. The pod's
// NetworkPolicy is programmed asynchronously after the pod starts (live on k3s:
// a build-labelled pod reached the internet and the Kubernetes API for its first
// ~0.7 s), so the gate dials a destination the policy denies until it stops
// answering, and only then lets the pod's next container, eventually the
// untrusted Dockerfile, start.
//
// The default probe is the Kubernetes API Service, which the kubelet names in
// every pod's environment and neither policy admits. A probe that still answers
// after --wait means the policy is not enforced at all (a CNI without
// every pod's environment and the build policy never admits. A probe that still
// answers after --wait means the policy is not enforced at all (a CNI without
// NetworkPolicy support, or k3s run with --disable-network-policy), and the
// build fails closed. A server passes --fail-open: an operator's
// --server-egress-allow-cidr may cover the node the API Service leads to, so a
// probe that keeps answering does not prove the fence is missing, and by then
// the policy has had --wait to land.
// build fails closed.
func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError)
fs.SetOutput(stderr)
probe := fs.String("probe", "", "host:port the pod's NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the gate gives up")
failOpen := fs.Bool("fail-open", false, "when --wait runs out, warn and let the pod go on instead of refusing it")
probe := fs.String("probe", "", "host:port the build NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the build is refused")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -61,12 +56,6 @@ func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
}
_ = conn.Close()
if time.Since(start) >= *wait {
if *failOpen {
fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s; starting anyway. Either this namespace's "+
"NetworkPolicy is not enforced (a CNI without NetworkPolicy support, or k3s started with "+
"--disable-network-policy), or an allowed CIDR admits the address behind it\n", *probe, *wait)
return 0
}
fmt.Fprintf(stderr, "felis egress-gate: %s still answers after %s: the build namespace's NetworkPolicy is not enforced "+
"(a CNI without NetworkPolicy support, or k3s started with --disable-network-policy); refusing to run the build\n",
*probe, *wait)
cmd/felis/egressgate_test.go
-34
View File
@@ -75,40 +75,6 @@ func TestEgressGateRefusesAnOpenNetwork(t *testing.T) {
}
}
// A server's gate waits out --wait all the same, then lets the pod start with a
// warning in its log.
func TestEgressGateFailOpenWaitsThenWarns(t *testing.T) {
shrinkEgressGate(t)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
_ = c.Close()
}
}()
var out, errb bytes.Buffer
start := time.Now()
if code := cmdEgressGate([]string{"--probe", ln.Addr().String(), "--wait", "150ms", "--fail-open"}, &out, &errb); code != 0 {
t.Fatalf("exit %d, want 0: %s", code, errb.String())
}
if waited := time.Since(start); waited < 150*time.Millisecond {
t.Errorf("gave up after %s, before --wait ran out", waited)
}
if !strings.Contains(errb.String(), ln.Addr().String()+" still answers after 150ms; starting anyway") {
t.Errorf("stderr = %q", errb.String())
}
if out.Len() != 0 {
t.Errorf("stdout = %q, want nothing: the lock was never seen", out.String())
}
}
func TestEgressGateDefaultsToTheKubernetesService(t *testing.T) {
shrinkEgressGate(t)
t.Setenv("KUBERNETES_SERVICE_HOST", "")
cmd/felis/export.go
-302
View File
@@ -1,302 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"hash"
"io"
"io/fs"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"felis.lolicon.best/internal/backup"
"felis.lolicon.best/internal/fileedit"
"felis.lolicon.best/internal/worldexport"
)
// cmdExport is the in-Pod entrypoint the export Job runs. internal/worldexport
// renders a Pod whose command is `/usr/local/bin/felis export`. It archives the
// mounted world, re-streams one archive from the mounted backup store, or sends
// one file or folder of the world, PUTs it to felis-api's internal face, and
// exits once felis-api says the owner's browser got all of it. It is NOT a
// user-facing command and is never invoked by hand.
//
// Like cmdRestore it holds no database credentials and never calls config.Load:
// felis-api made every decision (who may download what, that the server is
// stopped, which archive) before the Job existed. Its input is the flags below
// plus the one-time upload token in the environment, which opens this one
// export and nothing else.
//
// Whatever leaves goes through the same guards as the file editor
// (fileedit.Guard): the proxy forwarding secret, which every server on the
// install shares, never leaves, and server.properties leaves with its RCON
// password redacted. A backup is stored with both, since a restore must bring
// the world back whole, so it is filtered on the way out rather than handed
// over as stored.
//
// Exit status: 0 once felis-api answers 204 (the download completed), 1 when
// the export could not be read or handed over, a backup failed its digest
// check, or felis-api refused it (the browser never came or left early), 2 on
// bad flags. The last stderr line reaches the export's status and the jobs list.
func cmdExport(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("export", flag.ContinueOnError)
fs.SetOutput(stderr)
mode := fs.String("mode", "", "what to export: world, backup or files")
server := fs.String("server", "", "server name being exported (for logging)")
target := fs.String("target-url", "", "felis-api URL to PUT the export to")
ref := fs.String("ref", "", "backup only: absolute path to the archive on the backup mount")
backupRoot := fs.String("backup-root", "/backups", "backup only: mount path of the backup PVC (the ref must resolve under it)")
sum := fs.String("sha256", "", "backup only: the sha256 recorded when the archive was written; a mismatch fails the export before its end is sent")
worldsRoot := fs.String("worlds-root", "/world", "world and files: mount path of the world PVC")
path := fs.String("path", "", "files only: the file or folder to send, relative to the world root")
dir := fs.Bool("dir", false, "files only: the path is a folder, sent as a zip")
if err := fs.Parse(args); err != nil {
return 2
}
token := os.Getenv(worldexport.TokenEnv)
if *target == "" || token == "" {
fmt.Fprintf(stderr, "felis export: --target-url and %s are required\n", worldexport.TokenEnv)
return 2
}
limitHeapToCgroup()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
var err error
switch *mode {
case worldexport.ModeBackup:
if *ref == "" {
fmt.Fprintln(stderr, "felis export: --ref is required for a backup")
return 2
}
err = exportBackup(ctx, *target, token, *ref, *backupRoot, *sum, stdout)
case worldexport.ModeWorld:
err = exportWorld(ctx, *target, token, *worldsRoot, stdout)
case worldexport.ModeFiles:
if *path == "" {
fmt.Fprintln(stderr, "felis export: --path is required for files")
return 2
}
err = exportFiles(ctx, *target, token, *worldsRoot, *path, *dir, stdout)
default:
fmt.Fprintf(stderr, "felis export: --mode must be %s, %s or %s\n", worldexport.ModeWorld, worldexport.ModeBackup, worldexport.ModeFiles)
return 2
}
if err != nil {
fmt.Fprintf(stderr, "felis export: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "felis export: server=%s mode=%s downloaded\n", *server, *mode)
return 0
}
// archiveType is the content type of a world or backup export.
const archiveType = "application/gzip"
// errBackupDigest fails a backup export whose stored archive no longer hashes
// to what was recorded when it was written.
var errBackupDigest = errors.New("the backup archive does not match the sha256 recorded when it was written")
// exportBackup re-streams one stored archive through the export guards
// (backup.FilterTarGz with archiveFilter). Its length changes on the way, so it
// goes chunked. With want set, the stored bytes are hashed as they are read,
// and FilterTarGz reads them to their end before it closes its own archive: a
// mismatch aborts the upload while what felis-api has passed on still lacks
// its end, so the browser never keeps a complete-looking corrupt file.
func exportBackup(ctx context.Context, target, token, ref, root, want string, stdout io.Writer) error {
// Defense in depth, as in cmdRestore: the ref comes from felis-api, but this
// process opens it, so it confirms the ref stays on the backup mount.
if !refWithinRoot(ref, root) {
return fmt.Errorf("ref %q is not under backup root %q", ref, root)
}
f, err := os.Open(ref)
if err != nil {
return err
}
defer f.Close()
var src io.Reader = f
if want != "" {
src = &digestReader{r: f, sum: sha256.New(), want: want}
}
var withheld []string
err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
var err error
withheld, err = backup.FilterTarGz(ctx, w, src, archiveFilter)
return err
})
if errors.Is(err, errBackupDigest) {
return errBackupDigest // the jobs list shows it as it is, not wrapped as a read error
}
reportWithheld(stdout, len(withheld))
return err
}
// exportWorld archives the world straight into the request body: nothing is
// staged, so a world bigger than the Pod's memory or any scratch disk exports
// the same.
func exportWorld(ctx context.Context, target, token, root string, stdout io.Writer) error {
r, err := os.OpenRoot(root)
if err != nil {
return err
}
guard := fileedit.NewGuard(r)
r.Close()
var skipped, withheld []string
err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
var err error
skipped, withheld, err = backup.WriteTarGz(ctx, w, root, worldFilter(guard))
return err
})
if len(skipped) > 0 {
fmt.Fprintf(stdout, "felis export: left out %d entries a tar cannot hold (symbolic links, devices, sockets)\n", len(skipped))
}
reportWithheld(stdout, len(withheld))
return err
}
// exportFiles sends one file or folder of the world (fileedit.OpenDownload): a
// file with its exact length, a folder as a zip made as it streams. dir is
// what the owner saw at path when they asked.
func exportFiles(ctx context.Context, target, token, root, path string, dir bool, stdout io.Writer) error {
d, err := fileedit.OpenDownload(root, path, dir)
if err != nil {
return err
}
defer d.Close()
err = streamExport(ctx, target, token, d.ContentType, d.Size, func(w io.Writer) error { return d.WriteTo(ctx, w) })
if d.Skipped > 0 {
fmt.Fprintf(stdout, "felis export: left out %d entries a zip does not carry (symbolic links, devices, sockets)\n", d.Skipped)
}
reportWithheld(stdout, d.Withheld)
return err
}
func reportWithheld(stdout io.Writer, n int) {
if n > 0 {
fmt.Fprintf(stdout, "felis export: left out %d files that hold platform secrets\n", n)
}
}
// worldFilter guards a live world by file identity, so a link to a guarded
// file under another name is caught as well.
func worldFilter(g fileedit.Guard) backup.Filter {
return func(_ string, info fs.FileInfo) (bool, func([]byte) []byte) {
return guardAction(g.Rule(info))
}
}
// archiveFilter guards a stored archive, which has only names.
func archiveFilter(name string, _ fs.FileInfo) (bool, func([]byte) []byte) {
return guardAction(fileedit.ArchiveRule(name))
}
func guardAction(withhold, redact bool) (bool, func([]byte) []byte) {
if redact {
return withhold, fileedit.RedactProps
}
return withhold, nil
}
// digestReader passes r through, hashing it, and turns r's EOF into
// errBackupDigest when the bytes do not hash to want.
type digestReader struct {
r io.Reader
sum hash.Hash
want string
}
func (d *digestReader) Read(p []byte) (int, error) {
n, err := d.r.Read(p)
d.sum.Write(p[:n])
if err == io.EOF && !strings.EqualFold(hex.EncodeToString(d.sum.Sum(nil)), d.want) {
return n, errBackupDigest
}
return n, err
}
// streamExport runs write straight into the body of the PUT, hashing it as it
// goes. Once write has finished, the SHA-256 of all it wrote rides the
// request's trailer (worldexport.DigestTrailer), and felis-api holds back the
// last bytes from the browser until what it received hashes the same. An error
// from write aborts the chunked body before the trailer, and felis-api then
// cuts the browser's download off rather than end it; that error is the one
// reported, since the PUT's own error only wraps it. When the PUT ends first,
// write is stopped.
func streamExport(ctx context.Context, target, token, contentType string, size int64, write func(io.Writer) error) error {
pr, pw := io.Pipe()
trailer := http.Header{worldexport.DigestTrailer: nil}
werr := make(chan error, 1)
go func() {
sum := sha256.New()
err := write(io.MultiWriter(pw, sum))
if err == nil {
// Set before the body ends: the transport reads the trailer once it
// has read the body to its end.
trailer.Set(worldexport.DigestTrailer, "sha-256=:"+base64.StdEncoding.EncodeToString(sum.Sum(nil))+":")
}
pw.CloseWithError(err)
werr <- err
}()
err := putExport(ctx, target, token, contentType, pr, size, trailer)
pr.CloseWithError(io.ErrClosedPipe)
if w := <-werr; w != nil && !errors.Is(w, io.ErrClosedPipe) {
return w
}
return err
}
// putExport PUTs the export to felis-api. There is no retry: the token opens
// the export once, so a second attempt could only be refused. Redirects are
// refused because the request carries the token and the internal face never
// redirects. felis-api answers only after the whole download, which the Job's
// activeDeadlineSeconds bounds, so the header timeout is a backstop for a
// wedged endpoint and not the real limit.
//
// The body always goes chunked, which is what lets it end with a trailer; a
// size the Job knows (-1 when it does not) goes as worldexport.LengthHeader in
// place of Content-Length.
func putExport(ctx context.Context, target, token, contentType string, body io.Reader, size int64, trailer http.Header) error {
req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body)
if err != nil {
return err
}
req.ContentLength = -1
req.Trailer = trailer
if size >= 0 {
req.Header.Set(worldexport.LengthHeader, strconv.FormatInt(size, 10))
}
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", contentType)
client := &http.Client{
Transport: &http.Transport{ResponseHeaderTimeout: 10 * time.Minute},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
return nil
}
var e struct {
Error struct {
Message string `json:"message"`
} `json:"error"`
}
if json.NewDecoder(io.LimitReader(resp.Body, 4<<10)).Decode(&e) == nil && e.Error.Message != "" {
return fmt.Errorf("felis-api answered %s: %s", resp.Status, e.Error.Message)
}
return fmt.Errorf("felis-api answered %s", resp.Status)
}
cmd/felis/export_test.go
-532
View File
@@ -1,532 +0,0 @@
package main
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"slices"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/worldexport"
)
// exportReceiver stands in for felis-api's internal upload route: it records
// the PUT it gets (or the error reading it ended on) and answers with reply.
type exportReceiver struct {
srv *httptest.Server
hits atomic.Int32
req *http.Request
body []byte
readErr error
served chan struct{} // one send per request, once it is answered
}
func receiveExport(t *testing.T, reply func(w http.ResponseWriter)) *exportReceiver {
t.Helper()
rcv := &exportReceiver{served: make(chan struct{}, 1)}
rcv.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rcv.hits.Add(1)
rcv.req = r
rcv.body, rcv.readErr = io.ReadAll(r.Body)
reply(w)
rcv.served <- struct{}{}
}))
t.Cleanup(rcv.srv.Close)
return rcv
}
func noContent(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) }
// sentWhole fails unless the upload rcv got ended with the Content-Digest
// trailer of its own bytes, and declared length as its size (-1: none).
func sentWhole(t *testing.T, rcv *exportReceiver, length int64) {
t.Helper()
sum := sha256.Sum256(rcv.body)
want := "sha-256=:" + base64.StdEncoding.EncodeToString(sum[:]) + ":"
wantLength := ""
if length >= 0 {
wantLength = strconv.FormatInt(length, 10)
}
r := rcv.req
if rcv.readErr != nil || r.Trailer.Get(worldexport.DigestTrailer) != want || r.Header.Get(worldexport.LengthHeader) != wantLength ||
r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
t.Fatalf("upload read %v, trailer %v, %s %q, length %d, encoding %v; want trailer %q and %s %q, chunked",
rcv.readErr, r.Trailer, worldexport.LengthHeader, r.Header.Get(worldexport.LengthHeader), r.ContentLength, r.TransferEncoding,
want, worldexport.LengthHeader, wantLength)
}
}
func tarEntries(t *testing.T, archive []byte) map[string]string {
t.Helper()
gz, err := gzip.NewReader(bytes.NewReader(archive))
if err != nil {
t.Fatalf("not gzip: %v", err)
}
out := map[string]string{}
tr := tar.NewReader(gz)
for {
h, err := tr.Next()
if err == io.EOF {
return out
}
if err != nil {
t.Fatalf("tar: %v", err)
}
b, _ := io.ReadAll(tr)
out[h.Name] = string(b)
}
}
// writeTree writes name → body under root, making the folders on the way.
func writeTree(t *testing.T, root string, files map[string]string) {
t.Helper()
for name, body := range files {
p := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
}
// The two secrets a world holds, and what server.properties reads as once
// redacted.
const (
secretProps = "motd=hi\nrcon.password=hunter2\n"
redactedProps = "motd=hi\nrcon.password=<redacted by felis>\n"
forwardingKey = "secret: aVeryRealForwardingKey\n"
)
// secretWorld is a world holding both secrets, with a hard link to the
// forwarding secret under a name nothing would guard by.
func secretWorld(t *testing.T) string {
t.Helper()
root := t.TempDir()
writeTree(t, root, map[string]string{
"server.properties": secretProps,
"config/paper-global.yml": forwardingKey,
"world/region/r.0.0.mca": "chunks",
})
if err := os.MkdirAll(filepath.Join(root, "plugins"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(root, "plugins/copy.yml")); err != nil {
t.Fatal(err)
}
return root
}
func TestCmdExportWorld(t *testing.T) {
root := secretWorld(t)
if err := os.Symlink("server.properties", filepath.Join(root, "props-link")); err != nil {
t.Fatal(err)
}
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
code := cmdExport([]string{"--mode", "world", "--server", "survival", "--target-url", rcv.srv.URL + "/api/v1/internal/exports/ab",
"--worlds-root", root}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
r := rcv.req
if r.Method != http.MethodPut || r.URL.Path != "/api/v1/internal/exports/ab" || r.Header.Get("Authorization") != "Bearer tok" ||
r.Header.Get("Content-Type") != "application/gzip" || r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
}
want := map[string]string{
"server.properties": redactedProps, "config/": "", "plugins/": "",
"world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks",
}
if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
t.Fatalf("archive holds %v\nwant %v", got, want)
}
sentWhole(t, rcv, -1)
want2 := "felis export: left out 1 entries a tar cannot hold (symbolic links, devices, sockets)\n" +
"felis export: left out 2 files that hold platform secrets\n" +
"felis export: server=survival mode=world downloaded\n"
if stdout.String() != want2 {
t.Errorf("stdout = %q, want %q", stdout.String(), want2)
}
}
// A world root that cannot be opened fails before anything reaches felis-api.
func TestCmdExportWorldUnreadable(t *testing.T) {
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
if code != 1 || rcv.hits.Load() != 0 {
t.Fatalf("exit %d with %d requests, want 1 and none", code, rcv.hits.Load())
}
}
// An export that fails part-way must never reach felis-api as a complete body:
// the chunked upload is cut off, so felis-api aborts the browser's download,
// and the failure itself is what the Job reports.
func TestStreamExportWriteErrorAbortsTheUpload(t *testing.T) {
broken := errors.New("disk read failed")
rcv := receiveExport(t, noContent)
err := streamExport(context.Background(), rcv.srv.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
if _, err := w.Write(bytes.Repeat([]byte("x"), 100_000)); err != nil {
return err
}
return broken
})
if err != broken {
t.Fatalf("err = %v, want the write's own error, unwrapped", err)
}
select {
case <-rcv.served:
if rcv.readErr == nil {
t.Fatalf("felis-api read a complete %d-byte body from a failed export", len(rcv.body))
}
case <-time.After(5 * time.Second):
t.Fatal("the request never reached felis-api")
}
}
// When felis-api refuses first, its reason is reported, not the closed pipe
// that then stops the writer.
func TestStreamExportRefusalStopsTheWriter(t *testing.T) {
refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer refusing.Close()
stopped := make(chan error, 1)
err := streamExport(context.Background(), refusing.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
for {
if _, err := w.Write(make([]byte, 32<<10)); err != nil {
stopped <- err
return err
}
}
})
if err == nil || err.Error() != "felis-api answered 404 Not Found" {
t.Fatalf("err = %v, want felis-api's answer", err)
}
if werr := <-stopped; !errors.Is(werr, io.ErrClosedPipe) {
t.Fatalf("the writer stopped on %v, want the closed pipe", werr)
}
// A PUT that never starts leaves no transport to close the body: the writer
// is still stopped, and the export fails rather than hangs.
done := make(chan error, 1)
go func() {
done <- streamExport(context.Background(), "http://[::1", "tok", "application/gzip", -1, func(w io.Writer) error {
_, err := w.Write([]byte("x"))
return err
})
}()
select {
case err := <-done:
if err == nil || !strings.Contains(err.Error(), "missing ']'") {
t.Fatalf("err = %v, want the bad URL", err)
}
case <-time.After(5 * time.Second):
t.Fatal("an export whose PUT never started hung")
}
}
// storedBackup writes, at path, a gzip+tar like one the backup store holds:
// the world whole, both secrets included, and a region file that does not
// compress. It returns the archive's sha256.
func storedBackup(t *testing.T, path string) string {
t.Helper()
region := make([]byte, 64<<10)
if _, err := rand.Read(region); err != nil {
t.Fatal(err)
}
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for _, e := range []struct{ name, body string }{
{"server.properties", secretProps},
{"config/paper-global.yml", forwardingKey},
{"world/level.dat", "level"},
{"world/region/r.0.0.mca", string(region)},
} {
if err := tw.WriteHeader(&tar.Header{Name: e.name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(e.body))}); err != nil {
t.Fatal(err)
}
if _, err := io.WriteString(tw, e.body); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(buf.Bytes())
return hex.EncodeToString(sum[:])
}
func TestCmdExportBackup(t *testing.T) {
root := t.TempDir()
ref := filepath.Join(root, "survival-1.tar.gz")
sum := storedBackup(t, ref)
stored, err := os.ReadFile(ref)
if err != nil {
t.Fatal(err)
}
region := tarEntries(t, stored)["world/region/r.0.0.mca"]
args := func(url, ref string) []string {
return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
}
for name, extra := range map[string][]string{
"no digest recorded": nil,
"recorded digest matches": {"--sha256", sum},
} {
t.Run(name+": re-streamed through the guards", func(t *testing.T) {
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(append(args(rcv.srv.URL, ref), extra...), &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
r := rcv.req
if r.ContentLength != -1 || r.Header.Get("Content-Type") != "application/gzip" || r.Header.Get("Authorization") != "Bearer tok" {
t.Fatalf("length %d, headers %v", r.ContentLength, r.Header)
}
want := map[string]string{"server.properties": redactedProps, "world/level.dat": "level", "world/region/r.0.0.mca": region}
if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
t.Fatalf("archive holds %d entries, want exactly the redacted properties, level.dat and the region file", len(got))
}
sentWhole(t, rcv, -1)
if want := "felis export: left out 1 files that hold platform secrets\nfelis export: server=survival mode=backup downloaded\n"; stdout.String() != want {
t.Errorf("stdout = %q, want %q", stdout.String(), want)
}
})
}
// The stored bytes are checked as they stream, and the archive the Job sends
// is only closed once they are all read: a mismatch cuts the upload off
// short of its end, so felis-api never passes on a complete-looking copy.
t.Run("a digest mismatch cuts the upload off before its end", func(t *testing.T) {
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(append(args(rcv.srv.URL, ref), "--sha256", strings.Repeat("ab", 32)), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if want := "felis export: the backup archive does not match the sha256 recorded when it was written\n"; stderr.String() != want {
t.Fatalf("stderr = %q, want %q", stderr.String(), want)
}
select {
case <-rcv.served:
case <-time.After(5 * time.Second):
t.Fatal("the upload never reached felis-api")
}
if rcv.readErr == nil {
t.Fatalf("felis-api read a complete %d-byte body", len(rcv.body))
}
zr, err := gzip.NewReader(bytes.NewReader(rcv.body))
if err == nil {
_, err = io.ReadAll(zr)
}
if err == nil {
t.Fatal("what felis-api got is a complete archive")
}
})
t.Run("a ref outside the backup root is refused before any request", func(t *testing.T) {
outside := filepath.Join(t.TempDir(), "secret.tar.gz")
if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil {
t.Fatal(err)
}
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(rcv.srv.URL, outside), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if n := rcv.hits.Load(); n != 0 {
t.Fatalf("felis-api got %d requests", n)
}
})
t.Run("a refusal exits 1 with felis-api's reason", func(t *testing.T) {
rcv := receiveExport(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusGone)
io.WriteString(w, `{"error":{"code":"export_expired","message":"nobody opened the download"}}`)
})
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if got := stderr.String(); got != "felis export: felis-api answered 410 Gone: nobody opened the download\n" {
t.Fatalf("stderr = %q", got)
}
})
// The request carries the token and the internal face never redirects, so a
// redirect is refused rather than followed with the token attached. A 302 or
// 303 is the one net/http would follow on its own (as a GET, and to the same
// host with the Authorization header still on it).
for _, status := range []int{http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
t.Run("a redirect is not followed: "+strconv.Itoa(status), func(t *testing.T) {
elsewhere := receiveExport(t, noContent)
redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.srv.URL, status))
defer redirecting.Close()
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(redirecting.URL, ref), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if n := elsewhere.hits.Load(); n != 0 {
t.Fatalf("the redirect target got %d requests", n)
}
if got, want := stderr.String(), "felis export: felis-api answered "+strconv.Itoa(status)+" "+http.StatusText(status)+"\n"; got != want {
t.Fatalf("stderr = %q, want %q", got, want)
}
})
}
}
func TestCmdExportFiles(t *testing.T) {
root := secretWorld(t)
writeTree(t, root, map[string]string{"plugins/Essentials/config.yml": "x: 1"})
if err := os.Symlink("config.yml", filepath.Join(root, "plugins/Essentials/link.yml")); err != nil {
t.Fatal(err)
}
export := func(t *testing.T, path string, dir bool) (*exportReceiver, int, string, string) {
t.Helper()
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
args := []string{"--mode", "files", "--server", "survival", "--target-url", rcv.srv.URL, "--worlds-root", root, "--path", path}
if dir {
args = append(args, "--dir")
}
var stdout, stderr bytes.Buffer
code := cmdExport(args, &stdout, &stderr)
return rcv, code, stdout.String(), stderr.String()
}
for path, want := range map[string]string{
"world/region/r.0.0.mca": "chunks",
"server.properties": redactedProps,
} {
t.Run("a file goes with its exact length: "+path, func(t *testing.T) {
rcv, code, stdout, stderr := export(t, path, false)
if code != 0 || stdout != "felis export: server=survival mode=files downloaded\n" {
t.Fatalf("exit %d, stdout %q, stderr %q", code, stdout, stderr)
}
if string(rcv.body) != want || rcv.req.Header.Get("Content-Type") != "application/octet-stream" {
t.Fatalf("body %q, type %q; want %q", rcv.body, rcv.req.Header.Get("Content-Type"), want)
}
sentWhole(t, rcv, int64(len(want)))
})
}
t.Run("a folder goes as a zip, guarded", func(t *testing.T) {
rcv, code, stdout, stderr := export(t, "plugins", true)
if code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr)
}
if rcv.req.ContentLength != -1 || rcv.req.Header.Get("Content-Type") != "application/zip" {
t.Fatalf("length %d, type %q", rcv.req.ContentLength, rcv.req.Header.Get("Content-Type"))
}
zr, err := zip.NewReader(bytes.NewReader(rcv.body), int64(len(rcv.body)))
if err != nil {
t.Fatal(err)
}
var names []string
for _, f := range zr.File {
names = append(names, f.Name)
}
if want := []string{"plugins/", "plugins/Essentials/", "plugins/Essentials/config.yml"}; !slices.Equal(names, want) {
t.Fatalf("zip holds %v, want %v", names, want)
}
want := "felis export: left out 1 entries a zip does not carry (symbolic links, devices, sockets)\n" +
"felis export: left out 1 files that hold platform secrets\n" +
"felis export: server=survival mode=files downloaded\n"
if stdout != want {
t.Errorf("stdout = %q, want %q", stdout, want)
}
})
for _, c := range []struct {
path string
dir bool
want string
}{
{"config/paper-global.yml", false, "forwarding secret"},
{"plugins/copy.yml", false, "forwarding secret"},
{"plugins", false, "is a folder now"},
{"server.properties", true, "is not a folder now"},
} {
t.Run("refused before any request: "+c.path, func(t *testing.T) {
rcv, code, _, stderr := export(t, c.path, c.dir)
if code != 1 || rcv.hits.Load() != 0 || !strings.Contains(stderr, c.want) {
t.Fatalf("exit %d, %d requests, stderr %q; want 1, none, and %q", code, rcv.hits.Load(), stderr, c.want)
}
})
}
}
func TestCmdExportUsage(t *testing.T) {
for name, tc := range map[string]struct {
token string
args []string
}{
"no token": {"", []string{"--mode", "world", "--target-url", "http://api/x"}},
"no target": {"tok", []string{"--mode", "world"}},
"unknown mode": {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
"files without path": {"tok", []string{"--mode", "files", "--target-url", "http://api/x"}},
} {
t.Run(name, func(t *testing.T) {
t.Setenv(worldexport.TokenEnv, tc.token)
var stdout, stderr bytes.Buffer
if code := cmdExport(tc.args, &stdout, &stderr); code != 2 {
t.Fatalf("exit %d, want 2; stderr %q", code, stderr.String())
}
})
}
}
// TestCmdExportWiring: the Job's `felis export` reaches cmdExport, and
// felis-api's executor mounts the backup store at the path the archives were
// written under, since a backup's ref is an absolute path there.
func TestCmdExportWiring(t *testing.T) {
t.Setenv(worldexport.TokenEnv, "")
var stdout, stderr bytes.Buffer
if code := run([]string{"export"}, &stdout, &stderr); code != 2 ||
stderr.String() != "felis export: --target-url and "+worldexport.TokenEnv+" are required\n" {
t.Fatalf("felis export = %d, stderr %q", code, stderr.String())
}
cfg := &config.Config{}
cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups"
want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"}
if got := exportConfig(cfg, "felis:1", "felis-backups"); got != want {
t.Fatalf("exportConfig = %+v, want %+v", got, want)
}
}
cmd/felis/files.go
+19 -113
View File
@@ -1,15 +1,11 @@
package main
import (
"context"
"encoding/base64"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"felis.lolicon.best/internal/fileedit"
)
@@ -23,44 +19,32 @@ import (
// Like cmdRestore it deliberately holds NO database credentials and never calls
// config.Load: felis-api made the authorization decision (the caller owns this
// server, and the server is stopped so the RWO world volume is free); this process
// is the unprivileged hands that touch bytes. Its entire input is the flags
// below plus, for a write, the content variables and, for an upload, one token.
// Every isolation guarantee lives in the Pod spec (internal/fileedit/jobspec.go),
// and the path-containment guarantee lives in fileedit.Execute, which resolves
// every path through os.Root and therefore cannot be walked out of the world
// mount.
// is the unprivileged hands that touch bytes. Its entire input is the three flags
// below plus, for a write, one environment variable. Every isolation guarantee
// lives in the Pod spec (internal/fileedit/jobspec.go), and the path-containment
// guarantee lives in fileedit.Execute, which resolves the path through os.Root and
// therefore cannot be walked out of the world mount.
//
// Exit status carries a specific meaning that felis-api depends on: a CALLER-fault
// outcome — a path that escapes the root, a file that is missing or too large — is
// a SUCCESSFUL run that prints a Result carrying an error code, so the API can map
// it to a precise 4xx. A non-zero exit means the operation could not be attempted
// at all (the world mount is unreadable, an upload's bytes could not be fetched
// intact, the result unprintable), which the API reports as a 500.
// at all (the world mount is unreadable, the result unprintable), which the API
// reports as a 500.
func cmdFiles(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("files", flag.ContinueOnError)
fs.SetOutput(stderr)
op := fs.String("op", "", "operation: list, read, write, mkdir, delete, rename, upload or unzip")
op := fs.String("op", "", "operation: list, read, or write")
path := fs.String("path", "", "path to operate on, relative to the world root (empty = the root itself)")
worldsRoot := fs.String("worlds-root", "/data", "mount path of the world PVC; every path resolves under it")
expect := fs.String("expect-sha256", "", "write only: refuse unless the file's current SHA-256 (hex) is this")
createOnly := fs.Bool("create-only", false, "write only: refuse a path that already exists")
to := fs.String("to", "", "rename only: the destination path")
sourceURL := fs.String("source-url", "", "upload only: felis-api URL to fetch the bytes from")
size := fs.Int64("size", -1, "upload only: the byte count the fetched file must have")
sum := fs.String("sha256", "", "write and upload: the SHA-256 (hex) the content or the fetched file must have")
overwrite := fs.Bool("overwrite", false, "upload and unzip: replace files already there")
if err := fs.Parse(args); err != nil {
return 2
}
if *op == "" {
fmt.Fprintln(stderr, "felis files: --op is required")
fmt.Fprintln(stderr, "felis files: --op is required (list, read, or write)")
return 2
}
limitHeapToCgroup()
req := fileedit.Request{
Op: *op, Path: *path, To: *to, Expect: *expect, CreateOnly: *createOnly, Overwrite: *overwrite,
}
// New content arrives base64-encoded in the environment rather than in argv:
// a process's arguments are world-readable on the node (/proc/<pid>/cmdline),
@@ -68,46 +52,22 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
// secrets — an RCON password in server.properties is the obvious case. The
// encoding is what lets arbitrary bytes (CRLF endings, a BOM, a NUL) survive a
// channel that must be a valid string.
switch *op {
case fileedit.OpWrite:
// The content's SHA-256 comes with it, so bytes that changed on the way
// to this Job are refused rather than written (Request.ContentSHA256).
if *sum == "" {
fmt.Fprintln(stderr, "felis files: a write needs --sha256")
var content []byte
if *op == fileedit.OpWrite {
raw, ok := os.LookupEnv(fileedit.ContentEnv)
if !ok {
fmt.Fprintf(stderr, "felis files: a write needs %s in the environment\n", fileedit.ContentEnv)
return 2
}
content, err := fileedit.ContentFromEnv(os.LookupEnv)
decoded, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
fmt.Fprintf(stderr, "felis files: %v\n", err)
fmt.Fprintf(stderr, "felis files: %s is not valid base64: %v\n", fileedit.ContentEnv, err)
return 2
}
req.Content, req.ContentSHA256 = content, *sum
case fileedit.OpUpload:
token := os.Getenv(fileedit.UploadTokenEnv)
if *sourceURL == "" || token == "" {
fmt.Fprintf(stderr, "felis files: an upload needs --source-url and %s\n", fileedit.UploadTokenEnv)
return 2
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
req.Upload = &fileedit.Upload{
Size: *size, SHA256: *sum,
Open: func() (io.ReadCloser, error) { return fetchUpload(ctx, *sourceURL, token) },
Landed: func() {
if err := reportLanded(ctx, *sourceURL, token); err != nil {
// The file is in place; felis-api drops its copy when it
// has sat idle long enough, and the panel cancels it too.
fmt.Fprintf(stderr, "felis files: tell felis-api the upload landed: %v\n", err)
}
},
}
content = decoded
}
// An upload or an unzip (the only ops that report progress) can run long
// enough that felis-api does not wait on its Job, and the panel shows how far
// it has got from the latest of these lines (fileedit.K8sRunner.Ops).
req.Progress = fileedit.ThrottledProgress(stdout, time.Second, time.Now)
res, err := fileedit.Execute(*worldsRoot, req)
res, err := fileedit.Execute(*worldsRoot, *op, *path, content)
if err != nil {
// The operation could not be attempted — infrastructure, not caller fault.
fmt.Fprintf(stderr, "felis files: %v\n", err)
@@ -122,57 +82,3 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
}
return 0
}
// fetchUpload opens the staged upload on felis-api's internal face. There is no
// retry: the token opens the upload once (fileedit.Stage), so a second attempt
// could only be refused, and the caller retries the failed Job whole (a file
// sent in parts stays staged until its Job reports it landed, so that retry
// does not send it again). Redirects are refused because the request carries the
// token and the internal face never redirects; the header timeout catches a
// wedged endpoint, and the Job's activeDeadlineSeconds bounds the body.
func fetchUpload(ctx context.Context, url, token string) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+token)
client := &http.Client{
Transport: &http.Transport{ResponseHeaderTimeout: 30 * time.Second},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("GET returned %s", resp.Status)
}
return resp.Body, nil
}
// reportLanded tells felis-api the upload's file is in place (DELETE on the URL
// it was fetched from, with the same token), so it deletes the copy it staged.
// One try: the file has landed whatever the answer, and a copy nobody deletes
// is dropped once it has sat idle for fileedit.SessionIdle.
func reportLanded(ctx context.Context, url, token string) error {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+token)
client := &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
resp, err := client.Do(req)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
return fmt.Errorf("DELETE returned %s", resp.Status)
}
return nil
}
cmd/felis/files_test.go
-358
View File
@@ -1,358 +0,0 @@
package main
import (
"archive/zip"
"bytes"
"cmp"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"felis.lolicon.best/internal/fileedit"
)
// filesResult is the Result a `felis files` run printed on its marked line,
// the last it prints.
func filesResult(t *testing.T, stdout string) fileedit.Result {
t.Helper()
lines := strings.Split(strings.TrimSpace(stdout), "\n")
line, ok := strings.CutPrefix(lines[len(lines)-1], fileedit.ResultPrefix)
if !ok {
t.Fatalf("stdout has no result line: %q", stdout)
}
var res fileedit.Result
if err := json.Unmarshal([]byte(line), &res); err != nil {
t.Fatalf("result line %q: %v", line, err)
}
return res
}
// stagedSource is felis-api's internal face for one staged upload. It serves
// body to a GET carrying Bearer token and 404 to any other, and answers the
// DELETE that reports the file landed with landedCode (204 when unset),
// redirecting to landedTo when that is a redirect. reports counts those
// DELETEs, each with the token and at the path the bytes came from.
type stagedSource struct {
*httptest.Server
reports, strays atomic.Int32
landedCode int
landedTo string
}
func stagedUpload(t *testing.T, token string, body []byte) *stagedSource {
t.Helper()
s := &stagedSource{}
s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer "+token || r.URL.Path != "/u" {
s.strays.Add(1)
http.Error(w, "no such upload", http.StatusNotFound)
return
}
switch r.Method {
case http.MethodGet:
w.Write(body)
case http.MethodDelete:
s.reports.Add(1)
if s.landedTo != "" {
w.Header().Set("Location", s.landedTo)
}
w.WriteHeader(cmp.Or(s.landedCode, http.StatusNoContent))
default:
s.strays.Add(1)
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}))
t.Cleanup(s.Close)
return s
}
func uploadArgs(root, sourceURL string, body []byte) []string {
sum := sha256.Sum256(body)
return []string{
"--op", "upload", "--path", "plugins/a.jar", "--worlds-root", root,
"--source-url", sourceURL, "--size", "4", "--sha256", hex.EncodeToString(sum[:]),
}
}
// uploadRoot is a world with the plugins folder an upload lands in.
func uploadRoot(t *testing.T) string {
t.Helper()
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "plugins"), 0o755); err != nil {
t.Fatal(err)
}
return root
}
func TestCmdFilesUpload(t *testing.T) {
body := []byte("PK\x03\x04")
t.Run("fetches the staged bytes with its token and lands them", func(t *testing.T) {
root := uploadRoot(t)
srv := stagedUpload(t, "tok", body)
t.Setenv(fileedit.UploadTokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix+`{"done":4,"total":4}`+"\n") {
t.Fatalf("stdout %q does not start with the progress to the last byte", stdout.String())
}
if res := filesResult(t, stdout.String()); res.Code != "" {
t.Fatalf("result = %+v", res)
}
got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar"))
if err != nil || !bytes.Equal(got, body) {
t.Fatalf("landed %q, %v", got, err)
}
if n, strays := srv.reports.Load(), srv.strays.Load(); n != 1 || strays != 0 || stderr.Len() != 0 {
t.Fatalf("reported landed %d times, %d stray requests, stderr %q; want once", n, strays, stderr.String())
}
})
t.Run("a file already there is a result, and nothing is reported landed", func(t *testing.T) {
root := uploadRoot(t)
if err := os.WriteFile(filepath.Join(root, "plugins", "a.jar"), []byte("old!"), 0o644); err != nil {
t.Fatal(err)
}
srv := stagedUpload(t, "tok", body)
t.Setenv(fileedit.UploadTokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeExists || srv.reports.Load() != 0 {
t.Fatalf("result = %+v, reported landed %d times", res, srv.reports.Load())
}
})
// The file is in place whatever felis-api answers, so the Job still succeeds
// and says why the staged copy may linger. A redirect is not followed, since
// the request carries the token.
for name, tc := range map[string]struct {
code int
stderr string
}{
"refused": {http.StatusNotFound, "felis files: tell felis-api the upload landed: DELETE returned 404 Not Found\n"},
"redirected": {http.StatusFound, "felis files: tell felis-api the upload landed: DELETE returned 302 Found\n"},
} {
t.Run("a landed report "+name+" still lands the file", func(t *testing.T) {
var elsewhere atomic.Int32
away := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { elsewhere.Add(1) }))
defer away.Close()
root := uploadRoot(t)
srv := stagedUpload(t, "tok", body)
srv.landedCode, srv.landedTo = tc.code, away.URL+"/u"
t.Setenv(fileedit.UploadTokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != "" || stderr.String() != tc.stderr || elsewhere.Load() != 0 {
t.Fatalf("result = %+v, stderr %q, redirect followed %d times", res, stderr.String(), elsewhere.Load())
}
if got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar")); err != nil || !bytes.Equal(got, body) {
t.Fatalf("landed %q, %v", got, err)
}
})
}
// A refused fetch is the Job failing, never a Result: the API answers it with a
// 500 the caller retries whole.
t.Run("a refused fetch exits 1 and lands nothing", func(t *testing.T) {
root := uploadRoot(t)
srv := stagedUpload(t, "tok", body)
t.Setenv(fileedit.UploadTokenEnv, "wrong")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1; stdout %q", code, stdout.String())
}
if !strings.Contains(stderr.String(), "404") {
t.Fatalf("stderr %q does not name the status", stderr.String())
}
if srv.reports.Load() != 0 {
t.Fatal("a refused fetch was reported landed")
}
if _, err := os.Lstat(filepath.Join(root, "plugins", "a.jar")); !os.IsNotExist(err) {
t.Fatalf("a refused fetch left a file: %v", err)
}
})
// The request carries the token, and the internal face never redirects, so a
// redirect is refused rather than followed with the token attached.
t.Run("a redirect is not followed", func(t *testing.T) {
root := uploadRoot(t)
var hits atomic.Int32
elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Write(body)
}))
defer elsewhere.Close()
redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.URL+"/u", http.StatusFound))
defer redirecting.Close()
t.Setenv(fileedit.UploadTokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdFiles(uploadArgs(root, redirecting.URL+"/u", body), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if n := hits.Load(); n != 0 {
t.Fatalf("the redirect target was fetched %d times", n)
}
})
for name, tc := range map[string]struct {
token string
drop string
}{
"no token": {"", ""},
"no source URL": {"tok", "--source-url"},
} {
t.Run(name+" exits 2", func(t *testing.T) {
srv := stagedUpload(t, "tok", body)
t.Setenv(fileedit.UploadTokenEnv, tc.token)
args := uploadArgs(uploadRoot(t), srv.URL+"/u", body)
if tc.drop != "" {
for i, a := range args {
if a == tc.drop {
args = append(args[:i:i], args[i+2:]...)
break
}
}
}
var stdout, stderr bytes.Buffer
if code := cmdFiles(args, &stdout, &stderr); code != 2 {
t.Fatalf("exit %d, want 2", code)
}
})
}
}
func TestCmdFilesWrite(t *testing.T) {
root := t.TempDir()
content := []byte("[]\r\n")
sum := sha256.Sum256(content)
args := []string{"--op", "write", "--path", "ops.json", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}
t.Run("reassembles the content parts", func(t *testing.T) {
t.Setenv(fileedit.ContentPartsEnv, "1")
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
var stdout, stderr bytes.Buffer
if code := cmdFiles(args, &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != "" {
t.Fatalf("result = %+v", res)
}
if got, err := os.ReadFile(filepath.Join(root, "ops.json")); err != nil || !bytes.Equal(got, content) {
t.Fatalf("wrote %q, %v", got, err)
}
})
// Writing what did arrive of an incomplete spec would truncate the file.
t.Run("an incomplete content spec exits 2 and writes nothing", func(t *testing.T) {
t.Setenv(fileedit.ContentPartsEnv, "2")
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("x")))
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 2 {
t.Fatalf("exit %d, want 2", code)
}
if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
t.Fatalf("an incomplete spec wrote a file: %v", err)
}
})
// Without the content's SHA-256 the Job could not tell bytes changed on the
// way from the bytes felis-api sent.
t.Run("a write without its SHA-256 exits 2 and writes nothing", func(t *testing.T) {
t.Setenv(fileedit.ContentPartsEnv, "1")
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root}, &stdout, &stderr); code != 2 {
t.Fatalf("exit %d, want 2", code)
}
if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
t.Fatalf("a write without its SHA-256 wrote a file: %v", err)
}
})
t.Run("content that changed on the way is a result and writes nothing", func(t *testing.T) {
t.Setenv(fileedit.ContentPartsEnv, "1")
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("[]\n")))
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeDigestMismatch {
t.Fatalf("result = %+v, want %s", res, fileedit.CodeDigestMismatch)
}
if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
t.Fatalf("changed content wrote a file: %v", err)
}
})
}
// A caller-fault outcome is a successful run carrying a code, so felis-api can
// answer the precise 4xx instead of a 500.
func TestCmdFilesCallerFaultIsAResult(t *testing.T) {
root := t.TempDir()
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "mkdir", "--path", "../out", "--worlds-root", root}, &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeBadPath {
t.Fatalf("result = %+v, want code %s", res, fileedit.CodeBadPath)
}
stdout.Reset()
if code := cmdFiles([]string{"--worlds-root", root}, &stdout, &stderr); code != 2 {
t.Fatalf("no --op: exit %d, want 2", code)
}
}
// TestCmdFilesUnzip checks an unzip extracts next to the archive and reports its
// progress before its result, the same way an upload does.
func TestCmdFilesUnzip(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "maps"), 0o755); err != nil {
t.Fatal(err)
}
var zb bytes.Buffer
zw := zip.NewWriter(&zb)
for name, body := range map[string]string{"world/level.dat": "level", "world/region/r.0.0.mca": "region!"} {
w, err := zw.Create(name)
if err != nil {
t.Fatal(err)
}
io.WriteString(w, body)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "maps", "a.zip"), zb.Bytes(), 0o644); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
if code := cmdFiles([]string{"--op", "unzip", "--path", "maps/a.zip", "--worlds-root", root}, &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if res := filesResult(t, stdout.String()); res.Code != "" || res.Files != 2 || res.Bytes != 12 {
t.Fatalf("result = %+v", res)
}
if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix) ||
!strings.Contains(stdout.String(), fileedit.ProgressPrefix+`{"done":12,"total":12}`+"\n") {
t.Fatalf("stdout %q does not report the progress to the last byte", stdout.String())
}
got, err := os.ReadFile(filepath.Join(root, "maps", "world", "region", "r.0.0.mca"))
if err != nil || string(got) != "region!" {
t.Fatalf("extracted %q, %v", got, err)
}
}
cmd/felis/hostcreds.go
-84
View File
@@ -1,84 +0,0 @@
package main
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// Host copies of the credentials `felis setup` takes at the keyboard: the [smtp]
// relay password and the uploads bucket's keys. The cluster reads them from the
// felis-smtp and felis-uploads-s3 Secrets, and a Secret lives in k3s's datastore,
// which a reinstall (uninstall.sh keeps /etc/felis) or a host rebuilt from a
// database bundle's state/ starts empty. Each file holds the bare value, mode
// 0600, directly in /etc/felis beside secrets.env: every installer run applies
// the Secrets from these files, and every database bundle, so the off-site copy
// too, carries them.
const (
hostSMTPPasswordPath = "/etc/felis/smtp-password"
hostUploadsS3AccessKeyPath = "/etc/felis/uploads-s3-access-key"
hostUploadsS3SecretKeyPath = "/etc/felis/uploads-s3-secret-key"
)
// writeHostCredential replaces the file at path with value, mode 0600, through a
// temporary file in the same directory, so a crash leaves the old value or the
// new one and never a partial one.
func writeHostCredential(path, value string) error {
tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
if err != nil {
return err
}
tmpPath := tmp.Name()
defer os.Remove(tmpPath)
// CreateTemp already makes the file 0600; the Chmod states it rather than
// leaning on that.
if err := tmp.Chmod(0o600); err != nil {
_ = tmp.Close()
return err
}
if _, err := tmp.WriteString(value); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmpPath, path)
}
// readHostCredential returns the value in path; ok is false when there is no
// such file. An empty file is a value: the relay password of a relay without AUTH.
func readHostCredential(path string) (value string, ok bool, err error) {
b, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return "", false, nil
}
if err != nil {
return "", false, err
}
return string(b), true, nil
}
// relayPassword is the [smtp] relay password as the host holds it: the copy
// `felis setup` keeps at path, else, on an install from before that copy, the
// felis-smtp Secret in ns, whose absence means a relay without AUTH. cl is only
// used when the file is missing; a nil cl then reports errClusterUnreachable.
func relayPassword(ctx context.Context, path string, cl client.Client, ns string) (string, error) {
if pw, ok, err := readHostCredential(path); err != nil || ok {
return pw, err
}
if cl == nil {
return "", errClusterUnreachable
}
return smtpSecretPassword(ctx, cl, ns)
}
var errClusterUnreachable = errors.New("the cluster did not answer")
cmd/felis/hostcreds_test.go
-198
View File
@@ -1,198 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/watchdog"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
func TestHostCredentialFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "smtp-password")
if _, ok, err := readHostCredential(path); ok || err != nil {
t.Fatalf("a missing file read as ok=%v err=%v; want not there", ok, err)
}
// A copy an operator put there by hand, readable by everyone, is tightened.
if err := os.WriteFile(path, []byte("by hand"), 0o644); err != nil {
t.Fatal(err)
}
for _, v := range []string{"first secret", "a \"quoted\" $second\nsecret", ""} {
if err := writeHostCredential(path, v); err != nil {
t.Fatal(err)
}
got, ok, err := readHostCredential(path)
if err != nil || !ok || got != v {
t.Fatalf("read back (%q, %v, %v); want (%q, true, nil)", got, ok, err, v)
}
fi, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if mode := fi.Mode().Perm(); mode != 0o600 {
t.Fatalf("mode %v; want 0600", mode)
}
}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
if len(entries) != 1 {
t.Fatalf("the directory holds %d entries; want the credential alone, no leftover temporary file", len(entries))
}
if _, _, err := readHostCredential(dir); err == nil {
t.Fatal("an unreadable credential read as fine")
}
}
func smtpSecretClient(t *testing.T, password string) client.Client {
t.Helper()
return fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
Data: map[string][]byte{platform.SMTPSecretPasswordKey: []byte(password)},
}).Build()
}
func TestRelayPassword(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
host := filepath.Join(dir, "smtp-password")
cl := smtpSecretClient(t, "from-secret")
if pw, err := relayPassword(ctx, host, cl, "felis"); err != nil || pw != "from-secret" {
t.Fatalf("without a host copy = (%q, %v); want the Secret's", pw, err)
}
if _, err := relayPassword(ctx, host, nil, "felis"); !errors.Is(err, errClusterUnreachable) {
t.Fatalf("without a host copy or a cluster err = %v; want errClusterUnreachable", err)
}
if err := writeHostCredential(host, "from-host"); err != nil {
t.Fatal(err)
}
for _, c := range []client.Client{cl, nil} {
if pw, err := relayPassword(ctx, host, c, "felis"); err != nil || pw != "from-host" {
t.Fatalf("with a host copy (cluster %v) = (%q, %v); want the host copy", c != nil, pw, err)
}
}
if _, err := relayPassword(ctx, dir, cl, "felis"); err == nil {
t.Fatal("an unreadable host copy fell through to the Secret")
}
}
// The watchdog mails the most while the cluster is down: the host copy must
// reach it then, and without one the password the last good run cached stays.
func TestRefreshSMTPPassword(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
host := filepath.Join(dir, "smtp-password")
var stderr bytes.Buffer
state := &watchdog.State{SMTPPassword: "cached"}
refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
if state.SMTPPassword != "cached" || stderr.Len() != 0 {
t.Fatalf("cluster down, no host copy: password %q, stderr %q; want the cached one kept quietly", state.SMTPPassword, stderr.String())
}
refreshSMTPPassword(ctx, host, smtpSecretClient(t, "from-secret"), "felis", state, &stderr)
if state.SMTPPassword != "from-secret" {
t.Fatalf("cluster up, no host copy: password %q; want the Secret's", state.SMTPPassword)
}
if err := writeHostCredential(host, "from-host"); err != nil {
t.Fatal(err)
}
refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
if state.SMTPPassword != "from-host" {
t.Fatalf("cluster down, host copy: password %q; want the host copy", state.SMTPPassword)
}
refreshSMTPPassword(ctx, dir, nil, "felis", state, &stderr)
if state.SMTPPassword != "from-host" || !strings.Contains(stderr.String(), "keeping the cached one") {
t.Fatalf("unreadable host copy: password %q, stderr %q; want the cached one kept and the failure said", state.SMTPPassword, stderr.String())
}
}
func TestHostRecoveryMailerHostCopy(t *testing.T) {
ctx := context.Background()
// No kubeconfig anywhere: reaching for the cluster fails, so a pass proves
// the host copy was enough.
t.Setenv("KUBECONFIG", filepath.Join(t.TempDir(), "no-kubeconfig"))
dir := t.TempDir()
host := filepath.Join(dir, "smtp-password")
if err := writeHostCredential(host, "from-host"); err != nil {
t.Fatal(err)
}
off := false
c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_UNSET_RELAY_PW", RequireTLS: &off}
got, err := hostRecoveryMailer(c, host, "felis")(ctx)
if err != nil {
t.Fatalf("with the host copy: %v", err)
}
if relay, ok := got.(*mail.SMTP); !ok || relay.Password != "from-host" {
t.Fatalf("relay = %#v; want the host copy's password", got)
}
if _, err := hostRecoveryMailer(c, dir, "felis")(ctx); err == nil || !strings.Contains(err.Error(), "read the relay password") {
t.Fatalf("unreadable host copy: err = %v; want it named", err)
}
if _, err := hostRecoveryMailer(c, filepath.Join(dir, "none"), "felis")(ctx); err == nil || !strings.Contains(err.Error(), "reach the cluster") {
t.Fatalf("no host copy and no cluster: err = %v; want the cluster named", err)
}
}
// A whole watchdog run with the API server and PostgreSQL both down still
// takes the relay password from the host copy, so the outage mail can
// authenticate even when no earlier run cached it.
func TestWatchdogReadsHostCopyWhileClusterDown(t *testing.T) {
dir := t.TempDir()
t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
cfgPath := filepath.Join(dir, "felis.toml")
if err := os.WriteFile(cfgPath, []byte(`[database]
url = "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
[server]
root_domain = "example.com"
[archive]
store = "tarLocal"
[k8s]
egress_mode = "nodeport"
[smtp]
host = "127.0.0.1"
port = 1
from = "[email protected]"
username = "felis"
password_ref = "FELIS_TEST_UNSET_RELAY_PW"
`), 0o600); err != nil {
t.Fatal(err)
}
pwPath := filepath.Join(dir, "smtp-password")
if err := writeHostCredential(pwPath, "from-host"); err != nil {
t.Fatal(err)
}
statePath := filepath.Join(dir, "state.json")
var stdout, stderr bytes.Buffer
cmdWatchdog([]string{
"-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"),
"-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", pwPath, "-heartbeat-file", filepath.Join(dir, "no-heartbeat"),
}, &stdout, &stderr)
if !strings.Contains(stdout.String(), "kube-api") {
t.Fatalf("the run found the API server up; the test needs it down (stdout %s)", stdout.String())
}
state, err := watchdog.LoadState(statePath)
if err != nil {
t.Fatalf("load state: %v (stderr %s)", err, stderr.String())
}
if state.SMTPPassword != "from-host" {
t.Fatalf("cached relay password %q; want the host copy (stdout %s, stderr %s)", state.SMTPPassword, stdout.String(), stderr.String())
}
}
cmd/felis/imagebundle.go
-109
View File
@@ -1,109 +0,0 @@
package main
import (
"bufio"
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"strings"
"syscall"
"felis.lolicon.best/internal/imagepush"
)
// cmdImageBundle writes a release's image bundle: one OCI layout tar holding every
// image an install runs, for one platform, plus its listing (one "role name
// manifest-digest config-digest" line per image). deploy/build-release-artifacts.sh
// runs it in CI; deploy/bootstrap.sh imports the tar into k3s's containerd and
// pushes it into the platform registry with push-image --image.
//
// --layout role=name=path an image buildx wrote with --output type=oci
// --pull role=ref a digest-pinned public image, named repository@digest
//
// The tar and the listing are written beside their final paths and renamed into
// place, so a failed run leaves neither behind.
func cmdImageBundle(args []string, _, stderr io.Writer) int {
fs := flag.NewFlagSet("image-bundle", flag.ContinueOnError)
fs.SetOutput(stderr)
platform := fs.String("platform", "", "os/arch the bundle is for, e.g. linux/arm64")
out := fs.String("out", "", "path of the bundle tar to write")
list := fs.String("list", "", "path of the listing to write")
var images []imagepush.BundleImage
fs.Func("layout", "role=name=path of an OCI layout tar (repeatable)", func(v string) error {
role, rest, ok := strings.Cut(v, "=")
name, path, ok2 := strings.Cut(rest, "=")
if !ok || !ok2 || role == "" || name == "" || path == "" {
return fmt.Errorf("want role=name=path, got %q", v)
}
images = append(images, imagepush.BundleImage{Role: role, Name: name, Layout: path})
return nil
})
fs.Func("pull", "role=ref of a digest-pinned public image (repeatable)", func(v string) error {
role, ref, ok := strings.Cut(v, "=")
if !ok || role == "" || !strings.Contains(ref, "@sha256:") {
return fmt.Errorf("want role=ref with ref pinned by digest, got %q", v)
}
images = append(images, imagepush.BundleImage{Role: role, Name: imagepush.PinnedName(ref), Source: ref})
return nil
})
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if *platform == "" || *out == "" || *list == "" || len(images) == 0 {
fmt.Fprintln(stderr, "felis image-bundle: --platform, --out, --list and at least one --layout or --pull are required")
return 2
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := writeImageBundle(ctx, &imagepush.Source{Platform: *platform}, images, *out, *list); err != nil {
fmt.Fprintf(stderr, "felis image-bundle: %v\n", err)
return 1
}
return 0
}
func writeImageBundle(ctx context.Context, s *imagepush.Source, images []imagepush.BundleImage, out, list string) (err error) {
tmpOut, tmpList := out+".tmp", list+".tmp"
defer func() {
if err != nil {
os.Remove(tmpOut)
os.Remove(tmpList)
}
}()
f, err := os.Create(tmpOut)
if err != nil {
return err
}
w := bufio.NewWriterSize(f, 1<<20)
entries, err := imagepush.WriteBundle(ctx, s, images, w)
if err == nil {
err = w.Flush()
}
if err == nil {
err = f.Sync()
}
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
return err
}
var b strings.Builder
for _, e := range entries {
fmt.Fprintf(&b, "%s %s %s %s\n", e.Role, e.Name, e.Digest, e.Config)
}
if err := os.WriteFile(tmpList, []byte(b.String()), 0o644); err != nil {
return err
}
if err := os.Rename(tmpOut, out); err != nil {
return err
}
return os.Rename(tmpList, list)
}
cmd/felis/imagebundle_test.go
-131
View File
@@ -1,131 +0,0 @@
package main
import (
"archive/tar"
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
)
// writeTestLayout writes an OCI layout tar holding one linux/arch image with one
// layer, the way buildx --output type=oci leaves a single-platform build, and
// returns its path with the manifest and config digests.
func writeTestLayout(t *testing.T, dir, arch, layer string) (path, manifestDigest, configDigest string) {
t.Helper()
blobs := map[string][]byte{}
add := func(b []byte) (string, int) {
sum := sha256.Sum256(b)
d := "sha256:" + hex.EncodeToString(sum[:])
blobs[d] = b
return d, len(b)
}
cfgDigest, cfgSize := add([]byte(`{"architecture":"` + arch + `","os":"linux","rootfs":{"type":"layers"}}`))
layerDigest, layerSize := add([]byte(layer))
manifest := fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",`+
`"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":%q,"size":%d},`+
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":%q,"size":%d}]}`,
cfgDigest, cfgSize, layerDigest, layerSize)
mDigest, mSize := add([]byte(manifest))
index, _ := json.Marshal(map[string]any{
"schemaVersion": 2,
"manifests": []map[string]any{{
"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": mDigest, "size": mSize,
}},
})
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
put := func(name string, b []byte) {
tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(b)), Typeflag: tar.TypeReg})
tw.Write(b)
}
put("oci-layout", []byte(`{"imageLayoutVersion":"1.0.0"}`))
put("index.json", index)
for d, b := range blobs {
put("blobs/sha256/"+strings.TrimPrefix(d, "sha256:"), b)
}
tw.Close()
path = filepath.Join(dir, arch+"-"+layer+".tar")
if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil {
t.Fatal(err)
}
return path, mDigest, cfgDigest
}
func TestImageBundleWritesTheListingTheInstallerReads(t *testing.T) {
dir := t.TempDir()
limbo, limboDigest, limboConfig := writeTestLayout(t, dir, "arm64", "limbo")
lobby, lobbyDigest, lobbyConfig := writeTestLayout(t, dir, "arm64", "lobby")
out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
var stderr bytes.Buffer
code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo,
"--layout", "lobby=registry.felis.svc:5000/felis/lobby:demo=" + lobby,
}, nil, &stderr)
if code != 0 {
t.Fatalf("image-bundle = %d: %s", code, stderr.String())
}
// deploy/bootstrap.sh reads this with `read -r role name digest config`.
got, _ := os.ReadFile(list)
want := "limbo registry.felis.svc:5000/felis/limbo:demo " + limboDigest + " " + limboConfig + "\n" +
"lobby registry.felis.svc:5000/felis/lobby:demo " + lobbyDigest + " " + lobbyConfig + "\n"
if string(got) != want {
t.Errorf("listing:\n%s\nwant:\n%s", got, want)
}
if fi, err := os.Stat(out); err != nil || fi.Size() == 0 {
t.Errorf("bundle: %v", err)
}
if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
t.Errorf("left behind %v", leftovers)
}
}
func TestImageBundleLeavesNothingWhenAnImageIsRefused(t *testing.T) {
dir := t.TempDir()
amd, _, _ := writeTestLayout(t, dir, "amd64", "limbo")
out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
// The pair an earlier run wrote stays as it was: a listing beside a bundle it
// does not describe would have the installer look for images that are not there.
os.WriteFile(out, []byte("old bundle"), 0o644)
os.WriteFile(list, []byte("old listing\n"), 0o644)
var stderr bytes.Buffer
code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + amd}, nil, &stderr)
if code != 1 || !strings.Contains(stderr.String(), "is a linux/amd64 image") {
t.Fatalf("image-bundle = %d: %s", code, stderr.String())
}
if got, _ := os.ReadFile(out); string(got) != "old bundle" {
t.Errorf("bundle was replaced with %d bytes", len(got))
}
if got, _ := os.ReadFile(list); string(got) != "old listing\n" {
t.Errorf("listing was replaced with %q", got)
}
if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 {
t.Errorf("left behind %v", leftovers)
}
}
func TestPushImageReadsABundleByImageName(t *testing.T) {
dir := t.TempDir()
limbo, _, _ := writeTestLayout(t, dir, "arm64", "limbo")
out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt")
if code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list,
"--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo}, nil, &bytes.Buffer{}); code != 0 {
t.Fatal("image-bundle failed")
}
t.Setenv("FELIS_REGISTRY_USERNAME", "platform")
t.Setenv("FELIS_REGISTRY_PASSWORD", "x")
// The name is looked up in the bundle's index before the registry is contacted,
// so a name the bundle lacks fails here with what it does hold.
var stderr bytes.Buffer
code := cmdPushImage([]string{"--tar", out, "--image", "registry.felis.svc:5000/felis/lobby:demo",
"--ref", "127.0.0.1:1/felis/lobby:demo"}, &bytes.Buffer{}, &stderr)
if code != 1 || !strings.Contains(stderr.String(), "holds no image named registry.felis.svc:5000/felis/lobby:demo (it holds: registry.felis.svc:5000/felis/limbo:demo") {
t.Fatalf("push-image = %d: %s", code, stderr.String())
}
}
cmd/felis/main.go
-24
View File
@@ -6,32 +6,8 @@ package main
import (
"os"
"path/filepath"
)
func main() {
ensureHostBinDirOnPath()
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
}
// hostBinDir is where deploy/bootstrap.sh installs felis, k3s and cloudflared.
const hostBinDir = "/usr/local/bin"
// ensureHostBinDirOnPath appends hostBinDir to PATH when it is missing, so the
// k3s and cloudflared this binary execs are found beside it. sudo's secure_path
// on EL leaves /usr/local/bin out: `sudo /usr/local/bin/felis db backup` would
// otherwise run with no k3s to reach the database's pod through. Appended, so a
// PATH that names another k3s first keeps it.
func ensureHostBinDirOnPath() {
path := os.Getenv("PATH")
for _, dir := range filepath.SplitList(path) {
if dir == hostBinDir {
return
}
}
if path == "" {
os.Setenv("PATH", hostBinDir)
return
}
os.Setenv("PATH", path+string(os.PathListSeparator)+hostBinDir)
}
cmd/felis/main_test.go
-33
View File
@@ -1,33 +0,0 @@
package main
import (
"os"
"regexp"
"testing"
)
func TestEnsureHostBinDirOnPath(t *testing.T) {
for _, c := range []struct{ in, want string }{
{"/usr/sbin:/usr/bin", "/usr/sbin:/usr/bin:/usr/local/bin"},
{"/usr/local/bin:/usr/bin", "/usr/local/bin:/usr/bin"},
{"/opt/k3s:/usr/bin:/usr/local/bin", "/opt/k3s:/usr/bin:/usr/local/bin"},
{"", "/usr/local/bin"},
} {
t.Setenv("PATH", c.in)
ensureHostBinDirOnPath()
if got := os.Getenv("PATH"); got != c.want {
t.Errorf("PATH %q became %q, want %q", c.in, got, c.want)
}
}
}
// run is what the tests drive, so the PATH fix must sit in main, before it.
func TestMainFixesPathBeforeRunning(t *testing.T) {
b, err := os.ReadFile("main.go")
if err != nil {
t.Fatal(err)
}
if !regexp.MustCompile(`func main\(\) \{\n\tensureHostBinDirOnPath\(\)\n\tos\.Exit\(run\(`).Match(b) {
t.Fatal("main does not call ensureHostBinDirOnPath before run")
}
}
cmd/felis/manifests.go
+4 -42
View File
@@ -35,11 +35,6 @@ func (m *multiFlag) Set(v string) error {
// --velocity-cidr records the proxy host addresses allowed by the game NetworkPolicy.
// Kubernetes permits resident-node traffic regardless, but remote proxy deployments
// need an explicit CIDR, so the renderer refuses to guess.
//
// --only postgres renders just the control-plane database (platform.PostgresObjects),
// which the installer brings up before migrations, before it has anything else
// to render the full bundle with; it needs neither --felis-image nor
// --velocity-cidr.
func cmdManifests(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("manifests", flag.ContinueOnError)
fs.SetOutput(stderr)
@@ -51,11 +46,9 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
panelNodePort := fs.Int("panel-node-port", int(platform.DefaultPanelNodePort), "NodePort that exposes the built-in HTTPS panel/API origin")
felisImage := fs.String("felis-image", "", "container image the felis-api/operator Deployments run, also passed through as FELIS_IMAGE (REQUIRED)")
registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry 2.8.3, pinned by digest)")
postgresImage := fs.String("postgres-image", "", "control-plane database image (default: PostgreSQL 18.6, pinned by digest)")
only := fs.String("only", "", `render one part of the bundle instead of all of it; "postgres" is the control-plane database`)
backupPVC := fs.String("backup-pvc", "felis-backups", "name of the world-archive PVC this bundle renders in the Minecraft namespace and advertises to the backup/restore executors via FELIS_BACKUP_PVC (default: felis-backups; pass an empty value to render none, leaving backup/restore answering 503)")
worldsHostPath := fs.String("worlds-host-path", "", "node directory the reaper reads worlds from: each world PVC resolves as <path>/<pvc>, or as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (k3s storage root: /var/lib/rancher/k3s/storage); enables the reaper CronJob (requires --archive-local-path and a non-empty --backup-pvc)")
archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path. With the backup PVC alone it renders the retention-only CronJob, which deletes backups past their expiry and never touches a world")
archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
registryStorage := fs.String("registry-storage", "", "capacity the registry PVC requests (default 10Gi; k3s local-path does not enforce it)")
uploadsStorage := fs.String("uploads-storage", "", "capacity the uploads PVC requests (default 5Gi; k3s local-path does not enforce it)")
backupStorage := fs.String("backup-storage", "", "capacity the world-archive PVC requests (default 10Gi; k3s local-path does not enforce it)")
@@ -71,18 +64,6 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
if err := fs.Parse(args); err != nil {
return 2
}
switch *only {
case "":
case "postgres":
return renderManifests(stdout, stderr, platform.PostgresObjects(platform.Params{
ControlNamespace: *controlNS,
MinecraftNamespace: *minecraftNS,
PostgresImage: *postgresImage,
}))
default:
fmt.Fprintf(stderr, "felis manifests: --only %q: the one part that renders alone is \"postgres\"\n", *only)
return 2
}
// Keep proxy placement explicit. This matters for remote proxies and documents
// the expected source even when Velocity runs on the resident node.
@@ -157,22 +138,8 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
"<path>/<pvc>, or each candidate's archive fails and the world is preserved;\n"+
" - %s.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, pin)
} else {
switch {
case *archiveLocalPath != "" && *backupPVC == "":
fmt.Fprintln(stderr, "felis manifests: --archive-local-path names where the backup PVC is mounted, "+
"but --backup-pvc is empty (no archive store renders); drop one or the other")
return 2
case *archiveLocalPath != "":
fmt.Fprintln(stderr, "felis manifests: note: rendering the reaper CronJob retention-only: backups past "+
"their expiry are deleted daily, idle worlds are never archived or deleted "+
"(pass --worlds-host-path to reap them too)")
case *backupPVC != "":
fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered: backups are never expired, so "+
"the archive store only grows until the disk fills (pass --archive-local-path, equal to felis.toml "+
"[archive] local_path, for the retention-only CronJob, and --worlds-host-path as well to reap idle worlds)")
default:
fmt.Fprintln(stderr, "felis manifests: note: reaper CronJob not rendered (backups are disabled)")
}
fmt.Fprintln(stderr, "felis manifests: note: retention reaper CronJob not rendered "+
"(pass --worlds-host-path and --archive-local-path — the archive PVC defaults to felis-backups — to enable it)")
}
params := platform.Params{
@@ -184,7 +151,6 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
PanelNodePort: int32(*panelNodePort),
FelisImage: *felisImage,
RegistryImage: *registryImage,
PostgresImage: *postgresImage,
BackupPVC: *backupPVC,
WorldsHostPath: *worldsHostPath,
ReaperNode: *reaperNode,
@@ -203,11 +169,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "felis manifests: %v\n", err)
return 2
}
return renderManifests(stdout, stderr, platform.Objects(params))
}
func renderManifests(stdout, stderr io.Writer, objs []platform.Object) int {
out, err := platform.RenderObjects(objs)
out, err := platform.RenderYAML(params)
if err != nil {
fmt.Fprintf(stderr, "felis manifests: render: %v\n", err)
return 1
cmd/felis/manifests_test.go
+4 -73
View File
@@ -90,13 +90,12 @@ func TestManifestsRendersBundle(t *testing.T) {
t.Error("rendered bundle must not contain ClusterRole/ClusterRoleBinding")
}
// Without the retention flags, the reaper CronJob is not rendered and the
// generator says so on stderr, naming what that leaves: backups that never
// expire.
// generator says so on stderr.
if strings.Contains(text, "kind: CronJob") {
t.Error("no reaper CronJob must render without --archive-local-path")
t.Error("no reaper CronJob must render without --worlds-host-path")
}
if !strings.Contains(errBuf.String(), "not rendered") || !strings.Contains(errBuf.String(), "never expired") {
t.Errorf("expected a 'reaper not rendered, backups never expired' notice on stderr, got %q", errBuf.String())
if !strings.Contains(errBuf.String(), "not rendered") {
t.Errorf("expected a 'reaper not rendered' notice on stderr, got %q", errBuf.String())
}
}
@@ -145,40 +144,6 @@ func TestManifestsBackupPVCOptOut(t *testing.T) {
}
}
// TestManifestsRendersRetentionOnly: the archive store without a worlds root
// still gets the daily CronJob, retention-only, so backups past their expiry
// leave the store on an install that never reaps a world; the operator is told
// which of the two it got. An archive path with the store switched off is a
// mistake and fails loud.
func TestManifestsRendersRetentionOnly(t *testing.T) {
var out, errBuf bytes.Buffer
code := run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
"--archive-local-path", "/var/lib/felis/archives"}, &out, &errBuf)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
}
text := out.String()
for _, want := range []string{"kind: CronJob", "name: felis-reaper", "--retention-only", "claimName: felis-backups"} {
if !strings.Contains(text, want) {
t.Errorf("retention-only bundle missing %q", want)
}
}
if strings.Contains(text, "kind: PersistentVolume\n") || strings.Contains(text, "--worlds-root") {
t.Error("a retention-only bundle must not reach for a worlds root")
}
if !strings.Contains(errBuf.String(), "retention-only") {
t.Errorf("stderr must say the CronJob is retention-only, got %q", errBuf.String())
}
out.Reset()
errBuf.Reset()
code = run([]string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32",
"--archive-local-path", "/var/lib/felis/archives", "--backup-pvc="}, &out, &errBuf)
if code != 2 || out.Len() != 0 || !strings.Contains(errBuf.String(), "--backup-pvc is empty") {
t.Errorf("archive path without an archive store: exit=%d out=%d bytes stderr=%q, want a fail-loud 2", code, out.Len(), errBuf.String())
}
}
// TestManifestsRendersReaper proves the happy path with the full retention trio:
// a batch/v1 CronJob is emitted, named felis-reaper, mounting the backup PVC at the
// supplied archive path.
@@ -278,37 +243,3 @@ func TestManifestsStorageSizes(t *testing.T) {
t.Errorf("--registry-storage lots: exit %d, stderr %q; want a refusal naming the flag", code, errBuf.String())
}
}
// TestManifestsOnlyPostgres: the installer renders the database before it has
// an image or a proxy address for the rest of the bundle, so --only postgres
// must render without them, and render the database and nothing else (a stray
// Deployment in that apply would start without its identities).
func TestManifestsOnlyPostgres(t *testing.T) {
var out, errBuf bytes.Buffer
code := run([]string{"manifests", "--only", "postgres", "--control-namespace", "ctl", "--postgres-image", "example/pg:18@sha256:abc"}, &out, &errBuf)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
}
var kinds []string
for _, doc := range strings.Split(out.String(), "\n---\n") {
for _, line := range strings.Split(doc, "\n") {
if strings.HasPrefix(line, "kind: ") {
kinds = append(kinds, strings.TrimPrefix(line, "kind: "))
}
}
}
if got, want := strings.Join(kinds, ","), "Namespace,ConfigMap,NetworkPolicy,Deployment,Service"; got != want {
t.Errorf("rendered kinds %s, want %s", got, want)
}
for _, want := range []string{"name: felis-postgres", "namespace: ctl", "image: example/pg:18@sha256:abc"} {
if !strings.Contains(out.String(), want) {
t.Errorf("rendered database missing %q", want)
}
}
out.Reset()
errBuf.Reset()
if code := run([]string{"manifests", "--only", "registry"}, &out, &errBuf); code != 2 || out.Len() != 0 {
t.Errorf("--only registry: exit %d with %d bytes of YAML, want 2 and none", code, out.Len())
}
}
cmd/felis/memlimit.go
-48
View File
@@ -1,48 +0,0 @@
package main
import (
"os"
"runtime/debug"
"strconv"
"strings"
)
// cgroupMemoryFiles are where a container reads the memory it is allowed:
// cgroup v2 first, then v1.
var cgroupMemoryFiles = []string{"/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"}
// limitHeapToCgroup sets the Go heap's soft limit from the container's memory
// limit, so the collector works harder as a Job nears it and the kernel does not
// kill the Job first. An extraction or a folder zipped for download keeps a few
// hundred bytes per entry for as long as it runs; without the limit the heap
// grows to twice that before a collection, and a 256 MiB Job was killed at
// 400,000 entries whose live heap was 115 MB. GOMEMLIMIT set by hand wins.
func limitHeapToCgroup() {
if os.Getenv("GOMEMLIMIT") != "" {
return
}
for _, f := range cgroupMemoryFiles {
b, err := os.ReadFile(f)
if err != nil {
continue
}
if n, ok := softMemoryLimit(string(b)); ok {
debug.SetMemoryLimit(n)
}
return
}
}
// softMemoryLimit answers three fifths of the limit a cgroup memory file holds,
// or false for "max" (no limit) and anything unreadable. The rest is left for
// what the kernel charges the container beyond the Go heap: the page cache of
// the files it reads and writes, and the inodes it creates. Under a 256 MiB
// limit, 400,000 extracted entries peaked at 184 MB resident with the heap held
// to 150 MiB.
func softMemoryLimit(content string) (int64, bool) {
n, err := strconv.ParseInt(strings.TrimSpace(content), 10, 64)
if err != nil || n <= 0 {
return 0, false
}
return n / 5 * 3, true
}
cmd/felis/memlimit_test.go
-58
View File
@@ -1,58 +0,0 @@
package main
import (
"math"
"os"
"path/filepath"
"runtime/debug"
"testing"
)
func TestSoftMemoryLimit(t *testing.T) {
for _, c := range []struct {
in string
want int64
ok bool
}{
{"268435456\n", 161061273, true}, // 256 MiB, as memory.max holds it
{"max\n", 0, false},
{"0\n", 0, false},
{"-1", 0, false},
{"", 0, false},
} {
got, ok := softMemoryLimit(c.in)
if got != c.want || ok != c.ok {
t.Errorf("softMemoryLimit(%q) = %d %v, want %d %v", c.in, got, ok, c.want, c.ok)
}
}
}
// TestLimitHeapToCgroup checks the limit comes from the first cgroup file there
// is, and that GOMEMLIMIT set by hand leaves the heap alone.
func TestLimitHeapToCgroup(t *testing.T) {
prevFiles, prevLimit := cgroupMemoryFiles, debug.SetMemoryLimit(-1)
t.Cleanup(func() { cgroupMemoryFiles = prevFiles; debug.SetMemoryLimit(prevLimit) })
dir := t.TempDir()
v1, v1b := filepath.Join(dir, "v1"), filepath.Join(dir, "v1b")
if err := os.WriteFile(v1, []byte("268435456\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(v1b, []byte("536870912\n"), 0o600); err != nil {
t.Fatal(err)
}
cgroupMemoryFiles = []string{filepath.Join(dir, "missing"), v1, v1b}
t.Setenv("GOMEMLIMIT", "")
debug.SetMemoryLimit(math.MaxInt64)
limitHeapToCgroup()
if got := debug.SetMemoryLimit(-1); got != 161061273 {
t.Fatalf("limit = %d, want three fifths of 256 MiB", got)
}
t.Setenv("GOMEMLIMIT", "1GiB")
debug.SetMemoryLimit(math.MaxInt64)
limitHeapToCgroup()
if got := debug.SetMemoryLimit(-1); got != math.MaxInt64 {
t.Fatalf("limit = %d with GOMEMLIMIT set, want it left alone", got)
}
}
cmd/felis/migrate.go
+7 -75
View File
@@ -2,11 +2,9 @@ package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
@@ -20,8 +18,7 @@ import (
// database that already holds a schema and has migrations pending is bundled
// first (internal/dbbackup, label pre-migrate). A failed snapshot stops the
// upgrade; -no-backup is the explicit way past it, e.g. for an external
// database (no [database] deployment, so the host's own pg_dump runs) whose
// server is newer than that pg_dump.
// database whose server is newer than the host's pg_dump.
func cmdMigrate(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("migrate", flag.ContinueOnError)
fs.SetOutput(stderr)
@@ -61,7 +58,7 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
}
if !*noBackup {
path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database, *backupDir, stderr)
path, err := preMigrateBackup(ctx, drv, migrations, cfg.Database.URL, *backupDir, stderr)
if err != nil {
fmt.Fprintf(stderr, "felis migrate: pre-migration backup failed, nothing applied: %v\n", err)
fmt.Fprintln(stderr, " fix the backup, or re-run with -no-backup to migrate without one")
@@ -85,14 +82,10 @@ func cmdMigrate(args []string, stdout, stderr io.Writer) int {
return 0
}
// preMigrateStateDir is the host state a pre-migrate bundle carries; tests
// point it at a directory of their own.
var preMigrateStateDir = dbbackup.DefaultStateDir
// preMigrateBackup bundles the database when it already carries a schema and
// some of migrations are not applied yet, and returns the bundle's path ("" when
// there was nothing to protect: a fresh database, or nothing pending).
func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, db config.DatabaseConfig, dir string, log io.Writer) (string, error) {
func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.Migration, dbURL, dir string, log io.Writer) (string, error) {
if err := drv.EnsureVersionTable(ctx); err != nil {
return "", fmt.Errorf("ensure version table: %w", err)
}
@@ -103,22 +96,11 @@ func preMigrateBackup(ctx context.Context, drv store.Driver, migrations []store.
if len(done) == 0 || !hasPending(done, migrations) {
return "", nil
}
tools, err := dbTools(db)
if err != nil {
return "", err
}
path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelPreMigrate,
Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: preMigrateStateDir,
Version: resolvedVersion(), ExportServers: exportMinecraftServers, Log: log, Record: true,
return dbbackup.Backup(ctx, dbbackup.BackupOptions{
DatabaseURL: dbURL, Dir: dir, Label: dbbackup.LabelPreMigrate,
Keep: defaultKeep[dbbackup.LabelPreMigrate], StateDir: dbbackup.DefaultStateDir,
Version: resolvedVersion(), Log: log, Record: true,
})
if errors.Is(err, dbbackup.ErrServersMissing) {
// Rolling the migration back needs the database alone. Backup logged
// the gap, and the panel and the watchdog show it while this is the
// newest bundle.
return path, nil
}
return path, err
}
func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
@@ -129,53 +111,3 @@ func hasPending(done map[int]struct{}, migrations []store.Migration) bool {
}
return false
}
// openStore opens the business database for a command that reads and writes its
// tables, and refuses one whose schema this build was not written against: a newer
// Felis migrated it (a rolled-back binary), or, unless allowPending, migrations this
// build embeds have not run yet (a binary swapped in ahead of `felis migrate up`).
func openStore(ctx context.Context, url string, allowPending bool) (*store.PostgresDriver, error) {
drv, err := store.Open(ctx, url)
if err != nil {
return nil, err
}
return checkSchema(ctx, drv, allowPending)
}
// podDBWindow and podDBInterval bound how long a pod that has just started retries its
// first database dial while the network policy has yet to admit it
// (store.OpenRetrying). A minute is far past the sync lag and far inside every Job's
// deadline. Vars so a test can shrink them.
var (
podDBWindow = time.Minute
podDBInterval = time.Second
)
// openPodStore is openStore for felis-api and the reaper and backup Jobs, whose first
// dial comes milliseconds after their pod starts.
func openPodStore(ctx context.Context, url, prog string, stderr io.Writer) (*store.PostgresDriver, error) {
drv, err := store.OpenRetrying(ctx, url, podDBWindow, podDBInterval, func(err error) {
fmt.Fprintf(stderr, "felis %s: %v; retrying (a pod that has just started waits for the network policy to admit it)\n", prog, err)
})
if err != nil {
return nil, err
}
return checkSchema(ctx, drv, false)
}
// checkSchema closes drv and fails when its schema is not the one this build was
// written against (see openStore).
func checkSchema(ctx context.Context, drv *store.PostgresDriver, allowPending bool) (*store.PostgresDriver, error) {
s, err := store.ReadSchema(ctx, drv)
if err == nil {
err = s.Err()
if allowPending {
err = s.Newer()
}
}
if err != nil {
drv.Close()
return nil, err
}
return drv, nil
}
cmd/felis/migrate_test.go
-48
View File
@@ -1,48 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"net"
"strings"
"syscall"
"testing"
"time"
)
// openPodStore retries a refused first dial for podDBWindow, saying so on stderr
// under the calling command's name each time.
func TestOpenPodStoreRetriesARefusedDial(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
addr := ln.Addr().String()
ln.Close()
window, interval := podDBWindow, podDBInterval
podDBWindow, podDBInterval = 200*time.Millisecond, 20*time.Millisecond
t.Cleanup(func() { podDBWindow, podDBInterval = window, interval })
var stderr bytes.Buffer
start := time.Now()
_, err = openPodStore(context.Background(), fmt.Sprintf("postgres://felis@%s/felis?sslmode=disable", addr), "reaper", &stderr)
if !errors.Is(err, syscall.ECONNREFUSED) {
t.Fatalf("err = %v, want a refused dial", err)
}
if elapsed := time.Since(start); elapsed < podDBWindow {
t.Fatalf("gave up after %s, inside the %s window", elapsed, podDBWindow)
}
lines := strings.Split(strings.TrimSuffix(stderr.String(), "\n"), "\n")
if len(lines) < 2 || len(lines) > 11 {
t.Fatalf("%d retry lines in a 200ms window at 20ms:\n%s", len(lines), stderr.String())
}
for _, l := range lines {
if !strings.HasPrefix(l, "felis reaper: failed to connect to `user=felis database=felis`: ") ||
!strings.HasSuffix(l, "; retrying (a pod that has just started waits for the network policy to admit it)") {
t.Fatalf("retry line %q", l)
}
}
}
cmd/felis/mirrortools.go
+11 -3
View File
@@ -14,6 +14,7 @@ import (
"felis.lolicon.best/internal/build"
"felis.lolicon.best/internal/imagepush"
"felis.lolicon.best/internal/registrygate"
)
// defaultBuildToolsStatus is where mirror-build-tools records its last run; the
@@ -48,9 +49,16 @@ func cmdMirrorBuildTools(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
return 2
}
user, pass, err := registryWriteCredential(*secrets)
if err != nil {
fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
if err := loadEnvFile(*secrets); err != nil {
fmt.Fprintf(stderr, "felis mirror-build-tools: read %s: %v\n", *secrets, err)
return 1
}
user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
if pass == "" {
user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
}
if pass == "" {
fmt.Fprintln(stderr, "felis mirror-build-tools: no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
return 2
}
cmd/felis/offsite.go
+67 -457
View File
@@ -16,6 +16,7 @@ import (
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,34 +25,18 @@ import (
)
const offsiteUsage = `usage:
felis offsite sync [-config path] [-archive-dir dir] [-db-dir dir] [-registry host:port|off]
[-uploads-dir dir] [-status-file path]
felis offsite sync [-config path] [-archive-dir dir] [-db-dir dir] [-status-file path]
felis offsite status [-config path] [-status-file path]
felis offsite list [-config path]
felis offsite fetch-db [-config path | -endpoint url -bucket name [-region r] [-prefix p]]
[-dir dir] latest|<bundle>
felis offsite fetch-worlds [-config path] [-archive-dir dir]
felis offsite fetch-images [-config path] [-registry host:port] [-at version]
felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
felis offsite check-key [-config path]
felis offsite take-over [-config path] [-status-file path] [-yes]
felis offsite keygen
Every verb but keygen reads the bucket credentials and the encryption key from
the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY,
FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
-env-file (default /etc/felis/offsite.env).
check-key tells whether the key is the one the bucket's objects are sealed
with, writing nothing; it exits 3 when they are sealed with another key, and
sync then refuses to write or prune anything in the bucket.
take-over names the host that writes the bucket, writing nothing; it exits 4
when that is another host and this one never wrote it, and 5 when another
host took the bucket over from this one. A host built from another host's
backup (a rehearsal, or a rebuild) copies nothing into that host's bucket
until -yes makes it the writer; the host it replaces then stops copying and
says so.
`
// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
@@ -59,10 +44,9 @@ says so.
const defaultOffsiteEnvFile = "/etc/felis/offsite.env"
// cmdOffsite implements `felis offsite`: the off-site copy of the world
// archives, the database bundles, the registry's user images and the
// submission uploads (internal/offsite). felis-offsite.timer runs `sync`
// hourly on the host; the fetch verbs are the way back after the node is lost
// (docs/troubleshooting.md §16).
// archives and the database bundles (internal/offsite). felis-offsite.timer
// runs `sync` hourly on the host; the fetch verbs are the way back after the
// node is lost (docs/troubleshooting.md §16).
func cmdOffsite(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprint(stderr, offsiteUsage)
@@ -83,14 +67,6 @@ func cmdOffsite(args []string, stdout, stderr io.Writer) int {
return offsiteFetchDB(fs, rest, stdout, stderr)
case "fetch-worlds":
return offsiteFetchWorlds(fs, rest, stdout, stderr)
case "fetch-images":
return offsiteFetchImages(fs, rest, stdout, stderr)
case "fetch-uploads":
return offsiteFetchUploads(fs, rest, stdout, stderr)
case "check-key":
return offsiteCheckKey(fs, rest, stdout, stderr)
case "take-over":
return offsiteTakeOver(fs, rest, stdout, stderr)
case "keygen":
k, err := offsite.NewKey()
if err != nil {
@@ -211,10 +187,6 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
stateDir := fs.String("state-dir", dbbackup.DefaultStateDir, `host state directory bundled into the database bundle taken after archives are copied ("" for none)`)
registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC through the cluster)")
uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, `the submission uploads PVC, in the control-plane namespace ("" copies no uploads)`)
statusFile := fs.String("status-file", offsite.DefaultStatusFile, "where the result of this run is recorded for the watchdog and `status`")
if err := fs.Parse(args); err != nil {
return 2
@@ -224,38 +196,29 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
return 1
}
st, lease := startRun(env.cfg, env.key, *statusFile, time.Now())
res, err := runOffsiteSync(cfg, env, offsiteSources{
archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir, stateDir: *stateDir,
registry: offsiteRegistryEndpoint(*registry, cfg.Registry),
uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
}, &lease, stderr)
recordRun(&st, res, err, lease)
st := offsite.Status{
LastAttempt: time.Now().UTC(), Endpoint: env.cfg.Endpoint, Bucket: env.cfg.Bucket,
Prefix: env.cfg.Prefix, KeyID: offsite.KeyID(env.key),
}
if prev, _ := offsite.ReadStatus(*statusFile); prev != nil {
st.LastSuccess = prev.LastSuccess
}
res, err := runOffsiteSync(cfg, env, *archiveDir, *backupPVC, *dbDir, stderr)
st.Result = res
if err != nil {
st.LastError = err.Error()
} else {
st.LastSuccess = st.LastAttempt
}
if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
}
return reportRun(res, err, stdout, stderr)
}
// reportRun prints one pass's outcome and its exit code. A run stopped before
// it copied anything (a bucket that did not answer, another key's objects,
// another host writing the bucket) prints no counts: its zeros would read as
// an empty bucket.
func reportRun(res offsite.Result, err error, stdout, stderr io.Writer) int {
if err == nil || len(res.Errors) > 0 {
fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n",
res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
res.UploadsUploaded, res.UploadObjectsPruned,
res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes),
res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes))
}
fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; bucket holds %d worlds (%s) and %d bundles\n",
res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
res.DBUploaded, res.DBPruned, res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB)
for _, m := range res.WorldsMissing {
fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
}
for _, m := range res.ImagesIncomplete {
fmt.Fprintf(stderr, "felis offsite sync: registry image not whole: %s\n", m)
}
if err != nil {
fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
return 1
@@ -263,62 +226,8 @@ func reportRun(res offsite.Result, err error, stdout, stderr io.Writer) int {
return 0
}
// startRun begins a pass: its status record, in this release's format and
// carrying the last success over, and this host's lease, read from the record
// the last pass left.
func startRun(cfg config.OffsiteConfig, key []byte, statusFile string, now time.Time) (offsite.Status, offsite.Lease) {
st := offsite.Status{
LastAttempt: now.UTC(), Endpoint: cfg.Endpoint, Bucket: cfg.Bucket,
Prefix: cfg.Prefix, KeyID: offsite.KeyID(key), Format: offsite.StatusFormat,
}
if prev, _ := offsite.ReadStatus(statusFile); prev != nil {
st.LastSuccess = prev.LastSuccess
}
return st, offsite.HostLease(statusFile)
}
// recordRun puts one pass's outcome into its status record. A host that
// inherited the bucket from an older release keeps that until it has an id.
func recordRun(st *offsite.Status, res offsite.Result, err error, lease offsite.Lease) {
st.Result = res
if err != nil {
st.LastError = err.Error()
st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch)
var we *offsite.WriterError
if errors.As(err, &we) {
st.Standby = errors.Is(err, offsite.ErrStandby)
st.Displaced = errors.Is(err, offsite.ErrDisplaced)
st.Writer = we.Writer
}
} else {
st.LastSuccess = st.LastAttempt
}
if lease.Inherited {
id, _ := lease.ID()
st.Inherited = id == ""
}
}
// offsiteSources is where one sync pass reads from: the world archive volume
// (archiveDir, or the backupPVC's directory), the bundle directory (with the
// host state the pass bundles, stateDir), the registry's loopback endpoint and
// the uploads volume (uploadsDir, or the uploadsPVC's directory). An empty
// source is skipped.
type offsiteSources struct {
archiveDir, backupPVC string
dbDir, stateDir string
registry string
uploadsDir, uploadsPVC string
}
// offsiteRunLimit backstops one sync pass. Each upload has its own deadline,
// scaled to its size (internal/offsite), so a pass over a big archive may run
// for hours; the timer starts no second pass while one runs, and the unit's
// TimeoutStartSec sits above this.
const offsiteRunLimit = 23 * time.Hour
func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lease *offsite.Lease, log io.Writer) (offsite.Result, error) {
ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit)
func runOffsiteSync(cfg *config.Config, env *offsiteEnv, archiveDir, backupPVC, dbDir string, log io.Writer) (offsite.Result, error) {
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Minute)
defer cancel()
checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
err := env.bucket.Check(checkCtx)
@@ -326,117 +235,47 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lea
if err != nil {
return offsite.Result{}, err
}
archiveDir, uploadsDir := src.archiveDir, src.uploadsDir
if archiveDir == "" && src.backupPVC != "" {
dir, err := resolveVolumeDir(ctx, cfg.K8s.Namespace, src.backupPVC, archiveVolume, false, log)
if archiveDir == "" && backupPVC != "" {
dir, err := resolveArchiveDir(ctx, cfg.K8s.Namespace, backupPVC, false, log)
if err != nil {
return offsite.Result{}, err
}
archiveDir = dir
}
// An s3:// uploads store is off the host already; only a local one, on
// the uploads PVC, needs the copy.
if uploadsDir == "" && src.uploadsPVC != "" && isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
dir, err := resolveVolumeDir(ctx, platform.DefaultControlNamespace, src.uploadsPVC, uploadsVolume, false, log)
if err != nil {
return offsite.Result{}, err
}
uploadsDir = dir
}
drv, err := openStore(ctx, cfg.Database.URL, false)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
return offsite.Result{}, fmt.Errorf("open database: %w", err)
}
defer drv.Close()
s := offsiteSyncer(cfg, env, src, archiveDir, uploadsDir, lease, log)
s.Catalog = offsite.PGCatalog{DB: drv.DB()}
if src.registry != "" {
s.Images = newRegistryImages(src.registry)
s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL)
s := &offsite.Syncer{
Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key,
ArchiveDir: archiveDir, DBDir: dbDir, DBKeep: env.cfg.DBKeep, Log: log,
}
return s.Run(ctx)
}
// offsiteSyncer is the pass runOffsiteSync runs over the resolved archive and
// uploads directories, before its catalog and registry are attached. It
// snapshots the database into the bundle directory after copying archives, and
// sweeps world objects no backup records once they outlive every retention in
// [archive]; a retention that does not parse sweeps none.
func offsiteSyncer(cfg *config.Config, env *offsiteEnv, src offsiteSources, archiveDir, uploadsDir string, lease *offsite.Lease, log io.Writer) *offsite.Syncer {
s := &offsite.Syncer{
Bucket: env.bucket, Key: env.key,
ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log,
}
if src.dbDir != "" {
s.Snapshot = offsiteSnapshot(cfg.Database, src.dbDir, src.stateDir, log)
}
if rc, err := reaperConfig(cfg); err != nil {
fmt.Fprintf(log, "felis offsite: world objects no backup records are kept: %v\n", err)
} else {
s.OrphanAfter = max(rc.Retention, rc.ManualRetention, rc.ScheduledRetention)
}
return s
}
// offsiteSnapshot takes the bundle a pass sends after copying world archives
// (offsite.Syncer.Snapshot): what `felis db backup` takes, labelled offsite,
// with the newest one kept in dir. It is not recorded for the panel, whose
// backup card watches felis-db-backup.timer: snapshots come only when archives
// are copied, and would hide a daily timer that stopped. It requires the
// MinecraftServer objects: it becomes the newest bundle in the bucket, which a
// lost host restores from, and a pass that cannot take a whole one fails and
// tries again next hour.
func offsiteSnapshot(db config.DatabaseConfig, dir, stateDir string, log io.Writer) func(context.Context) error {
return func(ctx context.Context) error {
tools, err := dbTools(db)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
defer cancel()
path, err := dbbackup.Backup(ctx, dbbackup.BackupOptions{
DatabaseURL: db.URL, Tools: tools, Dir: dir, Label: dbbackup.LabelOffsite,
Keep: defaultKeep[dbbackup.LabelOffsite], StateDir: stateDir, Version: resolvedVersion(),
ExportServers: exportMinecraftServers, RequireServers: true, Log: log,
})
if err == nil {
fmt.Fprintf(log, "felis offsite: took database bundle %s, which lists the archives just copied\n", filepath.Base(path))
}
return err
}
}
// volumeKind names a PVC the off-site copy reads or restores, for messages,
// with the flag that bypasses finding it through the cluster.
type volumeKind struct{ what, dirFlag, empty string }
var (
archiveVolume = volumeKind{"archive volume", "-archive-dir", "no world has been archived"}
uploadsVolume = volumeKind{"uploads volume", "-uploads-dir", "no modpack has been uploaded"}
)
// resolveVolumeDir finds the host directory behind a PVC: a local-path volume
// is a directory on this node. A PVC still waiting for its first consumer
// holds nothing yet: without bind that is "" (nothing to copy), with bind it
// is bound first, for a fetch to write into.
func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind, bind bool, log io.Writer) (string, error) {
// resolveArchiveDir finds the host directory behind the world archive PVC: a
// local-path volume is a directory on this node. A PVC still waiting for its
// first consumer holds nothing yet: without bind that is "" (no archives),
// with bind it is bound first, for fetch-worlds to write into.
func resolveArchiveDir(ctx context.Context, ns, pvcName string, bind bool, log io.Writer) (string, error) {
if ns == "" {
ns = platform.DefaultMinecraftNamespace
}
cl, err := buildSystemServerClient()
if err != nil {
return "", fmt.Errorf("reach the cluster to find the %s (or pass %s): %w", kind.what, kind.dirFlag, err)
return "", fmt.Errorf("reach the cluster to find the archive volume (or pass -archive-dir): %w", err)
}
var pvc corev1.PersistentVolumeClaim
if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
return "", fmt.Errorf("%s %s/%s: %w", kind.what, ns, pvcName, err)
return "", fmt.Errorf("archive volume %s/%s: %w", ns, pvcName, err)
}
if pvc.Spec.VolumeName == "" {
if !bind {
fmt.Fprintf(log, "felis offsite: %s %s/%s is not bound yet; %s\n", kind.what, ns, pvcName, kind.empty)
fmt.Fprintf(log, "felis offsite: archive volume %s/%s is not bound yet; no world has been archived\n", ns, pvcName)
return "", nil
}
if err := bindVolume(ctx, cl, ns, pvcName, kind, log); err != nil {
if err := bindVolume(ctx, cl, ns, pvcName, log); err != nil {
return "", err
}
if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
@@ -445,7 +284,7 @@ func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind,
}
var pv corev1.PersistentVolume
if err := cl.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil {
return "", fmt.Errorf("%s %s: %w", kind.what, pvc.Spec.VolumeName, err)
return "", fmt.Errorf("archive volume %s: %w", pvc.Spec.VolumeName, err)
}
var dir string
switch {
@@ -454,10 +293,10 @@ func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind,
case pv.Spec.HostPath != nil:
dir = pv.Spec.HostPath.Path
default:
return "", fmt.Errorf("%s %s is not a directory on a node (local or hostPath); pass %s with where it is mounted on this host", kind.what, pv.Name, kind.dirFlag)
return "", fmt.Errorf("archive volume %s is not a directory on a node (local or hostPath); pass -archive-dir with where it is mounted on this host", pv.Name)
}
if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
return "", fmt.Errorf("%s %s is %s on its node, which is not a directory here; run this on the node that holds it, or pass %s", kind.what, pv.Name, dir, kind.dirFlag)
return "", fmt.Errorf("archive volume %s is %s on its node, which is not a directory here; run this on the node that holds it, or pass -archive-dir", pv.Name, dir)
}
return dir, nil
}
@@ -465,10 +304,10 @@ func resolveVolumeDir(ctx context.Context, ns, pvcName string, kind volumeKind,
// bindVolume runs a pod that mounts the PVC and exits, which is what makes a
// WaitForFirstConsumer volume (k3s local-path) get provisioned. The pod uses
// the control plane's own image, which every install already has.
func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind volumeKind, log io.Writer) error {
func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, log io.Writer) error {
var api appsv1.Deployment
if err := cl.Get(ctx, types.NamespacedName{Namespace: platform.DefaultControlNamespace, Name: "felis-api"}, &api); err != nil {
return fmt.Errorf("find the felis image to bind the %s with: %w", kind.what, err)
return fmt.Errorf("find the felis image to bind the archive volume with: %w", err)
}
if len(api.Spec.Template.Spec.Containers) == 0 {
return errors.New("felis-api has no container to take the image from")
@@ -476,9 +315,9 @@ func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind
image := api.Spec.Template.Spec.Containers[0].Image
pod := platform.VolumeBinderPod(ns, pvcName, image)
if err := cl.Create(ctx, pod); err != nil {
return fmt.Errorf("start a pod to bind the %s: %w", kind.what, err)
return fmt.Errorf("start a pod to bind the archive volume: %w", err)
}
fmt.Fprintf(log, "felis offsite: binding the %s %s/%s (pod %s)\n", kind.what, ns, pvcName, pod.Name)
fmt.Fprintf(log, "felis offsite: binding the archive volume %s/%s (pod %s)\n", ns, pvcName, pod.Name)
defer func() {
_ = cl.Delete(context.Background(), pod, client.PropagationPolicy(metav1.DeletePropagationBackground))
}()
@@ -494,7 +333,7 @@ func bindVolume(ctx context.Context, cl client.Client, ns, pvcName string, kind
case <-time.After(2 * time.Second):
}
}
return fmt.Errorf("the %s %s/%s did not bind within 3 minutes; see kubectl -n %s describe pod %s", kind.what, ns, pvcName, ns, pod.Name)
return fmt.Errorf("the archive volume %s/%s did not bind within 3 minutes; see kubectl -n %s describe pod %s", ns, pvcName, ns, pod.Name)
}
func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
@@ -509,7 +348,7 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
return 1
}
if !cfg.Offsite.Enabled() {
fmt.Fprintln(stdout, "off-site copy: not configured. World archives, database bundles, user images and uploaded modpacks exist on this machine only.")
fmt.Fprintln(stdout, "off-site copy: not configured. World archives and database bundles exist on this machine only.")
fmt.Fprintln(stdout, "See docs/troubleshooting.md §16, \"Keep a copy somewhere else\".")
return 1
}
@@ -539,49 +378,13 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
if st.LastError != "" {
fmt.Fprintf(stdout, "last error: %s\n", st.LastError)
}
if st.KeyMismatch {
fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile)
return 1
}
if st.Displaced && st.Writer != nil {
fmt.Fprintf(stdout, "\nThe last run was refused: %s took the bucket over (it last wrote it at %s), and this host copies nothing there any more. If that host is a rehearsal machine, take the bucket back: sudo felis offsite take-over -yes\n",
st.Writer, st.Writer.At.Local().Format(time.DateTime))
return 1
}
if st.Standby {
switch w := st.StandsBy(now); {
case w != nil:
fmt.Fprintf(stdout, "\nThis host stands by: %s writes the bucket (last at %s). This host was built from its backup, copies nothing into the bucket and, while that host keeps writing, mails no watchdog alert.", w, w.At.Local().Format(time.DateTime))
case st.Writer != nil:
fmt.Fprintf(stdout, "\nThis host copies nothing into the bucket: %s wrote it, last at %s, and this host was built from its backup.", st.Writer, st.Writer.At.Local().Format(time.DateTime))
default:
fmt.Fprint(stdout, "\nThis host copies nothing into the bucket: it holds copies this host did not write, and names no host writing it.")
}
fmt.Fprintln(stdout, " Once this host replaces that one for good: sudo felis offsite take-over -yes")
return 1
}
r := st.Result
fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
if r.ImageIndex != "" {
fmt.Fprintf(stdout, "images: %d in %d repositories (%s), registry index %s\n",
r.Images, r.ImageRepos, offsite.HumanBytes(r.RemoteImageBytes), r.ImageIndex)
} else {
fmt.Fprintln(stdout, "images: not copied (no in-cluster registry, or no sync has reached it yet)")
}
if r.UploadIndex != "" {
fmt.Fprintf(stdout, "uploads: %d submission contexts (%s), uploads index %s\n",
r.Uploads, offsite.HumanBytes(r.RemoteUploadBytes), r.UploadIndex)
} else {
fmt.Fprintln(stdout, "uploads: not copied (an s3:// uploads store, or no sync has reached the volume yet)")
}
fmt.Fprintf(stdout, "waiting: %d world archives not yet copied\n", r.WorldsPending)
for _, m := range r.WorldsMissing {
fmt.Fprintf(stdout, "missing: %s is recorded but not on the volume\n", m)
}
for _, m := range r.ImagesIncomplete {
fmt.Fprintf(stdout, "not whole: %s\n", m)
}
if st.LastSuccess.IsZero() || now.Sub(st.LastSuccess) > offsite.StaleAfter {
fmt.Fprintf(stdout, "\nThe last successful sync is older than %s: journalctl -u felis-offsite -n 50\n", dbbackup.Age(offsite.StaleAfter))
return 1
@@ -623,201 +426,18 @@ func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
return 1
}
printDBBundles(ctx, env.bucket, env.key, bundles, stdout)
fmt.Fprintf(stdout, "database bundles (%d, newest first):\n", len(bundles))
for _, b := range bundles {
fmt.Fprintf(stdout, " %s %s\n", b.Key, offsite.HumanBytes(b.Size))
}
var total int64
for _, w := range worlds {
total += w.Size
}
fmt.Fprintf(stdout, "world archives: %d (%s)\n", len(worlds), offsite.HumanBytes(total))
versions, err := offsite.ImageIndexes(ctx, env.bucket)
if err != nil {
fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "registry index versions (%d, newest first; restore one with fetch-images -at):\n", len(versions))
for i := len(versions) - 1; i >= 0; i-- {
x, err := offsite.LoadImageIndex(ctx, env.bucket, env.key, versions[i])
if err != nil {
fmt.Fprintf(stdout, " %s unreadable: %v\n", versions[i], err)
continue
}
fmt.Fprintf(stdout, " %s %d images in %d repositories\n", versions[i], x.Images(), len(x.Repositories))
}
uploads, err := offsite.UploadIndexes(ctx, env.bucket)
if err != nil {
fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "uploads index versions (%d, newest first; restore one with fetch-uploads -at):\n", len(uploads))
for i := len(uploads) - 1; i >= 0; i-- {
x, err := offsite.LoadUploadIndex(ctx, env.bucket, env.key, uploads[i])
if err != nil {
fmt.Fprintf(stdout, " %s unreadable: %v\n", uploads[i], err)
continue
}
fmt.Fprintf(stdout, " %s %d submission contexts (%s)\n", uploads[i], len(x.Contexts), offsite.HumanBytes(x.Bytes()))
}
return 0
}
// printDBBundles lists the database bundles with what each one's database
// held, read off the front of each, so a restore can pick one by its contents.
func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles []offsite.Object, stdout io.Writer) {
fmt.Fprintf(stdout, "database bundles (%d, newest first; restore one with fetch-db):\n", len(bundles))
for _, o := range bundles {
m, err := offsite.PeekDB(ctx, b, key, o.Key)
if err != nil {
fmt.Fprintf(stdout, " %s %s unreadable: %v\n", o.Key, offsite.HumanBytes(o.Size), err)
continue
}
gap := ""
if m.ServersError != "" {
gap = ", no MinecraftServer objects"
}
fmt.Fprintf(stdout, " %s %s %s%s\n", o.Key, offsite.HumanBytes(o.Size), m.Counts.String(), gap)
}
}
func offsiteCheckKey(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
if err := fs.Parse(args); err != nil {
return 2
}
_, env, err := loadOffsite(*cfgPath, *envFile)
if err != nil {
fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if err := env.bucket.Check(ctx); err != nil {
fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
return 1
}
return checkKey(ctx, env.bucket, env.key, stdout, stderr)
}
// checkKey is check-key once the bucket is open: 0 when the key fits, 3 when
// the bucket's objects are sealed with another one, 1 when it cannot tell.
func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr io.Writer) int {
fit, err := offsite.CheckKey(ctx, b, key)
if err != nil {
fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
if errors.Is(err, offsite.ErrKeyMismatch) {
return 3
}
return 1
}
id := offsite.KeyID(key)
switch fit {
case offsite.KeyRecorded:
fmt.Fprintf(stdout, "felis offsite check-key: the bucket records key id %s, this key's\n", id)
case offsite.KeyOpens:
fmt.Fprintf(stdout, "felis offsite check-key: the bucket's newest objects open with this key (key id %s); the next sync records it\n", id)
case offsite.KeyUnused:
fmt.Fprintf(stdout, "felis offsite check-key: the bucket holds no sealed object yet; the first sync records key id %s\n", id)
}
return 0
}
func offsiteTakeOver(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes; this host's id is kept next to it")
yes := fs.Bool("yes", false, "make this host the one that writes the bucket")
if err := fs.Parse(args); err != nil {
return 2
}
_, env, err := loadOffsite(*cfgPath, *envFile)
if err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if err := env.bucket.Check(ctx); err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
return 1
}
return takeOver(ctx, env.bucket, env.key, offsite.HostLease(*statusFile), *statusFile, *yes, time.Now(), stdout, stderr)
}
// takeOver is take-over once the bucket is open: without yes it says which
// host writes the bucket, 0 for this one (or none yet), 4 for another and 5
// for one that took the bucket over from this host; with
// yes it records this host as the writer. A key the bucket's objects refuse
// is 3, as in check-key: taking over a bucket this host cannot copy into
// would only stop the host that can.
func takeOver(ctx context.Context, b offsite.Bucket, key []byte, lease offsite.Lease, statusFile string, yes bool, now time.Time, stdout, stderr io.Writer) int {
fit, err := offsite.CheckKey(ctx, b, key)
if err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
if errors.Is(err, offsite.ErrKeyMismatch) {
return 3
}
return 1
}
role, w, err := lease.Plan(ctx, b, fit == offsite.KeyUnused)
if err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
return 1
}
switch role {
case offsite.RoleWrites:
id, _ := lease.ID()
fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket; nothing to take over\n", id)
return 0
case offsite.RoleClaims:
fmt.Fprintln(stdout, "felis offsite take-over: the bucket names no host writing it; this host's next sync records itself")
return 0
}
who := "another host"
if w != nil {
who = w.String()
fmt.Fprintf(stdout, "felis offsite take-over: %s writes the bucket, last at %s (%s ago)\n", w, w.At.Local().Format(time.DateTime), dbbackup.Age(now.Sub(w.At)))
} else {
fmt.Fprintln(stdout, "felis offsite take-over: the bucket holds copies this host did not write, and names no host writing it")
}
if !yes {
if role == offsite.RoleDisplaced {
fmt.Fprintf(stdout, "It took the bucket over from this host: this host copies nothing there any more, and its watchdog mails the owners about it. If that host is a rehearsal machine, take the bucket back:\n sudo felis offsite take-over -yes\n")
return 5
}
fmt.Fprintf(stdout, "This host was built from its backup and copies nothing into the bucket.\n")
fmt.Fprintf(stdout, "Taking it over makes this host the one that copies into the bucket and prunes it; %s stops at its next copy and mails its owners. Do it once that host is gone for good, or is a rehearsal machine you are done with:\n sudo felis offsite take-over -yes\n", who)
return 4
}
if _, err := lease.TakeOver(ctx, b, now); err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
return 1
}
// The refusal the last sync recorded is over: the watchdog mails again
// from now on, and status shows the next run's outcome.
if st, err := offsite.ReadStatus(statusFile); err == nil && st != nil && (st.Standby || st.Displaced) {
st.Standby, st.Displaced, st.Writer, st.LastError, st.Inherited = false, false, nil, "", false
if err := offsite.WriteStatus(statusFile, *st); err != nil {
fmt.Fprintf(stderr, "felis offsite take-over: record status: %v\n", err)
}
}
id, _ := lease.ID()
fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket now; %s stops at its next copy.\nStart the first copy: sudo systemctl start felis-offsite.service\n", id, who)
return 0
}
// keyHint explains an object the key cannot open when the bucket records
// another key's id, "" otherwise.
func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string {
if !errors.Is(err, offsite.ErrAuth) {
return ""
}
id, ierr := offsite.BucketKeyID(ctx, b)
if ierr != nil || id == "" || id == offsite.KeyID(key) {
return ""
}
return fmt.Sprintf("\n the bucket records key id %s, and this key is %s: set FELIS_OFFSITE_KEY to the key the bucket was written with", id, offsite.KeyID(key))
}
func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
@@ -860,47 +480,37 @@ func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) i
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
return fetchDB(ctx, env.bucket, env.key, arg, *dir, time.Now(), stdout, stderr)
}
// fetchDB is fetch-db once the bucket is open: arg is a bundle name or latest.
func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string, now time.Time, stdout, stderr io.Writer) int {
name := arg
if name == "latest" {
var err error
if name, _, err = offsite.ChooseDB(ctx, b, key); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
bundles, err := offsite.ListDB(ctx, env.bucket)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
return 1
}
if len(bundles) == 0 {
fmt.Fprintln(stderr, "felis offsite fetch-db: the bucket holds no database bundle")
return 1
}
name = bundles[0].Key
}
if _, _, ok := dbbackup.ParseBundleName(name); !ok {
fmt.Fprintf(stderr, "felis offsite fetch-db: %q is not a bundle name (felis-db-<stamp>-<label>.tar); see `felis offsite list`\n", name)
return 2
}
if err := os.MkdirAll(dir, 0o700); err != nil {
if err := os.MkdirAll(*dir, 0o700); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
return 1
}
dst := filepath.Join(dir, name)
if err := offsite.FetchObject(ctx, b, key, offsite.DBKey(name), dst, 0o600); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
dst := filepath.Join(*dir, name)
if err := offsite.FetchObject(ctx, env.bucket, env.key, offsite.DBKey(name), dst, 0o600); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
return 1
}
m, err := dbbackup.Verify(dst)
if err != nil {
if _, err := dbbackup.Verify(dst); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: fetched %s but it does not verify: %v\n", dst, err)
return 1
}
fmt.Fprintf(stdout, "felis offsite fetch-db: wrote %s (verified)\n", dst)
fmt.Fprintf(stdout, " taken %s (%s, %s ago)\n felis %s, schema %d\n holds %s\n",
m.CreatedAt.Format(time.RFC3339), m.Label, dbbackup.Age(now.Sub(m.CreatedAt)),
orUnknown(m.FelisVersion), m.SchemaVersion, m.Counts.String())
if m.Counts.Fresh() {
fmt.Fprintln(stdout, " This database holds no servers and at most one account, like a new install's. Check it is the state to restore before `felis db restore`.")
}
if m.ServersError != "" {
fmt.Fprintf(stdout, " This bundle lacks the MinecraftServer objects (%s): `felis db restore` brings back the database, and the servers come from k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap.\n", m.ServersError)
}
return 0
}
@@ -921,12 +531,12 @@ func offsiteFetchWorlds(fs *flag.FlagSet, args []string, stdout, stderr io.Write
defer cancel()
dir := *archiveDir
if dir == "" {
if dir, err = resolveVolumeDir(ctx, cfg.K8s.Namespace, *backupPVC, archiveVolume, true, stderr); err != nil {
if dir, err = resolveArchiveDir(ctx, cfg.K8s.Namespace, *backupPVC, true, stderr); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-worlds: %v\n", err)
return 1
}
}
drv, err := openStore(ctx, cfg.Database.URL, false)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-worlds: open database: %v\n", err)
return 1
cmd/felis/offsite_images.go
-213
View File
@@ -1,213 +0,0 @@
package main
import (
"context"
"database/sql"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"slices"
"strings"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/imagepush"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/registrygate"
"felis.lolicon.best/internal/registryprune"
)
// registryImages is the platform registry as the off-site copy sees it: read
// anonymously through the gate (the catalog, its manifest index, manifests and
// blobs) and, for a restore, written as the platform principal. Both go through
// the node's loopback hostPort, the way the installer pushes.
type registryImages struct {
host string
index *registryprune.Client
source *imagepush.Source
pusher *imagepush.Pusher
}
func newRegistryImages(endpoint string) *registryImages {
return &registryImages{
host: endpoint,
index: &registryprune.Client{Endpoint: "http://" + endpoint},
source: &imagepush.Source{Scheme: "http"},
}
}
func (r *registryImages) Repositories(ctx context.Context) ([]string, error) {
return r.index.Repositories(ctx)
}
func (r *registryImages) Revisions(ctx context.Context, repo string) ([]string, map[string]string, error) {
idx, err := r.index.Index(ctx, repo)
if err != nil {
return nil, nil, err
}
digests := make([]string, 0, len(idx.Revisions))
for _, rev := range idx.Revisions {
digests = append(digests, rev.Digest)
}
return digests, idx.Tags, nil
}
func (r *registryImages) Manifest(ctx context.Context, repo, digest string) ([]byte, string, error) {
body, mt, err := r.source.Manifest(ctx, r.host, repo, digest)
return body, mt, registryGone(err)
}
func (r *registryImages) Blob(ctx context.Context, repo, digest string) (io.ReadCloser, error) {
rc, err := r.source.Blob(ctx, r.host, repo, digest)
return rc, registryGone(err)
}
func (r *registryImages) PutBlob(ctx context.Context, repo, digest string, size int64, open func() (io.ReadCloser, error)) error {
return r.pusher.UploadBlob(ctx, r.host, repo, digest, size, open)
}
func (r *registryImages) PutManifest(ctx context.Context, repo, reference, mediaType string, body []byte) error {
_, err := r.pusher.PutManifest(ctx, r.host, repo, reference, mediaType, body)
return err
}
// imagePins lists, per repository of the registry refs spell as host, the
// digests the MinecraftServers' specs and the image whitelist pin: what a
// restored database and its servers will ask the registry for.
func imagePins(db *sql.DB, host string) func(ctx context.Context) (map[string][]string, error) {
return func(ctx context.Context) (map[string][]string, error) {
cl, err := buildSystemServerClient()
if err != nil {
return nil, fmt.Errorf("reach the cluster: %w", err)
}
var servers v1alpha1.MinecraftServerList
if err := cl.List(ctx, &servers); err != nil {
return nil, fmt.Errorf("list MinecraftServers: %w", err)
}
refs := make([]string, 0, len(servers.Items))
for _, s := range servers.Items {
refs = append(refs, s.Spec.Image)
}
rows, err := db.QueryContext(ctx, `SELECT image_ref FROM image_whitelist`)
if err != nil {
return nil, fmt.Errorf("read the image whitelist: %w", err)
}
defer rows.Close()
for rows.Next() {
var ref string
if err := rows.Scan(&ref); err != nil {
return nil, fmt.Errorf("read the image whitelist: %w", err)
}
refs = append(refs, ref)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read the image whitelist: %w", err)
}
pins := map[string][]string{}
for _, ref := range refs {
repo, _, digest, ok := registryprune.ParseRef(ref, host)
if ok && digest != "" && !slices.Contains(pins[repo], digest) {
pins[repo] = append(pins[repo], digest)
}
}
return pins, nil
}
}
// registryGone marks a 404 as a manifest or blob the registry no longer holds.
func registryGone(err error) error {
var se *imagepush.StatusError
if errors.As(err, &se) && se.Code == http.StatusNotFound {
return fmt.Errorf("%w: %v", offsite.ErrImageGone, err)
}
return err
}
// offsiteRegistryEndpoint is where the host reaches the registry whose images
// the off-site copy covers: flag when given ("off" for none), otherwise the
// loopback hostPort of the in-cluster registry [registry] url names. A
// registry outside the cluster is not this install's to copy.
func offsiteRegistryEndpoint(flag string, reg config.RegistryConfig) string {
switch flag {
case "off":
return ""
case "":
default:
return flag
}
host, _, _ := strings.Cut(reg.URL, "/")
name, _, _ := strings.Cut(host, ":")
if strings.HasSuffix(name, ".svc") || strings.HasSuffix(name, ".svc.cluster.local") {
return loopbackEndpoint(host)
}
return ""
}
// registryWriteCredential is the credential a host-side push uses:
// FELIS_REGISTRY_USERNAME/PASSWORD when set, otherwise the platform principal
// with REGISTRY_PLATFORM_TOKEN from the installer's secrets file.
func registryWriteCredential(secrets string) (string, string, error) {
if err := loadEnvFile(secrets); err != nil {
return "", "", fmt.Errorf("read %s: %w", secrets, err)
}
user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
if pass == "" {
user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
}
if pass == "" {
return "", "", errors.New("no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
}
return user, pass, nil
}
func offsiteFetchImages(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
registry := fs.String("registry", "", "host[:port] of the registry to push into (default: the loopback hostPort of the in-cluster registry)")
secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
at := fs.String("at", "", "registry index version to restore (default: the newest; `felis offsite list` shows them)")
if err := fs.Parse(args); err != nil {
return 2
}
cfg, env, err := loadOffsite(*cfgPath, *envFile)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
return 1
}
endpoint := offsiteRegistryEndpoint(*registry, cfg.Registry)
if endpoint == "" {
fmt.Fprintf(stderr, "felis offsite fetch-images: [registry] url %q is not the in-cluster registry; pass -registry host:port\n", cfg.Registry.URL)
return 2
}
user, pass, err := registryWriteCredential(*secrets)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
return 2
}
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
defer cancel()
stamp, idx, err := offsite.ChooseImageIndex(ctx, env.bucket, env.key, *at)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "felis offsite fetch-images: restoring registry index %s (%d repositories, %d images) into %s\n",
stamp, len(idx.Repositories), idx.Images(), endpoint)
target := newRegistryImages(endpoint)
target.pusher = &imagepush.Pusher{Scheme: "http", Username: user, Password: pass}
res, err := offsite.FetchImages(ctx, env.bucket, env.key, idx, target, stderr)
fmt.Fprintf(stdout, "felis offsite fetch-images: %d of %d repositories restored, %d images, %d tags, %d blobs pushed (%s)\n",
res.Repositories, len(idx.Repositories), res.Manifests, res.Tags, res.BlobsPushed, offsite.HumanBytes(res.BytesPushed))
for _, f := range res.Failures {
fmt.Fprintf(stderr, "felis offsite fetch-images: %s\n", f)
}
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-images: %v (a second run pushes only what is still missing)\n", err)
return 1
}
return 0
}
cmd/felis/offsite_test.go
-679
View File
@@ -1,25 +1,13 @@
package main
import (
"archive/tar"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/imagepush"
"felis.lolicon.best/internal/offsite"
)
@@ -106,670 +94,3 @@ func TestOffsiteFetchDBRejectsOddNames(t *testing.T) {
t.Fatalf("exit %d: %s", code, errb.String())
}
}
func TestOffsiteRegistryEndpoint(t *testing.T) {
for _, tc := range []struct{ flag, url, want string }{
{"", "registry.felis.svc:5000", "127.0.0.1:5000"},
{"", "registry.felis.svc.cluster.local:5001", "127.0.0.1:5001"},
{"", "ghcr.io/acme", ""},
{"", "", ""},
{"off", "registry.felis.svc:5000", ""},
{"10.0.0.5:5000", "ghcr.io/acme", "10.0.0.5:5000"},
} {
if got := offsiteRegistryEndpoint(tc.flag, config.RegistryConfig{URL: tc.url}); got != tc.want {
t.Errorf("offsiteRegistryEndpoint(%q, %q) = %q, want %q", tc.flag, tc.url, got, tc.want)
}
}
}
func TestRegistryGoneMarksNotFound(t *testing.T) {
if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 404}); !errors.Is(err, offsite.ErrImageGone) {
t.Fatalf("404 = %v, want ErrImageGone", err)
}
if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 503}); errors.Is(err, offsite.ErrImageGone) {
t.Fatalf("503 = %v, want it kept an ordinary failure", err)
}
}
// mapBucket is an in-memory offsite.Bucket.
type mapBucket map[string][]byte
func (b mapBucket) Put(_ context.Context, key string, r io.Reader, _ int64) error {
data, err := io.ReadAll(r)
b[key] = data
return err
}
func (b mapBucket) Get(_ context.Context, key string) (io.ReadCloser, error) {
data, ok := b[key]
if !ok {
return nil, offsite.ErrNotFound
}
return io.NopCloser(bytes.NewReader(data)), nil
}
func (b mapBucket) List(_ context.Context, prefix string) ([]offsite.Object, error) {
var out []offsite.Object
for k, v := range b {
if strings.HasPrefix(k, prefix) {
out = append(out, offsite.Object{Key: k, Size: int64(len(v))})
}
}
return out, nil
}
func (b mapBucket) Remove(_ context.Context, key string) error {
delete(b, key)
return nil
}
var fetchT0 = time.Date(2026, 9, 20, 3, 30, 0, 0, time.UTC)
// putBundle seals a bundle that verifies, taken daysAgo days before fetchT0,
// into b and returns its name.
func putBundle(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts) string {
t.Helper()
return putBundleWith(t, b, key, daysAgo, counts, "")
}
// putBundleWith is putBundle for a bundle whose server export failed with
// serversError, when that is not empty.
func putBundleWith(t *testing.T, b mapBucket, key []byte, daysAgo int, counts *dbbackup.Counts, serversError string) string {
t.Helper()
created := fetchT0.AddDate(0, 0, -daysAgo)
dump := []byte("PGDMP " + created.String())
sum := sha256.Sum256(dump)
manifest, err := json.Marshal(dbbackup.Manifest{
Format: 1, CreatedAt: created, Label: dbbackup.LabelDaily, FelisVersion: "v1.2.3", SchemaVersion: 21, Counts: counts, ServersError: serversError,
Files: []dbbackup.ManifestEntry{{Name: "db.dump", Size: int64(len(dump)), SHA256: hex.EncodeToString(sum[:]), Mode: 0o600}},
})
if err != nil {
t.Fatal(err)
}
var plain bytes.Buffer
tw := tar.NewWriter(&plain)
for _, f := range []struct {
name string
data []byte
}{{"MANIFEST.json", manifest}, {"db.dump", dump}} {
if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o600, Size: int64(len(f.data)), Typeflag: tar.TypeReg}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write(f.data); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
var sealed bytes.Buffer
if err := offsite.Encrypt(&sealed, &plain, key); err != nil {
t.Fatal(err)
}
name := dbbackup.BundleName(created, dbbackup.LabelDaily)
b[offsite.DBKey(name)] = sealed.Bytes()
return name
}
func TestOffsiteFetchDB(t *testing.T) {
rawKey, _ := offsite.NewKey()
key, _ := offsite.ParseKey(rawKey)
now := fetchT0.Add(2 * time.Hour)
fetch := func(b mapBucket, arg string) (dir string, code int, stdout, stderr string) {
dir = t.TempDir()
var out, errb bytes.Buffer
code = fetchDB(context.Background(), b, key, arg, dir, now, &out, &errb)
return dir, code, out.String(), errb.String()
}
fetched := func(t *testing.T, dir string) []string {
t.Helper()
var names []string
entries, _ := os.ReadDir(dir)
for _, e := range entries {
names = append(names, e.Name())
}
return names
}
t.Run("latest skips a rebuilt host's empty bundle", func(t *testing.T) {
b := mapBucket{}
full := putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
dir, code, out, errb := fetch(b, "latest")
if code != 1 || !strings.Contains(errb, empty) || !strings.Contains(errb, full+" (5 accounts, 3 servers)") {
t.Fatalf("exit %d, stdout %q, stderr %q; want a refusal naming %s", code, out, errb, full)
}
if got := fetched(t, dir); len(got) != 0 {
t.Errorf("a refused fetch wrote %v", got)
}
})
t.Run("latest takes the newest bundle and says what it holds", func(t *testing.T) {
b := mapBucket{}
putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 2})
newest := putBundle(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3})
dir, code, out, errb := fetch(b, "latest")
if code != 0 {
t.Fatalf("exit %d: %s", code, errb)
}
if got := fetched(t, dir); !slices.Equal(got, []string{newest}) {
t.Errorf("wrote %v, want %s", got, newest)
}
for _, want := range []string{
"wrote " + filepath.Join(dir, newest) + " (verified)",
"taken 2026-09-19T03:30:00Z (daily, 26h0m ago)",
"felis v1.2.3, schema 21",
"holds 5 accounts, 3 servers",
} {
if !strings.Contains(out, want) {
t.Errorf("stdout lacks %q:\n%s", want, out)
}
}
if strings.Contains(out, "new install") {
t.Errorf("a bundle with servers flagged as a new install's:\n%s", out)
}
})
t.Run("an empty bundle named outright is fetched with a warning", func(t *testing.T) {
b := mapBucket{}
putBundle(t, b, key, 3, &dbbackup.Counts{Users: 5, Servers: 3})
empty := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 1})
dir, code, out, errb := fetch(b, empty)
if code != 0 || !slices.Equal(fetched(t, dir), []string{empty}) {
t.Fatalf("exit %d, wrote %v: %s", code, fetched(t, dir), errb)
}
if !strings.Contains(out, "holds 1 account, 0 servers") || !strings.Contains(out, "like a new install's") {
t.Errorf("stdout = %s", out)
}
})
t.Run("a bundle without the servers says where they come from", func(t *testing.T) {
b := mapBucket{}
gapped := putBundleWith(t, b, key, 1, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused (tried 3 times)")
_, code, out, errb := fetch(b, gapped)
if code != 0 || !strings.Contains(out, "This bundle lacks the MinecraftServer objects (connection refused (tried 3 times))") ||
!strings.Contains(out, "k8s/minecraftservers.json in the newest bundle `felis offsite list` shows without that gap") {
t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
}
whole := putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
if _, _, out, _ := fetch(b, whole); strings.Contains(out, "lacks the MinecraftServer objects") {
t.Errorf("a whole bundle flagged:\n%s", out)
}
})
t.Run("a bundle from before counts says so", func(t *testing.T) {
b := mapBucket{}
old := putBundle(t, b, key, 0, nil)
_, code, out, errb := fetch(b, "latest")
if code != 0 || !strings.Contains(out, old) || !strings.Contains(out, "holds not recorded") || strings.Contains(out, "new install") {
t.Errorf("exit %d, stdout %q, stderr %q", code, out, errb)
}
})
}
func TestOffsiteCheckKey(t *testing.T) {
newKey := func() []byte {
raw, _ := offsite.NewKey()
k, _ := offsite.ParseKey(raw)
return k
}
key, other := newKey(), newKey()
marked := func(k []byte) mapBucket { return mapBucket{"felis-key-id": []byte(offsite.KeyID(k) + "\n")} }
unmarked := func(k []byte) mapBucket {
b := mapBucket{}
putBundle(t, b, k, 0, nil)
return b
}
for _, tc := range []struct {
what string
bucket mapBucket
code int
says []string
}{
{"the recorded key", marked(key), 0, []string{"records key id " + offsite.KeyID(key)}},
{"an unmarked bucket the key opens", unmarked(key), 0, []string{"open with this key", "the next sync records it"}},
{"an empty bucket", mapBucket{}, 0, []string{"no sealed object yet", "records key id " + offsite.KeyID(key)}},
{"another recorded key", marked(other), 3, []string{offsite.KeyID(other), offsite.KeyID(key), "FELIS_OFFSITE_KEY"}},
{"an unmarked bucket under another key", unmarked(other), 3, []string{"opens none of db/felis-db-"}},
{"a marker Felis did not write", mapBucket{"felis-key-id": []byte("hello")}, 1, []string{"not a key id"}},
} {
t.Run(tc.what, func(t *testing.T) {
before := len(tc.bucket)
var out, errb bytes.Buffer
code := checkKey(context.Background(), tc.bucket, key, &out, &errb)
if code != tc.code {
t.Fatalf("exit %d, want %d; stdout %q, stderr %q", code, tc.code, out.String(), errb.String())
}
said := out.String() + errb.String()
for _, s := range tc.says {
if !strings.Contains(said, s) {
t.Errorf("output lacks %q: %s", s, said)
}
}
if len(tc.bucket) != before {
t.Errorf("check-key wrote to the bucket: %d objects, had %d", len(tc.bucket), before)
}
})
}
// fetch-db names both ids when the bucket records another key.
b := marked(other)
name := putBundle(t, b, other, 0, &dbbackup.Counts{Users: 5, Servers: 3})
for _, arg := range []string{"latest", name} {
var out, errb bytes.Buffer
if code := fetchDB(context.Background(), b, key, arg, t.TempDir(), fetchT0, &out, &errb); code != 1 ||
!strings.Contains(errb.String(), "the bucket records key id "+offsite.KeyID(other)+", and this key is "+offsite.KeyID(key)) {
t.Errorf("fetch-db %s under another key: exit %d, stderr %q", arg, code, errb.String())
}
}
// A bundle the bucket lacks is not the key's fault.
var missOut, missErr bytes.Buffer
if code := fetchDB(context.Background(), b, key, "felis-db-20200101T000000Z-daily.tar", t.TempDir(), fetchT0, &missOut, &missErr); code != 1 || strings.Contains(missErr.String(), "records key id") {
t.Errorf("missing bundle: exit %d, stderr %q", code, missErr.String())
}
// A bundle damaged under the recorded key gets no such hint.
b = marked(key)
name = putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
b[offsite.DBKey(name)][60] ^= 1
var out, errb bytes.Buffer
if code := fetchDB(context.Background(), b, key, name, t.TempDir(), fetchT0, &out, &errb); code != 1 || strings.Contains(errb.String(), "records key id") {
t.Errorf("damaged bundle: exit %d, stderr %q", code, errb.String())
}
}
func TestRecordRun(t *testing.T) {
t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}
for _, tc := range []struct {
err error
mismatch, standby, displaced, success bool
}{
{nil, false, false, false, true},
{errors.New("list worlds/ in the bucket: connection reset"), false, false, false, false},
{fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false, false, false},
{&offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, false, true, false, false},
{&offsite.WriterError{Kind: offsite.ErrDisplaced, Writer: w}, false, false, true, false},
} {
st := offsite.Status{LastAttempt: t0}
recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err, offsite.Lease{})
if st.KeyMismatch != tc.mismatch || st.Standby != tc.standby || st.Displaced != tc.displaced ||
(st.Writer != nil) != (tc.standby || tc.displaced) || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 {
t.Errorf("err %v: status %+v", tc.err, st)
}
}
// A host that copied before writers were recorded keeps that claim over
// failed runs until it has an id.
l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Inherited: true}
st := offsite.Status{}
recordRun(&st, offsite.Result{}, errors.New("cannot reach bucket"), l)
if !st.Inherited {
t.Error("a failed run on a host with no id dropped the older release's claim")
}
writeTestFile(t, l.IDFile, "aaaaaaaaaaaaaaaa\n", 0o600)
st = offsite.Status{Inherited: true}
recordRun(&st, offsite.Result{}, nil, l)
if st.Inherited {
t.Error("a host with an id still carries the older release's claim")
}
}
// A refused run has copied nothing and listed nothing: its zero counts would
// tell the journal the bucket is empty. A pass that ran and failed a step
// shows what it did get to.
func TestReportRun(t *testing.T) {
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)}
for _, tc := range []struct {
name string
res offsite.Result
err error
code int
counts bool
}{
{"a pass", offsite.Result{DBUploaded: 1, RemoteDB: 3}, nil, 0, true},
{"a pass with a failed step", offsite.Result{RemoteDB: 3, Errors: []string{"copy db/x: timeout"}}, errors.New("1 of this run's steps failed; first: copy db/x: timeout"), 1, true},
{"standing by", offsite.Result{}, &offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, 1, false},
{"another key", offsite.Result{}, fmt.Errorf("%w: the bucket records key id 1111111111111111", offsite.ErrKeyMismatch), 1, false},
{"no bucket", offsite.Result{}, errors.New("bucket: access denied"), 1, false},
} {
t.Run(tc.name, func(t *testing.T) {
var out, errOut bytes.Buffer
code := reportRun(tc.res, tc.err, &out, &errOut)
if code != tc.code {
t.Errorf("exit %d, want %d", code, tc.code)
}
if got := strings.Contains(out.String(), "bucket holds 0 worlds (0 B), 3 bundles"); got != tc.counts {
t.Errorf("counts shown = %v, want %v: %q", got, tc.counts, out.String())
}
if !tc.counts && out.Len() > 0 {
t.Errorf("a refused run printed %q", out.String())
}
if tc.err != nil && !strings.Contains(errOut.String(), "felis offsite sync: "+tc.err.Error()) {
t.Errorf("stderr %q lacks the error", errOut.String())
}
})
}
}
func TestOffsiteTakeOver(t *testing.T) {
newKey := func() []byte {
raw, _ := offsite.NewKey()
k, _ := offsite.ParseKey(raw)
return k
}
key, other := newKey(), newKey()
const mine, theirs = "aaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbb"
t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
sealed := func(k []byte, writer string) mapBucket {
b := mapBucket{}
putBundle(t, b, k, 0, nil)
b["felis-key-id"] = []byte(offsite.KeyID(k) + "\n")
if writer != "" {
raw, _ := json.Marshal(offsite.Writer{HostID: writer, Host: "prod-1", At: t0.Add(-20 * time.Minute)})
b["felis-writer"] = raw
}
return b
}
lease := func(id string) offsite.Lease {
l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Host: "spare-1"}
if id != "" {
writeTestFile(t, l.IDFile, id+"\n", 0o600)
}
return l
}
run := func(b mapBucket, l offsite.Lease, statusFile string, yes bool) (int, string, string) {
t.Helper()
var out, errb bytes.Buffer
code := takeOver(context.Background(), b, key, l, statusFile, yes, t0, &out, &errb)
return code, out.String(), errb.String()
}
for _, tc := range []struct {
what string
bucket mapBucket
id string
code int
says []string
}{
{"this host writes the bucket", sealed(key, mine), mine, 0, []string{"this host (id " + mine + ") writes the bucket; nothing to take over"}},
{"an empty bucket", mapBucket{}, "", 0, []string{"names no host writing it; this host's next sync records itself"}},
{"a host built from the writer's backup", sealed(key, theirs), "", 4, []string{"host prod-1 (id " + theirs + ") writes the bucket, last at", "(20m ago)", "built from its backup", "sudo felis offsite take-over -yes"}},
{"another host's copies, no writer named", sealed(key, ""), "", 4, []string{"holds copies this host did not write, and names no host writing it", "take-over -yes"}},
{"a host another one took over from", sealed(key, theirs), mine, 5, []string{"took the bucket over from this host"}},
{"a key the bucket refuses", sealed(other, theirs), "", 3, []string{"sealed with another key"}},
} {
t.Run(tc.what, func(t *testing.T) {
before := string(tc.bucket["felis-writer"])
l := lease(tc.id)
code, out, errb := run(tc.bucket, l, filepath.Join(t.TempDir(), "status.json"), false)
if code != tc.code {
t.Fatalf("exit %d, want %d\n%s%s", code, tc.code, out, errb)
}
for _, s := range tc.says {
if !strings.Contains(out+errb, s) {
t.Errorf("output lacks %q:\n%s%s", s, out, errb)
}
}
if string(tc.bucket["felis-writer"]) != before {
t.Error("take-over without -yes wrote the bucket's writer")
}
if tc.id == "" {
if _, err := os.Stat(l.IDFile); !errors.Is(err, os.ErrNotExist) {
t.Errorf("take-over without -yes made this host an id: %v", err)
}
}
})
}
// -yes on a standby host: the bucket names it, and the refusal the last
// sync recorded is cleared, so the watchdog mails again at once.
b := sealed(key, theirs)
l := lease("")
statusFile := filepath.Join(t.TempDir(), "status.json")
lastSuccess := t0.Add(-48 * time.Hour)
if err := offsite.WriteStatus(statusFile, offsite.Status{
LastAttempt: t0.Add(-time.Hour), LastSuccess: lastSuccess, LastError: "offsite: another host writes this bucket", Format: offsite.StatusFormat,
Standby: true, Writer: &offsite.Writer{HostID: theirs, Host: "prod-1", At: t0}, Inherited: true,
}); err != nil {
t.Fatal(err)
}
code, out, errb := run(b, l, statusFile, true)
id, _ := l.ID()
if code != 0 || id == "" || !strings.Contains(out, "this host (id "+id+") writes the bucket now; host prod-1 (id "+theirs+") stops at its next copy") || !strings.Contains(out, "systemctl start felis-offsite.service") {
t.Fatalf("take-over -yes: exit %d, id %q\n%s%s", code, id, out, errb)
}
if w, err := offsite.BucketWriter(context.Background(), b); err != nil || w.HostID != id || w.Host != "spare-1" || !w.At.Equal(t0) {
t.Errorf("writer after take-over -yes = %+v, %v", w, err)
}
st, err := offsite.ReadStatus(statusFile)
if err != nil || st.Standby || st.Writer != nil || st.LastError != "" || st.Inherited || !st.LastSuccess.Equal(lastSuccess) {
t.Errorf("status after take-over -yes = %+v, %v; want the refusal cleared and the last success kept", st, err)
}
// -yes with a key the bucket refuses writes nothing.
b = sealed(other, theirs)
before := string(b["felis-writer"])
if code, _, _ := run(b, lease(""), filepath.Join(t.TempDir(), "status.json"), true); code != 3 || string(b["felis-writer"]) != before {
t.Errorf("take-over -yes under another key: exit %d, writer %s", code, b["felis-writer"])
}
}
func TestOffsiteStatusSaysWhoWritesTheBucket(t *testing.T) {
dir := t.TempDir()
cfg := filepath.Join(dir, "felis.toml")
writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
statusFile := filepath.Join(dir, "status.json")
now := time.Now()
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-30 * time.Minute)}
stale := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-offsite.WriterLive - time.Hour)}
for _, tc := range []struct {
what string
st offsite.Status
says []string
not string
}{
{"standing by for a live writer", offsite.Status{Standby: true, Writer: w}, []string{"This host stands by: host prod-1 (id bbbbbbbbbbbbbbbb) writes the bucket", "mails no watchdog alert", "take-over -yes"}, "wrote it, last at"},
{"standing by for a writer gone quiet", offsite.Status{Standby: true, Writer: stale}, []string{"copies nothing into the bucket: host prod-1 (id bbbbbbbbbbbbbbbb) wrote it, last at", "take-over -yes"}, "mails no watchdog alert"},
{"standing by, no writer named", offsite.Status{Standby: true}, []string{"names no host writing it", "take-over -yes"}, "stands by:"},
{"displaced", offsite.Status{Displaced: true, Writer: w}, []string{"host prod-1 (id bbbbbbbbbbbbbbbb) took the bucket over", "rehearsal machine", "take-over -yes"}, "stands by"},
} {
t.Run(tc.what, func(t *testing.T) {
tc.st.LastAttempt, tc.st.LastSuccess, tc.st.LastError = now.Add(-time.Minute), now.Add(-time.Hour), "offsite: another host writes this bucket"
if err := offsite.WriteStatus(statusFile, tc.st); err != nil {
t.Fatal(err)
}
var out, errb bytes.Buffer
code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
if code != 1 || strings.Contains(out.String(), "bucket holds:") || strings.Contains(out.String(), tc.not) {
t.Errorf("exit %d\n%s%s", code, out.String(), errb.String())
}
for _, s := range tc.says {
if !strings.Contains(out.String(), s) {
t.Errorf("output lacks %q:\n%s", s, out.String())
}
}
})
}
}
func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) {
dir := t.TempDir()
cfg := filepath.Join(dir, "felis.toml")
writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
statusFile := filepath.Join(dir, "status.json")
st := offsite.Status{LastAttempt: time.Now().Add(-time.Minute), LastSuccess: time.Now().Add(-time.Hour), KeyID: "0123456789abcdef"}
status := func() (int, string) {
t.Helper()
if err := offsite.WriteStatus(statusFile, st); err != nil {
t.Fatal(err)
}
var out, errb bytes.Buffer
code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
return code, out.String() + errb.String()
}
if code, out := status(); code != 0 || strings.Contains(out, "refused") {
t.Fatalf("a recent success: exit %d\n%s", code, out)
}
st.LastError, st.KeyMismatch = "offsite: the bucket's objects are sealed with another key", true
code, out := status()
if code != 1 || !strings.Contains(out, "The last run was refused") || !strings.Contains(out, "key id 0123456789abcdef") || strings.Contains(out, "bucket holds:") {
t.Fatalf("a refused run: exit %d\n%s", code, out)
}
}
func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
rawKey, _ := offsite.NewKey()
key, _ := offsite.ParseKey(rawKey)
otherRaw, _ := offsite.NewKey()
other, _ := offsite.ParseKey(otherRaw)
b := mapBucket{}
gapped := putBundleWith(t, b, key, 4, &dbbackup.Counts{Users: 5, Servers: 3}, "connection refused")
old := putBundle(t, b, key, 3, nil)
full := putBundle(t, b, key, 2, &dbbackup.Counts{Users: 5, Servers: 3})
sealedElsewhere := putBundle(t, b, other, 1, &dbbackup.Counts{Users: 5, Servers: 3})
empty := putBundle(t, b, key, 0, &dbbackup.Counts{})
bundles, err := offsite.ListDB(context.Background(), b)
if err != nil {
t.Fatal(err)
}
var out bytes.Buffer
printDBBundles(context.Background(), b, key, bundles, &out)
lines := strings.Split(strings.TrimSpace(out.String()), "\n")
want := []struct{ name, holds string }{
{empty, "0 accounts, 0 servers"},
{sealedElsewhere, "unreadable: offsite: object does not decrypt with this key"},
{full, "5 accounts, 3 servers"},
{old, "not recorded"},
{gapped, "5 accounts, 3 servers, no MinecraftServer objects"},
}
if len(lines) != len(want)+1 || !strings.HasPrefix(lines[0], "database bundles (5, newest first") {
t.Fatalf("output:\n%s", out.String())
}
for i, w := range want {
if l := lines[i+1]; !strings.HasPrefix(l, " "+w.name+" ") || !strings.Contains(l, w.holds) || strings.Contains(l, "no MinecraftServer") != (w.name == gapped) {
t.Errorf("line %d = %q, want %s with %q", i+1, l, w.name, w.holds)
}
}
}
// TestRestoredHostKeepsStandingBy walks the status file across runs: a host
// restored from the writer's backup stands by on its first run and on every
// run after it, a host an older release left copying claims the bucket once,
// and a failed first run after the upgrade keeps that claim.
func TestRestoredHostKeepsStandingBy(t *testing.T) {
rawKey, _ := offsite.NewKey()
key, _ := offsite.ParseKey(rawKey)
cfg := config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-backups"}
t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
standby := &offsite.WriterError{Kind: offsite.ErrStandby, Writer: &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}}
pass := func(statusFile string, at time.Time, err error) offsite.Lease {
t.Helper()
st, lease := startRun(cfg, key, statusFile, at)
recordRun(&st, offsite.Result{}, err, lease)
if werr := offsite.WriteStatus(statusFile, st); werr != nil {
t.Fatal(werr)
}
return lease
}
restored := filepath.Join(t.TempDir(), "status.json")
for i := range 3 {
if l := pass(restored, t0.Add(time.Duration(i)*time.Hour), standby); l.Inherited {
t.Fatalf("run %d of a restored host claims the bucket", i+1)
}
}
if st, _ := offsite.ReadStatus(restored); !st.Standby || st.Format != offsite.StatusFormat || st.KeyID != offsite.KeyID(key) || st.Bucket != "felis-backups" {
t.Errorf("restored host's status = %+v", st)
}
upgraded := filepath.Join(t.TempDir(), "status.json")
if err := offsite.WriteStatus(upgraded, offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-time.Hour)}); err != nil {
t.Fatal(err)
}
if l := pass(upgraded, t0, errors.New("cannot reach bucket")); !l.Inherited {
t.Fatal("the first run after the upgrade does not claim the bucket")
}
l := pass(upgraded, t0.Add(time.Hour), nil)
if !l.Inherited {
t.Fatal("a failed first run after the upgrade lost the claim")
}
if st, _ := offsite.ReadStatus(upgraded); !st.LastSuccess.Equal(t0.Add(time.Hour)) {
t.Errorf("upgraded host's status = %+v", st)
}
}
// TestOffsiteSyncerSnapshotsAndSweeps: the pass `offsite sync` runs takes its
// snapshot the way `felis db backup` does, in the database pod, into the
// bundle directory, keeping the newest one there; and it sweeps unrecorded
// world objects only past the longest retention [archive] gives any backup.
func TestOffsiteSyncerSnapshotsAndSweeps(t *testing.T) {
dir := newPodRig(t)
bundles := filepath.Join(dir, "bundles")
env := &offsiteEnv{cfg: config.OffsiteConfig{DBKeep: 5}}
var log bytes.Buffer
cfg := &config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "120d"}}
s := offsiteSyncer(cfg, env, offsiteSources{dbDir: bundles}, "/archives", "/uploads", nil, &log)
if s.DBDir != bundles || s.DBKeep != 5 || s.ArchiveDir != "/archives" || s.UploadsDir != "/uploads" {
t.Fatalf("syncer = %+v", s)
}
if s.OrphanAfter != 120*24*time.Hour {
t.Errorf("OrphanAfter = %s, want the 120d retention", s.OrphanAfter)
}
if s.Snapshot == nil {
t.Fatal("the pass takes no snapshot after copying archives")
}
for i := 0; i < 2; i++ {
if err := s.Snapshot(context.Background()); err != nil {
t.Fatalf("snapshot %d: %v", i, err)
}
}
got, err := dbbackup.List(bundles)
if err != nil || len(got) != 1 || got[0].Label != dbbackup.LabelOffsite {
t.Fatalf("bundle directory = %+v, %v; want the newest offsite bundle alone", got, err)
}
if _, err := dbbackupVerify(got[0].Path); err != nil {
t.Fatalf("the snapshot does not verify: %v", err)
}
// The MinecraftServer objects are exported alongside, as in the daily bundle.
argv, _ := os.ReadFile(filepath.Join(dir, "k3s.args"))
if ran := string(argv); !strings.Contains(ran, podExecPrefix+"pg_dump --format=custom") {
t.Errorf("k3s ran %q, want pg_dump in the pod", ran)
}
if !strings.Contains(string(bundleServers(t, got[0].Path)), `"name": "lobby"`) {
t.Errorf("the snapshot holds no MinecraftServer objects")
}
if !strings.Contains(log.String(), "took database bundle "+got[0].Name) {
t.Errorf("the snapshot is not logged:\n%s", log.String())
}
// It becomes the newest bundle in the bucket, so with the cluster away it
// fails, leaves no bundle, and the next pass tries again.
noServerExportWait(t)
writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
if err := s.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "export the MinecraftServer objects") {
t.Errorf("snapshot with the cluster away: %v, want a failure", err)
}
if again, _ := dbbackup.List(bundles); len(again) != 1 || again[0].Name != got[0].Name {
t.Errorf("bundle directory after the failed snapshot = %+v, want %s alone", again, got[0].Name)
}
for _, c := range []struct {
archive config.ArchiveConfig
want time.Duration
}{
{config.ArchiveConfig{}, 90 * 24 * time.Hour},
{config.ArchiveConfig{ScheduledRetention: "200d"}, 200 * 24 * time.Hour},
{config.ArchiveConfig{ManualRetention: "150d", Retention: "30d", ScheduledRetention: "60d"}, 150 * 24 * time.Hour},
} {
s := offsiteSyncer(&config.Config{Database: podDB, Archive: c.archive}, env, offsiteSources{dbDir: bundles}, "", "", nil, io.Discard)
if s.OrphanAfter != c.want {
t.Errorf("%+v: OrphanAfter = %s, want %s", c.archive, s.OrphanAfter, c.want)
}
}
log.Reset()
s = offsiteSyncer(&config.Config{Database: podDB, Archive: config.ArchiveConfig{Retention: "soon"}}, env, offsiteSources{}, "", "", nil, &log)
if s.OrphanAfter != 0 || !strings.Contains(log.String(), "world objects no backup records are kept") {
t.Errorf("a retention that does not parse: OrphanAfter %s, log %q; want no sweep, said", s.OrphanAfter, log.String())
}
if s.Snapshot != nil {
t.Error("a pass that copies no bundles takes a snapshot")
}
}
cmd/felis/offsite_uploads.go
-59
View File
@@ -1,59 +0,0 @@
package main
import (
"context"
"flag"
"fmt"
"io"
"time"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/platform"
)
func offsiteFetchUploads(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
uploadsDir := fs.String("uploads-dir", "", "host directory of the submission uploads volume (default: resolved from the uploads PVC, binding it if needed)")
uploadsPVC := fs.String("uploads-pvc", platform.UploadsPVCName, "the submission uploads PVC, in the control-plane namespace")
at := fs.String("at", "", "uploads index version to restore (default: the newest; `felis offsite list` shows them)")
if err := fs.Parse(args); err != nil {
return 2
}
cfg, env, err := loadOffsite(*cfgPath, *envFile)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
defer cancel()
stamp, idx, err := offsite.ChooseUploadIndex(ctx, env.bucket, env.key, *at)
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
return 1
}
dir := *uploadsDir
if dir == "" {
if !isLocalUploadsPath(cfg.Registry.UserUploadsContext) {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: [registry] user_uploads_context %q is not the uploads volume; pass -uploads-dir to restore into a directory anyway\n", cfg.Registry.UserUploadsContext)
return 2
}
if dir, err = resolveVolumeDir(ctx, platform.DefaultControlNamespace, *uploadsPVC, uploadsVolume, true, stderr); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v\n", err)
return 1
}
}
fmt.Fprintf(stdout, "felis offsite fetch-uploads: restoring uploads index %s (%d submission contexts, %s) into %s\n",
stamp, len(idx.Contexts), offsite.HumanBytes(idx.Bytes()), dir)
res, err := offsite.FetchUploads(ctx, env.bucket, env.key, idx, dir, platform.ControlPlaneUID, platform.ControlPlaneUID, stderr)
fmt.Fprintf(stdout, "felis offsite fetch-uploads: %d written (%s), %d already in place, %d failed\n",
res.Written, offsite.HumanBytes(res.Bytes), res.Present, len(res.Failures))
for _, f := range res.Failures {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %s\n", f)
}
if err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-uploads: %v (a second run writes only what is still missing)\n", err)
return 1
}
return 0
}
cmd/felis/operator.go
+2 -9
View File
@@ -117,15 +117,8 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
FelisImage: os.Getenv("FELIS_IMAGE"),
// Uncached: the maintenance-lock check lists Jobs only when a server is
// about to start, which does not justify a namespace-wide Job informer.
Jobs: mgr.GetAPIReader(),
// Uncached too: RCON Secrets are read by name, so the Role grants
// secrets:get without the list/watch an informer would need.
Secrets: mgr.GetAPIReader(),
// And pods: pod-0 is read by name, so the Role grants pods:get and no
// namespace-wide pod informer runs.
Pods: mgr.GetAPIReader(),
Recorder: mgr.GetEventRecorderFor("felis-operator"),
Watch: watch,
Jobs: mgr.GetAPIReader(),
Watch: watch,
}
if err := r.SetupWithManager(mgr); err != nil {
fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
cmd/felis/pinimages.go
+2 -91
View File
@@ -11,9 +11,7 @@ import (
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/imagepin"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -37,17 +35,12 @@ func cmdPinImages(args []string, stdout, stderr io.Writer) int {
namespace := fs.String("namespace", platform.DefaultMinecraftNamespace, "namespace the MinecraftServers live in")
registry := fs.String("registry", defaultRegistryURL, "registry host[:port] the image refs spell")
endpoint := fs.String("endpoint", "", "host[:port] to reach the registry at (default: 127.0.0.1 on the registry's port, its hostPort on this node)")
system := fs.String("system", "", "pin this system server ("+naming.SystemLoginServer+" or "+naming.SystemLobbyServer+") to the build its tag names now, instead of the user servers")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if *system != "" && *system != naming.SystemLoginServer && *system != naming.SystemLobbyServer {
fmt.Fprintf(stderr, "felis pin-images: --system takes %s or %s, not %q\n", naming.SystemLoginServer, naming.SystemLobbyServer, *system)
return 2
}
if *endpoint == "" {
*endpoint = loopbackEndpoint(*registry)
}
@@ -58,9 +51,6 @@ func cmdPinImages(args []string, stdout, stderr io.Writer) int {
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
if *system != "" {
return reportSystemPin(ctx, cl, *namespace, *system, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint}, stdout, stderr)
}
outcomes, err := pinUserServerImages(ctx, cl, *namespace, imagepin.Resolver{Registry: *registry, Endpoint: *endpoint})
if meta.IsNoMatchError(err) {
fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
@@ -87,84 +77,6 @@ func cmdPinImages(args []string, stdout, stderr io.Writer) int {
return exit
}
// reportSystemPin runs pinSystemServerImage for `felis pin-images --system` and
// prints what it did. Only a failed pin exits non-zero: the installer falls back
// to restarting the pod on its tag then.
func reportSystemPin(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver, stdout, stderr io.Writer) int {
o, err := pinSystemServerImage(ctx, cl, namespace, name, r)
switch {
case meta.IsNoMatchError(err):
fmt.Fprintln(stdout, "felis pin-images: no MinecraftServer CRD yet, so no server to pin")
return 0
case err == nil && o.err != nil:
err = o.err
}
if err != nil {
fmt.Fprintf(stderr, "felis pin-images: %s: %v\n", name, err)
return 1
}
switch {
case o.updated:
fmt.Fprintf(stdout, "felis pin-images: %s: %s; the operator rolls it onto that build\n", name, strings.Join(o.changes, ", "))
default:
fmt.Fprintf(stdout, "felis pin-images: %s: %s\n", name, o.skipped)
}
return 0
}
// pinSystemServerImage fixes a system server to the build its image tag names now,
// replacing the digest of an earlier build. The installer runs it after pushing a
// rebuilt login or lobby image, and the operator rolls the StatefulSet onto the new
// ref, so the build a system server runs is written in its spec and moves only when
// a build did. An image outside the platform registry, or one naming no tag to
// follow, is the admin's choice and is left alone; so is a server whose image an
// admin retargets while this runs.
func pinSystemServerImage(ctx context.Context, cl client.Client, namespace, name string, r imagepin.Resolver) (systemServerOutcome, error) {
var ms v1alpha1.MinecraftServer
if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &ms); err != nil {
if apierrors.IsNotFound(err) {
return systemServerOutcome{name: name, skipped: "not present yet; sudo felis setup creates it"}, nil
}
return systemServerOutcome{}, err
}
if ms.Labels[v1alpha1.LabelSystemRole] != name {
return systemServerOutcome{name: name, err: fmt.Errorf(
"MinecraftServer %s/%s is not marked as the Felis %q system server; left alone", namespace, name, name)}, nil
}
tagged := withoutDigest(ms.Spec.Image)
if !r.Covers(tagged) || !strings.Contains(tagged[strings.LastIndex(tagged, "/")+1:], ":") {
return systemServerOutcome{name: name, available: true, skipped: "runs " + ms.Spec.Image +
", which names no platform registry tag to follow; left alone"}, nil
}
pinned, err := r.Pin(ctx, tagged)
if err != nil {
return systemServerOutcome{name: name, err: fmt.Errorf("resolve %s: %w", tagged, err)}, nil
}
changed, err := patchOnConflictRetry(ctx, cl, &ms, func() bool {
if withoutDigest(ms.Spec.Image) != tagged || ms.Spec.Image == pinned {
return false
}
ms.Spec.Image = pinned
return true
})
if err != nil {
return systemServerOutcome{name: name, err: fmt.Errorf("patch %s: %w", name, err)}, nil
}
if !changed {
return systemServerOutcome{name: name, available: true, skipped: "already runs " + ms.Spec.Image}, nil
}
return systemServerOutcome{name: name, available: true, updated: true,
changes: []string{"spec.image pinned to " + pinned}}, nil
}
// withoutDigest drops the @sha256:… of a pinned ref, leaving the tag it came from.
func withoutDigest(ref string) string {
if i := strings.Index(ref, "@"); i >= 0 {
return ref[:i]
}
return ref
}
// loopbackEndpoint is the registry's port on 127.0.0.1: the registry Deployment
// binds it as a hostPort, and containerd's mirror and the installer's pushes use
// the same address.
@@ -176,9 +88,8 @@ func loopbackEndpoint(registry string) string {
}
// pinUserServerImages patches spec.image of every user server whose image the
// resolver covers and is not yet pinned. System servers are the installer's to move:
// it re-pins them with --system when it rolls them onto a new build
// (restart_existing_system_servers).
// resolver covers and is not yet pinned. System servers are left on their tags:
// the installer rebuilds and restarts them on purpose (restart_existing_system_servers).
// A server that is already pinned, or runs an image from elsewhere, produces no
// outcome, so a pinned fleet reports nothing. A running server restarts once as
// the operator rolls its StatefulSet onto the pinned ref, which is the build it
cmd/felis/reaper.go
+51 -75
View File
@@ -16,8 +16,10 @@ import (
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/backup"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/reaper"
"felis.lolicon.best/internal/store"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -31,17 +33,11 @@ import (
// cadence, and RunOnce is idempotent and restart-safe, so a missed or retried
// run simply converges. Only the tarLocal archive backend is wired in this
// build; the snapshot backends (§19) are a later integration.
//
// --retention-only runs the archive-store half alone (reaper.RunRetention):
// the CronJob an install without a worlds root gets, so backups past their
// expiry still leave the store there. It builds no Kubernetes client and reads
// no world.
func cmdReaper(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("reaper", flag.ContinueOnError)
fs.SetOutput(stderr)
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>, else the stock local-path <root>/<pv-name>_<ns>_<pvc-name>)")
retentionOnly := fs.Bool("retention-only", false, "only expire, read back and sweep the archive store: no server is evaluated and no world is read or deleted, so neither the worlds root nor the Kubernetes API is needed")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -60,16 +56,13 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
ctx := ctrl.SetupSignalHandler()
var cl client.Client
if !*retentionOnly {
scheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(v1alpha1.AddToScheme(scheme))
cl, err = client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
if err != nil {
fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
return 1
}
scheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(v1alpha1.AddToScheme(scheme))
cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
if err != nil {
fmt.Fprintf(stderr, "felis reaper: build k8s client: %v\n", err)
return 1
}
archiver, err := buildArchiver(ctx, cfg, *worldsRoot, cl)
@@ -78,7 +71,7 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
return 1
}
drv, err := openPodStore(ctx, cfg.Database.URL, "reaper", stderr)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
return 1
@@ -88,13 +81,9 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
r := &reaper.Reaper{
Cfg: rcfg,
Store: reaper.NewPGStore(drv.DB()),
Cluster: reaper.NewK8sCluster(cl, cfg.K8s.Namespace),
Archiver: archiver,
}
if *retentionOnly {
fmt.Fprintln(stderr, "felis reaper: retention only — no worlds root is configured, so idle worlds are neither archived nor released")
return reportReaperRun(r.RunRetention(ctx), stdout, stderr)
}
r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace)
// Pre-reap warnings go out by email when [smtp] is configured (the same
// relay and password_ref convention felis-api uses); without it the channel
@@ -125,7 +114,13 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
}
return email, nil
},
notifier: smtpRelay(cfg.SMTP, password),
notifier: &mail.SMTP{
Host: cfg.SMTP.Host,
Port: cfg.SMTP.Port,
From: cfg.SMTP.From,
Username: cfg.SMTP.Username,
Password: password,
},
}
} else {
fmt.Fprintln(stderr, "felis reaper: [smtp] not configured — pre-reap warnings are logged and NOT marked sent")
@@ -144,24 +139,14 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
// is kept, and without this nobody would learn that it is never reaped.
func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d released=%d deleted=%d awaiting_offsite=%d awaiting_stop=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
sum.Evaluated, sum.WorldsReaped, sum.Released, sum.ServersDeleted, sum.AwaitingOffsite, sum.AwaitingStop, sum.Warned, sum.Skipped, sum.StoreFull,
sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d\n",
sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.Warned, sum.Skipped, sum.StoreFull,
sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed)
if !sum.Failed() {
return 0
}
if sum.Skipped > 0 || sum.ExpireFailed > 0 {
fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
sum.Skipped, sum.StoreFull, sum.ExpireFailed)
}
if sum.Corrupt > 0 {
fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)
}
if sum.VerifyFailed > 0 || sum.SweepFailed {
fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",
sum.VerifyFailed, !sum.SweepFailed)
}
fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
sum.Skipped, sum.StoreFull, sum.ExpireFailed)
return 1
}
@@ -199,9 +184,8 @@ func (w *mailWarner) Warn(ctx context.Context, ownerID, server, remaining string
// reaperConfig derives the reaper's retention windows from felis.toml. The 15d
// idle deadline is fixed by §18; only the warning offsets, retention, the
// store soft-cap, the on-demand backup bounds and the scheduled restore points
// are configurable (§24). The backup Job and felis-api read the manual_* and
// scheduled_* keys through it too.
// store soft-cap and the on-demand backup bounds are configurable (§24). The
// backup Job and felis-api read the manual_* bounds through it too.
func reaperConfig(cfg *config.Config) (reaper.Config, error) {
rc := reaper.DefaultConfig()
if v := cfg.Archive.Retention; v != "" {
@@ -249,46 +233,20 @@ func reaperConfig(cfg *config.Config) (reaper.Config, error) {
}
rc.ManualCooldown = d
}
if v := cfg.Archive.ScheduledEvery; v != "" {
d, err := parseSpanDuration(v)
if err != nil || d < 0 {
return rc, fmt.Errorf("[archive] scheduled_every %q: want a span such as 1d (0s for none)", v)
}
rc.ScheduledEvery = d
}
switch n := cfg.Archive.ScheduledKeep; {
case n < 0:
return rc, fmt.Errorf("[archive] scheduled_keep %d: want 1 or more", n)
case n > 0:
rc.ScheduledKeep = n
}
if v := cfg.Archive.ScheduledRetention; v != "" {
d, err := parseSpanDuration(v)
if err != nil || d <= 0 {
return rc, fmt.Errorf("[archive] scheduled_retention %q: want a positive span such as 90d", v)
}
rc.ScheduledRetention = d
}
rc.RequireOffsite = cfg.Offsite.Enabled()
return rc, nil
}
// buildArchiver constructs the WorldArchiver. Only tarLocal is implemented in
// this build; the resolver maps each world PVC to its directory under worldsRoot
// (resolveWorldDir). A nil cl is the retention-only run, which never archives a
// world, so its archiver resolves none.
// (resolveWorldDir).
func buildArchiver(ctx context.Context, cfg *config.Config, worldsRoot string, cl client.Client) (backup.WorldArchiver, error) {
switch cfg.Archive.Store {
case "tarLocal":
t := &backup.TarLocal{BackupRoot: cfg.Archive.LocalPath}
if cl != nil {
t.Resolve = resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot)
} else {
t.Resolve = func(pvc string) (string, error) {
return "", fmt.Errorf("resolve world PVC %s: this run has no worlds root (retention only)", pvc)
}
}
return t, nil
return &backup.TarLocal{
BackupRoot: cfg.Archive.LocalPath,
Resolve: resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot),
}, nil
default:
return nil, fmt.Errorf("[archive] store %q is not implemented in this build (only tarLocal)", cfg.Archive.Store)
}
@@ -331,9 +289,27 @@ func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoo
}
}
// parseSpanDuration parses the human spans used in felis.toml's [archive] table
// (config.ParseSpan).
func parseSpanDuration(s string) (time.Duration, error) { return config.ParseSpan(s) }
// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
func parseSpanDuration(s string) (time.Duration, error) {
s = strings.TrimSpace(s)
switch {
case strings.HasSuffix(s, "mo"):
n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
if err != nil {
return 0, err
}
return time.Duration(n) * 30 * 24 * time.Hour, nil
case strings.HasSuffix(s, "d"):
n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
if err != nil {
return 0, err
}
return time.Duration(n) * 24 * time.Hour, nil
default:
return time.ParseDuration(s)
}
}
// parseByteSize parses a Kubernetes-style quantity ("200Gi", "10G") into bytes.
// An empty string means unlimited (0).
cmd/felis/reaper_test.go
+1 -45
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -28,22 +27,15 @@ func TestReportReaperRunFailsTheJob(t *testing.T) {
want int
}{
{"clean", reaper.Summary{Evaluated: 3, WorldsReaped: 1, AwaitingOffsite: 1}, 0},
{"retirements", reaper.Summary{Evaluated: 5, WorldsReaped: 3, Released: 2, ServersDeleted: 1}, 0},
{"waiting for a stop", reaper.Summary{Evaluated: 3, AwaitingStop: 1}, 0},
{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},
{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},
{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},
{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},
{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},
} {
var out, errb bytes.Buffer
if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {
t.Errorf("%s: exit %d, want %d", tc.name, got, tc.want)
}
if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") ||
!strings.Contains(out.String(), fmt.Sprintf(" released=%d deleted=%d ", tc.sum.Released, tc.sum.ServersDeleted)) {
if !strings.Contains(out.String(), "skipped=") || !strings.Contains(out.String(), "expire_failed=") {
t.Errorf("%s: summary line = %q", tc.name, out.String())
}
if (tc.want == 1) != (errb.Len() > 0) {
@@ -88,42 +80,6 @@ func TestReaperConfigManualKeys(t *testing.T) {
}
}
// TestReaperConfigScheduledKeys: the scheduled restore points default to one a
// day, seven per server and the reaper's 90 days, accept overrides ("0s" turns
// them off), and refuse values that would keep nothing or run backwards.
func TestReaperConfigScheduledKeys(t *testing.T) {
rc, err := reaperConfig(&config.Config{})
if err != nil {
t.Fatal(err)
}
if rc.ScheduledEvery != reaper.Day || rc.ScheduledKeep != 7 || rc.ScheduledRetention != 90*reaper.Day {
t.Fatalf("defaults = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
}
rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{
ScheduledEvery: "12h", ScheduledKeep: 3, ScheduledRetention: "14d"}})
if err != nil {
t.Fatal(err)
}
if rc.ScheduledEvery != 12*time.Hour || rc.ScheduledKeep != 3 || rc.ScheduledRetention != 14*reaper.Day {
t.Fatalf("overrides = %v / %d / %v", rc.ScheduledEvery, rc.ScheduledKeep, rc.ScheduledRetention)
}
rc, err = reaperConfig(&config.Config{Archive: config.ArchiveConfig{ScheduledEvery: "0s"}})
if err != nil || rc.ScheduledEvery != 0 {
t.Fatalf("scheduled_every 0s = %v, %v; want off", rc.ScheduledEvery, err)
}
for _, bad := range []config.ArchiveConfig{
{ScheduledEvery: "-1h"},
{ScheduledEvery: "daily"},
{ScheduledKeep: -1},
{ScheduledRetention: "0d"},
{ScheduledRetention: "forever"},
} {
if _, err := reaperConfig(&config.Config{Archive: bad}); err == nil {
t.Errorf("%+v was accepted", bad)
}
}
}
func TestResolveWorldDir(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
cmd/felis/registrygate.go
+2 -9
View File
@@ -131,8 +131,7 @@ func loopbackAddr(addr string) bool {
func cmdPushImage(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("push-image", flag.ContinueOnError)
fs.SetOutput(stderr)
tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path, or with --image an OCI layout tar")
image := fs.String("image", "", "push the image this name (io.containerd.image.name) marks in the OCI layout tar --tar, e.g. a release's image bundle")
tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path")
ref := fs.String("ref", "", "host/repository:tag to publish it as")
scheme := fs.String("scheme", "http", "registry scheme: http for the in-cluster registry, https otherwise")
if err := fs.Parse(args); err != nil {
@@ -155,13 +154,7 @@ func cmdPushImage(args []string, stdout, stderr io.Writer) int {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
p := &imagepush.Pusher{Scheme: *scheme, Username: user, Password: pass, Log: stderr}
var digest string
var err error
if *image != "" {
digest, err = p.PushLayout(ctx, *tarPath, *image, *ref)
} else {
digest, err = p.Push(ctx, *tarPath, *ref)
}
digest, err := p.Push(ctx, *tarPath, *ref)
if err != nil {
fmt.Fprintf(stderr, "felis push-image: %v\n", err)
return 1
cmd/felis/restore_test.go
+2 -2
View File
@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup
BackupRoot: backupRoot,
Resolve: func(string) (string, error) { return srcDir, nil },
}
got, err := ar.Archive(context.Background(), "survival", "world-survival-0")
got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0")
if err != nil {
t.Fatalf("Archive: %v", err)
}
return string(got.Ref), backupRoot
return string(got), backupRoot
}
// The restore subcommand must extract the archived world into the target world
cmd/felis/rotatetoken.go
-854
View File
@@ -1,854 +0,0 @@
package main
import (
"context"
"crypto/hmac"
"crypto/pbkdf2"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"flag"
"fmt"
"io"
"io/fs"
neturl "net/url"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/maintenance"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/operator"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/registrygate"
"github.com/BurntSushi/toml"
"github.com/jackc/pgx/v5"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// rotate-token replaces one credential the installer generated: an internal
// caller's token (naming.CallerTokens), the registry's write tokens, the
// Velocity forwarding secret or the database password. The new value goes into
// the installer's record first, so that whatever fails later a re-run of the
// installer puts it everywhere; then into the Secrets and files that carry it;
// then whatever read the old value at start restarts. Without -yes it prints
// what it would change and what that interrupts, and changes nothing.
const (
defaultSecretsEnvPath = "/etc/felis/secrets.env"
defaultLinkPropsPath = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
defaultForwardingPath = "/opt/felis/velocity/forwarding.secret"
velocityUnit = "felis-velocity"
apiDeployment = "felis-api"
registryDeployment = "registry"
kindRegistry = "registry"
kindForwarding = "forwarding"
kindDB = "db"
// proxyReloadWait is how long the host proxy gets, once felis-api has rolled,
// to show it re-read its token: it reads the file again on its next call to
// felis-api, and it calls every 15 seconds.
proxyReloadWait = 60 * time.Second
// apiRestartNote is in every plan: felis-api runs as one replica replaced in
// place, so its restart is a short outage of everything that talks to it.
apiRestartNote = "felis-api restarts (a single replica): the panel, sign-in and the proxy's calls are unavailable for the few seconds that takes"
)
// installerTokenKeys names each caller's token in the installer's secrets.env
// (deploy/bootstrap.sh load_or_make_secrets). A re-run of the installer applies
// these values to the Secrets, so a rotation that skipped the file would be
// undone by the next upgrade.
var installerTokenKeys = map[string]string{
"velocity": "SERVICE_TOKEN",
"limbo": "LIMBO_TOKEN",
"build": "BUILD_TOKEN",
"ops": "OPS_TOKEN",
}
// installerRegistryKeys names the registry principals' tokens in secrets.env,
// in registrygate.Principals order.
var installerRegistryKeys = map[string]string{
registrygate.PrincipalPlatform: "REGISTRY_PLATFORM_TOKEN",
registrygate.PrincipalBuild: "REGISTRY_BUILD_TOKEN",
registrygate.PrincipalPrune: "REGISTRY_PRUNE_TOKEN",
}
// installerForwardingKey and installerDBKey name the forwarding secret and the
// database password in secrets.env.
const (
installerForwardingKey = "FORWARDING_SECRET"
installerDBKey = "DB_PASSWORD"
)
type tokenRotator struct {
cl client.Client
controlNS string
minecraftNS string
buildNS string
// secretsEnv, linkProps and forwardingFile are the installer's record and the
// host proxy's felis-link.properties and forwarding.secret; a missing file is
// reported and skipped.
secretsEnv string
linkProps string
forwardingFile string
// hostTOML, podTOML and defaultTOML are the config copies that carry the
// database URL (defaultTOML only when it is a file of its own).
hostTOML string
podTOML string
defaultTOML string
newToken func() (string, error)
// rollout restarts a control-namespace Deployment and waits for it.
rollout func(ctx context.Context, deployment string) error
// restartUnit restarts a systemd unit on this host.
restartUnit func(ctx context.Context, unit string) error
// proxyLog is what the host proxy has logged since a moment, as far as it
// can be read.
proxyLog func(ctx context.Context, since time.Time) string
// alterRole stores a password verifier for a role of the database that runs
// as deployment ("namespace/name"); verifyDB connects with a URL.
alterRole func(ctx context.Context, deployment, role, verifier string) error
verifyDB func(ctx context.Context, url string) error
now func() time.Time
// reloadWait bounds the wait for the proxy to re-read its token, polled
// every pollEvery.
reloadWait time.Duration
pollEvery time.Duration
out io.Writer
}
func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("rotate-token", flag.ContinueOnError)
fs.SetOutput(stderr)
cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
forwarding := fs.String("forwarding-secret", defaultForwardingPath, "the host proxy's forwarding secret file (forwarding only)")
yes := fs.Bool("yes", false, "rotate; without it the plan is printed and nothing changes")
fs.Usage = func() {
fmt.Fprintf(stderr, "Usage: felis rotate-token [-yes] [flags] <%s>\n\n", strings.Join(rotationKinds(), "|"))
fmt.Fprintln(stderr, "Replaces one generated credential: the installer's record, the Secrets and files that carry it, then what reads it.")
fmt.Fprintln(stderr, "Without -yes it prints what would change and what that interrupts.")
fs.PrintDefaults()
}
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if fs.NArg() != 1 {
fs.Usage()
return 2
}
kind := fs.Arg(0)
if !knownRotation(kind) {
fmt.Fprintf(stderr, "felis rotate-token: unknown credential %q (one of %s)\n", kind, strings.Join(rotationKinds(), ", "))
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+kind+")")
return 1
}
cfg, err := config.Load(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
return 1
}
cl, err := buildSystemServerClient()
if err != nil {
fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
return 1
}
buildNS := cfg.Registry.BuildNamespace
if buildNS == "" {
buildNS = platform.DefaultBuildNamespace
}
controlNS := platform.DefaultControlNamespace
r := tokenRotator{
cl: cl,
controlNS: controlNS,
minecraftNS: cfg.K8s.Namespace,
buildNS: buildNS,
secretsEnv: *secretsEnv,
linkProps: *linkProps,
forwardingFile: *forwarding,
hostTOML: hostSetupConfigPath,
podTOML: podSetupConfigPath,
defaultTOML: defaultSetupConfigPath,
newToken: randomToken,
rollout: func(ctx context.Context, deployment string) error {
if err := kubectl(ctx, "-n", controlNS, "rollout", "restart", "deployment/"+deployment); err != nil {
return err
}
return kubectl(ctx, "-n", controlNS, "rollout", "status", "deployment/"+deployment, "--timeout=180s")
},
restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
proxyLog: journalSince,
alterRole: alterRoleInPod,
verifyDB: func(ctx context.Context, url string) error {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
conn, err := pgx.Connect(ctx, url)
if err != nil {
return err
}
return conn.Close(ctx)
},
now: time.Now,
reloadWait: proxyReloadWait,
pollEvery: 3 * time.Second,
out: stdout,
}
if err := r.rotate(context.Background(), kind, *yes); err != nil {
fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
return 1
}
return 0
}
func callerNames() []string {
names := make([]string, 0, len(naming.CallerTokens))
for _, ct := range naming.CallerTokens {
names = append(names, ct.Caller)
}
return names
}
func callerToken(name string) (naming.CallerToken, bool) {
for _, ct := range naming.CallerTokens {
if ct.Caller == name {
return ct, true
}
}
return naming.CallerToken{}, false
}
// rotationKinds is every credential rotate-token replaces: the callers' tokens
// and the installer's other generated secrets.
func rotationKinds() []string {
return append(callerNames(), kindRegistry, kindForwarding, kindDB)
}
func knownRotation(kind string) bool {
for _, k := range rotationKinds() {
if k == kind {
return true
}
}
return false
}
// randomToken is 32 random bytes in hex, the shape the installer generates.
func randomToken() (string, error) {
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
// tokenFingerprint is how the proxy names the token it reloaded in its log
// (plugins/shared FileToken.fingerprint): the first twelve hex digits of its
// SHA-256, enough to tell tokens apart and useless for finding one.
func tokenFingerprint(token string) string {
sum := sha256.Sum256([]byte(token))
return hex.EncodeToString(sum[:])[:12]
}
func (r tokenRotator) rotate(ctx context.Context, kind string, apply bool) error {
switch kind {
case kindRegistry:
return r.rotateRegistry(ctx, apply)
case kindForwarding:
return r.rotateForwarding(ctx, apply)
case kindDB:
return r.rotateDB(ctx, apply)
}
ct, ok := callerToken(kind)
if !ok {
return fmt.Errorf("unknown credential %q (one of %s)", kind, strings.Join(rotationKinds(), ", "))
}
return r.rotateCaller(ctx, ct, apply)
}
// confirm ends the plan: without apply it says nothing changed and how to go
// ahead, and reports false.
func (r tokenRotator) confirm(kind string, apply bool) bool {
if !apply {
fmt.Fprintf(r.out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes %s\n", kind)
return false
}
fmt.Fprintln(r.out, "\nRotating:")
return true
}
// record writes new values into the installer's secrets.env. It comes first in
// every rotation: from then on, whatever fails, a re-run of the installer puts
// the new values everywhere.
func (r tokenRotator) record(kv ...[2]string) error {
keys := make([]string, len(kv))
for i, p := range kv {
keys[i] = p[0]
}
switch err := setKeyValueLines(r.secretsEnv, "=", kv); {
case errors.Is(err, fs.ErrNotExist):
fmt.Fprintf(r.out, " - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
case err != nil:
return fmt.Errorf("record the new value in %s: %w", r.secretsEnv, err)
default:
fmt.Fprintf(r.out, " - %s: %s updated\n", r.secretsEnv, strings.Join(keys, ", "))
}
return nil
}
func (r tokenRotator) putSecret(ctx context.Context, ns, name string, data map[string][]byte) error {
if _, err := putSecretKeys(ctx, r.cl, ns, name, corev1.SecretTypeOpaque, data); err != nil {
return fmt.Errorf("write Secret %s/%s: %w", ns, name, err)
}
fmt.Fprintf(r.out, " - Secret %s/%s: updated\n", ns, name)
return nil
}
func (r tokenRotator) rollAPI(ctx context.Context) error {
if err := r.rollout(ctx, apiDeployment); err != nil {
return fmt.Errorf("roll felis-api: %w", err)
}
fmt.Fprintln(r.out, " - felis-api: rolled out on the new value")
return nil
}
// withMinecraft is the control namespace plus the minecraft namespace when that
// is a different one: where the Secrets game pods and Jobs mount are mirrored.
func (r tokenRotator) withMinecraft() []string {
if r.minecraftNS == "" || r.minecraftNS == r.controlNS {
return []string{r.controlNS}
}
return []string{r.controlNS, r.minecraftNS}
}
func (r tokenRotator) rotateCaller(ctx context.Context, ct naming.CallerToken, apply bool) error {
namespaces := []string{r.controlNS}
replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
if replica != "" && replica != r.controlNS {
namespaces = append(namespaces, replica)
}
key := installerTokenKeys[ct.Caller]
fmt.Fprintf(r.out, "felis rotate-token %s: a new internal token for the %s caller\n", ct.Caller, ct.Caller)
secrets := make([]string, len(namespaces))
for i, ns := range namespaces {
secrets[i] = "Secret " + ns + "/" + ct.Secret
}
writes := append([]string{r.secretsEnv + " (" + key + ")"}, secrets...)
hostProxy := ct.Caller == "velocity" && fileExists(r.linkProps)
if hostProxy {
writes = append(writes, r.linkProps+" (service-token)")
}
fmt.Fprintf(r.out, " - writes %s\n", strings.Join(writes, ", "))
fmt.Fprintf(r.out, " - %s\n", apiRestartNote)
switch ct.Caller {
case "velocity":
if hostProxy {
fmt.Fprintf(r.out, " - the proxy re-reads its token from the file and keeps its players; if it has not within %s of felis-api's restart, it is restarted, which disconnects everyone online\n", r.reloadWait)
} else {
fmt.Fprintf(r.out, " - no proxy on this host (%s): set service-token in your proxy's felis-link.properties to the value in Secret %s/%s afterwards; it re-reads the file without a restart\n",
r.linkProps, r.controlNS, ct.Secret)
}
case "limbo":
fmt.Fprintln(r.out, " - the login gate's pod restarts: a player signing in at that moment reconnects")
case "build":
fmt.Fprintln(r.out, " - a build fetching its context at that moment fails and can be submitted again")
case "ops":
fmt.Fprintln(r.out, " - felis backup-now presents the new token on its next run")
}
if !r.confirm(ct.Caller, apply) {
return nil
}
tok, err := r.newToken()
if err != nil {
return fmt.Errorf("generate a token: %w", err)
}
if err := r.record([2]string{key, tok}); err != nil {
return err
}
for _, ns := range namespaces {
if err := r.putSecret(ctx, ns, ct.Secret, map[string][]byte{naming.ServiceTokenSecretKey: []byte(tok)}); err != nil {
return err
}
}
// The proxy's file changes before felis-api rolls, so the file never falls
// behind the Secret; the proxy's log is read from just before the write,
// since it may pick the new token up before the rollout ends.
since := r.now()
if hostProxy {
if err := setProxyToken(r.linkProps, tok); err != nil {
return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
}
fmt.Fprintf(r.out, " - %s: service-token updated\n", r.linkProps)
}
if err := r.rollAPI(ctx); err != nil {
return err
}
switch ct.Caller {
case "velocity":
if !hostProxy {
break
}
if r.proxyReloaded(ctx, since, tokenFingerprint(tok)) {
fmt.Fprintf(r.out, " - %s: took the new token from its properties; players stayed connected\n", velocityUnit)
break
}
if err := r.restartUnit(ctx, velocityUnit); err != nil {
return fmt.Errorf("restart %s: %w", velocityUnit, err)
}
fmt.Fprintf(r.out, " - %s: had not taken the new token within %s, restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit, r.reloadWait)
case "limbo":
// Only the operator's server pods carry its managed-by label; a backup or
// restore Job's pod carries the server label too, and app.kubernetes.io ones.
if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), client.MatchingLabels{
v1alpha1.LabelServer: naming.SystemLoginServer,
v1alpha1.LabelManagedBy: operator.ManagedByValue,
}); err != nil {
return fmt.Errorf("restart the login gate: %w", err)
}
fmt.Fprintln(r.out, " - login gate: pod restarted to read the new token")
case "build":
fmt.Fprintln(r.out, " - builds: the next build Job reads the new token")
case "ops":
fmt.Fprintln(r.out, " - felis backup-now reads the new token on its next run")
}
return nil
}
// proxyReloaded waits up to reloadWait for the host proxy to log that it
// reloaded the token with this fingerprint (plugins/shared FileToken).
func (r tokenRotator) proxyReloaded(ctx context.Context, since time.Time, fingerprint string) bool {
want := "(fingerprint " + fingerprint + ")"
deadline := r.now().Add(r.reloadWait)
for {
if strings.Contains(r.proxyLog(ctx, since), want) {
return true
}
if !r.now().Before(deadline) {
return false
}
select {
case <-ctx.Done():
return false
case <-time.After(r.pollEvery):
}
}
}
// journalSince is the proxy unit's journal from the second since falls in. A
// journal that cannot be read reads as one without the line, which ends in the
// restart a rotation made before the proxy could reload.
func journalSince(ctx context.Context, since time.Time) string {
out, _ := exec.CommandContext(ctx, "journalctl", "-u", velocityUnit, "--since", "@"+strconv.FormatInt(since.Unix(), 10),
"-o", "cat", "--no-pager", "-q").Output()
return string(out)
}
// setProxyToken writes the proxy's token into felis-link.properties and puts
// the file's modification time back. The proxy re-reads its token by itself,
// while `felis domain check` and `felis domain set` read a file newer than the
// proxy's start as config it has not loaded (the installer's
// install_if_changed keeps the time for the same reason).
func setProxyToken(path, token string) error {
info, err := os.Stat(path)
if err != nil {
return err
}
if err := setKeyValueLine(path, "service-token", "=", token); err != nil {
return err
}
return os.Chtimes(path, time.Time{}, info.ModTime())
}
func (r tokenRotator) rotateRegistry(ctx context.Context, apply bool) error {
keys := make([]string, len(registrygate.Principals))
for i, p := range registrygate.Principals {
keys[i] = installerRegistryKeys[p]
}
fmt.Fprintf(r.out, "felis rotate-token %s: new write tokens for the image registry's principals (%s)\n", kindRegistry, strings.Join(registrygate.Principals, ", "))
fmt.Fprintf(r.out, " - writes %s (%s), Secret %s/%s, Secret %s/%s\n", r.secretsEnv, strings.Join(keys, ", "),
r.controlNS, naming.RegistryAuthSecretName, r.buildNS, naming.RegistryPushSecretName)
fmt.Fprintln(r.out, " - the registry restarts to load them: a build pushing its image at that moment fails and can be submitted again, and an image pull in that moment retries")
fmt.Fprintf(r.out, " - %s (it presents the prune token)\n", apiRestartNote)
if !r.confirm(kindRegistry, apply) {
return nil
}
tokens := map[string][]byte{}
kv := make([][2]string, 0, len(registrygate.Principals))
for _, p := range registrygate.Principals {
tok, err := r.newToken()
if err != nil {
return fmt.Errorf("generate a token: %w", err)
}
tokens[p] = []byte(tok)
kv = append(kv, [2]string{installerRegistryKeys[p], tok})
}
if err := r.record(kv...); err != nil {
return err
}
if err := r.putSecret(ctx, r.controlNS, naming.RegistryAuthSecretName, tokens); err != nil {
return err
}
if err := r.putSecret(ctx, r.buildNS, naming.RegistryPushSecretName, map[string][]byte{
naming.RegistryPushUsernameKey: []byte(registrygate.PrincipalBuild),
naming.RegistryPushPasswordKey: tokens[registrygate.PrincipalBuild],
}); err != nil {
return err
}
if err := r.rollout(ctx, registryDeployment); err != nil {
return fmt.Errorf("restart the registry: %w", err)
}
fmt.Fprintln(r.out, " - registry: restarted on the new tokens")
return r.rollAPI(ctx)
}
func (r tokenRotator) rotateForwarding(ctx context.Context, apply bool) error {
restart, held, err := r.gamePods(ctx)
if err != nil {
return err
}
hostProxy := fileExists(r.forwardingFile)
fmt.Fprintf(r.out, "felis rotate-token %s: a new Velocity forwarding secret, the key a server checks each player's identity with\n", kindForwarding)
secrets := []string{}
for _, ns := range r.withMinecraft() {
secrets = append(secrets, "Secret "+ns+"/"+naming.ForwardingSecretName)
}
writes := append([]string{r.secretsEnv + " (" + installerForwardingKey + ")"}, secrets...)
if hostProxy {
writes = append(writes, r.forwardingFile)
}
fmt.Fprintf(r.out, " - writes %s\n", strings.Join(writes, ", "))
fmt.Fprintf(r.out, " - every running server restarts to read it (%d now), saving its world on the way down, and the proxy restarts: everyone online is disconnected and can rejoin once their server is back\n", len(restart))
if len(held) > 0 {
fmt.Fprintf(r.out, " - left running, because a backup, restore or file write holds its world: %s. Players cannot join it until it restarts: stop and start it from the panel once that finishes\n", strings.Join(held, ", "))
}
if !hostProxy {
fmt.Fprintf(r.out, " - no proxy on this host (%s): put the value in Secret %s/%s into your proxy's forwarding secret file and restart it\n",
r.forwardingFile, r.controlNS, naming.ForwardingSecretName)
}
if !r.confirm(kindForwarding, apply) {
return nil
}
tok, err := r.newToken()
if err != nil {
return fmt.Errorf("generate a secret: %w", err)
}
if err := r.record([2]string{installerForwardingKey, tok}); err != nil {
return err
}
if hostProxy {
info, err := os.Stat(r.forwardingFile)
if err != nil {
return err
}
if err := replaceFileKeepingMode(r.forwardingFile, info, []byte(tok)); err != nil {
return fmt.Errorf("write %s: %w", r.forwardingFile, err)
}
fmt.Fprintf(r.out, " - %s: updated\n", r.forwardingFile)
}
for _, ns := range r.withMinecraft() {
if err := r.putSecret(ctx, ns, naming.ForwardingSecretName, map[string][]byte{naming.ForwardingSecretKey: []byte(tok)}); err != nil {
return err
}
}
// The servers go first: their pods read the Secret as they are recreated,
// and a player who rejoins through the restarted proxy meets a server on the
// new secret, or one still starting.
for i := range restart {
p := &restart[i]
if err := r.cl.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) {
return fmt.Errorf("restart %s: %w", p.Labels[v1alpha1.LabelServer], err)
}
}
fmt.Fprintf(r.out, " - servers: %d restarting on the new secret\n", len(restart))
if hostProxy {
if err := r.restartUnit(ctx, velocityUnit); err != nil {
return fmt.Errorf("restart %s: %w", velocityUnit, err)
}
fmt.Fprintf(r.out, " - %s: restarted on the new secret\n", velocityUnit)
}
return nil
}
// gamePods lists the running game server pods: those a rotation may restart,
// and the servers left alone because a backup, restore or file write holds
// their world (internal/maintenance), which a restart in the middle of would
// break.
func (r tokenRotator) gamePods(ctx context.Context) ([]corev1.Pod, []string, error) {
var pods corev1.PodList
// The operator's managed-by label is on its server pods alone (see rotateCaller).
if err := r.cl.List(ctx, &pods, client.InNamespace(r.minecraftNS), client.MatchingLabels{v1alpha1.LabelManagedBy: operator.ManagedByValue}); err != nil {
return nil, nil, fmt.Errorf("list the servers' pods: %w", err)
}
var jobs batchv1.JobList
if err := r.cl.List(ctx, &jobs, client.InNamespace(r.minecraftNS)); err != nil {
return nil, nil, fmt.Errorf("list the maintenance Jobs: %w", err)
}
var restart []corev1.Pod
var held []string
for _, p := range pods.Items {
if p.DeletionTimestamp != nil {
continue
}
server := p.Labels[v1alpha1.LabelServer]
var ms v1alpha1.MinecraftServer
if err := r.cl.Get(ctx, client.ObjectKey{Namespace: r.minecraftNS, Name: server}, &ms); client.IgnoreNotFound(err) != nil {
return nil, nil, fmt.Errorf("read server %s: %w", server, err)
}
if kind, ok := maintenance.Holder(server, ms.Annotations, jobs.Items, r.now()); ok {
held = append(held, server+" ("+kind+")")
continue
}
restart = append(restart, p)
}
return restart, held, nil
}
func (r tokenRotator) rotateDB(ctx context.Context, apply bool) error {
cfg, err := config.Load(r.hostTOML)
if err != nil {
return err
}
if cfg.Database.Deployment == "" {
return fmt.Errorf("[database] deployment is unset in %s, so the database is not the installer's felis-postgres: change the role's password where it runs, then in [database] url of each config copy", r.hostTOML)
}
u, err := neturl.Parse(cfg.Database.URL)
if err != nil || u.User == nil || u.User.Username() == "" {
return fmt.Errorf("[database] url in %s names no role", r.hostTOML)
}
role := u.User.Username()
targets, err := tomlTargetsOf(r.hostTOML, r.podTOML, r.defaultTOML)
if err != nil {
return err
}
paths := make([]string, len(targets))
for i, t := range targets {
paths[i] = t.path
}
secrets := []string{}
for _, ns := range r.withMinecraft() {
secrets = append(secrets, ns+"/"+platform.ConfigSecretName)
}
fmt.Fprintf(r.out, "felis rotate-token %s: a new password for the database role %q\n", kindDB, role)
fmt.Fprintf(r.out, " - writes %s (%s), the role in %s, [database] url in %s, Secret %s\n",
r.secretsEnv, installerDBKey, cfg.Database.Deployment, strings.Join(paths, " and "), strings.Join(secrets, " and "))
fmt.Fprintf(r.out, " - %s\n", apiRestartNote)
fmt.Fprintln(r.out, " - a backup, restore or file Job that connects in the seconds between the password change and felis-api's restart fails and can be run again; the host's timers read the new config on their next run")
if !r.confirm(kindDB, apply) {
return nil
}
password, err := r.newToken()
if err != nil {
return fmt.Errorf("generate a password: %w", err)
}
// Every config copy is edited in memory first, so one this cannot edit stops
// the rotation before the role's password changes.
edited := make([][]byte, len(targets))
var hostURL string
var podConfig []byte
for i, t := range targets {
raw, err := os.ReadFile(t.real)
if err != nil {
return err
}
var doc struct {
Database struct {
URL string `toml:"url"`
} `toml:"database"`
}
if _, err := toml.Decode(string(raw), &doc); err != nil {
return fmt.Errorf("%s: %w", t.path, err)
}
next, err := withPassword(doc.Database.URL, password)
if err != nil {
return fmt.Errorf("%s: %w", t.path, err)
}
if edited[i], err = editTOMLStrings(raw, []tomlStringEdit{{"database", "url", next}}); err != nil {
return fmt.Errorf("%s: %w; set the password in its [database] url by hand", t.path, err)
}
switch t.path {
case r.hostTOML:
hostURL = next
case r.podTOML:
podConfig = edited[i]
}
}
if podConfig == nil {
return fmt.Errorf("%s resolves to the same file as %s; the pods reach the database at another address and need a copy of their own", r.podTOML, r.hostTOML)
}
if err := r.record([2]string{installerDBKey, password}); err != nil {
return err
}
salt := make([]byte, 16)
if _, err := rand.Read(salt); err != nil {
return err
}
verifier, err := scramVerifier(password, salt, scramIterations)
if err != nil {
return err
}
if err := r.alterRole(ctx, cfg.Database.Deployment, role, verifier); err != nil {
return fmt.Errorf("set the role's password: %w", err)
}
if err := r.verifyDB(ctx, hostURL); err != nil {
return fmt.Errorf("the database does not accept the new password (%v); the config copies still hold the old one: run the installer again (sudo bash deploy/bootstrap.sh), which sets the password in %s everywhere", err, r.secretsEnv)
}
fmt.Fprintf(r.out, " - role %s: password changed, and the database accepts it\n", role)
for i, t := range targets {
info, err := os.Stat(t.real)
if err != nil {
return err
}
if err := replaceFileKeepingMode(t.real, info, edited[i]); err != nil {
return fmt.Errorf("write %s: %w", t.path, err)
}
fmt.Fprintf(r.out, " - %s: [database] url updated\n", t.path)
}
for _, ns := range r.withMinecraft() {
if err := r.putSecret(ctx, ns, platform.ConfigSecretName, map[string][]byte{platform.ConfigSecretKey: podConfig}); err != nil {
return err
}
}
return r.rollAPI(ctx)
}
// withPassword is a database URL with its password replaced.
func withPassword(raw, password string) (string, error) {
u, err := neturl.Parse(raw)
if err != nil || u.User == nil || u.User.Username() == "" {
return "", errors.New("its [database] url names no role")
}
u.User = neturl.UserPassword(u.User.Username(), password)
return u.String(), nil
}
// scramIterations is PostgreSQL's default scram_iterations.
const scramIterations = 4096
// scramVerifier is the SCRAM-SHA-256 verifier PostgreSQL stores for a password
// (RFC 5802 and RFC 7677, in the form libpq's PQencryptPasswordConn makes).
// ALTER ROLE stores a verifier as it is given, so the password itself never
// reaches the server, where a failing statement is logged with its text.
func scramVerifier(password string, salt []byte, iterations int) (string, error) {
salted, err := pbkdf2.Key(sha256.New, password, salt, iterations, sha256.Size)
if err != nil {
return "", err
}
mac := func(msg string) []byte {
h := hmac.New(sha256.New, salted)
h.Write([]byte(msg))
return h.Sum(nil)
}
stored := sha256.Sum256(mac("Client Key"))
b64 := base64.StdEncoding.EncodeToString
return fmt.Sprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, b64(salt), b64(stored[:]), b64(mac("Server Key"))), nil
}
// alterRoleInPod runs ALTER ROLE as the superuser inside the database's
// container, over its socket, with the statement on stdin.
func alterRoleInPod(ctx context.Context, deployment, role, verifier string) error {
ns, name, _ := strings.Cut(deployment, "/")
return kubectlWithInput(ctx, []byte(alterRoleSQL(role, verifier)), "-n", ns, "exec", "-i", "deploy/"+name, "-c", platform.PostgresContainer, "--",
"psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres")
}
// alterRoleSQL sets role's password to a SCRAM verifier, which holds no quote.
func alterRoleSQL(role, verifier string) string {
return "ALTER ROLE \"" + strings.ReplaceAll(role, `"`, `""`) + "\" WITH PASSWORD '" + verifier + "';\n"
}
// setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file
// (secrets.env, a .properties file), appending one when the key is absent. The
// file is replaced atomically and keeps its mode and owner: felis-link.properties
// is root:felis-velocity 0640, and the proxy must still be able to read it.
func setKeyValueLine(path, key, sep, value string) error {
return setKeyValueLines(path, sep, [][2]string{{key, value}})
}
// setKeyValueLines is setKeyValueLine for several keys in one rewrite.
func setKeyValueLines(path, sep string, kv [][2]string) error {
info, err := os.Stat(path)
if err != nil {
return err
}
raw, err := os.ReadFile(path)
if err != nil {
return err
}
lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
for _, p := range kv {
key, value := p[0], p[1]
found := false
for i, ln := range lines {
k, _, ok := strings.Cut(ln, sep)
if ok && strings.TrimSpace(k) == key {
lines[i] = key + sep + value
found = true
}
}
if !found {
lines = append(lines, key+sep+value)
}
}
return replaceFileKeepingMode(path, info, []byte(strings.Join(lines, "\n")+"\n"))
}
// replaceFileKeepingMode atomically replaces path with data, keeping the mode and
// owner info describes: these files are read by other users (the proxy's) and
// some hold credentials, so a rewrite must not widen or narrow who can read them.
func replaceFileKeepingMode(path string, info os.FileInfo, data []byte) error {
tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if err := tmp.Chmod(info.Mode().Perm()); err != nil {
tmp.Close()
return err
}
if st, ok := info.Sys().(*syscall.Stat_t); ok {
if err := tmp.Chown(int(st.Uid), int(st.Gid)); err != nil {
tmp.Close()
return err
}
}
if _, err := tmp.Write(data); err != nil {
tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}
cmd/felis/rotatetoken_test.go
-883
View File
@@ -1,883 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
type rotationRig struct {
r tokenRotator
cl client.Client
out *bytes.Buffer
events []string
dir string
clock time.Time
}
func tokenSecret(ns, name, val string) *corev1.Secret {
return keySecret(ns, name, "token", val)
}
func keySecret(ns, name, key, val string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name},
Data: map[string][]byte{key: []byte(val)},
}
}
// serverPod is a game server's pod as the operator labels it.
func serverPod(ns, name, server string) *corev1.Pod {
return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
Labels: map[string]string{
"felis.lolicon.best/server": server,
"felis.lolicon.best/managed-by": "felis-operator",
"felis.lolicon.best/component": "server",
}}}
}
// backupPod is a backup Job's pod as internal/backupjob labels it: it carries
// the server's label too, and no rotation may take it for the server's own.
func backupPod(ns, name, server string) *corev1.Pod {
return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
Labels: map[string]string{
"felis.lolicon.best/server": server,
"app.kubernetes.io/managed-by": "felis-backup",
"app.kubernetes.io/component": "world-backup",
}}}
}
func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
t.Helper()
rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir(), clock: time.Unix(1_800_000_000, 0)}
rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
rig.r = tokenRotator{
cl: rig.cl,
controlNS: "felis",
minecraftNS: "minecraft",
buildNS: "felis-build",
secretsEnv: filepath.Join(rig.dir, "secrets.env"),
linkProps: filepath.Join(rig.dir, "felis-link.properties"),
forwardingFile: filepath.Join(rig.dir, "forwarding.secret"),
hostTOML: filepath.Join(rig.dir, "felis.host.toml"),
podTOML: filepath.Join(rig.dir, "felis.pod.toml"),
defaultTOML: filepath.Join(rig.dir, "felis.toml"),
newToken: func() (string, error) { return "NEWTOKEN", nil },
rollout: func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
return nil
},
restartUnit: func(_ context.Context, unit string) error {
rig.events = append(rig.events, "restart "+unit)
return nil
},
proxyLog: func(context.Context, time.Time) string { return "" },
alterRole: func(context.Context, string, string, string) error {
rig.events = append(rig.events, "alter-role")
return nil
},
verifyDB: func(context.Context, string) error {
rig.events = append(rig.events, "verify-db")
return nil
},
// Every look at the clock moves it a second on, so a wait measured with it
// ends after a known number of looks.
now: func() time.Time {
rig.clock = rig.clock.Add(time.Second)
return rig.clock
},
reloadWait: 5 * time.Second,
out: rig.out,
}
return rig
}
func (rig *rotationRig) secretKey(t *testing.T, ns, name, key string) string {
t.Helper()
var s corev1.Secret
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
return "<missing>"
}
return string(s.Data[key])
}
func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
t.Helper()
return rig.secretKey(t, ns, name, "token")
}
func (rig *rotationRig) pods(t *testing.T) string {
t.Helper()
var pods corev1.PodList
if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
t.Fatal(err)
}
names := make([]string, len(pods.Items))
for i, p := range pods.Items {
names[i] = p.Name
}
return strings.Join(names, ",")
}
func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
t.Helper()
if err := os.WriteFile(path, []byte(body), mode); err != nil {
t.Fatal(err)
}
if err := os.Chmod(path, mode); err != nil {
t.Fatal(err)
}
}
func readTestFile(t *testing.T, path string) string {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
func TestRotateLimboToken(t *testing.T) {
rig := newRotationRig(t,
tokenSecret("felis", "felis-limbo-token", "old"),
tokenSecret("minecraft", "felis-limbo-token", "old"),
tokenSecret("felis", "felis-service-token", "proxy"),
serverPod("minecraft", "login-0", "login"),
serverPod("minecraft", "survival-0", "survival"),
backupPod("minecraft", "login-backup-x", "login"),
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600)
// felis-api must roll only after both copies hold the new value, and the login
// pod must still be there then: restarting it earlier would have it present
// the new token to an api that does not know it yet.
rig.r.rollout = func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" {
t.Errorf("api rolled while the control Secret held %q", got)
}
if got := rig.secret(t, "minecraft", "felis-limbo-token"); got != "NEWTOKEN" {
t.Errorf("api rolled while the minecraft replica held %q", got)
}
var pod corev1.Pod
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
t.Errorf("the login pod was restarted before the api rolled")
}
return nil
}
if err := rig.r.rotate(context.Background(), "limbo", true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
t.Errorf("secrets.env = %q", got)
}
if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 {
t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm())
}
if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" {
t.Errorf("the proxy's token changed to %q", got)
}
// The login pod restarted; a user server and the backup Job's pod, which
// carries the login server's label too, are untouched.
if got := rig.pods(t); got != "login-backup-x,survival-0" {
t.Errorf("pods left = %s, want login-backup-x,survival-0", got)
}
if strings.Join(rig.events, ",") != "roll felis-api" {
t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events)
}
if strings.Contains(rig.out.String(), "NEWTOKEN") {
t.Errorf("the new token was printed: %s", rig.out.String())
}
}
const testLinkProps = "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n"
func reloadLine(token string) string {
return "[12:00:00 INFO] [felis-link]: Felis: service-token reloaded from /x/felis-link.properties (fingerprint " + tokenFingerprint(token) + ")\n"
}
// The host proxy re-reads its token: the rotation waits for it to say so and
// leaves it running.
func TestRotateVelocityTokenReloadsTheHostProxy(t *testing.T) {
rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
if err := os.Chtimes(rig.r.linkProps, written, written); err != nil {
t.Fatal(err)
}
// The proxy logs its reload on its next call to felis-api, which may be
// while the api rolls: the log must be read from before that.
var reloadedAt time.Time
rig.r.rollout = func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
if got := readTestFile(t, rig.r.linkProps); !strings.Contains(got, "service-token=NEWTOKEN\n") {
t.Errorf("api rolled before the proxy's file held the new token: %q", got)
}
reloadedAt = rig.clock
return nil
}
looks := 0
rig.r.proxyLog = func(_ context.Context, since time.Time) string {
looks++
log := reloadLine("old") // an earlier rotation's
if looks >= 3 && !since.After(reloadedAt) {
log += reloadLine("NEWTOKEN")
}
return log
}
if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.linkProps); got != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
t.Errorf("felis-link.properties = %q", got)
}
info, _ := os.Stat(rig.r.linkProps)
if info.Mode().Perm() != 0o640 {
t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm())
}
if !info.ModTime().Equal(written) {
t.Errorf("properties mtime = %v, want it kept at %v (felis domain check would read the proxy as stale)", info.ModTime(), written)
}
if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
t.Errorf("control Secret = %q, want NEWTOKEN", got)
}
// The proxy's token has no replica: it must not appear in a workload namespace.
if got := rig.secret(t, "minecraft", "felis-service-token"); got != "<missing>" {
t.Errorf("rotation copied the proxy token into minecraft (%q)", got)
}
if strings.Join(rig.events, ",") != "roll felis-api" {
t.Errorf("events = %v, want the api roll and no proxy restart", rig.events)
}
if looks != 3 {
t.Errorf("the log was read %d times, want 3 (until the line appeared)", looks)
}
if out := rig.out.String(); !strings.Contains(out, "felis-velocity: took the new token from its properties; players stayed connected") {
t.Errorf("output does not say the players stayed: %s", out)
}
}
// A proxy that never logs the new fingerprint (an older plugin, a stuck proxy)
// is restarted once the wait is over.
func TestRotateVelocityTokenRestartsAProxyThatDoesNotReload(t *testing.T) {
rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
looks := 0
rig.r.proxyLog = func(context.Context, time.Time) string {
looks++
return reloadLine("old")
}
if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
t.Fatal(err)
}
if strings.Join(rig.events, ",") != "roll felis-api,restart felis-velocity" {
t.Errorf("events = %v, want the api roll then the proxy restart", rig.events)
}
// reloadWait is 5s and each look moves the clock a second.
if looks != 5 {
t.Errorf("the log was read %d times, want 5", looks)
}
if out := rig.out.String(); !strings.Contains(out, "felis-velocity: had not taken the new token within 5s, restarted") {
t.Errorf("output does not explain the restart: %s", out)
}
}
// The proxy and the Java plugin name a token by the same fingerprint
// (plugins/shared LinkConfigLoaderTest fileTokenFollowsTheFile).
func TestTokenFingerprintMatchesThePlugin(t *testing.T) {
if got := tokenFingerprint("new-token"); got != "348e9df2a42b" {
t.Errorf("tokenFingerprint(new-token) = %q, want 348e9df2a42b", got)
}
}
// An external proxy has no felis-link.properties here: the Secret still rotates,
// nothing is restarted on this host, and the output says where the value is
// without printing it.
func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) {
rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
t.Fatal(err)
}
if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
t.Errorf("control Secret = %q, want NEWTOKEN", got)
}
if strings.Join(rig.events, ",") != "roll felis-api" {
t.Errorf("events = %v, want no proxy restart", rig.events)
}
out := rig.out.String()
if !strings.Contains(out, "felis/felis-service-token") || strings.Contains(out, "NEWTOKEN") {
t.Errorf("output should point at the Secret without the value: %s", out)
}
}
func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old"))
// An install from before per-caller tokens has no BUILD_TOKEN line yet.
writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600)
if err := rig.r.rotate(context.Background(), "build", true); err != nil {
t.Fatal(err)
}
if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" {
t.Errorf("build replica = %q, want NEWTOKEN (created when absent)", got)
}
if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" {
t.Errorf("control Secret = %q, want NEWTOKEN", got)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
t.Errorf("secrets.env = %q", got)
}
}
// A failed api rollout stops the rotation before the caller restarts: the login
// gate keeps running on the old value the old api pods still accept.
func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) {
rig := newRotationRig(t,
tokenSecret("felis", "felis-limbo-token", "old"),
serverPod("minecraft", "login-0", "login"),
)
rig.r.rollout = func(context.Context, string) error { return errors.New("rollout timed out") }
err := rig.r.rotate(context.Background(), "limbo", true)
if err == nil || !strings.Contains(err.Error(), "rollout timed out") {
t.Fatalf("err = %v, want the rollout failure", err)
}
if got := rig.pods(t); got != "login-0" {
t.Errorf("pods left = %s: the login pod was restarted although the api never rolled", got)
}
}
func TestRotateRefusesAnUnknownCredential(t *testing.T) {
rig := newRotationRig(t)
writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600)
if err := rig.r.rotate(context.Background(), "admin", true); err == nil {
t.Fatal("rotated an unknown credential")
}
if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\n" {
t.Errorf("secrets.env = %q, want it untouched", got)
}
if len(rig.events) != 0 {
t.Errorf("events = %v, want nothing touched", rig.events)
}
}
const (
testHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
testPodTOML = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
)
// installedRig is a host as the installer leaves it, with every credential in
// place, for the plan test.
func installedRig(t *testing.T) *rotationRig {
t.Helper()
rig := newRotationRig(t,
tokenSecret("felis", "felis-service-token", "old"),
tokenSecret("felis", "felis-limbo-token", "old"),
tokenSecret("minecraft", "felis-limbo-token", "old"),
tokenSecret("felis", "felis-build-token", "old"),
tokenSecret("felis-build", "felis-build-token", "old"),
tokenSecret("felis", "felis-ops-token", "old"),
keySecret("felis", "felis-registry-auth", "platform", "old"),
keySecret("felis-build", "felis-registry-push", "password", "old"),
keySecret("felis", "felis-forwarding-secret", "secret", "old"),
keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
keySecret("felis", "felis-config", "felis.toml", testPodTOML),
keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
serverPod("minecraft", "login-0", "login"),
serverPod("minecraft", "survival-0", "survival"),
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=old\n", 0o600)
writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
return rig
}
// Without -yes every rotation prints its plan and changes nothing.
func TestRotatePlanChangesNothing(t *testing.T) {
for _, kind := range rotationKinds() {
t.Run(kind, func(t *testing.T) {
rig := installedRig(t)
files := map[string]string{}
for _, p := range []string{rig.r.secretsEnv, rig.r.linkProps, rig.r.forwardingFile, rig.r.hostTOML, rig.r.podTOML} {
files[p] = readTestFile(t, p)
}
rig.r.newToken = func() (string, error) {
t.Error("a plan generated a token")
return "NEWTOKEN", nil
}
if err := rig.r.rotate(context.Background(), kind, false); err != nil {
t.Fatal(err)
}
for p, before := range files {
if got := readTestFile(t, p); got != before {
t.Errorf("%s changed to %q", filepath.Base(p), got)
}
}
for _, s := range [][3]string{
{"felis", "felis-service-token", "token"}, {"felis", "felis-limbo-token", "token"},
{"felis-build", "felis-build-token", "token"}, {"felis", "felis-ops-token", "token"},
{"felis", "felis-registry-auth", "platform"}, {"felis-build", "felis-registry-push", "password"},
{"minecraft", "felis-forwarding-secret", "secret"},
} {
if got := rig.secretKey(t, s[0], s[1], s[2]); got != "old" {
t.Errorf("Secret %s/%s changed to %q", s[0], s[1], got)
}
}
if got := rig.secretKey(t, "minecraft", "felis-config", "felis.toml"); got != testPodTOML {
t.Errorf("felis-config changed to %q", got)
}
if len(rig.events) != 0 {
t.Errorf("events = %v, want none", rig.events)
}
if got := rig.pods(t); got != "login-0,survival-0" {
t.Errorf("pods left = %s, want all", got)
}
out := rig.out.String()
if !strings.HasSuffix(out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes "+kind+"\n") {
t.Errorf("the plan does not end with how to go ahead: %s", out)
}
// Each plan names the interruption it causes.
want := apiRestartNote
if kind == kindForwarding {
want = "everyone online is disconnected"
}
if !strings.Contains(out, want) {
t.Errorf("the plan does not say %q: %s", want, out)
}
})
}
}
func TestRotateRegistryTokens(t *testing.T) {
rig := newRotationRig(t,
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: "felis-registry-auth"},
Data: map[string][]byte{"platform": []byte("a"), "build": []byte("b"), "prune": []byte("c")}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis-build", Name: "felis-registry-push"},
Data: map[string][]byte{"username": []byte("build"), "password": []byte("b")}},
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=a\nREGISTRY_BUILD_TOKEN=b\nREGISTRY_PRUNE_TOKEN=c\n", 0o600)
n := 0
rig.r.newToken = func() (string, error) {
n++
return fmt.Sprintf("NEWTOKEN%d", n), nil
}
// The registry reads its tokens as it starts: it restarts after the Secret
// holds them, and felis-api (the prune token) after that.
rig.r.rollout = func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
if got := rig.secretKey(t, "felis", "felis-registry-auth", "prune"); got != "NEWTOKEN3" {
t.Errorf("%s rolled while the prune token was %q", deployment, got)
}
return nil
}
if err := rig.r.rotate(context.Background(), kindRegistry, true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=NEWTOKEN1\nREGISTRY_BUILD_TOKEN=NEWTOKEN2\nREGISTRY_PRUNE_TOKEN=NEWTOKEN3\n" {
t.Errorf("secrets.env = %q", got)
}
for key, want := range map[string]string{"platform": "NEWTOKEN1", "build": "NEWTOKEN2", "prune": "NEWTOKEN3"} {
if got := rig.secretKey(t, "felis", "felis-registry-auth", key); got != want {
t.Errorf("felis-registry-auth %s = %q, want %s", key, got, want)
}
}
if got := rig.secretKey(t, "felis-build", "felis-registry-push", "password"); got != "NEWTOKEN2" {
t.Errorf("the build Jobs' push password = %q, want the build token", got)
}
if got := rig.secretKey(t, "felis-build", "felis-registry-push", "username"); got != "build" {
t.Errorf("the build Jobs' push username = %q, want build", got)
}
if strings.Join(rig.events, ",") != "roll registry,roll felis-api" {
t.Errorf("events = %v, want the registry then felis-api", rig.events)
}
if strings.Contains(rig.out.String(), "NEWTOKEN") {
t.Errorf("a new token was printed: %s", rig.out.String())
}
}
func TestRotateForwardingSecret(t *testing.T) {
lock := time.Unix(1_800_000_000, 0)
rig := newRotationRig(t,
keySecret("felis", "felis-forwarding-secret", "secret", "old"),
keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
serverPod("minecraft", "survival-0", "survival"),
serverPod("minecraft", "lobby-0", "lobby"),
// creative is being backed up: a running Job holds its world.
serverPod("minecraft", "creative-0", "creative"),
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "creative-backup",
Labels: map[string]string{maintenance.LabelServer: "creative", maintenance.LabelManagedBy: "felis-backup"}}},
// survival's last backup is over; its finished pod stays until the Job's TTL.
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival-backup",
Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: "felis-backup"}},
Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}},
backupPod("minecraft", "survival-backup-x", "survival"),
// A pod already on its way out is neither counted nor deleted again.
func() *corev1.Pod {
p := serverPod("minecraft", "lobby-old", "lobby")
p.DeletionTimestamp = &metav1.Time{Time: lock}
p.Finalizers = []string{"felis.lolicon.best/test"}
return p
}(),
// skyblock's restore has just been admitted, its Job not created yet.
serverPod("minecraft", "skyblock-0", "skyblock"),
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "skyblock",
Annotations: map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lock)}}},
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival"}},
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nFORWARDING_SECRET=old\n", 0o600)
writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
// The proxy restarts after the servers were told to: a player who rejoins
// must not meet a server still on the old secret.
rig.r.restartUnit = func(_ context.Context, unit string) error {
rig.events = append(rig.events, "restart "+unit)
if got := rig.pods(t); strings.Contains(got, "survival-0") {
t.Errorf("the proxy restarted before the servers (pods %s)", got)
}
if got := readTestFile(t, rig.r.forwardingFile); got != "NEWTOKEN" {
t.Errorf("the proxy restarted on forwarding.secret %q", got)
}
return nil
}
if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nFORWARDING_SECRET=NEWTOKEN\n" {
t.Errorf("secrets.env = %q", got)
}
if info, _ := os.Stat(rig.r.forwardingFile); info.Mode().Perm() != 0o640 {
t.Errorf("forwarding.secret mode = %v, want 0640", info.Mode().Perm())
}
for _, ns := range []string{"felis", "minecraft"} {
if got := rig.secretKey(t, ns, "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
t.Errorf("Secret %s/felis-forwarding-secret = %q, want NEWTOKEN", ns, got)
}
}
if got := rig.pods(t); got != "creative-0,lobby-old,skyblock-0,survival-backup-x" {
t.Errorf("pods left = %s, want the held servers, the terminating pod and the backup's pod", got)
}
if strings.Join(rig.events, ",") != "restart felis-velocity" {
t.Errorf("events = %v, want only the proxy restart (felis-api does not hold this secret)", rig.events)
}
out := rig.out.String()
for _, want := range []string{
"every running server restarts to read it (2 now)",
"left running, because a backup, restore or file write holds its world: creative (backup), skyblock (restore).",
" - servers: 2 restarting on the new secret\n",
} {
if !strings.Contains(out, want) {
t.Errorf("output lacks %q: %s", want, out)
}
}
if strings.Contains(out, "NEWTOKEN") {
t.Errorf("the new secret was printed: %s", out)
}
}
func TestRotateForwardingSecretForAnExternalProxy(t *testing.T) {
rig := newRotationRig(t, keySecret("felis", "felis-forwarding-secret", "secret", "old"))
if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
t.Fatal(err)
}
if got := rig.secretKey(t, "felis", "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
t.Errorf("control Secret = %q, want NEWTOKEN", got)
}
if len(rig.events) != 0 {
t.Errorf("events = %v, want no proxy restart on this host", rig.events)
}
if out := rig.out.String(); !strings.Contains(out, "put the value in Secret felis/felis-forwarding-secret into your proxy's forwarding secret file") {
t.Errorf("output does not say where the value is: %s", out)
}
}
// dbRig is an installed host for the database rotation: felis.toml links to the
// host copy, as the installer makes it.
func dbRig(t *testing.T) *rotationRig {
t.Helper()
rig := newRotationRig(t,
keySecret("felis", "felis-config", "felis.toml", testPodTOML),
keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
)
writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n", 0o600)
writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
if err := os.Symlink("felis.host.toml", rig.r.defaultTOML); err != nil {
t.Fatal(err)
}
return rig
}
const (
wantHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
wantPodTOML = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
)
func TestRotateDatabasePassword(t *testing.T) {
rig := dbRig(t)
rig.r.alterRole = func(_ context.Context, deployment, role, verifier string) error {
rig.events = append(rig.events, "alter-role")
if deployment != "felis/felis-postgres" || role != "felis" {
t.Errorf("alterRole(%q, %q), want felis/felis-postgres and felis", deployment, role)
}
if !strings.Contains(readTestFile(t, rig.r.secretsEnv), "DB_PASSWORD=NEWTOKEN\n") {
t.Error("the role changed before secrets.env recorded the new password")
}
// The verifier is the new password's, under the salt it carries.
m := regexp.MustCompile(`^SCRAM-SHA-256\$4096:([^$]+)\$`).FindStringSubmatch(verifier)
if m == nil {
t.Fatalf("verifier %q is not a SCRAM-SHA-256 verifier", verifier)
}
salt, err := base64.StdEncoding.DecodeString(m[1])
if err != nil || len(salt) != 16 {
t.Fatalf("verifier salt %q: %v", m[1], err)
}
if want, _ := scramVerifier("NEWTOKEN", salt, 4096); verifier != want {
t.Errorf("verifier = %q, want %q", verifier, want)
}
return nil
}
// The configs change only once the database accepts the new password.
rig.r.verifyDB = func(_ context.Context, url string) error {
rig.events = append(rig.events, "verify-db")
if url != "postgres://felis:[email protected]:30432/felis?sslmode=disable" {
t.Errorf("verified with %q, want the host URL with the new password", url)
}
if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
t.Errorf("the host config changed before the database accepted the password: %q", got)
}
return nil
}
rig.r.rollout = func(_ context.Context, deployment string) error {
rig.events = append(rig.events, "roll "+deployment)
if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != wantPodTOML {
t.Errorf("felis-api rolled on felis-config %q", got)
}
return nil
}
if err := rig.r.rotate(context.Background(), kindDB, true); err != nil {
t.Fatal(err)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=NEWTOKEN\nSERVICE_TOKEN=s\n" {
t.Errorf("secrets.env = %q", got)
}
if got := readTestFile(t, rig.r.hostTOML); got != wantHostTOML {
t.Errorf("host config = %q", got)
}
if got := readTestFile(t, rig.r.podTOML); got != wantPodTOML {
t.Errorf("pod config = %q", got)
}
if info, err := os.Lstat(rig.r.defaultTOML); err != nil || info.Mode()&os.ModeSymlink == 0 {
t.Errorf("felis.toml is no longer the link to the host copy (%v)", err)
}
for _, ns := range []string{"felis", "minecraft"} {
if got := rig.secretKey(t, ns, "felis-config", "felis.toml"); got != wantPodTOML {
t.Errorf("Secret %s/felis-config = %q", ns, got)
}
}
if strings.Join(rig.events, ",") != "alter-role,verify-db,roll felis-api" {
t.Errorf("events = %v", rig.events)
}
if strings.Contains(rig.out.String(), "NEWTOKEN") {
t.Errorf("the new password was printed: %s", rig.out.String())
}
}
// A password the database does not accept leaves the configs on the old one
// and felis-api running: the installer's record, already new, is the way back.
func TestRotateDatabasePasswordStopsWhenTheDatabaseRefuses(t *testing.T) {
rig := dbRig(t)
rig.r.verifyDB = func(context.Context, string) error {
rig.events = append(rig.events, "verify-db")
return errors.New("password authentication failed")
}
err := rig.r.rotate(context.Background(), kindDB, true)
if err == nil || !strings.Contains(err.Error(), "password authentication failed") || !strings.Contains(err.Error(), "run the installer again") {
t.Fatalf("err = %v, want the refusal and the way back", err)
}
if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
t.Errorf("host config = %q, want it unchanged", got)
}
if got := readTestFile(t, rig.r.podTOML); got != testPodTOML {
t.Errorf("pod config = %q, want it unchanged", got)
}
if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != testPodTOML {
t.Errorf("felis-config = %q, want it unchanged", got)
}
if strings.Join(rig.events, ",") != "alter-role,verify-db" {
t.Errorf("events = %v, want no api roll", rig.events)
}
}
// Everything that can refuse does so before the installer's record or the
// role change; what the host config alone shows is refused by the plan too.
func TestRotateDatabasePasswordRefusesEarly(t *testing.T) {
for _, tc := range []struct {
name string
apply bool
setup func(t *testing.T, rig *rotationRig)
want string
}{
{"a database the installer does not run", false, func(t *testing.T, rig *rotationRig) {
writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "deployment = \"felis/felis-postgres\"\n", "", 1), 0o600)
}, "[database] deployment is unset"},
{"a database url without a role", false, func(t *testing.T, rig *rotationRig) {
writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "felis:oldpw@", "", 1), 0o600)
}, "names no role"},
{"a config copy the line editor cannot edit", true, func(t *testing.T, rig *rotationRig) {
writeTestFile(t, rig.r.podTOML, "[database]\nurl = \"\"\"\npostgres://felis:[email protected]:5432/felis\"\"\"\n", 0o600)
}, "set the password in its [database] url by hand"},
{"a pod copy that is the host copy", true, func(t *testing.T, rig *rotationRig) {
if err := os.Remove(rig.r.podTOML); err != nil {
t.Fatal(err)
}
if err := os.Symlink("felis.host.toml", rig.r.podTOML); err != nil {
t.Fatal(err)
}
}, "resolves to the same file as"},
} {
t.Run(tc.name, func(t *testing.T) {
rig := dbRig(t)
tc.setup(t, rig)
err := rig.r.rotate(context.Background(), kindDB, tc.apply)
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("err = %v, want %q", err, tc.want)
}
if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n" {
t.Errorf("secrets.env = %q, want it untouched", got)
}
if len(rig.events) != 0 {
t.Errorf("events = %v, want nothing done", rig.events)
}
})
}
}
// The RFC 7677 example (user "user", password "pencil"), with the stored and
// server keys computed by openssl's PBKDF2 and HMAC rather than this code.
func TestScramVerifier(t *testing.T) {
salt, _ := base64.StdEncoding.DecodeString("W22ZaJ0SNY7soEsUEjb6gQ==")
got, err := scramVerifier("pencil", salt, 4096)
if err != nil {
t.Fatal(err)
}
if want := "SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==$WG5d8oPm3OtcPnkdi4Uo7BkeZkBFzpcXkuLmtbsT4qY=:wfPLwcE6nTWhTAmQ7tl2KeoiWGPlZqQxSrmfPwDl2dU="; got != want {
t.Errorf("scramVerifier = %q\nwant %q", got, want)
}
}
func TestAlterRoleSQL(t *testing.T) {
if got := alterRoleSQL(`fe"lis`, "SCRAM-SHA-256$4096:c2FsdA==$a:b"); got != "ALTER ROLE \"fe\"\"lis\" WITH PASSWORD 'SCRAM-SHA-256$4096:c2FsdA==$a:b';\n" {
t.Errorf("alterRoleSQL = %q", got)
}
}
// installerSecretKeys is every secrets.env key a rotation writes.
func installerSecretKeys() map[string]bool {
keys := map[string]bool{installerForwardingKey: true, installerDBKey: true}
for _, k := range installerTokenKeys {
keys[k] = true
}
for _, k := range installerRegistryKeys {
keys[k] = true
}
return keys
}
// The installer and rotate-token must agree on where each caller's token lives:
// rotate-token writes installerTokenKeys into secrets.env, and the installer
// applies those same keys to the Secrets on its next run. A key the installer
// does not read would be silently reverted by the next upgrade.
func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
if err != nil {
t.Fatal(err)
}
script := string(raw)
for caller, key := range installerTokenKeys {
ct, ok := callerToken(caller)
if !ok {
t.Fatalf("installerTokenKeys names unknown caller %q", caller)
}
apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`)
if !apply.MatchString(script) {
t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key)
}
}
// The replicas the installer applies straight into the workload namespaces, so an
// upgrade has them before the new operator and build Jobs reference them.
for _, line := range []string{
`apply_literal_secret "$MINECRAFT_NS" felis-limbo-token token "$LIMBO_TOKEN"`,
`apply_literal_secret "$BUILD_NS" felis-build-token token "$BUILD_TOKEN"`,
`kube -n "$MINECRAFT_NS" delete secret felis-service-token --ignore-not-found`,
`kube -n "$BUILD_NS" delete secret felis-service-token --ignore-not-found`,
} {
if !strings.Contains(script, line) {
t.Errorf("bootstrap.sh lacks %s", line)
}
}
for _, ns := range []string{"MINECRAFT_NS", "BUILD_NS"} {
if strings.Contains(script, `apply_literal_secret "$`+ns+`" felis-service-token`) {
t.Errorf("bootstrap.sh still copies the proxy's token into %s", ns)
}
}
if len(installerTokenKeys) != len(callerNames()) {
t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames()))
}
}
// Every value the installer keeps in secrets.env has a rotation that rewrites
// that very key, and every key a rotation writes is one the installer
// generates, keeps and so re-applies on its next run.
func TestInstallerSecretsAreAllRotatable(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
if err != nil {
t.Fatal(err)
}
script := string(raw)
m := regexp.MustCompile(`(?s)write_file_atomic "\$SECRETS_ENV" 0600 <<EOF\n(.*?)\nEOF\n`).FindStringSubmatch(script)
if m == nil {
t.Fatal("bootstrap.sh: no secrets.env heredoc found")
}
persisted := map[string]bool{}
for _, line := range strings.Split(m[1], "\n") {
key, value, _ := strings.Cut(line, "=")
if value != "${"+key+"}" {
t.Errorf("secrets.env line %q is not KEY=${KEY}", line)
}
persisted[key] = true
}
rotated := installerSecretKeys()
for key := range persisted {
if !rotated[key] {
t.Errorf("secrets.env keeps %s, which no rotation replaces", key)
}
}
for key := range rotated {
if !persisted[key] {
t.Errorf("a rotation writes %s, which the installer does not keep in secrets.env", key)
}
if !regexp.MustCompile(`\n ` + key + `="\$\{` + key + `:-\$\(openssl rand -hex [0-9]+\)\}"\n`).MatchString(script) {
t.Errorf("bootstrap.sh does not generate %s when secrets.env lacks it", key)
}
}
}
cmd/felis/run.go
+3 -20
View File
@@ -13,19 +13,16 @@ Usage:
Commands:
migrate up Apply embedded database migrations under an advisory lock (snapshots the database first)
db Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check)
offsite Copy world archives, database bundles, user images and uploads to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|fetch-images|fetch-uploads|keygen)
offsite Copy world archives and database bundles to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|keygen)
operator Run the MinecraftServer controller-manager
api Run the felis-api HTTP server
nano Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)
reaper Run the world reaper / backup batch
restore Extract a world archive into a world volume (internal Job entrypoint)
backup Archive a world into the backup store and record it (internal Job entrypoint)
backup-now Archive every user server's world now, one at a time (or the named ones; -stop stops running ones first; prints the plan, -yes applies; requires root/sudo)
files List, read, write, mkdir, delete, rename, upload or unzip one path in a stopped server's world (internal Job entrypoint)
export Archive a stopped server's world, or read one of its backups, and hand it to felis-api for download (internal Job entrypoint)
egress-gate Hold a build or game server pod until its egress NetworkPolicy is enforced (internal init container entrypoint)
files List/read/write one file in a stopped server's world (internal Job entrypoint)
egress-gate Hold a build pod until its egress NetworkPolicy is enforced (internal Job entrypoint)
fetch-context Fetch and extract a submission's build context (internal Job entrypoint)
scan-gate Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
push-image Push a scanned image tarball to the registry (internal Job entrypoint)
mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
registry-gate Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
@@ -33,11 +30,6 @@ Commands:
apply Create a MinecraftServer CRD (direct K8s write; use -f server.json)
setup Run host bootstrap + first-run setup console (TUI; requires root/sudo)
converge Fill in fields a newer desired spec added to already-installed system servers
rotate-token Replace a generated credential and restart what reads it (velocity|limbo|build|ops|registry|forwarding|db; prints the plan, -yes applies; requires root/sudo)
domain Move the install to a new root domain on every surface that carries it, or check each one (set|check; requires root/sudo)
status Print the platform at a glance: node, control plane, proxy, servers, backups, host, open alerts (requires root/sudo)
doctor Run every health check once, grouped by area, with where to look next; mails nothing (requires root/sudo)
support-bundle Collect status, doctor, logs and cluster state into one redacted tar.gz to share when asking for help (requires root/sudo)
watchdog Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
version Print the build stamp of this binary
update Report which platform components have updates available
@@ -65,12 +57,9 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
"reaper": cmdReaper,
"restore": cmdRestore,
"backup": cmdBackup,
"backup-now": cmdBackupNow,
"files": cmdFiles,
"export": cmdExport,
"egress-gate": cmdEgressGate,
"fetch-context": cmdFetchContext,
"scan-gate": cmdScanGate,
"push-image": cmdPushImage,
"mirror-build-tools": cmdMirrorBuildTools,
"registry-gate": cmdRegistryGate,
@@ -78,20 +67,14 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
"apply": cmdApply,
"setup": cmdSetup,
"converge": cmdConverge,
"rotate-token": cmdRotateToken,
"domain": cmdDomain,
"breakGlass": cmdBreakGlass,
"bootstrap-assets": cmdBootstrapAssets,
"init-forwarding": cmdInitForwarding,
"init-volume": cmdInitVolume,
"pin-images": cmdPinImages,
"image-bundle": cmdImageBundle,
"version": cmdVersion,
"update": cmdUpdate,
"watchdog": cmdWatchdog,
"status": cmdStatus,
"doctor": cmdDoctor,
"support-bundle": cmdSupportBundle,
}
// run dispatches a subcommand. It is separate from main so the router is
cmd/felis/run_test.go
+1 -1
View File
@@ -43,7 +43,7 @@ func TestRunUnknownCommand(t *testing.T) {
// decision rather than an oversight.
var undocumentedCommands = map[string]bool{
"bootstrap-assets": true, "init-forwarding": true, "init-volume": true,
"pin-images": true, "image-bundle": true,
"pin-images": true,
}
// The usage text and the dispatch table must describe the same set of commands.
cmd/felis/scangate.go
-166
View File
@@ -1,166 +0,0 @@
package main
import (
"errors"
"flag"
"fmt"
"io"
"io/fs"
"os"
"strings"
"felis.lolicon.best/internal/build"
)
// maxScanDocument bounds each document scan-gate reads: a modpack report lists a
// few thousand packages, far below this. Tests shrink it.
var maxScanDocument int64 = 64 << 20
// cmdScanGate is the build pod's verdict step, after trivy wrote its full JSON
// report and trivy convert wrote the CycloneDX SBOM. It applies the scan policy
// to the report, prints the verdict and every blocking finding, then appends the
// verdict, the report and the SBOM to its log as the envelope felis-api keeps on
// the build (build.WriteScanEnvelope). It exits 1 when a finding blocks, which
// fails the pod before the push step runs, and 2 when the report cannot be read,
// so a scan that produced nothing usable never admits an image.
func cmdScanGate(args []string, stdout, stderr io.Writer) int {
fset := flag.NewFlagSet("scan-gate", flag.ContinueOnError)
fset.SetOutput(stderr)
reportPath := fset.String("report", "", "trivy JSON report (required)")
sbomPath := fset.String("sbom", "", "CycloneDX SBOM to keep with the report")
failOn := fset.String("fail-on", strings.Join(build.DefaultScanFailOn, ","), "comma-separated severities that block the image")
failUnfixed := fset.Bool("fail-unfixed", false, "block on vulnerabilities that have no fixed release too")
accept := fset.String("accept", "", "comma-separated vulnerability ids and secret rule ids that never block")
termLog := fset.String("termination-log", "/dev/termination-log", "where the one-line verdict goes for the pod status")
if err := fset.Parse(args); err != nil {
return 2
}
// A stray argument is a policy the gate would otherwise drop without a word
// (a comma split out of --fail-on, say).
if fset.NArg() > 0 {
fmt.Fprintf(stderr, "felis scan-gate: unexpected argument %q\n", fset.Arg(0))
return 2
}
sevs, err := build.ParseSeverities(*failOn)
if err != nil {
fmt.Fprintf(stderr, "felis scan-gate: --fail-on: %v\n", err)
return 2
}
accepted, err := build.ParseScanAccept(*accept)
if err != nil {
fmt.Fprintf(stderr, "felis scan-gate: --accept: %v\n", err)
return 2
}
if *reportPath == "" {
fmt.Fprintln(stderr, "felis scan-gate: --report is required")
return 2
}
fail := func(msg string) int {
fmt.Fprintln(stderr, "felis scan-gate: "+msg)
writeTerminationLog(*termLog, msg)
return 2
}
report, err := readScanDocument(*reportPath)
if err != nil {
return fail("the scan report is unreadable: " + err.Error())
}
policy := build.ScanPolicy{FailOn: sevs, FailUnfixed: *failUnfixed, Accept: accepted}
summary, err := build.Summarize(report, policy)
if err != nil {
return fail("the scan report is unreadable: " + err.Error())
}
env := build.ScanEnvelope{Summary: summary, Report: report}
if *sbomPath != "" {
switch sbom, err := readScanDocument(*sbomPath); {
case err == nil:
env.SBOM = sbom
case errors.Is(err, fs.ErrNotExist):
fmt.Fprintf(stdout, "felis scan-gate: no SBOM at %s; keeping the report alone\n", *sbomPath)
default:
return fail("the SBOM is unreadable: " + err.Error())
}
}
printScanVerdict(stdout, summary)
written, err := build.WriteScanEnvelope(stdout, env)
if err != nil {
return fail("could not write the scan envelope: " + err.Error())
}
for _, doc := range written.Summary.Omitted {
fmt.Fprintf(stderr, "felis scan-gate: the %s is too large to keep with the build and was left out\n", doc)
}
if summary.Blocked {
writeTerminationLog(*termLog, summary.Reason())
return 1
}
writeTerminationLog(*termLog, "the scan passed")
return 0
}
// printScanVerdict writes the human half of scan-gate's log.
func printScanVerdict(w io.Writer, s build.ScanSummary) {
var counts []string
for _, sev := range build.Severities {
counts = append(counts, fmt.Sprintf("%s %d", sev, s.Counts[sev]))
}
fmt.Fprintf(w, "felis scan-gate: %d packages; findings: %s\n", s.Packages, strings.Join(counts, ", "))
unfixed := "vulnerabilities with no fixed release do not block"
if s.Policy.FailUnfixed {
unfixed = "vulnerabilities with no fixed release block too"
}
fmt.Fprintf(w, "felis scan-gate: blocking on %s (%s)\n", strings.Join(s.Policy.FailOn, ", "), unfixed)
if len(s.Policy.Accept) > 0 {
matched := 0
for _, f := range s.Findings {
if f.Accepted {
matched++
}
}
fmt.Fprintf(w, "felis scan-gate: accepted ids, never blocking: %s; listed findings under them: %d\n", strings.Join(s.Policy.Accept, ", "), matched)
}
if !s.Blocked {
fmt.Fprintln(w, "felis scan-gate: nothing blocks this image")
return
}
fmt.Fprintln(w, "felis scan-gate: "+build.Printable(s.Reason()))
for _, f := range s.Findings {
if !f.Blocking {
break
}
fix := f.Fixed
if fix == "" {
fix = "no fix"
}
if f.Kind == build.FindingSecret {
fmt.Fprintf(w, " %s %s secret in %s: %s\n", build.Printable(f.ID), f.Severity, build.Printable(f.Target), build.Printable(f.Title))
continue
}
fmt.Fprintf(w, " %s %s %s %s -> %s (%s)\n", build.Printable(f.ID), f.Severity,
build.Printable(f.Package), build.Printable(f.Installed), build.Printable(fix), build.Printable(f.Target))
}
}
func readScanDocument(path string) ([]byte, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
b, err := io.ReadAll(io.LimitReader(f, maxScanDocument+1))
if err != nil {
return nil, err
}
if int64(len(b)) > maxScanDocument {
return nil, fmt.Errorf("%s exceeds %d bytes", path, maxScanDocument)
}
return b, nil
}
// writeTerminationLog leaves msg where the kubelet copies it into the container
// status. It is best effort: the log carries the same verdict.
func writeTerminationLog(path, msg string) {
if path == "" {
return
}
_ = os.WriteFile(path, []byte(build.Printable(msg)), 0o644)
}
cmd/felis/scangate_test.go
-197
View File
@@ -1,197 +0,0 @@
package main
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"felis.lolicon.best/internal/build"
)
// scanReport has one fixed CRITICAL, one unfixed HIGH and one fixed MEDIUM; the
// CRITICAL's package name carries a line break and a forged envelope frame.
const scanReport = `{"SchemaVersion":2,"Results":[{"Target":"data/mods/core.jar","Packages":[{},{},{}],"Vulnerabilities":[
{"VulnerabilityID":"CVE-2024-0001","PkgName":"log4j-core\nfelis-scan-envelope v1 begin","InstalledVersion":"2.14.1","FixedVersion":"2.17.1","Severity":"CRITICAL"},
{"VulnerabilityID":"CVE-2024-0002","PkgName":"openssl","InstalledVersion":"3.0.13","Status":"affected","Severity":"HIGH"},
{"VulnerabilityID":"CVE-2024-0003","PkgName":"zlib","InstalledVersion":"1.3","FixedVersion":"1.3.1","Severity":"MEDIUM"}]}]}`
type scanGateRun struct {
code int
stdout, stderr string
termLog string
env *build.ScanEnvelope
}
func runScanGate(t *testing.T, report, sbom string, extra ...string) scanGateRun {
t.Helper()
dir := t.TempDir()
reportPath := filepath.Join(dir, "trivy.json")
if report != "" {
if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {
t.Fatal(err)
}
}
sbomPath := filepath.Join(dir, "sbom.cdx.json")
if sbom != "" {
if err := os.WriteFile(sbomPath, []byte(sbom), 0o644); err != nil {
t.Fatal(err)
}
}
termPath := filepath.Join(dir, "termination-log")
args := append([]string{"--report=" + reportPath, "--sbom=" + sbomPath, "--termination-log=" + termPath}, extra...)
var stdout, stderr bytes.Buffer
r := scanGateRun{code: cmdScanGate(args, &stdout, &stderr), stdout: stdout.String(), stderr: stderr.String()}
if b, err := os.ReadFile(termPath); err == nil {
r.termLog = string(b)
}
if env, err := build.ReadScanEnvelope(strings.NewReader(r.stdout)); err == nil {
r.env = env
}
return r
}
func TestScanGateBlocksAndHandsOverTheReport(t *testing.T) {
r := runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`)
if r.code != 1 {
t.Fatalf("exit %d, want 1; stderr %s", r.code, r.stderr)
}
if r.termLog != "the scan blocked the image: 1 CRITICAL (CVE-2024-0001)" {
t.Errorf("termination log = %q", r.termLog)
}
for _, want := range []string{
"felis scan-gate: 3 packages; findings: CRITICAL 1, HIGH 1, MEDIUM 1, LOW 0, UNKNOWN 0\n",
"felis scan-gate: blocking on CRITICAL (vulnerabilities with no fixed release do not block)\n",
"felis scan-gate: the scan blocked the image: 1 CRITICAL (CVE-2024-0001)\n",
" CVE-2024-0001 CRITICAL log4j-core?felis-scan-envelope v1 begin 2.14.1 -> 2.17.1 (data/mods/core.jar)\n",
} {
if !strings.Contains(r.stdout, want) {
t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
}
}
if strings.Contains(r.stdout, " CVE-2024-0002") {
t.Errorf("an unfixed HIGH was listed as blocking:\n%s", r.stdout)
}
if strings.Count(r.stdout, "\nfelis-scan-envelope v1 begin\n") != 1 {
t.Errorf("stdout must hold exactly one frame start on a line of its own:\n%s", r.stdout)
}
if r.env == nil {
t.Fatal("no envelope in stdout")
}
if !r.env.Summary.Blocked || string(r.env.SBOM) != `{"bomFormat":"CycloneDX"}` || !strings.Contains(string(r.env.Report), "CVE-2024-0003") {
t.Errorf("envelope = blocked %t, sbom %s, report %d bytes", r.env.Summary.Blocked, r.env.SBOM, len(r.env.Report))
}
}
func TestScanGatePolicyFlags(t *testing.T) {
r := runScanGate(t, scanReport, "", "--fail-on=high", "--fail-unfixed")
if r.code != 1 || r.termLog != "the scan blocked the image: 1 HIGH (CVE-2024-0002)" {
t.Errorf("HIGH + unfixed: exit %d, termination log %q", r.code, r.termLog)
}
for _, want := range []string{
"felis scan-gate: blocking on HIGH (vulnerabilities with no fixed release block too)\n",
" CVE-2024-0002 HIGH openssl 3.0.13 -> no fix (data/mods/core.jar)\n",
} {
if !strings.Contains(r.stdout, want) {
t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
}
}
r = runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`, "--fail-on=LOW")
if r.code != 0 || r.termLog != "the scan passed" || !strings.Contains(r.stdout, "felis scan-gate: nothing blocks this image\n") {
t.Errorf("LOW only: exit %d, termination log %q, stdout:\n%s", r.code, r.termLog, r.stdout)
}
if r.env == nil || r.env.Summary.Blocked || r.env.SBOM == nil {
t.Errorf("a passing scan must still hand over its report and SBOM: %+v", r.env)
}
}
func TestScanGateAcceptedIDsNeverBlock(t *testing.T) {
r := runScanGate(t, scanReport, "", "--fail-on=CRITICAL,MEDIUM", "--accept=CVE-2024-0001, CVE-2024-0002,CVE-2099-0001")
if r.code != 1 || r.termLog != "the scan blocked the image: 1 MEDIUM (CVE-2024-0003)" {
t.Errorf("exit %d, termination log %q", r.code, r.termLog)
}
if !strings.Contains(r.stdout, "felis scan-gate: accepted ids, never blocking: CVE-2024-0001, CVE-2024-0002, CVE-2099-0001; listed findings under them: 2\n") {
t.Errorf("stdout:\n%s", r.stdout)
}
if r.env == nil || strings.Join(r.env.Summary.Policy.Accept, ",") != "CVE-2024-0001,CVE-2024-0002,CVE-2099-0001" {
t.Fatalf("envelope = %+v", r.env)
}
for _, f := range r.env.Summary.Findings {
if f.ID == "CVE-2024-0001" && (f.Blocking || !f.Accepted) {
t.Errorf("accepted finding = %+v", f)
}
}
r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001")
if r.code != 0 || r.termLog != "the scan passed" {
t.Errorf("only an accepted CRITICAL: exit %d, termination log %q", r.code, r.termLog)
}
}
func TestScanGateWithoutAnSBOMKeepsTheReport(t *testing.T) {
r := runScanGate(t, scanReport, "", "--fail-on=LOW")
if r.code != 0 || !strings.Contains(r.stdout, "felis scan-gate: no SBOM at ") {
t.Errorf("exit %d, stdout:\n%s", r.code, r.stdout)
}
if r.env == nil || r.env.SBOM != nil || r.env.Report == nil {
t.Errorf("envelope = %+v", r.env)
}
}
func TestScanGateListsABlockingSecret(t *testing.T) {
r := runScanGate(t, `{"SchemaVersion":2,"Results":[{"Target":"config/keys.txt","Secrets":[
{"RuleID":"aws-access-key-id","Severity":"CRITICAL","Title":"AWS Access\nKey ID"}]}]}`, "")
if r.code != 1 || r.termLog != "the scan blocked the image: 1 CRITICAL (aws-access-key-id)" {
t.Errorf("exit %d, termination log %q", r.code, r.termLog)
}
if !strings.Contains(r.stdout, " aws-access-key-id CRITICAL secret in config/keys.txt: AWS Access?Key ID\n") {
t.Errorf("stdout:\n%s", r.stdout)
}
}
func TestScanGateFailsClosed(t *testing.T) {
r := runScanGate(t, "", "")
if r.code != 2 || !strings.HasPrefix(r.termLog, "the scan report is unreadable: open ") || r.env != nil {
t.Errorf("missing report: exit %d, termination log %q", r.code, r.termLog)
}
r = runScanGate(t, `{"SchemaVersion":1}`, "")
if r.code != 2 || r.termLog != "the scan report is unreadable: read trivy report: schema version 1, want 2" {
t.Errorf("old schema: exit %d, termination log %q", r.code, r.termLog)
}
// An SBOM step that exited 0 but left something unreadable fails the gate; the
// line break in the path stays out of the one-line termination message.
sbomDir := filepath.Join(t.TempDir(), "sb\nom")
if err := os.Mkdir(sbomDir, 0o755); err != nil {
t.Fatal(err)
}
r = runScanGate(t, scanReport, "", "--sbom="+sbomDir)
if r.code != 2 || !strings.HasPrefix(r.termLog, "the SBOM is unreadable: read ") ||
!strings.HasSuffix(r.termLog, "/sb?om: is a directory") || r.env != nil {
t.Errorf("unreadable SBOM: exit %d, termination log %q", r.code, r.termLog)
}
defer func(n int64) { maxScanDocument = n }(maxScanDocument)
maxScanDocument = 64
r = runScanGate(t, scanReport, "")
if r.code != 2 || !strings.HasSuffix(r.termLog, "/trivy.json exceeds 64 bytes") || r.env != nil {
t.Errorf("oversized report: exit %d, termination log %q", r.code, r.termLog)
}
maxScanDocument = 64 << 20
r = runScanGate(t, scanReport, "", "--fail-on=SEVERE")
if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --fail-on: unknown severity "SEVERE"`) {
t.Errorf("bad severity: exit %d, stderr %q", r.code, r.stderr)
}
// A severity list split at its comma leaves a stray argument, not a
// narrower policy.
r = runScanGate(t, scanReport, "", "--fail-on=LOW", "CRITICAL")
if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: unexpected argument "CRITICAL"`) || r.env != nil {
t.Errorf("stray argument: exit %d, stderr %q", r.code, r.stderr)
}
r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001 CVE-2024-0003")
if r.code != 2 || !strings.Contains(r.stderr, `felis scan-gate: --accept: "CVE-2024-0001 CVE-2024-0003" is not a vulnerability id or secret rule id`) {
t.Errorf("bad accept list: exit %d, stderr %q", r.code, r.stderr)
}
var stdout, stderr bytes.Buffer
if code := cmdScanGate(nil, &stdout, &stderr); code != 2 || !strings.Contains(stderr.String(), "--report is required") {
t.Errorf("no report flag: exit %d, stderr %q", code, stderr.String())
}
}
cmd/felis/setup.go
+35 -12
View File
@@ -13,6 +13,7 @@ import (
"felis.lolicon.best/internal/api"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
)
@@ -112,7 +113,7 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
return 1
}
res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database.URL, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
if err != nil {
fmt.Fprintf(stderr, "felis setup: %v\n", err)
return 1
@@ -215,18 +216,18 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
"Re-run `sudo felis setup` on the control-plane host once the cluster is reachable", err)
}
// The login limbo authenticates to the felis-api INTERNAL face, so it needs the
// internal base URL, the root domain (to link players at the console), and its
// own token (felis-limbo-token). The first two are plain env baked into the pod
// here; the token is a Secret the operator injects by reference — but a
// secretKeyRef is namespace-local, so first replicate the token Secret from the
// control namespace into the minecraft namespace where the login pod runs. The control namespace is
// internal base URL, the root domain (to link players at the console), and the
// service token. The first two are plain env baked into the pod here; the token
// is a Secret the operator injects by reference — but a secretKeyRef is
// namespace-local, so first replicate the token Secret from the control namespace
// into the minecraft namespace where the login pod runs. The control namespace is
// the platform default (there is no felis.toml override for it); a deployment that
// renamed it must replicate the Secret by hand.
controlNS := platform.DefaultControlNamespace
apiBaseURL := platform.InternalAPIBaseURL(controlNS)
// These Secrets must land in the minecraft namespace before the pods that
// mount them are created: the login gate's token (login authenticates to
// felis-api with it), the Velocity forwarding secret (every backend verifies the proxy's
// mount them are created: the service token (login authenticates to felis-api
// with it), the Velocity forwarding secret (every backend verifies the proxy's
// signed handshake with it — without it the login gate would derive an OFFLINE
// UUID and the Owner would bind the wrong Minecraft identity), and felis-config
// (the on-demand BACKUP Job runs in the minecraft namespace and mounts it to
@@ -238,7 +239,31 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
if buildNS == "" {
buildNS = platform.DefaultBuildNamespace
}
secretOutcomes := provisionSecretReplicas(ctx, cl, controlNS, cfg.K8s.Namespace, buildNS)
secretOutcomes := []systemServerOutcome{
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false),
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
// refresh=true: felis-config is the rendered config, not a credential. The
// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
// must update it when the control plane's render has moved on (a stale copy
// e.g. keeps an old database URL after a credential rotation).
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
"felis-config", "felis.toml", "config", "minecraft ns", true),
// The reaper's pre-reap warning emails authenticate with the same relay
// password felis-api uses; the reaper pod runs in the minecraft namespace,
// where a secretKeyRef resolves only against a local mirror. Skipped while
// the relay is not configured yet — the "configure email" screen refreshes
// both mirrors when it applies.
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
"felis-smtp", "password", "smtp", "minecraft ns", false),
// The build namespace needs the same token: the build Job's fetch
// initContainer reads the submission context from the internal face. Best
// effort — a deployment that only installs the control plane simply never
// builds a user submission.
ensureSecretReplica(ctx, cl, controlNS, buildNS,
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "felis-build ns", false),
}
outcomes := ensureSystemServers(ctx, cl, cfg.K8s.Namespace, cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, apiBaseURL, cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
outcomes = append(secretOutcomes, outcomes...)
fmt.Fprintln(out, "\nfelis setup: login/lobby system servers (always-on, reaper-exempt):")
@@ -320,9 +345,7 @@ func openConfiguredSetup(ctx context.Context, cfgPath string) (*configuredSetup,
if err != nil {
return nil, &setupOpenError{stage: "load config", err: err}
}
// Pending migrations are the preflight's to apply; a newer schema is a rolled-back
// binary, and nothing this console writes would match it.
drv, err := openStore(ctx, cfg.Database.URL, true)
drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil {
return nil, &setupOpenError{stage: "open database", err: err}
}
cmd/felis/status.go
-438
View File
@@ -1,438 +0,0 @@
package main
import (
"bufio"
"context"
"errors"
"flag"
"fmt"
"io"
"net"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"text/tabwriter"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/store"
"felis.lolicon.best/internal/watchdog"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// cmdStatus prints the platform at a glance: the release and the node, the
// control plane's workloads, the game proxy, every server with its players
// and newest world backup, the database backups and the off-site copy, the
// host's disks and memory, and what the watchdog has open. It changes
// nothing, and a part that is down (the cluster, PostgreSQL) reads as such
// while the rest still prints. felis doctor says what is wrong and where to
// look.
func cmdStatus(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("status", flag.ContinueOnError)
fs.SetOutput(stderr)
unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis status: run as root (sudo felis status): it reads root-only state under /etc/felis and /var/lib/felis")
return 1
}
env, err := hostStatusEnv(*unitDir)
if err != nil {
fmt.Fprintf(stderr, "felis status: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
printStatus(ctx, env, stdout)
return 0
}
// statusEnv is what one status report reads the host through.
type statusEnv struct {
cfg *config.Config
// w is how felis-watchdog.service runs the watchdog: where the backups,
// the off-site record and the proxy are.
w watchdogFlags
cl client.Client // nil while the API server is unreachable
clErr error
backups func(ctx context.Context) (map[string]time.Time, error)
run func(ctx context.Context, name string, args ...string) ([]byte, error)
unitDir string
meminfo string
host string
now time.Time
}
// hostStatusEnv reads this host: the watchdog's settings, the configuration
// they name, and the cluster.
func hostStatusEnv(unitDir string) (statusEnv, error) {
w, _, err := watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
if err != nil {
return statusEnv{}, err
}
cfg, err := config.Load(w.cfgPath)
if err != nil {
return statusEnv{}, err
}
cl, clErr := buildSystemServerClient()
if clErr != nil {
cl = nil
}
host, _ := os.Hostname()
return statusEnv{
cfg: cfg, w: w, cl: cl, clErr: clErr,
backups: func(ctx context.Context) (map[string]time.Time, error) {
return newestWorldBackups(ctx, cfg.Database.URL)
},
run: hostCommand, unitDir: unitDir, meminfo: "/proc/meminfo", host: host, now: time.Now(),
}, nil
}
func printStatus(ctx context.Context, env statusEnv, out io.Writer) {
fmt.Fprintf(out, "felis %s on %s at %s\n", resolvedVersion(), env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
if env.cl == nil {
fmt.Fprintf(out, "cluster: unreachable (%v)\n", env.clErr)
} else {
statusCluster(ctx, env, out)
}
statusProxy(ctx, env, out)
statusServers(ctx, env, out)
statusBackups(env, out)
statusHost(env, out)
statusWatchdog(ctx, env, out)
}
func statusCluster(ctx context.Context, env statusEnv, out io.Writer) {
var nodes corev1.NodeList
if err := env.cl.List(ctx, &nodes); err != nil {
fmt.Fprintf(out, "cluster: unreachable (%v)\n", err)
return
}
for _, n := range nodes.Items {
ready := "NotReady"
for _, c := range n.Status.Conditions {
if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue {
ready = "Ready"
}
}
info := n.Status.NodeInfo
fmt.Fprintf(out, "node: %s %s, k3s %s, %s, kernel %s\n", n.Name, ready, info.KubeletVersion, info.OSImage, info.KernelVersion)
}
ns := env.w.controlNS
var deps appsv1.DeploymentList
var pods corev1.PodList
err := env.cl.List(ctx, &deps, client.InNamespace(ns))
if err == nil {
err = env.cl.List(ctx, &pods, client.InNamespace(ns))
}
fmt.Fprintf(out, "\ncontrol plane (namespace %s):\n", ns)
if err != nil {
fmt.Fprintf(out, " cannot list it: %v\n", err)
return
}
sort.Slice(deps.Items, func(i, j int) bool { return deps.Items[i].Name < deps.Items[j].Name })
tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
for _, d := range deps.Items {
want := int32(1)
if d.Spec.Replicas != nil {
want = *d.Spec.Replicas
}
image := "-"
if cs := d.Spec.Template.Spec.Containers; len(cs) > 0 {
image = shortImage(cs[0].Image)
}
fmt.Fprintf(tw, " %s\t%d/%d ready\t%s\trestarts %d\n", d.Name, d.Status.ReadyReplicas, want, image, podRestarts(d.Spec.Selector, pods.Items))
}
tw.Flush()
}
// podRestarts adds up the container restarts of the pods selector picks (the
// API server refuses a Deployment whose selector is empty).
func podRestarts(selector *metav1.LabelSelector, pods []corev1.Pod) int32 {
sel, err := metav1.LabelSelectorAsSelector(selector)
if err != nil {
return 0
}
var n int32
for _, p := range pods {
if !sel.Matches(labels.Set(p.Labels)) {
continue
}
for _, cs := range p.Status.ContainerStatuses {
n += cs.RestartCount
}
}
return n
}
// shortImage is an image reference without its registry and repository path,
// and with its digest cut to 12 characters.
func shortImage(ref string) string {
if i := strings.LastIndex(ref, "/"); i >= 0 {
ref = ref[i+1:]
}
if name, digest, ok := strings.Cut(ref, "@sha256:"); ok && len(digest) > 12 {
ref = name + "@" + digest[:12]
}
return ref
}
func statusProxy(ctx context.Context, env statusEnv, out io.Writer) {
var parts []string
if _, err := os.Stat(filepath.Join(env.unitDir, "felis-velocity.service")); err == nil {
parts = append(parts, "felis-velocity "+unitActiveState(ctx, doctorEnv{run: env.run}, "felis-velocity.service"))
}
if env.w.proxyAddr != "" {
d := net.Dialer{Timeout: 3 * time.Second}
if conn, err := d.DialContext(ctx, "tcp", env.w.proxyAddr); err != nil {
parts = append(parts, fmt.Sprintf("%s refuses connections (%v)", env.w.proxyAddr, err))
} else {
conn.Close()
parts = append(parts, env.w.proxyAddr+" accepts connections")
}
}
if len(parts) == 0 {
parts = append(parts, "not on this host")
}
fmt.Fprintf(out, "\nproxy: %s\n", strings.Join(parts, ", "))
}
func statusServers(ctx context.Context, env statusEnv, out io.Writer) {
ns := env.cfg.K8s.Namespace
if ns == "" {
ns = platform.DefaultMinecraftNamespace
}
if env.cl == nil {
fmt.Fprintf(out, "\nservers (namespace %s): unknown while the cluster is unreachable\n", ns)
return
}
var list v1alpha1.MinecraftServerList
if err := env.cl.List(ctx, &list, client.InNamespace(ns)); err != nil {
fmt.Fprintf(out, "\nservers (namespace %s): cannot list them: %v\n", ns, err)
return
}
newest, backupErr := env.backups(ctx)
running, online := 0, int32(0)
for _, ms := range list.Items {
if ms.Status.Phase == v1alpha1.PhaseRunning {
running++
online += ms.Status.Players.Online
}
}
fmt.Fprintf(out, "\nservers (namespace %s): %d, %d running, %d players online\n", ns, len(list.Items), running, online)
if len(list.Items) == 0 {
return
}
sort.Slice(list.Items, func(i, j int) bool { return list.Items[i].Name < list.Items[j].Name })
tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
fmt.Fprintln(tw, " NAME\tROLE\tDESIRED\tPHASE\tPLAYERS\tNEWEST WORLD BACKUP")
for _, ms := range list.Items {
role := ms.Labels[v1alpha1.LabelSystemRole]
if role == "" {
role = "-"
}
phase := string(ms.Status.Phase)
if phase == "" {
phase = "-"
}
players := "-"
if ms.Status.Phase == v1alpha1.PhaseRunning {
players = fmt.Sprintf("%d/%d", ms.Status.Players.Online, ms.Status.Players.Max)
}
backup := "none"
switch at, ok := newest[ms.Name]; {
case backupErr != nil:
backup = "?"
case ok:
backup = dbbackup.Age(env.now.Sub(at)) + " ago"
}
fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%s\t%s\n", ms.Name, role, orDash(string(ms.Spec.DesiredState)), phase, players, backup)
}
tw.Flush()
if backupErr != nil {
fmt.Fprintf(out, " world backups unknown: %v\n", backupErr)
}
}
func orDash(s string) string {
if s == "" {
return "-"
}
return s
}
// newestWorldBackups is when each server's newest world backup that a restore
// can use was taken.
func newestWorldBackups(ctx context.Context, url string) (map[string]time.Time, error) {
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
drv, err := store.Open(ctx, url)
if err != nil {
return nil, err
}
defer drv.Close()
rows, err := drv.DB().QueryContext(ctx, `SELECT server_name, max(created_at) FROM world_backups
WHERE status = 'present' AND corrupt_at IS NULL GROUP BY server_name`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]time.Time{}
for rows.Next() {
var name string
var at time.Time
if err := rows.Scan(&name, &at); err != nil {
return nil, err
}
out[name] = at
}
return out, rows.Err()
}
func statusBackups(env statusEnv, out io.Writer) {
fmt.Fprintln(out, "\nbackups:")
switch bundles, err := dbbackup.List(env.w.backupDir); {
case env.w.backupDir == "":
fmt.Fprintln(out, " database: not checked (felis-watchdog.service names no -backup-dir)")
case err != nil:
fmt.Fprintf(out, " database: cannot read %s: %v\n", env.w.backupDir, err)
case len(bundles) == 0:
fmt.Fprintf(out, " database: none in %s\n", env.w.backupDir)
default:
fmt.Fprintf(out, " database: newest %s, %s ago; %d bundles in %s\n",
bundles[0].Name, dbbackup.Age(env.now.Sub(bundles[0].Created)), len(bundles), env.w.backupDir)
}
if !env.cfg.Offsite.Enabled() {
fmt.Fprintln(out, " off-site: not configured, every backup is on this machine only")
return
}
switch st, err := offsite.ReadStatus(env.w.offsiteStatus); {
case err != nil:
fmt.Fprintf(out, " off-site: %v\n", err)
case st == nil:
fmt.Fprintln(out, " off-site: never synced")
case st.LastSuccess.IsZero():
fmt.Fprintf(out, " off-site: never succeeded; last attempt %s ago: %s\n", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
default:
line := fmt.Sprintf(" off-site: last good sync %s ago to %s", dbbackup.Age(env.now.Sub(st.LastSuccess)), st.Bucket)
if st.LastAttempt.After(st.LastSuccess) { // a failed run records no success
line += fmt.Sprintf("; the last attempt, %s ago, failed: %s", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
}
fmt.Fprintln(out, line)
}
}
func statusHost(env statusEnv, out io.Writer) {
fmt.Fprintln(out, "\nhost:")
seen := map[uint64]bool{}
for _, p := range splitList(env.w.diskPaths) {
var st syscall.Stat_t
if err := syscall.Stat(p, &st); err != nil {
continue
}
dev := uint64(st.Dev) // int32 on darwin
if seen[dev] {
continue
}
seen[dev] = true
var fs syscall.Statfs_t
if err := syscall.Statfs(p, &fs); err != nil || fs.Blocks == 0 {
continue
}
bsize := uint64(fs.Bsize) // uint32 on darwin
total, avail := uint64(fs.Blocks)*bsize, uint64(fs.Bavail)*bsize
fmt.Fprintf(out, " disk %s: %s free of %s (%.0f%% free)\n", p, offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)), float64(fs.Bavail)/float64(fs.Blocks)*100)
}
if total, avail, ok := readMeminfo(env.meminfo); ok {
fmt.Fprintf(out, " memory: %s available of %s\n", offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)))
}
}
// readMeminfo reads MemTotal and MemAvailable, in bytes, from a /proc/meminfo
// style file.
func readMeminfo(path string) (total, avail uint64, ok bool) {
f, err := os.Open(path)
if err != nil {
return 0, 0, false
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
fields := strings.Fields(sc.Text())
if len(fields) < 2 {
continue
}
v, _ := strconv.ParseUint(fields[1], 10, 64) // the kernel writes numbers
switch fields[0] {
case "MemTotal:":
total = v * 1024
case "MemAvailable:":
avail = v * 1024
}
}
return total, avail, total > 0
}
func statusWatchdog(ctx context.Context, env statusEnv, out io.Writer) {
fmt.Fprintln(out, "\nwatchdog:")
if _, err := os.Stat(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
fmt.Fprintln(out, " not installed: nothing checks this host")
return
}
line := " timer " + unitActiveState(ctx, doctorEnv{run: env.run}, "felis-watchdog.timer")
show, _ := env.run(ctx, "systemctl", "show", "--timestamp=unix", "-p", "Result", "-p", "ExecMainExitTimestamp", "felis-watchdog.service")
props := map[string]string{}
for _, ln := range strings.Split(string(show), "\n") {
if k, v, ok := strings.Cut(strings.TrimSpace(ln), "="); ok {
props[k] = v
}
}
// ExecMainExitTimestamp is empty until the service has run.
if sec, _ := strconv.ParseInt(strings.TrimPrefix(props["ExecMainExitTimestamp"], "@"), 10, 64); sec > 0 {
line += fmt.Sprintf(", last run %s ago (%s)", dbbackup.Age(env.now.Sub(time.Unix(sec, 0))), orDash(props["Result"]))
}
fmt.Fprintln(out, line)
state, err := watchdog.LoadState(watchdog.NewestState(env.w.statePath, env.w.fallbackState))
if err != nil {
fmt.Fprintf(out, " alerts: unknown (%v)\n", err)
return
}
var open []string
for key, a := range state.Alerts {
if !a.ClearedAt.IsZero() {
continue
}
if a.Notified.IsZero() {
open = append(open, fmt.Sprintf("%s (%s, seen %s ago, not mailed yet)", key, a.Severity, dbbackup.Age(env.now.Sub(a.FirstSeen))))
} else {
open = append(open, fmt.Sprintf("%s (%s, mailed %s ago)", key, a.Severity, dbbackup.Age(env.now.Sub(a.Notified))))
}
}
if len(open) == 0 {
fmt.Fprintln(out, " alerts: none open")
return
}
sort.Strings(open)
fmt.Fprintf(out, " alerts: %d open (sudo felis doctor says where to look)\n", len(open))
for _, o := range open {
fmt.Fprintf(out, " %s\n", o)
}
}
cmd/felis/status_test.go
-490
View File
@@ -1,490 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/watchdog"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)
// statusCluster is a one-node install: felis-api with a restarted pod, three
// servers (one of them the lobby), and a pod of another app whose restarts
// are not felis-api's.
func statusClusterObjects() []client.Object {
replicas := int32(1)
apiLabels := map[string]string{"app": "felis-api"}
return []client.Object{
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{Name: "felis-1"},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}},
NodeInfo: corev1.NodeSystemInfo{KubeletVersion: "v1.36.4+k3s1", OSImage: "CentOS Stream 9", KernelVersion: "5.14.0-630.el9.aarch64"},
},
},
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{MatchLabels: apiLabels},
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{
Name: "api", Image: "registry.felis.svc:5000/felis/felis-api:v1.4.0@sha256:0123456789abcdef0123456789abcdef",
}}}},
},
Status: appsv1.DeploymentStatus{ReadyReplicas: 1},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "felis-api-7d9", Namespace: "felis", Labels: apiLabels},
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 2}}},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "other-1", Namespace: "felis", Labels: map[string]string{"app": "other"}},
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "x", RestartCount: 5}}},
},
&v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseRunning, Players: v1alpha1.PlayersStatus{Online: 2, Max: 20}},
},
&v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: "creative", Namespace: "minecraft"},
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped},
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped},
},
&v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: "lobby", Namespace: "minecraft", Labels: map[string]string{v1alpha1.LabelSystemRole: "lobby"}},
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStarting},
},
// Another namespace's server is not this install's.
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "elsewhere", Namespace: "other"}},
}
}
// statusTestEnv is a host with the cluster above, a database backup nine hours
// old, an off-site copy last good two hours ago, and one alert open.
func statusTestEnv(t *testing.T) statusEnv {
t.Helper()
now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
dir := t.TempDir()
var w watchdogFlags
w.controlNS = "felis"
w.backupDir = filepath.Join(dir, "db-backups")
w.offsiteStatus = filepath.Join(dir, "offsite-status.json")
w.statePath = filepath.Join(dir, "state.json")
w.fallbackState = filepath.Join(dir, "fallback.json")
w.diskPaths = "/nonexistent-felis-status-test"
unitDir := filepath.Join(dir, "systemd")
for _, d := range []string{w.backupDir, unitDir} {
if err := os.MkdirAll(d, 0o700); err != nil {
t.Fatal(err)
}
}
writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-9*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-33*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
if err := offsite.WriteStatus(w.offsiteStatus, offsite.Status{
LastAttempt: now.Add(-2 * time.Hour), LastSuccess: now.Add(-2 * time.Hour), Bucket: "felis-dr", Endpoint: "https://s3.example.com",
}); err != nil {
t.Fatal(err)
}
state := &watchdog.State{Alerts: map[string]*watchdog.Alert{
"db-backup-servers": {Finding: watchdog.Finding{Key: "db-backup-servers", Severity: watchdog.Warning}, FirstSeen: now.Add(-3 * time.Hour), Notified: now.Add(-90 * time.Minute)},
"proxy": {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: now.Add(-time.Hour), Notified: now.Add(-time.Hour), ClearedAt: now.Add(-10 * time.Minute)},
"disk//var": {Finding: watchdog.Finding{Key: "disk//var", Severity: watchdog.Critical}, FirstSeen: now.Add(-4 * time.Minute)},
}}
if err := watchdog.SaveState(w.statePath, state); err != nil {
t.Fatal(err)
}
meminfo := filepath.Join(dir, "meminfo")
writeTestFile(t, meminfo, "MemTotal: 8000000 kB\nMemFree: 100000 kB\nMemAvailable: 2000000 kB\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.timer"), "[Unit]\n", 0o644)
cfg := &config.Config{Offsite: config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-dr"}}
return statusEnv{
cfg: cfg, w: w, cl: fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).Build(),
backups: func(context.Context) (map[string]time.Time, error) {
return map[string]time.Time{"survival": now.Add(-3 * time.Hour)}, nil
},
run: func(_ context.Context, name string, args ...string) ([]byte, error) {
switch strings.Join(append([]string{name}, args...), " ") {
case "systemctl is-active felis-watchdog.timer":
return []byte("active\n"), nil
case "systemctl show --timestamp=unix -p Result -p ExecMainExitTimestamp felis-watchdog.service":
return []byte("Result=success\nExecMainExitTimestamp=@" + strconv.FormatInt(now.Add(-70*time.Second).Unix(), 10) + "\n"), nil
}
return nil, errors.New("unexpected")
},
unitDir: unitDir, meminfo: meminfo, host: "felis-test", now: now,
}
}
// lineFields is the words of the first line of out that starts with prefix
// once trimmed.
func lineFields(out, prefix string) []string {
for _, l := range strings.Split(out, "\n") {
if strings.HasPrefix(strings.TrimSpace(l), prefix) {
return strings.Fields(l)
}
}
return nil
}
func TestStatusReport(t *testing.T) {
env := statusTestEnv(t)
var out bytes.Buffer
printStatus(context.Background(), env, &out)
got := out.String()
for _, want := range []string{
"on felis-test at 2026-09-27 12:00 UTC\n",
"node: felis-1 Ready, k3s v1.36.4+k3s1, CentOS Stream 9, kernel 5.14.0-630.el9.aarch64\n",
"\ncontrol plane (namespace felis):\n",
"\nproxy: not on this host\n",
"\nservers (namespace minecraft): 3, 1 running, 2 players online\n",
" database: newest " + dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily) + ", 9h0m ago; 2 bundles in " + env.w.backupDir + "\n",
" off-site: last good sync 2h0m ago to felis-dr\n",
" memory: 1.9 GiB available of 7.6 GiB\n",
" timer active, last run 1m ago (success)\n",
" alerts: 2 open (sudo felis doctor says where to look)\n" +
" db-backup-servers (warning, mailed 1h30m ago)\n" +
" disk//var (critical, seen 4m ago, not mailed yet)\n",
} {
if !strings.Contains(got, want) {
t.Errorf("status lacks %q:\n%s", want, got)
}
}
for prefix, want := range map[string]string{
"felis-api": "felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
"NAME": "NAME ROLE DESIRED PHASE PLAYERS NEWEST WORLD BACKUP",
"creative": "creative - Stopped Stopped - none",
"lobby": "lobby lobby Running Starting - none",
"survival": "survival - Running Running 2/20 3h0m ago",
} {
if f := strings.Join(lineFields(got, prefix), " "); f != want {
t.Errorf("row %q = %q, want %q\n%s", prefix, f, want, got)
}
}
if strings.Contains(got, "elsewhere") || strings.Contains(got, "other-1") {
t.Errorf("status shows what is not this install's:\n%s", got)
}
}
// Whatever is down reads as down, and the rest of the report still prints.
func TestStatusWithPartsDown(t *testing.T) {
env := statusTestEnv(t)
env.cl, env.clErr = nil, errors.New("connection refused")
env.cfg = &config.Config{}
var out bytes.Buffer
printStatus(context.Background(), env, &out)
for _, want := range []string{
"cluster: unreachable (connection refused)\n",
"\nservers (namespace minecraft): unknown while the cluster is unreachable\n",
" off-site: not configured, every backup is on this machine only\n",
" timer active, last run",
} {
if !strings.Contains(out.String(), want) {
t.Errorf("status lacks %q:\n%s", want, out.String())
}
}
env = statusTestEnv(t)
env.backups = func(context.Context) (map[string]time.Time, error) { return nil, errors.New("postgres is down") }
out.Reset()
printStatus(context.Background(), env, &out)
if f := strings.Join(lineFields(out.String(), "survival"), " "); f != "survival - Running Running 2/20 ?" {
t.Errorf("survival with PostgreSQL down = %q", f)
}
if !strings.Contains(out.String(), " world backups unknown: postgres is down\n") {
t.Errorf("status does not say why the backups are unknown:\n%s", out.String())
}
}
func TestShortImage(t *testing.T) {
for in, want := range map[string]string{
"registry.felis.svc:5000/felis/felis-api:v1.4.0": "felis-api:v1.4.0",
"docker.io/library/postgres:17@sha256:0123456789abcdef0123": "postgres:17@0123456789ab",
"felis-operator@sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba98765432": "felis-operator@fedcba987654",
"busybox": "busybox",
} {
if got := shortImage(in); got != want {
t.Errorf("shortImage(%q) = %q, want %q", in, got, want)
}
}
}
// A node that is not ready, a Deployment without replicas or containers, and
// lists the API server refuses each read as such.
func TestStatusClusterEdges(t *testing.T) {
env := statusTestEnv(t)
env.cfg = &config.Config{K8s: config.K8sConfig{Namespace: "games"}}
env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}, Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionFalse}, {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionTrue}}}},
&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-bare", Namespace: "felis"}},
&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "felis"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 4}}}},
).Build()
var out bytes.Buffer
printStatus(context.Background(), env, &out)
got := out.String()
for _, want := range []string{
"node: felis-1 NotReady, k3s , , kernel \n",
"\nservers (namespace games): 0, 0 running, 0 players online\n\nbackups:\n",
} {
if !strings.Contains(got, want) {
t.Errorf("status lacks %q:\n%s", want, got)
}
}
if f := strings.Join(lineFields(got, "felis-bare"), " "); f != "felis-bare 0/1 ready - restarts 0" {
t.Errorf("row felis-bare = %q, want its one replica wanted, no image, and no pod of its own:\n%s", f, got)
}
failing := func(what client.ObjectList) {
env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).
WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
if fmt.Sprintf("%T", list) == fmt.Sprintf("%T", what) {
return errors.New("forbidden")
}
return c.List(ctx, list, opts...)
}}).Build()
out.Reset()
printStatus(context.Background(), env, &out)
}
for _, tc := range []struct {
list client.ObjectList
want string
}{
{&corev1.NodeList{}, "cluster: unreachable (forbidden)\n\nproxy:"},
{&appsv1.DeploymentList{}, "\ncontrol plane (namespace felis):\n cannot list it: forbidden\n\nproxy:"},
{&corev1.PodList{}, "\ncontrol plane (namespace felis):\n cannot list it: forbidden\n\nproxy:"},
{&v1alpha1.MinecraftServerList{}, "\nservers (namespace games): cannot list them: forbidden\n\nbackups:"},
} {
failing(tc.list)
if !strings.Contains(out.String(), tc.want) {
t.Errorf("listing %T refused: status lacks %q:\n%s", tc.list, tc.want, out.String())
}
}
}
func TestStatusProxy(t *testing.T) {
env := statusTestEnv(t)
writeTestFile(t, filepath.Join(env.unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active"})
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
env.w.proxyAddr = ln.Addr().String()
var out bytes.Buffer
statusProxy(context.Background(), env, &out)
if want := "\nproxy: felis-velocity active, " + env.w.proxyAddr + " accepts connections\n"; out.String() != want {
t.Errorf("proxy listening: %q, want %q", out.String(), want)
}
ln.Close()
out.Reset()
statusProxy(context.Background(), env, &out)
if want := "\nproxy: felis-velocity active, " + env.w.proxyAddr + " refuses connections (dial tcp " + env.w.proxyAddr + ": "; !strings.HasPrefix(out.String(), want) {
t.Errorf("proxy gone: %q, want it to start %q", out.String(), want)
}
}
func TestStatusBackups(t *testing.T) {
env := statusTestEnv(t)
status := func() string {
var out bytes.Buffer
statusBackups(env, &out)
return out.String()
}
for _, tc := range []struct {
what, dir, want string
}{
{"no -backup-dir", "", " database: not checked (felis-watchdog.service names no -backup-dir)\n"},
{"an empty directory", t.TempDir(), " database: none in %s\n"},
{"a file where the directory should be", filepath.Join(env.w.backupDir, dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily)), " database: cannot read %s: "},
} {
env.w.backupDir = tc.dir
want := tc.want
if strings.Contains(want, "%s") {
want = fmt.Sprintf(want, tc.dir)
}
if got := status(); !strings.Contains(got, want) {
t.Errorf("%s: %q, want %q", tc.what, got, want)
}
}
for _, tc := range []struct {
what string
status *offsite.Status
raw string
want string
}{
{"never synced", nil, "", " off-site: never synced\n"},
{"never succeeded", &offsite.Status{LastAttempt: env.now.Add(-time.Hour), LastError: "403 Forbidden"}, "", " off-site: never succeeded; last attempt 1h0m ago: 403 Forbidden\n"},
{"the last attempt failed", &offsite.Status{LastAttempt: env.now.Add(-30 * time.Minute), LastSuccess: env.now.Add(-26 * time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
" off-site: last good sync 26h0m ago to felis-dr; the last attempt, 30m ago, failed: timeout\n"},
{"an error an earlier attempt left", &offsite.Status{LastAttempt: env.now.Add(-2 * time.Hour), LastSuccess: env.now.Add(-time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
" off-site: last good sync 1h0m ago to felis-dr\n"},
{"an unreadable record", nil, "{", " off-site: unexpected end of JSON input\n"},
} {
os.Remove(env.w.offsiteStatus)
switch {
case tc.status != nil:
if err := offsite.WriteStatus(env.w.offsiteStatus, *tc.status); err != nil {
t.Fatal(err)
}
case tc.raw != "":
writeTestFile(t, env.w.offsiteStatus, tc.raw, 0o600)
}
if got := status(); !strings.HasSuffix(got, tc.want) {
t.Errorf("%s: %q, want it to end %q", tc.what, got, tc.want)
}
}
}
func TestStatusHost(t *testing.T) {
env := statusTestEnv(t)
dir := t.TempDir()
env.w.diskPaths = "/nonexistent-felis-status-test," + dir + "," + dir
env.meminfo = filepath.Join(dir, "meminfo")
var out bytes.Buffer
statusHost(env, &out)
lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
if len(lines) != 3 || lines[0] != "" || lines[1] != "host:" || !strings.HasPrefix(lines[2], " disk "+dir+": ") || !strings.HasSuffix(lines[2], "% free)") {
t.Errorf("host: %q, want one line for %s (a path on a disk already shown, and one that is not there, print none) and no memory line", lines, dir)
}
writeTestFile(t, env.meminfo, "MemTotal: 4096 kB\ngarbage\nMemAvailable: 1024 kB\n", 0o644)
if total, avail, ok := readMeminfo(env.meminfo); !ok || total != 4096*1024 || avail != 1024*1024 {
t.Errorf("readMeminfo = %d, %d, %v", total, avail, ok)
}
writeTestFile(t, env.meminfo, "MemAvailable: 1024 kB\n", 0o644)
if _, _, ok := readMeminfo(env.meminfo); ok {
t.Error("readMeminfo without MemTotal: ok")
}
}
func TestStatusWatchdog(t *testing.T) {
env := statusTestEnv(t)
status := func() string {
var out bytes.Buffer
statusWatchdog(context.Background(), env, &out)
return out.String()
}
run := env.run
env.run = func(ctx context.Context, name string, args ...string) ([]byte, error) {
if len(args) > 0 && args[0] == "show" {
return []byte("Result=success\nExecMainExitTimestamp=\n"), nil
}
return run(ctx, name, args...)
}
if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: map[string]*watchdog.Alert{
"proxy": {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: env.now.Add(-time.Hour), ClearedAt: env.now.Add(-time.Minute)},
}}); err != nil {
t.Fatal(err)
}
if got, want := status(), "\nwatchdog:\n timer active\n alerts: none open\n"; got != want {
t.Errorf("a watchdog that has not run and has nothing open: %q, want %q", got, want)
}
writeTestFile(t, env.w.statePath, "{", 0o600)
if got := status(); !strings.Contains(got, "\n alerts: unknown (") {
t.Errorf("an unreadable state: %q", got)
}
if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
t.Fatal(err)
}
if got, want := status(), "\nwatchdog:\n not installed: nothing checks this host\n"; got != want {
t.Errorf("no watchdog timer: %q, want %q", got, want)
}
}
// Rows print by name in whatever order the API server lists them, a server
// the operator has not reconciled yet reads as dashes, and open alerts print
// by key in whatever order the state's map yields them.
func TestStatusOrder(t *testing.T) {
env := statusTestEnv(t)
objs := append(statusClusterObjects(),
&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-operator", Namespace: "felis"}},
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "fresh", Namespace: "minecraft"}})
env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
// The fake lists by name; the other way round, then.
if err := c.List(ctx, list, opts...); err != nil {
return err
}
items, err := meta.ExtractList(list)
if err != nil {
return err
}
slices.Reverse(items)
return meta.SetList(list, items)
}}).Build()
var out bytes.Buffer
printStatus(context.Background(), env, &out)
var rows []string
for _, l := range strings.Split(out.String(), "\n") {
if f := strings.Fields(l); len(f) > 0 && (strings.HasPrefix(f[0], "felis-") || slices.Contains([]string{"creative", "fresh", "lobby", "survival"}, f[0])) {
rows = append(rows, strings.Join(f, " "))
}
}
want := []string{
"felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
"felis-operator 0/1 ready - restarts 0",
"creative - Stopped Stopped - none",
"fresh - - - - none",
"lobby lobby Running Starting - none",
"survival - Running Running 2/20 3h0m ago",
}
if !slices.Equal(rows, want) {
t.Errorf("rows %q, want %q:\n%s", rows, want, out.String())
}
alerts := map[string]*watchdog.Alert{}
for i, key := range []string{"a", "b", "c", "d"} {
alerts[key] = &watchdog.Alert{Finding: watchdog.Finding{Key: key, Severity: watchdog.Warning}, FirstSeen: env.now.Add(-time.Duration(i+1) * time.Minute)}
}
if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: alerts}); err != nil {
t.Fatal(err)
}
wantAlerts := " alerts: 4 open (sudo felis doctor says where to look)\n" +
" a (warning, seen 1m ago, not mailed yet)\n b (warning, seen 2m ago, not mailed yet)\n" +
" c (warning, seen 3m ago, not mailed yet)\n d (warning, seen 4m ago, not mailed yet)\n"
// A map starts its walk at random: fifty walks all in order by chance
// is out of the question.
for range 50 {
out.Reset()
statusWatchdog(context.Background(), env, &out)
if !strings.HasSuffix(out.String(), wantAlerts) {
t.Fatalf("alerts %q, want them to end %q", out.String(), wantAlerts)
}
}
}
// A selector the API server would have refused picks no pods.
func TestPodRestartsBadSelector(t *testing.T) {
pods := []corev1.Pod{{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 3}}}}}
if n := podRestarts(&metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "app", Operator: "Near"}}}, pods); n != 0 {
t.Errorf("podRestarts with a bad selector = %d, want 0", n)
}
if n := podRestarts(&metav1.LabelSelector{}, pods); n != 3 {
t.Errorf("podRestarts with a selector that picks all = %d, want 3", n)
}
}
cmd/felis/supportbundle.go
-862
View File
@@ -1,862 +0,0 @@
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"errors"
"flag"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"text/tabwriter"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/watchdog"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"
)
// supportBundleDir is where felis support-bundle writes unless told otherwise.
const supportBundleDir = "/var/lib/felis/support"
// redacted stands in for every value the bundle leaves out.
const redacted = "<redacted>"
// minScrubLen is the shortest known secret value the bundle searches for: a
// shorter one would blank ordinary words, and every secret the installer
// generates is far longer.
const minScrubLen = 8
// hostSecretSources are the files on a Felis host that hold secrets; the
// bundle reads them only to take each value out of what it collects.
var hostSecretSources = secretSources{
envFiles: []string{"/etc/felis/secrets.env", defaultOffsiteEnvFile},
valueFiles: []string{hostSMTPPasswordPath, hostUploadsS3AccessKeyPath, hostUploadsS3SecretKeyPath, defaultHeartbeatFile, "/opt/felis/velocity/forwarding.secret"},
propsFiles: []string{"/opt/felis/velocity/plugins/felis-link/felis-link.properties"},
tokenFiles: []string{"/var/lib/rancher/k3s/server/token", "/var/lib/rancher/k3s/server/agent-token"},
}
// cmdSupportBundle collects what someone helping with this host needs into
// one tar.gz: felis status and felis doctor, the logs of the control plane,
// the builds and the systemd units, the cluster's workloads and events, and a
// summary of the configuration. It never collects a Secret, a ConfigMap, the
// contents of a configuration file, the database or a world, and takes every
// value of the host's secret files out of what it does collect. Game server
// logs, which carry player names, addresses and chat, only with -server-logs.
func cmdSupportBundle(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("support-bundle", flag.ContinueOnError)
fs.SetOutput(stderr)
outDir := fs.String("o", supportBundleDir, "directory to write the bundle to (created mode 0700 when missing)")
logLines := fs.Int64("log-lines", 2000, "lines kept from the end of each pod log and each unit's journal")
since := fs.Duration("since", 48*time.Hour, "how far back each unit's journal is read")
serverLogs := fs.Bool("server-logs", false, "also collect the game servers' own logs, which carry player names, IP addresses and chat")
unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if os.Geteuid() != 0 {
fmt.Fprintln(stderr, "felis support-bundle: run as root (sudo felis support-bundle): it reads root-only logs and state")
return 1
}
b := hostSupportBundle(*unitDir, *logLines, *since, *serverLogs)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
path, err := b.write(ctx, *outDir)
if err != nil {
fmt.Fprintf(stderr, "felis support-bundle: %v\n", err)
return 1
}
size := int64(0)
if st, err := os.Stat(path); err == nil {
size = st.Size()
}
fmt.Fprintf(stdout, "wrote %s (%s, mode 0600)\n", path, humanSize(size))
fmt.Fprintln(stdout, "MANIFEST.txt inside says what it holds and what was taken out. Read it through before you send it anywhere:")
fmt.Fprintln(stdout, "redaction finds this host's known secrets and the common ways a secret is logged, and a secret logged another way stays in.")
if !*serverLogs {
fmt.Fprintln(stdout, "Game server logs are left out; -server-logs adds them (player names, IP addresses, chat).")
}
return 0
}
func humanSize(n int64) string {
switch {
case n >= 1<<20:
return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
case n >= 1<<10:
return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10))
}
return fmt.Sprintf("%d B", n)
}
// shortDuration is d as Duration.String writes it, less the zero minutes and
// seconds after whole hours or minutes: 48h, and 90m as 1h30m.
func shortDuration(d time.Duration) string {
s := d.String()
if strings.HasSuffix(s, "m0s") {
s = strings.TrimSuffix(s, "0s")
}
if strings.HasSuffix(s, "h0m") {
s = strings.TrimSuffix(s, "0m")
}
return s
}
// supportBundle is one collection and what it reads the host through.
type supportBundle struct {
host string
now time.Time
unitDir string
// w is how felis-watchdog.service runs the watchdog, wErr why it could
// not be read (w then holds the defaults).
w watchdogFlags
wErr error
cfg *config.Config // nil when cfgErr
cfgErr error
cl client.Client // nil when clErr
clErr error
logs func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error)
run func(ctx context.Context, name string, args ...string) ([]byte, error)
backups func(ctx context.Context) (map[string]time.Time, error)
doctor func(ctx context.Context, out io.Writer)
secrets secretSources
logLines int64
since time.Duration
serverLogs bool
// Host files read whole, and the directory whose listing is kept.
meminfo, osRelease, procVersion, stateDir string
}
func hostSupportBundle(unitDir string, logLines int64, since time.Duration, serverLogs bool) *supportBundle {
host, _ := os.Hostname()
b := &supportBundle{
host: host, now: time.Now(), unitDir: unitDir, run: hostCommand, secrets: hostSecretSources,
logLines: logLines, since: since, serverLogs: serverLogs,
meminfo: "/proc/meminfo", osRelease: "/etc/os-release", procVersion: "/proc/version", stateDir: "/etc/felis",
}
var found bool
b.w, found, b.wErr = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
if b.wErr == nil && !found {
b.wErr = fmt.Errorf("%s is not installed; read the watchdog's defaults", filepath.Join(unitDir, "felis-watchdog.service"))
}
if b.cfg, b.cfgErr = config.Load(b.w.cfgPath); b.cfgErr == nil {
cfg := b.cfg
b.backups = func(ctx context.Context) (map[string]time.Time, error) {
return newestWorldBackups(ctx, cfg.Database.URL)
}
} else {
b.cfg = nil
}
if b.cl, b.clErr = buildSystemServerClient(); b.clErr != nil {
b.cl = nil
} else if rc, err := hostRESTConfig(); err != nil {
b.clErr = err
b.cl = nil
} else if cs, err := kubernetes.NewForConfig(rc); err != nil {
b.clErr = err
b.cl = nil
} else {
limit := int64(8 << 20)
b.logs = func(ctx context.Context, ns, pod, container string, previous bool) ([]byte, error) {
return cs.CoreV1().Pods(ns).GetLogs(pod, &corev1.PodLogOptions{
Container: container, Previous: previous, Timestamps: true, TailLines: &logLines, LimitBytes: &limit,
}).DoRaw(ctx)
}
}
b.doctor = func(ctx context.Context, out io.Writer) {
runDoctor(ctx, doctorEnv{unitDir: unitDir, run: hostCommand, now: b.now, host: host}, out)
}
return b
}
// bundleWriter streams scrubbed files into the archive and keeps what went
// wrong while collecting, for MANIFEST.txt.
type bundleWriter struct {
tw *tar.Writer
prefix string
now time.Time
scrub *scrubber
errs []string
}
// logText adds a log, or a report that quotes errors: known secrets and
// anything logged as one come out.
func (bw *bundleWriter) logText(name string, data []byte) error {
return bw.add(name, bw.scrub.text(data))
}
// plain adds a file whose secrets were already taken out by structure (the
// cluster's objects, the configuration summary) or that names none (the
// release, disk use, addresses): known secret values still come out, and
// nothing else is rewritten.
func (bw *bundleWriter) plain(name string, data []byte) error {
return bw.add(name, bw.scrub.values(data))
}
func (bw *bundleWriter) add(name string, data []byte) error {
hdr := &tar.Header{Name: path.Join(bw.prefix, name), Mode: 0o600, Size: int64(len(data)), ModTime: bw.now, Typeflag: tar.TypeReg}
if err := bw.tw.WriteHeader(hdr); err != nil {
return err
}
_, err := bw.tw.Write(data)
return err
}
func (bw *bundleWriter) failed(what string, err error) {
bw.errs = append(bw.errs, string(bw.scrub.text([]byte(fmt.Sprintf("%s: %v", what, err)))))
}
// write collects the bundle into dir and returns its path.
func (b *supportBundle) write(ctx context.Context, dir string) (string, error) {
if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
}
stamp := b.now.UTC().Format("20060102T150405Z")
base := fmt.Sprintf("felis-support-%s-%s", safeName(b.host), stamp)
final := filepath.Join(dir, base+".tar.gz")
f, err := os.CreateTemp(dir, "."+base+".*.partial") // mode 0600
if err != nil {
return "", err
}
keep := false
defer func() {
if !keep {
f.Close()
os.Remove(f.Name())
}
}()
gz := gzip.NewWriter(f)
bw := &bundleWriter{tw: tar.NewWriter(gz), prefix: base, now: b.now, scrub: b.scrubber()}
if err := b.collect(ctx, bw); err != nil {
return "", err
}
if err := bw.tw.Close(); err != nil {
return "", err
}
if err := gz.Close(); err != nil {
return "", err
}
if err := f.Sync(); err != nil {
return "", err
}
if err := f.Close(); err != nil {
return "", err
}
if err := os.Rename(f.Name(), final); err != nil {
return "", err
}
keep = true
return final, nil
}
// safeName keeps a host name usable in a file name.
func safeName(s string) string {
s = strings.Map(func(r rune) rune {
if r == '-' || r == '.' || r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
return r
}
return '_'
}, s)
if s == "" {
return "host"
}
return s
}
func (b *supportBundle) collect(ctx context.Context, bw *bundleWriter) error {
var buf bytes.Buffer
cmdVersion(nil, &buf, io.Discard)
for _, p := range []string{b.osRelease, b.procVersion} {
if raw, err := os.ReadFile(p); err == nil {
fmt.Fprintf(&buf, "\n# %s\n%s", p, raw)
}
}
if err := bw.plain("version.txt", buf.Bytes()); err != nil {
return err
}
buf.Reset()
switch {
case b.cfgErr != nil:
fmt.Fprintf(&buf, "felis status: the configuration did not load: %v\n", b.cfgErr)
default:
printStatus(ctx, statusEnv{
cfg: b.cfg, w: b.w, cl: b.cl, clErr: b.clErr, backups: b.backups, run: b.run,
unitDir: b.unitDir, meminfo: b.meminfo, host: b.host, now: b.now,
}, &buf)
}
if err := bw.logText("status.txt", buf.Bytes()); err != nil {
return err
}
buf.Reset()
b.doctor(ctx, &buf)
if err := bw.logText("doctor.txt", buf.Bytes()); err != nil {
return err
}
if b.cfg != nil {
if err := bw.plain("config.txt", configSummary(b.cfg, b.w.cfgPath)); err != nil {
return err
}
}
if err := b.collectHost(ctx, bw); err != nil {
return err
}
if err := b.collectJournal(ctx, bw); err != nil {
return err
}
if b.cl == nil {
bw.failed("cluster", b.clErr)
} else if err := b.collectCluster(ctx, bw); err != nil {
return err
}
return bw.add("MANIFEST.txt", b.manifest(bw))
}
func (b *supportBundle) collectHost(ctx context.Context, bw *bundleWriter) error {
commands := []struct {
name string
argv []string
}{
{"host/systemd-units.txt", []string{"systemctl", "list-units", "--all", "--no-pager", "--plain", "felis-*", "k3s.service"}},
{"host/systemd-timers.txt", []string{"systemctl", "list-timers", "--all", "--no-pager", "felis-*"}},
{"host/df.txt", []string{"df", "-h"}},
{"host/addresses.txt", []string{"ip", "-brief", "address"}},
}
for _, c := range commands {
out, err := b.run(ctx, c.argv[0], c.argv[1:]...)
if err != nil && len(out) == 0 {
bw.failed(strings.Join(c.argv, " "), err)
continue
}
if err := bw.plain(c.name, out); err != nil {
return err
}
}
if raw, err := os.ReadFile(b.meminfo); err == nil {
if err := bw.plain("host/meminfo.txt", raw); err != nil {
return err
}
}
listing, err := dirListing(b.stateDir)
if err != nil {
bw.failed("list "+b.stateDir, err)
return nil
}
return bw.plain("host/etc-felis.txt", listing)
}
// dirListing names every file under dir with its mode, size and time, and
// holds nothing of what is in them.
func dirListing(dir string) ([]byte, error) {
var buf bytes.Buffer
tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
fmt.Fprintf(tw, "# %s: names, modes, sizes and times only; no contents\n", dir)
err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
info, err := d.Info()
if err != nil {
return err
}
fmt.Fprintf(tw, "%s\t%d\t%s\t%s\n", info.Mode(), info.Size(), info.ModTime().UTC().Format(time.RFC3339), p)
return nil
})
tw.Flush()
return buf.Bytes(), err
}
func (b *supportBundle) collectJournal(ctx context.Context, bw *bundleWriter) error {
units, _ := filepath.Glob(filepath.Join(b.unitDir, "felis-*.service"))
if _, err := os.Stat(filepath.Join(b.unitDir, "k3s.service")); err == nil {
units = append(units, filepath.Join(b.unitDir, "k3s.service"))
}
since := "@" + strconv.FormatInt(b.now.Add(-b.since).Unix(), 10)
for _, u := range units {
unit := filepath.Base(u)
out, err := b.run(ctx, "journalctl", "-u", unit, "--since", since, "-n", strconv.FormatInt(b.logLines, 10), "--no-pager", "-o", "short-iso")
if err != nil && len(out) == 0 {
bw.failed("journalctl -u "+unit, err)
continue
}
if err := bw.logText("journal/"+strings.TrimSuffix(unit, ".service")+".log", out); err != nil {
return err
}
}
return nil
}
// bundleNamespaces are the namespaces whose objects and logs the bundle
// collects: the control plane, the builds and the game servers.
func (b *supportBundle) bundleNamespaces() (control, build, minecraft string) {
control, build, minecraft = b.w.controlNS, platform.DefaultBuildNamespace, platform.DefaultMinecraftNamespace
if b.cfg != nil {
if b.cfg.Registry.BuildNamespace != "" {
build = b.cfg.Registry.BuildNamespace
}
if b.cfg.K8s.Namespace != "" {
minecraft = b.cfg.K8s.Namespace
}
}
return control, build, minecraft
}
func (b *supportBundle) collectCluster(ctx context.Context, bw *bundleWriter) error {
control, build, minecraft := b.bundleNamespaces()
dump := func(name string, list client.ObjectList, opts ...client.ListOption) error {
if err := b.cl.List(ctx, list, opts...); err != nil {
bw.failed("list "+name, err)
return nil
}
redactList(list)
out, err := yaml.Marshal(list)
if err != nil {
bw.failed("encode "+name, err)
return nil
}
return bw.plain(name, out)
}
if err := dump("cluster/nodes.yaml", &corev1.NodeList{}); err != nil {
return err
}
if err := dump("cluster/persistentvolumes.yaml", &corev1.PersistentVolumeList{}); err != nil {
return err
}
if err := dump("cluster/minecraftservers.yaml", &v1alpha1.MinecraftServerList{}, client.InNamespace(minecraft)); err != nil {
return err
}
for _, ns := range []string{control, build, minecraft} {
for _, k := range []struct {
name string
list client.ObjectList
}{
{"pods", &corev1.PodList{}},
{"deployments", &appsv1.DeploymentList{}},
{"statefulsets", &appsv1.StatefulSetList{}},
{"jobs", &batchv1.JobList{}},
{"cronjobs", &batchv1.CronJobList{}},
{"services", &corev1.ServiceList{}},
{"persistentvolumeclaims", &corev1.PersistentVolumeClaimList{}},
{"networkpolicies", &networkingv1.NetworkPolicyList{}},
{"events", &corev1.EventList{}},
} {
if err := dump("cluster/"+ns+"/"+k.name+".yaml", k.list, client.InNamespace(ns)); err != nil {
return err
}
}
}
var all corev1.PodList
if err := b.cl.List(ctx, &all); err != nil {
bw.failed("list every pod", err)
} else if err := bw.plain("cluster/pods-all-namespaces.txt", podTable(all.Items, b.now)); err != nil {
return err
}
for _, ns := range []string{control, build, minecraft} {
var pods corev1.PodList
if err := b.cl.List(ctx, &pods, client.InNamespace(ns)); err != nil {
continue // already recorded by the dump above
}
for _, p := range pods.Items {
if err := b.collectPodLogs(ctx, bw, p, ns == minecraft && !b.serverLogs); err != nil {
return err
}
}
}
return nil
}
// collectPodLogs keeps the tail of each container's log, and of its previous
// run when it restarted. initOnly keeps only the init containers: a game
// server's own log carries player names, addresses and chat.
func (b *supportBundle) collectPodLogs(ctx context.Context, bw *bundleWriter, p corev1.Pod, initOnly bool) error {
type ctr struct {
name string
restarts int32
}
var ctrs []ctr
restarts := map[string]int32{}
for _, cs := range append(append([]corev1.ContainerStatus(nil), p.Status.InitContainerStatuses...), p.Status.ContainerStatuses...) {
restarts[cs.Name] = cs.RestartCount
}
for _, c := range p.Spec.InitContainers {
ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
}
if !initOnly {
for _, c := range p.Spec.Containers {
ctrs = append(ctrs, ctr{c.Name, restarts[c.Name]})
}
}
for _, c := range ctrs {
for _, previous := range []bool{false, true} {
if previous && c.restarts == 0 {
continue
}
name := fmt.Sprintf("logs/%s/%s/%s.log", p.Namespace, p.Name, c.name)
if previous {
name = fmt.Sprintf("logs/%s/%s/%s.previous.log", p.Namespace, p.Name, c.name)
}
out, err := b.logs(ctx, p.Namespace, p.Name, c.name, previous)
if err != nil {
bw.failed(name, err)
continue
}
if err := bw.logText(name, out); err != nil {
return err
}
}
}
return nil
}
// podTable is every pod on the node, one line each.
func podTable(pods []corev1.Pod, now time.Time) []byte {
sort.Slice(pods, func(i, j int) bool {
if pods[i].Namespace != pods[j].Namespace {
return pods[i].Namespace < pods[j].Namespace
}
return pods[i].Name < pods[j].Name
})
var buf bytes.Buffer
tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0)
fmt.Fprintln(tw, "NAMESPACE\tNAME\tPHASE\tREADY\tRESTARTS\tAGE")
for _, p := range pods {
var ready, restarts int32
for _, cs := range p.Status.ContainerStatuses {
if cs.Ready {
ready++
}
restarts += cs.RestartCount
}
age := "-"
if !p.CreationTimestamp.IsZero() {
age = now.Sub(p.CreationTimestamp.Time).Round(time.Minute).String()
}
fmt.Fprintf(tw, "%s\t%s\t%s\t%d/%d\t%d\t%s\n", p.Namespace, p.Name, p.Status.Phase, ready, len(p.Spec.Containers), restarts, age)
}
tw.Flush()
return buf.Bytes()
}
// redactList takes out of every object what the bundle must not carry: the
// literal env values of pod specs and of MinecraftServers (valueFrom
// references stay, naming the Secret without its contents), the
// last-applied-configuration annotation that repeats them, and managedFields.
func redactList(list client.ObjectList) {
items, err := meta.ExtractList(list)
if err != nil {
return
}
for _, it := range items {
if acc, err := meta.Accessor(it); err == nil {
acc.SetManagedFields(nil)
if ann := acc.GetAnnotations(); ann != nil {
delete(ann, corev1.LastAppliedConfigAnnotation)
acc.SetAnnotations(ann)
}
}
switch o := it.(type) {
case *corev1.Pod:
redactPodSpec(&o.Spec)
case *appsv1.Deployment:
redactPodSpec(&o.Spec.Template.Spec)
case *appsv1.StatefulSet:
redactPodSpec(&o.Spec.Template.Spec)
case *batchv1.Job:
redactPodSpec(&o.Spec.Template.Spec)
case *batchv1.CronJob:
redactPodSpec(&o.Spec.JobTemplate.Spec.Template.Spec)
case *v1alpha1.MinecraftServer:
for i := range o.Spec.Env {
if o.Spec.Env[i].Value != "" {
o.Spec.Env[i].Value = redacted
}
}
}
}
}
func redactPodSpec(s *corev1.PodSpec) {
blank := func(cs []corev1.Container) {
for i := range cs {
for j := range cs[i].Env {
if cs[i].Env[j].Value != "" {
cs[i].Env[j].Value = redacted
}
}
}
}
blank(s.InitContainers)
blank(s.Containers)
for i := range s.EphemeralContainers {
for j := range s.EphemeralContainers[i].Env {
if s.EphemeralContainers[i].Env[j].Value != "" {
s.EphemeralContainers[i].Env[j].Value = redacted
}
}
}
}
// configSummary is felis.toml without a single credential: the hostnames,
// namespaces and which features are on. Fields are picked one by one, so a
// field added later stays out until someone decides it is safe.
func configSummary(c *config.Config, path string) []byte {
var buf bytes.Buffer
line := func(k string, v any) { fmt.Fprintf(&buf, "%-34s %v\n", k, v) }
fmt.Fprintf(&buf, "# a summary of %s; no password, key or token is in it\n", path)
line("server.root_domain", c.Server.RootDomain)
line("server.listen", c.Server.Listen)
db := "(unparsable)"
if u, err := url.Parse(c.Database.URL); err == nil {
db = u.Scheme + "://" + u.User.Username() + "@" + u.Host + u.Path
}
line("database.url (no password)", db)
line("database.deployment", c.Database.Deployment)
line("velocity.public_ip", c.Velocity.PublicIP)
line("velocity.game_port", c.Velocity.GamePort)
line("velocity.login_image", c.Velocity.LoginImage)
line("velocity.lobby_image", c.Velocity.LobbyImage)
line("auth.panel_hostname", c.Auth.PanelHostname)
line("auth.admin_hostname", c.Auth.AdminHostname)
line("auth.client_ip_header", c.Auth.ClientIPHeader)
line("k8s.namespace", c.K8s.Namespace)
line("k8s.egress_mode", c.K8s.EgressMode)
line("k8s.metallb_pool", c.K8s.MetalLBPool)
line("registry.url", c.Registry.URL)
line("registry.build_namespace", c.Registry.BuildNamespace)
line("registry.trivy_db_repository", c.Registry.TrivyDBRepository)
line("registry.build_user_namespaces", c.Registry.BuildUserNamespaces)
line("registry.build_runtime_class", c.Registry.BuildRuntimeClass)
line("registry.max_concurrent_builds", c.Registry.MaxConcurrentBuilds)
line("registry.user_uploads_context", c.Registry.UserUploadsContext)
line("archive.store", c.Archive.Store)
line("archive.local_path", c.Archive.LocalPath)
line("archive.retention", c.Archive.Retention)
line("archive.scheduled_every", c.Archive.ScheduledEvery)
line("archive.scheduled_keep", c.Archive.ScheduledKeep)
line("offsite (configured)", c.Offsite.Enabled())
if c.Offsite.Enabled() {
line("offsite.endpoint", c.Offsite.Endpoint)
line("offsite.bucket", c.Offsite.Bucket)
line("offsite.prefix", c.Offsite.Prefix)
}
line("smtp.host", c.SMTP.Host)
if c.SMTP.Host != "" {
line("smtp.port", c.SMTP.Port)
line("smtp.require_tls", c.SMTP.TLSRequired())
line("smtp.max_per_hour", c.SMTP.MaxPerHour)
}
for i, s := range c.AuthSources {
line(fmt.Sprintf("auth_source[%d]", i), s.Tag+" "+s.Prefix+" "+s.URL)
}
return buf.Bytes()
}
func (b *supportBundle) manifest(bw *bundleWriter) []byte {
control, build, minecraft := b.bundleNamespaces()
var buf bytes.Buffer
fmt.Fprintf(&buf, "Felis support bundle\nhost %s, collected %s, felis %s\n\n", b.host, b.now.UTC().Format(time.RFC3339), resolvedVersion())
fmt.Fprintf(&buf, `What it holds:
status.txt, doctor.txt felis status and felis doctor at collection time
version.txt the release, the OS and the kernel
config.txt a summary of felis.toml: hostnames, namespaces, which features are on
host/ systemd units and timers, disk use, memory, addresses, and the names,
modes, sizes and times of the files under %s (not what is in them)
journal/ up to %d lines per Felis unit and k3s, from the last %s
logs/ up to %d lines of each container of the pods in %s and %s, and of the
init containers of the game server pods in %s; the previous run too
where a container restarted
cluster/ nodes, volumes, the MinecraftServers, and the pods, workloads, services,
volume claims, network policies and events of %s, %s and %s
`, b.stateDir, b.logLines, shortDuration(b.since), b.logLines, control, build, minecraft, control, build, minecraft)
if b.serverLogs {
fmt.Fprintln(&buf, "\nThe game servers' own logs are in logs/ (-server-logs): they carry player names, IP addresses and chat.")
} else {
fmt.Fprintln(&buf, "\nThe game servers' own logs are left out (they carry player names, IP addresses and chat; -server-logs adds them).")
}
fmt.Fprint(&buf, `
Never collected: Kubernetes Secrets and ConfigMaps, what is in /etc/felis or any configuration
file, the database, worlds, uploads.
Taken out:
`)
if len(bw.scrub.sources) > 0 {
fmt.Fprintf(&buf, " - %d secret values, wherever they appear, read from:\n", len(bw.scrub.vals))
for _, s := range bw.scrub.sources {
fmt.Fprintf(&buf, " %s\n", s)
}
} else {
fmt.Fprintln(&buf, " - no secret file was found on this host to take values from")
}
fmt.Fprint(&buf, ` - passwords in URLs, private keys, Bearer and Basic credentials
- in logs and command output, whatever follows password=, secret=, token=, api_key=,
access_key=, private_key= or credentials= (and the same with a colon)
- every literal env value in pod specs and MinecraftServers (valueFrom references stay)
Read it through before you send it anywhere: redaction finds this host's known secrets and the
common ways a secret is logged, and a secret logged another way stays in.
`)
if b.wErr != nil {
fmt.Fprintf(&buf, "\nThe watchdog's settings: %v\n", b.wErr)
}
if len(bw.errs) > 0 {
fmt.Fprintln(&buf, "\nNot collected:")
for _, e := range bw.errs {
fmt.Fprintf(&buf, " - %s\n", e)
}
}
// Its own words name what is redacted, and would be redacted themselves;
// what it quotes was scrubbed as it was recorded.
return buf.Bytes()
}
// secretSources are files that hold secrets, by how each is laid out.
type secretSources struct {
envFiles []string // KEY=VALUE lines, every value a secret (a commented-out one too)
valueFiles []string // one secret, the whole file
propsFiles []string // key=value lines; the keys naming a token, secret, password or key hold one
tokenFiles []string // k3s join tokens: the whole token and its secret part after the last ':'
}
// scrubber takes secrets out of what the bundle collects.
type scrubber struct {
vals []string // longest first, so a secret that contains another goes whole
sources []string // the files vals came from
}
var (
pemPrivateKey = regexp.MustCompile(`(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?-----END [A-Z0-9 ]*PRIVATE KEY-----`)
urlUserinfo = regexp.MustCompile(`([A-Za-z][A-Za-z0-9+.-]*://[^/\s:@]*:)[^/\s@]+@`)
authScheme = regexp.MustCompile(`(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}`)
secretAssign = regexp.MustCompile(`(?i)((?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|credentials?)[A-Za-z0-9_.-]*"?[ \t]*[=:][ \t]*"?)([^\s"',;&]{4,})`)
secretPropKey = regexp.MustCompile(`(?i)token|secret|password|key`)
)
func (b *supportBundle) scrubber() *scrubber {
s := &scrubber{}
s.readSources(b.secrets)
if b.cfg != nil {
if u, err := url.Parse(b.cfg.Database.URL); err == nil {
if pw, ok := u.User.Password(); ok {
s.add(pw)
}
}
for _, ref := range []string{
b.cfg.Velocity.ServiceTokenRef, b.cfg.SMTP.PasswordRef,
b.cfg.Offsite.AccessKeyRef, b.cfg.Offsite.SecretKeyRef, b.cfg.Offsite.KeyRef,
b.cfg.Registry.S3.AccessKeyRef, b.cfg.Registry.S3.SecretKeyRef,
b.cfg.Archive.S3.AccessKeyRef, b.cfg.Archive.S3.SecretKeyRef,
} {
if ref != "" {
s.add(os.Getenv(ref))
}
}
}
if st, err := watchdog.LoadState(watchdog.NewestState(b.w.statePath, b.w.fallbackState)); err == nil {
s.add(st.SMTPPassword)
}
return s
}
func (s *scrubber) add(v string) bool {
v = strings.TrimSpace(v)
if len(v) < minScrubLen {
return false
}
for _, have := range s.vals {
if have == v {
return true
}
}
s.vals = append(s.vals, v)
sort.SliceStable(s.vals, func(i, j int) bool { return len(s.vals[i]) > len(s.vals[j]) })
return true
}
func (s *scrubber) readSources(src secretSources) {
read := func(p string, take func(content string) bool) {
raw, err := os.ReadFile(p)
if err != nil {
return
}
if take(string(raw)) {
s.sources = append(s.sources, p)
}
}
keyValues := func(content string, keep func(key string) bool) bool {
found := false
for _, line := range strings.Split(content, "\n") {
k, v, ok := strings.Cut(line, "=")
if !ok || !keep(strings.TrimSpace(k)) {
continue
}
v = strings.TrimSpace(v)
if len(v) >= 2 && (v[0] == '\'' || v[0] == '"') && v[len(v)-1] == v[0] {
v = v[1 : len(v)-1]
}
found = s.add(v) || found
}
return found
}
for _, p := range src.envFiles {
read(p, func(c string) bool { return keyValues(c, func(string) bool { return true }) })
}
for _, p := range src.propsFiles {
read(p, func(c string) bool { return keyValues(c, secretPropKey.MatchString) })
}
for _, p := range src.valueFiles {
read(p, func(c string) bool { return s.add(c) })
}
for _, p := range src.tokenFiles {
read(p, func(c string) bool {
c = strings.TrimSpace(c)
whole := s.add(c)
part := false
if i := strings.LastIndex(c, ":"); i >= 0 {
part = s.add(c[i+1:])
}
return whole || part
})
}
}
// values takes every known secret value out of data.
func (s *scrubber) values(data []byte) []byte {
t := pemPrivateKey.ReplaceAllString(string(data), "<redacted private key>")
for _, v := range s.vals {
t = strings.ReplaceAll(t, v, redacted)
}
t = urlUserinfo.ReplaceAllString(t, "${1}"+redacted+"@")
t = authScheme.ReplaceAllString(t, "${1} "+redacted)
return []byte(t)
}
// text is values, and whatever a log names as a secret as well.
func (s *scrubber) text(data []byte) []byte {
return secretAssign.ReplaceAll(s.values(data), []byte("${1}"+redacted))
}
cmd/felis/supportbundle_test.go
-433
View File
@@ -1,433 +0,0 @@
package main
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/watchdog"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
// The secrets a bundle host holds, each of which must be nowhere in the bundle.
var plantedSecrets = map[string]string{
"secrets.env SERVICE_TOKEN": "svc-token-planted-0a1b2c3d",
"secrets.env DB_PASSWORD": "db-pass-planted-4e5f6a7b",
"offsite.env FELIS_OFFSITE_KEY": "offsite-key-planted+8c9d/0e1f=",
"smtp-password": "smtp-pass-planted-2a3b",
"felis-link.properties token": "props-token-planted-4c5d",
"k3s token secret part": "k3s-secret-part-planted-6e7f",
"watchdog state relay password": "cached-relay-pw-planted-8a9b",
"pod env literal": "env-literal-planted-0c1d",
"deployment env literal": "deploy-env-planted-2e3f",
"MinecraftServer env literal": "cr-env-planted-4a5b",
"last-applied annotation": "annotation-planted-6c7d",
"logged password": "hunter2-planted-8e9f",
"logged bearer": "bearerplanted0a1b2c3d",
"URL password": "urlpass-planted-4e5f",
"token in an error": "errtoken-planted-0f1e",
"felis.host.toml DB password": "cfg-db-pass-planted-1c2d",
"service token by its env ref": "env-ref-token-planted-3e4f",
"init container env literal": "init-env-planted-5a6b",
"ephemeral container env": "ephemeral-env-planted-7c8d",
"statefulset env literal": "sts-env-planted-9e0f",
"job env literal": "job-env-planted-1a2b",
"cronjob env literal": "cronjob-env-planted-3c4d",
"commented-out offsite key": "old-offsite-key-planted-5e6f",
"doctor quoting a password": "doctor-pw-planted-7a8b",
"status quoting a password": "status-pw-planted-9c0d",
"journal quoting a password": "journal-pw-planted-1e2f",
}
// bundleHost is a Felis host for felis support-bundle: its secret files, its
// watchdog unit and state, a cluster with a control-plane pod that restarted
// and a game server, and logs and a journal that name secrets.
func bundleHost(t *testing.T) *supportBundle {
t.Helper()
p := plantedSecrets
dir := t.TempDir()
etc := filepath.Join(dir, "etc-felis")
for _, d := range []string{etc, filepath.Join(dir, "systemd"), filepath.Join(dir, "k3s")} {
if err := os.MkdirAll(d, 0o700); err != nil {
t.Fatal(err)
}
}
writeTestFile(t, filepath.Join(etc, "secrets.env"), "SERVICE_TOKEN="+p["secrets.env SERVICE_TOKEN"]+"\nDB_PASSWORD="+p["secrets.env DB_PASSWORD"]+"\nSHORT=abc\n", 0o600)
writeTestFile(t, filepath.Join(etc, "offsite.env"), "# before the rotation\n# FELIS_OFFSITE_KEY="+p["commented-out offsite key"]+"\nFELIS_OFFSITE_KEY='"+p["offsite.env FELIS_OFFSITE_KEY"]+"'\n", 0o600)
writeTestFile(t, filepath.Join(etc, "smtp-password"), p["smtp-password"]+"\n", 0o600)
writeTestFile(t, filepath.Join(etc, "felis-link.properties"), "api-base-url=http://10.43.0.10:8081\nservice-token="+p["felis-link.properties token"]+"\nroot-domain=games.example.org\n", 0o640)
writeTestFile(t, filepath.Join(dir, "k3s", "token"), "K10deadbeefcafe::server:"+p["k3s token secret part"]+"\n", 0o600)
cfgPath := filepath.Join(etc, "felis.host.toml")
writeTestFile(t, cfgPath, "[database]\nurl = \"postgres://felis:"+p["felis.host.toml DB password"]+"@127.0.0.1:1/felis?sslmode=disable&connect_timeout=1\"\n"+
"[server]\nroot_domain = \"games.example.org\"\n[archive]\nstore = \"tarLocal\"\n[k8s]\negress_mode = \"nodeport\"\n"+
"[velocity]\nservice_token_ref = \"FELIS_BUNDLE_TEST_SERVICE_TOKEN\"\n", 0o600)
t.Setenv("FELIS_BUNDLE_TEST_SERVICE_TOKEN", p["service token by its env ref"])
statePath := filepath.Join(dir, "state.json")
if err := watchdog.SaveState(statePath, &watchdog.State{SMTPPassword: p["watchdog state relay password"]}); err != nil {
t.Fatal(err)
}
unitDir := filepath.Join(dir, "systemd")
writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+cfgPath+" -state "+statePath+"\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "felis-offsite.service"), "[Unit]\n", 0o644)
writeTestFile(t, filepath.Join(unitDir, "k3s.service"), "[Unit]\n", 0o644)
meminfo := filepath.Join(dir, "meminfo")
writeTestFile(t, meminfo, "MemTotal: 8000000 kB\nMemAvailable: 2000000 kB\n", 0o644)
cfg, err := config.Load(cfgPath)
if err != nil {
t.Fatal(err)
}
var w watchdogFlags
w, _, err = watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
if err != nil {
t.Fatal(err)
}
w.diskPaths = "/nonexistent-felis-bundle-test"
secretEnv := corev1.EnvVar{Name: "FELIS_SMTP_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "felis-smtp"}, Key: "password"}}}
replicas := int32(1)
cl := fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "felis-api-7d9", Namespace: "felis",
Annotations: map[string]string{corev1.LastAppliedConfigAnnotation: `{"env":"` + p["last-applied annotation"] + `"}`, "felis.lolicon.best/kept": "yes"},
ManagedFields: []metav1.ManagedFieldsEntry{{Manager: "kubectl-client-side-apply"}},
},
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{{Name: "wait-db", Image: "busybox", Env: []corev1.EnvVar{{Name: "FELIS_INIT_SETTING", Value: p["init container env literal"]}}}},
Containers: []corev1.Container{{Name: "api", Image: "felis-api:v1", Env: []corev1.EnvVar{
{Name: "FELIS_PLAIN_SETTING", Value: p["pod env literal"]}, secretEnv,
}}},
EphemeralContainers: []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "debug", Env: []corev1.EnvVar{{Name: "FELIS_DEBUG_SETTING", Value: p["ephemeral container env"]}}}}},
},
Status: corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 1, Ready: true}}},
},
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
Spec: appsv1.DeploymentSpec{Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "felis-api"}},
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "api", Env: []corev1.EnvVar{
{Name: "FELIS_DEPLOY_SETTING", Value: p["deployment env literal"]},
}}}}}},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "survival-0", Namespace: "minecraft"},
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{{Name: "prepare-data"}, {Name: "egress-gate"}},
Containers: []corev1.Container{{Name: "server"}},
},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
},
&v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
Spec: v1alpha1.MinecraftServerSpec{Env: []v1alpha1.EnvVar{{Name: "DISCORD_WEBHOOK", Value: p["MinecraftServer env literal"]}}},
},
&appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: "felis-postgres", Namespace: "felis"},
Spec: appsv1.StatefulSetSpec{Template: envTemplate("FELIS_STS_SETTING", p["statefulset env literal"])},
},
&batchv1.Job{
ObjectMeta: metav1.ObjectMeta{Name: "build-1", Namespace: "felis-build"},
Spec: batchv1.JobSpec{Template: envTemplate("FELIS_JOB_SETTING", p["job env literal"])},
},
&batchv1.CronJob{
ObjectMeta: metav1.ObjectMeta{Name: "felis-reaper", Namespace: "felis"},
Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: envTemplate("FELIS_CRON_SETTING", p["cronjob env literal"])}}},
},
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}},
).Build()
secretLog := fmt.Sprintf("started with SERVICE_TOKEN=%s\nlogin password=%s ok\nAuthorization: Bearer %s\ndial postgres://felis:%s@db:5432/felis\n",
p["secrets.env SERVICE_TOKEN"], p["logged password"], p["logged bearer"], p["URL password"])
return &supportBundle{
host: "felis-test", now: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC), unitDir: unitDir,
w: w, cfg: cfg, cl: cl,
logs: func(_ context.Context, ns, pod, container string, previous bool) ([]byte, error) {
if container == "wait-db" {
return nil, errors.New("container \"wait-db\" is waiting to start; token=" + p["token in an error"])
}
return []byte(fmt.Sprintf("log of %s/%s/%s previous=%v\n%s", ns, pod, container, previous, secretLog)), nil
},
run: func(_ context.Context, name string, args ...string) ([]byte, error) {
switch name {
case "journalctl":
return []byte("journal of " + args[1] + "\nFELIS_OFFSITE_KEY=" + p["offsite.env FELIS_OFFSITE_KEY"] + "\nrelay " + p["smtp-password"] + " refused\n" +
"relay login with the cached " + p["watchdog state relay password"] + "\ndatabase auth failed for " + p["felis.host.toml DB password"] +
"\nservice token " + p["service token by its env ref"] + " rejected\nnode joined with " + p["k3s token secret part"] + "\n" +
"old copies sealed with " + p["commented-out offsite key"] + "\nretrying with password=" + p["journal quoting a password"] + "\n"), nil
case "systemctl", "df", "ip":
return []byte(name + " output\n"), nil
}
return nil, errors.New("unexpected " + name)
},
backups: func(context.Context) (map[string]time.Time, error) {
return nil, errors.New("connect: password=" + p["status quoting a password"])
},
doctor: func(_ context.Context, out io.Writer) {
fmt.Fprintf(out, "doctor report; the k3s token K10deadbeefcafe::server:%s leaked here\nprobe said password=%s\n", p["k3s token secret part"], p["doctor quoting a password"])
},
secrets: secretSources{
envFiles: []string{filepath.Join(etc, "secrets.env"), filepath.Join(etc, "offsite.env"), filepath.Join(etc, "absent.env")},
valueFiles: []string{filepath.Join(etc, "smtp-password"), filepath.Join(etc, "uploads-s3-secret-key")},
propsFiles: []string{filepath.Join(etc, "felis-link.properties")},
tokenFiles: []string{filepath.Join(dir, "k3s", "token")},
},
logLines: 500, since: 48 * time.Hour,
meminfo: meminfo, osRelease: filepath.Join(dir, "os-release"), procVersion: filepath.Join(dir, "version"), stateDir: etc,
}
}
// envTemplate is a pod template whose one container sets name to a literal.
func envTemplate(name, value string) corev1.PodTemplateSpec {
return corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Env: []corev1.EnvVar{{Name: name, Value: value}}}}}}
}
// readBundle is every file in the bundle at path, by its name inside the
// bundle's directory, and each one's mode.
func readBundle(t *testing.T, path string) (map[string]string, map[string]int64) {
t.Helper()
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
t.Fatal(err)
}
tr := tar.NewReader(gz)
files, modes := map[string]string{}, map[string]int64{}
prefix := strings.TrimSuffix(filepath.Base(path), ".tar.gz") + "/"
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
name, ok := strings.CutPrefix(hdr.Name, prefix)
if !ok {
t.Errorf("%s is outside the bundle's directory %s", hdr.Name, prefix)
}
raw, err := io.ReadAll(tr)
if err != nil {
t.Fatal(err)
}
files[name], modes[name] = string(raw), hdr.Mode
}
return files, modes
}
func TestSupportBundle(t *testing.T) {
b := bundleHost(t)
out := filepath.Join(t.TempDir(), "support")
path, err := b.write(context.Background(), out)
if err != nil {
t.Fatal(err)
}
if want := filepath.Join(out, "felis-support-felis-test-20260927T120000Z.tar.gz"); path != want {
t.Errorf("path %s, want %s", path, want)
}
for p, want := range map[string]os.FileMode{out: 0o700 | os.ModeDir, path: 0o600} {
if st, err := os.Stat(p); err != nil || st.Mode() != want {
t.Errorf("%s: mode %v (err %v), want %v", p, st.Mode(), err, want)
}
}
if left, _ := filepath.Glob(filepath.Join(out, ".*partial")); len(left) != 0 {
t.Errorf("left behind %v", left)
}
files, modes := readBundle(t, path)
var names []string
for n := range files {
names = append(names, n)
if modes[n] != 0o600 {
t.Errorf("%s: mode %o in the archive, want 600", n, modes[n])
}
}
sort.Strings(names)
want := []string{
"MANIFEST.txt",
"cluster/felis-build/cronjobs.yaml", "cluster/felis-build/deployments.yaml", "cluster/felis-build/events.yaml", "cluster/felis-build/jobs.yaml",
"cluster/felis-build/networkpolicies.yaml", "cluster/felis-build/persistentvolumeclaims.yaml", "cluster/felis-build/pods.yaml",
"cluster/felis-build/services.yaml", "cluster/felis-build/statefulsets.yaml",
"cluster/felis/cronjobs.yaml", "cluster/felis/deployments.yaml", "cluster/felis/events.yaml", "cluster/felis/jobs.yaml",
"cluster/felis/networkpolicies.yaml", "cluster/felis/persistentvolumeclaims.yaml", "cluster/felis/pods.yaml",
"cluster/felis/services.yaml", "cluster/felis/statefulsets.yaml",
"cluster/minecraft/cronjobs.yaml", "cluster/minecraft/deployments.yaml", "cluster/minecraft/events.yaml", "cluster/minecraft/jobs.yaml",
"cluster/minecraft/networkpolicies.yaml", "cluster/minecraft/persistentvolumeclaims.yaml", "cluster/minecraft/pods.yaml",
"cluster/minecraft/services.yaml", "cluster/minecraft/statefulsets.yaml",
"cluster/minecraftservers.yaml", "cluster/nodes.yaml", "cluster/persistentvolumes.yaml", "cluster/pods-all-namespaces.txt",
"config.txt", "doctor.txt",
"host/addresses.txt", "host/df.txt", "host/etc-felis.txt", "host/meminfo.txt", "host/systemd-timers.txt", "host/systemd-units.txt",
"journal/felis-offsite.log", "journal/felis-watchdog.log", "journal/k3s.log",
"logs/felis/felis-api-7d9/api.log", "logs/felis/felis-api-7d9/api.previous.log",
"logs/minecraft/survival-0/egress-gate.log", "logs/minecraft/survival-0/prepare-data.log",
"status.txt", "version.txt",
}
if strings.Join(names, "\n") != strings.Join(want, "\n") {
t.Errorf("bundle holds\n %s\nwant\n %s", strings.Join(names, "\n "), strings.Join(want, "\n "))
}
for what, secret := range plantedSecrets {
for n, body := range files {
if strings.Contains(body, secret) {
t.Errorf("%s (%s) is in %s:\n%s", what, secret, n, body)
}
}
}
// What was taken out leaves what a reader needs around it.
for name, wants := range map[string][]string{
"logs/felis/felis-api-7d9/api.previous.log": {"log of felis/felis-api-7d9/api previous=true\n", "SERVICE_TOKEN=<redacted>\n", "login password=<redacted> ok\n", "Authorization: Bearer <redacted>\n", "postgres://felis:<redacted>@db:5432/felis\n"},
"journal/k3s.log": {"journal of k3s.service\n", "FELIS_OFFSITE_KEY=<redacted>\n", "relay <redacted> refused\n",
"relay login with the cached <redacted>\n", "database auth failed for <redacted>\n", "service token <redacted> rejected\n", "node joined with <redacted>\n"},
"cluster/felis/statefulsets.yaml": {"name: FELIS_STS_SETTING\n"},
"cluster/felis/cronjobs.yaml": {"name: FELIS_CRON_SETTING\n"},
"cluster/felis-build/jobs.yaml": {"name: FELIS_JOB_SETTING\n"},
"cluster/felis/pods.yaml": {"name: FELIS_PLAIN_SETTING\n value: <redacted>\n", "name: FELIS_SMTP_PASSWORD\n valueFrom:\n secretKeyRef:\n key: password\n name: felis-smtp\n", "felis.lolicon.best/kept: \"yes\"", "name: FELIS_INIT_SETTING\n", "name: FELIS_DEBUG_SETTING\n"},
"cluster/felis/deployments.yaml": {"name: FELIS_DEPLOY_SETTING\n value: <redacted>\n"},
"cluster/minecraftservers.yaml": {"name: DISCORD_WEBHOOK\n value: <redacted>\n"},
"config.txt": {fmt.Sprintf("%-34s %s\n", "server.root_domain", "games.example.org"), fmt.Sprintf("%-34s %s\n", "database.url (no password)", "postgres://[email protected]:1/felis")},
"doctor.txt": {"the k3s token <redacted> leaked here\nprobe said password=<redacted>\n"},
"status.txt": {" world backups unknown: connect: password=<redacted>\n"},
"host/etc-felis.txt": {"secrets.env\n", "smtp-password\n", "felis-link.properties\n"},
"MANIFEST.txt": {
"The game servers' own logs are left out",
" journal/ up to 500 lines per Felis unit and k3s, from the last 48h\n",
" - 11 secret values, wherever they appear, read from:\n",
"secrets.env\n", "offsite.env\n", "smtp-password\n", "felis-link.properties\n", "token\n",
"logs/felis/felis-api-7d9/wait-db.log: container \"wait-db\" is waiting to start; token=<redacted>\n",
// Its own words about what is redacted come through whole.
" - passwords in URLs, private keys, Bearer and Basic credentials\n",
"access_key=, private_key= or credentials= (and the same with a colon)\n",
"Read it through before you send it anywhere",
},
} {
for _, w := range wants {
if !strings.Contains(files[name], w) {
t.Errorf("%s lacks %q:\n%s", name, w, files[name])
}
}
}
for _, gone := range []string{"managedFields", "last-applied-configuration"} {
if strings.Contains(files["cluster/felis/pods.yaml"], gone) {
t.Errorf("pods.yaml keeps %s:\n%s", gone, files["cluster/felis/pods.yaml"])
}
}
if strings.Contains(files["MANIFEST.txt"], "absent.env") || strings.Contains(files["MANIFEST.txt"], "uploads-s3-secret-key") {
t.Errorf("MANIFEST names files this host does not have:\n%s", files["MANIFEST.txt"])
}
}
// -server-logs adds the game servers' own logs, and says so.
func TestSupportBundleServerLogs(t *testing.T) {
b := bundleHost(t)
b.serverLogs = true
path, err := b.write(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}
files, _ := readBundle(t, path)
if _, ok := files["logs/minecraft/survival-0/server.log"]; !ok {
t.Error("no server.log with -server-logs")
}
if !strings.Contains(files["MANIFEST.txt"], "The game servers' own logs are in logs/ (-server-logs)") {
t.Errorf("MANIFEST does not say server logs are in:\n%s", files["MANIFEST.txt"])
}
}
// With the cluster and the configuration gone the bundle still holds what the
// host shows, and MANIFEST says what it could not collect.
func TestSupportBundleWithTheClusterDown(t *testing.T) {
b := bundleHost(t)
b.cl, b.clErr = nil, errors.New("connection refused")
b.cfg, b.cfgErr = nil, errors.New("felis.host.toml: no such file")
b.wErr = errors.New("felis-watchdog.service is not installed; read the watchdog's defaults")
path, err := b.write(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}
files, _ := readBundle(t, path)
for _, n := range []string{"status.txt", "doctor.txt", "host/etc-felis.txt", "journal/k3s.log"} {
if _, ok := files[n]; !ok {
t.Errorf("no %s", n)
}
}
for n := range files {
if strings.HasPrefix(n, "cluster/") || strings.HasPrefix(n, "logs/") || n == "config.txt" {
t.Errorf("%s with no cluster and no configuration", n)
}
}
if !strings.Contains(files["status.txt"], "felis status: the configuration did not load: felis.host.toml: no such file") {
t.Errorf("status.txt:\n%s", files["status.txt"])
}
if !strings.Contains(files["MANIFEST.txt"], "\nThe watchdog's settings: felis-watchdog.service is not installed; read the watchdog's defaults\n") ||
!strings.Contains(files["MANIFEST.txt"], " - cluster: connection refused\n") {
t.Errorf("MANIFEST.txt:\n%s", files["MANIFEST.txt"])
}
}
func TestShortDuration(t *testing.T) {
for d, want := range map[time.Duration]string{
48 * time.Hour: "48h",
90 * time.Minute: "1h30m",
45 * time.Minute: "45m",
30 * time.Second: "30s",
time.Hour + 30*time.Second: "1h0m30s",
} {
if got := shortDuration(d); got != want {
t.Errorf("shortDuration(%v) = %q, want %q", d, got, want)
}
}
}
func TestScrubber(t *testing.T) {
s := &scrubber{}
for _, v := range []string{"known-secret-value", "known-secret-value-longer", "short", "known-secret-value"} {
s.add(v)
}
if got := strings.Join(s.vals, ","); got != "known-secret-value-longer,known-secret-value" {
t.Errorf("kept %q; want each value once, longest first, none under %d characters", s.vals, minScrubLen)
}
for in, want := range map[string]string{
"a known-secret-value-longer b": "a <redacted> b",
"a known-secret-value b": "a <redacted> b",
"password=abcd1234 next": "password=<redacted> next",
`{"token": "abcd1234"}`: `{"token": "<redacted>"}`,
"SMTP_PASSWORD: s3cr3t!x": "SMTP_PASSWORD: <redacted>",
"x-api-key=zzzz9999&q=1": "x-api-key=<redacted>&q=1",
"Authorization: Basic dXNlcjpwYXNzd29yZA==": "Authorization: Basic <redacted>",
"s3://AKIA:secretpart123@bucket/key": "s3://AKIA:<redacted>@bucket/key",
"https://user@host/path": "https://user@host/path",
"-----BEGIN EC PRIVATE KEY-----\nMHc\n-----END EC PRIVATE KEY-----": "<redacted private key>",
"tokens: 3": "tokens: 3",
"read the relay password (keeping the cached one)": "read the relay password (keeping the cached one)",
`secrets "felis-smtp" not found`: `secrets "felis-smtp" not found`,
} {
if got := string(s.text([]byte(in))); got != want {
t.Errorf("text(%q) = %q, want %q", in, got, want)
}
}
// Structured files keep what only looks like a secret.
if got := string(s.values([]byte("secretName: felis-forwarding-secret and known-secret-value"))); got != "secretName: felis-forwarding-secret and <redacted>" {
t.Errorf("values() = %q", got)
}
}
cmd/felis/systemservers.go
+45 -130
View File
@@ -15,9 +15,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -244,15 +242,6 @@ func buildSystemServerClient() (client.Client, error) {
if err := v1alpha1.AddToScheme(scheme); err != nil {
return nil, err
}
cfg, err := hostRESTConfig()
if err != nil {
return nil, err
}
return client.New(cfg, client.Options{Scheme: scheme})
}
// hostRESTConfig is the cluster connection buildSystemServerClient describes.
func hostRESTConfig() (*rest.Config, error) {
cfg, err := ctrl.GetConfig()
if err != nil {
cfg, err = clientcmd.BuildConfigFromFlags("", hostBootstrapKubeconfigPath)
@@ -260,7 +249,7 @@ func hostRESTConfig() (*rest.Config, error) {
return nil, fmt.Errorf("no reachable kubeconfig (tried in-cluster/$KUBECONFIG/~/.kube and %s): %w", hostBootstrapKubeconfigPath, err)
}
}
return cfg, nil
return client.New(cfg, client.Options{Scheme: scheme})
}
// systemServerOutcome records what ensureSystemServers did with one service so
@@ -403,48 +392,21 @@ var derivedSystemEnv = map[string]bool{
// operator every run.
func refreshDerivedEnv(ctx context.Context, cl client.Client, existing, desired *v1alpha1.MinecraftServer) (bool, error) {
want := derivedEnvWanted(desired)
changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
changed := false
for i, e := range existing.Spec.Env {
if v, ok := want[e.Name]; ok && v != e.Value {
existing.Spec.Env[i].Value = v
changed = true
}
changed := false
for i, e := range existing.Spec.Env {
if v, ok := want[e.Name]; ok && v != e.Value {
existing.Spec.Env[i].Value = v
changed = true
}
return changed
})
if err != nil {
}
if !changed {
return false, nil
}
if err := cl.Update(ctx, existing); err != nil {
return false, fmt.Errorf("refresh %s env: %w", existing.Name, err)
}
return changed, nil
}
// patchOnConflictRetry applies mutate to obj and sends only the difference, as a
// merge patch that carries the resourceVersion obj was read at. The operator writes
// status and felis-api patches spec.idle on these same objects, so a write can land
// between setup's read and its patch: the apiserver then answers 409, and this
// re-reads obj and runs mutate again on the fresh copy, up to retry.DefaultRetry's
// five attempts. The pinned resourceVersion is what keeps a list field such as
// spec.env safe — a merge patch replaces a list whole, and without the lock an
// entry added concurrently would be dropped. mutate reports whether it changed
// anything; nothing is sent when it did not. obj holds the stored object after.
func patchOnConflictRetry(ctx context.Context, cl client.Client, obj client.Object, mutate func() bool) (bool, error) {
key := client.ObjectKeyFromObject(obj)
changed, reread := false, false
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
if reread {
if err := cl.Get(ctx, key, obj); err != nil {
return err
}
}
reread = true
base := obj.DeepCopyObject().(client.Object)
if changed = mutate(); !changed {
return nil
}
return cl.Patch(ctx, obj, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}))
})
return changed, err
return true, nil
}
// derivedEnvWanted maps the derived env keys of desired onto their values.
@@ -507,25 +469,22 @@ func convergeSystemServers(ctx context.Context, cl client.Client, namespace, log
}
var changes []string
changed, err := patchOnConflictRetry(ctx, cl, &existing, func() bool {
changes = nil
if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
existing.Spec.Rcon = desired.Spec.Rcon
changes = append(changes, "spec.rcon")
}
if existing.Spec.Startup.HealthHTTPPort == 0 && desired.Spec.Startup.HealthHTTPPort != 0 {
existing.Spec.Startup.HealthHTTPPort = desired.Spec.Startup.HealthHTTPPort
changes = append(changes, "spec.startup.healthHTTPPort")
}
changes = append(changes, convergeDerivedEnv(&existing, desired)...)
return len(changes) > 0
})
if err != nil {
outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
if existing.Spec.Rcon == (v1alpha1.RconSpec{}) && desired.Spec.Rcon != (v1alpha1.RconSpec{}) {
existing.Spec.Rcon = desired.Spec.Rcon
changes = append(changes, "spec.rcon")
}
if existing.Spec.Startup.HealthHTTPPort == 0 && desired.Spec.Startup.HealthHTTPPort != 0 {
existing.Spec.Startup.HealthHTTPPort = desired.Spec.Startup.HealthHTTPPort
changes = append(changes, "spec.startup.healthHTTPPort")
}
changes = append(changes, convergeDerivedEnv(&existing, desired)...)
if len(changes) == 0 {
outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
continue
}
if !changed {
outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, skipped: "already converged"})
if err := cl.Update(ctx, &existing); err != nil {
outcomes = append(outcomes, systemServerOutcome{name: p.name, err: fmt.Errorf("converge %s: %w", p.name, err)})
continue
}
outcomes = append(outcomes, systemServerOutcome{name: p.name, available: true, updated: true, changes: changes})
@@ -629,51 +588,15 @@ func phaseOrPending(p v1alpha1.Phase) string {
return string(p)
}
// provisionSecretReplicas copies the Secrets workload pods mount from the control
// namespace into the namespaces those pods run in. The proxy's felis-service-token
// is not among them: it lives in the control namespace and on the host, and a copy
// anywhere else would open every game route to whoever reads that namespace.
func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, minecraftNS, buildNS string) []systemServerOutcome {
return []systemServerOutcome{
// refresh=true for both caller tokens: the control namespace holds the
// current value and `felis rotate-token` replaces it there, so a replica
// that differs is stale and the login gate would be turned away with it.
ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
naming.LimboTokenSecretName, naming.ServiceTokenSecretKey, "limbo-token", "minecraft ns", true),
ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false),
// refresh=true: felis-config is the rendered config, not a credential. The
// backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run
// must update it when the control plane's render has moved on (a stale copy
// e.g. keeps an old database URL after a credential rotation).
ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
"felis-config", "felis.toml", "config", "minecraft ns", true),
// The reaper's pre-reap warning emails authenticate with the same relay
// password felis-api uses; the reaper pod runs in the minecraft namespace,
// where a secretKeyRef resolves only against a local mirror. Skipped while
// the relay is not configured yet — the "configure email" screen refreshes
// both mirrors when it applies.
ensureSecretReplica(ctx, cl, controlNS, minecraftNS,
"felis-smtp", "password", "smtp", "minecraft ns", false),
// The build namespace needs the build token: the build Job's fetch
// initContainer reads the submission context from the internal face, and
// that is all this token opens. Best effort — a deployment that only
// installs the control plane simply never builds a user submission.
ensureSecretReplica(ctx, cl, controlNS, buildNS,
naming.BuildTokenSecretName, naming.ServiceTokenSecretKey, "build-token", "felis-build ns", true),
}
}
// ensureSecretReplica copies one Secret from the control namespace into a workload
// namespace (minecraft — or the build namespace, whose fetch initContainer reads the
// context from the felis-api internal face with the build token) so a pod can mount it
// context from the felis-api internal face with the same token) so a pod can mount it
// via secretKeyRef. A secretKeyRef is namespace-local, but those workloads do not run
// beside the control plane — so without this replica the secretKeyRef would dangle and
// wedge the pod in CreateContainerConfigError.
//
// Several Secrets need it, for different reasons: the caller tokens of the login
// limbo and the build Pod's context fetch (both authenticate to the felis-api
// internal face, each with its own token),
// Three Secrets need it, for different reasons: the service token (the login limbo and
// the build Pod's context fetch — both authenticate to the felis-api internal face),
// the Velocity modern-forwarding secret (every backend — it is how a backend knows
// a login really came from the proxy, and so that the player's UUID is Mojang-verified
// rather than offline-derived), and the SMTP relay password (the reaper's pre-reap
@@ -686,14 +609,12 @@ func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, m
// copies only Type and Data — never labels/annotations/ownerRefs — so the replica
// carries no accidental GC owner or managed-by lineage.
//
// refreshExisting switches a replica to refresh-in-place from the control namespace.
// The felis-config mirror uses it because that Secret is a rendered config and the
// workload Jobs that mount it (backup/restore/fileedit) plus the reaper silently
// misbehave on a stale copy — e.g. after a database credential rotation the control
// plane moves on while every backup Job keeps failing auth. The caller tokens use it
// because `felis rotate-token` replaces them in the control namespace, which makes a
// differing replica stale by definition. The forwarding and SMTP Secrets keep the
// never-overwrite rule.
// refreshExisting switches the felis-config mirror to refresh-in-place: that Secret is
// a rendered config, never a hand-rotated credential, and the workload Jobs that mount
// it (backup/restore/fileedit) plus the reaper silently misbehave on a stale copy —
// e.g. after a database credential rotation the control plane moves on while every
// backup Job keeps failing auth. Credential Secrets keep the never-overwrite rule so a
// rotated value survives; to rotate those, delete the replica and re-run setup.
func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label, where string, refreshExisting bool) systemServerOutcome {
name := label + " (" + where + ")"
validate := func(secret *corev1.Secret, location, skipped string) systemServerOutcome {
@@ -718,22 +639,16 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
if out := validate(&src, controlNamespace, ""); !out.available {
return out
}
changed, err := patchOnConflictRetry(ctx, cl, existing, func() bool {
if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
return false
}
if existing.Data == nil {
existing.Data = map[string][]byte{}
}
existing.Data[secretKey] = src.Data[secretKey]
return true
})
if err != nil {
return systemServerOutcome{name: name, err: err}
}
if !changed {
if bytes.Equal(existing.Data[secretKey], src.Data[secretKey]) {
return validate(existing, minecraftNamespace, "already current")
}
if existing.Data == nil {
existing.Data = map[string][]byte{}
}
existing.Data[secretKey] = src.Data[secretKey]
if err := cl.Update(ctx, existing); err != nil {
return systemServerOutcome{name: name, err: err}
}
return systemServerOutcome{name: name, updated: true, available: true}
}
if controlNamespace == minecraftNamespace {
@@ -797,7 +712,7 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace
func requiredProvisioningError(outcomes []systemServerOutcome) error {
required := map[string]struct{}{
"limbo-token (minecraft ns)": {},
"service-token (minecraft ns)": {},
"forwarding-secret (minecraft ns)": {},
naming.SystemLoginServer: {},
}
cmd/felis/systemservers_test.go
+18 -85
View File
@@ -142,21 +142,21 @@ func TestLoginSystemServerEnv(t *testing.T) {
// namespace (create-if-absent), so the operator's secretKeyRef on the backend pod
// resolves. It must not overwrite an existing replica, and must degrade gracefully
// when the source is missing or the namespaces coincide. Exercised here with the
// Velocity forwarding secret, which setup replicates in this never-overwrite mode.
// service token; setup runs it a second time for the Velocity forwarding secret.
func TestEnsureSecretReplica(t *testing.T) {
scheme := newSystemServerScheme(t)
ctx := context.Background()
srcSecret := func() *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "felis"},
ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "felis"},
Type: corev1.SecretTypeOpaque,
Data: map[string][]byte{naming.ForwardingSecretKey: []byte("s3cr3t")},
Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte("s3cr3t")},
}
}
replicate := func(cl client.Client, controlNS, mcNS string) systemServerOutcome {
return ensureSecretReplica(ctx, cl, controlNS, mcNS,
naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false)
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false)
}
t.Run("replicates when absent", func(t *testing.T) {
@@ -166,19 +166,19 @@ func TestEnsureSecretReplica(t *testing.T) {
t.Fatalf("outcome = %+v, want created", out)
}
var replica corev1.Secret
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
t.Fatalf("get replica: %v", err)
}
if string(replica.Data[naming.ForwardingSecretKey]) != "s3cr3t" {
t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ForwardingSecretKey])
if string(replica.Data[naming.ServiceTokenSecretKey]) != "s3cr3t" {
t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ServiceTokenSecretKey])
}
})
t.Run("does not overwrite existing replica", func(t *testing.T) {
existing := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
Type: corev1.SecretTypeOpaque,
Data: map[string][]byte{naming.ForwardingSecretKey: []byte("rotated")},
Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte("rotated")},
}
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), existing).Build()
out := replicate(cl, "felis", "minecraft")
@@ -186,11 +186,11 @@ func TestEnsureSecretReplica(t *testing.T) {
t.Fatalf("outcome = %+v, want skipped (not clobbered)", out)
}
var replica corev1.Secret
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil {
if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil {
t.Fatalf("get replica: %v", err)
}
if string(replica.Data[naming.ForwardingSecretKey]) != "rotated" {
t.Error("existing replica was overwritten — a hand-set value must survive")
if string(replica.Data[naming.ServiceTokenSecretKey]) != "rotated" {
t.Error("existing replica was overwritten — a rotated token must survive")
}
})
@@ -204,22 +204,22 @@ func TestEnsureSecretReplica(t *testing.T) {
t.Run("rejects a source with an empty required key", func(t *testing.T) {
bad := srcSecret()
bad.Data[naming.ForwardingSecretKey] = nil
bad.Data[naming.ServiceTokenSecretKey] = nil
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
out := replicate(cl, "felis", "minecraft")
if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
t.Fatalf("outcome = %+v, want unavailable required key", out)
}
})
t.Run("rejects an existing replica with an empty required key", func(t *testing.T) {
bad := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"},
ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"},
Data: map[string][]byte{},
}
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), bad).Build()
out := replicate(cl, "felis", "minecraft")
if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
t.Fatalf("outcome = %+v, want unavailable existing replica", out)
}
})
@@ -245,7 +245,7 @@ func TestEnsureSecretReplica(t *testing.T) {
bad.Data = nil
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build()
out := replicate(cl, "felis", "felis")
if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) {
if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) {
t.Fatalf("outcome = %+v, want unavailable required key", out)
}
})
@@ -334,7 +334,7 @@ func TestEnsureSecretReplicaRefresh(t *testing.T) {
func TestRequiredProvisioningError(t *testing.T) {
ready := []systemServerOutcome{
{name: "limbo-token (minecraft ns)", available: true},
{name: "service-token (minecraft ns)", available: true},
{name: "forwarding-secret (minecraft ns)", available: true},
{name: naming.SystemLoginServer, available: true},
{name: naming.SystemLobbyServer, skipped: "image not configured"},
@@ -349,14 +349,6 @@ func TestRequiredProvisioningError(t *testing.T) {
t.Fatalf("missing forwarding secret = %v, want named error", err)
}
// The login gate cannot reach felis-api without its token, so setup must not
// report success while that replica is missing.
noToken := append([]systemServerOutcome(nil), ready...)
noToken[0] = systemServerOutcome{name: "limbo-token (minecraft ns)", skipped: "source missing"}
if err := requiredProvisioningError(noToken); err == nil || !strings.Contains(err.Error(), "limbo-token") {
t.Fatalf("missing limbo token = %v, want named error", err)
}
failed := append([]systemServerOutcome(nil), ready...)
failed[3] = systemServerOutcome{name: naming.SystemLobbyServer, err: context.DeadlineExceeded}
if err := requiredProvisioningError(failed); err == nil || !strings.Contains(err.Error(), naming.SystemLobbyServer) {
@@ -670,62 +662,3 @@ func TestEnsureSystemServersRefreshesDerivedEnv(t *testing.T) {
}
})
}
// Setup's replicas carry each workload its own caller token and nothing more: the
// login gate gets felis-limbo-token in the minecraft namespace, the build Jobs get
// felis-build-token in the build namespace, a replica left stale by a rotation is
// brought up to date, and the proxy's felis-service-token is copied nowhere.
func TestProvisionSecretReplicasCarryCallerTokens(t *testing.T) {
scheme := newSystemServerScheme(t)
ctx := context.Background()
secret := func(ns, name, key, val string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
Type: corev1.SecretTypeOpaque,
Data: map[string][]byte{key: []byte(val)},
}
}
cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
secret("felis", "felis-service-token", "token", "proxy-tok"),
secret("felis", "felis-limbo-token", "token", "limbo-new"),
secret("felis", "felis-build-token", "token", "build-tok"),
secret("felis", "felis-ops-token", "token", "ops-tok"),
secret("felis", naming.ForwardingSecretName, naming.ForwardingSecretKey, "fwd"),
// What a rotation leaves behind before setup runs again.
secret("minecraft", "felis-limbo-token", "token", "limbo-old"),
).Build()
outcomes := provisionSecretReplicas(ctx, cl, "felis", "minecraft", "felis-build")
for _, o := range outcomes {
if o.err != nil {
t.Fatalf("%s: %v", o.name, o.err)
}
}
read := func(ns, name string) (string, bool) {
var s corev1.Secret
if err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
return "", false
}
return string(s.Data["token"]), true
}
if got, _ := read("minecraft", "felis-limbo-token"); got != "limbo-new" {
t.Errorf("minecraft/felis-limbo-token = %q, want the rotated limbo-new", got)
}
if got, _ := read("felis-build", "felis-build-token"); got != "build-tok" {
t.Errorf("felis-build/felis-build-token = %q, want build-tok", got)
}
for _, ns := range []string{"minecraft", "felis-build"} {
for _, name := range []string{"felis-service-token", "felis-ops-token"} {
if _, ok := read(ns, name); ok {
t.Errorf("%s/%s was replicated; only the control namespace holds it", ns, name)
}
}
}
if _, ok := read("minecraft", "felis-build-token"); ok {
t.Error("the build token was copied into the minecraft namespace")
}
if _, ok := read("felis-build", "felis-limbo-token"); ok {
t.Error("the limbo token was copied into the build namespace")
}
}
cmd/felis/tui_alerts_test.go
-202
View File
@@ -1,202 +0,0 @@
package main
import (
"context"
"errors"
"path/filepath"
"strings"
"testing"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"felis.lolicon.best/internal/config"
)
// TestAlertRouteLines: the summary's alert rows say where the watchdog's
// alerts go, and flag every route that reaches no one.
func TestAlertRouteLines(t *testing.T) {
cases := []struct {
name string
r alertRoute
alerts string
alertsOK bool
beat string
beatOK bool
}{
{
name: "fresh install",
alerts: "only logged: no email relay (press e)",
beat: "none: no outside check (troubleshooting.md §14)",
},
{
name: "relay, no verified owner",
r: alertRoute{relay: "smtp.example.com:587", heartbeat: "https://hc-ping.com/..."},
alerts: "only logged: no verified Owner email (panel → Account)",
beat: "https://hc-ping.com/... every 2 minutes", beatOK: true,
},
{
name: "relay, owners unreadable",
r: alertRoute{relay: "smtp.example.com:587", lookupErr: errors.New("connection refused")},
alerts: "via smtp.example.com:587; could not read the Owner addresses",
beat: "none: no outside check (troubleshooting.md §14)",
},
{
name: "owners but no relay",
r: alertRoute{recipients: []string{"[email protected]"}},
alerts: "only logged: no email relay (press e)",
beat: "none: no outside check (troubleshooting.md §14)",
},
{
name: "mailed",
r: alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]", "[email protected]"}, heartbeatErr: errors.New("/etc/felis/watchdog-heartbeat-url: the heartbeat URL is not an http:// or https:// URL")},
alerts: "mailed to [email protected], [email protected] via smtp.example.com:587", alertsOK: true,
beat: "unreadable: /etc/felis/watchdog-heartbeat-url: the heartbeat URL is not an http:// or https:// URL",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if line, ok := c.r.alertsLine(); line != c.alerts || ok != c.alertsOK {
t.Errorf("alerts = %q, %v; want %q, %v", line, ok, c.alerts, c.alertsOK)
}
if line, ok := c.r.heartbeatLine(); line != c.beat || ok != c.beatOK {
t.Errorf("heartbeat = %q, %v; want %q, %v", line, ok, c.beat, c.beatOK)
}
})
}
}
// TestSummaryAlertRows: the rows show on the card, a route that needs action
// marked so it reads without colour, and stay off a summary with no route.
func TestSummaryAlertRows(t *testing.T) {
m := &summaryModel{ownerUsername: "owner", alerts: &alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]"}}}
v := m.View()
for _, want := range []string{"alerts mailed to [email protected] via smtp.example.com:587", "heartbeat ⚠ none: no outside check"} {
if !strings.Contains(v, want) {
t.Errorf("summary lacks %q:\n%s", want, v)
}
}
if strings.Contains(v, "⚠ mailed") {
t.Errorf("a route that reaches the owners is marked:\n%s", v)
}
m.alerts = &alertRoute{heartbeat: "https://hc-ping.com/..."}
v = m.View()
for _, want := range []string{"alerts ⚠ only logged: no email relay (press e)", "heartbeat https://hc-ping.com/... every 2 minutes"} {
if !strings.Contains(v, want) {
t.Errorf("summary lacks %q:\n%s", want, v)
}
}
m.alerts = nil
if v = m.View(); strings.Contains(v, "alerts") || strings.Contains(v, "heartbeat") {
t.Errorf("a summary with no route shows alert rows:\n%s", v)
}
}
// TestRootSummaryReadsAlertRoute: the summary and the re-run status screen
// read the route each time they open, so a relay configured with e shows at
// once.
func TestRootSummaryReadsAlertRoute(t *testing.T) {
m := newTestRoot(false, consoleModeSetup, "")
reads := 0
route := alertRoute{}
m.alertRoute = func(context.Context) alertRoute {
reads++
return route
}
m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk"})
sum, ok := m.screen.(*summaryModel)
if !ok || sum.alerts == nil || sum.alerts.relay != "" {
t.Fatalf("summary after storage: %T %+v", m.screen, sum)
}
route = alertRoute{relay: "smtp.example.com:587", recipients: []string{"[email protected]"}}
m = drive(t, m, smtpResultMsg{configured: true})
sum, ok = m.screen.(*summaryModel)
if !ok || sum.alerts == nil || sum.alerts.relay != "smtp.example.com:587" {
t.Fatalf("summary after configuring email: %T %+v", m.screen, sum)
}
if reads != 2 {
t.Errorf("route read %d times, want 2", reads)
}
status := newTestRoot(true, consoleModeSetup, "")
status.alertRoute = m.alertRoute
status.showStatus()
if sum, ok := status.screen.(*summaryModel); !ok || sum.alerts == nil || sum.alerts.relay != "smtp.example.com:587" {
t.Fatalf("status screen: %T %+v", status.screen, status.screen)
}
}
// TestHostAlertRoute reads the relay from the host config and the heartbeat
// file, shows the heartbeat by its host only, and reports a database it cannot
// reach.
func TestHostAlertRoute(t *testing.T) {
dir := t.TempDir()
cfg := filepath.Join(dir, "felis.host.toml")
writeTestFile(t, cfg, testWatchdogConfig+testWatchdogSMTP, 0o600)
beat := filepath.Join(dir, "watchdog-heartbeat-url")
writeTestFile(t, beat, "https://hc-ping.com/check-key\n", 0o600)
dbURL := "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
r := hostAlertRoute(context.Background(), cfg, dbURL, beat)
if r.relay != "smtp.config.example:2525" {
t.Errorf("relay = %q", r.relay)
}
if r.heartbeat != "https://hc-ping.com/..." || r.heartbeatErr != nil {
t.Errorf("heartbeat = %q, %v", r.heartbeat, r.heartbeatErr)
}
if r.lookupErr == nil {
t.Errorf("an unreachable database reads as %v", r.recipients)
}
noRelay := filepath.Join(dir, "no-relay.toml")
writeTestFile(t, noRelay, testWatchdogConfig, 0o600)
writeTestFile(t, beat, "hc-ping.com/check-key\n", 0o600)
r = hostAlertRoute(context.Background(), noRelay, dbURL, beat)
if r.relay != "" {
t.Errorf("no [smtp]: relay = %q", r.relay)
}
if r.heartbeatErr == nil || strings.Contains(r.heartbeatErr.Error(), "check-key") {
t.Errorf("a bad heartbeat file: %v", r.heartbeatErr)
}
r = hostAlertRoute(context.Background(), noRelay, dbURL, filepath.Join(dir, "none"))
if r.heartbeat != "" || r.heartbeatErr != nil {
t.Errorf("no heartbeat file: %q, %v", r.heartbeat, r.heartbeatErr)
}
}
// TestSummaryWithAlertsFitsTerminal: the two rows keep the summary inside the
// terminal, with the longest route lines.
func TestSummaryWithAlertsFitsTerminal(t *testing.T) {
for _, w := range []int{60, 80, 90} {
for _, h := range []int{24, 30, 45} {
m := newTestRoot(false, consoleModeSetup, "")
m.alertRoute = func(context.Context) alertRoute {
return alertRoute{relay: "smtp.example.com:587", lookupErr: errors.New("dial tcp 127.0.0.1:5432: connect: connection refused")}
}
m = drive(t, m, tea.WindowSizeMsg{Width: w, Height: h})
m = drive(t, m, preflightDoneMsg{})
m = drive(t, m, ownerResultMsg{username: "owner", setupTokenURL: "https://op.console.example.com/setup?token=t0ken"})
m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.example.com"})
m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"})
if _, ok := m.screen.(*summaryModel); !ok {
t.Fatalf("screen = %T, want the summary", m.screen)
}
if got := lipgloss.Height(m.View()); got > h {
t.Errorf("terminal %dx%d: summary with alert rows = %d rows (exceeds height)", w, h, got)
}
}
}
}
// TestConsoleRootReadsHostAlertRoute: the console the host runs gives its
// summary this host's route, read against its database.
func TestConsoleRootReadsHostAlertRoute(t *testing.T) {
db := config.DatabaseConfig{URL: "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"}
rm := newConsoleRoot(context.Background(), &fakeOwnerStore{}, db, "felis.example.com", "admin.felis.example.com", "panel.felis.example.com", "", "minecraft", "root", false, consoleModeSetup, recoveryConfig{})
if rm.alertRoute == nil {
t.Fatal("the console's summary reads no alert route")
}
if r := rm.alertRoute(context.Background()); r.lookupErr == nil {
t.Errorf("the route did not query the console's database: %+v", r)
}
}
cmd/felis/tui_migration.go
+13 -6
View File
@@ -6,7 +6,6 @@ import (
"io"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/dbbackup"
"felis.lolicon.best/internal/store"
)
@@ -14,10 +13,10 @@ import (
// applyMigrations runs any pending schema migrations and returns the resulting
// applied count. Used by the preflight stage to self-heal a freshly bootstrapped
// (or upgraded) database.
func applyMigrations(db config.DatabaseConfig) (int, error) {
func applyMigrations(dbURL string) (int, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
drv, err := store.Open(ctx, db.URL)
drv, err := store.Open(ctx, dbURL)
if err != nil {
return 0, fmt.Errorf("open db: %w", err)
}
@@ -28,15 +27,23 @@ func applyMigrations(db config.DatabaseConfig) (int, error) {
}
// Same guard as `felis migrate up`: never roll a populated database forward
// without a snapshot to roll back to.
if _, err := preMigrateBackup(ctx, drv, migrations, db, dbbackup.DefaultDir, io.Discard); err != nil {
if _, err := preMigrateBackup(ctx, drv, migrations, dbURL, dbbackup.DefaultDir, io.Discard); err != nil {
return 0, fmt.Errorf("pre-migration backup: %w", err)
}
if _, err := store.Up(ctx, drv, migrations); err != nil {
return 0, err
}
s, err := store.ReadSchema(ctx, drv)
applied, err := drv.AppliedVersions(ctx)
if err != nil {
return 0, err
}
return s.Applied, nil
return len(applied), nil
}
func totalMigrations() (int, error) {
migrations, err := store.LoadMigrations()
if err != nil {
return 0, err
}
return len(migrations), nil
}
cmd/felis/tui_owner.go
+28 -147
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"strings"
"time"
"felis.lolicon.best/internal/api"
@@ -15,8 +14,9 @@ import (
)
type owAuthMsg struct {
start recoveryStart
err error
matched string
ok bool
err error
}
type owProvisionMsg struct {
@@ -29,15 +29,14 @@ type owStep int
const (
owAuth owStep = iota
owOverride
owCode // typing the recovery code mailed to the named admin
owProvision
owWorking
owDone
owError
)
// ownerModel collects the owner account. The input phases (admin name, recovery
// code, root override, owner details) are huh forms; the async phases (verifying,
// ownerModel collects the owner account. The input phases (admin auth, root
// override, owner details) are huh forms; the async phases (verifying,
// provisioning) show a spinner; the done phase shows the credential card. The
// outward contract is unchanged: it emits an ownerResultMsg when finished.
//
@@ -56,18 +55,6 @@ type ownerModel struct {
mode string // "bootstrap", "recovery", "root_override"
accountable string
attempt string
recovery recoveryConfig
// Recovery proof: the admin the typed name resolved to, the code mailed to it,
// and once settled either how it was proven or why the run fell back to the
// override.
admin *api.StaffUser
code *recoveryCode
codeNote string // "wrong code" line shown above a rebuilt code form
verifiedBy string
codeSentTo string
skip string // otpSkip*
skipDetail string
step owStep
form *huh.Form
@@ -79,7 +66,6 @@ type ownerModel struct {
// huh-bound form values
authUser string
codeInput string
overrideTok string
ownerUser string
ownerEmail string
@@ -138,12 +124,6 @@ func newOperatorModel(ctx context.Context, store ownerStore, osUser string) *own
return m
}
// withRecovery hands the model the relay its recovery codes go through.
func (m *ownerModel) withRecovery(r recoveryConfig) *ownerModel {
m.recovery = r
return m
}
func (m *ownerModel) Init() tea.Cmd { return m.form.Init() }
// subject is the human label for the account being provisioned, branching every
@@ -172,7 +152,7 @@ func (m *ownerModel) sized(f *huh.Form) *huh.Form {
}
func (m *ownerModel) isFormStep() bool {
return m.step == owAuth || m.step == owOverride || m.step == owCode || m.step == owProvision
return m.step == owAuth || m.step == owOverride || m.step == owProvision
}
func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
@@ -181,15 +161,16 @@ func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
if msg.err != nil {
return m, m.failCmd(msg.err)
}
m.admin = msg.start.admin
if msg.start.code != nil {
m.code = msg.start.code
m.codeNote = ""
m.step = owCode
m.form = m.sized(m.buildCodeForm())
if msg.ok {
m.mode = "recovery"
m.accountable = msg.matched
m.step = owProvision
m.form = m.sized(m.buildProvisionForm())
return m, m.form.Init()
}
return m.toOverride(msg.start.skip, msg.start.detail)
m.step = owOverride
m.form = m.sized(m.buildOverrideForm())
return m, m.form.Init()
case owProvisionMsg:
if msg.err != nil {
@@ -240,9 +221,7 @@ func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
case "ctrl+c":
return m, tea.Quit
case "esc":
if m.step == owOverride || m.step == owCode {
// Start over: a code mailed for the old attempt dies with it.
m.admin, m.code, m.skip, m.skipDetail = nil, nil, "", ""
if m.step == owOverride {
m.step = owAuth
m.form = m.sized(m.buildAuthForm())
return m, m.form.Init()
@@ -274,41 +253,18 @@ func (m *ownerModel) onFormComplete() (tea.Model, tea.Cmd) {
case owAuth:
m.attempt = strings.TrimSpace(m.authUser)
m.step = owWorking
m.working = "Sending a recovery code…"
user, rc, osUser, op := m.authUser, m.recovery, m.osUser, m.operation
m.working = "Verifying admin…"
user := m.authUser
return m, tea.Batch(m.sp.Tick, func() tea.Msg {
start, err := beginRecovery(m.ctx, m.store, rc, user, osUser, op)
return owAuthMsg{start: start, err: err}
matched, ok, err := authenticateAdmin(m.ctx, m.store, user)
return owAuthMsg{matched: matched, ok: ok, err: err}
})
case owCode:
typed := strings.TrimSpace(m.codeInput)
m.codeInput = ""
if typed == breakGlassOverrideToken {
m.skip, m.skipDetail = otpSkipByOperator, ""
m.code = nil
return m.proceedAsRoot()
}
switch m.code.check(typed, m.recovery.clock()) {
case codeAccepted:
m.mode = "recovery"
m.accountable = m.admin.Username
m.verifiedBy = verifiedByEmailOTP
m.codeSentTo = m.admin.Email
m.code = nil
m.step = owProvision
m.form = m.sized(m.buildProvisionForm())
return m, m.form.Init()
case codeWrong:
m.codeNote = fmt.Sprintf("That code is wrong. %d attempts left.", m.code.attemptsLeft())
m.form = m.sized(m.buildCodeForm())
return m, m.form.Init()
case codeExpired:
return m.toOverride(otpSkipCodeExpired, "")
default:
return m.toOverride(otpSkipCodeRejected, fmt.Sprintf("%d wrong codes", recoveryCodeAttempts))
}
case owOverride:
return m.proceedAsRoot()
m.mode = "root_override"
m.accountable = m.osUser
m.step = owProvision
m.form = m.sized(m.buildProvisionForm())
return m, m.form.Init()
case owProvision:
m.username = strings.TrimSpace(m.ownerUser)
m.step = owWorking
@@ -318,24 +274,6 @@ func (m *ownerModel) onFormComplete() (tea.Model, tea.Cmd) {
return m, nil
}
// toOverride records why no code proved an admin and asks for the typed OVERRIDE.
func (m *ownerModel) toOverride(skip, detail string) (tea.Model, tea.Cmd) {
m.skip, m.skipDetail = skip, detail
m.code = nil
m.step = owOverride
m.form = m.sized(m.buildOverrideForm())
return m, m.form.Init()
}
// proceedAsRoot is the typed OVERRIDE: the run goes on as the OS user, unverified.
func (m *ownerModel) proceedAsRoot() (tea.Model, tea.Cmd) {
m.mode = "root_override"
m.accountable = m.osUser
m.step = owProvision
m.form = m.sized(m.buildProvisionForm())
return m, m.form.Init()
}
func (m *ownerModel) provisionCmd() tea.Cmd {
op := breakGlassOp{
mode: m.mode,
@@ -344,10 +282,6 @@ func (m *ownerModel) provisionCmd() tea.Cmd {
ownerUsername: m.username,
ownerEmail: m.ownerEmail,
attemptedAdmin: m.attempt,
verifiedBy: m.verifiedBy,
codeSentTo: m.codeSentTo,
otpSkipped: m.skip,
otpSkipDetail: m.skipDetail,
}
// performAddOperator and performBreakGlass share a signature; the operation
// discriminator selects which one runs. The operator path is insert-only and
@@ -386,7 +320,7 @@ func (m *ownerModel) buildAuthForm() *huh.Form {
return m.sized(newFelisForm(huh.NewGroup(
huh.NewNote().
Title("Admin authentication").
Description("A staff account already exists. Name yours: a one-time code goes to its verified email address."),
Description("A staff account already exists. Identify yourself to continue."),
huh.NewInput().
Title("Admin username").
Value(&m.authUser).
@@ -394,64 +328,11 @@ func (m *ownerModel) buildAuthForm() *huh.Form {
)))
}
func (m *ownerModel) buildCodeForm() *huh.Form {
desc := fmt.Sprintf("A recovery code went to %s, the verified address of %q. It works for %d minutes.\n\n"+
"No mail? Type %s to go on as OS user %q with root authority; the audit log records that as an unverified override. Esc starts over.",
maskEmail(m.admin.Email), m.admin.Username, int(recoveryCodeTTL/time.Minute), breakGlassOverrideToken, m.osUser)
if m.codeNote != "" {
desc = m.codeNote + "\n\n" + desc
}
return m.sized(newFelisForm(huh.NewGroup(
huh.NewNote().Title("Email verification").Description(desc),
huh.NewInput().
Title("Recovery code").
Value(&m.codeInput).
Validate(func(s string) error {
s = strings.TrimSpace(s)
if s == breakGlassOverrideToken || isRecoveryCodeShape(s) {
return nil
}
return errors.New("enter the 6-digit code, or " + breakGlassOverrideToken)
}),
)))
}
func isRecoveryCodeShape(s string) bool {
if len(s) != 6 {
return false
}
for _, c := range s {
if c < '0' || c > '9' {
return false
}
}
return true
}
// overrideReason says why no code proved an admin, first line of the override form.
func (m *ownerModel) overrideReason() string {
switch m.skip {
case otpSkipUnknownAdmin:
return fmt.Sprintf("No staff account is named %q.", m.attempt)
case otpSkipNoVerifiedEmail:
return fmt.Sprintf("%q has no verified email address, so no recovery code can reach it.", m.admin.Username)
case otpSkipNoRelay:
return "No mail relay can send a recovery code: " + m.skipDetail + "."
case otpSkipSendFailed:
return "The recovery code could not be sent: " + m.skipDetail + "."
case otpSkipCodeExpired:
return "The recovery code expired."
case otpSkipCodeRejected:
return fmt.Sprintf("%d wrong codes; that code no longer works.", recoveryCodeAttempts)
}
return "No admin was verified."
}
func (m *ownerModel) buildOverrideForm() *huh.Form {
return m.sized(newFelisForm(huh.NewGroup(
huh.NewNote().
Title("Root override").
Description(m.overrideReason()+"\n\n"+fmt.Sprintf("Proceed as OS user %q with root authority by typing the confirmation token. The audit log records this run as an unverified root override and the reason above. Esc starts over.", m.osUser)),
Description(fmt.Sprintf("That credential did not match. Proceed as OS user %q with root authority by typing the confirmation token.", m.osUser)),
huh.NewInput().
Title("Type "+breakGlassOverrideToken+" to confirm").
Value(&m.overrideTok).
@@ -471,14 +352,14 @@ func (m *ownerModel) buildProvisionForm() *huh.Form {
desc := fmt.Sprintf("Create the first Owner — recorded as OS user %q.", m.osUser)
switch m.mode {
case "recovery":
desc = fmt.Sprintf("Verified as %q by an email code.", m.accountable)
desc = fmt.Sprintf("Authenticated as %q.", m.accountable)
case "root_override":
desc = "Root override — the Owner will be reset."
}
if m.operation == bgAddOperator {
switch m.mode {
case "recovery":
desc = fmt.Sprintf("Add an Operator — verified as %q by an email code.", m.accountable)
desc = fmt.Sprintf("Add an Operator — authenticated as %q.", m.accountable)
case "root_override":
desc = "Add an Operator (root override)."
}
cmd/felis/tui_postgres.go
+10 -4
View File
@@ -72,14 +72,20 @@ func parseDBURL(url string) (dbCfg, error) {
}, nil
}
// readSchema lines the database's recorded migrations up with this build's.
func readSchema(dbURL string) (store.Schema, error) {
func countMigrations(dbURL string) (int, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
drv, err := store.Open(ctx, dbURL)
if err != nil {
return store.Schema{}, err
return 0, err
}
defer drv.Close()
return store.ReadSchema(ctx, drv)
if err := drv.EnsureVersionTable(ctx); err != nil {
return 0, err
}
applied, err := drv.AppliedVersions(ctx)
if err != nil {
return 0, err
}
return len(applied), nil
}
cmd/felis/tui_preflight.go
+11 -16
View File
@@ -3,8 +3,6 @@ package main
import (
"fmt"
"felis.lolicon.best/internal/config"
"github.com/charmbracelet/bubbles/spinner"
tea "github.com/charmbracelet/bubbletea"
)
@@ -16,7 +14,7 @@ import (
// of the wizard can assume a healthy backend. It is non-interactive — it runs
// to completion and reports back to the root via preflightDoneMsg.
type preflightModel struct {
db config.DatabaseConfig
dbURL string
rootDomain string
adminHost string
@@ -48,7 +46,6 @@ type pfDBMsg struct{ err error }
type pfMigCheckMsg struct {
applied int
total int
pending bool
err error
}
@@ -59,11 +56,11 @@ type pfMigApplyMsg struct {
type pfPanelMsg struct{ err error }
func newPreflightModel(db config.DatabaseConfig, rootDomain, adminHostname string) *preflightModel {
func newPreflightModel(dbURL, rootDomain, adminHostname string) *preflightModel {
sp := spinner.New()
sp.Spinner = spinner.Dot
sp.Style = tuiLabel
return &preflightModel{db: db, rootDomain: rootDomain, adminHost: adminHostname, sp: sp, state: pfCheckDB}
return &preflightModel{dbURL: dbURL, rootDomain: rootDomain, adminHost: adminHostname, sp: sp, state: pfCheckDB}
}
func (m *preflightModel) Init() tea.Cmd {
@@ -87,7 +84,7 @@ func (m *preflightModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, nil
}
m.applied, m.total = msg.applied, msg.total
if msg.pending {
if msg.applied < msg.total {
m.state = pfApplyMig
return m, m.applyMigrations()
}
@@ -202,25 +199,23 @@ func (m *preflightModel) errText() string {
}
func (m *preflightModel) checkDB() tea.Cmd {
return func() tea.Msg { return pfDBMsg{err: checkPostgres(m.db.URL)} }
return func() tea.Msg { return pfDBMsg{err: checkPostgres(m.dbURL)} }
}
func (m *preflightModel) checkMigrations() tea.Cmd {
return func() tea.Msg {
// The sets, not their sizes: a database a newer release migrated can hold as
// many rows as this build has migrations, and must stop here rather than be
// "healed" by an older binary.
s, err := readSchema(m.db.URL)
if err == nil {
err = s.Newer()
applied, err := countMigrations(m.dbURL)
if err != nil {
return pfMigCheckMsg{err: err}
}
return pfMigCheckMsg{applied: s.Applied, total: s.Total, pending: len(s.Pending) > 0, err: err}
total, err := totalMigrations()
return pfMigCheckMsg{applied: applied, total: total, err: err}
}
}
func (m *preflightModel) applyMigrations() tea.Cmd {
return func() tea.Msg {
applied, err := applyMigrations(m.db)
applied, err := applyMigrations(m.dbURL)
return pfMigApplyMsg{applied: applied, err: err}
}
}
cmd/felis/tui_root.go
+6 -23
View File
@@ -5,7 +5,6 @@ import (
"strings"
"felis.lolicon.best/internal/cfsetup"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/platform"
tea "github.com/charmbracelet/bubbletea"
@@ -134,7 +133,7 @@ type rootModel struct {
err error
mode consoleMode
db config.DatabaseConfig
dbURL string
store ownerStore
osUser string
rootDomain string
@@ -143,17 +142,13 @@ type rootModel struct {
accessAud string
namespace string // minecraft workload namespace (cfg.K8s.Namespace); target of the halt op
adminExists bool
recovery recoveryConfig // how the account operations mail a recovery code
// alertRoute reads where the watchdog's alerts go for the summary; nil
// leaves those rows out.
alertRoute func(context.Context) alertRoute
}
func newRootModel(ctx context.Context, store ownerStore, db config.DatabaseConfig, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) *rootModel {
func newRootModel(ctx context.Context, store ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) *rootModel {
rm := &rootModel{
ctx: ctx,
reviewing: -1,
db: db,
dbURL: dbURL,
store: store,
osUser: osUser,
rootDomain: rootDomain,
@@ -184,7 +179,7 @@ func newRootModel(ctx context.Context, store ownerStore, db config.DatabaseConfi
}
} else {
rm.stage = stagePreflight
rm.screen = newPreflightModel(db, rootDomain, adminHostname)
rm.screen = newPreflightModel(dbURL, rootDomain, adminHostname)
}
return rm
}
@@ -226,7 +221,7 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.stage = stageOwner
switch msg.op {
case bgAddOperator:
return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser).withRecovery(m.recovery))
return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser))
case bgHaltServer:
return m.adopt(newHaltModel(m.ctx, m.store, m.namespace, m.osUser))
case bgSyncBackup:
@@ -234,7 +229,7 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
// Service + service token the peer dials live in the control namespace.
return m.adopt(newBackupModel(m.ctx, m.namespace, platform.DefaultControlNamespace, m.osUser))
default:
return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists).withRecovery(m.recovery))
return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists))
}
case haltResultMsg:
@@ -559,7 +554,6 @@ func (m *rootModel) showSummary() (tea.Model, tea.Cmd) {
routedHosts: routed,
localHint: m.result.connectMethod == connectLocal,
alreadySetUp: m.result.alreadySetUp,
alerts: m.readAlertRoute(),
})
}
@@ -580,20 +574,9 @@ func (m *rootModel) showStatus() (tea.Model, tea.Cmd) {
accessLabel: accessLabel,
alreadySetUp: true,
localHint: m.accessAud == "" && rootDomainEmbeddedIP(m.rootDomain) != "",
alerts: m.readAlertRoute(),
})
}
// readAlertRoute reads the alert route afresh, so the summary shows a relay the
// Owner just configured with e.
func (m *rootModel) readAlertRoute() *alertRoute {
if m.alertRoute == nil {
return nil
}
r := m.alertRoute(m.ctx)
return &r
}
func panelURLFor(method connectMethod, panelHostname, rootDomain, adminHostname string) string {
if method != connectLocal && panelHostname != "" {
return "https://" + panelHostname
cmd/felis/tui_root_test.go
+1 -3
View File
@@ -5,8 +5,6 @@ import (
"strings"
"testing"
"felis.lolicon.best/internal/config"
tea "github.com/charmbracelet/bubbletea"
)
@@ -32,7 +30,7 @@ func newTestRoot(adminExists bool, mode consoleMode, accessAud string) *rootMode
return newRootModel(
context.Background(),
&fakeOwnerStore{},
config.DatabaseConfig{URL: "postgres://localhost/felis"},
"postgres://localhost/felis",
"felis.example.com",
"admin.felis.example.com",
"panel.felis.example.com",
cmd/felis/tui_smtp.go
+16 -38
View File
@@ -9,6 +9,7 @@ import (
"strings"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/platform"
"github.com/charmbracelet/bubbles/spinner"
@@ -276,14 +277,11 @@ func validateSMTPFrom(s string) error {
}
// currentSMTPInputs reads the relay already recorded in felis.toml so the form
// pre-fills the non-secret fields. The password (the felis-smtp Secret and its
// host copy) is deliberately never read back — it must be re-entered to change.
// pre-fills the non-secret fields. The password lives only in the felis-smtp
// Secret and is deliberately never read back — it must be re-entered to change.
// Any read error falls back to a blank form rather than blocking reconfig.
func currentSMTPInputs() smtpInputs { return smtpInputsFrom(hostSetupConfigPath) }
// smtpInputsFrom is currentSMTPInputs for the config at path.
func smtpInputsFrom(path string) smtpInputs {
cfg, err := config.Load(path)
func currentSMTPInputs() smtpInputs {
cfg, err := config.Load(hostSetupConfigPath)
if err != nil || cfg.SMTP.Host == "" {
return smtpInputs{}
}
@@ -298,8 +296,7 @@ func smtpInputsFrom(path string) smtpInputs {
// applySMTPConfig proves the relay works, then persists it and rolls felis-api:
// Ping (a full transaction — connect/STARTTLS/AUTH/MAIL FROM/RCPT/DATA, which
// delivers one self-test message to the From address) → [smtp] into both config
// files → the password into /etc/felis/smtp-password (hostcreds.go) → the
// felis-smtp Secret → the config Secret → rollout. A failed Ping
// files → the felis-smtp Secret → the config Secret → rollout. A failed Ping
// leaves the install untouched, so a bad relay dies at the keyboard, not at a
// player's OTP.
//
@@ -314,31 +311,28 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
if err != nil {
return fmt.Errorf("port %q is not a number", in.port)
}
var prev config.SMTPConfig
if cur, err := config.Load(hostSetupConfigPath); err == nil {
prev = cur.SMTP
}
// Ping under the posture felis api will send with, so a relay without
// STARTTLS is turned down here rather than at a player's first code.
if err := smtpRelay(setupSMTPConfig(in, port, prev), in.password).Ping(ctx); err != nil {
relay := &mail.SMTP{Host: in.host, Port: port, From: in.from, Username: in.username, Password: in.password}
if err := relay.Ping(ctx); err != nil {
return err
}
smtpCfg := config.SMTPConfig{
Host: in.host,
Port: port,
From: in.from,
Username: in.username,
PasswordRef: platform.SMTPPasswordEnv,
}
for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} {
cfg, err := config.Load(path)
if err != nil {
return err
}
cfg.SMTP = setupSMTPConfig(in, port, cfg.SMTP)
cfg.SMTP = smtpCfg
if err := writeConfig(path, cfg); err != nil {
return err
}
}
// The host copy first: should the Secret fail, the next installer run applies
// it from this file.
if err := writeHostCredential(hostSMTPPasswordPath, in.password); err != nil {
return fmt.Errorf("keep the relay password in %s: %w", hostSMTPPasswordPath, err)
}
if err := applySMTPSecret(ctx, in.password); err != nil {
return err
}
@@ -357,22 +351,6 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
return kubectl(ctx, "-n", "felis", "rollout", "status", "deployment/felis-api", "--timeout=180s")
}
// setupSMTPConfig is the [smtp] block this screen writes: the relay it just
// proved, plus the keys only an operator sets by hand (require_tls,
// max_per_hour), carried over from the block it replaces so reconfiguring the
// relay does not quietly reset them.
func setupSMTPConfig(in smtpInputs, port int, prev config.SMTPConfig) config.SMTPConfig {
return config.SMTPConfig{
Host: in.host,
Port: port,
From: in.from,
Username: in.username,
PasswordRef: platform.SMTPPasswordEnv,
MaxPerHour: prev.MaxPerHour,
RequireTLS: prev.RequireTLS,
}
}
// smtpSecretManifest renders the felis-smtp Secret for the given namespace, the
// one the receiving Deployment/CronJob resolves its secretKeyRef against (felis
// for felis-api, the workload namespace for the reaper's mirror). The namespace
cmd/felis/tui_smtp_test.go
-39
View File
@@ -1,12 +1,9 @@
package main
import (
"reflect"
"strings"
"testing"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/mail"
"sigs.k8s.io/yaml"
)
@@ -38,39 +35,3 @@ func TestSMTPSecretManifestCarriesTargetNamespace(t *testing.T) {
}
}
}
// TestSMTPRelayCarriesTLSPosture: the relay felis api, the reaper and the
// watchdog send through refuses plaintext for a remote host and allows it for
// one on this host, as [smtp] says.
func TestSMTPRelayCarriesTLSPosture(t *testing.T) {
remote := smtpRelay(config.SMTPConfig{Host: "smtp.example.net", Port: 587, From: "[email protected]", Username: "felis"}, "pw")
want := &mail.SMTP{Host: "smtp.example.net", Port: 587, From: "[email protected]", Username: "felis", Password: "pw", RequireTLS: true}
if !reflect.DeepEqual(remote, want) {
t.Errorf("remote relay = %+v, want %+v", remote, want)
}
if local := smtpRelay(config.SMTPConfig{Host: "127.0.0.1", Port: 25, From: "[email protected]"}, ""); local.RequireTLS {
t.Error("a relay on this host must not require TLS by default")
}
}
// TestSetupSMTPConfigKeepsHandSetKeys: re-running the email screen replaces the
// relay but keeps require_tls and max_per_hour, which only an operator sets.
func TestSetupSMTPConfigKeepsHandSetKeys(t *testing.T) {
off := false
prev := config.SMTPConfig{Host: "old.example.net", Port: 25, From: "[email protected]", MaxPerHour: 500, RequireTLS: &off}
in := smtpInputs{host: "smtp.example.net", from: "[email protected]", username: "felis"}
got := setupSMTPConfig(in, 465, prev)
if got.Host != "smtp.example.net" || got.Port != 465 || got.From != "[email protected]" ||
got.Username != "felis" || got.PasswordRef != "FELIS_SMTP_PASSWORD" {
t.Errorf("relay fields = %+v", got)
}
if got.MaxPerHour != 500 {
t.Errorf("max_per_hour = %d, want 500", got.MaxPerHour)
}
if got.RequireTLS == nil || *got.RequireTLS {
t.Errorf("require_tls = %v, want the operator's false", got.RequireTLS)
}
if fresh := setupSMTPConfig(in, 587, config.SMTPConfig{}); fresh.RequireTLS != nil || fresh.MaxPerHour != 0 {
t.Errorf("first setup = %+v, want require_tls and max_per_hour unset", fresh)
}
}
cmd/felis/tui_storage_apply.go
+3 -13
View File
@@ -53,17 +53,8 @@ func applyStorageConfig(ctx context.Context, method storageMethod, in s3Inputs)
return err
}
// S3: land the credentials in their own Secret BEFORE the roll, so the optional
// env refs resolve on the fresh pod, and on the host before that, so the next
// installer run can apply the Secret again (hostcreds.go). Local needs no Secret.
// env refs resolve on the fresh pod. Local needs no Secret.
if method == storageS3 {
for _, c := range []struct{ path, value string }{
{hostUploadsS3AccessKeyPath, in.accessKey},
{hostUploadsS3SecretKeyPath, in.secretKey},
} {
if err := writeHostCredential(c.path, c.value); err != nil {
return fmt.Errorf("keep the bucket credentials in %s: %w", c.path, err)
}
}
if err := applyUploadsS3Secret(ctx, in.accessKey, in.secretKey); err != nil {
return err
}
@@ -79,9 +70,8 @@ func applyStorageConfig(ctx context.Context, method storageMethod, in s3Inputs)
// currentStorageInputs reads the storage backend already recorded in felis.toml so
// the reconfigure flow can pre-select the method and pre-fill the non-secret S3
// fields (endpoint/bucket/region). The credentials (the felis-uploads-s3 Secret
// and its host copies) are deliberately never read back — they must be re-entered
// to change.
// fields (endpoint/bucket/region). Credentials live only in the felis-uploads-s3
// Secret and are deliberately never read back — they must be re-entered to change.
// Any read error falls back to a blank local default rather than blocking reconfig.
func currentStorageInputs() (storageMethod, s3Inputs) {
cfg, err := config.Load(hostSetupConfigPath)
cmd/felis/tui_summary.go
-77
View File
@@ -1,9 +1,7 @@
package main
import (
"context"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
)
@@ -22,8 +20,6 @@ type summaryModel struct {
routedHosts []string
alreadySetUp bool // re-run: Owner pre-existed
localHint bool // show the self-signed-cert note
// alerts is where the watchdog's alerts go; nil leaves the rows out.
alerts *alertRoute
}
func (m *summaryModel) Init() tea.Cmd { return nil }
@@ -77,12 +73,6 @@ func (m *summaryModel) View() string {
if m.panelURL != "" {
card.WriteString(tuiLabel.Render("panel ") + m.panelURL + "\n")
}
if m.alerts != nil {
line, ok := m.alerts.alertsLine()
card.WriteString(routeRow("alerts ", line, ok))
line, ok = m.alerts.heartbeatLine()
card.WriteString(routeRow("heartbeat ", line, ok))
}
b.WriteString(tuiCardStyle.Render(strings.TrimRight(card.String(), "\n")) + "\n\n")
b.WriteString(tuiHint.Render("ℹ Everything else — servers, users, plugins — is configured in the panel. You won't need this console again.") + "\n")
@@ -93,70 +83,3 @@ func (m *summaryModel) View() string {
b.WriteString("\n" + tuiAction("c", "change connection", "s", "change storage", "e", "configure email", "enter/esc", "exit"))
return b.String()
}
// alertRoute is where this host's watchdog alerts go, as the summary shows it:
// by mail through the [smtp] relay to the Owners' verified addresses, and the
// heartbeat that notices the host itself going down (docs/troubleshooting.md
// §14). Setup runs mail-less by design, so a fresh install has neither; the
// summary says so where the Owner can press e.
type alertRoute struct {
relay string // "host:port", "" with no [smtp] relay
recipients []string // enabled Owners' verified addresses
lookupErr error // the recipients could not be read
heartbeat string // the heartbeat URL's scheme and host, "" with none
heartbeatErr error // the heartbeat file does not read
}
// hostAlertRoute reads the route from the host config at cfgPath, the database
// at dbURL and the heartbeat file at heartbeatPath: what the next watchdog run
// uses.
func hostAlertRoute(ctx context.Context, cfgPath, dbURL, heartbeatPath string) alertRoute {
var r alertRoute
if in := smtpInputsFrom(cfgPath); in.host != "" {
r.relay = in.host + ":" + in.port
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
r.recipients, r.lookupErr = ownerEmails(ctx, dbURL)
u, err := readHeartbeatURL(heartbeatPath)
switch {
case err != nil:
r.heartbeatErr = err
case u != "":
r.heartbeat = redactURL(u)
}
return r
}
// alertsLine is the summary's alerts row; ok is false when the alerts reach no one.
func (r alertRoute) alertsLine() (line string, ok bool) {
switch {
case r.relay == "":
return "only logged: no email relay (press e)", false
case r.lookupErr != nil:
return "via " + r.relay + "; could not read the Owner addresses", false
case len(r.recipients) == 0:
return "only logged: no verified Owner email (panel → Account)", false
}
return "mailed to " + strings.Join(r.recipients, ", ") + " via " + r.relay, true
}
// heartbeatLine is the summary's heartbeat row; ok is false with no heartbeat.
func (r alertRoute) heartbeatLine() (line string, ok bool) {
switch {
case r.heartbeatErr != nil:
return "unreadable: " + r.heartbeatErr.Error(), false
case r.heartbeat == "":
return "none: no outside check (troubleshooting.md §14)", false
}
return r.heartbeat + " every 2 minutes", true
}
// routeRow renders one alert row, marked and in the warning style when it
// needs action: the mark reads without colour too.
func routeRow(label, line string, ok bool) string {
if !ok {
line = tuiWarn.Render("⚠ " + line)
}
return tuiLabel.Render(label) + line + "\n"
}
cmd/felis/update.go
+20 -239
View File
@@ -2,8 +2,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
@@ -11,9 +9,6 @@ import (
"strings"
"time"
"github.com/jackc/pgx/v5"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/updater"
"felis.lolicon.best/internal/updates"
)
@@ -39,8 +34,6 @@ const updateTimeout = 60 * time.Second
type updateTarget struct {
// selector is the flag name without dashes.
selector string
// help is the flag's usage line.
help string
// component is the updates planner's name for this piece, or "" when the planner
// deliberately does not track it (Minecraft, which is pinned).
component string
@@ -48,11 +41,9 @@ type updateTarget struct {
note string
// command is the exact, already-tested way to apply it.
command string
// installer marks a command that re-runs the installer, which the trailer explains.
installer bool
}
// installerRerun is the tested apply path for every selector Felis installs: re-run the
// installerRerun is the tested apply path for every planner-backed selector: re-run the
// installer. It is idempotent, and it is the only path that fetches a newer version --
// `felis setup` skips its host-bootstrap phase on a completed install (all four install
// markers already exist), so there it opens the config console and moves no component,
@@ -67,10 +58,6 @@ type updateTarget struct {
// below points at the README's token'd form for that case.
const installerRerun = "curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/{ref}/deploy/bootstrap.sh | sudo bash"
// installerRerunDeps is the same re-run with FELIS_UPGRADE_DEPS=1, which lets it move an
// installed k3s and cloudflared to the versions the release pins.
const installerRerunDeps = "curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/{ref}/deploy/bootstrap.sh | sudo FELIS_UPGRADE_DEPS=1 bash"
// updateTargets is the selector table. panel and plugins both resolve to felis-api
// because they are not separately versioned: the panel is compiled into the felis
// binary with //go:embed, and the plugin jars are built from this same repo in the
@@ -78,63 +65,24 @@ const installerRerunDeps = "curl -fsSL https://raw.githubusercontent.com/FelisMC
var updateTargets = []updateTarget{
{
selector: "panel",
help: "select the panel + control plane (felis-api)",
component: "felis-api",
note: "the panel is embedded in the felis binary (//go:embed), so updating it means rebuilding the felis image and rolling felis-api",
command: installerRerun,
installer: true,
},
{
selector: "velocity",
help: "select the Velocity proxy",
component: "velocity",
note: "re-runs install_velocity: the build the release pins in deploy/game-stack.lock (FELIS_VELOCITY_VERSION=<minor> takes that minor's newest build instead), sha256-checked, atomic jar install, then restarts felis-velocity only if the jar or its config changed",
command: installerRerun,
installer: true,
},
{
selector: "plugins",
help: "select the Felis plugin jars (velocity/paper/limbo)",
component: "felis-api",
note: "felis-velocity.jar is a host-file swap, but felis-paper.jar and felis-limbo.jar are baked into the lobby/limbo images and need a rebuild + re-mirror into the in-cluster registry (the installer re-run does both)",
command: installerRerun,
installer: true,
},
{
selector: "k3s",
help: "select k3s",
component: "k3s",
note: "FELIS_UPGRADE_DEPS=1 moves k3s to the version the Felis release pins, which can trail the newest upstream; it moves one minor version at a time and refuses a larger jump. Running game servers keep running while k3s restarts",
command: installerRerunDeps,
installer: true,
},
{
selector: "cloudflared",
help: "select cloudflared",
component: "cloudflared",
note: "FELIS_UPGRADE_DEPS=1 swaps the binary for the sha256-pinned build the Felis release names and restarts cloudflared-felis; the panel's tunnel drops for a few seconds",
command: installerRerunDeps,
installer: true,
},
{
selector: "jre",
help: "select the Temurin JRE Velocity runs on",
component: "jre",
note: "the installer installs the Temurin build the Felis release pins (sha256-checked) and restarts felis-velocity when it changed; a newer upstream build reaches the host with a release that pins it",
command: installerRerun,
installer: true,
},
{
selector: "postgres",
help: "select PostgreSQL",
component: "postgresql",
note: "PostgreSQL runs as the felis-postgres Deployment from the image the Felis release pins by digest; a newer minor reaches the host with a release that moves the pin, and the installer re-run restarts the database on it (a few seconds without the API). A new major is a dump and restore: docs/operations.md §4",
command: installerRerun,
installer: true,
},
{
selector: "mc",
help: "select Minecraft (pinned; reported only)",
component: "", // never tracked: see the pin note below
note: "Minecraft is pinned by policy (\"能不动的就别动\") and Felis never proposes a version change for it. A server's version is a property of that server's image — change it on the server, not through a platform update",
command: "",
@@ -144,30 +92,26 @@ var updateTargets = []updateTarget{
// cmdUpdate reports what can be updated and what is already current.
//
// Bare `felis update` prints the status of every tracked component. Selector flags
// (--panel/--velocity/--plugins/--k3s/--cloudflared/--jre/--postgres/--mc/--all) narrow that report to the components
// (--panel/--velocity/--mc/--plugins/--all) narrow that report to the components
// they name AND print how to apply each one. --force additionally prints the apply
// instruction for a selected component that is already up to date, for the
// reinstall/repair case.
//
// It never applies anything and never mutates the node, so unlike setup/breakGlass
// it needs no root, apart from PostgreSQL's version, which the felis-postgres container
// answers through the cluster's admin kubeconfig. The versions it reads come from this
// host: k3s, cloudflared and PostgreSQL answer `--version`, Velocity's version is read out of the installed jar's
// manifest, the JRE's out of its release file, and felis-api's is this binary's own
// build stamp — the same value `felis version` prints, which is what the user asked
// to be the source of truth.
// it needs no root. The versions it reads come from this host: k3s and cloudflared
// answer `--version`, Velocity's version is read out of the installed jar's
// manifest, and felis-api's is this binary's own build stamp — the same value
// `felis version` prints, which is what the user asked to be the source of truth.
func cmdUpdate(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("update", flag.ContinueOnError)
fs.SetOutput(stderr)
flags := map[string]*bool{}
for _, t := range updateTargets {
flags[t.selector] = fs.Bool(t.selector, false, t.help)
}
panel := fs.Bool("panel", false, "select the panel + control plane (felis-api)")
velocity := fs.Bool("velocity", false, "select the Velocity proxy")
mc := fs.Bool("mc", false, "select Minecraft (pinned; reported only)")
plugins := fs.Bool("plugins", false, "select the Felis plugin jars (velocity/paper/limbo)")
all := fs.Bool("all", false, "select every component above")
force := fs.Bool("force", false, "print the apply command for a selected component even when it is already up to date")
velocityJar := fs.String("velocity-jar", updater.DefaultVelocityJarPath, "path to the installed Velocity jar to read the current version from")
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml, read for the maintenance window the panel stores")
record := fs.Bool("record", false, "also store this check for the panel's Updates page (felis-update-check.timer runs it daily)")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -177,8 +121,8 @@ func cmdUpdate(args []string, stdout, stderr io.Writer) int {
}
selected := map[string]bool{}
for sel, on := range flags {
if *on || *all {
for sel, on := range map[string]bool{"panel": *panel, "velocity": *velocity, "mc": *mc, "plugins": *plugins} {
if on || *all {
selected[sel] = true
}
}
@@ -186,10 +130,9 @@ func cmdUpdate(args []string, stdout, stderr io.Writer) int {
ctx, cancel := context.WithTimeout(context.Background(), updateTimeout)
defer cancel()
src := updater.NewRoutingSource(updater.Topology())
rn := &updater.Runner{
Gatherer: updater.NewHostGatherer(resolvedVersion(), *velocityJar),
Source: src,
Source: updater.NewRoutingSource(updater.Topology()),
// Notifier and Applier stay nil on purpose: a human typing this command IS the
// notification, and nothing here applies. The zero Window below means every
// Scheduled component degrades to a notify, so the report can never claim an
@@ -201,104 +144,13 @@ func cmdUpdate(args []string, stdout, stderr io.Writer) int {
return 1
}
now := time.Now()
win, winErr := readUpdateWindow(ctx, *cfgPath)
fmt.Fprint(stdout, renderWindowLine(win, winErr, now))
fmt.Fprint(stdout, renderUpdateReport(res, selected))
fmt.Fprint(stdout, renderNotes(src.Notes(), selected))
if len(selected) > 0 {
if winErr == nil && !win.Start.IsZero() && !win.Contains(now) {
fmt.Fprint(stdout, "Warning: this is outside the maintenance window; the commands below take effect as soon as you run them.\n")
}
fmt.Fprint(stdout, renderApplyGuidance(res, selected, *force))
}
if *record {
// A fresh context: the discovery pass may have spent most of updateTimeout.
rctx, rcancel := context.WithTimeout(context.Background(), updateWindowTimeout)
defer rcancel()
if err := recordUpdateStatus(rctx, *cfgPath, buildStatusReport(res, src.Notes(), resolvedVersion(), now)); err != nil {
fmt.Fprintf(stderr, "felis update: record the check for the panel: %v\n", err)
return 1
}
fmt.Fprint(stdout, "Recorded this check for the panel's Updates page.\n")
}
return 0
}
// buildStatusReport turns one run into the record the panel shows: every planned
// component in plan order, then each component whose installed version could not
// be read, by name. A component the feed could not answer for is StateUnknown with
// the reason, never StateCurrent: the panel must not call a component current when
// nobody could check.
func buildStatusReport(res updater.Result, notes map[string]string, felis string, now time.Time) updates.StatusReport {
selectorOf := map[string]string{}
for _, t := range updateTargets {
if t.component != "" && selectorOf[t.component] == "" {
selectorOf[t.component] = t.selector
}
}
rep := updates.StatusReport{CheckedAt: now.UTC(), Felis: felis, Components: []updates.ComponentStatus{}}
for _, a := range res.RunResult.Plan {
cs := updates.ComponentStatus{
Name: a.Component,
Current: a.Current.String(),
Selector: selectorOf[a.Component],
Note: notes[a.Component],
}
switch {
case a.Kind == updates.ActionPinned:
cs.State = updates.StatePinned
case a.Kind == updates.ActionNotify || a.Kind == updates.ActionApply:
cs.State = updates.StateAvailable
cs.Latest = a.Latest.String()
case a.LatestKnown:
cs.State = updates.StateCurrent
default:
cs.State = updates.StateUnknown
if err := res.RunResult.SourceErrors[a.Component]; err != nil {
cs.Error = err.Error()
}
}
rep.Components = append(rep.Components, cs)
}
names := make([]string, 0, len(res.GatherErrors))
for name := range res.GatherErrors {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
rep.Components = append(rep.Components, updates.ComponentStatus{
Name: name,
State: updates.StateUnreadable,
Selector: selectorOf[name],
Note: notes[name],
Error: res.GatherErrors[name].Error(),
})
}
return rep
}
// recordUpdateStatus upserts rep into platform_settings[updates.StatusKey], the
// row the API serves to the panel's Updates page.
func recordUpdateStatus(ctx context.Context, cfgPath string, rep updates.StatusReport) error {
cfg, err := config.Load(cfgPath)
if err != nil {
return err
}
v, err := json.Marshal(rep)
if err != nil {
return err
}
conn, err := pgx.Connect(ctx, cfg.Database.URL)
if err != nil {
return err
}
defer conn.Close(context.Background())
_, err = conn.Exec(ctx, `INSERT INTO platform_settings (key, value) VALUES ($1, $2::jsonb)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`, updates.StatusKey, string(v))
return err
}
// renderUpdateReport renders the component status table. With no selectors it shows
// every tracked component; with selectors it shows only the components those
// selectors name, so `felis update --velocity` is a focused answer rather than the
@@ -347,23 +199,6 @@ func renderUpdateReport(res updater.Result, selected map[string]bool) string {
return b.String()
}
// renderNotes prints what the release lookups learned beyond the versions (today: a
// PostgreSQL major past its end of life), for the components the selectors show.
func renderNotes(notes map[string]string, selected map[string]bool) string {
names := make([]string, 0, len(notes))
for name := range notes {
if len(selected) == 0 || selectedCovers(selected, name) {
names = append(names, name)
}
}
sort.Strings(names)
var b strings.Builder
for _, name := range names {
fmt.Fprintf(&b, "%-13s note: %s\n", name, notes[name])
}
return b.String()
}
// writeErrs appends one explanatory line per failed component, in a stable order so
// the output does not shuffle between runs, honouring the active selector filter.
func writeErrs(b *strings.Builder, label string, errs map[string]error, selected map[string]bool) {
@@ -399,7 +234,7 @@ func renderApplyGuidance(res updater.Result, selected map[string]bool, force boo
}
var b strings.Builder
var offeredInstaller bool
var offeredCommand bool
for _, t := range updateTargets {
if !selected[t.selector] {
continue
@@ -427,7 +262,7 @@ func renderApplyGuidance(res updater.Result, selected map[string]bool, force boo
fmt.Fprintf(&b, " note: cannot tell whether %s is current — its latest version could not be discovered (see above); this reinstalls it either way\n", t.component)
}
fmt.Fprintf(&b, " run: %s\n", strings.ReplaceAll(t.command, "{ref}", installerRef(byComponent)))
offeredInstaller = offeredInstaller || t.installer
offeredCommand = true
}
// Only explain the command when one was actually offered; a --mc-only run has
// nothing to run and the trailer would be a non-sequitur.
@@ -435,13 +270,13 @@ func renderApplyGuidance(res updater.Result, selected map[string]bool, force boo
// One trailer serves every selector now: setup is not an apply path at all on a
// completed install (shouldRunHostBootstrapBeforeConfig only enters the host
// bootstrap while an install marker is missing), so the installer re-run is the one
// worked path for every component Felis installs and there is no per-component exception left
// worked path for all three components and there is no per-component exception left
// to scope. Two caveats stay because following the advice without them bites real
// hosts: the channel is not persisted anywhere (a bare re-run on a main host quietly
// moves it onto releases), and the private repo's one-liner needs the read token
// back in the environment before it can resolve anything.
if offeredInstaller {
b.WriteString("\nRe-running the installer applies each installer command above: it fetches the newest version on\nthe channel in effect and re-applies the bundle (release is the default). The channel\nis not persisted, so pass FELIS_VERSION_BOOTSTRAP=dev if this host tracks main. While\nthis repo is private, the one-liner above 404s without a token; the README's install\nsection has the token'd form that works. felis setup is not this path: on a completed\ninstall it opens the config console and installs nothing newer. Restart game servers\nafterwards.\n")
if offeredCommand {
b.WriteString("\nRe-running the installer applies everything above: it fetches the newest version on\nthe channel in effect and re-applies the bundle (release is the default). The channel\nis not persisted, so pass FELIS_VERSION_BOOTSTRAP=dev if this host tracks main. While\nthis repo is private, the one-liner above 404s without a token; the README's install\nsection has the token'd form that works. felis setup is not this path: on a completed\ninstall it opens the config console and installs nothing newer. Restart game servers\nafterwards.\n")
}
return b.String()
}
@@ -464,66 +299,12 @@ func installerRef(byComponent map[string]updates.Action) string {
}
// isReleaseTag reports whether v was read from a stable vX.Y.Z tag, the only refs
// release.yml publishes a binary for. A source build stamps v0.0.0+g<commit>, which
// names no tag, so build metadata disqualifies a version too.
// release.yml publishes a binary for.
func isReleaseTag(v updates.Version) bool {
s := v.String()
if !strings.HasPrefix(s, "v") || v.IsPrerelease() || strings.Contains(s, "+") {
if !strings.HasPrefix(s, "v") || v.IsPrerelease() {
return false
}
_, err := updates.Parse(s)
return err == nil
}
// updateWindowTimeout bounds the maintenance-window read, so an unreachable
// database costs the report a line and never the report itself.
const updateWindowTimeout = 3 * time.Second
// readUpdateWindow reads the maintenance window the panel stores
// (platform_settings "update_window"). Felis applies nothing on its own: this
// command is the window's consumer, showing it and warning before an apply
// outside it. A missing row is an unset window.
func readUpdateWindow(ctx context.Context, cfgPath string) (updates.Window, error) {
cfg, err := config.Load(cfgPath)
if err != nil {
return updates.Window{}, err
}
ctx, cancel := context.WithTimeout(ctx, updateWindowTimeout)
defer cancel()
conn, err := pgx.Connect(ctx, cfg.Database.URL)
if err != nil {
return updates.Window{}, err
}
defer conn.Close(context.Background())
var raw []byte
err = conn.QueryRow(ctx, `SELECT value FROM platform_settings WHERE key = 'update_window'`).Scan(&raw)
if errors.Is(err, pgx.ErrNoRows) {
return updates.Window{}, nil
}
if err != nil {
return updates.Window{}, err
}
var w updates.Window
if err := json.Unmarshal(raw, &w); err != nil {
return updates.Window{}, fmt.Errorf("stored window: %w", err)
}
return w, nil
}
// renderWindowLine is the report's first line: where now sits against the
// maintenance window.
func renderWindowLine(w updates.Window, err error, now time.Time) string {
const layout = "2006-01-02 15:04 MST"
switch {
case err != nil:
return fmt.Sprintf("Maintenance window: unknown (%v).\n", err)
case w.Start.IsZero() || w.End.IsZero():
return "Maintenance window: not set; apply whenever suits you.\n"
case w.Contains(now):
return fmt.Sprintf("Maintenance window: open now, until %s.\n", w.End.Local().Format(layout))
case now.Before(w.Start):
return fmt.Sprintf("Maintenance window: opens %s, until %s. Felis applies nothing on its own; run the apply commands inside it.\n", w.Start.Local().Format(layout), w.End.Local().Format(layout))
default:
return fmt.Sprintf("Maintenance window: ended %s; set a new one in the panel before applying.\n", w.End.Local().Format(layout))
}
}
cmd/felis/update_test.go
-132
View File
@@ -1,11 +1,9 @@
package main
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/updater"
"felis.lolicon.best/internal/updates"
@@ -201,133 +199,3 @@ func TestApplyGuidanceReadsTheInstallerAtTheReleaseTag(t *testing.T) {
}
}
}
// Every selector in the table is a flag: the FlagSet is built from the table.
func TestUpdateSelectorsAreFlags(t *testing.T) {
var out, errb strings.Builder
if code := cmdUpdate([]string{"-h"}, &out, &errb); code != 2 {
t.Fatalf("-h exit = %d, want 2", code)
}
for _, target := range updateTargets {
if !strings.Contains(errb.String(), "-"+target.selector+"\n") {
t.Errorf("usage has no -%s flag:\n%s", target.selector, errb.String())
}
}
}
// k3s and cloudflared move only when the re-run is told to; the release pins the JRE
// build and the PostgreSQL image, so their guidance is the plain re-run.
func TestApplyGuidanceForHostDependencies(t *testing.T) {
notify := func(c string) updater.Result {
return planResult([]updates.Action{{Component: c, Kind: updates.ActionNotify, LatestKnown: true}})
}
for _, sel := range []string{"k3s", "cloudflared"} {
out := renderApplyGuidance(notify(sel), map[string]bool{sel: true}, false)
if !strings.Contains(out, "sudo FELIS_UPGRADE_DEPS=1 bash") || !strings.Contains(out, "Re-running the installer") {
t.Errorf("--%s guidance must re-run the installer with FELIS_UPGRADE_DEPS=1:\n%s", sel, out)
}
}
jre := renderApplyGuidance(notify("jre"), map[string]bool{"jre": true}, false)
if !strings.Contains(jre, "| sudo bash") || strings.Contains(jre, "FELIS_UPGRADE_DEPS") {
t.Errorf("--jre guidance is the plain installer re-run:\n%s", jre)
}
pg := renderApplyGuidance(notify("postgresql"), map[string]bool{"postgres": true}, false)
if !strings.Contains(pg, "| sudo bash") || strings.Contains(pg, "FELIS_UPGRADE_DEPS") || strings.Contains(pg, "apt-get") || strings.Contains(pg, "systemctl") {
t.Errorf("--postgres guidance is the plain installer re-run that moves the image pin:\n%s", pg)
}
}
func TestRenderNotesHonoursSelectors(t *testing.T) {
notes := map[string]string{"postgresql": "PostgreSQL 13 reached end of life on 2025-11-13"}
if out := renderNotes(notes, nil); !strings.Contains(out, "postgresql") || !strings.Contains(out, "note: PostgreSQL 13 reached end of life") {
t.Errorf("unfiltered notes = %q", out)
}
if out := renderNotes(notes, map[string]bool{"postgres": true}); !strings.Contains(out, "end of life") {
t.Errorf("--postgres must show its note, got %q", out)
}
if out := renderNotes(notes, map[string]bool{"velocity": true}); out != "" {
t.Errorf("--velocity must not show the postgresql note, got %q", out)
}
}
// A source build's v0.0.0+g<commit> names no tag, so the installer one-liner has to
// fall back to main instead of a 404ing ref.
func TestInstallerRefNamesATag(t *testing.T) {
v := func(s string) updates.Version {
t.Helper()
x, err := updates.Parse(s)
if err != nil {
t.Fatal(err)
}
return x
}
cases := []struct {
name string
api updates.Action
want string
}{
{"newest release", updates.Action{Current: v("v1.2.0"), Latest: v("v1.3.0"), LatestKnown: true}, "v1.3.0"},
{"feed down, host on a release", updates.Action{Current: v("v1.2.0")}, "v1.2.0"},
{"source build", updates.Action{Current: v("v0.0.0+gunknown")}, "main"},
{"source build with commit", updates.Action{Current: v("v0.0.0+g1a2b3c4")}, "main"},
{"prerelease", updates.Action{Current: v("v1.3.0-rc.1")}, "main"},
}
for _, c := range cases {
if got := installerRef(map[string]updates.Action{"felis-api": c.api}); got != c.want {
t.Errorf("%s: installerRef = %q, want %q", c.name, got, c.want)
}
}
if got := installerRef(nil); got != "main" {
t.Errorf("no felis-api row: installerRef = %q, want main", got)
}
}
func mustVersion(t *testing.T, s string) updates.Version {
t.Helper()
v, err := updates.Parse(s)
if err != nil {
t.Fatalf("parse %q: %v", s, err)
}
return v
}
// The record the panel shows keeps every component with a state that cannot be
// mistaken: a feed failure is "unknown" with its reason, an unreadable install is
// listed after the plan, and each row carries the selector that prints its apply.
func TestBuildStatusReport(t *testing.T) {
res := planResult([]updates.Action{
{Component: "felis-api", Current: mustVersion(t, "v0.4.0"), Latest: mustVersion(t, "v0.5.0"), LatestKnown: true, Kind: updates.ActionNotify},
{Component: "velocity", Current: mustVersion(t, "3.4.0"), Latest: mustVersion(t, "3.4.0"), LatestKnown: true, Kind: updates.ActionNone},
{Component: "k3s", Current: mustVersion(t, "v1.36.2+k3s1"), Kind: updates.ActionNone},
{Component: "cloudflared", Current: mustVersion(t, "2026.6.1"), Latest: mustVersion(t, "2026.9.0"), LatestKnown: true, Kind: updates.ActionApply},
{Component: "mc-lobby", Current: mustVersion(t, "1.21.4"), Kind: updates.ActionPinned},
})
res.RunResult.SourceErrors["k3s"] = errors.New("github: HTTP 403")
res.GatherErrors["postgresql"] = errors.New("psql: not found")
res.GatherErrors["jre"] = errors.New("release file missing")
notes := map[string]string{"postgresql": "PostgreSQL 13 is past its end of life", "velocity": "pinned minor 3.4"}
now := time.Date(2026, 9, 25, 3, 4, 5, 0, time.FixedZone("CST", 8*3600))
b, err := json.Marshal(buildStatusReport(res, notes, "v0.4.0", now))
if err != nil {
t.Fatal(err)
}
want := `{"checked_at":"2026-09-24T19:04:05Z","felis":"v0.4.0","components":[` +
`{"name":"felis-api","current":"v0.4.0","latest":"v0.5.0","state":"available","selector":"panel"},` +
`{"name":"velocity","current":"3.4.0","state":"current","selector":"velocity","note":"pinned minor 3.4"},` +
`{"name":"k3s","current":"v1.36.2+k3s1","state":"unknown","selector":"k3s","error":"github: HTTP 403"},` +
`{"name":"cloudflared","current":"2026.6.1","latest":"2026.9.0","state":"available","selector":"cloudflared"},` +
`{"name":"mc-lobby","current":"1.21.4","state":"pinned"},` +
`{"name":"jre","state":"unreadable","selector":"jre","error":"release file missing"},` +
`{"name":"postgresql","state":"unreadable","selector":"postgres","note":"PostgreSQL 13 is past its end of life","error":"psql: not found"}]}`
if string(b) != want {
t.Errorf("status report =\n%s\nwant\n%s", b, want)
}
// Nothing tracked still records an empty list, so the panel can tell "checked,
// nothing to show" from a report that never arrived.
b, _ = json.Marshal(buildStatusReport(planResult(nil), nil, "v0.4.0", now))
if !strings.Contains(string(b), `"components":[]`) {
t.Errorf("an empty check = %s, want an empty components list", b)
}
}
cmd/felis/update_window_test.go
-41
View File
@@ -1,41 +0,0 @@
package main
import (
"errors"
"testing"
"time"
"felis.lolicon.best/internal/updates"
)
func TestRenderWindowLinePlacesNowAgainstTheWindow(t *testing.T) {
prev := time.Local
time.Local = time.FixedZone("CST", 8*3600)
t.Cleanup(func() { time.Local = prev })
w := updates.Window{
Start: time.Date(2026, 9, 26, 18, 0, 0, 0, time.UTC),
End: time.Date(2026, 9, 26, 20, 0, 0, 0, time.UTC),
}
cases := []struct {
name string
w updates.Window
err error
now time.Time
want string
}{
{"unreadable", updates.Window{}, errors.New("connection refused"), w.Start, "Maintenance window: unknown (connection refused).\n"},
{"unset", updates.Window{}, nil, w.Start, "Maintenance window: not set; apply whenever suits you.\n"},
{"half set", updates.Window{Start: w.Start}, nil, w.Start, "Maintenance window: not set; apply whenever suits you.\n"},
{"at the opening instant", w, nil, w.Start, "Maintenance window: open now, until 2026-09-27 04:00 CST.\n"},
{"before", w, nil, w.Start.Add(-time.Minute), "Maintenance window: opens 2026-09-27 02:00 CST, until 2026-09-27 04:00 CST. Felis applies nothing on its own; run the apply commands inside it.\n"},
{"at the closing instant", w, nil, w.End, "Maintenance window: ended 2026-09-27 04:00 CST; set a new one in the panel before applying.\n"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := renderWindowLine(tc.w, tc.err, tc.now); got != tc.want {
t.Fatalf("got %q\nwant %q", got, tc.want)
}
})
}
}
cmd/felis/watchdog.go
+107 -348
View File
@@ -2,12 +2,12 @@ package main
import (
"context"
"database/sql"
"errors"
"flag"
"fmt"
"io"
"net"
"net/http"
"os"
"strings"
"time"
@@ -29,134 +29,42 @@ const proxyFor = 3 * time.Minute
// cmdWatchdog runs one pass of the platform watchdog (internal/watchdog): it
// checks the cluster, PostgreSQL, the game proxy, the database backups and the
// host (disks, memory, its address and clock), prints every finding, and mails
// the platform owners what came due.
// host, prints every finding, and mails the platform owners what came due.
// deploy/bootstrap.sh runs it every two minutes from felis-watchdog.timer.
func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
fs.SetOutput(stderr)
var w watchdogFlags
w.register(fs)
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")
quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/postgresql,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)
controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")
offsiteStatus := fs.String("offsite-status", offsite.DefaultStatusFile, "the record `felis offsite sync` leaves, checked when [offsite] is configured")
toolsStatus := fs.String("build-tools-status", defaultBuildToolsStatus, "the record `felis mirror-build-tools` leaves, checked when builds scan against the registry's DB copy")
dryRun := fs.Bool("dry-run", false, "print every finding and the mail that is due; send nothing and keep the state as it was")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
now := time.Now()
if w.unitFailed {
return watchdogUnitFailed(unitFailedRun{
cfgPath: w.cfgPath, statePath: w.statePath, fallbackPath: w.fallbackState, quietPath: w.quietPath,
offsiteStatus: w.offsiteStatus, heartbeatFile: w.heartbeatFile,
result: os.Getenv("MONITOR_SERVICE_RESULT"), exitStatus: os.Getenv("MONITOR_EXIT_STATUS"),
send: watchdogSender, client: http.DefaultClient, now: now,
}, stdout, stderr)
}
cfg, err := config.Load(w.cfgPath)
cfg, err := config.Load(*cfgPath)
if err != nil {
fmt.Fprintf(stderr, "felis watchdog: %v\n", err)
return 1
}
loadPath := watchdog.NewestState(w.statePath, w.fallbackState)
state, aside, err := watchdog.RecoverState(loadPath, now)
state, err := watchdog.LoadState(*statePath)
if err != nil {
fmt.Fprintf(stderr, "felis watchdog: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
now := time.Now()
var report watchdog.Report
if aside != "" {
fmt.Fprintf(stderr, "felis watchdog: %s was unreadable; moved it to %s and started over\n", loadPath, aside)
report.Findings = append(report.Findings, watchdog.StateSetAside(aside))
}
cl, owners, ownersErr := watchdogProbes(ctx, w, cfg, now, &report)
if cfg.SMTP.Host != "" {
refreshSMTPPassword(ctx, w.smtpPasswordFile, cl, w.controlNS, state, stderr)
}
state.Relay = cachedRelay(cfg.SMTP)
if ownersErr == nil {
state.Recipients = owners
}
if len(report.Findings) == 0 {
fmt.Fprintln(stdout, "felis watchdog: every check passed")
}
for _, f := range report.Findings {
fmt.Fprintf(stdout, "felis watchdog: [%s] %s: %s\n", f.Severity, f.Key, f.SummaryEN)
}
plan := state.Observe(report, now)
host, _ := os.Hostname()
subject, body := plan.Message(host, now)
beat := heartbeat{
standby: standsBy(cfg.Offsite.Enabled(), w.offsiteStatus),
quiet: now.Before(watchdog.QuietUntil(w.quietPath)),
}
if beat.url, err = readHeartbeatURL(w.heartbeatFile); err != nil {
fmt.Fprintf(stderr, "felis watchdog: %v; pinging no heartbeat\n", err)
}
if w.dryRun {
if plan.Empty() {
fmt.Fprintln(stdout, "felis watchdog: nothing is due to be mailed")
} else {
fmt.Fprintf(stdout, "felis watchdog: due to be mailed to %s:\nSubject: %s\n\n%s", strings.Join(state.Recipients, ", "), subject, strings.ReplaceAll(body, "\r\n", "\n"))
}
if beat.url != "" {
fmt.Fprintf(stdout, "felis watchdog: a run pings the heartbeat at %s\n", redactURL(beat.url))
}
return 0
}
m := configMailer(cfg.SMTP, state.SMTPPassword, watchdogSender)
unheard, mailFailed := m.deliver(ctx, state, plan, subject, body, mailHold(w.quietPath, cfg.Offsite.Enabled(), w.offsiteStatus, now), now, stdout, stderr)
saveErr := watchdog.SaveStateOr(w.statePath, w.fallbackState, state)
if saveErr != nil {
fmt.Fprintf(stderr, "felis watchdog: save state: %v\n", saveErr)
}
beat.report = failureReport(unheard, mailFailed, saveErr, state.Open(), report)
beat.fail = beat.report != ""
beat.send(http.DefaultClient, stdout, stderr)
if mailFailed || saveErr != nil {
return 1
}
return 0
}
// watchdogFlags are felis watchdog's flags. felis doctor reads them back from
// the ExecStart= line of felis-watchdog.service, so it checks what the timer's
// runs check, with the same paths.
type watchdogFlags struct {
cfgPath, statePath, fallbackState, smtpPasswordFile, quietPath string
backupDir, diskPaths, certDirs, proxyAddr, nodeIP, controlNS string
offsiteStatus, toolsStatus, heartbeatFile string
dryRun, unitFailed bool
}
func (w *watchdogFlags) register(fs *flag.FlagSet) {
fs.StringVar(&w.cfgPath, "config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
fs.StringVar(&w.statePath, "state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")
fs.StringVar(&w.fallbackState, "fallback-state", watchdog.FallbackStatePath, "where a run keeps its state while -state cannot be written, so what it mailed is not mailed again (tmpfs: until the host restarts; \"\" keeps none)")
fs.StringVar(&w.smtpPasswordFile, "smtp-password-file", hostSMTPPasswordPath, "the relay password `felis setup` keeps on the host; the felis-smtp Secret stands in while it is missing")
fs.StringVar(&w.quietPath, "quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
fs.StringVar(&w.backupDir, "backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
fs.StringVar(&w.diskPaths, "disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
fs.StringVar(&w.certDirs, "k3s-cert-dirs", strings.Join(watchdog.K3sCertDirs, ","), `k3s certificate directories whose *.crt files must not be near expiry ("" skips the check)`)
fs.StringVar(&w.proxyAddr, "proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)
fs.StringVar(&w.nodeIP, "node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`)
fs.StringVar(&w.controlNS, "control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")
fs.StringVar(&w.offsiteStatus, "offsite-status", offsite.DefaultStatusFile, "the record `felis offsite sync` leaves, checked when [offsite] is configured")
fs.StringVar(&w.toolsStatus, "build-tools-status", defaultBuildToolsStatus, "the record `felis mirror-build-tools` leaves, checked when builds scan against the registry's DB copy")
fs.BoolVar(&w.dryRun, "dry-run", false, "print every finding and the mail that is due; send nothing and keep the state as it was")
fs.StringVar(&w.heartbeatFile, "heartbeat-file", defaultHeartbeatFile, "file holding the heartbeat URL each run pings, a dead man's switch at a monitoring service that alerts when the pings stop (no file pings nothing)")
fs.BoolVar(&w.unitFailed, "unit-failed", false, "report a failed run of felis-watchdog.service instead of checking; felis-watchdog-failed.service runs this through OnFailure=")
}
// watchdogProbes is one pass of every check, appended to report. cl is the
// cluster client, nil while the API server is unreachable; owners is who the
// alerts go to, and ownersErr why PostgreSQL did not say.
func watchdogProbes(ctx context.Context, w watchdogFlags, cfg *config.Config, now time.Time, report *watchdog.Report) (cl client.Client, owners []string, ownersErr error) {
add := func(f *watchdog.Finding) {
if f != nil {
report.Findings = append(report.Findings, *f)
@@ -171,237 +79,90 @@ func watchdogProbes(ctx context.Context, w watchdogFlags, cfg *config.Config, no
cl, err := buildSystemServerClient()
var found []watchdog.Finding
if err == nil {
found, err = watchdog.Cluster{Client: cl, ControlNamespace: w.controlNS, MinecraftNamespace: minecraftNS}.Check(ctx, now)
found, err = watchdog.Cluster{Client: cl, ControlNamespace: *controlNS, MinecraftNamespace: minecraftNS}.Check(ctx, now)
}
if err != nil {
cl = nil
f := watchdog.KubeAPIDown(err)
add(&f)
report.Unknown = append(report.Unknown, watchdog.ClusterPrefixes...)
} else {
report.Findings = append(report.Findings, found...)
if cfg.SMTP.Host != "" {
refreshSMTPPassword(ctx, cl, *controlNS, state, stderr)
}
}
if owners, ownersErr = ownerEmails(ctx, cfg.Database.URL); ownersErr != nil {
f := watchdog.PostgresDown(ownersErr)
if recipients, err := ownerEmails(ctx, cfg.Database.URL); err != nil {
f := watchdog.PostgresDown(err)
add(&f)
} else {
state.Recipients = recipients
}
if w.proxyAddr != "" {
add(proxyFinding(ctx, w.proxyAddr))
if *proxyAddr != "" {
add(proxyFinding(ctx, *proxyAddr))
}
if w.backupDir != "" {
add(watchdog.BackupFinding(w.backupDir, now))
if *backupDir != "" {
add(watchdog.BackupFinding(*backupDir, now))
}
if cfg.Offsite.Enabled() {
add(watchdog.OffsiteFinding(w.offsiteStatus, now))
add(watchdog.OffsiteFinding(*offsiteStatus, now))
}
if usesMirroredScanDB(cfg) {
add(watchdog.ScanDBFinding(w.toolsStatus, now))
add(watchdog.ScanDBFinding(*toolsStatus, now))
}
report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(w.diskPaths))...)
report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(*diskPaths))...)
add(watchdog.MemoryFinding("/proc/meminfo"))
add(watchdog.CertFinding(splitList(w.certDirs), now))
if w.nodeIP != "" {
if held, err := watchdog.HostAddresses(); err == nil {
add(watchdog.AddressFinding(w.nodeIP, held))
if len(report.Findings) == 0 {
fmt.Fprintln(stdout, "felis watchdog: every check passed")
}
for _, f := range report.Findings {
fmt.Fprintf(stdout, "felis watchdog: [%s] %s: %s\n", f.Severity, f.Key, f.SummaryEN)
}
plan := state.Observe(report, now)
host, _ := os.Hostname()
subject, body := plan.Message(host, now)
if *dryRun {
if plan.Empty() {
fmt.Fprintln(stdout, "felis watchdog: nothing is due to be mailed")
} else {
fmt.Fprintf(stdout, "felis watchdog: due to be mailed to %s:\nSubject: %s\n\n%s", strings.Join(state.Recipients, ", "), subject, strings.ReplaceAll(body, "\r\n", "\n"))
}
return 0
}
add(watchdog.ClockFinding(watchdog.ClockStatus()))
return cl, owners, ownersErr
}
// failureReport is what the heartbeat's failure ping carries, "" when the run
// pings success: the alerts this run knows of reach no one (a mail that
// failed, or no relay or recipient while something is open), or the state did
// not save to its file: kept on tmpfs it holds until the host restarts, which
// forgets what was mailed, and with nowhere to keep it the next run mails the
// same alerts again.
func failureReport(unheard string, mailFailed bool, saveErr error, open bool, r watchdog.Report) string {
var why []string
if unheard != "" && (mailFailed || open) {
why = append(why, "the alerts reach no one: "+unheard)
save := func() int {
if err := watchdog.SaveState(*statePath, state); err != nil {
fmt.Fprintf(stderr, "felis watchdog: save state: %v\n", err)
return 1
}
return 0
}
if saveErr != nil {
why = append(why, "the watchdog state did not save: "+saveErr.Error())
if plan.Empty() {
return save()
}
if len(why) == 0 {
return ""
if until := watchdog.QuietUntil(*quietPath); now.Before(until) {
fmt.Fprintf(stdout, "felis watchdog: quiet until %s (installer running); holding this mail: %s\n", until.UTC().Format(time.RFC3339), subject)
return save()
}
return strings.Join(why, "\n") + "\n\n" + findingLines(r)
}
// findingLines is the report as the journal shows it.
func findingLines(r watchdog.Report) string {
var b strings.Builder
for _, f := range r.Findings {
fmt.Fprintf(&b, "[%s] %s: %s\n", f.Severity, f.Key, f.SummaryEN)
}
return b.String()
}
// mailer is how a run reaches the owners.
type mailer struct {
relay *watchdog.Relay // nil: no [smtp] relay
password string
send func(*mail.SMTP) alertSender
}
// deliver mails plan to the owners unless hold says why it waits, and commits
// it once it reached them, or once it is logged because nothing can reach them.
// unheard is why the owners hear nothing of this run's alerts, "" when they do;
// mailFailed is a mail that did not go out, left uncommitted so the same
// alerts come due again next run.
func (m mailer) deliver(ctx context.Context, state *watchdog.State, plan watchdog.Plan, subject, body, hold string, now time.Time, stdout, stderr io.Writer) (unheard string, mailFailed bool) {
switch {
case m.relay == nil:
unheard = "no [smtp] relay is configured"
case cfg.SMTP.Host == "":
fmt.Fprintf(stdout, "felis watchdog: no [smtp] relay configured, so this is logged only: %s\n", subject)
case len(state.Recipients) == 0:
unheard = "no owner account has a verified email"
}
switch {
case plan.Empty():
case hold != "":
fmt.Fprintf(stdout, "felis watchdog: %s; holding this mail: %s\n", hold, subject)
case unheard != "":
fmt.Fprintf(stdout, "felis watchdog: %s, so this is logged only: %s\n", unheard, subject)
state.Commit(plan, now)
fmt.Fprintf(stdout, "felis watchdog: no owner account has a verified email, so this is logged only: %s\n", subject)
default:
relay := &mail.SMTP{Host: m.relay.Host, Port: m.relay.Port, From: m.relay.From, Username: m.relay.Username, Password: m.password, RequireTLS: m.relay.RequireTLS}
if err := sendAlert(ctx, m.send(relay), state.Recipients, subject, body); err != nil {
if err := sendAlert(ctx, cfg, state, subject, body); err != nil {
// Not committed: the same alerts come due again next run.
fmt.Fprintf(stderr, "felis watchdog: mail %q: %v\n", subject, err)
return "the alert mail failed: " + err.Error(), true
save()
return 1
}
fmt.Fprintf(stdout, "felis watchdog: mailed %s: %s\n", strings.Join(state.Recipients, ", "), subject)
state.Commit(plan, now)
}
return unheard, false
}
// configMailer reaches the owners through the relay felis.toml configures.
func configMailer(c config.SMTPConfig, cachedPassword string, send func(*mail.SMTP) alertSender) mailer {
return mailer{relay: cachedRelay(c), password: smtpPassword(c, cachedPassword), send: send}
}
// cachedRelay is what State.Relay keeps of c, nil when no relay is configured.
func cachedRelay(c config.SMTPConfig) *watchdog.Relay {
if c.Host == "" {
return nil
}
return &watchdog.Relay{Host: c.Host, Port: c.Port, From: c.From, Username: c.Username, RequireTLS: c.TLSRequired()}
}
// smtpPassword is the relay password a mail signs in with: the env var [smtp]
// password_ref names when it is set, else the one the state caches.
func smtpPassword(c config.SMTPConfig, cached string) string {
if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" {
return os.Getenv(ref)
}
return cached
}
// unitFailedRun is one run of felis-watchdog-failed.service.
type unitFailedRun struct {
cfgPath, statePath, fallbackPath, quietPath, offsiteStatus, heartbeatFile string
// result and exitStatus are what systemd hands an OnFailure= unit
// (MONITOR_SERVICE_RESULT, MONITOR_EXIT_STATUS; systemd 251 and later).
result, exitStatus string
send func(*mail.SMTP) alertSender
client *http.Client
now time.Time
}
// watchdogUnitFailed is felis-watchdog-failed.service, which systemd starts
// through OnFailure= when a run of felis-watchdog.service fails: a crash, a
// felis.toml that no longer loads, a hang past the unit's timeout, a mail
// that did not go out. Such a run checks and mails nothing, so this records
// the failure as the alert watchdog/run, due after five failed runs in a row
// and cleared by the next run that succeeds; mails it through the relay the
// last good run cached when felis.toml does not load; and pings the
// heartbeat's failure endpoint. Every other alert keeps its state.
func watchdogUnitFailed(r unitFailedRun, stdout, stderr io.Writer) int {
detail := failureDetail(r.result, r.exitStatus)
cfg, cfgErr := config.Load(r.cfgPath)
if cfgErr != nil {
detail += "; " + cfgErr.Error()
}
fmt.Fprintf(stdout, "felis watchdog: felis-watchdog.service failed: %s\n", detail)
offsiteOn := cfgErr != nil || cfg.Offsite.Enabled()
beat := heartbeat{
fail: true,
report: "felis-watchdog.service failed: " + detail,
standby: standsBy(offsiteOn, r.offsiteStatus),
quiet: r.now.Before(watchdog.QuietUntil(r.quietPath)),
}
var err error
if beat.url, err = readHeartbeatURL(r.heartbeatFile); err != nil {
fmt.Fprintf(stderr, "felis watchdog: %v; pinging no heartbeat\n", err)
}
state, err := watchdog.LoadState(watchdog.NewestState(r.statePath, r.fallbackPath))
if err != nil {
// The next run that gets that far moves a state that does not parse
// aside (watchdog.RecoverState).
fmt.Fprintf(stderr, "felis watchdog: %v; mailing nothing\n", err)
beat.report += "\nThe watchdog state does not load either, so nothing was mailed: " + err.Error()
beat.send(r.client, stdout, stderr)
return 1
}
plan := state.Observe(watchdog.Report{Findings: []watchdog.Finding{watchdog.WatchdogFailed(detail)}, Unknown: []string{""}}, r.now)
host, _ := os.Hostname()
subject, body := plan.Message(host, r.now)
m := mailer{relay: state.Relay, password: state.SMTPPassword, send: r.send}
if cfgErr == nil {
m = configMailer(cfg.SMTP, state.SMTPPassword, r.send)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
unheard, mailFailed := m.deliver(ctx, state, plan, subject, body, mailHold(r.quietPath, offsiteOn, r.offsiteStatus, r.now), r.now, stdout, stderr)
code := 0
if mailFailed {
code = 1
}
if err := watchdog.SaveStateOr(r.statePath, r.fallbackPath, state); err != nil {
fmt.Fprintf(stderr, "felis watchdog: save state: %v\n", err)
code = 1
}
if unheard != "" {
beat.report += "\nThe alerts reach no one: " + unheard
}
beat.send(r.client, stdout, stderr)
return code
}
// failureDetail names how felis-watchdog.service failed.
func failureDetail(result, exitStatus string) string {
switch {
case result == "":
return "systemd named no cause (journalctl -u felis-watchdog -n 50)"
case exitStatus == "":
return "result " + result
}
return "result " + result + ", exit status " + exitStatus
}
// mailHold is why this run's mail waits, "" when it goes out: the installer's
// quiet window, or this host standing by for another host's off-site bucket
// (offsite.Status.StandsBy). A standby host is a rehearsal, or a rebuild not
// yet taken over, and the owners its restored database names are that host's,
// which mails them itself.
func mailHold(quietPath string, offsiteOn bool, offsiteStatus string, now time.Time) string {
if until := watchdog.QuietUntil(quietPath); now.Before(until) {
return fmt.Sprintf("quiet until %s (installer running)", until.UTC().Format(time.RFC3339))
}
if !offsiteOn {
return ""
}
st, err := offsite.ReadStatus(offsiteStatus)
if err != nil {
return ""
}
if w := st.StandsBy(now); w != nil {
return fmt.Sprintf("this host stands by for %s, which wrote the off-site bucket at %s and mails its owners itself", w, w.At.UTC().Format(time.RFC3339))
}
return ""
state.Commit(plan, now)
return save()
}
// usesMirroredScanDB reports whether build scans read the vulnerability DB copy
@@ -415,40 +176,24 @@ func usesMirroredScanDB(cfg *config.Config) bool {
return repo == "" || strings.HasPrefix(repo, cfg.Registry.URL+"/mirror/")
}
// refreshSMTPPassword caches the relay password (relayPassword: the host copy at
// path, else the felis-smtp Secret), or forgets it when the Secret is gone (a
// relay without AUTH). The host copy is read even while the cluster is down, the
// time an alert matters most; without one, a down cluster keeps the cached
// password. An env var named by [smtp] password_ref, when set, wins at send time
// instead.
func refreshSMTPPassword(ctx context.Context, path string, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) {
password, err := relayPassword(ctx, path, cl, ns)
switch {
case errors.Is(err, errClusterUnreachable):
return
case err != nil:
fmt.Fprintf(stderr, "felis watchdog: read the relay password (keeping the cached one): %v\n", err)
return
}
state.SMTPPassword = password
}
// smtpSecretPassword reads the relay password from the felis-smtp Secret. A missing
// Secret is a relay without AUTH and reads as "".
func smtpSecretPassword(ctx context.Context, cl client.Client, ns string) (string, error) {
// refreshSMTPPassword caches the relay password from the felis-smtp Secret, or
// forgets it when the Secret is gone (a relay without AUTH). An env var named by
// [smtp] password_ref, when set, wins at send time instead.
func refreshSMTPPassword(ctx context.Context, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) {
var sec corev1.Secret
err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: platform.SMTPSecretName}, &sec)
if apierrors.IsNotFound(err) {
return "", nil
switch {
case apierrors.IsNotFound(err):
state.SMTPPassword = ""
case err != nil:
fmt.Fprintf(stderr, "felis watchdog: read %s/%s (keeping the cached relay password): %v\n", ns, platform.SMTPSecretName, err)
default:
state.SMTPPassword = string(sec.Data[platform.SMTPSecretPasswordKey])
}
if err != nil {
return "", err
}
return string(sec.Data[platform.SMTPSecretPasswordKey]), nil
}
// ownerEmails pings PostgreSQL and returns the owners an alert goes to
// (watchdog.OwnerEmails).
// ownerEmails pings PostgreSQL and returns the verified addresses of the
// enabled owner accounts, the people who can act on an alert.
func ownerEmails(ctx context.Context, url string) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
@@ -457,7 +202,24 @@ func ownerEmails(ctx context.Context, url string) ([]string, error) {
return nil, err
}
defer drv.Close()
return watchdog.OwnerEmails(ctx, drv.DB())
rows, err := drv.DB().QueryContext(ctx,
`SELECT email FROM users
WHERE role = 'owner' AND email_verified AND COALESCE(email, '') <> ''
AND NOT disabled AND deleted_at IS NULL
ORDER BY email`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []string
for rows.Next() {
var email sql.NullString
if err := rows.Scan(&email); err != nil {
return nil, err
}
out = append(out, email.String)
}
return out, rows.Err()
}
// proxyFinding dials the game proxy; players reach every server through it.
@@ -476,24 +238,21 @@ func proxyFinding(ctx context.Context, addr string) *watchdog.Finding {
}
}
// alertSender mails one alert; smtpSender is the real one.
type alertSender func(ctx context.Context, to, subject, body string) error
func smtpSender(relay *mail.SMTP) alertSender { return relay.SendNotice }
// watchdogSender is what a run mails through; tests stand a recorder in.
var watchdogSender = smtpSender
// sendAlert mails subject/body to every recipient; it fails only when no
// recipient got it.
func sendAlert(ctx context.Context, send alertSender, recipients []string, subject, body string) error {
func sendAlert(ctx context.Context, cfg *config.Config, state *watchdog.State, subject, body string) error {
password := state.SMTPPassword
if ref := cfg.SMTP.PasswordRef; ref != "" && os.Getenv(ref) != "" {
password = os.Getenv(ref)
}
relay := &mail.SMTP{Host: cfg.SMTP.Host, Port: cfg.SMTP.Port, From: cfg.SMTP.From, Username: cfg.SMTP.Username, Password: password}
var errs []error
for _, to := range recipients {
if err := send(ctx, to, subject, body); err != nil {
for _, to := range state.Recipients {
if err := relay.SendNotice(ctx, to, subject, body); err != nil {
errs = append(errs, fmt.Errorf("%s: %w", to, err))
}
}
if len(errs) == len(recipients) {
if len(errs) == len(state.Recipients) {
return errors.Join(errs...)
}
return nil
cmd/felis/watchdog_heartbeat.go
-164
View File
@@ -1,164 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
"felis.lolicon.best/internal/offsite"
)
// defaultHeartbeatFile holds the heartbeat URL deploy/bootstrap.sh writes from
// FELIS_WATCHDOG_HEARTBEAT_URL. It lives in /etc/felis, so a host rebuilt from
// a bundle pings the same check once it takes the off-site bucket over.
const defaultHeartbeatFile = "/etc/felis/watchdog-heartbeat-url"
const (
// heartbeatTimeout bounds one ping; a monitoring service slower than that
// is as good as down for the run.
heartbeatTimeout = 10 * time.Second
// heartbeatReportMax caps the report a failure ping carries: monitoring
// services keep about the first 10 KB of a ping's body.
heartbeatReportMax = 10000
)
// heartbeat is the ping one run sends to a dead man's switch at a monitoring
// service (Healthchecks.io, and the services that copy its API), which alerts
// its own users when the pings stop: the host down, the timer gone, the
// watchdog failing before it can mail. No check on the host can report those.
// A run whose alerts reach the owners GETs url; one whose alerts reach no one
// POSTs report to url/fail, or withholds the ping when url has a query, where
// no /fail can be added and the missing ping trips the check instead.
type heartbeat struct {
url string // "" pings nothing
fail bool
report string
// standby is a host standing by for another host's off-site bucket: a
// rehearsal, or a rebuild not taken over. It pings nothing, or it would
// keep the check of the host that writes the bucket green after that host
// died.
standby bool
// quiet is the installer's quiet window, which restarts things on
// purpose: no failure is pinged.
quiet bool
}
// send pings the heartbeat and logs the outcome.
func (b heartbeat) send(cl *http.Client, stdout, stderr io.Writer) {
switch {
case b.url == "":
return
case b.standby:
fmt.Fprintln(stdout, "felis watchdog: this host stands by for the off-site bucket; pinging no heartbeat (the host that writes the bucket pings it)")
return
case b.fail && b.quiet:
fmt.Fprintln(stdout, "felis watchdog: quiet while the installer runs; withholding the failure ping")
return
}
sent, err := b.ping(cl)
switch {
case err != nil:
fmt.Fprintf(stderr, "felis watchdog: heartbeat: %v\n", err)
case !sent:
fmt.Fprintf(stdout, "felis watchdog: withholding the heartbeat ping (the URL has a query, so it has no /fail endpoint)\n")
case b.fail:
fmt.Fprintf(stdout, "felis watchdog: pinged the heartbeat's failure endpoint at %s\n", redactURL(b.url))
}
}
// ping sends the request; sent is false when a failure withholds it.
func (b heartbeat) ping(cl *http.Client) (sent bool, err error) {
ctx, cancel := context.WithTimeout(context.Background(), heartbeatTimeout)
defer cancel()
method, target, body := http.MethodGet, b.url, io.Reader(nil)
if b.fail {
if strings.Contains(b.url, "?") {
return false, nil
}
method, target = http.MethodPost, strings.TrimSuffix(b.url, "/")+"/fail"
body = strings.NewReader(clipUTF8(b.report, heartbeatReportMax))
}
req, err := http.NewRequestWithContext(ctx, method, target, body)
if err != nil {
return false, fmt.Errorf("%s %s: bad URL", method, redactURL(target))
}
resp, err := cl.Do(req)
if err != nil {
// A url.Error quotes the whole URL, the check's key among it.
var ue *url.Error
if errors.As(err, &ue) {
err = ue.Err
}
return false, fmt.Errorf("%s %s: %w", method, redactURL(target), err)
}
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return false, fmt.Errorf("%s %s: %s", method, redactURL(target), resp.Status)
}
return true, nil
}
// clipUTF8 cuts s to at most n bytes without splitting a character.
func clipUTF8(s string, n int) string {
if len(s) <= n {
return s
}
return strings.ToValidUTF8(s[:n], "")
}
// readHeartbeatURL reads the heartbeat URL from path; no file is no heartbeat.
func readHeartbeatURL(path string) (string, error) {
if path == "" {
return "", nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return "", nil
}
if err != nil {
return "", err
}
s := strings.TrimSpace(string(raw))
if err := checkHeartbeatURL(s); err != nil {
return "", fmt.Errorf("%s: %w", path, err)
}
return s, nil
}
// checkHeartbeatURL accepts an http(s) URL with a host. Its error leaves the
// URL out: the path is the check's key, which anyone who reads it can ping in
// the host's name.
func checkHeartbeatURL(s string) error {
u, err := url.Parse(s)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || strings.ContainsAny(s, " \t\r\n") {
return errors.New("the heartbeat URL is not an http:// or https:// URL")
}
return nil
}
// redactURL is a heartbeat URL as logs show it: its scheme and host.
func redactURL(s string) string {
u, err := url.Parse(s)
if err != nil || u.Host == "" {
return "(the heartbeat URL)"
}
return u.Scheme + "://" + u.Host + "/..."
}
// standsBy reports whether this host stands by for another host's off-site
// bucket (offsite.Status.Standby), however old that record is: the
// installer's take-over or the host's first write ends it.
func standsBy(offsiteOn bool, statusPath string) bool {
if !offsiteOn {
return false
}
st, err := offsite.ReadStatus(statusPath)
return err == nil && st != nil && st.Standby
}
cmd/felis/watchdog_heartbeat_test.go
-615
View File
@@ -1,615 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"unicode/utf8"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/watchdog"
)
// pingLog is a monitoring service that records every ping it gets.
type pingLog struct {
mu sync.Mutex
pings []string // "METHOD path?query"
bodies []string
status int
}
func newPingServer(t *testing.T) (*pingLog, *httptest.Server) {
t.Helper()
l := &pingLog{status: http.StatusOK}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
l.mu.Lock()
defer l.mu.Unlock()
p := r.Method + " " + r.URL.Path
if r.URL.RawQuery != "" {
p += "?" + r.URL.RawQuery
}
l.pings = append(l.pings, p)
l.bodies = append(l.bodies, string(body))
w.WriteHeader(l.status)
}))
t.Cleanup(srv.Close)
return l, srv
}
func (l *pingLog) got() ([]string, []string) {
l.mu.Lock()
defer l.mu.Unlock()
return append([]string(nil), l.pings...), append([]string(nil), l.bodies...)
}
// TestHeartbeatSend: a run whose alerts reach the owners GETs the URL, one
// whose alerts reach no one POSTs its report to /fail, and a standby host, a
// failure in the installer's quiet window and a query URL that has no /fail
// send nothing.
func TestHeartbeatSend(t *testing.T) {
const key = "/ping/5f1e0c2a-check-key"
for _, tc := range []struct {
what string
beat heartbeat
path string // appended to the server URL
want string // the ping, "" for none
wantBody string
wantOut string
}{
{what: "success", beat: heartbeat{}, path: key, want: "GET " + key},
{what: "failure", beat: heartbeat{fail: true, report: "the alerts reach no one: x"}, path: key, want: "POST " + key + "/fail", wantBody: "the alerts reach no one: x", wantOut: "pinged the heartbeat's failure endpoint"},
{what: "failure, URL with a trailing slash", beat: heartbeat{fail: true, report: "r"}, path: key + "/", want: "POST " + key + "/fail", wantBody: "r"},
{what: "success, URL with a query", beat: heartbeat{}, path: key + "?rid=7", want: "GET " + key + "?rid=7"},
{what: "failure, URL with a query", beat: heartbeat{fail: true, report: "r"}, path: key + "?rid=7", wantOut: "withholding the heartbeat ping"},
{what: "standby, success", beat: heartbeat{standby: true}, path: key, wantOut: "stands by for the off-site bucket"},
{what: "standby, failure", beat: heartbeat{standby: true, fail: true}, path: key, wantOut: "stands by for the off-site bucket"},
{what: "quiet, failure", beat: heartbeat{quiet: true, fail: true}, path: key, wantOut: "withholding the failure ping"},
{what: "quiet, success", beat: heartbeat{quiet: true}, path: key, want: "GET " + key},
} {
log, srv := newPingServer(t)
b := tc.beat
b.url = srv.URL + tc.path
var stdout, stderr bytes.Buffer
b.send(srv.Client(), &stdout, &stderr)
pings, bodies := log.got()
switch {
case tc.want == "" && len(pings) != 0:
t.Errorf("%s: pinged %v, want nothing", tc.what, pings)
case tc.want != "" && (len(pings) != 1 || pings[0] != tc.want):
t.Errorf("%s: pinged %v, want %q", tc.what, pings, tc.want)
case tc.want != "" && bodies[0] != tc.wantBody:
t.Errorf("%s: body %q, want %q", tc.what, bodies[0], tc.wantBody)
}
if !strings.Contains(stdout.String(), tc.wantOut) || stderr.Len() != 0 {
t.Errorf("%s: stdout %q (want %q), stderr %q", tc.what, stdout.String(), tc.wantOut, stderr.String())
}
}
}
// TestHeartbeatErrors: a ping the service refuses or that cannot connect is
// logged without the URL's path, the check's key.
func TestHeartbeatErrors(t *testing.T) {
const key = "5f1e0c2a-check-key"
log, srv := newPingServer(t)
log.status = http.StatusNotFound
var stdout, stderr bytes.Buffer
heartbeat{url: srv.URL + "/" + key}.send(srv.Client(), &stdout, &stderr)
if got := stderr.String(); !strings.Contains(got, "heartbeat: GET http://127.0.0.1") || !strings.Contains(got, "404") || strings.Contains(got, key) {
t.Errorf("refused ping: stderr %q; want the host and status, not the key", got)
}
closed := httptest.NewServer(http.NotFoundHandler())
dead := closed.URL + "/" + key
closed.Close()
stderr.Reset()
heartbeat{url: dead, fail: true, report: "r"}.send(http.DefaultClient, &stdout, &stderr)
if got := stderr.String(); !strings.Contains(got, "heartbeat: POST http://127.0.0.1") || strings.Contains(got, key) {
t.Errorf("unreachable service: stderr %q; want the host, not the key", got)
}
}
// TestHeartbeatReportClipped: a long report is cut to what the service keeps,
// on a character boundary.
func TestHeartbeatReportClipped(t *testing.T) {
log, srv := newPingServer(t)
report := strings.Repeat("磁盘", heartbeatReportMax)
heartbeat{url: srv.URL + "/k", fail: true, report: report}.send(srv.Client(), io.Discard, io.Discard)
_, bodies := log.got()
if len(bodies) != 1 || len(bodies[0]) > heartbeatReportMax || len(bodies[0]) < heartbeatReportMax-3 || !utf8.ValidString(bodies[0]) {
t.Fatalf("clipped body: %d pings, %d bytes, valid UTF-8 %v", len(bodies), len(bodies[0]), utf8.ValidString(bodies[0]))
}
if got := clipUTF8("short", heartbeatReportMax); got != "short" {
t.Errorf("clipUTF8(short) = %q", got)
}
}
func TestReadHeartbeatURL(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "watchdog-heartbeat-url")
if u, err := readHeartbeatURL(path); u != "" || err != nil {
t.Fatalf("no file: %q, %v", u, err)
}
if u, err := readHeartbeatURL(""); u != "" || err != nil {
t.Fatalf("no path: %q, %v", u, err)
}
writeTestFile(t, path, "https://hc-ping.com/5f1e0c2a\n", 0o600)
if u, err := readHeartbeatURL(path); u != "https://hc-ping.com/5f1e0c2a" || err != nil {
t.Fatalf("good file: %q, %v", u, err)
}
for _, bad := range []string{"", "hc-ping.com/5f1e0c2a", "ftp://hc-ping.com/5f1e0c2a", "https:///5f1e0c2a", "https://hc-ping.com/5f1e 0c2a"} {
writeTestFile(t, path, bad, 0o600)
u, err := readHeartbeatURL(path)
if u != "" || err == nil || !strings.Contains(err.Error(), path) || (bad != "" && strings.Contains(err.Error(), "5f1e")) {
t.Errorf("%q: %q, %v; want an error naming the file and not the URL", bad, u, err)
}
}
}
func TestRedactURL(t *testing.T) {
for in, want := range map[string]string{
"https://hc-ping.com/5f1e0c2a": "https://hc-ping.com/...",
"http://user:[email protected]:8080/k?x=1": "http://status.example:8080/...",
"not a url": "(the heartbeat URL)",
} {
if got := redactURL(in); got != want {
t.Errorf("redactURL(%q) = %q, want %q", in, got, want)
}
}
}
// TestStandsBy: a standby record keeps the host from pinging however old it
// is; a displaced host, the writer, or a host without [offsite] pings.
func TestStandsBy(t *testing.T) {
status := filepath.Join(t.TempDir(), "status.json")
if standsBy(true, status) {
t.Fatal("no status file stands by")
}
old := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Now().Add(-30 * 24 * time.Hour)}
for _, tc := range []struct {
what string
st offsite.Status
offsiteOn bool
want bool
}{
{"standing by for a writer gone quiet a month ago", offsite.Status{Standby: true, Writer: old}, true, true},
{"standing by, [offsite] since removed", offsite.Status{Standby: true, Writer: old}, false, false},
{"displaced", offsite.Status{Displaced: true, Writer: old}, true, false},
{"the writer itself", offsite.Status{LastSuccess: time.Now()}, true, false},
} {
if err := offsite.WriteStatus(status, tc.st); err != nil {
t.Fatal(err)
}
if got := standsBy(tc.offsiteOn, status); got != tc.want {
t.Errorf("%s: standsBy = %v, want %v", tc.what, got, tc.want)
}
}
}
func TestFailureReport(t *testing.T) {
r := watchdog.Report{Findings: []watchdog.Finding{{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "PostgreSQL is down"}}}
for _, tc := range []struct {
what string
unheard string
mailFailed bool
saveErr error
open bool
want string // "" for a success ping
}{
{what: "all good", open: true},
{what: "no relay, nothing mailed yet", unheard: "no [smtp] relay is configured"},
{what: "no relay, an alert open", unheard: "no [smtp] relay is configured", open: true, want: "the alerts reach no one: no [smtp] relay is configured"},
{what: "the mail failed", unheard: "the alert mail failed: refused", mailFailed: true, want: "the alerts reach no one: the alert mail failed: refused"},
{what: "the state did not save", saveErr: errors.New("disk full"), want: "the watchdog state did not save: disk full"},
} {
got := failureReport(tc.unheard, tc.mailFailed, tc.saveErr, tc.open, r)
if (got == "") != (tc.want == "") || !strings.Contains(got, tc.want) || (got != "" && !strings.Contains(got, "[critical] postgres: PostgreSQL is down")) {
t.Errorf("%s: report %q, want %q and the findings", tc.what, got, tc.want)
}
}
}
// alertRecorder records the mail a run hands the relay.
type alertRecorder struct {
relays []*mail.SMTP
sent []string // "to: subject"
fail map[string]bool
}
func (f *alertRecorder) sender(relay *mail.SMTP) alertSender {
f.relays = append(f.relays, relay)
return func(_ context.Context, to, subject, _ string) error {
if f.fail[to] {
return errors.New("550 refused")
}
f.sent = append(f.sent, to+": "+subject)
return nil
}
}
// TestMailerDeliver: a plan goes out and is committed; held, it waits
// uncommitted; with no relay or recipient it is logged and committed; a mail
// no recipient got is left uncommitted to come due again.
func TestMailerDeliver(t *testing.T) {
now := time.Now()
relay := &watchdog.Relay{Host: "smtp.example.com", Port: 587, From: "[email protected]", Username: "felis"}
for _, tc := range []struct {
what string
relay *watchdog.Relay
recipients []string
hold string
fail map[string]bool
wantUnheard string
wantFailed bool
committed bool
sent int
}{
{what: "mailed", relay: relay, recipients: []string{"[email protected]", "[email protected]"}, committed: true, sent: 2},
{what: "one recipient refused", relay: relay, recipients: []string{"[email protected]", "[email protected]"}, fail: map[string]bool{"[email protected]": true}, committed: true, sent: 1},
{what: "every recipient refused", relay: relay, recipients: []string{"[email protected]"}, fail: map[string]bool{"[email protected]": true}, wantUnheard: "the alert mail failed", wantFailed: true},
{what: "held", relay: relay, recipients: []string{"[email protected]"}, hold: "quiet until later"},
{what: "no relay", recipients: []string{"[email protected]"}, wantUnheard: "no [smtp] relay is configured", committed: true},
{what: "no recipient", relay: relay, wantUnheard: "no owner account has a verified email", committed: true},
} {
state := &watchdog.State{Recipients: tc.recipients}
plan := state.Observe(watchdog.Report{Findings: []watchdog.Finding{{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "down"}}}, now)
f := &alertRecorder{fail: tc.fail}
m := mailer{relay: tc.relay, password: "relay-pw", send: f.sender}
var stdout, stderr bytes.Buffer
unheard, failed := m.deliver(context.Background(), state, plan, "subject", "body", tc.hold, now, &stdout, &stderr)
if !strings.HasPrefix(unheard, tc.wantUnheard) || (tc.wantUnheard == "") != (unheard == "") || failed != tc.wantFailed {
t.Errorf("%s: unheard %q, failed %v; want %q, %v", tc.what, unheard, failed, tc.wantUnheard, tc.wantFailed)
}
if got := !state.Alerts["postgres"].Notified.IsZero(); got != tc.committed {
t.Errorf("%s: committed %v, want %v", tc.what, got, tc.committed)
}
if len(f.sent) != tc.sent {
t.Errorf("%s: sent %v, want %d mails", tc.what, f.sent, tc.sent)
}
if len(f.relays) > 0 {
if got := *f.relays[0]; got != (mail.SMTP{Host: "smtp.example.com", Port: 587, From: "[email protected]", Username: "felis", Password: "relay-pw"}) {
t.Errorf("%s: relay %+v", tc.what, got)
}
}
}
}
// TestCachedRelay: the cache keeps the relay's coordinates and its TLS rule,
// and nothing when no relay is configured.
func TestCachedRelay(t *testing.T) {
if r := cachedRelay(testSMTPConfig("")); r != nil {
t.Fatalf("no relay cached %+v", r)
}
got := cachedRelay(testSMTPConfig("smtp.example.com"))
if got == nil || *got != (watchdog.Relay{Host: "smtp.example.com", Port: 2525, From: "[email protected]", Username: "felis", RequireTLS: true}) {
t.Fatalf("cachedRelay = %+v", got)
}
if got := cachedRelay(testSMTPConfig("127.0.0.1")); got == nil || got.RequireTLS {
t.Fatalf("a relay on this host: %+v, want TLS not required", got)
}
}
func testSMTPConfig(host string) config.SMTPConfig {
if host == "" {
return config.SMTPConfig{}
}
return config.SMTPConfig{Host: host, Port: 2525, From: "[email protected]", Username: "felis"}
}
// TestSMTPPassword: the env var password_ref names wins over the cache once
// it is set.
func TestSMTPPassword(t *testing.T) {
c := testSMTPConfig("smtp.example.com")
c.PasswordRef = "FELIS_TEST_WATCHDOG_RELAY_PW"
t.Setenv(c.PasswordRef, "")
if got := smtpPassword(c, "cached"); got != "cached" {
t.Errorf("env var unset: %q", got)
}
t.Setenv(c.PasswordRef, "from-env")
if got := smtpPassword(c, "cached"); got != "from-env" {
t.Errorf("env var set: %q", got)
}
}
const testWatchdogConfig = `[database]
url = "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
[server]
root_domain = "example.com"
[archive]
store = "tarLocal"
[k8s]
egress_mode = "nodeport"
`
const testWatchdogSMTP = `[smtp]
host = "smtp.config.example"
port = 2525
from = "[email protected]"
username = "felis"
password_ref = "FELIS_TEST_WATCHDOG_RELAY_PW"
`
// unitFailedFixture is a host whose watchdog last ran well: one alert open,
// one owner and the relay cached.
func unitFailedFixture(t *testing.T, cfg string) (unitFailedRun, *alertRecorder, *pingLog) {
t.Helper()
dir := t.TempDir()
log, srv := newPingServer(t)
beatFile := filepath.Join(dir, "watchdog-heartbeat-url")
writeTestFile(t, beatFile, srv.URL+"/check-key\n", 0o600)
cfgPath := filepath.Join(dir, "felis.toml")
writeTestFile(t, cfgPath, cfg, 0o600)
statePath := filepath.Join(dir, "state.json")
state := &watchdog.State{
Recipients: []string{"[email protected]"},
SMTPPassword: "cached-pw",
Relay: &watchdog.Relay{Host: "smtp.cached.example", Port: 587, From: "[email protected]", RequireTLS: true},
}
mem := watchdog.Finding{Key: "memory", Severity: watchdog.Warning, SummaryEN: "memory low"}
state.Commit(state.Observe(watchdog.Report{Findings: []watchdog.Finding{mem}}, time.Now().Add(-time.Hour)), time.Now().Add(-time.Hour))
if err := watchdog.SaveState(statePath, state); err != nil {
t.Fatal(err)
}
f := &alertRecorder{}
return unitFailedRun{
cfgPath: cfgPath, statePath: statePath, fallbackPath: filepath.Join(t.TempDir(), "watchdog-state.json"), quietPath: filepath.Join(dir, "quiet"),
offsiteStatus: filepath.Join(dir, "offsite-status.json"), heartbeatFile: beatFile,
result: "exit-code", exitStatus: "1", send: f.sender, client: srv.Client(), now: time.Now(),
}, f, log
}
// TestWatchdogUnitFailedBrokenConfig: failed runs of a watchdog whose
// felis.toml no longer loads are mailed through the relay the last good run
// cached, after five in a row; every one pings /fail; the open alert keeps
// its state.
func TestWatchdogUnitFailedBrokenConfig(t *testing.T) {
r, f, log := unitFailedFixture(t, "[database\n")
start := r.now
for i := 0; i <= 5; i++ {
r.now = start.Add(time.Duration(i) * 2 * time.Minute)
var stdout, stderr bytes.Buffer
if code := watchdogUnitFailed(r, &stdout, &stderr); code != 0 {
t.Fatalf("run %d: exit %d (stdout %s, stderr %s)", i, code, stdout.String(), stderr.String())
}
if i < 5 && len(f.sent) != 0 {
t.Fatalf("run %d, %v after the first failure: mailed %v, want nothing yet", i, r.now.Sub(start), f.sent)
}
}
if len(f.sent) != 1 || !strings.Contains(f.sent[0], "[email protected]: ") {
t.Fatalf("mailed %v, want one alert to the cached owner", f.sent)
}
if got := *f.relays[0]; got.Host != "smtp.cached.example" || got.Password != "cached-pw" || !got.RequireTLS {
t.Fatalf("relay %+v, want the cached one", got)
}
pings, bodies := log.got()
if len(pings) != 6 || pings[0] != "POST /check-key/fail" || !strings.Contains(bodies[0], "felis-watchdog.service failed: result exit-code, exit status 1; ") {
t.Fatalf("pings %v, bodies %q", pings, bodies)
}
state, err := watchdog.LoadState(r.statePath)
if err != nil {
t.Fatal(err)
}
if a := state.Alerts["watchdog/run"]; a == nil || a.Notified.IsZero() || !strings.Contains(a.SummaryEN, "result exit-code, exit status 1") {
t.Fatalf("watchdog/run alert = %+v", a)
}
if a := state.Alerts["memory"]; a == nil || !a.ClearedAt.IsZero() || !a.Notified.Before(start) {
t.Fatalf("memory alert = %+v, want it untouched", a)
}
}
// TestWatchdogUnitFailedStateThatDoesNotSave: with the state file read-only,
// each report keeps the state in the fallback and the next one reads it, so
// the failure is mailed once, after five in a row, and never again.
func TestWatchdogUnitFailedStateThatDoesNotSave(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root writes into a read-only directory")
}
r, f, _ := unitFailedFixture(t, "[database\n")
dir := filepath.Dir(r.statePath)
if err := os.Chmod(dir, 0o500); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.Chmod(dir, 0o700) })
start := r.now
for i := 0; i <= 8; i++ {
r.now = start.Add(time.Duration(i) * 2 * time.Minute)
var stdout, stderr bytes.Buffer
if code := watchdogUnitFailed(r, &stdout, &stderr); code != 1 || !strings.Contains(stderr.String(), "; kept in "+r.fallbackPath) {
t.Fatalf("run %d: exit %d, stderr %s; want exit 1 and the state kept in the fallback", i, code, stderr.String())
}
if want := min(max(i-4, 0), 1); len(f.sent) != want {
t.Fatalf("run %d, %v after the first failure: mailed %d, want %d", i, r.now.Sub(start), len(f.sent), want)
}
}
}
// TestWatchdogUnitFailedConfigRelay: with felis.toml loading, the relay is
// the configured one and signs in with the env var password_ref names.
func TestWatchdogUnitFailedConfigRelay(t *testing.T) {
r, f, _ := unitFailedFixture(t, testWatchdogConfig+testWatchdogSMTP)
t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
start := r.now
for _, at := range []time.Duration{0, 10 * time.Minute} {
r.now = start.Add(at)
var stdout, stderr bytes.Buffer
if code := watchdogUnitFailed(r, &stdout, &stderr); code != 0 {
t.Fatalf("exit %d (stdout %s, stderr %s)", code, stdout.String(), stderr.String())
}
}
if len(f.relays) != 1 || f.relays[0].Host != "smtp.config.example" || f.relays[0].Port != 2525 || f.relays[0].Password != "env-pw" {
t.Fatalf("relays %+v, want the configured one with the env password", f.relays)
}
}
// TestWatchdogUnitFailedHeld: in the installer's quiet window the alert waits
// and no failure is pinged; a host standing by for the off-site bucket pings
// nothing either.
func TestWatchdogUnitFailedHeld(t *testing.T) {
r, f, log := unitFailedFixture(t, "[database\n")
writeTestFile(t, r.quietPath, fmt.Sprintf("%d\n", r.now.Add(time.Hour).Unix()), 0o644)
start := r.now
for _, at := range []time.Duration{0, 10 * time.Minute} {
r.now = start.Add(at)
watchdogUnitFailed(r, io.Discard, io.Discard)
}
if pings, _ := log.got(); len(f.sent) != 0 || len(pings) != 0 {
t.Fatalf("quiet window: mailed %v, pinged %v", f.sent, pings)
}
os.Remove(r.quietPath)
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: r.now.Add(-time.Hour)}
if err := offsite.WriteStatus(r.offsiteStatus, offsite.Status{Standby: true, Writer: w}); err != nil {
t.Fatal(err)
}
var stdout bytes.Buffer
watchdogUnitFailed(r, &stdout, io.Discard)
if pings, _ := log.got(); len(f.sent) != 0 || len(pings) != 0 || !strings.Contains(stdout.String(), "stands by for host prod-1") {
t.Fatalf("standby: mailed %v, pinged %v, stdout %s", f.sent, pings, stdout.String())
}
}
// TestWatchdogUnitFailedStateUnreadable: with no state to mail from, the
// failure still reaches the heartbeat.
func TestWatchdogUnitFailedStateUnreadable(t *testing.T) {
r, f, log := unitFailedFixture(t, "[database\n")
writeTestFile(t, r.statePath, "{", 0o600)
if code := watchdogUnitFailed(r, io.Discard, io.Discard); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
pings, bodies := log.got()
if len(f.sent) != 0 || len(pings) != 1 || pings[0] != "POST /check-key/fail" || !strings.Contains(bodies[0], "state does not load") {
t.Fatalf("mailed %v, pinged %v %q", f.sent, pings, bodies)
}
}
func TestFailureDetail(t *testing.T) {
for _, tc := range [][3]string{
{"exit-code", "1", "result exit-code, exit status 1"},
{"timeout", "", "result timeout"},
{"", "", "systemd named no cause (journalctl -u felis-watchdog -n 50)"},
} {
if got := failureDetail(tc[0], tc[1]); got != tc[2] {
t.Errorf("failureDetail(%q, %q) = %q, want %q", tc[0], tc[1], got, tc[2])
}
}
}
// TestWatchdogRunHeartbeat runs whole watchdog passes with the API server and
// PostgreSQL down and the relay a recorder: a pass whose alerts reach the
// owners pings success; one whose mail fails, or that has no relay while an
// alert is open, posts its report to /fail; a host standing by for the
// off-site bucket pings nothing; a state file that does not parse is moved
// aside and reported.
func TestWatchdogRunHeartbeat(t *testing.T) {
due := func(statePath string) {
// PostgreSQL has been down for an hour and nobody was told yet.
s := &watchdog.State{Recipients: []string{"[email protected]"}, SMTPPassword: "cached-pw"}
s.Observe(watchdog.Report{Findings: []watchdog.Finding{watchdog.PostgresDown(errors.New("refused"))}}, time.Now().Add(-time.Hour))
if err := watchdog.SaveState(statePath, s); err != nil {
t.Fatal(err)
}
}
open := func(statePath string) {
// The owners were told of PostgreSQL half an hour ago.
s := &watchdog.State{Recipients: []string{"[email protected]"}}
at := time.Now().Add(-30 * time.Minute)
r := watchdog.Report{Findings: []watchdog.Finding{watchdog.PostgresDown(errors.New("refused"))}}
s.Observe(r, at.Add(-time.Hour))
s.Commit(s.Observe(r, at), at)
if err := watchdog.SaveState(statePath, s); err != nil {
t.Fatal(err)
}
}
const offsiteTable = "[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis\"\n"
t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
for _, tc := range []struct {
what string
cfg string
state func(path string)
standby bool
quiet bool
refused bool
wantCode int
want string // the ping, "" for none
wantBody string
wantOut string
wantSent int
}{
{what: "nothing due", cfg: testWatchdogConfig + testWatchdogSMTP, want: "GET /check-key"},
{what: "mailed", cfg: testWatchdogConfig + testWatchdogSMTP, state: due, want: "GET /check-key", wantSent: 1},
{what: "the mail fails", cfg: testWatchdogConfig + testWatchdogSMTP, state: due, refused: true, wantCode: 1, want: "POST /check-key/fail", wantBody: "the alerts reach no one: the alert mail failed: [email protected]: 550 refused"},
{what: "no relay, an alert open", cfg: testWatchdogConfig, state: due, want: "POST /check-key/fail", wantBody: "the alerts reach no one: no [smtp] relay is configured"},
{what: "no relay, an alert open, the installer running", cfg: testWatchdogConfig, state: open, quiet: true, wantOut: "withholding the failure ping"},
{what: "standing by for the off-site bucket", cfg: testWatchdogConfig + testWatchdogSMTP + offsiteTable, state: due, standby: true, wantOut: "stands by for the off-site bucket"},
{what: "a state that does not parse", cfg: testWatchdogConfig, state: func(p string) { writeTestFile(t, p, "{", 0o600) }, want: "GET /check-key", wantOut: "[warning] watchdog/state: the watchdog's state file was unreadable and was moved to "},
} {
dir := t.TempDir()
t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
log, srv := newPingServer(t)
beatFile := filepath.Join(dir, "watchdog-heartbeat-url")
writeTestFile(t, beatFile, srv.URL+"/check-key\n", 0o600)
cfgPath := filepath.Join(dir, "felis.toml")
writeTestFile(t, cfgPath, tc.cfg, 0o600)
statePath := filepath.Join(dir, "state.json")
if tc.state != nil {
tc.state(statePath)
}
statusPath := filepath.Join(dir, "offsite-status.json")
if tc.standby {
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Now().Add(-time.Hour)}
if err := offsite.WriteStatus(statusPath, offsite.Status{Standby: true, Writer: w}); err != nil {
t.Fatal(err)
}
}
if tc.quiet {
writeTestFile(t, filepath.Join(dir, "quiet"), fmt.Sprintf("%d\n", time.Now().Add(time.Hour).Unix()), 0o644)
}
rec := &alertRecorder{}
if tc.refused {
rec.fail = map[string]bool{"[email protected]": true}
}
watchdogSender = rec.sender
var stdout, stderr bytes.Buffer
code := cmdWatchdog([]string{
"-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"),
"-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", filepath.Join(dir, "smtp-password"),
"-offsite-status", statusPath, "-heartbeat-file", beatFile,
}, &stdout, &stderr)
watchdogSender = smtpSender
pings, bodies := log.got()
fail := code != tc.wantCode || len(rec.sent) != tc.wantSent || !strings.Contains(stdout.String(), tc.wantOut)
if tc.want == "" {
fail = fail || len(pings) != 0
} else {
fail = fail || len(pings) != 1 || pings[0] != tc.want || !strings.Contains(bodies[0], tc.wantBody)
}
if fail {
t.Errorf("%s: exit %d, mailed %v, pings %v %q; want exit %d, %d mails, %q with %q\nstdout %s\nstderr %s", tc.what, code, rec.sent, pings, bodies, tc.wantCode, tc.wantSent, tc.want, tc.wantBody, stdout.String(), stderr.String())
}
if tc.wantSent > 0 {
if got := *rec.relays[0]; got.Host != "smtp.config.example" || got.Port != 2525 || got.Password != "env-pw" {
t.Errorf("%s: relay %+v, want the configured one with the env password", tc.what, got)
}
if s, err := watchdog.LoadState(statePath); err != nil || s.Relay == nil || s.Relay.Host != "smtp.config.example" {
t.Errorf("%s: the state caches relay %+v (%v), want the configured one", tc.what, s.Relay, err)
}
}
if strings.Contains(tc.wantOut, "watchdog/state") {
if aside, _ := filepath.Glob(statePath + ".unreadable-*"); len(aside) != 1 {
t.Errorf("%s: moved aside %v, want one file", tc.what, aside)
}
}
}
}
cmd/felis/watchdog_test.go
-289
View File
@@ -1,20 +1,10 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io/fs"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/offsite"
"felis.lolicon.best/internal/watchdog"
)
// TestProxyFinding: a listening proxy is healthy; a closed port is the critical
@@ -50,282 +40,3 @@ func TestSplitList(t *testing.T) {
t.Fatalf("splitList = %q", got)
}
}
// TestMailHold: mail waits through the installer's quiet window, and on a host
// standing by for another host's off-site bucket while that host writes it.
func TestMailHold(t *testing.T) {
dir := t.TempDir()
quiet, status := filepath.Join(dir, "quiet"), filepath.Join(dir, "status.json")
now := time.Now()
if got := mailHold(quiet, true, status, now); got != "" {
t.Errorf("no quiet file, no status: %q", got)
}
writeTestFile(t, quiet, fmt.Sprintf("%d\n", now.Add(time.Hour).Unix()), 0o644)
if got := mailHold(quiet, false, status, now); !strings.Contains(got, "quiet until") {
t.Errorf("inside the quiet window: %q", got)
}
os.Remove(quiet)
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-40 * time.Minute)}
for _, tc := range []struct {
what string
st offsite.Status
offsiteOn bool
held bool
}{
{"standing by for a live writer", offsite.Status{Standby: true, Writer: w}, true, true},
{"standing by, [offsite] since removed", offsite.Status{Standby: true, Writer: w}, false, false},
{"standing by for a writer gone quiet", offsite.Status{Standby: true, Writer: &offsite.Writer{HostID: w.HostID, Host: w.Host, At: now.Add(-offsite.WriterLive - time.Minute)}}, true, false},
{"standing by, no writer named", offsite.Status{Standby: true}, true, false},
{"displaced", offsite.Status{Displaced: true, Writer: w}, true, false},
{"the writer itself", offsite.Status{LastSuccess: now}, true, false},
} {
if err := offsite.WriteStatus(status, tc.st); err != nil {
t.Fatal(err)
}
got := mailHold(quiet, tc.offsiteOn, status, now)
if (got != "") != tc.held || (tc.held && !strings.Contains(got, "stands by for host prod-1 (id bbbbbbbbbbbbbbbb)")) {
t.Errorf("%s: hold = %q, want held %v", tc.what, got, tc.held)
}
}
writeTestFile(t, status, "{", 0o600)
if got := mailHold(quiet, true, status, now); got != "" {
t.Errorf("an unreadable status held the mail: %q", got)
}
}
// watchdogHost is a host whole watchdog passes run on: the API server and PostgreSQL
// are down (no kubeconfig, nothing on the database port), the relay is a recorder and
// the heartbeat URL points at a ping log.
type watchdogHost struct {
dir, statePath string
// fallbackPath stands in for /run/felis, in a directory of its own.
fallbackPath string
args []string
rec *alertRecorder
pings *pingLog
url string
}
func newWatchdogHost(t *testing.T, cfg string, state *watchdog.State) *watchdogHost {
t.Helper()
dir := t.TempDir()
t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
pings, srv := newPingServer(t)
h := &watchdogHost{dir: dir, statePath: filepath.Join(dir, "state.json"), fallbackPath: filepath.Join(t.TempDir(), "watchdog-state.json"),
rec: &alertRecorder{}, pings: pings, url: srv.URL}
writeTestFile(t, filepath.Join(dir, "felis.toml"), cfg, 0o600)
writeTestFile(t, filepath.Join(dir, "watchdog-heartbeat-url"), srv.URL+"/check-key\n", 0o600)
if state != nil {
if err := watchdog.SaveState(h.statePath, state); err != nil {
t.Fatal(err)
}
}
h.args = []string{
"-config", filepath.Join(dir, "felis.toml"), "-state", h.statePath, "-fallback-state", h.fallbackPath, "-quiet-file", filepath.Join(dir, "quiet"),
"-backup-dir", "", "-disk-paths", dir, "-k3s-cert-dirs", "", "-smtp-password-file", filepath.Join(dir, "smtp-password"),
"-offsite-status", filepath.Join(dir, "offsite-status.json"), "-build-tools-status", filepath.Join(dir, "build-tools.json"),
"-heartbeat-file", filepath.Join(dir, "watchdog-heartbeat-url"),
}
return h
}
// run is one pass; flags given here override the host's own.
func (h *watchdogHost) run(flags ...string) (code int, stdout, stderr string) {
watchdogSender = h.rec.sender
defer func() { watchdogSender = smtpSender }()
var out, errOut bytes.Buffer
code = cmdWatchdog(append(append([]string(nil), h.args...), flags...), &out, &errOut)
return code, out.String(), errOut.String()
}
// duePostgres is a state whose owner has not yet been told of PostgreSQL, down
// for an hour.
func duePostgres(cachedPassword string) *watchdog.State {
s := &watchdog.State{Recipients: []string{"[email protected]"}, SMTPPassword: cachedPassword}
s.Observe(watchdog.Report{Findings: []watchdog.Finding{watchdog.PostgresDown(errors.New("refused"))}}, time.Now().Add(-time.Hour))
return s
}
// TestWatchdogRunKeepsClusterAlertsWhileTheAPIIsDown: with the API server down,
// the alerts under the cluster checks keep their state, since nothing looked
// at them; an alert of a check that did run and found nothing reads as cleared.
func TestWatchdogRunKeepsClusterAlertsWhileTheAPIIsDown(t *testing.T) {
told := time.Now().Add(-30 * time.Minute)
cluster := []string{"deployment/felis-api", "node/felis-1/NotReady", "server-failed/lobby"}
var r watchdog.Report
for _, key := range append([]string{"proxy"}, cluster...) {
r.Findings = append(r.Findings, watchdog.Finding{Key: key, Severity: watchdog.Critical, SummaryEN: key + " is down"})
}
s := &watchdog.State{Recipients: []string{"[email protected]"}}
s.Commit(s.Observe(r, told), told)
h := newWatchdogHost(t, testWatchdogConfig, s)
code, stdout, stderr := h.run()
if code != 0 || len(h.rec.relays) != 0 {
t.Fatalf("exit %d, mailed %v\nstdout %s\nstderr %s", code, h.rec.sent, stdout, stderr)
}
for _, want := range []string{"felis watchdog: [critical] kube-api: ", "felis watchdog: [critical] postgres: "} {
if !strings.Contains(stdout, want) {
t.Errorf("stdout lacks %q:\n%s", want, stdout)
}
}
got, err := watchdog.LoadState(h.statePath)
if err != nil {
t.Fatal(err)
}
for _, key := range cluster {
if a := got.Alerts[key]; a == nil || !a.ClearedAt.IsZero() || !a.Notified.Equal(told) {
t.Errorf("%s with the API server down: %+v, want it kept open as mailed at %s", key, a, told)
}
}
if a := got.Alerts["proxy"]; a == nil || a.ClearedAt.IsZero() {
t.Errorf("proxy, whose check ran and found nothing: %+v, want it cleared", a)
}
}
// TestWatchdogDryRunMailsAndSavesNothing: a dry run prints the mail that is due
// and the heartbeat it would ping, and sends, pings and saves nothing.
func TestWatchdogDryRunMailsAndSavesNothing(t *testing.T) {
t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
h := newWatchdogHost(t, testWatchdogConfig+testWatchdogSMTP, duePostgres(""))
before, err := os.ReadFile(h.statePath)
if err != nil {
t.Fatal(err)
}
code, stdout, stderr := h.run("-dry-run")
after, err := os.ReadFile(h.statePath)
if err != nil {
t.Fatal(err)
}
pings, _ := h.pings.got()
if code != 0 || len(h.rec.relays) != 0 || len(pings) != 0 || !bytes.Equal(before, after) {
t.Errorf("dry run: exit %d, relays %d, pings %v, state changed %v\nstdout %s\nstderr %s", code, len(h.rec.relays), pings, !bytes.Equal(before, after), stdout, stderr)
}
host, _ := os.Hostname()
for _, want := range []string{
"felis watchdog: due to be mailed to [email protected]:\nSubject: Felis 严重告警（" + host + "）：1 项异常 · 1 firing\n\n",
"felis watchdog: a run pings the heartbeat at " + h.url + "/...\n",
} {
if !strings.Contains(stdout, want) {
t.Errorf("stdout lacks %q:\n%s", want, stdout)
}
}
if strings.Contains(stdout, "\r") {
t.Errorf("the mail body printed with CRLF:\n%q", stdout)
}
h = newWatchdogHost(t, testWatchdogConfig, nil)
code, stdout, stderr = h.run("-dry-run")
if code != 0 || !strings.Contains(stdout, "felis watchdog: nothing is due to be mailed\n") {
t.Errorf("dry run with nothing due: exit %d\nstdout %s\nstderr %s", code, stdout, stderr)
}
if _, err := os.Stat(h.statePath); !errors.Is(err, fs.ErrNotExist) {
t.Errorf("a dry run wrote the state (%v)", err)
}
}
// TestWatchdogRunSignsInWithTheHostRelayPassword: a pass caches the relay
// password from the host copy even while the cluster is down, and signs in with
// it; with no host copy and no cluster it keeps the one it had.
func TestWatchdogRunSignsInWithTheHostRelayPassword(t *testing.T) {
const smtpNoRef = "[smtp]\nhost = \"smtp.config.example\"\nport = 2525\nfrom = \"[email protected]\"\nusername = \"felis\"\n"
for _, tc := range []struct{ what, hostCopy, want string }{
{"the host copy", "host-pw", "host-pw"},
{"no host copy, the cluster down", "", "cached-pw"},
} {
h := newWatchdogHost(t, testWatchdogConfig+smtpNoRef, duePostgres("cached-pw"))
if tc.hostCopy != "" {
writeTestFile(t, filepath.Join(h.dir, "smtp-password"), tc.hostCopy, 0o600)
}
code, stdout, stderr := h.run()
if code != 0 || len(h.rec.relays) != 1 || h.rec.relays[0].Password != tc.want {
t.Errorf("%s: exit %d, relays %+v; want one mail signed in with %q\nstdout %s\nstderr %s", tc.what, code, h.rec.relays, tc.want, stdout, stderr)
continue
}
if s, err := watchdog.LoadState(h.statePath); err != nil || s.SMTPPassword != tc.want {
t.Errorf("%s: the state caches another password (%v)", tc.what, err)
}
}
}
// TestWatchdogRunChecksWhatIsConfigured: the proxy, the database backups, the
// off-site copy and the build lane's scan DB are each checked when the host
// has them, and only then.
func TestWatchdogRunChecksWhatIsConfigured(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
closed := ln.Addr().String()
ln.Close()
const offsiteTable = "[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis\"\n"
const registry = "[registry]\nurl = \"registry.felis.svc:5000\"\n"
optional := []string{"proxy", "db-backup", "offsite", "scan-db"}
for _, tc := range []struct {
what string
cfg string
flags func(dir string) []string
want string // the one optional check that reports, "" for none
}{
{what: "none of them", cfg: testWatchdogConfig},
{what: "a proxy address", cfg: testWatchdogConfig, flags: func(string) []string { return []string{"-proxy-addr", closed} }, want: "proxy"},
{what: "a backup directory", cfg: testWatchdogConfig, flags: func(dir string) []string { return []string{"-backup-dir", dir} }, want: "db-backup"},
{what: "[offsite]", cfg: testWatchdogConfig + offsiteTable, want: "offsite"},
{what: "a registry with the default scan DB", cfg: testWatchdogConfig + registry, want: "scan-db"},
{what: "a registry with its scan DB under mirror/", cfg: testWatchdogConfig + registry + "trivy_db_repository = \"registry.felis.svc:5000/mirror/trivy-db\"\n", want: "scan-db"},
{what: "a registry with the scan DB elsewhere", cfg: testWatchdogConfig + registry + "trivy_db_repository = \"ghcr.io/aquasecurity/trivy-db\"\n"},
} {
h := newWatchdogHost(t, tc.cfg, nil)
var flags []string
if tc.flags != nil {
flags = tc.flags(t.TempDir())
}
code, stdout, stderr := h.run(append(flags, "-dry-run")...)
var reported []string
for _, key := range optional {
if strings.Contains(stdout, "] "+key+": ") {
reported = append(reported, key)
}
}
if code != 0 || strings.Join(reported, ",") != tc.want {
t.Errorf("%s: exit %d, reported %v; want %q\nstdout %s\nstderr %s", tc.what, code, reported, tc.want, stdout, stderr)
}
}
}
// TestWatchdogRunStateThatDoesNotSave: a pass whose state does not save mails
// as usual, keeps its state in the fallback, exits 1 and posts the failure to
// the heartbeat. The next pass reads the fallback and mails nothing again; the
// first one whose state saves drops the fallback.
func TestWatchdogRunStateThatDoesNotSave(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root writes into a read-only directory")
}
t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
h := newWatchdogHost(t, testWatchdogConfig+testWatchdogSMTP, duePostgres(""))
if err := os.Chmod(h.dir, 0o500); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.Chmod(h.dir, 0o700) })
kept := "; kept in " + h.fallbackPath + " until the host restarts"
for i := range 2 {
code, stdout, stderr := h.run()
pings, bodies := h.pings.got()
if code != 1 || len(h.rec.sent) != 1 || !strings.Contains(stderr, "felis watchdog: save state: ") || !strings.Contains(stderr, kept) ||
len(pings) != i+1 || pings[i] != "POST /check-key/fail" || !strings.HasPrefix(bodies[i], "the watchdog state did not save: ") {
t.Fatalf("pass %d: exit %d, mailed %v, pings %v %q; want exit 1, one mail in all and the save failure posted to /fail\nstdout %s\nstderr %s", i, code, h.rec.sent, pings, bodies, stdout, stderr)
}
}
if err := os.Chmod(h.dir, 0o700); err != nil {
t.Fatal(err)
}
code, stdout, stderr := h.run()
if _, err := os.Stat(h.fallbackPath); code != 0 || len(h.rec.sent) != 1 || !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("once the state saves: exit %d, mailed %v, fallback %v; want exit 0, no new mail, the fallback gone\nstdout %s\nstderr %s", code, h.rec.sent, err, stdout, stderr)
}
if s, err := watchdog.LoadState(h.statePath); err != nil || s.Alerts["postgres"] == nil || s.Alerts["postgres"].Notified.IsZero() {
t.Fatalf("saved state = %+v, %v; want the mailed PostgreSQL alert", s, err)
}
}
deploy/bootstrap.sh
+493 -2526
View File
File diff suppressed because it is too large. Load diff
deploy/bootstrap_test.sh
+179 -3024
View File
File diff suppressed because it is too large. Load diff
deploy/build-release-artifacts.sh
-198
View File
@@ -1,198 +0,0 @@
#!/usr/bin/env bash
# Builds everything a Felis release installs, for every architecture, into one directory:
#
# felis-linux-<arch> the felis binary, panel included
# felis-image-felis-linux-<arch>.tar the control-plane image (OCI layout tars:
# felis-image-game-linux-<arch>.tar the limbo, lobby and paper images `ctr images import`
# felis-image-base-linux-<arch>.tar the registry and PostgreSQL reads them as-is)
# felis-images-linux-<arch>.txt one "bundle role name manifest-digest config-digest"
# line per image in the three tars
# felis-velocity.jar the proxy plugin (JVM bytecode, one for every arch)
# SHA256SUMS the sha256 of every file above
#
# Every name above is a contract with deploy/bootstrap.sh, which installs from a release's
# assets (or from a directory like this one, FELIS_ARTIFACT_DIR) instead of building on the
# host: with them a host needs neither Docker, Gradle, Go nor Docker Hub. The images are split
# in three because they change at different rates: the control plane with every release, the
# game images when game-stack.lock or a plugin changes, the base images almost never. An
# upgrade downloads only the tars holding an image the host does not have yet.
#
# .github/workflows/release.yml runs this on a tag and publishes the directory; e2e.yml runs
# it on a branch and installs from the directory.
#
# Needs docker with buildx, and binfmt/QEMU for the architectures other than the builder's:
# the game images' runtime stages run apt-get on the target platform. The builder's own felis
# binary writes the image tars, so Go is not needed here either.
#
# Usage: deploy/build-release-artifacts.sh <version> <out-dir>
# FELIS_RELEASE_ARCHES the architectures to build (default "amd64 arm64")
set -Eeuo pipefail
log() { printf '\033[1;36m[release]\033[0m %s\n' "$*"; }
die() { printf '\033[1;31m[fail]\033[0m %s\n' "$*" >&2; exit 1; }
[ "$#" -eq 2 ] || die "usage: $0 <version> <out-dir>"
VERSION="$1"
OUT="$2"
ARCHES="${FELIS_RELEASE_ARCHES:-amd64 arm64}"
cd "$(dirname "$0")/.."
# The Gradle image the plugin builds run in: deploy/{limbo,lobby}/Dockerfile and bootstrap's
# Velocity build name the same one (bootstrap_asset_test.go holds them together).
PLUGIN_BUILD_IMAGE="gradle:9.8.0-jdk25@sha256:2b2fc1b1dfc3604a2acc916839f36eb5ee48fd7f232427fc5faca224c73bcb01"
# The names and pins bootstrap uses, read from bootstrap itself so the two cannot drift.
bootstrap_value() {
local v
v="$(sed -n "s/^$1=\"\(.*\)\"\$/\1/p" deploy/bootstrap.sh)"
[ -n "$v" ] || die "deploy/bootstrap.sh sets no $1"
printf '%s' "$v"
}
REGISTRY_URL="$(bootstrap_value REGISTRY_URL)"
REGISTRY_IMAGE="$(bootstrap_value REGISTRY_IMAGE)"
POSTGRES_IMAGE="$(bootstrap_value POSTGRES_IMAGE)"
# The final stage of the repo Dockerfile, the base every felis image runs on.
FELIS_BASE_IMAGE="$(awk '$1 == "FROM" && $2 ~ /^gcr\.io\/distroless\// { print $2 }' Dockerfile)"
[ -n "$FELIS_BASE_IMAGE" ] || die "the Dockerfile names no distroless base"
# lock_value reads one KEY=value line of deploy/game-stack.lock, which bootstrap reads the
# same way: never sourced, values limited to URL and version characters.
lock_value() {
local v
v="$(awk -F= -v k="$1" '$1 == k { print substr($0, length(k) + 2); exit }' deploy/game-stack.lock)"
case "$v" in
''|*[!A-Za-z0-9._:/+%-]*) die "deploy/game-stack.lock: $1 is missing or malformed" ;;
esac
printf '%s' "$v"
}
# image_tag_for_version is bootstrap's: the tag bootstrap names this release's image with.
image_tag_for_version() {
local v
v="$(printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '-')"
case "$v" in
""|dev|[!A-Za-z0-9_]*) printf 'demo' ;;
*) printf '%s' "${v:0:128}" ;;
esac
}
host_arch() {
case "$(uname -m)" in
x86_64|amd64) printf 'amd64' ;;
aarch64|arm64) printf 'arm64' ;;
*) die "unsupported builder architecture $(uname -m)" ;;
esac
}
mkdir -p "$OUT"
[ -z "$(ls -A "$OUT")" ] || die "${OUT} is not empty; SHA256SUMS must describe exactly what one run built"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
platforms=""
for arch in $ARCHES; do
case "$arch" in amd64|arm64) ;; *) die "unsupported architecture ${arch}" ;; esac
platforms="${platforms:+${platforms},}linux/${arch}"
done
# ---- the felis binaries --------------------------------------------------------------
# Through the repo Dockerfile, the one recipe that runs the panel build before go build:
# a plain `go build` compiles against the placeholder panel and ships it.
log "building felis ${VERSION} for ${platforms}"
docker buildx build --platform "$platforms" \
--build-arg FELIS_VERSION="$VERSION" \
--output "type=local,dest=${WORK}/bin" .
for arch in $ARCHES; do
src="${WORK}/bin/usr/local/bin/felis"
# buildx nests the output per platform only when it builds more than one.
[ -f "${WORK}/bin/linux_${arch}/usr/local/bin/felis" ] && src="${WORK}/bin/linux_${arch}/usr/local/bin/felis"
install -m 0755 "$src" "${OUT}/felis-linux-${arch}"
# By ELF machine, never by running it: binfmt would run the wrong architecture happily.
case "$arch" in
amd64) want='x86-64' ;;
arm64) want='ARM aarch64' ;;
esac
file "${OUT}/felis-linux-${arch}" | grep -q "$want" \
|| die "felis-linux-${arch} is not a ${want} ELF: TARGETARCH did not reach the go build"
done
HOST_FELIS="${OUT}/felis-linux-$(host_arch)"
[ -x "$HOST_FELIS" ] || die "no felis binary for the builder's own architecture ($(host_arch)); add it to FELIS_RELEASE_ARCHES"
got="$("$HOST_FELIS" version | head -n 1)"
[ "$got" = "felis ${VERSION}" ] || die "felis reports '${got}', want 'felis ${VERSION}': the version stamp did not reach the binary"
# ---- the Velocity plugin -------------------------------------------------------------
# The command bootstrap's source path runs, on a copy of the tree so the checkout stays clean.
log "building felis-velocity.jar in ${PLUGIN_BUILD_IMAGE%%@*}"
mkdir -p "${WORK}/velocity"
tar -C . --exclude=build --exclude=.gradle -cf - plugins/velocity plugins/shared | tar -C "${WORK}/velocity" -xf -
docker run --rm --user "$(id -u):$(id -g)" -e HOME=/tmp -e GRADLE_USER_HOME=/tmp/gradle \
-v "${WORK}/velocity:/src" -w /src/plugins/velocity \
"$PLUGIN_BUILD_IMAGE" gradle --no-daemon clean build
jars=( "${WORK}"/velocity/plugins/velocity/build/libs/felis-velocity-*.jar )
[ "${#jars[@]}" -eq 1 ] && [ -f "${jars[0]}" ] || die "the felis-velocity build must produce exactly one plugin jar"
install -m 0644 "${jars[0]}" "${OUT}/felis-velocity.jar"
# ---- the images ----------------------------------------------------------------------
FELIS_IMAGE="${REGISTRY_URL}/felis/felis:$(image_tag_for_version "$VERSION")"
game_build_args=(
--build-arg "LIMBO_JAR_URL=$(lock_value LIMBO_JAR_URL)"
--build-arg "LIMBO_JAR_SHA256=$(lock_value LIMBO_JAR_SHA256)"
--build-arg "LIMBO_SCHEM_URL=$(lock_value LIMBO_SCHEM_URL)"
--build-arg "LIMBO_SCHEM_SHA256=$(lock_value LIMBO_SCHEM_SHA256)"
--build-arg "LIMBO_VERSION=$(lock_value LIMBO_VERSION)"
--build-arg "PAPER_JAR_URL=$(lock_value PAPER_JAR_URL)"
--build-arg "PAPER_JAR_SHA256=$(lock_value PAPER_JAR_SHA256)"
--build-arg "LUCKPERMS_JAR_URL=$(lock_value LUCKPERMS_JAR_URL)"
--build-arg "LUCKPERMS_JAR_SHA256=$(lock_value LUCKPERMS_JAR_SHA256)"
)
# oci_image <arch> <dest> <docker build args...> builds one image for one platform into an
# OCI layout tar. No attestations: the bundle carries images only.
oci_image() {
local arch="$1" dest="$2"
shift 2
docker buildx build --platform "linux/${arch}" --provenance=false --sbom=false \
--output "type=oci,dest=${dest}" "$@"
}
# bundle <arch> <group> <image-bundle flags...> writes one image tar and appends its lines
# to the architecture's listing.
bundle() {
local arch="$1" group="$2" name
shift 2
name="felis-image-${group}-linux-${arch}.tar"
"$HOST_FELIS" image-bundle --platform "linux/${arch}" \
--out "${OUT}/${name}" --list "${WORK}/${name}.txt" "$@"
awk -v b="$name" '{ print b, $0 }' "${WORK}/${name}.txt" >> "${OUT}/felis-images-linux-${arch}.txt"
}
for arch in $ARCHES; do
# The control-plane image wraps the released binary itself, byte for byte, the way
# bootstrap wraps a downloaded binary.
log "building the linux/${arch} images"
mkdir -p "${WORK}/felis-${arch}"
cp "${OUT}/felis-linux-${arch}" "${WORK}/felis-${arch}/felis"
cat > "${WORK}/felis-${arch}/Dockerfile" <<EOF
FROM ${FELIS_BASE_IMAGE}
ENV PATH=/usr/local/bin:/usr/bin:/bin
COPY --chmod=0755 felis /usr/local/bin/felis
USER 65532:65532
ENTRYPOINT ["/usr/local/bin/felis"]
EOF
oci_image "$arch" "${WORK}/felis-${arch}.tar" "${WORK}/felis-${arch}"
for role in limbo lobby paper; do
oci_image "$arch" "${WORK}/${role}-${arch}.tar" -f "deploy/${role}/Dockerfile" "${game_build_args[@]}" .
done
bundle "$arch" felis --layout "felis=${FELIS_IMAGE}=${WORK}/felis-${arch}.tar"
bundle "$arch" game \
--layout "limbo=${REGISTRY_URL}/felis/limbo:demo=${WORK}/limbo-${arch}.tar" \
--layout "lobby=${REGISTRY_URL}/felis/lobby:demo=${WORK}/lobby-${arch}.tar" \
--layout "paper=${REGISTRY_URL}/felis/paper:demo=${WORK}/paper-${arch}.tar"
bundle "$arch" base --pull "registry=${REGISTRY_IMAGE}" --pull "postgres=${POSTGRES_IMAGE}"
rm -f "${WORK}"/*-"${arch}".tar
done
log "writing SHA256SUMS"
(cd "$OUT" && sha256sum -- *) > "${WORK}/SHA256SUMS"
mv "${WORK}/SHA256SUMS" "${OUT}/SHA256SUMS"
ls -l "$OUT"
deploy/crd/felis.lolicon.best_minecraftservers.yaml
+3 -43
View File
@@ -101,8 +101,6 @@ spec:
EmptySecondsBeforeStop is how long the server may sit empty before the
operator scales it down.
format: int32
maximum: 604800
minimum: 0
type: integer
type: object
image:
@@ -129,8 +127,6 @@ spec:
description: TerminationGracePeriodSeconds is the pod grace period
(default 300).
format: int64
maximum: 3600
minimum: 0
type: integer
type: object
motd:
@@ -163,8 +159,6 @@ spec:
port:
description: Port is the RCON TCP port (default 25575).
format: int32
maximum: 65535
minimum: 0
type: integer
secretRef:
description: SecretRef points at the Secret holding the RCON password.
@@ -178,10 +172,6 @@ spec:
- name
type: object
type: object
x-kubernetes-validations:
- message: the allow-rcon NetworkPolicy admits only port 25575; leave
port unset
rule: '!has(self.port) || self.port == 0 || self.port == 25575'
reaperExempt:
description: ReaperExempt opts this server out of the world reaper
entirely (spec §18).
@@ -260,22 +250,16 @@ spec:
(notably LOOHP/Limbo) where the felis-limbo plugin reports true
readiness only after the first server tick.
format: int32
maximum: 65535
minimum: 0
type: integer
readinessTimeoutSeconds:
description: ReadinessTimeoutSeconds is the budget for the first
successful RCON probe.
format: int32
maximum: 86400
minimum: 0
type: integer
timeoutSeconds:
description: TimeoutSeconds is the overall budget before the server
is marked Failed.
format: int32
maximum: 86400
minimum: 0
type: integer
type: object
storage:
@@ -301,13 +285,6 @@ spec:
status:
description: MinecraftServerStatus is the observed state (spec §4 status.*).
properties:
autoRestarts:
description: |-
AutoRestarts counts how often the operator recreated the pod of a start
that timed out (at most 3, with a doubling backoff); reaching Ready or
stopping resets it.
format: int32
type: integer
conditions:
description: Conditions are the standard metav1 conditions (Ready,
RconReached, ...).
@@ -370,10 +347,8 @@ spec:
description: |-
EmptySince is when the operator first observed 0 online players during
a Running phase (spec §8 idle auto-stop). It is reset when a player joins
or the server restarts or stops, so the empty-duration counter starts fresh
each time the server becomes unoccupied. A zero sampled within three minutes
of a run's first ready probe stamps nothing: the players it came up for may
not be in yet.
or the server stops, so the empty-duration counter starts fresh each time
the server becomes unoccupied.
format: date-time
type: string
endpoint:
@@ -386,11 +361,6 @@ spec:
description: Mode is "direct" or "fallback".
type: string
type: object
lastAutoRestartAt:
description: LastAutoRestartAt is when the operator last recreated
the pod.
format: date-time
type: string
liveMotd:
description: LiveMotd is the MOTD currently advertised for the active
phase.
@@ -422,9 +392,7 @@ spec:
status ping is never sufficient — spec §5).
type: boolean
readySignalAt:
description: |-
ReadySignalAt is when the first RCON probe of the current run succeeded.
Every Starting or Stopping pass clears it, so each start is measured once.
description: ReadySignalAt is when the first RCON probe succeeded.
format: date-time
type: string
startRequestedAt:
@@ -437,14 +405,6 @@ spec:
because the two endpoints fall in different reconcile passes.
format: date-time
type: string
stopNoticeAt:
description: |-
StopNoticeAt is when the operator told the players on a server that it is
about to stop (desiredState flipped to Stopped with players online). The stop
itself waits until StopNoticeWindow has passed since then; the stamp is cleared
once the server is scaled down, or when desiredState goes back to Running first.
format: date-time
type: string
type: object
type: object
served: true
deploy/e2e_check.sh
-244
View File
@@ -1,244 +0,0 @@
#!/bin/bash
# Checks a host that deploy/bootstrap.sh just installed (or re-ran on). The e2e workflow
# runs it on a fresh GitHub runner after each of its two installer runs:
#
# sudo bash deploy/e2e_check.sh install # after the first run
# sudo bash deploy/e2e_check.sh rerun # after the same commit ran again
# sudo bash deploy/e2e_check.sh release # after the newest release installed
# sudo bash deploy/e2e_check.sh upgrade # after this commit ran over a release
#
# It asks what an operator's first minutes ask: the binary runs, the control plane and its
# database are rolled out and ready, a database backup can be taken and restored, the panel
# answers on its NodePort, the proxy answers a Minecraft status ping, the host timers are
# all there and waiting, and the backup and watchdog runs the installer made succeeded.
# A rerun must also leave the proxy running (it restarts only when what it runs changed)
# and keep every earlier answer.
set -euo pipefail
phase="${1:?usage: e2e_check.sh install|rerun|release|upgrade}"
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
KUBECTL=(/usr/local/bin/k3s kubectl)
PID_FILE=/var/tmp/felis-e2e-velocity.pid
fails=0
pass() { printf 'PASS %s\n' "$*"; }
fail() { printf 'FAIL %s\n' "$*"; fails=$((fails + 1)); }
check() { # label command...
local label="$1"
shift
if "$@"; then pass "$label"; else fail "$label"; fi
}
check "felis version runs" sh -c '/usr/local/bin/felis version | grep -q "^felis "'
deploys=(felis-api felis-operator registry)
# A release may still run the database on the host; the upgrade moves it into felis-postgres.
if [ "$phase" != release ] || "${KUBECTL[@]}" -n felis get deploy/felis-postgres >/dev/null 2>&1; then
deploys=(felis-postgres "${deploys[@]}")
fi
for d in "${deploys[@]}"; do
check "deployment ${d} is rolled out" "${KUBECTL[@]}" -n felis rollout status "deploy/${d}" --timeout=180s
done
node_ip="$(ip -4 route get 1.1.1.1 | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}')"
check "the panel serves its page on the NodePort" \
sh -c "curl -skf --retry 10 --retry-delay 3 --retry-all-errors https://${node_ip}:30443/ | grep -qi '<html'"
# Readiness lives on the internal face only; a Service ClusterIP routes from the node.
internal="$("${KUBECTL[@]}" -n felis get svc felis-api-internal -o jsonpath='{.spec.clusterIP}:{.spec.ports[0].port}')"
check "felis-api is ready (database and cluster reachable)" \
curl -sf --retry 10 --retry-delay 3 --retry-all-errors -o /dev/null "http://${internal}/readyz"
for unit in k3s felis-velocity; do
check "${unit} is active" systemctl is-active --quiet "$unit"
done
# pod_psql runs one statement in the database's pod, over its socket, as the felis role on
# the felis database.
pod_psql() {
"${KUBECTL[@]}" -n felis exec -i deploy/felis-postgres -c postgres -- \
psql -X -q -At -v ON_ERROR_STOP=1 -U felis -d felis -c "$1"
}
# restore_drill walks troubleshooting.md's "Restore on the same host": refused while the
# control plane is connected; with it scaled to 0 the bundle comes back (a row written after
# it is gone) and the database it replaced is kept; migrate up runs; the control plane serves
# again.
restore_drill() { # dir bundle
local dir="$1" bundle="$2" out rc
local sel="app.kubernetes.io/part-of=felis-control-plane,app.kubernetes.io/component in (api,operator)"
if ! pod_psql "INSERT INTO platform_settings (key, value) VALUES ('e2e_restore_drill', '1')" >/dev/null; then
fail "write a row after the bundle"
return
fi
rc=0
out="$(/usr/local/bin/felis db restore -dir "$dir" -yes "$bundle" 2>&1)" || rc=$?
if [ "$rc" -eq 1 ] && grep -q "other clients are connected to the database" <<<"$out"; then
pass "felis db restore refuses while the control plane is connected"
else
fail "felis db restore refuses while the control plane is connected (exit ${rc}): ${out}"
fi
"${KUBECTL[@]}" -n felis scale deployment felis-api felis-operator --replicas=0 >/dev/null
for _ in $(seq 60); do
[ -z "$("${KUBECTL[@]}" -n felis get pods -l "$sel" -o name)" ] && break
sleep 2
done
rc=0
out="$(/usr/local/bin/felis db restore -dir "$dir" -yes "$bundle" 2>&1)" || rc=$?
if [ "$rc" -eq 0 ]; then
pass "felis db restore replays the bundle with the control plane scaled to 0"
else
fail "felis db restore replays the bundle with the control plane scaled to 0 (exit ${rc}): ${out}"
fi
check "the restore dropped the row written after the bundle" \
test "$(pod_psql "SELECT count(*) FROM platform_settings WHERE key = 'e2e_restore_drill'")" = 0
check "the restore kept the database it replaced in a pre-restore bundle" \
sh -c "ls '${dir}' | grep -q -- '-pre-restore\.tar\$'"
check "felis migrate up runs on the restored database" \
/usr/local/bin/felis migrate up -config /etc/felis/felis.host.toml
"${KUBECTL[@]}" -n felis scale deployment felis-api felis-operator --replicas=1 >/dev/null
for d in felis-api felis-operator; do
check "deployment ${d} is rolled out again after the restore" "${KUBECTL[@]}" -n felis rollout status "deploy/${d}" --timeout=180s
done
check "felis-api is ready on the restored database" \
curl -sf --retry 10 --retry-delay 3 --retry-all-errors -o /dev/null "http://${internal}/readyz"
}
# The database runs in k3s; a release may still run it on the host, and the upgrade moved
# it. The host has no PostgreSQL client: a bundle that verifies proves felis reaches the
# database's pod through kubectl exec, and that pg_dump there reads every table.
if [ "$phase" != release ]; then
check "the host's own postgresql is stopped" sh -c '! systemctl is-active --quiet postgresql'
bundle_dir="$(mktemp -d)"
if out="$(/usr/local/bin/felis db backup -dir "$bundle_dir" -state-dir "" -no-servers 2>&1)"; then
bundle="$(printf '%s\n' "$out" | sed -n 's/^felis db backup: wrote //p' | tail -n 1)"
check "felis db backup writes a bundle that verifies" /usr/local/bin/felis db verify "$bundle"
# A rerun keeps what the install left; the install and the upgrade restore it.
[ "$phase" = rerun ] || restore_drill "$bundle_dir" "$bundle"
else
fail "felis db backup writes a bundle: ${out}"
fi
rm -rf "$bundle_dir"
fi
# The daily reaper is what deletes expired archives. Run it once from its CronJob: the API
# accepts a pod template naming a ServiceAccount that does not exist, and only the Job's
# pod creation fails, so rendering it proves nothing. A release may carry exactly that bug.
if [ "$phase" != release ]; then
reaper_job="felis-e2e-reaper-${phase}"
"${KUBECTL[@]}" -n minecraft delete job "$reaper_job" --ignore-not-found >/dev/null
if "${KUBECTL[@]}" -n minecraft create job --from=cronjob/felis-reaper "$reaper_job" >/dev/null; then
check "the reaper CronJob runs to completion" \
"${KUBECTL[@]}" -n minecraft wait --for=condition=complete "job/${reaper_job}" --timeout=180s
"${KUBECTL[@]}" -n minecraft delete job "$reaper_job" --ignore-not-found >/dev/null
else
fail "a Job can be created from the reaper CronJob"
fi
fi
# A release may predate a timer. What this commit installs has every timer bootstrap.sh
# names, enabled and waiting, and no felis timer it does not name. The off-site copy's
# comes only with an [offsite] bucket, which no e2e host has.
if [ "$phase" != release ]; then
timers="$(sed -n 's|^[A-Z_]*_TIMER="/etc/systemd/system/\(felis-[a-z-]*\.timer\)"$|\1|p' "$(dirname "$0")/bootstrap.sh")"
check "bootstrap.sh names the felis timers" test -n "$timers"
for timer in $timers; do
if [ "$timer" = felis-offsite.timer ]; then
check "${timer} is not installed without an [offsite] bucket" test ! -e "/etc/systemd/system/${timer}"
continue
fi
check "${timer} is enabled" systemctl is-enabled --quiet "$timer"
check "${timer} is waiting" systemctl is-active --quiet "$timer"
done
for timer in $(systemctl list-unit-files --no-legend 'felis-*.timer' | awk '{print $1}'); do
check "${timer} is a timer bootstrap.sh installs" grep -qxF "$timer" <<<"$timers"
done
# The installer runs the backup and the watchdog once itself and only warns when that
# fails: a unit that cannot run (a flag the binary lacks, a path its sandbox hides) shows
# here. Without an [smtp] relay the watchdog has no mail to fail on.
for unit in felis-db-backup.service felis-watchdog.service; do
check "${unit} has run" test "$(systemctl show -p ExecMainStartTimestampMonotonic --value "$unit")" != 0
check "${unit}'s last run succeeded" test "$(systemctl show -p Result --value "$unit")" = success
done
# A failed watchdog run starts the unit its OnFailure= names, which must be installed.
on_failure="$(systemctl show -p OnFailure --value felis-watchdog.service)"
check "felis-watchdog.service names an OnFailure= unit" test -n "$on_failure"
for unit in $on_failure; do
check "${unit} is installed" test "$(systemctl show -p LoadState --value "$unit")" = loaded
done
fi
# A status ping is the proxy's own answer (ping passthrough is off), so it proves the JRE,
# Velocity and its config without a login gate or a Mojang account.
ping_proxy() {
python3 - <<'EOF'
import json, socket, struct, sys
def varint(n):
out = b""
n &= 0xFFFFFFFF
while True:
b, n = n & 0x7F, n >> 7
if n:
out += bytes([b | 0x80])
else:
return out + bytes([b])
def read_varint(s):
n = 0
for i in range(5):
b = s.recv(1)
if not b:
raise EOFError("connection closed")
n |= (b[0] & 0x7F) << (7 * i)
if not b[0] & 0x80:
return n
raise ValueError("varint too long")
host, port = "127.0.0.1", 25565
s = socket.create_connection((host, port), timeout=10)
hs = varint(0) + varint(767) + varint(len(host)) + host.encode() + struct.pack(">H", port) + varint(1)
s.sendall(varint(len(hs)) + hs + varint(1) + varint(0))
read_varint(s)
read_varint(s)
size = read_varint(s)
data = b""
while len(data) < size:
chunk = s.recv(size - len(data))
if not chunk:
raise EOFError("short status response")
data += chunk
print(json.loads(data)["version"]["name"])
EOF
}
# The installer returns once the unit is started; the JVM binds the port a few
# seconds later.
for _ in $(seq 30); do
if version="$(ping_proxy 2>&1)"; then
break
fi
sleep 2
done
if version="$(ping_proxy 2>&1)"; then
pass "the proxy answers a status ping (${version})"
else
fail "the proxy answers a status ping: ${version}"
fi
pid="$(systemctl show -p MainPID --value felis-velocity)"
case "$phase" in
install | release) printf '%s\n' "$pid" > "$PID_FILE" ;;
rerun)
if [ "$pid" = "$(cat "$PID_FILE" 2>/dev/null)" ]; then
pass "the rerun left the proxy running (pid ${pid})"
else
fail "the rerun restarted the proxy (pid $(cat "$PID_FILE" 2>/dev/null || echo '?') -> ${pid}) though nothing it runs changed"
fi
;;
upgrade) ;; # a new release may well change what the proxy runs
*) fail "unknown phase ${phase}" ;;
esac
if [ "$fails" -eq 0 ]; then
echo "ALL PASS (${phase})"
else
echo "${fails} FAILED (${phase})"
exit 1
fi
deploy/e2e_release.sh
-112
View File
@@ -1,112 +0,0 @@
#!/bin/bash
# The newest published release, for the e2e workflow's readme and upgrade jobs, and what
# their installer logs must show about how that release reached the host:
#
# bash deploy/e2e_release.sh find # tag, binary, sums into $GITHUB_OUTPUT
# bash deploy/e2e_release.sh check-own LOG # the release's own installer (upgrade)
# bash deploy/e2e_release.sh check-readme LOG # this commit's installer on its default
# # channel, the README's command (readme)
#
# The checks read TAG, BINARY and SUMS from the environment, as find wrote them. The
# runners are x86_64, so the binary asset is felis-linux-amd64. deploy/e2e_release_test.sh
# holds this script's own checks, against bootstrap.sh's own messages.
set -euo pipefail
ASSET=felis-linux-amd64
HOST_BIN=/usr/local/bin/felis
fails=0
pass() { printf 'PASS %s\n' "$*"; }
fail() { printf 'FAIL %s\n' "$*"; fails=$((fails + 1)); }
has() { # label fixed-string log
if grep -qF -- "$2" "$3"; then pass "$1"; else fail "$1: no line with <$2> in $3"; fi
}
has_re() { # label regex log
if grep -qE -- "$2" "$3"; then pass "$1"; else fail "$1: no line matching <$2> in $3"; fi
}
lacks_re() { # label regex log
local hit
if hit="$(grep -E -m 1 -- "$2" "$3")"; then fail "$1: ${hit}"; else pass "$1"; fi
}
# find_release: `gh release view` with no tag answers with the newest release that is not a
# prerelease, the one the installer's release channel resolves.
#
# An empty tag skips the readme and upgrade jobs, green. Only gh's own `release not found`
# means there is no release; any other failure (a token it refused, a rate limit, a network
# error) fails the step, or every release's upgrade would go untested without a word.
find_release() {
local lines err rc=0 tag names binary="" sums=""
err="$(mktemp)"
lines="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName,assets --jq '.tagName, .assets[].name' 2>"$err")" || rc=$?
if [ "$rc" -ne 0 ] && grep -qx 'release not found' "$err"; then
rm -f "$err"
echo "::notice::no published release yet; the readme and upgrade jobs have nothing to install"
printf 'tag=\nbinary=\nsums=\n' >> "${GITHUB_OUTPUT:-/dev/stdout}"
return 0
fi
if [ "$rc" -ne 0 ]; then
echo "::error::gh release view failed (exit ${rc}), so the newest release is unknown: $(tr '\n' ' ' < "$err")"
rm -f "$err"
fails=$((fails + 1))
return
fi
rm -f "$err"
tag="$(printf '%s\n' "$lines" | head -n 1)"
names="$(printf '%s\n' "$lines" | tail -n +2)"
if [ -z "$tag" ]; then
echo "::error::gh release view answered without a tag"
fails=$((fails + 1))
return
fi
if printf '%s\n' "$names" | grep -qxF "$ASSET"; then binary=yes; fi
if printf '%s\n' "$names" | grep -qxF SHA256SUMS; then sums=yes; fi
printf 'tag=%s\nbinary=%s\nsums=%s\n' "$tag" "$binary" "$sums" >> "${GITHUB_OUTPUT:-/dev/stdout}"
}
# check_own: a release's installer, however old, downloads the release's binary when the
# release publishes one, and says so in the same words.
check_own() {
local log="$1"
if [ "${BINARY:-}" != yes ]; then
echo "::notice::release ${TAG} publishes no ${ASSET}; its installer builds it from source"
return 0
fi
has "the release's installer installed the release's binary" "installed ${ASSET} ${TAG} at ${HOST_BIN}" "$log"
}
# check_readme: this commit's installer takes everything from a release that publishes its
# SHA256SUMS and builds nothing on the host; from one without, it builds the tag from source
# and says why.
check_readme() {
local log="$1" role
if [ "${BINARY:-}" = yes ] && [ "${SUMS:-}" = yes ]; then
has "the binary is the release's" "installed ${ASSET} ${TAG} at ${HOST_BIN}" "$log"
has "the images and plugin come from the release" "release ${TAG}'s prebuilt images and Velocity plugin are installed as published" "$log"
for role in felis limbo lobby paper; do
has_re "felis/${role} is the release's" "felis/${role}:[^ ]* is the release's" "$log"
done
has_re "the registry image is the release's" "docker.io/library/registry@sha256:[0-9a-f]* is the release's" "$log"
has_re "the postgres image is the release's" "docker.io/library/postgres@sha256:[0-9a-f]* is the release's" "$log"
has "felis-velocity.jar is the release's" "felis-velocity.jar is the release's" "$log"
lacks_re "nothing fell back to a build or a pull" "on this host instead|from Docker Hub instead|building felis from source" "$log"
lacks_re "Docker was left alone" "docker already installed|installing docker|docker running" "$log"
elif [ "${BINARY:-}" = yes ]; then
has "the source build says why" "release ${TAG} publishes no SHA256SUMS, so ${ASSET} cannot be verified; building ${TAG} from source on this host instead" "$log"
echo "::warning::release ${TAG} publishes no SHA256SUMS, so the README's install builds ${TAG} from source on the host, Docker included; a release cut by release.yml puts it on the assets"
else
has "the source build says why" "release ${TAG} publishes no usable ${ASSET}; building ${TAG} from source on this host instead" "$log"
echo "::warning::release ${TAG} publishes no ${ASSET}, so the README's install builds ${TAG} from source on the host, Docker included; a release cut by release.yml puts it on the assets"
fi
}
case "${1:-}" in
find) find_release ;;
check-own) check_own "${2:?usage: e2e_release.sh check-own LOG}" ;;
check-readme) check_readme "${2:?usage: e2e_release.sh check-readme LOG}" ;;
*)
echo "usage: e2e_release.sh find | check-own LOG | check-readme LOG" >&2
exit 2
;;
esac
[ "$fails" -eq 0 ] || exit 1
deploy/e2e_release_test.sh
-187
View File
@@ -1,187 +0,0 @@
#!/bin/bash
# Checks for deploy/e2e_release.sh. Run it as: bash deploy/e2e_release_test.sh
#
# The installer logs it reads are built from bootstrap.sh's own ok/warn messages, expanded
# with a release's values, so rewording one of them there fails here rather than in a
# two-hour e2e run. gh is a stub that prints what a release listing would.
set -u
here="$(dirname "$0")"
ER="${1:-${here}/e2e_release.sh}"
BS="${2:-${here}/bootstrap.sh}"
[ -f "$ER" ] || { echo "no such script: $ER"; exit 1; }
[ -f "$BS" ] || { echo "no such script: $BS"; exit 1; }
fails=0
expect() { # label needle haystack
case "$3" in
*"$2"*) echo "PASS $1" ;;
*) echo "FAIL $1: expected <$2> in:"; echo "$3"; fails=$((fails + 1)) ;;
esac
}
status() { # label want got
if [ "$2" = "$3" ]; then echo "PASS $1"; else echo "FAIL $1: exit $3, want $2"; fails=$((fails + 1)); fi
}
root="$(mktemp -d)"
trap 'rm -rf "$root"' EXIT
# msg <marker> prints bootstrap.sh's first ok/warn message holding <marker>, expanded with
# the variables below the way the installer expands it. They are read only through that eval.
# shellcheck disable=SC2034
{
tag=v1.2.3
FELIS_REF="$tag"
v="$tag"
asset=felis-linux-amd64
name="$asset"
HOST_BIN=/usr/local/bin/felis
digest=sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
}
msg() {
local line body
line="$(grep -F -- "$1" "$BS" | grep -E '^[[:space:]]*(ok|warn) "' | head -n 1)"
if [ -z "$line" ]; then
echo "FAIL bootstrap.sh prints no message holding <$1>" >&2
fails=$((fails + 1))
return
fi
body="${line#*\"}"
body="${body%\"*}"
eval "printf '%s\n' \"${body}\""
}
# unusable <marker> prints the warning of bootstrap.sh's first artifact_unusable call holding
# <marker>; artifact_unusable joins its two arguments with "; ".
unusable() {
local line
line="$(grep -F -- "$1" "$BS" | grep -E '^[[:space:]]*artifact_unusable "' | head -n 1)"
if [ -z "$line" ]; then
echo "FAIL bootstrap.sh has no artifact_unusable call holding <$1>" >&2
fails=$((fails + 1))
return
fi
artifact_unusable() { printf '%s; %s\n' "$1" "$2"; }
eval "$line"
}
image_line() { # target
# shellcheck disable=SC2034 # read by msg's eval
local target="$1"
msg 'is the release'"'"'s (${digest:7:12})'
}
# What an install from a complete release prints, in the installer's order.
{
msg 'ok "installed ${asset} ${FELIS_REF} at ${HOST_BIN}"'
msg 'matches release ${tag}'"'"'s SHA256SUMS'
msg 'prebuilt images and Velocity plugin are installed as published'
for t in felis/felis:v1.2.3 felis/limbo:v1.2.3 felis/lobby:v1.2.3 felis/paper:v1.2.3 \
docker.io/library/registry@sha256:aa docker.io/library/postgres@sha256:bb; do
image_line "$t"
done
msg 'felis-velocity.jar is the release'"'"'s"'
} > "$root/complete.log"
readme() { # log binary sums
TAG="$tag" BINARY="$2" SUMS="$3" bash "$ER" check-readme "$1" 2>&1
}
own() { # log binary
TAG="$tag" BINARY="$2" bash "$ER" check-own "$1" 2>&1
}
out="$(readme "$root/complete.log" yes yes)"
status "a complete release's install passes" 0 $?
expect " and every image is checked" "PASS felis/paper is the release's" "$out"
grep -v 'felis/limbo:' "$root/complete.log" > "$root/nolimbo.log"
out="$(readme "$root/nolimbo.log" yes yes)"
status "an image missing from the log fails" 1 $?
expect " and names it" "FAIL felis/limbo is the release's" "$out"
{
cat "$root/complete.log"
name=felis-images-linux-amd64.txt unusable '} cannot be used" "building its images on this host instead"'
} > "$root/fallback.log"
out="$(readme "$root/fallback.log" yes yes)"
status "an image built on the host fails" 1 $?
expect " and quotes the fallback" "building its images on this host instead" "$out"
{ cat "$root/complete.log"; msg 'ok "docker already installed"'; } > "$root/docker.log"
out="$(readme "$root/docker.log" yes yes)"
status "Docker touched fails" 1 $?
sed 's/installed felis-linux-amd64 v1.2.3/installed felis-linux-amd64 v1.2.2/' "$root/complete.log" > "$root/other.log"
out="$(readme "$root/other.log" yes yes)"
status "another release's binary fails" 1 $?
# A release that publishes its binary but no SHA256SUMS: this commit's installer builds the
# tag from source and says so.
msg 'publishes no SHA256SUMS, so ${name} cannot be verified' > "$root/nosums.log"
out="$(readme "$root/nosums.log" yes "")"
status "a release without SHA256SUMS passes when the fallback is announced" 0 $?
expect " and warns in the run" "::warning::release v1.2.3 publishes no SHA256SUMS" "$out"
out="$(readme "$root/complete.log" yes "")"
status "a release without SHA256SUMS fails when nothing announced the fallback" 1 $?
msg 'publishes no usable ${asset}' > "$root/nobinary.log"
out="$(readme "$root/nobinary.log" "" "")"
status "a release without a binary passes when the fallback is announced" 0 $?
out="$(readme "$root/nosums.log" "" "")"
status "a release without a binary fails when the log says otherwise" 1 $?
# check-own: the release's own installer, which may predate SHA256SUMS.
out="$(own "$root/complete.log" yes)"
status "the release's installer downloading its binary passes" 0 $?
out="$(own "$root/nobinary.log" yes)"
status "the release's installer building from source fails" 1 $?
out="$(own "$root/other.log" yes)"
status "the release's installer downloading another release fails" 1 $?
out="$(own "$root/nobinary.log" "")"
status "a release without a binary asks nothing of its installer" 0 $?
# find: the listing gh answers with, in the step's outputs, exactly. GH_FAIL is what a
# failing gh prints on stderr before it exits 1.
mkdir -p "$root/bin"
cat > "$root/bin/gh" <<'STUB'
#!/bin/sh
if [ -n "${GH_FAIL:-}" ]; then
printf '%s\n' "$GH_FAIL" >&2
exit 1
fi
printf '%s\n' $GH_LISTING
STUB
chmod +x "$root/bin/gh"
find_run() { # listing [gh's error]: prints what find says; its outputs land in $root/out
: > "$root/out"
GH_LISTING="$1" GH_FAIL="${2:-}" GITHUB_REPOSITORY=FelisMC/Felis GITHUB_OUTPUT="$root/out" PATH="$root/bin:$PATH" bash "$ER" find 2>&1
}
same() { # label want got
if [ "$2" = "$3" ]; then echo "PASS $1"; else printf 'FAIL %s: got\n%s\nwant\n%s\n' "$1" "$3" "$2"; fails=$((fails + 1)); fi
}
find_run "v1.2.3 felis-linux-amd64 felis-linux-arm64 SHA256SUMS felis-velocity.jar" >/dev/null
same "find: a complete release" "$(printf 'tag=v1.2.3\nbinary=yes\nsums=yes')" "$(cat "$root/out")"
find_run "v0.1.0 felis-linux-amd64 felis-linux-arm64" >/dev/null
same "find: a release without SHA256SUMS" "$(printf 'tag=v0.1.0\nbinary=yes\nsums=')" "$(cat "$root/out")"
find_run "v0.1.0 felis-linux-arm64 felis-linux-amd64.cdx.json SHA256SUMS.sig" >/dev/null
same "find: only exact asset names count" "$(printf 'tag=v0.1.0\nbinary=\nsums=')" "$(cat "$root/out")"
out="$(find_run "" "release not found")"
status "find: no release passes" 0 $?
same " with an empty tag, which skips the release jobs" "$(printf 'tag=\nbinary=\nsums=')" "$(cat "$root/out")"
same " and says so" "::notice::no published release yet; the readme and upgrade jobs have nothing to install" "$out"
# Anything else gh fails on leaves the newest release unknown: the step fails, and writes no
# tag that would skip the jobs.
for e in "HTTP 401: Bad credentials (https://api.github.com/graphql)" \
"API rate limit exceeded for installation ID 1." \
"error connecting to api.github.com"; do
out="$(find_run "" "$e")"
status "find: gh failing with <$e> fails" 1 $?
same " and quotes gh" "::error::gh release view failed (exit 1), so the newest release is unknown: $e " "$out"
same " and writes no outputs" "" "$(cat "$root/out")"
done
out="$(find_run "")"
status "find: gh answering nothing fails" 1 $?
same " and says so" "::error::gh release view answered without a tag" "$out"
same " and writes no outputs" "" "$(cat "$root/out")"
echo
if [ "$fails" -eq 0 ]; then echo "ALL PASS"; else echo "${fails} FAILED"; exit 1; fi
deploy/e2e_seed.sh
-208
View File
@@ -1,208 +0,0 @@
#!/bin/bash
# Seeds the database of the release the e2e upgrade job installed, and checks after the
# upgrade that every seeded row came through unchanged. The job runs it around the upgrade:
#
# sudo bash deploy/e2e_seed.sh seed # after the release installed
# sudo bash deploy/e2e_seed.sh check # after this commit ran over it
#
# A fresh install's database holds only the rows its migrations write, so without the seed
# the upgrade moves and migrates an almost empty database: the move's row-count comparison
# compares zeros and the pending migrations never meet an existing row. The seed writes the
# columns the oldest release has (v0.1.0), so it applies to every release since, and the
# check reads the same columns back: a migration that rewrites one of them on purpose
# updates the snapshot query here.
#
# Every row is inert. Nothing in it is a credential anyone holds: the session and the
# challenge carry the hash of bytes nobody kept and are spent already. Nothing is due for
# the reaper or the operator, and the retention sweep keeps spent rows for 30 days after
# they were spent. servers stays empty: its rows mirror MinecraftServer objects the
# operator and the reaper reconcile, and a row without one would not stay as written.
set -euo pipefail
phase="${1:?usage: e2e_seed.sh seed|check}"
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
KUBECTL=(/usr/local/bin/k3s kubectl)
STATE_DIR=/var/tmp/felis-e2e-seed
PG_MOVED_MARKER=/var/lib/felis/postgres-moved
TABLES=(users account_links quotas sessions webauthn_challenges platform_settings audit_logs
world_backups image_builds image_submissions account_migrations op_login_requests)
fails=0
pass() { printf 'PASS %s\n' "$*"; }
fail() { printf 'FAIL %s\n' "$*"; fails=$((fails + 1)); }
# db_where says where the platform's database lives: in felis-postgres once the install
# has it, else in the host PostgreSQL a release before it ran.
db_where() {
if "${KUBECTL[@]}" -n felis get deploy/felis-postgres >/dev/null 2>&1; then
echo pod
else
echo host
fi
}
# db_psql runs the SQL on its stdin in the felis database at <where>.
db_psql() { # where
case "$1" in
pod)
"${KUBECTL[@]}" -n felis exec -i deploy/felis-postgres -c postgres -- \
psql -X -q -At -v ON_ERROR_STOP=1 -U felis -d felis
;;
# From /, which the postgres user can always enter: psql warns about any other cwd.
host) (cd / && runuser -u postgres -- psql -X -q -At -v ON_ERROR_STOP=1 -d felis) ;;
esac
}
# snapshot prints every seeded row, one table after the other, each in key order. The host
# server's time zone is the host's and the pod's is UTC, so timestamps print in UTC.
snapshot() { # where
db_psql "$1" <<'EOF'
SET TimeZone = 'UTC';
SELECT 'users ' || row(id, username, email, role, email_verified, disabled, deleted_at, created_at, updated_at)::text
FROM users WHERE id LIKE 'e2e-seed-%' ORDER BY id;
SELECT 'account_links ' || row(user_id, mc_uuid, auth_source, verified_at)::text
FROM account_links WHERE user_id LIKE 'e2e-seed-%' ORDER BY mc_uuid;
SELECT 'quotas ' || row(user_id, max_servers, max_cpu_milli, max_memory_mb, max_storage_gb, updated_by, updated_at)::text
FROM quotas WHERE user_id LIKE 'e2e-seed-%' ORDER BY user_id;
SELECT 'sessions ' || row(token_hash, user_id, created_at, expires_at, revoked_at)::text
FROM sessions WHERE user_id LIKE 'e2e-seed-%' ORDER BY token_hash;
SELECT 'webauthn_challenges ' || row(id, user_id, purpose, session_data, expires_at, consumed_at, created_at)::text
FROM webauthn_challenges WHERE id LIKE 'e2e-seed-%' ORDER BY id;
SELECT 'platform_settings ' || row(key, value, updated_at)::text
FROM platform_settings WHERE key = 'e2e_seed';
SELECT 'audit_logs ' || row(id, actor, source, action, server_name, request_id, payload, created_at)::text
FROM audit_logs WHERE actor = 'e2e-seed-staff' ORDER BY id;
SELECT 'world_backups ' || row(id, server_name, former_owner, backup_ref, size_bytes, reason, status, created_at, expires_at, deleted_at)::text
FROM world_backups WHERE id LIKE 'e2e-seed-%' ORDER BY id;
SELECT 'image_builds ' || row(id, image_ref, status, dockerfile, context_ref, base_image, requested_by, job_name, log_ref, error, created_at, finished_at)::text
FROM image_builds WHERE id LIKE 'e2e-seed-%' ORDER BY id;
SELECT 'image_submissions ' || row(id, submitted_by, display_name, context_ref, status, image_ref, build_id, reviewed_by, reject_reason, created_at, reviewed_at)::text
FROM image_submissions WHERE id LIKE 'e2e-seed-%' ORDER BY id;
SELECT 'account_migrations ' || row(id, source_user_id, target_user_id, state, confirm_factor, confirmed_at, code_hash, code_expires_at, redeemed_at, created_at, updated_at)::text
FROM account_migrations WHERE id LIKE 'e2e-seed-%' ORDER BY id;
SELECT 'op_login_requests ' || row(id, user_id, email, expires_at, created_at, consumed_at, approved_at, approved_by)::text
FROM op_login_requests WHERE id LIKE 'e2e-seed-%' ORDER BY id;
EOF
}
# seed writes the rows in one transaction and keeps where they went and what they read as.
# Text carries quotes, backslashes, control characters and non-ASCII, and the bytea a NUL,
# so a dump or restore that mangles any of them shows in the check. Rows that the retention
# sweep would judge by age are dated now.
seed() {
local where before t
where="$(db_where)"
if ! db_psql "$where" <<'EOF'; then
BEGIN;
INSERT INTO users (id, username, email, role, email_verified, disabled, created_at, updated_at) VALUES
('e2e-seed-player', 'e2e_seed_player', '[email protected]', 'user', true, false,
'2026-01-02 03:04:05.678901+00', '2026-01-02 03:04:05.678901+00'),
('e2e-seed-staff', 'e2e_seed_staff', '[email protected]', 'admin', false, true,
'2026-01-03 00:00:00+00', '2026-01-03 00:00:00+00');
INSERT INTO account_links (user_id, mc_uuid, auth_source, verified_at) VALUES
('e2e-seed-player', '5eed0000-e2e0-4000-8000-000000000001', 'thirdparty', '2026-01-02 04:00:00+00');
INSERT INTO quotas (user_id, max_servers, max_cpu_milli, max_memory_mb, max_storage_gb, updated_by, updated_at) VALUES
('e2e-seed-player', 2, 4000, 8192, NULL, 'e2e-seed-staff', '2026-01-04 00:00:00+00');
INSERT INTO sessions (token_hash, user_id, created_at, expires_at, revoked_at) VALUES
(encode(sha256(convert_to(gen_random_uuid()::text, 'UTF8')), 'hex'), 'e2e-seed-player',
now(), now() + interval '30 days', now());
INSERT INTO webauthn_challenges (id, user_id, purpose, session_data, expires_at, consumed_at, created_at) VALUES
('e2e-seed-challenge', 'e2e-seed-player', 'passkey_register', '\x00ff0a0d5c27'::bytea,
now() + interval '5 minutes', now(), now());
INSERT INTO platform_settings (key, value, updated_at) VALUES
('e2e_seed', '{"text": "quote '' dq \" backslash \\ tab\t newline\n 猫 🐱", "n": 1.50, "list": [1, null, true, {"k": "v"}]}',
'2026-01-05 00:00:00+00');
INSERT INTO audit_logs (actor, source, action, server_name, request_id, payload, created_at) VALUES
('e2e-seed-staff', 'panel', 'e2e.seed', NULL, 'e2e-seed-req-1', '{"reason": "种子 \"quoted\"", "ids": [1, 2, 3]}', now()),
('e2e-seed-staff', 'cli', 'e2e.seed', 'e2e-seed-world', 'e2e-seed-req-2', NULL, now());
INSERT INTO world_backups (id, server_name, former_owner, backup_ref, size_bytes, reason, status, created_at, expires_at, deleted_at) VALUES
('e2e-seed-backup', 'e2e-seed-world', 'e2e-seed-player', 'e2e-seed/world.tar.zst', 123456789012, 'manual', 'deleted',
'2026-01-06 00:00:00+00', '2026-04-06 00:00:00+00', '2026-04-07 00:00:00+00');
INSERT INTO image_builds (id, image_ref, status, dockerfile, context_ref, requested_by, error, created_at, finished_at) VALUES
('e2e-seed-build', 'registry.felis.svc:5000/user-uploads/e2e-seed:latest', 'failed',
E'FROM scratch\nLABEL note="e2e seed"\n', 'e2e-seed/context', 'e2e-seed-staff', 'e2e seed: never built',
'2026-01-07 00:00:00+00', '2026-01-07 00:01:00+00');
INSERT INTO image_submissions (id, submitted_by, display_name, context_ref, status, reviewed_by, reject_reason, created_at, reviewed_at) VALUES
('e2e-seed-submission', 'e2e-seed-player', '种子整合包 e2e', 'e2e-seed/submission', 'rejected', 'e2e-seed-staff', 'e2e seed',
'2026-01-08 00:00:00+00', '2026-01-08 01:00:00+00');
INSERT INTO account_migrations (id, source_user_id, target_user_id, state, confirm_factor, confirmed_at, redeemed_at, created_at, updated_at) VALUES
('e2e-seed-migration', 'e2e-seed-staff', 'e2e-seed-player', 'redeemed', 'email_otp', '2026-01-09 00:00:00+00',
'2026-01-09 00:05:00+00', '2026-01-09 00:00:00+00', '2026-01-09 00:05:00+00');
INSERT INTO op_login_requests (id, user_id, email, expires_at, created_at, consumed_at, approved_at, approved_by) VALUES
('e2e-seed-oplogin', 'e2e-seed-staff', '[email protected]', now() + interval '10 minutes', now(), now(), now(), 'e2e-seed-staff');
COMMIT;
EOF
fail "seed the release's database (${where})"
return
fi
mkdir -p "$STATE_DIR"
printf '%s\n' "$where" > "${STATE_DIR}/where"
if ! before="$(snapshot "$where")"; then
fail "read the seeded rows back"
return
fi
printf '%s\n' "$before" > "${STATE_DIR}/before"
for t in "${TABLES[@]}"; do
if grep -q "^${t} " <<<"$before"; then
pass "seeded ${t} (${where})"
else
fail "seeded ${t} (${where}): the snapshot has no row of it"
fi
done
}
# check_upgrade reads the seeded rows from felis-postgres and compares them with what the
# release's database held. A database the upgrade moved off the host also left the marker.
check_upgrade() {
local where after seq
if [ ! -s "${STATE_DIR}/before" ]; then
fail "the seed step left its snapshot in ${STATE_DIR}/before"
return
fi
where="$(db_where)"
if [ "$where" != pod ]; then
fail "the database lives in felis-postgres after the upgrade"
return
fi
if [ "$(cat "${STATE_DIR}/where")" = host ]; then
if [ -f "$PG_MOVED_MARKER" ]; then
pass "the upgrade moved the seeded host database into felis-postgres"
else
fail "the upgrade moved the seeded host database into felis-postgres: no ${PG_MOVED_MARKER}"
fi
fi
if ! after="$(snapshot pod)"; then
fail "read the seeded rows from felis-postgres"
return
fi
if [ "$after" = "$(cat "${STATE_DIR}/before")" ]; then
pass "every seeded row came through the upgrade unchanged"
else
fail "the seeded rows changed across the upgrade (< release, > felis-postgres):"
diff "${STATE_DIR}/before" <(printf '%s\n' "$after") || true
fi
# A restore that loses a sequence's position hands out ids that are taken: the next
# audit row would fail on its primary key.
seq="$(db_psql pod <<<"SELECT coalesce(pg_sequence_last_value(pg_get_serial_sequence('audit_logs', 'id')::regclass), 0) >= (SELECT max(id) FROM audit_logs)")" || seq=error
if [ "$seq" = t ]; then
pass "audit_logs' id sequence is past every seeded id"
else
fail "audit_logs' id sequence is past every seeded id (${seq})"
fi
}
case "$phase" in
seed) seed ;;
check) check_upgrade ;;
*)
echo "usage: e2e_seed.sh seed|check" >&2
exit 2
;;
esac
if [ "$fails" -eq 0 ]; then
echo "ALL PASS (seed ${phase})"
exit 0
fi
echo "${fails} FAILED (seed ${phase})"
exit 1
Loaded 100 of 708 files, more files were not shown because too many files have changed in this diff. Show more