diff --git a/cmd/felis/manifests.go b/cmd/felis/manifests.go index 51fb333..a92d9bf 100644 --- a/cmd/felis/manifests.go +++ b/cmd/felis/manifests.go @@ -100,20 +100,24 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int { "(the archive store; default felis-backups)") return 2 } - // The reaper WILL render. Two deployment preconditions this generator cannot - // check would SILENTLY turn retention into a no-op if unmet — surface them as - // loudly as the fail-closed cases above, so an operator is never left with a - // reaper that reaps an empty directory. (Both are also in the WorldsHostPath - // flag/field docs, but nobody deploying from stdout reads those.) + // The reaper WILL render. Three deployment preconditions this generator cannot + // check would silently turn retention into a no-op (or a permission-denied + // loop) if unmet — surface them as loudly as the fail-closed cases above, so + // an operator is never left with a reaper that reaps nothing. (All three are + // also in the WorldsHostPath flag/field docs, but nobody deploying from stdout + // reads those.) fmt.Fprintf(stderr, "felis manifests: note: rendering the retention reaper CronJob (worlds hostPath %q). "+ - "Two preconditions are NOT verified here:\n"+ + "Three preconditions are NOT verified here:\n"+ " - the node's world volumes must actually live below %s: the reaper resolves a world as "+ "%s/, then as the stock local-path directory /__ (what k3s "+ "writes under /var/lib/rancher/k3s/storage). Any other provisioner needs its volumes exposed as "+ "/, or each candidate's archive fails and the world is preserved;\n"+ + " - the reaper pod runs as uid 1000 and must be able to traverse %s (k3s ships its storage root "+ + "0700 root:root — the installer grants `setfacl -m u:1000:x` or o+x; a manual install must do the "+ + "same or every archive fails with permission denied and the world is preserved);\n"+ " - the CronJob sets NO nodeSelector: a single-node starter pins it to the worlds implicitly, but "+ "on a multi-node cluster you MUST add a nodeSelector for the node holding the worlds, or the reaper "+ - "may schedule where the hostPath is empty.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath) + "may schedule where the hostPath is empty.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, *worldsHostPath) } else { fmt.Fprintln(stderr, "felis manifests: note: retention reaper CronJob not rendered "+ "(pass --worlds-host-path and --archive-local-path — the archive PVC defaults to felis-backups — to enable it)") diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index ed61e44..9cce91c 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -71,7 +71,9 @@ # FELIS_WORLDS_HOST_PATH node directory holding the world volumes (on the k3s this # installer provisions: /var/lib/rancher/k3s/storage). Setting it # enables the daily retention reaper, which archives and then deletes -# worlds idle beyond the retention window (default: unset = no reaper) +# worlds idle beyond the retention window, and grants the reaper's +# uid (1000) traverse access to that root — k3s ships it 0700 +# root:root (default: unset = no reaper) # PKG_LOCK_TIMEOUT seconds to wait for package-manager locks (default: 900) # APT_LOCK_TIMEOUT legacy alias for PKG_LOCK_TIMEOUT set -Eeuo pipefail @@ -2175,6 +2177,20 @@ deploy_bundle() { # always travels with it because it must equal the [archive] local_path written above. if [ -n "$FELIS_WORLDS_HOST_PATH" ]; then log "retention enabled: the daily reaper will read worlds from ${FELIS_WORLDS_HOST_PATH}" + # The reaper pod runs as the tree's non-root uid (1000, platform.workloads.nonRootUID) + # and must traverse into the per-volume directories under this root. k3s's own storage + # root ships 0700 root:root, so grant traverse — an ACL entry when the host has setfacl, + # otherwise the equivalent o+x. Traverse only: no listing either way, and the per-volume + # directories themselves are world-accessible (local-path creates them 0777). + if [ -d "$FELIS_WORLDS_HOST_PATH" ]; then + if command -v setfacl >/dev/null 2>&1; then + setfacl -m u:1000:x "$FELIS_WORLDS_HOST_PATH" || chmod o+x "$FELIS_WORLDS_HOST_PATH" + else + chmod o+x "$FELIS_WORLDS_HOST_PATH" + fi + else + warn "worlds root ${FELIS_WORLDS_HOST_PATH} does not exist yet; the reaper CronJob cannot start until it does (hostPath type Directory)" + fi manifest_args+=(--worlds-host-path "$FELIS_WORLDS_HOST_PATH" --archive-local-path "$FELIS_ARCHIVE_LOCAL_PATH") fi "$HOST_BIN" manifests "${manifest_args[@]}" | kube apply -f - diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index aebf5e7..ad73c42 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -579,9 +579,9 @@ expect "a failed fetch into an existing checkout names the token" "set FELIS_GIT # honest 503), and FELIS_WORLDS_HOST_PATH must turn into the reaper's two flags or an # operator's retention enablement silently renders no CronJob. Extracted, not retyped. -mblock="$(awk '/^ local -a manifest_args=\(/,/kube apply -f -/' "$BS")" +mblock="$(awk '/^ log "rendering \+ applying the control-plane bundle"/,/kube apply -f -/' "$BS")" [ -n "$mblock" ] || { echo "FAIL: no manifest_args block found in $BS"; exit 1; } -[ "$(printf '%s\n' "$mblock" | wc -l)" -lt 30 ] \ +[ "$(printf '%s\n' "$mblock" | wc -l)" -lt 40 ] \ || { echo "FAIL: the extracted block is not the manifest_args block -- did it move?"; exit 1; } run_bundle_flags() { # backup-pvc worlds-host-path @@ -589,8 +589,10 @@ run_bundle_flags() { # backup-pvc worlds-host-path FELIS_BACKUP_PVC="$1" FELIS_WORLDS_HOST_PATH="$2" FELIS_ARCHIVE_LOCAL_PATH=/var/lib/felis/archives \ HOST_BIN=myManifests bash -c ' log() { :; } + warn() { printf "WARN: %s\n" "$*"; } kube() { cat; } myManifests() { printf "%s\n" "$@"; } + setfacl() { printf "SETFACL %s\n" "$*"; } run_bundle() { '"$mblock"' } @@ -612,6 +614,13 @@ expect "enabling retention passes the worlds root" "--worlds-host-path /var/lib/rancher/k3s/storage" "$out" expect "enabling retention passes the archive mount that must match felis.toml" "--archive-local-path /var/lib/felis/archives" "$out" +expect "a missing worlds root is warned about, not silently skipped" "WARN: worlds root /var/lib/rancher/k3s/storage does not exist yet" "$out" + +# The reaper pod is non-root (uid 1000) and k3s ships the storage root 0700 root:root, so +# the installer must grant traverse or every archive dies with permission denied. +wdir="$(mktemp -d)" +out="$(run_bundle_flags felis-backups "$wdir")" +expect "enabling retention grants the reaper uid traverse on the worlds root" "SETFACL -m u:1000:x $wdir" "$out" # --------------------------------------------------------------------------------------- if [ "$fails" -eq 0 ]; then diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c9fe44a..35375a5 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -468,6 +468,26 @@ Only `TarLocal` (tar+gzip) archiving is implemented; VolumeSnapshot/Longhorn backends return `not implemented in this build`. The live PVC delete / Postgres store paths are [INTEGRATION-ONLY]. +### Where worlds are read from (hostPath resolution) + +The CronJob mounts `--worlds-host-path` read-only at `/worlds`; the resolver +runs `cmd/felis/reaper.resolveWorldDir`: it looks for `/`, then for +the stock local-path directory `/__` derived from +the live PVC's `spec.volumeName` (never a glob — a leftover directory of a +deleted PV must not stand in for the world the PVC currently binds). Pointing +the flag at k3s's storage root (`/var/lib/rancher/k3s/storage`) is therefore the +supported way to enable retention on a stock install. Two deployment facts the +resolver cannot fix: + +- **Permissions.** The reaper pod runs as uid 1000, while k3s creates its + storage root `0700 root:root`. Without traverse (`setfacl -m u:1000:x`, or + `chmod o+x`; the installer applies this when `FELIS_WORLDS_HOST_PATH` is set) + every walk fails `permission denied` / `lstat …: permission denied` and the + world is **preserved**, never reaped — a silent no-op with ERROR logs. +- **Node placement.** Multi-node clusters: the world's directory exists only on + the node holding its volume, and the CronJob sets no `nodeSelector`, so add + one (single-node starters are pinned implicitly). + --- ## 11. Idle auto-stop never fires; player count always shows 0