Loading cmd/felis/manifests.go +2 −2 Changes for cmd/felis/manifests.go: 2 added lines, 2 removed lines. Original line number Diff line number Diff line Loading @@ -124,8 +124,8 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int { "multi-node cluster you MUST pass --reaper-node <name> (or add a nodeSelector) for the node holding the " + "worlds, or the reaper may schedule where the hostPath is empty" if *reaperNode != "" { pin = fmt.Sprintf("the CronJob is pinned to node %q via kubernetes.io/hostname — keep this pointed at the "+ "node that actually holds the world volumes", *reaperNode) pin = fmt.Sprintf("the CronJob and its worlds-root PV are pinned to node %q via kubernetes.io/hostname — "+ "keep this pointed at the node that actually holds the world volumes", *reaperNode) } fmt.Fprintf(stderr, "felis manifests: note: rendering the retention reaper CronJob (worlds hostPath %q). "+ "These points are NOT verified here:\n"+ Loading deploy/bootstrap.sh +4 −12 Changes for deploy/bootstrap.sh: 4 added lines, 12 removed lines. Original line number Diff line number Diff line Loading @@ -2496,18 +2496,10 @@ deploy_bundle() { # always travels with it because it must equal the [archive] local_path written above. if [ -n "$FELIS_WORLDS_HOST_PATH" ]; then log "retention enabled: the daily reaper will read worlds from ${FELIS_WORLDS_HOST_PATH}" # The reaper pod runs as the tree's non-root uid (1000, platform.workloads.nonRootUID) # and must traverse into the per-volume directories under this root. k3s's own storage # root ships 0700 root:root, so grant traverse — an ACL entry when the host has setfacl, # otherwise the equivalent o+x. Traverse only: no listing either way, and the per-volume # directories themselves are world-accessible (local-path creates them 0777). if [ -d "$FELIS_WORLDS_HOST_PATH" ]; then if command -v setfacl >/dev/null 2>&1; then setfacl -m u:1000:x "$FELIS_WORLDS_HOST_PATH" || chmod o+x "$FELIS_WORLDS_HOST_PATH" else chmod o+x "$FELIS_WORLDS_HOST_PATH" fi else # The reaper reads this root as root with DAC_OVERRIDE (platform.reaperPodSecurityContext) # through a static hostPath PV, so the host directory keeps k3s's own 0700 root:root and # needs no extra grant. It must exist, though: the PV declares type Directory. if [ ! -d "$FELIS_WORLDS_HOST_PATH" ]; then warn "worlds root ${FELIS_WORLDS_HOST_PATH} does not exist yet; the reaper CronJob cannot start until it does (hostPath type Directory)" fi manifest_args+=(--worlds-host-path "$FELIS_WORLDS_HOST_PATH" --archive-local-path "$FELIS_ARCHIVE_LOCAL_PATH") Loading deploy/bootstrap_test.sh +7 −3 Changes for deploy/bootstrap_test.sh: 7 added lines, 3 removed lines. Original line number Diff line number Diff line Loading @@ -670,6 +670,7 @@ run_bundle_flags() { # backup-pvc worlds-host-path kube() { cat; } myManifests() { printf "%s\n" "$@"; } setfacl() { printf "SETFACL %s\n" "$*"; } chmod() { printf "CHMOD %s\n" "$*"; } node_global_cidrs() { printf "203.0.113.7/32\n2001:db8::7/128\n"; } run_bundle() { '"$mblock"' Loading Loading @@ -723,11 +724,14 @@ missing="/tmp/felis-worlds-root-must-not-exist-$$" out="$(run_bundle_flags felis-backups "$missing")" expect "a missing worlds root is warned about, not silently skipped" "WARN: worlds root $missing does not exist yet" "$out" # The reaper pod is non-root (uid 1000) and k3s ships the storage root 0700 root:root, so # the installer must grant traverse or every archive dies with permission denied. # The reaper reads the root as root with DAC_OVERRIDE through a static PV, so an existing # root is left exactly as k3s shipped it: uid 1000 is now the game servers' uid, and a # traverse grant for it on the node's storage root would serve nothing but them. wdir="$(mktemp -d)" out="$(run_bundle_flags felis-backups "$wdir")" expect "enabling retention grants the reaper uid traverse on the worlds root" "SETFACL -m u:1000:x $wdir" "$out" case "$out" in *SETFACL*|*CHMOD*|*WARN*) echo "FAIL: an existing worlds root must get no grant and no warning: $out"; fails=$((fails + 1)) ;; esac # --- the registry mirror writer ----------------------------------------------------------- # k3s only consults registries.yaml at agent start, so a CONTENT change must restart k3s and Loading docs/troubleshooting.md +38 −13 Changes for docs/troubleshooting.md: 38 added lines, 13 removed lines. Original line number Diff line number Diff line Loading @@ -91,6 +91,26 @@ kubectl describe pod <pod> # look at Events + container State Fix the StorageClass name or capacity. [INTEGRATION-ONLY.] - **Container crash-looping before the readiness port opens** → check container logs; this is a backend/entrypoint problem, not a Felis problem. - **Stuck in `Init:` or `AccessDeniedException` / `Permission denied` in the log** → the pod runs as uid/gid **1000** (`naming.GameUID`) with every capability dropped, whatever `USER` the image declares. Before the server starts, the `prepare-data` initContainer (`felis init-volume`, root with only `CHOWN` + `DAC_OVERRIDE`) hands every world entry not yet owned by 1000:1000 to that uid, so a world written by an older root-run release or extracted by a restore Job is fixed on its next start: `kubectl logs <pod> -c prepare-data` prints how many entries it changed and lists up to 20 it could not. An image that writes outside `/data` and `/tmp` (a directory baked into the image as root) cannot run as uid 1000; rebuild it to keep its state under `/data`. [GO-TESTED: `TestBuildStatefulSetRunsGameAsNonRoot`, `TestChownTreeHandsOverMismatchedEntries`; INTEGRATION-ONLY for the walk on a live volume.] - **`FailedCreate … violates PodSecurity "baseline"`** on the StatefulSet or a Job → the minecraft namespace enforces the PodSecurity `baseline` profile (`pod-security.kubernetes.io/enforce=baseline`, set by the install bundle). Everything Felis renders there fits it; a pod that is refused was edited or created outside Felis (hostPath, hostPort, privileged, extra capabilities). `kubectl get events -n minecraft --field-selector reason=FailedCreate` names the field. [GO-TESTED: `TestObjects_MinecraftNamespaceEnforcesBaseline`.] ### 1b. `RconSecretUnavailable` — RCON secret missing or malformed Loading Loading @@ -681,7 +701,13 @@ store paths are [INTEGRATION-ONLY]. ### Where worlds are read from (hostPath resolution) The CronJob mounts `--worlds-host-path` read-only at `/worlds`; the resolver The CronJob mounts `--worlds-host-path` read-only at `/worlds` through a static PersistentVolume (`felis-worlds-root-<digest>`, hostPath type `Directory`, `Retain`, pre-bound to the same-named PVC in the minecraft namespace), since the namespace's PodSecurity baseline refuses an inline hostPath in any pod. A re-install with a different worlds root or `--reaper-node` renders a new pair under a new digest; the old PV/PVC pair is left behind unused and can be deleted by hand (Retain: deleting it never touches the directory). The resolver runs `cmd/felis/reaper.resolveWorldDir`: it looks for `<root>/<pvc>`, then for the stock local-path directory `<root>/<pv-name>_<ns>_<pvc-name>` derived from the live PVC's `spec.volumeName` (never a glob — a leftover directory of a Loading @@ -691,19 +717,18 @@ supported way to enable retention on a stock install. Two deployment facts the resolver cannot fix: - **Permissions.** The reaper Pod runs as **root** and carries `DAC_OVERRIDE`: worlds are written by the game image's own UID (root for every Paper image we ship), and Paper saves `level.dat` mode-0600, so any fixed non-root identity (the previous uid-1000 convention, and the ACL setup that went with it) could neither walk the tree nor read the files — every archive failed `open …/level.dat: permission denied` and the same defect failed on-demand backups/restores. Root is the same identity the game container itself runs as (see the operator's forwarding-init note); `DAC_OVERRIDE` extends the archive to game images with a different UID. If a world is still **preserved** while a reap was expected, it is now a different cause: check the run's ERROR logs for the resolver's `lstat` messages before suspecting permissions. k3s's storage root is `0700 root:root`, worlds are written by the game uid (1000) — or by root, for a world an older release wrote — and Paper saves `level.dat` mode-0600, so a fixed non-root identity (the previous uid-1000 convention, and the ACL setup that went with it) could neither walk the tree nor read the files — every archive failed `open …/level.dat: permission denied`. The installer no longer grants uid 1000 any access to the storage root: that uid is now the game servers'. If a world is still **preserved** while a reap was expected, it is a different cause: check the run's ERROR logs for the resolver's `lstat` messages before suspecting permissions. - **Node placement.** Multi-node clusters: the world's directory exists only on the node holding its volume, and the CronJob sets no `nodeSelector`, so add one (single-node starters are pinned implicitly). the node holding its volume, so pass `--reaper-node`; it pins both the CronJob's pod and the PV (single-node starters are pinned implicitly). --- Loading internal/platform/bundle.go +39 −1 Changes for internal/platform/bundle.go: 39 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -3,6 +3,7 @@ package platform import ( "bytes" "fmt" "maps" "felis.lolicon.best/internal/build" "felis.lolicon.best/internal/restore" Loading Loading @@ -51,7 +52,11 @@ func Objects(p Params) []Object { // namespaceSelectors match on. (K8s ≥1.21 adds this label automatically, but // rendering it makes the bundle self-contained and the selectors provable.) for _, ns := range distinctNamespaces(p) { objs = append(objs, namespaceObject(ns)) nsObj := namespaceObject(ns) if ns == p.MinecraftNamespace && minecraftNamespaceIsOwn(p) { maps.Copy(nsObj.Labels, minecraftPodSecurityLabels) } objs = append(objs, nsObj) } // Control-plane RBAC: SAs, then Roles, then RoleBindings. Loading Loading @@ -143,6 +148,39 @@ func distinctNamespaces(p Params) []string { return out } // minecraftPodSecurityLabels put the minecraft namespace under the PodSecurity // admission baseline profile. Everything Felis runs there fits it: game servers run // as naming.GameUID with every capability dropped, their prepare-data init and the // file/backup/restore Jobs run as root holding at most CHOWN and DAC_OVERRIDE (both // on baseline's allow-list), and the reaper reaches the node's worlds-root through a // static PV rather than an inline hostPath. What baseline then refuses — privileged // containers, host namespaces and ports, inline hostPath, extra capabilities — is // exactly what a pod smuggled in through any other write path to this namespace // would need to reach the node. // // warn repeats the enforced level so a StatefulSet or Job that would render a // refused pod reports it at apply time, instead of the controller failing to create // pods quietly. audit records restricted-profile violations for the path toward // restricted (only the root prepare-data init and root Jobs stand in its way). var minecraftPodSecurityLabels = map[string]string{ "pod-security.kubernetes.io/enforce": "baseline", "pod-security.kubernetes.io/enforce-version": "latest", "pod-security.kubernetes.io/warn": "baseline", "pod-security.kubernetes.io/warn-version": "latest", "pod-security.kubernetes.io/audit": "restricted", "pod-security.kubernetes.io/audit-version": "latest", } // minecraftNamespaceIsOwn reports whether the minecraft namespace is shared with // no other component. The registry (hostPort) and the build Jobs do not fit the // baseline profile, so the labels go on only when neither lives there, and the // control plane's namespace is never labelled from here. func minecraftNamespaceIsOwn(p Params) bool { return p.MinecraftNamespace != p.ControlNamespace && p.MinecraftNamespace != p.BuildNamespace && p.MinecraftNamespace != p.RegistryNamespace } // namespaceObject renders a Namespace carrying the immutable name label the // NetworkPolicy namespaceSelectors key on. func namespaceObject(name string) *corev1.Namespace { Loading Loading
cmd/felis/manifests.go +2 −2 Changes for cmd/felis/manifests.go: 2 added lines, 2 removed lines. Original line number Diff line number Diff line Loading @@ -124,8 +124,8 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int { "multi-node cluster you MUST pass --reaper-node <name> (or add a nodeSelector) for the node holding the " + "worlds, or the reaper may schedule where the hostPath is empty" if *reaperNode != "" { pin = fmt.Sprintf("the CronJob is pinned to node %q via kubernetes.io/hostname — keep this pointed at the "+ "node that actually holds the world volumes", *reaperNode) pin = fmt.Sprintf("the CronJob and its worlds-root PV are pinned to node %q via kubernetes.io/hostname — "+ "keep this pointed at the node that actually holds the world volumes", *reaperNode) } fmt.Fprintf(stderr, "felis manifests: note: rendering the retention reaper CronJob (worlds hostPath %q). "+ "These points are NOT verified here:\n"+ Loading
deploy/bootstrap.sh +4 −12 Changes for deploy/bootstrap.sh: 4 added lines, 12 removed lines. Original line number Diff line number Diff line Loading @@ -2496,18 +2496,10 @@ deploy_bundle() { # always travels with it because it must equal the [archive] local_path written above. if [ -n "$FELIS_WORLDS_HOST_PATH" ]; then log "retention enabled: the daily reaper will read worlds from ${FELIS_WORLDS_HOST_PATH}" # The reaper pod runs as the tree's non-root uid (1000, platform.workloads.nonRootUID) # and must traverse into the per-volume directories under this root. k3s's own storage # root ships 0700 root:root, so grant traverse — an ACL entry when the host has setfacl, # otherwise the equivalent o+x. Traverse only: no listing either way, and the per-volume # directories themselves are world-accessible (local-path creates them 0777). if [ -d "$FELIS_WORLDS_HOST_PATH" ]; then if command -v setfacl >/dev/null 2>&1; then setfacl -m u:1000:x "$FELIS_WORLDS_HOST_PATH" || chmod o+x "$FELIS_WORLDS_HOST_PATH" else chmod o+x "$FELIS_WORLDS_HOST_PATH" fi else # The reaper reads this root as root with DAC_OVERRIDE (platform.reaperPodSecurityContext) # through a static hostPath PV, so the host directory keeps k3s's own 0700 root:root and # needs no extra grant. It must exist, though: the PV declares type Directory. if [ ! -d "$FELIS_WORLDS_HOST_PATH" ]; then warn "worlds root ${FELIS_WORLDS_HOST_PATH} does not exist yet; the reaper CronJob cannot start until it does (hostPath type Directory)" fi manifest_args+=(--worlds-host-path "$FELIS_WORLDS_HOST_PATH" --archive-local-path "$FELIS_ARCHIVE_LOCAL_PATH") Loading
deploy/bootstrap_test.sh +7 −3 Changes for deploy/bootstrap_test.sh: 7 added lines, 3 removed lines. Original line number Diff line number Diff line Loading @@ -670,6 +670,7 @@ run_bundle_flags() { # backup-pvc worlds-host-path kube() { cat; } myManifests() { printf "%s\n" "$@"; } setfacl() { printf "SETFACL %s\n" "$*"; } chmod() { printf "CHMOD %s\n" "$*"; } node_global_cidrs() { printf "203.0.113.7/32\n2001:db8::7/128\n"; } run_bundle() { '"$mblock"' Loading Loading @@ -723,11 +724,14 @@ missing="/tmp/felis-worlds-root-must-not-exist-$$" out="$(run_bundle_flags felis-backups "$missing")" expect "a missing worlds root is warned about, not silently skipped" "WARN: worlds root $missing does not exist yet" "$out" # The reaper pod is non-root (uid 1000) and k3s ships the storage root 0700 root:root, so # the installer must grant traverse or every archive dies with permission denied. # The reaper reads the root as root with DAC_OVERRIDE through a static PV, so an existing # root is left exactly as k3s shipped it: uid 1000 is now the game servers' uid, and a # traverse grant for it on the node's storage root would serve nothing but them. wdir="$(mktemp -d)" out="$(run_bundle_flags felis-backups "$wdir")" expect "enabling retention grants the reaper uid traverse on the worlds root" "SETFACL -m u:1000:x $wdir" "$out" case "$out" in *SETFACL*|*CHMOD*|*WARN*) echo "FAIL: an existing worlds root must get no grant and no warning: $out"; fails=$((fails + 1)) ;; esac # --- the registry mirror writer ----------------------------------------------------------- # k3s only consults registries.yaml at agent start, so a CONTENT change must restart k3s and Loading
docs/troubleshooting.md +38 −13 Changes for docs/troubleshooting.md: 38 added lines, 13 removed lines. Original line number Diff line number Diff line Loading @@ -91,6 +91,26 @@ kubectl describe pod <pod> # look at Events + container State Fix the StorageClass name or capacity. [INTEGRATION-ONLY.] - **Container crash-looping before the readiness port opens** → check container logs; this is a backend/entrypoint problem, not a Felis problem. - **Stuck in `Init:` or `AccessDeniedException` / `Permission denied` in the log** → the pod runs as uid/gid **1000** (`naming.GameUID`) with every capability dropped, whatever `USER` the image declares. Before the server starts, the `prepare-data` initContainer (`felis init-volume`, root with only `CHOWN` + `DAC_OVERRIDE`) hands every world entry not yet owned by 1000:1000 to that uid, so a world written by an older root-run release or extracted by a restore Job is fixed on its next start: `kubectl logs <pod> -c prepare-data` prints how many entries it changed and lists up to 20 it could not. An image that writes outside `/data` and `/tmp` (a directory baked into the image as root) cannot run as uid 1000; rebuild it to keep its state under `/data`. [GO-TESTED: `TestBuildStatefulSetRunsGameAsNonRoot`, `TestChownTreeHandsOverMismatchedEntries`; INTEGRATION-ONLY for the walk on a live volume.] - **`FailedCreate … violates PodSecurity "baseline"`** on the StatefulSet or a Job → the minecraft namespace enforces the PodSecurity `baseline` profile (`pod-security.kubernetes.io/enforce=baseline`, set by the install bundle). Everything Felis renders there fits it; a pod that is refused was edited or created outside Felis (hostPath, hostPort, privileged, extra capabilities). `kubectl get events -n minecraft --field-selector reason=FailedCreate` names the field. [GO-TESTED: `TestObjects_MinecraftNamespaceEnforcesBaseline`.] ### 1b. `RconSecretUnavailable` — RCON secret missing or malformed Loading Loading @@ -681,7 +701,13 @@ store paths are [INTEGRATION-ONLY]. ### Where worlds are read from (hostPath resolution) The CronJob mounts `--worlds-host-path` read-only at `/worlds`; the resolver The CronJob mounts `--worlds-host-path` read-only at `/worlds` through a static PersistentVolume (`felis-worlds-root-<digest>`, hostPath type `Directory`, `Retain`, pre-bound to the same-named PVC in the minecraft namespace), since the namespace's PodSecurity baseline refuses an inline hostPath in any pod. A re-install with a different worlds root or `--reaper-node` renders a new pair under a new digest; the old PV/PVC pair is left behind unused and can be deleted by hand (Retain: deleting it never touches the directory). The resolver runs `cmd/felis/reaper.resolveWorldDir`: it looks for `<root>/<pvc>`, then for the stock local-path directory `<root>/<pv-name>_<ns>_<pvc-name>` derived from the live PVC's `spec.volumeName` (never a glob — a leftover directory of a Loading @@ -691,19 +717,18 @@ supported way to enable retention on a stock install. Two deployment facts the resolver cannot fix: - **Permissions.** The reaper Pod runs as **root** and carries `DAC_OVERRIDE`: worlds are written by the game image's own UID (root for every Paper image we ship), and Paper saves `level.dat` mode-0600, so any fixed non-root identity (the previous uid-1000 convention, and the ACL setup that went with it) could neither walk the tree nor read the files — every archive failed `open …/level.dat: permission denied` and the same defect failed on-demand backups/restores. Root is the same identity the game container itself runs as (see the operator's forwarding-init note); `DAC_OVERRIDE` extends the archive to game images with a different UID. If a world is still **preserved** while a reap was expected, it is now a different cause: check the run's ERROR logs for the resolver's `lstat` messages before suspecting permissions. k3s's storage root is `0700 root:root`, worlds are written by the game uid (1000) — or by root, for a world an older release wrote — and Paper saves `level.dat` mode-0600, so a fixed non-root identity (the previous uid-1000 convention, and the ACL setup that went with it) could neither walk the tree nor read the files — every archive failed `open …/level.dat: permission denied`. The installer no longer grants uid 1000 any access to the storage root: that uid is now the game servers'. If a world is still **preserved** while a reap was expected, it is a different cause: check the run's ERROR logs for the resolver's `lstat` messages before suspecting permissions. - **Node placement.** Multi-node clusters: the world's directory exists only on the node holding its volume, and the CronJob sets no `nodeSelector`, so add one (single-node starters are pinned implicitly). the node holding its volume, so pass `--reaper-node`; it pins both the CronJob's pod and the PV (single-node starters are pinned implicitly). --- Loading
internal/platform/bundle.go +39 −1 Changes for internal/platform/bundle.go: 39 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -3,6 +3,7 @@ package platform import ( "bytes" "fmt" "maps" "felis.lolicon.best/internal/build" "felis.lolicon.best/internal/restore" Loading Loading @@ -51,7 +52,11 @@ func Objects(p Params) []Object { // namespaceSelectors match on. (K8s ≥1.21 adds this label automatically, but // rendering it makes the bundle self-contained and the selectors provable.) for _, ns := range distinctNamespaces(p) { objs = append(objs, namespaceObject(ns)) nsObj := namespaceObject(ns) if ns == p.MinecraftNamespace && minecraftNamespaceIsOwn(p) { maps.Copy(nsObj.Labels, minecraftPodSecurityLabels) } objs = append(objs, nsObj) } // Control-plane RBAC: SAs, then Roles, then RoleBindings. Loading Loading @@ -143,6 +148,39 @@ func distinctNamespaces(p Params) []string { return out } // minecraftPodSecurityLabels put the minecraft namespace under the PodSecurity // admission baseline profile. Everything Felis runs there fits it: game servers run // as naming.GameUID with every capability dropped, their prepare-data init and the // file/backup/restore Jobs run as root holding at most CHOWN and DAC_OVERRIDE (both // on baseline's allow-list), and the reaper reaches the node's worlds-root through a // static PV rather than an inline hostPath. What baseline then refuses — privileged // containers, host namespaces and ports, inline hostPath, extra capabilities — is // exactly what a pod smuggled in through any other write path to this namespace // would need to reach the node. // // warn repeats the enforced level so a StatefulSet or Job that would render a // refused pod reports it at apply time, instead of the controller failing to create // pods quietly. audit records restricted-profile violations for the path toward // restricted (only the root prepare-data init and root Jobs stand in its way). var minecraftPodSecurityLabels = map[string]string{ "pod-security.kubernetes.io/enforce": "baseline", "pod-security.kubernetes.io/enforce-version": "latest", "pod-security.kubernetes.io/warn": "baseline", "pod-security.kubernetes.io/warn-version": "latest", "pod-security.kubernetes.io/audit": "restricted", "pod-security.kubernetes.io/audit-version": "latest", } // minecraftNamespaceIsOwn reports whether the minecraft namespace is shared with // no other component. The registry (hostPort) and the build Jobs do not fit the // baseline profile, so the labels go on only when neither lives there, and the // control plane's namespace is never labelled from here. func minecraftNamespaceIsOwn(p Params) bool { return p.MinecraftNamespace != p.ControlNamespace && p.MinecraftNamespace != p.BuildNamespace && p.MinecraftNamespace != p.RegistryNamespace } // namespaceObject renders a Namespace carrying the immutable name label the // NetworkPolicy namespaceSelectors key on. func namespaceObject(name string) *corev1.Namespace { Loading