diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 6eecd26..9901185 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -59,7 +59,10 @@ # FELIS_GITHUB_TOKEN GitHub token; REQUIRED while the repo is private # FELIS_REF branch/tag/sha — pins the build, overrides the channel, and forces a # source build (naming a ref asks for that tree, not a published asset) -# FELIS_IMAGE local image tag (default: felis:demo — never :latest) +# FELIS_IMAGE control-plane image ref (default: +# registry.felis.svc:5000/felis/felis:demo — never :latest; +# anything not under the registry is used as-is but is NOT +# mirrored into it, so it has no pull source after an image GC) # FELIS_ROOT_DOMAIN deployment root domain (default: .nip.io) # FELIS_PANEL_NODEPORT local HTTPS panel/API NodePort (default: 30443) # FELIS_EGRESS_MODE loadbalancer|nodeport (default: nodeport — no MetalLB on a demo box) @@ -114,7 +117,15 @@ export FELIS_GITHUB_TOKEN # Set by resolve_install_ref/stamp_version and linked into the binary as main.version. FELIS_VERSION="" FELIS_VERSION_BASE="" -FELIS_IMAGE="${FELIS_IMAGE:-felis:demo}" +# Where the installer parks every image it builds, so kubelet can re-pull one the +# image GC has collected (the disk-pressure drill's dead end: ImagePullBackOff with +# nothing to pull from). The node pulls through the loopback hostPort the registry +# Deployment binds (configure_registry_mirror below); pushes go through +# REGISTRY_PUSH_HOST — docker treats 127.0.0.1 as insecure by default, so the +# daemon needs no insecure-registries entry for it. +REGISTRY_URL="registry.felis.svc:5000" +REGISTRY_PUSH_HOST="127.0.0.1:${REGISTRY_URL##*:}" +FELIS_IMAGE="${FELIS_IMAGE:-${REGISTRY_URL}/felis/felis:demo}" FELIS_EGRESS_MODE="${FELIS_EGRESS_MODE:-nodeport}" FELIS_PANEL_NODEPORT="${FELIS_PANEL_NODEPORT:-30443}" # World-archive storage. The installer renders this PVC (minecraft namespace) and felis-api @@ -156,12 +167,12 @@ PKG_LOCK_TIMEOUT="${PKG_LOCK_TIMEOUT:-${APT_LOCK_TIMEOUT:-900}}" APT_LOCK_TIMEOUT="${APT_LOCK_TIMEOUT:-$PKG_LOCK_TIMEOUT}" # --- the game stack: proxy on the host, the two always-on backends in k3s --- -FELIS_LIMBO_IMAGE="${FELIS_LIMBO_IMAGE:-felis-limbo:demo}" -FELIS_LOBBY_IMAGE="${FELIS_LOBBY_IMAGE:-felis-lobby:demo}" +FELIS_LIMBO_IMAGE="${FELIS_LIMBO_IMAGE:-${REGISTRY_URL}/felis/limbo:demo}" +FELIS_LOBBY_IMAGE="${FELIS_LOBBY_IMAGE:-${REGISTRY_URL}/felis/lobby:demo}" # Plain Paper base recommended for a user's own server (deploy/paper). Not a system # server — forwarding is applied by the operator's init-forwarding initContainer, so it # needs no secret. Seeded recommended in 0019_recommended_paper.sql. -FELIS_PAPER_IMAGE="${FELIS_PAPER_IMAGE:-felis-paper:demo}" +FELIS_PAPER_IMAGE="${FELIS_PAPER_IMAGE:-${REGISTRY_URL}/felis/paper:demo}" # The Velocity MINOR is pinned, not discovered. PaperMC's Fill v3 groups velocity # builds by version group, and "newest across all groups" today means 4.0.0-SNAPSHOT — # an UNRELEASED proxy (the 4.0.0 group has zero published builds) that needs a Java 25 @@ -214,7 +225,6 @@ POD_CIDR="10.42.0.0/16" # k3s default cluster CIDR SERVICE_CIDR="10.43.0.0/16" # k3s default service CIDR DB_NAME="felis" DB_USER="felis" -REGISTRY_URL="registry.felis.svc:5000" STATE_DIR="/etc/felis" SECRETS_ENV="${STATE_DIR}/secrets.env" @@ -233,6 +243,10 @@ VELOCITY_SERVICE="/etc/systemd/system/felis-velocity.service" JRE_DIR="/opt/felis/jre" K3S_BIN_DIR="${K3S_BIN_DIR:-/usr/local/bin}" K3S_BIN="${K3S_BIN_DIR}/k3s" +# k3s's containerd mirror config, written by configure_registry_mirror. A variable +# (not just the literal path) so bootstrap_test.sh can point the writer at a +# scratch file. +K3S_REGISTRIES_FILE="/etc/rancher/k3s/registries.yaml" APT_LOCK_FILES=( /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock @@ -813,6 +827,13 @@ install_k3s() { systemctl enable --now k3s export KUBECONFIG=/etc/rancher/k3s/k3s.yaml log "waiting for the node to become Ready" + wait_for_node_ready +} + +# Waits for the (single) node to report Ready. Shared by the k3s install and the +# registry-mirror restart below: both restart the agent, and a bootstrap that +# proceeds early fails later with a misleading "not found"/timeout instead. +wait_for_node_ready() { local i for i in $(seq 1 60); do if kube get nodes 2>/dev/null | grep -q ' Ready '; then @@ -825,6 +846,78 @@ install_k3s() { die "k3s node did not become Ready in time" } +# --------------------------------------------------------------------------- +# 4b. The in-cluster registry: the node-side pull path, the registry's own +# image, and the hosting of every image this installer builds. +# +# Kubelet's image GC collects an unused image under disk pressure (drilled: +# the game images were collected and ImagePullBackOff had nothing to pull +# from). The fix is a pull source that is always there — the registry the +# bundle already renders. Two node-level facts make that work: +# * kubelet cannot reach the registry Service VIP (the live stack answered +# "Empty reply"), so containerd is told to go through the loopback +# hostPort the registry Deployment binds (the Deployment renders it) — +# that is configure_registry_mirror below; +# * the registry's own image (registry:2) must already be in containerd +# before the registry Deployment can start at all — +# import_registry_image below caches it. +# After deploy_bundle, push_images_to_registry mirrors the built images into +# the registry, so containerd's imported copies are a first-boot cache +# rather than the only copy. +# --------------------------------------------------------------------------- + +# The node's containerd cannot dial the registry Service VIP, so pulls arrive +# over the loopback hostPort the registry Deployment binds. k3s reads this file +# when the agent starts and regenerates containerd's certs.d from it — no +# restart, no effect — so a CONTENT change restarts k3s; an identical file +# (every re-run) restarts nothing. K3S_REGISTRIES_FILE is a variable so +# bootstrap_test.sh can point the function at a scratch file. +configure_registry_mirror() { + local file="$K3S_REGISTRIES_FILE" tmp + tmp="$(mktemp)" + remember_temp "$tmp" + cat > "$tmp" < http://${REGISTRY_PUSH_HOST})" + return 0 + fi + mkdir -p "$(dirname "$file")" + mv "$tmp" "$file" + log "restarting k3s to load the registry mirror (${REGISTRY_URL} -> http://${REGISTRY_PUSH_HOST})" + systemctl restart k3s + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + wait_for_node_ready +} + +# The registry Deployment runs registry:2 (platform.defaultRegistryImage; the +# renderer's default — this script never passes --registry-image). On a box +# that cannot reach Docker Hub the Deployment can never start without a local +# copy, so the installer caches one whenever it can. Best-effort by design: if +# the pull fails the registry rollout still fails loudly at deploy_bundle, with +# the regular diagnostics — but for every box that CAN pull, the image is +# fetched exactly once, here, instead of at first pod start. +import_registry_image() { + # The name containerd normalizes "registry:2" to after any docker-save import. + if k3s_cmd ctr images ls -q 2>/dev/null | grep -qx 'docker.io/library/registry:2'; then + ok "registry image registry:2 already in k3s containerd" + return 0 + fi + log "importing the registry's own image (registry:2) into k3s containerd" + systemctl start docker + if docker pull registry:2 && docker save registry:2 | k3s_cmd ctr images import -; then + ok "registry image registry:2 imported" + else + warn "could not import registry:2: the in-cluster registry will start only if the node can pull it from Docker Hub; on an air-gapped box import it by hand (docs/troubleshooting.md §8e)" + fi + systemctl stop docker docker.socket 2>/dev/null || true +} + # --------------------------------------------------------------------------- # 5. Source/binary + image build + containerd import # --------------------------------------------------------------------------- @@ -2223,10 +2316,48 @@ restart_existing_control_plane() { if [ "$had_operator" = "1" ]; then kube -n "$CONTROL_NS" rollout restart deployment/felis-operator; fi } -# The login/lobby images use local mutable tags. Importing a replacement updates -# containerd, but an existing StatefulSet template is byte-for-byte unchanged and -# Kubernetes will not roll it. Recreate only the two always-on system pods so a -# convergent bootstrap actually starts the images it just imported. +# push_image_to_registry re-tags a locally built image for the node's +# loopback push endpoint and uploads it. The registry keys a repository by the +# path AFTER the host, so pushing 127.0.0.1:5000/felis/felis:demo lands exactly +# where a later kubelet pull of registry.felis.svc:5000/felis/felis:demo (the +# mirror rewrites the host) will look. A ref not under REGISTRY_URL is not +# mirrored — warn, don't fail: the install is still self-consistent, that image +# just has no pull source once GC collects its containerd copy. +push_image_to_registry() { + local ref="$1" push_ref + case "$ref" in + "${REGISTRY_URL}/"*) + push_ref="${REGISTRY_PUSH_HOST}/${ref#"${REGISTRY_URL}/"}" + ;; + *) + warn "not mirroring ${ref} into the internal registry: it is not under ${REGISTRY_URL}; once the image GC collects that tag, nothing can re-pull it" + return 0 + ;; + esac + systemctl start docker + log "mirroring ${ref} into the internal registry" + docker tag "$ref" "$push_ref" || die "could not tag ${ref} as ${push_ref} — is docker healthy?" + docker push "$push_ref" || die "could not mirror ${ref} into the internal registry — check the registry Deployment/pod and its PVC" + docker rmi "$push_ref" >/dev/null 2>&1 || true + systemctl stop docker docker.socket 2>/dev/null || true +} + +# Every image this installer builds is hosted in the registry, so the copies it +# imported into containerd are a first-boot cache, not the only copy: kubelet +# re-pulls from the registry after any image GC. Runs AFTER deploy_bundle — the +# registry it pushes into does not exist before that. +push_images_to_registry() { + local img + for img in "$FELIS_IMAGE" "$FELIS_LIMBO_IMAGE" "$FELIS_LOBBY_IMAGE" "$FELIS_PAPER_IMAGE"; do + [ -n "$img" ] || continue + push_image_to_registry "$img" + done +} + +# The login/lobby images use mutable :demo tags. Importing/pushing a replacement +# updates containerd, but an existing StatefulSet template is byte-for-byte +# unchanged and Kubernetes will not roll it. Recreate only the two always-on +# system pods so a convergent bootstrap actually starts the images it just built. restart_existing_system_servers() { local name pods for name in "$LOGIN_SERVER" "$LOBBY_SERVER"; do @@ -2631,6 +2762,11 @@ main() { ensure_panel_tls_cert install_docker install_k3s + # The registry mirror must exist before the bundle's pods start pulling (and + # before any re-run's rollouts); the registry's own image must be in containerd + # before its Deployment can start at all. + configure_registry_mirror + import_registry_image # Three ways to end up with a felis binary, in preference order. The release download is # the only one that skips compiling: it is the CI artifact for this exact tag, panel # included. Both other arms leave HAVE_PREBUILT_BINARY unset where a source build is what @@ -2648,6 +2784,9 @@ main() { configure_postgres run_migrations deploy_bundle + # AFTER deploy_bundle: the registry the built images are mirrored into is part + # of that bundle. + push_images_to_registry restart_existing_system_servers # After deploy_bundle: the proxy dials felis-api's internal ClusterIP, which does not # exist until the bundle is applied. diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index ad73c42..219bf74 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -622,6 +622,109 @@ wdir="$(mktemp -d)" out="$(run_bundle_flags felis-backups "$wdir")" expect "enabling retention grants the reaper uid traverse on the worlds root" "SETFACL -m u:1000:x $wdir" "$out" +# --- the registry mirror writer ----------------------------------------------------------- +# k3s only consults registries.yaml at agent start, so a CONTENT change must restart k3s and +# an identical file (every re-run) must restart nothing. The k3s restart is the expensive, +# disruptive half of the pair -- getting the idempotence wrong bounces the whole cluster on +# every installer re-run, so both halves are pinned here against the extracted function. + +cmblock="$(awk '/^configure_registry_mirror\(\) \{/,/^}/' "$BS")" +[ -n "$cmblock" ] || { echo "FAIL: no configure_registry_mirror found in $BS"; exit 1; } + +run_mirror() { # scratch-file + K3S_REGISTRIES_FILE="$1" REGISTRY_URL=registry.felis.svc:5000 REGISTRY_PUSH_HOST=127.0.0.1:5000 \ + bash -c ' + log() { printf "LOG: %s\n" "$*"; } + ok() { printf "OK: %s\n" "$*"; } + die() { printf "DIE: %s\n" "$*"; exit 1; } + warn() { printf "WARN: %s\n" "$*"; } + remember_temp() { :; } + systemctl() { printf "SYSTEMCTL %s\n" "$*"; } + kube() { printf "n Ready \n"; } + wait_for_node_ready() { kube get nodes | grep -q " Ready " && ok "k3s node Ready"; } + '"$cmblock"' + configure_registry_mirror' +} + +mfile="$(mktemp -u)" +out="$(run_mirror "$mfile")" +expect "a missing registries.yaml is written" "\"registry.felis.svc:5000\":" "$(cat "$mfile" 2>/dev/null)" +expect "the mirror endpoint is the node loopback push/pull host" "\"http://127.0.0.1:5000\"" "$(cat "$mfile" 2>/dev/null)" +expect "a content change restarts k3s" "SYSTEMCTL restart k3s" "$out" + +out="$(run_mirror "$mfile")" +expect "an identical registries.yaml is recognised" "already configured" "$out" +case "$out" in + *"SYSTEMCTL restart"*) echo "FAIL: a re-run with identical content must not restart k3s"; fails=$((fails + 1)) ;; +esac + +printf 'mirrors: {}\n' >"$mfile" +out="$(run_mirror "$mfile")" +expect "changed content restarts k3s again" "SYSTEMCTL restart k3s" "$out" +rm -f "$mfile" + +# --- image mirroring ---------------------------------------------------------------------- +# The registry keys a repository by the path AFTER the host, so the push must swap the +# registry host for the node's loopback endpoint and nothing else. A ref outside the +# registry must be warned about, not silently pushed somewhere unintended. + +pblock="$(awk '/^push_image_to_registry\(\) \{/,/^}/' "$BS")" +[ -n "$pblock" ] || { echo "FAIL: no push_image_to_registry found in $BS"; exit 1; } + +run_push() { # ref [docker-push-exit] + REF="$1" PUSH_EXIT="${2:-0}" \ + REGISTRY_URL=registry.felis.svc:5000 REGISTRY_PUSH_HOST=127.0.0.1:5000 \ + bash -c ' + log() { printf "LOG: %s\n" "$*"; } + warn() { printf "WARN: %s\n" "$*"; } + die() { printf "DIE: %s\n" "$*"; exit 1; } + ok() { :; } + systemctl() { :; } + docker() { + case "$1" in + push) printf "DOCKER %s\n" "$*"; return "$PUSH_EXIT" ;; + *) printf "DOCKER %s\n" "$*" ;; + esac + } + '"$pblock"' + push_image_to_registry "$REF"' +} + +out="$(run_push registry.felis.svc:5000/felis/felis:demo)" +expect "a registry ref is re-tagged onto the node loopback endpoint" \ + "DOCKER tag registry.felis.svc:5000/felis/felis:demo 127.0.0.1:5000/felis/felis:demo" "$out" +expect "and pushed to exactly that endpoint" "DOCKER push 127.0.0.1:5000/felis/felis:demo" "$out" + +out="$(run_push registry.felis.svc:50000/felis/felis:demo)" +expect "a ref outside the registry is refused with a warning" "WARN: not mirroring" "$out" +case "$out" in + *"DOCKER push"*) echo "FAIL: a non-registry ref must not be pushed"; fails=$((fails + 1)) ;; +esac + +out="$(run_push registry.felis.svc:5000/felis/felis:demo 1)" +expect "a failed push fails the install loudly" "DIE: could not mirror" "$out" + +# --- the registry's own image must not be re-pulled on every run -------------------------- +iblock="$(awk '/^import_registry_image\(\) \{/,/^}/' "$BS")" +[ -n "$iblock" ] || { echo "FAIL: no import_registry_image found in $BS"; exit 1; } + +out="$( + bash -c ' + log() { printf "LOG: %s\n" "$*"; } + ok() { printf "OK: %s\n" "$*"; } + warn() { printf "WARN: %s\n" "$*"; } + die() { printf "DIE: %s\n" "$*"; exit 1; } + systemctl() { :; } + k3s_cmd() { case "$*" in "ctr images ls -q") printf "docker.io/library/registry:2\n" ;; esac; } + docker() { printf "DOCKER %s\n" "$*"; return 1; } + '"$iblock"' + import_registry_image' +)" +expect "an already-imported registry:2 is left alone" "already in k3s containerd" "$out" +case "$out" in + *DOCKER*) echo "FAIL: a present registry:2 must not trigger a docker pull"; fails=$((fails + 1)) ;; +esac + # --------------------------------------------------------------------------------------- if [ "$fails" -eq 0 ]; then echo "ALL PASS" diff --git a/internal/build/pgstore.go b/internal/build/pgstore.go index 32a7c1b..5b9154b 100644 --- a/internal/build/pgstore.go +++ b/internal/build/pgstore.go @@ -183,11 +183,11 @@ func (s *PGStore) ListImages(ctx context.Context) ([]Image, error) { // Image it constructed (source=external) without re-reading the row, so a sticky // source here would report a value the database does not hold. // -// Note that the demote branch is unreachable for the ONLY recommended row Felis -// currently seeds: the caller validates with ValidateImageRef first, which refuses -// a bare local containerd tag, and 0018's felis-lobby:demo is exactly that. The -// branch is written for the host-qualified recommendations this list grows into, -// not for today's single seed. +// Note that this demote branch is REACHABLE for today's recommended rows: 0021 +// re-pointed the seeds at host-qualified registry refs (registry..svc:5000/…), +// which ValidateImageRef accepts — so an admin re-admitting one of those refs +// demotes the curated row, by design. It was dead only while the seeds were bare +// local containerd tags (0018's felis-lobby:demo), which the validation refuses. func (s *PGStore) AddExternalImage(ctx context.Context, img Image) error { const q = `INSERT INTO image_whitelist (image_ref, source, added_by, enabled, added_at) diff --git a/internal/config/config.go b/internal/config/config.go index 7509c1a..d06dd8b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -126,8 +126,9 @@ type RegistryConfig struct { // install there IS no such reach (the build egress policy allows only DNS, // the internal registry and explicit package mirrors), so the operator must // point these at whatever their box can actually pull — typically images - // imported into the node's containerd alongside the felis image. Empty keeps - // the default. + // mirrored into the in-cluster registry (docs/troubleshooting.md §8e); a + // bare node-containerd import does not survive an image GC, there is no pull + // source for it. Empty keeps the default. KanikoImage string `toml:"kaniko_image"` TrivyImage string `toml:"trivy_image"` BuildCPULimit string `toml:"build_cpu_limit"` diff --git a/internal/store/migrations/0021_registry_recommended_images.sql b/internal/store/migrations/0021_registry_recommended_images.sql new file mode 100644 index 0000000..87e25e0 --- /dev/null +++ b/internal/store/migrations/0021_registry_recommended_images.sql @@ -0,0 +1,43 @@ +-- Re-point the platform-seeded recommended images at the internal registry. +-- +-- 0018/0019 seeded 'felis-lobby:demo' and 'felis-paper:demo' — the bare local +-- containerd tags the bootstrap of that day imported. The installer now builds +-- every image under registry..svc:5000/felis/... and mirrors it into the +-- in-cluster registry, which is what lets kubelet re-pull an image the image GC +-- collected (drilled: disk pressure → game images collected → +-- ImagePullBackOff with no pull source). A bare local tag has no pull source at +-- all once its containerd copy is collected, so a user server created from a +-- recommended row would strand the same way. Re-point the seeds at the refs the +-- installer now builds — these MUST stay identical to deploy/bootstrap.sh's +-- FELIS_LOBBY_IMAGE / FELIS_PAPER_IMAGE defaults. +-- +-- Only source='recommended' rows are touched, and only while the ref still IS +-- the old seed: a built/external row, or a recommended row an admin re-pointed +-- by hand, is theirs to keep. enabled is preserved either way — a disabled seed +-- stays disabled, just under its durable name. +-- +-- Collision handling: image_ref is the primary key, so an UPDATE would abort if +-- the new ref already exists (e.g. an admin added it by hand). Keep whichever +-- row exists and drop the stale old one — never a duplicate, never an aborted +-- migration. +UPDATE image_whitelist +SET image_ref = 'registry.felis.svc:5000/felis/lobby:demo' +WHERE image_ref = 'felis-lobby:demo' + AND source = 'recommended' + AND NOT EXISTS (SELECT 1 FROM image_whitelist WHERE image_ref = 'registry.felis.svc:5000/felis/lobby:demo'); + +DELETE FROM image_whitelist +WHERE image_ref = 'felis-lobby:demo' + AND source = 'recommended' + AND EXISTS (SELECT 1 FROM image_whitelist WHERE image_ref = 'registry.felis.svc:5000/felis/lobby:demo'); + +UPDATE image_whitelist +SET image_ref = 'registry.felis.svc:5000/felis/paper:demo' +WHERE image_ref = 'felis-paper:demo' + AND source = 'recommended' + AND NOT EXISTS (SELECT 1 FROM image_whitelist WHERE image_ref = 'registry.felis.svc:5000/felis/paper:demo'); + +DELETE FROM image_whitelist +WHERE image_ref = 'felis-paper:demo' + AND source = 'recommended' + AND EXISTS (SELECT 1 FROM image_whitelist WHERE image_ref = 'registry.felis.svc:5000/felis/paper:demo');