diff --git a/cmd/felis/manifests.go b/cmd/felis/manifests.go index b17a01d..1187eed 100644 --- a/cmd/felis/manifests.go +++ b/cmd/felis/manifests.go @@ -45,7 +45,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int { registryPort := fs.Int("registry-port", 5000, "port the in-cluster registry listens on") panelNodePort := fs.Int("panel-node-port", int(platform.DefaultPanelNodePort), "NodePort that exposes the built-in HTTPS panel/API origin") felisImage := fs.String("felis-image", "", "container image the felis-api/operator Deployments run, also passed through as FELIS_IMAGE (REQUIRED)") - registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry:2)") + registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry 2.8.3, pinned by digest)") backupPVC := fs.String("backup-pvc", "felis-backups", "name of the world-archive PVC this bundle renders in the Minecraft namespace and advertises to the backup/restore executors via FELIS_BACKUP_PVC (default: felis-backups; pass an empty value to render none, leaving backup/restore answering 503)") worldsHostPath := fs.String("worlds-host-path", "", "node directory the reaper reads worlds from: each world PVC resolves as /, or as the stock local-path directory /__ (k3s storage root: /var/lib/rancher/k3s/storage); enables the reaper CronJob (requires --archive-local-path and a non-empty --backup-pvc)") archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path") diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 565f168..215da2b 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -49,13 +49,20 @@ # FELIS_GO_VERSION Go toolchain used to build the nano binary (default: 1.26.4) # FELIS_GO_SHA256 sha256 of that version's linux tarball for this host's architecture. # REQUIRED for a non-default FELIS_GO_VERSION; the default's is pinned. +# FELIS_K3S_VERSION k3s release a fresh install gets (default: v1.36.4+k3s1). An +# installed k3s is left alone. +# FELIS_CLOUDFLARED_VERSION / FELIS_CLOUDFLARED_SHA256 cloudflared release installed +# when none is present (default: 2026.9.1, digests pinned); the +# sha256 is REQUIRED for any other version # FELIS_REPO_URL git URL to build from (raw script mode only) # FELIS_VERSION_BOOTSTRAP release|dev — which version to install (default: release). # release DOWNLOADS the prebuilt felis binary published for the newest # tag (panel included — it is go:embed'ed into that same binary) and # builds only a thin image around it; dev clones and compiles. If the # asset is missing or this architecture has none, release warns and falls -# back to compiling the SAME tag. The game stack is always built here. +# back to compiling the SAME tag. The downloaded binary must match +# the release's SHA256SUMS before it is run; a release without one +# is compiled from source too. The game stack is always built here. # FELIS_GITHUB_TOKEN GitHub token; REQUIRED while the repo is private # FELIS_REF branch/tag/sha — pins the build, overrides the channel, and forces a # source build (naming a ref asks for that tree, not a published asset) @@ -197,6 +204,24 @@ GO_PINNED_SHA256_AMD64="1153d3d50e0ac764b447adfe05c2bcf08e889d42a02e0fe0259bd47f GO_PINNED_SHA256_ARM64="ef758ae7c6cf9267c9c0ef080b8965f453d89ab2d25d9eb22de4405925238768" FELIS_GO_VERSION="${FELIS_GO_VERSION:-$GO_PINNED_VERSION}" FELIS_GO_SHA256="${FELIS_GO_SHA256:-}" +# cloudflared runs as root on the edge, so it gets the same treatment: a pinned release and +# the sha256 GitHub lists for each asset. A different FELIS_CLOUDFLARED_VERSION has to bring +# its own FELIS_CLOUDFLARED_SHA256. install_cloudflared only runs when the binary is absent; +# upgrading an installed one is `felis update`'s report plus a manual swap. +CLOUDFLARED_PINNED_VERSION="2026.9.1" +CLOUDFLARED_PINNED_SHA256_AMD64="03f1f25d1cc93b9ad6c60569d44060bc4f17ed97075760ed8cfca4b12dcd68cc" +CLOUDFLARED_PINNED_SHA256_ARM64="3d97437c71848bd8df68041e12436b484a661d95073ea1937f01a845ce88faa3" +CLOUDFLARED_PINNED_SHA256_ARM="093ffa3638ab2b636de63c43a8c68f96a69cf71f9699dd8277a91b160b0f4fc0" +FELIS_CLOUDFLARED_VERSION="${FELIS_CLOUDFLARED_VERSION:-$CLOUDFLARED_PINNED_VERSION}" +FELIS_CLOUDFLARED_SHA256="${FELIS_CLOUDFLARED_SHA256:-}" +# The k3s release a fresh install gets, and the tag its install script is read from. The +# script checks the k3s binary against that release's sha256sum file, so pinning the tag +# pins both. An installed k3s is never touched; see docs/troubleshooting.md for upgrades. +FELIS_K3S_VERSION="${FELIS_K3S_VERSION:-v1.36.4+k3s1}" +# The in-cluster registry's image, by digest. It must equal platform.defaultRegistryImage +# (internal/platform/identities.go, TestBootstrapPinsTheRegistryImage): the renderer puts +# that ref in the Deployment, and this script caches and pins the same ref in containerd. +REGISTRY_IMAGE="docker.io/library/registry:2.8.3@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373" PKG_LOCK_TIMEOUT="${PKG_LOCK_TIMEOUT:-${APT_LOCK_TIMEOUT:-900}}" APT_LOCK_TIMEOUT="${APT_LOCK_TIMEOUT:-$PKG_LOCK_TIMEOUT}" @@ -804,19 +829,26 @@ install_cloudflared() { ok "cloudflared already installed" return 0 fi - local machine arch url tmp + local machine arch url tmp want have machine="$(uname -m)" case "$machine" in - x86_64|amd64) arch="amd64" ;; - aarch64|arm64) arch="arm64" ;; - armv7l|armv6l) arch="arm" ;; + x86_64|amd64) arch="amd64"; want="$CLOUDFLARED_PINNED_SHA256_AMD64" ;; + aarch64|arm64) arch="arm64"; want="$CLOUDFLARED_PINNED_SHA256_ARM64" ;; + armv7l|armv6l) arch="arm"; want="$CLOUDFLARED_PINNED_SHA256_ARM" ;; *) die "unsupported architecture for cloudflared: ${machine}" ;; esac - url="https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${arch}" + [ "$FELIS_CLOUDFLARED_VERSION" = "$CLOUDFLARED_PINNED_VERSION" ] || want="$FELIS_CLOUDFLARED_SHA256" + [ -n "$want" ] || die "no pinned sha256 for cloudflared ${FELIS_CLOUDFLARED_VERSION}; set FELIS_CLOUDFLARED_SHA256 to the digest of cloudflared-linux-${arch} on that GitHub release" + url="https://github.com/cloudflare/cloudflared/releases/download/${FELIS_CLOUDFLARED_VERSION}/cloudflared-linux-${arch}" tmp="$(mktemp)" remember_temp "$tmp" - log "installing cloudflared (${arch})" - curl -fsSL "$url" -o "$tmp" + log "installing cloudflared ${FELIS_CLOUDFLARED_VERSION} (${arch})" + curl -fsSL --retry 5 --retry-delay 2 "$url" -o "$tmp" + have="$(sha256sum <"$tmp" | cut -d' ' -f1)" + if [ "$have" != "$(printf '%s' "$want" | tr 'A-Z' 'a-z')" ]; then + rm -f "$tmp" + die "cloudflared-linux-${arch} ${FELIS_CLOUDFLARED_VERSION} hashes to ${have}, expected ${want}; refusing to install it" + fi install -m 0755 "$tmp" /usr/local/bin/cloudflared rm -f "$tmp" ok "cloudflared installed ($(cloudflared --version | head -n 1))" @@ -947,8 +979,11 @@ install_k3s() { if [ -x "$K3S_BIN" ]; then ok "k3s already installed at ${K3S_BIN}" else - log "installing k3s into ${K3S_BIN_DIR} (no traefik/servicelb/metrics-server)" - curl -sfL https://get.k3s.io | \ + log "installing k3s ${FELIS_K3S_VERSION} into ${K3S_BIN_DIR} (no traefik/servicelb/metrics-server)" + # The script from the release's own tag rather than get.k3s.io, which serves whatever + # master holds today. '+' is literal in a URL path, so the tag needs no escaping. + curl -sfL --retry 5 --retry-delay 2 "https://raw.githubusercontent.com/k3s-io/k3s/${FELIS_K3S_VERSION}/install.sh" | \ + INSTALL_K3S_VERSION="$FELIS_K3S_VERSION" \ INSTALL_K3S_BIN_DIR="$K3S_BIN_DIR" \ INSTALL_K3S_EXEC="--disable traefik --disable servicelb --disable metrics-server --write-kubeconfig-mode 644" \ sh - @@ -990,7 +1025,7 @@ wait_for_node_ready() { # "Empty reply"), so containerd is told to go through the loopback # hostPort the registry Deployment binds (the Deployment renders it) — # that is configure_registry_mirror below; -# * the registry's own image (registry:2) must already be in containerd +# * the registry's own image (REGISTRY_IMAGE) must already be in containerd # before the registry Deployment can start at all — # import_registry_image below caches it. # After deploy_bundle, push_images_to_registry mirrors the built images into @@ -1027,30 +1062,37 @@ EOF wait_for_node_ready } -# The registry Deployment runs registry:2 (platform.defaultRegistryImage; the +# The registry Deployment runs REGISTRY_IMAGE (platform.defaultRegistryImage; the # renderer's default — this script never passes --registry-image). On a box # that cannot reach Docker Hub the Deployment can never start without a local # copy, so the installer caches one whenever it can. Best-effort by design: if # the pull fails the registry rollout still fails loudly at deploy_bundle, with # the regular diagnostics — but for every box that CAN pull, the image is # fetched exactly once, here, instead of at first pod start. +# +# crictl pulls through CRI, the same call kubelet makes, so containerd records the +# digest ref the Deployment names and kubelet finds it. A docker save/import round +# trip rewrites the manifest and would leave a copy the digest ref never matches. import_registry_image() { - # The name containerd normalizes "registry:2" to after any docker-save import. - if k3s_cmd ctr images ls -q 2>/dev/null | grep -qx 'docker.io/library/registry:2'; then - ok "registry image registry:2 already in k3s containerd" + if k3s_cmd ctr images ls -q 2>/dev/null | grep -qxF "$(registry_image_containerd_ref)"; then + ok "registry image ${REGISTRY_IMAGE} already in k3s containerd" return 0 fi - log "importing the registry's own image (registry:2) into k3s containerd" - systemctl start docker - if docker pull registry:2 && docker save registry:2 | k3s_cmd ctr images import -; then - ok "registry image registry:2 imported" + log "pulling the registry's own image (${REGISTRY_IMAGE}) into k3s containerd" + if k3s_cmd crictl pull "$REGISTRY_IMAGE" >/dev/null; then + ok "registry image ${REGISTRY_IMAGE} pulled" else - warn "could not import registry:2: the in-cluster registry will start only if the node can pull it from Docker Hub; on an air-gapped box import it by hand (docs/troubleshooting.md §8e)" + warn "could not pull ${REGISTRY_IMAGE}: the in-cluster registry will start only if the node can pull it from Docker Hub; on an air-gapped box import it by hand (docs/troubleshooting.md §8e)" fi - systemctl stop docker docker.socket 2>/dev/null || true } -# The registry pod runs registry:2 and, as its registry-gate sidecar, the felis +# registry_image_containerd_ref prints the name containerd lists REGISTRY_IMAGE under +# once CRI has pulled it: the repository and the digest, with the tag dropped. +registry_image_containerd_ref() { + printf '%s@%s\n' "${REGISTRY_IMAGE%%:*}" "${REGISTRY_IMAGE#*@}" +} + +# The registry pod runs REGISTRY_IMAGE and, as its registry-gate sidecar, the felis # image — neither of which can be pulled from the registry they make up. A kubelet # image GC that collected either would leave the registry, and every pull through # it, dead until someone re-imported by hand. containerd reports an image labelled @@ -1058,14 +1100,16 @@ import_registry_image() { # removes a pinned image. Older felis/felis tags are unpinned first, so upgrades # do not pile up pinned images forever. pin_registry_images() { - local ref + local ref registry_ref + registry_ref="$(registry_image_containerd_ref)" while read -r ref; do case "$ref" in - "$FELIS_IMAGE") ;; - */felis/felis:*) k3s_cmd ctr images label "$ref" io.cri-containerd.pinned= >/dev/null 2>&1 || true ;; + "$FELIS_IMAGE"|"$registry_ref") ;; + */felis/felis:*|docker.io/library/registry[:@]*) + k3s_cmd ctr images label "$ref" io.cri-containerd.pinned= >/dev/null 2>&1 || true ;; esac done < <(k3s_cmd ctr images ls -q 2>/dev/null || true) - for ref in "$FELIS_IMAGE" docker.io/library/registry:2; do + for ref in "$FELIS_IMAGE" "$registry_ref"; do if k3s_cmd ctr images label "$ref" io.cri-containerd.pinned=pinned >/dev/null 2>&1; then ok "pinned ${ref} in containerd (exempt from kubelet image GC)" else @@ -1215,6 +1259,35 @@ download_release_asset() { fi } +# verify_release_checksum checks the downloaded asset $2 (at $3) against the SHA256SUMS +# file release.yml publishes next to it on tag $1. It returns non-zero, with a warning, +# when the release has no SHA256SUMS (a tag cut before release.yml wrote one, or a release +# still uploading), when the file does not list the asset, or when the hash differs. +verify_release_checksum() { + local tag="$1" name="$2" file="$3" sums want have + sums="$(mktemp)" + remember_temp "$sums" + if ! download_release_asset "$tag" SHA256SUMS "$sums"; then + rm -f "$sums" + warn "release ${tag} publishes no SHA256SUMS, so ${name} cannot be verified; building ${tag} from source on this host instead" + return 1 + fi + # sha256sum's text-mode line is " ", binary mode " *". + want="$(awk -v n="$name" '$2 == n || $2 == "*" n { print $1; exit }' "$sums")" + rm -f "$sums" + if [ -z "$want" ]; then + warn "release ${tag}'s SHA256SUMS does not list ${name}; building ${tag} from source on this host instead" + return 1 + fi + # Hash stdin, never the path: the same sha256sum escaping install_via_plugins avoids. + have="$(sha256sum <"$file" | cut -d' ' -f1)" + if [ "$have" != "$want" ]; then + warn "downloaded ${name} hashes to ${have}, but release ${tag}'s SHA256SUMS says ${want}; discarding it and building ${tag} from source on this host instead" + return 1 + fi + ok "${name} matches release ${tag}'s SHA256SUMS" +} + # use_release_binary reports whether this run should install a prebuilt binary instead of # compiling one. Only the plain release channel qualifies: FELIS_SKIP_FETCH means "build # exactly what I staged" and a pinned FELIS_REF means "build that tree", both of which are @@ -1265,6 +1338,15 @@ download_release_binary() { warn "release ${FELIS_REF} publishes no usable ${asset}; building ${FELIS_REF} from source on this host instead" return 1 fi + + # The hash comes BEFORE the exec below: until the file matches the release's SHA256SUMS it + # is unverified bytes, and running it as root to ask its version would hand root to + # whoever could swap the asset or sit on the download path. Missing or unmatched both + # fall back to the source build of the same tag, which trusts only the git fetch. + if ! verify_release_checksum "$FELIS_REF" "$asset" "$tmp"; then + rm -f "$tmp" + return 1 + fi chmod 0755 "$tmp" # Run it once. This single exec subsumes three checks that would otherwise each need their diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index 21cb55f..572a2cd 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -635,6 +635,83 @@ gtmp="$(printf '%s\n' "$out" | sed -n 's/^TEMP: //p')" expect "the Go download is staged in a directory the cleanup removes" \ "CURL: ${gtmp:-}/go1.26.4.linux-amd64.tar.gz" "$out" +# --- a release binary is hashed against SHA256SUMS before anything runs it --------------- +# download_release_binary executes the asset as root to read its version stamp, so the +# checksum has to come first, and every failure has to fall back to the source build. + +vblock="$(awk '/^verify_release_checksum\(\) \{/,/^}/' "$BS")" +[ -n "$vblock" ] || { echo "FAIL: no verify_release_checksum found in $BS"; exit 1; } +asset_file="$sdir/felis-linux-amd64" +printf 'stand-in felis binary\n' > "$asset_file" +asum="$(sha256sum <"$asset_file" | cut -d' ' -f1)" + +run_verify() { # SHA256SUMS-content ("" = the release has none) + SUMS="$1" TMPDIR="$sdir" bash -c ' + ok() { printf "OK: %s\n" "$*"; } + warn() { printf "WARN: %s\n" "$*"; } + remember_temp() { :; } + download_release_asset() { [ -n "$SUMS" ] || return 1; printf "%s" "$SUMS" > "$3"; } + '"$vblock"' + verify_release_checksum v9.9.9 felis-linux-amd64 '"$asset_file"' && echo VERIFIED' +} +expect "a listed, matching binary is accepted" "VERIFIED" \ + "$(run_verify "$(printf '%s felis-linux-arm64\n%s felis-linux-amd64\n' deadbeef "$asum")")" +expect "a binary-mode SHA256SUMS line is read too" "VERIFIED" \ + "$(run_verify "$(printf '%s *felis-linux-amd64\n' "$asum")")" +out="$(run_verify "$(printf '%s felis-linux-amd64\n' deadbeef)")" +expect "a hash mismatch is refused and names both hashes" "hashes to ${asum}, but release v9.9.9's SHA256SUMS says deadbeef" "$out" +case "$out" in *VERIFIED*) echo "FAIL: a mismatched binary must not verify"; fails=$((fails + 1)) ;; esac +out="$(run_verify "$(printf '%s felis-linux-amd64.sig\n' "$asum")")" +expect "a SHA256SUMS that does not list the asset is refused" "does not list felis-linux-amd64" "$out" +case "$out" in *VERIFIED*) echo "FAIL: an unlisted binary must not verify"; fails=$((fails + 1)) ;; esac +out="$(run_verify "")" +expect "a release without SHA256SUMS falls back to a source build" "publishes no SHA256SUMS" "$out" +case "$out" in *VERIFIED*) echo "FAIL: a release without SHA256SUMS must not verify"; fails=$((fails + 1)) ;; esac + +dblock="$(awk '/^download_release_binary\(\) \{/,/^}/' "$BS")" +[ -n "$dblock" ] || { echo "FAIL: no download_release_binary found in $BS"; exit 1; } +v="$(printf '%s\n' "$dblock" | grep -n 'verify_release_checksum' | head -1 | cut -d: -f1)" +x="$(printf '%s\n' "$dblock" | grep -n '"\$tmp" version' | head -1 | cut -d: -f1)" +[ -n "$v" ] && [ -n "$x" ] && [ "$v" -lt "$x" ] \ + && echo "PASS the release binary is verified before it is executed" \ + || { echo "FAIL: download_release_binary must call verify_release_checksum before running the binary (lines: $v $x)"; fails=$((fails + 1)); } + +# --- cloudflared is a pinned release, checked before it is installed --------------------- +cfblock="$(awk '/^install_cloudflared\(\) \{/,/^}/' "$BS")" +[ -n "$cfblock" ] || { echo "FAIL: no install_cloudflared found in $BS"; exit 1; } +cfsum="$(printf 'stand-in cloudflared\n' | sha256sum | cut -d' ' -f1)" +run_cf() { # FELIS_CLOUDFLARED_VERSION pinned-amd64-digest [FELIS_CLOUDFLARED_SHA256] + FELIS_CLOUDFLARED_VERSION="$1" CLOUDFLARED_PINNED_VERSION=2026.9.1 CLOUDFLARED_PINNED_SHA256_AMD64="$2" \ + CLOUDFLARED_PINNED_SHA256_ARM64=unused CLOUDFLARED_PINNED_SHA256_ARM=unused \ + FELIS_CLOUDFLARED_SHA256="${3:-}" TMPDIR="$sdir" bash -c ' + die() { printf "DIE: %s\n" "$*"; exit 1; } + log() { printf "LOG: %s\n" "$*"; } + ok() { printf "OK: %s\n" "$*"; } + remember_temp() { :; } + command() { return 1; } + uname() { echo x86_64; } + cloudflared() { echo "cloudflared version test"; } + curl() { printf "CURL: %s\n" "$*"; while [ "$#" -gt 1 ] && [ "$1" != "-o" ]; do shift; done + printf "stand-in cloudflared\n" > "$2"; } + install() { printf "INSTALL: %s\n" "$*"; } + '"$cfblock"' + install_cloudflared' +} +out="$(run_cf 2026.9.1 "$cfsum")" +expect "cloudflared comes from the pinned release, never latest" "releases/download/2026.9.1/cloudflared-linux-amd64" "$out" +expect "a matching cloudflared is installed" "INSTALL: -m 0755" "$out" +out="$(run_cf 2026.9.1 deadbeef)" +expect "a cloudflared that does not match the pin is refused" "DIE: cloudflared-linux-amd64 2026.9.1 hashes to ${cfsum}, expected deadbeef" "$out" +case "$out" in *INSTALL:*) echo "FAIL: a refused cloudflared must not be installed"; fails=$((fails + 1)) ;; esac +expect "another cloudflared version needs its own digest" "DIE: no pinned sha256 for cloudflared 2027.1.0" "$(run_cf 2027.1.0 "$cfsum")" +expect "another cloudflared version installs with its digest" "INSTALL: -m 0755" "$(run_cf 2027.1.0 deadbeef "$cfsum")" + +# k3s: the install script is read from the pinned tag, and told the same version. +kblock="$(awk '/^install_k3s\(\) \{/,/^}/' "$BS")" +expect "k3s's install script comes from the pinned tag" 'raw.githubusercontent.com/k3s-io/k3s/${FELIS_K3S_VERSION}/install.sh' "$kblock" +expect "k3s's install script is told the pinned version" 'INSTALL_K3S_VERSION="$FELIS_K3S_VERSION"' "$kblock" +case "$kblock" in *"https://get.k3s.io"*) echo "FAIL: get.k3s.io serves master's script; read it from the pinned tag"; fails=$((fails + 1)) ;; esac + # --- a private repo without a token fails with the hint instead of prompting ------------- # git asks for credentials on /dev/tty, where a piped install would sit waiting. Every # network git call goes through git_auth, so the switch belongs there. @@ -685,7 +762,7 @@ expect "a failed fetch into an existing checkout names the token" "set FELIS_GIT mblock="$(awk '/^ log "rendering \+ applying the control-plane bundle"/,/kube apply -f -/' "$BS")" [ -n "$mblock" ] || { echo "FAIL: no manifest_args block found in $BS"; exit 1; } -[ "$(printf '%s\n' "$mblock" | wc -l)" -lt 40 ] \ +[ "$(printf '%s\n' "$mblock" | wc -l)" -lt 60 ] \ || { echo "FAIL: the extracted block is not the manifest_args block -- did it move?"; exit 1; } run_bundle_flags() { # backup-pvc worlds-host-path @@ -699,6 +776,7 @@ run_bundle_flags() { # backup-pvc worlds-host-path setfacl() { printf "SETFACL %s\n" "$*"; } chmod() { printf "CHMOD %s\n" "$*"; } node_global_cidrs() { printf "203.0.113.7/32\n2001:db8::7/128\n"; } + pvc_size() { case "$2" in registry) printf "20Gi\n" ;; esac; } run_bundle() { '"$mblock"' } @@ -712,6 +790,12 @@ case "$out" in *--worlds-host-path*) echo "FAIL: no reaper flags may render without FELIS_WORLDS_HOST_PATH"; fails=$((fails + 1)) ;; esac +expect "a known registry size reaches the renderer" "--registry-storage +20Gi" "$out" +case "$out" in + *--uploads-storage*|*--backup-storage*) echo "FAIL: an unset size must leave the renderer's default alone"; fails=$((fails + 1)) ;; +esac + out="$(run_bundle_flags '' '')" expect "an emptied FELIS_BACKUP_PVC is the explicit no-backup shape" "--backup-pvc=" "$out" # Game server egress excludes every private range already; the node's own public @@ -922,24 +1006,33 @@ expect "the throwaway config is registered for EXIT cleanup" "TEMP /" "$out" # both against kubelet image GC, and unpin a previous felis tag. pnblock="$(awk '/^pin_registry_images\(\) \{/,/^}/' "$BS")" [ -n "$pnblock" ] || { echo "FAIL: no pin_registry_images found in $BS"; exit 1; } +rrblock="$(awk '/^registry_image_containerd_ref\(\) \{/,/^}/' "$BS")" +[ -n "$rrblock" ] || { echo "FAIL: no registry_image_containerd_ref found in $BS"; exit 1; } +regimage="$(grep -m1 '^REGISTRY_IMAGE=' "$BS" | cut -d'"' -f2)" +regdigest="${regimage#*@}" calls="$(mktemp)" out="$( - CALLS="$calls" FELIS_IMAGE=registry.felis.svc:5000/felis/felis:v2 bash -c ' + CALLS="$calls" REGISTRY_IMAGE="$regimage" REGDIGEST="$regdigest" FELIS_IMAGE=registry.felis.svc:5000/felis/felis:v2 bash -c ' ok() { printf "OK: %s\n" "$*"; } warn() { printf "WARN: %s\n" "$*"; } k3s_cmd() { case "$*" in - "ctr images ls -q") printf "registry.felis.svc:5000/felis/felis:v1\nregistry.felis.svc:5000/felis/felis:v2\ndocker.io/library/registry:2\nregistry.felis.svc:5000/felis/limbo:demo\n" ;; + "ctr images ls -q") printf "registry.felis.svc:5000/felis/felis:v1\nregistry.felis.svc:5000/felis/felis:v2\ndocker.io/library/registry:2\ndocker.io/library/registry@%s\nregistry.felis.svc:5000/felis/limbo:demo\n" "$REGDIGEST" ;; *) printf "CTR %s\n" "$*" >>"$CALLS" ;; esac } + '"$rrblock"' '"$pnblock"' pin_registry_images' )$(printf '\n'; cat "$calls")" rm -f "$calls" expect "the running felis image is pinned" "CTR ctr images label registry.felis.svc:5000/felis/felis:v2 io.cri-containerd.pinned=pinned" "$out" -expect "registry:2 is pinned" "CTR ctr images label docker.io/library/registry:2 io.cri-containerd.pinned=pinned" "$out" +expect "the registry image is pinned by digest" "CTR ctr images label docker.io/library/registry@${regdigest} io.cri-containerd.pinned=pinned" "$out" expect "a previous felis tag is unpinned" "CTR ctr images label registry.felis.svc:5000/felis/felis:v1 io.cri-containerd.pinned=" "$out" +expect "the old registry:2 tag is unpinned" "CTR ctr images label docker.io/library/registry:2 io.cri-containerd.pinned=" "$out" +case "$out" in + *"registry@${regdigest} io.cri-containerd.pinned="$'\n'*) echo "FAIL: the current registry image must not be unpinned"; fails=$((fails + 1)) ;; +esac case "$out" in *"limbo:demo io.cri"*) echo "FAIL: only the registry pod's images may be pinned or unpinned"; fails=$((fails + 1)) ;; esac @@ -986,22 +1079,24 @@ p="$(line_of pin_user_server_images)"; b="$(line_of build_game_stack)"; u="$(lin iblock="$(awk '/^import_registry_image\(\) \{/,/^}/' "$BS")" [ -n "$iblock" ] || { echo "FAIL: no import_registry_image found in $BS"; exit 1; } -out="$( - bash -c ' +run_import() { # listed-refs + LISTED="$1" REGISTRY_IMAGE="$regimage" bash -c ' log() { printf "LOG: %s\n" "$*"; } ok() { printf "OK: %s\n" "$*"; } warn() { printf "WARN: %s\n" "$*"; } die() { printf "DIE: %s\n" "$*"; exit 1; } - systemctl() { :; } - k3s_cmd() { case "$*" in "ctr images ls -q") printf "docker.io/library/registry:2\n" ;; esac; } - docker() { printf "DOCKER %s\n" "$*"; return 1; } + k3s_cmd() { case "$*" in "ctr images ls -q") printf "%b" "$LISTED" ;; *) printf "PULL %s\n" "$*" >&2 ;; esac; } + '"$rrblock"' '"$iblock"' - import_registry_image' -)" -expect "an already-imported registry:2 is left alone" "already in k3s containerd" "$out" + import_registry_image' 2>&1 +} +out="$(run_import "docker.io/library/registry@${regdigest}\n")" +expect "an already-pulled registry image is left alone" "already in k3s containerd" "$out" case "$out" in - *DOCKER*) echo "FAIL: a present registry:2 must not trigger a docker pull"; fails=$((fails + 1)) ;; + *PULL*) echo "FAIL: a present registry image must not be pulled again"; fails=$((fails + 1)) ;; esac +out="$(run_import "docker.io/library/registry:2\n")" +expect "only the pinned digest counts as present; the old tag is pulled over" "PULL crictl pull ${regimage}" "$out" # --- installer re-runs refresh the workload namespace's felis-config copy --------------- # The backup/restore/fileedit Jobs and the reaper mount the workload namespace's own diff --git a/internal/platform/bootstrap_pin_test.go b/internal/platform/bootstrap_pin_test.go new file mode 100644 index 0000000..9a68df1 --- /dev/null +++ b/internal/platform/bootstrap_pin_test.go @@ -0,0 +1,28 @@ +package platform + +import ( + "os" + "regexp" + "testing" +) + +// The renderer puts defaultRegistryImage in the registry Deployment, and bootstrap +// caches and GC-pins REGISTRY_IMAGE in containerd. If the two drift, kubelet looks +// for an image nobody cached or pinned, and an air-gapped box cannot start its +// registry at all. +func TestBootstrapPinsTheRegistryImage(t *testing.T) { + b, err := os.ReadFile("../../deploy/bootstrap.sh") + if err != nil { + t.Fatal(err) + } + m := regexp.MustCompile(`(?m)^REGISTRY_IMAGE="([^"]+)"$`).FindSubmatch(b) + if m == nil { + t.Fatal("deploy/bootstrap.sh has no REGISTRY_IMAGE=\"...\" line") + } + if got := string(m[1]); got != defaultRegistryImage { + t.Fatalf("bootstrap REGISTRY_IMAGE = %q, renderer default = %q", got, defaultRegistryImage) + } + if !regexp.MustCompile(`@sha256:[0-9a-f]{64}$`).MatchString(defaultRegistryImage) { + t.Fatalf("defaultRegistryImage %q is not pinned by digest", defaultRegistryImage) + } +} diff --git a/internal/platform/identities.go b/internal/platform/identities.go index be039c1..a83aa1f 100644 --- a/internal/platform/identities.go +++ b/internal/platform/identities.go @@ -65,8 +65,11 @@ const ( // defaultRegistryImage is the upstream CNCF Distribution registry. It is an // official, stable image and the only registry implementation the build/restore - // subsystems are exercised against (registry..svc:5000). - defaultRegistryImage = "registry:2" + // subsystems are exercised against (registry..svc:5000). Pinned by digest so + // a re-tag upstream cannot change what holds every image on the box; the tag is + // for humans. deploy/bootstrap.sh's REGISTRY_IMAGE caches and GC-pins this exact + // ref and must name the same one (TestBootstrapPinsTheRegistryImage). + defaultRegistryImage = "docker.io/library/registry:2.8.3@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373" ) // Params parameterises the install bundle. Namespaces and the registry location @@ -121,7 +124,7 @@ type Params struct { // passes this value through as FELIS_IMAGE so the restore executor launches its // `felis restore` Job using the very same image. FelisImage string - // RegistryImage is the in-cluster registry image. Defaults to registry:2. + // RegistryImage is the in-cluster registry image. Defaults to registry 2.8.3, by digest. RegistryImage string // BackupPVC is the name of the world-archive PersistentVolumeClaim. The bundle // RENDERS this PVC (backupPVC in workloads.go, Minecraft namespace — where every