From a4fa16ae69ac74945d4db1a22d05a52f6b27d959 Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Thu, 24 Sep 2026 23:52:30 +0800 Subject: [PATCH] =?UTF-8?q?feat(bootstrap):=20=E6=8E=A7=E5=88=B6=E9=9D=A2?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E6=8C=89=E7=89=88=E6=9C=AC=E6=89=93=20tag?= =?UTF-8?q?=EF=BC=8C=E5=8D=87=E7=BA=A7=E5=90=8E=20rollout=20undo=20?= =?UTF-8?q?=E5=8F=AF=E5=9B=9E=E5=88=B0=E4=B8=8A=E4=B8=80=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/bootstrap.sh | 93 ++++++++++++++++++++++++++++++++++------ deploy/bootstrap_test.sh | 68 +++++++++++++++++++++++++++++ docs/troubleshooting.md | 26 ++++++++--- 3 files changed, 168 insertions(+), 19 deletions(-) diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 215da2b..960f43d 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -67,7 +67,9 @@ # FELIS_REF branch/tag/sha — pins the build, overrides the channel, and forces a # source build (naming a ref asks for that tree, not a published asset) # FELIS_IMAGE control-plane image ref (default: -# registry.felis.svc:5000/felis/felis:demo — never :latest; +# registry.felis.svc:5000/felis/felis:, so each +# release has its own tag and `kubectl rollout undo` returns to the +# previous one; :demo when the version is unknown — never :latest; # anything not under the registry is used as-is but is NOT # mirrored into it, so it has no pull source after an image GC) # FELIS_ROOT_DOMAIN deployment root domain (default: .nip.io) @@ -117,6 +119,9 @@ if [ -n "$FELIS_REF" ]; then FELIS_REF_PINNED=1; fi # release download. It is what the image build, the CRD apply and the game stack key off: # all three only need "is there a binary and no checkout", never "which route got us here". HAVE_PREBUILT_BINARY="" +# The image the control plane ran before this run moved it (deploy_bundle), for the rollback +# hint in summary. +PREVIOUS_FELIS_IMAGE="" # Optional GitHub credential, needed while this repository is private: GitHub answers # 404 (not 403) for a repo the caller cannot see, so without it both the release lookup # and the clone fail as "not found". Exported because git's credential helper below runs @@ -136,7 +141,9 @@ FELIS_VERSION_BASE="" # daemon needs no insecure-registries entry for it. REGISTRY_URL="registry.felis.svc:5000" REGISTRY_PUSH_HOST="127.0.0.1:${REGISTRY_URL##*:}" -FELIS_IMAGE="${FELIS_IMAGE:-${REGISTRY_URL}/felis/felis:demo}" +# Empty unless the operator names one: resolve_felis_image derives the tag from the version +# this run installs, which is only known once the binary or the checkout is. +FELIS_IMAGE="${FELIS_IMAGE:-}" FELIS_EGRESS_MODE="${FELIS_EGRESS_MODE:-nodeport}" FELIS_PANEL_NODEPORT="${FELIS_PANEL_NODEPORT:-30443}" # World-archive storage. The installer renders this PVC (minecraft namespace) and felis-api @@ -1554,6 +1561,39 @@ build_image_from_source() { chmod 0755 "$HOST_BIN" } +# resolve_felis_image names the control-plane image after the release it carries +# (registry.felis.svc:5000/felis/felis:v1.2.3) unless FELIS_IMAGE was given. One tag per +# release is what makes `kubectl rollout undo` a rollback: the previous ReplicaSet names the +# previous tag, and the registry keeps the newest five of them (its pruner, §9). Under one +# mutable tag the undo re-created the pods on the image the upgrade had just written over it. +# +# The version is the release tag on the download path, the stamp on a source build +# (v1.2.3+gabc1234 becomes the tag v1.2.3-gabc1234: '+' is not allowed in a tag), and the +# binary's own report on the setup-console path, which skips both. A rerun of the same version +# reuses its tag; deploy_bundle restarts the pods onto the rebuilt image then. +resolve_felis_image() { + local v + [ -z "$FELIS_IMAGE" ] || return 0 + v="$FELIS_VERSION" + if [ -z "$v" ] && [ -n "$HAVE_PREBUILT_BINARY" ]; then + v="$("$HOST_BIN" version 2>/dev/null | head -n 1 || true)" + v="${v#felis }" + fi + FELIS_IMAGE="${REGISTRY_URL}/felis/felis:$(image_tag_for_version "$v")" + ok "control-plane image: ${FELIS_IMAGE}" +} + +# image_tag_for_version turns a felis version into an image tag: every character a tag may not +# hold becomes '-'. An unknown version ("dev" is what an unstamped binary reports) is :demo. +image_tag_for_version() { + local v + v="$(printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '-')" + case "$v" in + ""|dev|[!A-Za-z0-9_]*) printf 'demo' ;; + *) printf '%s' "${v:0:128}" ;; + esac +} + build_image() { systemctl start docker # Keyed on the binary, not on the route that produced it: the TUI hand-off and a release @@ -2894,12 +2934,16 @@ EOF } deploy_bundle() { - local had_api=0 had_operator=0 + local prev_api prev_operator export KUBECONFIG=/etc/rancher/k3s/k3s.yaml write_felis_toml "${STATE_DIR}/felis.pod.toml" "${NODE_IP}" - kube -n "$CONTROL_NS" get deployment felis-api >/dev/null 2>&1 && had_api=1 - kube -n "$CONTROL_NS" get deployment felis-operator >/dev/null 2>&1 && had_operator=1 + prev_api="$(deployment_image felis-api api)" + prev_operator="$(deployment_image felis-operator operator)" + if [ -n "$prev_api" ] && [ "$prev_api" != "$FELIS_IMAGE" ]; then + PREVIOUS_FELIS_IMAGE="$prev_api" + printf '%s\n' "$prev_api" > "${STATE_DIR}/previous-felis-image" + fi # Always the embedded copy. It is byte-identical to deploy/crd/ (bootstrap_asset.go embeds # that very file), it needs no checkout — which the release-download path does not have — @@ -2972,7 +3016,7 @@ deploy_bundle() { if [ -n "$size" ]; then manifest_args+=(--backup-storage "$size"); fi fi "$HOST_BIN" manifests "${manifest_args[@]}" | kube apply -f - - restart_existing_control_plane "$had_api" "$had_operator" + restart_existing_control_plane "$prev_api" "$prev_operator" log "waiting for control-plane rollouts" local d @@ -3002,17 +3046,32 @@ pvc_size() { printf '%s' "$have" } -restart_existing_control_plane() { - local had_api="$1" had_operator="$2" - [ "$had_api$had_operator" != "00" ] || return 0 +# deployment_image prints the image that container of a +# control-plane Deployment runs now, or nothing when the Deployment does not exist yet. +deployment_image() { + kube -n "$CONTROL_NS" get deployment "$1" \ + -o "jsonpath={.spec.template.spec.containers[?(@.name==\"$2\")].image}" 2>/dev/null || true +} - log "restarting existing control-plane deployments to pick up ${FELIS_IMAGE}" +# restart_existing_control_plane restarts the Deployments the bundle apply left as they were: +# those that already ran FELIS_IMAGE, whose tag now names a rebuilt image (a rerun of the same +# version, or a FELIS_IMAGE the operator reuses). A Deployment whose image changed is rolling +# from the apply already, and must not be restarted on top: the restart is a second template +# change, so `rollout undo` would step back to the new image instead of the previous release. +restart_existing_control_plane() { + local prev_api="$1" prev_operator="$2" # `if`, not `[ test ] && cmd`: as the LAST command of the function the and-list returns 1 # when the test is false, which becomes the function's exit status and kills the whole # install under `set -Eeuo pipefail` — right after the bundle is applied and before the - # rollout wait. Fires on any host carrying felis-api without felis-operator. - if [ "$had_api" = "1" ]; then kube -n "$CONTROL_NS" rollout restart deployment/felis-api; fi - if [ "$had_operator" = "1" ]; then kube -n "$CONTROL_NS" rollout restart deployment/felis-operator; fi + # rollout wait. + if [ "$prev_api" = "$FELIS_IMAGE" ]; then + log "restarting felis-api onto the rebuilt ${FELIS_IMAGE}" + kube -n "$CONTROL_NS" rollout restart deployment/felis-api + fi + if [ "$prev_operator" = "$FELIS_IMAGE" ]; then + log "restarting felis-operator onto the rebuilt ${FELIS_IMAGE}" + kube -n "$CONTROL_NS" rollout restart deployment/felis-operator + fi } # push_image_to_registry re-tags a locally built image for the node's @@ -3183,6 +3242,13 @@ summary() { log "the proxy in Minecraft — that is what makes the Owner's admin identity a real" log "Mojang account rather than a password." log "Use 'sudo felis breakGlass' only for emergency local Owner recovery/reset." + if [ -n "${PREVIOUS_FELIS_IMAGE:-}" ]; then + echo + log "The control plane moved from ${PREVIOUS_FELIS_IMAGE} to ${FELIS_IMAGE}" + log "(also recorded in ${STATE_DIR}/previous-felis-image). To go back to it:" + log " kubectl -n ${CONTROL_NS} rollout undo deployment/felis-api deployment/felis-operator" + log "The database stays migrated; docs/troubleshooting.md §16 has the full rollback." + fi echo summary_offsite echo @@ -3538,6 +3604,7 @@ main() { else fetch_source fi + resolve_felis_image build_image # After build_image imported the felis image: the registry pod's gate runs it. pin_registry_images diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index 572a2cd..dffe56a 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -1515,6 +1515,74 @@ else fi rm -rf "$odir" "$ofile" +# --- the control-plane image is tagged by release, so rollout undo is a rollback -------- +# Under one mutable tag `kubectl rollout undo` re-created the pods on the image the upgrade had +# just written over it. The tag must follow the version, and an upgrade must not restart the +# Deployments on top of the apply's own roll (that second revision is what undo would reach). + +imgblock="$(awk '/^resolve_felis_image\(\) \{/,/^}/' "$BS"; awk '/^image_tag_for_version\(\) \{/,/^}/' "$BS")" +[ -n "$imgblock" ] || { echo "FAIL: no resolve_felis_image found in $BS"; exit 1; } +[ "$(printf '%s\n' "$imgblock" | wc -l)" -lt 30 ] \ + || { echo "FAIL: the extracted block is not resolve_felis_image -- did it move?"; exit 1; } + +run_image() { # FELIS_IMAGE FELIS_VERSION HAVE_PREBUILT_BINARY binary-version + FELIS_IMAGE="$1" FELIS_VERSION="$2" HAVE_PREBUILT_BINARY="$3" BIN_VERSION="$4" \ + REGISTRY_URL=registry.felis.svc:5000 HOST_BIN=fakeFelis bash -c ' + ok() { :; } + fakeFelis() { printf "felis %s\n" "$BIN_VERSION"; } + '"$imgblock"' + resolve_felis_image + printf "%s\n" "$FELIS_IMAGE"' +} + +expect "a release install is tagged with its release" "registry.felis.svc:5000/felis/felis:v1.2.3" \ + "$(run_image '' v1.2.3 '' '')" +expect "a source build's stamp becomes a legal tag" "registry.felis.svc:5000/felis/felis:v1.2.3-gabc1234" \ + "$(run_image '' 'v1.2.3+gabc1234' '' '')" +expect "the setup console path asks the binary it installed" "registry.felis.svc:5000/felis/felis:v1.4.0" \ + "$(run_image '' '' 1 v1.4.0)" +expect "an unstamped binary keeps the old tag" "registry.felis.svc:5000/felis/felis:demo" \ + "$(run_image '' '' 1 dev)" +expect "an unknown version keeps the old tag" "registry.felis.svc:5000/felis/felis:demo" \ + "$(run_image '' '' '' '')" +expect "an explicit FELIS_IMAGE is used as given" "reg.example/felis:mine" \ + "$(run_image reg.example/felis:mine v1.2.3 '' '')" + +rsblock="$(awk '/^restart_existing_control_plane\(\) \{/,/^}/' "$BS")" +[ -n "$rsblock" ] || { echo "FAIL: no restart_existing_control_plane found in $BS"; exit 1; } +run_restart() { # prev-api prev-operator + FELIS_IMAGE=reg/felis/felis:v2 CONTROL_NS=felis bash -c ' + set -Eeuo pipefail + log() { :; } + kube() { printf "KUBE %s\n" "$*"; } + '"$rsblock"' + restart_existing_control_plane "$1" "$2" + echo DONE' _ "$1" "$2" +} + +out="$(run_restart reg/felis/felis:v1 reg/felis/felis:v1)" +case "$out" in + *"rollout restart"*) echo "FAIL an upgrade restarted the control plane on top of the apply's roll"; fails=$((fails + 1)) ;; + *DONE*) echo "PASS an upgrade leaves the roll to the apply" ;; + *) echo "FAIL restart_existing_control_plane died on an upgrade: $out"; fails=$((fails + 1)) ;; +esac +out="$(run_restart reg/felis/felis:v2 reg/felis/felis:v2)" +expect "a rerun of the same tag restarts felis-api onto the rebuilt image" "KUBE -n felis rollout restart deployment/felis-api" "$out" +expect "a rerun of the same tag restarts felis-operator too" "KUBE -n felis rollout restart deployment/felis-operator" "$out" +out="$(run_restart '' '')" +case "$out" in + *"rollout restart"*) echo "FAIL a first install restarted Deployments that did not exist"; fails=$((fails + 1)) ;; + *DONE*) echo "PASS a first install restarts nothing" ;; + *) echo "FAIL restart_existing_control_plane died on a first install: $out"; fails=$((fails + 1)) ;; +esac + +mainblock="$(awk '/^main\(\) \{/,/^}/' "$BS")" +case "$mainblock" in + *"resolve_felis_image + build_image"*) echo "PASS the image is named before it is built" ;; + *) echo "FAIL main must call resolve_felis_image right before build_image"; fails=$((fails + 1)) ;; +esac + # --------------------------------------------------------------------------------------- if [ "$fails" -eq 0 ]; then diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index a2d3cd5..ab665f1 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1330,17 +1330,31 @@ attestation. Check a downloaded binary with `gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`; it names the workflow run and commit that built it. -Roll back with: +Each release runs under its own image tag, `felis/felis:v1.2.3` (a source +build's stamp `v1.2.3+gabc1234` becomes `v1.2.3-gabc1234`; a build that reports +no version uses `:demo`). At the end of an upgrade the installer prints the tag +it moved away from, and keeps it in `/etc/felis/previous-felis-image`. Roll +back with: ``` -kubectl -n felis rollout undo deploy/felis-api +kubectl -n felis rollout undo deployment/felis-api deployment/felis-operator kubectl -n felis rollout status deploy/felis-api ``` -(the same for `felis-operator` and `registry`). `rollout undo` returns to the -previous ReplicaSet, whose image is normally still on the node; if the image GC -collected it, the registry re-serves it automatically (§13b) for every tag the -installer built — only hand-built tags need a manual re-mirror. +`rollout undo` returns each Deployment to its previous ReplicaSet, which names +the previous release's tag. That image is normally still on the node; if the +image GC collected it, the registry re-serves it (§13b): the pruner keeps the +five newest `felis/felis` tags and every image a game pod still runs. A rerun +of the same version reuses its tag and restarts both Deployments onto the +rebuilt image, so after such a rerun undo lands on that same tag again. The +next installer run re-applies the bundle and moves the image to whatever that +run installs. + +A platform upgrade leaves running game servers alone. Their init containers +run the felis image too; a running server keeps the one it started with and +picks up the new release on its next start. A release that changes the game +pod in any other way still restarts running servers once, as a server edit +does. `rollout undo` reverts the image only. The upgrade's database migrations stay applied; when they are the problem, restore the `pre-migrate` bundle the upgrade