feat(bootstrap): 控制面镜像按版本打 tag,升级后 rollout undo 可回到上一版本
This commit is contained in:
3 files changed
+168
-19
No files matched your search
+80
-13
@@ -67,7 +67,9 @@
|
|||||||
# FELIS_REF branch/tag/sha — pins the build, overrides the channel, and forces a
|
# FELIS_REF branch/tag/sha — pins the build, overrides the channel, and forces a
|
||||||
# source build (naming a ref asks for that tree, not a published asset)
|
# source build (naming a ref asks for that tree, not a published asset)
|
||||||
# FELIS_IMAGE control-plane image ref (default:
|
# FELIS_IMAGE control-plane image ref (default:
|
||||||
# registry.felis.svc:5000/felis/felis:demo — never :latest;
|
# registry.felis.svc:5000/felis/felis:<the felis version>, so each
|
||||||
|
# release has its own tag and `kubectl rollout undo` returns to the
|
||||||
|
# previous one; :demo when the version is unknown — never :latest;
|
||||||
# anything not under the registry is used as-is but is NOT
|
# anything not under the registry is used as-is but is NOT
|
||||||
# mirrored into it, so it has no pull source after an image GC)
|
# mirrored into it, so it has no pull source after an image GC)
|
||||||
# FELIS_ROOT_DOMAIN deployment root domain (default: <node-ip>.nip.io)
|
# FELIS_ROOT_DOMAIN deployment root domain (default: <node-ip>.nip.io)
|
||||||
@@ -117,6 +119,9 @@ if [ -n "$FELIS_REF" ]; then FELIS_REF_PINNED=1; fi
|
|||||||
# release download. It is what the image build, the CRD apply and the game stack key off:
|
# release download. It is what the image build, the CRD apply and the game stack key off:
|
||||||
# all three only need "is there a binary and no checkout", never "which route got us here".
|
# all three only need "is there a binary and no checkout", never "which route got us here".
|
||||||
HAVE_PREBUILT_BINARY=""
|
HAVE_PREBUILT_BINARY=""
|
||||||
|
# The image the control plane ran before this run moved it (deploy_bundle), for the rollback
|
||||||
|
# hint in summary.
|
||||||
|
PREVIOUS_FELIS_IMAGE=""
|
||||||
# Optional GitHub credential, needed while this repository is private: GitHub answers
|
# Optional GitHub credential, needed while this repository is private: GitHub answers
|
||||||
# 404 (not 403) for a repo the caller cannot see, so without it both the release lookup
|
# 404 (not 403) for a repo the caller cannot see, so without it both the release lookup
|
||||||
# and the clone fail as "not found". Exported because git's credential helper below runs
|
# and the clone fail as "not found". Exported because git's credential helper below runs
|
||||||
@@ -136,7 +141,9 @@ FELIS_VERSION_BASE=""
|
|||||||
# daemon needs no insecure-registries entry for it.
|
# daemon needs no insecure-registries entry for it.
|
||||||
REGISTRY_URL="registry.felis.svc:5000"
|
REGISTRY_URL="registry.felis.svc:5000"
|
||||||
REGISTRY_PUSH_HOST="127.0.0.1:${REGISTRY_URL##*:}"
|
REGISTRY_PUSH_HOST="127.0.0.1:${REGISTRY_URL##*:}"
|
||||||
FELIS_IMAGE="${FELIS_IMAGE:-${REGISTRY_URL}/felis/felis:demo}"
|
# Empty unless the operator names one: resolve_felis_image derives the tag from the version
|
||||||
|
# this run installs, which is only known once the binary or the checkout is.
|
||||||
|
FELIS_IMAGE="${FELIS_IMAGE:-}"
|
||||||
FELIS_EGRESS_MODE="${FELIS_EGRESS_MODE:-nodeport}"
|
FELIS_EGRESS_MODE="${FELIS_EGRESS_MODE:-nodeport}"
|
||||||
FELIS_PANEL_NODEPORT="${FELIS_PANEL_NODEPORT:-30443}"
|
FELIS_PANEL_NODEPORT="${FELIS_PANEL_NODEPORT:-30443}"
|
||||||
# World-archive storage. The installer renders this PVC (minecraft namespace) and felis-api
|
# World-archive storage. The installer renders this PVC (minecraft namespace) and felis-api
|
||||||
@@ -1554,6 +1561,39 @@ build_image_from_source() {
|
|||||||
chmod 0755 "$HOST_BIN"
|
chmod 0755 "$HOST_BIN"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# resolve_felis_image names the control-plane image after the release it carries
|
||||||
|
# (registry.felis.svc:5000/felis/felis:v1.2.3) unless FELIS_IMAGE was given. One tag per
|
||||||
|
# release is what makes `kubectl rollout undo` a rollback: the previous ReplicaSet names the
|
||||||
|
# previous tag, and the registry keeps the newest five of them (its pruner, §9). Under one
|
||||||
|
# mutable tag the undo re-created the pods on the image the upgrade had just written over it.
|
||||||
|
#
|
||||||
|
# The version is the release tag on the download path, the stamp on a source build
|
||||||
|
# (v1.2.3+gabc1234 becomes the tag v1.2.3-gabc1234: '+' is not allowed in a tag), and the
|
||||||
|
# binary's own report on the setup-console path, which skips both. A rerun of the same version
|
||||||
|
# reuses its tag; deploy_bundle restarts the pods onto the rebuilt image then.
|
||||||
|
resolve_felis_image() {
|
||||||
|
local v
|
||||||
|
[ -z "$FELIS_IMAGE" ] || return 0
|
||||||
|
v="$FELIS_VERSION"
|
||||||
|
if [ -z "$v" ] && [ -n "$HAVE_PREBUILT_BINARY" ]; then
|
||||||
|
v="$("$HOST_BIN" version 2>/dev/null | head -n 1 || true)"
|
||||||
|
v="${v#felis }"
|
||||||
|
fi
|
||||||
|
FELIS_IMAGE="${REGISTRY_URL}/felis/felis:$(image_tag_for_version "$v")"
|
||||||
|
ok "control-plane image: ${FELIS_IMAGE}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# image_tag_for_version turns a felis version into an image tag: every character a tag may not
|
||||||
|
# hold becomes '-'. An unknown version ("dev" is what an unstamped binary reports) is :demo.
|
||||||
|
image_tag_for_version() {
|
||||||
|
local v
|
||||||
|
v="$(printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '-')"
|
||||||
|
case "$v" in
|
||||||
|
""|dev|[!A-Za-z0-9_]*) printf 'demo' ;;
|
||||||
|
*) printf '%s' "${v:0:128}" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
build_image() {
|
build_image() {
|
||||||
systemctl start docker
|
systemctl start docker
|
||||||
# Keyed on the binary, not on the route that produced it: the TUI hand-off and a release
|
# Keyed on the binary, not on the route that produced it: the TUI hand-off and a release
|
||||||
@@ -2894,12 +2934,16 @@ EOF
|
|||||||
}
|
}
|
||||||
|
|
||||||
deploy_bundle() {
|
deploy_bundle() {
|
||||||
local had_api=0 had_operator=0
|
local prev_api prev_operator
|
||||||
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
||||||
write_felis_toml "${STATE_DIR}/felis.pod.toml" "${NODE_IP}"
|
write_felis_toml "${STATE_DIR}/felis.pod.toml" "${NODE_IP}"
|
||||||
|
|
||||||
kube -n "$CONTROL_NS" get deployment felis-api >/dev/null 2>&1 && had_api=1
|
prev_api="$(deployment_image felis-api api)"
|
||||||
kube -n "$CONTROL_NS" get deployment felis-operator >/dev/null 2>&1 && had_operator=1
|
prev_operator="$(deployment_image felis-operator operator)"
|
||||||
|
if [ -n "$prev_api" ] && [ "$prev_api" != "$FELIS_IMAGE" ]; then
|
||||||
|
PREVIOUS_FELIS_IMAGE="$prev_api"
|
||||||
|
printf '%s\n' "$prev_api" > "${STATE_DIR}/previous-felis-image"
|
||||||
|
fi
|
||||||
|
|
||||||
# Always the embedded copy. It is byte-identical to deploy/crd/ (bootstrap_asset.go embeds
|
# Always the embedded copy. It is byte-identical to deploy/crd/ (bootstrap_asset.go embeds
|
||||||
# that very file), it needs no checkout — which the release-download path does not have —
|
# that very file), it needs no checkout — which the release-download path does not have —
|
||||||
@@ -2972,7 +3016,7 @@ deploy_bundle() {
|
|||||||
if [ -n "$size" ]; then manifest_args+=(--backup-storage "$size"); fi
|
if [ -n "$size" ]; then manifest_args+=(--backup-storage "$size"); fi
|
||||||
fi
|
fi
|
||||||
"$HOST_BIN" manifests "${manifest_args[@]}" | kube apply -f -
|
"$HOST_BIN" manifests "${manifest_args[@]}" | kube apply -f -
|
||||||
restart_existing_control_plane "$had_api" "$had_operator"
|
restart_existing_control_plane "$prev_api" "$prev_operator"
|
||||||
|
|
||||||
log "waiting for control-plane rollouts"
|
log "waiting for control-plane rollouts"
|
||||||
local d
|
local d
|
||||||
@@ -3002,17 +3046,32 @@ pvc_size() {
|
|||||||
printf '%s' "$have"
|
printf '%s' "$have"
|
||||||
}
|
}
|
||||||
|
|
||||||
restart_existing_control_plane() {
|
# deployment_image <deployment> <container> prints the image that container of a
|
||||||
local had_api="$1" had_operator="$2"
|
# control-plane Deployment runs now, or nothing when the Deployment does not exist yet.
|
||||||
[ "$had_api$had_operator" != "00" ] || return 0
|
deployment_image() {
|
||||||
|
kube -n "$CONTROL_NS" get deployment "$1" \
|
||||||
|
-o "jsonpath={.spec.template.spec.containers[?(@.name==\"$2\")].image}" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
log "restarting existing control-plane deployments to pick up ${FELIS_IMAGE}"
|
# restart_existing_control_plane restarts the Deployments the bundle apply left as they were:
|
||||||
|
# those that already ran FELIS_IMAGE, whose tag now names a rebuilt image (a rerun of the same
|
||||||
|
# version, or a FELIS_IMAGE the operator reuses). A Deployment whose image changed is rolling
|
||||||
|
# from the apply already, and must not be restarted on top: the restart is a second template
|
||||||
|
# change, so `rollout undo` would step back to the new image instead of the previous release.
|
||||||
|
restart_existing_control_plane() {
|
||||||
|
local prev_api="$1" prev_operator="$2"
|
||||||
# `if`, not `[ test ] && cmd`: as the LAST command of the function the and-list returns 1
|
# `if`, not `[ test ] && cmd`: as the LAST command of the function the and-list returns 1
|
||||||
# when the test is false, which becomes the function's exit status and kills the whole
|
# when the test is false, which becomes the function's exit status and kills the whole
|
||||||
# install under `set -Eeuo pipefail` — right after the bundle is applied and before the
|
# install under `set -Eeuo pipefail` — right after the bundle is applied and before the
|
||||||
# rollout wait. Fires on any host carrying felis-api without felis-operator.
|
# rollout wait.
|
||||||
if [ "$had_api" = "1" ]; then kube -n "$CONTROL_NS" rollout restart deployment/felis-api; fi
|
if [ "$prev_api" = "$FELIS_IMAGE" ]; then
|
||||||
if [ "$had_operator" = "1" ]; then kube -n "$CONTROL_NS" rollout restart deployment/felis-operator; fi
|
log "restarting felis-api onto the rebuilt ${FELIS_IMAGE}"
|
||||||
|
kube -n "$CONTROL_NS" rollout restart deployment/felis-api
|
||||||
|
fi
|
||||||
|
if [ "$prev_operator" = "$FELIS_IMAGE" ]; then
|
||||||
|
log "restarting felis-operator onto the rebuilt ${FELIS_IMAGE}"
|
||||||
|
kube -n "$CONTROL_NS" rollout restart deployment/felis-operator
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# push_image_to_registry <ref> re-tags a locally built image for the node's
|
# push_image_to_registry <ref> re-tags a locally built image for the node's
|
||||||
@@ -3183,6 +3242,13 @@ summary() {
|
|||||||
log "the proxy in Minecraft — that is what makes the Owner's admin identity a real"
|
log "the proxy in Minecraft — that is what makes the Owner's admin identity a real"
|
||||||
log "Mojang account rather than a password."
|
log "Mojang account rather than a password."
|
||||||
log "Use 'sudo felis breakGlass' only for emergency local Owner recovery/reset."
|
log "Use 'sudo felis breakGlass' only for emergency local Owner recovery/reset."
|
||||||
|
if [ -n "${PREVIOUS_FELIS_IMAGE:-}" ]; then
|
||||||
|
echo
|
||||||
|
log "The control plane moved from ${PREVIOUS_FELIS_IMAGE} to ${FELIS_IMAGE}"
|
||||||
|
log "(also recorded in ${STATE_DIR}/previous-felis-image). To go back to it:"
|
||||||
|
log " kubectl -n ${CONTROL_NS} rollout undo deployment/felis-api deployment/felis-operator"
|
||||||
|
log "The database stays migrated; docs/troubleshooting.md §16 has the full rollback."
|
||||||
|
fi
|
||||||
echo
|
echo
|
||||||
summary_offsite
|
summary_offsite
|
||||||
echo
|
echo
|
||||||
@@ -3538,6 +3604,7 @@ main() {
|
|||||||
else
|
else
|
||||||
fetch_source
|
fetch_source
|
||||||
fi
|
fi
|
||||||
|
resolve_felis_image
|
||||||
build_image
|
build_image
|
||||||
# After build_image imported the felis image: the registry pod's gate runs it.
|
# After build_image imported the felis image: the registry pod's gate runs it.
|
||||||
pin_registry_images
|
pin_registry_images
|
||||||
|
|||||||
@@ -1515,6 +1515,74 @@ else
|
|||||||
fi
|
fi
|
||||||
rm -rf "$odir" "$ofile"
|
rm -rf "$odir" "$ofile"
|
||||||
|
|
||||||
|
# --- the control-plane image is tagged by release, so rollout undo is a rollback --------
|
||||||
|
# Under one mutable tag `kubectl rollout undo` re-created the pods on the image the upgrade had
|
||||||
|
# just written over it. The tag must follow the version, and an upgrade must not restart the
|
||||||
|
# Deployments on top of the apply's own roll (that second revision is what undo would reach).
|
||||||
|
|
||||||
|
imgblock="$(awk '/^resolve_felis_image\(\) \{/,/^}/' "$BS"; awk '/^image_tag_for_version\(\) \{/,/^}/' "$BS")"
|
||||||
|
[ -n "$imgblock" ] || { echo "FAIL: no resolve_felis_image found in $BS"; exit 1; }
|
||||||
|
[ "$(printf '%s\n' "$imgblock" | wc -l)" -lt 30 ] \
|
||||||
|
|| { echo "FAIL: the extracted block is not resolve_felis_image -- did it move?"; exit 1; }
|
||||||
|
|
||||||
|
run_image() { # FELIS_IMAGE FELIS_VERSION HAVE_PREBUILT_BINARY binary-version
|
||||||
|
FELIS_IMAGE="$1" FELIS_VERSION="$2" HAVE_PREBUILT_BINARY="$3" BIN_VERSION="$4" \
|
||||||
|
REGISTRY_URL=registry.felis.svc:5000 HOST_BIN=fakeFelis bash -c '
|
||||||
|
ok() { :; }
|
||||||
|
fakeFelis() { printf "felis %s\n" "$BIN_VERSION"; }
|
||||||
|
'"$imgblock"'
|
||||||
|
resolve_felis_image
|
||||||
|
printf "%s\n" "$FELIS_IMAGE"'
|
||||||
|
}
|
||||||
|
|
||||||
|
expect "a release install is tagged with its release" "registry.felis.svc:5000/felis/felis:v1.2.3" \
|
||||||
|
"$(run_image '' v1.2.3 '' '')"
|
||||||
|
expect "a source build's stamp becomes a legal tag" "registry.felis.svc:5000/felis/felis:v1.2.3-gabc1234" \
|
||||||
|
"$(run_image '' 'v1.2.3+gabc1234' '' '')"
|
||||||
|
expect "the setup console path asks the binary it installed" "registry.felis.svc:5000/felis/felis:v1.4.0" \
|
||||||
|
"$(run_image '' '' 1 v1.4.0)"
|
||||||
|
expect "an unstamped binary keeps the old tag" "registry.felis.svc:5000/felis/felis:demo" \
|
||||||
|
"$(run_image '' '' 1 dev)"
|
||||||
|
expect "an unknown version keeps the old tag" "registry.felis.svc:5000/felis/felis:demo" \
|
||||||
|
"$(run_image '' '' '' '')"
|
||||||
|
expect "an explicit FELIS_IMAGE is used as given" "reg.example/felis:mine" \
|
||||||
|
"$(run_image reg.example/felis:mine v1.2.3 '' '')"
|
||||||
|
|
||||||
|
rsblock="$(awk '/^restart_existing_control_plane\(\) \{/,/^}/' "$BS")"
|
||||||
|
[ -n "$rsblock" ] || { echo "FAIL: no restart_existing_control_plane found in $BS"; exit 1; }
|
||||||
|
run_restart() { # prev-api prev-operator
|
||||||
|
FELIS_IMAGE=reg/felis/felis:v2 CONTROL_NS=felis bash -c '
|
||||||
|
set -Eeuo pipefail
|
||||||
|
log() { :; }
|
||||||
|
kube() { printf "KUBE %s\n" "$*"; }
|
||||||
|
'"$rsblock"'
|
||||||
|
restart_existing_control_plane "$1" "$2"
|
||||||
|
echo DONE' _ "$1" "$2"
|
||||||
|
}
|
||||||
|
|
||||||
|
out="$(run_restart reg/felis/felis:v1 reg/felis/felis:v1)"
|
||||||
|
case "$out" in
|
||||||
|
*"rollout restart"*) echo "FAIL an upgrade restarted the control plane on top of the apply's roll"; fails=$((fails + 1)) ;;
|
||||||
|
*DONE*) echo "PASS an upgrade leaves the roll to the apply" ;;
|
||||||
|
*) echo "FAIL restart_existing_control_plane died on an upgrade: $out"; fails=$((fails + 1)) ;;
|
||||||
|
esac
|
||||||
|
out="$(run_restart reg/felis/felis:v2 reg/felis/felis:v2)"
|
||||||
|
expect "a rerun of the same tag restarts felis-api onto the rebuilt image" "KUBE -n felis rollout restart deployment/felis-api" "$out"
|
||||||
|
expect "a rerun of the same tag restarts felis-operator too" "KUBE -n felis rollout restart deployment/felis-operator" "$out"
|
||||||
|
out="$(run_restart '' '')"
|
||||||
|
case "$out" in
|
||||||
|
*"rollout restart"*) echo "FAIL a first install restarted Deployments that did not exist"; fails=$((fails + 1)) ;;
|
||||||
|
*DONE*) echo "PASS a first install restarts nothing" ;;
|
||||||
|
*) echo "FAIL restart_existing_control_plane died on a first install: $out"; fails=$((fails + 1)) ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
mainblock="$(awk '/^main\(\) \{/,/^}/' "$BS")"
|
||||||
|
case "$mainblock" in
|
||||||
|
*"resolve_felis_image
|
||||||
|
build_image"*) echo "PASS the image is named before it is built" ;;
|
||||||
|
*) echo "FAIL main must call resolve_felis_image right before build_image"; fails=$((fails + 1)) ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------------------
|
||||||
if [ "$fails" -eq 0 ]; then
|
if [ "$fails" -eq 0 ]; then
|
||||||
|
|||||||
+20
-6
@@ -1330,17 +1330,31 @@ attestation. Check a downloaded binary with
|
|||||||
`gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`; it names the
|
`gh attestation verify felis-linux-amd64 --repo FelisMC/Felis`; it names the
|
||||||
workflow run and commit that built it.
|
workflow run and commit that built it.
|
||||||
|
|
||||||
Roll back with:
|
Each release runs under its own image tag, `felis/felis:v1.2.3` (a source
|
||||||
|
build's stamp `v1.2.3+gabc1234` becomes `v1.2.3-gabc1234`; a build that reports
|
||||||
|
no version uses `:demo`). At the end of an upgrade the installer prints the tag
|
||||||
|
it moved away from, and keeps it in `/etc/felis/previous-felis-image`. Roll
|
||||||
|
back with:
|
||||||
|
|
||||||
```
|
```
|
||||||
kubectl -n felis rollout undo deploy/felis-api
|
kubectl -n felis rollout undo deployment/felis-api deployment/felis-operator
|
||||||
kubectl -n felis rollout status deploy/felis-api
|
kubectl -n felis rollout status deploy/felis-api
|
||||||
```
|
```
|
||||||
|
|
||||||
(the same for `felis-operator` and `registry`). `rollout undo` returns to the
|
`rollout undo` returns each Deployment to its previous ReplicaSet, which names
|
||||||
previous ReplicaSet, whose image is normally still on the node; if the image GC
|
the previous release's tag. That image is normally still on the node; if the
|
||||||
collected it, the registry re-serves it automatically (§13b) for every tag the
|
image GC collected it, the registry re-serves it (§13b): the pruner keeps the
|
||||||
installer built — only hand-built tags need a manual re-mirror.
|
five newest `felis/felis` tags and every image a game pod still runs. A rerun
|
||||||
|
of the same version reuses its tag and restarts both Deployments onto the
|
||||||
|
rebuilt image, so after such a rerun undo lands on that same tag again. The
|
||||||
|
next installer run re-applies the bundle and moves the image to whatever that
|
||||||
|
run installs.
|
||||||
|
|
||||||
|
A platform upgrade leaves running game servers alone. Their init containers
|
||||||
|
run the felis image too; a running server keeps the one it started with and
|
||||||
|
picks up the new release on its next start. A release that changes the game
|
||||||
|
pod in any other way still restarts running servers once, as a server edit
|
||||||
|
does.
|
||||||
|
|
||||||
`rollout undo` reverts the image only. The upgrade's database migrations stay
|
`rollout undo` reverts the image only. The upgrade's database migrations stay
|
||||||
applied; when they are the problem, restore the `pre-migrate` bundle the upgrade
|
applied; when they are the problem, restore the `pre-migrate` bundle the upgrade
|
||||||
|
|||||||
Reference in new issue
Block a user