From 3e61fde06d8be9914785ca264acb8ec8a6c80997 Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Fri, 25 Sep 2026 00:43:02 +0800 Subject: [PATCH] =?UTF-8?q?fix(bootstrap):=20=E5=85=B3=E9=97=AD=20BuildKit?= =?UTF-8?q?=20=E9=BB=98=E8=AE=A4=20attestation=EF=BC=8C=E6=97=A0=E6=94=B9?= =?UTF-8?q?=E5=8A=A8=E9=87=8D=E8=B7=91=E4=B8=8D=E5=86=8D=E9=87=8D=E5=90=AF?= =?UTF-8?q?=20login/lobby=EF=BC=9B=E5=90=8C=E7=89=88=E6=9C=AC=E9=87=8D?= =?UTF-8?q?=E8=B7=91=E5=90=8C=E6=97=B6=E9=87=8D=E5=90=AF=20registry=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/bootstrap.sh | 21 ++++++++++++++++++--- deploy/bootstrap_test.sh | 21 ++++++++++++++++----- docs/troubleshooting.md | 4 ++-- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index b94d9ad..87cdcc7 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -400,6 +400,13 @@ ZYPPER_BACKGROUND_SERVICES=( PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export PATH +# BuildKit attaches a provenance attestation to every build by default, and it records the +# build's start time. On Docker's containerd image store the image id is the digest of the +# index that carries it, so an unchanged rebuild would get a new id each run: the login and +# lobby pods would restart on every rerun, and each run would push another versioned tag. +# Without it the id is the manifest digest, which an all-cached build reproduces. +export BUILDX_NO_DEFAULT_ATTESTATIONS=1 + # --------------------------------------------------------------------------- # Logging # --------------------------------------------------------------------------- @@ -3254,12 +3261,13 @@ EOF } deploy_bundle() { - local prev_api prev_operator + local prev_api prev_operator prev_gate export KUBECONFIG=/etc/rancher/k3s/k3s.yaml write_felis_toml "${STATE_DIR}/felis.pod.toml" "${NODE_IP}" prev_api="$(deployment_image felis-api api)" prev_operator="$(deployment_image felis-operator operator)" + prev_gate="$(deployment_image registry registry-gate)" if [ -n "$prev_api" ] && [ "$prev_api" != "$FELIS_IMAGE" ]; then PREVIOUS_FELIS_IMAGE="$prev_api" printf '%s\n' "$prev_api" > "${STATE_DIR}/previous-felis-image" @@ -3336,7 +3344,7 @@ deploy_bundle() { if [ -n "$size" ]; then manifest_args+=(--backup-storage "$size"); fi fi "$HOST_BIN" manifests "${manifest_args[@]}" | kube apply -f - - restart_existing_control_plane "$prev_api" "$prev_operator" + restart_existing_control_plane "$prev_api" "$prev_operator" "$prev_gate" log "waiting for control-plane rollouts" local d @@ -3378,8 +3386,11 @@ deployment_image() { # version, or a FELIS_IMAGE the operator reuses). A Deployment whose image changed is rolling # from the apply already, and must not be restarted on top: the restart is a second template # change, so `rollout undo` would step back to the new image instead of the previous release. +# +# The registry pod runs the same binary in its registry-gate and registry-gc containers, so it +# follows the same rule; the rollout wait below covers it before anything is pushed. restart_existing_control_plane() { - local prev_api="$1" prev_operator="$2" + local prev_api="$1" prev_operator="$2" prev_gate="${3:-}" # `if`, not `[ test ] && cmd`: as the LAST command of the function the and-list returns 1 # when the test is false, which becomes the function's exit status and kills the whole # install under `set -Eeuo pipefail` — right after the bundle is applied and before the @@ -3392,6 +3403,10 @@ restart_existing_control_plane() { log "restarting felis-operator onto the rebuilt ${FELIS_IMAGE}" kube -n "$CONTROL_NS" rollout restart deployment/felis-operator fi + if [ "$prev_gate" = "$FELIS_IMAGE" ]; then + log "restarting the registry's gate onto the rebuilt ${FELIS_IMAGE}" + kube -n "$CONTROL_NS" rollout restart deployment/registry + fi } # push_image_to_registry re-tags a locally built image for the node's diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index 4b6602f..c54ce3e 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -1670,25 +1670,26 @@ expect "an explicit FELIS_IMAGE is used as given" "reg.example/felis:mine" \ rsblock="$(awk '/^restart_existing_control_plane\(\) \{/,/^}/' "$BS")" [ -n "$rsblock" ] || { echo "FAIL: no restart_existing_control_plane found in $BS"; exit 1; } -run_restart() { # prev-api prev-operator +run_restart() { # prev-api prev-operator [prev-gate] FELIS_IMAGE=reg/felis/felis:v2 CONTROL_NS=felis bash -c ' set -Eeuo pipefail log() { :; } kube() { printf "KUBE %s\n" "$*"; } '"$rsblock"' - restart_existing_control_plane "$1" "$2" - echo DONE' _ "$1" "$2" + restart_existing_control_plane "$1" "$2" "$3" + echo DONE' _ "$1" "$2" "${3:-}" } -out="$(run_restart reg/felis/felis:v1 reg/felis/felis:v1)" +out="$(run_restart reg/felis/felis:v1 reg/felis/felis:v1 reg/felis/felis:v1)" case "$out" in *"rollout restart"*) echo "FAIL an upgrade restarted the control plane on top of the apply's roll"; fails=$((fails + 1)) ;; *DONE*) echo "PASS an upgrade leaves the roll to the apply" ;; *) echo "FAIL restart_existing_control_plane died on an upgrade: $out"; fails=$((fails + 1)) ;; esac -out="$(run_restart reg/felis/felis:v2 reg/felis/felis:v2)" +out="$(run_restart reg/felis/felis:v2 reg/felis/felis:v2 reg/felis/felis:v2)" expect "a rerun of the same tag restarts felis-api onto the rebuilt image" "KUBE -n felis rollout restart deployment/felis-api" "$out" expect "a rerun of the same tag restarts felis-operator too" "KUBE -n felis rollout restart deployment/felis-operator" "$out" +expect "a rerun of the same tag restarts the registry's gate too" "KUBE -n felis rollout restart deployment/registry" "$out" out="$(run_restart '' '')" case "$out" in *"rollout restart"*) echo "FAIL a first install restarted Deployments that did not exist"; fails=$((fails + 1)) ;; @@ -1902,6 +1903,16 @@ expect "a v6 node address gets a v6 rule" "tcp dport 5432 ip6 saddr 2001:db8::7 rm -rf "$fwdir" +# --- reproducible image ids --------------------------------------------------------------- +# restart_existing_system_servers compares image ids across runs; a default BuildKit +# provenance attestation (it carries a timestamp) would make every rebuild look new. +attest_line="$(grep -n '^export BUILDX_NO_DEFAULT_ATTESTATIONS=1$' "$BS" | cut -d: -f1 | head -1)" +build_line="$(grep -n '^ docker build ' "$BS" | cut -d: -f1 | head -1)" +if [ -n "$attest_line" ] && [ -n "$build_line" ] && [ "$attest_line" -lt "$build_line" ]; then + echo "PASS default build attestations are off before the first docker build" +else + echo "FAIL BUILDX_NO_DEFAULT_ATTESTATIONS=1 must be exported before the first docker build"; fails=$((fails + 1)) +fi # --------------------------------------------------------------------------------------- diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 8bf867c..d7414cc 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1366,9 +1366,9 @@ runs changed: | Component | Restarted when | |---|---| | `felis-velocity` (the proxy) | its unit, the JRE, `velocity.jar`, `velocity.toml`, the forwarding secret, the felis-link settings or a plugin jar changed, or it was not running. The fingerprint lives in `/etc/felis/velocity.fingerprint`; delete it to force a restart. | -| login and lobby pods | the rebuilt limbo or lobby image has a new image ID (`/etc/felis/system-server-images`). Each restarts on its own. | +| login and lobby pods | the rebuilt limbo or lobby image has a new image ID (`/etc/felis/system-server-images`). Each restarts on its own. The installer turns off BuildKit's default provenance attestation (`BUILDX_NO_DEFAULT_ATTESTATIONS=1`): it records the build time, which would give every rebuild a new ID. | | PostgreSQL | first install only (`listen_addresses` needs a restart). A rerun reloads the configuration, which keeps connections open. | -| felis-api, felis-operator | the image tag changed (an upgrade), or a same-version rerun rebuilt it. | +| felis-api, felis-operator, the registry pod (its gate and GC containers run the felis binary) | the image tag changed (an upgrade), or a same-version rerun rebuilt it. | **PostgreSQL across reruns.** On hosts without firewalld the installer loads an nftables table, `inet felis_postgres`, from `felis-postgres-firewall.service`: