diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 735388b..169d8f5 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -527,6 +527,12 @@ K3S_IMAGES_DIR="/var/lib/rancher/k3s/agent/images" # stores the journal persistently. JOURNALD_DROPIN="/etc/systemd/journald.conf.d/50-felis.conf" JOURNAL_DIR="/var/log/journal" +# dpkg's journal of a run in progress: not empty after a dpkg run was cut off +# (finish_interrupted_dpkg). +DPKG_UPDATES_DIR="/var/lib/dpkg/updates" +# Held for the whole run (acquire_run_lock), so a second installer started while one is +# still going stops at once instead of working the same files and cluster beside it. +RUN_LOCK_FILE="/run/felis-bootstrap.lock" APT_LOCK_FILES=( /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock @@ -658,6 +664,19 @@ trap 'on_error "$LINENO" "$?"' ERR trap cleanup EXIT k3s_cmd() { [ -x "$K3S_BIN" ] || die "k3s binary not found at ${K3S_BIN}"; "$K3S_BIN" "$@"; } + +# acquire_run_lock takes RUN_LOCK_FILE for the rest of the run. A rerun started while an +# earlier one still runs (in tmux after the SSH session dropped, in a second terminal) +# would otherwise install the same packages, rewrite the same files and apply the same +# cluster objects beside it. The lock goes with the process, however it ends. +acquire_run_lock() { + if ! command -v flock >/dev/null 2>&1; then + warn "flock not found; nothing stops a second installer run beside this one" + return 0 + fi + exec 9>"$RUN_LOCK_FILE" + flock -n 9 || die "another installer run is still going on this host; wait for it to finish, then rerun (ps -ef | grep bootstrap)" +} kube() { k3s_cmd kubectl "$@"; } # Create or update a single-key Secret without putting the value in kubectl's argv. @@ -1566,10 +1585,23 @@ pkg_install() { esac } +# finish_interrupted_dpkg completes a dpkg run that was cut off: an earlier install killed +# halfway through a package, a reboot during unattended-upgrades. Until then apt-get refuses +# everything with "dpkg was interrupted, you must manually run 'dpkg --configure -a'", so +# each rerun failed exactly as the run before it. +finish_interrupted_dpkg() { + [ -n "$(ls -A "$DPKG_UPDATES_DIR" 2>/dev/null)" ] || [ -n "$(dpkg --audit 2>/dev/null)" ] || return 0 + warn "an earlier dpkg run was cut off; finishing it (dpkg --configure -a)" + wait_for_pkg_locks + NEEDRESTART_SUSPEND=1 DEBIAN_FRONTEND=noninteractive \ + dpkg --force-confdef --force-confold --configure -a \ + || die "dpkg --configure -a failed; fix the package it names, then rerun the installer" +} + pkg_refresh_once() { [ -n "${_PKG_REFRESHED:-}" ] && return 0 case "$PKG" in - apt) apt_get update -y ;; + apt) finish_interrupted_dpkg; apt_get update -y ;; dnf|yum) : ;; # dnf/yum refresh metadata on demand zypper) wait_for_pkg_locks; zypper --non-interactive refresh ;; # Arch supports only whole-system upgrades (-Sy alone leaves a partial upgrade), so the @@ -1968,7 +2000,17 @@ install_k3s() { write_k3s_service_dropin local installer_ran=0 - if [ -x "$K3S_BIN" ]; then + if [ -x "$K3S_BIN" ] && [ ! -f "$K3S_UNIT_FILE" ]; then + # k3s's installer moves the binary into place before it writes k3s.service, so a run + # cut short between the two left a k3s nothing starts, and every rerun stopped at + # `systemctl enable`. Its installer run again around the binary in place finishes the + # job: no download, no version change, the cluster's data untouched. + "$K3S_BIN" --version >/dev/null 2>&1 \ + || die "${K3S_BIN} does not run and k3s.service is missing: an earlier k3s install was cut short. Remove ${K3S_BIN} and rerun to install k3s ${FELIS_K3S_VERSION}" + log "k3s.service is missing beside ${K3S_BIN}: an earlier k3s install was cut short; finishing it around the binary in place" + run_k3s_installer --keep-binary + installer_ran=1 + elif [ -x "$K3S_BIN" ]; then local current current="$("$K3S_BIN" --version 2>/dev/null | awk 'NR == 1 { print $3 }')" if [ "$current" = "$FELIS_K3S_VERSION" ]; then @@ -2135,8 +2177,13 @@ strip_k3s_kubeconfig_mode_flag() { # The script from the release's own tag rather than get.k3s.io, which serves whatever # master holds today. '+' is literal in a URL path, so the tag needs no escaping. On an # installed k3s the same script replaces the binary in place and restarts the service. -run_k3s_installer() { +# --keep-binary keeps the k3s already at K3S_BIN (INSTALL_K3S_SKIP_DOWNLOAD=binary) and +# still does everything after it, the SELinux policy included. +run_k3s_installer() { # [--keep-binary] + local skip="" + [ "${1:-}" != --keep-binary ] || skip=binary curl -sfL --retry 5 --retry-delay 2 "https://raw.githubusercontent.com/k3s-io/k3s/${FELIS_K3S_VERSION}/install.sh" | \ + INSTALL_K3S_SKIP_DOWNLOAD="$skip" \ INSTALL_K3S_VERSION="$FELIS_K3S_VERSION" \ INSTALL_K3S_BIN_DIR="$K3S_BIN_DIR" \ INSTALL_K3S_EXEC="--disable traefik --disable servicelb --disable metrics-server" \ @@ -2892,6 +2939,9 @@ fetch_source() { # whole: git clone refuses a non-empty destination, which stopped the rerun outright. log "cloning ${FELIS_REPO_URL} (${FELIS_REF})" mkdir -p "$(dirname "$SRC_DIR")" + # A run killed outright (no EXIT trap) leaves its half-made checkout; the run lock + # means none of these belongs to a run still going. + rm -rf -- "${SRC_DIR}".new.* work="$(mktemp -d "${SRC_DIR}.new.XXXXXX")" chmod 755 "$work" if ! { git -C "$work" init -q && git -C "$work" remote add origin "$FELIS_REPO_URL" \ @@ -5798,6 +5848,9 @@ summary_next() { start_setup_console() { [ "$SETUP_CONSOLE" = 1 ] || return 0 resume_package_background_timers + # The install is done, and felis setup can start the installer itself (its host + # bootstrap step): an inherited run lock would stop that run as a second one. + exec 9>&- "$HOST_BIN" setup <"$SETUP_TTY" \ || warn "the setup console exited with status $?; run 'sudo felis setup' to come back to it" } @@ -6222,6 +6275,7 @@ EOF_MIRROR } main() { + acquire_run_lock ensure_k3s_on_path resolve_nano_listen validate_settings diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index d1510ea..93ce436 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -1237,14 +1237,25 @@ kblock="$(awk '/^run_k3s_installer\(\) \{/,/^}/' "$BS")" expect "k3s's install script comes from the pinned tag" 'raw.githubusercontent.com/k3s-io/k3s/${FELIS_K3S_VERSION}/install.sh' "$kblock" expect "k3s's install script is told the pinned version" 'INSTALL_K3S_VERSION="$FELIS_K3S_VERSION"' "$kblock" case "$kblock" in *"https://get.k3s.io"*) echo "FAIL: get.k3s.io serves master's script; read it from the pinned tag"; fails=$((fails + 1)) ;; esac +run_kinst() { # [--keep-binary] + bash -c ' + curl() { :; } + sh() { printf "SKIP=[%s] VERSION=%s\n" "$INSTALL_K3S_SKIP_DOWNLOAD" "$INSTALL_K3S_VERSION"; } + FELIS_K3S_VERSION=v1.36.4+k3s1 K3S_BIN_DIR=/usr/local/bin + '"$kblock"' + run_k3s_installer "$@"' _ "$@" +} +expect "finishing a cut-short k3s install keeps the binary in place" "SKIP=[binary] VERSION=v1.36.4+k3s1" "$(run_kinst --keep-binary)" +expect "an install or an upgrade downloads k3s" "SKIP=[] VERSION=v1.36.4+k3s1" "$(run_kinst)" # An installed k3s moves only under FELIS_UPGRADE_DEPS=1, one minor version at a time and # never backwards; the refusal names the release to go through first. kfake="$sdir/k3s" +: > "$sdir/k3s.service" run_k3s() { # installed-version pinned-version [FELIS_UPGRADE_DEPS] printf '#!/bin/sh\necho "k3s version %s (0123abcd)"\necho "go version go1.26"\n' "$1" > "$kfake" chmod +x "$kfake" - K3S_BIN="$kfake" FELIS_K3S_VERSION="$2" FELIS_UPGRADE_DEPS="${3:-0}" bash -c ' + K3S_BIN="$kfake" K3S_UNIT_FILE="$sdir/k3s.service" FELIS_K3S_VERSION="$2" FELIS_UPGRADE_DEPS="${3:-0}" bash -c ' die() { printf "DIE: %s\n" "$*"; exit 1; } log() { printf "LOG: %s\n" "$*"; } ok() { printf "OK: %s\n" "$*"; } @@ -1383,8 +1394,13 @@ case "$out" in esac expect "a rerun over a shallow checkout takes an abbreviated commit id" "AT ${first} one" \ "$(run_real_fetch "$gdir/src" "$first")" +# A run killed outright (no EXIT trap) leaves its half-made checkout beside SRC_DIR. +mkdir -p "$gdir/fresh.new.Ab12Cd" +echo killed >"$gdir/fresh.new.Ab12Cd/a.txt" expect "a fresh clone takes an abbreviated commit id" "AT ${first} one" \ "$(run_real_fetch "$gdir/fresh" "$first")" +[ ! -e "$gdir/fresh.new.Ab12Cd" ] && echo "PASS and clears the half-made checkout a killed run left" \ + || { echo "FAIL the half-made checkout a killed run left is still there"; fails=$((fails + 1)); } # A full clone left a local main behind origin's; with the shallow fetch failing, the whole # history arm must still land on origin's main. git clone -q "file://$gdir/up" "$gdir/stale" @@ -2537,6 +2553,7 @@ printf 'keys from the terminal\n' > "$next_dir/tty" cat > "$next_dir/felis" <<'XEOF' #!/bin/sh echo "felis $*" >> "$NEXT_JOURNAL" +if (: >&9) 2>/dev/null; then echo "fd 9 open" >> "$NEXT_JOURNAL"; fi cat > "$NEXT_STDIN" exit "${SETUP_EXIT:-0}" XEOF @@ -2547,10 +2564,11 @@ chmod +x "$next_dir/felis" run_next() { : > "$next_dir/journal" : > "$next_dir/stdin" - PG_ANSWER="$1" NEXT_JOURNAL="$next_dir/journal" NEXT_STDIN="$next_dir/stdin" \ + PG_ANSWER="$1" NEXT_JOURNAL="$next_dir/journal" NEXT_STDIN="$next_dir/stdin" NEXT_LOCK="$next_dir/lock" \ HOST_BIN="$next_dir/felis" SETUP_TTY="$next_dir/tty" bash -c ' set -Eeuo pipefail trap "echo TRAP" ERR + exec 9>"$NEXT_LOCK" # the run lock acquire_run_lock holds log() { printf "LOG: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; } auth_hostname() { printf "%s" "$2"; } pg_exec() { echo "pg_exec $*" >> "$NEXT_JOURNAL"; [ "$PG_ANSWER" != fail ] || return 1; echo "$PG_ANSWER"; } @@ -2578,6 +2596,12 @@ expect "with no Owner on a terminal the summary says the setup console starts" " expect " and how it binds the Owner" "has you join 10.0.0.5:25565 in Minecraft" "$out" if started; then echo "PASS and starts it"; else echo "FAIL the setup console did not start: $out"; fails=$((fails + 1)); fi expect " on the terminal's keys" "keys from the terminal" "$(cat "$next_dir/stdin")" +# felis setup can run the installer itself (its host bootstrap step), which would take an +# inherited run lock for a second installer run. +case "$(cat "$next_dir/journal")" in + *"fd 9 open"*) echo "FAIL the setup console must not inherit the installer's run lock"; fails=$((fails + 1)) ;; + *) echo "PASS without the installer's run lock" ;; +esac case "$(cat "$next_dir/journal")" in *"systemctl start apt-daily.timer"*"felis setup"*) echo "PASS after the package timers are back" ;; *) echo "FAIL the package timers must restart before the setup console: $(cat "$next_dir/journal")"; fails=$((fails + 1)) ;; @@ -3190,6 +3214,96 @@ esac printf '16\n' > "$pmdir/PG_VERSION" expect "an Arch rerun holds PostgreSQL at the cluster's version" "PACMAN -Syu --noconfirm --ignore postgresql" "$(run_refresh)" rm -rf "$pmdir" +expect "an apt refresh first finishes a dpkg run that was cut off" "FINISH +APT_GET update -y" "$(PKG=apt bash -c ' + finish_interrupted_dpkg() { echo FINISH; } + apt_get() { echo "APT_GET $*"; } + '"$rfblock"' + pkg_refresh_once')" + +# An install killed halfway through a package leaves dpkg's journal behind, and apt-get then +# refuses everything until `dpkg --configure -a` runs: the rerun runs it. +fdblock="$(awk '/^finish_interrupted_dpkg\(\) \{/,/^}/' "$BS")" +[ -n "$fdblock" ] || { echo "FAIL: no finish_interrupted_dpkg found in $BS"; exit 1; } +[ "$(printf '%s\n' "$fdblock" | wc -l)" -lt 15 ] \ + || { echo "FAIL: the extracted block is not finish_interrupted_dpkg -- did its closing brace move?"; exit 1; } +dpdir="$(mktemp -d)" +run_finish_dpkg() { # $1: what dpkg --audit prints, $2: dpkg --configure's exit status + AUDIT="$1" DPKG_RC="${2:-0}" DPKG_UPDATES_DIR="$dpdir/updates" bash -c ' + set -Eeuo pipefail + warn() { printf "WARN: %s\n" "$*"; }; die() { printf "DIE: %s\n" "$*"; exit 1; } + wait_for_pkg_locks() { echo LOCKS; } + dpkg() { + if [ "$1" = --audit ]; then printf "%s" "$AUDIT"; return 0; fi + printf "DPKG: %s (frontend=%s)\n" "$*" "${DEBIAN_FRONTEND:-}" + return "$DPKG_RC" + } + '"$fdblock"' + finish_interrupted_dpkg + echo DONE' 2>&1 +} +out="$(run_finish_dpkg "")" +expect "a host without dpkg's journal carries on" "DONE" "$out" +case "$out" in *DPKG:*) echo "FAIL a host without dpkg's journal must not run dpkg --configure: $out"; fails=$((fails + 1)) ;; *) echo "PASS without running dpkg --configure" ;; esac +mkdir -p "$dpdir/updates" +case "$(run_finish_dpkg "")" in *DPKG:*) echo "FAIL an empty dpkg journal must not run dpkg --configure"; fails=$((fails + 1)) ;; *) echo "PASS an empty dpkg journal runs nothing" ;; esac +: > "$dpdir/updates/0007" +out="$(run_finish_dpkg "")" +expect "a dpkg run cut off mid-package is finished first" "LOCKS +DPKG: --force-confdef --force-confold --configure -a (frontend=noninteractive)" "$out" +expect " and the install carries on" "DONE" "$out" +out="$(run_finish_dpkg "" 1)" +expect "a dpkg run that cannot be finished stops the install" "DIE: dpkg --configure -a failed" "$out" +case "$out" in *DONE*) echo "FAIL a dpkg run that cannot be finished must stop the install"; fails=$((fails + 1)) ;; esac +rm -f "$dpdir/updates/0007" +expect "a package dpkg --audit names as half configured is finished too" "DPKG: --force-confdef --force-confold --configure -a" \ + "$(run_finish_dpkg "The following packages are only half configured")" +rm -rf "$dpdir" + +# A second installer started while one still runs (in tmux after the SSH session dropped) +# stops at once. +alblock="$(awk '/^acquire_run_lock\(\) \{/,/^}/' "$BS")" +[ -n "$alblock" ] || { echo "FAIL: no acquire_run_lock found in $BS"; exit 1; } +[ "$(printf '%s\n' "$alblock" | wc -l)" -lt 15 ] \ + || { echo "FAIL: the extracted block is not acquire_run_lock -- did its closing brace move?"; exit 1; } +case "$(awk '/^main\(\) \{/,/^}/' "$BS" | sed -n 2p)" in + " acquire_run_lock") echo "PASS the installer takes its run lock before anything else" ;; + *) echo "FAIL main must start with acquire_run_lock"; fails=$((fails + 1)) ;; +esac +lkdir="$(mktemp -d)" +run_lock() { # $1: flock's exit status ("" = no flock on the host) + FLOCK_RC="$1" RUN_LOCK_FILE="$lkdir/run.lock" NOPATH="$lkdir/nopath" bash -c ' + set -Eeuo pipefail + warn() { printf "WARN: %s\n" "$*"; }; die() { printf "DIE: %s\n" "$*"; exit 1; } + if [ -n "$FLOCK_RC" ]; then + flock() { printf "FLOCK: %s\n" "$*"; if (: >&9) 2>/dev/null; then echo "FD9 OPEN"; fi; return "$FLOCK_RC"; } + else + PATH="$NOPATH" + fi + '"$alblock"' + acquire_run_lock + echo CONTINUES' 2>&1 +} +out="$(run_lock 0)" +expect "the installer takes its run lock on the lock file" "FLOCK: -n 9 +FD9 OPEN +CONTINUES" "$out" +[ -e "$lkdir/run.lock" ] && echo "PASS which it creates" || { echo "FAIL the run lock file was not created"; fails=$((fails + 1)); } +out="$(run_lock 1)" +expect "a run lock held by another run stops the install" "DIE: another installer run is still going on this host" "$out" +case "$out" in *CONTINUES*) echo "FAIL a second installer run must stop: $out"; fails=$((fails + 1)) ;; esac +expect "a host without flock warns and carries on" "WARN: flock not found" "$(run_lock "")" +if command -v flock >/dev/null 2>&1; then + out="$(flock "$lkdir/held.lock" env RUN_LOCK_FILE="$lkdir/held.lock" bash -c ' + die() { printf "DIE: %s\n" "$*"; exit 1; }; warn() { :; } + '"$alblock"' + acquire_run_lock + echo CONTINUES' 2>&1)" + expect "a lock another process holds stops the install (real flock)" "DIE: another installer run" "$out" +else + echo "SKIP a lock another process holds (real flock): flock is not installed here" +fi +rm -rf "$lkdir" @@ -3532,21 +3646,29 @@ iblock="$(awk '/^install_k3s\(\) \{/,/^}/' "$BS")" iblock="$iblock $(awk '/^version_newer\(\) \{/,/^}/' "$BS") $(awk '/^k3s_upgrade_allowed\(\) \{/,/^}/' "$BS")" -run_install_k3s() { # $1: installed version ("" = none), $2: config drop-in changed (0|1), $3: FELIS_UPGRADE_DEPS, $4: service drop-in changed (0|1) - INSTALLED="$1" CHANGED="$2" UPGRADE="${3:-0}" SVC_CHANGED="${4:-0}" KDIR="$kdir" bash -c ' +run_install_k3s() { # $1: installed version ("" = none, "broken" = a binary that does not run), $2: config drop-in changed (0|1), $3: FELIS_UPGRADE_DEPS, $4: service drop-in changed (0|1), $5: k3s.service present (1|0) + INSTALLED="$1" CHANGED="$2" UPGRADE="${3:-0}" SVC_CHANGED="${4:-0}" UNIT="${5:-1}" KDIR="$kdir" bash -c ' set -Eeuo pipefail ok() { echo "OK: $*"; }; log() { echo "LOG: $*"; }; warn() { echo "WARN: $*"; } die() { echo "DIE: $*"; exit 1; } FELIS_K3S_VERSION=v1.36.4+k3s1 FELIS_UPGRADE_DEPS="$UPGRADE" K3S_BIN_DIR="$KDIR" K3S_BIN="$KDIR/k3s-under-test" K3S_CONFIG_DROPIN=/etc/rancher/k3s/config.yaml.d/50-felis.yaml K3S_KUBECONFIG=/etc/rancher/k3s/k3s.yaml - K3S_SERVICE_DROPIN=/etc/systemd/system/k3s.service.d/50-felis.conf - rm -f "$K3S_BIN" - if [ -n "$INSTALLED" ]; then printf "#!/bin/sh\necho \"k3s version %s (abc)\"\n" "$INSTALLED" > "$K3S_BIN"; chmod +x "$K3S_BIN"; fi + K3S_SERVICE_DROPIN=/etc/systemd/system/k3s.service.d/50-felis.conf K3S_UNIT_FILE="$KDIR/k3s.service-under-test" + rm -f "$K3S_BIN" "$K3S_UNIT_FILE" + [ "$UNIT" = 0 ] || : > "$K3S_UNIT_FILE" + case "$INSTALLED" in + "") ;; + broken) printf "#!/bin/sh\nexit 1\n" > "$K3S_BIN"; chmod +x "$K3S_BIN" ;; + *) printf "#!/bin/sh\necho \"k3s version %s (abc)\"\n" "$INSTALLED" > "$K3S_BIN"; chmod +x "$K3S_BIN" ;; + esac configure_k3s_firewall() { :; } write_k3s_config() { [ "$CHANGED" = 0 ] || K3S_RESTART_NEEDED=1; } write_k3s_service_dropin() { echo "SERVICE-DROPIN"; [ "$SVC_CHANGED" = 0 ] || K3S_RESTART_NEEDED=1; } strip_k3s_kubeconfig_mode_flag() { :; } - run_k3s_installer() { echo "INSTALLER"; printf "#!/bin/sh\n" > "$K3S_BIN"; command chmod +x "$K3S_BIN"; } + run_k3s_installer() { + echo "INSTALLER${1:+ $1}" + [ "${1:-}" = --keep-binary ] || { printf "#!/bin/sh\n" > "$K3S_BIN"; command chmod +x "$K3S_BIN"; } + } stage_k3s_airgap_images() { echo "STAGE"; } systemctl() { echo "SYSTEMCTL: $*"; } wait_for_node_ready() { echo "READY"; } @@ -3573,6 +3695,18 @@ case "$out" in *"restart k3s"*) echo "FAIL the k3s installer already started k3s out="$(run_install_k3s v1.35.2+k3s1 1 1)" expect "an upgrade runs the k3s installer" "INSTALLER" "$out" case "$out" in *"restart k3s"*) echo "FAIL the upgrade already restarted k3s on the new settings; no second restart"; fails=$((fails + 1)) ;; *) echo "PASS an upgraded k3s is not restarted a second time" ;; esac +# k3s's installer puts the binary in place before it writes k3s.service: a run cut short +# between the two is finished around that binary, at its own version. +out="$(run_install_k3s v1.36.4+k3s1 1 0 0 0)" +expect "a k3s binary without k3s.service has its installer finish around it" "INSTALLER --keep-binary +SYSTEMCTL: enable --now k3s" "$out" +case "$out" in *"restart k3s"*) echo "FAIL the finishing installer already started k3s; no second restart"; fails=$((fails + 1)) ;; *) echo "PASS and k3s is not restarted a second time" ;; esac +expect "an older binary without k3s.service is finished at its own version" "INSTALLER --keep-binary" \ + "$(run_install_k3s v1.35.2+k3s1 0 1 0 0)" +out="$(run_install_k3s broken 0 0 0 0)" +expect "a k3s binary that does not run, without k3s.service, stops the install" \ + "does not run and k3s.service is missing: an earlier k3s install was cut short" "$out" +case "$out" in *INSTALLER*) echo "FAIL a k3s binary that does not run must not be installed around"; fails=$((fails + 1)) ;; *) echo "PASS and the installer is not run around it" ;; esac rm -rf "$kdir" jblock="$(awk '/^ensure_persistent_journal\(\) \{/,/^}/' "$BS")"