From 9d0253a75b77f3b59f87b96732cc98b7e9f89c4e Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Sun, 27 Sep 2026 09:58:25 +0800 Subject: [PATCH] =?UTF-8?q?fix(bootstrap):=20release=20=E9=99=84=E4=BB=B6?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E9=87=8D=E8=AF=95=E4=B8=89=E6=AC=A1=E5=B9=B6?= =?UTF-8?q?=E6=A3=80=E6=B5=8B=E5=8D=A1=E6=AD=BB=EF=BC=8C=E5=9B=9E=E9=80=80?= =?UTF-8?q?=E6=9C=AC=E6=9C=BA=E6=9E=84=E5=BB=BA=E5=89=8D=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E7=A3=81=E7=9B=98=EF=BC=8C=E6=89=80=E6=9C=89=20curl=20?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=B8=A6=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/bootstrap.sh | 70 ++++++++++++++----- deploy/bootstrap_test.sh | 141 +++++++++++++++++++++++++++++++++++++++ docs/operations.md | 4 +- docs/troubleshooting.md | 11 ++- 4 files changed, 206 insertions(+), 20 deletions(-) diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index b2601bb..e1056b3 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -1730,7 +1730,7 @@ install_docker_apt() { log "installing docker apt repository" install -m 0755 -d /etc/apt/keyrings - curl -fsSL "https://download.docker.com/linux/${repo_os}/gpg" -o "$keyring" \ + curl -fsSL --retry 5 --retry-delay 2 "https://download.docker.com/linux/${repo_os}/gpg" -o "$keyring" \ || die "failed to download Docker GPG key for ${repo_os}" chmod a+r "$keyring" @@ -1774,7 +1774,7 @@ install_docker_rpm() { log "installing docker rpm repository" mkdir -p "$(dirname "$repo_file")" - curl -fsSL "$repo_url" -o "$repo_file" \ + curl -fsSL --retry 5 --retry-delay 2 "$repo_url" -o "$repo_file" \ || die "failed to download Docker repo file: ${repo_url}" pkg_install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin } @@ -1810,6 +1810,7 @@ install_docker() { # is built here, so an install from a release's assets never installs Docker at all. ensure_docker() { if [ -z "$DOCKER_INSTALLED" ]; then + host_builds_expected || unplanned_build_room install_docker DOCKER_INSTALLED=1 else @@ -1817,6 +1818,26 @@ ensure_docker() { fi } +# unplanned_build_room runs before the first build of a run whose preflight counted on none: +# a release asset that could not be downloaded or checked (the warnings before it name +# which) is built here instead, with Docker's images and build cache under +# /var/lib/containerd. When that filesystem lacks preflight_disk's room for them the install +# stops here, before Docker is installed, instead of filling the disk halfway through a +# build; FELIS_PREFLIGHT=warn goes on, as it does past preflight's own problems. +unplanned_build_room() { + local path=/var/lib/containerd need=8192 line mount avail problem + path_populated "$path" && need=2048 + line="$(df -Pk "$(existing_ancestor "$path")" 2>/dev/null | awk 'NR == 2 { print $6, $4 }')" || return 0 + read -r mount avail <<<"$line" + [ -n "$avail" ] && [ "$avail" -lt $((need * 1024)) ] || return 0 + problem="building here what the release did not supply takes about ${need} MiB under ${path}, and ${mount} has $((avail / 1024)) MiB free" + if [ "$FELIS_PREFLIGHT" = warn ]; then + warn "${problem}; FELIS_PREFLIGHT=warn, building anyway" + return 0 + fi + die "${problem}; nothing has been built. Rerun the installer once the release's assets download, free space on ${mount}, or set FELIS_PREFLIGHT=warn to build anyway" +} + # stop_docker hands back the ~150 MiB the docker daemon holds once a step is done with it; the # next step that builds starts it again. A Docker this run never started is left alone. stop_docker() { @@ -2319,28 +2340,41 @@ github_asset_id() { # # -f is not cosmetic: without it curl writes GitHub's error JSON into the output file and # still exits 0. +# +# curl's --retry covers transient HTTP statuses and timeouts, and --speed-limit makes a +# transfer stalled under 1 KiB/s for a minute one of those. A connection reset mid-stream +# is outside its retry set, and on an image bundle of several hundred MiB it is the likely +# failure, so the outer loop tries the whole download twice more (meta_get's reasoning): +# what the caller falls back to is a build on this host that preflight may not have +# counted on (ensure_docker checks the room for it). download_release_asset() { - local tag="$1" name="$2" dest="$3" id ua url rc=0 + local tag="$1" name="$2" dest="$3" id ua url rc attempt ua="felis-bootstrap (+${FELIS_REPO_URL})" # Loaded here, in this shell, so the lookup's subshell below finds it cached. load_release_json "$tag" || return 1 id="$(github_asset_id "$tag" "$name")" || return 1 url="https://api.github.com/repos/$(repo_slug)/releases/assets/${id}" log "downloading ${name} from release ${tag}" - if [ -n "$FELIS_GITHUB_TOKEN" ]; then - printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \ - | curl -fsSL --retry 5 --retry-delay 2 --config - \ - -A "$ua" -H "Accept: application/octet-stream" -o "$dest" "$url" || rc=$? - else - curl -fsSL --retry 5 --retry-delay 2 \ - -A "$ua" -H "Accept: application/octet-stream" -o "$dest" "$url" || rc=$? - fi - # curl -f leaves a PARTIAL file behind when a transfer dies mid-stream, so a failed - # download must not hand the caller something it could mistake for a complete one. - if [ "$rc" -ne 0 ] || [ ! -s "$dest" ]; then + for attempt in 1 2 3; do + rc=0 + if [ -n "$FELIS_GITHUB_TOKEN" ]; then + printf 'header = "Authorization: Bearer %s"\n' "$FELIS_GITHUB_TOKEN" \ + | curl -fsSL --retry 5 --retry-delay 2 --speed-limit 1024 --speed-time 60 --config - \ + -A "$ua" -H "Accept: application/octet-stream" -o "$dest" "$url" || rc=$? + else + curl -fsSL --retry 5 --retry-delay 2 --speed-limit 1024 --speed-time 60 \ + -A "$ua" -H "Accept: application/octet-stream" -o "$dest" "$url" || rc=$? + fi + [ "$rc" -eq 0 ] && [ -s "$dest" ] && return 0 + # curl -f leaves a PARTIAL file behind when a transfer dies mid-stream, so a failed + # download must not hand the caller something it could mistake for a complete one. rm -f "$dest" - return 1 - fi + if [ "$attempt" -lt 3 ]; then + warn "downloading ${name} failed (attempt ${attempt}/3, curl exit ${rc}); retrying" + sleep 5 + fi + done + return 1 } # verify_release_checksum checks the downloaded asset $2 (at $3) against the SHA256SUMS @@ -3446,7 +3480,7 @@ install_via_plugins() { log "downloading ${name} ${version}" tmp="$(mktemp "${VELOCITY_DIR}/.${name}.jar.XXXXXX")" remember_temp "$tmp" - curl -fsSL "$url" -o "$tmp" || die "failed to download ${name} ${version}: ${url}" + curl -fsSL --retry 5 --retry-delay 2 "$url" -o "$tmp" || die "failed to download ${name} ${version}: ${url}" have="$(sha256sum <"$tmp" | cut -d' ' -f1)" [ "$have" = "$want" ] \ || die "${name} ${version} checksum mismatch: got ${have}, expected ${want}" @@ -5568,7 +5602,7 @@ install_go_toolchain() { # A private directory, not a fixed /tmp name another local user could have planted first. tmp="$(mktemp -d)" remember_temp "$tmp" - curl -fsSL "$url" -o "${tmp}/${tarball}" || die "failed to download the Go toolchain: ${url}" + curl -fsSL --retry 5 --retry-delay 2 "$url" -o "${tmp}/${tarball}" || die "failed to download the Go toolchain: ${url}" # Checked before the old toolchain is removed, so a refusal leaves the host as it was. # Hash stdin, never the path — same reason as install_via_plugins. have="$(sha256sum <"${tmp}/${tarball}" | cut -d' ' -f1)" diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index c299adc..9708b73 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -810,6 +810,147 @@ gtmp="$(printf '%s\n' "$out" | sed -n 's/^TEMP: //p')" expect "the Go download is staged in a directory the cleanup removes" \ "CURL: ${gtmp:-}/go1.26.4.linux-amd64.tar.gz" "$out" +# --- every download retries -------------------------------------------------------------- +# curl without --retry gives up on the first 503 or timeout: one blip in the ViaVersion, Go +# or Docker repository download stopped a fresh install halfway. +noretry="$(grep -nE 'curl -(fsSL|sfL)' "$BS" | grep -vE '^[0-9]+:[[:space:]]*#|<(raw-)?url>' | grep -v -- '--retry' || true)" +if [ -z "$noretry" ]; then + echo "PASS every curl download in the installer retries" +else + echo "FAIL curl downloads without --retry:"; printf '%s\n' "$noretry"; fails=$((fails + 1)) +fi + +# --- a release asset's download is tried three times ------------------------------------- +# curl's --retry does not cover a connection reset mid-stream, the likely failure on an +# image bundle of several hundred MiB, and what the installer falls back to is a build on +# the host that preflight may not have counted on. A release that does not list the asset +# (still uploading) falls back at once: no download can find it. + +dlblock="$(awk '/^download_release_asset\(\) \{/,/^}/' "$BS")" +[ -n "$dlblock" ] || { echo "FAIL: no download_release_asset found in $BS"; exit 1; } +[ "$(printf '%s\n' "$dlblock" | wc -l)" -lt 40 ] \ + || { echo "FAIL: the extracted block is not download_release_asset -- did its closing brace move?"; exit 1; } +dldir="$(mktemp -d)" +run_dl() { # [script]; DL_FAILS curl calls fail (exit ${DL_RC:-56}) after writing part of the file + rm -f "$dldir/calls" "$dldir/asset" + DLDIR="$dldir" bash -c ' + set -Eeuo pipefail + log() { printf "LOG: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; } + sleep() { printf "SLEEP: %s\n" "$*"; } + load_release_json() { :; } + github_asset_id() { [ -z "${DL_UNLISTED:-}" ] || return 1; echo 4242; } + repo_slug() { echo FelisMC/Felis; } + curl() { + local out="" a stdin="" + printf "%s\n" "$*" >> "$DLDIR/calls" + [ "${*: -1}" = https://api.github.com/repos/FelisMC/Felis/releases/assets/4242 ] || { echo "curl: wrong url ${*: -1}" >&2; return 3; } + for a in "$@"; do [ "$a" != - ] || stdin="$(command cat)"; done + [ -z "$stdin" ] || printf "%s\n" "$stdin" > "$DLDIR/stdin" + while [ "$#" -gt 0 ]; do [ "$1" = -o ] && out="$2"; shift; done + if [ "$(wc -l < "$DLDIR/calls")" -le "${DL_FAILS:-0}" ]; then + printf "half a bund" > "$out"; return "${DL_RC:-56}" + fi + [ -n "${DL_EMPTY:-}" ] && { : > "$out"; return 0; } + printf "the whole bundle\n" > "$out" + } + FELIS_REPO_URL=https://github.com/FelisMC/Felis FELIS_GITHUB_TOKEN="${DL_TOKEN:-}" + '"$dlblock"' + if download_release_asset v9.9.9 felis-image-felis-linux-amd64.tar "$DLDIR/asset"; then echo FETCHED; else echo "GAVE UP"; fi' 2>&1 +} +calls() { [ -f "$dldir/calls" ] && wc -l < "$dldir/calls" | tr -d ' ' || echo 0; } + +out="$(DL_FAILS=2 run_dl)" +expect "a download reset twice mid-stream gets the asset on the third try" "FETCHED" "$out" +expect "the third try's file is the whole asset" "the whole bundle" "$(cat "$dldir/asset" 2>/dev/null)" +expect "each failed try is reported with curl's exit code" "WARN: downloading felis-image-felis-linux-amd64.tar failed (attempt 2/3, curl exit 56); retrying" "$out" +expect "the tries are spaced" "SLEEP: 5" "$out" +expect "a download that works on the third try takes three" 3 "$(calls)" + +out="$(DL_FAILS=9 run_dl)" +expect "a download that keeps failing gives up" "GAVE UP" "$out" +expect "it gives up after three tries" 3 "$(calls)" +if [ -e "$dldir/asset" ]; then + echo "FAIL a failed download left part of the asset behind"; fails=$((fails + 1)) +else + echo "PASS a failed download leaves no part of the asset behind" +fi +out="$(DL_EMPTY=1 run_dl)" +expect "an empty file is a failed download" "GAVE UP" "$out" +expect "an empty file is tried again" 3 "$(calls)" + +out="$(DL_UNLISTED=1 DL_FAILS=9 run_dl)" +expect "a release that does not list the asset gives up" "GAVE UP" "$out" +if [ "$(calls) $(printf '%s\n' "$out" | grep -c SLEEP)" = "0 0" ]; then + echo "PASS an unlisted asset is not downloaded, nor waited for" +else + echo "FAIL an unlisted asset was tried $(calls) times"; fails=$((fails + 1)) +fi + +out="$(run_dl)" +expect "a download that works takes one try" 1 "$(calls)" +expect "a stalled transfer counts as a failure curl retries" "--speed-limit 1024 --speed-time 60" "$(cat "$dldir/calls")" +DL_TOKEN=tok-secret run_dl >/dev/null +case "$(cat "$dldir/calls")" in + *tok-secret*) echo "FAIL the token reached curl's argv"; fails=$((fails + 1)) ;; + *) echo "PASS the token stays out of curl's argv" ;; +esac +expect "the token goes to curl on stdin" 'header = "Authorization: Bearer tok-secret"' "$(cat "$dldir/stdin" 2>/dev/null)" +expect "a download with a token counts a stalled transfer as a failure too" "--speed-limit 1024 --speed-time 60" "$(cat "$dldir/calls")" +rm -rf "$dldir" + +# --- a build preflight did not count on checks its room first --------------------------- +# A release install budgets no Docker; an asset it then cannot use is built here, with +# Docker's images and build cache under /var/lib/containerd. Without the room the install +# must stop before Docker is installed, not fill the disk halfway through a build. + +edblock="$(awk '/^ensure_docker\(\) \{/,/^}/' "$BS"; awk '/^unplanned_build_room\(\) \{/,/^}/' "$BS")" +case "$edblock" in *"install_docker"*"FELIS_PREFLIGHT=warn to build anyway"*) ;; *) echo "FAIL: ensure_docker / unplanned_build_room not found in $BS"; exit 1 ;; esac +run_ed() { # [DOCKER_INSTALLED]; ED_FREE_MIB free on the filesystem holding /var/lib/containerd + DOCKER_INSTALLED="${1:-}" bash -c ' + set -Eeuo pipefail + ok() { printf "OK: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; }; die() { printf "DIE: %s\n" "$*"; exit 1; } + install_docker() { echo INSTALL; } + systemctl() { echo "SYSTEMCTL $*"; } + path_populated() { [ -n "${ED_POPULATED:-}" ]; } + existing_ancestor() { echo /; } + df() { echo DF >&2; echo "Filesystem 1024-blocks Used Available Capacity Mounted" + echo "/dev/vda1 41943040 1 $(( ${ED_FREE_MIB:-60000} * 1024 )) 50% /"; } + bootstrap_from_tui() { return 1; } + '"$(awk '/^use_release_binary\(\) \{/,/^}/' "$BS")"' + '"$(awk '/^release_assets_expected\(\) \{/,/^}/' "$BS")"' + '"$(awk '/^host_builds_expected\(\) \{/,/^}/' "$BS")"' + FELIS_SKIP_FETCH="" FELIS_REF_PINNED="" FELIS_ARTIFACT_DIR="" FELIS_GAME_STACK=pinned + FELIS_VERSION_BOOTSTRAP="${ED_CHANNEL:-release}" FELIS_PREFLIGHT="${ED_MODE:-strict}" + '"$edblock"' + ensure_docker; echo "installed=$DOCKER_INSTALLED"' 2>&1 +} + +out="$(ED_FREE_MIB=1000 run_ed)" +expect "a release install without room for an unplanned build stops" \ + "DIE: building here what the release did not supply takes about 8192 MiB under /var/lib/containerd, and / has 1000 MiB free; nothing has been built" "$out" +case "$out" in + *INSTALL*) echo "FAIL Docker was installed on a host without room to build"; fails=$((fails + 1)) ;; + *) echo "PASS the stop comes before Docker is installed" ;; +esac +expect "a release install with the room builds" "INSTALL" "$(ED_FREE_MIB=9000 run_ed)" +expect "Docker's own cache needs only the rerun's room" "INSTALL" "$(ED_POPULATED=1 ED_FREE_MIB=3000 run_ed)" +expect "a cache on a nearly full disk still stops" "takes about 2048 MiB" "$(ED_POPULATED=1 ED_FREE_MIB=1500 run_ed)" +out="$(ED_CHANNEL=dev ED_FREE_MIB=1000 run_ed)" +expect "a source build's room is preflight's to check" "installed=1" "$out" +case "$out" in + *DF*) echo "FAIL a planned build measured the disk again"; fails=$((fails + 1)) ;; + *) echo "PASS a planned build is not measured again" ;; +esac +out="$(ED_MODE=warn ED_FREE_MIB=1000 run_ed)" +expect "FELIS_PREFLIGHT=warn builds anyway, saying so" "WARN: building here what the release did not supply takes about 8192 MiB" "$out" +expect "FELIS_PREFLIGHT=warn installs Docker" "INSTALL" "$out" +out="$(ED_FREE_MIB=1000 run_ed 1)" +expect "a Docker this run installed is started again" "SYSTEMCTL start docker" "$out" +case "$out" in + *DF*|*DIE*) echo "FAIL a restart of Docker checked the room again"; fails=$((fails + 1)) ;; + *) echo "PASS a restart of Docker is not checked again" ;; +esac + # --- a release binary is hashed against SHA256SUMS before anything runs it --------------- # download_release_binary executes the asset as root to read its version stamp, so the # checksum has to come first, and every failure has to fall back to the source build. diff --git a/docs/operations.md b/docs/operations.md index b93b33a..f0278ce 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -136,7 +136,9 @@ the images are in the registry, when: - the release publishes no `SHA256SUMS` (one cut before release assets existed, or still uploading), or an asset is missing, fails its checksum or is malformed. Only that image is built (the registry and PostgreSQL images are pulled from Docker Hub instead), and a - warning names it; troubleshooting §15c lists the messages. + warning names it; troubleshooting §15c lists the messages. Each download is tried + three times first, and a host without the room for the build stops before installing + Docker (troubleshooting §15c). `FELIS_ARTIFACT_DIR=` installs from a directory instead of the release: a release's assets downloaded there (every `felis-*` file and `SHA256SUMS`), or the directory diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e3e1682..c104f75 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1969,7 +1969,16 @@ A release install takes its images and the Velocity plugin from the release's as (operations §1, "Where the binary and the images come from"). When one cannot be used the installer names it and the reason, and builds that image with Docker instead (the registry and PostgreSQL images are pulled from Docker Hub). Nothing unchecked is used either way -**[SH-TESTED]**. +**[SH-TESTED]**. Each download is tried three times, five seconds apart, before it counts as +failed, and a transfer that stalls under 1 KiB/s for a minute is cut off and tried again. + +Preflight counts no Docker builds for a release install, so the first build it falls back +to checks the room first: Docker's images and build cache take about 8 GiB under +`/var/lib/containerd` (2 GiB when Docker already has a cache there). Without it the install +stops before Docker is installed, with `building here what the release did not supply takes +about … MiB under /var/lib/containerd, and … has … MiB free; nothing has been built`. Rerun +once the asset downloads (the messages above it say which one failed), free that space, or +set `FELIS_PREFLIGHT=warn` to build anyway **[SH-TESTED]**. | Message | Meaning | What to do | |---|---|---|