diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6074dd4..823497e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,7 @@ jobs: - run: sh deploy/bootstrap_test.sh - run: sh deploy/uninstall_test.sh - run: bash deploy/e2e_release_test.sh + - run: bash deploy/e2e_upstream_test.sh # The shipped alert rules (deploy/alerts): promtool parses them and runs their unit tests, # which pin when each alert fires and that it stays quiet before. internal/metrics' diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f30d3da..11a21d0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -21,6 +21,11 @@ # This runs on pushes that touch what gets installed, by hand, and weekly (a moving # upstream: apt mirrors, k3s's install script and release assets, Adoptium). # deploy/e2e_check.sh holds the assertions, deploy/e2e_seed.sh the upgrade's seed and its check. +# +# An installer that stops on an upstream download refused or dropped (a GitHub 403, a 5xx, a +# timeout) ends its job green, with a warning on the run: each install step hands a failed +# run's log to deploy/e2e_upstream.sh, which sets E2E_UPSTREAM_SKIP for the job's later +# steps. Every other installer failure, a 404 included, fails the job. name: e2e on: @@ -43,8 +48,8 @@ on: permissions: contents: read -# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` fails the step -# when the installer fails; the default shell reports tee's status. +# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` carries the +# installer's status to deploy/e2e_upstream.sh; the default shell reports tee's status. defaults: run: shell: bash @@ -107,9 +112,10 @@ jobs: run: sudo ufw --force enable - name: Install - run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log + run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log || bash deploy/e2e_upstream.sh install.log $? - name: Check the install + if: env.E2E_UPSTREAM_SKIP != '1' run: | sudo bash deploy/e2e_check.sh install for tag in felis-k3s-pods felis-k3s-services felis-panel felis-proxy; do @@ -124,11 +130,13 @@ jobs: grep -q "felis-velocity.jar is the release's" install.log - name: Rerun the same assets - run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log + if: env.E2E_UPSTREAM_SKIP != '1' + run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log || bash deploy/e2e_upstream.sh rerun.log $? # containerd and the registry already hold every image under the digest the listing # names, so nothing is imported or uploaded twice. - name: Check the rerun + if: env.E2E_UPSTREAM_SKIP != '1' run: | sudo bash deploy/e2e_check.sh rerun grep -q 'felis-velocity unchanged; left running' rerun.log @@ -160,8 +168,9 @@ jobs: # The README's command on a fresh host: this commit's installer, as main serves it, on its # default channel with nothing pinned. It resolves the newest release and installs that # release's binary, images and plugin from its assets, each checked against its - # SHA256SUMS; the private repo's token is the only thing added. FELIS_INSTALL_MODE picks the - # mode the setup console would ask for. + # SHA256SUMS. The workflow's token is the only thing added: it lifts GitHub's limit of 60 API + # calls an hour for an address without one. FELIS_INSTALL_MODE picks the mode the setup + # console would ask for. readme: runs-on: ubuntu-24.04 timeout-minutes: 120 @@ -181,12 +190,12 @@ jobs: if: steps.release.outputs.tag != '' env: TOKEN: ${{ github.token }} - run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log + run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log || bash deploy/e2e_upstream.sh readme.log $? # The binary is the release's, so the phase is `release`: its database backup and # timers are the release's to have or lack. - name: Check the install - if: steps.release.outputs.tag != '' + if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1' env: TAG: ${{ steps.release.outputs.tag }} BINARY: ${{ steps.release.outputs.binary }} @@ -246,10 +255,10 @@ jobs: TOKEN: ${{ github.token }} run: | git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh - sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log + sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log || bash deploy/e2e_upstream.sh release.log $? - name: Check the release install - if: steps.release.outputs.tag != '' + if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1' env: TAG: ${{ steps.release.outputs.tag }} BINARY: ${{ steps.release.outputs.binary }} @@ -260,17 +269,17 @@ jobs: # A fresh install's database holds only what its migrations wrote: the seed gives the # upgrade's move and its pending migrations existing rows to carry. - name: Seed the release's database - if: steps.release.outputs.tag != '' + if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1' run: sudo bash deploy/e2e_seed.sh seed - name: Upgrade to this commit - if: steps.release.outputs.tag != '' - run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log + if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1' + run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log || bash deploy/e2e_upstream.sh upgrade.log $? # Both checks run and report: the seeded rows go last, after the restore drill has # also put them back from a bundle. - name: Check the upgrade - if: steps.release.outputs.tag != '' + if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1' run: | rc=0 sudo bash deploy/e2e_check.sh upgrade || rc=1 @@ -313,9 +322,10 @@ jobs: sudo chown -R root:root /opt/felis/src - name: Install - run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log + run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log || bash deploy/e2e_upstream.sh source.log $? - name: Check the install + if: env.E2E_UPSTREAM_SKIP != '1' run: sudo bash deploy/e2e_check.sh install - name: Diagnostics diff --git a/deploy/e2e_upstream.sh b/deploy/e2e_upstream.sh new file mode 100644 index 0000000..0a542a9 --- /dev/null +++ b/deploy/e2e_upstream.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Whether a failed installer run in the e2e workflow stopped on an upstream download that +# was refused or dropped, which says nothing about the commit under test. Each install step +# hands its log and the installer's status here when the installer fails: +# +# sudo ... bash deploy/bootstrap.sh 2>&1 | tee install.log || bash deploy/e2e_upstream.sh install.log $? +# +# It reads the installer's last ERR-trap line ("bootstrap failed near line N (exit E)") and +# the line just before it. When that line is curl's own error for the same status, and the +# error is one a mirror or GitHub answers with on a bad minute (a connection refused, reset +# or timed out, a 403, 408, 429 or 5xx), it leaves a warning and E2E_UPSTREAM_SKIP=1 in +# $GITHUB_ENV and exits 0: the job's later steps are gated on that variable, so the job ends +# green with the warning on the run. Anything else exits with the installer's status. A 404 +# is a URL or a version the installer names, which a commit can break, so it fails. +# +# deploy/e2e_upstream_test.sh holds its checks, against bootstrap.sh's own trap message. +set -euo pipefail + +log="${1:?usage: e2e_upstream.sh LOG STATUS}" +status="${2:?usage: e2e_upstream.sh LOG STATUS}" + +# The last run of trap lines and the line before it. bash may fire the trap again for each +# function the failure unwinds through, and the EXIT cleanup can print after it. The log +# keeps the installer's colour codes, so nothing is anchored on the left. +code="" before="" +{ read -r code && IFS= read -r before; } < <(awk ' + /bootstrap failed near line [0-9]+ \(exit [0-9]+\)$/ { + code = $0; sub(/.*\(exit /, "", code); sub(/\)$/, "", code) + found = code; foundprev = last + next + } + { last = $0 } + END { if (found != "") { print found; print foundprev } } +' "$log") || true + +upstream="" +case "$before" in + *"curl: (${code}) "*) + case "$code" in + # Could not resolve or connect, an HTTP/2 or TLS failure, a transfer cut short, no + # reply, a send or receive failure, a timeout. + 5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92) upstream=yes ;; + # -f's HTTP error; curl before 7.75 appends the reason phrase. + 22) if [[ "$before" =~ returned\ error:\ (403|408|429|5[0-9][0-9])([^0-9]|$) ]]; then upstream=yes; fi ;; + esac + ;; +esac + +if [ -n "$upstream" ]; then + echo "::warning::the installer stopped on an upstream download (curl: ${before#*curl: }); this job skips its remaining steps" + echo "E2E_UPSTREAM_SKIP=1" >> "${GITHUB_ENV:-/dev/null}" + exit 0 +fi +echo "the installer failed (exit ${status}) on something other than a refused upstream download; its log is above" +[ "$status" -ne 0 ] 2>/dev/null || status=1 +exit "$status" diff --git a/deploy/e2e_upstream_test.sh b/deploy/e2e_upstream_test.sh new file mode 100644 index 0000000..6dd9eec --- /dev/null +++ b/deploy/e2e_upstream_test.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# Checks for deploy/e2e_upstream.sh. Run it as: bash deploy/e2e_upstream_test.sh +# +# The trap line in the logs is bootstrap.sh's own on_error message, printed the way its warn +# prints it, so rewording it there fails here rather than turning every refused download +# back into a red e2e run. The curl lines are curl's own wording. +set -u + +here="$(dirname "$0")" +EU="${1:-${here}/e2e_upstream.sh}" +BS="${2:-${here}/bootstrap.sh}" +[ -f "$EU" ] || { echo "no such script: $EU"; exit 1; } +[ -f "$BS" ] || { echo "no such script: $BS"; exit 1; } +fails=0 + +expect() { # label needle haystack + case "$3" in + *"$2"*) echo "PASS $1" ;; + *) echo "FAIL $1: expected <$2> in:"; echo "$3"; fails=$((fails + 1)) ;; + esac +} +status() { # label want got + if [ "$2" = "$3" ]; then echo "PASS $1"; else echo "FAIL $1: exit $3, want $2"; fails=$((fails + 1)); fi +} +same() { # label want got + if [ "$2" = "$3" ]; then echo "PASS $1"; else printf 'FAIL %s: got <%s>, want <%s>\n' "$1" "$3" "$2"; fails=$((fails + 1)); fi +} + +root="$(mktemp -d)" +trap 'rm -rf "$root"' EXIT + +trap_body="$(grep -E '^[[:space:]]*warn "bootstrap failed near line' "$BS" | head -n 1)" +if [ -z "$trap_body" ]; then + echo "FAIL bootstrap.sh's on_error prints no 'bootstrap failed near line' warning" + fails=$((fails + 1)) +fi +trap_body="${trap_body#*\"}" +trap_body="${trap_body%\"*}" +trapped() { # line code: on_error's warning as the installer prints it + # shellcheck disable=SC2034 # read by the eval + local line="$1" code="$2" + printf '\033[1;33m[warn]\033[0m %s\n' "$(eval "printf '%s' \"${trap_body}\"")" +} +step() { printf '\033[1;36m[felis]\033[0m %s\n' "$1"; } + +run() { # log status: prints what the script says; $root/env is its GITHUB_ENV + : > "$root/env" + GITHUB_ENV="$root/env" bash "$EU" "$1" "$2" 2>&1 +} + +# The upgrade job's v0.2.0 install on 2026-10-02: GitHub refused cloudflared's download. +{ + step "release channel: v0.2.0" + step "installing cloudflared 2026.9.1 (amd64)" + echo "curl: (22) The requested URL returned error: 403" + trapped 1727 22 +} > "$root/403.log" +out="$(run "$root/403.log" 22)" +status "a 403 on a download skips" 0 $? +expect " with a warning that quotes curl" "::warning::the installer stopped on an upstream download (curl: (22) The requested URL returned error: 403)" "$out" +same " and gates the later steps" "E2E_UPSTREAM_SKIP=1" "$(cat "$root/env")" + +for e in "(6) Could not resolve host: github.com" \ + "(7) Failed to connect to github.com port 443 after 130 ms: Couldn't connect to server" \ + "(28) Operation timed out after 300000 milliseconds with 0 out of 0 bytes received" \ + "(35) OpenSSL SSL_connect: SSL_ERROR_SYSCALL in connection to objects.githubusercontent.com:443" \ + "(56) Recv failure: Connection reset by peer" \ + "(22) The requested URL returned error: 429" \ + "(22) The requested URL returned error: 503" \ + "(22) The requested URL returned error: 502 Bad Gateway"; do + code="${e#(}" + code="${code%%)*}" + { step "installing k3s"; echo "curl: $e"; trapped 900 "$code"; } > "$root/e.log" + out="$(run "$root/e.log" "$code")" + status "curl: $e skips" 0 $? +done + +# What the installer prints after the trap, as it exits, changes nothing. +{ + cat "$root/403.log" + printf '\033[1;33m[warn]\033[0m %s\n' "restored the previous felis binary at /usr/local/bin/felis; the database was not migrated, so rerunning the installer picks up where this run stopped" +} > "$root/cleanup.log" +out="$(run "$root/cleanup.log" 22)" +status "warnings after the trap still skip" 0 $? + +# set -E: the trap again for a function the failure unwound through. +{ cat "$root/403.log"; trapped 1790 22; } > "$root/twice.log" +out="$(run "$root/twice.log" 22)" +status "the trap fired twice still skips" 0 $? + +# A 404 is a URL or a version the installer names. +{ step "installing cloudflared 2026.9.1 (amd64)"; echo "curl: (22) The requested URL returned error: 404"; trapped 1727 22; } > "$root/404.log" +out="$(run "$root/404.log" 22)" +status "a 404 fails with the installer's status" 22 $? +expect " and says so" "the installer failed (exit 22) on something other than a refused upstream download" "$out" +same " and gates nothing" "" "$(cat "$root/env")" + +for e in "401" "400" "410"; do + { echo "curl: (22) The requested URL returned error: $e"; trapped 1727 22; } > "$root/e.log" + out="$(run "$root/e.log" 22)" + status "a $e fails" 22 $? +done + +# A refused download the installer got past, then a failure of its own that happens to +# exit with curl's status: only the line before the trap counts. +{ + echo "curl: (22) The requested URL returned error: 403" + step "building felis from source" + trapped 4100 22 +} > "$root/later.log" +out="$(run "$root/later.log" 22)" +status "a refused download earlier in the log fails" 22 $? + +# curl's error on the line before, but the installer stopped with another status. +{ echo "curl: (22) The requested URL returned error: 403"; trapped 1727 1; } > "$root/mismatch.log" +out="$(run "$root/mismatch.log" 1)" +status "a trap for another status fails" 1 $? + +# The installer's own die, which the trap never sees. +{ step "installing k3s"; printf '\033[1;31m[fail]\033[0m %s\n' "k3s did not become ready"; } > "$root/die.log" +out="$(run "$root/die.log" 1)" +status "the installer's own failure fails" 1 $? +same " and gates nothing" "" "$(cat "$root/env")" + +: > "$root/empty.log" +out="$(run "$root/empty.log" 141)" +status "an empty log fails with the installer's status" 141 $? + +echo +if [ "$fails" -eq 0 ]; then echo "ALL PASS"; else echo "${fails} FAILED"; exit 1; fi