ci(e2e): 上游下载被拒时跳过

.github/workflows/ci.yml

@@ -151,6 +151,7 @@ jobs:
       - run: sh deploy/bootstrap_test.sh
       - run: sh deploy/uninstall_test.sh
       - run: bash deploy/e2e_release_test.sh
+      - run: bash deploy/e2e_upstream_test.sh
 
   # The shipped alert rules (deploy/alerts): promtool parses them and runs their unit tests,
   # which pin when each alert fires and that it stays quiet before. internal/metrics'

.github/workflows/e2e.yml

@@ -21,6 +21,11 @@
 # This runs on pushes that touch what gets installed, by hand, and weekly (a moving
 # upstream: apt mirrors, k3s's install script and release assets, Adoptium).
 # deploy/e2e_check.sh holds the assertions, deploy/e2e_seed.sh the upgrade's seed and its check.
+#
+# An installer that stops on an upstream download refused or dropped (a GitHub 403, a 5xx, a
+# timeout) ends its job green, with a warning on the run: each install step hands a failed
+# run's log to deploy/e2e_upstream.sh, which sets E2E_UPSTREAM_SKIP for the job's later
+# steps. Every other installer failure, a 404 included, fails the job.
 name: e2e
 
 on:
@@ -43,8 +48,8 @@ on:
 permissions:
   contents: read
 
-# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` fails the step
-# when the installer fails; the default shell reports tee's status.
+# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` carries the
+# installer's status to deploy/e2e_upstream.sh; the default shell reports tee's status.
 defaults:
   run:
     shell: bash
@@ -107,9 +112,10 @@ jobs:
         run: sudo ufw --force enable
 
       - name: Install
-        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log || bash deploy/e2e_upstream.sh install.log $?
 
       - name: Check the install
+        if: env.E2E_UPSTREAM_SKIP != '1'
         run: |
           sudo bash deploy/e2e_check.sh install
           for tag in felis-k3s-pods felis-k3s-services felis-panel felis-proxy; do
@@ -124,11 +130,13 @@ jobs:
           grep -q "felis-velocity.jar is the release's" install.log
 
       - name: Rerun the same assets
-        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log
+        if: env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log || bash deploy/e2e_upstream.sh rerun.log $?
 
       # containerd and the registry already hold every image under the digest the listing
       # names, so nothing is imported or uploaded twice.
       - name: Check the rerun
+        if: env.E2E_UPSTREAM_SKIP != '1'
         run: |
           sudo bash deploy/e2e_check.sh rerun
           grep -q 'felis-velocity unchanged; left running' rerun.log
@@ -160,8 +168,9 @@ jobs:
   # The README's command on a fresh host: this commit's installer, as main serves it, on its
   # default channel with nothing pinned. It resolves the newest release and installs that
   # release's binary, images and plugin from its assets, each checked against its
-  # SHA256SUMS; the private repo's token is the only thing added. FELIS_INSTALL_MODE picks the
-  # mode the setup console would ask for.
+  # SHA256SUMS. The workflow's token is the only thing added: it lifts GitHub's limit of 60 API
+  # calls an hour for an address without one. FELIS_INSTALL_MODE picks the mode the setup
+  # console would ask for.
   readme:
     runs-on: ubuntu-24.04
     timeout-minutes: 120
@@ -181,12 +190,12 @@ jobs:
         if: steps.release.outputs.tag != ''
         env:
           TOKEN: ${{ github.token }}
-        run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log
+        run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log || bash deploy/e2e_upstream.sh readme.log $?
 
       # The binary is the release's, so the phase is `release`: its database backup and
       # timers are the release's to have or lack.
       - name: Check the install
-        if: steps.release.outputs.tag != ''
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
         env:
           TAG: ${{ steps.release.outputs.tag }}
           BINARY: ${{ steps.release.outputs.binary }}
@@ -246,10 +255,10 @@ jobs:
           TOKEN: ${{ github.token }}
         run: |
           git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh
-          sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log
+          sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log || bash deploy/e2e_upstream.sh release.log $?
 
       - name: Check the release install
-        if: steps.release.outputs.tag != ''
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
         env:
           TAG: ${{ steps.release.outputs.tag }}
           BINARY: ${{ steps.release.outputs.binary }}
@@ -260,17 +269,17 @@ jobs:
       # A fresh install's database holds only what its migrations wrote: the seed gives the
       # upgrade's move and its pending migrations existing rows to carry.
       - name: Seed the release's database
-        if: steps.release.outputs.tag != ''
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
         run: sudo bash deploy/e2e_seed.sh seed
 
       - name: Upgrade to this commit
-        if: steps.release.outputs.tag != ''
-        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
+        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log || bash deploy/e2e_upstream.sh upgrade.log $?
 
       # Both checks run and report: the seeded rows go last, after the restore drill has
       # also put them back from a bundle.
       - name: Check the upgrade
-        if: steps.release.outputs.tag != ''
+        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
         run: |
           rc=0
           sudo bash deploy/e2e_check.sh upgrade || rc=1
@@ -313,9 +322,10 @@ jobs:
           sudo chown -R root:root /opt/felis/src
 
       - name: Install
-        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log
+        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log || bash deploy/e2e_upstream.sh source.log $?
 
       - name: Check the install
+        if: env.E2E_UPSTREAM_SKIP != '1'
         run: sudo bash deploy/e2e_check.sh install
 
       - name: Diagnostics

deploy/e2e_upstream.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# Whether a failed installer run in the e2e workflow stopped on an upstream download that
+# was refused or dropped, which says nothing about the commit under test. Each install step
+# hands its log and the installer's status here when the installer fails:
+#
+#   sudo ... bash deploy/bootstrap.sh 2>&1 | tee install.log || bash deploy/e2e_upstream.sh install.log $?
+#
+# It reads the installer's last ERR-trap line ("bootstrap failed near line N (exit E)") and
+# the line just before it. When that line is curl's own error for the same status, and the
+# error is one a mirror or GitHub answers with on a bad minute (a connection refused, reset
+# or timed out, a 403, 408, 429 or 5xx), it leaves a warning and E2E_UPSTREAM_SKIP=1 in
+# $GITHUB_ENV and exits 0: the job's later steps are gated on that variable, so the job ends
+# green with the warning on the run. Anything else exits with the installer's status. A 404
+# is a URL or a version the installer names, which a commit can break, so it fails.
+#
+# deploy/e2e_upstream_test.sh holds its checks, against bootstrap.sh's own trap message.
+set -euo pipefail
+
+log="${1:?usage: e2e_upstream.sh LOG STATUS}"
+status="${2:?usage: e2e_upstream.sh LOG STATUS}"
+
+# The last run of trap lines and the line before it. bash may fire the trap again for each
+# function the failure unwinds through, and the EXIT cleanup can print after it. The log
+# keeps the installer's colour codes, so nothing is anchored on the left.
+code="" before=""
+{ read -r code && IFS= read -r before; } < <(awk '
+  /bootstrap failed near line [0-9]+ \(exit [0-9]+\)$/ {
+    code = $0; sub(/.*\(exit /, "", code); sub(/\)$/, "", code)
+    found = code; foundprev = last
+    next
+  }
+  { last = $0 }
+  END { if (found != "") { print found; print foundprev } }
+' "$log") || true
+
+upstream=""
+case "$before" in
+  *"curl: (${code}) "*)
+    case "$code" in
+      # Could not resolve or connect, an HTTP/2 or TLS failure, a transfer cut short, no
+      # reply, a send or receive failure, a timeout.
+      5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92) upstream=yes ;;
+      # -f's HTTP error; curl before 7.75 appends the reason phrase.
+      22) if [[ "$before" =~ returned\ error:\ (403|408|429|5[0-9][0-9])([^0-9]|$) ]]; then upstream=yes; fi ;;
+    esac
+    ;;
+esac
+
+if [ -n "$upstream" ]; then
+  echo "::warning::the installer stopped on an upstream download (curl: ${before#*curl: }); this job skips its remaining steps"
+  echo "E2E_UPSTREAM_SKIP=1" >> "${GITHUB_ENV:-/dev/null}"
+  exit 0
+fi
+echo "the installer failed (exit ${status}) on something other than a refused upstream download; its log is above"
+[ "$status" -ne 0 ] 2>/dev/null || status=1
+exit "$status"

deploy/e2e_upstream_test.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# Checks for deploy/e2e_upstream.sh. Run it as: bash deploy/e2e_upstream_test.sh
+#
+# The trap line in the logs is bootstrap.sh's own on_error message, printed the way its warn
+# prints it, so rewording it there fails here rather than turning every refused download
+# back into a red e2e run. The curl lines are curl's own wording.
+set -u
+
+here="$(dirname "$0")"
+EU="${1:-${here}/e2e_upstream.sh}"
+BS="${2:-${here}/bootstrap.sh}"
+[ -f "$EU" ] || { echo "no such script: $EU"; exit 1; }
+[ -f "$BS" ] || { echo "no such script: $BS"; exit 1; }
+fails=0
+
+expect() { # label needle haystack
+  case "$3" in
+    *"$2"*) echo "PASS $1" ;;
+    *) echo "FAIL $1: expected <$2> in:"; echo "$3"; fails=$((fails + 1)) ;;
+  esac
+}
+status() { # label want got
+  if [ "$2" = "$3" ]; then echo "PASS $1"; else echo "FAIL $1: exit $3, want $2"; fails=$((fails + 1)); fi
+}
+same() { # label want got
+  if [ "$2" = "$3" ]; then echo "PASS $1"; else printf 'FAIL %s: got <%s>, want <%s>\n' "$1" "$3" "$2"; fails=$((fails + 1)); fi
+}
+
+root="$(mktemp -d)"
+trap 'rm -rf "$root"' EXIT
+
+trap_body="$(grep -E '^[[:space:]]*warn "bootstrap failed near line' "$BS" | head -n 1)"
+if [ -z "$trap_body" ]; then
+  echo "FAIL bootstrap.sh's on_error prints no 'bootstrap failed near line' warning"
+  fails=$((fails + 1))
+fi
+trap_body="${trap_body#*\"}"
+trap_body="${trap_body%\"*}"
+trapped() { # line code: on_error's warning as the installer prints it
+  # shellcheck disable=SC2034 # read by the eval
+  local line="$1" code="$2"
+  printf '\033[1;33m[warn]\033[0m %s\n' "$(eval "printf '%s' \"${trap_body}\"")"
+}
+step() { printf '\033[1;36m[felis]\033[0m %s\n' "$1"; }
+
+run() { # log status: prints what the script says; $root/env is its GITHUB_ENV
+  : > "$root/env"
+  GITHUB_ENV="$root/env" bash "$EU" "$1" "$2" 2>&1
+}
+
+# The upgrade job's v0.2.0 install on 2026-10-02: GitHub refused cloudflared's download.
+{
+  step "release channel: v0.2.0"
+  step "installing cloudflared 2026.9.1 (amd64)"
+  echo "curl: (22) The requested URL returned error: 403"
+  trapped 1727 22
+} > "$root/403.log"
+out="$(run "$root/403.log" 22)"
+status "a 403 on a download skips" 0 $?
+expect "  with a warning that quotes curl" "::warning::the installer stopped on an upstream download (curl: (22) The requested URL returned error: 403)" "$out"
+same "  and gates the later steps" "E2E_UPSTREAM_SKIP=1" "$(cat "$root/env")"
+
+for e in "(6) Could not resolve host: github.com" \
+    "(7) Failed to connect to github.com port 443 after 130 ms: Couldn't connect to server" \
+    "(28) Operation timed out after 300000 milliseconds with 0 out of 0 bytes received" \
+    "(35) OpenSSL SSL_connect: SSL_ERROR_SYSCALL in connection to objects.githubusercontent.com:443" \
+    "(56) Recv failure: Connection reset by peer" \
+    "(22) The requested URL returned error: 429" \
+    "(22) The requested URL returned error: 503" \
+    "(22) The requested URL returned error: 502 Bad Gateway"; do
+  code="${e#(}"
+  code="${code%%)*}"
+  { step "installing k3s"; echo "curl: $e"; trapped 900 "$code"; } > "$root/e.log"
+  out="$(run "$root/e.log" "$code")"
+  status "curl: $e skips" 0 $?
+done
+
+# What the installer prints after the trap, as it exits, changes nothing.
+{
+  cat "$root/403.log"
+  printf '\033[1;33m[warn]\033[0m %s\n' "restored the previous felis binary at /usr/local/bin/felis; the database was not migrated, so rerunning the installer picks up where this run stopped"
+} > "$root/cleanup.log"
+out="$(run "$root/cleanup.log" 22)"
+status "warnings after the trap still skip" 0 $?
+
+# set -E: the trap again for a function the failure unwound through.
+{ cat "$root/403.log"; trapped 1790 22; } > "$root/twice.log"
+out="$(run "$root/twice.log" 22)"
+status "the trap fired twice still skips" 0 $?
+
+# A 404 is a URL or a version the installer names.
+{ step "installing cloudflared 2026.9.1 (amd64)"; echo "curl: (22) The requested URL returned error: 404"; trapped 1727 22; } > "$root/404.log"
+out="$(run "$root/404.log" 22)"
+status "a 404 fails with the installer's status" 22 $?
+expect "  and says so" "the installer failed (exit 22) on something other than a refused upstream download" "$out"
+same "  and gates nothing" "" "$(cat "$root/env")"
+
+for e in "401" "400" "410"; do
+  { echo "curl: (22) The requested URL returned error: $e"; trapped 1727 22; } > "$root/e.log"
+  out="$(run "$root/e.log" 22)"
+  status "a $e fails" 22 $?
+done
+
+# A refused download the installer got past, then a failure of its own that happens to
+# exit with curl's status: only the line before the trap counts.
+{
+  echo "curl: (22) The requested URL returned error: 403"
+  step "building felis from source"
+  trapped 4100 22
+} > "$root/later.log"
+out="$(run "$root/later.log" 22)"
+status "a refused download earlier in the log fails" 22 $?
+
+# curl's error on the line before, but the installer stopped with another status.
+{ echo "curl: (22) The requested URL returned error: 403"; trapped 1727 1; } > "$root/mismatch.log"
+out="$(run "$root/mismatch.log" 1)"
+status "a trap for another status fails" 1 $?
+
+# The installer's own die, which the trap never sees.
+{ step "installing k3s"; printf '\033[1;31m[fail]\033[0m %s\n' "k3s did not become ready"; } > "$root/die.log"
+out="$(run "$root/die.log" 1)"
+status "the installer's own failure fails" 1 $?
+same "  and gates nothing" "" "$(cat "$root/env")"
+
+: > "$root/empty.log"
+out="$(run "$root/empty.log" 141)"
+status "an empty log fails with the installer's status" 141 $?
+
+echo
+if [ "$fails" -eq 0 ]; then echo "ALL PASS"; else echo "${fails} FAILED"; exit 1; fi