Unverified Commit 94cbd7e1 authored by Lemon-miaow's avatar Lemon-miaow
Browse files

ci(e2e): 上游下载被拒时跳过

parent ce8c7893
Loading
Loading
Loading
Loading
+1 −0
Changes for .github/workflows/ci.yml: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -151,6 +151,7 @@ jobs:
      - run: sh deploy/bootstrap_test.sh
      - run: sh deploy/uninstall_test.sh
      - run: bash deploy/e2e_release_test.sh
      - run: bash deploy/e2e_upstream_test.sh

  # The shipped alert rules (deploy/alerts): promtool parses them and runs their unit tests,
  # which pin when each alert fires and that it stays quiet before. internal/metrics'
+25 −15
Changes for .github/workflows/e2e.yml: 25 added lines, 15 removed lines.
Original line number Diff line number Diff line
@@ -21,6 +21,11 @@
# This runs on pushes that touch what gets installed, by hand, and weekly (a moving
# upstream: apt mirrors, k3s's install script and release assets, Adoptium).
# deploy/e2e_check.sh holds the assertions, deploy/e2e_seed.sh the upgrade's seed and its check.
#
# An installer that stops on an upstream download refused or dropped (a GitHub 403, a 5xx, a
# timeout) ends its job green, with a warning on the run: each install step hands a failed
# run's log to deploy/e2e_upstream.sh, which sets E2E_UPSTREAM_SKIP for the job's later
# steps. Every other installer failure, a 404 included, fails the job.
name: e2e

on:
@@ -43,8 +48,8 @@ on:
permissions:
  contents: read

# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` fails the step
# when the installer fails; the default shell reports tee's status.
# An explicit bash runs with -o pipefail, so `bootstrap.sh | tee install.log` carries the
# installer's status to deploy/e2e_upstream.sh; the default shell reports tee's status.
defaults:
  run:
    shell: bash
@@ -107,9 +112,10 @@ jobs:
        run: sudo ufw --force enable

      - name: Install
        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log
        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log || bash deploy/e2e_upstream.sh install.log $?

      - name: Check the install
        if: env.E2E_UPSTREAM_SKIP != '1'
        run: |
          sudo bash deploy/e2e_check.sh install
          for tag in felis-k3s-pods felis-k3s-services felis-panel felis-proxy; do
@@ -124,11 +130,13 @@ jobs:
          grep -q "felis-velocity.jar is the release's" install.log

      - name: Rerun the same assets
        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log
        if: env.E2E_UPSTREAM_SKIP != '1'
        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee rerun.log || bash deploy/e2e_upstream.sh rerun.log $?

      # containerd and the registry already hold every image under the digest the listing
      # names, so nothing is imported or uploaded twice.
      - name: Check the rerun
        if: env.E2E_UPSTREAM_SKIP != '1'
        run: |
          sudo bash deploy/e2e_check.sh rerun
          grep -q 'felis-velocity unchanged; left running' rerun.log
@@ -160,8 +168,9 @@ jobs:
  # The README's command on a fresh host: this commit's installer, as main serves it, on its
  # default channel with nothing pinned. It resolves the newest release and installs that
  # release's binary, images and plugin from its assets, each checked against its
  # SHA256SUMS; the private repo's token is the only thing added. FELIS_INSTALL_MODE picks the
  # mode the setup console would ask for.
  # SHA256SUMS. The workflow's token is the only thing added: it lifts GitHub's limit of 60 API
  # calls an hour for an address without one. FELIS_INSTALL_MODE picks the mode the setup
  # console would ask for.
  readme:
    runs-on: ubuntu-24.04
    timeout-minutes: 120
@@ -181,12 +190,12 @@ jobs:
        if: steps.release.outputs.tag != ''
        env:
          TOKEN: ${{ github.token }}
        run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log
        run: sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee readme.log || bash deploy/e2e_upstream.sh readme.log $?

      # The binary is the release's, so the phase is `release`: its database backup and
      # timers are the release's to have or lack.
      - name: Check the install
        if: steps.release.outputs.tag != ''
        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
        env:
          TAG: ${{ steps.release.outputs.tag }}
          BINARY: ${{ steps.release.outputs.binary }}
@@ -246,10 +255,10 @@ jobs:
          TOKEN: ${{ github.token }}
        run: |
          git show "${TAG}:deploy/bootstrap.sh" > release-bootstrap.sh
          sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log
          sudo FELIS_GITHUB_TOKEN="$TOKEN" FELIS_RELEASE="$TAG" FELIS_INSTALL_MODE=full bash release-bootstrap.sh 2>&1 | tee release.log || bash deploy/e2e_upstream.sh release.log $?

      - name: Check the release install
        if: steps.release.outputs.tag != ''
        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
        env:
          TAG: ${{ steps.release.outputs.tag }}
          BINARY: ${{ steps.release.outputs.binary }}
@@ -260,17 +269,17 @@ jobs:
      # A fresh install's database holds only what its migrations wrote: the seed gives the
      # upgrade's move and its pending migrations existing rows to carry.
      - name: Seed the release's database
        if: steps.release.outputs.tag != ''
        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
        run: sudo bash deploy/e2e_seed.sh seed

      - name: Upgrade to this commit
        if: steps.release.outputs.tag != ''
        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log
        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
        run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee upgrade.log || bash deploy/e2e_upstream.sh upgrade.log $?

      # Both checks run and report: the seeded rows go last, after the restore drill has
      # also put them back from a bundle.
      - name: Check the upgrade
        if: steps.release.outputs.tag != ''
        if: steps.release.outputs.tag != '' && env.E2E_UPSTREAM_SKIP != '1'
        run: |
          rc=0
          sudo bash deploy/e2e_check.sh upgrade || rc=1
@@ -313,9 +322,10 @@ jobs:
          sudo chown -R root:root /opt/felis/src

      - name: Install
        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log
        run: sudo FELIS_SKIP_FETCH=1 FELIS_INSTALL_MODE=full bash /opt/felis/src/deploy/bootstrap.sh 2>&1 | tee source.log || bash deploy/e2e_upstream.sh source.log $?

      - name: Check the install
        if: env.E2E_UPSTREAM_SKIP != '1'
        run: sudo bash deploy/e2e_check.sh install

      - name: Diagnostics

deploy/e2e_upstream.sh

0 → 100644
+56 −0
Changes for deploy/e2e_upstream.sh: 56 added lines, 0 removed lines.
Original line number Diff line number Diff line
#!/bin/bash
# Whether a failed installer run in the e2e workflow stopped on an upstream download that
# was refused or dropped, which says nothing about the commit under test. Each install step
# hands its log and the installer's status here when the installer fails:
#
#   sudo ... bash deploy/bootstrap.sh 2>&1 | tee install.log || bash deploy/e2e_upstream.sh install.log $?
#
# It reads the installer's last ERR-trap line ("bootstrap failed near line N (exit E)") and
# the line just before it. When that line is curl's own error for the same status, and the
# error is one a mirror or GitHub answers with on a bad minute (a connection refused, reset
# or timed out, a 403, 408, 429 or 5xx), it leaves a warning and E2E_UPSTREAM_SKIP=1 in
# $GITHUB_ENV and exits 0: the job's later steps are gated on that variable, so the job ends
# green with the warning on the run. Anything else exits with the installer's status. A 404
# is a URL or a version the installer names, which a commit can break, so it fails.
#
# deploy/e2e_upstream_test.sh holds its checks, against bootstrap.sh's own trap message.
set -euo pipefail

log="${1:?usage: e2e_upstream.sh LOG STATUS}"
status="${2:?usage: e2e_upstream.sh LOG STATUS}"

# The last run of trap lines and the line before it. bash may fire the trap again for each
# function the failure unwinds through, and the EXIT cleanup can print after it. The log
# keeps the installer's colour codes, so nothing is anchored on the left.
code="" before=""
{ read -r code && IFS= read -r before; } < <(awk '
  /bootstrap failed near line [0-9]+ \(exit [0-9]+\)$/ {
    code = $0; sub(/.*\(exit /, "", code); sub(/\)$/, "", code)
    found = code; foundprev = last
    next
  }
  { last = $0 }
  END { if (found != "") { print found; print foundprev } }
' "$log") || true

upstream=""
case "$before" in
  *"curl: (${code}) "*)
    case "$code" in
      # Could not resolve or connect, an HTTP/2 or TLS failure, a transfer cut short, no
      # reply, a send or receive failure, a timeout.
      5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92) upstream=yes ;;
      # -f's HTTP error; curl before 7.75 appends the reason phrase.
      22) if [[ "$before" =~ returned\ error:\ (403|408|429|5[0-9][0-9])([^0-9]|$) ]]; then upstream=yes; fi ;;
    esac
    ;;
esac

if [ -n "$upstream" ]; then
  echo "::warning::the installer stopped on an upstream download (curl: ${before#*curl: }); this job skips its remaining steps"
  echo "E2E_UPSTREAM_SKIP=1" >> "${GITHUB_ENV:-/dev/null}"
  exit 0
fi
echo "the installer failed (exit ${status}) on something other than a refused upstream download; its log is above"
[ "$status" -ne 0 ] 2>/dev/null || status=1
exit "$status"
+130 −0
Changes for deploy/e2e_upstream_test.sh: 130 added lines, 0 removed lines.
Original line number Diff line number Diff line
#!/bin/bash
# Checks for deploy/e2e_upstream.sh. Run it as: bash deploy/e2e_upstream_test.sh
#
# The trap line in the logs is bootstrap.sh's own on_error message, printed the way its warn
# prints it, so rewording it there fails here rather than turning every refused download
# back into a red e2e run. The curl lines are curl's own wording.
set -u

here="$(dirname "$0")"
EU="${1:-${here}/e2e_upstream.sh}"
BS="${2:-${here}/bootstrap.sh}"
[ -f "$EU" ] || { echo "no such script: $EU"; exit 1; }
[ -f "$BS" ] || { echo "no such script: $BS"; exit 1; }
fails=0

expect() { # label needle haystack
  case "$3" in
    *"$2"*) echo "PASS $1" ;;
    *) echo "FAIL $1: expected <$2> in:"; echo "$3"; fails=$((fails + 1)) ;;
  esac
}
status() { # label want got
  if [ "$2" = "$3" ]; then echo "PASS $1"; else echo "FAIL $1: exit $3, want $2"; fails=$((fails + 1)); fi
}
same() { # label want got
  if [ "$2" = "$3" ]; then echo "PASS $1"; else printf 'FAIL %s: got <%s>, want <%s>\n' "$1" "$3" "$2"; fails=$((fails + 1)); fi
}

root="$(mktemp -d)"
trap 'rm -rf "$root"' EXIT

trap_body="$(grep -E '^[[:space:]]*warn "bootstrap failed near line' "$BS" | head -n 1)"
if [ -z "$trap_body" ]; then
  echo "FAIL bootstrap.sh's on_error prints no 'bootstrap failed near line' warning"
  fails=$((fails + 1))
fi
trap_body="${trap_body#*\"}"
trap_body="${trap_body%\"*}"
trapped() { # line code: on_error's warning as the installer prints it
  # shellcheck disable=SC2034 # read by the eval
  local line="$1" code="$2"
  printf '\033[1;33m[warn]\033[0m %s\n' "$(eval "printf '%s' \"${trap_body}\"")"
}
step() { printf '\033[1;36m[felis]\033[0m %s\n' "$1"; }

run() { # log status: prints what the script says; $root/env is its GITHUB_ENV
  : > "$root/env"
  GITHUB_ENV="$root/env" bash "$EU" "$1" "$2" 2>&1
}

# The upgrade job's v0.2.0 install on 2026-10-02: GitHub refused cloudflared's download.
{
  step "release channel: v0.2.0"
  step "installing cloudflared 2026.9.1 (amd64)"
  echo "curl: (22) The requested URL returned error: 403"
  trapped 1727 22
} > "$root/403.log"
out="$(run "$root/403.log" 22)"
status "a 403 on a download skips" 0 $?
expect "  with a warning that quotes curl" "::warning::the installer stopped on an upstream download (curl: (22) The requested URL returned error: 403)" "$out"
same "  and gates the later steps" "E2E_UPSTREAM_SKIP=1" "$(cat "$root/env")"

for e in "(6) Could not resolve host: github.com" \
    "(7) Failed to connect to github.com port 443 after 130 ms: Couldn't connect to server" \
    "(28) Operation timed out after 300000 milliseconds with 0 out of 0 bytes received" \
    "(35) OpenSSL SSL_connect: SSL_ERROR_SYSCALL in connection to objects.githubusercontent.com:443" \
    "(56) Recv failure: Connection reset by peer" \
    "(22) The requested URL returned error: 429" \
    "(22) The requested URL returned error: 503" \
    "(22) The requested URL returned error: 502 Bad Gateway"; do
  code="${e#(}"
  code="${code%%)*}"
  { step "installing k3s"; echo "curl: $e"; trapped 900 "$code"; } > "$root/e.log"
  out="$(run "$root/e.log" "$code")"
  status "curl: $e skips" 0 $?
done

# What the installer prints after the trap, as it exits, changes nothing.
{
  cat "$root/403.log"
  printf '\033[1;33m[warn]\033[0m %s\n' "restored the previous felis binary at /usr/local/bin/felis; the database was not migrated, so rerunning the installer picks up where this run stopped"
} > "$root/cleanup.log"
out="$(run "$root/cleanup.log" 22)"
status "warnings after the trap still skip" 0 $?

# set -E: the trap again for a function the failure unwound through.
{ cat "$root/403.log"; trapped 1790 22; } > "$root/twice.log"
out="$(run "$root/twice.log" 22)"
status "the trap fired twice still skips" 0 $?

# A 404 is a URL or a version the installer names.
{ step "installing cloudflared 2026.9.1 (amd64)"; echo "curl: (22) The requested URL returned error: 404"; trapped 1727 22; } > "$root/404.log"
out="$(run "$root/404.log" 22)"
status "a 404 fails with the installer's status" 22 $?
expect "  and says so" "the installer failed (exit 22) on something other than a refused upstream download" "$out"
same "  and gates nothing" "" "$(cat "$root/env")"

for e in "401" "400" "410"; do
  { echo "curl: (22) The requested URL returned error: $e"; trapped 1727 22; } > "$root/e.log"
  out="$(run "$root/e.log" 22)"
  status "a $e fails" 22 $?
done

# A refused download the installer got past, then a failure of its own that happens to
# exit with curl's status: only the line before the trap counts.
{
  echo "curl: (22) The requested URL returned error: 403"
  step "building felis from source"
  trapped 4100 22
} > "$root/later.log"
out="$(run "$root/later.log" 22)"
status "a refused download earlier in the log fails" 22 $?

# curl's error on the line before, but the installer stopped with another status.
{ echo "curl: (22) The requested URL returned error: 403"; trapped 1727 1; } > "$root/mismatch.log"
out="$(run "$root/mismatch.log" 1)"
status "a trap for another status fails" 1 $?

# The installer's own die, which the trap never sees.
{ step "installing k3s"; printf '\033[1;31m[fail]\033[0m %s\n' "k3s did not become ready"; } > "$root/die.log"
out="$(run "$root/die.log" 1)"
status "the installer's own failure fails" 1 $?
same "  and gates nothing" "" "$(cat "$root/env")"

: > "$root/empty.log"
out="$(run "$root/empty.log" 141)"
status "an empty log fails with the installer's status" 141 $?

echo
if [ "$fails" -eq 0 ]; then echo "ALL PASS"; else echo "${fails} FAILED"; exit 1; fi