diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 8194e67..095f02c 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -1314,18 +1314,27 @@ preflight_hosts() { fi } -# host_reachable reports whether an HTTPS connection to $1 can be made: any answer -# counts, a 404 included. Without curl (a minimal image, before install_base) a bare -# TCP connect stands in, unless a proxy is configured, which only curl would use. +# host_reachable reports whether an HTTPS connection to $1 can be made: a TLS handshake that +# completes, which any answer (a 404 included) comes after, and which a server that then sits +# on the request has made too. It tries three times, two seconds apart, as the downloads it +# stands for retry: one dropped probe must not stop an install. Without curl (a minimal +# image, before install_base) a bare TCP connect stands in, unless a proxy is configured, +# which only curl would use. host_reachable() { - if command -v curl >/dev/null 2>&1; then - local code - code="$(curl -s -o /dev/null --connect-timeout 5 --max-time 15 -w '%{http_code}' "https://$1/" 2>/dev/null)" || true - [ -n "$code" ] && [ "$code" != 000 ] - return - fi - [ -z "${https_proxy:-}${HTTPS_PROXY:-}" ] || return 0 - timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/443"' "$1" 2>/dev/null + local try tls + for try in 1 2 3; do + [ "$try" = 1 ] || sleep 2 + if command -v curl >/dev/null 2>&1; then + # The handshake's time, 0.000000 until one completes, whatever curl then exits with. + tls="$(curl -s -o /dev/null --connect-timeout 5 --max-time 15 -w '%{time_appconnect}' "https://$1/" 2>/dev/null)" || true + [ -n "${tls//[0.]/}" ] && return 0 + elif [ -n "${https_proxy:-}${HTTPS_PROXY:-}" ]; then + return 0 + elif timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/443"' "$1" 2>/dev/null; then + return 0 + fi + done + return 1 } preflight_outbound() { diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index ecf461a..107af14 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -2746,11 +2746,21 @@ run_pf() { *) printf "%s\n" "${PF_ADDRS:-}" ;; esac } + # curl -w "%{time_appconnect}": a host in PF_DOWN never answers, one in PF_FLAKY drops + # its first probe, one in PF_STALL completes TLS and then times out. Each probe of those + # is counted in $PF_ROOT/tries.. curl() { local host="${!#}" host="${host#https://}"; host="${host%/}" - case " ${PF_DOWN:-} " in *" ${host} "*) echo 000 ;; *) echo 404 ;; esac + case " ${PF_DOWN:-} ${PF_FLAKY:-} ${PF_STALL:-} " in *" ${host} "*) printf x >> "$PF_ROOT/tries.${host}" ;; esac + case " ${PF_FLAKY:-} " in *" ${host} "*) + [ "$(cat "$PF_ROOT/tries.${host}")" = x ] && printf 0.000000 && return 7 + printf 0.300000; return ;; + esac + case " ${PF_STALL:-} " in *" ${host} "*) printf 0.481676; return 28 ;; esac + case " ${PF_DOWN:-} " in *" ${host} "*) printf 0.000000; return 7 ;; *) printf 0.300000 ;; esac } + sleep() { echo "SLEEP: $*"; } bootstrap_from_tui() { [ -n "${PF_TUI:-}" ]; } '"$(awk '/^use_release_binary\(\) \{/,/^}/' "$BS")"' FELIS_GAME_PORT=25565 FELIS_PANEL_NODEPORT=30443 REGISTRY_URL=registry.felis.svc:5000 PG_HOST_PORT=15432 @@ -2834,6 +2844,21 @@ expect "and the install goes on" "WENT ON" "$out" out="$(PF_DOWN="github.com fill-data.papermc.io" run_pf)" expect "unreachable download hosts are named together" "cannot reach github.com fill-data.papermc.io over HTTPS" "$out" +# The VM's rerun after an upgrade: one probe of fill-data.papermc.io timed out, a second a +# minute later answered in under a second, and preflight had refused the install. +rm -f "$pfroot"/tries.* +out="$(PF_FLAKY=fill-data.papermc.io run_pf)" +expect "a download host that drops one probe is tried again" "WENT ON" "$out" +expect "two seconds later" "SLEEP: 2" "$out" +[ "$(cat "$pfroot/tries.fill-data.papermc.io")" = xx ] && echo "PASS and not again once it answered" \ + || { echo "FAIL: a host that answered its second probe was probed $(wc -c < "$pfroot/tries.fill-data.papermc.io") times"; fails=$((fails + 1)); } +out="$(PF_STALL=fill-data.papermc.io run_pf)" +expect "a host that completes TLS and then sits on the request is reachable" "WENT ON" "$out" +rm -f "$pfroot"/tries.* +out="$(PF_DOWN=fill-data.papermc.io run_pf)" +expect "a host that never answers is refused" "cannot reach fill-data.papermc.io over HTTPS" "$out" +[ "$(cat "$pfroot/tries.fill-data.papermc.io")" = xxx ] && echo "PASS after three probes" \ + || { echo "FAIL: a host that never answered was probed $(wc -c < "$pfroot/tries.fill-data.papermc.io") times"; fails=$((fails + 1)); } # An install from a release's assets builds nothing: no Docker cache under /var/lib/containerd, # and Docker Hub only as the fallback for an image the release cannot supply. The downloaded diff --git a/docs/operations.md b/docs/operations.md index 153d62a..55f965f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -60,9 +60,10 @@ once, then stops with nothing touched **[SH-TESTED]**: (a Docker network there is the usual case); a wider route such as a `10.0.0.0/8` VPN is a warning; - HTTPS to the hosts it downloads from: GitHub and PaperMC's download API always, Docker - Hub when it builds images on the host. Installing a release, an unreachable Docker Hub - is a warning (it is needed only if an asset turns out unusable); from - `FELIS_ARTIFACT_DIR` it is not checked. + Hub when it builds images on the host. A host counts as reachable once a TLS handshake + with it completes, and each gets three tries two seconds apart. Installing a release, an + unreachable Docker Hub is a warning (it is needed only if an asset turns out unusable); + from `FELIS_ARTIFACT_DIR` it is not checked. `FELIS_PREFLIGHT=warn` reports the same problems as warnings and installs anyway, for a host the checks misjudge.