diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index de39277..089ab5c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -3,9 +3,10 @@ # # artifacts this commit's release assets, built by deploy/build-release-artifacts.sh the # way release.yml builds a tag's (amd64 only, the runners' architecture) -# install a full install from those assets, the way a release installs, then the same -# assets again (a rerun must converge without restarting what did not change, -# importing or uploading an image again, or touching Docker) +# install a full install from those assets, the way a release installs, on a host with +# ufw enabled, then the same assets again (a rerun must converge without +# restarting what did not change, importing or uploading an image again, or +# touching Docker) # readme the README's one-line install as a new host runs it today: this commit's # installer on its default channel, which installs the newest published # release's binary, images and plugin from that release's assets; skipped until @@ -98,12 +99,21 @@ jobs: # whether bootstrap reached for it. From FELIS_ARTIFACT_DIR every image and the plugin # come out of the assets, so it must not have. (`! grep` is spelled `if grep ... exit 1` # below because bash -e ignores a failing `!` command.) + # ufw, enabled on many Ubuntu hosts, drops every inbound packet no rule admits, the + # pods' traffic to the API server among them; the runner ships it disabled. The + # runner's own traffic is outbound, which ufw lets through. + - name: Enable ufw + run: sudo ufw --force enable + - name: Install run: sudo FELIS_ARTIFACT_DIR="$GITHUB_WORKSPACE/dist" FELIS_INSTALL_MODE=full bash deploy/bootstrap.sh 2>&1 | tee install.log - name: Check the install run: | sudo bash deploy/e2e_check.sh install + for tag in felis-k3s-pods felis-k3s-services felis-panel felis-proxy; do + sudo ufw status | grep -qE "# ${tag} *$" + done if grep -E 'docker already installed|installing docker|docker running' install.log; then exit 1; fi for role in felis limbo lobby paper; do grep -q "felis/${role}:[^ ]* is the release's" install.log diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 2175f29..0347fde 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -1786,10 +1786,31 @@ stop_docker() { # 4. k3s — single node, trimmed for RAM. NetworkPolicy stays ENABLED on purpose: # Felis's minecraft fence (default-deny + allow-rcon/allow-game) is a core # security claim, so we must NOT pass --disable-network-policy. -# On SUSE-family hosts firewalld ships active by default; open the required -# rules rather than disabling the firewall. +# On SUSE-family hosts firewalld ships active by default, and Ubuntu and Debian +# hosts often enable ufw; open the required rules rather than disabling either. # --------------------------------------------------------------------------- + +# ufw_active reports whether ufw is enabled. Enabled, it drops every inbound packet no rule +# admits, the pods' traffic to the API server among them, so an install that left it alone +# waited out its first rollout and died with "did not complete". ufw translates its status +# line, hence the C locale. +ufw_active() { + command -v ufw >/dev/null 2>&1 || return 1 + [ "$(LC_ALL=C ufw status 2>/dev/null | head -n 1)" = "Status: active" ] +} + +# configure_k3s_firewall opens what k3s needs. For ufw that is what k3s's documentation asks: +# the pod and service ranges, plus the panel's NodePort as firewalld gets it. The API +# server's 6443 stays closed to the network, since pods reach it from POD_CIDR. Each ufw rule +# carries a felis- comment, which is how deploy/uninstall.sh finds it again; `ufw allow` +# skips a rule it already has, so a rerun adds nothing. configure_k3s_firewall() { + if ufw_active; then + log "configuring ufw for k3s" + ufw allow from "$POD_CIDR" comment felis-k3s-pods >/dev/null + ufw allow from "$SERVICE_CIDR" comment felis-k3s-services >/dev/null + ufw allow "${FELIS_PANEL_NODEPORT}/tcp" comment felis-panel >/dev/null + fi command -v firewall-cmd >/dev/null 2>&1 || return 0 systemctl is-active --quiet firewalld || return 0 @@ -3773,6 +3794,10 @@ velocity_fingerprint() { } configure_velocity_firewall() { + if ufw_active; then + log "opening ufw port ${FELIS_GAME_PORT}/tcp for the Minecraft proxy" + ufw allow "${FELIS_GAME_PORT}/tcp" comment felis-proxy >/dev/null + fi command -v firewall-cmd >/dev/null 2>&1 || return 0 systemctl is-active --quiet firewalld || return 0 log "opening firewalld port ${FELIS_GAME_PORT}/tcp for the Minecraft proxy" @@ -5483,9 +5508,17 @@ configure_nano_firewall() { ok "nano listens on ${FELIS_NANO_LISTEN} (loopback); no firewall port opened" return 0 fi + local port="${FELIS_NANO_LISTEN##*:}" family=ipv4 + if ufw_active; then + if [ -n "$FELIS_NANO_PROXY_CIDR" ]; then + log "opening ufw port ${port}/tcp to ${FELIS_NANO_PROXY_CIDR} only" + ufw allow proto tcp from "$FELIS_NANO_PROXY_CIDR" to any port "$port" comment felis-nano >/dev/null + else + warn "no FELIS_NANO_PROXY_CIDR, so ufw keeps ${port}/tcp closed; the summary shows how to admit your proxy" + fi + fi command -v firewall-cmd >/dev/null 2>&1 || return 0 systemctl is-active --quiet firewalld || return 0 - local port="${FELIS_NANO_LISTEN##*:}" family=ipv4 # hasJoined takes no token, so the port is opened to the proxy alone. Earlier installers # opened it to every source, and a re-run must not leave that behind. A rule for a previous # FELIS_NANO_PROXY_CIDR is not tracked; it stays until removed by hand. @@ -5560,16 +5593,17 @@ summary_nano() { log "Proxy on another machine? Re-run with the address on the sudo line (sudo drops" log "exported variables):" log " curl -fsSL /deploy/bootstrap.sh | sudo FELIS_NANO_LISTEN=:${port} FELIS_NANO_PROXY_CIDR=/32 bash" - log "firewalld then admits ${port}/tcp ONLY from that proxy — hasJoined takes no auth token," + log "firewalld or ufw then admits ${port}/tcp ONLY from that proxy — hasJoined takes no auth token," log "so an internet-facing one is a free auth relay burning your Mojang egress IP." elif [ -n "$FELIS_NANO_PROXY_CIDR" ]; then - log "Bound to ${FELIS_NANO_LISTEN}. firewalld, where it runs, admits ${port}/tcp only from" + log "Bound to ${FELIS_NANO_LISTEN}. firewalld or ufw, where it runs, admits ${port}/tcp only from" log "${FELIS_NANO_PROXY_CIDR}; any other firewall in front of this host must do the same." else log "WARNING: bound to ${FELIS_NANO_LISTEN} with no FELIS_NANO_PROXY_CIDR. hasJoined takes no auth" log "token, so admit ${port}/tcp from your proxy alone, or anyone can relay their logins" - log "through you. firewalld, where it runs, keeps the port closed until you add:" + log "through you. firewalld or ufw, where it runs, keeps the port closed until you add one of:" log " firewall-cmd --permanent --add-rich-rule='rule family=\"ipv4\" source address=\"/32\" port port=\"${port}\" protocol=\"tcp\" accept' && firewall-cmd --reload" + log " ufw allow proto tcp from /32 to any port ${port} comment felis-nano" fi log "Then edit ${STATE_DIR}/felis.toml to add your [[auth_source]] roots and run:" log " sudo systemctl restart felis-nano" diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index e7684a0..97fe94e 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -585,8 +585,24 @@ fblock="$(awk '/^configure_nano_firewall\(\) \{/,/^}/' "$BS")" [ "$(printf '%s\n' "$fblock" | wc -l)" -lt 40 ] \ || { echo "FAIL: the extracted block is not the function -- did its closing brace move?"; exit 1; } -run_fw() { # listen proxy-cidr port-already-open(0|1) - FELIS_NANO_LISTEN="$1" FELIS_NANO_PROXY_CIDR="$2" OPEN="$3" bash -c ' +# ufw_stub is ufw for a run: UFW=active or inactive, its status line translated outside the +# C locale as ufw translates it; every other call is printed to stderr, which the installer +# leaves alone when it silences ufw's "Rule added". +ufw_stub=' + ufw() { + case "$*" in + status) + if [ "${UFW:-inactive}" != active ]; then echo "Status: inactive" + elif [ "${LC_ALL:-}" = C ]; then echo "Status: active" + else echo "状态:激活"; fi ;; + *) printf "UFW: %s\n" "$*" >&2 ;; + esac + }' +ublock="$(awk '/^ufw_active\(\) \{/,/^}/' "$BS")" +[ -n "$ublock" ] || { echo "FAIL: no ufw_active found in $BS"; exit 1; } + +run_fw() { # listen proxy-cidr port-already-open(0|1) [ufw-state] + FELIS_NANO_LISTEN="$1" FELIS_NANO_PROXY_CIDR="$2" OPEN="$3" UFW="${4:-inactive}" bash -c ' ok() { printf "OK: %s\n" "$*"; } log() { printf "LOG: %s\n" "$*"; } warn() { printf "WARN: %s\n" "$*"; } @@ -595,6 +611,8 @@ run_fw() { # listen proxy-cidr port-already-open(0|1) case "$*" in *--query-port=*) [ "$OPEN" = 1 ]; return ;; esac printf "FW: %s\n" "$*" } + '"$ufw_stub"' + '"$ublock"' '"$kblock"' '"$fblock"' configure_nano_firewall' 2>&1 @@ -627,6 +645,61 @@ case "$(run_fw 127.0.0.1:8081 10.0.0.7/32 1)" in *) echo "PASS a loopback bind leaves firewalld alone" ;; esac +# ufw, enabled on many Ubuntu and Debian hosts, drops what no rule admits. +out="$(run_fw 0.0.0.0:8081 10.0.0.7/32 0 active)" +expect "ufw admits the nano port from the proxy CIDR alone" \ + "UFW: allow proto tcp from 10.0.0.7/32 to any port 8081 comment felis-nano" "$out" +out="$(run_fw 0.0.0.0:8081 '' 0 active)" +expect "no proxy CIDR says ufw keeps the port closed" "WARN: no FELIS_NANO_PROXY_CIDR, so ufw keeps 8081/tcp closed" "$out" +case "$out" in + *"UFW: allow"*) echo "FAIL no proxy CIDR must add no ufw rule:"; echo "$out"; fails=$((fails + 1)) ;; + *) echo "PASS no proxy CIDR adds no ufw rule" ;; +esac +for args in "127.0.0.1:8081 10.0.0.7/32 0 active" "0.0.0.0:8081 10.0.0.7/32 0 inactive"; do + # shellcheck disable=SC2086 # the words are the arguments + case "$(run_fw $args)" in + *UFW:*) echo "FAIL run_fw $args must leave ufw alone"; fails=$((fails + 1)) ;; + *) echo "PASS run_fw $args leaves ufw alone" ;; + esac +done + +k3fblock="$(awk '/^configure_k3s_firewall\(\) \{/,/^}/' "$BS")" +[ -n "$k3fblock" ] || { echo "FAIL: no configure_k3s_firewall found in $BS"; exit 1; } +vfblock="$(awk '/^configure_velocity_firewall\(\) \{/,/^}/' "$BS")" +[ -n "$vfblock" ] || { echo "FAIL: no configure_velocity_firewall found in $BS"; exit 1; } +run_host_fw() { # ufw-state firewalld(0|1) + UFW="$1" FIREWALLD="$2" POD_CIDR=10.42.0.0/16 SERVICE_CIDR=10.43.0.0/16 FELIS_PANEL_NODEPORT=30443 \ + FELIS_GAME_PORT=25565 bash -c ' + log() { printf "LOG: %s\n" "$*"; } + systemctl() { [ "$FIREWALLD" = 1 ]; } + firewall-cmd() { printf "FW: %s\n" "$*"; } + '"$ufw_stub"' + '"$ublock"' + '"$k3fblock"' + '"$vfblock"' + configure_k3s_firewall + configure_velocity_firewall' 2>&1 +} +out="$(run_host_fw active 0)" +expect "ufw admits the pods" "UFW: allow from 10.42.0.0/16 comment felis-k3s-pods" "$out" +expect "ufw admits the services" "UFW: allow from 10.43.0.0/16 comment felis-k3s-services" "$out" +expect "ufw opens the panel NodePort" "UFW: allow 30443/tcp comment felis-panel" "$out" +expect "ufw opens the game port" "UFW: allow 25565/tcp comment felis-proxy" "$out" +case "$out" in + *"UFW: allow 6443"*) echo "FAIL ufw must not open the API server to the network:"; echo "$out"; fails=$((fails + 1)) ;; + *) echo "PASS ufw keeps the API server closed to the network" ;; +esac +case "$out" in + *FW:*--add*) echo "FAIL an inactive firewalld must be left alone:"; echo "$out"; fails=$((fails + 1)) ;; + *) echo "PASS ufw alone leaves firewalld alone" ;; +esac +out="$(run_host_fw inactive 1)" +case "$out" in + *UFW:*) echo "FAIL an inactive ufw must be left alone:"; echo "$out"; fails=$((fails + 1)) ;; + *) echo "PASS an inactive ufw is left alone" ;; +esac +expect " while firewalld still gets the pods" "FW: --permanent --zone=trusted --add-source=10.42.0.0/16" "$out" + expect "a routable bind with no proxy CIDR is warned about" "WARNING: bound to 10.0.0.5:8081 with no FELIS_NANO_PROXY_CIDR" \ "$(run_summary 10.0.0.5:8081)" expect "a routable bind with a proxy CIDR names it" "admits 8081/tcp only from" \ diff --git a/deploy/uninstall.sh b/deploy/uninstall.sh index 523c0b5..810147a 100644 --- a/deploy/uninstall.sh +++ b/deploy/uninstall.sh @@ -133,7 +133,7 @@ confirm() { print_plan() { log "this will remove from $(uname -n):" log " the felis-* systemd units, cloudflared-felis.service, the ${VELOCITY_USER} user," - log " ${OPT_DIR}, ${HOST_BIN}, the felis_postgres and felis_edge nftables tables and the firewalld openings" + log " ${OPT_DIR}, ${HOST_BIN}, the felis_postgres and felis_edge nftables tables and the firewalld and ufw openings" case "$K3S_MODE" in remove) log " k3s, with everything in it (${K3S_BIN_DIR}/k3s-uninstall.sh)" ;; keep) log " Felis's namespaces (${FELIS_NAMESPACES[*]}) and the ${FELIS_CRD} CRD; k3s stays" ;; @@ -244,6 +244,35 @@ remove_firewalld_rules() { # game-port nano-port fi } +# remove_ufw_rules takes back the ufw rules the installer added, found by their felis- +# comments (configure_k3s_firewall and its neighbours in bootstrap.sh). The k3s ranges stay +# when k3s does, and go with it or when it is already gone. ufw lists a rule's IPv6 twin +# under its own number, and each delete renumbers the rules after it, so they go from the +# highest number down. +remove_ufw_rules() { + command -v ufw >/dev/null 2>&1 || return 0 + local status nums n keep_k3s=1 + status="$(LC_ALL=C ufw status numbered 2>/dev/null)" || return 0 + [ "$(printf '%s\n' "$status" | head -n 1)" = "Status: active" ] || return 0 + [ "$K3S_MODE" = keep ] || keep_k3s="" + nums="$(printf '%s\n' "$status" | awk -v keep_k3s="$keep_k3s" ' + match($0, /# felis-[a-z0-9-]+ *$/) { + tag = substr($0, RSTART + 2) + sub(/ +$/, "", tag) + if (keep_k3s != "" && tag ~ /^felis-k3s-/) next + if (match($0, /^\[ *[0-9]+\]/)) { + n = substr($0, RSTART + 1, RLENGTH - 2) + gsub(/ /, "", n) + print n + } + }' | sort -rn)" + [ -n "$nums" ] || return 0 + for n in $nums; do + ufw --force delete "$n" >/dev/null + done + ok "ufw rules removed" +} + # retain_volumes_in_cluster keeps every volume Felis's claims are bound to when the # namespaces go: local-path deletes a Delete-policy volume's directory with its claim. retain_volumes_in_cluster() { @@ -496,6 +525,7 @@ main() { esac remove_nft_tables remove_firewalld_rules "$game" "$nano" + remove_ufw_rules remove_host_files purge_database purge_images diff --git a/deploy/uninstall_test.sh b/deploy/uninstall_test.sh index e2ae492..b155565 100644 --- a/deploy/uninstall_test.sh +++ b/deploy/uninstall_test.sh @@ -114,6 +114,14 @@ run_uninstall() { esac ;; esac } + # UFW_STATUS: what `ufw status numbered` answers; ufw translates it outside the C locale. + ufw() { + case "$*" in + "status numbered") + if [ "${LC_ALL:-}" = C ]; then printf "%s\n" "${UFW_STATUS:-Status: inactive}"; else echo "状态:激活"; fi ;; + *) echo "UFW $*" >> "$calls" ;; + esac + } docker() { echo "DOCKER $*" >> "$calls"; } userdel() { echo "USERDEL $*" >> "$calls"; } uname() { echo testhost; } @@ -313,6 +321,57 @@ for u in $(sed -n 's|^[A-Z_]*="/etc/systemd/system/\([^"]*\)"$|\1|p' "$(dirname expect "the uninstaller removes $u" " $u" " $(printf '%s' "$units" | tr '\n' ' ')" done +# --- ufw: the rules bootstrap added, by the comments bootstrap gives them ------------------ +# The listing is what `ufw status numbered` prints once bootstrap has run: a rule of the +# operator's own first (commented, as an operator may), then each felis- rule bootstrap.sh can +# add and its IPv6 twin. +tags="$(grep -o 'comment felis-[a-z0-9-]*' "$(dirname "$US")/bootstrap.sh" | awk '{ print $2 }' | sort -u)" +case " $(printf '%s ' $tags)" in + *" felis-k3s-pods "*" felis-proxy "*) echo "PASS bootstrap tags its ufw rules" ;; + *) echo "FAIL bootstrap.sh adds no felis-k3s-pods and felis-proxy ufw rules: <$tags>"; fails=$((fails + 1)) ;; +esac +listing="Status: active + + To Action From + -- ------ ---- +[ 1] 22/tcp ALLOW IN Anywhere # ssh" +n=1 +for twin in "" " (v6)"; do + for t in $tags; do + n=$((n + 1)) + listing="${listing} +$(printf '[%2d] Rule%-22s ALLOW IN Anywhere%-19s # %s' "$n" "$twin" "$twin" "$t")" + done +done +listing="${listing} +$(printf '[%2d] 22/tcp (v6) ALLOW IN Anywhere (v6)' "$((n + 1))")" +# numbers : the rule numbers whose line matches, highest first. +numbers() { printf '%s\n' "$1" | grep -E "$2" | sed 's/^\[ *\([0-9]*\)\].*/\1/' | sort -rn | paste -sd ' ' -; } +deletes() { grep '^UFW --force delete' "$root/calls" | awk '{ print $4 }' | paste -sd ' ' -; } + +fresh_host +out="$(UFW_STATUS="$listing" run_uninstall "default felis minecraft" --yes)" +want="$(numbers "$listing" '# felis-')" +[ -n "$want" ] && [ "$(deletes)" = "$want" ] \ + && echo "PASS a Felis-only cluster's uninstall deletes every felis- ufw rule, highest first" \ + || { echo "FAIL a Felis-only cluster: deleted <$(deletes)>, want <$want>"; fails=$((fails + 1)); } +case " $(deletes) " in + *" 1 "* | *" $((n + 1)) "*) echo "FAIL the operator's own ufw rules were deleted: $(deletes)"; fails=$((fails + 1)) ;; + *) echo "PASS the operator's own ufw rules stay" ;; +esac +expect " and says so" "ufw rules removed" "$out" + +fresh_host +UFW_STATUS="$listing" run_uninstall "default kube-system felis minecraft felis-build shop" --yes >/dev/null +[ "$(deletes)" = "$(numbers "$listing" '# felis-' | tr ' ' '\n' | grep -vxE "$(numbers "$listing" '# felis-k3s-' | tr ' ' '|')" | paste -sd ' ' -)" ] \ + && echo "PASS a k3s that stays keeps its pod and service ranges in ufw" \ + || { echo "FAIL a kept k3s: deleted <$(deletes)>, listing:"; echo "$listing"; fails=$((fails + 1)); } + +fresh_host +UFW_STATUS="Status: inactive" run_uninstall "default felis minecraft" --yes >/dev/null +[ -z "$(deletes)" ] && echo "PASS an inactive ufw is left alone" \ + || { echo "FAIL an inactive ufw: deleted <$(deletes)>"; fails=$((fails + 1)); } + # The database's cluster and the move's marker are where bootstrap put them, or keep-data # and purge act on a directory that is not there. paths="$(FELIS_UNINSTALL_SOURCED=1 bash -c '. "$0"; printf "%s %s\n" "$PG_DATA_DIR" "$PG_MOVED_MARKER"' "$US")" diff --git a/docs/operations.md b/docs/operations.md index f390713..b93b33a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -68,6 +68,17 @@ once, then stops with nothing touched **[SH-TESTED]**: `FELIS_PREFLIGHT=warn` reports the same problems as warnings and installs anyway, for a host the checks misjudge. +A host firewall is opened, never turned off **[SH-TESTED]**. With firewalld active the +installer adds the panel NodePort, the game port and 6443, and puts k3s's pod and service +ranges in the trusted zone. With ufw enabled (common on Ubuntu and Debian, and enabled in +the CI install **[CI]**) it admits `10.42.0.0/16` and `10.43.0.0/16`, the panel NodePort and +the game port, each rule commented `felis-…`; 6443 stays closed to the network, since pods +reach the API server from their own range. Felis-nano opens its port to +`FELIS_NANO_PROXY_CIDR` alone in either. `uninstall.sh` removes these again, the k3s ranges +only when k3s goes too. Any other firewall in front of the host must admit the same: +dropped pod traffic shows up as the first rollout timing out ("control-plane rollout did +not complete"). + Two things the host must keep for as long as the install lives: - **Its address.** The install is bound to the IPv4 address it was made on (the k3s @@ -266,8 +277,8 @@ Both modes remove the `felis-*` systemd units and `cloudflared-felis.service`, t Velocity user, `/opt/felis`, `/usr/local/bin/felis`, the release assets an interrupted install left in `/var/lib/felis/artifacts`, the installer's cloudflared binary (unless another unit runs it), the `felis_edge` nftables table (and `felis_postgres`, which -releases before the database moved into k3s loaded) and the firewalld ports the installer -opened. k3s goes with k3s's own `k3s-uninstall.sh` when the cluster holds nothing but +releases before the database moved into k3s loaded), the firewalld ports the installer +opened and its `felis-`-commented ufw rules. k3s goes with k3s's own `k3s-uninstall.sh` when the cluster holds nothing but Felis's namespaces; when it runs anything else only `felis`, `minecraft`, `felis-build` and the MinecraftServer CRD are deleted. `--keep-k3s` and `--remove-k3s` override that choice.