From 1f2a6130ac39db8c21c4b62aeb113ed40c3edb26 Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Sat, 26 Sep 2026 11:30:31 +0800 Subject: [PATCH] =?UTF-8?q?feat(bootstrap):=20=E8=A3=85=E6=9C=BA=E5=89=8D?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E6=80=A7=E9=A2=84=E6=A3=80=E5=B9=B3=E5=8F=B0?= =?UTF-8?q?/=E5=86=85=E5=AD=98/=E7=A3=81=E7=9B=98/=E7=AB=AF=E5=8F=A3/?= =?UTF-8?q?=E7=BD=91=E6=AE=B5/=E5=A4=96=E7=BD=91=EF=BC=8C=E9=97=AE?= =?UTF-8?q?=E9=A2=98=E5=85=A8=E5=88=97=E5=87=BA=E5=90=8E=E5=86=8D=E5=81=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 + README_EN.md | 2 + deploy/bootstrap.sh | 304 ++++++++++++++++++++++++++++++++++++++- deploy/bootstrap_test.sh | 158 +++++++++++++++++++- docs/operations.md | 21 +++ 5 files changed, 483 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c0be761..ef17c88 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap 脚本将自动安装 K3s、部署控制平面并启动设置向导。完成后浏览器访问已配置的域名进入控制面板即可使用。 +动手之前，脚本先检查内存、磁盘、端口、网段冲突、已有的 Kubernetes 和外网连通，把所有问题一次列出并停下，主机上什么都没改（检查项见 [运维手册 §1](docs/operations.md#1-supported-hosts)）。 + > **本仓库当前为私有**，上面这条会返回 404。请改用带凭据的形式；安装器自身也需要同一个 token > 去解析并下载 release，所以用 `sudo -E` 把它带进去： > diff --git a/README_EN.md b/README_EN.md index 49a82c1..abe9c5f 100644 --- a/README_EN.md +++ b/README_EN.md @@ -38,6 +38,8 @@ curl -fsSL https://raw.githubusercontent.com/FelisMC/Felis/main/deploy/bootstrap The script installs K3s, deploys the control plane, and launches a setup wizard. Once done, open your browser at the configured domain. +Before it changes anything, the script checks RAM, disk, ports, network-range clashes, any Kubernetes already there and outbound access, lists every problem at once and stops with the host untouched (the checks are in [operations §1](docs/operations.md#1-supported-hosts)). + > **This repository is currently private**, so the command above returns 404. Use the > credentialed form instead; the installer itself needs the same token to resolve and > download the release, so pass it through with `sudo -E`: diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 86884b5..f61a958 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -115,6 +115,10 @@ # system journal persistent so logs survive a reboot (default: 1) # FELIS_JOURNAL_MAX_USE the persistent journal's size cap, journald's SystemMaxUse # written K|M|G (default: 1G) +# FELIS_PREFLIGHT strict|warn — before touching the host the installer checks RAM, +# disk, ports, other Kubernetes, the pod network ranges and the hosts it +# downloads from, and stops on any problem with the full list (default: +# strict). warn reports them and installs anyway. # PKG_LOCK_TIMEOUT seconds to wait for package-manager locks (default: 900) # APT_LOCK_TIMEOUT legacy alias for PKG_LOCK_TIMEOUT set -Eeuo pipefail @@ -217,6 +221,9 @@ FELIS_OFFSITE_REGION="${FELIS_OFFSITE_REGION:-}" FELIS_OFFSITE_PREFIX="${FELIS_OFFSITE_PREFIX:-}" FELIS_OFFSITE_DB_KEEP="${FELIS_OFFSITE_DB_KEEP:-}" INSTALL_MODE="${FELIS_INSTALL_MODE:-}" +# strict stops the install on any preflight problem (preflight below); warn reports them +# and goes on, for a host the checks misjudge. +FELIS_PREFLIGHT="${FELIS_PREFLIGHT:-strict}" # Loopback by default: hasJoined is an unauthenticated endpoint by protocol (Velocity # sends no token), so a public bind is a free auth relay — anyone can point their own # proxy at it and spend YOUR egress IP on Mojang, until Mojang rate-limits you and your @@ -787,6 +794,10 @@ validate_settings() { validate_listen FELIS_NANO_LISTEN "$FELIS_NANO_LISTEN" validate_cidr FELIS_NANO_PROXY_CIDR "$FELIS_NANO_PROXY_CIDR" validate_offsite_settings + case "$FELIS_PREFLIGHT" in + strict|warn) ;; + *) die "FELIS_PREFLIGHT must be strict or warn (got '${FELIS_PREFLIGHT}')" ;; + esac case "$FELIS_GAME_STACK" in pinned|latest) ;; *) die "FELIS_GAME_STACK must be pinned or latest (got '${FELIS_GAME_STACK}')" ;; @@ -952,6 +963,293 @@ warn_dynamic_node_ip() { fi } +# --------------------------------------------------------------------------- +# Preflight: what would stop the install halfway, checked before the host is touched. +# Every problem is collected and reported together, so a host that needs three fixes +# costs one rerun rather than three; warnings print as they are found and never stop +# the run. FELIS_PREFLIGHT=warn turns the problems into warnings too. +# --------------------------------------------------------------------------- +PREFLIGHT_PROBLEMS=() +preflight_fail() { PREFLIGHT_PROBLEMS+=("$*"); } + +# The smallest host the full stack runs on: k3s, PostgreSQL, the control plane, the +# registry, the proxy (1G heap by default) and the login and lobby servers. A "2 GB" +# VPS reports ~1.9 GiB once the kernel has taken its share; ensure_swap adds swap below +# 2 GiB. Below the recommendation it runs, with little room for players' own servers. +PREFLIGHT_MIN_RAM_KB=1835008 # 1.75 GiB +PREFLIGHT_RECOMMENDED_RAM_KB=3670016 # 3.5 GiB + +# ipv4_to_int prints a dotted quad as a 32-bit integer; anything else fails. +ipv4_to_int() { + local a b c d n + IFS=. read -r a b c d <<<"$1" + for n in "$a" "$b" "$c" "$d"; do + case "$n" in ""|*[!0-9]*) return 1 ;; esac + [ "$n" -le 255 ] || return 1 + done + printf '%s\n' "$(( (a << 24) | (b << 16) | (c << 8) | d ))" +} + +# cidr_overlap reports whether two IPv4 prefixes (a bare address is a /32) share an +# address. +cidr_overlap() { # a/n b/m + local an=32 bn=32 ai bi n mask + case "$1" in */*) an="${1#*/}" ;; esac + case "$2" in */*) bn="${2#*/}" ;; esac + ai="$(ipv4_to_int "${1%/*}")" || return 1 + bi="$(ipv4_to_int "${2%/*}")" || return 1 + n=$(( an < bn ? an : bn )) + mask=$(( n == 0 ? 0 : (0xFFFFFFFF << (32 - n)) & 0xFFFFFFFF )) + [ $(( ai & mask )) -eq $(( bi & mask )) ] +} + +# cgroup_controllers prints the enabled cgroup controllers (v2, else v1). +cgroup_controllers() { + if [ -r /sys/fs/cgroup/cgroup.controllers ]; then + cat /sys/fs/cgroup/cgroup.controllers + elif [ -r /proc/cgroups ]; then + awk '$4 == 1 { print $1 }' /proc/cgroups | tr '\n' ' ' + fi +} + +systemd_is_init() { [ -d /run/systemd/system ]; } + +mem_total_kb() { awk '/^MemTotal:/ { print $2 }' /proc/meminfo; } + +preflight_platform() { + case "$(uname -m)" in + x86_64|amd64|aarch64|arm64) ;; + *) preflight_fail "this host is $(uname -m); Felis publishes its images for amd64 and arm64 only" ;; + esac + systemd_is_init \ + || preflight_fail "systemd is not running as init; the installer manages k3s, the proxy and its timers as systemd units" + # k3s refuses to start without the memory controller. Raspberry Pi OS ships it off. + local controllers + controllers="$(cgroup_controllers)" + case " $controllers " in + *" memory "*) ;; + *) preflight_fail "the memory cgroup controller is off, and k3s will not start without it (on a Raspberry Pi add 'cgroup_memory=1 cgroup_enable=memory' to /boot/firmware/cmdline.txt and reboot)" ;; + esac +} + +preflight_memory() { + local mem_kb + mem_kb="$(mem_total_kb)" + [ -n "$mem_kb" ] || return 0 + if [ "$mem_kb" -lt "$PREFLIGHT_MIN_RAM_KB" ]; then + preflight_fail "this host has $((mem_kb / 1024)) MiB of RAM; the full stack needs at least $((PREFLIGHT_MIN_RAM_KB / 1024)) MiB (a 2 GB host), 4 GB to run players' servers beside it. Felis-nano (FELIS_INSTALL_MODE=nano) fits in far less" + elif [ "$mem_kb" -lt "$PREFLIGHT_RECOMMENDED_RAM_KB" ]; then + warn "preflight: $((mem_kb / 1024)) MiB of RAM runs the platform with little room for players' servers; 4 GB is the comfortable size" + fi +} + +# existing_ancestor prints the nearest directory of $1 that exists, for df. +existing_ancestor() { + local p="$1" + while [ ! -e "$p" ]; do p="$(dirname "$p")"; done + printf '%s\n' "$p" +} + +# path_populated reports whether directory $1 exists with something in it. +path_populated() { [ -n "$(ls -A "$1" 2>/dev/null)" ]; } + +# preflight_disk checks each filesystem the install writes to against what it will +# write there, in MiB: k3s's images and volumes, the database and its bundles under +# /var/lib/felis, the sources, toolchains and proxy under /opt/felis, and Docker's image +# builds (operations.md §2 has the measured sizes). A directory that already holds +# something (a rerun, a reused k3s, Docker's cache from an earlier install) needs only +# the room for what changes. +preflight_disk() { + local spec path need_empty need_populated need line rows="" mount size used avail + for spec in "/var/lib/rancher 10240 3072" "/var/lib/felis 2048 1024" "/opt/felis 3072 1024" \ + "/var/lib/containerd 8192 2048"; do + read -r path need_empty need_populated <<<"$spec" + need="$need_empty" + path_populated "$path" && need="$need_populated" + line="$(df -Pk "$(existing_ancestor "$path")" 2>/dev/null | awk 'NR == 2 { print $6, $2, $3, $4 }')" || continue + [ -n "$line" ] && rows="${rows}${line} $((need * 1024))"$'\n' + done + # One line per filesystem, with what lands on it summed. + rows="$(printf '%s' "$rows" | awk 'NF == 5 { + if (!($1 in need)) order[++n] = $1 + size[$1] = $2; used[$1] = $3; avail[$1] = $4; need[$1] += $5 + } + END { for (i = 1; i <= n; i++) { m = order[i]; print m, size[m], used[m], avail[m], need[m] } }')" + while read -r mount size used avail need; do + [ -n "$mount" ] || continue + if [ "$avail" -lt "$need" ]; then + preflight_fail "${mount} has $((avail / 1024)) MiB free; this install writes about $((need / 1024)) MiB there (k3s under /var/lib/rancher, the image builds under /var/lib/containerd, the database under /var/lib/felis, sources under /opt/felis)" + continue + fi + # k3s's image GC starts collecting at 85% and the kubelet evicts pods below 5% free. + if [ "$size" -gt 0 ] && [ $(( (used + need) * 100 / size )) -ge 85 ]; then + warn "preflight: ${mount} will be over 85% full after the install; k3s starts deleting cached images there and evicts pods near 95%" + fi + done <<<"$rows" +} + +# pid_unit prints the systemd service a process runs in, or nothing. +pid_unit() { + sed -n 's#^.*/\([^/]*\.service\)\(/.*\)\{0,1\}$#\1#p' "/proc/$1/cgroup" 2>/dev/null | tail -n 1 +} + +# port_listeners prints "comm pid unit" for each process listening on TCP port $1. +port_listeners() { + local out pair comm pid + out="$(ss -Hltnp "sport = :$1" 2>/dev/null)" || return 0 + [ -n "$out" ] || return 0 + pair="$(grep -oE '\("[^"]+",pid=[0-9]+' <<<"$out" | sort -u)" || true + if [ -z "$pair" ]; then + printf '%s\n' "? ? ?" + return 0 + fi + while IFS= read -r pair; do + comm="${pair#(\"}"; comm="${comm%%\"*}" + pid="${pair##*pid=}" + printf '%s %s %s\n' "$comm" "$pid" "$(pid_unit "$pid")" + done <<<"$pair" +} + +# preflight_ports refuses a port another program already holds. Listeners of the units +# this installer runs are what a rerun finds, and pass. +preflight_ports() { + command -v ss >/dev/null 2>&1 || { warn "preflight: ss not found; ports are not checked"; return 0; } + local spec port unit label comm pid owner + for spec in \ + "${FELIS_GAME_PORT} felis-velocity.service the Minecraft proxy (FELIS_GAME_PORT)" \ + "6443 k3s.service the Kubernetes API" "6444 k3s.service k3s's supervisor" \ + "10248 k3s.service the kubelet" "10249 k3s.service kube-proxy" "10250 k3s.service the kubelet" \ + "10256 k3s.service kube-proxy" "10257 k3s.service the controller manager" "10259 k3s.service the scheduler" \ + "${FELIS_PANEL_NODEPORT} k3s.service the panel (FELIS_PANEL_NODEPORT)" \ + "${REGISTRY_URL##*:} k3s.service the image registry's loopback port"; do + read -r port unit label <<<"$spec" + while read -r comm pid owner; do + [ -n "$comm" ] || continue + [ "$owner" = "$unit" ] && continue + if [ "$comm" = "?" ]; then + preflight_fail "port ${port} (${label}) is already taken by a process ss cannot name" + else + preflight_fail "port ${port} (${label}) is already taken by ${comm} (pid ${pid}${owner:+, ${owner}}); stop it or move it" + fi + done < <(port_listeners "$port") + done +} + +# preflight_cluster refuses a host that already runs another Kubernetes, or is a k3s +# agent: k3s would fight it for 6443, 10250 and the iptables chains. A k3s server is +# reused as it is. +preflight_cluster() { + local unit units + units="$(systemctl list-units --all --plain --no-legend --type=service 2>/dev/null | awk '{ print $1 }')" || units="" + for unit in rke2-server.service rke2-agent.service k0scontroller.service k0sworker.service \ + snap.microk8s.daemon-kubelite.service kubelet.service k3s-agent.service; do + grep -qx "$unit" <<<"$units" || continue + systemctl is-active --quiet "$unit" 2>/dev/null || continue + case "$unit" in + k3s-agent.service) preflight_fail "this host is a k3s agent (k3s-agent.service); Felis installs a single-node k3s server" ;; + *) preflight_fail "another Kubernetes runs here (${unit}); Felis installs its own k3s, which would fight it for ports 6443 and 10250" ;; + esac + done + if [ -x "$K3S_BIN" ] && [ ! -f "$BOOTSTRAP_DONE" ]; then + log "preflight: k3s is already installed at ${K3S_BIN}; Felis adds its namespaces to that cluster" + fi +} + +# preflight_networks refuses addresses k3s's pod and service ranges would shadow: the +# node's own address, or a network (a Docker bridge, a LAN, a VPN) routed inside them. +# A wider route that merely contains them, a 10.0.0.0/8 VPN say, keeps working for +# everything outside the two ranges, so it is a warning. +preflight_networks() { + local cidr routes line dst rest dev len + for cidr in "$POD_CIDR" "$SERVICE_CIDR"; do + if cidr_overlap "$NODE_IP" "$cidr"; then + preflight_fail "this host's address ${NODE_IP} is inside k3s's range ${cidr}; the cluster cannot route to its own node" + fi + done + routes="$(ip -4 route show 2>/dev/null)" || return 0 + while read -r dst rest; do + case "$dst" in + ""|default) continue ;; + blackhole|unreachable|prohibit|throw|local|broadcast) read -r dst rest <<<"$rest" ;; + esac + line="$dst $rest" + dev="$(awk '{ for (i = 1; i < NF; i++) if ($i == "dev") { print $(i + 1); exit } }' <<<"$line")" + case "$dev" in cni0|flannel.1|flannel-wg|kube-ipvs0) continue ;; esac + len=32 + case "$dst" in */*) len="${dst#*/}" ;; esac + for cidr in "$POD_CIDR" "$SERVICE_CIDR"; do + cidr_overlap "$dst" "$cidr" || continue + if [ "$len" -lt "${cidr#*/}" ]; then + warn "preflight: the route ${dst}${dev:+ via ${dev}} covers k3s's ${cidr}; hosts in ${cidr} on that network will be unreachable from here" + else + preflight_fail "the network ${dst}${dev:+ on ${dev}} lies inside k3s's ${cidr}; pods and that network would be confused (move the Docker network or LAN, or reinstall k3s with other ranges)" + fi + done + done <<<"$routes" +} + +# preflight_hosts prints the hosts this run downloads from, one per line. Package +# mirrors are left out: the package manager names its own. +preflight_hosts() { + printf '%s\n' github.com + if ! bootstrap_from_tui && [ -z "${FELIS_SKIP_FETCH:-}" ] && [ -z "$FELIS_REF_PINNED" ]; then + printf '%s\n' api.github.com + fi + [ -x "$K3S_BIN" ] || printf '%s\n' raw.githubusercontent.com + [ -n "$FELIS_VELOCITY_FORK_JAR" ] || printf '%s\n' fill-data.papermc.io + printf '%s\n' registry-1.docker.io +} + +# host_reachable reports whether an HTTPS connection to $1 can be made: any answer +# counts, a 404 included. Without curl (a minimal image, before install_base) a bare +# TCP connect stands in, unless a proxy is configured, which only curl would use. +host_reachable() { + if command -v curl >/dev/null 2>&1; then + local code + code="$(curl -s -o /dev/null --connect-timeout 5 --max-time 15 -w '%{http_code}' "https://$1/" 2>/dev/null)" || true + [ -n "$code" ] && [ "$code" != 000 ] + return + fi + [ -z "${https_proxy:-}${HTTPS_PROXY:-}" ] || return 0 + timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/443"' "$1" 2>/dev/null +} + +preflight_outbound() { + local host unreachable=() + while IFS= read -r host; do + [ -n "$host" ] || continue + host_reachable "$host" || unreachable+=("$host") + done < <(preflight_hosts) + [ "${#unreachable[@]}" -eq 0 ] && return 0 + preflight_fail "cannot reach ${unreachable[*]} over HTTPS; the install downloads from there (check DNS, the firewall, or set https_proxy)" +} + +preflight() { + log "preflight: checking this host before anything is changed" + PREFLIGHT_PROBLEMS=() + preflight_platform + preflight_memory + preflight_disk + preflight_ports + preflight_cluster + preflight_networks + preflight_outbound + warn_dynamic_node_ip + local n="${#PREFLIGHT_PROBLEMS[@]}" p + if [ "$n" -eq 0 ]; then + ok "preflight passed" + return 0 + fi + if [ "$FELIS_PREFLIGHT" = warn ]; then + for p in "${PREFLIGHT_PROBLEMS[@]}"; do warn "preflight: $p"; done + warn "preflight: FELIS_PREFLIGHT=warn, going on despite ${n} problem(s)" + return 0 + fi + printf '\033[1;31m[fail]\033[0m preflight found %s problem(s); nothing on this host has been changed:\n' "$n" >&2 + for p in "${PREFLIGHT_PROBLEMS[@]}"; do printf ' - %s\n' "$p" >&2; done + die "fix them and rerun the installer (FELIS_PREFLIGHT=warn installs anyway)" +} + pkg_install() { case "$PKG" in apt) apt_get install -y "$@" ;; @@ -4518,10 +4816,12 @@ main() { main_nano return fi + detect_node_ip + # Before the first change to the host: a problem found here costs a rerun, one found + # halfway through costs an install to unwind. + preflight quiet_watchdog pause_package_background_timers - detect_node_ip - warn_dynamic_node_ip ensure_swap install_base ensure_time_sync diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index c1b27f2..6c29330 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -2738,9 +2738,163 @@ for v in 1g G 01G 1.5G 1GB 2T 1024 ""; do expect "journal cap '$v' is refused" "DIE: FELIS_JOURNAL_MAX_USE must be written" "$(check_max_use "$v")" done -order="$(awk '/^main\(\) \{/,/^}/' "$BS" | grep -nE '^[[:space:]]*(detect_node_ip|warn_dynamic_node_ip|install_base|ensure_time_sync|ensure_persistent_journal|install_k3s)$' | sed 's/^[0-9]*:[[:space:]]*//' | tr '\n' ' ')" +order="$(awk '/^main\(\) \{/,/^}/' "$BS" | grep -nE '^[[:space:]]*(detect_node_ip|install_base|ensure_time_sync|ensure_persistent_journal|install_k3s)$' | sed 's/^[0-9]*:[[:space:]]*//' | tr '\n' ' ')" expect "main checks the address, then turns on NTP and the journal once packages install, before k3s" \ - "detect_node_ip warn_dynamic_node_ip install_base ensure_time_sync ensure_persistent_journal install_k3s " "$order" + "detect_node_ip install_base ensure_time_sync ensure_persistent_journal install_k3s " "$order" +expect "preflight is what warns about a leased address" "warn_dynamic_node_ip" "$(awk '/^preflight\(\) \{/,/^}/' "$BS")" + +# --- preflight --------------------------------------------------------------------------- +# The whole preflight section runs against a stubbed host: every fact it reads (RAM, +# df, ss, systemctl, routes, curl) is answered from variables, so each case below is one +# changed fact and the verdict it must change. +pfblock="$(awk '/^PREFLIGHT_PROBLEMS=\(\)$/,/^preflight\(\) \{/' "$BS"; awk '/^preflight\(\) \{/,/^}/' "$BS" | tail -n +2)" +case "$pfblock" in *"preflight_outbound"*"FELIS_PREFLIGHT=warn installs anyway"*) ;; *) echo "FAIL: preflight section not found in $BS"; exit 1 ;; esac +wdblock="$(awk '/^warn_dynamic_node_ip\(\) \{/,/^}/' "$BS")" +pfroot="$(mktemp -d)" +# run_pf runs preflight with the stubbed facts in the environment; each defaults to a +# healthy host: 8 GiB RAM, one 100 GiB filesystem with 60 GiB free, no listeners, no +# other cluster, a LAN route, every download host answering. +run_pf() { + PF_ROOT="$pfroot" bash -c ' + set -Eeuo pipefail + log() { echo "LOG: $*"; } + ok() { echo "OK: $*"; } + warn() { echo "WARN: $*"; } + die() { echo "DIE: $*"; exit 1; } + uname() { echo "${PF_ARCH:-x86_64}"; } + df() { # -Pk path: the longest mount in PF_DF that prefixes path + local row + row="$(printf "%s\n" "${PF_DF:-/ 104857600 41943040 62914560}" | awk -v p="$2" " + { if (index(p, \$1) == 1 && length(\$1) > best) { best = length(\$1); r = \$0 } } END { print r }")" + echo "Filesystem 1024-blocks Used Available Capacity Mounted" + set -- $row + echo "/dev/x $2 $3 $4 50% $1" + } + ss() { # -Hltnp "sport = :PORT" + local port="${2##*:}" + printf "%s\n" "${PF_SS:-}" | grep -F ":${port} " || true + } + systemctl() { + case "$1" in + list-units) printf "%s\n" ${PF_ACTIVE:-} ${PF_STOPPED:-} ;; + is-active) case " ${PF_ACTIVE:-} " in *" ${3:-} "*) return 0 ;; esac; return 1 ;; + esac + } + ip() { + case "$*" in + "-4 route show") printf "%s\n" "${PF_ROUTES:-default via 192.168.1.1 dev eth0 +192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.20}" ;; + *) printf "%s\n" "${PF_ADDRS:-}" ;; + esac + } + curl() { + local host="${!#}" + host="${host#https://}"; host="${host%/}" + case " ${PF_DOWN:-} " in *" ${host} "*) echo 000 ;; *) echo 404 ;; esac + } + bootstrap_from_tui() { return 1; } + FELIS_GAME_PORT=25565 FELIS_PANEL_NODEPORT=30443 REGISTRY_URL=registry.felis.svc:5000 + POD_CIDR=10.42.0.0/16 SERVICE_CIDR=10.43.0.0/16 NODE_IP="${PF_NODE_IP:-192.168.1.20}" + K3S_BIN="$PF_ROOT/k3s" BOOTSTRAP_DONE="$PF_ROOT/bootstrap.done" + FELIS_REF_PINNED="" FELIS_VELOCITY_FORK_JAR="" FELIS_PREFLIGHT="${PF_MODE:-strict}" + '"$wdblock"' + '"$pfblock"' + # The host readers the section defines, answered from the same facts. + systemd_is_init() { [ "${PF_SYSTEMD:-1}" = 1 ]; } + cgroup_controllers() { echo "${PF_CGROUPS:-cpuset cpu io memory pids}"; } + mem_total_kb() { echo "${PF_MEM_KB:-8000000}"; } + pid_unit() { printf "%s\n" "${PF_UNITS:-}" | awk -v p="$1" "\$1 == p { print \$2 }"; } + existing_ancestor() { printf "%s\n" "$1"; } + path_populated() { case " ${PF_POPULATED:-} " in *" $1 "*) return 0 ;; esac; return 1; } + preflight; echo "WENT ON"' 2>&1 +} +out="$(run_pf)" +expect "a healthy host passes preflight" "OK: preflight passed" "$out" +expect "and the install goes on" "WENT ON" "$out" + +out="$(PF_MEM_KB=1000000 run_pf)" +expect "a 1 GB host is refused" "976 MiB of RAM; the full stack needs at least 1792 MiB" "$out" +expect "and nano is named as what fits it" "FELIS_INSTALL_MODE=nano" "$out" +out="$(PF_MEM_KB=1950000 run_pf)" +expect "a 2 GB host installs" "WENT ON" "$out" +expect "with a warning about room for servers" "WARN: preflight: 1904 MiB of RAM runs the platform" "$out" + +# 20 GiB free on the root filesystem: every path lands there and a first install writes +# 10 + 2 + 3 + 8 GiB, so only the sum refuses it. +out="$(PF_DF="/ 104857600 83886080 20971520" run_pf)" +expect "a first install that fits each budget but not their sum is refused" "/ has 20480 MiB free; this install writes about 23552 MiB there" "$out" +out="$(PF_DF="/ 104857600 83886080 20971520 +/var/lib/rancher 52428800 1048576 51380224" run_pf)" +expect "the same host with k3s on its own disk passes" "OK: preflight passed" "$out" +out="$(PF_POPULATED="/var/lib/rancher /var/lib/felis /opt/felis /var/lib/containerd" PF_DF="/ 104857600 96468992 8388608" run_pf)" +expect "a rerun needs only room for new images" "WENT ON" "$out" +expect "and warns when that crosses k3s's image-GC line" "WARN: preflight: / will be over 85% full" "$out" +# The VM after a failed purge: Docker's cache stayed, k3s and /opt/felis went. +out="$(PF_POPULATED="/var/lib/felis /var/lib/containerd" PF_DF="/ 39755776 21827584 17928192" run_pf)" +expect "Docker's cache left by an earlier install is counted as there" "WENT ON" "$out" +out="$(PF_DF="/ 39755776 21827584 17928192" run_pf)" +expect "the same free space is too little for a bare host" "/ has 17508 MiB free; this install writes about 23552 MiB there" "$out" + +ss_line() { printf 'LISTEN 0 4096 0.0.0.0:%s 0.0.0.0:* users:(("%s",pid=%s,fd=7))' "$1" "$2" "$3"; } +out="$(PF_SS="$(ss_line 25565 java 900) +$(ss_line 6443 k3s-server 812)" PF_UNITS="900 felis-velocity.service +812 k3s.service" run_pf)" +expect "the installer's own proxy and k3s pass on a rerun" "OK: preflight passed" "$out" +out="$(PF_SS="$(ss_line 25565 java 901)" PF_UNITS="901 minecraft.service" run_pf)" +expect "someone else's server on the game port is refused" "port 25565 (the Minecraft proxy (FELIS_GAME_PORT)) is already taken by java (pid 901, minecraft.service)" "$out" +out="$(PF_SS="$(ss_line 5000 python3 77)" run_pf)" +expect "a program on the registry's loopback port is refused" "port 5000 (the image registry's loopback port) is already taken by python3 (pid 77)" "$out" + +out="$(PF_ACTIVE="kubelet.service" run_pf)" +expect "a kubeadm node is refused" "another Kubernetes runs here (kubelet.service)" "$out" +out="$(PF_STOPPED="kubelet.service" run_pf)" +expect "a kubelet left installed but stopped is no obstacle" "OK: preflight passed" "$out" +out="$(PF_ACTIVE="k3s-agent.service" run_pf)" +expect "a k3s agent is refused" "this host is a k3s agent" "$out" +: > "$pfroot/k3s"; chmod +x "$pfroot/k3s" +out="$(run_pf)" +expect "an existing k3s server is reused" "LOG: preflight: k3s is already installed" "$out" +rm -f "$pfroot/k3s" + +out="$(PF_NODE_IP=10.42.7.9 run_pf)" +expect "a node address inside the pod range is refused" "10.42.7.9 is inside k3s's range 10.42.0.0/16" "$out" +out="$(PF_ROUTES="default via 192.168.1.1 dev eth0 +10.42.0.0/24 dev cni0 proto kernel scope link src 10.42.0.1 +10.43.0.0/16 dev docker0 proto kernel scope link src 10.43.0.1 linkdown" run_pf)" +expect "a Docker network inside the service range is refused" "the network 10.43.0.0/16 on docker0 lies inside k3s's 10.43.0.0/16" "$out" +case "$out" in *"10.42.0.0/24"*) echo "FAIL k3s's own cni0 route must not count against it"; fails=$((fails + 1)) ;; *) echo "PASS k3s's own cni0 route is its own" ;; esac +out="$(PF_ROUTES="default via 192.168.1.1 dev eth0 +10.0.0.0/8 via 172.20.0.1 dev tun0" run_pf)" +expect "a VPN route around both ranges only warns" "WARN: preflight: the route 10.0.0.0/8 via tun0 covers k3s's 10.42.0.0/16" "$out" +expect "and the install goes on" "WENT ON" "$out" + +out="$(PF_DOWN="github.com fill-data.papermc.io" run_pf)" +expect "unreachable download hosts are named together" "cannot reach github.com fill-data.papermc.io over HTTPS" "$out" + +# Three problems, one report, nothing done. +out="$(PF_MEM_KB=1000000 PF_ARCH=armv7l PF_DOWN=github.com run_pf)" +expect "every problem is in the one report" "preflight found 3 problem(s); nothing on this host has been changed" "$out" +expect "the architecture is one of them" "this host is armv7l" "$out" +case "$out" in *"WENT ON"*) echo "FAIL a failed preflight must stop the install"; fails=$((fails + 1)) ;; *) echo "PASS a failed preflight stops the install" ;; esac +out="$(PF_MEM_KB=1000000 PF_MODE=warn run_pf)" +expect "FELIS_PREFLIGHT=warn reports the problem" "WARN: preflight: this host has 976 MiB" "$out" +expect "and goes on" "WENT ON" "$out" +out="$(PF_CGROUPS="cpuset cpu io pids" run_pf)" +expect "a host without the memory controller is refused" "the memory cgroup controller is off" "$out" +rm -rf "$pfroot" + +ciblock="$(awk '/^ipv4_to_int\(\) \{/,/^}/' "$BS"; awk '/^cidr_overlap\(\) \{/,/^}/' "$BS")" +overlap() { bash -c "$ciblock"' + if cidr_overlap "$0" "$1"; then echo yes; else echo no; fi' "$1" "$2"; } +expect "a /24 inside the pod range overlaps" yes "$(overlap 10.42.5.0/24 10.42.0.0/16)" +expect "the next /16 does not" no "$(overlap 10.44.0.0/16 10.42.0.0/16)" +expect "a /8 around the service range overlaps" yes "$(overlap 10.0.0.0/8 10.43.0.0/16)" +expect "a bare address is a /32" no "$(overlap 10.41.255.255 10.42.0.0/16)" +expect "and is inside its own range" yes "$(overlap 10.42.0.1 10.42.0.0/16)" + +pforder="$(awk '/^main\(\) \{/,/^}/' "$BS" | grep -nE '^[[:space:]]*(detect_node_ip|preflight|quiet_watchdog|pause_package_background_timers|ensure_swap|install_base)$' | sed 's/^[0-9]*:[[:space:]]*//' | tr '\n' ' ')" +expect "preflight runs once the node address is known and before the first change" \ + "detect_node_ip preflight quiet_watchdog pause_package_background_timers ensure_swap install_base " "$pforder" # --------------------------------------------------------------------------------------- if [ "$fails" -eq 0 ]; then diff --git a/docs/operations.md b/docs/operations.md index 8a18281..c98bf90 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -39,6 +39,27 @@ cloudflared is left as it is, see §4): 32-bit hosts are not supported: there is no k3s, JRE or Go build the installer will fetch for them. +Before it changes anything the installer checks the host and reports every problem at +once, then stops with nothing touched **[SH-TESTED]**: + +- the architecture, systemd as init, and the memory cgroup controller k3s needs; +- RAM: under 1.75 GiB is refused (a "2 GB" VPS passes), under 3.5 GiB is a warning; +- free disk on each filesystem it writes to, summed when they share one: about 23 GiB + on a bare host, 7 GiB for a rerun, a directory that already holds data (Docker's cache, + a reused k3s) counting at the rerun size; a filesystem that would end over 85%, where + k3s starts deleting cached images, is a warning; +- the ports it will listen on: the game port, the panel NodePort, k3s's 6443/6444 and + 10248–10259 and the registry's loopback 5000. A port held by the installer's own + proxy or k3s is a rerun and passes; +- another Kubernetes (kubelet, RKE2, k0s, MicroK8s) or a k3s agent on the host; +- the node address or a routed network inside k3s's `10.42.0.0/16` and `10.43.0.0/16` + (a Docker network there is the usual case); a wider route such as a `10.0.0.0/8` VPN + is a warning; +- HTTPS to the hosts it downloads from (GitHub, PaperMC's download API, Docker Hub). + +`FELIS_PREFLIGHT=warn` reports the same problems as warnings and installs anyway, for a +host the checks misjudge. + Two things the host must keep for as long as the install lives: - **Its address.** The install is bound to the IPv4 address it was made on (the