diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 17ad001..7162417 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -87,7 +87,9 @@ # previous one; :demo when the version is unknown — never :latest; # anything not under the registry is used as-is but is NOT # mirrored into it, so it has no pull source after an image GC) -# FELIS_ROOT_DOMAIN deployment root domain (default: .nip.io) +# FELIS_ROOT_DOMAIN deployment root domain (default: .nip.io; first install +# only -- a rerun keeps the installed one, and sudo felis domain set +# moves it) # FELIS_PANEL_NODEPORT local HTTPS panel/API NodePort (default: 30443) # FELIS_EGRESS_MODE loadbalancer|nodeport (default: nodeport — no MetalLB on a demo box) # FELIS_BACKUP_PVC world-archive PVC the installer renders and felis-api hands to its @@ -914,19 +916,19 @@ detect_node_ip() { # installer re-derived the domain from scratch every time and defaulted to nip.io, so # re-running it on a live install -- the only way to move felis-api to a newer release, # and what `felis update` points operators at -- rewrote root_domain, panel_hostname and - # admin_hostname to nip.io names. ensure_panel_tls_cert is write-once and kept serving a - # certificate for the OLD hostnames, so the console stopped matching its own cert, with - # no re-domain flow to recover through. Secrets never had this problem: - # load_or_make_secrets has always sourced secrets.env before generating anything. + # admin_hostname to nip.io names while the write-once panel certificate kept the old + # ones. Secrets never had this problem: load_or_make_secrets has always sourced + # secrets.env before generating anything. + # + # A different FELIS_ROOT_DOMAIN on an installed host is refused. The name is on more + # surfaces than this script rewrites -- the panel certificate, the felis-api-tls and + # felis-config Secrets, the proxy's felis-link.properties, the login gate's CR env, the + # Cloudflare tunnel -- and a half-moved install serves a certificate for the old names. + # `felis domain set` moves all of them and `felis domain check` proves each one. local persisted persisted="$(persisted_root_domain)" if [ -n "${FELIS_ROOT_DOMAIN:-}" ] && [ -n "$persisted" ] && [ "$FELIS_ROOT_DOMAIN" != "$persisted" ]; then - # Deliberate re-domain. Allowed -- there is no other route to it -- but it is not a - # thing this script finishes: the panel certificate, the two secrets, the velocity - # config and the login CR all still carry the old name. - warn "FELIS_ROOT_DOMAIN (${FELIS_ROOT_DOMAIN}) differs from the installed ${persisted}." - warn "This re-domains the install. The write-once panel certificate is NOT reissued and" - warn "will keep the old hostnames; the proxy and login config need the same treatment." + die "FELIS_ROOT_DOMAIN (${FELIS_ROOT_DOMAIN}) differs from the installed ${persisted}. The installer keeps the installed domain; to move the install, rerun it without FELIS_ROOT_DOMAIN, then run: sudo felis domain set ${FELIS_ROOT_DOMAIN} (docs/operations.md, Changing the root domain)" fi FELIS_ROOT_DOMAIN="${FELIS_ROOT_DOMAIN:-${persisted:-${NODE_IP}.nip.io}}" if [ -n "$persisted" ] && [ "$FELIS_ROOT_DOMAIN" = "$persisted" ]; then @@ -2645,8 +2647,10 @@ felis_internal_ip() { } write_velocity_config() { - local api_ip tmp + local api_ip tmp panel_host admin_host api_ip="$(felis_internal_ip)" + panel_host="$(auth_hostname panel_hostname "console.${FELIS_ROOT_DOMAIN}")" + admin_host="$(auth_hostname admin_hostname "op.console.${FELIS_ROOT_DOMAIN}")" prepare_velocity_layout tmp="$(mktemp -d)" @@ -2715,8 +2719,8 @@ EOF api-base-url=http://${api_ip}:8081 service-token=${SERVICE_TOKEN} root-domain=${FELIS_ROOT_DOMAIN} -panel-hostname=console.${FELIS_ROOT_DOMAIN} -admin-hostname=op.console.${FELIS_ROOT_DOMAIN} +panel-hostname=${panel_host} +admin-hostname=${admin_host} login-server=${LOGIN_SERVER} lobby-server=${LOBBY_SERVER} EOF @@ -3111,8 +3115,9 @@ ensure_panel_tls_cert() { return 0 fi - local cn conf - cn="op.console.${FELIS_ROOT_DOMAIN}" + local cn panel_host conf + cn="$(auth_hostname admin_hostname "op.console.${FELIS_ROOT_DOMAIN}")" + panel_host="$(auth_hostname panel_hostname "console.${FELIS_ROOT_DOMAIN}")" conf="$(mktemp)" remember_temp "$conf" cat > "$conf" < / op.console.. A wholesale rewrite dropped +# all of them on every re-run: behind Cloudflare the rate limit fell back to the tunnel's +# address, one bucket for everyone. Header-and-keys only, like persisted_smtp_block, and +# the same first-readable-file rule. +persisted_auth_lines() { + local f out + for f in "${STATE_DIR}/felis.host.toml" "${STATE_DIR}/felis.pod.toml"; do + [ -r "$f" ] || continue + out="$(awk ' + /^[[:space:]]*\[/ { + if (inauth) exit + inauth = ($0 ~ /^[[:space:]]*\[auth\][[:space:]]*$/) + next + } + inauth && /^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*=/ { print } + ' "$f")" + [ -n "$out" ] || continue + printf '%s\n' "$out" + return 0 + done +} + +# auth_hostname echoes the installed value of an [auth] hostname key, or the default +# derived from the root domain: `auth_hostname panel_hostname "console.${FELIS_ROOT_DOMAIN}"`. +# The lines are read into a variable before matching, as in auth_lines. +auth_hostname() { + local lines v + lines="$(persisted_auth_lines)" + v="$(awk -F'"' -v k="$1" '$1 ~ "^[[:space:]]*" k "[[:space:]]*=[[:space:]]*$" { print $2; exit }' <<<"$lines")" + printf '%s' "${v:-$2}" +} + +# auth_lines is the body of the [auth] section this run writes: the carried keys, after +# the two hostnames derived from the root domain when the carry lacks them. A first +# install gets exactly the two derived lines; a re-run reproduces the carried section. +# The keys are matched in a here-string, never `printf | grep -q`: grep exits at the +# match, printf dies of SIGPIPE on the lines after it, pipefail fails the test, and a +# second admin_hostname then breaks the file (see postgres_installed). +auth_lines() { + local carried + carried="$(persisted_auth_lines)" + grep -Eq '^[[:space:]]*admin_hostname[[:space:]]*=' <<<"$carried" || + printf 'admin_hostname = "op.console.%s"\n' "$FELIS_ROOT_DOMAIN" + grep -Eq '^[[:space:]]*panel_hostname[[:space:]]*=' <<<"$carried" || + printf 'panel_hostname = "console.%s"\n' "$FELIS_ROOT_DOMAIN" + if [ -n "$carried" ]; then printf '%s\n' "$carried"; fi +} + # persisted_auth_source_blocks echoes the [[auth_source]] tables an earlier run left # behind, or the LittleSkin default when there is no earlier felis.toml at all. The # list is the operator's: it is the only way to add or drop a Yggdrasil root on a full @@ -3301,12 +3358,16 @@ offsite_enabled() { } write_felis_toml() { - local target="$1" db_host="$2" smtp_block auth_source_blocks registry_block archive_block offsite_section + local target="$1" db_host="$2" smtp_block auth_body auth_source_blocks registry_block archive_block offsite_section smtp_block="$(persisted_smtp_block)" if [ -n "$smtp_block" ]; then log "carrying forward the configured [smtp] relay" smtp_block="${smtp_block}"$'\n' # keep a blank line before the next section fi + if [ -n "$(persisted_auth_lines)" ]; then + log "carrying forward the configured [auth] keys" + fi + auth_body="$(auth_lines)" auth_source_blocks="$(persisted_auth_source_blocks)" registry_block="$(persisted_registry_block)" if [ -n "$registry_block" ]; then @@ -3326,8 +3387,9 @@ write_felis_toml() { fi cat > "$target" < [server] listen = "0.0.0.0:8080" root_domain = "${FELIS_ROOT_DOMAIN}" @@ -3356,8 +3418,7 @@ store = "tarLocal" local_path = "${FELIS_ARCHIVE_LOCAL_PATH}" ${archive_block} ${offsite_section}[auth] -admin_hostname = "op.console.${FELIS_ROOT_DOMAIN}" -panel_hostname = "console.${FELIS_ROOT_DOMAIN}" +${auth_body} ${smtp_block} # Third-party Yggdrasil sources federated by the hasJoined multiplexer. Mojang is @@ -4092,8 +4153,8 @@ summary() { echo systemctl --no-pager --full status felis-velocity 2>/dev/null | head -n 4 || true echo - log "Player panel: https://console.${FELIS_ROOT_DOMAIN} — served on 443 once your edge/Cloudflare Tunnel routes it here." - log "Operator console (Op/Admin/Owner): https://op.console.${FELIS_ROOT_DOMAIN} — the Owner runs 'felis setup' and onboards here." + log "Player panel: https://$(auth_hostname panel_hostname "console.${FELIS_ROOT_DOMAIN}") — served on 443 once your edge/Cloudflare Tunnel routes it here." + log "Operator console (Op/Admin/Owner): https://$(auth_hostname admin_hostname "op.console.${FELIS_ROOT_DOMAIN}") — the Owner runs 'felis setup' and onboards here." log "Before the edge is ready: direct + self-signed at https://${NODE_IP}:${FELIS_PANEL_NODEPORT} (browser will warn on first visit)." log "Minecraft address: ${NODE_IP}:${FELIS_GAME_PORT} (point mc.${FELIS_ROOT_DOMAIN} here)" log "The proxy authenticates against Mojang and forwards the verified profile to the" diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index 5723b16..e9ae696 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -1375,12 +1375,16 @@ prblock="$(awk '/^persisted_registry_block\(\) \{/,/^}/' "$BS")" pablock="$(awk '/^persisted_archive_block\(\) \{/,/^}/' "$BS")" poblock="$(awk '/^persisted_offsite_block\(\) \{/,/^}/' "$BS")" oblock="$(awk '/^offsite_block\(\) \{/,/^}/' "$BS")" -{ [ -n "$wrblock" ] && [ -n "$prblock" ] && [ -n "$pablock" ] && [ -n "$poblock" ] && [ -n "$oblock" ]; } \ - || { echo "FAIL: write_felis_toml / persisted_{registry,archive,offsite}_block / offsite_block not found in $BS"; exit 1; } +palblock="$(awk '/^persisted_auth_lines\(\) \{/,/^}/' "$BS")" +ahblock="$(awk '/^auth_hostname\(\) \{/,/^}/' "$BS")" +alblock="$(awk '/^auth_lines\(\) \{/,/^}/' "$BS")" +{ [ -n "$wrblock" ] && [ -n "$prblock" ] && [ -n "$pablock" ] && [ -n "$poblock" ] && [ -n "$oblock" ] && + [ -n "$palblock" ] && [ -n "$ahblock" ] && [ -n "$alblock" ]; } \ + || { echo "FAIL: write_felis_toml / persisted_{registry,archive,offsite}_block / offsite_block / the [auth] helpers not found in $BS"; exit 1; } # The blocks quote themselves (the awk program uses single quotes), so they are # sourced from a file instead of being spliced into a single-quoted bash -c. fnfile="$(mktemp)" -printf '%s\n%s\n%s\n%s\n%s\n' "$prblock" "$pablock" "$poblock" "$oblock" "$wrblock" > "$fnfile" +printf '%s\n' "$prblock" "$pablock" "$poblock" "$oblock" "$palblock" "$ahblock" "$alblock" "$wrblock" > "$fnfile" rdir="$(mktemp -d)" cat > "$rdir/felis.host.toml" <<'TOML' @@ -1413,8 +1417,10 @@ endpoint = "https://objects.example" bucket = "felis-offsite" TOML -run_write() { # out-file - STATE_DIR="$rdir" OUT_TOML="$1" FNFILE="$fnfile" bash -c ' +run_write() { # out-file [state-dir]; under the installer's shell options and ERR trap + STATE_DIR="${2:-$rdir}" OUT_TOML="$1" FNFILE="$fnfile" bash -c ' + set -Eeuo pipefail + trap '\''echo "ERR near line $LINENO (exit $?)" >&2'\'' ERR log() { :; } persisted_smtp_block() { :; } persisted_auth_source_blocks() { :; } @@ -1467,6 +1473,164 @@ else fails=$((fails + 1)) fi +# --- installer re-runs keep [auth]; a different root domain is refused ------------------ +# The Cloudflare edge setup writes access_jwt_aud and client_ip_header into [auth] (the +# header is what the sign-in rate limit keys on), and an operator may serve the admin +# console on a name of their own. A re-run rewrote [auth] from the root domain and dropped +# all of it. The hostnames also feed the proxy's felis-link.properties and the panel +# certificate, which must follow the carried names. + +adir="$(mktemp -d)" +cat > "$adir/felis.host.toml" <<'TOML' +[server] +root_domain = "r.example.com" + +[auth] +admin_hostname = "ops.example.org" +panel_hostname = "console.r.example.com" +access_jwt_aud = "aud-0123" +client_ip_header = "CF-Connecting-IP" + +[smtp] +host = "mail.example" +TOML +err="$(run_write "$adir/out.toml" "$adir" 2>&1)" +if [ -z "$err" ]; then + echo "PASS writing the config runs nothing (no command substitution in the template)" +else + echo "FAIL: writing the config printed:"; printf '%s\n' "$err"; fails=$((fails + 1)) +fi +out="$(cat "$adir/out.toml")" +expect "a re-run keeps every [auth] key, the edge's and the operator's" '[auth] +admin_hostname = "ops.example.org" +panel_hostname = "console.r.example.com" +access_jwt_aud = "aud-0123" +client_ip_header = "CF-Connecting-IP" + +' "$out" +case "$out" in + *op.console.r.example.com*) + echo "FAIL: a derived admin hostname was written next to the carried one:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;; + *mail.example*) + echo "FAIL: the [auth] carry ran into the next section:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;; + *) echo "PASS the carried names replace the derived ones and the carry stops at [smtp]" ;; +esac +cp "$adir/out.toml" "$adir/felis.host.toml" +run_write "$adir/out2.toml" "$adir" +if cmp -s "$adir/out.toml" "$adir/out2.toml"; then + echo "PASS a carried-forward [auth] converges (the second re-run is a no-op)" +else + echo "FAIL: carrying [auth] is not idempotent"; diff "$adir/out.toml" "$adir/out2.toml" | head + fails=$((fails + 1)) +fi + +printf '[server]\nroot_domain = "r.example.com"\n\n[auth]\naccess_jwt_aud = "aud-0123"\n' > "$adir/felis.host.toml" +run_write "$adir/out.toml" "$adir" +expect "an [auth] without the hostnames gets the derived ones and keeps the rest" '[auth] +admin_hostname = "op.console.r.example.com" +panel_hostname = "console.r.example.com" +access_jwt_aud = "aud-0123"' "$(cat "$adir/out.toml")" +rm -f "$adir/felis.host.toml" +run_write "$adir/out.toml" "$adir" +expect "a first install writes the two derived hostnames" '[auth] +admin_hostname = "op.console.r.example.com" +panel_hostname = "console.r.example.com" + +' "$(cat "$adir/out.toml")" + +# The proxy's link properties and the panel certificate take the carried names. +wvblock="$(awk '/^write_velocity_config\(\) \{/,/^}/' "$BS")" +ptblock="$(awk '/^ensure_panel_tls_cert\(\) \{/,/^}/' "$BS")" +{ [ -n "$wvblock" ] && [ -n "$ptblock" ]; } \ + || { echo "FAIL: write_velocity_config / ensure_panel_tls_cert not found in $BS"; exit 1; } +printf '%s\n' "$wvblock" "$ptblock" >> "$fnfile" +cat > "$adir/felis.host.toml" <<'TOML' +[auth] +admin_hostname = "ops.example.org" +panel_hostname = "play.example.org" +TOML +run_surfaces() { # state-dir out-dir; under the installer's shell options and ERR trap + STATE_DIR="$1" VOUT="$2" TMPDIR="$2" FNFILE="$fnfile" bash -c ' + set -Eeuo pipefail + trap '\''echo "ERR near line $LINENO (exit $?)" >&2'\'' ERR + log() { :; }; ok() { :; }; remember_temp() { :; } + felis_internal_ip() { printf 10.43.0.1; } + prepare_velocity_layout() { :; } + atomic_install_file() { cp "$1" "$VOUT/$(basename "$2")"; } + openssl() { for a in "$@"; do :; done; cp "$a" "$VOUT/openssl.cnf"; touch "$VOUT/k" "$VOUT/c"; } + . "$FNFILE" + FELIS_ROOT_DOMAIN=r.example.com FORWARDING_SECRET=f SERVICE_TOKEN=t LOGIN_SERVER=login \ + LOBBY_SERVER=lobby FELIS_GAME_PORT=25565 VELOCITY_DIR=/v VELOCITY_USER=v NODE_IP=10.0.0.5 \ + PANEL_TLS_CERT="$VOUT/c" PANEL_TLS_KEY="$VOUT/k" + write_velocity_config + ensure_panel_tls_cert' +} +mkdir "$adir/v" +run_surfaces "$adir" "$adir/v" +out="$(cat "$adir/v/felis-link.properties" 2>&1)" +expect "the proxy routes the carried panel hostname" "panel-hostname=play.example.org" "$out" +expect "the proxy routes the carried admin hostname" "admin-hostname=ops.example.org" "$out" +expect "the proxy keeps the root domain" "root-domain=r.example.com" "$out" +out="$(cat "$adir/v/openssl.cnf" 2>&1)" +expect "a regenerated panel certificate names the carried hostnames" 'DNS.1 = ops.example.org +DNS.2 = play.example.org' "$out" + +# The hostnames on the section's first lines and the section far past one pipe buffer: +# a `printf | grep -q` or `| awk exit` over it has the reader exit while printf is still +# writing, which is certain here and a scheduling race on a real host. The config must +# still carry each name once, and +# the helpers must fail nothing along the way (the installer logs every failed command). +{ printf '[auth]\nadmin_hostname = "ops.example.org"\npanel_hostname = "play.example.org"\n' + seq 1 200000 | sed 's/.*/k& = "v"/'; } > "$adir/felis.host.toml" +err="$(run_write "$adir/out.toml" "$adir" 2>&1)" +for key in admin_hostname panel_hostname; do + got="$(grep "^${key} = " "$adir/out.toml")" + case "$got" in + *$'\n'*) echo "FAIL: a long carried [auth] wrote ${key} twice:"; printf '%s\n' "$got"; fails=$((fails + 1)) ;; + "") echo "FAIL: a long carried [auth] lost ${key}"; fails=$((fails + 1)) ;; + *) echo "PASS a long carried [auth] writes ${key} once: ${got}" ;; + esac +done +expect "a long carried [auth] keeps the operator's admin name" 'admin_hostname = "ops.example.org"' "$(grep '^admin_hostname' "$adir/out.toml")" +rm -rf "$adir/v"; mkdir "$adir/v" +err="${err}$(run_surfaces "$adir" "$adir/v" 2>&1)" +expect "a long carried [auth] still routes the carried panel hostname" "panel-hostname=play.example.org" "$(cat "$adir/v/felis-link.properties" 2>&1)" +if [ -z "$err" ]; then + echo "PASS reading a long [auth] fails no command" +else + echo "FAIL: reading a long [auth] printed:"; printf '%s\n' "$err" | head -5; fails=$((fails + 1)) +fi + +# A different FELIS_ROOT_DOMAIN on an installed host is refused with the way through. +dnblock="$(awk '/^detect_node_ip\(\) \{/,/^}/' "$BS")" +prdblock="$(awk '/^persisted_root_domain\(\) \{/,/^}/' "$BS")" +{ [ -n "$dnblock" ] && [ -n "$prdblock" ]; } || { echo "FAIL: detect_node_ip / persisted_root_domain not found in $BS"; exit 1; } +printf '%s\n' "$prdblock" "$dnblock" > "$fnfile" +run_detect() { # state-dir [FELIS_ROOT_DOMAIN] + STATE_DIR="$1" RD="${2:-}" FNFILE="$fnfile" bash -c ' + die() { printf "DIE: %s\n" "$*"; exit 1; } + log() { printf "LOG: %s\n" "$*"; } + warn() { printf "WARN: %s\n" "$*"; } + ip() { echo "1.1.1.1 via 10.0.0.1 dev eth0 src 10.0.0.5 uid 0"; } + . "$FNFILE" + if [ -n "$RD" ]; then FELIS_ROOT_DOMAIN="$RD"; fi + detect_node_ip + echo "ROOT=$FELIS_ROOT_DOMAIN"' +} +printf '[server]\nroot_domain = "r.example.com"\n' > "$adir/felis.host.toml" +out="$(run_detect "$adir" new.example.net)" +expect "a different FELIS_ROOT_DOMAIN is refused" "DIE: FELIS_ROOT_DOMAIN (new.example.net) differs from the installed r.example.com" "$out" +expect "the refusal names the command that moves the install" "sudo felis domain set new.example.net" "$out" +case "$out" in + *ROOT=*) echo "FAIL: the installer went on after refusing:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;; +esac +expect "the installed domain is reused when named again" "ROOT=r.example.com" "$(run_detect "$adir" r.example.com)" +expect "the installed domain is reused when unset" "ROOT=r.example.com" "$(run_detect "$adir")" +rm -f "$adir/felis.host.toml" +expect "a first install takes FELIS_ROOT_DOMAIN" "ROOT=new.example.net" "$(run_detect "$adir" new.example.net)" +expect "a first install defaults to nip.io" "ROOT=10.0.0.5.nip.io" "$(run_detect "$adir")" + +rm -rf "$adir" rm -f "$fnfile" # --- database backups: the pre-migration snapshot and the daily timer -------------------- diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f8b3f73..e63b236 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1367,7 +1367,9 @@ the router or as a static address (`nmcli con mod ipv4.method manual ipv4.addresses / ipv4.gateway ipv4.dns && nmcli con up ` on Rocky), then restart felis-api (`sudo k3s kubectl -n felis rollout restart deploy/felis-api`) and the proxy (`sudo systemctl restart felis-velocity`). Moving an install -to a new address is a reinstall onto a restored backup (docs/operations.md §5). +to a new address is a reinstall onto a restored backup (docs/operations.md §5). A +default `.nip.io` root domain names the old address too; once the install runs on +its new address, `sudo felis domain set .nip.io` moves it (docs/operations.md §6). The node **name** is pinned. Every local-path volume (worlds, registry, uploads, backups) is bound to its node by name, and k3s takes the name from the