perf(bootstrap): k3s 以 GOGC=50 运行,Velocity 1G 以内改用串行回收和 C1,基础设施空载压到约 1 GB

This commit is contained in:
Lemon-miaow committed 2026-09-29 19:06:37 +08:00
1 parent a25553413c
commit 26e817f2c9
5 files changed
+200 -40

No files matched your search

+53 -9
View File
@@ -502,6 +502,9 @@ K3S_REGISTRIES_FILE="/etc/rancher/k3s/registries.yaml"
# client certificate are variables for the same reason as the file above. # client certificate are variables for the same reason as the file above.
K3S_CONFIG_DROPIN="/etc/rancher/k3s/config.yaml.d/50-felis.yaml" K3S_CONFIG_DROPIN="/etc/rancher/k3s/config.yaml.d/50-felis.yaml"
K3S_UNIT_FILE="/etc/systemd/system/k3s.service" K3S_UNIT_FILE="/etc/systemd/system/k3s.service"
# The installer's environment for the k3s service (write_k3s_service_dropin); k3s's own
# installer rewrites the unit and its .env file, never this.
K3S_SERVICE_DROPIN="/etc/systemd/system/k3s.service.d/50-felis.conf"
K3S_KUBECONFIG="/etc/rancher/k3s/k3s.yaml" K3S_KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
K3S_KUBELET_CERT="/var/lib/rancher/k3s/agent/client-kubelet.crt" K3S_KUBELET_CERT="/var/lib/rancher/k3s/agent/client-kubelet.crt"
# Where k3s imports image tarballs from as it starts (stage_k3s_airgap_images). # Where k3s imports image tarballs from as it starts (stage_k3s_airgap_images).
@@ -1937,9 +1940,10 @@ configure_k3s_firewall() {
install_k3s() { install_k3s() {
configure_k3s_firewall configure_k3s_firewall
# Before the installer runs: a fresh k3s reads the drop-in on its first start. # Before the installer runs: a fresh k3s reads both drop-ins on its first start.
K3S_RESTART_NEEDED=0 K3S_RESTART_NEEDED=0
write_k3s_config write_k3s_config
write_k3s_service_dropin
local installer_ran=0 local installer_ran=0
if [ -x "$K3S_BIN" ]; then if [ -x "$K3S_BIN" ]; then
@@ -1969,7 +1973,7 @@ install_k3s() {
# The installer restarts k3s itself; otherwise a changed drop-in or unit takes a # The installer restarts k3s itself; otherwise a changed drop-in or unit takes a
# restart to load. Pods keep running across it (k3s leaves the containers be). # restart to load. Pods keep running across it (k3s leaves the containers be).
if [ "$K3S_RESTART_NEEDED" = 1 ] && [ "$installer_ran" = 0 ]; then if [ "$K3S_RESTART_NEEDED" = 1 ] && [ "$installer_ran" = 0 ]; then
log "restarting k3s to load its new settings (${K3S_CONFIG_DROPIN})" log "restarting k3s to load its new settings (${K3S_CONFIG_DROPIN}, ${K3S_SERVICE_DROPIN})"
systemctl restart k3s systemctl restart k3s
fi fi
export KUBECONFIG="$K3S_KUBECONFIG" export KUBECONFIG="$K3S_KUBECONFIG"
@@ -2039,6 +2043,37 @@ write_k3s_config() {
log "wrote ${file}${name:+ (node name pinned to ${name})}" log "wrote ${file}${name:+ (node name pinned to ${name})}"
} }
# write_k3s_service_dropin runs k3s, and the containerd it starts with its own environment,
# with the Go collector at half the default heap growth (GOGC=50). An idle k3s holds about
# 150 MiB live and by default lets its heap reach twice that before collecting; at 50 it
# collects at one and a half times. Measured on the verification host: k3s 430 -> 370 MiB and
# its containerd 114 -> 104 MiB, for about 2% of one core more while idle. It sets
# K3S_RESTART_NEEDED when the file changed, since k3s reads its environment only as it starts.
write_k3s_service_dropin() {
local file="$K3S_SERVICE_DROPIN" tmp
mkdir -p "$(dirname "$file")"
# Beside its destination, like write_k3s_config's; systemd reads only *.conf from the
# directory, so the temp name is never loaded.
tmp="$(mktemp "${file}.XXXXXX")"
remember_temp "$tmp"
{
echo "# Written by the Felis installer (deploy/bootstrap.sh); a rerun rewrites it."
echo "[Service]"
echo "Environment=GOGC=50"
} > "$tmp"
if [ -f "$file" ] && cmp -s "$tmp" "$file"; then
rm -f "$tmp"
restore_label "$file"
ok "k3s service environment already current"
return 0
fi
chmod 0644 "$tmp"
mv "$tmp" "$file"
systemctl daemon-reload
K3S_RESTART_NEEDED=1
log "wrote ${file} (GOGC=50)"
}
# A command-line flag outranks every config file, and k3s's installer writes # A command-line flag outranks every config file, and k3s's installer writes
# INSTALL_K3S_EXEC into the unit's ExecStart one quoted word per line, so installs from # INSTALL_K3S_EXEC into the unit's ExecStart one quoted word per line, so installs from
# before the drop-in keep "'--write-kubeconfig-mode' \" followed by "'644' \" there. # before the drop-in keep "'--write-kubeconfig-mode' \" followed by "'644' \" there.
@@ -3865,12 +3900,21 @@ install_velocity_service() {
# sees it -- unquoted, that spelling would hand java a stray "legacy112" argument and the unit # sees it -- unquoted, that spelling would hand java a stray "legacy112" argument and the unit
# would not start. Quoting keeps the whole property one argv item. # would not start. Quoting keeps the whole property one argv item.
local legacy_forwarding_servers="${FELIS_LEGACY_FORWARDING_SERVERS}" local legacy_forwarding_servers="${FELIS_LEGACY_FORWARDING_SERVERS}"
# The heap starts small and is not pre-touched. The proxy with its Via plugins holds about 50M # The heap starts small and is not pre-touched: the proxy with its Via plugins holds about 50M
# live; a pre-touched 512M start kept ~0.7 GB resident on an idle network, ~0.25 GB without # live, and a pre-touched 512M start kept ~0.7 GB resident on an idle network. Up to a 1G
# (measured on the verification host). The heap grows toward -Xmx as players arrive, and the # ceiling (the default, sized for about 100 players) it runs the serial collector and only the
# periodic collection hands the growth back once they have left. FELIS_VELOCITY_XMX is at # C1 compiler, 173 MiB idle against 267 MiB under G1 (both measured on the verification host);
# least 256M, so the start never exceeds the ceiling. # with that little live, a young collection takes milliseconds, and the proxy's compression
local xmx="$FELIS_VELOCITY_XMX" xms="64M" # and encryption run in Velocity's native library whichever compiler is on. A larger ceiling
# is for a network where a serial full collection over a big heap would stall every player at
# once, so it keeps G1, whose periodic collection hands the growth back once players have left.
# FELIS_VELOCITY_XMX is at least 256M, so the start never exceeds the ceiling.
local xmx="$FELIS_VELOCITY_XMX" jvm
if [ "$(heap_megabytes "$xmx")" -le 1024 ]; then
jvm="-Xms16M -Xmx${xmx} -XX:+UseSerialGC -XX:TieredStopAtLevel=1"
else
jvm="-Xms64M -Xmx${xmx} -XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:G1PeriodicGCInterval=60000"
fi
cat > "$VELOCITY_SERVICE" <<EOF cat > "$VELOCITY_SERVICE" <<EOF
[Unit] [Unit]
Description=Felis Velocity proxy (Mojang authentication + modern forwarding) Description=Felis Velocity proxy (Mojang authentication + modern forwarding)
@@ -3882,7 +3926,7 @@ Type=simple
User=${VELOCITY_USER} User=${VELOCITY_USER}
Group=${VELOCITY_USER} Group=${VELOCITY_USER}
WorkingDirectory=${VELOCITY_DIR} WorkingDirectory=${VELOCITY_DIR}
ExecStart=${JRE_DIR}/bin/java -Xms${xms} -Xmx${xmx} -XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:G1PeriodicGCInterval=60000 -Dmojang.sessionserver=http://${api_ip}:8081/session/minecraft/hasJoined "-Dfelis.legacy-forwarding.servers=${legacy_forwarding_servers}" -jar ${VELOCITY_DIR}/velocity.jar ExecStart=${JRE_DIR}/bin/java ${jvm} -Dmojang.sessionserver=http://${api_ip}:8081/session/minecraft/hasJoined "-Dfelis.legacy-forwarding.servers=${legacy_forwarding_servers}" -jar ${VELOCITY_DIR}/velocity.jar
Restart=on-failure Restart=on-failure
RestartSec=5 RestartSec=5
NoNewPrivileges=yes NoNewPrivileges=yes
+82 -12
View File
@@ -1250,6 +1250,7 @@ run_k3s() { # installed-version pinned-version [FELIS_UPGRADE_DEPS]
ok() { printf "OK: %s\n" "$*"; } ok() { printf "OK: %s\n" "$*"; }
configure_k3s_firewall() { :; } configure_k3s_firewall() { :; }
write_k3s_config() { :; } write_k3s_config() { :; }
write_k3s_service_dropin() { :; }
strip_k3s_kubeconfig_mode_flag() { :; } strip_k3s_kubeconfig_mode_flag() { :; }
run_k3s_installer() { printf "INSTALLER: %s\n" "$FELIS_K3S_VERSION"; } run_k3s_installer() { printf "INSTALLER: %s\n" "$FELIS_K3S_VERSION"; }
stage_k3s_airgap_images() { echo STAGE; } stage_k3s_airgap_images() { echo STAGE; }
@@ -2895,11 +2896,11 @@ run_velocity_service() { # is-active(0|1) [heap]
} }
out="$(run_velocity_service 1)" out="$(run_velocity_service 1)"
expect "a proxy with no recorded start is restarted" "SYSTEMCTL restart felis-velocity" "$out" expect "a proxy with no recorded start is restarted" "SYSTEMCTL restart felis-velocity" "$out"
expect "the default heap starts at 64M and may grow to 1G" "java -Xms64M -Xmx1G " "$(cat "$vdir/unit")" expect "the default 1G ceiling runs the serial collector and C1 from a 16M start" \
"java -Xms16M -Xmx1G -XX:+UseSerialGC -XX:TieredStopAtLevel=1 -Dmojang" "$(cat "$vdir/unit")"
case "$(cat "$vdir/unit")" in case "$(cat "$vdir/unit")" in
*AlwaysPreTouch*) echo "FAIL the proxy pre-touches its heap, holding all of -Xms from the start"; fails=$((fails + 1)) ;; *AlwaysPreTouch*|*UseG1GC*) echo "FAIL a 1G proxy pre-touches its heap or runs G1: $(grep ExecStart "$vdir/unit")"; fails=$((fails + 1)) ;;
*"-XX:G1PeriodicGCInterval="*) echo "PASS the proxy neither pre-touches its heap nor keeps growth it no longer uses" ;; *) echo "PASS a 1G proxy neither pre-touches its heap nor runs G1" ;;
*) echo "FAIL the proxy has no periodic collection to hand back an idle heap"; fails=$((fails + 1)) ;;
esac esac
[ -s "$vdir/fp" ] && echo "PASS the restart records what the proxy runs" \ [ -s "$vdir/fp" ] && echo "PASS the restart records what the proxy runs" \
|| { echo "FAIL no fingerprint was recorded after the restart"; fails=$((fails + 1)); } || { echo "FAIL no fingerprint was recorded after the restart"; fails=$((fails + 1)); }
@@ -2915,9 +2916,19 @@ expect "a stopped proxy is started whatever the fingerprint" "SYSTEMCTL restart
printf 'JAVA_VERSION="25.0.1"\n' > "$vdir/jre/release" printf 'JAVA_VERSION="25.0.1"\n' > "$vdir/jre/release"
expect "a patched JRE restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1)" expect "a patched JRE restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1)"
expect "a new heap size restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1 3G)" expect "a new heap size restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1 3G)"
expect "the unit carries the new ceiling" "java -Xms64M -Xmx3G " "$(cat "$vdir/unit")" expect "a ceiling above 1G runs G1, whose periodic collection hands idle growth back" \
"java -Xms64M -Xmx3G -XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:G1PeriodicGCInterval=60000 -Dmojang" "$(cat "$vdir/unit")"
case "$(cat "$vdir/unit")" in
*AlwaysPreTouch*|*UseSerialGC*|*TieredStopAtLevel*) echo "FAIL a 3G proxy carries a small proxy's flags or pre-touches: $(grep ExecStart "$vdir/unit")"; fails=$((fails + 1)) ;;
*) echo "PASS a 3G proxy keeps G1 and both compilers" ;;
esac
run_velocity_service 1 1024M >/dev/null
expect "1024M is still a small proxy" "java -Xms16M -Xmx1024M -XX:+UseSerialGC " "$(cat "$vdir/unit")"
run_velocity_service 1 1025M >/dev/null
expect "a megabyte past 1G is a large one" "java -Xms64M -Xmx1025M -XX:+UseG1GC " "$(cat "$vdir/unit")"
run_velocity_service 1 384M >/dev/null run_velocity_service 1 384M >/dev/null
expect "a small ceiling keeps the same small start" "java -Xms64M -Xmx384M " "$(cat "$vdir/unit")" expect "a small ceiling runs the small proxy's flags" \
"java -Xms16M -Xmx384M -XX:+UseSerialGC -XX:TieredStopAtLevel=1 -Dmojang" "$(cat "$vdir/unit")"
# `felis rotate-token velocity` rewrites service-token, which the plugin re-reads by itself: # `felis rotate-token velocity` rewrites service-token, which the plugin re-reads by itself:
# that line alone changing leaves the proxy running, and any other change restarts it. # that line alone changing leaves the proxy running, and any other change restarts it.
props="$vdir/v/plugins/felis-link/felis-link.properties" props="$vdir/v/plugins/felis-link/felis-link.properties"
@@ -3015,10 +3026,7 @@ expect "a heap in megabytes is kept" "768" "$(heap 768m)"
for bad in 1 1K 0G 01G G -1G 1.5G 9999999G; do for bad in 1 1K 0G 01G G -1G 1.5G 9999999G; do
expect "the heap spelling '$bad' is refused" "0" "$(heap "$bad")" expect "the heap spelling '$bad' is refused" "0" "$(heap "$bad")"
done done
case "$(awk '/^install_velocity_service\(\) \{/,/^}/' "$BS")" in # The unit written for each ceiling is checked with the rerun tests above.
*'-Xms${xms} -Xmx${xmx} '*) echo "PASS the proxy unit takes its heap from FELIS_VELOCITY_XMX" ;;
*) echo "FAIL the proxy unit's heap is not FELIS_VELOCITY_XMX"; fails=$((fails + 1)) ;;
esac
# --- reproducible image ids --------------------------------------------------------------- # --- reproducible image ids ---------------------------------------------------------------
# restart_existing_system_servers compares image ids across runs; a default BuildKit # restart_existing_system_servers compares image ids across runs; a default BuildKit
@@ -3230,6 +3238,63 @@ out="$(run_k3s_config "$kdir/broken.crt" "$kdir/k3s" renamed-host)"
expect "a certificate openssl cannot parse is a warning, not a failed install" "WARN: could not read this node's k3s name" "$out" expect "a certificate openssl cannot parse is a warning, not a failed install" "WARN: could not read this node's k3s name" "$out"
expect "and the drop-in is still written" 'write-kubeconfig-mode: "0600"' "$(cat "$dropin")" expect "and the drop-in is still written" 'write-kubeconfig-mode: "0600"' "$(cat "$dropin")"
# k3s's Go collector runs at GOGC=50 through a systemd drop-in, written beside itself and
# loaded (with a k3s restart) only when it changed.
svblock="$(awk '/^write_k3s_service_dropin\(\) \{/,/^}/' "$BS")"
[ -n "$svblock" ] || { echo "FAIL: no write_k3s_service_dropin found in $BS"; exit 1; }
svdropin="$kdir/k3s.service.d/50-felis.conf"
run_k3s_service_dropin() {
DROPIN="$svdropin" bash -c '
set -Eeuo pipefail
ok() { echo "OK: $*"; }; log() { echo "LOG: $*"; }
remember_temp() { :; }
restore_label() { echo "RELABEL: $*"; }
systemctl() { echo "SYSTEMCTL: $*"; }
mktemp() { local p; p="$(command mktemp "$@")"; echo "MKTEMP: $(dirname "$p")" >&2; echo "$p"; }
K3S_SERVICE_DROPIN="$DROPIN"
'"$svblock"'
K3S_RESTART_NEEDED=0
write_k3s_service_dropin
echo "RESTART=$K3S_RESTART_NEEDED"' 2>&1
}
out="$(run_k3s_service_dropin)"
if [ "$(cat "$svdropin")" = '# Written by the Felis installer (deploy/bootstrap.sh); a rerun rewrites it.
[Service]
Environment=GOGC=50' ]; then
echo "PASS k3s runs its Go collector at GOGC=50"
else
echo "FAIL the k3s service drop-in is:"; cat "$svdropin"; fails=$((fails + 1))
fi
expect "a new service drop-in is loaded" "SYSTEMCTL: daemon-reload" "$out"
expect "a new service drop-in asks for a k3s restart" "RESTART=1" "$out"
if [ "$(printf '%s\n' "$out" | sed -n 's/^MKTEMP: //p' | sort -u)" = "$kdir/k3s.service.d" ]; then
echo "PASS the service drop-in is made beside itself, never under /tmp"
else
echo "FAIL temporary files for the service drop-in were made in: $(printf '%s\n' "$out" | sed -n 's/^MKTEMP: //p')"; fails=$((fails + 1))
fi
if [ "$(stat -c %a "$svdropin" 2>/dev/null || stat -f %Lp "$svdropin")" = 644 ]; then
echo "PASS the service drop-in is readable like the unit it extends"
else
echo "FAIL the service drop-in must be 0644"; fails=$((fails + 1))
fi
out="$(run_k3s_service_dropin)"
expect "an unchanged service drop-in restarts nothing" "RESTART=0" "$out"
expect "an unchanged service drop-in says so" "OK: k3s service environment already current" "$out"
case "$out" in
*daemon-reload*) echo "FAIL an unchanged service drop-in must not reload systemd"; fails=$((fails + 1)) ;;
*) echo "PASS an unchanged service drop-in reloads nothing" ;;
esac
expect "an unchanged service drop-in is still relabelled" "RELABEL: $svdropin" "$out"
if [ "$(ls "$kdir/k3s.service.d")" = "50-felis.conf" ]; then
echo "PASS no temporary file is left beside the service drop-in"
else
echo "FAIL k3s.service.d holds: $(ls "$kdir/k3s.service.d")"; fails=$((fails + 1))
fi
printf '[Service]\nEnvironment=GOGC=100\n' > "$svdropin"
out="$(run_k3s_service_dropin)"
expect "an edited service drop-in is put back" "Environment=GOGC=50" "$(cat "$svdropin")"
expect "and k3s is restarted onto it" "RESTART=1" "$out"
sblock="$(awk '/^strip_k3s_kubeconfig_mode_flag\(\) \{/,/^}/' "$BS")" sblock="$(awk '/^strip_k3s_kubeconfig_mode_flag\(\) \{/,/^}/' "$BS")"
[ -n "$sblock" ] || { echo "FAIL: no strip_k3s_kubeconfig_mode_flag found in $BS"; exit 1; } [ -n "$sblock" ] || { echo "FAIL: no strip_k3s_kubeconfig_mode_flag found in $BS"; exit 1; }
# ExecStart as k3s's installer writes it (copied off an install made with the old flag). # ExecStart as k3s's installer writes it (copied off an install made with the old flag).
@@ -3290,17 +3355,19 @@ iblock="$(awk '/^install_k3s\(\) \{/,/^}/' "$BS")"
iblock="$iblock iblock="$iblock
$(awk '/^version_newer\(\) \{/,/^}/' "$BS") $(awk '/^version_newer\(\) \{/,/^}/' "$BS")
$(awk '/^k3s_upgrade_allowed\(\) \{/,/^}/' "$BS")" $(awk '/^k3s_upgrade_allowed\(\) \{/,/^}/' "$BS")"
run_install_k3s() { # $1: installed version ("" = none), $2: drop-in changed (0|1), $3: FELIS_UPGRADE_DEPS run_install_k3s() { # $1: installed version ("" = none), $2: config drop-in changed (0|1), $3: FELIS_UPGRADE_DEPS, $4: service drop-in changed (0|1)
INSTALLED="$1" CHANGED="$2" UPGRADE="${3:-0}" KDIR="$kdir" bash -c ' INSTALLED="$1" CHANGED="$2" UPGRADE="${3:-0}" SVC_CHANGED="${4:-0}" KDIR="$kdir" bash -c '
set -Eeuo pipefail set -Eeuo pipefail
ok() { echo "OK: $*"; }; log() { echo "LOG: $*"; }; warn() { echo "WARN: $*"; } ok() { echo "OK: $*"; }; log() { echo "LOG: $*"; }; warn() { echo "WARN: $*"; }
die() { echo "DIE: $*"; exit 1; } die() { echo "DIE: $*"; exit 1; }
FELIS_K3S_VERSION=v1.36.4+k3s1 FELIS_UPGRADE_DEPS="$UPGRADE" K3S_BIN_DIR="$KDIR" K3S_BIN="$KDIR/k3s-under-test" FELIS_K3S_VERSION=v1.36.4+k3s1 FELIS_UPGRADE_DEPS="$UPGRADE" K3S_BIN_DIR="$KDIR" K3S_BIN="$KDIR/k3s-under-test"
K3S_CONFIG_DROPIN=/etc/rancher/k3s/config.yaml.d/50-felis.yaml K3S_KUBECONFIG=/etc/rancher/k3s/k3s.yaml K3S_CONFIG_DROPIN=/etc/rancher/k3s/config.yaml.d/50-felis.yaml K3S_KUBECONFIG=/etc/rancher/k3s/k3s.yaml
K3S_SERVICE_DROPIN=/etc/systemd/system/k3s.service.d/50-felis.conf
rm -f "$K3S_BIN" rm -f "$K3S_BIN"
if [ -n "$INSTALLED" ]; then printf "#!/bin/sh\necho \"k3s version %s (abc)\"\n" "$INSTALLED" > "$K3S_BIN"; chmod +x "$K3S_BIN"; fi if [ -n "$INSTALLED" ]; then printf "#!/bin/sh\necho \"k3s version %s (abc)\"\n" "$INSTALLED" > "$K3S_BIN"; chmod +x "$K3S_BIN"; fi
configure_k3s_firewall() { :; } configure_k3s_firewall() { :; }
write_k3s_config() { [ "$CHANGED" = 0 ] || K3S_RESTART_NEEDED=1; } write_k3s_config() { [ "$CHANGED" = 0 ] || K3S_RESTART_NEEDED=1; }
write_k3s_service_dropin() { echo "SERVICE-DROPIN"; [ "$SVC_CHANGED" = 0 ] || K3S_RESTART_NEEDED=1; }
strip_k3s_kubeconfig_mode_flag() { :; } strip_k3s_kubeconfig_mode_flag() { :; }
run_k3s_installer() { echo "INSTALLER"; printf "#!/bin/sh\n" > "$K3S_BIN"; command chmod +x "$K3S_BIN"; } run_k3s_installer() { echo "INSTALLER"; printf "#!/bin/sh\n" > "$K3S_BIN"; command chmod +x "$K3S_BIN"; }
stage_k3s_airgap_images() { echo "STAGE"; } stage_k3s_airgap_images() { echo "STAGE"; }
@@ -3316,9 +3383,12 @@ expect "the admin kubeconfig is made root-only once the node is up" "READY
CHMOD: 0600 /etc/rancher/k3s/k3s.yaml" "$out" CHMOD: 0600 /etc/rancher/k3s/k3s.yaml" "$out"
out="$(run_install_k3s v1.36.4+k3s1 0)" out="$(run_install_k3s v1.36.4+k3s1 0)"
case "$out" in *"restart k3s"*) echo "FAIL unchanged k3s settings must not restart k3s"; fails=$((fails + 1)) ;; *) echo "PASS unchanged k3s settings restart nothing" ;; esac case "$out" in *"restart k3s"*) echo "FAIL unchanged k3s settings must not restart k3s"; fails=$((fails + 1)) ;; *) echo "PASS unchanged k3s settings restart nothing" ;; esac
expect "a new service environment alone restarts a running k3s" "SYSTEMCTL: restart k3s" "$(run_install_k3s v1.36.4+k3s1 0 0 1)"
out="$(run_install_k3s "" 1)" out="$(run_install_k3s "" 1)"
expect "a fresh host runs the k3s installer and waits for the node" "INSTALLER expect "a fresh host runs the k3s installer and waits for the node" "INSTALLER
SYSTEMCTL: enable --now k3s" "$out" SYSTEMCTL: enable --now k3s" "$out"
expect "a fresh k3s has its service environment before its first start" "SERVICE-DROPIN INSTALLER " \
"$(printf '%s\n' "$out" | grep -E '^(SERVICE-DROPIN|INSTALLER)$' | tr '\n' ' ')"
expect "a fresh host stages k3s's images before k3s first starts" "STAGE expect "a fresh host stages k3s's images before k3s first starts" "STAGE
INSTALLER" "$out" INSTALLER" "$out"
expect "a fresh host's kubeconfig is made root-only too" "CHMOD: 0600 /etc/rancher/k3s/k3s.yaml" "$out" expect "a fresh host's kubeconfig is made root-only too" "CHMOD: 0600 /etc/rancher/k3s/k3s.yaml" "$out"
+13
View File
@@ -321,6 +321,16 @@ stop_database_pod() {
|| warn "${PG_DEPLOYMENT} did not stop within 2 minutes; its cluster recovers from its WAL on the next start" || warn "${PG_DEPLOYMENT} did not stop within 2 minutes; its cluster recovers from its WAL on the next start"
} }
# remove_k3s_service_dropin deletes the environment the installer gives the k3s service
# (bootstrap.sh, write_k3s_service_dropin). k3s-uninstall.sh removes the unit and its .env
# file and leaves the unit's drop-in directory.
remove_k3s_service_dropin() {
[ -f "${UNIT_DIR}/k3s.service.d/50-felis.conf" ] || return 0
rm -f "${UNIT_DIR}/k3s.service.d/50-felis.conf"
rmdir "${UNIT_DIR}/k3s.service.d" 2>/dev/null || true
ok "k3s service environment removed"
}
remove_k3s() { remove_k3s() {
local stamp local stamp
[ "$PURGE" = 1 ] || stop_database_pod [ "$PURGE" = 1 ] || stop_database_pod
@@ -336,6 +346,7 @@ remove_k3s() {
if [ -x "${K3S_BIN_DIR}/k3s-uninstall.sh" ]; then if [ -x "${K3S_BIN_DIR}/k3s-uninstall.sh" ]; then
log "running k3s-uninstall.sh" log "running k3s-uninstall.sh"
"${K3S_BIN_DIR}/k3s-uninstall.sh" >/dev/null 2>&1 || warn "k3s-uninstall.sh reported an error; check /var/lib/rancher and /etc/rancher" "${K3S_BIN_DIR}/k3s-uninstall.sh" >/dev/null 2>&1 || warn "k3s-uninstall.sh reported an error; check /var/lib/rancher and /etc/rancher"
remove_k3s_service_dropin
ok "k3s removed" ok "k3s removed"
else else
warn "k3s is at ${K3S_BIN_DIR}/k3s but ${K3S_BIN_DIR}/k3s-uninstall.sh is missing; remove k3s by hand" warn "k3s is at ${K3S_BIN_DIR}/k3s but ${K3S_BIN_DIR}/k3s-uninstall.sh is missing; remove k3s by hand"
@@ -522,6 +533,8 @@ main() {
case "$K3S_MODE" in case "$K3S_MODE" in
remove) remove_k3s ;; remove) remove_k3s ;;
keep) remove_from_cluster ;; keep) remove_from_cluster ;;
# k3s was removed some other way; what it left of the installer's goes too.
absent) remove_k3s_service_dropin ;;
esac esac
remove_nft_tables remove_nft_tables
remove_firewalld_rules "$game" "$nano" remove_firewalld_rules "$game" "$nano"
+20
View File
@@ -35,6 +35,8 @@ fresh_host() {
for u in felis-db-backup.timer felis-db-backup.service felis-velocity.service felis-postgres-firewall.service; do for u in felis-db-backup.timer felis-db-backup.service felis-velocity.service felis-postgres-firewall.service; do
printf '[Unit]\n' > "$root/h/units/$u" printf '[Unit]\n' > "$root/h/units/$u"
done done
mkdir -p "$root/h/units/k3s.service.d"
printf '[Service]\nEnvironment=GOGC=50\n' > "$root/h/units/k3s.service.d/50-felis.conf"
printf '[Service]\nExecStart=/usr/local/bin/cloudflared --config %s tunnel run\n' "$root/h/etc/cloudflared.yml" \ printf '[Service]\nExecStart=/usr/local/bin/cloudflared --config %s tunnel run\n' "$root/h/etc/cloudflared.yml" \
> "$root/h/units/cloudflared-felis.service" > "$root/h/units/cloudflared-felis.service"
printf 'tunnel: abc\ncredentials-file: %s\n' "$root/h/cf/abc.json" > "$root/h/etc/cloudflared.yml" printf 'tunnel: abc\ncredentials-file: %s\n' "$root/h/cf/abc.json" > "$root/h/etc/cloudflared.yml"
@@ -155,6 +157,9 @@ refute "keep-data leaves the database alone" "DROP DATABASE" "$calls"
|| { echo "FAIL /opt/felis or the host binary is still there"; fails=$((fails + 1)); } || { echo "FAIL /opt/felis or the host binary is still there"; fails=$((fails + 1)); }
[ -z "$(ls "$root/h/units")" ] && echo "PASS every Felis unit file is removed" \ [ -z "$(ls "$root/h/units")" ] && echo "PASS every Felis unit file is removed" \
|| { echo "FAIL units left: $(ls "$root/h/units")"; fails=$((fails + 1)); } || { echo "FAIL units left: $(ls "$root/h/units")"; fails=$((fails + 1)); }
[ ! -e "$root/h/units/k3s.service.d" ] \
&& echo "PASS the k3s service environment k3s's uninstaller leaves is removed with its directory" \
|| { echo "FAIL the k3s service drop-in is still there: $(ls -R "$root/h/units")"; fails=$((fails + 1)); }
expect "the timers are disabled" "SYSTEMCTL disable --now felis-db-backup.timer" "$calls" expect "the timers are disabled" "SYSTEMCTL disable --now felis-db-backup.timer" "$calls"
expect "the velocity user is removed" "USERDEL felis-velocity" "$calls" expect "the velocity user is removed" "USERDEL felis-velocity" "$calls"
expect "the run ends pointing at the reinstall steps" "Reinstall on top of kept data" "$out" expect "the run ends pointing at the reinstall steps" "Reinstall on top of kept data" "$out"
@@ -190,6 +195,9 @@ expect "Felis's volumes are retained before their claims go" 'KUBE patch pv pvc-
refute "a volume of another namespace is not touched" "patch pv pvc-9" "$calls" refute "a volume of another namespace is not touched" "patch pv pvc-9" "$calls"
expect "Felis's namespaces are deleted" "KUBE delete namespace felis minecraft felis-build" "$calls" expect "Felis's namespaces are deleted" "KUBE delete namespace felis minecraft felis-build" "$calls"
expect "the CRD is deleted" "KUBE delete crd minecraftservers.felis.lolicon.best" "$calls" expect "the CRD is deleted" "KUBE delete crd minecraftservers.felis.lolicon.best" "$calls"
[ -f "$root/h/units/k3s.service.d/50-felis.conf" ] \
&& echo "PASS a k3s that stays keeps its service environment, like its config" \
|| { echo "FAIL the kept k3s lost its service drop-in"; fails=$((fails + 1)); }
out="$(fresh_host; run_uninstall down --yes)" out="$(fresh_host; run_uninstall down --yes)"
expect "a k3s that does not answer stops the run" "k3s does not answer" "$out" expect "a k3s that does not answer stops the run" "k3s does not answer" "$out"
@@ -198,6 +206,18 @@ run_uninstall down --yes --keep-k3s >/dev/null
calls="$(cat "$root/calls")" calls="$(cat "$root/calls")"
refute "--keep-k3s never runs k3s's uninstaller" "RUN k3s-uninstall.sh" "$calls" refute "--keep-k3s never runs k3s's uninstaller" "RUN k3s-uninstall.sh" "$calls"
# --- k3s already gone ----------------------------------------------------------------------
fresh_host
rm -f "$root/h/bin/k3s"
printf '[Service]\nLimitNOFILE=4096\n' > "$root/h/units/k3s.service.d/90-admin.conf"
run_uninstall down --yes >/dev/null
[ ! -e "$root/h/units/k3s.service.d/50-felis.conf" ] \
&& echo "PASS a k3s removed some other way does not leave the installer's service environment" \
|| { echo "FAIL the k3s service drop-in outlived k3s"; fails=$((fails + 1)); }
[ -f "$root/h/units/k3s.service.d/90-admin.conf" ] \
&& echo "PASS a drop-in someone else wrote beside it stays, with the directory" \
|| { echo "FAIL the uninstall removed a k3s drop-in it did not write"; fails=$((fails + 1)); }
# --- purge ------------------------------------------------------------------------------- # --- purge -------------------------------------------------------------------------------
fresh_host fresh_host
out="$(run_uninstall "default felis minecraft" --purge --yes)" out="$(run_uninstall "default felis minecraft" --purge --yes)"
+32 -19
View File
@@ -216,21 +216,30 @@ to 0 for about 8 minutes on the reference VM.
### What the platform itself uses ### What the platform itself uses
Measured on the verification host (4 vCPU, 5.5 GB RAM, 6 GB swap, CentOS Stream 9 Measured on the verification host (4 vCPU, 5.5 GB RAM, 6 GB swap, CentOS Stream 9
aarch64) with the control plane, the login and lobby system servers and one idle Paper aarch64) on an idle network, as each process's proportional set size (PSS: a page shared
server running **[VM-VERIFIED]**: by several processes is split among them; `/proc/<pid>/smaps_rollup`) **[VM-VERIFIED]**:
| Process | Resident memory | | Process | Memory (PSS) |
|---|---| |---|---|
| k3s (server, kubelet, containerd) | ~1.1 GB | | k3s (API server, controllers, scheduler, kubelet) | ~370 MiB |
| Velocity (`-Xms64M -Xmx1G`, idle; it grows with players) | ~0.25 GB | | k3s's containerd and the pods' shims | ~170 MiB |
| lobby (Paper, pod limit 1 GiB) | ~0.7–0.85 GB | | CoreDNS and the local-path volume provisioner | ~105 MiB |
| login (Limbo, pod limit 512 MiB) | ~0.16 GB | | Velocity (`-Xms16M -Xmx1G`, idle; it grows with players) | ~175 MiB |
| felis-api, felis-operator, registry gate | ~50 MB each | | felis-api, felis-operator, registry gate | ~85 MiB together |
| PostgreSQL (the felis-postgres pod) | ~30 MB plus page cache | | Image registry | ~25 MiB |
| **Total in use** | **~2.9 GB** | | PostgreSQL (the felis-postgres pod) | ~40 MiB plus page cache |
| **Infrastructure total** | **~1 GB** |
Every game server adds the memory its owner gave it: the pod's limit equals its request, The installer runs k3s, and the containerd it starts, with Go's collector at half its
and the JVM heap is derived from it (§1a). Quotas cap it per user (panel → 管理 → 配额). default heap growth (`GOGC=50`, in `/etc/systemd/system/k3s.service.d/50-felis.conf`): an
idle k3s holds about 150 MiB live and would otherwise let its heap reach twice that before
collecting. It saves about 70 MiB for about 2% of one core. An install from before this
picks it up on its next installer run, which restarts k3s; the pods keep running.
The login (Limbo, pod limit 512 MiB, ~0.16 GB) and lobby (Paper, pod limit 1 GiB,
~0.7–0.85 GB) system servers come on top, and every game server adds the memory its owner
gave it: the pod's limit equals its request, and the JVM heap is derived from it (§1a).
Quotas cap it per user (panel → 管理 → 配额).
A release install builds nothing (§1). When the installer builds on the host its peak is A release install builds nothing (§1). When the installer builds on the host its peak is
the image builds (Docker plus a Gradle container). Afterwards it stops Docker, and Docker's the image builds (Docker plus a Gradle container). Afterwards it stops Docker, and Docker's
@@ -249,15 +258,19 @@ adds a 2 GiB `/swapfile`.
The player-count rows are planning figures, not measurements: a Minecraft server's cost The player-count rows are planning figures, not measurements: a Minecraft server's cost
depends mostly on what its players do (view distance, redstone, mods). Size RAM as the depends mostly on what its players do (view distance, redstone, mods). Size RAM as the
platform's ~3 GB plus the sum of the servers you expect to run at once, then add a infrastructure's ~1 GB and the login and lobby servers' ~1 GB, plus the sum of the servers
quarter for the page cache and PostgreSQL. Velocity itself needs little per player; raise you expect to run at once, then add a quarter for the page cache and PostgreSQL. Velocity
its heap when `journalctl -u felis-velocity` shows long GC pauses or `OutOfMemoryError`. itself needs little per player; raise its heap when `journalctl -u felis-velocity` shows
long GC pauses or `OutOfMemoryError`.
`FELIS_VELOCITY_XMX` (default `1G`, at least `256M`, written `<n>M` or `<n>G`) is read on `FELIS_VELOCITY_XMX` (default `1G`, at least `256M`, written `<n>M` or `<n>G`) is read on
every installer run. The heap starts at 64M and grows toward the maximum as players arrive; every installer run. Up to 1G the heap starts at 16M and the proxy runs the serial collector
a periodic collection hands the growth back once they have left. Changing it rewrites the and only the C1 compiler: its plugins hold about 50M live, so a collection takes
unit, and the rerun restarts the proxy, which disconnects everyone online; do it in a quiet milliseconds, and compression and encryption run in Velocity's native library. Above 1G it
hour **[VM-VERIFIED]**: runs G1 from a 64M start, since a serial full collection over a large heap would stall
every player at once, and a periodic collection hands the growth back once players have
left. Changing it rewrites the unit, and the rerun restarts the proxy, which disconnects
everyone online; do it in a quiet hour **[VM-VERIFIED]**:
``` ```
curl -fsSL <raw-url>/deploy/bootstrap.sh | sudo FELIS_VELOCITY_XMX=2G bash curl -fsSL <raw-url>/deploy/bootstrap.sh | sudo FELIS_VELOCITY_XMX=2G bash