perf(bootstrap): k3s 以 GOGC=50 运行,Velocity 1G 以内改用串行回收和 C1,基础设施空载压到约 1 GB
This commit is contained in:
5 files changed
+200
-40
No files matched your search
+53
-9
@@ -502,6 +502,9 @@ K3S_REGISTRIES_FILE="/etc/rancher/k3s/registries.yaml"
|
|||||||
# client certificate are variables for the same reason as the file above.
|
# client certificate are variables for the same reason as the file above.
|
||||||
K3S_CONFIG_DROPIN="/etc/rancher/k3s/config.yaml.d/50-felis.yaml"
|
K3S_CONFIG_DROPIN="/etc/rancher/k3s/config.yaml.d/50-felis.yaml"
|
||||||
K3S_UNIT_FILE="/etc/systemd/system/k3s.service"
|
K3S_UNIT_FILE="/etc/systemd/system/k3s.service"
|
||||||
|
# The installer's environment for the k3s service (write_k3s_service_dropin); k3s's own
|
||||||
|
# installer rewrites the unit and its .env file, never this.
|
||||||
|
K3S_SERVICE_DROPIN="/etc/systemd/system/k3s.service.d/50-felis.conf"
|
||||||
K3S_KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
|
K3S_KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
|
||||||
K3S_KUBELET_CERT="/var/lib/rancher/k3s/agent/client-kubelet.crt"
|
K3S_KUBELET_CERT="/var/lib/rancher/k3s/agent/client-kubelet.crt"
|
||||||
# Where k3s imports image tarballs from as it starts (stage_k3s_airgap_images).
|
# Where k3s imports image tarballs from as it starts (stage_k3s_airgap_images).
|
||||||
@@ -1937,9 +1940,10 @@ configure_k3s_firewall() {
|
|||||||
|
|
||||||
install_k3s() {
|
install_k3s() {
|
||||||
configure_k3s_firewall
|
configure_k3s_firewall
|
||||||
# Before the installer runs: a fresh k3s reads the drop-in on its first start.
|
# Before the installer runs: a fresh k3s reads both drop-ins on its first start.
|
||||||
K3S_RESTART_NEEDED=0
|
K3S_RESTART_NEEDED=0
|
||||||
write_k3s_config
|
write_k3s_config
|
||||||
|
write_k3s_service_dropin
|
||||||
|
|
||||||
local installer_ran=0
|
local installer_ran=0
|
||||||
if [ -x "$K3S_BIN" ]; then
|
if [ -x "$K3S_BIN" ]; then
|
||||||
@@ -1969,7 +1973,7 @@ install_k3s() {
|
|||||||
# The installer restarts k3s itself; otherwise a changed drop-in or unit takes a
|
# The installer restarts k3s itself; otherwise a changed drop-in or unit takes a
|
||||||
# restart to load. Pods keep running across it (k3s leaves the containers be).
|
# restart to load. Pods keep running across it (k3s leaves the containers be).
|
||||||
if [ "$K3S_RESTART_NEEDED" = 1 ] && [ "$installer_ran" = 0 ]; then
|
if [ "$K3S_RESTART_NEEDED" = 1 ] && [ "$installer_ran" = 0 ]; then
|
||||||
log "restarting k3s to load its new settings (${K3S_CONFIG_DROPIN})"
|
log "restarting k3s to load its new settings (${K3S_CONFIG_DROPIN}, ${K3S_SERVICE_DROPIN})"
|
||||||
systemctl restart k3s
|
systemctl restart k3s
|
||||||
fi
|
fi
|
||||||
export KUBECONFIG="$K3S_KUBECONFIG"
|
export KUBECONFIG="$K3S_KUBECONFIG"
|
||||||
@@ -2039,6 +2043,37 @@ write_k3s_config() {
|
|||||||
log "wrote ${file}${name:+ (node name pinned to ${name})}"
|
log "wrote ${file}${name:+ (node name pinned to ${name})}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# write_k3s_service_dropin runs k3s, and the containerd it starts with its own environment,
|
||||||
|
# with the Go collector at half the default heap growth (GOGC=50). An idle k3s holds about
|
||||||
|
# 150 MiB live and by default lets its heap reach twice that before collecting; at 50 it
|
||||||
|
# collects at one and a half times. Measured on the verification host: k3s 430 -> 370 MiB and
|
||||||
|
# its containerd 114 -> 104 MiB, for about 2% of one core more while idle. It sets
|
||||||
|
# K3S_RESTART_NEEDED when the file changed, since k3s reads its environment only as it starts.
|
||||||
|
write_k3s_service_dropin() {
|
||||||
|
local file="$K3S_SERVICE_DROPIN" tmp
|
||||||
|
mkdir -p "$(dirname "$file")"
|
||||||
|
# Beside its destination, like write_k3s_config's; systemd reads only *.conf from the
|
||||||
|
# directory, so the temp name is never loaded.
|
||||||
|
tmp="$(mktemp "${file}.XXXXXX")"
|
||||||
|
remember_temp "$tmp"
|
||||||
|
{
|
||||||
|
echo "# Written by the Felis installer (deploy/bootstrap.sh); a rerun rewrites it."
|
||||||
|
echo "[Service]"
|
||||||
|
echo "Environment=GOGC=50"
|
||||||
|
} > "$tmp"
|
||||||
|
if [ -f "$file" ] && cmp -s "$tmp" "$file"; then
|
||||||
|
rm -f "$tmp"
|
||||||
|
restore_label "$file"
|
||||||
|
ok "k3s service environment already current"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
chmod 0644 "$tmp"
|
||||||
|
mv "$tmp" "$file"
|
||||||
|
systemctl daemon-reload
|
||||||
|
K3S_RESTART_NEEDED=1
|
||||||
|
log "wrote ${file} (GOGC=50)"
|
||||||
|
}
|
||||||
|
|
||||||
# A command-line flag outranks every config file, and k3s's installer writes
|
# A command-line flag outranks every config file, and k3s's installer writes
|
||||||
# INSTALL_K3S_EXEC into the unit's ExecStart one quoted word per line, so installs from
|
# INSTALL_K3S_EXEC into the unit's ExecStart one quoted word per line, so installs from
|
||||||
# before the drop-in keep "'--write-kubeconfig-mode' \" followed by "'644' \" there.
|
# before the drop-in keep "'--write-kubeconfig-mode' \" followed by "'644' \" there.
|
||||||
@@ -3865,12 +3900,21 @@ install_velocity_service() {
|
|||||||
# sees it -- unquoted, that spelling would hand java a stray "legacy112" argument and the unit
|
# sees it -- unquoted, that spelling would hand java a stray "legacy112" argument and the unit
|
||||||
# would not start. Quoting keeps the whole property one argv item.
|
# would not start. Quoting keeps the whole property one argv item.
|
||||||
local legacy_forwarding_servers="${FELIS_LEGACY_FORWARDING_SERVERS}"
|
local legacy_forwarding_servers="${FELIS_LEGACY_FORWARDING_SERVERS}"
|
||||||
# The heap starts small and is not pre-touched. The proxy with its Via plugins holds about 50M
|
# The heap starts small and is not pre-touched: the proxy with its Via plugins holds about 50M
|
||||||
# live; a pre-touched 512M start kept ~0.7 GB resident on an idle network, ~0.25 GB without
|
# live, and a pre-touched 512M start kept ~0.7 GB resident on an idle network. Up to a 1G
|
||||||
# (measured on the verification host). The heap grows toward -Xmx as players arrive, and the
|
# ceiling (the default, sized for about 100 players) it runs the serial collector and only the
|
||||||
# periodic collection hands the growth back once they have left. FELIS_VELOCITY_XMX is at
|
# C1 compiler, 173 MiB idle against 267 MiB under G1 (both measured on the verification host);
|
||||||
# least 256M, so the start never exceeds the ceiling.
|
# with that little live, a young collection takes milliseconds, and the proxy's compression
|
||||||
local xmx="$FELIS_VELOCITY_XMX" xms="64M"
|
# and encryption run in Velocity's native library whichever compiler is on. A larger ceiling
|
||||||
|
# is for a network where a serial full collection over a big heap would stall every player at
|
||||||
|
# once, so it keeps G1, whose periodic collection hands the growth back once players have left.
|
||||||
|
# FELIS_VELOCITY_XMX is at least 256M, so the start never exceeds the ceiling.
|
||||||
|
local xmx="$FELIS_VELOCITY_XMX" jvm
|
||||||
|
if [ "$(heap_megabytes "$xmx")" -le 1024 ]; then
|
||||||
|
jvm="-Xms16M -Xmx${xmx} -XX:+UseSerialGC -XX:TieredStopAtLevel=1"
|
||||||
|
else
|
||||||
|
jvm="-Xms64M -Xmx${xmx} -XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:G1PeriodicGCInterval=60000"
|
||||||
|
fi
|
||||||
cat > "$VELOCITY_SERVICE" <<EOF
|
cat > "$VELOCITY_SERVICE" <<EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=Felis Velocity proxy (Mojang authentication + modern forwarding)
|
Description=Felis Velocity proxy (Mojang authentication + modern forwarding)
|
||||||
@@ -3882,7 +3926,7 @@ Type=simple
|
|||||||
User=${VELOCITY_USER}
|
User=${VELOCITY_USER}
|
||||||
Group=${VELOCITY_USER}
|
Group=${VELOCITY_USER}
|
||||||
WorkingDirectory=${VELOCITY_DIR}
|
WorkingDirectory=${VELOCITY_DIR}
|
||||||
ExecStart=${JRE_DIR}/bin/java -Xms${xms} -Xmx${xmx} -XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:G1PeriodicGCInterval=60000 -Dmojang.sessionserver=http://${api_ip}:8081/session/minecraft/hasJoined "-Dfelis.legacy-forwarding.servers=${legacy_forwarding_servers}" -jar ${VELOCITY_DIR}/velocity.jar
|
ExecStart=${JRE_DIR}/bin/java ${jvm} -Dmojang.sessionserver=http://${api_ip}:8081/session/minecraft/hasJoined "-Dfelis.legacy-forwarding.servers=${legacy_forwarding_servers}" -jar ${VELOCITY_DIR}/velocity.jar
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=5
|
RestartSec=5
|
||||||
NoNewPrivileges=yes
|
NoNewPrivileges=yes
|
||||||
|
|||||||
+82
-12
@@ -1250,6 +1250,7 @@ run_k3s() { # installed-version pinned-version [FELIS_UPGRADE_DEPS]
|
|||||||
ok() { printf "OK: %s\n" "$*"; }
|
ok() { printf "OK: %s\n" "$*"; }
|
||||||
configure_k3s_firewall() { :; }
|
configure_k3s_firewall() { :; }
|
||||||
write_k3s_config() { :; }
|
write_k3s_config() { :; }
|
||||||
|
write_k3s_service_dropin() { :; }
|
||||||
strip_k3s_kubeconfig_mode_flag() { :; }
|
strip_k3s_kubeconfig_mode_flag() { :; }
|
||||||
run_k3s_installer() { printf "INSTALLER: %s\n" "$FELIS_K3S_VERSION"; }
|
run_k3s_installer() { printf "INSTALLER: %s\n" "$FELIS_K3S_VERSION"; }
|
||||||
stage_k3s_airgap_images() { echo STAGE; }
|
stage_k3s_airgap_images() { echo STAGE; }
|
||||||
@@ -2895,11 +2896,11 @@ run_velocity_service() { # is-active(0|1) [heap]
|
|||||||
}
|
}
|
||||||
out="$(run_velocity_service 1)"
|
out="$(run_velocity_service 1)"
|
||||||
expect "a proxy with no recorded start is restarted" "SYSTEMCTL restart felis-velocity" "$out"
|
expect "a proxy with no recorded start is restarted" "SYSTEMCTL restart felis-velocity" "$out"
|
||||||
expect "the default heap starts at 64M and may grow to 1G" "java -Xms64M -Xmx1G " "$(cat "$vdir/unit")"
|
expect "the default 1G ceiling runs the serial collector and C1 from a 16M start" \
|
||||||
|
"java -Xms16M -Xmx1G -XX:+UseSerialGC -XX:TieredStopAtLevel=1 -Dmojang" "$(cat "$vdir/unit")"
|
||||||
case "$(cat "$vdir/unit")" in
|
case "$(cat "$vdir/unit")" in
|
||||||
*AlwaysPreTouch*) echo "FAIL the proxy pre-touches its heap, holding all of -Xms from the start"; fails=$((fails + 1)) ;;
|
*AlwaysPreTouch*|*UseG1GC*) echo "FAIL a 1G proxy pre-touches its heap or runs G1: $(grep ExecStart "$vdir/unit")"; fails=$((fails + 1)) ;;
|
||||||
*"-XX:G1PeriodicGCInterval="*) echo "PASS the proxy neither pre-touches its heap nor keeps growth it no longer uses" ;;
|
*) echo "PASS a 1G proxy neither pre-touches its heap nor runs G1" ;;
|
||||||
*) echo "FAIL the proxy has no periodic collection to hand back an idle heap"; fails=$((fails + 1)) ;;
|
|
||||||
esac
|
esac
|
||||||
[ -s "$vdir/fp" ] && echo "PASS the restart records what the proxy runs" \
|
[ -s "$vdir/fp" ] && echo "PASS the restart records what the proxy runs" \
|
||||||
|| { echo "FAIL no fingerprint was recorded after the restart"; fails=$((fails + 1)); }
|
|| { echo "FAIL no fingerprint was recorded after the restart"; fails=$((fails + 1)); }
|
||||||
@@ -2915,9 +2916,19 @@ expect "a stopped proxy is started whatever the fingerprint" "SYSTEMCTL restart
|
|||||||
printf 'JAVA_VERSION="25.0.1"\n' > "$vdir/jre/release"
|
printf 'JAVA_VERSION="25.0.1"\n' > "$vdir/jre/release"
|
||||||
expect "a patched JRE restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1)"
|
expect "a patched JRE restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1)"
|
||||||
expect "a new heap size restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1 3G)"
|
expect "a new heap size restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1 3G)"
|
||||||
expect "the unit carries the new ceiling" "java -Xms64M -Xmx3G " "$(cat "$vdir/unit")"
|
expect "a ceiling above 1G runs G1, whose periodic collection hands idle growth back" \
|
||||||
|
"java -Xms64M -Xmx3G -XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:G1PeriodicGCInterval=60000 -Dmojang" "$(cat "$vdir/unit")"
|
||||||
|
case "$(cat "$vdir/unit")" in
|
||||||
|
*AlwaysPreTouch*|*UseSerialGC*|*TieredStopAtLevel*) echo "FAIL a 3G proxy carries a small proxy's flags or pre-touches: $(grep ExecStart "$vdir/unit")"; fails=$((fails + 1)) ;;
|
||||||
|
*) echo "PASS a 3G proxy keeps G1 and both compilers" ;;
|
||||||
|
esac
|
||||||
|
run_velocity_service 1 1024M >/dev/null
|
||||||
|
expect "1024M is still a small proxy" "java -Xms16M -Xmx1024M -XX:+UseSerialGC " "$(cat "$vdir/unit")"
|
||||||
|
run_velocity_service 1 1025M >/dev/null
|
||||||
|
expect "a megabyte past 1G is a large one" "java -Xms64M -Xmx1025M -XX:+UseG1GC " "$(cat "$vdir/unit")"
|
||||||
run_velocity_service 1 384M >/dev/null
|
run_velocity_service 1 384M >/dev/null
|
||||||
expect "a small ceiling keeps the same small start" "java -Xms64M -Xmx384M " "$(cat "$vdir/unit")"
|
expect "a small ceiling runs the small proxy's flags" \
|
||||||
|
"java -Xms16M -Xmx384M -XX:+UseSerialGC -XX:TieredStopAtLevel=1 -Dmojang" "$(cat "$vdir/unit")"
|
||||||
# `felis rotate-token velocity` rewrites service-token, which the plugin re-reads by itself:
|
# `felis rotate-token velocity` rewrites service-token, which the plugin re-reads by itself:
|
||||||
# that line alone changing leaves the proxy running, and any other change restarts it.
|
# that line alone changing leaves the proxy running, and any other change restarts it.
|
||||||
props="$vdir/v/plugins/felis-link/felis-link.properties"
|
props="$vdir/v/plugins/felis-link/felis-link.properties"
|
||||||
@@ -3015,10 +3026,7 @@ expect "a heap in megabytes is kept" "768" "$(heap 768m)"
|
|||||||
for bad in 1 1K 0G 01G G -1G 1.5G 9999999G; do
|
for bad in 1 1K 0G 01G G -1G 1.5G 9999999G; do
|
||||||
expect "the heap spelling '$bad' is refused" "0" "$(heap "$bad")"
|
expect "the heap spelling '$bad' is refused" "0" "$(heap "$bad")"
|
||||||
done
|
done
|
||||||
case "$(awk '/^install_velocity_service\(\) \{/,/^}/' "$BS")" in
|
# The unit written for each ceiling is checked with the rerun tests above.
|
||||||
*'-Xms${xms} -Xmx${xmx} '*) echo "PASS the proxy unit takes its heap from FELIS_VELOCITY_XMX" ;;
|
|
||||||
*) echo "FAIL the proxy unit's heap is not FELIS_VELOCITY_XMX"; fails=$((fails + 1)) ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# --- reproducible image ids ---------------------------------------------------------------
|
# --- reproducible image ids ---------------------------------------------------------------
|
||||||
# restart_existing_system_servers compares image ids across runs; a default BuildKit
|
# restart_existing_system_servers compares image ids across runs; a default BuildKit
|
||||||
@@ -3230,6 +3238,63 @@ out="$(run_k3s_config "$kdir/broken.crt" "$kdir/k3s" renamed-host)"
|
|||||||
expect "a certificate openssl cannot parse is a warning, not a failed install" "WARN: could not read this node's k3s name" "$out"
|
expect "a certificate openssl cannot parse is a warning, not a failed install" "WARN: could not read this node's k3s name" "$out"
|
||||||
expect "and the drop-in is still written" 'write-kubeconfig-mode: "0600"' "$(cat "$dropin")"
|
expect "and the drop-in is still written" 'write-kubeconfig-mode: "0600"' "$(cat "$dropin")"
|
||||||
|
|
||||||
|
# k3s's Go collector runs at GOGC=50 through a systemd drop-in, written beside itself and
|
||||||
|
# loaded (with a k3s restart) only when it changed.
|
||||||
|
svblock="$(awk '/^write_k3s_service_dropin\(\) \{/,/^}/' "$BS")"
|
||||||
|
[ -n "$svblock" ] || { echo "FAIL: no write_k3s_service_dropin found in $BS"; exit 1; }
|
||||||
|
svdropin="$kdir/k3s.service.d/50-felis.conf"
|
||||||
|
run_k3s_service_dropin() {
|
||||||
|
DROPIN="$svdropin" bash -c '
|
||||||
|
set -Eeuo pipefail
|
||||||
|
ok() { echo "OK: $*"; }; log() { echo "LOG: $*"; }
|
||||||
|
remember_temp() { :; }
|
||||||
|
restore_label() { echo "RELABEL: $*"; }
|
||||||
|
systemctl() { echo "SYSTEMCTL: $*"; }
|
||||||
|
mktemp() { local p; p="$(command mktemp "$@")"; echo "MKTEMP: $(dirname "$p")" >&2; echo "$p"; }
|
||||||
|
K3S_SERVICE_DROPIN="$DROPIN"
|
||||||
|
'"$svblock"'
|
||||||
|
K3S_RESTART_NEEDED=0
|
||||||
|
write_k3s_service_dropin
|
||||||
|
echo "RESTART=$K3S_RESTART_NEEDED"' 2>&1
|
||||||
|
}
|
||||||
|
out="$(run_k3s_service_dropin)"
|
||||||
|
if [ "$(cat "$svdropin")" = '# Written by the Felis installer (deploy/bootstrap.sh); a rerun rewrites it.
|
||||||
|
[Service]
|
||||||
|
Environment=GOGC=50' ]; then
|
||||||
|
echo "PASS k3s runs its Go collector at GOGC=50"
|
||||||
|
else
|
||||||
|
echo "FAIL the k3s service drop-in is:"; cat "$svdropin"; fails=$((fails + 1))
|
||||||
|
fi
|
||||||
|
expect "a new service drop-in is loaded" "SYSTEMCTL: daemon-reload" "$out"
|
||||||
|
expect "a new service drop-in asks for a k3s restart" "RESTART=1" "$out"
|
||||||
|
if [ "$(printf '%s\n' "$out" | sed -n 's/^MKTEMP: //p' | sort -u)" = "$kdir/k3s.service.d" ]; then
|
||||||
|
echo "PASS the service drop-in is made beside itself, never under /tmp"
|
||||||
|
else
|
||||||
|
echo "FAIL temporary files for the service drop-in were made in: $(printf '%s\n' "$out" | sed -n 's/^MKTEMP: //p')"; fails=$((fails + 1))
|
||||||
|
fi
|
||||||
|
if [ "$(stat -c %a "$svdropin" 2>/dev/null || stat -f %Lp "$svdropin")" = 644 ]; then
|
||||||
|
echo "PASS the service drop-in is readable like the unit it extends"
|
||||||
|
else
|
||||||
|
echo "FAIL the service drop-in must be 0644"; fails=$((fails + 1))
|
||||||
|
fi
|
||||||
|
out="$(run_k3s_service_dropin)"
|
||||||
|
expect "an unchanged service drop-in restarts nothing" "RESTART=0" "$out"
|
||||||
|
expect "an unchanged service drop-in says so" "OK: k3s service environment already current" "$out"
|
||||||
|
case "$out" in
|
||||||
|
*daemon-reload*) echo "FAIL an unchanged service drop-in must not reload systemd"; fails=$((fails + 1)) ;;
|
||||||
|
*) echo "PASS an unchanged service drop-in reloads nothing" ;;
|
||||||
|
esac
|
||||||
|
expect "an unchanged service drop-in is still relabelled" "RELABEL: $svdropin" "$out"
|
||||||
|
if [ "$(ls "$kdir/k3s.service.d")" = "50-felis.conf" ]; then
|
||||||
|
echo "PASS no temporary file is left beside the service drop-in"
|
||||||
|
else
|
||||||
|
echo "FAIL k3s.service.d holds: $(ls "$kdir/k3s.service.d")"; fails=$((fails + 1))
|
||||||
|
fi
|
||||||
|
printf '[Service]\nEnvironment=GOGC=100\n' > "$svdropin"
|
||||||
|
out="$(run_k3s_service_dropin)"
|
||||||
|
expect "an edited service drop-in is put back" "Environment=GOGC=50" "$(cat "$svdropin")"
|
||||||
|
expect "and k3s is restarted onto it" "RESTART=1" "$out"
|
||||||
|
|
||||||
sblock="$(awk '/^strip_k3s_kubeconfig_mode_flag\(\) \{/,/^}/' "$BS")"
|
sblock="$(awk '/^strip_k3s_kubeconfig_mode_flag\(\) \{/,/^}/' "$BS")"
|
||||||
[ -n "$sblock" ] || { echo "FAIL: no strip_k3s_kubeconfig_mode_flag found in $BS"; exit 1; }
|
[ -n "$sblock" ] || { echo "FAIL: no strip_k3s_kubeconfig_mode_flag found in $BS"; exit 1; }
|
||||||
# ExecStart as k3s's installer writes it (copied off an install made with the old flag).
|
# ExecStart as k3s's installer writes it (copied off an install made with the old flag).
|
||||||
@@ -3290,17 +3355,19 @@ iblock="$(awk '/^install_k3s\(\) \{/,/^}/' "$BS")"
|
|||||||
iblock="$iblock
|
iblock="$iblock
|
||||||
$(awk '/^version_newer\(\) \{/,/^}/' "$BS")
|
$(awk '/^version_newer\(\) \{/,/^}/' "$BS")
|
||||||
$(awk '/^k3s_upgrade_allowed\(\) \{/,/^}/' "$BS")"
|
$(awk '/^k3s_upgrade_allowed\(\) \{/,/^}/' "$BS")"
|
||||||
run_install_k3s() { # $1: installed version ("" = none), $2: drop-in changed (0|1), $3: FELIS_UPGRADE_DEPS
|
run_install_k3s() { # $1: installed version ("" = none), $2: config drop-in changed (0|1), $3: FELIS_UPGRADE_DEPS, $4: service drop-in changed (0|1)
|
||||||
INSTALLED="$1" CHANGED="$2" UPGRADE="${3:-0}" KDIR="$kdir" bash -c '
|
INSTALLED="$1" CHANGED="$2" UPGRADE="${3:-0}" SVC_CHANGED="${4:-0}" KDIR="$kdir" bash -c '
|
||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
ok() { echo "OK: $*"; }; log() { echo "LOG: $*"; }; warn() { echo "WARN: $*"; }
|
ok() { echo "OK: $*"; }; log() { echo "LOG: $*"; }; warn() { echo "WARN: $*"; }
|
||||||
die() { echo "DIE: $*"; exit 1; }
|
die() { echo "DIE: $*"; exit 1; }
|
||||||
FELIS_K3S_VERSION=v1.36.4+k3s1 FELIS_UPGRADE_DEPS="$UPGRADE" K3S_BIN_DIR="$KDIR" K3S_BIN="$KDIR/k3s-under-test"
|
FELIS_K3S_VERSION=v1.36.4+k3s1 FELIS_UPGRADE_DEPS="$UPGRADE" K3S_BIN_DIR="$KDIR" K3S_BIN="$KDIR/k3s-under-test"
|
||||||
K3S_CONFIG_DROPIN=/etc/rancher/k3s/config.yaml.d/50-felis.yaml K3S_KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
K3S_CONFIG_DROPIN=/etc/rancher/k3s/config.yaml.d/50-felis.yaml K3S_KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
||||||
|
K3S_SERVICE_DROPIN=/etc/systemd/system/k3s.service.d/50-felis.conf
|
||||||
rm -f "$K3S_BIN"
|
rm -f "$K3S_BIN"
|
||||||
if [ -n "$INSTALLED" ]; then printf "#!/bin/sh\necho \"k3s version %s (abc)\"\n" "$INSTALLED" > "$K3S_BIN"; chmod +x "$K3S_BIN"; fi
|
if [ -n "$INSTALLED" ]; then printf "#!/bin/sh\necho \"k3s version %s (abc)\"\n" "$INSTALLED" > "$K3S_BIN"; chmod +x "$K3S_BIN"; fi
|
||||||
configure_k3s_firewall() { :; }
|
configure_k3s_firewall() { :; }
|
||||||
write_k3s_config() { [ "$CHANGED" = 0 ] || K3S_RESTART_NEEDED=1; }
|
write_k3s_config() { [ "$CHANGED" = 0 ] || K3S_RESTART_NEEDED=1; }
|
||||||
|
write_k3s_service_dropin() { echo "SERVICE-DROPIN"; [ "$SVC_CHANGED" = 0 ] || K3S_RESTART_NEEDED=1; }
|
||||||
strip_k3s_kubeconfig_mode_flag() { :; }
|
strip_k3s_kubeconfig_mode_flag() { :; }
|
||||||
run_k3s_installer() { echo "INSTALLER"; printf "#!/bin/sh\n" > "$K3S_BIN"; command chmod +x "$K3S_BIN"; }
|
run_k3s_installer() { echo "INSTALLER"; printf "#!/bin/sh\n" > "$K3S_BIN"; command chmod +x "$K3S_BIN"; }
|
||||||
stage_k3s_airgap_images() { echo "STAGE"; }
|
stage_k3s_airgap_images() { echo "STAGE"; }
|
||||||
@@ -3316,9 +3383,12 @@ expect "the admin kubeconfig is made root-only once the node is up" "READY
|
|||||||
CHMOD: 0600 /etc/rancher/k3s/k3s.yaml" "$out"
|
CHMOD: 0600 /etc/rancher/k3s/k3s.yaml" "$out"
|
||||||
out="$(run_install_k3s v1.36.4+k3s1 0)"
|
out="$(run_install_k3s v1.36.4+k3s1 0)"
|
||||||
case "$out" in *"restart k3s"*) echo "FAIL unchanged k3s settings must not restart k3s"; fails=$((fails + 1)) ;; *) echo "PASS unchanged k3s settings restart nothing" ;; esac
|
case "$out" in *"restart k3s"*) echo "FAIL unchanged k3s settings must not restart k3s"; fails=$((fails + 1)) ;; *) echo "PASS unchanged k3s settings restart nothing" ;; esac
|
||||||
|
expect "a new service environment alone restarts a running k3s" "SYSTEMCTL: restart k3s" "$(run_install_k3s v1.36.4+k3s1 0 0 1)"
|
||||||
out="$(run_install_k3s "" 1)"
|
out="$(run_install_k3s "" 1)"
|
||||||
expect "a fresh host runs the k3s installer and waits for the node" "INSTALLER
|
expect "a fresh host runs the k3s installer and waits for the node" "INSTALLER
|
||||||
SYSTEMCTL: enable --now k3s" "$out"
|
SYSTEMCTL: enable --now k3s" "$out"
|
||||||
|
expect "a fresh k3s has its service environment before its first start" "SERVICE-DROPIN INSTALLER " \
|
||||||
|
"$(printf '%s\n' "$out" | grep -E '^(SERVICE-DROPIN|INSTALLER)$' | tr '\n' ' ')"
|
||||||
expect "a fresh host stages k3s's images before k3s first starts" "STAGE
|
expect "a fresh host stages k3s's images before k3s first starts" "STAGE
|
||||||
INSTALLER" "$out"
|
INSTALLER" "$out"
|
||||||
expect "a fresh host's kubeconfig is made root-only too" "CHMOD: 0600 /etc/rancher/k3s/k3s.yaml" "$out"
|
expect "a fresh host's kubeconfig is made root-only too" "CHMOD: 0600 /etc/rancher/k3s/k3s.yaml" "$out"
|
||||||
|
|||||||
@@ -321,6 +321,16 @@ stop_database_pod() {
|
|||||||
|| warn "${PG_DEPLOYMENT} did not stop within 2 minutes; its cluster recovers from its WAL on the next start"
|
|| warn "${PG_DEPLOYMENT} did not stop within 2 minutes; its cluster recovers from its WAL on the next start"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# remove_k3s_service_dropin deletes the environment the installer gives the k3s service
|
||||||
|
# (bootstrap.sh, write_k3s_service_dropin). k3s-uninstall.sh removes the unit and its .env
|
||||||
|
# file and leaves the unit's drop-in directory.
|
||||||
|
remove_k3s_service_dropin() {
|
||||||
|
[ -f "${UNIT_DIR}/k3s.service.d/50-felis.conf" ] || return 0
|
||||||
|
rm -f "${UNIT_DIR}/k3s.service.d/50-felis.conf"
|
||||||
|
rmdir "${UNIT_DIR}/k3s.service.d" 2>/dev/null || true
|
||||||
|
ok "k3s service environment removed"
|
||||||
|
}
|
||||||
|
|
||||||
remove_k3s() {
|
remove_k3s() {
|
||||||
local stamp
|
local stamp
|
||||||
[ "$PURGE" = 1 ] || stop_database_pod
|
[ "$PURGE" = 1 ] || stop_database_pod
|
||||||
@@ -336,6 +346,7 @@ remove_k3s() {
|
|||||||
if [ -x "${K3S_BIN_DIR}/k3s-uninstall.sh" ]; then
|
if [ -x "${K3S_BIN_DIR}/k3s-uninstall.sh" ]; then
|
||||||
log "running k3s-uninstall.sh"
|
log "running k3s-uninstall.sh"
|
||||||
"${K3S_BIN_DIR}/k3s-uninstall.sh" >/dev/null 2>&1 || warn "k3s-uninstall.sh reported an error; check /var/lib/rancher and /etc/rancher"
|
"${K3S_BIN_DIR}/k3s-uninstall.sh" >/dev/null 2>&1 || warn "k3s-uninstall.sh reported an error; check /var/lib/rancher and /etc/rancher"
|
||||||
|
remove_k3s_service_dropin
|
||||||
ok "k3s removed"
|
ok "k3s removed"
|
||||||
else
|
else
|
||||||
warn "k3s is at ${K3S_BIN_DIR}/k3s but ${K3S_BIN_DIR}/k3s-uninstall.sh is missing; remove k3s by hand"
|
warn "k3s is at ${K3S_BIN_DIR}/k3s but ${K3S_BIN_DIR}/k3s-uninstall.sh is missing; remove k3s by hand"
|
||||||
@@ -522,6 +533,8 @@ main() {
|
|||||||
case "$K3S_MODE" in
|
case "$K3S_MODE" in
|
||||||
remove) remove_k3s ;;
|
remove) remove_k3s ;;
|
||||||
keep) remove_from_cluster ;;
|
keep) remove_from_cluster ;;
|
||||||
|
# k3s was removed some other way; what it left of the installer's goes too.
|
||||||
|
absent) remove_k3s_service_dropin ;;
|
||||||
esac
|
esac
|
||||||
remove_nft_tables
|
remove_nft_tables
|
||||||
remove_firewalld_rules "$game" "$nano"
|
remove_firewalld_rules "$game" "$nano"
|
||||||
|
|||||||
@@ -35,6 +35,8 @@ fresh_host() {
|
|||||||
for u in felis-db-backup.timer felis-db-backup.service felis-velocity.service felis-postgres-firewall.service; do
|
for u in felis-db-backup.timer felis-db-backup.service felis-velocity.service felis-postgres-firewall.service; do
|
||||||
printf '[Unit]\n' > "$root/h/units/$u"
|
printf '[Unit]\n' > "$root/h/units/$u"
|
||||||
done
|
done
|
||||||
|
mkdir -p "$root/h/units/k3s.service.d"
|
||||||
|
printf '[Service]\nEnvironment=GOGC=50\n' > "$root/h/units/k3s.service.d/50-felis.conf"
|
||||||
printf '[Service]\nExecStart=/usr/local/bin/cloudflared --config %s tunnel run\n' "$root/h/etc/cloudflared.yml" \
|
printf '[Service]\nExecStart=/usr/local/bin/cloudflared --config %s tunnel run\n' "$root/h/etc/cloudflared.yml" \
|
||||||
> "$root/h/units/cloudflared-felis.service"
|
> "$root/h/units/cloudflared-felis.service"
|
||||||
printf 'tunnel: abc\ncredentials-file: %s\n' "$root/h/cf/abc.json" > "$root/h/etc/cloudflared.yml"
|
printf 'tunnel: abc\ncredentials-file: %s\n' "$root/h/cf/abc.json" > "$root/h/etc/cloudflared.yml"
|
||||||
@@ -155,6 +157,9 @@ refute "keep-data leaves the database alone" "DROP DATABASE" "$calls"
|
|||||||
|| { echo "FAIL /opt/felis or the host binary is still there"; fails=$((fails + 1)); }
|
|| { echo "FAIL /opt/felis or the host binary is still there"; fails=$((fails + 1)); }
|
||||||
[ -z "$(ls "$root/h/units")" ] && echo "PASS every Felis unit file is removed" \
|
[ -z "$(ls "$root/h/units")" ] && echo "PASS every Felis unit file is removed" \
|
||||||
|| { echo "FAIL units left: $(ls "$root/h/units")"; fails=$((fails + 1)); }
|
|| { echo "FAIL units left: $(ls "$root/h/units")"; fails=$((fails + 1)); }
|
||||||
|
[ ! -e "$root/h/units/k3s.service.d" ] \
|
||||||
|
&& echo "PASS the k3s service environment k3s's uninstaller leaves is removed with its directory" \
|
||||||
|
|| { echo "FAIL the k3s service drop-in is still there: $(ls -R "$root/h/units")"; fails=$((fails + 1)); }
|
||||||
expect "the timers are disabled" "SYSTEMCTL disable --now felis-db-backup.timer" "$calls"
|
expect "the timers are disabled" "SYSTEMCTL disable --now felis-db-backup.timer" "$calls"
|
||||||
expect "the velocity user is removed" "USERDEL felis-velocity" "$calls"
|
expect "the velocity user is removed" "USERDEL felis-velocity" "$calls"
|
||||||
expect "the run ends pointing at the reinstall steps" "Reinstall on top of kept data" "$out"
|
expect "the run ends pointing at the reinstall steps" "Reinstall on top of kept data" "$out"
|
||||||
@@ -190,6 +195,9 @@ expect "Felis's volumes are retained before their claims go" 'KUBE patch pv pvc-
|
|||||||
refute "a volume of another namespace is not touched" "patch pv pvc-9" "$calls"
|
refute "a volume of another namespace is not touched" "patch pv pvc-9" "$calls"
|
||||||
expect "Felis's namespaces are deleted" "KUBE delete namespace felis minecraft felis-build" "$calls"
|
expect "Felis's namespaces are deleted" "KUBE delete namespace felis minecraft felis-build" "$calls"
|
||||||
expect "the CRD is deleted" "KUBE delete crd minecraftservers.felis.lolicon.best" "$calls"
|
expect "the CRD is deleted" "KUBE delete crd minecraftservers.felis.lolicon.best" "$calls"
|
||||||
|
[ -f "$root/h/units/k3s.service.d/50-felis.conf" ] \
|
||||||
|
&& echo "PASS a k3s that stays keeps its service environment, like its config" \
|
||||||
|
|| { echo "FAIL the kept k3s lost its service drop-in"; fails=$((fails + 1)); }
|
||||||
|
|
||||||
out="$(fresh_host; run_uninstall down --yes)"
|
out="$(fresh_host; run_uninstall down --yes)"
|
||||||
expect "a k3s that does not answer stops the run" "k3s does not answer" "$out"
|
expect "a k3s that does not answer stops the run" "k3s does not answer" "$out"
|
||||||
@@ -198,6 +206,18 @@ run_uninstall down --yes --keep-k3s >/dev/null
|
|||||||
calls="$(cat "$root/calls")"
|
calls="$(cat "$root/calls")"
|
||||||
refute "--keep-k3s never runs k3s's uninstaller" "RUN k3s-uninstall.sh" "$calls"
|
refute "--keep-k3s never runs k3s's uninstaller" "RUN k3s-uninstall.sh" "$calls"
|
||||||
|
|
||||||
|
# --- k3s already gone ----------------------------------------------------------------------
|
||||||
|
fresh_host
|
||||||
|
rm -f "$root/h/bin/k3s"
|
||||||
|
printf '[Service]\nLimitNOFILE=4096\n' > "$root/h/units/k3s.service.d/90-admin.conf"
|
||||||
|
run_uninstall down --yes >/dev/null
|
||||||
|
[ ! -e "$root/h/units/k3s.service.d/50-felis.conf" ] \
|
||||||
|
&& echo "PASS a k3s removed some other way does not leave the installer's service environment" \
|
||||||
|
|| { echo "FAIL the k3s service drop-in outlived k3s"; fails=$((fails + 1)); }
|
||||||
|
[ -f "$root/h/units/k3s.service.d/90-admin.conf" ] \
|
||||||
|
&& echo "PASS a drop-in someone else wrote beside it stays, with the directory" \
|
||||||
|
|| { echo "FAIL the uninstall removed a k3s drop-in it did not write"; fails=$((fails + 1)); }
|
||||||
|
|
||||||
# --- purge -------------------------------------------------------------------------------
|
# --- purge -------------------------------------------------------------------------------
|
||||||
fresh_host
|
fresh_host
|
||||||
out="$(run_uninstall "default felis minecraft" --purge --yes)"
|
out="$(run_uninstall "default felis minecraft" --purge --yes)"
|
||||||
|
|||||||
+32
-19
@@ -216,21 +216,30 @@ to 0 for about 8 minutes on the reference VM.
|
|||||||
### What the platform itself uses
|
### What the platform itself uses
|
||||||
|
|
||||||
Measured on the verification host (4 vCPU, 5.5 GB RAM, 6 GB swap, CentOS Stream 9
|
Measured on the verification host (4 vCPU, 5.5 GB RAM, 6 GB swap, CentOS Stream 9
|
||||||
aarch64) with the control plane, the login and lobby system servers and one idle Paper
|
aarch64) on an idle network, as each process's proportional set size (PSS: a page shared
|
||||||
server running **[VM-VERIFIED]**:
|
by several processes is split among them; `/proc/<pid>/smaps_rollup`) **[VM-VERIFIED]**:
|
||||||
|
|
||||||
| Process | Resident memory |
|
| Process | Memory (PSS) |
|
||||||
|---|---|
|
|---|---|
|
||||||
| k3s (server, kubelet, containerd) | ~1.1 GB |
|
| k3s (API server, controllers, scheduler, kubelet) | ~370 MiB |
|
||||||
| Velocity (`-Xms64M -Xmx1G`, idle; it grows with players) | ~0.25 GB |
|
| k3s's containerd and the pods' shims | ~170 MiB |
|
||||||
| lobby (Paper, pod limit 1 GiB) | ~0.7–0.85 GB |
|
| CoreDNS and the local-path volume provisioner | ~105 MiB |
|
||||||
| login (Limbo, pod limit 512 MiB) | ~0.16 GB |
|
| Velocity (`-Xms16M -Xmx1G`, idle; it grows with players) | ~175 MiB |
|
||||||
| felis-api, felis-operator, registry gate | ~50 MB each |
|
| felis-api, felis-operator, registry gate | ~85 MiB together |
|
||||||
| PostgreSQL (the felis-postgres pod) | ~30 MB plus page cache |
|
| Image registry | ~25 MiB |
|
||||||
| **Total in use** | **~2.9 GB** |
|
| PostgreSQL (the felis-postgres pod) | ~40 MiB plus page cache |
|
||||||
|
| **Infrastructure total** | **~1 GB** |
|
||||||
|
|
||||||
Every game server adds the memory its owner gave it: the pod's limit equals its request,
|
The installer runs k3s, and the containerd it starts, with Go's collector at half its
|
||||||
and the JVM heap is derived from it (§1a). Quotas cap it per user (panel → 管理 → 配额).
|
default heap growth (`GOGC=50`, in `/etc/systemd/system/k3s.service.d/50-felis.conf`): an
|
||||||
|
idle k3s holds about 150 MiB live and would otherwise let its heap reach twice that before
|
||||||
|
collecting. It saves about 70 MiB for about 2% of one core. An install from before this
|
||||||
|
picks it up on its next installer run, which restarts k3s; the pods keep running.
|
||||||
|
|
||||||
|
The login (Limbo, pod limit 512 MiB, ~0.16 GB) and lobby (Paper, pod limit 1 GiB,
|
||||||
|
~0.7–0.85 GB) system servers come on top, and every game server adds the memory its owner
|
||||||
|
gave it: the pod's limit equals its request, and the JVM heap is derived from it (§1a).
|
||||||
|
Quotas cap it per user (panel → 管理 → 配额).
|
||||||
|
|
||||||
A release install builds nothing (§1). When the installer builds on the host its peak is
|
A release install builds nothing (§1). When the installer builds on the host its peak is
|
||||||
the image builds (Docker plus a Gradle container). Afterwards it stops Docker, and Docker's
|
the image builds (Docker plus a Gradle container). Afterwards it stops Docker, and Docker's
|
||||||
@@ -249,15 +258,19 @@ adds a 2 GiB `/swapfile`.
|
|||||||
|
|
||||||
The player-count rows are planning figures, not measurements: a Minecraft server's cost
|
The player-count rows are planning figures, not measurements: a Minecraft server's cost
|
||||||
depends mostly on what its players do (view distance, redstone, mods). Size RAM as the
|
depends mostly on what its players do (view distance, redstone, mods). Size RAM as the
|
||||||
platform's ~3 GB plus the sum of the servers you expect to run at once, then add a
|
infrastructure's ~1 GB and the login and lobby servers' ~1 GB, plus the sum of the servers
|
||||||
quarter for the page cache and PostgreSQL. Velocity itself needs little per player; raise
|
you expect to run at once, then add a quarter for the page cache and PostgreSQL. Velocity
|
||||||
its heap when `journalctl -u felis-velocity` shows long GC pauses or `OutOfMemoryError`.
|
itself needs little per player; raise its heap when `journalctl -u felis-velocity` shows
|
||||||
|
long GC pauses or `OutOfMemoryError`.
|
||||||
|
|
||||||
`FELIS_VELOCITY_XMX` (default `1G`, at least `256M`, written `<n>M` or `<n>G`) is read on
|
`FELIS_VELOCITY_XMX` (default `1G`, at least `256M`, written `<n>M` or `<n>G`) is read on
|
||||||
every installer run. The heap starts at 64M and grows toward the maximum as players arrive;
|
every installer run. Up to 1G the heap starts at 16M and the proxy runs the serial collector
|
||||||
a periodic collection hands the growth back once they have left. Changing it rewrites the
|
and only the C1 compiler: its plugins hold about 50M live, so a collection takes
|
||||||
unit, and the rerun restarts the proxy, which disconnects everyone online; do it in a quiet
|
milliseconds, and compression and encryption run in Velocity's native library. Above 1G it
|
||||||
hour **[VM-VERIFIED]**:
|
runs G1 from a 64M start, since a serial full collection over a large heap would stall
|
||||||
|
every player at once, and a periodic collection hands the growth back once players have
|
||||||
|
left. Changing it rewrites the unit, and the rerun restarts the proxy, which disconnects
|
||||||
|
everyone online; do it in a quiet hour **[VM-VERIFIED]**:
|
||||||
|
|
||||||
```
|
```
|
||||||
curl -fsSL <raw-url>/deploy/bootstrap.sh | sudo FELIS_VELOCITY_XMX=2G bash
|
curl -fsSL <raw-url>/deploy/bootstrap.sh | sudo FELIS_VELOCITY_XMX=2G bash
|
||||||
|
|||||||
Reference in new issue
Block a user