FelisMC/Felis
2
0 Fork 0
Code Issues Pull Requests 6 Actions Packages Projects Releases Wiki Activity

fix(watchdog): 外部心跳、巡检失败兜底告警、状态文件损坏自动移开

Browse Source
This commit is contained in:
Lemon-miaow committed 2026-09-27 08:40:32 +08:00
1 parent 149ab0be66
commit 82ffa55a8f
11 files changed
+1360 -44
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

deploy/bootstrap.sh
+96 -6
View File
@@ -266,6 +266,13 @@ FELIS_OFFSITE_BUCKET="${FELIS_OFFSITE_BUCKET:-}"
FELIS_OFFSITE_REGION="${FELIS_OFFSITE_REGION:-}"
FELIS_OFFSITE_PREFIX="${FELIS_OFFSITE_PREFIX:-}"
FELIS_OFFSITE_DB_KEEP="${FELIS_OFFSITE_DB_KEEP:-}"
# The watchdog's heartbeat (troubleshooting §14): the ping URL of a check at a monitoring
# service such as Healthchecks.io, a dead man's switch. Every watchdog run pings it, and
# the service mails its own users when the pings stop or report failure: the host down,
# the watchdog broken, alerts that reach no one. Nothing on this host can report its own
# death. The URL is kept in /etc/felis/watchdog-heartbeat-url, mode 0600, as its path is
# the key that pings the check; off removes it, and a re-run without it keeps it.
FELIS_WATCHDOG_HEARTBEAT_URL="${FELIS_WATCHDOG_HEARTBEAT_URL:-}"
INSTALL_MODE="${FELIS_INSTALL_MODE:-}"
# strict stops the install on any preflight problem (preflight below); warn reports them
# and goes on, for a host the checks misjudge.
@@ -416,6 +423,8 @@ NANO_SERVICE="/etc/systemd/system/felis-nano.service"
DB_BACKUP_SERVICE="/etc/systemd/system/felis-db-backup.service"
DB_BACKUP_TIMER="/etc/systemd/system/felis-db-backup.timer"
WATCHDOG_SERVICE="/etc/systemd/system/felis-watchdog.service"
# OnFailure= of felis-watchdog.service: reports a run that failed (felis watchdog -unit-failed).
WATCHDOG_FAILED_SERVICE="/etc/systemd/system/felis-watchdog-failed.service"
WATCHDOG_TIMER="/etc/systemd/system/felis-watchdog.timer"
UPDATE_CHECK_SERVICE="/etc/systemd/system/felis-update-check.service"
UPDATE_CHECK_TIMER="/etc/systemd/system/felis-update-check.timer"
@@ -436,6 +445,8 @@ BUILD_TOOLS_STATUS="/var/lib/felis/build-tools/status.json"
# restarts the control plane and the system servers on purpose. cleanup removes it; the
# time in it is the backstop for an installer killed before its EXIT trap runs.
WATCHDOG_QUIET_FILE="/run/felis/watchdog-quiet-until"
# FELIS_WATCHDOG_HEARTBEAT_URL, where felis watchdog reads it by default.
WATCHDOG_HEARTBEAT_FILE="${STATE_DIR}/watchdog-heartbeat-url"
VELOCITY_DIR="/opt/felis/velocity"
VELOCITY_USER="felis-velocity"
VELOCITY_SERVICE="/etc/systemd/system/felis-velocity.service"
@@ -926,6 +937,7 @@ validate_settings() {
validate_listen FELIS_NANO_LISTEN "$FELIS_NANO_LISTEN"
validate_cidr FELIS_NANO_PROXY_CIDR "$FELIS_NANO_PROXY_CIDR"
validate_offsite_settings
validate_heartbeat_url
case "$FELIS_PREFLIGHT" in
strict|warn) ;;
*) die "FELIS_PREFLIGHT must be strict or warn (got '${FELIS_PREFLIGHT}')" ;;
@@ -1028,6 +1040,18 @@ validate_offsite_settings() {
fi
}
# validate_heartbeat_url checks FELIS_WATCHDOG_HEARTBEAT_URL before anything is
# installed: a check's http(s) ping URL, or off. The messages leave the URL out: its path
# is the key that pings the check.
validate_heartbeat_url() {
case "$FELIS_WATCHDOG_HEARTBEAT_URL" in
"" | off) ;;
*[[:space:]]* | *\"* | *\'* | *\\*) die "FELIS_WATCHDOG_HEARTBEAT_URL must not contain spaces, quotes or backslashes" ;;
http://[!/]* | https://[!/]*) ;;
*) die "FELIS_WATCHDOG_HEARTBEAT_URL must be the http:// or https:// ping URL of a monitoring service's check, or off" ;;
esac
}
# ---------------------------------------------------------------------------
# 0. Privilege & host facts
# ---------------------------------------------------------------------------
@@ -4685,11 +4709,6 @@ summary_offsite() {
fi
}
# The platform watchdog: every two minutes it checks the control plane, the login gate,
# the fleet, PostgreSQL, the game proxy, the database backups and the host's disks and
# memory, and mails the owners (their verified addresses, over the [smtp] relay) what
# has stayed wrong long enough to matter. It runs on the host so a k3s that is down is
# still reported. The first run happens now, so a broken unit shows up in this install.
# The daily version check. Felis applies no update on its own; `felis update --record`
# compares what this host runs with the newest upstream releases and stores the result,
# which the panel's Updates page shows with the command that applies each update. It runs
@@ -4730,16 +4749,29 @@ EOF
ok "version check: daily; the panel's Updates page shows what has a newer release (journalctl -u felis-update-check)"
}
# The platform watchdog: every two minutes it checks the control plane, the login gate,
# the fleet, PostgreSQL, the game proxy, the database backups and the host's disks and
# memory, and mails the owners (their verified addresses, over the [smtp] relay) what
# has stayed wrong long enough to matter. It runs on the host so a k3s that is down is
# still reported. The first run happens now, so a broken unit shows up in this install.
# A run that fails starts felis-watchdog-failed.service (OnFailure=), which mails the
# failure once five runs in a row failed, through the relay the last good run cached, and
# pings the heartbeat's failure endpoint. The heartbeat (FELIS_WATCHDOG_HEARTBEAT_URL) is
# what notices a host that is down or a watchdog that no longer runs at all. The units
# name no heartbeat flag: felis watchdog reads WATCHDOG_HEARTBEAT_FILE by default, so an
# older binary put back under these units still runs.
install_watchdog_timer() {
local disks="/,/var/lib/rancher/k3s,/var/lib/felis" path
for path in "$FELIS_WORLDS_HOST_PATH" "$FELIS_ARCHIVE_LOCAL_PATH" "$FELIS_DB_BACKUP_DIR"; do
if [ -n "$path" ]; then disks="${disks},${path}"; fi
done
write_heartbeat_url
install -d -m 0700 "$(dirname "$WATCHDOG_STATE")"
cat > "$WATCHDOG_SERVICE" <<EOF
[Unit]
Description=Felis platform watchdog (health checks, owner alert mail)
After=network-online.target k3s.service
OnFailure=felis-watchdog-failed.service
[Service]
Type=oneshot
@@ -4749,6 +4781,19 @@ Nice=5
PrivateTmp=yes
NoNewPrivileges=yes
ProtectSystem=full
EOF
cat > "$WATCHDOG_FAILED_SERVICE" <<EOF
[Unit]
Description=Felis platform watchdog failure report (owner alert mail, heartbeat failure ping)
[Service]
Type=oneshot
ExecStart=${HOST_BIN} watchdog -unit-failed -config ${STATE_DIR}/felis.host.toml -state ${WATCHDOG_STATE} -quiet-file ${WATCHDOG_QUIET_FILE}
TimeoutStartSec=2min
Nice=5
PrivateTmp=yes
NoNewPrivileges=yes
ProtectSystem=full
EOF
cat > "$WATCHDOG_TIMER" <<EOF
[Unit]
@@ -4764,14 +4809,58 @@ WantedBy=timers.target
EOF
systemctl daemon-reload
systemctl enable --now felis-watchdog.timer
local reach="mails the owners' verified addresses"
if [ -f "$WATCHDOG_HEARTBEAT_FILE" ]; then
reach="${reach} and pings the heartbeat at $(heartbeat_host)"
fi
if systemctl start felis-watchdog.service; then
ok "watchdog: checks every 2 minutes and mails the owners' verified addresses (journalctl -u felis-watchdog)"
ok "watchdog: checks every 2 minutes and ${reach} (journalctl -u felis-watchdog)"
else
journalctl -u felis-watchdog.service -n 20 --no-pager >&2 || true
warn "the first watchdog run failed (log above); nothing will be mailed until it runs: sudo systemctl start felis-watchdog.service"
fi
}
# write_heartbeat_url keeps FELIS_WATCHDOG_HEARTBEAT_URL in WATCHDOG_HEARTBEAT_FILE, mode
# 0600 and replaced whole; off removes the file, and no value keeps it as it is.
write_heartbeat_url() {
local tmp
case "$FELIS_WATCHDOG_HEARTBEAT_URL" in
"") return 0 ;;
off)
rm -f -- "$WATCHDOG_HEARTBEAT_FILE"
return 0
;;
esac
# mktemp creates the file 0600, before the key is in it.
tmp="$(mktemp "${WATCHDOG_HEARTBEAT_FILE}.XXXXXX")"
printf '%s\n' "$FELIS_WATCHDOG_HEARTBEAT_URL" > "$tmp"
mv -f -- "$tmp" "$WATCHDOG_HEARTBEAT_FILE"
}
# heartbeat_host is the heartbeat URL as the install shows it: its scheme and host.
heartbeat_host() {
local url rest host
url="$(head -n 1 "$WATCHDOG_HEARTBEAT_FILE")"
rest="${url#*://}"
host="${rest%%/*}"
host="${host%%\?*}"
host="${host##*@}"
printf '%s://%s/...' "${url%%://*}" "$host"
}
# summary_heartbeat closes the install on the heartbeat: without one, nothing off this
# machine notices it going down.
summary_heartbeat() {
if [ -f "$WATCHDOG_HEARTBEAT_FILE" ]; then
log "Heartbeat: every watchdog run pings $(heartbeat_host); that service mails you when the pings stop."
return 0
fi
warn "NO HEARTBEAT: nothing off this machine notices it going down or its watchdog stopping."
warn "Create a check at a monitoring service (Healthchecks.io or alike; period 2 min, grace 10 min),"
warn "then re-run with FELIS_WATCHDOG_HEARTBEAT_URL=<its ping URL> (docs/troubleshooting.md §14)."
}
# The build lane's tools: kaniko and trivy (pinned by digest in internal/build/tools.go)
# and Trivy's vulnerability and Java DBs, copied into the registry's mirror/ where build
# Jobs pull them; the build namespace has no internet egress. The timer refreshes the DBs
@@ -5374,6 +5463,7 @@ summary() {
fi
echo
summary_offsite
summary_heartbeat
echo
}
deploy/bootstrap_test.sh
+88 -2
View File
@@ -1837,6 +1837,7 @@ run_timer() { # exit status of the first backup
FELIS_DB_BACKUP_METRICS=/var/lib/node_exporter/textfile_collector/felis_db_backup.prom \
HOST_BIN=/usr/local/bin/felis STATE_DIR=/etc/felis bash -c '
ok() { printf "OK: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; }
log() { printf "LOG: %s\n" "$*"; }; die() { printf "DIE: %s\n" "$*"; exit 1; }
systemctl() { printf "SYSTEMCTL: %s\n" "$*"; [ "$1" != start ] || return "$FIRST"; }
journalctl() { printf "JOURNAL: pg_dump: connection refused\n"; }
'"$tblock"'
@@ -1894,23 +1895,36 @@ esac
wblock="$(awk '/^install_watchdog_timer\(\) \{/,/^}/' "$BS")"
[ -n "$wblock" ] || { echo "FAIL: no install_watchdog_timer found in $BS"; exit 1; }
[ "$(printf '%s\n' "$wblock" | wc -l)" -lt 60 ] \
[ "$(printf '%s\n' "$wblock" | wc -l)" -lt 90 ] \
|| { echo "FAIL: the extracted block is not install_watchdog_timer -- did its closing brace move?"; exit 1; }
qblock="$(awk '/^quiet_watchdog\(\) \{/,/^}/' "$BS")"
[ -n "$qblock" ] || { echo "FAIL: no quiet_watchdog found in $BS"; exit 1; }
hbblock=""
for fn in write_heartbeat_url heartbeat_host summary_heartbeat validate_heartbeat_url; do
blk="$(awk "/^${fn}\\(\\) \\{/,/^}/" "$BS")"
[ -n "$blk" ] || { echo "FAIL: no ${fn} found in $BS"; exit 1; }
[ "$(printf '%s\n' "$blk" | wc -l)" -lt 30 ] \
|| { echo "FAIL: the extracted block is not ${fn} -- did its closing brace move?"; exit 1; }
hbblock="${hbblock}${blk}
"
done
tdir="$(mktemp -d)"
run_watchdog_timer() { # $1: exit status of the first run, $2: FELIS_WORLDS_HOST_PATH, $3: NODE_IP
FIRST="$1" FELIS_WORLDS_HOST_PATH="$2" NODE_IP="${3:-}" WATCHDOG_SERVICE="$tdir/felis-watchdog.service" WATCHDOG_TIMER="$tdir/felis-watchdog.timer" \
WATCHDOG_FAILED_SERVICE="$tdir/felis-watchdog-failed.service" WATCHDOG_HEARTBEAT_FILE="$tdir/watchdog-heartbeat-url" \
WATCHDOG_STATE="$tdir/watchdog/state.json" WATCHDOG_QUIET_FILE=/run/felis/watchdog-quiet-until \
FELIS_DB_BACKUP_DIR=/var/lib/felis/db-backups FELIS_ARCHIVE_LOCAL_PATH=/var/lib/felis/archives FELIS_GAME_PORT=25577 \
HOST_BIN=/usr/local/bin/felis STATE_DIR=/etc/felis bash -c '
set -Eeuo pipefail
ok() { printf "OK: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; }
systemctl() { printf "SYSTEMCTL: %s\n" "$*"; [ "$1" != start ] || return "$FIRST"; }
log() { printf "LOG: %s\n" "$*"; }; die() { printf "DIE: %s\n" "$*"; exit 1; }
journalctl() { printf "JOURNAL: parse /etc/felis/felis.host.toml\n"; }
FELIS_WATCHDOG_HEARTBEAT_URL="${FELIS_WATCHDOG_HEARTBEAT_URL:-}"
'"$hbblock"'
'"$wblock"'
install_watchdog_timer' 2>&1
eval "${SCRIPT:-install_watchdog_timer}"' 2>&1
}
out="$(run_watchdog_timer 0 "")"
@@ -1930,6 +1944,78 @@ else
echo "FAIL the watchdog state directory must be 0700"; fails=$((fails + 1))
fi
# A run that fails is reported by felis-watchdog-failed.service, and the heartbeat URL is
# kept private and shown by its host only. The units name no heartbeat flag, so a binary
# from before it still runs under them.
failed_unit="$(cat "$tdir/felis-watchdog-failed.service")"
expect "a failed run starts the failure report" "OnFailure=felis-watchdog-failed.service" "$unit"
expect "the failure report runs the host binary against the watchdog's state" \
"ExecStart=/usr/local/bin/felis watchdog -unit-failed -config /etc/felis/felis.host.toml -state $tdir/watchdog/state.json -quiet-file /run/felis/watchdog-quiet-until
" "$failed_unit"
expect "the failure report is a oneshot" "Type=oneshot" "$failed_unit"
expect "a wedged failure report is killed" "TimeoutStartSec=2min" "$failed_unit"
case "$unit$failed_unit" in
*-heartbeat*) echo "FAIL the watchdog units must name no heartbeat flag (an older binary refuses it)"; fails=$((fails + 1)) ;;
*) echo "PASS the watchdog units name no heartbeat flag" ;;
esac
case "$out" in
*"pings the heartbeat"*) echo "FAIL without a heartbeat URL the install must claim no heartbeat: $out"; fails=$((fails + 1)) ;;
*) echo "PASS without a heartbeat URL the install claims none" ;;
esac
[ ! -e "$tdir/watchdog-heartbeat-url" ] && echo "PASS no heartbeat URL writes no file" \
|| { echo "FAIL no heartbeat URL must write no file"; fails=$((fails + 1)); }
out="$(FELIS_WATCHDOG_HEARTBEAT_URL=https://hc-ping.com/5f1e0c2a-key run_watchdog_timer 0 "")"
expect "the heartbeat URL is kept where felis watchdog reads it" "https://hc-ping.com/5f1e0c2a-key" "$(cat "$tdir/watchdog-heartbeat-url" 2>&1)"
if [ "$(stat -c %a "$tdir/watchdog-heartbeat-url" 2>/dev/null || stat -f %Lp "$tdir/watchdog-heartbeat-url")" = 600 ]; then
echo "PASS the heartbeat URL file is private (its path pings the check)"
else
echo "FAIL the heartbeat URL file must be 0600"; fails=$((fails + 1))
fi
expect "the install says the watchdog pings the heartbeat" \
"OK: watchdog: checks every 2 minutes and mails the owners' verified addresses and pings the heartbeat at https://hc-ping.com/... (journalctl -u felis-watchdog)" "$out"
case "$out" in
*5f1e0c2a*) echo "FAIL the install printed the heartbeat URL's key: $out"; fails=$((fails + 1)) ;;
*) echo "PASS the install shows the heartbeat by its host only" ;;
esac
out="$(run_watchdog_timer 0 "")"
expect "a re-run without the variable keeps the heartbeat" "https://hc-ping.com/5f1e0c2a-key" "$(cat "$tdir/watchdog-heartbeat-url" 2>&1)"
expect "a re-run without the variable still reports the heartbeat" "and pings the heartbeat at https://hc-ping.com/..." "$out"
out="$(SCRIPT=summary_heartbeat run_watchdog_timer 0 "")"
expect "the summary names the heartbeat by its host" "LOG: Heartbeat: every watchdog run pings https://hc-ping.com/...;" "$out"
out="$(FELIS_WATCHDOG_HEARTBEAT_URL=off run_watchdog_timer 0 "")"
[ ! -e "$tdir/watchdog-heartbeat-url" ] && echo "PASS off removes the heartbeat" \
|| { echo "FAIL FELIS_WATCHDOG_HEARTBEAT_URL=off must remove the file"; fails=$((fails + 1)); }
leftover="$(find "$tdir" -maxdepth 1 -name 'watchdog-heartbeat-url.*')"
[ -z "$leftover" ] && echo "PASS writing the heartbeat URL leaves no temporary file" \
|| { echo "FAIL temporary files left: $leftover"; fails=$((fails + 1)); }
out="$(SCRIPT=summary_heartbeat run_watchdog_timer 0 "")"
expect "without a heartbeat the summary says so loudly" "WARN: NO HEARTBEAT: nothing off this machine notices it going down or its watchdog stopping." "$out"
expect "and says how to add one" "then re-run with FELIS_WATCHDOG_HEARTBEAT_URL=<its ping URL> (docs/troubleshooting.md §14)." "$out"
out="$(FELIS_WATCHDOG_HEARTBEAT_URL='https://user:[email protected]:8443?rid=5f1e0c2a' SCRIPT='write_heartbeat_url; heartbeat_host' run_watchdog_timer 0 "")"
expect "the host shown drops the sign-in and the query" "https://status.example:8443/..." "$out"
rm -f "$tdir/watchdog-heartbeat-url"
for good in "" off https://hc-ping.com/5f1e0c2a http://10.0.0.5:8000/ping/5f1e0c2a; do
out="$(FELIS_WATCHDOG_HEARTBEAT_URL="$good" SCRIPT='validate_heartbeat_url; echo fine' run_watchdog_timer 0 "")"
expect "the heartbeat URL <$good> is accepted" "fine" "$out"
done
for bad in hc-ping.com/5f1e0c2a ftp://hc-ping.com/5f1e0c2a https:///5f1e0c2a 'https://hc-ping.com/5f1e 0c2a' 'https://hc-ping.com/5f1e"0c2a' "$(printf 'https://hc-ping.com/5f1e\n0c2a')"; do
out="$(FELIS_WATCHDOG_HEARTBEAT_URL="$bad" SCRIPT='validate_heartbeat_url; echo fine' run_watchdog_timer 0 "")"
case "$out" in
*"DIE: FELIS_WATCHDOG_HEARTBEAT_URL must"*fine*|*5f1e*) echo "FAIL the heartbeat URL <$bad>: $out"; fails=$((fails + 1)) ;;
*"DIE: FELIS_WATCHDOG_HEARTBEAT_URL must"*) echo "PASS the heartbeat URL <$bad> is refused without being echoed" ;;
*) echo "FAIL the heartbeat URL <$bad> was accepted: $out"; fails=$((fails + 1)) ;;
esac
done
case "$(awk '/^validate_settings\(\) \{/,/^}/' "$BS")" in
*validate_heartbeat_url*) echo "PASS the heartbeat URL is checked before anything is installed" ;;
*) echo "FAIL validate_settings must call validate_heartbeat_url"; fails=$((fails + 1)) ;;
esac
case "$(awk '/^summary\(\) \{/,/^}/' "$BS")" in
*summary_offsite*summary_heartbeat*) echo "PASS the install's summary ends on the heartbeat" ;;
*) echo "FAIL summary must call summary_heartbeat"; fails=$((fails + 1)) ;;
esac
out="$(run_watchdog_timer 0 /srv/worlds)"
expect "a custom worlds root is watched for free space" "-disk-paths /,/var/lib/rancher/k3s,/var/lib/felis,/srv/worlds," "$(cat "$tdir/felis-watchdog.service")"
case "$unit" in
deploy/uninstall.sh
+1 -1
View File
@@ -60,7 +60,7 @@ FELIS_CRD="minecraftservers.felis.lolicon.best"
# service that is already gone.
FELIS_UNITS=(
felis-db-backup.timer felis-watchdog.timer felis-offsite.timer felis-build-tools.timer felis-update-check.timer
felis-db-backup.service felis-watchdog.service felis-offsite.service felis-build-tools.service felis-update-check.service
felis-db-backup.service felis-watchdog.service felis-watchdog-failed.service felis-offsite.service felis-build-tools.service felis-update-check.service
felis-velocity.service felis-nano.service cloudflared-felis.service
felis-postgres-firewall.service
)