From 34a4dffc50e7295bf3c692da8853a6bae51ae2e7 Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Sat, 26 Sep 2026 12:14:24 +0800 Subject: [PATCH] =?UTF-8?q?feat(watchdog):=20=E7=9B=91=E6=8E=A7=20felis-po?= =?UTF-8?q?stgres=20=E9=83=A8=E7=BD=B2=E5=B9=B6=E6=9B=B4=E6=96=B0=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E6=8E=92=E9=9A=9C=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/felis/watchdog.go | 2 +- internal/watchdog/host.go | 8 ++++---- internal/watchdog/probes.go | 8 ++++++-- internal/watchdog/probes_test.go | 6 +++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cmd/felis/watchdog.go b/cmd/felis/watchdog.go index ce17598..f5ca98f 100644 --- a/cmd/felis/watchdog.go +++ b/cmd/felis/watchdog.go @@ -38,7 +38,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int { statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)") quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose") backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`) - diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/postgresql,/var/lib/felis", "comma-separated paths whose filesystems must keep free space") + diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space") proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`) nodeIP := fs.String("node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`) controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane") diff --git a/internal/watchdog/host.go b/internal/watchdog/host.go index 4cd83a3..2483bee 100644 --- a/internal/watchdog/host.go +++ b/internal/watchdog/host.go @@ -23,8 +23,8 @@ const ( ) // AddressFinding reports that this host no longer holds want, the node address -// the installer wrote into the database connection string, pg_hba, the network -// policies, the panel certificate and (by default) the nip.io root domain. held +// the installer gave k3s and wrote into the network policies, the panel +// certificate and (by default) the nip.io root domain. held // is every address on the host's interfaces. An empty or unparsable want skips // the check. func AddressFinding(want string, held []net.IP) *Finding { @@ -47,9 +47,9 @@ func AddressFinding(want string, held []net.IP) *Finding { } return &Finding{ Key: "host-address", Severity: Critical, For: addressFor, - Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s（现在是：%s）：数据库连接、pg_hba、网络策略和面板证书仍指向旧地址", + Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s（现在是：%s）：k3s 节点、网络策略和面板证书仍指向旧地址", want, current), - SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the database connection, pg_hba, the network policies and the panel certificate still point at it", + SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the k3s node, the network policies and the panel certificate still point at it", want, current), Hint: "give the host its old address back (a DHCP reservation or a static address); docs/troubleshooting.md §13c", } diff --git a/internal/watchdog/probes.go b/internal/watchdog/probes.go index 74ef4fd..fce910f 100644 --- a/internal/watchdog/probes.go +++ b/internal/watchdog/probes.go @@ -62,6 +62,7 @@ const ( var controlDeployments = []struct { name, impact, impactEN string }{ + {platform.PostgresName, "数据库不可用：登录、面板和服务器管理都会失败", "the control-plane database is down: sign-in, the panel and server management fail"}, {platform.SAAPI, "面板、登录验证和内部接口都不可用", "the panel, sign-in and the internal API are down"}, {platform.SAOperator, "服务器无法启动、停止或更新状态", "no server can start, stop or report status"}, {"registry", "游戏镜像拉取和构建都会失败", "game image pulls and builds fail"}, @@ -301,13 +302,16 @@ func KubeAPIDown(err error) Finding { } } -// PostgresDown is the finding for a database that did not answer. +// PostgresDown is the finding for a database that did not answer. It runs as +// the felis-postgres Deployment; the host reaches it on the loopback hostPort. func PostgresDown(err error) Finding { + ns := platform.DefaultControlNamespace return Finding{ Key: "postgres", Severity: Critical, For: postgresFor, Summary: "PostgreSQL 无法连接：登录、面板和服务器管理都会失败", SummaryEN: "PostgreSQL is unreachable: sign-in, the panel and server management fail", - Hint: fmt.Sprintf("systemctl status postgresql; journalctl -u postgresql -n 100 (%v)", err), + Hint: fmt.Sprintf("k3s kubectl -n %s get pods -l app.kubernetes.io/component=%s; k3s kubectl -n %s logs deploy/%s --tail=100 (%v)", + ns, platform.ComponentPostgres, ns, platform.PostgresName, err), } } diff --git a/internal/watchdog/probes_test.go b/internal/watchdog/probes_test.go index 2b343d9..72d7f04 100644 --- a/internal/watchdog/probes_test.go +++ b/internal/watchdog/probes_test.go @@ -103,7 +103,7 @@ func TestClusterCheck(t *testing.T) { t.Fatalf("Check: %v", err) } want := []string{ - "deployment/felis-operator", "deployment/registry", + "deployment/felis-operator", "deployment/felis-postgres", "deployment/registry", "job-failed/backup-survival-abc", "node/n1/DiskPressure", "reaper-stale", "server-failed/broken", "system-server/login", } @@ -120,6 +120,10 @@ func TestClusterCheck(t *testing.T) { if !strings.Contains(f.SummaryEN, "missing") { t.Errorf("registry finding = %+v, want missing", f) } + case "deployment/felis-postgres": + if f.Severity != Critical || !strings.Contains(f.SummaryEN, "database is down") { + t.Errorf("database finding = %+v, want a critical database outage", f) + } case "job-failed/backup-survival-abc": if !f.Event || !strings.Contains(f.SummaryEN, "world backup Job") { t.Errorf("job finding = %+v", f)