feat(watchdog): 监控 felis-postgres 部署并更新数据库排障提示

cmd/felis/watchdog.go

@@ -38,7 +38,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")
 	quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
 	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
-	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/postgresql,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
+	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
 	proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)
 	nodeIP := fs.String("node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`)
 	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")

internal/watchdog/host.go

@@ -23,8 +23,8 @@ const (
 )
 
 // AddressFinding reports that this host no longer holds want, the node address
-// the installer wrote into the database connection string, pg_hba, the network
+// the installer gave k3s and wrote into the network policies, the panel
-// policies, the panel certificate and (by default) the nip.io root domain. held
+// certificate and (by default) the nip.io root domain. held
 // is every address on the host's interfaces. An empty or unparsable want skips
 // the check.
 func AddressFinding(want string, held []net.IP) *Finding {
@@ -47,9 +47,9 @@ func AddressFinding(want string, held []net.IP) *Finding {
 	}
 	return &Finding{
 		Key: "host-address", Severity: Critical, For: addressFor,
-		Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s（现在是：%s）：数据库连接、pg_hba、网络策略和面板证书仍指向旧地址",
+		Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s（现在是：%s）：k3s 节点、网络策略和面板证书仍指向旧地址",
 			want, current),
-		SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the database connection, pg_hba, the network policies and the panel certificate still point at it",
+		SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the k3s node, the network policies and the panel certificate still point at it",
 			want, current),
 		Hint: "give the host its old address back (a DHCP reservation or a static address); docs/troubleshooting.md §13c",
 	}

internal/watchdog/probes.go

@@ -62,6 +62,7 @@ const (
 var controlDeployments = []struct {
 	name, impact, impactEN string
 }{
+	{platform.PostgresName, "数据库不可用：登录、面板和服务器管理都会失败", "the control-plane database is down: sign-in, the panel and server management fail"},
 	{platform.SAAPI, "面板、登录验证和内部接口都不可用", "the panel, sign-in and the internal API are down"},
 	{platform.SAOperator, "服务器无法启动、停止或更新状态", "no server can start, stop or report status"},
 	{"registry", "游戏镜像拉取和构建都会失败", "game image pulls and builds fail"},
@@ -301,13 +302,16 @@ func KubeAPIDown(err error) Finding {
 	}
 }
 
-// PostgresDown is the finding for a database that did not answer.
+// PostgresDown is the finding for a database that did not answer. It runs as
+// the felis-postgres Deployment; the host reaches it on the loopback hostPort.
 func PostgresDown(err error) Finding {
+	ns := platform.DefaultControlNamespace
 	return Finding{
 		Key: "postgres", Severity: Critical, For: postgresFor,
 		Summary:   "PostgreSQL 无法连接：登录、面板和服务器管理都会失败",
 		SummaryEN: "PostgreSQL is unreachable: sign-in, the panel and server management fail",
-		Hint:      fmt.Sprintf("systemctl status postgresql; journalctl -u postgresql -n 100 (%v)", err),
+		Hint: fmt.Sprintf("k3s kubectl -n %s get pods -l app.kubernetes.io/component=%s; k3s kubectl -n %s logs deploy/%s --tail=100 (%v)",
+			ns, platform.ComponentPostgres, ns, platform.PostgresName, err),
 	}
 }
 

internal/watchdog/probes_test.go

@@ -103,7 +103,7 @@ func TestClusterCheck(t *testing.T) {
 		t.Fatalf("Check: %v", err)
 	}
 	want := []string{
-		"deployment/felis-operator", "deployment/registry",
+		"deployment/felis-operator", "deployment/felis-postgres", "deployment/registry",
 		"job-failed/backup-survival-abc", "node/n1/DiskPressure", "reaper-stale",
 		"server-failed/broken", "system-server/login",
 	}
@@ -120,6 +120,10 @@ func TestClusterCheck(t *testing.T) {
 			if !strings.Contains(f.SummaryEN, "missing") {
 				t.Errorf("registry finding = %+v, want missing", f)
 			}
+		case "deployment/felis-postgres":
+			if f.Severity != Critical || !strings.Contains(f.SummaryEN, "database is down") {
+				t.Errorf("database finding = %+v, want a critical database outage", f)
+			}
 		case "job-failed/backup-survival-abc":
 			if !f.Event || !strings.Contains(f.SummaryEN, "world backup Job") {
 				t.Errorf("job finding = %+v", f)