feat(watchdog): 监控 felis-postgres 部署并更新数据库排障提示 (34a4dffc) · Commits · FelisMC / Felis · GitLab

cmd/felis/watchdog.go

+1 −1

Changes for cmd/felis/watchdog.go: 1 added line, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -38,7 +38,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
		statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")
		quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
		backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
		diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/postgresql,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
		diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
		proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)
		nodeIP := fs.String("node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`)
		controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")

internal/watchdog/host.go

+4 −4

Changes for internal/watchdog/host.go: 4 added lines, 4 removed lines.
Original line number	Diff line number	Diff line
		@@ -23,8 +23,8 @@ const (
		)

		// AddressFinding reports that this host no longer holds want, the node address
		// the installer wrote into the database connection string, pg_hba, the network
		// policies, the panel certificate and (by default) the nip.io root domain. held
		// the installer gave k3s and wrote into the network policies, the panel
		// certificate and (by default) the nip.io root domain. held
		// is every address on the host's interfaces. An empty or unparsable want skips
		// the check.
		func AddressFinding(want string, held []net.IP) *Finding {
		@@ -47,9 +47,9 @@ func AddressFinding(want string, held []net.IP) *Finding {
		}
		return &Finding{
		Key: "host-address", Severity: Critical, For: addressFor,
		Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s（现在是：%s）：数据库连接、pg_hba、网络策略和面板证书仍指向旧地址",
		Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s（现在是：%s）：k3s 节点、网络策略和面板证书仍指向旧地址",
		want, current),
		SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the database connection, pg_hba, the network policies and the panel certificate still point at it",
		SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the k3s node, the network policies and the panel certificate still point at it",
		want, current),
		Hint: "give the host its old address back (a DHCP reservation or a static address); docs/troubleshooting.md §13c",
		}

internal/watchdog/probes.go

+6 −2

Changes for internal/watchdog/probes.go: 6 added lines, 2 removed lines.
Original line number	Diff line number	Diff line
		@@ -62,6 +62,7 @@ const (
		var controlDeployments = []struct {
		name, impact, impactEN string
		}{
		{platform.PostgresName, "数据库不可用：登录、面板和服务器管理都会失败", "the control-plane database is down: sign-in, the panel and server management fail"},
		{platform.SAAPI, "面板、登录验证和内部接口都不可用", "the panel, sign-in and the internal API are down"},
		{platform.SAOperator, "服务器无法启动、停止或更新状态", "no server can start, stop or report status"},
		{"registry", "游戏镜像拉取和构建都会失败", "game image pulls and builds fail"},
		@@ -301,13 +302,16 @@ func KubeAPIDown(err error) Finding {
		}
		}

		// PostgresDown is the finding for a database that did not answer.
		// PostgresDown is the finding for a database that did not answer. It runs as
		// the felis-postgres Deployment; the host reaches it on the loopback hostPort.
		func PostgresDown(err error) Finding {
		ns := platform.DefaultControlNamespace
		return Finding{
		Key: "postgres", Severity: Critical, For: postgresFor,
		Summary: "PostgreSQL 无法连接：登录、面板和服务器管理都会失败",
		SummaryEN: "PostgreSQL is unreachable: sign-in, the panel and server management fail",
		Hint: fmt.Sprintf("systemctl status postgresql; journalctl -u postgresql -n 100 (%v)", err),
		Hint: fmt.Sprintf("k3s kubectl -n %s get pods -l app.kubernetes.io/component=%s; k3s kubectl -n %s logs deploy/%s --tail=100 (%v)",
		ns, platform.ComponentPostgres, ns, platform.PostgresName, err),
		}
		}

internal/watchdog/probes_test.go

+5 −1

Changes for internal/watchdog/probes_test.go: 5 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -103,7 +103,7 @@ func TestClusterCheck(t *testing.T) {
		t.Fatalf("Check: %v", err)
		}
		want := []string{
		"deployment/felis-operator", "deployment/registry",
		"deployment/felis-operator", "deployment/felis-postgres", "deployment/registry",
		"job-failed/backup-survival-abc", "node/n1/DiskPressure", "reaper-stale",
		"server-failed/broken", "system-server/login",
		}
		@@ -120,6 +120,10 @@ func TestClusterCheck(t *testing.T) {
		if !strings.Contains(f.SummaryEN, "missing") {
		t.Errorf("registry finding = %+v, want missing", f)
		}
		case "deployment/felis-postgres":
		if f.Severity != Critical \|\| !strings.Contains(f.SummaryEN, "database is down") {
		t.Errorf("database finding = %+v, want a critical database outage", f)
		}
		case "job-failed/backup-survival-abc":
		if !f.Event \|\| !strings.Contains(f.SummaryEN, "world backup Job") {
		t.Errorf("job finding = %+v", f)