feat(watchdog): 监控 felis-postgres 部署并更新数据库排障提示
This commit is contained in:
4 files changed
+16
-8
No files matched your search
@@ -23,8 +23,8 @@ const (
|
||||
)
|
||||
|
||||
// AddressFinding reports that this host no longer holds want, the node address
|
||||
// the installer wrote into the database connection string, pg_hba, the network
|
||||
// policies, the panel certificate and (by default) the nip.io root domain. held
|
||||
// the installer gave k3s and wrote into the network policies, the panel
|
||||
// certificate and (by default) the nip.io root domain. held
|
||||
// is every address on the host's interfaces. An empty or unparsable want skips
|
||||
// the check.
|
||||
func AddressFinding(want string, held []net.IP) *Finding {
|
||||
@@ -47,9 +47,9 @@ func AddressFinding(want string, held []net.IP) *Finding {
|
||||
}
|
||||
return &Finding{
|
||||
Key: "host-address", Severity: Critical, For: addressFor,
|
||||
Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s(现在是:%s):数据库连接、pg_hba、网络策略和面板证书仍指向旧地址",
|
||||
Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s(现在是:%s):k3s 节点、网络策略和面板证书仍指向旧地址",
|
||||
want, current),
|
||||
SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the database connection, pg_hba, the network policies and the panel certificate still point at it",
|
||||
SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the k3s node, the network policies and the panel certificate still point at it",
|
||||
want, current),
|
||||
Hint: "give the host its old address back (a DHCP reservation or a static address); docs/troubleshooting.md §13c",
|
||||
}
|
||||
|
||||
@@ -62,6 +62,7 @@ const (
|
||||
var controlDeployments = []struct {
|
||||
name, impact, impactEN string
|
||||
}{
|
||||
{platform.PostgresName, "数据库不可用:登录、面板和服务器管理都会失败", "the control-plane database is down: sign-in, the panel and server management fail"},
|
||||
{platform.SAAPI, "面板、登录验证和内部接口都不可用", "the panel, sign-in and the internal API are down"},
|
||||
{platform.SAOperator, "服务器无法启动、停止或更新状态", "no server can start, stop or report status"},
|
||||
{"registry", "游戏镜像拉取和构建都会失败", "game image pulls and builds fail"},
|
||||
@@ -301,13 +302,16 @@ func KubeAPIDown(err error) Finding {
|
||||
}
|
||||
}
|
||||
|
||||
// PostgresDown is the finding for a database that did not answer.
|
||||
// PostgresDown is the finding for a database that did not answer. It runs as
|
||||
// the felis-postgres Deployment; the host reaches it on the loopback hostPort.
|
||||
func PostgresDown(err error) Finding {
|
||||
ns := platform.DefaultControlNamespace
|
||||
return Finding{
|
||||
Key: "postgres", Severity: Critical, For: postgresFor,
|
||||
Summary: "PostgreSQL 无法连接:登录、面板和服务器管理都会失败",
|
||||
SummaryEN: "PostgreSQL is unreachable: sign-in, the panel and server management fail",
|
||||
Hint: fmt.Sprintf("systemctl status postgresql; journalctl -u postgresql -n 100 (%v)", err),
|
||||
Hint: fmt.Sprintf("k3s kubectl -n %s get pods -l app.kubernetes.io/component=%s; k3s kubectl -n %s logs deploy/%s --tail=100 (%v)",
|
||||
ns, platform.ComponentPostgres, ns, platform.PostgresName, err),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ func TestClusterCheck(t *testing.T) {
|
||||
t.Fatalf("Check: %v", err)
|
||||
}
|
||||
want := []string{
|
||||
"deployment/felis-operator", "deployment/registry",
|
||||
"deployment/felis-operator", "deployment/felis-postgres", "deployment/registry",
|
||||
"job-failed/backup-survival-abc", "node/n1/DiskPressure", "reaper-stale",
|
||||
"server-failed/broken", "system-server/login",
|
||||
}
|
||||
@@ -120,6 +120,10 @@ func TestClusterCheck(t *testing.T) {
|
||||
if !strings.Contains(f.SummaryEN, "missing") {
|
||||
t.Errorf("registry finding = %+v, want missing", f)
|
||||
}
|
||||
case "deployment/felis-postgres":
|
||||
if f.Severity != Critical || !strings.Contains(f.SummaryEN, "database is down") {
|
||||
t.Errorf("database finding = %+v, want a critical database outage", f)
|
||||
}
|
||||
case "job-failed/backup-survival-abc":
|
||||
if !f.Event || !strings.Contains(f.SummaryEN, "world backup Job") {
|
||||
t.Errorf("job finding = %+v", f)
|
||||
|
||||
Reference in new issue
Block a user