test(watchdog): 整轮巡检测试覆盖集群告警保持、dry-run、主机 relay 密码、按配置检查与状态存不下，收件人查询移入 watchdog 并加 pgint 契约

2026-09-27 12:33:11 +08:00
1 parent f9312118c9
commit fe48319866
@@ -2,7 +2,6 @@ package main

 import (
 	"context"
-	"database/sql"
 	"errors"
 	"flag"
 	"fmt"
@@ -424,8 +423,8 @@ func smtpSecretPassword(ctx context.Context, cl client.Client, ns string) (strin
 	return string(sec.Data[platform.SMTPSecretPasswordKey]), nil
 }

-// ownerEmails pings PostgreSQL and returns the verified addresses of the
-// enabled owner accounts, the people who can act on an alert.
+// ownerEmails pings PostgreSQL and returns the owners an alert goes to
+// (watchdog.OwnerEmails).
 func ownerEmails(ctx context.Context, url string) ([]string, error) {
 	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
 	defer cancel()
@@ -434,24 +433,7 @@ func ownerEmails(ctx context.Context, url string) ([]string, error) {
 		return nil, err
 	}
 	defer drv.Close()
-	rows, err := drv.DB().QueryContext(ctx,
-		`SELECT email FROM users
-		 WHERE role = 'owner' AND email_verified AND COALESCE(email, '') <> ''
-		   AND NOT disabled AND deleted_at IS NULL
-		 ORDER BY email`)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var out []string
-	for rows.Next() {
-		var email sql.NullString
-		if err := rows.Scan(&email); err != nil {
-			return nil, err
-		}
-		out = append(out, email.String)
-	}
-	return out, rows.Err()
+	return watchdog.OwnerEmails(ctx, drv.DB())
 }

 // proxyFinding dials the game proxy; players reach every server through it.
@@ -1,8 +1,11 @@
 package main

 import (
+	"bytes"
 	"context"
+	"errors"
 	"fmt"
+	"io/fs"
 	"net"
 	"os"
 	"path/filepath"
@@ -11,6 +14,7 @@ import (
 	"time"

 	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/watchdog"
 )

 // TestProxyFinding: a listening proxy is healthy; a closed port is the critical
@@ -89,3 +93,221 @@ func TestMailHold(t *testing.T) {
 		t.Errorf("an unreadable status held the mail: %q", got)
 	}
 }
+
+// watchdogHost is a host whole watchdog passes run on: the API server and PostgreSQL
+// are down (no kubeconfig, nothing on the database port), the relay is a recorder and
+// the heartbeat URL points at a ping log.
+type watchdogHost struct {
+	dir, statePath string
+	args           []string
+	rec            *alertRecorder
+	pings          *pingLog
+	url            string
+}
+
+func newWatchdogHost(t *testing.T, cfg string, state *watchdog.State) *watchdogHost {
+	t.Helper()
+	dir := t.TempDir()
+	t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
+	pings, srv := newPingServer(t)
+	h := &watchdogHost{dir: dir, statePath: filepath.Join(dir, "state.json"), rec: &alertRecorder{}, pings: pings, url: srv.URL}
+	writeTestFile(t, filepath.Join(dir, "felis.toml"), cfg, 0o600)
+	writeTestFile(t, filepath.Join(dir, "watchdog-heartbeat-url"), srv.URL+"/check-key\n", 0o600)
+	if state != nil {
+		if err := watchdog.SaveState(h.statePath, state); err != nil {
+			t.Fatal(err)
+		}
+	}
+	h.args = []string{
+		"-config", filepath.Join(dir, "felis.toml"), "-state", h.statePath, "-quiet-file", filepath.Join(dir, "quiet"),
+		"-backup-dir", "", "-disk-paths", dir, "-k3s-cert-dirs", "", "-smtp-password-file", filepath.Join(dir, "smtp-password"),
+		"-offsite-status", filepath.Join(dir, "offsite-status.json"), "-build-tools-status", filepath.Join(dir, "build-tools.json"),
+		"-heartbeat-file", filepath.Join(dir, "watchdog-heartbeat-url"),
+	}
+	return h
+}
+
+// run is one pass; flags given here override the host's own.
+func (h *watchdogHost) run(flags ...string) (code int, stdout, stderr string) {
+	watchdogSender = h.rec.sender
+	defer func() { watchdogSender = smtpSender }()
+	var out, errOut bytes.Buffer
+	code = cmdWatchdog(append(append([]string(nil), h.args...), flags...), &out, &errOut)
+	return code, out.String(), errOut.String()
+}
+
+// duePostgres is a state whose owner has not yet been told of PostgreSQL, down
+// for an hour.
+func duePostgres(cachedPassword string) *watchdog.State {
+	s := &watchdog.State{Recipients: []string{"[email protected]"}, SMTPPassword: cachedPassword}
+	s.Observe(watchdog.Report{Findings: []watchdog.Finding{watchdog.PostgresDown(errors.New("refused"))}}, time.Now().Add(-time.Hour))
+	return s
+}
+
+// TestWatchdogRunKeepsClusterAlertsWhileTheAPIIsDown: with the API server down,
+// the alerts under the cluster checks keep their state, since nothing looked
+// at them; an alert of a check that did run and found nothing reads as cleared.
+func TestWatchdogRunKeepsClusterAlertsWhileTheAPIIsDown(t *testing.T) {
+	told := time.Now().Add(-30 * time.Minute)
+	cluster := []string{"deployment/felis-api", "node/felis-1/NotReady", "server-failed/lobby"}
+	var r watchdog.Report
+	for _, key := range append([]string{"proxy"}, cluster...) {
+		r.Findings = append(r.Findings, watchdog.Finding{Key: key, Severity: watchdog.Critical, SummaryEN: key + " is down"})
+	}
+	s := &watchdog.State{Recipients: []string{"[email protected]"}}
+	s.Commit(s.Observe(r, told), told)
+	h := newWatchdogHost(t, testWatchdogConfig, s)
+
+	code, stdout, stderr := h.run()
+	if code != 0 || len(h.rec.relays) != 0 {
+		t.Fatalf("exit %d, mailed %v\nstdout %s\nstderr %s", code, h.rec.sent, stdout, stderr)
+	}
+	for _, want := range []string{"felis watchdog: [critical] kube-api: ", "felis watchdog: [critical] postgres: "} {
+		if !strings.Contains(stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, stdout)
+		}
+	}
+	got, err := watchdog.LoadState(h.statePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, key := range cluster {
+		if a := got.Alerts[key]; a == nil || !a.ClearedAt.IsZero() || !a.Notified.Equal(told) {
+			t.Errorf("%s with the API server down: %+v, want it kept open as mailed at %s", key, a, told)
+		}
+	}
+	if a := got.Alerts["proxy"]; a == nil || a.ClearedAt.IsZero() {
+		t.Errorf("proxy, whose check ran and found nothing: %+v, want it cleared", a)
+	}
+}
+
+// TestWatchdogDryRunMailsAndSavesNothing: a dry run prints the mail that is due
+// and the heartbeat it would ping, and sends, pings and saves nothing.
+func TestWatchdogDryRunMailsAndSavesNothing(t *testing.T) {
+	t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
+	h := newWatchdogHost(t, testWatchdogConfig+testWatchdogSMTP, duePostgres(""))
+	before, err := os.ReadFile(h.statePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	code, stdout, stderr := h.run("-dry-run")
+	after, err := os.ReadFile(h.statePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pings, _ := h.pings.got()
+	if code != 0 || len(h.rec.relays) != 0 || len(pings) != 0 || !bytes.Equal(before, after) {
+		t.Errorf("dry run: exit %d, relays %d, pings %v, state changed %v\nstdout %s\nstderr %s", code, len(h.rec.relays), pings, !bytes.Equal(before, after), stdout, stderr)
+	}
+	host, _ := os.Hostname()
+	for _, want := range []string{
+		"felis watchdog: due to be mailed to [email protected]:\nSubject: Felis 严重告警（" + host + "）：1 项异常 · 1 firing\n\n",
+		"felis watchdog: a run pings the heartbeat at " + h.url + "/...\n",
+	} {
+		if !strings.Contains(stdout, want) {
+			t.Errorf("stdout lacks %q:\n%s", want, stdout)
+		}
+	}
+	if strings.Contains(stdout, "\r") {
+		t.Errorf("the mail body printed with CRLF:\n%q", stdout)
+	}
+
+	h = newWatchdogHost(t, testWatchdogConfig, nil)
+	code, stdout, stderr = h.run("-dry-run")
+	if code != 0 || !strings.Contains(stdout, "felis watchdog: nothing is due to be mailed\n") {
+		t.Errorf("dry run with nothing due: exit %d\nstdout %s\nstderr %s", code, stdout, stderr)
+	}
+	if _, err := os.Stat(h.statePath); !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("a dry run wrote the state (%v)", err)
+	}
+}
+
+// TestWatchdogRunSignsInWithTheHostRelayPassword: a pass caches the relay
+// password from the host copy even while the cluster is down, and signs in with
+// it; with no host copy and no cluster it keeps the one it had.
+func TestWatchdogRunSignsInWithTheHostRelayPassword(t *testing.T) {
+	const smtpNoRef = "[smtp]\nhost = \"smtp.config.example\"\nport = 2525\nfrom = \"[email protected]\"\nusername = \"felis\"\n"
+	for _, tc := range []struct{ what, hostCopy, want string }{
+		{"the host copy", "host-pw", "host-pw"},
+		{"no host copy, the cluster down", "", "cached-pw"},
+	} {
+		h := newWatchdogHost(t, testWatchdogConfig+smtpNoRef, duePostgres("cached-pw"))
+		if tc.hostCopy != "" {
+			writeTestFile(t, filepath.Join(h.dir, "smtp-password"), tc.hostCopy, 0o600)
+		}
+		code, stdout, stderr := h.run()
+		if code != 0 || len(h.rec.relays) != 1 || h.rec.relays[0].Password != tc.want {
+			t.Errorf("%s: exit %d, relays %+v; want one mail signed in with %q\nstdout %s\nstderr %s", tc.what, code, h.rec.relays, tc.want, stdout, stderr)
+			continue
+		}
+		if s, err := watchdog.LoadState(h.statePath); err != nil || s.SMTPPassword != tc.want {
+			t.Errorf("%s: the state caches another password (%v)", tc.what, err)
+		}
+	}
+}
+
+// TestWatchdogRunChecksWhatIsConfigured: the proxy, the database backups, the
+// off-site copy and the build lane's scan DB are each checked when the host
+// has them, and only then.
+func TestWatchdogRunChecksWhatIsConfigured(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	closed := ln.Addr().String()
+	ln.Close()
+	const offsiteTable = "[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis\"\n"
+	const registry = "[registry]\nurl = \"registry.felis.svc:5000\"\n"
+	optional := []string{"proxy", "db-backup", "offsite", "scan-db"}
+	for _, tc := range []struct {
+		what  string
+		cfg   string
+		flags func(dir string) []string
+		want  string // the one optional check that reports, "" for none
+	}{
+		{what: "none of them", cfg: testWatchdogConfig},
+		{what: "a proxy address", cfg: testWatchdogConfig, flags: func(string) []string { return []string{"-proxy-addr", closed} }, want: "proxy"},
+		{what: "a backup directory", cfg: testWatchdogConfig, flags: func(dir string) []string { return []string{"-backup-dir", dir} }, want: "db-backup"},
+		{what: "[offsite]", cfg: testWatchdogConfig + offsiteTable, want: "offsite"},
+		{what: "a registry with the default scan DB", cfg: testWatchdogConfig + registry, want: "scan-db"},
+		{what: "a registry with its scan DB under mirror/", cfg: testWatchdogConfig + registry + "trivy_db_repository = \"registry.felis.svc:5000/mirror/trivy-db\"\n", want: "scan-db"},
+		{what: "a registry with the scan DB elsewhere", cfg: testWatchdogConfig + registry + "trivy_db_repository = \"ghcr.io/aquasecurity/trivy-db\"\n"},
+	} {
+		h := newWatchdogHost(t, tc.cfg, nil)
+		var flags []string
+		if tc.flags != nil {
+			flags = tc.flags(t.TempDir())
+		}
+		code, stdout, stderr := h.run(append(flags, "-dry-run")...)
+		var reported []string
+		for _, key := range optional {
+			if strings.Contains(stdout, "] "+key+": ") {
+				reported = append(reported, key)
+			}
+		}
+		if code != 0 || strings.Join(reported, ",") != tc.want {
+			t.Errorf("%s: exit %d, reported %v; want %q\nstdout %s\nstderr %s", tc.what, code, reported, tc.want, stdout, stderr)
+		}
+	}
+}
+
+// TestWatchdogRunStateThatDoesNotSave: a pass whose state does not save mails
+// as usual, exits 1 and posts the failure to the heartbeat, since the next run
+// mails the same alerts again.
+func TestWatchdogRunStateThatDoesNotSave(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("root writes into a read-only directory")
+	}
+	t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
+	h := newWatchdogHost(t, testWatchdogConfig+testWatchdogSMTP, duePostgres(""))
+	if err := os.Chmod(h.dir, 0o500); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { os.Chmod(h.dir, 0o700) })
+	code, stdout, stderr := h.run()
+	pings, bodies := h.pings.got()
+	if code != 1 || len(h.rec.sent) != 1 || !strings.Contains(stderr, "felis watchdog: save state: ") ||
+		len(pings) != 1 || pings[0] != "POST /check-key/fail" || !strings.HasPrefix(bodies[0], "the watchdog state did not save: ") {
+		t.Errorf("exit %d, mailed %v, pings %v %q; want exit 1, one mail and the save failure posted to /fail\nstdout %s\nstderr %s", code, h.rec.sent, pings, bodies, stdout, stderr)
+	}
+}