diff --git a/cmd/felis/watchdog.go b/cmd/felis/watchdog.go index 8487b30..8c44687 100644 --- a/cmd/felis/watchdog.go +++ b/cmd/felis/watchdog.go @@ -2,7 +2,6 @@ package main import ( "context" - "database/sql" "errors" "flag" "fmt" @@ -424,8 +423,8 @@ func smtpSecretPassword(ctx context.Context, cl client.Client, ns string) (strin return string(sec.Data[platform.SMTPSecretPasswordKey]), nil } -// ownerEmails pings PostgreSQL and returns the verified addresses of the -// enabled owner accounts, the people who can act on an alert. +// ownerEmails pings PostgreSQL and returns the owners an alert goes to +// (watchdog.OwnerEmails). func ownerEmails(ctx context.Context, url string) ([]string, error) { ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() @@ -434,24 +433,7 @@ func ownerEmails(ctx context.Context, url string) ([]string, error) { return nil, err } defer drv.Close() - rows, err := drv.DB().QueryContext(ctx, - `SELECT email FROM users - WHERE role = 'owner' AND email_verified AND COALESCE(email, '') <> '' - AND NOT disabled AND deleted_at IS NULL - ORDER BY email`) - if err != nil { - return nil, err - } - defer rows.Close() - var out []string - for rows.Next() { - var email sql.NullString - if err := rows.Scan(&email); err != nil { - return nil, err - } - out = append(out, email.String) - } - return out, rows.Err() + return watchdog.OwnerEmails(ctx, drv.DB()) } // proxyFinding dials the game proxy; players reach every server through it. diff --git a/cmd/felis/watchdog_test.go b/cmd/felis/watchdog_test.go index d3e0acf..cd1f538 100644 --- a/cmd/felis/watchdog_test.go +++ b/cmd/felis/watchdog_test.go @@ -1,8 +1,11 @@ package main import ( + "bytes" "context" + "errors" "fmt" + "io/fs" "net" "os" "path/filepath" @@ -11,6 +14,7 @@ import ( "time" "felis.lolicon.best/internal/offsite" + "felis.lolicon.best/internal/watchdog" ) // TestProxyFinding: a listening proxy is healthy; a closed port is the critical @@ -89,3 +93,221 @@ func TestMailHold(t *testing.T) { t.Errorf("an unreadable status held the mail: %q", got) } } + +// watchdogHost is a host whole watchdog passes run on: the API server and PostgreSQL +// are down (no kubeconfig, nothing on the database port), the relay is a recorder and +// the heartbeat URL points at a ping log. +type watchdogHost struct { + dir, statePath string + args []string + rec *alertRecorder + pings *pingLog + url string +} + +func newWatchdogHost(t *testing.T, cfg string, state *watchdog.State) *watchdogHost { + t.Helper() + dir := t.TempDir() + t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig")) + pings, srv := newPingServer(t) + h := &watchdogHost{dir: dir, statePath: filepath.Join(dir, "state.json"), rec: &alertRecorder{}, pings: pings, url: srv.URL} + writeTestFile(t, filepath.Join(dir, "felis.toml"), cfg, 0o600) + writeTestFile(t, filepath.Join(dir, "watchdog-heartbeat-url"), srv.URL+"/check-key\n", 0o600) + if state != nil { + if err := watchdog.SaveState(h.statePath, state); err != nil { + t.Fatal(err) + } + } + h.args = []string{ + "-config", filepath.Join(dir, "felis.toml"), "-state", h.statePath, "-quiet-file", filepath.Join(dir, "quiet"), + "-backup-dir", "", "-disk-paths", dir, "-k3s-cert-dirs", "", "-smtp-password-file", filepath.Join(dir, "smtp-password"), + "-offsite-status", filepath.Join(dir, "offsite-status.json"), "-build-tools-status", filepath.Join(dir, "build-tools.json"), + "-heartbeat-file", filepath.Join(dir, "watchdog-heartbeat-url"), + } + return h +} + +// run is one pass; flags given here override the host's own. +func (h *watchdogHost) run(flags ...string) (code int, stdout, stderr string) { + watchdogSender = h.rec.sender + defer func() { watchdogSender = smtpSender }() + var out, errOut bytes.Buffer + code = cmdWatchdog(append(append([]string(nil), h.args...), flags...), &out, &errOut) + return code, out.String(), errOut.String() +} + +// duePostgres is a state whose owner has not yet been told of PostgreSQL, down +// for an hour. +func duePostgres(cachedPassword string) *watchdog.State { + s := &watchdog.State{Recipients: []string{"owner@example.com"}, SMTPPassword: cachedPassword} + s.Observe(watchdog.Report{Findings: []watchdog.Finding{watchdog.PostgresDown(errors.New("refused"))}}, time.Now().Add(-time.Hour)) + return s +} + +// TestWatchdogRunKeepsClusterAlertsWhileTheAPIIsDown: with the API server down, +// the alerts under the cluster checks keep their state, since nothing looked +// at them; an alert of a check that did run and found nothing reads as cleared. +func TestWatchdogRunKeepsClusterAlertsWhileTheAPIIsDown(t *testing.T) { + told := time.Now().Add(-30 * time.Minute) + cluster := []string{"deployment/felis-api", "node/felis-1/NotReady", "server-failed/lobby"} + var r watchdog.Report + for _, key := range append([]string{"proxy"}, cluster...) { + r.Findings = append(r.Findings, watchdog.Finding{Key: key, Severity: watchdog.Critical, SummaryEN: key + " is down"}) + } + s := &watchdog.State{Recipients: []string{"owner@example.com"}} + s.Commit(s.Observe(r, told), told) + h := newWatchdogHost(t, testWatchdogConfig, s) + + code, stdout, stderr := h.run() + if code != 0 || len(h.rec.relays) != 0 { + t.Fatalf("exit %d, mailed %v\nstdout %s\nstderr %s", code, h.rec.sent, stdout, stderr) + } + for _, want := range []string{"felis watchdog: [critical] kube-api: ", "felis watchdog: [critical] postgres: "} { + if !strings.Contains(stdout, want) { + t.Errorf("stdout lacks %q:\n%s", want, stdout) + } + } + got, err := watchdog.LoadState(h.statePath) + if err != nil { + t.Fatal(err) + } + for _, key := range cluster { + if a := got.Alerts[key]; a == nil || !a.ClearedAt.IsZero() || !a.Notified.Equal(told) { + t.Errorf("%s with the API server down: %+v, want it kept open as mailed at %s", key, a, told) + } + } + if a := got.Alerts["proxy"]; a == nil || a.ClearedAt.IsZero() { + t.Errorf("proxy, whose check ran and found nothing: %+v, want it cleared", a) + } +} + +// TestWatchdogDryRunMailsAndSavesNothing: a dry run prints the mail that is due +// and the heartbeat it would ping, and sends, pings and saves nothing. +func TestWatchdogDryRunMailsAndSavesNothing(t *testing.T) { + t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw") + h := newWatchdogHost(t, testWatchdogConfig+testWatchdogSMTP, duePostgres("")) + before, err := os.ReadFile(h.statePath) + if err != nil { + t.Fatal(err) + } + code, stdout, stderr := h.run("-dry-run") + after, err := os.ReadFile(h.statePath) + if err != nil { + t.Fatal(err) + } + pings, _ := h.pings.got() + if code != 0 || len(h.rec.relays) != 0 || len(pings) != 0 || !bytes.Equal(before, after) { + t.Errorf("dry run: exit %d, relays %d, pings %v, state changed %v\nstdout %s\nstderr %s", code, len(h.rec.relays), pings, !bytes.Equal(before, after), stdout, stderr) + } + host, _ := os.Hostname() + for _, want := range []string{ + "felis watchdog: due to be mailed to owner@example.com:\nSubject: Felis 严重告警（" + host + "）：1 项异常 · 1 firing\n\n", + "felis watchdog: a run pings the heartbeat at " + h.url + "/...\n", + } { + if !strings.Contains(stdout, want) { + t.Errorf("stdout lacks %q:\n%s", want, stdout) + } + } + if strings.Contains(stdout, "\r") { + t.Errorf("the mail body printed with CRLF:\n%q", stdout) + } + + h = newWatchdogHost(t, testWatchdogConfig, nil) + code, stdout, stderr = h.run("-dry-run") + if code != 0 || !strings.Contains(stdout, "felis watchdog: nothing is due to be mailed\n") { + t.Errorf("dry run with nothing due: exit %d\nstdout %s\nstderr %s", code, stdout, stderr) + } + if _, err := os.Stat(h.statePath); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("a dry run wrote the state (%v)", err) + } +} + +// TestWatchdogRunSignsInWithTheHostRelayPassword: a pass caches the relay +// password from the host copy even while the cluster is down, and signs in with +// it; with no host copy and no cluster it keeps the one it had. +func TestWatchdogRunSignsInWithTheHostRelayPassword(t *testing.T) { + const smtpNoRef = "[smtp]\nhost = \"smtp.config.example\"\nport = 2525\nfrom = \"felis@example.com\"\nusername = \"felis\"\n" + for _, tc := range []struct{ what, hostCopy, want string }{ + {"the host copy", "host-pw", "host-pw"}, + {"no host copy, the cluster down", "", "cached-pw"}, + } { + h := newWatchdogHost(t, testWatchdogConfig+smtpNoRef, duePostgres("cached-pw")) + if tc.hostCopy != "" { + writeTestFile(t, filepath.Join(h.dir, "smtp-password"), tc.hostCopy, 0o600) + } + code, stdout, stderr := h.run() + if code != 0 || len(h.rec.relays) != 1 || h.rec.relays[0].Password != tc.want { + t.Errorf("%s: exit %d, relays %+v; want one mail signed in with %q\nstdout %s\nstderr %s", tc.what, code, h.rec.relays, tc.want, stdout, stderr) + continue + } + if s, err := watchdog.LoadState(h.statePath); err != nil || s.SMTPPassword != tc.want { + t.Errorf("%s: the state caches another password (%v)", tc.what, err) + } + } +} + +// TestWatchdogRunChecksWhatIsConfigured: the proxy, the database backups, the +// off-site copy and the build lane's scan DB are each checked when the host +// has them, and only then. +func TestWatchdogRunChecksWhatIsConfigured(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + closed := ln.Addr().String() + ln.Close() + const offsiteTable = "[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis\"\n" + const registry = "[registry]\nurl = \"registry.felis.svc:5000\"\n" + optional := []string{"proxy", "db-backup", "offsite", "scan-db"} + for _, tc := range []struct { + what string + cfg string + flags func(dir string) []string + want string // the one optional check that reports, "" for none + }{ + {what: "none of them", cfg: testWatchdogConfig}, + {what: "a proxy address", cfg: testWatchdogConfig, flags: func(string) []string { return []string{"-proxy-addr", closed} }, want: "proxy"}, + {what: "a backup directory", cfg: testWatchdogConfig, flags: func(dir string) []string { return []string{"-backup-dir", dir} }, want: "db-backup"}, + {what: "[offsite]", cfg: testWatchdogConfig + offsiteTable, want: "offsite"}, + {what: "a registry with the default scan DB", cfg: testWatchdogConfig + registry, want: "scan-db"}, + {what: "a registry with its scan DB under mirror/", cfg: testWatchdogConfig + registry + "trivy_db_repository = \"registry.felis.svc:5000/mirror/trivy-db\"\n", want: "scan-db"}, + {what: "a registry with the scan DB elsewhere", cfg: testWatchdogConfig + registry + "trivy_db_repository = \"ghcr.io/aquasecurity/trivy-db\"\n"}, + } { + h := newWatchdogHost(t, tc.cfg, nil) + var flags []string + if tc.flags != nil { + flags = tc.flags(t.TempDir()) + } + code, stdout, stderr := h.run(append(flags, "-dry-run")...) + var reported []string + for _, key := range optional { + if strings.Contains(stdout, "] "+key+": ") { + reported = append(reported, key) + } + } + if code != 0 || strings.Join(reported, ",") != tc.want { + t.Errorf("%s: exit %d, reported %v; want %q\nstdout %s\nstderr %s", tc.what, code, reported, tc.want, stdout, stderr) + } + } +} + +// TestWatchdogRunStateThatDoesNotSave: a pass whose state does not save mails +// as usual, exits 1 and posts the failure to the heartbeat, since the next run +// mails the same alerts again. +func TestWatchdogRunStateThatDoesNotSave(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root writes into a read-only directory") + } + t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw") + h := newWatchdogHost(t, testWatchdogConfig+testWatchdogSMTP, duePostgres("")) + if err := os.Chmod(h.dir, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(h.dir, 0o700) }) + code, stdout, stderr := h.run() + pings, bodies := h.pings.got() + if code != 1 || len(h.rec.sent) != 1 || !strings.Contains(stderr, "felis watchdog: save state: ") || + len(pings) != 1 || pings[0] != "POST /check-key/fail" || !strings.HasPrefix(bodies[0], "the watchdog state did not save: ") { + t.Errorf("exit %d, mailed %v, pings %v %q; want exit 1, one mail and the save failure posted to /fail\nstdout %s\nstderr %s", code, h.rec.sent, pings, bodies, stdout, stderr) + } +} diff --git a/internal/pgint/watchdog_test.go b/internal/pgint/watchdog_test.go new file mode 100644 index 0000000..e1bc9d7 --- /dev/null +++ b/internal/pgint/watchdog_test.go @@ -0,0 +1,54 @@ +//go:build pgint + +package pgint + +import ( + "context" + "slices" + "strings" + "testing" + + "felis.lolicon.best/internal/watchdog" +) + +// TestWatchdogOwnerEmails: the watchdog mails the verified address of every enabled +// owner account, in order, and no other account's. +func TestWatchdogOwnerEmails(t *testing.T) { + ctx := context.Background() + tag := suffix(t) + addr := func(who string) string { return who + "-" + tag + "@example.com" } + for _, u := range []struct { + who, role, email string + verified, off bool + deleted bool + }{ + {who: "zoe", role: "owner", email: addr("zoe"), verified: true}, + {who: "amy", role: "owner", email: addr("amy"), verified: true}, + {who: "unverified", role: "owner", email: addr("unverified")}, + {who: "disabled", role: "owner", email: addr("disabled"), verified: true, off: true}, + {who: "deleted", role: "owner", email: addr("deleted"), verified: true, deleted: true}, + {who: "admin", role: "admin", email: addr("admin"), verified: true}, + {who: "user", role: "user", email: addr("user"), verified: true}, + {who: "no-address", role: "owner", verified: true}, + } { + var email any + if u.email != "" { + email = u.email + } + if _, err := db.ExecContext(ctx, + `INSERT INTO users (id, username, role, email, email_verified, disabled, deleted_at) + VALUES ($1, $1, $2, $3, $4, $5, CASE WHEN $6 THEN now() END)`, + "pgint-"+u.who+"-"+tag, u.role, email, u.verified, u.off, u.deleted); err != nil { + t.Fatalf("insert %s: %v", u.who, err) + } + } + got, err := watchdog.OwnerEmails(ctx, db) + if err != nil { + t.Fatal(err) + } + // Other suites share the schema; their accounts carry other tags. + mine := slices.DeleteFunc(got, func(e string) bool { return !strings.Contains(e, tag) }) + if want := []string{addr("amy"), addr("zoe")}; !slices.Equal(mine, want) { + t.Fatalf("OwnerEmails = %q, want %q", mine, want) + } +} diff --git a/internal/watchdog/owners.go b/internal/watchdog/owners.go new file mode 100644 index 0000000..15878e6 --- /dev/null +++ b/internal/watchdog/owners.go @@ -0,0 +1,30 @@ +package watchdog + +import ( + "context" + "database/sql" +) + +// OwnerEmails returns the verified addresses of the enabled owner accounts, the +// people who can act on an alert, in order. internal/pgint holds its contract +// against the real schema. +func OwnerEmails(ctx context.Context, db *sql.DB) ([]string, error) { + rows, err := db.QueryContext(ctx, + `SELECT email FROM users + WHERE role = 'owner' AND email_verified AND COALESCE(email, '') <> '' + AND NOT disabled AND deleted_at IS NULL + ORDER BY email`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var email string + if err := rows.Scan(&email); err != nil { + return nil, err + } + out = append(out, email) + } + return out, rows.Err() +}