+3
−21
+222
−0
| Original line number | Diff line number | Diff line |
|---|---|---|
package main
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"io/fs"
|
||
"net"
|
||
"os"
|
||
"path/filepath"
|
||
| @@ -11,6 +14,7 @@ import ( | ||
"time"
|
||
"felis.lolicon.best/internal/offsite"
|
||
"felis.lolicon.best/internal/watchdog"
|
||
)
|
||
// TestProxyFinding: a listening proxy is healthy; a closed port is the critical
|
||
| @@ -89,3 +93,221 @@ func TestMailHold(t *testing.T) { | ||
t.Errorf("an unreadable status held the mail: %q", got)
|
||
}
|
||
}
|
||
// watchdogHost is a host whole watchdog passes run on: the API server and PostgreSQL
|
||
// are down (no kubeconfig, nothing on the database port), the relay is a recorder and
|
||
// the heartbeat URL points at a ping log.
|
||
type watchdogHost struct {
|
||
dir, statePath string
|
||
args []string
|
||
rec *alertRecorder
|
||
pings *pingLog
|
||
url string
|
||
}
|
||
func newWatchdogHost(t *testing.T, cfg string, state *watchdog.State) *watchdogHost {
|
||
t.Helper()
|
||
dir := t.TempDir()
|
||
t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
|
||
pings, srv := newPingServer(t)
|
||
h := &watchdogHost{dir: dir, statePath: filepath.Join(dir, "state.json"), rec: &alertRecorder{}, pings: pings, url: srv.URL}
|
||
writeTestFile(t, filepath.Join(dir, "felis.toml"), cfg, 0o600)
|
||
writeTestFile(t, filepath.Join(dir, "watchdog-heartbeat-url"), srv.URL+"/check-key\n", 0o600)
|
||
if state != nil {
|
||
if err := watchdog.SaveState(h.statePath, state); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
h.args = []string{
|
||
"-config", filepath.Join(dir, "felis.toml"), "-state", h.statePath, "-quiet-file", filepath.Join(dir, "quiet"),
|
||
"-backup-dir", "", "-disk-paths", dir, "-k3s-cert-dirs", "", "-smtp-password-file", filepath.Join(dir, "smtp-password"),
|
||
"-offsite-status", filepath.Join(dir, "offsite-status.json"), "-build-tools-status", filepath.Join(dir, "build-tools.json"),
|
||
"-heartbeat-file", filepath.Join(dir, "watchdog-heartbeat-url"),
|
||
}
|
||
return h
|
||
}
|
||
// run is one pass; flags given here override the host's own.
|
||
func (h *watchdogHost) run(flags ...string) (code int, stdout, stderr string) {
|
||
watchdogSender = h.rec.sender
|
||
defer func() { watchdogSender = smtpSender }()
|
||
var out, errOut bytes.Buffer
|
||
code = cmdWatchdog(append(append([]string(nil), h.args...), flags...), &out, &errOut)
|
||
return code, out.String(), errOut.String()
|
||
}
|
||
// duePostgres is a state whose owner has not yet been told of PostgreSQL, down
|
||
// for an hour.
|
||
func duePostgres(cachedPassword string) *watchdog.State {
|
||
s := &watchdog.State{Recipients: []string{"[email protected]"}, SMTPPassword: cachedPassword}
|
||
s.Observe(watchdog.Report{Findings: []watchdog.Finding{watchdog.PostgresDown(errors.New("refused"))}}, time.Now().Add(-time.Hour))
|
||
return s
|
||
}
|
||
// TestWatchdogRunKeepsClusterAlertsWhileTheAPIIsDown: with the API server down,
|
||
// the alerts under the cluster checks keep their state, since nothing looked
|
||
// at them; an alert of a check that did run and found nothing reads as cleared.
|
||
func TestWatchdogRunKeepsClusterAlertsWhileTheAPIIsDown(t *testing.T) {
|
||
told := time.Now().Add(-30 * time.Minute)
|
||
cluster := []string{"deployment/felis-api", "node/felis-1/NotReady", "server-failed/lobby"}
|
||
var r watchdog.Report
|
||
for _, key := range append([]string{"proxy"}, cluster...) {
|
||
r.Findings = append(r.Findings, watchdog.Finding{Key: key, Severity: watchdog.Critical, SummaryEN: key + " is down"})
|
||
}
|
||
s := &watchdog.State{Recipients: []string{"[email protected]"}}
|
||
s.Commit(s.Observe(r, told), told)
|
||
h := newWatchdogHost(t, testWatchdogConfig, s)
|
||
code, stdout, stderr := h.run()
|
||
if code != 0 || len(h.rec.relays) != 0 {
|
||
t.Fatalf("exit %d, mailed %v\nstdout %s\nstderr %s", code, h.rec.sent, stdout, stderr)
|
||
}
|
||
for _, want := range []string{"felis watchdog: [critical] kube-api: ", "felis watchdog: [critical] postgres: "} {
|
||
if !strings.Contains(stdout, want) {
|
||
t.Errorf("stdout lacks %q:\n%s", want, stdout)
|
||
}
|
||
}
|
||
got, err := watchdog.LoadState(h.statePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, key := range cluster {
|
||
if a := got.Alerts[key]; a == nil || !a.ClearedAt.IsZero() || !a.Notified.Equal(told) {
|
||
t.Errorf("%s with the API server down: %+v, want it kept open as mailed at %s", key, a, told)
|
||
}
|
||
}
|
||
if a := got.Alerts["proxy"]; a == nil || a.ClearedAt.IsZero() {
|
||
t.Errorf("proxy, whose check ran and found nothing: %+v, want it cleared", a)
|
||
}
|
||
}
|
||
// TestWatchdogDryRunMailsAndSavesNothing: a dry run prints the mail that is due
|
||
// and the heartbeat it would ping, and sends, pings and saves nothing.
|
||
func TestWatchdogDryRunMailsAndSavesNothing(t *testing.T) {
|
||
t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
|
||
h := newWatchdogHost(t, testWatchdogConfig+testWatchdogSMTP, duePostgres(""))
|
||
before, err := os.ReadFile(h.statePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
code, stdout, stderr := h.run("-dry-run")
|
||
after, err := os.ReadFile(h.statePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
pings, _ := h.pings.got()
|
||
if code != 0 || len(h.rec.relays) != 0 || len(pings) != 0 || !bytes.Equal(before, after) {
|
||
t.Errorf("dry run: exit %d, relays %d, pings %v, state changed %v\nstdout %s\nstderr %s", code, len(h.rec.relays), pings, !bytes.Equal(before, after), stdout, stderr)
|
||
}
|
||
host, _ := os.Hostname()
|
||
for _, want := range []string{
|
||
"felis watchdog: due to be mailed to [email protected]:\nSubject: Felis 严重告警（" + host + "）：1 项异常 · 1 firing\n\n",
|
||
"felis watchdog: a run pings the heartbeat at " + h.url + "/...\n",
|
||
} {
|
||
if !strings.Contains(stdout, want) {
|
||
t.Errorf("stdout lacks %q:\n%s", want, stdout)
|
||
}
|
||
}
|
||
if strings.Contains(stdout, "\r") {
|
||
t.Errorf("the mail body printed with CRLF:\n%q", stdout)
|
||
}
|
||
h = newWatchdogHost(t, testWatchdogConfig, nil)
|
||
code, stdout, stderr = h.run("-dry-run")
|
||
if code != 0 || !strings.Contains(stdout, "felis watchdog: nothing is due to be mailed\n") {
|
||
t.Errorf("dry run with nothing due: exit %d\nstdout %s\nstderr %s", code, stdout, stderr)
|
||
}
|
||
if _, err := os.Stat(h.statePath); !errors.Is(err, fs.ErrNotExist) {
|
||
t.Errorf("a dry run wrote the state (%v)", err)
|
||
}
|
||
}
|
||
// TestWatchdogRunSignsInWithTheHostRelayPassword: a pass caches the relay
|
||
// password from the host copy even while the cluster is down, and signs in with
|
||
// it; with no host copy and no cluster it keeps the one it had.
|
||
func TestWatchdogRunSignsInWithTheHostRelayPassword(t *testing.T) {
|
||
const smtpNoRef = "[smtp]\nhost = \"smtp.config.example\"\nport = 2525\nfrom = \"[email protected]\"\nusername = \"felis\"\n"
|
||
for _, tc := range []struct{ what, hostCopy, want string }{
|
||
{"the host copy", "host-pw", "host-pw"},
|
||
{"no host copy, the cluster down", "", "cached-pw"},
|
||
} {
|
||
h := newWatchdogHost(t, testWatchdogConfig+smtpNoRef, duePostgres("cached-pw"))
|
||
if tc.hostCopy != "" {
|
||
writeTestFile(t, filepath.Join(h.dir, "smtp-password"), tc.hostCopy, 0o600)
|
||
}
|
||
code, stdout, stderr := h.run()
|
||
if code != 0 || len(h.rec.relays) != 1 || h.rec.relays[0].Password != tc.want {
|
||
t.Errorf("%s: exit %d, relays %+v; want one mail signed in with %q\nstdout %s\nstderr %s", tc.what, code, h.rec.relays, tc.want, stdout, stderr)
|
||
continue
|
||
}
|
||
if s, err := watchdog.LoadState(h.statePath); err != nil || s.SMTPPassword != tc.want {
|
||
t.Errorf("%s: the state caches another password (%v)", tc.what, err)
|
||
}
|
||
}
|
||
}
|
||
// TestWatchdogRunChecksWhatIsConfigured: the proxy, the database backups, the
|
||
// off-site copy and the build lane's scan DB are each checked when the host
|
||
// has them, and only then.
|
||
func TestWatchdogRunChecksWhatIsConfigured(t *testing.T) {
|
||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
closed := ln.Addr().String()
|
||
ln.Close()
|
||
const offsiteTable = "[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis\"\n"
|
||
const registry = "[registry]\nurl = \"registry.felis.svc:5000\"\n"
|
||
optional := []string{"proxy", "db-backup", "offsite", "scan-db"}
|
||
for _, tc := range []struct {
|
||
what string
|
||
cfg string
|
||
flags func(dir string) []string
|
||
want string // the one optional check that reports, "" for none
|
||
}{
|
||
{what: "none of them", cfg: testWatchdogConfig},
|
||
{what: "a proxy address", cfg: testWatchdogConfig, flags: func(string) []string { return []string{"-proxy-addr", closed} }, want: "proxy"},
|
||
{what: "a backup directory", cfg: testWatchdogConfig, flags: func(dir string) []string { return []string{"-backup-dir", dir} }, want: "db-backup"},
|
||
{what: "[offsite]", cfg: testWatchdogConfig + offsiteTable, want: "offsite"},
|
||
{what: "a registry with the default scan DB", cfg: testWatchdogConfig + registry, want: "scan-db"},
|
||
{what: "a registry with its scan DB under mirror/", cfg: testWatchdogConfig + registry + "trivy_db_repository = \"registry.felis.svc:5000/mirror/trivy-db\"\n", want: "scan-db"},
|
||
{what: "a registry with the scan DB elsewhere", cfg: testWatchdogConfig + registry + "trivy_db_repository = \"ghcr.io/aquasecurity/trivy-db\"\n"},
|
||
} {
|
||
h := newWatchdogHost(t, tc.cfg, nil)
|
||
var flags []string
|
||
if tc.flags != nil {
|
||
flags = tc.flags(t.TempDir())
|
||
}
|
||
code, stdout, stderr := h.run(append(flags, "-dry-run")...)
|
||
var reported []string
|
||
for _, key := range optional {
|
||
if strings.Contains(stdout, "] "+key+": ") {
|
||
reported = append(reported, key)
|
||
}
|
||
}
|
||
if code != 0 || strings.Join(reported, ",") != tc.want {
|
||
t.Errorf("%s: exit %d, reported %v; want %q\nstdout %s\nstderr %s", tc.what, code, reported, tc.want, stdout, stderr)
|
||
}
|
||
}
|
||
}
|
||
// TestWatchdogRunStateThatDoesNotSave: a pass whose state does not save mails
|
||
// as usual, exits 1 and posts the failure to the heartbeat, since the next run
|
||
// mails the same alerts again.
|
||
func TestWatchdogRunStateThatDoesNotSave(t *testing.T) {
|
||
if os.Geteuid() == 0 {
|
||
t.Skip("root writes into a read-only directory")
|
||
}
|
||
t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
|
||
h := newWatchdogHost(t, testWatchdogConfig+testWatchdogSMTP, duePostgres(""))
|
||
if err := os.Chmod(h.dir, 0o500); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() { os.Chmod(h.dir, 0o700) })
|
||
code, stdout, stderr := h.run()
|
||
pings, bodies := h.pings.got()
|
||
if code != 1 || len(h.rec.sent) != 1 || !strings.Contains(stderr, "felis watchdog: save state: ") ||
|
||
len(pings) != 1 || pings[0] != "POST /check-key/fail" || !strings.HasPrefix(bodies[0], "the watchdog state did not save: ") {
|
||
t.Errorf("exit %d, mailed %v, pings %v %q; want exit 1, one mail and the save failure posted to /fail\nstdout %s\nstderr %s", code, h.rec.sent, pings, bodies, stdout, stderr)
|
||
}
|
||
} |
||
internal/pgint/watchdog_test.go
0 → 100644
+54
−0
internal/watchdog/owners.go
0 → 100644
+30
−0
Loading