fix(watchdog): 外部心跳、巡检失败兜底告警、状态文件损坏自动移开
This commit is contained in:
11 files changed
+1360
-44
No files matched your search
@@ -0,0 +1,589 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"felis.lolicon.best/internal/config"
|
||||
"felis.lolicon.best/internal/mail"
|
||||
"felis.lolicon.best/internal/offsite"
|
||||
"felis.lolicon.best/internal/watchdog"
|
||||
)
|
||||
|
||||
// pingLog is a monitoring service that records every ping it gets.
|
||||
type pingLog struct {
|
||||
mu sync.Mutex
|
||||
pings []string // "METHOD path?query"
|
||||
bodies []string
|
||||
status int
|
||||
}
|
||||
|
||||
func newPingServer(t *testing.T) (*pingLog, *httptest.Server) {
|
||||
t.Helper()
|
||||
l := &pingLog{status: http.StatusOK}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
p := r.Method + " " + r.URL.Path
|
||||
if r.URL.RawQuery != "" {
|
||||
p += "?" + r.URL.RawQuery
|
||||
}
|
||||
l.pings = append(l.pings, p)
|
||||
l.bodies = append(l.bodies, string(body))
|
||||
w.WriteHeader(l.status)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return l, srv
|
||||
}
|
||||
|
||||
func (l *pingLog) got() ([]string, []string) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return append([]string(nil), l.pings...), append([]string(nil), l.bodies...)
|
||||
}
|
||||
|
||||
// TestHeartbeatSend: a run whose alerts reach the owners GETs the URL, one
|
||||
// whose alerts reach no one POSTs its report to /fail, and a standby host, a
|
||||
// failure in the installer's quiet window and a query URL that has no /fail
|
||||
// send nothing.
|
||||
func TestHeartbeatSend(t *testing.T) {
|
||||
const key = "/ping/5f1e0c2a-check-key"
|
||||
for _, tc := range []struct {
|
||||
what string
|
||||
beat heartbeat
|
||||
path string // appended to the server URL
|
||||
want string // the ping, "" for none
|
||||
wantBody string
|
||||
wantOut string
|
||||
}{
|
||||
{what: "success", beat: heartbeat{}, path: key, want: "GET " + key},
|
||||
{what: "failure", beat: heartbeat{fail: true, report: "the alerts reach no one: x"}, path: key, want: "POST " + key + "/fail", wantBody: "the alerts reach no one: x", wantOut: "pinged the heartbeat's failure endpoint"},
|
||||
{what: "failure, URL with a trailing slash", beat: heartbeat{fail: true, report: "r"}, path: key + "/", want: "POST " + key + "/fail", wantBody: "r"},
|
||||
{what: "success, URL with a query", beat: heartbeat{}, path: key + "?rid=7", want: "GET " + key + "?rid=7"},
|
||||
{what: "failure, URL with a query", beat: heartbeat{fail: true, report: "r"}, path: key + "?rid=7", wantOut: "withholding the heartbeat ping"},
|
||||
{what: "standby, success", beat: heartbeat{standby: true}, path: key, wantOut: "stands by for the off-site bucket"},
|
||||
{what: "standby, failure", beat: heartbeat{standby: true, fail: true}, path: key, wantOut: "stands by for the off-site bucket"},
|
||||
{what: "quiet, failure", beat: heartbeat{quiet: true, fail: true}, path: key, wantOut: "withholding the failure ping"},
|
||||
{what: "quiet, success", beat: heartbeat{quiet: true}, path: key, want: "GET " + key},
|
||||
} {
|
||||
log, srv := newPingServer(t)
|
||||
b := tc.beat
|
||||
b.url = srv.URL + tc.path
|
||||
var stdout, stderr bytes.Buffer
|
||||
b.send(srv.Client(), &stdout, &stderr)
|
||||
pings, bodies := log.got()
|
||||
switch {
|
||||
case tc.want == "" && len(pings) != 0:
|
||||
t.Errorf("%s: pinged %v, want nothing", tc.what, pings)
|
||||
case tc.want != "" && (len(pings) != 1 || pings[0] != tc.want):
|
||||
t.Errorf("%s: pinged %v, want %q", tc.what, pings, tc.want)
|
||||
case tc.want != "" && bodies[0] != tc.wantBody:
|
||||
t.Errorf("%s: body %q, want %q", tc.what, bodies[0], tc.wantBody)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), tc.wantOut) || stderr.Len() != 0 {
|
||||
t.Errorf("%s: stdout %q (want %q), stderr %q", tc.what, stdout.String(), tc.wantOut, stderr.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeartbeatErrors: a ping the service refuses or that cannot connect is
|
||||
// logged without the URL's path, the check's key.
|
||||
func TestHeartbeatErrors(t *testing.T) {
|
||||
const key = "5f1e0c2a-check-key"
|
||||
log, srv := newPingServer(t)
|
||||
log.status = http.StatusNotFound
|
||||
var stdout, stderr bytes.Buffer
|
||||
heartbeat{url: srv.URL + "/" + key}.send(srv.Client(), &stdout, &stderr)
|
||||
if got := stderr.String(); !strings.Contains(got, "heartbeat: GET http://127.0.0.1") || !strings.Contains(got, "404") || strings.Contains(got, key) {
|
||||
t.Errorf("refused ping: stderr %q; want the host and status, not the key", got)
|
||||
}
|
||||
|
||||
closed := httptest.NewServer(http.NotFoundHandler())
|
||||
dead := closed.URL + "/" + key
|
||||
closed.Close()
|
||||
stderr.Reset()
|
||||
heartbeat{url: dead, fail: true, report: "r"}.send(http.DefaultClient, &stdout, &stderr)
|
||||
if got := stderr.String(); !strings.Contains(got, "heartbeat: POST http://127.0.0.1") || strings.Contains(got, key) {
|
||||
t.Errorf("unreachable service: stderr %q; want the host, not the key", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeartbeatReportClipped: a long report is cut to what the service keeps,
|
||||
// on a character boundary.
|
||||
func TestHeartbeatReportClipped(t *testing.T) {
|
||||
log, srv := newPingServer(t)
|
||||
report := strings.Repeat("磁盘", heartbeatReportMax)
|
||||
heartbeat{url: srv.URL + "/k", fail: true, report: report}.send(srv.Client(), io.Discard, io.Discard)
|
||||
_, bodies := log.got()
|
||||
if len(bodies) != 1 || len(bodies[0]) > heartbeatReportMax || len(bodies[0]) < heartbeatReportMax-3 || !utf8.ValidString(bodies[0]) {
|
||||
t.Fatalf("clipped body: %d pings, %d bytes, valid UTF-8 %v", len(bodies), len(bodies[0]), utf8.ValidString(bodies[0]))
|
||||
}
|
||||
if got := clipUTF8("short", heartbeatReportMax); got != "short" {
|
||||
t.Errorf("clipUTF8(short) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadHeartbeatURL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "watchdog-heartbeat-url")
|
||||
if u, err := readHeartbeatURL(path); u != "" || err != nil {
|
||||
t.Fatalf("no file: %q, %v", u, err)
|
||||
}
|
||||
if u, err := readHeartbeatURL(""); u != "" || err != nil {
|
||||
t.Fatalf("no path: %q, %v", u, err)
|
||||
}
|
||||
writeTestFile(t, path, "https://hc-ping.com/5f1e0c2a\n", 0o600)
|
||||
if u, err := readHeartbeatURL(path); u != "https://hc-ping.com/5f1e0c2a" || err != nil {
|
||||
t.Fatalf("good file: %q, %v", u, err)
|
||||
}
|
||||
for _, bad := range []string{"", "hc-ping.com/5f1e0c2a", "ftp://hc-ping.com/5f1e0c2a", "https:///5f1e0c2a", "https://hc-ping.com/5f1e 0c2a"} {
|
||||
writeTestFile(t, path, bad, 0o600)
|
||||
u, err := readHeartbeatURL(path)
|
||||
if u != "" || err == nil || !strings.Contains(err.Error(), path) || (bad != "" && strings.Contains(err.Error(), "5f1e")) {
|
||||
t.Errorf("%q: %q, %v; want an error naming the file and not the URL", bad, u, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactURL(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"https://hc-ping.com/5f1e0c2a": "https://hc-ping.com/...",
|
||||
"http://user:[email protected]:8080/k?x=1": "http://status.example:8080/...",
|
||||
"not a url": "(the heartbeat URL)",
|
||||
} {
|
||||
if got := redactURL(in); got != want {
|
||||
t.Errorf("redactURL(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStandsBy: a standby record keeps the host from pinging however old it
|
||||
// is; a displaced host, the writer, or a host without [offsite] pings.
|
||||
func TestStandsBy(t *testing.T) {
|
||||
status := filepath.Join(t.TempDir(), "status.json")
|
||||
if standsBy(true, status) {
|
||||
t.Fatal("no status file stands by")
|
||||
}
|
||||
old := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Now().Add(-30 * 24 * time.Hour)}
|
||||
for _, tc := range []struct {
|
||||
what string
|
||||
st offsite.Status
|
||||
offsiteOn bool
|
||||
want bool
|
||||
}{
|
||||
{"standing by for a writer gone quiet a month ago", offsite.Status{Standby: true, Writer: old}, true, true},
|
||||
{"standing by, [offsite] since removed", offsite.Status{Standby: true, Writer: old}, false, false},
|
||||
{"displaced", offsite.Status{Displaced: true, Writer: old}, true, false},
|
||||
{"the writer itself", offsite.Status{LastSuccess: time.Now()}, true, false},
|
||||
} {
|
||||
if err := offsite.WriteStatus(status, tc.st); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := standsBy(tc.offsiteOn, status); got != tc.want {
|
||||
t.Errorf("%s: standsBy = %v, want %v", tc.what, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailureReport(t *testing.T) {
|
||||
r := watchdog.Report{Findings: []watchdog.Finding{{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "PostgreSQL is down"}}}
|
||||
for _, tc := range []struct {
|
||||
what string
|
||||
unheard string
|
||||
mailFailed bool
|
||||
saveErr error
|
||||
open bool
|
||||
want string // "" for a success ping
|
||||
}{
|
||||
{what: "all good", open: true},
|
||||
{what: "no relay, nothing mailed yet", unheard: "no [smtp] relay is configured"},
|
||||
{what: "no relay, an alert open", unheard: "no [smtp] relay is configured", open: true, want: "the alerts reach no one: no [smtp] relay is configured"},
|
||||
{what: "the mail failed", unheard: "the alert mail failed: refused", mailFailed: true, want: "the alerts reach no one: the alert mail failed: refused"},
|
||||
{what: "the state did not save", saveErr: errors.New("disk full"), want: "the watchdog state did not save: disk full"},
|
||||
} {
|
||||
got := failureReport(tc.unheard, tc.mailFailed, tc.saveErr, tc.open, r)
|
||||
if (got == "") != (tc.want == "") || !strings.Contains(got, tc.want) || (got != "" && !strings.Contains(got, "[critical] postgres: PostgreSQL is down")) {
|
||||
t.Errorf("%s: report %q, want %q and the findings", tc.what, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// alertRecorder records the mail a run hands the relay.
|
||||
type alertRecorder struct {
|
||||
relays []*mail.SMTP
|
||||
sent []string // "to: subject"
|
||||
fail map[string]bool
|
||||
}
|
||||
|
||||
func (f *alertRecorder) sender(relay *mail.SMTP) alertSender {
|
||||
f.relays = append(f.relays, relay)
|
||||
return func(_ context.Context, to, subject, _ string) error {
|
||||
if f.fail[to] {
|
||||
return errors.New("550 refused")
|
||||
}
|
||||
f.sent = append(f.sent, to+": "+subject)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// TestMailerDeliver: a plan goes out and is committed; held, it waits
|
||||
// uncommitted; with no relay or recipient it is logged and committed; a mail
|
||||
// no recipient got is left uncommitted to come due again.
|
||||
func TestMailerDeliver(t *testing.T) {
|
||||
now := time.Now()
|
||||
relay := &watchdog.Relay{Host: "smtp.example.com", Port: 587, From: "[email protected]", Username: "felis"}
|
||||
for _, tc := range []struct {
|
||||
what string
|
||||
relay *watchdog.Relay
|
||||
recipients []string
|
||||
hold string
|
||||
fail map[string]bool
|
||||
wantUnheard string
|
||||
wantFailed bool
|
||||
committed bool
|
||||
sent int
|
||||
}{
|
||||
{what: "mailed", relay: relay, recipients: []string{"[email protected]", "[email protected]"}, committed: true, sent: 2},
|
||||
{what: "one recipient refused", relay: relay, recipients: []string{"[email protected]", "[email protected]"}, fail: map[string]bool{"[email protected]": true}, committed: true, sent: 1},
|
||||
{what: "every recipient refused", relay: relay, recipients: []string{"[email protected]"}, fail: map[string]bool{"[email protected]": true}, wantUnheard: "the alert mail failed", wantFailed: true},
|
||||
{what: "held", relay: relay, recipients: []string{"[email protected]"}, hold: "quiet until later"},
|
||||
{what: "no relay", recipients: []string{"[email protected]"}, wantUnheard: "no [smtp] relay is configured", committed: true},
|
||||
{what: "no recipient", relay: relay, wantUnheard: "no owner account has a verified email", committed: true},
|
||||
} {
|
||||
state := &watchdog.State{Recipients: tc.recipients}
|
||||
plan := state.Observe(watchdog.Report{Findings: []watchdog.Finding{{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "down"}}}, now)
|
||||
f := &alertRecorder{fail: tc.fail}
|
||||
m := mailer{relay: tc.relay, password: "relay-pw", send: f.sender}
|
||||
var stdout, stderr bytes.Buffer
|
||||
unheard, failed := m.deliver(context.Background(), state, plan, "subject", "body", tc.hold, now, &stdout, &stderr)
|
||||
if !strings.HasPrefix(unheard, tc.wantUnheard) || (tc.wantUnheard == "") != (unheard == "") || failed != tc.wantFailed {
|
||||
t.Errorf("%s: unheard %q, failed %v; want %q, %v", tc.what, unheard, failed, tc.wantUnheard, tc.wantFailed)
|
||||
}
|
||||
if got := !state.Alerts["postgres"].Notified.IsZero(); got != tc.committed {
|
||||
t.Errorf("%s: committed %v, want %v", tc.what, got, tc.committed)
|
||||
}
|
||||
if len(f.sent) != tc.sent {
|
||||
t.Errorf("%s: sent %v, want %d mails", tc.what, f.sent, tc.sent)
|
||||
}
|
||||
if len(f.relays) > 0 {
|
||||
if got := *f.relays[0]; got != (mail.SMTP{Host: "smtp.example.com", Port: 587, From: "[email protected]", Username: "felis", Password: "relay-pw"}) {
|
||||
t.Errorf("%s: relay %+v", tc.what, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCachedRelay: the cache keeps the relay's coordinates and its TLS rule,
|
||||
// and nothing when no relay is configured.
|
||||
func TestCachedRelay(t *testing.T) {
|
||||
if r := cachedRelay(testSMTPConfig("")); r != nil {
|
||||
t.Fatalf("no relay cached %+v", r)
|
||||
}
|
||||
got := cachedRelay(testSMTPConfig("smtp.example.com"))
|
||||
if got == nil || *got != (watchdog.Relay{Host: "smtp.example.com", Port: 2525, From: "[email protected]", Username: "felis", RequireTLS: true}) {
|
||||
t.Fatalf("cachedRelay = %+v", got)
|
||||
}
|
||||
if got := cachedRelay(testSMTPConfig("127.0.0.1")); got == nil || got.RequireTLS {
|
||||
t.Fatalf("a relay on this host: %+v, want TLS not required", got)
|
||||
}
|
||||
}
|
||||
|
||||
func testSMTPConfig(host string) config.SMTPConfig {
|
||||
if host == "" {
|
||||
return config.SMTPConfig{}
|
||||
}
|
||||
return config.SMTPConfig{Host: host, Port: 2525, From: "[email protected]", Username: "felis"}
|
||||
}
|
||||
|
||||
// TestSMTPPassword: the env var password_ref names wins over the cache once
|
||||
// it is set.
|
||||
func TestSMTPPassword(t *testing.T) {
|
||||
c := testSMTPConfig("smtp.example.com")
|
||||
c.PasswordRef = "FELIS_TEST_WATCHDOG_RELAY_PW"
|
||||
t.Setenv(c.PasswordRef, "")
|
||||
if got := smtpPassword(c, "cached"); got != "cached" {
|
||||
t.Errorf("env var unset: %q", got)
|
||||
}
|
||||
t.Setenv(c.PasswordRef, "from-env")
|
||||
if got := smtpPassword(c, "cached"); got != "from-env" {
|
||||
t.Errorf("env var set: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
const testWatchdogConfig = `[database]
|
||||
url = "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
|
||||
[server]
|
||||
root_domain = "example.com"
|
||||
[archive]
|
||||
store = "tarLocal"
|
||||
[k8s]
|
||||
egress_mode = "nodeport"
|
||||
`
|
||||
|
||||
const testWatchdogSMTP = `[smtp]
|
||||
host = "smtp.config.example"
|
||||
port = 2525
|
||||
from = "[email protected]"
|
||||
username = "felis"
|
||||
password_ref = "FELIS_TEST_WATCHDOG_RELAY_PW"
|
||||
`
|
||||
|
||||
// unitFailedFixture is a host whose watchdog last ran well: one alert open,
|
||||
// one owner and the relay cached.
|
||||
func unitFailedFixture(t *testing.T, cfg string) (unitFailedRun, *alertRecorder, *pingLog) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
log, srv := newPingServer(t)
|
||||
beatFile := filepath.Join(dir, "watchdog-heartbeat-url")
|
||||
writeTestFile(t, beatFile, srv.URL+"/check-key\n", 0o600)
|
||||
cfgPath := filepath.Join(dir, "felis.toml")
|
||||
writeTestFile(t, cfgPath, cfg, 0o600)
|
||||
statePath := filepath.Join(dir, "state.json")
|
||||
state := &watchdog.State{
|
||||
Recipients: []string{"[email protected]"},
|
||||
SMTPPassword: "cached-pw",
|
||||
Relay: &watchdog.Relay{Host: "smtp.cached.example", Port: 587, From: "[email protected]", RequireTLS: true},
|
||||
}
|
||||
mem := watchdog.Finding{Key: "memory", Severity: watchdog.Warning, SummaryEN: "memory low"}
|
||||
state.Commit(state.Observe(watchdog.Report{Findings: []watchdog.Finding{mem}}, time.Now().Add(-time.Hour)), time.Now().Add(-time.Hour))
|
||||
if err := watchdog.SaveState(statePath, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := &alertRecorder{}
|
||||
return unitFailedRun{
|
||||
cfgPath: cfgPath, statePath: statePath, quietPath: filepath.Join(dir, "quiet"),
|
||||
offsiteStatus: filepath.Join(dir, "offsite-status.json"), heartbeatFile: beatFile,
|
||||
result: "exit-code", exitStatus: "1", send: f.sender, client: srv.Client(), now: time.Now(),
|
||||
}, f, log
|
||||
}
|
||||
|
||||
// TestWatchdogUnitFailedBrokenConfig: failed runs of a watchdog whose
|
||||
// felis.toml no longer loads are mailed through the relay the last good run
|
||||
// cached, after five in a row; every one pings /fail; the open alert keeps
|
||||
// its state.
|
||||
func TestWatchdogUnitFailedBrokenConfig(t *testing.T) {
|
||||
r, f, log := unitFailedFixture(t, "[database\n")
|
||||
start := r.now
|
||||
for i := 0; i <= 5; i++ {
|
||||
r.now = start.Add(time.Duration(i) * 2 * time.Minute)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := watchdogUnitFailed(r, &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("run %d: exit %d (stdout %s, stderr %s)", i, code, stdout.String(), stderr.String())
|
||||
}
|
||||
if i < 5 && len(f.sent) != 0 {
|
||||
t.Fatalf("run %d, %v after the first failure: mailed %v, want nothing yet", i, r.now.Sub(start), f.sent)
|
||||
}
|
||||
}
|
||||
if len(f.sent) != 1 || !strings.Contains(f.sent[0], "[email protected]: ") {
|
||||
t.Fatalf("mailed %v, want one alert to the cached owner", f.sent)
|
||||
}
|
||||
if got := *f.relays[0]; got.Host != "smtp.cached.example" || got.Password != "cached-pw" || !got.RequireTLS {
|
||||
t.Fatalf("relay %+v, want the cached one", got)
|
||||
}
|
||||
pings, bodies := log.got()
|
||||
if len(pings) != 6 || pings[0] != "POST /check-key/fail" || !strings.Contains(bodies[0], "felis-watchdog.service failed: result exit-code, exit status 1; ") {
|
||||
t.Fatalf("pings %v, bodies %q", pings, bodies)
|
||||
}
|
||||
state, err := watchdog.LoadState(r.statePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := state.Alerts["watchdog/run"]; a == nil || a.Notified.IsZero() || !strings.Contains(a.SummaryEN, "result exit-code, exit status 1") {
|
||||
t.Fatalf("watchdog/run alert = %+v", a)
|
||||
}
|
||||
if a := state.Alerts["memory"]; a == nil || !a.ClearedAt.IsZero() || !a.Notified.Before(start) {
|
||||
t.Fatalf("memory alert = %+v, want it untouched", a)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchdogUnitFailedConfigRelay: with felis.toml loading, the relay is
|
||||
// the configured one and signs in with the env var password_ref names.
|
||||
func TestWatchdogUnitFailedConfigRelay(t *testing.T) {
|
||||
r, f, _ := unitFailedFixture(t, testWatchdogConfig+testWatchdogSMTP)
|
||||
t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
|
||||
start := r.now
|
||||
for _, at := range []time.Duration{0, 10 * time.Minute} {
|
||||
r.now = start.Add(at)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := watchdogUnitFailed(r, &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("exit %d (stdout %s, stderr %s)", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
if len(f.relays) != 1 || f.relays[0].Host != "smtp.config.example" || f.relays[0].Port != 2525 || f.relays[0].Password != "env-pw" {
|
||||
t.Fatalf("relays %+v, want the configured one with the env password", f.relays)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchdogUnitFailedHeld: in the installer's quiet window the alert waits
|
||||
// and no failure is pinged; a host standing by for the off-site bucket pings
|
||||
// nothing either.
|
||||
func TestWatchdogUnitFailedHeld(t *testing.T) {
|
||||
r, f, log := unitFailedFixture(t, "[database\n")
|
||||
writeTestFile(t, r.quietPath, fmt.Sprintf("%d\n", r.now.Add(time.Hour).Unix()), 0o644)
|
||||
start := r.now
|
||||
for _, at := range []time.Duration{0, 10 * time.Minute} {
|
||||
r.now = start.Add(at)
|
||||
watchdogUnitFailed(r, io.Discard, io.Discard)
|
||||
}
|
||||
if pings, _ := log.got(); len(f.sent) != 0 || len(pings) != 0 {
|
||||
t.Fatalf("quiet window: mailed %v, pinged %v", f.sent, pings)
|
||||
}
|
||||
|
||||
os.Remove(r.quietPath)
|
||||
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: r.now.Add(-time.Hour)}
|
||||
if err := offsite.WriteStatus(r.offsiteStatus, offsite.Status{Standby: true, Writer: w}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
watchdogUnitFailed(r, &stdout, io.Discard)
|
||||
if pings, _ := log.got(); len(f.sent) != 0 || len(pings) != 0 || !strings.Contains(stdout.String(), "stands by for host prod-1") {
|
||||
t.Fatalf("standby: mailed %v, pinged %v, stdout %s", f.sent, pings, stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchdogUnitFailedStateUnreadable: with no state to mail from, the
|
||||
// failure still reaches the heartbeat.
|
||||
func TestWatchdogUnitFailedStateUnreadable(t *testing.T) {
|
||||
r, f, log := unitFailedFixture(t, "[database\n")
|
||||
writeTestFile(t, r.statePath, "{", 0o600)
|
||||
if code := watchdogUnitFailed(r, io.Discard, io.Discard); code != 1 {
|
||||
t.Fatalf("exit %d, want 1", code)
|
||||
}
|
||||
pings, bodies := log.got()
|
||||
if len(f.sent) != 0 || len(pings) != 1 || pings[0] != "POST /check-key/fail" || !strings.Contains(bodies[0], "state does not load") {
|
||||
t.Fatalf("mailed %v, pinged %v %q", f.sent, pings, bodies)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailureDetail(t *testing.T) {
|
||||
for _, tc := range [][3]string{
|
||||
{"exit-code", "1", "result exit-code, exit status 1"},
|
||||
{"timeout", "", "result timeout"},
|
||||
{"", "", "systemd named no cause (journalctl -u felis-watchdog -n 50)"},
|
||||
} {
|
||||
if got := failureDetail(tc[0], tc[1]); got != tc[2] {
|
||||
t.Errorf("failureDetail(%q, %q) = %q, want %q", tc[0], tc[1], got, tc[2])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchdogRunHeartbeat runs whole watchdog passes with the API server and
|
||||
// PostgreSQL down and the relay a recorder: a pass whose alerts reach the
|
||||
// owners pings success; one whose mail fails, or that has no relay while an
|
||||
// alert is open, posts its report to /fail; a host standing by for the
|
||||
// off-site bucket pings nothing; a state file that does not parse is moved
|
||||
// aside and reported.
|
||||
func TestWatchdogRunHeartbeat(t *testing.T) {
|
||||
due := func(statePath string) {
|
||||
// PostgreSQL has been down for an hour and nobody was told yet.
|
||||
s := &watchdog.State{Recipients: []string{"[email protected]"}, SMTPPassword: "cached-pw"}
|
||||
s.Observe(watchdog.Report{Findings: []watchdog.Finding{watchdog.PostgresDown(errors.New("refused"))}}, time.Now().Add(-time.Hour))
|
||||
if err := watchdog.SaveState(statePath, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
open := func(statePath string) {
|
||||
// The owners were told of PostgreSQL half an hour ago.
|
||||
s := &watchdog.State{Recipients: []string{"[email protected]"}}
|
||||
at := time.Now().Add(-30 * time.Minute)
|
||||
r := watchdog.Report{Findings: []watchdog.Finding{watchdog.PostgresDown(errors.New("refused"))}}
|
||||
s.Observe(r, at.Add(-time.Hour))
|
||||
s.Commit(s.Observe(r, at), at)
|
||||
if err := watchdog.SaveState(statePath, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const offsiteTable = "[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis\"\n"
|
||||
t.Setenv("FELIS_TEST_WATCHDOG_RELAY_PW", "env-pw")
|
||||
for _, tc := range []struct {
|
||||
what string
|
||||
cfg string
|
||||
state func(path string)
|
||||
standby bool
|
||||
quiet bool
|
||||
refused bool
|
||||
wantCode int
|
||||
want string // the ping, "" for none
|
||||
wantBody string
|
||||
wantOut string
|
||||
wantSent int
|
||||
}{
|
||||
{what: "nothing due", cfg: testWatchdogConfig + testWatchdogSMTP, want: "GET /check-key"},
|
||||
{what: "mailed", cfg: testWatchdogConfig + testWatchdogSMTP, state: due, want: "GET /check-key", wantSent: 1},
|
||||
{what: "the mail fails", cfg: testWatchdogConfig + testWatchdogSMTP, state: due, refused: true, wantCode: 1, want: "POST /check-key/fail", wantBody: "the alerts reach no one: the alert mail failed: [email protected]: 550 refused"},
|
||||
{what: "no relay, an alert open", cfg: testWatchdogConfig, state: due, want: "POST /check-key/fail", wantBody: "the alerts reach no one: no [smtp] relay is configured"},
|
||||
{what: "no relay, an alert open, the installer running", cfg: testWatchdogConfig, state: open, quiet: true, wantOut: "withholding the failure ping"},
|
||||
{what: "standing by for the off-site bucket", cfg: testWatchdogConfig + testWatchdogSMTP + offsiteTable, state: due, standby: true, wantOut: "stands by for the off-site bucket"},
|
||||
{what: "a state that does not parse", cfg: testWatchdogConfig, state: func(p string) { writeTestFile(t, p, "{", 0o600) }, want: "GET /check-key", wantOut: "[warning] watchdog/state: the watchdog's state file was unreadable and was moved to "},
|
||||
} {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
|
||||
log, srv := newPingServer(t)
|
||||
beatFile := filepath.Join(dir, "watchdog-heartbeat-url")
|
||||
writeTestFile(t, beatFile, srv.URL+"/check-key\n", 0o600)
|
||||
cfgPath := filepath.Join(dir, "felis.toml")
|
||||
writeTestFile(t, cfgPath, tc.cfg, 0o600)
|
||||
statePath := filepath.Join(dir, "state.json")
|
||||
if tc.state != nil {
|
||||
tc.state(statePath)
|
||||
}
|
||||
statusPath := filepath.Join(dir, "offsite-status.json")
|
||||
if tc.standby {
|
||||
w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Now().Add(-time.Hour)}
|
||||
if err := offsite.WriteStatus(statusPath, offsite.Status{Standby: true, Writer: w}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.quiet {
|
||||
writeTestFile(t, filepath.Join(dir, "quiet"), fmt.Sprintf("%d\n", time.Now().Add(time.Hour).Unix()), 0o644)
|
||||
}
|
||||
rec := &alertRecorder{}
|
||||
if tc.refused {
|
||||
rec.fail = map[string]bool{"[email protected]": true}
|
||||
}
|
||||
watchdogSender = rec.sender
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := cmdWatchdog([]string{
|
||||
"-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"),
|
||||
"-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", filepath.Join(dir, "smtp-password"),
|
||||
"-offsite-status", statusPath, "-heartbeat-file", beatFile,
|
||||
}, &stdout, &stderr)
|
||||
watchdogSender = smtpSender
|
||||
pings, bodies := log.got()
|
||||
fail := code != tc.wantCode || len(rec.sent) != tc.wantSent || !strings.Contains(stdout.String(), tc.wantOut)
|
||||
if tc.want == "" {
|
||||
fail = fail || len(pings) != 0
|
||||
} else {
|
||||
fail = fail || len(pings) != 1 || pings[0] != tc.want || !strings.Contains(bodies[0], tc.wantBody)
|
||||
}
|
||||
if fail {
|
||||
t.Errorf("%s: exit %d, mailed %v, pings %v %q; want exit %d, %d mails, %q with %q\nstdout %s\nstderr %s", tc.what, code, rec.sent, pings, bodies, tc.wantCode, tc.wantSent, tc.want, tc.wantBody, stdout.String(), stderr.String())
|
||||
}
|
||||
if tc.wantSent > 0 {
|
||||
if got := *rec.relays[0]; got.Host != "smtp.config.example" || got.Port != 2525 || got.Password != "env-pw" {
|
||||
t.Errorf("%s: relay %+v, want the configured one with the env password", tc.what, got)
|
||||
}
|
||||
if s, err := watchdog.LoadState(statePath); err != nil || s.Relay == nil || s.Relay.Host != "smtp.config.example" {
|
||||
t.Errorf("%s: the state caches relay %+v (%v), want the configured one", tc.what, s.Relay, err)
|
||||
}
|
||||
}
|
||||
if strings.Contains(tc.wantOut, "watchdog/state") {
|
||||
if aside, _ := filepath.Glob(statePath + ".unreadable-*"); len(aside) != 1 {
|
||||
t.Errorf("%s: moved aside %v, want one file", tc.what, aside)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user