feat(cli): 新增 felis status、doctor 和 support-bundle，一屏看全平台、按区域列出问题、打出脱敏诊断包
This commit is contained in:
12 files changed
+3237
-86
No files matched your search
@@ -0,0 +1,423 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/config"
|
||||
"felis.lolicon.best/internal/watchdog"
|
||||
)
|
||||
|
||||
// bootstrapWatchdogExecStart is the ExecStart= line deploy/bootstrap.sh writes
|
||||
// into felis-watchdog.service, with its variables filled in as an install
|
||||
// fills them.
|
||||
func bootstrapWatchdogExecStart(t *testing.T, vars map[string]string) string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../deploy/bootstrap.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var line string
|
||||
for _, l := range strings.Split(string(raw), "\n") {
|
||||
if strings.HasPrefix(l, "ExecStart=${HOST_BIN} watchdog -config") {
|
||||
line = l
|
||||
}
|
||||
}
|
||||
if line == "" {
|
||||
t.Fatal("deploy/bootstrap.sh writes no `ExecStart=${HOST_BIN} watchdog -config` line")
|
||||
}
|
||||
line = strings.ReplaceAll(line, "${NODE_IP:+ -node-ip ${NODE_IP}}", " -node-ip "+vars["NODE_IP"])
|
||||
line = regexp.MustCompile(`\$\{([A-Za-z_]+)\}`).ReplaceAllStringFunc(line, func(m string) string {
|
||||
v, ok := vars[m[2:len(m)-1]]
|
||||
if !ok {
|
||||
t.Fatalf("bootstrap's watchdog ExecStart= uses %s, which this test does not fill in", m)
|
||||
}
|
||||
return v
|
||||
})
|
||||
return line
|
||||
}
|
||||
|
||||
// felis doctor reads the watchdog's settings from the unit the installer
|
||||
// writes, so it checks the paths the timer's runs check.
|
||||
func TestWatchdogUnitFlagsReadsTheInstallersUnit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
exec := bootstrapWatchdogExecStart(t, map[string]string{
|
||||
"HOST_BIN": "/usr/local/bin/felis", "STATE_DIR": "/srv/felis-etc", "WATCHDOG_STATE": "/srv/watchdog/state.json",
|
||||
"WATCHDOG_QUIET_FILE": "/srv/quiet-until", "FELIS_DB_BACKUP_DIR": "/srv/db-backups", "FELIS_GAME_PORT": "25577",
|
||||
"disks": "/,/srv/data", "NODE_IP": "10.0.0.5",
|
||||
})
|
||||
unit := filepath.Join(dir, "felis-watchdog.service")
|
||||
writeTestFile(t, unit, "[Unit]\nDescription=Felis watchdog\n\n[Service]\nType=oneshot\n"+exec+"\nTimeoutStartSec=3min\n", 0o644)
|
||||
|
||||
w, found, err := watchdogUnitFlags(unit)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("found %v, err %v", found, err)
|
||||
}
|
||||
got := []string{w.cfgPath, w.statePath, w.quietPath, w.backupDir, w.proxyAddr, w.diskPaths, w.nodeIP, w.heartbeatFile, w.controlNS}
|
||||
want := []string{"/srv/felis-etc/felis.host.toml", "/srv/watchdog/state.json", "/srv/quiet-until", "/srv/db-backups", "127.0.0.1:25577", "/,/srv/data", "10.0.0.5", defaultHeartbeatFile, "felis"}
|
||||
if strings.Join(got, "|") != strings.Join(want, "|") {
|
||||
t.Errorf("read\n %q\nwant\n %q", got, want)
|
||||
}
|
||||
|
||||
w, found, err = watchdogUnitFlags(filepath.Join(dir, "missing.service"))
|
||||
if err != nil || found || w.cfgPath != "/etc/felis/felis.toml" || w.backupDir != "/var/lib/felis/db-backups" {
|
||||
t.Errorf("no unit: found %v, err %v, config %q, backups %q; want the watchdog's defaults", found, err, w.cfgPath, w.backupDir)
|
||||
}
|
||||
|
||||
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis version\n", 0o644)
|
||||
if _, found, err := watchdogUnitFlags(unit); !found || err == nil || !strings.Contains(err.Error(), "runs no `felis watchdog`") {
|
||||
t.Errorf("a unit that runs something else: found %v, err %v", found, err)
|
||||
}
|
||||
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
|
||||
if _, _, err := watchdogUnitFlags(unit); err == nil {
|
||||
t.Error("a flag this binary does not know was accepted")
|
||||
}
|
||||
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -backup-dir \"\"\t-proxy-addr '127.0.0.1:1' -disk-paths \"/a b\"\n", 0o644)
|
||||
if w, _, err := watchdogUnitFlags(unit); err != nil || w.backupDir != "" || w.proxyAddr != "127.0.0.1:1" || w.diskPaths != "/a b" {
|
||||
t.Errorf("quoted words: backups %q, proxy %q, disks %q, err %v", w.backupDir, w.proxyAddr, w.diskPaths, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindingArea(t *testing.T) {
|
||||
for key, want := range map[string]string{
|
||||
"kube-api": "cluster",
|
||||
"deployment/felis-api": "cluster",
|
||||
"system-server/lobby": "cluster",
|
||||
"server-failed/survival": "cluster",
|
||||
"job-failed/reaper-123": "cluster",
|
||||
"reaper-stale": "cluster",
|
||||
"node/felis-1/NotReady": "cluster",
|
||||
"postgres": "postgres",
|
||||
"proxy": "proxy",
|
||||
"db-backup": "db-backup",
|
||||
"db-backup-servers": "db-backup",
|
||||
"offsite": "offsite",
|
||||
"scan-db": "scan-db",
|
||||
"disk//var/lib/felis": "disk",
|
||||
"memory": "memory",
|
||||
"k3s-certs": "k3s-certs",
|
||||
"host-address": "host-address",
|
||||
"clock": "clock",
|
||||
"config": "config",
|
||||
"unit/felis-offsite.service": "systemd",
|
||||
"timer/felis-db-backup.timer": "systemd",
|
||||
"watchdog/unit": "alerts",
|
||||
"watchdog/heartbeat": "alerts",
|
||||
"alerts/relay": "alerts",
|
||||
"alerts/recipients": "alerts",
|
||||
"something-a-later-release-reported": "something-a-later-release-reported",
|
||||
} {
|
||||
if got := findingArea(key); got != want {
|
||||
t.Errorf("findingArea(%q) = %q, want %q", key, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertReachFindings(t *testing.T) {
|
||||
relay := &config.Config{SMTP: config.SMTPConfig{Host: "smtp.example.com"}}
|
||||
keys := func(fs []watchdog.Finding) string {
|
||||
var k []string
|
||||
for _, f := range fs {
|
||||
k = append(k, f.Key)
|
||||
}
|
||||
return strings.Join(k, ",")
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
what string
|
||||
cfg *config.Config
|
||||
owners []string
|
||||
ownersErr error
|
||||
want string
|
||||
}{
|
||||
{"a relay and an owner", relay, []string{"[email protected]"}, nil, ""},
|
||||
{"no relay", &config.Config{}, []string{"[email protected]"}, nil, "alerts/relay"},
|
||||
{"no owner with an address", relay, nil, nil, "alerts/recipients"},
|
||||
{"PostgreSQL down: its own finding says so", relay, nil, errors.New("refused"), ""},
|
||||
{"neither", &config.Config{}, nil, nil, "alerts/relay,alerts/recipients"},
|
||||
} {
|
||||
if got := keys(alertReachFindings(tc.cfg, tc.owners, tc.ownersErr)); got != tc.want {
|
||||
t.Errorf("%s: %q, want %q", tc.what, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fakeSystemctl answers list-units with failed and is-active from states;
|
||||
// a unit missing from states is "inactive", as systemctl says, and one whose
|
||||
// state is "" gets no answer.
|
||||
func fakeSystemctl(failed string, states map[string]string) func(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
return func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
if name != "systemctl" || len(args) == 0 {
|
||||
return nil, errors.New("unexpected command " + name)
|
||||
}
|
||||
switch args[0] {
|
||||
case "list-units":
|
||||
return []byte(failed), nil
|
||||
case "is-active":
|
||||
if s, ok := states[args[1]]; ok && s == "" {
|
||||
return nil, errors.New("signal: killed")
|
||||
} else if ok {
|
||||
return []byte(s + "\n"), nil
|
||||
}
|
||||
return []byte("inactive\n"), errors.New("exit status 3")
|
||||
}
|
||||
return nil, errors.New("unexpected systemctl " + args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnitFindings(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, f := range []string{"k3s.service", "felis-velocity.service", "felis-db-backup.timer", "felis-offsite.timer", "felis-offsite.service"} {
|
||||
writeTestFile(t, filepath.Join(dir, f), "[Unit]\n", 0o644)
|
||||
}
|
||||
env := doctorEnv{unitDir: dir, run: fakeSystemctl(
|
||||
"felis-offsite.service loaded failed failed Felis off-site copy\nfelis-velocity.service loaded failed failed Velocity\n",
|
||||
map[string]string{"k3s.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "inactive"},
|
||||
)}
|
||||
var got []string
|
||||
for _, f := range unitFindings(context.Background(), env) {
|
||||
got = append(got, string(f.Severity)+" "+f.Key+": "+f.SummaryEN)
|
||||
}
|
||||
want := []string{
|
||||
"critical unit/felis-offsite.service: felis-offsite.service failed",
|
||||
"critical unit/felis-velocity.service: felis-velocity.service failed",
|
||||
"warning timer/felis-offsite.timer: felis-offsite.timer is inactive: the job it starts no longer runs",
|
||||
}
|
||||
if strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("findings:\n%s\nwant (felis-nano.service has no unit file here, and a failed unit is reported once):\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
|
||||
env.run = fakeSystemctl("", map[string]string{"k3s.service": "activating", "felis-velocity.service": "active", "felis-db-backup.timer": "active", "felis-offsite.timer": "active"})
|
||||
got = nil
|
||||
for _, f := range unitFindings(context.Background(), env) {
|
||||
got = append(got, f.Key+": "+f.SummaryEN)
|
||||
}
|
||||
if strings.Join(got, "\n") != "unit/k3s.service: k3s.service is activating" {
|
||||
t.Errorf("findings %q, want only k3s.service, which is not active", got)
|
||||
}
|
||||
|
||||
env.run = fakeSystemctl("", map[string]string{"k3s.service": "active", "felis-velocity.service": "active", "felis-db-backup.timer": "", "felis-offsite.timer": "active"})
|
||||
got = nil
|
||||
for _, f := range unitFindings(context.Background(), env) {
|
||||
got = append(got, f.Key+": "+f.SummaryEN)
|
||||
}
|
||||
if want := "timer/felis-db-backup.timer: felis-db-backup.timer is in an unknown state (systemctl is-active: signal: killed): the job it starts no longer runs"; strings.Join(got, "\n") != want {
|
||||
t.Errorf("findings %q, want %q", got, want)
|
||||
}
|
||||
|
||||
env.run = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("no systemctl") }
|
||||
if fs := unitFindings(context.Background(), env); len(fs) != 1 || fs[0].Key != "unit/systemctl" || fs[0].Severity != watchdog.Warning {
|
||||
t.Errorf("without systemctl: %+v, want the one unit/systemctl warning", fs)
|
||||
}
|
||||
}
|
||||
|
||||
// quoteArgs writes args as an ExecStart= line does, each word in quotes so an
|
||||
// empty one survives.
|
||||
func quoteArgs(args []string) string {
|
||||
q := make([]string, len(args))
|
||||
for i, a := range args {
|
||||
q[i] = `"` + a + `"`
|
||||
}
|
||||
return strings.Join(q, " ")
|
||||
}
|
||||
|
||||
// doctorHost is a host with the installer's watchdog unit, whose API server
|
||||
// and PostgreSQL are down.
|
||||
func doctorHost(t *testing.T, cfg string, extraFlags string) (env doctorEnv, h *watchdogHost) {
|
||||
t.Helper()
|
||||
h = newWatchdogHost(t, cfg, nil)
|
||||
unitDir := t.TempDir()
|
||||
writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.service"),
|
||||
"[Service]\nType=oneshot\nExecStart=/usr/local/bin/felis watchdog "+quoteArgs(h.args)+
|
||||
// Off this machine's disk, whose free space is not the test's.
|
||||
` -disk-paths "/nonexistent-felis-doctor-test"`+extraFlags+"\n", 0o644)
|
||||
writeTestFile(t, filepath.Join(unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
|
||||
writeTestFile(t, filepath.Join(unitDir, "felis-offsite.timer"), "[Unit]\n", 0o644)
|
||||
return doctorEnv{
|
||||
unitDir: unitDir, now: time.Now(), host: "felis-test",
|
||||
run: fakeSystemctl("felis-db-backup.service loaded failed failed Felis database backup\n",
|
||||
map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"}),
|
||||
}, h
|
||||
}
|
||||
|
||||
func TestDoctorReportsByArea(t *testing.T) {
|
||||
env, _ := doctorHost(t, testWatchdogConfig, "")
|
||||
var out bytes.Buffer
|
||||
code := runDoctor(context.Background(), env, &out)
|
||||
got := out.String()
|
||||
if code != 1 {
|
||||
t.Errorf("exit %d, want 1 with problems found", code)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"felis doctor on felis-test at ",
|
||||
"checks run as " + filepath.Join(env.unitDir, "felis-watchdog.service") + " runs them (config ",
|
||||
"✓ configuration\n",
|
||||
"✗ Kubernetes cluster\n critical kube-api: ",
|
||||
"✗ PostgreSQL\n critical postgres: ",
|
||||
"- game proxy: not checked, no -proxy-addr\n",
|
||||
"- database backups: not checked, no -backup-dir\n",
|
||||
"- off-site copy: not checked, not configured\n",
|
||||
"- build scan database: not checked, builds do not scan against the registry's copy\n",
|
||||
"- k3s certificates: not checked, no -k3s-cert-dirs\n",
|
||||
"- node address: not checked, no -node-ip\n",
|
||||
"✗ systemd units and timers\n critical unit/felis-db-backup.service: felis-db-backup.service failed\n" +
|
||||
" → journalctl -u felis-db-backup.service -n 100 --no-pager; once fixed, sudo systemctl reset-failed felis-db-backup.service",
|
||||
"! alerting\n warning alerts/relay: no [smtp] relay is configured",
|
||||
"\n4 problem(s): 3 critical, 1 warning(s)\n",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("report lacks %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, "alerts/recipients") {
|
||||
t.Errorf("reported no recipients although PostgreSQL, which names them, is down:\n%s", got)
|
||||
}
|
||||
if strings.Contains(got, "note: no heartbeat URL") {
|
||||
t.Errorf("the host has a heartbeat URL:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A doctor run is only a look: whatever is due to be mailed stays due, no
|
||||
// heartbeat is pinged, and the watchdog's state is left as it was.
|
||||
func TestDoctorMailsPingsAndSavesNothing(t *testing.T) {
|
||||
cfg := testWatchdogConfig + "[smtp]\nhost = \"smtp.example.com\"\nport = 587\nfrom = \"[email protected]\"\n"
|
||||
env, h := doctorHost(t, cfg, "")
|
||||
if err := watchdog.SaveState(h.statePath, duePostgres("cached-pw")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := os.ReadFile(h.statePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
watchdogSender = h.rec.sender
|
||||
defer func() { watchdogSender = smtpSender }()
|
||||
var out bytes.Buffer
|
||||
runDoctor(context.Background(), env, &out)
|
||||
if !strings.Contains(out.String(), "critical postgres: ") {
|
||||
t.Fatalf("the due PostgreSQL alert was not seen:\n%s", out.String())
|
||||
}
|
||||
if len(h.rec.sent) != 0 || len(h.rec.relays) != 0 {
|
||||
t.Errorf("mailed %v", h.rec.sent)
|
||||
}
|
||||
if n := len(h.pings.pings); n != 0 {
|
||||
t.Errorf("pinged the heartbeat %d times", n)
|
||||
}
|
||||
after, err := os.ReadFile(h.statePath)
|
||||
if err != nil || !bytes.Equal(before, after) {
|
||||
t.Errorf("the watchdog state changed (err %v)", err)
|
||||
}
|
||||
if _, err := os.Stat(h.fallbackPath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Errorf("a fallback state was written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoctorNotes(t *testing.T) {
|
||||
env, h := doctorHost(t, testWatchdogConfig, "")
|
||||
if err := os.Remove(filepath.Join(h.dir, "watchdog-heartbeat-url")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
until := env.now.Add(10 * time.Minute).Unix()
|
||||
writeTestFile(t, filepath.Join(h.dir, "quiet"), strconv.FormatInt(until, 10)+"\n", 0o644)
|
||||
var out bytes.Buffer
|
||||
runDoctor(context.Background(), env, &out)
|
||||
for _, want := range []string{
|
||||
"\nnote: no heartbeat URL is set: ",
|
||||
"\nnote: the watchdog mails nothing until " + time.Unix(until, 0).UTC().Format("2006-01-02 15:04 UTC") + " (" + filepath.Join(h.dir, "quiet") + ")",
|
||||
} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("report lacks %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
writeTestFile(t, filepath.Join(h.dir, "watchdog-heartbeat-url"), "not a url\n", 0o600)
|
||||
out.Reset()
|
||||
runDoctor(context.Background(), env, &out)
|
||||
if !strings.Contains(out.String(), "warning watchdog/heartbeat: the heartbeat URL is unusable") {
|
||||
t.Errorf("an unusable heartbeat URL is not reported:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Without the watchdog's unit the doctor still checks, with the watchdog's
|
||||
// defaults, and says the host has no watchdog; a configuration that does not
|
||||
// load leaves the checks that need it unchecked and the host's own checks on.
|
||||
func TestDoctorWithoutUnitOrConfig(t *testing.T) {
|
||||
env, _ := doctorHost(t, testWatchdogConfig, "")
|
||||
if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeTestFile(t, filepath.Join(env.unitDir, "felis-watchdog.service"), "[Service]\nExecStart=/usr/local/bin/felis watchdog -config "+filepath.Join(t.TempDir(), "absent.toml")+"\n", 0o644)
|
||||
env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active", "felis-offsite.timer": "active"})
|
||||
var out bytes.Buffer
|
||||
if code := runDoctor(context.Background(), env, &out); code != 1 {
|
||||
t.Errorf("exit %d, want 1", code)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"✗ configuration\n critical config: ",
|
||||
"- Kubernetes cluster: not checked, the configuration did not load\n",
|
||||
"- PostgreSQL: not checked, the configuration did not load\n",
|
||||
"- disk space: not checked, the configuration did not load\n",
|
||||
"✓ systemd units and timers\n",
|
||||
"- alerting: not checked, the configuration did not load\n",
|
||||
} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("report lacks %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.service")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out.Reset()
|
||||
runDoctor(context.Background(), env, &out)
|
||||
if !strings.Contains(out.String(), "critical watchdog/unit: "+filepath.Join(env.unitDir, "felis-watchdog.service")+" is not installed") ||
|
||||
!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
|
||||
t.Errorf("a host without the watchdog's unit:\n%s", out.String())
|
||||
}
|
||||
|
||||
unit := filepath.Join(env.unitDir, "felis-watchdog.service")
|
||||
writeTestFile(t, unit, "[Service]\nExecStart=/usr/local/bin/felis watchdog -no-such-flag x\n", 0o644)
|
||||
out.Reset()
|
||||
runDoctor(context.Background(), env, &out)
|
||||
if !strings.Contains(out.String(), "critical watchdog/unit: cannot read the watchdog's settings: "+unit+": flag provided but not defined: -no-such-flag\n") ||
|
||||
!strings.Contains(out.String(), "checks run with the watchdog's defaults (config /etc/felis/felis.toml)") {
|
||||
t.Errorf("a watchdog unit this binary cannot read:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintDoctorReport(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if code := printDoctorReport(&out, nil, map[string]string{"proxy": "no -proxy-addr"}, nil); code != 0 {
|
||||
t.Errorf("exit %d with nothing found, want 0", code)
|
||||
}
|
||||
want := "✓ configuration\n✓ Kubernetes cluster\n✓ PostgreSQL\n- game proxy: not checked, no -proxy-addr\n✓ database backups\n✓ off-site copy\n" +
|
||||
"✓ build scan database\n✓ disk space\n✓ memory\n✓ k3s certificates\n✓ node address\n✓ clock\n✓ systemd units and timers\n✓ alerting\n" +
|
||||
"\nno problems found\n"
|
||||
if out.String() != want {
|
||||
t.Errorf("report:\n%s\nwant:\n%s", out.String(), want)
|
||||
}
|
||||
|
||||
out.Reset()
|
||||
code := printDoctorReport(&out, []watchdog.Finding{
|
||||
{Key: "unit/systemctl", Severity: watchdog.Warning, SummaryEN: "systemctl list-units failed"},
|
||||
{Key: "postgres", Severity: watchdog.Critical, SummaryEN: "PostgreSQL is down", Hint: "kubectl -n felis get pods"},
|
||||
}, map[string]string{"postgres": "a skip loses to what was found"}, []string{"a note"})
|
||||
if code != 1 {
|
||||
t.Errorf("exit %d with problems, want 1", code)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"✗ PostgreSQL\n critical postgres: PostgreSQL is down\n → kubectl -n felis get pods\n✓ game proxy\n",
|
||||
"! systemd units and timers\n warning unit/systemctl: systemctl list-units failed\n✓ alerting\n",
|
||||
"✓ alerting\n\nnote: a note\n\n2 problem(s): 1 critical, 1 warning(s)\n",
|
||||
} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("report lacks %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user