package watchdog import ( "os" "path/filepath" "strings" "testing" "time" ) var t0 = time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC) func finding(key string, sev Severity, forDur time.Duration) Finding { return Finding{Key: key, Severity: sev, For: forDur, Summary: key + " 中文", SummaryEN: key + " en"} } func keys(l []Alert) []string { var out []string for _, a := range l { out = append(out, a.Key) } return out } // run observes r at now and commits whatever came due, like a run whose mail // went out. func run(s *State, r Report, now time.Time) Plan { p := s.Observe(r, now) s.Commit(p, now) return p } // TestObserveLifecycle walks one condition through pending, firing, the daily // reminder, clearing, and the resolved notice after it stayed clear. func TestObserveLifecycle(t *testing.T) { s := &State{} f := finding("deployment/felis-api", Critical, 5*time.Minute) down := Report{Findings: []Finding{f}} if p := run(s, down, t0); !p.Empty() { t.Fatalf("first sight mailed %+v, want it pending for 5m", p) } if p := run(s, down, t0.Add(4*time.Minute)); !p.Empty() { t.Fatalf("4m in mailed %v", keys(p.Firing)) } p := run(s, down, t0.Add(6*time.Minute)) if len(p.Firing) != 1 || p.Firing[0].Key != f.Key || !p.Firing[0].FirstSeen.Equal(t0) { t.Fatalf("6m in: firing = %+v, want %s since t0", p.Firing, f.Key) } if p := run(s, down, t0.Add(8*time.Minute)); !p.Empty() { t.Fatalf("already mailed, mailed again: %+v", p) } p = run(s, down, t0.Add(6*time.Minute+remindEvery)) if len(p.Reminders) != 1 || len(p.Firing) != 0 { t.Fatalf("a day later: %+v, want one reminder", p) } up := Report{} later := t0.Add(7*time.Minute + remindEvery) if p := run(s, up, later); !p.Empty() { t.Fatalf("just cleared, mailed %+v; want resolveAfter to pass first", p) } p = run(s, up, later.Add(resolveAfter)) if len(p.Resolved) != 1 || p.Resolved[0].Key != f.Key { t.Fatalf("resolved = %v, want %s", keys(p.Resolved), f.Key) } if len(s.Alerts) != 0 { t.Errorf("state after resolve = %v, want empty", s.Alerts) } } // TestObserveFlapWithinResolveWindow: a condition that returns before // resolveAfter is neither resolved nor mailed as new. func TestObserveFlapWithinResolveWindow(t *testing.T) { s := &State{} f := finding("disk//", Warning, 0) run(s, Report{Findings: []Finding{f}}, t0) run(s, Report{}, t0.Add(time.Minute)) if p := run(s, Report{Findings: []Finding{f}}, t0.Add(3*time.Minute)); !p.Empty() { t.Fatalf("flap mailed %+v", p) } if a := s.Alerts[f.Key]; a == nil || !a.ClearedAt.IsZero() { t.Fatalf("alert after flap = %+v, want firing again", a) } } // TestObservePendingNeverMailed: a condition that heals inside its For is // dropped without a word. func TestObservePendingNeverMailed(t *testing.T) { s := &State{} run(s, Report{Findings: []Finding{finding("deployment/registry", Critical, 5*time.Minute)}}, t0) if p := run(s, Report{}, t0.Add(2*time.Minute)); !p.Empty() || len(s.Alerts) != 0 { t.Fatalf("healed pending alert: plan %+v state %v", p, s.Alerts) } } // TestObserveEscalation: a warning that turns critical is mailed again at once. func TestObserveEscalation(t *testing.T) { s := &State{} run(s, Report{Findings: []Finding{finding("disk//", Warning, 0)}}, t0) p := run(s, Report{Findings: []Finding{finding("disk//", Critical, 5*time.Minute)}}, t0.Add(time.Minute)) if len(p.Firing) != 1 || p.Firing[0].Severity != Critical { t.Fatalf("escalation = %+v, want the critical finding mailed", p.Firing) } if p := run(s, Report{Findings: []Finding{finding("disk//", Critical, 5*time.Minute)}}, t0.Add(2*time.Minute)); !p.Empty() { t.Fatalf("critical mailed twice: %+v", p) } } // TestObserveEventOnce: a failed Job is mailed once, never reminded, and leaves // without a resolved notice. func TestObserveEventOnce(t *testing.T) { s := &State{} ev := finding("job-failed/backup-survival-x", Warning, 0) ev.Event = true if p := run(s, Report{Findings: []Finding{ev}}, t0); len(p.Firing) != 1 { t.Fatalf("event not mailed: %+v", p) } if p := run(s, Report{Findings: []Finding{ev}}, t0.Add(remindEvery+time.Hour)); !p.Empty() { t.Fatalf("event reminded: %+v", p) } if p := run(s, Report{}, t0.Add(remindEvery+2*time.Hour)); !p.Empty() || len(s.Alerts) != 0 { t.Fatalf("event gone: plan %+v state %v, want silent removal", p, s.Alerts) } } // TestObserveUnknownCarriesOver: while the API server is down, cluster alerts // neither resolve nor restart their clocks. func TestObserveUnknownCarriesOver(t *testing.T) { s := &State{} f := finding("system-server/login", Critical, 0) run(s, Report{Findings: []Finding{f}}, t0) api := finding("kube-api", Critical, 5*time.Minute) for i := 1; i <= 3; i++ { p := run(s, Report{Findings: []Finding{api}, Unknown: ClusterPrefixes}, t0.Add(time.Duration(i)*resolveAfter)) if len(p.Resolved) != 0 { t.Fatalf("run %d resolved %v while the API was down", i, keys(p.Resolved)) } } if a := s.Alerts[f.Key]; a == nil || !a.ClearedAt.IsZero() { t.Fatalf("login alert = %+v, want still firing", a) } } // TestObserveUncommittedRetries: a plan whose mail failed comes due again. func TestObserveUncommittedRetries(t *testing.T) { s := &State{} f := finding("postgres", Critical, 0) if p := s.Observe(Report{Findings: []Finding{f}}, t0); len(p.Firing) != 1 { t.Fatalf("not due: %+v", p) } if p := s.Observe(Report{Findings: []Finding{f}}, t0.Add(2*time.Minute)); len(p.Firing) != 1 || !p.Firing[0].FirstSeen.Equal(t0) { t.Fatalf("after a failed send: %+v, want the same alert due again since t0", p.Firing) } } func TestMessage(t *testing.T) { s := &State{} run(s, Report{Findings: []Finding{finding("memory", Warning, 0)}}, t0) p := run(s, Report{Findings: []Finding{finding("memory", Warning, 0), finding("postgres", Critical, 0)}}, t0.Add(time.Minute)) subject, body := p.Message("node-1", t0.Add(time.Minute)) for _, want := range []string{"严重告警", "node-1", "1 项异常", "1 firing"} { if !strings.Contains(subject, want) { t.Errorf("subject %q lacks %q", subject, want) } } for _, want := range []string{"postgres 中文", "postgres en", "== 其他仍在进行的告警 / also still firing ==", "memory 中文 / memory en", "journalctl -u felis-watchdog"} { if !strings.Contains(body, want) { t.Errorf("body lacks %q:\n%s", want, body) } } } func TestStateRoundTripPrivate(t *testing.T) { path := filepath.Join(t.TempDir(), "watchdog", "state.json") s := &State{SMTPPassword: "secret", Recipients: []string{"owner@example.com"}} run(s, Report{Findings: []Finding{finding("memory", Warning, time.Hour)}}, t0) if err := SaveState(path, s); err != nil { t.Fatalf("SaveState: %v", err) } info, err := os.Stat(path) if err != nil || info.Mode().Perm() != 0o600 { t.Fatalf("state file mode = %v (%v), want 0600", info.Mode(), err) } got, err := LoadState(path) if err != nil || got.SMTPPassword != "secret" || !got.Alerts["memory"].FirstSeen.Equal(t0) { t.Fatalf("LoadState = %+v, %v", got, err) } fresh, err := LoadState(filepath.Join(t.TempDir(), "missing.json")) if err != nil || fresh.Alerts == nil { t.Fatalf("missing state = %+v, %v", fresh, err) } } func TestQuietUntil(t *testing.T) { dir := t.TempDir() marker := filepath.Join(dir, "quiet") if !QuietUntil(marker).IsZero() { t.Error("missing marker should mean no quiet period") } if err := os.WriteFile(marker, []byte("1790000000\n"), 0o644); err != nil { t.Fatal(err) } if got := QuietUntil(marker); got.Unix() != 1790000000 { t.Errorf("QuietUntil = %v", got) } }