fix(watchdog): 外部心跳、巡检失败兜底告警、状态文件损坏自动移开

internal/watchdog/probes.go

@@ -38,6 +38,8 @@ const (
 	diskLowFor      = 15 * time.Minute
 	diskCriticalFor = 5 * time.Minute
 	memoryLowFor    = 15 * time.Minute
+	// watchdogFailedFor is five failed runs in a row.
+	watchdogFailedFor = 10 * time.Minute
 
 	// jobFailureWindow is how far back a failed Job is still news.
 	jobFailureWindow = 24 * time.Hour
@@ -292,6 +294,30 @@ func nodeFindings(n *corev1.Node) []Finding {
 	return out
 }
 
+// WatchdogFailed is the watchdog's own run failing (felis-watchdog-failed.service,
+// through OnFailure=): while it fails no other check runs and nothing else is
+// mailed. detail is how the run ended. It waits watchdogFailedFor, so one run cut
+// short by a slow host mails nobody.
+func WatchdogFailed(detail string) Finding {
+	return Finding{
+		Key: "watchdog/run", Severity: Critical, For: watchdogFailedFor,
+		Summary:   "平台巡检本身运行失败，其他检查都没有执行，出了别的问题也不会有告警：" + detail,
+		SummaryEN: "the platform watchdog itself fails, so no other check runs and nothing else is mailed: " + detail,
+		Hint:      "journalctl -u felis-watchdog -n 50; sudo felis watchdog -dry-run (docs/troubleshooting.md §14)",
+	}
+}
+
+// StateSetAside is a state file RecoverState moved aside: the alerts it held
+// start over, so a condition still firing is mailed again as new.
+func StateSetAside(aside string) Finding {
+	return Finding{
+		Key: "watchdog/state", Severity: Warning, Event: true,
+		Summary:   fmt.Sprintf("平台巡检的状态文件无法读取，已移到 %s 并重新开始：仍未恢复的告警会当作新告警再发一次", aside),
+		SummaryEN: fmt.Sprintf("the watchdog's state file was unreadable and was moved to %s; its alerts start over, so one still firing is mailed again as new", aside),
+		Hint:      "df -h /var/lib/felis (a full disk cuts writes short)",
+	}
+}
+
 // KubeAPIDown is the finding for an API server that did not answer.
 func KubeAPIDown(err error) Finding {
 	return Finding{

internal/watchdog/watchdog.go

@@ -83,6 +83,30 @@ type State struct {
 	// still be mailed.
 	Recipients   []string `json:"recipients,omitempty"`
 	SMTPPassword string   `json:"smtp_password,omitempty"`
+	// Relay is the [smtp] relay of the last run that could read felis.toml,
+	// so a run of the watchdog that failed can still be mailed after the
+	// config stopped loading; nil when there is none.
+	Relay *Relay `json:"relay,omitempty"`
+}
+
+// Relay is the part of [smtp] a mail needs, besides the password.
+type Relay struct {
+	Host       string `json:"host"`
+	Port       int    `json:"port"`
+	From       string `json:"from"`
+	Username   string `json:"username,omitempty"`
+	RequireTLS bool   `json:"require_tls"`
+}
+
+// Open reports whether the owners were told of a condition that still holds:
+// an alert mailed (or logged) and not seen gone since, one-off events aside.
+func (s *State) Open() bool {
+	for _, a := range s.Alerts {
+		if !a.Notified.IsZero() && a.ClearedAt.IsZero() && !a.Event {
+			return true
+		}
+	}
+	return false
 }
 
 const (
@@ -276,6 +300,9 @@ func without(all []Alert, skip ...[]Alert) []Alert {
 	return out
 }
 
+// ErrBadState is a state file that is not the JSON the watchdog writes.
+var ErrBadState = errors.New("watchdog: the state file is not one the watchdog wrote")
+
 // LoadState reads the state file; a missing file is a fresh state.
 func LoadState(path string) (*State, error) {
 	raw, err := os.ReadFile(path)
@@ -287,7 +314,7 @@ func LoadState(path string) (*State, error) {
 	}
 	var s State
 	if err := json.Unmarshal(raw, &s); err != nil {
-		return nil, fmt.Errorf("parse %s: %w", path, err)
+		return nil, fmt.Errorf("%w: %s: %v", ErrBadState, path, err)
 	}
 	if s.Alerts == nil {
 		s.Alerts = map[string]*Alert{}
@@ -295,6 +322,22 @@ func LoadState(path string) (*State, error) {
 	return &s, nil
 }
 
+// RecoverState is LoadState for a run that must go on: a state file that does
+// not parse (a disk that filled mid-write, a hand edit) is renamed aside and
+// the run starts from a fresh state, rather than every later run failing on
+// it and mailing nothing. aside is where it went, "" when nothing was moved.
+func RecoverState(path string, now time.Time) (s *State, aside string, err error) {
+	s, err = LoadState(path)
+	if !errors.Is(err, ErrBadState) {
+		return s, "", err
+	}
+	aside = fmt.Sprintf("%s.unreadable-%d", path, now.Unix())
+	if rerr := os.Rename(path, aside); rerr != nil {
+		return nil, "", fmt.Errorf("%w (and could not move it aside: %v)", err, rerr)
+	}
+	return &State{Alerts: map[string]*Alert{}}, aside, nil
+}
+
 // SaveState writes s atomically, readable by root only: it caches the relay
 // password.
 func SaveState(path string, s *State) error {

internal/watchdog/watchdog_test.go

@@ -1,6 +1,7 @@
 package watchdog
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -205,3 +206,65 @@ func TestQuietUntil(t *testing.T) {
 		t.Errorf("QuietUntil = %v", got)
 	}
 }
+
+// TestRecoverState: a state file that does not parse is moved aside, whole,
+// and the run starts over; a good or missing one is loaded as LoadState does.
+func TestRecoverState(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "state.json")
+	if err := os.WriteFile(path, []byte(`{"alerts": {"memo`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	s, aside, err := RecoverState(path, t0)
+	if err != nil || s == nil || s.Alerts == nil || len(s.Alerts) != 0 {
+		t.Fatalf("RecoverState(bad) = %+v, %q, %v; want a fresh state", s, aside, err)
+	}
+	if want := path + ".unreadable-" + fmt.Sprint(t0.Unix()); aside != want {
+		t.Fatalf("aside = %q, want %q", aside, want)
+	}
+	if raw, err := os.ReadFile(aside); err != nil || string(raw) != `{"alerts": {"memo` {
+		t.Fatalf("the moved file = %q, %v; want the bad state kept as it was", raw, err)
+	}
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		t.Fatalf("the bad state is still at %s (%v)", path, err)
+	}
+
+	good := &State{Recipients: []string{"[email protected]"}}
+	run(good, Report{Findings: []Finding{finding("memory", Warning, 0)}}, t0)
+	if err := SaveState(path, good); err != nil {
+		t.Fatal(err)
+	}
+	s, aside, err = RecoverState(path, t0)
+	if err != nil || aside != "" || s.Alerts["memory"] == nil || len(s.Recipients) != 1 {
+		t.Fatalf("RecoverState(good) = %+v, %q, %v", s, aside, err)
+	}
+	s, aside, err = RecoverState(filepath.Join(dir, "missing.json"), t0)
+	if err != nil || aside != "" || s.Alerts == nil {
+		t.Fatalf("RecoverState(missing) = %+v, %q, %v", s, aside, err)
+	}
+}
+
+// TestStateOpen: open is a condition the owners were told of that still holds.
+func TestStateOpen(t *testing.T) {
+	s := &State{}
+	f := finding("memory", Warning, time.Hour)
+	run(s, Report{Findings: []Finding{f}}, t0)
+	if s.Open() {
+		t.Fatal("a pending alert, never mailed, counts as open")
+	}
+	run(s, Report{Findings: []Finding{f}}, t0.Add(time.Hour))
+	if !s.Open() {
+		t.Fatal("a mailed alert still firing is not open")
+	}
+	run(s, Report{}, t0.Add(time.Hour+time.Minute))
+	if s.Open() {
+		t.Fatal("an alert seen gone counts as open")
+	}
+	ev := finding("watchdog/state", Warning, 0)
+	ev.Event = true
+	e := &State{}
+	run(e, Report{Findings: []Finding{ev}}, t0)
+	if e.Open() {
+		t.Fatal("a one-off event counts as open")
+	}
+}