fix(watchdog): 状态文件写不进时把状态留在 /run 下的后备文件，下一轮读较新的一份，同一封告警不再每 2 分钟重发

cmd/felis/watchdog.go

@@ -37,6 +37,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
 	statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")
+	fallbackState := fs.String("fallback-state", watchdog.FallbackStatePath, "where a run keeps its state while -state cannot be written, so what it mailed is not mailed again (tmpfs: until the host restarts; \"\" keeps none)")
 	smtpPasswordFile := fs.String("smtp-password-file", hostSMTPPasswordPath, "the relay password `felis setup` keeps on the host; the felis-smtp Secret stands in while it is missing")
 	quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
 	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
@@ -59,7 +60,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	now := time.Now()
 	if *unitFailed {
 		return watchdogUnitFailed(unitFailedRun{
-			cfgPath: *cfgPath, statePath: *statePath, quietPath: *quietPath,
+			cfgPath: *cfgPath, statePath: *statePath, fallbackPath: *fallbackState, quietPath: *quietPath,
 			offsiteStatus: *offsiteStatus, heartbeatFile: *heartbeatFile,
 			result: os.Getenv("MONITOR_SERVICE_RESULT"), exitStatus: os.Getenv("MONITOR_EXIT_STATUS"),
 			send: watchdogSender, client: http.DefaultClient, now: now,
@@ -70,7 +71,8 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis watchdog: %v\n", err)
 		return 1
 	}
-	state, aside, err := watchdog.RecoverState(*statePath, now)
+	loadPath := watchdog.NewestState(*statePath, *fallbackState)
+	state, aside, err := watchdog.RecoverState(loadPath, now)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis watchdog: %v\n", err)
 		return 1
@@ -85,7 +87,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 		}
 	}
 	if aside != "" {
-		fmt.Fprintf(stderr, "felis watchdog: %s was unreadable; moved it to %s and started over\n", *statePath, aside)
+		fmt.Fprintf(stderr, "felis watchdog: %s was unreadable; moved it to %s and started over\n", loadPath, aside)
 		f := watchdog.StateSetAside(aside)
 		add(&f)
 	}
@@ -176,7 +178,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 
 	m := configMailer(cfg.SMTP, state.SMTPPassword, watchdogSender)
 	unheard, mailFailed := m.deliver(ctx, state, plan, subject, body, mailHold(*quietPath, cfg.Offsite.Enabled(), *offsiteStatus, now), now, stdout, stderr)
-	saveErr := watchdog.SaveState(*statePath, state)
+	saveErr := watchdog.SaveStateOr(*statePath, *fallbackState, state)
 	if saveErr != nil {
 		fmt.Fprintf(stderr, "felis watchdog: save state: %v\n", saveErr)
 	}
@@ -192,7 +194,9 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 // failureReport is what the heartbeat's failure ping carries, "" when the run
 // pings success: the alerts this run knows of reach no one (a mail that
 // failed, or no relay or recipient while something is open), or the state did
-// not save and the next run mails the same alerts again.
+// not save to its file: kept on tmpfs it holds until the host restarts, which
+// forgets what was mailed, and with nowhere to keep it the next run mails the
+// same alerts again.
 func failureReport(unheard string, mailFailed bool, saveErr error, open bool, r watchdog.Report) string {
 	var why []string
 	if unheard != "" && (mailFailed || open) {
@@ -278,7 +282,7 @@ func smtpPassword(c config.SMTPConfig, cached string) string {
 
 // unitFailedRun is one run of felis-watchdog-failed.service.
 type unitFailedRun struct {
-	cfgPath, statePath, quietPath, offsiteStatus, heartbeatFile string
+	cfgPath, statePath, fallbackPath, quietPath, offsiteStatus, heartbeatFile string
 	// result and exitStatus are what systemd hands an OnFailure= unit
 	// (MONITOR_SERVICE_RESULT, MONITOR_EXIT_STATUS; systemd 251 and later).
 	result, exitStatus string
@@ -313,7 +317,7 @@ func watchdogUnitFailed(r unitFailedRun, stdout, stderr io.Writer) int {
 	if beat.url, err = readHeartbeatURL(r.heartbeatFile); err != nil {
 		fmt.Fprintf(stderr, "felis watchdog: %v; pinging no heartbeat\n", err)
 	}
-	state, err := watchdog.LoadState(r.statePath)
+	state, err := watchdog.LoadState(watchdog.NewestState(r.statePath, r.fallbackPath))
 	if err != nil {
 		// The next run that gets that far moves a state that does not parse
 		// aside (watchdog.RecoverState).
@@ -336,7 +340,7 @@ func watchdogUnitFailed(r unitFailedRun, stdout, stderr io.Writer) int {
 	if mailFailed {
 		code = 1
 	}
-	if err := watchdog.SaveState(r.statePath, state); err != nil {
+	if err := watchdog.SaveStateOr(r.statePath, r.fallbackPath, state); err != nil {
 		fmt.Fprintf(stderr, "felis watchdog: save state: %v\n", err)
 		code = 1
 	}

cmd/felis/watchdog_heartbeat_test.go

@@ -364,7 +364,7 @@ func unitFailedFixture(t *testing.T, cfg string) (unitFailedRun, *alertRecorder,
 	}
 	f := &alertRecorder{}
 	return unitFailedRun{
-		cfgPath: cfgPath, statePath: statePath, quietPath: filepath.Join(dir, "quiet"),
+		cfgPath: cfgPath, statePath: statePath, fallbackPath: filepath.Join(t.TempDir(), "watchdog-state.json"), quietPath: filepath.Join(dir, "quiet"),
 		offsiteStatus: filepath.Join(dir, "offsite-status.json"), heartbeatFile: beatFile,
 		result: "exit-code", exitStatus: "1", send: f.sender, client: srv.Client(), now: time.Now(),
 	}, f, log
@@ -409,6 +409,32 @@ func TestWatchdogUnitFailedBrokenConfig(t *testing.T) {
 	}
 }
 
+// TestWatchdogUnitFailedStateThatDoesNotSave: with the state file read-only,
+// each report keeps the state in the fallback and the next one reads it, so
+// the failure is mailed once, after five in a row, and never again.
+func TestWatchdogUnitFailedStateThatDoesNotSave(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("root writes into a read-only directory")
+	}
+	r, f, _ := unitFailedFixture(t, "[database\n")
+	dir := filepath.Dir(r.statePath)
+	if err := os.Chmod(dir, 0o500); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { os.Chmod(dir, 0o700) })
+	start := r.now
+	for i := 0; i <= 8; i++ {
+		r.now = start.Add(time.Duration(i) * 2 * time.Minute)
+		var stdout, stderr bytes.Buffer
+		if code := watchdogUnitFailed(r, &stdout, &stderr); code != 1 || !strings.Contains(stderr.String(), "; kept in "+r.fallbackPath) {
+			t.Fatalf("run %d: exit %d, stderr %s; want exit 1 and the state kept in the fallback", i, code, stderr.String())
+		}
+		if want := min(max(i-4, 0), 1); len(f.sent) != want {
+			t.Fatalf("run %d, %v after the first failure: mailed %d, want %d", i, r.now.Sub(start), len(f.sent), want)
+		}
+	}
+}
+
 // TestWatchdogUnitFailedConfigRelay: with felis.toml loading, the relay is
 // the configured one and signs in with the env var password_ref names.
 func TestWatchdogUnitFailedConfigRelay(t *testing.T) {

cmd/felis/watchdog_test.go

@@ -99,10 +99,12 @@ func TestMailHold(t *testing.T) {
 // the heartbeat URL points at a ping log.
 type watchdogHost struct {
 	dir, statePath string
-	args           []string
-	rec            *alertRecorder
-	pings          *pingLog
-	url            string
+	// fallbackPath stands in for /run/felis, in a directory of its own.
+	fallbackPath string
+	args         []string
+	rec          *alertRecorder
+	pings        *pingLog
+	url          string
 }
 
 func newWatchdogHost(t *testing.T, cfg string, state *watchdog.State) *watchdogHost {
@@ -110,7 +112,8 @@ func newWatchdogHost(t *testing.T, cfg string, state *watchdog.State) *watchdogH
 	dir := t.TempDir()
 	t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
 	pings, srv := newPingServer(t)
-	h := &watchdogHost{dir: dir, statePath: filepath.Join(dir, "state.json"), rec: &alertRecorder{}, pings: pings, url: srv.URL}
+	h := &watchdogHost{dir: dir, statePath: filepath.Join(dir, "state.json"), fallbackPath: filepath.Join(t.TempDir(), "watchdog-state.json"),
+		rec: &alertRecorder{}, pings: pings, url: srv.URL}
 	writeTestFile(t, filepath.Join(dir, "felis.toml"), cfg, 0o600)
 	writeTestFile(t, filepath.Join(dir, "watchdog-heartbeat-url"), srv.URL+"/check-key\n", 0o600)
 	if state != nil {
@@ -119,7 +122,7 @@ func newWatchdogHost(t *testing.T, cfg string, state *watchdog.State) *watchdogH
 		}
 	}
 	h.args = []string{
-		"-config", filepath.Join(dir, "felis.toml"), "-state", h.statePath, "-quiet-file", filepath.Join(dir, "quiet"),
+		"-config", filepath.Join(dir, "felis.toml"), "-state", h.statePath, "-fallback-state", h.fallbackPath, "-quiet-file", filepath.Join(dir, "quiet"),
 		"-backup-dir", "", "-disk-paths", dir, "-k3s-cert-dirs", "", "-smtp-password-file", filepath.Join(dir, "smtp-password"),
 		"-offsite-status", filepath.Join(dir, "offsite-status.json"), "-build-tools-status", filepath.Join(dir, "build-tools.json"),
 		"-heartbeat-file", filepath.Join(dir, "watchdog-heartbeat-url"),
@@ -292,8 +295,9 @@ func TestWatchdogRunChecksWhatIsConfigured(t *testing.T) {
 }
 
 // TestWatchdogRunStateThatDoesNotSave: a pass whose state does not save mails
-// as usual, exits 1 and posts the failure to the heartbeat, since the next run
-// mails the same alerts again.
+// as usual, keeps its state in the fallback, exits 1 and posts the failure to
+// the heartbeat. The next pass reads the fallback and mails nothing again; the
+// first one whose state saves drops the fallback.
 func TestWatchdogRunStateThatDoesNotSave(t *testing.T) {
 	if os.Geteuid() == 0 {
 		t.Skip("root writes into a read-only directory")
@@ -304,10 +308,24 @@ func TestWatchdogRunStateThatDoesNotSave(t *testing.T) {
 		t.Fatal(err)
 	}
 	t.Cleanup(func() { os.Chmod(h.dir, 0o700) })
+	kept := "; kept in " + h.fallbackPath + " until the host restarts"
+	for i := range 2 {
+		code, stdout, stderr := h.run()
+		pings, bodies := h.pings.got()
+		if code != 1 || len(h.rec.sent) != 1 || !strings.Contains(stderr, "felis watchdog: save state: ") || !strings.Contains(stderr, kept) ||
+			len(pings) != i+1 || pings[i] != "POST /check-key/fail" || !strings.HasPrefix(bodies[i], "the watchdog state did not save: ") {
+			t.Fatalf("pass %d: exit %d, mailed %v, pings %v %q; want exit 1, one mail in all and the save failure posted to /fail\nstdout %s\nstderr %s", i, code, h.rec.sent, pings, bodies, stdout, stderr)
+		}
+	}
+
+	if err := os.Chmod(h.dir, 0o700); err != nil {
+		t.Fatal(err)
+	}
 	code, stdout, stderr := h.run()
-	pings, bodies := h.pings.got()
-	if code != 1 || len(h.rec.sent) != 1 || !strings.Contains(stderr, "felis watchdog: save state: ") ||
-		len(pings) != 1 || pings[0] != "POST /check-key/fail" || !strings.HasPrefix(bodies[0], "the watchdog state did not save: ") {
-		t.Errorf("exit %d, mailed %v, pings %v %q; want exit 1, one mail and the save failure posted to /fail\nstdout %s\nstderr %s", code, h.rec.sent, pings, bodies, stdout, stderr)
+	if _, err := os.Stat(h.fallbackPath); code != 0 || len(h.rec.sent) != 1 || !errors.Is(err, fs.ErrNotExist) {
+		t.Fatalf("once the state saves: exit %d, mailed %v, fallback %v; want exit 0, no new mail, the fallback gone\nstdout %s\nstderr %s", code, h.rec.sent, err, stdout, stderr)
+	}
+	if s, err := watchdog.LoadState(h.statePath); err != nil || s.Alerts["postgres"] == nil || s.Alerts["postgres"].Notified.IsZero() {
+		t.Fatalf("saved state = %+v, %v; want the mailed PostgreSQL alert", s, err)
 	}
 }