fix(watchdog): 状态文件写不进时把状态留在 /run 下的后备文件，下一轮读较新的一份，同一封告警不再每 2 分钟重发

	fs.SetOutput(stderr)

	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")

	statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")

	fallbackState := fs.String("fallback-state", watchdog.FallbackStatePath, "where a run keeps its state while -state cannot be written, so what it mailed is not mailed again (tmpfs: until the host restarts; \"\" keeps none)")

	smtpPasswordFile := fs.String("smtp-password-file", hostSMTPPasswordPath, "the relay password `felis setup` keeps on the host; the felis-smtp Secret stands in while it is missing")

	quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")

	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)

	now := time.Now()

	if *unitFailed {

		return watchdogUnitFailed(unitFailedRun{

			cfgPath: *cfgPath, statePath: *statePath, quietPath: *quietPath,

			cfgPath: *cfgPath, statePath: *statePath, fallbackPath: *fallbackState, quietPath: *quietPath,

			offsiteStatus: *offsiteStatus, heartbeatFile: *heartbeatFile,

			result: os.Getenv("MONITOR_SERVICE_RESULT"), exitStatus: os.Getenv("MONITOR_EXIT_STATUS"),

			send: watchdogSender, client: http.DefaultClient, now: now,

		fmt.Fprintf(stderr, "felis watchdog: %v\n", err)

		return 1

	}

	state, aside, err := watchdog.RecoverState(*statePath, now)

	loadPath := watchdog.NewestState(*statePath, *fallbackState)

	state, aside, err := watchdog.RecoverState(loadPath, now)

	if err != nil {

		fmt.Fprintf(stderr, "felis watchdog: %v\n", err)

		return 1

		}

	}

	if aside != "" {

		fmt.Fprintf(stderr, "felis watchdog: %s was unreadable; moved it to %s and started over\n", *statePath, aside)

		fmt.Fprintf(stderr, "felis watchdog: %s was unreadable; moved it to %s and started over\n", loadPath, aside)

		f := watchdog.StateSetAside(aside)

		add(&f)

	}

	m := configMailer(cfg.SMTP, state.SMTPPassword, watchdogSender)

	unheard, mailFailed := m.deliver(ctx, state, plan, subject, body, mailHold(*quietPath, cfg.Offsite.Enabled(), *offsiteStatus, now), now, stdout, stderr)

	saveErr := watchdog.SaveState(*statePath, state)

	saveErr := watchdog.SaveStateOr(*statePath, *fallbackState, state)

	if saveErr != nil {

		fmt.Fprintf(stderr, "felis watchdog: save state: %v\n", saveErr)

	}

// failureReport is what the heartbeat's failure ping carries, "" when the run

// pings success: the alerts this run knows of reach no one (a mail that

// failed, or no relay or recipient while something is open), or the state did

// not save and the next run mails the same alerts again.

// not save to its file: kept on tmpfs it holds until the host restarts, which

// forgets what was mailed, and with nowhere to keep it the next run mails the

// same alerts again.

func failureReport(unheard string, mailFailed bool, saveErr error, open bool, r watchdog.Report) string {

	var why []string

	if unheard != "" && (mailFailed || open) {

// unitFailedRun is one run of felis-watchdog-failed.service.

type unitFailedRun struct {

	cfgPath, statePath, quietPath, offsiteStatus, heartbeatFile string

	cfgPath, statePath, fallbackPath, quietPath, offsiteStatus, heartbeatFile string

	// result and exitStatus are what systemd hands an OnFailure= unit

	// (MONITOR_SERVICE_RESULT, MONITOR_EXIT_STATUS; systemd 251 and later).

	result, exitStatus string

	if beat.url, err = readHeartbeatURL(r.heartbeatFile); err != nil {

		fmt.Fprintf(stderr, "felis watchdog: %v; pinging no heartbeat\n", err)

	}

	state, err := watchdog.LoadState(r.statePath)

	state, err := watchdog.LoadState(watchdog.NewestState(r.statePath, r.fallbackPath))

	if err != nil {

		// The next run that gets that far moves a state that does not parse

		// aside (watchdog.RecoverState).

	if mailFailed {

		code = 1

	}

	if err := watchdog.SaveState(r.statePath, state); err != nil {

	if err := watchdog.SaveStateOr(r.statePath, r.fallbackPath, state); err != nil {

		fmt.Fprintf(stderr, "felis watchdog: save state: %v\n", err)

		code = 1

	}

	}

	f := &alertRecorder{}

	return unitFailedRun{

		cfgPath: cfgPath, statePath: statePath, quietPath: filepath.Join(dir, "quiet"),

		cfgPath: cfgPath, statePath: statePath, fallbackPath: filepath.Join(t.TempDir(), "watchdog-state.json"), quietPath: filepath.Join(dir, "quiet"),

		offsiteStatus: filepath.Join(dir, "offsite-status.json"), heartbeatFile: beatFile,

		result: "exit-code", exitStatus: "1", send: f.sender, client: srv.Client(), now: time.Now(),

	}, f, log

	}

}

// TestWatchdogUnitFailedStateThatDoesNotSave: with the state file read-only,

// each report keeps the state in the fallback and the next one reads it, so

// the failure is mailed once, after five in a row, and never again.

func TestWatchdogUnitFailedStateThatDoesNotSave(t *testing.T) {

	if os.Geteuid() == 0 {

		t.Skip("root writes into a read-only directory")

	}

	r, f, _ := unitFailedFixture(t, "[database\n")

	dir := filepath.Dir(r.statePath)

	if err := os.Chmod(dir, 0o500); err != nil {

		t.Fatal(err)

	}

	t.Cleanup(func() { os.Chmod(dir, 0o700) })

	start := r.now

	for i := 0; i <= 8; i++ {

		r.now = start.Add(time.Duration(i) * 2 * time.Minute)

		var stdout, stderr bytes.Buffer

		if code := watchdogUnitFailed(r, &stdout, &stderr); code != 1 || !strings.Contains(stderr.String(), "; kept in "+r.fallbackPath) {

			t.Fatalf("run %d: exit %d, stderr %s; want exit 1 and the state kept in the fallback", i, code, stderr.String())

		}

		if want := min(max(i-4, 0), 1); len(f.sent) != want {

			t.Fatalf("run %d, %v after the first failure: mailed %d, want %d", i, r.now.Sub(start), len(f.sent), want)

		}

	}

}

// TestWatchdogUnitFailedConfigRelay: with felis.toml loading, the relay is

// the configured one and signs in with the env var password_ref names.

func TestWatchdogUnitFailedConfigRelay(t *testing.T) {

// the heartbeat URL points at a ping log.

type watchdogHost struct {

	dir, statePath string

	// fallbackPath stands in for /run/felis, in a directory of its own.

	fallbackPath string

	args         []string

	rec          *alertRecorder

	pings        *pingLog

	dir := t.TempDir()

	t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))

	pings, srv := newPingServer(t)

	h := &watchdogHost{dir: dir, statePath: filepath.Join(dir, "state.json"), rec: &alertRecorder{}, pings: pings, url: srv.URL}

	h := &watchdogHost{dir: dir, statePath: filepath.Join(dir, "state.json"), fallbackPath: filepath.Join(t.TempDir(), "watchdog-state.json"),

		rec: &alertRecorder{}, pings: pings, url: srv.URL}

	writeTestFile(t, filepath.Join(dir, "felis.toml"), cfg, 0o600)

	writeTestFile(t, filepath.Join(dir, "watchdog-heartbeat-url"), srv.URL+"/check-key\n", 0o600)

	if state != nil {

		}

	}

	h.args = []string{

		"-config", filepath.Join(dir, "felis.toml"), "-state", h.statePath, "-quiet-file", filepath.Join(dir, "quiet"),

		"-config", filepath.Join(dir, "felis.toml"), "-state", h.statePath, "-fallback-state", h.fallbackPath, "-quiet-file", filepath.Join(dir, "quiet"),

		"-backup-dir", "", "-disk-paths", dir, "-k3s-cert-dirs", "", "-smtp-password-file", filepath.Join(dir, "smtp-password"),

		"-offsite-status", filepath.Join(dir, "offsite-status.json"), "-build-tools-status", filepath.Join(dir, "build-tools.json"),

		"-heartbeat-file", filepath.Join(dir, "watchdog-heartbeat-url"),

}

// TestWatchdogRunStateThatDoesNotSave: a pass whose state does not save mails

// as usual, exits 1 and posts the failure to the heartbeat, since the next run

// mails the same alerts again.

// as usual, keeps its state in the fallback, exits 1 and posts the failure to

// the heartbeat. The next pass reads the fallback and mails nothing again; the

// first one whose state saves drops the fallback.

func TestWatchdogRunStateThatDoesNotSave(t *testing.T) {

	if os.Geteuid() == 0 {

		t.Skip("root writes into a read-only directory")

		t.Fatal(err)

	}

	t.Cleanup(func() { os.Chmod(h.dir, 0o700) })

	kept := "; kept in " + h.fallbackPath + " until the host restarts"

	for i := range 2 {

		code, stdout, stderr := h.run()

		pings, bodies := h.pings.got()

	if code != 1 || len(h.rec.sent) != 1 || !strings.Contains(stderr, "felis watchdog: save state: ") ||

		len(pings) != 1 || pings[0] != "POST /check-key/fail" || !strings.HasPrefix(bodies[0], "the watchdog state did not save: ") {

		t.Errorf("exit %d, mailed %v, pings %v %q; want exit 1, one mail and the save failure posted to /fail\nstdout %s\nstderr %s", code, h.rec.sent, pings, bodies, stdout, stderr)

		if code != 1 || len(h.rec.sent) != 1 || !strings.Contains(stderr, "felis watchdog: save state: ") || !strings.Contains(stderr, kept) ||

			len(pings) != i+1 || pings[i] != "POST /check-key/fail" || !strings.HasPrefix(bodies[i], "the watchdog state did not save: ") {

			t.Fatalf("pass %d: exit %d, mailed %v, pings %v %q; want exit 1, one mail in all and the save failure posted to /fail\nstdout %s\nstderr %s", i, code, h.rec.sent, pings, bodies, stdout, stderr)

		}

	}

	if err := os.Chmod(h.dir, 0o700); err != nil {

		t.Fatal(err)

	}

	code, stdout, stderr := h.run()

	if _, err := os.Stat(h.fallbackPath); code != 0 || len(h.rec.sent) != 1 || !errors.Is(err, fs.ErrNotExist) {

		t.Fatalf("once the state saves: exit %d, mailed %v, fallback %v; want exit 0, no new mail, the fallback gone\nstdout %s\nstderr %s", code, h.rec.sent, err, stdout, stderr)

	}

	if s, err := watchdog.LoadState(h.statePath); err != nil || s.Alerts["postgres"] == nil || s.Alerts["postgres"].Notified.IsZero() {

		t.Fatalf("saved state = %+v, %v; want the mailed PostgreSQL alert", s, err)

	}

}

power loss right after a save or a hand edit usually causes this; check

`df -h /var/lib/felis`, then delete the set-aside copy. [GO-TESTED: `TestRecoverState`]

A state file that cannot be written (a full or read-only `/var/lib/felis`)

leaves the run's state in `/run/felis/watchdog-state.json` (tmpfs, root-only).

The next run reads whichever of the two was written last, so an alert that was

mailed is not mailed again every two minutes. Each such run logs `save state:

...; kept in /run/felis/watchdog-state.json until the host restarts`, exits 1

and pings the heartbeat's `/fail`; after five in a row `watchdog/run` is mailed

once. The first run whose state file saves again deletes the tmpfs copy. A

restart before that forgets what was mailed, so the open problems are mailed

again as new. [GO-TESTED: `TestSaveStateOr`, `TestWatchdogRunStateThatDoesNotSave`,

`TestWatchdogUnitFailedStateThatDoesNotSave`]

```bash

journalctl -u felis-watchdog-failed -n 20   # what the fallback reported and pinged

systemctl status felis-watchdog             # the failed run's result

	return os.Rename(tmp.Name(), path)

}

// FallbackStatePath is where a run keeps its state when the state file cannot be

// written (a full or read-only /var/lib): tmpfs, which lasts until the host

// restarts, next to the installer's quiet marker.

const FallbackStatePath = "/run/felis/watchdog-state.json"

// NewestState is the state file a run reads: fallback when a run wrote it after

// path (its save to path failed, SaveStateOr), else path.

func NewestState(path, fallback string) string {

	fb, err := os.Stat(fallback) // "" is no fallback: it does not stat

	if err != nil {

		return path

	}

	if st, err := os.Stat(path); err == nil && !fb.ModTime().After(st.ModTime()) {

		return path

	}

	return fallback

}

// SaveStateOr saves s to path and drops fallback. When path cannot be written it

// saves s to fallback instead, so the next run still knows what this one mailed

// and does not mail it again every two minutes. The error is path's either way,

// saying where the state went.

func SaveStateOr(path, fallback string, s *State) error {

	err := SaveState(path, s)

	if err == nil {

		if fallback != "" {

			os.Remove(fallback)

		}

		return nil

	}

	if fallback == "" {

		return err

	}

	if ferr := SaveState(fallback, s); ferr != nil {

		return fmt.Errorf("%w; nor in %s: %v", err, fallback, ferr)

	}

	return fmt.Errorf("%w; kept in %s until the host restarts", err, fallback)

}

// QuietUntil reads the maintenance marker the installer writes while it

// restarts things on purpose: a Unix timestamp, before which nothing is mailed.

// A missing or unreadable marker means no quiet period.