fix(setup): SMTP 密码和上传桶密钥同时存进 /etc/felis，安装器每次从这里重建 Secret，看门狗和 breakGlass 在 k3s 宕机时也能读到

cmd/felis/breakglass.go

@@ -156,7 +156,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 	// Recovery mails its code through [smtp]; the relay is opened only if a code is
 	// asked for.
 	host, _ := os.Hostname()
-	recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, platform.DefaultControlNamespace), host: host}
+	recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, hostSMTPPasswordPath, platform.DefaultControlNamespace), host: host}
 
 	res, err := runBreakGlassTUI(ctx, repo, cfg.Database, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists, recovery)
 	if err != nil {

cmd/felis/breakglass_otp.go

@@ -227,9 +227,11 @@ func maskEmail(email string) string {
 }
 
 // hostRecoveryMailer opens the [smtp] relay from the host the way the watchdog does:
-// the password is the env var password_ref names when that is set, else the
-// felis-smtp Secret, whose absence means a relay without AUTH.
-func hostRecoveryMailer(c config.SMTPConfig, controlNS string) func(context.Context) (recoveryMailer, error) {
+// the password is the env var password_ref names when that is set, else the host
+// copy at passwordPath, else the felis-smtp Secret, whose absence means a relay
+// without AUTH. The cluster is reached only when the host copy is missing, so a
+// break-glass on a host whose k3s is down still gets its code.
+func hostRecoveryMailer(c config.SMTPConfig, passwordPath, controlNS string) func(context.Context) (recoveryMailer, error) {
 	return func(ctx context.Context) (recoveryMailer, error) {
 		if strings.TrimSpace(c.Host) == "" {
 			return nil, errors.New("[smtp] is not configured in felis.toml")
@@ -237,6 +239,11 @@ func hostRecoveryMailer(c config.SMTPConfig, controlNS string) func(context.Cont
 		if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" {
 			return smtpRelay(c, os.Getenv(ref)), nil
 		}
+		if password, ok, err := readHostCredential(passwordPath); err != nil {
+			return nil, fmt.Errorf("read the relay password: %w", err)
+		} else if ok {
+			return smtpRelay(c, password), nil
+		}
 		cl, err := buildSystemServerClient()
 		if err != nil {
 			return nil, fmt.Errorf("reach the cluster for the relay password: %w", err)

cmd/felis/breakglass_otp_test.go

@@ -246,7 +246,7 @@ func TestHostRecoveryMailer(t *testing.T) {
 	ctx := context.Background()
 
 	t.Run("no [smtp] host is no relay", func(t *testing.T) {
-		_, err := hostRecoveryMailer(config.SMTPConfig{}, "felis")(ctx)
+		_, err := hostRecoveryMailer(config.SMTPConfig{}, hostSMTPPasswordPath, "felis")(ctx)
 		if err == nil || !strings.Contains(err.Error(), "[smtp]") {
 			t.Fatalf("err = %v, want it to name [smtp]", err)
 		}
@@ -256,7 +256,7 @@ func TestHostRecoveryMailer(t *testing.T) {
 		t.Setenv("FELIS_TEST_RELAY_PW", "from-env")
 		off := false
 		c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_RELAY_PW", RequireTLS: &off}
-		got, err := hostRecoveryMailer(c, "felis")(ctx)
+		got, err := hostRecoveryMailer(c, hostSMTPPasswordPath, "felis")(ctx)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/felis/hostcreds.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// Host copies of the credentials `felis setup` takes at the keyboard: the [smtp]
+// relay password and the uploads bucket's keys. The cluster reads them from the
+// felis-smtp and felis-uploads-s3 Secrets, and a Secret lives in k3s's datastore,
+// which a reinstall (uninstall.sh keeps /etc/felis) or a host rebuilt from a
+// database bundle's state/ starts empty. Each file holds the bare value, mode
+// 0600, directly in /etc/felis beside secrets.env: every installer run applies
+// the Secrets from these files, and every database bundle, so the off-site copy
+// too, carries them.
+const (
+	hostSMTPPasswordPath       = "/etc/felis/smtp-password"
+	hostUploadsS3AccessKeyPath = "/etc/felis/uploads-s3-access-key"
+	hostUploadsS3SecretKeyPath = "/etc/felis/uploads-s3-secret-key"
+)
+
+// writeHostCredential replaces the file at path with value, mode 0600, through a
+// temporary file in the same directory, so a crash leaves the old value or the
+// new one and never a partial one.
+func writeHostCredential(path, value string) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	defer os.Remove(tmpPath)
+	// CreateTemp already makes the file 0600; the Chmod states it rather than
+	// leaning on that.
+	if err := tmp.Chmod(0o600); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if _, err := tmp.WriteString(value); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpPath, path)
+}
+
+// readHostCredential returns the value in path; ok is false when there is no
+// such file. An empty file is a value: the relay password of a relay without AUTH.
+func readHostCredential(path string) (value string, ok bool, err error) {
+	b, err := os.ReadFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return string(b), true, nil
+}
+
+// relayPassword is the [smtp] relay password as the host holds it: the copy
+// `felis setup` keeps at path, else, on an install from before that copy, the
+// felis-smtp Secret in ns, whose absence means a relay without AUTH. cl is only
+// used when the file is missing; a nil cl then reports errClusterUnreachable.
+func relayPassword(ctx context.Context, path string, cl client.Client, ns string) (string, error) {
+	if pw, ok, err := readHostCredential(path); err != nil || ok {
+		return pw, err
+	}
+	if cl == nil {
+		return "", errClusterUnreachable
+	}
+	return smtpSecretPassword(ctx, cl, ns)
+}
+
+var errClusterUnreachable = errors.New("the cluster did not answer")

cmd/felis/hostcreds_test.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/watchdog"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func TestHostCredentialFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "smtp-password")
+
+	if _, ok, err := readHostCredential(path); ok || err != nil {
+		t.Fatalf("a missing file read as ok=%v err=%v; want not there", ok, err)
+	}
+	// A copy an operator put there by hand, readable by everyone, is tightened.
+	if err := os.WriteFile(path, []byte("by hand"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range []string{"first secret", "a \"quoted\" $second\nsecret", ""} {
+		if err := writeHostCredential(path, v); err != nil {
+			t.Fatal(err)
+		}
+		got, ok, err := readHostCredential(path)
+		if err != nil || !ok || got != v {
+			t.Fatalf("read back (%q, %v, %v); want (%q, true, nil)", got, ok, err, v)
+		}
+		fi, err := os.Stat(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if mode := fi.Mode().Perm(); mode != 0o600 {
+			t.Fatalf("mode %v; want 0600", mode)
+		}
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 1 {
+		t.Fatalf("the directory holds %d entries; want the credential alone, no leftover temporary file", len(entries))
+	}
+
+	if _, _, err := readHostCredential(dir); err == nil {
+		t.Fatal("an unreadable credential read as fine")
+	}
+}
+
+func smtpSecretClient(t *testing.T, password string) client.Client {
+	t.Helper()
+	return fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
+		Data:       map[string][]byte{platform.SMTPSecretPasswordKey: []byte(password)},
+	}).Build()
+}
+
+func TestRelayPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	cl := smtpSecretClient(t, "from-secret")
+
+	if pw, err := relayPassword(ctx, host, cl, "felis"); err != nil || pw != "from-secret" {
+		t.Fatalf("without a host copy = (%q, %v); want the Secret's", pw, err)
+	}
+	if _, err := relayPassword(ctx, host, nil, "felis"); !errors.Is(err, errClusterUnreachable) {
+		t.Fatalf("without a host copy or a cluster err = %v; want errClusterUnreachable", err)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []client.Client{cl, nil} {
+		if pw, err := relayPassword(ctx, host, c, "felis"); err != nil || pw != "from-host" {
+			t.Fatalf("with a host copy (cluster %v) = (%q, %v); want the host copy", c != nil, pw, err)
+		}
+	}
+	if _, err := relayPassword(ctx, dir, cl, "felis"); err == nil {
+		t.Fatal("an unreadable host copy fell through to the Secret")
+	}
+}
+
+// The watchdog mails the most while the cluster is down: the host copy must
+// reach it then, and without one the password the last good run cached stays.
+func TestRefreshSMTPPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	var stderr bytes.Buffer
+
+	state := &watchdog.State{SMTPPassword: "cached"}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "cached" || stderr.Len() != 0 {
+		t.Fatalf("cluster down, no host copy: password %q, stderr %q; want the cached one kept quietly", state.SMTPPassword, stderr.String())
+	}
+	refreshSMTPPassword(ctx, host, smtpSecretClient(t, "from-secret"), "felis", state, &stderr)
+	if state.SMTPPassword != "from-secret" {
+		t.Fatalf("cluster up, no host copy: password %q; want the Secret's", state.SMTPPassword)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cluster down, host copy: password %q; want the host copy", state.SMTPPassword)
+	}
+	refreshSMTPPassword(ctx, dir, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" || !strings.Contains(stderr.String(), "keeping the cached one") {
+		t.Fatalf("unreadable host copy: password %q, stderr %q; want the cached one kept and the failure said", state.SMTPPassword, stderr.String())
+	}
+}
+
+func TestHostRecoveryMailerHostCopy(t *testing.T) {
+	ctx := context.Background()
+	// No kubeconfig anywhere: reaching for the cluster fails, so a pass proves
+	// the host copy was enough.
+	t.Setenv("KUBECONFIG", filepath.Join(t.TempDir(), "no-kubeconfig"))
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	off := false
+	c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_UNSET_RELAY_PW", RequireTLS: &off}
+
+	got, err := hostRecoveryMailer(c, host, "felis")(ctx)
+	if err != nil {
+		t.Fatalf("with the host copy: %v", err)
+	}
+	if relay, ok := got.(*mail.SMTP); !ok || relay.Password != "from-host" {
+		t.Fatalf("relay = %#v; want the host copy's password", got)
+	}
+	if _, err := hostRecoveryMailer(c, dir, "felis")(ctx); err == nil || !strings.Contains(err.Error(), "read the relay password") {
+		t.Fatalf("unreadable host copy: err = %v; want it named", err)
+	}
+	if _, err := hostRecoveryMailer(c, filepath.Join(dir, "none"), "felis")(ctx); err == nil || !strings.Contains(err.Error(), "reach the cluster") {
+		t.Fatalf("no host copy and no cluster: err = %v; want the cluster named", err)
+	}
+}
+
+// A whole watchdog run with the API server and PostgreSQL both down still
+// takes the relay password from the host copy, so the outage mail can
+// authenticate even when no earlier run cached it.
+func TestWatchdogReadsHostCopyWhileClusterDown(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
+	cfgPath := filepath.Join(dir, "felis.toml")
+	if err := os.WriteFile(cfgPath, []byte(`[database]
+url = "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
+[server]
+root_domain = "example.com"
+[archive]
+store = "tarLocal"
+[k8s]
+egress_mode = "nodeport"
+[smtp]
+host = "127.0.0.1"
+port = 1
+from = "[email protected]"
+username = "felis"
+password_ref = "FELIS_TEST_UNSET_RELAY_PW"
+`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	pwPath := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(pwPath, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	statePath := filepath.Join(dir, "state.json")
+	var stdout, stderr bytes.Buffer
+	cmdWatchdog([]string{
+		"-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"),
+		"-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", pwPath,
+	}, &stdout, &stderr)
+	if !strings.Contains(stdout.String(), "kube-api") {
+		t.Fatalf("the run found the API server up; the test needs it down (stdout %s)", stdout.String())
+	}
+	state, err := watchdog.LoadState(statePath)
+	if err != nil {
+		t.Fatalf("load state: %v (stderr %s)", err, stderr.String())
+	}
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cached relay password %q; want the host copy (stdout %s, stderr %s)", state.SMTPPassword, stdout.String(), stderr.String())
+	}
+}

cmd/felis/tui_smtp.go

@@ -276,8 +276,8 @@ func validateSMTPFrom(s string) error {
 }
 
 // currentSMTPInputs reads the relay already recorded in felis.toml so the form
-// pre-fills the non-secret fields. The password lives only in the felis-smtp
-// Secret and is deliberately never read back — it must be re-entered to change.
+// pre-fills the non-secret fields. The password (the felis-smtp Secret and its
+// host copy) is deliberately never read back — it must be re-entered to change.
 // Any read error falls back to a blank form rather than blocking reconfig.
 func currentSMTPInputs() smtpInputs {
 	cfg, err := config.Load(hostSetupConfigPath)
@@ -295,7 +295,8 @@ func currentSMTPInputs() smtpInputs {
 // applySMTPConfig proves the relay works, then persists it and rolls felis-api:
 // Ping (a full transaction — connect/STARTTLS/AUTH/MAIL FROM/RCPT/DATA, which
 // delivers one self-test message to the From address) → [smtp] into both config
-// files → the felis-smtp Secret → the config Secret → rollout. A failed Ping
+// files → the password into /etc/felis/smtp-password (hostcreds.go) → the
+// felis-smtp Secret → the config Secret → rollout. A failed Ping
 // leaves the install untouched, so a bad relay dies at the keyboard, not at a
 // player's OTP.
 //
@@ -330,6 +331,11 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
 			return err
 		}
 	}
+	// The host copy first: should the Secret fail, the next installer run applies
+	// it from this file.
+	if err := writeHostCredential(hostSMTPPasswordPath, in.password); err != nil {
+		return fmt.Errorf("keep the relay password in %s: %w", hostSMTPPasswordPath, err)
+	}
 	if err := applySMTPSecret(ctx, in.password); err != nil {
 		return err
 	}

cmd/felis/tui_storage_apply.go

@@ -53,8 +53,17 @@ func applyStorageConfig(ctx context.Context, method storageMethod, in s3Inputs)
 		return err
 	}
 	// S3: land the credentials in their own Secret BEFORE the roll, so the optional
-	// env refs resolve on the fresh pod. Local needs no Secret.
+	// env refs resolve on the fresh pod, and on the host before that, so the next
+	// installer run can apply the Secret again (hostcreds.go). Local needs no Secret.
 	if method == storageS3 {
+		for _, c := range []struct{ path, value string }{
+			{hostUploadsS3AccessKeyPath, in.accessKey},
+			{hostUploadsS3SecretKeyPath, in.secretKey},
+		} {
+			if err := writeHostCredential(c.path, c.value); err != nil {
+				return fmt.Errorf("keep the bucket credentials in %s: %w", c.path, err)
+			}
+		}
 		if err := applyUploadsS3Secret(ctx, in.accessKey, in.secretKey); err != nil {
 			return err
 		}
@@ -70,8 +79,9 @@ func applyStorageConfig(ctx context.Context, method storageMethod, in s3Inputs)
 
 // currentStorageInputs reads the storage backend already recorded in felis.toml so
 // the reconfigure flow can pre-select the method and pre-fill the non-secret S3
-// fields (endpoint/bucket/region). Credentials live only in the felis-uploads-s3
-// Secret and are deliberately never read back — they must be re-entered to change.
+// fields (endpoint/bucket/region). The credentials (the felis-uploads-s3 Secret
+// and its host copies) are deliberately never read back — they must be re-entered
+// to change.
 // Any read error falls back to a blank local default rather than blocking reconfig.
 func currentStorageInputs() (storageMethod, s3Inputs) {
 	cfg, err := config.Load(hostSetupConfigPath)

cmd/felis/watchdog.go

@@ -36,6 +36,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
 	statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")
+	smtpPasswordFile := fs.String("smtp-password-file", hostSMTPPasswordPath, "the relay password `felis setup` keeps on the host; the felis-smtp Secret stands in while it is missing")
 	quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
 	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
 	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
@@ -88,9 +89,13 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 		report.Unknown = append(report.Unknown, watchdog.ClusterPrefixes...)
 	} else {
 		report.Findings = append(report.Findings, found...)
-		if cfg.SMTP.Host != "" {
-			refreshSMTPPassword(ctx, cl, *controlNS, state, stderr)
+	}
+	if cfg.SMTP.Host != "" {
+		var secrets client.Client
+		if err == nil {
+			secrets = cl
 		}
+		refreshSMTPPassword(ctx, *smtpPasswordFile, secrets, *controlNS, state, stderr)
 	}
 
 	if recipients, err := ownerEmails(ctx, cfg.Database.URL); err != nil {
@@ -183,13 +188,19 @@ func usesMirroredScanDB(cfg *config.Config) bool {
 	return repo == "" || strings.HasPrefix(repo, cfg.Registry.URL+"/mirror/")
 }
 
-// refreshSMTPPassword caches the relay password from the felis-smtp Secret, or
-// forgets it when the Secret is gone (a relay without AUTH). An env var named by
-// [smtp] password_ref, when set, wins at send time instead.
-func refreshSMTPPassword(ctx context.Context, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) {
-	password, err := smtpSecretPassword(ctx, cl, ns)
-	if err != nil {
-		fmt.Fprintf(stderr, "felis watchdog: read %s/%s (keeping the cached relay password): %v\n", ns, platform.SMTPSecretName, err)
+// refreshSMTPPassword caches the relay password (relayPassword: the host copy at
+// path, else the felis-smtp Secret), or forgets it when the Secret is gone (a
+// relay without AUTH). The host copy is read even while the cluster is down, the
+// time an alert matters most; without one, a down cluster keeps the cached
+// password. An env var named by [smtp] password_ref, when set, wins at send time
+// instead.
+func refreshSMTPPassword(ctx context.Context, path string, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) {
+	password, err := relayPassword(ctx, path, cl, ns)
+	switch {
+	case errors.Is(err, errClusterUnreachable):
+		return
+	case err != nil:
+		fmt.Fprintf(stderr, "felis watchdog: read the relay password (keeping the cached one): %v\n", err)
 		return
 	}
 	state.SMTPPassword = password