fix(setup): SMTP 密码和上传桶密钥同时存进 /etc/felis，安装器每次从这里重建 Secret，看门狗和 breakGlass 在 k3s 宕机时也能读到

cmd/felis/breakglass.go

@@ -156,7 +156,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
 	// Recovery mails its code through [smtp]; the relay is opened only if a code is
 	// asked for.
 	host, _ := os.Hostname()
-	recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, platform.DefaultControlNamespace), host: host}
+	recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, hostSMTPPasswordPath, platform.DefaultControlNamespace), host: host}
 
 	res, err := runBreakGlassTUI(ctx, repo, cfg.Database, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists, recovery)
 	if err != nil {

cmd/felis/breakglass_otp.go

@@ -227,9 +227,11 @@ func maskEmail(email string) string {
 }
 
 // hostRecoveryMailer opens the [smtp] relay from the host the way the watchdog does:
-// the password is the env var password_ref names when that is set, else the
+// the password is the env var password_ref names when that is set, else the host
-// felis-smtp Secret, whose absence means a relay without AUTH.
+// copy at passwordPath, else the felis-smtp Secret, whose absence means a relay
-func hostRecoveryMailer(c config.SMTPConfig, controlNS string) func(context.Context) (recoveryMailer, error) {
+// without AUTH. The cluster is reached only when the host copy is missing, so a
+// break-glass on a host whose k3s is down still gets its code.
+func hostRecoveryMailer(c config.SMTPConfig, passwordPath, controlNS string) func(context.Context) (recoveryMailer, error) {
 	return func(ctx context.Context) (recoveryMailer, error) {
 		if strings.TrimSpace(c.Host) == "" {
 			return nil, errors.New("[smtp] is not configured in felis.toml")
@@ -237,6 +239,11 @@ func hostRecoveryMailer(c config.SMTPConfig, controlNS string) func(context.Cont
 		if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" {
 			return smtpRelay(c, os.Getenv(ref)), nil
 		}
+		if password, ok, err := readHostCredential(passwordPath); err != nil {
+			return nil, fmt.Errorf("read the relay password: %w", err)
+		} else if ok {
+			return smtpRelay(c, password), nil
+		}
 		cl, err := buildSystemServerClient()
 		if err != nil {
 			return nil, fmt.Errorf("reach the cluster for the relay password: %w", err)

cmd/felis/breakglass_otp_test.go

@@ -246,7 +246,7 @@ func TestHostRecoveryMailer(t *testing.T) {
 	ctx := context.Background()
 
 	t.Run("no [smtp] host is no relay", func(t *testing.T) {
-		_, err := hostRecoveryMailer(config.SMTPConfig{}, "felis")(ctx)
+		_, err := hostRecoveryMailer(config.SMTPConfig{}, hostSMTPPasswordPath, "felis")(ctx)
 		if err == nil || !strings.Contains(err.Error(), "[smtp]") {
 			t.Fatalf("err = %v, want it to name [smtp]", err)
 		}
@@ -256,7 +256,7 @@ func TestHostRecoveryMailer(t *testing.T) {
 		t.Setenv("FELIS_TEST_RELAY_PW", "from-env")
 		off := false
 		c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_RELAY_PW", RequireTLS: &off}
-		got, err := hostRecoveryMailer(c, "felis")(ctx)
+		got, err := hostRecoveryMailer(c, hostSMTPPasswordPath, "felis")(ctx)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/felis/hostcreds.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// Host copies of the credentials `felis setup` takes at the keyboard: the [smtp]
+// relay password and the uploads bucket's keys. The cluster reads them from the
+// felis-smtp and felis-uploads-s3 Secrets, and a Secret lives in k3s's datastore,
+// which a reinstall (uninstall.sh keeps /etc/felis) or a host rebuilt from a
+// database bundle's state/ starts empty. Each file holds the bare value, mode
+// 0600, directly in /etc/felis beside secrets.env: every installer run applies
+// the Secrets from these files, and every database bundle, so the off-site copy
+// too, carries them.
+const (
+	hostSMTPPasswordPath       = "/etc/felis/smtp-password"
+	hostUploadsS3AccessKeyPath = "/etc/felis/uploads-s3-access-key"
+	hostUploadsS3SecretKeyPath = "/etc/felis/uploads-s3-secret-key"
+)
+
+// writeHostCredential replaces the file at path with value, mode 0600, through a
+// temporary file in the same directory, so a crash leaves the old value or the
+// new one and never a partial one.
+func writeHostCredential(path, value string) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	defer os.Remove(tmpPath)
+	// CreateTemp already makes the file 0600; the Chmod states it rather than
+	// leaning on that.
+	if err := tmp.Chmod(0o600); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if _, err := tmp.WriteString(value); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpPath, path)
+}
+
+// readHostCredential returns the value in path; ok is false when there is no
+// such file. An empty file is a value: the relay password of a relay without AUTH.
+func readHostCredential(path string) (value string, ok bool, err error) {
+	b, err := os.ReadFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return string(b), true, nil
+}
+
+// relayPassword is the [smtp] relay password as the host holds it: the copy
+// `felis setup` keeps at path, else, on an install from before that copy, the
+// felis-smtp Secret in ns, whose absence means a relay without AUTH. cl is only
+// used when the file is missing; a nil cl then reports errClusterUnreachable.
+func relayPassword(ctx context.Context, path string, cl client.Client, ns string) (string, error) {
+	if pw, ok, err := readHostCredential(path); err != nil || ok {
+		return pw, err
+	}
+	if cl == nil {
+		return "", errClusterUnreachable
+	}
+	return smtpSecretPassword(ctx, cl, ns)
+}
+
+var errClusterUnreachable = errors.New("the cluster did not answer")

cmd/felis/hostcreds_test.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/mail"
+	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/watchdog"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func TestHostCredentialFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "smtp-password")
+
+	if _, ok, err := readHostCredential(path); ok || err != nil {
+		t.Fatalf("a missing file read as ok=%v err=%v; want not there", ok, err)
+	}
+	// A copy an operator put there by hand, readable by everyone, is tightened.
+	if err := os.WriteFile(path, []byte("by hand"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range []string{"first secret", "a \"quoted\" $second\nsecret", ""} {
+		if err := writeHostCredential(path, v); err != nil {
+			t.Fatal(err)
+		}
+		got, ok, err := readHostCredential(path)
+		if err != nil || !ok || got != v {
+			t.Fatalf("read back (%q, %v, %v); want (%q, true, nil)", got, ok, err, v)
+		}
+		fi, err := os.Stat(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if mode := fi.Mode().Perm(); mode != 0o600 {
+			t.Fatalf("mode %v; want 0600", mode)
+		}
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 1 {
+		t.Fatalf("the directory holds %d entries; want the credential alone, no leftover temporary file", len(entries))
+	}
+
+	if _, _, err := readHostCredential(dir); err == nil {
+		t.Fatal("an unreadable credential read as fine")
+	}
+}
+
+func smtpSecretClient(t *testing.T, password string) client.Client {
+	t.Helper()
+	return fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName},
+		Data:       map[string][]byte{platform.SMTPSecretPasswordKey: []byte(password)},
+	}).Build()
+}
+
+func TestRelayPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	cl := smtpSecretClient(t, "from-secret")
+
+	if pw, err := relayPassword(ctx, host, cl, "felis"); err != nil || pw != "from-secret" {
+		t.Fatalf("without a host copy = (%q, %v); want the Secret's", pw, err)
+	}
+	if _, err := relayPassword(ctx, host, nil, "felis"); !errors.Is(err, errClusterUnreachable) {
+		t.Fatalf("without a host copy or a cluster err = %v; want errClusterUnreachable", err)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []client.Client{cl, nil} {
+		if pw, err := relayPassword(ctx, host, c, "felis"); err != nil || pw != "from-host" {
+			t.Fatalf("with a host copy (cluster %v) = (%q, %v); want the host copy", c != nil, pw, err)
+		}
+	}
+	if _, err := relayPassword(ctx, dir, cl, "felis"); err == nil {
+		t.Fatal("an unreadable host copy fell through to the Secret")
+	}
+}
+
+// The watchdog mails the most while the cluster is down: the host copy must
+// reach it then, and without one the password the last good run cached stays.
+func TestRefreshSMTPPassword(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	var stderr bytes.Buffer
+
+	state := &watchdog.State{SMTPPassword: "cached"}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "cached" || stderr.Len() != 0 {
+		t.Fatalf("cluster down, no host copy: password %q, stderr %q; want the cached one kept quietly", state.SMTPPassword, stderr.String())
+	}
+	refreshSMTPPassword(ctx, host, smtpSecretClient(t, "from-secret"), "felis", state, &stderr)
+	if state.SMTPPassword != "from-secret" {
+		t.Fatalf("cluster up, no host copy: password %q; want the Secret's", state.SMTPPassword)
+	}
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cluster down, host copy: password %q; want the host copy", state.SMTPPassword)
+	}
+	refreshSMTPPassword(ctx, dir, nil, "felis", state, &stderr)
+	if state.SMTPPassword != "from-host" || !strings.Contains(stderr.String(), "keeping the cached one") {
+		t.Fatalf("unreadable host copy: password %q, stderr %q; want the cached one kept and the failure said", state.SMTPPassword, stderr.String())
+	}
+}
+
+func TestHostRecoveryMailerHostCopy(t *testing.T) {
+	ctx := context.Background()
+	// No kubeconfig anywhere: reaching for the cluster fails, so a pass proves
+	// the host copy was enough.
+	t.Setenv("KUBECONFIG", filepath.Join(t.TempDir(), "no-kubeconfig"))
+	dir := t.TempDir()
+	host := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(host, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	off := false
+	c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "[email protected]", Username: "felis", PasswordRef: "FELIS_TEST_UNSET_RELAY_PW", RequireTLS: &off}
+
+	got, err := hostRecoveryMailer(c, host, "felis")(ctx)
+	if err != nil {
+		t.Fatalf("with the host copy: %v", err)
+	}
+	if relay, ok := got.(*mail.SMTP); !ok || relay.Password != "from-host" {
+		t.Fatalf("relay = %#v; want the host copy's password", got)
+	}
+	if _, err := hostRecoveryMailer(c, dir, "felis")(ctx); err == nil || !strings.Contains(err.Error(), "read the relay password") {
+		t.Fatalf("unreadable host copy: err = %v; want it named", err)
+	}
+	if _, err := hostRecoveryMailer(c, filepath.Join(dir, "none"), "felis")(ctx); err == nil || !strings.Contains(err.Error(), "reach the cluster") {
+		t.Fatalf("no host copy and no cluster: err = %v; want the cluster named", err)
+	}
+}
+
+// A whole watchdog run with the API server and PostgreSQL both down still
+// takes the relay password from the host copy, so the outage mail can
+// authenticate even when no earlier run cached it.
+func TestWatchdogReadsHostCopyWhileClusterDown(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig"))
+	cfgPath := filepath.Join(dir, "felis.toml")
+	if err := os.WriteFile(cfgPath, []byte(`[database]
+url = "postgres://felis:[email protected]:1/felis?sslmode=disable&connect_timeout=2"
+[server]
+root_domain = "example.com"
+[archive]
+store = "tarLocal"
+[k8s]
+egress_mode = "nodeport"
+[smtp]
+host = "127.0.0.1"
+port = 1
+from = "[email protected]"
+username = "felis"
+password_ref = "FELIS_TEST_UNSET_RELAY_PW"
+`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	pwPath := filepath.Join(dir, "smtp-password")
+	if err := writeHostCredential(pwPath, "from-host"); err != nil {
+		t.Fatal(err)
+	}
+	statePath := filepath.Join(dir, "state.json")
+	var stdout, stderr bytes.Buffer
+	cmdWatchdog([]string{
+		"-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"),
+		"-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", pwPath,
+	}, &stdout, &stderr)
+	if !strings.Contains(stdout.String(), "kube-api") {
+		t.Fatalf("the run found the API server up; the test needs it down (stdout %s)", stdout.String())
+	}
+	state, err := watchdog.LoadState(statePath)
+	if err != nil {
+		t.Fatalf("load state: %v (stderr %s)", err, stderr.String())
+	}
+	if state.SMTPPassword != "from-host" {
+		t.Fatalf("cached relay password %q; want the host copy (stdout %s, stderr %s)", state.SMTPPassword, stdout.String(), stderr.String())
+	}
+}

cmd/felis/tui_smtp.go

@@ -276,8 +276,8 @@ func validateSMTPFrom(s string) error {
 }
 
 // currentSMTPInputs reads the relay already recorded in felis.toml so the form
-// pre-fills the non-secret fields. The password lives only in the felis-smtp
+// pre-fills the non-secret fields. The password (the felis-smtp Secret and its
-// Secret and is deliberately never read back — it must be re-entered to change.
+// host copy) is deliberately never read back — it must be re-entered to change.
 // Any read error falls back to a blank form rather than blocking reconfig.
 func currentSMTPInputs() smtpInputs {
 	cfg, err := config.Load(hostSetupConfigPath)
@@ -295,7 +295,8 @@ func currentSMTPInputs() smtpInputs {
 // applySMTPConfig proves the relay works, then persists it and rolls felis-api:
 // Ping (a full transaction — connect/STARTTLS/AUTH/MAIL FROM/RCPT/DATA, which
 // delivers one self-test message to the From address) → [smtp] into both config
-// files → the felis-smtp Secret → the config Secret → rollout. A failed Ping
+// files → the password into /etc/felis/smtp-password (hostcreds.go) → the
+// felis-smtp Secret → the config Secret → rollout. A failed Ping
 // leaves the install untouched, so a bad relay dies at the keyboard, not at a
 // player's OTP.
 //
@@ -330,6 +331,11 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
 			return err
 		}
 	}
+	// The host copy first: should the Secret fail, the next installer run applies
+	// it from this file.
+	if err := writeHostCredential(hostSMTPPasswordPath, in.password); err != nil {
+		return fmt.Errorf("keep the relay password in %s: %w", hostSMTPPasswordPath, err)
+	}
 	if err := applySMTPSecret(ctx, in.password); err != nil {
 		return err
 	}

cmd/felis/tui_storage_apply.go

@@ -53,8 +53,17 @@ func applyStorageConfig(ctx context.Context, method storageMethod, in s3Inputs)
 		return err
 	}
 	// S3: land the credentials in their own Secret BEFORE the roll, so the optional
-	// env refs resolve on the fresh pod. Local needs no Secret.
+	// env refs resolve on the fresh pod, and on the host before that, so the next
+	// installer run can apply the Secret again (hostcreds.go). Local needs no Secret.
 	if method == storageS3 {
+		for _, c := range []struct{ path, value string }{
+			{hostUploadsS3AccessKeyPath, in.accessKey},
+			{hostUploadsS3SecretKeyPath, in.secretKey},
+		} {
+			if err := writeHostCredential(c.path, c.value); err != nil {
+				return fmt.Errorf("keep the bucket credentials in %s: %w", c.path, err)
+			}
+		}
 		if err := applyUploadsS3Secret(ctx, in.accessKey, in.secretKey); err != nil {
 			return err
 		}
@@ -70,8 +79,9 @@ func applyStorageConfig(ctx context.Context, method storageMethod, in s3Inputs)
 
 // currentStorageInputs reads the storage backend already recorded in felis.toml so
 // the reconfigure flow can pre-select the method and pre-fill the non-secret S3
-// fields (endpoint/bucket/region). Credentials live only in the felis-uploads-s3
+// fields (endpoint/bucket/region). The credentials (the felis-uploads-s3 Secret
-// Secret and are deliberately never read back — they must be re-entered to change.
+// and its host copies) are deliberately never read back — they must be re-entered
+// to change.
 // Any read error falls back to a blank local default rather than blocking reconfig.
 func currentStorageInputs() (storageMethod, s3Inputs) {
 	cfg, err := config.Load(hostSetupConfigPath)

cmd/felis/watchdog.go

@@ -36,6 +36,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	fs.SetOutput(stderr)
 	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)")
 	statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)")
+	smtpPasswordFile := fs.String("smtp-password-file", hostSMTPPasswordPath, "the relay password `felis setup` keeps on the host; the felis-smtp Secret stands in while it is missing")
 	quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
 	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
 	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
@@ -88,9 +89,13 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 		report.Unknown = append(report.Unknown, watchdog.ClusterPrefixes...)
 	} else {
 		report.Findings = append(report.Findings, found...)
-		if cfg.SMTP.Host != "" {
-			refreshSMTPPassword(ctx, cl, *controlNS, state, stderr)
 	}
+	if cfg.SMTP.Host != "" {
+		var secrets client.Client
+		if err == nil {
+			secrets = cl
+		}
+		refreshSMTPPassword(ctx, *smtpPasswordFile, secrets, *controlNS, state, stderr)
 	}
 
 	if recipients, err := ownerEmails(ctx, cfg.Database.URL); err != nil {
@@ -183,13 +188,19 @@ func usesMirroredScanDB(cfg *config.Config) bool {
 	return repo == "" || strings.HasPrefix(repo, cfg.Registry.URL+"/mirror/")
 }
 
-// refreshSMTPPassword caches the relay password from the felis-smtp Secret, or
+// refreshSMTPPassword caches the relay password (relayPassword: the host copy at
-// forgets it when the Secret is gone (a relay without AUTH). An env var named by
+// path, else the felis-smtp Secret), or forgets it when the Secret is gone (a
-// [smtp] password_ref, when set, wins at send time instead.
+// relay without AUTH). The host copy is read even while the cluster is down, the
-func refreshSMTPPassword(ctx context.Context, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) {
+// time an alert matters most; without one, a down cluster keeps the cached
-	password, err := smtpSecretPassword(ctx, cl, ns)
+// password. An env var named by [smtp] password_ref, when set, wins at send time
-	if err != nil {
+// instead.
-		fmt.Fprintf(stderr, "felis watchdog: read %s/%s (keeping the cached relay password): %v\n", ns, platform.SMTPSecretName, err)
+func refreshSMTPPassword(ctx context.Context, path string, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) {
+	password, err := relayPassword(ctx, path, cl, ns)
+	switch {
+	case errors.Is(err, errClusterUnreachable):
+		return
+	case err != nil:
+		fmt.Fprintf(stderr, "felis watchdog: read the relay password (keeping the cached one): %v\n", err)
 		return
 	}
 	state.SMTPPassword = password

deploy/bootstrap.sh

@@ -414,6 +414,12 @@ UPDATE_CHECK_SERVICE="/etc/systemd/system/felis-update-check.service"
 UPDATE_CHECK_TIMER="/etc/systemd/system/felis-update-check.timer"
 WATCHDOG_STATE="/var/lib/felis/watchdog/state.json"
 OFFSITE_ENV="${STATE_DIR}/offsite.env"
+# Host copies of the credentials `felis setup` takes at the keyboard, one bare value per
+# file, mode 0600 (cmd/felis/hostcreds.go); apply_setup_credential_secrets applies their
+# Secrets from them on every run.
+SMTP_PASSWORD_FILE="${STATE_DIR}/smtp-password"
+UPLOADS_S3_ACCESS_KEY_FILE="${STATE_DIR}/uploads-s3-access-key"
+UPLOADS_S3_SECRET_KEY_FILE="${STATE_DIR}/uploads-s3-secret-key"
 OFFSITE_SERVICE="/etc/systemd/system/felis-offsite.service"
 OFFSITE_TIMER="/etc/systemd/system/felis-offsite.timer"
 BUILD_TOOLS_SERVICE="/etc/systemd/system/felis-build-tools.service"
@@ -666,6 +672,54 @@ apply_registry_secrets() {
   rm -rf -- "$dir"
 }
 
+# secret_key_to_file keeps one key of a Secret in path, mode 0600, when path does not
+# exist yet. An install from before the host copies had the setup screens' credentials
+# only in the cluster; this is the run that moves them onto the host. A missing Secret
+# leaves path missing. The value goes from kubectl into the file, never into argv or
+# the log.
+secret_key_to_file() {
+  local namespace="$1" name="$2" key="$3" path="$4" encoded tmp
+  [ -e "$path" ] && return 0
+  encoded="$(kube -n "$namespace" get secret "$name" -o "jsonpath={.data.${key}}" 2>/dev/null)" || return 0
+  tmp="$(umask 077; mktemp "${path}.XXXXXX")"
+  remember_temp "$tmp"
+  printf '%s' "$encoded" | base64 -d > "$tmp"
+  mv -f -- "$tmp" "$path"
+}
+
+# apply_setup_credential_secrets applies the Secrets behind `felis setup`'s email and
+# uploads-bucket screens from their host copies: felis-smtp in the control namespace and
+# in the workload one (the reaper's warning mails read that copy), felis-uploads-s3 in the
+# control namespace. A reinstall that kept /etc/felis, or a host rebuilt from a bundle's
+# state/, starts k3s with no Secrets at all; without this, every sign-in code and alert
+# would go out without AUTH and the relay would turn it away. The host copy wins over the
+# cluster's: `felis setup` writes both, so they differ only after a hand edit of the
+# Secret. A relay or bucket whose credentials are on neither side is reported, so the
+# operator enters them again before the first code fails.
+apply_setup_credential_secrets() {
+  local ns
+  secret_key_to_file "$CONTROL_NS" felis-smtp password "$SMTP_PASSWORD_FILE"
+  secret_key_to_file "$CONTROL_NS" felis-uploads-s3 access_key_id "$UPLOADS_S3_ACCESS_KEY_FILE"
+  secret_key_to_file "$CONTROL_NS" felis-uploads-s3 secret_access_key "$UPLOADS_S3_SECRET_KEY_FILE"
+  if [ -f "$SMTP_PASSWORD_FILE" ]; then
+    for ns in "$CONTROL_NS" "$MINECRAFT_NS"; do
+      kube -n "$ns" create secret generic felis-smtp \
+        --from-file=password="$SMTP_PASSWORD_FILE" \
+        --dry-run=client -o yaml | kube apply -f -
+    done
+  elif persisted_smtp_block | grep -Eq '^[[:space:]]*username[[:space:]]*=[[:space:]]*"[^"]'; then
+    warn "[smtp] signs in with a username, but its password is in neither ${SMTP_PASSWORD_FILE} nor the cluster: sign-in codes and alerts go out without AUTH until it is entered again (sudo felis setup, then e to configure email)"
+  fi
+  if [ -f "$UPLOADS_S3_ACCESS_KEY_FILE" ] && [ -f "$UPLOADS_S3_SECRET_KEY_FILE" ]; then
+    kube -n "$CONTROL_NS" create secret generic felis-uploads-s3 \
+      --from-file=access_key_id="$UPLOADS_S3_ACCESS_KEY_FILE" \
+      --from-file=secret_access_key="$UPLOADS_S3_SECRET_KEY_FILE" \
+      --dry-run=client -o yaml | kube apply -f -
+  elif persisted_registry_block | grep -Eq '^[[:space:]]*user_uploads_context[[:space:]]*=[[:space:]]*"[sS]3://'; then
+    warn "uploads go to an S3 bucket, but its keys are in neither ${UPLOADS_S3_ACCESS_KEY_FILE} / ${UPLOADS_S3_SECRET_KEY_FILE} nor the cluster: felis-api refuses modpack uploads until they are entered again (sudo felis setup, then s to change storage)"
+  fi
+}
+
 # node_global_cidrs prints one host-length CIDR per global address on this node.
 # Game server egress already excludes every private range; this adds the node's
 # public addresses, which would otherwise let a server dial the panel NodePort,
@@ -4777,7 +4831,7 @@ deploy_bundle() {
     kube create namespace "$ns" --dry-run=client -o yaml | kube apply -f -
   done
 
-  log "provisioning felis-config + internal caller tokens + felis-forwarding-secret + registry credentials + panel TLS secrets (out-of-band, never in the bundle)"
+  log "provisioning felis-config + internal caller tokens + felis-forwarding-secret + registry credentials + mail relay and uploads bucket credentials + panel TLS secrets (out-of-band, never in the bundle)"
   apply_felis_config_secrets
   # felis-api mounts all four caller tokens from the control namespace. The login
   # gate's and the build Job's are also applied into the namespace their pods run in
@@ -4797,6 +4851,7 @@ deploy_bundle() {
   # "less secure", it is unjoinable.
   apply_literal_secret "$CONTROL_NS" felis-forwarding-secret secret "$FORWARDING_SECRET"
   apply_registry_secrets
+  apply_setup_credential_secrets
   kube -n "$CONTROL_NS" create secret tls felis-api-tls \
     --cert="$PANEL_TLS_CERT" \
     --key="$PANEL_TLS_KEY" \

deploy/bootstrap_test.sh

@@ -3833,6 +3833,112 @@ expect "a FELIS_ARTIFACT_DIR with SHA256SUMS is accepted" "VALID" "$(run_vs "$ad
 expect "FELIS_ARTIFACT_DIR and FELIS_REF together are refused" "DIE: FELIS_ARTIFACT_DIR and FELIS_REF both name what to install" "$(run_vs "$adir" 1)"
 rm -rf "$adir"
 
+# --- the setup screens' credentials come back from /etc/felis ---------------------------
+# `felis setup` keeps the relay password and the uploads bucket's keys in /etc/felis as
+# well as in their Secrets. A reinstall, or a host rebuilt from a bundle's state/, starts
+# k3s empty, so every run applies the Secrets from those files; an install from before
+# them moves the Secrets' values into the files first. No value may reach kubectl's argv.
+for f in secret_key_to_file apply_setup_credential_secrets; do
+  [ -n "$(bsfn "$f")" ] || { echo "FAIL: no ${f} in $BS"; exit 1; }
+  [ "$(bsfn "$f" | wc -l)" -lt 45 ] \
+    || { echo "FAIL: the extracted ${f} is not the function -- did its closing brace move?"; exit 1; }
+done
+grep -q '^  apply_setup_credential_secrets$' "$BS" \
+  && echo "PASS every run applies the setup screens' Secrets" \
+  || { echo "FAIL: the install no longer calls apply_setup_credential_secrets"; fails=$((fails + 1)); }
+credir="$(mktemp -d)"
+credcalls="$(mktemp)"
+run_creds() { # secrets-in-cluster(0/1) smtp-username uploads-context
+  : > "$credcalls"
+  HAVE="$1" SMTPUSER="$2" UPCTX="$3" CALLS="$credcalls" STATE_DIR="$credir" CONTROL_NS=felis MINECRAFT_NS=minecraft bash -c '
+    set -Eeuo pipefail
+    SMTP_PASSWORD_FILE="${STATE_DIR}/smtp-password"
+    UPLOADS_S3_ACCESS_KEY_FILE="${STATE_DIR}/uploads-s3-access-key"
+    UPLOADS_S3_SECRET_KEY_FILE="${STATE_DIR}/uploads-s3-secret-key"
+    remember_temp() { :; }
+    warn() { printf "WARN: %s\n" "$*"; }
+    persisted_smtp_block() { printf "[smtp]\nhost = \"mail.example.com\"\nusername = \"%s\"\n" "$SMTPUSER"; }
+    persisted_registry_block() { printf "user_uploads_context = \"%s\"\n" "$UPCTX"; }
+    kube() {
+      case "$*" in
+        *" get secret "*)
+          printf "GET %s\n" "$*" >>"$CALLS"
+          [ "$HAVE" = 1 ] || return 1
+          case "$*" in
+            *felis-smtp*) printf "cmVsYXkgcHcvMSt4" ;;          # relay pw/1+x
+            *access_key_id*) printf "QUtJQU9MRA==" ;;           # AKIAOLD
+            *secret_access_key*) printf "b2xkL3NlY3JldCtrZXk=" ;; # old/secret+key
+          esac ;;
+        "apply -f -") cat >/dev/null; printf "APPLY\n" >>"$CALLS" ;;
+        *) printf "KUBE %s\n" "$*" >>"$CALLS" ;;
+      esac
+    }
+    '"$(bsfn secret_key_to_file)"'
+    '"$(bsfn apply_setup_credential_secrets)"'
+    apply_setup_credential_secrets' 2>&1
+  cat "$credcalls"
+}
+applies() { printf '%s\n' "$1" | grep -c '^APPLY$'; }
+
+out="$(run_creds 1 felis s3://uploads)"
+[ "$(cat "$credir/smtp-password")" = "relay pw/1+x" ] \
+  && [ "$(cat "$credir/uploads-s3-access-key")" = "AKIAOLD" ] \
+  && [ "$(cat "$credir/uploads-s3-secret-key")" = "old/secret+key" ] \
+  && echo "PASS an install from before the host copies moves the Secrets' values onto the host" \
+  || { echo "FAIL: the Secrets' values did not land in ${credir}:"; printf '%s\n' "$out"; fails=$((fails + 1)); }
+for f in smtp-password uploads-s3-access-key uploads-s3-secret-key; do
+  case "$(ls -l "$credir/$f" | cut -c1-10)" in
+    -rw-------) echo "PASS ${f} is root's alone" ;;
+    *) echo "FAIL: ${f} is $(ls -l "$credir/$f" | cut -c1-10), want -rw-------"; fails=$((fails + 1)) ;;
+  esac
+done
+[ -z "$(find "$credir" -name '*.??????' | head -1)" ] && echo "PASS the move leaves no temporary file in /etc/felis" \
+  || { echo "FAIL: a temporary file stayed behind: $(ls -a "$credir")"; fails=$((fails + 1)); }
+expect "the relay password is applied to the control plane" \
+  "KUBE -n felis create secret generic felis-smtp --from-file=password=${credir}/smtp-password --dry-run=client -o yaml" "$out"
+expect "and to the workload namespace the reaper mails from" \
+  "KUBE -n minecraft create secret generic felis-smtp --from-file=password=${credir}/smtp-password --dry-run=client -o yaml" "$out"
+expect "the bucket keys are applied to the control plane" \
+  "KUBE -n felis create secret generic felis-uploads-s3 --from-file=access_key_id=${credir}/uploads-s3-access-key --from-file=secret_access_key=${credir}/uploads-s3-secret-key --dry-run=client -o yaml" "$out"
+[ "$(applies "$out")" = 3 ] && echo "PASS all three Secrets are piped to kubectl apply" \
+  || { echo "FAIL: expected 3 applies:"; printf '%s\n' "$out"; fails=$((fails + 1)); }
+case "$out" in
+  *"relay pw"*|*AKIAOLD*|*"old/secret"*|*WARN*) echo "FAIL: a credential reached argv or the log, or a warning fired:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;;
+  *) echo "PASS no credential reaches kubectl's argv or the log" ;;
+esac
+
+printf 'new pw' > "$credir/smtp-password"
+out="$(run_creds 1 felis s3://uploads)"
+[ "$(cat "$credir/smtp-password")" = "new pw" ] && ! printf '%s\n' "$out" | grep -q '^GET' \
+  && echo "PASS the host copy wins over the cluster's" \
+  || { echo "FAIL: the cluster's value replaced the host copy:"; printf '%s\n' "$out"; fails=$((fails + 1)); }
+[ "$(applies "$out")" = 3 ] && echo "PASS a re-run applies the Secrets from the host copies" \
+  || { echo "FAIL: expected 3 applies on the re-run:"; printf '%s\n' "$out"; fails=$((fails + 1)); }
+
+rm -f "$credir"/*
+out="$(run_creds 0 felis s3://uploads)"
+expect "a relay that signs in with no password anywhere is reported" \
+  "WARN: [smtp] signs in with a username, but its password is in neither ${credir}/smtp-password nor the cluster" "$out"
+expect "a bucket with no keys anywhere is reported" \
+  "WARN: uploads go to an S3 bucket, but its keys are in neither" "$out"
+[ "$(applies "$out")" = 0 ] && [ -z "$(ls -A "$credir")" ] \
+  && echo "PASS nothing is applied or written without a value" \
+  || { echo "FAIL: something was applied or written without a value:"; printf '%s\n' "$out"; ls -A "$credir"; fails=$((fails + 1)); }
+
+out="$(run_creds 0 "" /var/lib/felis/uploads)"
+case "$out" in
+  *WARN*|*APPLY*) echo "FAIL: a relay without AUTH and local uploads were reported or applied:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;;
+  *) echo "PASS a relay without AUTH and local uploads need no credentials" ;;
+esac
+
+printf 'AKIAONLY' > "$credir/uploads-s3-access-key"
+out="$(run_creds 0 "" s3://uploads)"
+case "$out" in
+  *"create secret generic felis-uploads-s3"*) echo "FAIL: a bucket Secret was applied from one key alone:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;;
+  *) expect "one bucket key alone is reported as missing keys" "WARN: uploads go to an S3 bucket" "$out" ;;
+esac
+rm -rf "$credir" "$credcalls"
+
 # ---------------------------------------------------------------------------------------
 if [ "$fails" -eq 0 ]; then
   echo "ALL PASS"

docs/operations.md

@@ -273,7 +273,7 @@ and the MinecraftServer CRD are deleted.
 | Final database bundle | taken first (`felis db backup -label manual`); a failure stops the uninstall before anything is removed. `--no-backup` skips it | none |
 | The database (`/var/lib/felis/postgres`) | kept; felis-postgres is stopped cleanly before k3s goes | deleted with `/var/lib/felis` |
 | A host PostgreSQL an earlier release ran the database on | kept as it is: stopped after the move into k3s (below, §4), with its old copy of `felis` | its `felis` database and role are dropped (the server is started for that and stopped again), and `listen_addresses` and `pg_hba.conf` go back to how they were. Checked before anything is removed: a role that still owns another database (the `felis_pgint` the PG contract tests use, CONTRIBUTING.md) or holds grants elsewhere stops the purge up front with the list and the `ALTER DATABASE … OWNER TO postgres` to run |
-| `/etc/felis` (secrets, `felis.toml`, `offsite.env`, tunnel config) | kept; `bootstrap.done` and the per-run records go | deleted, with the tunnel's credentials file |
+| `/etc/felis` (secrets, `felis.toml`, `offsite.env`, the mail relay password and uploads bucket keys `felis setup` took, tunnel config) | kept; `bootstrap.done` and the per-run records go | deleted, with the tunnel's credentials file |
 | `/var/lib/felis` (the database, its bundles) | kept | deleted |
 | Worlds, archives, registry, uploads | moved to `/var/lib/felis/retained/k3s-storage-<stamp>/` (with `--keep-k3s`: their volumes switch to `Retain` and stay in place) | deleted |
 | Felis images, Docker build cache | kept | deleted |

docs/troubleshooting.md

@@ -1101,6 +1101,20 @@ control-namespace Secret alone is not. [GO-TESTED: the delivered/retried/
 suppressed matrix in `internal/reaper`; live-drilled end to end against a local
 SMTP sink.]
 
+The screen also keeps the password in `/etc/felis/smtp-password` (mode 0600),
+and the uploads screen keeps the bucket keys in `/etc/felis/uploads-s3-access-key`
+and `uploads-s3-secret-key`. Secrets live in k3s's datastore, which a reinstall
+or a host rebuilt from a bundle's `state/` starts empty, so **every installer
+run applies `felis-smtp` (both namespaces) and `felis-uploads-s3` from these
+files**. The files win: a hand edit of either Secret lasts until the next
+installer run, so change a credential through `felis setup`. An install from
+before these files gets them from the Secrets on its first re-run. When
+`[smtp]` has a `username` but no password is on either side, or uploads go to
+`s3://` without both keys, the installer says so and names the `felis setup`
+screen that takes them again. [SH-TESTED: `deploy/bootstrap_test.sh`, the move
+from the Secrets, the host copy winning, both warnings, no value in kubectl's
+argv. GO-TESTED: the host copy's mode and replacement.]
+
 ### Genuine false-delete risk vectors
 
 - **Stale `last_active_at`.** The keep-alive is `RecordJoin`, called from the
@@ -1510,9 +1524,12 @@ How it mails:
   delay is never mailed; one that turns critical is mailed again at once.
 - **During an install** nothing is mailed. `bootstrap.sh` writes
   `/run/felis/watchdog-quiet-until` and removes it when it exits.
-- **Caching:** the relay password and the recipient list are cached in
+- **Caching:** the relay password comes from `/etc/felis/smtp-password`, which
-  `/var/lib/felis/watchdog/state.json` (root-only). An outage of PostgreSQL or
+  the watchdog reads even while the API server is down (an install without that
-  of the API server can therefore still be mailed.
+  file reads the `felis-smtp` Secret instead). It and the recipient list are
+  cached in `/var/lib/felis/watchdog/state.json` (root-only). An outage of
+  PostgreSQL or of the API server can therefore still be mailed. [GO-TESTED: a
+  run with the API server and PostgreSQL both down caches the host copy.]
 - **No relay or no verified owner address:** each alert is written to the
   journal only.
 
@@ -1893,7 +1910,7 @@ along). One bundle is `felis-db-<UTC stamp>-<label>.tar`:
 |---|---|
 | `MANIFEST.json` | version, schema version, `pg_dump --version`, sha256 of every member |
 | `db.dump` | `pg_dump --format=custom` of the `felis` database |
-| `state/etc/felis/...` | every file in `/etc/felis`: `secrets.env` (DB password, session/forwarding secrets, registry tokens), `felis.host.toml`, `felis.pod.toml`, the `felis.toml` symlink, `offsite.env` (bucket credentials and encryption key), the panel TLS pair, and the installer's own markers (`system-server-images`, `velocity.fingerprint`). `bootstrap.done` is left out on purpose |
+| `state/etc/felis/...` | every file in `/etc/felis`: `secrets.env` (DB password, session/forwarding secrets, registry tokens), `felis.host.toml`, `felis.pod.toml`, the `felis.toml` symlink, `offsite.env` (bucket credentials and encryption key), `smtp-password`, `uploads-s3-access-key` and `uploads-s3-secret-key` (the mail relay password and the uploads bucket keys `felis setup` took), the panel TLS pair, and the installer's own markers (`system-server-images`, `velocity.fingerprint`). `bootstrap.done` is left out on purpose |
 | `k8s/minecraftservers.json` | every MinecraftServer, status and server-side metadata stripped, ready for `kubectl apply` (best effort: when the cluster did not answer, the manifest records why) |
 
 next to a `.sha256` sidecar in `sha256sum` format. **A bundle contains the
@@ -2030,7 +2047,7 @@ off-site bucket (next sections) plus a fresh install. What the host holds:
 | Data | On the host | In the bucket | Brought back by | Lost at most |
 |---|---|---|---|---|
 | Control-plane database (accounts, passkeys, ownership, quotas, audit, submissions, the `world_backups` index) | felis-postgres, `/var/lib/felis/postgres` | every bundle, copied within the hour of being written | `fetch-db`, `db restore` | changes since the newest bundle: up to a day plus an hour with the daily timer |
-| Host state (`/etc/felis`: secrets, both `felis.toml` copies, `offsite.env`, panel TLS pair) | `/etc/felis` | inside every bundle | `tar -x` of the bundle's `state/` | as the database |
+| Host state (`/etc/felis`: secrets, both `felis.toml` copies, `offsite.env`, the mail relay password and uploads bucket keys, panel TLS pair) | `/etc/felis` | inside every bundle | `tar -x` of the bundle's `state/` | as the database |
 | MinecraftServer objects | k3s | inside every bundle (`k8s/minecraftservers.json`) | `kubectl apply` | as the database |
 | World archives (reaper, "Back up now", pre-restore snapshots) | `felis-backups` volume | each one within the hour | `fetch-worlds` | archives written in the last hour |
 | Live worlds | `world-*` volumes under `/var/lib/rancher/k3s/storage` | **only as their archives** | a restore from the newest archive (§10) | everything since that world's newest archive |
@@ -2088,7 +2105,8 @@ host yourself, plus the off-site encryption key if the copy is in the bucket.
    `object does not decrypt with this key` and writes nothing. For a copy you
    made yourself, check it with `sha256sum -c felis-db-....tar.sha256`.
 2. Put the old host's state in place **before** installing, so the installer
-   reuses the same DB password, session secret, forwarding secret and the
+   reuses the same DB password, session secret, forwarding secret, the mail
+   relay password and uploads bucket keys `felis setup` took, and the
    `[offsite]` bucket with its credentials and key (`offsite.env`):
 
    ```
@@ -2414,8 +2432,10 @@ Once a staff account exists, `sudo felis breakGlass` asks which admin or owner
 is breaking the glass and mails that account's verified address a six-digit
 code through the same `[smtp]` relay as the sign-in codes. The code works for
 10 minutes and five wrong ones end it. The console reads the relay password
-the way the watchdog does (the `password_ref` env var, else the `felis-smtp`
+the way the watchdog does (the `password_ref` env var, else
-Secret, else no AUTH), and the mail skips the API's `max_per_hour` budget.
+`/etc/felis/smtp-password`, else the `felis-smtp` Secret, else no AUTH), so a
+host whose k3s is down still gets its code, and the mail skips the API's
+`max_per_hour` budget.
 Only the right code makes the run a `recovery` attributed to that account; the
 mail says which host and OS user asked, so an admin who did not ask learns
 that root there is in other hands.

internal/platform/workloads.go

@@ -130,7 +130,8 @@ const (
 	UploadsLocalPath = "/var/lib/felis/uploads"
 	// UploadsS3SecretName is the out-of-band Secret carrying the S3 credentials for
 	// an s3:// user_uploads_context. felis-api mounts its keys into env (optionally)
-	// and the setup wizard creates it. Like configSecretName it is NEVER rendered
+	// and the setup wizard creates it, keeping a host copy in /etc/felis that every
+	// installer run applies it from again. Like configSecretName it is NEVER rendered
 	// into the bundle — the credentials are the same red line.
 	UploadsS3SecretName      = "felis-uploads-s3"
 	UploadsS3SecretAccessKey = "access_key_id"
@@ -143,7 +144,8 @@ const (
 
 	// SMTPSecretName is the out-of-band Secret carrying the [smtp] relay password.
 	// Same red line as the S3 credentials: the setup wizard's "configure email"
-	// step creates it, felis-api reads it via SMTPPasswordEnv (optionally — a
+	// step creates it (and keeps a host copy in /etc/felis that every installer run
+	// applies it from again), felis-api reads it via SMTPPasswordEnv (optionally — a
 	// mailer-less install has no such Secret and still starts, falling back to
 	// logging codes), and it is never rendered into the bundle.
 	SMTPSecretName        = "felis-smtp"