diff --git a/cmd/felis/breakglass.go b/cmd/felis/breakglass.go index 57ec012..328421e 100644 --- a/cmd/felis/breakglass.go +++ b/cmd/felis/breakglass.go @@ -156,7 +156,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int { // Recovery mails its code through [smtp]; the relay is opened only if a code is // asked for. host, _ := os.Hostname() - recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, platform.DefaultControlNamespace), host: host} + recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, hostSMTPPasswordPath, platform.DefaultControlNamespace), host: host} res, err := runBreakGlassTUI(ctx, repo, cfg.Database, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists, recovery) if err != nil { diff --git a/cmd/felis/breakglass_otp.go b/cmd/felis/breakglass_otp.go index e89cd55..49c282a 100644 --- a/cmd/felis/breakglass_otp.go +++ b/cmd/felis/breakglass_otp.go @@ -227,9 +227,11 @@ func maskEmail(email string) string { } // hostRecoveryMailer opens the [smtp] relay from the host the way the watchdog does: -// the password is the env var password_ref names when that is set, else the -// felis-smtp Secret, whose absence means a relay without AUTH. -func hostRecoveryMailer(c config.SMTPConfig, controlNS string) func(context.Context) (recoveryMailer, error) { +// the password is the env var password_ref names when that is set, else the host +// copy at passwordPath, else the felis-smtp Secret, whose absence means a relay +// without AUTH. The cluster is reached only when the host copy is missing, so a +// break-glass on a host whose k3s is down still gets its code. +func hostRecoveryMailer(c config.SMTPConfig, passwordPath, controlNS string) func(context.Context) (recoveryMailer, error) { return func(ctx context.Context) (recoveryMailer, error) { if strings.TrimSpace(c.Host) == "" { return nil, errors.New("[smtp] is not configured in felis.toml") @@ -237,6 +239,11 @@ func hostRecoveryMailer(c config.SMTPConfig, controlNS string) func(context.Cont if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" { return smtpRelay(c, os.Getenv(ref)), nil } + if password, ok, err := readHostCredential(passwordPath); err != nil { + return nil, fmt.Errorf("read the relay password: %w", err) + } else if ok { + return smtpRelay(c, password), nil + } cl, err := buildSystemServerClient() if err != nil { return nil, fmt.Errorf("reach the cluster for the relay password: %w", err) diff --git a/cmd/felis/breakglass_otp_test.go b/cmd/felis/breakglass_otp_test.go index bb9dab8..6a98e94 100644 --- a/cmd/felis/breakglass_otp_test.go +++ b/cmd/felis/breakglass_otp_test.go @@ -246,7 +246,7 @@ func TestHostRecoveryMailer(t *testing.T) { ctx := context.Background() t.Run("no [smtp] host is no relay", func(t *testing.T) { - _, err := hostRecoveryMailer(config.SMTPConfig{}, "felis")(ctx) + _, err := hostRecoveryMailer(config.SMTPConfig{}, hostSMTPPasswordPath, "felis")(ctx) if err == nil || !strings.Contains(err.Error(), "[smtp]") { t.Fatalf("err = %v, want it to name [smtp]", err) } @@ -256,7 +256,7 @@ func TestHostRecoveryMailer(t *testing.T) { t.Setenv("FELIS_TEST_RELAY_PW", "from-env") off := false c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "felis@example.com", Username: "felis", PasswordRef: "FELIS_TEST_RELAY_PW", RequireTLS: &off} - got, err := hostRecoveryMailer(c, "felis")(ctx) + got, err := hostRecoveryMailer(c, hostSMTPPasswordPath, "felis")(ctx) if err != nil { t.Fatal(err) } diff --git a/cmd/felis/hostcreds.go b/cmd/felis/hostcreds.go new file mode 100644 index 0000000..630c611 --- /dev/null +++ b/cmd/felis/hostcreds.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Host copies of the credentials `felis setup` takes at the keyboard: the [smtp] +// relay password and the uploads bucket's keys. The cluster reads them from the +// felis-smtp and felis-uploads-s3 Secrets, and a Secret lives in k3s's datastore, +// which a reinstall (uninstall.sh keeps /etc/felis) or a host rebuilt from a +// database bundle's state/ starts empty. Each file holds the bare value, mode +// 0600, directly in /etc/felis beside secrets.env: every installer run applies +// the Secrets from these files, and every database bundle, so the off-site copy +// too, carries them. +const ( + hostSMTPPasswordPath = "/etc/felis/smtp-password" + hostUploadsS3AccessKeyPath = "/etc/felis/uploads-s3-access-key" + hostUploadsS3SecretKeyPath = "/etc/felis/uploads-s3-secret-key" +) + +// writeHostCredential replaces the file at path with value, mode 0600, through a +// temporary file in the same directory, so a crash leaves the old value or the +// new one and never a partial one. +func writeHostCredential(path, value string) error { + tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*") + if err != nil { + return err + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + // CreateTemp already makes the file 0600; the Chmod states it rather than + // leaning on that. + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return err + } + if _, err := tmp.WriteString(value); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpPath, path) +} + +// readHostCredential returns the value in path; ok is false when there is no +// such file. An empty file is a value: the relay password of a relay without AUTH. +func readHostCredential(path string) (value string, ok bool, err error) { + b, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return "", false, nil + } + if err != nil { + return "", false, err + } + return string(b), true, nil +} + +// relayPassword is the [smtp] relay password as the host holds it: the copy +// `felis setup` keeps at path, else, on an install from before that copy, the +// felis-smtp Secret in ns, whose absence means a relay without AUTH. cl is only +// used when the file is missing; a nil cl then reports errClusterUnreachable. +func relayPassword(ctx context.Context, path string, cl client.Client, ns string) (string, error) { + if pw, ok, err := readHostCredential(path); err != nil || ok { + return pw, err + } + if cl == nil { + return "", errClusterUnreachable + } + return smtpSecretPassword(ctx, cl, ns) +} + +var errClusterUnreachable = errors.New("the cluster did not answer") diff --git a/cmd/felis/hostcreds_test.go b/cmd/felis/hostcreds_test.go new file mode 100644 index 0000000..6bfe150 --- /dev/null +++ b/cmd/felis/hostcreds_test.go @@ -0,0 +1,198 @@ +package main + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/mail" + "felis.lolicon.best/internal/platform" + "felis.lolicon.best/internal/watchdog" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestHostCredentialFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "smtp-password") + + if _, ok, err := readHostCredential(path); ok || err != nil { + t.Fatalf("a missing file read as ok=%v err=%v; want not there", ok, err) + } + // A copy an operator put there by hand, readable by everyone, is tightened. + if err := os.WriteFile(path, []byte("by hand"), 0o644); err != nil { + t.Fatal(err) + } + for _, v := range []string{"first secret", "a \"quoted\" $second\nsecret", ""} { + if err := writeHostCredential(path, v); err != nil { + t.Fatal(err) + } + got, ok, err := readHostCredential(path) + if err != nil || !ok || got != v { + t.Fatalf("read back (%q, %v, %v); want (%q, true, nil)", got, ok, err, v) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if mode := fi.Mode().Perm(); mode != 0o600 { + t.Fatalf("mode %v; want 0600", mode) + } + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("the directory holds %d entries; want the credential alone, no leftover temporary file", len(entries)) + } + + if _, _, err := readHostCredential(dir); err == nil { + t.Fatal("an unreadable credential read as fine") + } +} + +func smtpSecretClient(t *testing.T, password string) client.Client { + t.Helper() + return fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName}, + Data: map[string][]byte{platform.SMTPSecretPasswordKey: []byte(password)}, + }).Build() +} + +func TestRelayPassword(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + host := filepath.Join(dir, "smtp-password") + cl := smtpSecretClient(t, "from-secret") + + if pw, err := relayPassword(ctx, host, cl, "felis"); err != nil || pw != "from-secret" { + t.Fatalf("without a host copy = (%q, %v); want the Secret's", pw, err) + } + if _, err := relayPassword(ctx, host, nil, "felis"); !errors.Is(err, errClusterUnreachable) { + t.Fatalf("without a host copy or a cluster err = %v; want errClusterUnreachable", err) + } + if err := writeHostCredential(host, "from-host"); err != nil { + t.Fatal(err) + } + for _, c := range []client.Client{cl, nil} { + if pw, err := relayPassword(ctx, host, c, "felis"); err != nil || pw != "from-host" { + t.Fatalf("with a host copy (cluster %v) = (%q, %v); want the host copy", c != nil, pw, err) + } + } + if _, err := relayPassword(ctx, dir, cl, "felis"); err == nil { + t.Fatal("an unreadable host copy fell through to the Secret") + } +} + +// The watchdog mails the most while the cluster is down: the host copy must +// reach it then, and without one the password the last good run cached stays. +func TestRefreshSMTPPassword(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + host := filepath.Join(dir, "smtp-password") + var stderr bytes.Buffer + + state := &watchdog.State{SMTPPassword: "cached"} + refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr) + if state.SMTPPassword != "cached" || stderr.Len() != 0 { + t.Fatalf("cluster down, no host copy: password %q, stderr %q; want the cached one kept quietly", state.SMTPPassword, stderr.String()) + } + refreshSMTPPassword(ctx, host, smtpSecretClient(t, "from-secret"), "felis", state, &stderr) + if state.SMTPPassword != "from-secret" { + t.Fatalf("cluster up, no host copy: password %q; want the Secret's", state.SMTPPassword) + } + if err := writeHostCredential(host, "from-host"); err != nil { + t.Fatal(err) + } + refreshSMTPPassword(ctx, host, nil, "felis", state, &stderr) + if state.SMTPPassword != "from-host" { + t.Fatalf("cluster down, host copy: password %q; want the host copy", state.SMTPPassword) + } + refreshSMTPPassword(ctx, dir, nil, "felis", state, &stderr) + if state.SMTPPassword != "from-host" || !strings.Contains(stderr.String(), "keeping the cached one") { + t.Fatalf("unreadable host copy: password %q, stderr %q; want the cached one kept and the failure said", state.SMTPPassword, stderr.String()) + } +} + +func TestHostRecoveryMailerHostCopy(t *testing.T) { + ctx := context.Background() + // No kubeconfig anywhere: reaching for the cluster fails, so a pass proves + // the host copy was enough. + t.Setenv("KUBECONFIG", filepath.Join(t.TempDir(), "no-kubeconfig")) + dir := t.TempDir() + host := filepath.Join(dir, "smtp-password") + if err := writeHostCredential(host, "from-host"); err != nil { + t.Fatal(err) + } + off := false + c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "felis@example.com", Username: "felis", PasswordRef: "FELIS_TEST_UNSET_RELAY_PW", RequireTLS: &off} + + got, err := hostRecoveryMailer(c, host, "felis")(ctx) + if err != nil { + t.Fatalf("with the host copy: %v", err) + } + if relay, ok := got.(*mail.SMTP); !ok || relay.Password != "from-host" { + t.Fatalf("relay = %#v; want the host copy's password", got) + } + if _, err := hostRecoveryMailer(c, dir, "felis")(ctx); err == nil || !strings.Contains(err.Error(), "read the relay password") { + t.Fatalf("unreadable host copy: err = %v; want it named", err) + } + if _, err := hostRecoveryMailer(c, filepath.Join(dir, "none"), "felis")(ctx); err == nil || !strings.Contains(err.Error(), "reach the cluster") { + t.Fatalf("no host copy and no cluster: err = %v; want the cluster named", err) + } +} + +// A whole watchdog run with the API server and PostgreSQL both down still +// takes the relay password from the host copy, so the outage mail can +// authenticate even when no earlier run cached it. +func TestWatchdogReadsHostCopyWhileClusterDown(t *testing.T) { + dir := t.TempDir() + t.Setenv("KUBECONFIG", filepath.Join(dir, "no-kubeconfig")) + cfgPath := filepath.Join(dir, "felis.toml") + if err := os.WriteFile(cfgPath, []byte(`[database] +url = "postgres://felis:pw@127.0.0.1:1/felis?sslmode=disable&connect_timeout=2" +[server] +root_domain = "example.com" +[archive] +store = "tarLocal" +[k8s] +egress_mode = "nodeport" +[smtp] +host = "127.0.0.1" +port = 1 +from = "felis@example.com" +username = "felis" +password_ref = "FELIS_TEST_UNSET_RELAY_PW" +`), 0o600); err != nil { + t.Fatal(err) + } + pwPath := filepath.Join(dir, "smtp-password") + if err := writeHostCredential(pwPath, "from-host"); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(dir, "state.json") + var stdout, stderr bytes.Buffer + cmdWatchdog([]string{ + "-config", cfgPath, "-state", statePath, "-quiet-file", filepath.Join(dir, "quiet"), + "-backup-dir", "", "-disk-paths", dir, "-smtp-password-file", pwPath, + }, &stdout, &stderr) + if !strings.Contains(stdout.String(), "kube-api") { + t.Fatalf("the run found the API server up; the test needs it down (stdout %s)", stdout.String()) + } + state, err := watchdog.LoadState(statePath) + if err != nil { + t.Fatalf("load state: %v (stderr %s)", err, stderr.String()) + } + if state.SMTPPassword != "from-host" { + t.Fatalf("cached relay password %q; want the host copy (stdout %s, stderr %s)", state.SMTPPassword, stdout.String(), stderr.String()) + } +} diff --git a/cmd/felis/tui_smtp.go b/cmd/felis/tui_smtp.go index a00dd17..0bfac85 100644 --- a/cmd/felis/tui_smtp.go +++ b/cmd/felis/tui_smtp.go @@ -276,8 +276,8 @@ func validateSMTPFrom(s string) error { } // currentSMTPInputs reads the relay already recorded in felis.toml so the form -// pre-fills the non-secret fields. The password lives only in the felis-smtp -// Secret and is deliberately never read back — it must be re-entered to change. +// pre-fills the non-secret fields. The password (the felis-smtp Secret and its +// host copy) is deliberately never read back — it must be re-entered to change. // Any read error falls back to a blank form rather than blocking reconfig. func currentSMTPInputs() smtpInputs { cfg, err := config.Load(hostSetupConfigPath) @@ -295,7 +295,8 @@ func currentSMTPInputs() smtpInputs { // applySMTPConfig proves the relay works, then persists it and rolls felis-api: // Ping (a full transaction — connect/STARTTLS/AUTH/MAIL FROM/RCPT/DATA, which // delivers one self-test message to the From address) → [smtp] into both config -// files → the felis-smtp Secret → the config Secret → rollout. A failed Ping +// files → the password into /etc/felis/smtp-password (hostcreds.go) → the +// felis-smtp Secret → the config Secret → rollout. A failed Ping // leaves the install untouched, so a bad relay dies at the keyboard, not at a // player's OTP. // @@ -330,6 +331,11 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error { return err } } + // The host copy first: should the Secret fail, the next installer run applies + // it from this file. + if err := writeHostCredential(hostSMTPPasswordPath, in.password); err != nil { + return fmt.Errorf("keep the relay password in %s: %w", hostSMTPPasswordPath, err) + } if err := applySMTPSecret(ctx, in.password); err != nil { return err } diff --git a/cmd/felis/tui_storage_apply.go b/cmd/felis/tui_storage_apply.go index d217d50..8a9a692 100644 --- a/cmd/felis/tui_storage_apply.go +++ b/cmd/felis/tui_storage_apply.go @@ -53,8 +53,17 @@ func applyStorageConfig(ctx context.Context, method storageMethod, in s3Inputs) return err } // S3: land the credentials in their own Secret BEFORE the roll, so the optional - // env refs resolve on the fresh pod. Local needs no Secret. + // env refs resolve on the fresh pod, and on the host before that, so the next + // installer run can apply the Secret again (hostcreds.go). Local needs no Secret. if method == storageS3 { + for _, c := range []struct{ path, value string }{ + {hostUploadsS3AccessKeyPath, in.accessKey}, + {hostUploadsS3SecretKeyPath, in.secretKey}, + } { + if err := writeHostCredential(c.path, c.value); err != nil { + return fmt.Errorf("keep the bucket credentials in %s: %w", c.path, err) + } + } if err := applyUploadsS3Secret(ctx, in.accessKey, in.secretKey); err != nil { return err } @@ -70,8 +79,9 @@ func applyStorageConfig(ctx context.Context, method storageMethod, in s3Inputs) // currentStorageInputs reads the storage backend already recorded in felis.toml so // the reconfigure flow can pre-select the method and pre-fill the non-secret S3 -// fields (endpoint/bucket/region). Credentials live only in the felis-uploads-s3 -// Secret and are deliberately never read back — they must be re-entered to change. +// fields (endpoint/bucket/region). The credentials (the felis-uploads-s3 Secret +// and its host copies) are deliberately never read back — they must be re-entered +// to change. // Any read error falls back to a blank local default rather than blocking reconfig. func currentStorageInputs() (storageMethod, s3Inputs) { cfg, err := config.Load(hostSetupConfigPath) diff --git a/cmd/felis/watchdog.go b/cmd/felis/watchdog.go index f5ca98f..bc7ede3 100644 --- a/cmd/felis/watchdog.go +++ b/cmd/felis/watchdog.go @@ -36,6 +36,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int { fs.SetOutput(stderr) cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy, which reaches PostgreSQL on 127.0.0.1)") statePath := fs.String("state", "/var/lib/felis/watchdog/state.json", "state kept between runs (root only: it caches the relay password)") + smtpPasswordFile := fs.String("smtp-password-file", hostSMTPPasswordPath, "the relay password `felis setup` keeps on the host; the felis-smtp Secret stands in while it is missing") quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose") backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`) diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space") @@ -88,9 +89,13 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int { report.Unknown = append(report.Unknown, watchdog.ClusterPrefixes...) } else { report.Findings = append(report.Findings, found...) - if cfg.SMTP.Host != "" { - refreshSMTPPassword(ctx, cl, *controlNS, state, stderr) + } + if cfg.SMTP.Host != "" { + var secrets client.Client + if err == nil { + secrets = cl } + refreshSMTPPassword(ctx, *smtpPasswordFile, secrets, *controlNS, state, stderr) } if recipients, err := ownerEmails(ctx, cfg.Database.URL); err != nil { @@ -183,13 +188,19 @@ func usesMirroredScanDB(cfg *config.Config) bool { return repo == "" || strings.HasPrefix(repo, cfg.Registry.URL+"/mirror/") } -// refreshSMTPPassword caches the relay password from the felis-smtp Secret, or -// forgets it when the Secret is gone (a relay without AUTH). An env var named by -// [smtp] password_ref, when set, wins at send time instead. -func refreshSMTPPassword(ctx context.Context, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) { - password, err := smtpSecretPassword(ctx, cl, ns) - if err != nil { - fmt.Fprintf(stderr, "felis watchdog: read %s/%s (keeping the cached relay password): %v\n", ns, platform.SMTPSecretName, err) +// refreshSMTPPassword caches the relay password (relayPassword: the host copy at +// path, else the felis-smtp Secret), or forgets it when the Secret is gone (a +// relay without AUTH). The host copy is read even while the cluster is down, the +// time an alert matters most; without one, a down cluster keeps the cached +// password. An env var named by [smtp] password_ref, when set, wins at send time +// instead. +func refreshSMTPPassword(ctx context.Context, path string, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) { + password, err := relayPassword(ctx, path, cl, ns) + switch { + case errors.Is(err, errClusterUnreachable): + return + case err != nil: + fmt.Fprintf(stderr, "felis watchdog: read the relay password (keeping the cached one): %v\n", err) return } state.SMTPPassword = password diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 84b6888..2fa6594 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -414,6 +414,12 @@ UPDATE_CHECK_SERVICE="/etc/systemd/system/felis-update-check.service" UPDATE_CHECK_TIMER="/etc/systemd/system/felis-update-check.timer" WATCHDOG_STATE="/var/lib/felis/watchdog/state.json" OFFSITE_ENV="${STATE_DIR}/offsite.env" +# Host copies of the credentials `felis setup` takes at the keyboard, one bare value per +# file, mode 0600 (cmd/felis/hostcreds.go); apply_setup_credential_secrets applies their +# Secrets from them on every run. +SMTP_PASSWORD_FILE="${STATE_DIR}/smtp-password" +UPLOADS_S3_ACCESS_KEY_FILE="${STATE_DIR}/uploads-s3-access-key" +UPLOADS_S3_SECRET_KEY_FILE="${STATE_DIR}/uploads-s3-secret-key" OFFSITE_SERVICE="/etc/systemd/system/felis-offsite.service" OFFSITE_TIMER="/etc/systemd/system/felis-offsite.timer" BUILD_TOOLS_SERVICE="/etc/systemd/system/felis-build-tools.service" @@ -666,6 +672,54 @@ apply_registry_secrets() { rm -rf -- "$dir" } +# secret_key_to_file keeps one key of a Secret in path, mode 0600, when path does not +# exist yet. An install from before the host copies had the setup screens' credentials +# only in the cluster; this is the run that moves them onto the host. A missing Secret +# leaves path missing. The value goes from kubectl into the file, never into argv or +# the log. +secret_key_to_file() { + local namespace="$1" name="$2" key="$3" path="$4" encoded tmp + [ -e "$path" ] && return 0 + encoded="$(kube -n "$namespace" get secret "$name" -o "jsonpath={.data.${key}}" 2>/dev/null)" || return 0 + tmp="$(umask 077; mktemp "${path}.XXXXXX")" + remember_temp "$tmp" + printf '%s' "$encoded" | base64 -d > "$tmp" + mv -f -- "$tmp" "$path" +} + +# apply_setup_credential_secrets applies the Secrets behind `felis setup`'s email and +# uploads-bucket screens from their host copies: felis-smtp in the control namespace and +# in the workload one (the reaper's warning mails read that copy), felis-uploads-s3 in the +# control namespace. A reinstall that kept /etc/felis, or a host rebuilt from a bundle's +# state/, starts k3s with no Secrets at all; without this, every sign-in code and alert +# would go out without AUTH and the relay would turn it away. The host copy wins over the +# cluster's: `felis setup` writes both, so they differ only after a hand edit of the +# Secret. A relay or bucket whose credentials are on neither side is reported, so the +# operator enters them again before the first code fails. +apply_setup_credential_secrets() { + local ns + secret_key_to_file "$CONTROL_NS" felis-smtp password "$SMTP_PASSWORD_FILE" + secret_key_to_file "$CONTROL_NS" felis-uploads-s3 access_key_id "$UPLOADS_S3_ACCESS_KEY_FILE" + secret_key_to_file "$CONTROL_NS" felis-uploads-s3 secret_access_key "$UPLOADS_S3_SECRET_KEY_FILE" + if [ -f "$SMTP_PASSWORD_FILE" ]; then + for ns in "$CONTROL_NS" "$MINECRAFT_NS"; do + kube -n "$ns" create secret generic felis-smtp \ + --from-file=password="$SMTP_PASSWORD_FILE" \ + --dry-run=client -o yaml | kube apply -f - + done + elif persisted_smtp_block | grep -Eq '^[[:space:]]*username[[:space:]]*=[[:space:]]*"[^"]'; then + warn "[smtp] signs in with a username, but its password is in neither ${SMTP_PASSWORD_FILE} nor the cluster: sign-in codes and alerts go out without AUTH until it is entered again (sudo felis setup, then e to configure email)" + fi + if [ -f "$UPLOADS_S3_ACCESS_KEY_FILE" ] && [ -f "$UPLOADS_S3_SECRET_KEY_FILE" ]; then + kube -n "$CONTROL_NS" create secret generic felis-uploads-s3 \ + --from-file=access_key_id="$UPLOADS_S3_ACCESS_KEY_FILE" \ + --from-file=secret_access_key="$UPLOADS_S3_SECRET_KEY_FILE" \ + --dry-run=client -o yaml | kube apply -f - + elif persisted_registry_block | grep -Eq '^[[:space:]]*user_uploads_context[[:space:]]*=[[:space:]]*"[sS]3://'; then + warn "uploads go to an S3 bucket, but its keys are in neither ${UPLOADS_S3_ACCESS_KEY_FILE} / ${UPLOADS_S3_SECRET_KEY_FILE} nor the cluster: felis-api refuses modpack uploads until they are entered again (sudo felis setup, then s to change storage)" + fi +} + # node_global_cidrs prints one host-length CIDR per global address on this node. # Game server egress already excludes every private range; this adds the node's # public addresses, which would otherwise let a server dial the panel NodePort, @@ -4777,7 +4831,7 @@ deploy_bundle() { kube create namespace "$ns" --dry-run=client -o yaml | kube apply -f - done - log "provisioning felis-config + internal caller tokens + felis-forwarding-secret + registry credentials + panel TLS secrets (out-of-band, never in the bundle)" + log "provisioning felis-config + internal caller tokens + felis-forwarding-secret + registry credentials + mail relay and uploads bucket credentials + panel TLS secrets (out-of-band, never in the bundle)" apply_felis_config_secrets # felis-api mounts all four caller tokens from the control namespace. The login # gate's and the build Job's are also applied into the namespace their pods run in @@ -4797,6 +4851,7 @@ deploy_bundle() { # "less secure", it is unjoinable. apply_literal_secret "$CONTROL_NS" felis-forwarding-secret secret "$FORWARDING_SECRET" apply_registry_secrets + apply_setup_credential_secrets kube -n "$CONTROL_NS" create secret tls felis-api-tls \ --cert="$PANEL_TLS_CERT" \ --key="$PANEL_TLS_KEY" \ diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index 58ed683..4529634 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -3833,6 +3833,112 @@ expect "a FELIS_ARTIFACT_DIR with SHA256SUMS is accepted" "VALID" "$(run_vs "$ad expect "FELIS_ARTIFACT_DIR and FELIS_REF together are refused" "DIE: FELIS_ARTIFACT_DIR and FELIS_REF both name what to install" "$(run_vs "$adir" 1)" rm -rf "$adir" +# --- the setup screens' credentials come back from /etc/felis --------------------------- +# `felis setup` keeps the relay password and the uploads bucket's keys in /etc/felis as +# well as in their Secrets. A reinstall, or a host rebuilt from a bundle's state/, starts +# k3s empty, so every run applies the Secrets from those files; an install from before +# them moves the Secrets' values into the files first. No value may reach kubectl's argv. +for f in secret_key_to_file apply_setup_credential_secrets; do + [ -n "$(bsfn "$f")" ] || { echo "FAIL: no ${f} in $BS"; exit 1; } + [ "$(bsfn "$f" | wc -l)" -lt 45 ] \ + || { echo "FAIL: the extracted ${f} is not the function -- did its closing brace move?"; exit 1; } +done +grep -q '^ apply_setup_credential_secrets$' "$BS" \ + && echo "PASS every run applies the setup screens' Secrets" \ + || { echo "FAIL: the install no longer calls apply_setup_credential_secrets"; fails=$((fails + 1)); } +credir="$(mktemp -d)" +credcalls="$(mktemp)" +run_creds() { # secrets-in-cluster(0/1) smtp-username uploads-context + : > "$credcalls" + HAVE="$1" SMTPUSER="$2" UPCTX="$3" CALLS="$credcalls" STATE_DIR="$credir" CONTROL_NS=felis MINECRAFT_NS=minecraft bash -c ' + set -Eeuo pipefail + SMTP_PASSWORD_FILE="${STATE_DIR}/smtp-password" + UPLOADS_S3_ACCESS_KEY_FILE="${STATE_DIR}/uploads-s3-access-key" + UPLOADS_S3_SECRET_KEY_FILE="${STATE_DIR}/uploads-s3-secret-key" + remember_temp() { :; } + warn() { printf "WARN: %s\n" "$*"; } + persisted_smtp_block() { printf "[smtp]\nhost = \"mail.example.com\"\nusername = \"%s\"\n" "$SMTPUSER"; } + persisted_registry_block() { printf "user_uploads_context = \"%s\"\n" "$UPCTX"; } + kube() { + case "$*" in + *" get secret "*) + printf "GET %s\n" "$*" >>"$CALLS" + [ "$HAVE" = 1 ] || return 1 + case "$*" in + *felis-smtp*) printf "cmVsYXkgcHcvMSt4" ;; # relay pw/1+x + *access_key_id*) printf "QUtJQU9MRA==" ;; # AKIAOLD + *secret_access_key*) printf "b2xkL3NlY3JldCtrZXk=" ;; # old/secret+key + esac ;; + "apply -f -") cat >/dev/null; printf "APPLY\n" >>"$CALLS" ;; + *) printf "KUBE %s\n" "$*" >>"$CALLS" ;; + esac + } + '"$(bsfn secret_key_to_file)"' + '"$(bsfn apply_setup_credential_secrets)"' + apply_setup_credential_secrets' 2>&1 + cat "$credcalls" +} +applies() { printf '%s\n' "$1" | grep -c '^APPLY$'; } + +out="$(run_creds 1 felis s3://uploads)" +[ "$(cat "$credir/smtp-password")" = "relay pw/1+x" ] \ + && [ "$(cat "$credir/uploads-s3-access-key")" = "AKIAOLD" ] \ + && [ "$(cat "$credir/uploads-s3-secret-key")" = "old/secret+key" ] \ + && echo "PASS an install from before the host copies moves the Secrets' values onto the host" \ + || { echo "FAIL: the Secrets' values did not land in ${credir}:"; printf '%s\n' "$out"; fails=$((fails + 1)); } +for f in smtp-password uploads-s3-access-key uploads-s3-secret-key; do + case "$(ls -l "$credir/$f" | cut -c1-10)" in + -rw-------) echo "PASS ${f} is root's alone" ;; + *) echo "FAIL: ${f} is $(ls -l "$credir/$f" | cut -c1-10), want -rw-------"; fails=$((fails + 1)) ;; + esac +done +[ -z "$(find "$credir" -name '*.??????' | head -1)" ] && echo "PASS the move leaves no temporary file in /etc/felis" \ + || { echo "FAIL: a temporary file stayed behind: $(ls -a "$credir")"; fails=$((fails + 1)); } +expect "the relay password is applied to the control plane" \ + "KUBE -n felis create secret generic felis-smtp --from-file=password=${credir}/smtp-password --dry-run=client -o yaml" "$out" +expect "and to the workload namespace the reaper mails from" \ + "KUBE -n minecraft create secret generic felis-smtp --from-file=password=${credir}/smtp-password --dry-run=client -o yaml" "$out" +expect "the bucket keys are applied to the control plane" \ + "KUBE -n felis create secret generic felis-uploads-s3 --from-file=access_key_id=${credir}/uploads-s3-access-key --from-file=secret_access_key=${credir}/uploads-s3-secret-key --dry-run=client -o yaml" "$out" +[ "$(applies "$out")" = 3 ] && echo "PASS all three Secrets are piped to kubectl apply" \ + || { echo "FAIL: expected 3 applies:"; printf '%s\n' "$out"; fails=$((fails + 1)); } +case "$out" in + *"relay pw"*|*AKIAOLD*|*"old/secret"*|*WARN*) echo "FAIL: a credential reached argv or the log, or a warning fired:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;; + *) echo "PASS no credential reaches kubectl's argv or the log" ;; +esac + +printf 'new pw' > "$credir/smtp-password" +out="$(run_creds 1 felis s3://uploads)" +[ "$(cat "$credir/smtp-password")" = "new pw" ] && ! printf '%s\n' "$out" | grep -q '^GET' \ + && echo "PASS the host copy wins over the cluster's" \ + || { echo "FAIL: the cluster's value replaced the host copy:"; printf '%s\n' "$out"; fails=$((fails + 1)); } +[ "$(applies "$out")" = 3 ] && echo "PASS a re-run applies the Secrets from the host copies" \ + || { echo "FAIL: expected 3 applies on the re-run:"; printf '%s\n' "$out"; fails=$((fails + 1)); } + +rm -f "$credir"/* +out="$(run_creds 0 felis s3://uploads)" +expect "a relay that signs in with no password anywhere is reported" \ + "WARN: [smtp] signs in with a username, but its password is in neither ${credir}/smtp-password nor the cluster" "$out" +expect "a bucket with no keys anywhere is reported" \ + "WARN: uploads go to an S3 bucket, but its keys are in neither" "$out" +[ "$(applies "$out")" = 0 ] && [ -z "$(ls -A "$credir")" ] \ + && echo "PASS nothing is applied or written without a value" \ + || { echo "FAIL: something was applied or written without a value:"; printf '%s\n' "$out"; ls -A "$credir"; fails=$((fails + 1)); } + +out="$(run_creds 0 "" /var/lib/felis/uploads)" +case "$out" in + *WARN*|*APPLY*) echo "FAIL: a relay without AUTH and local uploads were reported or applied:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;; + *) echo "PASS a relay without AUTH and local uploads need no credentials" ;; +esac + +printf 'AKIAONLY' > "$credir/uploads-s3-access-key" +out="$(run_creds 0 "" s3://uploads)" +case "$out" in + *"create secret generic felis-uploads-s3"*) echo "FAIL: a bucket Secret was applied from one key alone:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;; + *) expect "one bucket key alone is reported as missing keys" "WARN: uploads go to an S3 bucket" "$out" ;; +esac +rm -rf "$credir" "$credcalls" + # --------------------------------------------------------------------------------------- if [ "$fails" -eq 0 ]; then echo "ALL PASS" diff --git a/docs/operations.md b/docs/operations.md index 798a2b3..90f3ff1 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -273,7 +273,7 @@ and the MinecraftServer CRD are deleted. | Final database bundle | taken first (`felis db backup -label manual`); a failure stops the uninstall before anything is removed. `--no-backup` skips it | none | | The database (`/var/lib/felis/postgres`) | kept; felis-postgres is stopped cleanly before k3s goes | deleted with `/var/lib/felis` | | A host PostgreSQL an earlier release ran the database on | kept as it is: stopped after the move into k3s (below, §4), with its old copy of `felis` | its `felis` database and role are dropped (the server is started for that and stopped again), and `listen_addresses` and `pg_hba.conf` go back to how they were. Checked before anything is removed: a role that still owns another database (the `felis_pgint` the PG contract tests use, CONTRIBUTING.md) or holds grants elsewhere stops the purge up front with the list and the `ALTER DATABASE … OWNER TO postgres` to run | -| `/etc/felis` (secrets, `felis.toml`, `offsite.env`, tunnel config) | kept; `bootstrap.done` and the per-run records go | deleted, with the tunnel's credentials file | +| `/etc/felis` (secrets, `felis.toml`, `offsite.env`, the mail relay password and uploads bucket keys `felis setup` took, tunnel config) | kept; `bootstrap.done` and the per-run records go | deleted, with the tunnel's credentials file | | `/var/lib/felis` (the database, its bundles) | kept | deleted | | Worlds, archives, registry, uploads | moved to `/var/lib/felis/retained/k3s-storage-/` (with `--keep-k3s`: their volumes switch to `Retain` and stay in place) | deleted | | Felis images, Docker build cache | kept | deleted | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 03c1e1b..06b7e7b 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1101,6 +1101,20 @@ control-namespace Secret alone is not. [GO-TESTED: the delivered/retried/ suppressed matrix in `internal/reaper`; live-drilled end to end against a local SMTP sink.] +The screen also keeps the password in `/etc/felis/smtp-password` (mode 0600), +and the uploads screen keeps the bucket keys in `/etc/felis/uploads-s3-access-key` +and `uploads-s3-secret-key`. Secrets live in k3s's datastore, which a reinstall +or a host rebuilt from a bundle's `state/` starts empty, so **every installer +run applies `felis-smtp` (both namespaces) and `felis-uploads-s3` from these +files**. The files win: a hand edit of either Secret lasts until the next +installer run, so change a credential through `felis setup`. An install from +before these files gets them from the Secrets on its first re-run. When +`[smtp]` has a `username` but no password is on either side, or uploads go to +`s3://` without both keys, the installer says so and names the `felis setup` +screen that takes them again. [SH-TESTED: `deploy/bootstrap_test.sh`, the move +from the Secrets, the host copy winning, both warnings, no value in kubectl's +argv. GO-TESTED: the host copy's mode and replacement.] + ### Genuine false-delete risk vectors - **Stale `last_active_at`.** The keep-alive is `RecordJoin`, called from the @@ -1510,9 +1524,12 @@ How it mails: delay is never mailed; one that turns critical is mailed again at once. - **During an install** nothing is mailed. `bootstrap.sh` writes `/run/felis/watchdog-quiet-until` and removes it when it exits. -- **Caching:** the relay password and the recipient list are cached in - `/var/lib/felis/watchdog/state.json` (root-only). An outage of PostgreSQL or - of the API server can therefore still be mailed. +- **Caching:** the relay password comes from `/etc/felis/smtp-password`, which + the watchdog reads even while the API server is down (an install without that + file reads the `felis-smtp` Secret instead). It and the recipient list are + cached in `/var/lib/felis/watchdog/state.json` (root-only). An outage of + PostgreSQL or of the API server can therefore still be mailed. [GO-TESTED: a + run with the API server and PostgreSQL both down caches the host copy.] - **No relay or no verified owner address:** each alert is written to the journal only. @@ -1893,7 +1910,7 @@ along). One bundle is `felis-db--