From 8bc3997a8b9a611430d242a059b733bb533506bb Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Sat, 26 Sep 2026 08:27:00 +0800 Subject: [PATCH] =?UTF-8?q?feat(breakglass):=20=E6=81=A2=E5=A4=8D=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E7=94=A8=E9=82=AE=E4=BB=B6=E9=AA=8C=E8=AF=81=E7=A0=81?= =?UTF-8?q?=E8=AF=81=E6=98=8E=E7=AE=A1=E7=90=86=E5=91=98=E8=BA=AB=E4=BB=BD?= =?UTF-8?q?=EF=BC=8C=E5=8F=91=E4=B8=8D=E5=87=BA=E6=88=96=E9=AA=8C=E4=B8=8D?= =?UTF-8?q?=E8=BF=87=E8=B5=B0=E5=B8=A6=E5=8E=9F=E5=9B=A0=E5=AE=A1=E8=AE=A1?= =?UTF-8?q?=E7=9A=84=20OVERRIDE=20(#13)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/felis/breakglass.go | 118 ++++---- cmd/felis/breakglass_otp.go | 252 ++++++++++++++++ cmd/felis/breakglass_otp_test.go | 498 +++++++++++++++++++++++++++++++ cmd/felis/breakglass_test.go | 96 +++--- cmd/felis/tui_owner.go | 175 +++++++++-- cmd/felis/tui_root.go | 5 +- cmd/felis/watchdog.go | 24 +- docs/troubleshooting.md | 37 +++ 8 files changed, 1071 insertions(+), 134 deletions(-) create mode 100644 cmd/felis/breakglass_otp.go create mode 100644 cmd/felis/breakglass_otp_test.go diff --git a/cmd/felis/breakglass.go b/cmd/felis/breakglass.go index df22cc5..751e0e7 100644 --- a/cmd/felis/breakglass.go +++ b/cmd/felis/breakglass.go @@ -17,6 +17,7 @@ import ( "felis.lolicon.best/internal/api" "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/platform" "felis.lolicon.best/internal/store" tea "github.com/charmbracelet/bubbletea" @@ -33,13 +34,15 @@ import ( // // Root is necessary but NOT sufficient for accountability: root is machine // authority, not a human identity, so the console additionally captures WHO is -// breaking the glass. When a staff account already exists it asks the operator to -// authenticate as an existing admin (the verified identity is the accountable -// actor); when none exists yet it bootstraps the first Owner from the typed -// credential and attributes the act to the OS user. The audit row records the -// difference. This attribution is best-effort, not tamper-proof — whoever runs -// this is root and can edit Postgres directly — but it produces an honest trail -// for an honest operator, which is the point. +// breaking the glass. When a staff account already exists the operator names one +// and types the one-time code the console mails to its verified address +// (breakglass_otp.go); that account is then the accountable actor. When no code can +// be sent or proven, the typed OVERRIDE proceeds as the OS user and the audit row +// says why. When no staff account exists yet it bootstraps the first Owner and +// attributes the act to the OS user. The audit row records which of these +// happened. This attribution is best-effort, not tamper-proof — whoever runs this +// is root and can edit Postgres directly — but it produces an honest trail for an +// honest operator, which is the point. // // When a staff account already exists the console opens on a thin top-level menu // (menuModel) so that operations are peers, not tails of one wizard. Two account @@ -62,7 +65,7 @@ import ( // suspension for the interactive `cloudflared tunnel login` browser consent. // breakGlassOverrideToken is the literal an operator must type to proceed when no -// admin credential could be verified. Requiring an explicit, deliberate word (not a +// admin could be verified by a mailed code. Requiring an explicit, deliberate word (not a // bare Enter) keeps the unverified root override from happening by reflex. const breakGlassOverrideToken = "OVERRIDE" @@ -142,7 +145,7 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int { repo := api.NewPGRepo(drv.DB()) // Decide bootstrap (no admin yet → typed credential mints the first Owner) vs - // recovery (an admin exists → the operator must authenticate as one) BEFORE the + // recovery (an admin exists → the operator proves one with a mailed code) BEFORE the // alt-screen TUI takes over, so a database fault surfaces as a plain error. adminExists, err := repo.AdminExists(ctx) if err != nil { @@ -150,7 +153,12 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int { return 1 } - res, err := runBreakGlassTUI(ctx, repo, cfg.Database.URL, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists) + // Recovery mails its code through [smtp]; the relay is opened only if a code is + // asked for. + host, _ := os.Hostname() + recovery := recoveryConfig{open: hostRecoveryMailer(cfg.SMTP, platform.DefaultControlNamespace), host: host} + + res, err := runBreakGlassTUI(ctx, repo, cfg.Database.URL, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists, recovery) if err != nil { fmt.Fprintf(stderr, "felis breakGlass: %v\n", err) return 1 @@ -174,6 +182,9 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stdout, "\nfelis breakGlass: Owner account %q provisioned; local session sign-in is ENABLED.\n", res.username) } fmt.Fprintf(stdout, "Recorded as %q (mode: %s, os user: %s).\n", res.accountable, res.mode, res.osUser) + if res.mode == "root_override" { + fmt.Fprintln(stdout, "No admin was proven by an email code; the audit row records this run as an unverified root override and why.") + } if res.setupTokenURL != "" { fmt.Fprintf(stdout, "One-time setup URL (opens a lockdown session to verify email / enroll passkey):\n\n %s\n\n", res.setupTokenURL) } @@ -258,29 +269,6 @@ func newOwnerID() string { return "usr-" + hex.EncodeToString(b[:]) } -// authenticateAdmin resolves a typed admin username for recovery-mode attribution. -// Password verification is gone (passwordless design); Phase 3 replaces this with -// email-OTP recovery. For now it confirms the named admin exists. -func authenticateAdmin(ctx context.Context, s ownerStore, username string) (matched string, ok bool, err error) { - username = strings.TrimSpace(username) - if username == "" { - return "", false, nil - } - u, err := s.UserByUsername(ctx, username) - if errors.Is(err, api.ErrNotFound) { - return "", false, nil - } - if err != nil { - return "", false, err - } - // Staff means admin OR owner: recovery attribution must accept the Owner (the - // primary break-glass identity), not just plain admins. - if u.Role != "admin" && u.Role != "owner" { - return "", false, nil - } - return u.Username, true, nil -} - // provisionOwner mints or resets the single Owner account direct-to-Postgres, // passwordless. The account is role=owner with no password — the Owner completes // passwordless login setup via the web setup-token flow after `felis setup`. @@ -371,6 +359,10 @@ type breakGlassOp struct { ownerUsername string ownerEmail string attemptedAdmin string // recovery / override: the admin username the operator typed + verifiedBy string // recovery: how the admin was proven (verifiedByEmailOTP) + codeSentTo string // recovery: the address the proving code went to + otpSkipped string // root_override: why no code proved an admin (otpSkip*) + otpSkipDetail string // root_override: what failed, when something did } // breakGlassOutcome is what performBreakGlass reports back to the TUI. @@ -494,16 +486,7 @@ func auditSetupMCBind(ctx context.Context, s ownerStore, osUser, mcUUID, authSou // does not fail the recovery if this write fails — and intentionally honest: it // records attribution, it does not prove it (a malicious root can edit the row). func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error { - payload := map[string]any{ - "mode": op.mode, - "owner": op.ownerUsername, - "os_user": op.osUser, - "verified": op.mode == "recovery", - } - if op.attemptedAdmin != "" { - payload["admin_account"] = op.attemptedAdmin - } - blob, err := json.Marshal(payload) + blob, err := json.Marshal(breakGlassPayload(op, "owner")) if err != nil { return err } @@ -515,6 +498,33 @@ func auditBreakGlass(ctx context.Context, s ownerStore, op breakGlassOp) error { }) } +// breakGlassPayload is the who/how both account audits carry, with the account the +// run wrote under subjectKey. verified is true only for a run a mailed code proved; +// such a run names the address the code went to, and an override names why no code +// proved anyone. +func breakGlassPayload(op breakGlassOp, subjectKey string) map[string]any { + payload := map[string]any{ + "mode": op.mode, + subjectKey: op.ownerUsername, + "os_user": op.osUser, + "verified": op.verifiedBy != "", + } + if op.attemptedAdmin != "" { + payload["admin_account"] = op.attemptedAdmin + } + if op.verifiedBy != "" { + payload["verified_by"] = op.verifiedBy + payload["code_sent_to"] = op.codeSentTo + } + if op.otpSkipped != "" { + payload["otp_skipped"] = op.otpSkipped + if op.otpSkipDetail != "" { + payload["otp_skip_detail"] = op.otpSkipDetail + } + } + return payload +} + // performAddOperator mints a NEW Operator account and records a best-effort // accountability row. It mirrors performBreakGlass — passwordless — with two // deliberate differences. (1) It provisions insert-only (provisionOperator), so it @@ -538,16 +548,7 @@ func performAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) (bre // break_glass.operator_create action, naming the new account under an "operator" key // rather than "owner". func auditAddOperator(ctx context.Context, s ownerStore, op breakGlassOp) error { - payload := map[string]any{ - "mode": op.mode, - "operator": op.ownerUsername, - "os_user": op.osUser, - "verified": op.mode == "recovery", - } - if op.attemptedAdmin != "" { - payload["admin_account"] = op.attemptedAdmin - } - blob, err := json.Marshal(payload) + blob, err := json.Marshal(breakGlassPayload(op, "operator")) if err != nil { return err } @@ -626,16 +627,19 @@ const ( cloudflareAPITokenDocsURL = "https://developers.cloudflare.com/fundamentals/api/how-to/account-owned-token-template/" ) -func runBreakGlassTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) { - return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass) +func runBreakGlassTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, recovery recoveryConfig) (breakGlassResult, error) { + return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass, recovery) } +// runSetupTUI never reaches recovery: setup with a staff account present lands on +// the status screen, so it has no relay to hand over. func runSetupTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) { - return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup) + return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup, recoveryConfig{}) } -func runConsoleTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) (breakGlassResult, error) { +func runConsoleTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode, recovery recoveryConfig) (breakGlassResult, error) { rm := newRootModel(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode) + rm.recovery = recovery final, err := tea.NewProgram(rm, tea.WithAltScreen()).Run() if err != nil { return breakGlassResult{}, err diff --git a/cmd/felis/breakglass_otp.go b/cmd/felis/breakglass_otp.go new file mode 100644 index 0000000..e89cd55 --- /dev/null +++ b/cmd/felis/breakglass_otp.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "errors" + "fmt" + "math/big" + "os" + "strings" + "time" + + "felis.lolicon.best/internal/api" + "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/platform" +) + +// Recovery mode proves who is breaking the glass (#13). Naming a staff account is +// where it starts: the console then mails a one-time code to that account's verified +// address, and only that code, typed within recoveryCodeTTL, makes the run a +// recovery attributed to the account. Every other ending — no such account, no +// verified address, no relay, a send that fails, a wrong or late code, or the +// operator giving up on the mail — leads to the typed OVERRIDE, which the audit row +// records as an unverified root_override together with the reason (otp_skipped). +// The code goes through the same [smtp] relay as the panel's login codes, so with +// that relay down recovery still works, as an override that says why. + +const ( + recoveryCodeTTL = 10 * time.Minute + recoveryCodeAttempts = 5 +) + +// The reasons a run fell back to the override, recorded as otp_skipped. +const ( + otpSkipUnknownAdmin = "unknown_admin" + otpSkipNoVerifiedEmail = "no_verified_email" + otpSkipNoRelay = "no_relay" + otpSkipSendFailed = "send_failed" + otpSkipCodeExpired = "code_expired" + otpSkipCodeRejected = "code_rejected" + otpSkipByOperator = "operator_skipped" +) + +// verifiedByEmailOTP is the audit's verified_by for a recovery the mailed code proved. +const verifiedByEmailOTP = "email_otp" + +// recoveryMailer is the one relay call a recovery code needs; *mail.SMTP has it. +type recoveryMailer interface { + SendNotice(ctx context.Context, email, subject, body string) error +} + +// recoveryConfig is what the console needs to mail a recovery code. open resolves +// the relay only when a code is about to go out, so a console used to halt a server +// never touches [smtp] or the cluster; its error says why no relay is available. +// host names this machine in the mail. The zero value has no relay. +type recoveryConfig struct { + open func(ctx context.Context) (recoveryMailer, error) + host string + now func() time.Time +} + +func (r recoveryConfig) clock() time.Time { + if r.now != nil { + return r.now() + } + return time.Now() +} + +// recoveryCode is one mailed code: its value, when it stops working, and how many +// wrong codes were typed against it. +type recoveryCode struct { + value string + expires time.Time + failures int +} + +func newRecoveryCode(now time.Time) (*recoveryCode, error) { + n, err := rand.Int(rand.Reader, big.NewInt(1_000_000)) + if err != nil { + return nil, fmt.Errorf("generate recovery code: %w", err) + } + return &recoveryCode{value: fmt.Sprintf("%06d", n.Int64()), expires: now.Add(recoveryCodeTTL)}, nil +} + +type codeVerdict int + +const ( + codeAccepted codeVerdict = iota + codeWrong + codeExpired + codeExhausted +) + +// check compares a typed code in constant time. Each wrong code counts; the one +// that reaches recoveryCodeAttempts exhausts the code, which then accepts nothing, +// and neither does an expired one. +func (c *recoveryCode) check(typed string, now time.Time) codeVerdict { + if c.failures >= recoveryCodeAttempts { + return codeExhausted + } + if !now.Before(c.expires) { + return codeExpired + } + if subtle.ConstantTimeCompare([]byte(strings.TrimSpace(typed)), []byte(c.value)) == 1 { + return codeAccepted + } + c.failures++ + if c.failures >= recoveryCodeAttempts { + return codeExhausted + } + return codeWrong +} + +func (c *recoveryCode) attemptsLeft() int { return recoveryCodeAttempts - c.failures } + +// recoveryStart is where naming an admin led: a code on its way to that admin, or +// the reason the run has to fall back to the override. +type recoveryStart struct { + admin *api.StaffUser // the named staff account; nil when none matched + code *recoveryCode // set when the code went out + skip string // otpSkip* when it did not + detail string // what failed, for the override screen and the audit row +} + +// resolveAdmin loads the staff account (admin or owner) a typed username names, or +// nil when there is none. +func resolveAdmin(ctx context.Context, s ownerStore, username string) (*api.StaffUser, error) { + username = strings.TrimSpace(username) + if username == "" { + return nil, nil + } + u, err := s.UserByUsername(ctx, username) + if errors.Is(err, api.ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + // Staff means admin or owner: the Owner is the primary break-glass identity. + if u.Role != "admin" && u.Role != "owner" { + return nil, nil + } + return u, nil +} + +// beginRecovery resolves the named admin and mails it a recovery code. Only a +// datastore or entropy fault is an error; every other way the code cannot go out is +// a recoveryStart with skip set. +func beginRecovery(ctx context.Context, s ownerStore, rc recoveryConfig, username, osUser string, op bgOperation) (recoveryStart, error) { + admin, err := resolveAdmin(ctx, s, username) + if err != nil { + return recoveryStart{}, err + } + if admin == nil { + return recoveryStart{skip: otpSkipUnknownAdmin}, nil + } + st := recoveryStart{admin: admin} + // An address nobody ever proved vouches for nobody. + email := strings.TrimSpace(admin.Email) + if email == "" || !admin.EmailVerified { + st.skip = otpSkipNoVerifiedEmail + return st, nil + } + if rc.open == nil { + st.skip, st.detail = otpSkipNoRelay, "this console has no mail relay" + return st, nil + } + relay, err := rc.open(ctx) + if err != nil { + st.skip, st.detail = otpSkipNoRelay, err.Error() + return st, nil + } + code, err := newRecoveryCode(rc.clock()) + if err != nil { + return recoveryStart{}, err + } + sendCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + subject, body := recoveryMail(code.value, rc.host, osUser, admin.Username, op) + if err := relay.SendNotice(sendCtx, email, subject, body); err != nil { + st.skip, st.detail = otpSkipSendFailed, err.Error() + return st, nil + } + st.code = code + return st, nil +} + +// recoveryMail words the code mail. It says where, by whom and for what the console +// was opened, so an admin who did not ask for it learns that root on that machine is +// in someone else's hands. +func recoveryMail(code, host, osUser, admin string, op bgOperation) (subject, body string) { + what, whatZH := "reset the Owner account", "重置 Owner 账号" + if op == bgAddOperator { + what, whatZH = "add an Operator account", "添加 Operator 账号" + } + if host == "" { + host = "the Felis host" + } + minutes := int(recoveryCodeTTL / time.Minute) + subject = "Felis break-glass recovery code / 紧急恢复验证码" + body = fmt.Sprintf(`Someone with root on %[1]s (OS user %[2]s) opened felis breakGlass and named your staff account %[3]q to %[4]s. + +Recovery code: %[6]s +It works for %[7]d minutes. + +If this was not you, root on that machine is in someone else's hands: change its credentials and read the audit log for break_glass entries. + +有人在 %[1]s 上以 root 身份（系统用户 %[2]s）打开了 felis breakGlass，指名你的管理员账号 %[3]q 来%[5]s。 + +恢复验证码：%[6]s +%[7]d 分钟内有效。 + +如果不是你本人，这台机器的 root 已落入他人之手：请更换它的凭据，并查看审计日志中的 break_glass 记录。 +`, host, osUser, admin, what, whatZH, code, minutes) + return subject, body +} + +// maskEmail keeps the first character of the local part and the domain, enough for +// the operator to recognise the address without putting it on screen whole. +func maskEmail(email string) string { + at := strings.LastIndex(email, "@") + if at <= 0 { + return "***" + } + return email[:1] + strings.Repeat("*", max(at-1, 3)) + email[at:] +} + +// hostRecoveryMailer opens the [smtp] relay from the host the way the watchdog does: +// the password is the env var password_ref names when that is set, else the +// felis-smtp Secret, whose absence means a relay without AUTH. +func hostRecoveryMailer(c config.SMTPConfig, controlNS string) func(context.Context) (recoveryMailer, error) { + return func(ctx context.Context) (recoveryMailer, error) { + if strings.TrimSpace(c.Host) == "" { + return nil, errors.New("[smtp] is not configured in felis.toml") + } + if ref := c.PasswordRef; ref != "" && os.Getenv(ref) != "" { + return smtpRelay(c, os.Getenv(ref)), nil + } + cl, err := buildSystemServerClient() + if err != nil { + return nil, fmt.Errorf("reach the cluster for the relay password: %w", err) + } + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + password, err := smtpSecretPassword(ctx, cl, controlNS) + if err != nil { + return nil, fmt.Errorf("read the relay password from %s/%s: %w", controlNS, platform.SMTPSecretName, err) + } + return smtpRelay(c, password), nil + } +} diff --git a/cmd/felis/breakglass_otp_test.go b/cmd/felis/breakglass_otp_test.go new file mode 100644 index 0000000..bb9dab8 --- /dev/null +++ b/cmd/felis/breakglass_otp_test.go @@ -0,0 +1,498 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "felis.lolicon.best/internal/api" + "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/mail" + "felis.lolicon.best/internal/platform" + + tea "github.com/charmbracelet/bubbletea" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +// These tests cover the recovery proof (#13): the mailed code's rules, where +// naming an admin leads, what the mail says, how the console walks from a name to +// a proven (or overridden) run, and what the audit row then records. No mail +// leaves the process: the relay is a fake that keeps what it was handed. + +type sentMail struct{ to, subject, body string } + +type fakeRelay struct { + sent []sentMail + err error +} + +func (r *fakeRelay) SendNotice(_ context.Context, to, subject, body string) error { + if r.err != nil { + return r.err + } + r.sent = append(r.sent, sentMail{to, subject, body}) + return nil +} + +// sentCode pulls the code out of the one mail the relay carried. +func (r *fakeRelay) sentCode(t *testing.T) string { + t.Helper() + if len(r.sent) != 1 { + t.Fatalf("relay carried %d mails, want 1", len(r.sent)) + } + _, after, ok := strings.Cut(r.sent[0].body, "Recovery code: ") + if !ok || len(after) < 6 { + t.Fatalf("mail carries no recovery code:\n%s", r.sent[0].body) + } + return after[:6] +} + +func relayConfig(r *fakeRelay, now func() time.Time) recoveryConfig { + return recoveryConfig{ + open: func(context.Context) (recoveryMailer, error) { return r, nil }, + host: "felis-host-1", + now: now, + } +} + +func verifiedAdmin(username, email string) *api.StaffUser { + return &api.StaffUser{ID: "usr-" + username, Username: username, Role: "owner", Email: email, EmailVerified: true} +} + +func TestRecoveryCodeRules(t *testing.T) { + t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC) + + t.Run("a fresh code is six digits and lives for the TTL", func(t *testing.T) { + seen := map[string]bool{} + for i := 0; i < 20; i++ { + c, err := newRecoveryCode(t0) + if err != nil { + t.Fatal(err) + } + if !isRecoveryCodeShape(c.value) { + t.Fatalf("code %q is not six digits", c.value) + } + if !c.expires.Equal(t0.Add(recoveryCodeTTL)) { + t.Fatalf("expires = %v, want %v", c.expires, t0.Add(recoveryCodeTTL)) + } + seen[c.value] = true + } + if len(seen) < 2 { + t.Error("twenty codes were all the same") + } + }) + + t.Run("the right code is accepted, surrounding space ignored", func(t *testing.T) { + c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)} + if v := c.check(" 042917 ", t0); v != codeAccepted { + t.Errorf("check = %v, want accepted", v) + } + }) + + t.Run("wrong codes count down and the last one exhausts it", func(t *testing.T) { + c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)} + for i := 1; i < recoveryCodeAttempts; i++ { + if v := c.check("000000", t0); v != codeWrong { + t.Fatalf("wrong code %d: check = %v, want wrong", i, v) + } + if left := c.attemptsLeft(); left != recoveryCodeAttempts-i { + t.Fatalf("after %d wrong codes attemptsLeft = %d, want %d", i, left, recoveryCodeAttempts-i) + } + } + if v := c.check("000000", t0); v != codeExhausted { + t.Fatalf("wrong code %d: check = %v, want exhausted", recoveryCodeAttempts, v) + } + if v := c.check("042917", t0); v != codeExhausted { + t.Errorf("the right code after exhaustion: check = %v, want exhausted", v) + } + }) + + t.Run("an expired code accepts nothing", func(t *testing.T) { + c := &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)} + if v := c.check("042917", t0.Add(recoveryCodeTTL-time.Second)); v != codeAccepted { + t.Fatalf("a second before expiry: check = %v, want accepted", v) + } + c = &recoveryCode{value: "042917", expires: t0.Add(recoveryCodeTTL)} + if v := c.check("042917", t0.Add(recoveryCodeTTL)); v != codeExpired { + t.Errorf("at expiry: check = %v, want expired", v) + } + }) +} + +func TestBeginRecovery(t *testing.T) { + ctx := context.Background() + t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC) + clock := func() time.Time { return t0 } + + t.Run("mails a code to the named admin's verified address", func(t *testing.T) { + f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "root@example.com")}} + r := &fakeRelay{} + st, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgAddOperator) + if err != nil { + t.Fatal(err) + } + if st.code == nil || st.skip != "" { + t.Fatalf("start = %+v, want a code and no skip", st) + } + if st.admin == nil || st.admin.Username != "root" { + t.Fatalf("start.admin = %+v, want root", st.admin) + } + if got := r.sentCode(t); got != st.code.value { + t.Errorf("mailed code %q, want the code the console checks (%q)", got, st.code.value) + } + m := r.sent[0] + if m.to != "root@example.com" { + t.Errorf("mail went to %q, want root@example.com", m.to) + } + for _, want := range []string{"felis-host-1", "OS user alice", `"root"`, "add an Operator account", "添加 Operator 账号", "10 minutes"} { + if !strings.Contains(m.body, want) { + t.Errorf("mail body lacks %q:\n%s", want, m.body) + } + } + if !st.code.expires.Equal(t0.Add(recoveryCodeTTL)) { + t.Errorf("code expires %v, want %v", st.code.expires, t0.Add(recoveryCodeTTL)) + } + }) + + t.Run("the Owner reset is named as such", func(t *testing.T) { + f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": verifiedAdmin("root", "root@example.com")}} + r := &fakeRelay{} + if _, err := beginRecovery(ctx, f, relayConfig(r, clock), "root", "alice", bgProvisionOwner); err != nil { + t.Fatal(err) + } + if body := r.sent[0].body; !strings.Contains(body, "reset the Owner account") || !strings.Contains(body, "重置 Owner 账号") { + t.Errorf("mail body does not name the Owner reset:\n%s", body) + } + }) + + // Each way the code cannot go out ends in a skip reason and no mail. + unverified := verifiedAdmin("root", "root@example.com") + unverified.EmailVerified = false + noEmail := verifiedAdmin("root", "") + cases := []struct { + name string + user *api.StaffUser + rc func(r *fakeRelay) recoveryConfig + skip string + detail string + wantAdmin bool + relayError error + }{ + {name: "unknown admin", user: nil, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipUnknownAdmin}, + {name: "unverified address", user: unverified, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true}, + {name: "no address", user: noEmail, rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, skip: otpSkipNoVerifiedEmail, wantAdmin: true}, + {name: "no relay wired", user: verifiedAdmin("root", "root@example.com"), rc: func(*fakeRelay) recoveryConfig { return recoveryConfig{} }, skip: otpSkipNoRelay, detail: "this console has no mail relay", wantAdmin: true}, + {name: "relay cannot open", user: verifiedAdmin("root", "root@example.com"), rc: func(*fakeRelay) recoveryConfig { + return recoveryConfig{open: func(context.Context) (recoveryMailer, error) { + return nil, errors.New("[smtp] is not configured in felis.toml") + }} + }, skip: otpSkipNoRelay, detail: "[smtp] is not configured in felis.toml", wantAdmin: true}, + {name: "send fails", user: verifiedAdmin("root", "root@example.com"), rc: func(r *fakeRelay) recoveryConfig { return relayConfig(r, clock) }, + skip: otpSkipSendFailed, detail: "554 relay refused", wantAdmin: true, relayError: errors.New("554 relay refused")}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := &fakeOwnerStore{users: map[string]*api.StaffUser{}} + if tc.user != nil { + f.users["root"] = tc.user + } + r := &fakeRelay{err: tc.relayError} + st, err := beginRecovery(ctx, f, tc.rc(r), "root", "alice", bgProvisionOwner) + if err != nil { + t.Fatal(err) + } + if st.code != nil || st.skip != tc.skip || st.detail != tc.detail { + t.Errorf("start = {code:%v skip:%q detail:%q}, want no code, skip %q, detail %q", st.code, st.skip, st.detail, tc.skip, tc.detail) + } + if (st.admin != nil) != tc.wantAdmin { + t.Errorf("start.admin = %+v, want present=%v", st.admin, tc.wantAdmin) + } + if len(r.sent) != 0 { + t.Errorf("relay carried %d mails, want none", len(r.sent)) + } + }) + } + + t.Run("a datastore fault is an error", func(t *testing.T) { + f := &fakeOwnerStore{userErr: errors.New("db down")} + if _, err := beginRecovery(ctx, f, relayConfig(&fakeRelay{}, clock), "root", "alice", bgProvisionOwner); err == nil { + t.Fatal("want the store fault") + } + }) +} + +func TestMaskEmail(t *testing.T) { + for in, want := range map[string]string{ + "alice@example.com": "a****@example.com", + "al@example.com": "a***@example.com", + "@example.com": "***", + "nonsense": "***", + } { + if got := maskEmail(in); got != want { + t.Errorf("maskEmail(%q) = %q, want %q", in, got, want) + } + } +} + +func TestHostRecoveryMailer(t *testing.T) { + ctx := context.Background() + + t.Run("no [smtp] host is no relay", func(t *testing.T) { + _, err := hostRecoveryMailer(config.SMTPConfig{}, "felis")(ctx) + if err == nil || !strings.Contains(err.Error(), "[smtp]") { + t.Fatalf("err = %v, want it to name [smtp]", err) + } + }) + + t.Run("the password_ref env var supplies the password", func(t *testing.T) { + t.Setenv("FELIS_TEST_RELAY_PW", "from-env") + off := false + c := config.SMTPConfig{Host: "mail.example.com", Port: 2525, From: "felis@example.com", Username: "felis", PasswordRef: "FELIS_TEST_RELAY_PW", RequireTLS: &off} + got, err := hostRecoveryMailer(c, "felis")(ctx) + if err != nil { + t.Fatal(err) + } + relay, ok := got.(*mail.SMTP) + if !ok { + t.Fatalf("relay is %T, want *mail.SMTP", got) + } + if relay.Password != "from-env" || relay.Host != "mail.example.com" || relay.Port != 2525 || relay.From != "felis@example.com" || relay.Username != "felis" || relay.RequireTLS { + t.Errorf("relay = %+v, want the [smtp] fields with the env password and TLS as configured", *relay) + } + }) +} + +func TestSMTPSecretPassword(t *testing.T) { + ctx := context.Background() + scheme := haltScheme(t) + + t.Run("reads the password key", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.SMTPSecretName}, + Data: map[string][]byte{platform.SMTPSecretPasswordKey: []byte("s3cret")}, + }).Build() + if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "s3cret" { + t.Errorf("smtpSecretPassword = (%q, %v), want (s3cret, nil)", pw, err) + } + }) + + t.Run("a missing Secret is a relay without AUTH", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + if pw, err := smtpSecretPassword(ctx, cl, "felis"); err != nil || pw != "" { + t.Errorf("smtpSecretPassword = (%q, %v), want (\"\", nil)", pw, err) + } + }) + + t.Run("any other read failure is an error", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return apierrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, platform.SMTPSecretName, errors.New("rbac")) + }, + }).Build() + if _, err := smtpSecretPassword(ctx, cl, "felis"); err == nil { + t.Fatal("want the read failure") + } + }) +} + +// recoveryModel is an Owner-reset console for admin "root" (verified address +// root@example.com), run by OS user alice, with the fake relay behind it. +func recoveryModel(t *testing.T, f *fakeOwnerStore, r *fakeRelay, now func() time.Time) *ownerModel { + t.Helper() + if f.users == nil { + f.users = map[string]*api.StaffUser{"root": verifiedAdmin("root", "root@example.com")} + } + return newOwnerModel(context.Background(), f, "alice", true).withRecovery(relayConfig(r, now)) +} + +// nameAdmin submits the admin-name form and feeds the result of the send back in. +func nameAdmin(t *testing.T, m *ownerModel, name string) *ownerModel { + t.Helper() + m.authUser = name + _, cmd := m.onFormComplete() + if m.step != owWorking { + t.Fatalf("after naming the admin step = %v, want owWorking", m.step) + } + msg := findMsg[owAuthMsg](t, cmd) + next, _ := m.Update(msg) + return next.(*ownerModel) +} + +// findMsg runs a (possibly batched) command and returns the first T it produces. +func findMsg[T any](t *testing.T, cmd tea.Cmd) T { + t.Helper() + var zero T + if cmd == nil { + t.Fatalf("no command, want one producing %T", zero) + } + switch msg := cmd().(type) { + case T: + return msg + case tea.BatchMsg: + for _, c := range msg { + if c == nil { + continue + } + if got, ok := c().(T); ok { + return got + } + } + } + t.Fatalf("command produced no %T", zero) + return zero +} + +// typeCode submits the code form with typed. +func typeCode(t *testing.T, m *ownerModel, typed string) { + t.Helper() + if m.step != owCode { + t.Fatalf("step = %v, want owCode", m.step) + } + m.codeInput = typed + m.onFormComplete() +} + +// provisionAudit finishes the run as an Owner reset and returns its audit payload. +func provisionAudit(t *testing.T, m *ownerModel, f *fakeOwnerStore) (api.AuditEntry, map[string]any) { + t.Helper() + if m.step != owProvision { + t.Fatalf("step = %v, want owProvision", m.step) + } + m.username = "owner" + msg := m.provisionCmd()().(owProvisionMsg) + if msg.err != nil { + t.Fatalf("provision: %v", msg.err) + } + return auditOf(t, f) +} + +func TestRecoveryConsoleFlow(t *testing.T) { + t0 := time.Date(2026, 9, 26, 8, 0, 0, 0, time.UTC) + fixed := func() time.Time { return t0 } + + t.Run("the mailed code proves the admin and the audit says so", func(t *testing.T) { + f, r := &fakeOwnerStore{}, &fakeRelay{} + m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root") + typeCode(t, m, r.sentCode(t)) + if m.mode != "recovery" || m.accountable != "root" { + t.Fatalf("mode=%q accountable=%q, want recovery attributed to root", m.mode, m.accountable) + } + e, payload := provisionAudit(t, m, f) + if e.Actor != "root" || e.Action != "break_glass.recovery" { + t.Errorf("audit = %+v, want actor=root action=break_glass.recovery", e) + } + if payload["verified"] != true || payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "root@example.com" || payload["os_user"] != "alice" { + t.Errorf("payload = %v, want verified by email_otp to root@example.com, os_user alice", payload) + } + }) + + t.Run("a wrong code asks again, and the last wrong one leads to the override", func(t *testing.T) { + f, r := &fakeOwnerStore{}, &fakeRelay{} + m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root") + wrong := "000000" + if r.sentCode(t) == wrong { + wrong = "111111" + } + for i := 1; i < recoveryCodeAttempts; i++ { + typeCode(t, m, wrong) + if m.step != owCode || !strings.Contains(m.codeNote, "wrong") { + t.Fatalf("wrong code %d: step=%v note=%q, want the code form again with a note", i, m.step, m.codeNote) + } + } + typeCode(t, m, wrong) + if m.step != owOverride || m.skip != otpSkipCodeRejected { + t.Fatalf("after %d wrong codes step=%v skip=%q, want the override for code_rejected", recoveryCodeAttempts, m.step, m.skip) + } + m.onFormComplete() // OVERRIDE typed + e, payload := provisionAudit(t, m, f) + if e.Actor != "alice" || e.Action != "break_glass.root_override" { + t.Errorf("audit = %+v, want actor=alice action=break_glass.root_override", e) + } + if payload["verified"] != false || payload["otp_skipped"] != otpSkipCodeRejected || payload["admin_account"] != "root" { + t.Errorf("payload = %v, want unverified, otp_skipped=code_rejected, admin_account=root", payload) + } + }) + + t.Run("a late code leads to the override", func(t *testing.T) { + now := t0 + f, r := &fakeOwnerStore{}, &fakeRelay{} + m := nameAdmin(t, recoveryModel(t, f, r, func() time.Time { return now }), "root") + now = t0.Add(recoveryCodeTTL) + typeCode(t, m, r.sentCode(t)) + if m.step != owOverride || m.skip != otpSkipCodeExpired { + t.Fatalf("step=%v skip=%q, want the override for code_expired", m.step, m.skip) + } + }) + + t.Run("OVERRIDE at the code prompt goes on unverified, saying the operator skipped", func(t *testing.T) { + f, r := &fakeOwnerStore{}, &fakeRelay{} + m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root") + typeCode(t, m, breakGlassOverrideToken) + if m.mode != "root_override" || m.accountable != "alice" { + t.Fatalf("mode=%q accountable=%q, want root_override as alice", m.mode, m.accountable) + } + _, payload := provisionAudit(t, m, f) + if payload["verified"] != false || payload["otp_skipped"] != otpSkipByOperator { + t.Errorf("payload = %v, want unverified, otp_skipped=operator_skipped", payload) + } + }) + + t.Run("a relay failure leads to the override naming it", func(t *testing.T) { + f, r := &fakeOwnerStore{}, &fakeRelay{err: errors.New("dial tcp 10.0.0.9:587: connect: connection refused")} + m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root") + if m.step != owOverride || m.skip != otpSkipSendFailed { + t.Fatalf("step=%v skip=%q, want the override for send_failed", m.step, m.skip) + } + if reason := m.overrideReason(); !strings.Contains(reason, "connection refused") { + t.Errorf("override reason %q does not name the failure", reason) + } + m.onFormComplete() + _, payload := provisionAudit(t, m, f) + if payload["otp_skipped"] != otpSkipSendFailed || !strings.Contains(payload["otp_skip_detail"].(string), "connection refused") { + t.Errorf("payload = %v, want otp_skipped=send_failed with the relay's error", payload) + } + }) + + t.Run("esc at the code prompt starts over and forgets the code", func(t *testing.T) { + f, r := &fakeOwnerStore{}, &fakeRelay{} + m := nameAdmin(t, recoveryModel(t, f, r, fixed), "root") + code := r.sentCode(t) + next, _ := m.Update(key(tea.KeyEsc)) + m = next.(*ownerModel) + if m.step != owAuth || m.code != nil || m.admin != nil { + t.Fatalf("after esc step=%v code=%v admin=%v, want owAuth with the attempt forgotten", m.step, m.code, m.admin) + } + // The old code cannot be replayed: the next name mails a new one. + m = nameAdmin(t, m, "root") + if len(r.sent) != 2 { + t.Fatalf("relay carried %d mails, want a second one for the new attempt", len(r.sent)) + } + _, fresh, _ := strings.Cut(r.sent[1].body, "Recovery code: ") + if m.code.value != fresh[:6] { + t.Errorf("the console checks %q, want the newly mailed %q (old one was %q)", m.code.value, fresh[:6], code) + } + }) +} + +func TestRootHandsRecoveryToAccountOperations(t *testing.T) { + for _, op := range []bgOperation{bgProvisionOwner, bgAddOperator} { + m := newTestRoot(true, consoleModeBreakGlass, "") + m.recovery = recoveryConfig{host: "felis-host-1"} + m = drive(t, m, menuChoiceMsg{op: op}) + om, ok := m.screen.(*ownerModel) + if !ok { + t.Fatalf("op %v: screen = %T, want *ownerModel", op, m.screen) + } + if om.recovery.host != "felis-host-1" { + t.Errorf("op %v: the account screen has no relay config; its codes could never go out", op) + } + } +} diff --git a/cmd/felis/breakglass_test.go b/cmd/felis/breakglass_test.go index 1e8d837..b30f599 100644 --- a/cmd/felis/breakglass_test.go +++ b/cmd/felis/breakglass_test.go @@ -269,71 +269,57 @@ func TestEnableLocalAuth(t *testing.T) { } } -func TestAuthenticateAdmin(t *testing.T) { +func TestResolveAdmin(t *testing.T) { ctx := context.Background() - // Password verification is gone (passwordless design): authenticateAdmin now only - // resolves the named admin so recovery can attribute the audit to a real identity. - // The security boundary is the break-glass root gate, not a typed secret. + // resolveAdmin only finds the staff account a typed name points at; proving the + // operator holds it is the mailed code's job (beginRecovery). - t.Run("resolves an existing admin for attribution", func(t *testing.T) { + t.Run("resolves an existing admin", func(t *testing.T) { f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": mkAdmin("root")}} - matched, ok, err := authenticateAdmin(ctx, f, "root") + u, err := resolveAdmin(ctx, f, " root ") if err != nil { - t.Fatalf("authenticateAdmin: %v", err) + t.Fatalf("resolveAdmin: %v", err) } - if !ok { - t.Fatal("ok = false, want true for an existing admin") - } - if matched != "root" { - t.Errorf("matched = %q, want root", matched) + if u == nil || u.Username != "root" { + t.Fatalf("resolveAdmin = %+v, want the root admin", u) } }) - t.Run("a non-admin role can never attribute a break-glass", func(t *testing.T) { + t.Run("a non-admin role is no staff account", func(t *testing.T) { player := mkAdmin("alice") player.Role = "user" // a player row is not staff f := &fakeOwnerStore{users: map[string]*api.StaffUser{"alice": player}} - _, ok, err := authenticateAdmin(ctx, f, "alice") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ok { - t.Error("ok = true, want false for a non-admin role") + if u, err := resolveAdmin(ctx, f, "alice"); u != nil || err != nil { + t.Errorf("resolveAdmin(player) = (%+v, %v), want (nil, nil)", u, err) } }) - t.Run("the owner role attributes like an admin", func(t *testing.T) { + t.Run("the owner role counts as staff", func(t *testing.T) { owner := mkAdmin("root") owner.Role = "owner" // the platform owner is staff too (migration 0011) f := &fakeOwnerStore{users: map[string]*api.StaffUser{"root": owner}} - matched, ok, err := authenticateAdmin(ctx, f, "root") - if err != nil || !ok || matched != "root" { - t.Fatalf("authenticateAdmin(owner) = (%q, %v, %v), want (root, true, nil)", matched, ok, err) + if u, err := resolveAdmin(ctx, f, "root"); err != nil || u == nil { + t.Fatalf("resolveAdmin(owner) = (%+v, %v), want the owner", u, err) } }) - t.Run("an unknown user is a non-match, not an error", func(t *testing.T) { - f := &fakeOwnerStore{} - _, ok, err := authenticateAdmin(ctx, f, "nobody") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ok { - t.Error("ok = true, want false for an unknown user") + t.Run("an unknown user is nil, not an error", func(t *testing.T) { + if u, err := resolveAdmin(ctx, &fakeOwnerStore{}, "nobody"); u != nil || err != nil { + t.Errorf("resolveAdmin(unknown) = (%+v, %v), want (nil, nil)", u, err) } }) - t.Run("an empty username is a non-match with no store call", func(t *testing.T) { + t.Run("an empty username makes no store call", func(t *testing.T) { f := &fakeOwnerStore{userErr: errors.New("must not be called")} - if _, ok, err := authenticateAdmin(ctx, f, ""); ok || err != nil { - t.Errorf("empty username: ok=%v err=%v, want false,nil", ok, err) + if u, err := resolveAdmin(ctx, f, " "); u != nil || err != nil { + t.Errorf("empty username: (%+v, %v), want (nil, nil)", u, err) } }) t.Run("a datastore fault is surfaced", func(t *testing.T) { f := &fakeOwnerStore{userErr: errors.New("db down")} - if _, _, err := authenticateAdmin(ctx, f, "root"); err == nil { + if _, err := resolveAdmin(ctx, f, "root"); err == nil { t.Fatal("want error when the store fails") } }) @@ -401,6 +387,8 @@ func TestPerformBreakGlass(t *testing.T) { osUser: "alice", ownerUsername: "owner", attemptedAdmin: "root", + verifiedBy: verifiedByEmailOTP, + codeSentTo: "root@example.com", } out, err := performBreakGlass(ctx, f, op) if err != nil { @@ -425,6 +413,23 @@ func TestPerformBreakGlass(t *testing.T) { if payload["admin_account"] != "root" { t.Errorf("payload.admin_account = %v, want root", payload["admin_account"]) } + if payload["verified_by"] != verifiedByEmailOTP || payload["code_sent_to"] != "root@example.com" { + t.Errorf("payload = %v, want verified_by=email_otp code_sent_to=root@example.com", payload) + } + if _, present := payload["otp_skipped"]; present { + t.Error("a proven recovery carries no otp_skipped") + } + }) + + t.Run("recovery without a proof is recorded unverified", func(t *testing.T) { + f := &fakeOwnerStore{} + op := breakGlassOp{mode: "recovery", accountable: "root", osUser: "alice", ownerUsername: "owner", attemptedAdmin: "root"} + if _, err := performBreakGlass(ctx, f, op); err != nil { + t.Fatalf("performBreakGlass: %v", err) + } + if _, payload := auditOf(t, f); payload["verified"] != false { + t.Errorf("payload.verified = %v, want false: only a mailed code verifies", payload["verified"]) + } }) t.Run("root override records an unverified row attributed to the OS user", func(t *testing.T) { @@ -435,6 +440,8 @@ func TestPerformBreakGlass(t *testing.T) { osUser: "alice", ownerUsername: "owner", attemptedAdmin: "typo-admin", + otpSkipped: otpSkipSendFailed, + otpSkipDetail: "dial tcp: connection refused", } if _, err := performBreakGlass(ctx, f, op); err != nil { t.Fatalf("performBreakGlass: %v", err) @@ -450,6 +457,13 @@ func TestPerformBreakGlass(t *testing.T) { if payload["admin_account"] != "typo-admin" { t.Errorf("payload.admin_account = %v, want typo-admin", payload["admin_account"]) } + // Why no code proved anyone is part of the record. + if payload["otp_skipped"] != otpSkipSendFailed || payload["otp_skip_detail"] != "dial tcp: connection refused" { + t.Errorf("payload = %v, want otp_skipped=send_failed with its detail", payload) + } + if _, present := payload["verified_by"]; present { + t.Error("an override carries no verified_by") + } }) t.Run("an audit failure does not fail the recovery", func(t *testing.T) { @@ -616,6 +630,8 @@ func TestPerformAddOperator(t *testing.T) { ownerUsername: "ops-jordan", ownerEmail: "jordan@example.com", attemptedAdmin: "root", + verifiedBy: verifiedByEmailOTP, + codeSentTo: "root@example.com", } out, err := performAddOperator(ctx, f, op) if err != nil { @@ -642,14 +658,14 @@ func TestPerformAddOperator(t *testing.T) { if _, present := payload["owner"]; present { t.Error("payload.owner present, want the new account under the operator key") } - if payload["verified"] != true || payload["admin_account"] != "root" { - t.Errorf("payload = %v, want verified=true admin_account=root", payload) + if payload["verified"] != true || payload["admin_account"] != "root" || payload["verified_by"] != verifiedByEmailOTP { + t.Errorf("payload = %v, want verified=true admin_account=root verified_by=email_otp", payload) } }) t.Run("root override records an unverified operator row", func(t *testing.T) { f := &fakeOwnerStore{} - op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin"} + op := breakGlassOp{mode: "root_override", accountable: "alice", osUser: "alice", ownerUsername: "ops", attemptedAdmin: "typo-admin", otpSkipped: otpSkipUnknownAdmin} if _, err := performAddOperator(ctx, f, op); err != nil { t.Fatalf("performAddOperator: %v", err) } @@ -657,8 +673,8 @@ func TestPerformAddOperator(t *testing.T) { t.Fatalf("want 1 insert, got %d", len(f.inserts)) } _, payload := auditOf(t, f) - if payload["verified"] != false { - t.Errorf("payload.verified = %v, want false for root_override", payload["verified"]) + if payload["verified"] != false || payload["otp_skipped"] != otpSkipUnknownAdmin { + t.Errorf("payload = %v, want verified=false otp_skipped=unknown_admin", payload) } }) diff --git a/cmd/felis/tui_owner.go b/cmd/felis/tui_owner.go index be6fe55..6f512a2 100644 --- a/cmd/felis/tui_owner.go +++ b/cmd/felis/tui_owner.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "time" "felis.lolicon.best/internal/api" @@ -14,9 +15,8 @@ import ( ) type owAuthMsg struct { - matched string - ok bool - err error + start recoveryStart + err error } type owProvisionMsg struct { @@ -29,14 +29,15 @@ type owStep int const ( owAuth owStep = iota owOverride + owCode // typing the recovery code mailed to the named admin owProvision owWorking owDone owError ) -// ownerModel collects the owner account. The input phases (admin auth, root -// override, owner details) are huh forms; the async phases (verifying, +// ownerModel collects the owner account. The input phases (admin name, recovery +// code, root override, owner details) are huh forms; the async phases (verifying, // provisioning) show a spinner; the done phase shows the credential card. The // outward contract is unchanged: it emits an ownerResultMsg when finished. // @@ -55,6 +56,18 @@ type ownerModel struct { mode string // "bootstrap", "recovery", "root_override" accountable string attempt string + recovery recoveryConfig + + // Recovery proof: the admin the typed name resolved to, the code mailed to it, + // and once settled either how it was proven or why the run fell back to the + // override. + admin *api.StaffUser + code *recoveryCode + codeNote string // "wrong code" line shown above a rebuilt code form + verifiedBy string + codeSentTo string + skip string // otpSkip* + skipDetail string step owStep form *huh.Form @@ -66,6 +79,7 @@ type ownerModel struct { // huh-bound form values authUser string + codeInput string overrideTok string ownerUser string ownerEmail string @@ -124,6 +138,12 @@ func newOperatorModel(ctx context.Context, store ownerStore, osUser string) *own return m } +// withRecovery hands the model the relay its recovery codes go through. +func (m *ownerModel) withRecovery(r recoveryConfig) *ownerModel { + m.recovery = r + return m +} + func (m *ownerModel) Init() tea.Cmd { return m.form.Init() } // subject is the human label for the account being provisioned, branching every @@ -152,7 +172,7 @@ func (m *ownerModel) sized(f *huh.Form) *huh.Form { } func (m *ownerModel) isFormStep() bool { - return m.step == owAuth || m.step == owOverride || m.step == owProvision + return m.step == owAuth || m.step == owOverride || m.step == owCode || m.step == owProvision } func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { @@ -161,16 +181,15 @@ func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if msg.err != nil { return m, m.failCmd(msg.err) } - if msg.ok { - m.mode = "recovery" - m.accountable = msg.matched - m.step = owProvision - m.form = m.sized(m.buildProvisionForm()) + m.admin = msg.start.admin + if msg.start.code != nil { + m.code = msg.start.code + m.codeNote = "" + m.step = owCode + m.form = m.sized(m.buildCodeForm()) return m, m.form.Init() } - m.step = owOverride - m.form = m.sized(m.buildOverrideForm()) - return m, m.form.Init() + return m.toOverride(msg.start.skip, msg.start.detail) case owProvisionMsg: if msg.err != nil { @@ -221,7 +240,9 @@ func (m *ownerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case "ctrl+c": return m, tea.Quit case "esc": - if m.step == owOverride { + if m.step == owOverride || m.step == owCode { + // Start over: a code mailed for the old attempt dies with it. + m.admin, m.code, m.skip, m.skipDetail = nil, nil, "", "" m.step = owAuth m.form = m.sized(m.buildAuthForm()) return m, m.form.Init() @@ -253,18 +274,41 @@ func (m *ownerModel) onFormComplete() (tea.Model, tea.Cmd) { case owAuth: m.attempt = strings.TrimSpace(m.authUser) m.step = owWorking - m.working = "Verifying admin…" - user := m.authUser + m.working = "Sending a recovery code…" + user, rc, osUser, op := m.authUser, m.recovery, m.osUser, m.operation return m, tea.Batch(m.sp.Tick, func() tea.Msg { - matched, ok, err := authenticateAdmin(m.ctx, m.store, user) - return owAuthMsg{matched: matched, ok: ok, err: err} + start, err := beginRecovery(m.ctx, m.store, rc, user, osUser, op) + return owAuthMsg{start: start, err: err} }) + case owCode: + typed := strings.TrimSpace(m.codeInput) + m.codeInput = "" + if typed == breakGlassOverrideToken { + m.skip, m.skipDetail = otpSkipByOperator, "" + m.code = nil + return m.proceedAsRoot() + } + switch m.code.check(typed, m.recovery.clock()) { + case codeAccepted: + m.mode = "recovery" + m.accountable = m.admin.Username + m.verifiedBy = verifiedByEmailOTP + m.codeSentTo = m.admin.Email + m.code = nil + m.step = owProvision + m.form = m.sized(m.buildProvisionForm()) + return m, m.form.Init() + case codeWrong: + m.codeNote = fmt.Sprintf("That code is wrong. %d attempts left.", m.code.attemptsLeft()) + m.form = m.sized(m.buildCodeForm()) + return m, m.form.Init() + case codeExpired: + return m.toOverride(otpSkipCodeExpired, "") + default: + return m.toOverride(otpSkipCodeRejected, fmt.Sprintf("%d wrong codes", recoveryCodeAttempts)) + } case owOverride: - m.mode = "root_override" - m.accountable = m.osUser - m.step = owProvision - m.form = m.sized(m.buildProvisionForm()) - return m, m.form.Init() + return m.proceedAsRoot() case owProvision: m.username = strings.TrimSpace(m.ownerUser) m.step = owWorking @@ -274,6 +318,24 @@ func (m *ownerModel) onFormComplete() (tea.Model, tea.Cmd) { return m, nil } +// toOverride records why no code proved an admin and asks for the typed OVERRIDE. +func (m *ownerModel) toOverride(skip, detail string) (tea.Model, tea.Cmd) { + m.skip, m.skipDetail = skip, detail + m.code = nil + m.step = owOverride + m.form = m.sized(m.buildOverrideForm()) + return m, m.form.Init() +} + +// proceedAsRoot is the typed OVERRIDE: the run goes on as the OS user, unverified. +func (m *ownerModel) proceedAsRoot() (tea.Model, tea.Cmd) { + m.mode = "root_override" + m.accountable = m.osUser + m.step = owProvision + m.form = m.sized(m.buildProvisionForm()) + return m, m.form.Init() +} + func (m *ownerModel) provisionCmd() tea.Cmd { op := breakGlassOp{ mode: m.mode, @@ -282,6 +344,10 @@ func (m *ownerModel) provisionCmd() tea.Cmd { ownerUsername: m.username, ownerEmail: m.ownerEmail, attemptedAdmin: m.attempt, + verifiedBy: m.verifiedBy, + codeSentTo: m.codeSentTo, + otpSkipped: m.skip, + otpSkipDetail: m.skipDetail, } // performAddOperator and performBreakGlass share a signature; the operation // discriminator selects which one runs. The operator path is insert-only and @@ -320,7 +386,7 @@ func (m *ownerModel) buildAuthForm() *huh.Form { return m.sized(newFelisForm(huh.NewGroup( huh.NewNote(). Title("Admin authentication"). - Description("A staff account already exists. Identify yourself to continue."), + Description("A staff account already exists. Name yours: a one-time code goes to its verified email address."), huh.NewInput(). Title("Admin username"). Value(&m.authUser). @@ -328,11 +394,64 @@ func (m *ownerModel) buildAuthForm() *huh.Form { ))) } +func (m *ownerModel) buildCodeForm() *huh.Form { + desc := fmt.Sprintf("A recovery code went to %s, the verified address of %q. It works for %d minutes.\n\n"+ + "No mail? Type %s to go on as OS user %q with root authority; the audit log records that as an unverified override. Esc starts over.", + maskEmail(m.admin.Email), m.admin.Username, int(recoveryCodeTTL/time.Minute), breakGlassOverrideToken, m.osUser) + if m.codeNote != "" { + desc = m.codeNote + "\n\n" + desc + } + return m.sized(newFelisForm(huh.NewGroup( + huh.NewNote().Title("Email verification").Description(desc), + huh.NewInput(). + Title("Recovery code"). + Value(&m.codeInput). + Validate(func(s string) error { + s = strings.TrimSpace(s) + if s == breakGlassOverrideToken || isRecoveryCodeShape(s) { + return nil + } + return errors.New("enter the 6-digit code, or " + breakGlassOverrideToken) + }), + ))) +} + +func isRecoveryCodeShape(s string) bool { + if len(s) != 6 { + return false + } + for _, c := range s { + if c < '0' || c > '9' { + return false + } + } + return true +} + +// overrideReason says why no code proved an admin, first line of the override form. +func (m *ownerModel) overrideReason() string { + switch m.skip { + case otpSkipUnknownAdmin: + return fmt.Sprintf("No staff account is named %q.", m.attempt) + case otpSkipNoVerifiedEmail: + return fmt.Sprintf("%q has no verified email address, so no recovery code can reach it.", m.admin.Username) + case otpSkipNoRelay: + return "No mail relay can send a recovery code: " + m.skipDetail + "." + case otpSkipSendFailed: + return "The recovery code could not be sent: " + m.skipDetail + "." + case otpSkipCodeExpired: + return "The recovery code expired." + case otpSkipCodeRejected: + return fmt.Sprintf("%d wrong codes; that code no longer works.", recoveryCodeAttempts) + } + return "No admin was verified." +} + func (m *ownerModel) buildOverrideForm() *huh.Form { return m.sized(newFelisForm(huh.NewGroup( huh.NewNote(). Title("Root override"). - Description(fmt.Sprintf("That credential did not match. Proceed as OS user %q with root authority by typing the confirmation token.", m.osUser)), + Description(m.overrideReason()+"\n\n"+fmt.Sprintf("Proceed as OS user %q with root authority by typing the confirmation token. The audit log records this run as an unverified root override and the reason above. Esc starts over.", m.osUser)), huh.NewInput(). Title("Type "+breakGlassOverrideToken+" to confirm"). Value(&m.overrideTok). @@ -352,14 +471,14 @@ func (m *ownerModel) buildProvisionForm() *huh.Form { desc := fmt.Sprintf("Create the first Owner — recorded as OS user %q.", m.osUser) switch m.mode { case "recovery": - desc = fmt.Sprintf("Authenticated as %q.", m.accountable) + desc = fmt.Sprintf("Verified as %q by an email code.", m.accountable) case "root_override": desc = "Root override — the Owner will be reset." } if m.operation == bgAddOperator { switch m.mode { case "recovery": - desc = fmt.Sprintf("Add an Operator — authenticated as %q.", m.accountable) + desc = fmt.Sprintf("Add an Operator — verified as %q by an email code.", m.accountable) case "root_override": desc = "Add an Operator (root override)." } diff --git a/cmd/felis/tui_root.go b/cmd/felis/tui_root.go index 11eebeb..4a40a04 100644 --- a/cmd/felis/tui_root.go +++ b/cmd/felis/tui_root.go @@ -142,6 +142,7 @@ type rootModel struct { accessAud string namespace string // minecraft workload namespace (cfg.K8s.Namespace); target of the halt op adminExists bool + recovery recoveryConfig // how the account operations mail a recovery code } func newRootModel(ctx context.Context, store ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) *rootModel { @@ -221,7 +222,7 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.stage = stageOwner switch msg.op { case bgAddOperator: - return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser)) + return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser).withRecovery(m.recovery)) case bgHaltServer: return m.adopt(newHaltModel(m.ctx, m.store, m.namespace, m.osUser)) case bgSyncBackup: @@ -229,7 +230,7 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // Service + service token the peer dials live in the control namespace. return m.adopt(newBackupModel(m.ctx, m.namespace, platform.DefaultControlNamespace, m.osUser)) default: - return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists)) + return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists).withRecovery(m.recovery)) } case haltResultMsg: diff --git a/cmd/felis/watchdog.go b/cmd/felis/watchdog.go index 39e1719..ce17598 100644 --- a/cmd/felis/watchdog.go +++ b/cmd/felis/watchdog.go @@ -187,16 +187,26 @@ func usesMirroredScanDB(cfg *config.Config) bool { // forgets it when the Secret is gone (a relay without AUTH). An env var named by // [smtp] password_ref, when set, wins at send time instead. func refreshSMTPPassword(ctx context.Context, cl client.Client, ns string, state *watchdog.State, stderr io.Writer) { + password, err := smtpSecretPassword(ctx, cl, ns) + if err != nil { + fmt.Fprintf(stderr, "felis watchdog: read %s/%s (keeping the cached relay password): %v\n", ns, platform.SMTPSecretName, err) + return + } + state.SMTPPassword = password +} + +// smtpSecretPassword reads the relay password from the felis-smtp Secret. A missing +// Secret is a relay without AUTH and reads as "". +func smtpSecretPassword(ctx context.Context, cl client.Client, ns string) (string, error) { var sec corev1.Secret err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: platform.SMTPSecretName}, &sec) - switch { - case apierrors.IsNotFound(err): - state.SMTPPassword = "" - case err != nil: - fmt.Fprintf(stderr, "felis watchdog: read %s/%s (keeping the cached relay password): %v\n", ns, platform.SMTPSecretName, err) - default: - state.SMTPPassword = string(sec.Data[platform.SMTPSecretPasswordKey]) + if apierrors.IsNotFound(err) { + return "", nil } + if err != nil { + return "", err + } + return string(sec.Data[platform.SMTPSecretPasswordKey]), nil } // ownerEmails pings PostgreSQL and returns the verified addresses of the diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c86316e..f8b3f73 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2270,6 +2270,42 @@ sudo felis db audit-export -until 2026-01-01 -out /root/audit-2025.jsonl window is `[since, until)`. Each line is one row as JSON, oldest first. The file is created `0600` and an existing file is never overwritten. +### `felis breakGlass`: the recovery code and the OVERRIDE [VM-VERIFIED] + +Once a staff account exists, `sudo felis breakGlass` asks which admin or owner +is breaking the glass and mails that account's verified address a six-digit +code through the same `[smtp]` relay as the sign-in codes. The code works for +10 minutes and five wrong ones end it. The console reads the relay password +the way the watchdog does (the `password_ref` env var, else the `felis-smtp` +Secret, else no AUTH), and the mail skips the API's `max_per_hour` budget. +Only the right code makes the run a `recovery` attributed to that account; the +mail says which host and OS user asked, so an admin who did not ask learns +that root there is in other hands. + +Every other ending leads to the typed `OVERRIDE`, and the screen says why: +the name matches no staff account, the account has no verified address, no +relay (`[smtp] is not configured in felis.toml`, or the Secret could not be +read), the relay refused the mail, the code expired or took five wrong tries, +or the operator typed `OVERRIDE` at the code prompt. Esc on either screen +starts over with a new code. With the relay down recovery still works, as an +unverified override that records the reason. + +The audit row is `break_glass.recovery` or `break_glass.root_override` +(`break_glass.operator_create` for a new Operator account), source +`break-glass`. `verified` is true only for a run a code proved, which also +carries `verified_by: email_otp` and `code_sent_to`. An override carries +`otp_skipped` (`unknown_admin`, `no_verified_email`, `no_relay`, +`send_failed`, `code_expired`, `code_rejected`, `operator_skipped`) and, where +something failed, `otp_skip_detail`. Root can edit the row afterwards, so it +records attribution without proving it. + +```sh +sudo -u postgres psql felis -c " + SELECT created_at, action, actor, payload->>'verified' AS verified, + payload->>'otp_skipped' AS skipped, payload->>'otp_skip_detail' AS detail + FROM audit_logs WHERE source = 'break-glass' ORDER BY created_at DESC LIMIT 20;" +``` + ### Optional: a Cloudflare rate limiting rule in front The limits above live in the API, so they hold on any edge. Behind Cloudflare @@ -2319,4 +2355,5 @@ for 10 seconds (the Free plan's limits). | Right code refused; `otp_account_locked` / `FelisOTPAccountLocked` | §17 | | `FelisSignInFailures` / who is guessing, from where | §17 | | `FelisAuditWriteFailing` | §17 | +| `felis breakGlass` sends no code / shows `Root override`; `otp_skipped` in the audit | §17 | | How long sessions, codes and audit rows are kept; export audit rows | §17 |