Loading cmd/felis/api.go +22 −0 Changes for cmd/felis/api.go: 22 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -300,8 +300,20 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { // for legitimate multi-tab / multi-server watching, while capping how many // upstream follow connections a single caller can tie up if their streams stall. MaxStreamsPerPrincipal: 16, // Public sign-in doors, per client address: a person signing in makes a // handful of calls, so 20 at once refilled at 20 a minute never bites a // real user and still turns a spray into a trickle. The client address // is the edge's header when the install names one (config.AuthConfig). AuthDoorLimit: api.RateLimit{Burst: 20, PerMinute: 20}, ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(), MailLimit: mailLimit(cfg.SMTP.MaxPerHour), } fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)") if a.ClientIPHeader != "" { fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader) } else { fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)") } // Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned // identity anchor (正版优先); config can only append namespace-rewritten third-party Loading Loading @@ -546,3 +558,13 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) { } } } // mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket: // the hourly cap as the refill rate, with a quarter of it (at least 5) allowed // at once so a burst of real sign-ins is not queued behind the average. func mailLimit(perHour int) api.RateLimit { if perHour <= 0 { perHour = config.DefaultMailPerHour } return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60} } cmd/felis/tui_edge_apply.go +10 −5 Changes for cmd/felis/tui_edge_apply.go: 10 added lines, 5 removed lines. Original line number Diff line number Diff line Loading @@ -32,7 +32,9 @@ func applyCloudflareEdge(ctx context.Context, result *cfsetup.Result, panelHost, if adminHost == "" { return fmt.Errorf("admin hostname is required") } if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud); err != nil { // cloudflared is the only way in once the NodePort is fenced, so the // visitor address it writes can key the sign-in rate limit. if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud, "CF-Connecting-IP"); err != nil { return err } if err := applyFelisConfigSecret(ctx); err != nil { Loading Loading @@ -78,7 +80,8 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error { if adminHost == "" { return fmt.Errorf("admin hostname is required") } if err := writeConnectionConfig(panelHost, adminHost, ""); err != nil { // Caddy, nginx and Traefik all append the peer they saw to X-Forwarded-For. if err := writeConnectionConfig(panelHost, adminHost, "", "X-Forwarded-For"); err != nil { return err } if err := applyFelisConfigSecret(ctx); err != nil { Loading @@ -93,16 +96,17 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error { // writeConnectionConfig stamps the chosen hostnames (and optional Access audience) // into both the host and pod config files. An empty aud clears any prior // Cloudflare audience, which is correct when switching to a non-Access front. func writeConnectionConfig(panelHost, adminHost, aud string) error { // clientIPHeader is the header that front writes the visitor address into. func writeConnectionConfig(panelHost, adminHost, aud, clientIPHeader string) error { for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} { if err := updateAuthConfig(path, panelHost, adminHost, aud); err != nil { if err := updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader); err != nil { return err } } return nil } func updateAuthConfig(path, panelHost, adminHost, aud string) error { func updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader string) error { cfg, err := config.Load(path) if err != nil { return err Loading @@ -112,6 +116,7 @@ func updateAuthConfig(path, panelHost, adminHost, aud string) error { } cfg.Auth.AdminHostname = adminHost cfg.Auth.AccessJWTAud = aud cfg.Auth.ClientIPHeader = clientIPHeader return writeConfig(path, cfg) } Loading deploy/alerts/felis-alerts.yaml +49 −1 Changes for deploy/alerts/felis-alerts.yaml: 49 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -4,7 +4,9 @@ # # felis_* series come from two processes: # - felis-operator pod :8080/metrics → felis_servers_total, felis_start_duration_seconds # - felis-api internal :8081/metrics → felis_image_build_failures_total # - felis-api internal :8081/metrics → felis_image_build_failures_total, # felis_mail_total, felis_rate_limited_total, # felis_auth_otp_lockouts_total # - node-exporter textfile collector → felis_db_backup_* (felis-db-backup.timer) # node_* / kube_* series come from node-exporter / kube-state-metrics. groups: Loading Loading @@ -94,3 +96,49 @@ groups: --collector.textfile.directory at the directory of FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) (troubleshooting §16). - name: felis.auth.rules rules: - alert: FelisMailBudgetExhausted expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0 labels: severity: warning annotations: summary: "the install-wide mail budget refused mail" description: >- felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is spent, and every sign-in code is refused with 429 mail_rate_limited until it refills. Check felis_rate_limited_total for a flood before raising the budget (troubleshooting §17). - alert: FelisMailDeliveryFailing expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0 labels: severity: warning annotations: summary: "the SMTP relay refused mail in the last 15m" description: >- felis_mail_total{result="failed"} increased: sign-in codes are not being delivered (502 mail_undeliverable). The relay's reason is in the felis-api log (troubleshooting §17). - alert: FelisSignInFlood expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10 for: 10m labels: severity: warning annotations: summary: "sign-in doors refusing over 10 requests a minute" description: >- The per-address sign-in limit has been refusing callers for 10 minutes. A script is hammering the auth doors; if real users report rate_limited at once instead, [auth] client_ip_header is missing and everyone shares the proxy's address (troubleshooting §17). - alert: FelisOTPAccountLocked expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0 labels: severity: warning annotations: summary: "an account's email-code sign-in locked after 10 wrong codes" description: >- Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}). The audit log names the account (action auth.otp.locked); the owner was mailed. Unless they fumbled codes, someone is guessing at it (troubleshooting §17). deploy/alerts/felis-alerts_test.yml +85 −0 Changes for deploy/alerts/felis-alerts_test.yml: 85 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -151,3 +151,88 @@ tests: --collector.textfile.directory at the directory of FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) (troubleshooting §16). - name: sign-in mail budget and relay interval: 1m input_series: # Created at zero on start; the budget refuses one mail at t=3m. - series: 'felis_mail_total{kind="otp",result="throttled",job="felis-api"}' values: '0 0 0 1x30' - series: 'felis_mail_total{kind="otp",result="failed",job="felis-api"}' values: '0x33' alert_rule_test: - eval_time: 2m alertname: FelisMailBudgetExhausted exp_alerts: [] - eval_time: 5m alertname: FelisMailBudgetExhausted exp_alerts: - exp_labels: severity: warning exp_annotations: summary: "the install-wide mail budget refused mail" description: >- felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is spent, and every sign-in code is refused with 429 mail_rate_limited until it refills. Check felis_rate_limited_total for a flood before raising the budget (troubleshooting §17). - eval_time: 5m alertname: FelisMailDeliveryFailing exp_alerts: [] - name: sign-in flood interval: 1m input_series: # 30 refusals a minute from t=0; a lone refused script at 2/min stays quiet. - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}' values: '0+30x40' alert_rule_test: - eval_time: 10m alertname: FelisSignInFlood exp_alerts: [] - eval_time: 20m alertname: FelisSignInFlood exp_alerts: - exp_labels: severity: warning exp_annotations: summary: "sign-in doors refusing over 10 requests a minute" description: >- The per-address sign-in limit has been refusing callers for 10 minutes. A script is hammering the auth doors; if real users report rate_limited at once instead, [auth] client_ip_header is missing and everyone shares the proxy's address (troubleshooting §17). - name: sign-in trickle stays quiet interval: 1m input_series: - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}' values: '0+2x40' alert_rule_test: - eval_time: 30m alertname: FelisSignInFlood exp_alerts: [] - name: account email-code lock interval: 1m input_series: - series: 'felis_auth_otp_lockouts_total{purpose="login_email",job="felis-api"}' values: '0 0 1x90' - series: 'felis_auth_otp_lockouts_total{purpose="op_login",job="felis-api"}' values: '0x92' alert_rule_test: - eval_time: 1m alertname: FelisOTPAccountLocked exp_alerts: [] - eval_time: 10m alertname: FelisOTPAccountLocked exp_alerts: - exp_labels: severity: warning purpose: login_email exp_annotations: summary: "an account's email-code sign-in locked after 10 wrong codes" description: >- Someone entered 10 wrong codes for one account within 24h (login_email). The audit log names the account (action auth.otp.locked); the owner was mailed. Unless they fumbled codes, someone is guessing at it (troubleshooting §17). - eval_time: 90m alertname: FelisOTPAccountLocked exp_alerts: [] deploy/alerts/felis-prometheusrule.yaml +46 −0 Changes for deploy/alerts/felis-prometheusrule.yaml: 46 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -98,3 +98,49 @@ spec: --collector.textfile.directory at the directory of FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) (troubleshooting §16). - name: felis.auth.rules rules: - alert: FelisMailBudgetExhausted expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0 labels: severity: warning annotations: summary: "the install-wide mail budget refused mail" description: >- felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is spent, and every sign-in code is refused with 429 mail_rate_limited until it refills. Check felis_rate_limited_total for a flood before raising the budget (troubleshooting §17). - alert: FelisMailDeliveryFailing expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0 labels: severity: warning annotations: summary: "the SMTP relay refused mail in the last 15m" description: >- felis_mail_total{result="failed"} increased: sign-in codes are not being delivered (502 mail_undeliverable). The relay's reason is in the felis-api log (troubleshooting §17). - alert: FelisSignInFlood expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10 for: 10m labels: severity: warning annotations: summary: "sign-in doors refusing over 10 requests a minute" description: >- The per-address sign-in limit has been refusing callers for 10 minutes. A script is hammering the auth doors; if real users report rate_limited at once instead, [auth] client_ip_header is missing and everyone shares the proxy's address (troubleshooting §17). - alert: FelisOTPAccountLocked expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0 labels: severity: warning annotations: summary: "an account's email-code sign-in locked after 10 wrong codes" description: >- Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}). The audit log names the account (action auth.otp.locked); the owner was mailed. Unless they fumbled codes, someone is guessing at it (troubleshooting §17). Loading
cmd/felis/api.go +22 −0 Changes for cmd/felis/api.go: 22 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -300,8 +300,20 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { // for legitimate multi-tab / multi-server watching, while capping how many // upstream follow connections a single caller can tie up if their streams stall. MaxStreamsPerPrincipal: 16, // Public sign-in doors, per client address: a person signing in makes a // handful of calls, so 20 at once refilled at 20 a minute never bites a // real user and still turns a spray into a trickle. The client address // is the edge's header when the install names one (config.AuthConfig). AuthDoorLimit: api.RateLimit{Burst: 20, PerMinute: 20}, ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(), MailLimit: mailLimit(cfg.SMTP.MaxPerHour), } fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)") if a.ClientIPHeader != "" { fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader) } else { fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)") } // Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned // identity anchor (正版优先); config can only append namespace-rewritten third-party Loading Loading @@ -546,3 +558,13 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) { } } } // mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket: // the hourly cap as the refill rate, with a quarter of it (at least 5) allowed // at once so a burst of real sign-ins is not queued behind the average. func mailLimit(perHour int) api.RateLimit { if perHour <= 0 { perHour = config.DefaultMailPerHour } return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60} }
cmd/felis/tui_edge_apply.go +10 −5 Changes for cmd/felis/tui_edge_apply.go: 10 added lines, 5 removed lines. Original line number Diff line number Diff line Loading @@ -32,7 +32,9 @@ func applyCloudflareEdge(ctx context.Context, result *cfsetup.Result, panelHost, if adminHost == "" { return fmt.Errorf("admin hostname is required") } if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud); err != nil { // cloudflared is the only way in once the NodePort is fenced, so the // visitor address it writes can key the sign-in rate limit. if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud, "CF-Connecting-IP"); err != nil { return err } if err := applyFelisConfigSecret(ctx); err != nil { Loading Loading @@ -78,7 +80,8 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error { if adminHost == "" { return fmt.Errorf("admin hostname is required") } if err := writeConnectionConfig(panelHost, adminHost, ""); err != nil { // Caddy, nginx and Traefik all append the peer they saw to X-Forwarded-For. if err := writeConnectionConfig(panelHost, adminHost, "", "X-Forwarded-For"); err != nil { return err } if err := applyFelisConfigSecret(ctx); err != nil { Loading @@ -93,16 +96,17 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error { // writeConnectionConfig stamps the chosen hostnames (and optional Access audience) // into both the host and pod config files. An empty aud clears any prior // Cloudflare audience, which is correct when switching to a non-Access front. func writeConnectionConfig(panelHost, adminHost, aud string) error { // clientIPHeader is the header that front writes the visitor address into. func writeConnectionConfig(panelHost, adminHost, aud, clientIPHeader string) error { for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} { if err := updateAuthConfig(path, panelHost, adminHost, aud); err != nil { if err := updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader); err != nil { return err } } return nil } func updateAuthConfig(path, panelHost, adminHost, aud string) error { func updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader string) error { cfg, err := config.Load(path) if err != nil { return err Loading @@ -112,6 +116,7 @@ func updateAuthConfig(path, panelHost, adminHost, aud string) error { } cfg.Auth.AdminHostname = adminHost cfg.Auth.AccessJWTAud = aud cfg.Auth.ClientIPHeader = clientIPHeader return writeConfig(path, cfg) } Loading
deploy/alerts/felis-alerts.yaml +49 −1 Changes for deploy/alerts/felis-alerts.yaml: 49 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -4,7 +4,9 @@ # # felis_* series come from two processes: # - felis-operator pod :8080/metrics → felis_servers_total, felis_start_duration_seconds # - felis-api internal :8081/metrics → felis_image_build_failures_total # - felis-api internal :8081/metrics → felis_image_build_failures_total, # felis_mail_total, felis_rate_limited_total, # felis_auth_otp_lockouts_total # - node-exporter textfile collector → felis_db_backup_* (felis-db-backup.timer) # node_* / kube_* series come from node-exporter / kube-state-metrics. groups: Loading Loading @@ -94,3 +96,49 @@ groups: --collector.textfile.directory at the directory of FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) (troubleshooting §16). - name: felis.auth.rules rules: - alert: FelisMailBudgetExhausted expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0 labels: severity: warning annotations: summary: "the install-wide mail budget refused mail" description: >- felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is spent, and every sign-in code is refused with 429 mail_rate_limited until it refills. Check felis_rate_limited_total for a flood before raising the budget (troubleshooting §17). - alert: FelisMailDeliveryFailing expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0 labels: severity: warning annotations: summary: "the SMTP relay refused mail in the last 15m" description: >- felis_mail_total{result="failed"} increased: sign-in codes are not being delivered (502 mail_undeliverable). The relay's reason is in the felis-api log (troubleshooting §17). - alert: FelisSignInFlood expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10 for: 10m labels: severity: warning annotations: summary: "sign-in doors refusing over 10 requests a minute" description: >- The per-address sign-in limit has been refusing callers for 10 minutes. A script is hammering the auth doors; if real users report rate_limited at once instead, [auth] client_ip_header is missing and everyone shares the proxy's address (troubleshooting §17). - alert: FelisOTPAccountLocked expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0 labels: severity: warning annotations: summary: "an account's email-code sign-in locked after 10 wrong codes" description: >- Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}). The audit log names the account (action auth.otp.locked); the owner was mailed. Unless they fumbled codes, someone is guessing at it (troubleshooting §17).
deploy/alerts/felis-alerts_test.yml +85 −0 Changes for deploy/alerts/felis-alerts_test.yml: 85 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -151,3 +151,88 @@ tests: --collector.textfile.directory at the directory of FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) (troubleshooting §16). - name: sign-in mail budget and relay interval: 1m input_series: # Created at zero on start; the budget refuses one mail at t=3m. - series: 'felis_mail_total{kind="otp",result="throttled",job="felis-api"}' values: '0 0 0 1x30' - series: 'felis_mail_total{kind="otp",result="failed",job="felis-api"}' values: '0x33' alert_rule_test: - eval_time: 2m alertname: FelisMailBudgetExhausted exp_alerts: [] - eval_time: 5m alertname: FelisMailBudgetExhausted exp_alerts: - exp_labels: severity: warning exp_annotations: summary: "the install-wide mail budget refused mail" description: >- felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is spent, and every sign-in code is refused with 429 mail_rate_limited until it refills. Check felis_rate_limited_total for a flood before raising the budget (troubleshooting §17). - eval_time: 5m alertname: FelisMailDeliveryFailing exp_alerts: [] - name: sign-in flood interval: 1m input_series: # 30 refusals a minute from t=0; a lone refused script at 2/min stays quiet. - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}' values: '0+30x40' alert_rule_test: - eval_time: 10m alertname: FelisSignInFlood exp_alerts: [] - eval_time: 20m alertname: FelisSignInFlood exp_alerts: - exp_labels: severity: warning exp_annotations: summary: "sign-in doors refusing over 10 requests a minute" description: >- The per-address sign-in limit has been refusing callers for 10 minutes. A script is hammering the auth doors; if real users report rate_limited at once instead, [auth] client_ip_header is missing and everyone shares the proxy's address (troubleshooting §17). - name: sign-in trickle stays quiet interval: 1m input_series: - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}' values: '0+2x40' alert_rule_test: - eval_time: 30m alertname: FelisSignInFlood exp_alerts: [] - name: account email-code lock interval: 1m input_series: - series: 'felis_auth_otp_lockouts_total{purpose="login_email",job="felis-api"}' values: '0 0 1x90' - series: 'felis_auth_otp_lockouts_total{purpose="op_login",job="felis-api"}' values: '0x92' alert_rule_test: - eval_time: 1m alertname: FelisOTPAccountLocked exp_alerts: [] - eval_time: 10m alertname: FelisOTPAccountLocked exp_alerts: - exp_labels: severity: warning purpose: login_email exp_annotations: summary: "an account's email-code sign-in locked after 10 wrong codes" description: >- Someone entered 10 wrong codes for one account within 24h (login_email). The audit log names the account (action auth.otp.locked); the owner was mailed. Unless they fumbled codes, someone is guessing at it (troubleshooting §17). - eval_time: 90m alertname: FelisOTPAccountLocked exp_alerts: []
deploy/alerts/felis-prometheusrule.yaml +46 −0 Changes for deploy/alerts/felis-prometheusrule.yaml: 46 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -98,3 +98,49 @@ spec: --collector.textfile.directory at the directory of FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) (troubleshooting §16). - name: felis.auth.rules rules: - alert: FelisMailBudgetExhausted expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0 labels: severity: warning annotations: summary: "the install-wide mail budget refused mail" description: >- felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is spent, and every sign-in code is refused with 429 mail_rate_limited until it refills. Check felis_rate_limited_total for a flood before raising the budget (troubleshooting §17). - alert: FelisMailDeliveryFailing expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0 labels: severity: warning annotations: summary: "the SMTP relay refused mail in the last 15m" description: >- felis_mail_total{result="failed"} increased: sign-in codes are not being delivered (502 mail_undeliverable). The relay's reason is in the felis-api log (troubleshooting §17). - alert: FelisSignInFlood expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10 for: 10m labels: severity: warning annotations: summary: "sign-in doors refusing over 10 requests a minute" description: >- The per-address sign-in limit has been refusing callers for 10 minutes. A script is hammering the auth doors; if real users report rate_limited at once instead, [auth] client_ip_header is missing and everyone shares the proxy's address (troubleshooting §17). - alert: FelisOTPAccountLocked expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0 labels: severity: warning annotations: summary: "an account's email-code sign-in locked after 10 wrong codes" description: >- Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}). The audit log names the account (action auth.otp.locked); the owner was mailed. Unless they fumbled codes, someone is guessing at it (troubleshooting §17).
