diff --git a/cmd/felis/api.go b/cmd/felis/api.go index d0343a6..f281b18 100644 --- a/cmd/felis/api.go +++ b/cmd/felis/api.go @@ -300,8 +300,20 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { // for legitimate multi-tab / multi-server watching, while capping how many // upstream follow connections a single caller can tie up if their streams stall. MaxStreamsPerPrincipal: 16, + // Public sign-in doors, per client address: a person signing in makes a + // handful of calls, so 20 at once refilled at 20 a minute never bites a + // real user and still turns a spray into a trickle. The client address + // is the edge's header when the install names one (config.AuthConfig). + AuthDoorLimit: api.RateLimit{Burst: 20, PerMinute: 20}, + ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(), + MailLimit: mailLimit(cfg.SMTP.MaxPerHour), } fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)") + if a.ClientIPHeader != "" { + fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader) + } else { + fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)") + } // Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned // identity anchor (正版优先); config can only append namespace-rewritten third-party @@ -546,3 +558,13 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) { } } } + +// mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket: +// the hourly cap as the refill rate, with a quarter of it (at least 5) allowed +// at once so a burst of real sign-ins is not queued behind the average. +func mailLimit(perHour int) api.RateLimit { + if perHour <= 0 { + perHour = config.DefaultMailPerHour + } + return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60} +} diff --git a/cmd/felis/tui_edge_apply.go b/cmd/felis/tui_edge_apply.go index 107b304..cc8e68a 100644 --- a/cmd/felis/tui_edge_apply.go +++ b/cmd/felis/tui_edge_apply.go @@ -32,7 +32,9 @@ func applyCloudflareEdge(ctx context.Context, result *cfsetup.Result, panelHost, if adminHost == "" { return fmt.Errorf("admin hostname is required") } - if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud); err != nil { + // cloudflared is the only way in once the NodePort is fenced, so the + // visitor address it writes can key the sign-in rate limit. + if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud, "CF-Connecting-IP"); err != nil { return err } if err := applyFelisConfigSecret(ctx); err != nil { @@ -78,7 +80,8 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error { if adminHost == "" { return fmt.Errorf("admin hostname is required") } - if err := writeConnectionConfig(panelHost, adminHost, ""); err != nil { + // Caddy, nginx and Traefik all append the peer they saw to X-Forwarded-For. + if err := writeConnectionConfig(panelHost, adminHost, "", "X-Forwarded-For"); err != nil { return err } if err := applyFelisConfigSecret(ctx); err != nil { @@ -93,16 +96,17 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error { // writeConnectionConfig stamps the chosen hostnames (and optional Access audience) // into both the host and pod config files. An empty aud clears any prior // Cloudflare audience, which is correct when switching to a non-Access front. -func writeConnectionConfig(panelHost, adminHost, aud string) error { +// clientIPHeader is the header that front writes the visitor address into. +func writeConnectionConfig(panelHost, adminHost, aud, clientIPHeader string) error { for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} { - if err := updateAuthConfig(path, panelHost, adminHost, aud); err != nil { + if err := updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader); err != nil { return err } } return nil } -func updateAuthConfig(path, panelHost, adminHost, aud string) error { +func updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader string) error { cfg, err := config.Load(path) if err != nil { return err @@ -112,6 +116,7 @@ func updateAuthConfig(path, panelHost, adminHost, aud string) error { } cfg.Auth.AdminHostname = adminHost cfg.Auth.AccessJWTAud = aud + cfg.Auth.ClientIPHeader = clientIPHeader return writeConfig(path, cfg) } diff --git a/deploy/alerts/felis-alerts.yaml b/deploy/alerts/felis-alerts.yaml index f4fc9c0..929ae9f 100644 --- a/deploy/alerts/felis-alerts.yaml +++ b/deploy/alerts/felis-alerts.yaml @@ -4,7 +4,9 @@ # # felis_* series come from two processes: # - felis-operator pod :8080/metrics → felis_servers_total, felis_start_duration_seconds -# - felis-api internal :8081/metrics → felis_image_build_failures_total +# - felis-api internal :8081/metrics → felis_image_build_failures_total, +# felis_mail_total, felis_rate_limited_total, +# felis_auth_otp_lockouts_total # - node-exporter textfile collector → felis_db_backup_* (felis-db-backup.timer) # node_* / kube_* series come from node-exporter / kube-state-metrics. groups: @@ -94,3 +96,49 @@ groups: --collector.textfile.directory at the directory of FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) (troubleshooting §16). + - name: felis.auth.rules + rules: + - alert: FelisMailBudgetExhausted + expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0 + labels: + severity: warning + annotations: + summary: "the install-wide mail budget refused mail" + description: >- + felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is + spent, and every sign-in code is refused with 429 mail_rate_limited until + it refills. Check felis_rate_limited_total for a flood before raising the + budget (troubleshooting §17). + - alert: FelisMailDeliveryFailing + expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0 + labels: + severity: warning + annotations: + summary: "the SMTP relay refused mail in the last 15m" + description: >- + felis_mail_total{result="failed"} increased: sign-in codes are not being + delivered (502 mail_undeliverable). The relay's reason is in the + felis-api log (troubleshooting §17). + - alert: FelisSignInFlood + expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10 + for: 10m + labels: + severity: warning + annotations: + summary: "sign-in doors refusing over 10 requests a minute" + description: >- + The per-address sign-in limit has been refusing callers for 10 minutes. + A script is hammering the auth doors; if real users report rate_limited + at once instead, [auth] client_ip_header is missing and everyone shares + the proxy's address (troubleshooting §17). + - alert: FelisOTPAccountLocked + expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0 + labels: + severity: warning + annotations: + summary: "an account's email-code sign-in locked after 10 wrong codes" + description: >- + Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}). + The audit log names the account (action auth.otp.locked); the owner was + mailed. Unless they fumbled codes, someone is guessing at it + (troubleshooting §17). diff --git a/deploy/alerts/felis-alerts_test.yml b/deploy/alerts/felis-alerts_test.yml index 24071ef..322ae22 100644 --- a/deploy/alerts/felis-alerts_test.yml +++ b/deploy/alerts/felis-alerts_test.yml @@ -151,3 +151,88 @@ tests: --collector.textfile.directory at the directory of FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) (troubleshooting §16). + - name: sign-in mail budget and relay + interval: 1m + input_series: + # Created at zero on start; the budget refuses one mail at t=3m. + - series: 'felis_mail_total{kind="otp",result="throttled",job="felis-api"}' + values: '0 0 0 1x30' + - series: 'felis_mail_total{kind="otp",result="failed",job="felis-api"}' + values: '0x33' + alert_rule_test: + - eval_time: 2m + alertname: FelisMailBudgetExhausted + exp_alerts: [] + - eval_time: 5m + alertname: FelisMailBudgetExhausted + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + summary: "the install-wide mail budget refused mail" + description: >- + felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is + spent, and every sign-in code is refused with 429 mail_rate_limited until + it refills. Check felis_rate_limited_total for a flood before raising the + budget (troubleshooting §17). + - eval_time: 5m + alertname: FelisMailDeliveryFailing + exp_alerts: [] + - name: sign-in flood + interval: 1m + input_series: + # 30 refusals a minute from t=0; a lone refused script at 2/min stays quiet. + - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}' + values: '0+30x40' + alert_rule_test: + - eval_time: 10m + alertname: FelisSignInFlood + exp_alerts: [] + - eval_time: 20m + alertname: FelisSignInFlood + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + summary: "sign-in doors refusing over 10 requests a minute" + description: >- + The per-address sign-in limit has been refusing callers for 10 minutes. + A script is hammering the auth doors; if real users report rate_limited + at once instead, [auth] client_ip_header is missing and everyone shares + the proxy's address (troubleshooting §17). + - name: sign-in trickle stays quiet + interval: 1m + input_series: + - series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}' + values: '0+2x40' + alert_rule_test: + - eval_time: 30m + alertname: FelisSignInFlood + exp_alerts: [] + - name: account email-code lock + interval: 1m + input_series: + - series: 'felis_auth_otp_lockouts_total{purpose="login_email",job="felis-api"}' + values: '0 0 1x90' + - series: 'felis_auth_otp_lockouts_total{purpose="op_login",job="felis-api"}' + values: '0x92' + alert_rule_test: + - eval_time: 1m + alertname: FelisOTPAccountLocked + exp_alerts: [] + - eval_time: 10m + alertname: FelisOTPAccountLocked + exp_alerts: + - exp_labels: + severity: warning + purpose: login_email + exp_annotations: + summary: "an account's email-code sign-in locked after 10 wrong codes" + description: >- + Someone entered 10 wrong codes for one account within 24h (login_email). + The audit log names the account (action auth.otp.locked); the owner was + mailed. Unless they fumbled codes, someone is guessing at it + (troubleshooting §17). + - eval_time: 90m + alertname: FelisOTPAccountLocked + exp_alerts: [] diff --git a/deploy/alerts/felis-prometheusrule.yaml b/deploy/alerts/felis-prometheusrule.yaml index 01cb6bb..0b68db9 100644 --- a/deploy/alerts/felis-prometheusrule.yaml +++ b/deploy/alerts/felis-prometheusrule.yaml @@ -98,3 +98,49 @@ spec: --collector.textfile.directory at the directory of FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) (troubleshooting §16). + - name: felis.auth.rules + rules: + - alert: FelisMailBudgetExhausted + expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0 + labels: + severity: warning + annotations: + summary: "the install-wide mail budget refused mail" + description: >- + felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is + spent, and every sign-in code is refused with 429 mail_rate_limited until + it refills. Check felis_rate_limited_total for a flood before raising the + budget (troubleshooting §17). + - alert: FelisMailDeliveryFailing + expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0 + labels: + severity: warning + annotations: + summary: "the SMTP relay refused mail in the last 15m" + description: >- + felis_mail_total{result="failed"} increased: sign-in codes are not being + delivered (502 mail_undeliverable). The relay's reason is in the + felis-api log (troubleshooting §17). + - alert: FelisSignInFlood + expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10 + for: 10m + labels: + severity: warning + annotations: + summary: "sign-in doors refusing over 10 requests a minute" + description: >- + The per-address sign-in limit has been refusing callers for 10 minutes. + A script is hammering the auth doors; if real users report rate_limited + at once instead, [auth] client_ip_header is missing and everyone shares + the proxy's address (troubleshooting §17). + - alert: FelisOTPAccountLocked + expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0 + labels: + severity: warning + annotations: + summary: "an account's email-code sign-in locked after 10 wrong codes" + description: >- + Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}). + The audit log names the account (action auth.otp.locked); the owner was + mailed. Unless they fumbled codes, someone is guessing at it + (troubleshooting §17). diff --git a/docs/openapi.yaml b/docs/openapi.yaml index f53d27d..13afb3d 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -147,6 +147,19 @@ components: content: application/json: schema: { $ref: '#/components/schemas/Error' } + RateLimited: + description: > + This client address called the public sign-in doors faster than the per-address + limit allows (code rate_limited); Retry-After gives the seconds until the next + call is admitted. The address is the visitor header the install's edge writes + ([auth] client_ip_header: CF-Connecting-IP behind the Cloudflare tunnel), else + the TCP peer; IPv6 clients share one limit per /64. + headers: + Retry-After: + schema: { type: integer } + content: + application/json: + schema: { $ref: '#/components/schemas/Error' } AccessResult: description: The structured access mutation succeeded; the raw RCON reply is in output. content: @@ -1875,8 +1888,8 @@ paths: array. It never reveals staffness: methods are computed identically for every resolved account (no role branch), so a staff and a player address in the same credential state return byte-identical bodies. passkey is offered only when a - verifier is wired. Sends no mail and mutates nothing; not rate-limited at the app - layer (volumetric abuse is bounded at the edge). Gated on local_auth_enabled. + verifier is wired. Sends no mail and mutates nothing; bounded by the per-address + sign-in rate limit (429 rate_limited). Gated on local_auth_enabled. x-felis-face: [external] x-felis-tier: public security: [] @@ -1918,6 +1931,8 @@ paths: content: application/json: schema: { $ref: '#/components/schemas/Error' } + '429': + $ref: '#/components/responses/RateLimited' /api/v1/auth/passkey/login/begin: post: @@ -1973,7 +1988,9 @@ paths: application/json: schema: { $ref: '#/components/schemas/Error' } '429': - description: A passkey login for this recipient was started too recently (otp_resend_cooldown). + description: >- + A passkey login for this recipient was started too recently (otp_resend_cooldown); + or this client address called the sign-in doors too often (rate_limited, with Retry-After). content: application/json: schema: { $ref: '#/components/schemas/Error' } @@ -2045,6 +2062,8 @@ paths: content: application/json: schema: { $ref: '#/components/schemas/Error' } + '429': + $ref: '#/components/responses/RateLimited' '503': description: No passkey verifier is wired on this deployment (passkey_unavailable). content: @@ -2066,9 +2085,9 @@ paths: userHandle inside the signed assertion at finish. The challenge cannot be user-keyed, so it is stashed under login_id in a non-user-keyed store and echoed back at finish. Mounted Public and gated on local_auth_enabled. There is no - recipient or principal to key a per-caller cooldown on (that volumetric limiting - is delegated to the edge), so the server-side brake is a hard global cap on live - challenges (429 too_many_challenges). Inert for a credential until its owner + recipient or principal to key a per-caller cooldown on, so one client is bounded + by the per-address sign-in rate limit (429 rate_limited) and the table by a hard + global cap on live challenges (429 too_many_challenges). Inert for a credential until its owner enrolls a resident passkey; email-OTP and username-first passkey remain the fallbacks, so no authenticator is ever locked out. x-felis-face: [external] @@ -2114,8 +2133,9 @@ paths: schema: { $ref: '#/components/schemas/Error' } '429': description: >- - Too many discoverable logins are in flight server-wide; the global cap is hit - (too_many_challenges). No per-recipient signal is leaked — the cap is global. + Too many discoverable logins are in flight server-wide (too_many_challenges; + the cap is global, so no per-recipient signal leaks); or this client address + called the sign-in doors too often (rate_limited, with Retry-After). content: application/json: schema: { $ref: '#/components/schemas/Error' } @@ -2192,6 +2212,8 @@ paths: content: application/json: schema: { $ref: '#/components/schemas/Error' } + '429': + $ref: '#/components/responses/RateLimited' '503': description: No passkey verifier is wired on this deployment (passkey_unavailable). content: @@ -2253,7 +2275,10 @@ paths: application/json: schema: { $ref: '#/components/schemas/Error' } '429': - description: A code for this recipient was requested too recently (otp_resend_cooldown). + description: >- + A code for this recipient was requested too recently (otp_resend_cooldown); + or this client address called the sign-in doors too often (rate_limited, with Retry-After); + or the install-wide mail budget is spent (mail_rate_limited, with Retry-After). content: application/json: schema: { $ref: '#/components/schemas/Error' } @@ -2318,6 +2343,8 @@ paths: content: application/json: schema: { $ref: '#/components/schemas/Error' } + '429': + $ref: '#/components/responses/RateLimited' /api/v1/auth/op-login/start: post: @@ -2373,7 +2400,10 @@ paths: application/json: schema: { $ref: '#/components/schemas/Error' } '429': - description: A code for this recipient was requested too recently (otp_resend_cooldown). + description: >- + A code for this recipient was requested too recently (otp_resend_cooldown); + or this client address called the sign-in doors too often (rate_limited, with Retry-After); + or the install-wide mail budget is spent (mail_rate_limited, with Retry-After). content: application/json: schema: { $ref: '#/components/schemas/Error' } @@ -2467,6 +2497,8 @@ paths: content: application/json: schema: { $ref: '#/components/schemas/Error' } + '429': + $ref: '#/components/responses/RateLimited' /api/v1/auth/setup/redeem: post: @@ -2524,6 +2556,8 @@ paths: content: application/json: schema: { $ref: '#/components/schemas/Error' } + '429': + $ref: '#/components/responses/RateLimited' /api/v1/auth/setup/status: get: @@ -2640,6 +2674,8 @@ paths: content: application/json: schema: { $ref: '#/components/schemas/Error' } + '429': + $ref: '#/components/responses/RateLimited' /api/v1/me: get: @@ -3778,7 +3814,8 @@ paths: description: >- Resend requested before the cooldown elapsed (otp_resend_cooldown); or the account spent its daily wrong-code budget (otp_account_locked, with - Retry-After). + Retry-After); or the install-wide mail budget is spent + (mail_rate_limited, with Retry-After). content: application/json: schema: { $ref: '#/components/schemas/Error' } @@ -4091,7 +4128,8 @@ paths: description: >- Resend requested before the cooldown elapsed (otp_resend_cooldown), or the account's daily wrong-code budget is spent (otp_account_locked, with - Retry-After). + Retry-After); or the install-wide mail budget is spent + (mail_rate_limited, with Retry-After). content: application/json: schema: { $ref: '#/components/schemas/Error' } diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 5e4f56d..538cdcc 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -931,7 +931,9 @@ The series come from two processes: `felis_start_duration_seconds` (no Service; scrape pod-scoped, e.g. a PodMonitor targeting port `metrics`). - `felis-api` internal face `:8081/metrics` (Service `felis-api-internal`) — - `felis_image_build_failures_total`. Unauthenticated like the probes; + `felis_image_build_failures_total`, and the sign-in series of §17 + (`felis_mail_total`, `felis_rate_limited_total`, + `felis_auth_otp_lockouts_total`). Unauthenticated like the probes; ClusterIP-only, and the external face never serves it. - `felis_reaper_worlds_deleted_total` is produced inside the one-shot reaper CronJob, which exits long before any scrape interval — without a pushgateway @@ -941,8 +943,10 @@ The series come from two processes: ### Alert rules `deploy/alerts/` ships ready-made rules: build failures, slow starts, node -disk/memory thresholds, the kubelet `DiskPressure` condition, and control-plane -database backup freshness (§16; needs node-exporter's textfile collector). +disk/memory thresholds, the kubelet `DiskPressure` condition, control-plane +database backup freshness (§16; needs node-exporter's textfile collector), and +sign-in abuse: the mail budget, relay failures, throttled floods and account +code locks (§17). - Plain Prometheus: add `felis-alerts.yaml` to `rule_files`. Check and unit-test it standalone with `promtool check rules felis-alerts.yaml` and @@ -1167,6 +1171,75 @@ Skips the pre-migration snapshot (`migrate up -no-backup`). The installer warns loudly when it is set. Use it only when the snapshot cannot work and you have another backup, e.g. an external database newer than the host's `pg_dump`. +## 17. Sign-in refused with 429, mail budget, account code locks + +The public sign-in doors (`/api/v1/auth/*` except logout and the op-login +status poll) have three limits of their own. Each answers 429 with a +`Retry-After` header and a distinct error code. + +### `rate_limited`: one address called the doors too often + +Each client address gets 20 calls at once, refilled at 20 a minute, shared +across every door. A person signing in makes three or four calls, so this only +bites scripts. IPv6 clients share one limit per /64. Refusals count in +`felis_rate_limited_total{scope="auth_door"}`; `FelisSignInFlood` fires when +more than 10 a minute are refused for 10 minutes. + +The address comes from `[auth] client_ip_header`: + +- Behind the Cloudflare tunnel it is `CF-Connecting-IP`. The edge setup writes + it, and an install with an `access_jwt_aud` implies it. The header is + trustworthy there because the same setup fences the panel NodePort to + loopback, so every request reaching the API came through cloudflared. +- Behind your own reverse proxy it is `X-Forwarded-For` (the rightmost entry, + the one your proxy appended). Firewall the NodePort so only the proxy reaches + it, or a direct caller can write any address it likes. +- Unset, the TCP peer is used. Behind any proxy every visitor then shares the + proxy's address and one limit, so **everyone gets `rate_limited` at once**. + The `felis api` log says at start which it keys on (`sign-in rate limit keys + on ...`). Set the header in `/etc/felis/felis.toml` and + `/etc/felis/felis.pod.toml`, then `felis converge`. + +### `mail_rate_limited`: the install-wide mail budget is spent + +Every code and notice the API mails spends one token of a single budget, +`[smtp] max_per_hour` (default 120; a quarter of it may go at once), so a flood +cannot burn the relay's quota and get the sending account suspended. While it +is spent, every address gets the same 429 and nothing reaches the relay. +`felis_mail_total{result="throttled"}` counts refusals and +`FelisMailBudgetExhausted` fires on the first one. Look at +`felis_rate_limited_total` first: a flood shows there. If sign-ins are real, +raise `max_per_hour` to what your relay allows. + +`FelisMailDeliveryFailing` is the other half: the relay itself refused mail +(`felis_mail_total{result="failed"}`, 502 `mail_undeliverable` to the caller). +The relay's reason is in the `felis-api` log. + +### `otp_account_locked`: ten wrong codes in 24 hours + +Ten wrong email codes for one account within 24 hours, counted across every +code it was sent, lock that account's email-code sign-in until 24 hours after +the first miss. The public doors answer a locked account exactly like a wrong +code, and the owner gets one mail saying so. Signed-in doors (email +verification, migration step-up) answer 429 `otp_account_locked`. Passkey +sign-in keeps working. Each lock is audited as `auth.otp.locked` and counted in +`felis_auth_otp_lockouts_total{purpose}` (`FelisOTPAccountLocked`). + +To lift a lock early once you have confirmed the owner locked themselves out: + +```sh +sudo -u postgres psql felis -c \ + "DELETE FROM otp_failure_windows WHERE user_id = (SELECT id FROM users WHERE username = '');" +``` + +### Optional: a Cloudflare rate limiting rule in front + +The limits above live in the API, so they hold on any edge. Behind Cloudflare +you can also stop floods before they reach the tunnel: Security → WAF → Rate +limiting rules, match URI Path starts with `/api/v1/auth/` on the console and +op.console hostnames, count by IP, 30 requests per 10 seconds, action Block +for 10 seconds (the Free plan's limits). + --- ## Quick reference: symptom → section @@ -1198,3 +1271,6 @@ another backup, e.g. an external database newer than the host's `pg_dump`. | `pre-migration backup failed, nothing applied` during an upgrade | §16 | | Undo a mistaken change / restore the control-plane database | §16 | | Host lost: rebuild from a database bundle | §16 | +| Sign-in 429 `rate_limited` for everyone at once | §17 | +| 429 `mail_rate_limited` / `FelisMailBudgetExhausted` | §17 | +| Right code refused; `otp_account_locked` / `FelisOTPAccountLocked` | §17 | diff --git a/internal/api/api.go b/internal/api/api.go index 902a319..6699e7c 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -158,6 +158,16 @@ type API struct { // Consumed by handleHasJoined (handlers_hasjoined.go). AuthSources []AuthSource + // AuthDoorLimit bounds how often one client address may call the public + // pre-session auth doors (ratelimit.go). MailLimit bounds all mail the API + // sends, install-wide. Zero values disable them; cmd/felis wires both. + AuthDoorLimit RateLimit + MailLimit RateLimit + // ClientIPHeader names the header the install's edge writes the client + // address into (CF-Connecting-IP behind the Cloudflare tunnel, + // X-Forwarded-For behind an operator proxy). Empty means the TCP peer. + ClientIPHeader string + // Now is the clock, injectable for tests. Defaults to time.Now. Now func() time.Time @@ -172,6 +182,11 @@ type API struct { streamCapOnce sync.Once streamCap *streamLimiter + + authDoorOnce sync.Once + authDoorBuckets *bucketSet + mailOnce sync.Once + mailBuckets *bucketSet } // panelURL returns the public player-console origin ("https://console."), @@ -286,6 +301,11 @@ type apiRoute struct { // whose EmailVerified is false is restricted to these routes only. SetupAllowed bool + // AuthDoor marks a public pre-session auth door: it is rate limited per + // client address (throttleAuthDoor). The op-login status poll is left off, + // since the browser calls it every few seconds while it waits. + AuthDoor bool + h http.HandlerFunc } @@ -381,21 +401,21 @@ func (a *API) externalAPIRoutes() []apiRoute { // counter-slice to the anti-enumeration doors — the ONE sanctioned place existence // is disclosed — but it never reveals staffness (methods computed with no role // branch, so a staff and a player address in the same state are indistinguishable). - {Method: "POST", Pattern: "/api/v1/auth/options", Public: true, h: a.handleAuthOptions}, - {Method: "POST", Pattern: "/api/v1/auth/setup/redeem", Public: true, h: a.handleSetupRedeem}, + {Method: "POST", Pattern: "/api/v1/auth/options", Public: true, AuthDoor: true, h: a.handleAuthOptions}, + {Method: "POST", Pattern: "/api/v1/auth/setup/redeem", Public: true, AuthDoor: true, h: a.handleSetupRedeem}, {Method: "GET", Pattern: "/api/v1/auth/setup/status", SetupAllowed: true, h: a.handleSetupStatus}, - {Method: "POST", Pattern: "/api/v1/auth/passkey/login/begin", Public: true, h: a.handlePasskeyLoginBegin}, - {Method: "POST", Pattern: "/api/v1/auth/passkey/login/finish", Public: true, h: a.handlePasskeyLoginFinish}, + {Method: "POST", Pattern: "/api/v1/auth/passkey/login/begin", Public: true, AuthDoor: true, h: a.handlePasskeyLoginBegin}, + {Method: "POST", Pattern: "/api/v1/auth/passkey/login/finish", Public: true, AuthDoor: true, h: a.handlePasskeyLoginFinish}, // Discoverable ("usernameless") passkey login (task #40): the from-zero sibling of the // email-first pair above — no identifier typed, the account is resolved from the // userHandle inside the signed assertion (handlers_passkey_discoverable.go). - {Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/begin", Public: true, h: a.handlePasskeyLoginDiscoverableBegin}, - {Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/finish", Public: true, h: a.handlePasskeyLoginDiscoverableFinish}, - {Method: "POST", Pattern: "/api/v1/auth/email/start", Public: true, h: a.handleLoginEmailStart}, - {Method: "POST", Pattern: "/api/v1/auth/email/verify", Public: true, h: a.handleLoginEmailVerify}, - {Method: "POST", Pattern: "/api/v1/auth/op-login/start", Public: true, h: a.handleOpLoginStart}, + {Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/begin", Public: true, AuthDoor: true, h: a.handlePasskeyLoginDiscoverableBegin}, + {Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/finish", Public: true, AuthDoor: true, h: a.handlePasskeyLoginDiscoverableFinish}, + {Method: "POST", Pattern: "/api/v1/auth/email/start", Public: true, AuthDoor: true, h: a.handleLoginEmailStart}, + {Method: "POST", Pattern: "/api/v1/auth/email/verify", Public: true, AuthDoor: true, h: a.handleLoginEmailVerify}, + {Method: "POST", Pattern: "/api/v1/auth/op-login/start", Public: true, AuthDoor: true, h: a.handleOpLoginStart}, {Method: "GET", Pattern: "/api/v1/auth/op-login/status/{id}", Public: true, h: a.handleOpLoginStatus}, - {Method: "POST", Pattern: "/api/v1/auth/op-login/finish", Public: true, h: a.handleOpLoginFinish}, + {Method: "POST", Pattern: "/api/v1/auth/op-login/finish", Public: true, AuthDoor: true, h: a.handleOpLoginFinish}, // Player-console bootstrap (console-tier access model): the account-less // player's door into console.. Public — like login there is no prior // principal — and session-minting, but the artifact it consumes is a one-time @@ -403,7 +423,7 @@ func (a *API) externalAPIRoutes() []apiRoute { // possession already proves a Minecraft identity. A code whose UUID belongs to // staff is refused (403) so this never yields an admin session; op.console stays // behind Zero Trust (handlers_onboard.go). - {Method: "POST", Pattern: "/api/v1/auth/bind", Public: true, h: a.handleBindRedeem}, + {Method: "POST", Pattern: "/api/v1/auth/bind", Public: true, AuthDoor: true, h: a.handleBindRedeem}, // App-auth tier: operations on your own servers (spec §14). {Method: "POST", Pattern: "/api/v1/servers/{name}/wake", h: a.handleWake}, @@ -614,7 +634,11 @@ func (a *API) buildFace(routes []apiRoute, guard func(http.Handler) http.Handler for _, rt := range routes { pattern := rt.Method + " " + rt.Pattern if rt.Public { - mux.HandleFunc(pattern, rt.h) + h := rt.h + if rt.AuthDoor { + h = a.throttleAuthDoor(h) + } + mux.HandleFunc(pattern, h) continue } h := rt.h @@ -727,10 +751,35 @@ func principalFromContext(ctx context.Context) *Principal { // reserve/release pair closes the intra-replica concurrent burst (the bug fixed in // #35); cross-replica bounding would need a shared store (out of scope for the // single-replica demo). +// +// Entries older than the longest window the limiter has been asked about can +// no longer block anything, so checks sweep them out (at most once per +// bucketSweepEvery). Without that, every distinct address typed into a public +// door, whose neutral branch keeps its reservation, stayed in the map for the +// life of the process. type cooldownLimiter struct { - mu sync.Mutex - now func() time.Time - last map[string]time.Time + mu sync.Mutex + now func() time.Time + last map[string]time.Time + maxWindow time.Duration + swept time.Time +} + +// noteWindow widens the retention to window and sweeps stale entries when due. +// The caller holds mu. +func (c *cooldownLimiter) noteWindow(window time.Duration, now time.Time) { + if window > c.maxWindow { + c.maxWindow = window + } + if c.maxWindow <= 0 || now.Sub(c.swept) < bucketSweepEvery { + return + } + c.swept = now + for k, t := range c.last { + if now.Sub(t) >= c.maxWindow { + delete(c.last, k) + } + } } // allowed reports whether name may wake now WITHOUT recording the attempt. A @@ -745,7 +794,9 @@ func (c *cooldownLimiter) allowed(name string, window time.Duration) bool { } c.mu.Lock() defer c.mu.Unlock() - if last, ok := c.last[name]; ok && c.now().Sub(last) < window { + now := c.now() + c.noteWindow(window, now) + if last, ok := c.last[name]; ok && now.Sub(last) < window { return false } return true @@ -778,10 +829,11 @@ func (c *cooldownLimiter) reserve(name string, window time.Duration) (time.Time, } c.mu.Lock() defer c.mu.Unlock() - if last, ok := c.last[name]; ok && c.now().Sub(last) < window { + t := c.now() + c.noteWindow(window, t) + if last, ok := c.last[name]; ok && t.Sub(last) < window { return time.Time{}, false } - t := c.now() c.last[name] = t return t, true } diff --git a/internal/api/errors.go b/internal/api/errors.go index 9dea2ed..9bc3f56 100644 --- a/internal/api/errors.go +++ b/internal/api/errors.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "net/http" + "strconv" "time" ) @@ -130,10 +131,19 @@ type apiError struct { status int code string msg string + // wait, when positive, is sent as Retry-After (whole seconds, rounded up). + wait time.Duration } func (e *apiError) Error() string { return e.msg } +// retryAfter returns a copy of e that tells the client when to retry. +func (e *apiError) retryAfter(d time.Duration) *apiError { + c := *e + c.wait = d + return &c +} + // newError builds an apiError with a formatted message. func newError(status int, code, format string, a ...any) *apiError { return &apiError{status: status, code: code, msg: fmt.Sprintf(format, a...)} @@ -176,6 +186,9 @@ func writeError(w http.ResponseWriter, r *http.Request, err error) { r.Method, r.URL.Path, requestIDFromContext(r.Context()), err) ae = newError(http.StatusInternalServerError, "internal", "internal error") } + if ae.wait > 0 { + w.Header().Set("Retry-After", strconv.FormatInt(int64((ae.wait+time.Second-1)/time.Second), 10)) + } body := map[string]any{ "error": map[string]string{ "code": ae.code, diff --git a/internal/api/handlers_auth_email.go b/internal/api/handlers_auth_email.go index 6e75964..ce1f9e9 100644 --- a/internal/api/handlers_auth_email.go +++ b/internal/api/handlers_auth_email.go @@ -24,12 +24,9 @@ import ( // this separator). // - No principal. The throttle cannot key off a user id (there is none yet); it // keys off the typed recipient address, the same anti-bomb dimension the onboard -// start uses. Per-source (client-IP) aggregate limiting is deliberately NOT done -// here: cooldownLimiter is a one-per-window primitive, so keying it on client IP -// would false-positive on shared egress (CGNAT / office NAT), and behind -// Cloudflare RemoteAddr is the proxy anyway. The only real harm — bombing one -// mailbox — is already bounded per recipient; volumetric per-source limiting -// belongs at the edge. +// start uses. Volume from one client is bounded separately by the per-address +// token bucket every public auth door sits behind (throttleAuthDoor), and total +// mail by the install-wide mail budget (ratelimit.go). // - Refuse staff. Like handleBindRedeem this public door provably never mints a // session for an admin identity: op.console stays behind Zero Trust (and its own // in-game approval gate). The refusal happens only AFTER a valid code is @@ -84,6 +81,13 @@ func (a *API) handleLoginEmailStart(w http.ResponseWriter, r *http.Request) { return } + // The install-wide mail budget is checked before the address is resolved, + // so while it is spent every address gets the same 429. + if err := a.checkMailBudget(); err != nil { + writeError(w, r, err) + return + } + // Atomically reserve the per-recipient cooldown BEFORE any work, so a burst of // truly concurrent starts yields exactly one winner and each admitted send is one // real, non-idempotent email. The key is namespaced apart from the onboard door's diff --git a/internal/api/handlers_auth_options.go b/internal/api/handlers_auth_options.go index dea7853..a96c19d 100644 --- a/internal/api/handlers_auth_options.go +++ b/internal/api/handlers_auth_options.go @@ -16,11 +16,10 @@ import ( // address has an account precisely because THIS endpoint is the one sanctioned place // existence is revealed. An empty methods array means "no (verified) account". That // makes it a mass-enumeration surface by design — an accepted product decision, the -// same one the email door's header records. It is bounded only at the edge: the -// handler sends no mail and mutates nothing, so a per-recipient cooldown would merely -// block a legitimate retry, and per-source (client-IP) limiting is the edge's job -// (behind Cloudflare RemoteAddr is the proxy, and CGNAT would false-positive) — see -// the handlers_auth_email.go header for the same reasoning. +// same one the email door's header records. The handler sends no mail and mutates +// nothing, so a per-recipient cooldown would merely block a legitimate retry; what +// bounds enumeration is the per-client-address token bucket shared by every public +// auth door (throttleAuthDoor in ratelimit.go). // // It never reveals STAFFNESS. Methods are computed by the SAME rule for every resolved // account — no role branch, no operator hint — so a staff email and a player email in diff --git a/internal/api/handlers_email_otp.go b/internal/api/handlers_email_otp.go index a9fc1b6..1b9502c 100644 --- a/internal/api/handlers_email_otp.go +++ b/internal/api/handlers_email_otp.go @@ -11,6 +11,8 @@ import ( "net/http" "strings" "time" + + "felis.lolicon.best/internal/metrics" ) // Player email verification (spec §B2 onboarding). Forced web onboarding proves a @@ -250,12 +252,21 @@ func (a *API) handleEmailOTPVerify(w http.ResponseWriter, r *http.Request) { // deliverOTP hands the code to the configured Mailer, or — when none is wired (the // demo) — logs it server-side as a KNOWN-LIMITATION. The code is logged ONLY in the // no-mailer fallback and ONLY to the server log; it is never put in an HTTP response. +// +// Every real send spends one token of the install-wide mail budget (mailGate); +// a spent budget is a 429 mail_rate_limited and nothing reaches the relay. func (a *API) deliverOTP(ctx context.Context, email, code string) error { if a.Mailer == nil { log.Printf("email-otp: no Mailer configured; code for %s is %s (KNOWN-LIMITATION: demo has no SMTP)", email, code) return nil } + if ok, wait := a.mailGate().take(mailGateKey); !ok { + metrics.MailTotal.WithLabelValues("otp", "throttled").Inc() + log.Printf("api: OTP mail refused by the install-wide mail budget (request_id=%s)", requestIDFromContext(ctx)) + return errMailRateLimited(wait) + } if err := a.Mailer.SendOTP(ctx, email, code); err != nil { + metrics.MailTotal.WithLabelValues("otp", "failed").Inc() // Mapped here rather than at each of the four call sites, so every door that // mails a code answers the same way. A relay refusal is neither the caller's // fault nor a bug in Felis, and a bare 500 says neither — it reads as "the @@ -269,6 +280,7 @@ func (a *API) deliverOTP(ctx context.Context, email, code string) error { return newError(http.StatusBadGateway, "mail_undeliverable", "the mail relay refused this message; ask the server operator to check the SMTP settings") } + metrics.MailTotal.WithLabelValues("otp", "sent").Inc() return nil } diff --git a/internal/api/handlers_op_login.go b/internal/api/handlers_op_login.go index 2e840ba..35c5110 100644 --- a/internal/api/handlers_op_login.go +++ b/internal/api/handlers_op_login.go @@ -86,6 +86,13 @@ func (a *API) handleOpLoginStart(w http.ResponseWriter, r *http.Request) { return } + // The install-wide mail budget is checked before the address is resolved, + // so while it is spent every address gets the same 429. + if err := a.checkMailBudget(); err != nil { + writeError(w, r, err) + return + } + // Per-recipient cooldown reserved BEFORE any work, identical to the console email // door: one winner per window, and the neutral (non-staff) branch keeps the // reservation too so probing an address is throttled exactly like a real send. The diff --git a/internal/api/handlers_passkey_discoverable.go b/internal/api/handlers_passkey_discoverable.go index 564811e..db0c178 100644 --- a/internal/api/handlers_passkey_discoverable.go +++ b/internal/api/handlers_passkey_discoverable.go @@ -21,10 +21,10 @@ import ( // // Anti-abuse divergence from the email-first door: that door reserves a per-recipient cooldown // (a.otpLimiter) keyed on the typed email. A usernameless begin has no recipient OR principal to -// key a fair per-caller limit on, so — matching the stance in handlers_auth_email.go (behind -// Cloudflare RemoteAddr is the proxy; CGNAT false-positives) — volumetric per-source limiting is -// left to the edge, and the server-side bound is a hard global cap on live challenges enforced -// atomically in CreateDiscoverableChallenge (ErrTooManyDiscoverableChallenges → 429). +// key a fair per-caller limit on, so one client is bounded by the per-address token bucket every +// public auth door sits behind (throttleAuthDoor), and the table by a hard global cap on live +// challenges enforced atomically in CreateDiscoverableChallenge (ErrTooManyDiscoverableChallenges +// → 429). // handlePasskeyLoginDiscoverableBegin starts a usernameless assertion ceremony (Public, // pre-session). It has no request body — the whole point is that the caller supplies no diff --git a/internal/api/otp_lock.go b/internal/api/otp_lock.go index d09e8e0..7189955 100644 --- a/internal/api/otp_lock.go +++ b/internal/api/otp_lock.go @@ -7,7 +7,6 @@ import ( "fmt" "log" "net/http" - "strconv" "time" "felis.lolicon.best/internal/metrics" @@ -35,14 +34,9 @@ var otpDoorName = map[string][2]string{ // writeOTPAccountLocked answers a signed-in door whose budget is spent. func writeOTPAccountLocked(w http.ResponseWriter, r *http.Request, until, now time.Time) { - secs := int64(until.Sub(now).Round(time.Second) / time.Second) - if secs < 1 { - secs = 1 - } - w.Header().Set("Retry-After", strconv.FormatInt(secs, 10)) writeError(w, r, newError(http.StatusTooManyRequests, "otp_account_locked", "too many wrong codes on this account; email codes work again after %s", - until.UTC().Format(time.RFC3339))) + until.UTC().Format(time.RFC3339)).retryAfter(max(until.Sub(now), time.Second))) } // noteOTPLock handles a redeem that met the account lock. Only the guess that @@ -83,10 +77,18 @@ func (a *API) noteOTPLock(r *http.Request, err error, userID, purpose string) { log.Printf("auth: no notice mailer; user %s was not told their %s is locked", userID, purpose) return } + if ok, _ := a.mailGate().take(mailGateKey); !ok { + metrics.MailTotal.WithLabelValues("notice", "throttled").Inc() + log.Printf("auth: mail budget spent; user %s was not told their %s is locked", userID, purpose) + return + } subject, body := otpLockNotice(door, lock.Until) if err := sender.SendNotice(ctx, u.Email, subject, body); err != nil { + metrics.MailTotal.WithLabelValues("notice", "failed").Inc() log.Printf("auth: otp lock notice to user %s failed: %v", userID, err) + return } + metrics.MailTotal.WithLabelValues("notice", "sent").Inc() } // otpLockNotice renders the bilingual lock notice. diff --git a/internal/api/otp_lock_test.go b/internal/api/otp_lock_test.go index f01944d..606022d 100644 --- a/internal/api/otp_lock_test.go +++ b/internal/api/otp_lock_test.go @@ -3,9 +3,12 @@ package api import ( "context" "net/http" + "slices" "strings" "testing" "time" + + "felis.lolicon.best/internal/metrics" ) // The per-code attempt cap resets on every resend; these pin the account-level @@ -181,3 +184,14 @@ func TestOTPLockNoticeNamesDoorAndTime(t *testing.T) { } } } + +// The lockout alert sees a purpose only if metrics pre-creates its series. +func TestOTPPurposesMatchMetricLabels(t *testing.T) { + got := []string{otpPurposeOnboard, otpPurposeLogin, otpPurposeOpLogin, otpPurposeMigrate} + want := slices.Clone(metrics.OTPPurposes) + slices.Sort(got) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Fatalf("otp purposes %v, metrics.OTPPurposes %v", got, want) + } +} diff --git a/internal/api/pgrepo.go b/internal/api/pgrepo.go index 77e0467..6cd3a58 100644 --- a/internal/api/pgrepo.go +++ b/internal/api/pgrepo.go @@ -1218,9 +1218,8 @@ func (p *PGRepo) ConsumePasskeyChallengeByUser(ctx context.Context, userID, purp // exist, a new begin is refused (ErrTooManyDiscoverableChallenges → 429). The cap is generous — // a login challenge lives only passkeyChallengeTTL (5 min) and each row is ~1 KB — so real // concurrency never approaches it, while an abusive begin-flood is bounded to a few MB instead -// of growing without limit. Volumetric per-IP limiting is the edge's job (handlers_auth_email.go): -// behind Cloudflare RemoteAddr is the proxy, and a usernameless door has no recipient to key a -// fair per-caller limit on. +// of growing without limit. One client's volume is bounded before it gets here, by the +// per-address token bucket in front of every public auth door (ratelimit.go). const maxLiveDiscoverableChallenges = 4096 // CreateDiscoverableChallenge stashes a discoverable-login ceremony under an opaque handle, diff --git a/internal/api/ratelimit.go b/internal/api/ratelimit.go new file mode 100644 index 0000000..d5a50c7 --- /dev/null +++ b/internal/api/ratelimit.go @@ -0,0 +1,247 @@ +package api + +import ( + "math" + "net" + "net/http" + "net/netip" + "strings" + "sync" + "time" + + "felis.lolicon.best/internal/metrics" +) + +// Volumetric limits for the public auth doors and for outbound mail. +// +// The per-recipient OTP cooldown (otpLimiter) stops one mailbox being bombed, +// and the per-account wrong-code budget stops one account being guessed. Neither +// bounds a caller who sprays many addresses or many accounts, so two more limits +// sit in front of them: +// +// - authDoorGate: a token bucket per client address over every pre-session +// auth door (options, login, op-login, bind, setup redeem). The client +// address comes from the edge's header only when the install says which +// header its edge writes (ClientIPHeader); with Cloudflare that header is +// CF-Connecting-IP, which is trustworthy because the edge setup fences the +// panel NodePort to loopback, so every request reaching the origin came +// through cloudflared. IPv6 clients are bucketed per /64, the unit one +// subscriber is handed. +// - mailGate: one install-wide bucket over every mail the API sends (codes +// and lock notices), so no flood can burn the SMTP relay's quota and get +// the sending account suspended. The public doors check it before they +// resolve the address, so a spent budget answers every address alike. + +// RateLimit is a token bucket: Burst calls at once, refilled at PerMinute. The +// zero value disables the limit. +type RateLimit struct { + Burst int + PerMinute float64 +} + +func (l RateLimit) enabled() bool { return l.Burst > 0 && l.PerMinute > 0 } + +// bucketSweepEvery is how often idle buckets are dropped; bucketMaxKeys bounds +// the map between sweeps. Past the bound, new keys share one overflow bucket, +// so a spray of fresh source addresses throttles itself instead of growing the +// map. +const ( + bucketSweepEvery = time.Minute + bucketMaxKeys = 50_000 + bucketOverflow = "\x00overflow" +) + +type tokenBucket struct { + tokens float64 + at time.Time +} + +// bucketSet is a set of token buckets keyed by caller. A missing key is a full +// bucket, so a bucket that has refilled completely carries no information and +// is dropped by the sweep; memory is bounded by the keys active in the last +// refill period. +type bucketSet struct { + mu sync.Mutex + now func() time.Time + limit RateLimit + buckets map[string]*tokenBucket + swept time.Time + maxKeys int +} + +func newBucketSet(limit RateLimit, now func() time.Time) *bucketSet { + return &bucketSet{now: now, limit: limit, buckets: map[string]*tokenBucket{}, maxKeys: bucketMaxKeys} +} + +// refill brings b up to now. The caller holds mu. +func (s *bucketSet) refill(b *tokenBucket, now time.Time) { + if elapsed := now.Sub(b.at); elapsed > 0 { + b.tokens = math.Min(float64(s.limit.Burst), b.tokens+elapsed.Minutes()*s.limit.PerMinute) + } + b.at = now +} + +// sweep drops full buckets at most once per bucketSweepEvery, or at once when +// force is set. The caller holds mu. +func (s *bucketSet) sweep(now time.Time, force bool) { + if !force && now.Sub(s.swept) < bucketSweepEvery { + return + } + s.swept = now + for k, b := range s.buckets { + s.refill(b, now) + if b.tokens >= float64(s.limit.Burst) { + delete(s.buckets, k) + } + } +} + +// bucket returns key's bucket, creating a full one. The caller holds mu. +func (s *bucketSet) bucket(key string, now time.Time) *tokenBucket { + s.sweep(now, false) + if b, ok := s.buckets[key]; ok { + s.refill(b, now) + return b + } + if len(s.buckets) >= s.maxKeys { + s.sweep(now, true) + if len(s.buckets) >= s.maxKeys { + key = bucketOverflow + if b, ok := s.buckets[key]; ok { + s.refill(b, now) + return b + } + } + } + b := &tokenBucket{tokens: float64(s.limit.Burst), at: now} + s.buckets[key] = b + return b +} + +// take spends one token from key's bucket. When none is left it reports how +// long until one is. A disabled limit always admits. +func (s *bucketSet) take(key string) (bool, time.Duration) { + if s == nil || !s.limit.enabled() { + return true, 0 + } + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + b := s.bucket(key, now) + if b.tokens >= 1 { + b.tokens-- + return true, 0 + } + return false, s.wait(b) +} + +// peek reports whether key's bucket holds a token, without spending it. +func (s *bucketSet) peek(key string) (bool, time.Duration) { + if s == nil || !s.limit.enabled() { + return true, 0 + } + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + b, ok := s.buckets[key] + if !ok { + return true, 0 + } + s.refill(b, now) + if b.tokens >= 1 { + return true, 0 + } + return false, s.wait(b) +} + +// wait is how long until b holds one token. The caller holds mu. +func (s *bucketSet) wait(b *tokenBucket) time.Duration { + missing := 1 - b.tokens + return time.Duration(math.Ceil(missing / s.limit.PerMinute * float64(time.Minute))) +} + +func (a *API) authDoorGate() *bucketSet { + a.authDoorOnce.Do(func() { a.authDoorBuckets = newBucketSet(a.AuthDoorLimit, a.now) }) + return a.authDoorBuckets +} + +func (a *API) mailGate() *bucketSet { + a.mailOnce.Do(func() { a.mailBuckets = newBucketSet(a.MailLimit, a.now) }) + return a.mailBuckets +} + +// mailGateKey is the single install-wide mail bucket. +const mailGateKey = "mail" + +// throttleAuthDoor applies the per-source bucket to one public auth door. +func (a *API) throttleAuthDoor(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ok, wait := a.authDoorGate().take(sourceKey(a.clientIP(r))) + if !ok { + metrics.RateLimitedTotal.WithLabelValues("auth_door").Inc() + writeError(w, r, newError(http.StatusTooManyRequests, "rate_limited", + "too many sign-in requests from this network; try again shortly").retryAfter(wait)) + return + } + h(w, r) + } +} + +// errMailRateLimited answers a door whose mail the install-wide budget refuses. +func errMailRateLimited(wait time.Duration) *apiError { + return newError(http.StatusTooManyRequests, "mail_rate_limited", + "this server is sending too much mail right now; try again shortly").retryAfter(wait) +} + +// checkMailBudget is the public doors' pre-resolution check: it refuses every +// address alike while the budget is spent, so the refusal says nothing about +// whether the address has an account. +func (a *API) checkMailBudget() error { + if a.Mailer == nil { + return nil + } + if ok, wait := a.mailGate().peek(mailGateKey); !ok { + metrics.MailTotal.WithLabelValues("otp", "throttled").Inc() + return errMailRateLimited(wait) + } + return nil +} + +// clientIP is the caller's address: the edge's header when the install names +// one and the request carries a parseable value, else the TCP peer. For +// X-Forwarded-For the rightmost entry is used, the one the trusted proxy +// appended itself. +func (a *API) clientIP(r *http.Request) netip.Addr { + if name := a.ClientIPHeader; name != "" { + if v := r.Header.Get(name); v != "" { + if strings.EqualFold(name, "X-Forwarded-For") { + if i := strings.LastIndexByte(v, ','); i >= 0 { + v = v[i+1:] + } + } + if ip, err := netip.ParseAddr(strings.TrimSpace(v)); err == nil { + return ip.Unmap() + } + } + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host = r.RemoteAddr + } + ip, _ := netip.ParseAddr(host) + return ip.Unmap() +} + +// sourceKey buckets an address: IPv4 per host, IPv6 per /64. An unparseable +// address shares one key. +func sourceKey(ip netip.Addr) string { + switch { + case !ip.IsValid(): + return "unknown" + case ip.Is4(): + return ip.String() + default: + p, _ := ip.Prefix(64) + return p.String() + } +} diff --git a/internal/api/ratelimit_test.go b/internal/api/ratelimit_test.go new file mode 100644 index 0000000..82d4b67 --- /dev/null +++ b/internal/api/ratelimit_test.go @@ -0,0 +1,269 @@ +package api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" + "time" +) + +// These pin the two volumetric limits in front of the public sign-in doors: +// the per-client-address bucket (keyed on the edge's visitor header only when +// the install names it) and the install-wide mail budget, which must refuse +// every address alike so it never becomes an existence oracle. They also pin +// that neither the buckets nor the OTP cooldown map grow without bound. + +func TestBucketSetBurstRefillAndWait(t *testing.T) { + clock := time.Unix(1_700_000_000, 0) + s := newBucketSet(RateLimit{Burst: 3, PerMinute: 6}, func() time.Time { return clock }) + for i := 0; i < 3; i++ { + if ok, _ := s.take("a"); !ok { + t.Fatalf("take %d refused inside the burst", i+1) + } + } + ok, wait := s.take("a") + if ok || wait != 10*time.Second { + t.Fatalf("4th take = %v wait %v, want refused with 10s wait (6/min)", ok, wait) + } + if ok, _ := s.take("b"); !ok { + t.Fatal("another key shares a's bucket") + } + if ok, _ := s.peek("a"); ok { + t.Fatal("peek admitted an empty bucket") + } + clock = clock.Add(10 * time.Second) + if ok, _ := s.peek("a"); !ok { + t.Fatal("peek refused after one token refilled") + } + if ok, _ := s.take("a"); !ok { + t.Fatal("take refused after one token refilled (peek must not spend it)") + } + if ok, _ := s.take("a"); ok { + t.Fatal("a second token appeared from nowhere") + } +} + +func TestBucketSetDropsIdleKeysAndCapsTheMap(t *testing.T) { + clock := time.Unix(1_700_000_000, 0) + s := newBucketSet(RateLimit{Burst: 2, PerMinute: 60}, func() time.Time { return clock }) + s.maxKeys = 100 + for i := 0; i < 100; i++ { + s.take(fmt.Sprintf("10.0.0.%d", i)) + } + // At the cap with every bucket still draining: fresh keys share the + // overflow bucket instead of growing the map. + s.take("fresh-1") + s.take("fresh-2") + if ok, _ := s.take("fresh-3"); ok { + t.Fatal("overflow bucket admitted past its burst") + } + if n := len(s.buckets); n != 101 { + t.Fatalf("map holds %d buckets, want 100 + overflow", n) + } + // Once they refill, the sweep drops them all. + clock = clock.Add(2 * bucketSweepEvery) + s.take("later") + if n := len(s.buckets); n != 1 { + t.Fatalf("after refill the map holds %d buckets, want only the new one", n) + } +} + +func TestDisabledLimitAdmitsEverything(t *testing.T) { + var nilSet *bucketSet + if ok, _ := nilSet.take("x"); !ok { + t.Fatal("nil set refused") + } + s := newBucketSet(RateLimit{}, time.Now) + for i := 0; i < 1000; i++ { + if ok, _ := s.take("x"); !ok { + t.Fatal("zero limit refused") + } + } +} + +func TestClientIPTrustsOnlyTheNamedHeader(t *testing.T) { + req := func(remote string, h map[string]string) *http.Request { + r := httptest.NewRequest("POST", "/", nil) + r.RemoteAddr = remote + for k, v := range h { + r.Header.Set(k, v) + } + return r + } + for _, tc := range []struct { + name string + header string + r *http.Request + want string + }{ + {"no header configured ignores CF-Connecting-IP", "", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "203.0.113.9"}), "10.42.0.1"}, + {"cloudflare header", "CF-Connecting-IP", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "203.0.113.9"}), "203.0.113.9"}, + {"cloudflare header absent falls back to peer", "CF-Connecting-IP", req("10.42.0.1:5000", nil), "10.42.0.1"}, + {"garbage header falls back to peer", "CF-Connecting-IP", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "not-an-ip"}), "10.42.0.1"}, + {"X-Forwarded-For takes the proxy-appended rightmost hop", "X-Forwarded-For", req("10.42.0.1:5000", map[string]string{"X-Forwarded-For": "1.1.1.1, 198.51.100.7"}), "198.51.100.7"}, + {"v4-mapped v6 is unmapped", "CF-Connecting-IP", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "::ffff:203.0.113.9"}), "203.0.113.9"}, + } { + t.Run(tc.name, func(t *testing.T) { + a := &API{ClientIPHeader: tc.header} + if got := a.clientIP(tc.r).String(); got != tc.want { + t.Fatalf("clientIP = %s, want %s", got, tc.want) + } + }) + } +} + +func TestSourceKeyGroupsIPv6By64(t *testing.T) { + a := sourceKey(netip.MustParseAddr("2001:db8:1:2::1")) + b := sourceKey(netip.MustParseAddr("2001:db8:1:2:ffff::9")) + c := sourceKey(netip.MustParseAddr("2001:db8:1:3::1")) + if a != b || a == c { + t.Fatalf("keys %q %q %q: want one per /64", a, b, c) + } + if k := sourceKey(netip.MustParseAddr("203.0.113.9")); k != "203.0.113.9" { + t.Fatalf("v4 key = %q", k) + } +} + +func TestAuthDoorsThrottlePerClientAddress(t *testing.T) { + api, _, _ := seedLoginEmailAPI(t) + api.AuthDoorLimit = RateLimit{Burst: 3, PerMinute: 3} + api.ClientIPHeader = "CF-Connecting-IP" + eh := api.ExternalHandler() + from := func(ip string) map[string]string { + return map[string]string{"Content-Type": "application/json", "CF-Connecting-IP": ip} + } + for i := 0; i < 3; i++ { + if w := do(eh, "POST", "/api/v1/auth/options", `{"email":"x@example.net"}`, from("203.0.113.9")); w.Code != http.StatusOK { + t.Fatalf("call %d = %d (%s)", i+1, w.Code, w.Body.String()) + } + } + // The bucket spans every door: a different door from the same address is + // refused too. + w := do(eh, "POST", "/api/v1/auth/email/verify", `{"email":"x@example.net","code":"000000"}`, from("203.0.113.9")) + if c, _ := errEnvelope(t, w); w.Code != http.StatusTooManyRequests || c != "rate_limited" { + t.Fatalf("4th call = %d %s, want 429 rate_limited", w.Code, c) + } + if ra := w.Header().Get("Retry-After"); ra != "20" { + t.Fatalf("Retry-After = %q, want 20 (3/min)", ra) + } + if w := do(eh, "POST", "/api/v1/auth/options", `{"email":"x@example.net"}`, from("198.51.100.7")); w.Code != http.StatusOK { + t.Fatalf("another address was throttled: %d", w.Code) + } + // The op-login poll and logout are not doors: a waiting browser polls. + for i := 0; i < 5; i++ { + if w := do(eh, "GET", "/api/v1/auth/op-login/status/abc", "", from("203.0.113.9")); w.Code == http.StatusTooManyRequests { + t.Fatal("op-login status poll was throttled") + } + } +} + +func TestAuthDoorsIgnoreVisitorHeaderUnlessConfigured(t *testing.T) { + api, _, _ := seedLoginEmailAPI(t) + api.AuthDoorLimit = RateLimit{Burst: 2, PerMinute: 2} + eh := api.ExternalHandler() + // Without client_ip_header a forged CF-Connecting-IP must not mint fresh + // buckets: every call below comes from httptest's one peer address. + for i := 0; i < 2; i++ { + do(eh, "POST", "/api/v1/auth/options", `{"email":"x@example.net"}`, + map[string]string{"Content-Type": "application/json", "CF-Connecting-IP": fmt.Sprintf("203.0.113.%d", i)}) + } + w := do(eh, "POST", "/api/v1/auth/options", `{"email":"x@example.net"}`, + map[string]string{"Content-Type": "application/json", "CF-Connecting-IP": "203.0.113.200"}) + if w.Code != http.StatusTooManyRequests { + t.Fatalf("forged visitor header escaped the limit: %d", w.Code) + } +} + +func TestMailBudgetRefusesEveryAddressAlike(t *testing.T) { + api, repo, mailer := seedLoginEmailAPI(t) + repo.staff["second"] = &StaffUser{ID: "u2", Username: "second", Email: "second@example.net", Role: "user", EmailVerified: true} + api.MailLimit = RateLimit{Burst: 1, PerMinute: 1} + clock := time.Unix(1_700_000_000, 0) + api.Now = func() time.Time { return clock } + eh := api.ExternalHandler() + start := func(email string) *httptest.ResponseRecorder { + return do(eh, "POST", "/api/v1/auth/email/start", `{"email":"`+email+`"}`, jsonHeader) + } + + if w := start("player@example.net"); w.Code != http.StatusAccepted || mailer.calls != 1 { + t.Fatalf("first send = %d, %d mails", w.Code, mailer.calls) + } + // Budget spent: a real account and an unknown address get the same 429, + // and nothing reaches the relay. + for _, email := range []string{"second@example.net", "nobody@example.net"} { + w := start(email) + if c, _ := errEnvelope(t, w); w.Code != http.StatusTooManyRequests || c != "mail_rate_limited" || w.Header().Get("Retry-After") == "" { + t.Fatalf("%s while the budget is spent = %d %s (Retry-After %q)", email, w.Code, c, w.Header().Get("Retry-After")) + } + } + if mailer.calls != 1 { + t.Fatalf("a spent budget still mailed: %d sends", mailer.calls) + } + // The refused starts did not burn their recipients' cooldowns. + clock = clock.Add(time.Minute) + if w := start("second@example.net"); w.Code != http.StatusAccepted || mailer.calls != 2 { + t.Fatalf("after refill = %d, %d mails", w.Code, mailer.calls) + } +} + +func TestSignedInDoorMailBudget(t *testing.T) { + repo := newFakeRepo() + repo.staff["player"] = &StaffUser{ID: "u1", Username: "player"} + api := newTestAPI(repo, newFakeCluster()) + api.External = staticExternal{p: &Principal{UserID: "u1", Email: "u1@example.net", Role: "user"}} + mailer := &captureMailer{} + api.Mailer = mailer + // One mail per two minutes, so stepping past the per-principal resend + // cooldown leaves the mail budget still empty. + api.MailLimit = RateLimit{Burst: 1, PerMinute: 0.5} + clock := time.Unix(1_700_000_000, 0) + api.Now = func() time.Time { return clock } + eh := api.ExternalHandler() + if w := do(eh, "POST", "/api/v1/account/email/start", `{"email":"a@example.net"}`, nil); w.Code != http.StatusAccepted { + t.Fatalf("first = %d (%s)", w.Code, w.Body.String()) + } + clock = clock.Add(otpResendCooldown + time.Second) + w := do(eh, "POST", "/api/v1/account/email/start", `{"email":"b@example.net"}`, nil) + if c, _ := errEnvelope(t, w); w.Code != http.StatusTooManyRequests || c != "mail_rate_limited" { + t.Fatalf("second = %d %s, want 429 mail_rate_limited", w.Code, c) + } + if mailer.calls != 1 { + t.Fatalf("mails = %d", mailer.calls) + } +} + +func TestCooldownLimiterForgetsExpiredKeys(t *testing.T) { + clock := time.Unix(1_700_000_000, 0) + c := &cooldownLimiter{now: func() time.Time { return clock }, last: map[string]time.Time{}} + for i := 0; i < 500; i++ { + if _, ok := c.reserve(fmt.Sprintf("login:email:user%d@example.net", i), time.Minute); !ok { + t.Fatalf("reserve %d refused", i) + } + } + if _, ok := c.reserve("login:email:user7@example.net", time.Minute); ok { + t.Fatal("a live reservation was forgotten") + } + clock = clock.Add(time.Minute + bucketSweepEvery) + c.reserve("login:email:fresh@example.net", time.Minute) + if n := len(c.last); n != 1 { + t.Fatalf("after every window ended the map holds %d keys, want 1", n) + } +} + +func TestMailLimitCountsNotices(t *testing.T) { + // The lock notice spends the same budget as codes; with none left it is + // skipped rather than sent. + api, _, _ := seedLoginEmailAPI(t) + mailer := ¬iceMailer{} + api.Mailer = mailer + api.MailLimit = RateLimit{Burst: 1, PerMinute: 1} + api.mailGate().take(mailGateKey) + r := httptest.NewRequest("POST", "/", strings.NewReader("")) + api.noteOTPLock(r, &OTPAccountLockedError{Until: api.now().Add(time.Hour), JustLocked: true}, "u1", otpPurposeLogin) + if len(mailer.notices) != 0 { + t.Fatalf("notice sent past a spent budget: %q", mailer.notices) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index c6fae61..b0f86fd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -71,8 +71,17 @@ type SMTPConfig struct { // Username is the AUTH identity; empty means the relay needs no AUTH. Username string `toml:"username"` PasswordRef string `toml:"password_ref"` + // MaxPerHour caps the mail the API sends install-wide (codes and notices), + // so a flood cannot spend the relay's quota and get the account suspended. + // 0 means DefaultMailPerHour. Size it to the relay's own limit. + MaxPerHour int `toml:"max_per_hour"` } +// DefaultMailPerHour is the install-wide mail cap when smtp.max_per_hour is +// unset: far above a community's normal sign-in mail, far below the daily +// quota of common relays. +const DefaultMailPerHour = 120 + // ServerConfig is the [server] table. type ServerConfig struct { Listen string `toml:"listen"` @@ -106,6 +115,24 @@ type AuthConfig struct { AdminHostname string `toml:"admin_hostname"` PanelHostname string `toml:"panel_hostname"` AccessJWTAud string `toml:"access_jwt_aud"` + // ClientIPHeader names the header the edge writes the visitor's address + // into: CF-Connecting-IP behind the Cloudflare tunnel (the edge setup + // writes it), X-Forwarded-For behind an operator's reverse proxy. The API + // keys its per-client sign-in rate limit on it. Empty means the TCP peer, + // except that an install with an Access audience (set only by the + // Cloudflare edge setup) implies CF-Connecting-IP. + ClientIPHeader string `toml:"client_ip_header"` +} + +// EffectiveClientIPHeader resolves ClientIPHeader with its Cloudflare default. +func (a AuthConfig) EffectiveClientIPHeader() string { + if a.ClientIPHeader != "" { + return a.ClientIPHeader + } + if a.AccessJWTAud != "" { + return "CF-Connecting-IP" + } + return "" } // K8sConfig is the [k8s] table. @@ -368,6 +395,12 @@ func (c *Config) Validate() error { return fmt.Errorf("config: [smtp] port %d must be 1-65535 (587 STARTTLS, 465 implicit TLS)", c.SMTP.Port) } } + if c.SMTP.MaxPerHour < 0 { + return fmt.Errorf("config: [smtp] max_per_hour %d must be positive (0 means the default %d)", c.SMTP.MaxPerHour, DefaultMailPerHour) + } + if h := c.Auth.ClientIPHeader; strings.ContainsAny(h, " :\t\r\n") { + return fmt.Errorf("config: [auth] client_ip_header %q must be a bare header name such as CF-Connecting-IP or X-Forwarded-For", h) + } return c.validateAuthSources() } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d6d69a4..cf42df1 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -574,3 +574,38 @@ host = "smtp.example.net" t.Fatal("expected error when [smtp] host is set without a from address") } } + +// TestClientIPHeaderAndMailCap pins the two knobs behind the sign-in rate +// limits: the visitor-address header (explicit, or implied by an Access +// audience that only the Cloudflare edge setup writes) and the mail cap. +func TestClientIPHeaderAndMailCap(t *testing.T) { + base := ` +[server] +root_domain = "mc.example.net" +[database] +url = "postgres://felis@db/felis" +` + for _, tc := range []struct { + name, auth, want string + }{ + {"unset", "", ""}, + {"cloudflare audience implies CF-Connecting-IP", "access_jwt_aud = \"aud123\"\n", "CF-Connecting-IP"}, + {"explicit wins", "access_jwt_aud = \"aud123\"\nclient_ip_header = \"X-Forwarded-For\"\n", "X-Forwarded-For"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg, err := config.Load(writeTOML(t, base+"[auth]\n"+tc.auth)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if got := cfg.Auth.EffectiveClientIPHeader(); got != tc.want { + t.Fatalf("EffectiveClientIPHeader = %q, want %q", got, tc.want) + } + }) + } + if _, err := config.Load(writeTOML(t, base+"[auth]\nclient_ip_header = \"CF-Connecting-IP: 1.2.3.4\"\n")); err == nil { + t.Fatal("a header line with a value was accepted as a header name") + } + if _, err := config.Load(writeTOML(t, base+"[smtp]\nhost = \"smtp.example.net\"\nfrom = \"f@example.net\"\nmax_per_hour = -1\n")); err == nil { + t.Fatal("negative max_per_hour accepted") + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 0baf7fc..5a4d5b7 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -64,8 +64,44 @@ var ( Name: "auth_otp_lockouts_total", Help: "Email-code doors locked after too many wrong codes, by purpose.", }, []string{"purpose"}) + + // MailTotal counts mail the API tried to send, by kind (otp, notice) and + // result: sent, failed (the relay refused it) or throttled (the + // install-wide mail budget refused it before it reached the relay). + MailTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "mail_total", + Help: "Mail the API tried to send, by kind and result (sent, failed, throttled).", + }, []string{"kind", "result"}) + + // RateLimitedTotal counts requests refused by a volumetric limit, by scope + // (auth_door: one client address calling the public sign-in doors too fast). + RateLimitedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "rate_limited_total", + Help: "Requests refused by a volumetric rate limit, by scope.", + }, []string{"scope"}) ) +// OTPPurposes are the email-code doors OTPLockoutsTotal is labelled by. +var OTPPurposes = []string{"onboard_email", "login_email", "op_login", "migrate_confirm"} + +// The sign-in alerts watch these counters with increase(). A labelled child +// that does not exist yet has no sample before its first event, so increase() +// would miss exactly the first lockout or throttle; every child the alerts use +// is created at zero up front. +func init() { + for _, kind := range []string{"otp", "notice"} { + for _, result := range []string{"sent", "failed", "throttled"} { + MailTotal.WithLabelValues(kind, result) + } + } + RateLimitedTotal.WithLabelValues("auth_door") + for _, p := range OTPPurposes { + OTPLockoutsTotal.WithLabelValues(p) + } +} + // SyncServerGauge republishes felis_servers_total from a full snapshot of the // fleet's per-server states. states holds one entry per MinecraftServer the // operator knows about (its desiredState). @@ -97,6 +133,8 @@ func Collectors() []prometheus.Collector { ImageBuildFailuresTotal, ReaperWorldsDeletedTotal, OTPLockoutsTotal, + MailTotal, + RateLimitedTotal, } } diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 42bbade..9a96cbc 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -116,3 +116,26 @@ func TestCountersRecordExpectedValues(t *testing.T) { t.Errorf("start_duration_seconds collected %d metrics, want 1 histogram", n) } } + +// The sign-in alerts use increase(), which needs a zero sample before the first +// event; every child they watch must be exposed before anything is counted. +func TestSignInSeriesStartAtZero(t *testing.T) { + want := map[string]int{ + "felis_mail_total": 6, + "felis_rate_limited_total": 1, + "felis_auth_otp_lockouts_total": len(OTPPurposes), + } + for _, c := range []prometheus.Collector{MailTotal, RateLimitedTotal, OTPLockoutsTotal} { + reg := prometheus.NewRegistry() + reg.MustRegister(c) + mfs, err := reg.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + for _, mf := range mfs { + if n := len(mf.GetMetric()); n < want[mf.GetName()] { + t.Errorf("%s exposes %d children, want at least %d", mf.GetName(), n, want[mf.GetName()]) + } + } + } +} diff --git a/panel/src/i18n/resources/en-US/errors.json b/panel/src/i18n/resources/en-US/errors.json index 8656c93..c96b614 100644 --- a/panel/src/i18n/resources/en-US/errors.json +++ b/panel/src/i18n/resources/en-US/errors.json @@ -23,6 +23,8 @@ "otp_resend_cooldown": "Verification code requested too frequently, please try again later.", "otp_locked": "Too many incorrect attempts, please request a new verification code.", "otp_account_locked": "Too many wrong codes in 24 hours — email codes for this account are paused. Try again later or use a passkey.", + "rate_limited": "Too many sign-in attempts from your network — wait a minute and try again.", + "mail_rate_limited": "This server is sending too much mail right now, so codes are paused. Try again shortly or use a passkey.", "passkey_challenge_invalid": "Passkey challenge is invalid or expired, please try again.", "invalid_attestation": "Could not verify this Passkey, please try again.", "passkey_already_bound": "This Passkey is already bound to another account.", diff --git a/panel/src/i18n/resources/zh-CN/errors.json b/panel/src/i18n/resources/zh-CN/errors.json index c1c8d47..725534f 100644 --- a/panel/src/i18n/resources/zh-CN/errors.json +++ b/panel/src/i18n/resources/zh-CN/errors.json @@ -23,6 +23,8 @@ "otp_resend_cooldown": "验证码发送频繁,请稍后再试。", "otp_locked": "验证码错误次数过多,请重新获取验证码。", "otp_account_locked": "24 小时内验证码错误次数已达上限,此账号的邮箱验证码暂时停用,请稍后再试或改用 Passkey。", + "rate_limited": "登录请求过于频繁,请稍等一分钟再试。", + "mail_rate_limited": "服务器当前发信过多,验证码暂时发不出去,请稍后再试或改用 Passkey。", "passkey_challenge_invalid": "验证挑战无效或已过期,请重试。", "invalid_attestation": "无法验证此 Passkey,请重试。", "passkey_already_bound": "此 Passkey 已被其他账户绑定。", diff --git a/panel/src/lib/api.ts b/panel/src/lib/api.ts index b057924..b2fd60f 100644 --- a/panel/src/lib/api.ts +++ b/panel/src/lib/api.ts @@ -679,6 +679,12 @@ export function humanizeError(e: unknown): string { return t("otp_locked"); case "otp_account_locked": return t("otp_account_locked"); + // Volumetric limits on the sign-in doors (internal/api/ratelimit.go): one + // network calling too fast, or the install-wide mail budget spent. + case "rate_limited": + return t("rate_limited"); + case "mail_rate_limited": + return t("mail_rate_limited"); case "passkey_challenge_invalid": return t("passkey_challenge_invalid"); case "invalid_attestation":