fix(auth): 公开登录门按来源限速并设全站发信上限,冷却表定期清理

This commit is contained in:
Lemon-miaow committed 2026-09-24 15:51:42 +08:00
1 parent 15f729ffea
commit c4e4953f3d
26 files changed
+1138 -61

No files matched your search

+22
View File
@@ -300,8 +300,20 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// for legitimate multi-tab / multi-server watching, while capping how many // for legitimate multi-tab / multi-server watching, while capping how many
// upstream follow connections a single caller can tie up if their streams stall. // upstream follow connections a single caller can tie up if their streams stall.
MaxStreamsPerPrincipal: 16, MaxStreamsPerPrincipal: 16,
// Public sign-in doors, per client address: a person signing in makes a
// handful of calls, so 20 at once refilled at 20 a minute never bites a
// real user and still turns a spray into a trickle. The client address
// is the edge's header when the install names one (config.AuthConfig).
AuthDoorLimit: api.RateLimit{Burst: 20, PerMinute: 20},
ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(),
MailLimit: mailLimit(cfg.SMTP.MaxPerHour),
} }
fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)") fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)")
if a.ClientIPHeader != "" {
fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader)
} else {
fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)")
}
// Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned // Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned
// identity anchor (正版优先); config can only append namespace-rewritten third-party // identity anchor (正版优先); config can only append namespace-rewritten third-party
@@ -546,3 +558,13 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
} }
} }
} }
// mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket:
// the hourly cap as the refill rate, with a quarter of it (at least 5) allowed
// at once so a burst of real sign-ins is not queued behind the average.
func mailLimit(perHour int) api.RateLimit {
if perHour <= 0 {
perHour = config.DefaultMailPerHour
}
return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60}
}
+10 -5
View File
@@ -32,7 +32,9 @@ func applyCloudflareEdge(ctx context.Context, result *cfsetup.Result, panelHost,
if adminHost == "" { if adminHost == "" {
return fmt.Errorf("admin hostname is required") return fmt.Errorf("admin hostname is required")
} }
if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud); err != nil { // cloudflared is the only way in once the NodePort is fenced, so the
// visitor address it writes can key the sign-in rate limit.
if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud, "CF-Connecting-IP"); err != nil {
return err return err
} }
if err := applyFelisConfigSecret(ctx); err != nil { if err := applyFelisConfigSecret(ctx); err != nil {
@@ -78,7 +80,8 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
if adminHost == "" { if adminHost == "" {
return fmt.Errorf("admin hostname is required") return fmt.Errorf("admin hostname is required")
} }
if err := writeConnectionConfig(panelHost, adminHost, ""); err != nil { // Caddy, nginx and Traefik all append the peer they saw to X-Forwarded-For.
if err := writeConnectionConfig(panelHost, adminHost, "", "X-Forwarded-For"); err != nil {
return err return err
} }
if err := applyFelisConfigSecret(ctx); err != nil { if err := applyFelisConfigSecret(ctx); err != nil {
@@ -93,16 +96,17 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
// writeConnectionConfig stamps the chosen hostnames (and optional Access audience) // writeConnectionConfig stamps the chosen hostnames (and optional Access audience)
// into both the host and pod config files. An empty aud clears any prior // into both the host and pod config files. An empty aud clears any prior
// Cloudflare audience, which is correct when switching to a non-Access front. // Cloudflare audience, which is correct when switching to a non-Access front.
func writeConnectionConfig(panelHost, adminHost, aud string) error { // clientIPHeader is the header that front writes the visitor address into.
func writeConnectionConfig(panelHost, adminHost, aud, clientIPHeader string) error {
for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} { for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} {
if err := updateAuthConfig(path, panelHost, adminHost, aud); err != nil { if err := updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader); err != nil {
return err return err
} }
} }
return nil return nil
} }
func updateAuthConfig(path, panelHost, adminHost, aud string) error { func updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader string) error {
cfg, err := config.Load(path) cfg, err := config.Load(path)
if err != nil { if err != nil {
return err return err
@@ -112,6 +116,7 @@ func updateAuthConfig(path, panelHost, adminHost, aud string) error {
} }
cfg.Auth.AdminHostname = adminHost cfg.Auth.AdminHostname = adminHost
cfg.Auth.AccessJWTAud = aud cfg.Auth.AccessJWTAud = aud
cfg.Auth.ClientIPHeader = clientIPHeader
return writeConfig(path, cfg) return writeConfig(path, cfg)
} }
+49 -1
View File
@@ -4,7 +4,9 @@
# #
# felis_* series come from two processes: # felis_* series come from two processes:
# - felis-operator pod :8080/metrics → felis_servers_total, felis_start_duration_seconds # - felis-operator pod :8080/metrics → felis_servers_total, felis_start_duration_seconds
# - felis-api internal :8081/metrics → felis_image_build_failures_total # - felis-api internal :8081/metrics → felis_image_build_failures_total,
# felis_mail_total, felis_rate_limited_total,
# felis_auth_otp_lockouts_total
# - node-exporter textfile collector → felis_db_backup_* (felis-db-backup.timer) # - node-exporter textfile collector → felis_db_backup_* (felis-db-backup.timer)
# node_* / kube_* series come from node-exporter / kube-state-metrics. # node_* / kube_* series come from node-exporter / kube-state-metrics.
groups: groups:
@@ -94,3 +96,49 @@ groups:
--collector.textfile.directory at the directory of --collector.textfile.directory at the directory of
FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
(troubleshooting §16). (troubleshooting §16).
- name: felis.auth.rules
rules:
- alert: FelisMailBudgetExhausted
expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0
labels:
severity: warning
annotations:
summary: "the install-wide mail budget refused mail"
description: >-
felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
spent, and every sign-in code is refused with 429 mail_rate_limited until
it refills. Check felis_rate_limited_total for a flood before raising the
budget (troubleshooting §17).
- alert: FelisMailDeliveryFailing
expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0
labels:
severity: warning
annotations:
summary: "the SMTP relay refused mail in the last 15m"
description: >-
felis_mail_total{result="failed"} increased: sign-in codes are not being
delivered (502 mail_undeliverable). The relay's reason is in the
felis-api log (troubleshooting §17).
- alert: FelisSignInFlood
expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10
for: 10m
labels:
severity: warning
annotations:
summary: "sign-in doors refusing over 10 requests a minute"
description: >-
The per-address sign-in limit has been refusing callers for 10 minutes.
A script is hammering the auth doors; if real users report rate_limited
at once instead, [auth] client_ip_header is missing and everyone shares
the proxy's address (troubleshooting §17).
- alert: FelisOTPAccountLocked
expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0
labels:
severity: warning
annotations:
summary: "an account's email-code sign-in locked after 10 wrong codes"
description: >-
Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}).
The audit log names the account (action auth.otp.locked); the owner was
mailed. Unless they fumbled codes, someone is guessing at it
(troubleshooting §17).
+85
View File
@@ -151,3 +151,88 @@ tests:
--collector.textfile.directory at the directory of --collector.textfile.directory at the directory of
FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
(troubleshooting §16). (troubleshooting §16).
- name: sign-in mail budget and relay
interval: 1m
input_series:
# Created at zero on start; the budget refuses one mail at t=3m.
- series: 'felis_mail_total{kind="otp",result="throttled",job="felis-api"}'
values: '0 0 0 1x30'
- series: 'felis_mail_total{kind="otp",result="failed",job="felis-api"}'
values: '0x33'
alert_rule_test:
- eval_time: 2m
alertname: FelisMailBudgetExhausted
exp_alerts: []
- eval_time: 5m
alertname: FelisMailBudgetExhausted
exp_alerts:
- exp_labels:
severity: warning
exp_annotations:
summary: "the install-wide mail budget refused mail"
description: >-
felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
spent, and every sign-in code is refused with 429 mail_rate_limited until
it refills. Check felis_rate_limited_total for a flood before raising the
budget (troubleshooting §17).
- eval_time: 5m
alertname: FelisMailDeliveryFailing
exp_alerts: []
- name: sign-in flood
interval: 1m
input_series:
# 30 refusals a minute from t=0; a lone refused script at 2/min stays quiet.
- series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}'
values: '0+30x40'
alert_rule_test:
- eval_time: 10m
alertname: FelisSignInFlood
exp_alerts: []
- eval_time: 20m
alertname: FelisSignInFlood
exp_alerts:
- exp_labels:
severity: warning
exp_annotations:
summary: "sign-in doors refusing over 10 requests a minute"
description: >-
The per-address sign-in limit has been refusing callers for 10 minutes.
A script is hammering the auth doors; if real users report rate_limited
at once instead, [auth] client_ip_header is missing and everyone shares
the proxy's address (troubleshooting §17).
- name: sign-in trickle stays quiet
interval: 1m
input_series:
- series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}'
values: '0+2x40'
alert_rule_test:
- eval_time: 30m
alertname: FelisSignInFlood
exp_alerts: []
- name: account email-code lock
interval: 1m
input_series:
- series: 'felis_auth_otp_lockouts_total{purpose="login_email",job="felis-api"}'
values: '0 0 1x90'
- series: 'felis_auth_otp_lockouts_total{purpose="op_login",job="felis-api"}'
values: '0x92'
alert_rule_test:
- eval_time: 1m
alertname: FelisOTPAccountLocked
exp_alerts: []
- eval_time: 10m
alertname: FelisOTPAccountLocked
exp_alerts:
- exp_labels:
severity: warning
purpose: login_email
exp_annotations:
summary: "an account's email-code sign-in locked after 10 wrong codes"
description: >-
Someone entered 10 wrong codes for one account within 24h (login_email).
The audit log names the account (action auth.otp.locked); the owner was
mailed. Unless they fumbled codes, someone is guessing at it
(troubleshooting §17).
- eval_time: 90m
alertname: FelisOTPAccountLocked
exp_alerts: []
+46
View File
@@ -98,3 +98,49 @@ spec:
--collector.textfile.directory at the directory of --collector.textfile.directory at the directory of
FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector) FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
(troubleshooting §16). (troubleshooting §16).
- name: felis.auth.rules
rules:
- alert: FelisMailBudgetExhausted
expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0
labels:
severity: warning
annotations:
summary: "the install-wide mail budget refused mail"
description: >-
felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
spent, and every sign-in code is refused with 429 mail_rate_limited until
it refills. Check felis_rate_limited_total for a flood before raising the
budget (troubleshooting §17).
- alert: FelisMailDeliveryFailing
expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0
labels:
severity: warning
annotations:
summary: "the SMTP relay refused mail in the last 15m"
description: >-
felis_mail_total{result="failed"} increased: sign-in codes are not being
delivered (502 mail_undeliverable). The relay's reason is in the
felis-api log (troubleshooting §17).
- alert: FelisSignInFlood
expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10
for: 10m
labels:
severity: warning
annotations:
summary: "sign-in doors refusing over 10 requests a minute"
description: >-
The per-address sign-in limit has been refusing callers for 10 minutes.
A script is hammering the auth doors; if real users report rate_limited
at once instead, [auth] client_ip_header is missing and everyone shares
the proxy's address (troubleshooting §17).
- alert: FelisOTPAccountLocked
expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0
labels:
severity: warning
annotations:
summary: "an account's email-code sign-in locked after 10 wrong codes"
description: >-
Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}).
The audit log names the account (action auth.otp.locked); the owner was
mailed. Unless they fumbled codes, someone is guessing at it
(troubleshooting §17).
+50 -12
View File
@@ -147,6 +147,19 @@ components:
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
RateLimited:
description: >
This client address called the public sign-in doors faster than the per-address
limit allows (code rate_limited); Retry-After gives the seconds until the next
call is admitted. The address is the visitor header the install's edge writes
([auth] client_ip_header: CF-Connecting-IP behind the Cloudflare tunnel), else
the TCP peer; IPv6 clients share one limit per /64.
headers:
Retry-After:
schema: { type: integer }
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
AccessResult: AccessResult:
description: The structured access mutation succeeded; the raw RCON reply is in output. description: The structured access mutation succeeded; the raw RCON reply is in output.
content: content:
@@ -1875,8 +1888,8 @@ paths:
array. It never reveals staffness: methods are computed identically for every array. It never reveals staffness: methods are computed identically for every
resolved account (no role branch), so a staff and a player address in the same resolved account (no role branch), so a staff and a player address in the same
credential state return byte-identical bodies. passkey is offered only when a credential state return byte-identical bodies. passkey is offered only when a
verifier is wired. Sends no mail and mutates nothing; not rate-limited at the app verifier is wired. Sends no mail and mutates nothing; bounded by the per-address
layer (volumetric abuse is bounded at the edge). Gated on local_auth_enabled. sign-in rate limit (429 rate_limited). Gated on local_auth_enabled.
x-felis-face: [external] x-felis-face: [external]
x-felis-tier: public x-felis-tier: public
security: [] security: []
@@ -1918,6 +1931,8 @@ paths:
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429':
$ref: '#/components/responses/RateLimited'
/api/v1/auth/passkey/login/begin: /api/v1/auth/passkey/login/begin:
post: post:
@@ -1973,7 +1988,9 @@ paths:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429': '429':
description: A passkey login for this recipient was started too recently (otp_resend_cooldown). description: >-
A passkey login for this recipient was started too recently (otp_resend_cooldown);
or this client address called the sign-in doors too often (rate_limited, with Retry-After).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
@@ -2045,6 +2062,8 @@ paths:
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429':
$ref: '#/components/responses/RateLimited'
'503': '503':
description: No passkey verifier is wired on this deployment (passkey_unavailable). description: No passkey verifier is wired on this deployment (passkey_unavailable).
content: content:
@@ -2066,9 +2085,9 @@ paths:
userHandle inside the signed assertion at finish. The challenge cannot be userHandle inside the signed assertion at finish. The challenge cannot be
user-keyed, so it is stashed under login_id in a non-user-keyed store and echoed user-keyed, so it is stashed under login_id in a non-user-keyed store and echoed
back at finish. Mounted Public and gated on local_auth_enabled. There is no back at finish. Mounted Public and gated on local_auth_enabled. There is no
recipient or principal to key a per-caller cooldown on (that volumetric limiting recipient or principal to key a per-caller cooldown on, so one client is bounded
is delegated to the edge), so the server-side brake is a hard global cap on live by the per-address sign-in rate limit (429 rate_limited) and the table by a hard
challenges (429 too_many_challenges). Inert for a credential until its owner global cap on live challenges (429 too_many_challenges). Inert for a credential until its owner
enrolls a resident passkey; email-OTP and username-first passkey remain the enrolls a resident passkey; email-OTP and username-first passkey remain the
fallbacks, so no authenticator is ever locked out. fallbacks, so no authenticator is ever locked out.
x-felis-face: [external] x-felis-face: [external]
@@ -2114,8 +2133,9 @@ paths:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429': '429':
description: >- description: >-
Too many discoverable logins are in flight server-wide; the global cap is hit Too many discoverable logins are in flight server-wide (too_many_challenges;
(too_many_challenges). No per-recipient signal is leaked — the cap is global. the cap is global, so no per-recipient signal leaks); or this client address
called the sign-in doors too often (rate_limited, with Retry-After).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
@@ -2192,6 +2212,8 @@ paths:
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429':
$ref: '#/components/responses/RateLimited'
'503': '503':
description: No passkey verifier is wired on this deployment (passkey_unavailable). description: No passkey verifier is wired on this deployment (passkey_unavailable).
content: content:
@@ -2253,7 +2275,10 @@ paths:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429': '429':
description: A code for this recipient was requested too recently (otp_resend_cooldown). description: >-
A code for this recipient was requested too recently (otp_resend_cooldown);
or this client address called the sign-in doors too often (rate_limited, with Retry-After);
or the install-wide mail budget is spent (mail_rate_limited, with Retry-After).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
@@ -2318,6 +2343,8 @@ paths:
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429':
$ref: '#/components/responses/RateLimited'
/api/v1/auth/op-login/start: /api/v1/auth/op-login/start:
post: post:
@@ -2373,7 +2400,10 @@ paths:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429': '429':
description: A code for this recipient was requested too recently (otp_resend_cooldown). description: >-
A code for this recipient was requested too recently (otp_resend_cooldown);
or this client address called the sign-in doors too often (rate_limited, with Retry-After);
or the install-wide mail budget is spent (mail_rate_limited, with Retry-After).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
@@ -2467,6 +2497,8 @@ paths:
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429':
$ref: '#/components/responses/RateLimited'
/api/v1/auth/setup/redeem: /api/v1/auth/setup/redeem:
post: post:
@@ -2524,6 +2556,8 @@ paths:
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429':
$ref: '#/components/responses/RateLimited'
/api/v1/auth/setup/status: /api/v1/auth/setup/status:
get: get:
@@ -2640,6 +2674,8 @@ paths:
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'429':
$ref: '#/components/responses/RateLimited'
/api/v1/me: /api/v1/me:
get: get:
@@ -3778,7 +3814,8 @@ paths:
description: >- description: >-
Resend requested before the cooldown elapsed (otp_resend_cooldown); or the Resend requested before the cooldown elapsed (otp_resend_cooldown); or the
account spent its daily wrong-code budget (otp_account_locked, with account spent its daily wrong-code budget (otp_account_locked, with
Retry-After). Retry-After); or the install-wide mail budget is spent
(mail_rate_limited, with Retry-After).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
@@ -4091,7 +4128,8 @@ paths:
description: >- description: >-
Resend requested before the cooldown elapsed (otp_resend_cooldown), or the Resend requested before the cooldown elapsed (otp_resend_cooldown), or the
account's daily wrong-code budget is spent (otp_account_locked, with account's daily wrong-code budget is spent (otp_account_locked, with
Retry-After). Retry-After); or the install-wide mail budget is spent
(mail_rate_limited, with Retry-After).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
+79 -3
View File
@@ -931,7 +931,9 @@ The series come from two processes:
`felis_start_duration_seconds` (no Service; scrape pod-scoped, e.g. a `felis_start_duration_seconds` (no Service; scrape pod-scoped, e.g. a
PodMonitor targeting port `metrics`). PodMonitor targeting port `metrics`).
- `felis-api` internal face `:8081/metrics` (Service `felis-api-internal`) — - `felis-api` internal face `:8081/metrics` (Service `felis-api-internal`) —
`felis_image_build_failures_total`. Unauthenticated like the probes; `felis_image_build_failures_total`, and the sign-in series of §17
(`felis_mail_total`, `felis_rate_limited_total`,
`felis_auth_otp_lockouts_total`). Unauthenticated like the probes;
ClusterIP-only, and the external face never serves it. ClusterIP-only, and the external face never serves it.
- `felis_reaper_worlds_deleted_total` is produced inside the one-shot reaper - `felis_reaper_worlds_deleted_total` is produced inside the one-shot reaper
CronJob, which exits long before any scrape interval — without a pushgateway CronJob, which exits long before any scrape interval — without a pushgateway
@@ -941,8 +943,10 @@ The series come from two processes:
### Alert rules ### Alert rules
`deploy/alerts/` ships ready-made rules: build failures, slow starts, node `deploy/alerts/` ships ready-made rules: build failures, slow starts, node
disk/memory thresholds, the kubelet `DiskPressure` condition, and control-plane disk/memory thresholds, the kubelet `DiskPressure` condition, control-plane
database backup freshness (§16; needs node-exporter's textfile collector). database backup freshness (§16; needs node-exporter's textfile collector), and
sign-in abuse: the mail budget, relay failures, throttled floods and account
code locks (§17).
- Plain Prometheus: add `felis-alerts.yaml` to `rule_files`. Check and unit-test - Plain Prometheus: add `felis-alerts.yaml` to `rule_files`. Check and unit-test
it standalone with `promtool check rules felis-alerts.yaml` and it standalone with `promtool check rules felis-alerts.yaml` and
@@ -1167,6 +1171,75 @@ Skips the pre-migration snapshot (`migrate up -no-backup`). The installer warns
loudly when it is set. Use it only when the snapshot cannot work and you have loudly when it is set. Use it only when the snapshot cannot work and you have
another backup, e.g. an external database newer than the host's `pg_dump`. another backup, e.g. an external database newer than the host's `pg_dump`.
## 17. Sign-in refused with 429, mail budget, account code locks
The public sign-in doors (`/api/v1/auth/*` except logout and the op-login
status poll) have three limits of their own. Each answers 429 with a
`Retry-After` header and a distinct error code.
### `rate_limited`: one address called the doors too often
Each client address gets 20 calls at once, refilled at 20 a minute, shared
across every door. A person signing in makes three or four calls, so this only
bites scripts. IPv6 clients share one limit per /64. Refusals count in
`felis_rate_limited_total{scope="auth_door"}`; `FelisSignInFlood` fires when
more than 10 a minute are refused for 10 minutes.
The address comes from `[auth] client_ip_header`:
- Behind the Cloudflare tunnel it is `CF-Connecting-IP`. The edge setup writes
it, and an install with an `access_jwt_aud` implies it. The header is
trustworthy there because the same setup fences the panel NodePort to
loopback, so every request reaching the API came through cloudflared.
- Behind your own reverse proxy it is `X-Forwarded-For` (the rightmost entry,
the one your proxy appended). Firewall the NodePort so only the proxy reaches
it, or a direct caller can write any address it likes.
- Unset, the TCP peer is used. Behind any proxy every visitor then shares the
proxy's address and one limit, so **everyone gets `rate_limited` at once**.
The `felis api` log says at start which it keys on (`sign-in rate limit keys
on ...`). Set the header in `/etc/felis/felis.toml` and
`/etc/felis/felis.pod.toml`, then `felis converge`.
### `mail_rate_limited`: the install-wide mail budget is spent
Every code and notice the API mails spends one token of a single budget,
`[smtp] max_per_hour` (default 120; a quarter of it may go at once), so a flood
cannot burn the relay's quota and get the sending account suspended. While it
is spent, every address gets the same 429 and nothing reaches the relay.
`felis_mail_total{result="throttled"}` counts refusals and
`FelisMailBudgetExhausted` fires on the first one. Look at
`felis_rate_limited_total` first: a flood shows there. If sign-ins are real,
raise `max_per_hour` to what your relay allows.
`FelisMailDeliveryFailing` is the other half: the relay itself refused mail
(`felis_mail_total{result="failed"}`, 502 `mail_undeliverable` to the caller).
The relay's reason is in the `felis-api` log.
### `otp_account_locked`: ten wrong codes in 24 hours
Ten wrong email codes for one account within 24 hours, counted across every
code it was sent, lock that account's email-code sign-in until 24 hours after
the first miss. The public doors answer a locked account exactly like a wrong
code, and the owner gets one mail saying so. Signed-in doors (email
verification, migration step-up) answer 429 `otp_account_locked`. Passkey
sign-in keeps working. Each lock is audited as `auth.otp.locked` and counted in
`felis_auth_otp_lockouts_total{purpose}` (`FelisOTPAccountLocked`).
To lift a lock early once you have confirmed the owner locked themselves out:
```sh
sudo -u postgres psql felis -c \
"DELETE FROM otp_failure_windows WHERE user_id = (SELECT id FROM users WHERE username = '<name>');"
```
### Optional: a Cloudflare rate limiting rule in front
The limits above live in the API, so they hold on any edge. Behind Cloudflare
you can also stop floods before they reach the tunnel: Security → WAF → Rate
limiting rules, match URI Path starts with `/api/v1/auth/` on the console and
op.console hostnames, count by IP, 30 requests per 10 seconds, action Block
for 10 seconds (the Free plan's limits).
--- ---
## Quick reference: symptom → section ## Quick reference: symptom → section
@@ -1198,3 +1271,6 @@ another backup, e.g. an external database newer than the host's `pg_dump`.
| `pre-migration backup failed, nothing applied` during an upgrade | §16 | | `pre-migration backup failed, nothing applied` during an upgrade | §16 |
| Undo a mistaken change / restore the control-plane database | §16 | | Undo a mistaken change / restore the control-plane database | §16 |
| Host lost: rebuild from a database bundle | §16 | | Host lost: rebuild from a database bundle | §16 |
| Sign-in 429 `rate_limited` for everyone at once | §17 |
| 429 `mail_rate_limited` / `FelisMailBudgetExhausted` | §17 |
| Right code refused; `otp_account_locked` / `FelisOTPAccountLocked` | §17 |
+67 -15
View File
@@ -158,6 +158,16 @@ type API struct {
// Consumed by handleHasJoined (handlers_hasjoined.go). // Consumed by handleHasJoined (handlers_hasjoined.go).
AuthSources []AuthSource AuthSources []AuthSource
// AuthDoorLimit bounds how often one client address may call the public
// pre-session auth doors (ratelimit.go). MailLimit bounds all mail the API
// sends, install-wide. Zero values disable them; cmd/felis wires both.
AuthDoorLimit RateLimit
MailLimit RateLimit
// ClientIPHeader names the header the install's edge writes the client
// address into (CF-Connecting-IP behind the Cloudflare tunnel,
// X-Forwarded-For behind an operator proxy). Empty means the TCP peer.
ClientIPHeader string
// Now is the clock, injectable for tests. Defaults to time.Now. // Now is the clock, injectable for tests. Defaults to time.Now.
Now func() time.Time Now func() time.Time
@@ -172,6 +182,11 @@ type API struct {
streamCapOnce sync.Once streamCapOnce sync.Once
streamCap *streamLimiter streamCap *streamLimiter
authDoorOnce sync.Once
authDoorBuckets *bucketSet
mailOnce sync.Once
mailBuckets *bucketSet
} }
// panelURL returns the public player-console origin ("https://console.<root>"), // panelURL returns the public player-console origin ("https://console.<root>"),
@@ -286,6 +301,11 @@ type apiRoute struct {
// whose EmailVerified is false is restricted to these routes only. // whose EmailVerified is false is restricted to these routes only.
SetupAllowed bool SetupAllowed bool
// AuthDoor marks a public pre-session auth door: it is rate limited per
// client address (throttleAuthDoor). The op-login status poll is left off,
// since the browser calls it every few seconds while it waits.
AuthDoor bool
h http.HandlerFunc h http.HandlerFunc
} }
@@ -381,21 +401,21 @@ func (a *API) externalAPIRoutes() []apiRoute {
// counter-slice to the anti-enumeration doors — the ONE sanctioned place existence // counter-slice to the anti-enumeration doors — the ONE sanctioned place existence
// is disclosed — but it never reveals staffness (methods computed with no role // is disclosed — but it never reveals staffness (methods computed with no role
// branch, so a staff and a player address in the same state are indistinguishable). // branch, so a staff and a player address in the same state are indistinguishable).
{Method: "POST", Pattern: "/api/v1/auth/options", Public: true, h: a.handleAuthOptions}, {Method: "POST", Pattern: "/api/v1/auth/options", Public: true, AuthDoor: true, h: a.handleAuthOptions},
{Method: "POST", Pattern: "/api/v1/auth/setup/redeem", Public: true, h: a.handleSetupRedeem}, {Method: "POST", Pattern: "/api/v1/auth/setup/redeem", Public: true, AuthDoor: true, h: a.handleSetupRedeem},
{Method: "GET", Pattern: "/api/v1/auth/setup/status", SetupAllowed: true, h: a.handleSetupStatus}, {Method: "GET", Pattern: "/api/v1/auth/setup/status", SetupAllowed: true, h: a.handleSetupStatus},
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/begin", Public: true, h: a.handlePasskeyLoginBegin}, {Method: "POST", Pattern: "/api/v1/auth/passkey/login/begin", Public: true, AuthDoor: true, h: a.handlePasskeyLoginBegin},
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/finish", Public: true, h: a.handlePasskeyLoginFinish}, {Method: "POST", Pattern: "/api/v1/auth/passkey/login/finish", Public: true, AuthDoor: true, h: a.handlePasskeyLoginFinish},
// Discoverable ("usernameless") passkey login (task #40): the from-zero sibling of the // Discoverable ("usernameless") passkey login (task #40): the from-zero sibling of the
// email-first pair above — no identifier typed, the account is resolved from the // email-first pair above — no identifier typed, the account is resolved from the
// userHandle inside the signed assertion (handlers_passkey_discoverable.go). // userHandle inside the signed assertion (handlers_passkey_discoverable.go).
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/begin", Public: true, h: a.handlePasskeyLoginDiscoverableBegin}, {Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/begin", Public: true, AuthDoor: true, h: a.handlePasskeyLoginDiscoverableBegin},
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/finish", Public: true, h: a.handlePasskeyLoginDiscoverableFinish}, {Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/finish", Public: true, AuthDoor: true, h: a.handlePasskeyLoginDiscoverableFinish},
{Method: "POST", Pattern: "/api/v1/auth/email/start", Public: true, h: a.handleLoginEmailStart}, {Method: "POST", Pattern: "/api/v1/auth/email/start", Public: true, AuthDoor: true, h: a.handleLoginEmailStart},
{Method: "POST", Pattern: "/api/v1/auth/email/verify", Public: true, h: a.handleLoginEmailVerify}, {Method: "POST", Pattern: "/api/v1/auth/email/verify", Public: true, AuthDoor: true, h: a.handleLoginEmailVerify},
{Method: "POST", Pattern: "/api/v1/auth/op-login/start", Public: true, h: a.handleOpLoginStart}, {Method: "POST", Pattern: "/api/v1/auth/op-login/start", Public: true, AuthDoor: true, h: a.handleOpLoginStart},
{Method: "GET", Pattern: "/api/v1/auth/op-login/status/{id}", Public: true, h: a.handleOpLoginStatus}, {Method: "GET", Pattern: "/api/v1/auth/op-login/status/{id}", Public: true, h: a.handleOpLoginStatus},
{Method: "POST", Pattern: "/api/v1/auth/op-login/finish", Public: true, h: a.handleOpLoginFinish}, {Method: "POST", Pattern: "/api/v1/auth/op-login/finish", Public: true, AuthDoor: true, h: a.handleOpLoginFinish},
// Player-console bootstrap (console-tier access model): the account-less // Player-console bootstrap (console-tier access model): the account-less
// player's door into console.<root_domain>. Public — like login there is no prior // player's door into console.<root_domain>. Public — like login there is no prior
// principal — and session-minting, but the artifact it consumes is a one-time // principal — and session-minting, but the artifact it consumes is a one-time
@@ -403,7 +423,7 @@ func (a *API) externalAPIRoutes() []apiRoute {
// possession already proves a Minecraft identity. A code whose UUID belongs to // possession already proves a Minecraft identity. A code whose UUID belongs to
// staff is refused (403) so this never yields an admin session; op.console stays // staff is refused (403) so this never yields an admin session; op.console stays
// behind Zero Trust (handlers_onboard.go). // behind Zero Trust (handlers_onboard.go).
{Method: "POST", Pattern: "/api/v1/auth/bind", Public: true, h: a.handleBindRedeem}, {Method: "POST", Pattern: "/api/v1/auth/bind", Public: true, AuthDoor: true, h: a.handleBindRedeem},
// App-auth tier: operations on your own servers (spec §14). // App-auth tier: operations on your own servers (spec §14).
{Method: "POST", Pattern: "/api/v1/servers/{name}/wake", h: a.handleWake}, {Method: "POST", Pattern: "/api/v1/servers/{name}/wake", h: a.handleWake},
@@ -614,7 +634,11 @@ func (a *API) buildFace(routes []apiRoute, guard func(http.Handler) http.Handler
for _, rt := range routes { for _, rt := range routes {
pattern := rt.Method + " " + rt.Pattern pattern := rt.Method + " " + rt.Pattern
if rt.Public { if rt.Public {
mux.HandleFunc(pattern, rt.h) h := rt.h
if rt.AuthDoor {
h = a.throttleAuthDoor(h)
}
mux.HandleFunc(pattern, h)
continue continue
} }
h := rt.h h := rt.h
@@ -727,10 +751,35 @@ func principalFromContext(ctx context.Context) *Principal {
// reserve/release pair closes the intra-replica concurrent burst (the bug fixed in // reserve/release pair closes the intra-replica concurrent burst (the bug fixed in
// #35); cross-replica bounding would need a shared store (out of scope for the // #35); cross-replica bounding would need a shared store (out of scope for the
// single-replica demo). // single-replica demo).
//
// Entries older than the longest window the limiter has been asked about can
// no longer block anything, so checks sweep them out (at most once per
// bucketSweepEvery). Without that, every distinct address typed into a public
// door, whose neutral branch keeps its reservation, stayed in the map for the
// life of the process.
type cooldownLimiter struct { type cooldownLimiter struct {
mu sync.Mutex mu sync.Mutex
now func() time.Time now func() time.Time
last map[string]time.Time last map[string]time.Time
maxWindow time.Duration
swept time.Time
}
// noteWindow widens the retention to window and sweeps stale entries when due.
// The caller holds mu.
func (c *cooldownLimiter) noteWindow(window time.Duration, now time.Time) {
if window > c.maxWindow {
c.maxWindow = window
}
if c.maxWindow <= 0 || now.Sub(c.swept) < bucketSweepEvery {
return
}
c.swept = now
for k, t := range c.last {
if now.Sub(t) >= c.maxWindow {
delete(c.last, k)
}
}
} }
// allowed reports whether name may wake now WITHOUT recording the attempt. A // allowed reports whether name may wake now WITHOUT recording the attempt. A
@@ -745,7 +794,9 @@ func (c *cooldownLimiter) allowed(name string, window time.Duration) bool {
} }
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
if last, ok := c.last[name]; ok && c.now().Sub(last) < window { now := c.now()
c.noteWindow(window, now)
if last, ok := c.last[name]; ok && now.Sub(last) < window {
return false return false
} }
return true return true
@@ -778,10 +829,11 @@ func (c *cooldownLimiter) reserve(name string, window time.Duration) (time.Time,
} }
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
if last, ok := c.last[name]; ok && c.now().Sub(last) < window { t := c.now()
c.noteWindow(window, t)
if last, ok := c.last[name]; ok && t.Sub(last) < window {
return time.Time{}, false return time.Time{}, false
} }
t := c.now()
c.last[name] = t c.last[name] = t
return t, true return t, true
} }
+13
View File
@@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"log" "log"
"net/http" "net/http"
"strconv"
"time" "time"
) )
@@ -130,10 +131,19 @@ type apiError struct {
status int status int
code string code string
msg string msg string
// wait, when positive, is sent as Retry-After (whole seconds, rounded up).
wait time.Duration
} }
func (e *apiError) Error() string { return e.msg } func (e *apiError) Error() string { return e.msg }
// retryAfter returns a copy of e that tells the client when to retry.
func (e *apiError) retryAfter(d time.Duration) *apiError {
c := *e
c.wait = d
return &c
}
// newError builds an apiError with a formatted message. // newError builds an apiError with a formatted message.
func newError(status int, code, format string, a ...any) *apiError { func newError(status int, code, format string, a ...any) *apiError {
return &apiError{status: status, code: code, msg: fmt.Sprintf(format, a...)} return &apiError{status: status, code: code, msg: fmt.Sprintf(format, a...)}
@@ -176,6 +186,9 @@ func writeError(w http.ResponseWriter, r *http.Request, err error) {
r.Method, r.URL.Path, requestIDFromContext(r.Context()), err) r.Method, r.URL.Path, requestIDFromContext(r.Context()), err)
ae = newError(http.StatusInternalServerError, "internal", "internal error") ae = newError(http.StatusInternalServerError, "internal", "internal error")
} }
if ae.wait > 0 {
w.Header().Set("Retry-After", strconv.FormatInt(int64((ae.wait+time.Second-1)/time.Second), 10))
}
body := map[string]any{ body := map[string]any{
"error": map[string]string{ "error": map[string]string{
"code": ae.code, "code": ae.code,
+10 -6
View File
@@ -24,12 +24,9 @@ import (
// this separator). // this separator).
// - No principal. The throttle cannot key off a user id (there is none yet); it // - No principal. The throttle cannot key off a user id (there is none yet); it
// keys off the typed recipient address, the same anti-bomb dimension the onboard // keys off the typed recipient address, the same anti-bomb dimension the onboard
// start uses. Per-source (client-IP) aggregate limiting is deliberately NOT done // start uses. Volume from one client is bounded separately by the per-address
// here: cooldownLimiter is a one-per-window primitive, so keying it on client IP // token bucket every public auth door sits behind (throttleAuthDoor), and total
// would false-positive on shared egress (CGNAT / office NAT), and behind // mail by the install-wide mail budget (ratelimit.go).
// Cloudflare RemoteAddr is the proxy anyway. The only real harm — bombing one
// mailbox — is already bounded per recipient; volumetric per-source limiting
// belongs at the edge.
// - Refuse staff. Like handleBindRedeem this public door provably never mints a // - Refuse staff. Like handleBindRedeem this public door provably never mints a
// session for an admin identity: op.console stays behind Zero Trust (and its own // session for an admin identity: op.console stays behind Zero Trust (and its own
// in-game approval gate). The refusal happens only AFTER a valid code is // in-game approval gate). The refusal happens only AFTER a valid code is
@@ -84,6 +81,13 @@ func (a *API) handleLoginEmailStart(w http.ResponseWriter, r *http.Request) {
return return
} }
// The install-wide mail budget is checked before the address is resolved,
// so while it is spent every address gets the same 429.
if err := a.checkMailBudget(); err != nil {
writeError(w, r, err)
return
}
// Atomically reserve the per-recipient cooldown BEFORE any work, so a burst of // Atomically reserve the per-recipient cooldown BEFORE any work, so a burst of
// truly concurrent starts yields exactly one winner and each admitted send is one // truly concurrent starts yields exactly one winner and each admitted send is one
// real, non-idempotent email. The key is namespaced apart from the onboard door's // real, non-idempotent email. The key is namespaced apart from the onboard door's
+4 -5
View File
@@ -16,11 +16,10 @@ import (
// address has an account precisely because THIS endpoint is the one sanctioned place // address has an account precisely because THIS endpoint is the one sanctioned place
// existence is revealed. An empty methods array means "no (verified) account". That // existence is revealed. An empty methods array means "no (verified) account". That
// makes it a mass-enumeration surface by design — an accepted product decision, the // makes it a mass-enumeration surface by design — an accepted product decision, the
// same one the email door's header records. It is bounded only at the edge: the // same one the email door's header records. The handler sends no mail and mutates
// handler sends no mail and mutates nothing, so a per-recipient cooldown would merely // nothing, so a per-recipient cooldown would merely block a legitimate retry; what
// block a legitimate retry, and per-source (client-IP) limiting is the edge's job // bounds enumeration is the per-client-address token bucket shared by every public
// (behind Cloudflare RemoteAddr is the proxy, and CGNAT would false-positive) — see // auth door (throttleAuthDoor in ratelimit.go).
// the handlers_auth_email.go header for the same reasoning.
// //
// It never reveals STAFFNESS. Methods are computed by the SAME rule for every resolved // It never reveals STAFFNESS. Methods are computed by the SAME rule for every resolved
// account — no role branch, no operator hint — so a staff email and a player email in // account — no role branch, no operator hint — so a staff email and a player email in
+12
View File
@@ -11,6 +11,8 @@ import (
"net/http" "net/http"
"strings" "strings"
"time" "time"
"felis.lolicon.best/internal/metrics"
) )
// Player email verification (spec §B2 onboarding). Forced web onboarding proves a // Player email verification (spec §B2 onboarding). Forced web onboarding proves a
@@ -250,12 +252,21 @@ func (a *API) handleEmailOTPVerify(w http.ResponseWriter, r *http.Request) {
// deliverOTP hands the code to the configured Mailer, or — when none is wired (the // deliverOTP hands the code to the configured Mailer, or — when none is wired (the
// demo) — logs it server-side as a KNOWN-LIMITATION. The code is logged ONLY in the // demo) — logs it server-side as a KNOWN-LIMITATION. The code is logged ONLY in the
// no-mailer fallback and ONLY to the server log; it is never put in an HTTP response. // no-mailer fallback and ONLY to the server log; it is never put in an HTTP response.
//
// Every real send spends one token of the install-wide mail budget (mailGate);
// a spent budget is a 429 mail_rate_limited and nothing reaches the relay.
func (a *API) deliverOTP(ctx context.Context, email, code string) error { func (a *API) deliverOTP(ctx context.Context, email, code string) error {
if a.Mailer == nil { if a.Mailer == nil {
log.Printf("email-otp: no Mailer configured; code for %s is %s (KNOWN-LIMITATION: demo has no SMTP)", email, code) log.Printf("email-otp: no Mailer configured; code for %s is %s (KNOWN-LIMITATION: demo has no SMTP)", email, code)
return nil return nil
} }
if ok, wait := a.mailGate().take(mailGateKey); !ok {
metrics.MailTotal.WithLabelValues("otp", "throttled").Inc()
log.Printf("api: OTP mail refused by the install-wide mail budget (request_id=%s)", requestIDFromContext(ctx))
return errMailRateLimited(wait)
}
if err := a.Mailer.SendOTP(ctx, email, code); err != nil { if err := a.Mailer.SendOTP(ctx, email, code); err != nil {
metrics.MailTotal.WithLabelValues("otp", "failed").Inc()
// Mapped here rather than at each of the four call sites, so every door that // Mapped here rather than at each of the four call sites, so every door that
// mails a code answers the same way. A relay refusal is neither the caller's // mails a code answers the same way. A relay refusal is neither the caller's
// fault nor a bug in Felis, and a bare 500 says neither — it reads as "the // fault nor a bug in Felis, and a bare 500 says neither — it reads as "the
@@ -269,6 +280,7 @@ func (a *API) deliverOTP(ctx context.Context, email, code string) error {
return newError(http.StatusBadGateway, "mail_undeliverable", return newError(http.StatusBadGateway, "mail_undeliverable",
"the mail relay refused this message; ask the server operator to check the SMTP settings") "the mail relay refused this message; ask the server operator to check the SMTP settings")
} }
metrics.MailTotal.WithLabelValues("otp", "sent").Inc()
return nil return nil
} }
+7
View File
@@ -86,6 +86,13 @@ func (a *API) handleOpLoginStart(w http.ResponseWriter, r *http.Request) {
return return
} }
// The install-wide mail budget is checked before the address is resolved,
// so while it is spent every address gets the same 429.
if err := a.checkMailBudget(); err != nil {
writeError(w, r, err)
return
}
// Per-recipient cooldown reserved BEFORE any work, identical to the console email // Per-recipient cooldown reserved BEFORE any work, identical to the console email
// door: one winner per window, and the neutral (non-staff) branch keeps the // door: one winner per window, and the neutral (non-staff) branch keeps the
// reservation too so probing an address is throttled exactly like a real send. The // reservation too so probing an address is throttled exactly like a real send. The
@@ -21,10 +21,10 @@ import (
// //
// Anti-abuse divergence from the email-first door: that door reserves a per-recipient cooldown // Anti-abuse divergence from the email-first door: that door reserves a per-recipient cooldown
// (a.otpLimiter) keyed on the typed email. A usernameless begin has no recipient OR principal to // (a.otpLimiter) keyed on the typed email. A usernameless begin has no recipient OR principal to
// key a fair per-caller limit on, so — matching the stance in handlers_auth_email.go (behind // key a fair per-caller limit on, so one client is bounded by the per-address token bucket every
// Cloudflare RemoteAddr is the proxy; CGNAT false-positives) — volumetric per-source limiting is // public auth door sits behind (throttleAuthDoor), and the table by a hard global cap on live
// left to the edge, and the server-side bound is a hard global cap on live challenges enforced // challenges enforced atomically in CreateDiscoverableChallenge (ErrTooManyDiscoverableChallenges
// atomically in CreateDiscoverableChallenge (ErrTooManyDiscoverableChallenges → 429). // → 429).
// handlePasskeyLoginDiscoverableBegin starts a usernameless assertion ceremony (Public, // handlePasskeyLoginDiscoverableBegin starts a usernameless assertion ceremony (Public,
// pre-session). It has no request body — the whole point is that the caller supplies no // pre-session). It has no request body — the whole point is that the caller supplies no
+9 -7
View File
@@ -7,7 +7,6 @@ import (
"fmt" "fmt"
"log" "log"
"net/http" "net/http"
"strconv"
"time" "time"
"felis.lolicon.best/internal/metrics" "felis.lolicon.best/internal/metrics"
@@ -35,14 +34,9 @@ var otpDoorName = map[string][2]string{
// writeOTPAccountLocked answers a signed-in door whose budget is spent. // writeOTPAccountLocked answers a signed-in door whose budget is spent.
func writeOTPAccountLocked(w http.ResponseWriter, r *http.Request, until, now time.Time) { func writeOTPAccountLocked(w http.ResponseWriter, r *http.Request, until, now time.Time) {
secs := int64(until.Sub(now).Round(time.Second) / time.Second)
if secs < 1 {
secs = 1
}
w.Header().Set("Retry-After", strconv.FormatInt(secs, 10))
writeError(w, r, newError(http.StatusTooManyRequests, "otp_account_locked", writeError(w, r, newError(http.StatusTooManyRequests, "otp_account_locked",
"too many wrong codes on this account; email codes work again after %s", "too many wrong codes on this account; email codes work again after %s",
until.UTC().Format(time.RFC3339))) until.UTC().Format(time.RFC3339)).retryAfter(max(until.Sub(now), time.Second)))
} }
// noteOTPLock handles a redeem that met the account lock. Only the guess that // noteOTPLock handles a redeem that met the account lock. Only the guess that
@@ -83,10 +77,18 @@ func (a *API) noteOTPLock(r *http.Request, err error, userID, purpose string) {
log.Printf("auth: no notice mailer; user %s was not told their %s is locked", userID, purpose) log.Printf("auth: no notice mailer; user %s was not told their %s is locked", userID, purpose)
return return
} }
if ok, _ := a.mailGate().take(mailGateKey); !ok {
metrics.MailTotal.WithLabelValues("notice", "throttled").Inc()
log.Printf("auth: mail budget spent; user %s was not told their %s is locked", userID, purpose)
return
}
subject, body := otpLockNotice(door, lock.Until) subject, body := otpLockNotice(door, lock.Until)
if err := sender.SendNotice(ctx, u.Email, subject, body); err != nil { if err := sender.SendNotice(ctx, u.Email, subject, body); err != nil {
metrics.MailTotal.WithLabelValues("notice", "failed").Inc()
log.Printf("auth: otp lock notice to user %s failed: %v", userID, err) log.Printf("auth: otp lock notice to user %s failed: %v", userID, err)
return
} }
metrics.MailTotal.WithLabelValues("notice", "sent").Inc()
} }
// otpLockNotice renders the bilingual lock notice. // otpLockNotice renders the bilingual lock notice.
+14
View File
@@ -3,9 +3,12 @@ package api
import ( import (
"context" "context"
"net/http" "net/http"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
"felis.lolicon.best/internal/metrics"
) )
// The per-code attempt cap resets on every resend; these pin the account-level // The per-code attempt cap resets on every resend; these pin the account-level
@@ -181,3 +184,14 @@ func TestOTPLockNoticeNamesDoorAndTime(t *testing.T) {
} }
} }
} }
// The lockout alert sees a purpose only if metrics pre-creates its series.
func TestOTPPurposesMatchMetricLabels(t *testing.T) {
got := []string{otpPurposeOnboard, otpPurposeLogin, otpPurposeOpLogin, otpPurposeMigrate}
want := slices.Clone(metrics.OTPPurposes)
slices.Sort(got)
slices.Sort(want)
if !slices.Equal(got, want) {
t.Fatalf("otp purposes %v, metrics.OTPPurposes %v", got, want)
}
}
+2 -3
View File
@@ -1218,9 +1218,8 @@ func (p *PGRepo) ConsumePasskeyChallengeByUser(ctx context.Context, userID, purp
// exist, a new begin is refused (ErrTooManyDiscoverableChallenges → 429). The cap is generous — // exist, a new begin is refused (ErrTooManyDiscoverableChallenges → 429). The cap is generous —
// a login challenge lives only passkeyChallengeTTL (5 min) and each row is ~1 KB — so real // a login challenge lives only passkeyChallengeTTL (5 min) and each row is ~1 KB — so real
// concurrency never approaches it, while an abusive begin-flood is bounded to a few MB instead // concurrency never approaches it, while an abusive begin-flood is bounded to a few MB instead
// of growing without limit. Volumetric per-IP limiting is the edge's job (handlers_auth_email.go): // of growing without limit. One client's volume is bounded before it gets here, by the
// behind Cloudflare RemoteAddr is the proxy, and a usernameless door has no recipient to key a // per-address token bucket in front of every public auth door (ratelimit.go).
// fair per-caller limit on.
const maxLiveDiscoverableChallenges = 4096 const maxLiveDiscoverableChallenges = 4096
// CreateDiscoverableChallenge stashes a discoverable-login ceremony under an opaque handle, // CreateDiscoverableChallenge stashes a discoverable-login ceremony under an opaque handle,
+247
View File
@@ -0,0 +1,247 @@
package api
import (
"math"
"net"
"net/http"
"net/netip"
"strings"
"sync"
"time"
"felis.lolicon.best/internal/metrics"
)
// Volumetric limits for the public auth doors and for outbound mail.
//
// The per-recipient OTP cooldown (otpLimiter) stops one mailbox being bombed,
// and the per-account wrong-code budget stops one account being guessed. Neither
// bounds a caller who sprays many addresses or many accounts, so two more limits
// sit in front of them:
//
// - authDoorGate: a token bucket per client address over every pre-session
// auth door (options, login, op-login, bind, setup redeem). The client
// address comes from the edge's header only when the install says which
// header its edge writes (ClientIPHeader); with Cloudflare that header is
// CF-Connecting-IP, which is trustworthy because the edge setup fences the
// panel NodePort to loopback, so every request reaching the origin came
// through cloudflared. IPv6 clients are bucketed per /64, the unit one
// subscriber is handed.
// - mailGate: one install-wide bucket over every mail the API sends (codes
// and lock notices), so no flood can burn the SMTP relay's quota and get
// the sending account suspended. The public doors check it before they
// resolve the address, so a spent budget answers every address alike.
// RateLimit is a token bucket: Burst calls at once, refilled at PerMinute. The
// zero value disables the limit.
type RateLimit struct {
Burst int
PerMinute float64
}
func (l RateLimit) enabled() bool { return l.Burst > 0 && l.PerMinute > 0 }
// bucketSweepEvery is how often idle buckets are dropped; bucketMaxKeys bounds
// the map between sweeps. Past the bound, new keys share one overflow bucket,
// so a spray of fresh source addresses throttles itself instead of growing the
// map.
const (
bucketSweepEvery = time.Minute
bucketMaxKeys = 50_000
bucketOverflow = "\x00overflow"
)
type tokenBucket struct {
tokens float64
at time.Time
}
// bucketSet is a set of token buckets keyed by caller. A missing key is a full
// bucket, so a bucket that has refilled completely carries no information and
// is dropped by the sweep; memory is bounded by the keys active in the last
// refill period.
type bucketSet struct {
mu sync.Mutex
now func() time.Time
limit RateLimit
buckets map[string]*tokenBucket
swept time.Time
maxKeys int
}
func newBucketSet(limit RateLimit, now func() time.Time) *bucketSet {
return &bucketSet{now: now, limit: limit, buckets: map[string]*tokenBucket{}, maxKeys: bucketMaxKeys}
}
// refill brings b up to now. The caller holds mu.
func (s *bucketSet) refill(b *tokenBucket, now time.Time) {
if elapsed := now.Sub(b.at); elapsed > 0 {
b.tokens = math.Min(float64(s.limit.Burst), b.tokens+elapsed.Minutes()*s.limit.PerMinute)
}
b.at = now
}
// sweep drops full buckets at most once per bucketSweepEvery, or at once when
// force is set. The caller holds mu.
func (s *bucketSet) sweep(now time.Time, force bool) {
if !force && now.Sub(s.swept) < bucketSweepEvery {
return
}
s.swept = now
for k, b := range s.buckets {
s.refill(b, now)
if b.tokens >= float64(s.limit.Burst) {
delete(s.buckets, k)
}
}
}
// bucket returns key's bucket, creating a full one. The caller holds mu.
func (s *bucketSet) bucket(key string, now time.Time) *tokenBucket {
s.sweep(now, false)
if b, ok := s.buckets[key]; ok {
s.refill(b, now)
return b
}
if len(s.buckets) >= s.maxKeys {
s.sweep(now, true)
if len(s.buckets) >= s.maxKeys {
key = bucketOverflow
if b, ok := s.buckets[key]; ok {
s.refill(b, now)
return b
}
}
}
b := &tokenBucket{tokens: float64(s.limit.Burst), at: now}
s.buckets[key] = b
return b
}
// take spends one token from key's bucket. When none is left it reports how
// long until one is. A disabled limit always admits.
func (s *bucketSet) take(key string) (bool, time.Duration) {
if s == nil || !s.limit.enabled() {
return true, 0
}
s.mu.Lock()
defer s.mu.Unlock()
now := s.now()
b := s.bucket(key, now)
if b.tokens >= 1 {
b.tokens--
return true, 0
}
return false, s.wait(b)
}
// peek reports whether key's bucket holds a token, without spending it.
func (s *bucketSet) peek(key string) (bool, time.Duration) {
if s == nil || !s.limit.enabled() {
return true, 0
}
s.mu.Lock()
defer s.mu.Unlock()
now := s.now()
b, ok := s.buckets[key]
if !ok {
return true, 0
}
s.refill(b, now)
if b.tokens >= 1 {
return true, 0
}
return false, s.wait(b)
}
// wait is how long until b holds one token. The caller holds mu.
func (s *bucketSet) wait(b *tokenBucket) time.Duration {
missing := 1 - b.tokens
return time.Duration(math.Ceil(missing / s.limit.PerMinute * float64(time.Minute)))
}
func (a *API) authDoorGate() *bucketSet {
a.authDoorOnce.Do(func() { a.authDoorBuckets = newBucketSet(a.AuthDoorLimit, a.now) })
return a.authDoorBuckets
}
func (a *API) mailGate() *bucketSet {
a.mailOnce.Do(func() { a.mailBuckets = newBucketSet(a.MailLimit, a.now) })
return a.mailBuckets
}
// mailGateKey is the single install-wide mail bucket.
const mailGateKey = "mail"
// throttleAuthDoor applies the per-source bucket to one public auth door.
func (a *API) throttleAuthDoor(h http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ok, wait := a.authDoorGate().take(sourceKey(a.clientIP(r)))
if !ok {
metrics.RateLimitedTotal.WithLabelValues("auth_door").Inc()
writeError(w, r, newError(http.StatusTooManyRequests, "rate_limited",
"too many sign-in requests from this network; try again shortly").retryAfter(wait))
return
}
h(w, r)
}
}
// errMailRateLimited answers a door whose mail the install-wide budget refuses.
func errMailRateLimited(wait time.Duration) *apiError {
return newError(http.StatusTooManyRequests, "mail_rate_limited",
"this server is sending too much mail right now; try again shortly").retryAfter(wait)
}
// checkMailBudget is the public doors' pre-resolution check: it refuses every
// address alike while the budget is spent, so the refusal says nothing about
// whether the address has an account.
func (a *API) checkMailBudget() error {
if a.Mailer == nil {
return nil
}
if ok, wait := a.mailGate().peek(mailGateKey); !ok {
metrics.MailTotal.WithLabelValues("otp", "throttled").Inc()
return errMailRateLimited(wait)
}
return nil
}
// clientIP is the caller's address: the edge's header when the install names
// one and the request carries a parseable value, else the TCP peer. For
// X-Forwarded-For the rightmost entry is used, the one the trusted proxy
// appended itself.
func (a *API) clientIP(r *http.Request) netip.Addr {
if name := a.ClientIPHeader; name != "" {
if v := r.Header.Get(name); v != "" {
if strings.EqualFold(name, "X-Forwarded-For") {
if i := strings.LastIndexByte(v, ','); i >= 0 {
v = v[i+1:]
}
}
if ip, err := netip.ParseAddr(strings.TrimSpace(v)); err == nil {
return ip.Unmap()
}
}
}
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host = r.RemoteAddr
}
ip, _ := netip.ParseAddr(host)
return ip.Unmap()
}
// sourceKey buckets an address: IPv4 per host, IPv6 per /64. An unparseable
// address shares one key.
func sourceKey(ip netip.Addr) string {
switch {
case !ip.IsValid():
return "unknown"
case ip.Is4():
return ip.String()
default:
p, _ := ip.Prefix(64)
return p.String()
}
}
+269
View File
@@ -0,0 +1,269 @@
package api
import (
"fmt"
"net/http"
"net/http/httptest"
"net/netip"
"strings"
"testing"
"time"
)
// These pin the two volumetric limits in front of the public sign-in doors:
// the per-client-address bucket (keyed on the edge's visitor header only when
// the install names it) and the install-wide mail budget, which must refuse
// every address alike so it never becomes an existence oracle. They also pin
// that neither the buckets nor the OTP cooldown map grow without bound.
func TestBucketSetBurstRefillAndWait(t *testing.T) {
clock := time.Unix(1_700_000_000, 0)
s := newBucketSet(RateLimit{Burst: 3, PerMinute: 6}, func() time.Time { return clock })
for i := 0; i < 3; i++ {
if ok, _ := s.take("a"); !ok {
t.Fatalf("take %d refused inside the burst", i+1)
}
}
ok, wait := s.take("a")
if ok || wait != 10*time.Second {
t.Fatalf("4th take = %v wait %v, want refused with 10s wait (6/min)", ok, wait)
}
if ok, _ := s.take("b"); !ok {
t.Fatal("another key shares a's bucket")
}
if ok, _ := s.peek("a"); ok {
t.Fatal("peek admitted an empty bucket")
}
clock = clock.Add(10 * time.Second)
if ok, _ := s.peek("a"); !ok {
t.Fatal("peek refused after one token refilled")
}
if ok, _ := s.take("a"); !ok {
t.Fatal("take refused after one token refilled (peek must not spend it)")
}
if ok, _ := s.take("a"); ok {
t.Fatal("a second token appeared from nowhere")
}
}
func TestBucketSetDropsIdleKeysAndCapsTheMap(t *testing.T) {
clock := time.Unix(1_700_000_000, 0)
s := newBucketSet(RateLimit{Burst: 2, PerMinute: 60}, func() time.Time { return clock })
s.maxKeys = 100
for i := 0; i < 100; i++ {
s.take(fmt.Sprintf("10.0.0.%d", i))
}
// At the cap with every bucket still draining: fresh keys share the
// overflow bucket instead of growing the map.
s.take("fresh-1")
s.take("fresh-2")
if ok, _ := s.take("fresh-3"); ok {
t.Fatal("overflow bucket admitted past its burst")
}
if n := len(s.buckets); n != 101 {
t.Fatalf("map holds %d buckets, want 100 + overflow", n)
}
// Once they refill, the sweep drops them all.
clock = clock.Add(2 * bucketSweepEvery)
s.take("later")
if n := len(s.buckets); n != 1 {
t.Fatalf("after refill the map holds %d buckets, want only the new one", n)
}
}
func TestDisabledLimitAdmitsEverything(t *testing.T) {
var nilSet *bucketSet
if ok, _ := nilSet.take("x"); !ok {
t.Fatal("nil set refused")
}
s := newBucketSet(RateLimit{}, time.Now)
for i := 0; i < 1000; i++ {
if ok, _ := s.take("x"); !ok {
t.Fatal("zero limit refused")
}
}
}
func TestClientIPTrustsOnlyTheNamedHeader(t *testing.T) {
req := func(remote string, h map[string]string) *http.Request {
r := httptest.NewRequest("POST", "/", nil)
r.RemoteAddr = remote
for k, v := range h {
r.Header.Set(k, v)
}
return r
}
for _, tc := range []struct {
name string
header string
r *http.Request
want string
}{
{"no header configured ignores CF-Connecting-IP", "", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "203.0.113.9"}), "10.42.0.1"},
{"cloudflare header", "CF-Connecting-IP", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "203.0.113.9"}), "203.0.113.9"},
{"cloudflare header absent falls back to peer", "CF-Connecting-IP", req("10.42.0.1:5000", nil), "10.42.0.1"},
{"garbage header falls back to peer", "CF-Connecting-IP", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "not-an-ip"}), "10.42.0.1"},
{"X-Forwarded-For takes the proxy-appended rightmost hop", "X-Forwarded-For", req("10.42.0.1:5000", map[string]string{"X-Forwarded-For": "1.1.1.1, 198.51.100.7"}), "198.51.100.7"},
{"v4-mapped v6 is unmapped", "CF-Connecting-IP", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "::ffff:203.0.113.9"}), "203.0.113.9"},
} {
t.Run(tc.name, func(t *testing.T) {
a := &API{ClientIPHeader: tc.header}
if got := a.clientIP(tc.r).String(); got != tc.want {
t.Fatalf("clientIP = %s, want %s", got, tc.want)
}
})
}
}
func TestSourceKeyGroupsIPv6By64(t *testing.T) {
a := sourceKey(netip.MustParseAddr("2001:db8:1:2::1"))
b := sourceKey(netip.MustParseAddr("2001:db8:1:2:ffff::9"))
c := sourceKey(netip.MustParseAddr("2001:db8:1:3::1"))
if a != b || a == c {
t.Fatalf("keys %q %q %q: want one per /64", a, b, c)
}
if k := sourceKey(netip.MustParseAddr("203.0.113.9")); k != "203.0.113.9" {
t.Fatalf("v4 key = %q", k)
}
}
func TestAuthDoorsThrottlePerClientAddress(t *testing.T) {
api, _, _ := seedLoginEmailAPI(t)
api.AuthDoorLimit = RateLimit{Burst: 3, PerMinute: 3}
api.ClientIPHeader = "CF-Connecting-IP"
eh := api.ExternalHandler()
from := func(ip string) map[string]string {
return map[string]string{"Content-Type": "application/json", "CF-Connecting-IP": ip}
}
for i := 0; i < 3; i++ {
if w := do(eh, "POST", "/api/v1/auth/options", `{"email":"[email protected]"}`, from("203.0.113.9")); w.Code != http.StatusOK {
t.Fatalf("call %d = %d (%s)", i+1, w.Code, w.Body.String())
}
}
// The bucket spans every door: a different door from the same address is
// refused too.
w := do(eh, "POST", "/api/v1/auth/email/verify", `{"email":"[email protected]","code":"000000"}`, from("203.0.113.9"))
if c, _ := errEnvelope(t, w); w.Code != http.StatusTooManyRequests || c != "rate_limited" {
t.Fatalf("4th call = %d %s, want 429 rate_limited", w.Code, c)
}
if ra := w.Header().Get("Retry-After"); ra != "20" {
t.Fatalf("Retry-After = %q, want 20 (3/min)", ra)
}
if w := do(eh, "POST", "/api/v1/auth/options", `{"email":"[email protected]"}`, from("198.51.100.7")); w.Code != http.StatusOK {
t.Fatalf("another address was throttled: %d", w.Code)
}
// The op-login poll and logout are not doors: a waiting browser polls.
for i := 0; i < 5; i++ {
if w := do(eh, "GET", "/api/v1/auth/op-login/status/abc", "", from("203.0.113.9")); w.Code == http.StatusTooManyRequests {
t.Fatal("op-login status poll was throttled")
}
}
}
func TestAuthDoorsIgnoreVisitorHeaderUnlessConfigured(t *testing.T) {
api, _, _ := seedLoginEmailAPI(t)
api.AuthDoorLimit = RateLimit{Burst: 2, PerMinute: 2}
eh := api.ExternalHandler()
// Without client_ip_header a forged CF-Connecting-IP must not mint fresh
// buckets: every call below comes from httptest's one peer address.
for i := 0; i < 2; i++ {
do(eh, "POST", "/api/v1/auth/options", `{"email":"[email protected]"}`,
map[string]string{"Content-Type": "application/json", "CF-Connecting-IP": fmt.Sprintf("203.0.113.%d", i)})
}
w := do(eh, "POST", "/api/v1/auth/options", `{"email":"[email protected]"}`,
map[string]string{"Content-Type": "application/json", "CF-Connecting-IP": "203.0.113.200"})
if w.Code != http.StatusTooManyRequests {
t.Fatalf("forged visitor header escaped the limit: %d", w.Code)
}
}
func TestMailBudgetRefusesEveryAddressAlike(t *testing.T) {
api, repo, mailer := seedLoginEmailAPI(t)
repo.staff["second"] = &StaffUser{ID: "u2", Username: "second", Email: "[email protected]", Role: "user", EmailVerified: true}
api.MailLimit = RateLimit{Burst: 1, PerMinute: 1}
clock := time.Unix(1_700_000_000, 0)
api.Now = func() time.Time { return clock }
eh := api.ExternalHandler()
start := func(email string) *httptest.ResponseRecorder {
return do(eh, "POST", "/api/v1/auth/email/start", `{"email":"`+email+`"}`, jsonHeader)
}
if w := start("[email protected]"); w.Code != http.StatusAccepted || mailer.calls != 1 {
t.Fatalf("first send = %d, %d mails", w.Code, mailer.calls)
}
// Budget spent: a real account and an unknown address get the same 429,
// and nothing reaches the relay.
for _, email := range []string{"[email protected]", "[email protected]"} {
w := start(email)
if c, _ := errEnvelope(t, w); w.Code != http.StatusTooManyRequests || c != "mail_rate_limited" || w.Header().Get("Retry-After") == "" {
t.Fatalf("%s while the budget is spent = %d %s (Retry-After %q)", email, w.Code, c, w.Header().Get("Retry-After"))
}
}
if mailer.calls != 1 {
t.Fatalf("a spent budget still mailed: %d sends", mailer.calls)
}
// The refused starts did not burn their recipients' cooldowns.
clock = clock.Add(time.Minute)
if w := start("[email protected]"); w.Code != http.StatusAccepted || mailer.calls != 2 {
t.Fatalf("after refill = %d, %d mails", w.Code, mailer.calls)
}
}
func TestSignedInDoorMailBudget(t *testing.T) {
repo := newFakeRepo()
repo.staff["player"] = &StaffUser{ID: "u1", Username: "player"}
api := newTestAPI(repo, newFakeCluster())
api.External = staticExternal{p: &Principal{UserID: "u1", Email: "[email protected]", Role: "user"}}
mailer := &captureMailer{}
api.Mailer = mailer
// One mail per two minutes, so stepping past the per-principal resend
// cooldown leaves the mail budget still empty.
api.MailLimit = RateLimit{Burst: 1, PerMinute: 0.5}
clock := time.Unix(1_700_000_000, 0)
api.Now = func() time.Time { return clock }
eh := api.ExternalHandler()
if w := do(eh, "POST", "/api/v1/account/email/start", `{"email":"[email protected]"}`, nil); w.Code != http.StatusAccepted {
t.Fatalf("first = %d (%s)", w.Code, w.Body.String())
}
clock = clock.Add(otpResendCooldown + time.Second)
w := do(eh, "POST", "/api/v1/account/email/start", `{"email":"[email protected]"}`, nil)
if c, _ := errEnvelope(t, w); w.Code != http.StatusTooManyRequests || c != "mail_rate_limited" {
t.Fatalf("second = %d %s, want 429 mail_rate_limited", w.Code, c)
}
if mailer.calls != 1 {
t.Fatalf("mails = %d", mailer.calls)
}
}
func TestCooldownLimiterForgetsExpiredKeys(t *testing.T) {
clock := time.Unix(1_700_000_000, 0)
c := &cooldownLimiter{now: func() time.Time { return clock }, last: map[string]time.Time{}}
for i := 0; i < 500; i++ {
if _, ok := c.reserve(fmt.Sprintf("login:email:user%[email protected]", i), time.Minute); !ok {
t.Fatalf("reserve %d refused", i)
}
}
if _, ok := c.reserve("login:email:[email protected]", time.Minute); ok {
t.Fatal("a live reservation was forgotten")
}
clock = clock.Add(time.Minute + bucketSweepEvery)
c.reserve("login:email:[email protected]", time.Minute)
if n := len(c.last); n != 1 {
t.Fatalf("after every window ended the map holds %d keys, want 1", n)
}
}
func TestMailLimitCountsNotices(t *testing.T) {
// The lock notice spends the same budget as codes; with none left it is
// skipped rather than sent.
api, _, _ := seedLoginEmailAPI(t)
mailer := &noticeMailer{}
api.Mailer = mailer
api.MailLimit = RateLimit{Burst: 1, PerMinute: 1}
api.mailGate().take(mailGateKey)
r := httptest.NewRequest("POST", "/", strings.NewReader(""))
api.noteOTPLock(r, &OTPAccountLockedError{Until: api.now().Add(time.Hour), JustLocked: true}, "u1", otpPurposeLogin)
if len(mailer.notices) != 0 {
t.Fatalf("notice sent past a spent budget: %q", mailer.notices)
}
}
+33
View File
@@ -71,8 +71,17 @@ type SMTPConfig struct {
// Username is the AUTH identity; empty means the relay needs no AUTH. // Username is the AUTH identity; empty means the relay needs no AUTH.
Username string `toml:"username"` Username string `toml:"username"`
PasswordRef string `toml:"password_ref"` PasswordRef string `toml:"password_ref"`
// MaxPerHour caps the mail the API sends install-wide (codes and notices),
// so a flood cannot spend the relay's quota and get the account suspended.
// 0 means DefaultMailPerHour. Size it to the relay's own limit.
MaxPerHour int `toml:"max_per_hour"`
} }
// DefaultMailPerHour is the install-wide mail cap when smtp.max_per_hour is
// unset: far above a community's normal sign-in mail, far below the daily
// quota of common relays.
const DefaultMailPerHour = 120
// ServerConfig is the [server] table. // ServerConfig is the [server] table.
type ServerConfig struct { type ServerConfig struct {
Listen string `toml:"listen"` Listen string `toml:"listen"`
@@ -106,6 +115,24 @@ type AuthConfig struct {
AdminHostname string `toml:"admin_hostname"` AdminHostname string `toml:"admin_hostname"`
PanelHostname string `toml:"panel_hostname"` PanelHostname string `toml:"panel_hostname"`
AccessJWTAud string `toml:"access_jwt_aud"` AccessJWTAud string `toml:"access_jwt_aud"`
// ClientIPHeader names the header the edge writes the visitor's address
// into: CF-Connecting-IP behind the Cloudflare tunnel (the edge setup
// writes it), X-Forwarded-For behind an operator's reverse proxy. The API
// keys its per-client sign-in rate limit on it. Empty means the TCP peer,
// except that an install with an Access audience (set only by the
// Cloudflare edge setup) implies CF-Connecting-IP.
ClientIPHeader string `toml:"client_ip_header"`
}
// EffectiveClientIPHeader resolves ClientIPHeader with its Cloudflare default.
func (a AuthConfig) EffectiveClientIPHeader() string {
if a.ClientIPHeader != "" {
return a.ClientIPHeader
}
if a.AccessJWTAud != "" {
return "CF-Connecting-IP"
}
return ""
} }
// K8sConfig is the [k8s] table. // K8sConfig is the [k8s] table.
@@ -368,6 +395,12 @@ func (c *Config) Validate() error {
return fmt.Errorf("config: [smtp] port %d must be 1-65535 (587 STARTTLS, 465 implicit TLS)", c.SMTP.Port) return fmt.Errorf("config: [smtp] port %d must be 1-65535 (587 STARTTLS, 465 implicit TLS)", c.SMTP.Port)
} }
} }
if c.SMTP.MaxPerHour < 0 {
return fmt.Errorf("config: [smtp] max_per_hour %d must be positive (0 means the default %d)", c.SMTP.MaxPerHour, DefaultMailPerHour)
}
if h := c.Auth.ClientIPHeader; strings.ContainsAny(h, " :\t\r\n") {
return fmt.Errorf("config: [auth] client_ip_header %q must be a bare header name such as CF-Connecting-IP or X-Forwarded-For", h)
}
return c.validateAuthSources() return c.validateAuthSources()
} }
+35
View File
@@ -574,3 +574,38 @@ host = "smtp.example.net"
t.Fatal("expected error when [smtp] host is set without a from address") t.Fatal("expected error when [smtp] host is set without a from address")
} }
} }
// TestClientIPHeaderAndMailCap pins the two knobs behind the sign-in rate
// limits: the visitor-address header (explicit, or implied by an Access
// audience that only the Cloudflare edge setup writes) and the mail cap.
func TestClientIPHeaderAndMailCap(t *testing.T) {
base := `
[server]
root_domain = "mc.example.net"
[database]
url = "postgres://felis@db/felis"
`
for _, tc := range []struct {
name, auth, want string
}{
{"unset", "", ""},
{"cloudflare audience implies CF-Connecting-IP", "access_jwt_aud = \"aud123\"\n", "CF-Connecting-IP"},
{"explicit wins", "access_jwt_aud = \"aud123\"\nclient_ip_header = \"X-Forwarded-For\"\n", "X-Forwarded-For"},
} {
t.Run(tc.name, func(t *testing.T) {
cfg, err := config.Load(writeTOML(t, base+"[auth]\n"+tc.auth))
if err != nil {
t.Fatalf("Load: %v", err)
}
if got := cfg.Auth.EffectiveClientIPHeader(); got != tc.want {
t.Fatalf("EffectiveClientIPHeader = %q, want %q", got, tc.want)
}
})
}
if _, err := config.Load(writeTOML(t, base+"[auth]\nclient_ip_header = \"CF-Connecting-IP: 1.2.3.4\"\n")); err == nil {
t.Fatal("a header line with a value was accepted as a header name")
}
if _, err := config.Load(writeTOML(t, base+"[smtp]\nhost = \"smtp.example.net\"\nfrom = \"[email protected]\"\nmax_per_hour = -1\n")); err == nil {
t.Fatal("negative max_per_hour accepted")
}
}
+38
View File
@@ -64,8 +64,44 @@ var (
Name: "auth_otp_lockouts_total", Name: "auth_otp_lockouts_total",
Help: "Email-code doors locked after too many wrong codes, by purpose.", Help: "Email-code doors locked after too many wrong codes, by purpose.",
}, []string{"purpose"}) }, []string{"purpose"})
// MailTotal counts mail the API tried to send, by kind (otp, notice) and
// result: sent, failed (the relay refused it) or throttled (the
// install-wide mail budget refused it before it reached the relay).
MailTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Name: "mail_total",
Help: "Mail the API tried to send, by kind and result (sent, failed, throttled).",
}, []string{"kind", "result"})
// RateLimitedTotal counts requests refused by a volumetric limit, by scope
// (auth_door: one client address calling the public sign-in doors too fast).
RateLimitedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Name: "rate_limited_total",
Help: "Requests refused by a volumetric rate limit, by scope.",
}, []string{"scope"})
) )
// OTPPurposes are the email-code doors OTPLockoutsTotal is labelled by.
var OTPPurposes = []string{"onboard_email", "login_email", "op_login", "migrate_confirm"}
// The sign-in alerts watch these counters with increase(). A labelled child
// that does not exist yet has no sample before its first event, so increase()
// would miss exactly the first lockout or throttle; every child the alerts use
// is created at zero up front.
func init() {
for _, kind := range []string{"otp", "notice"} {
for _, result := range []string{"sent", "failed", "throttled"} {
MailTotal.WithLabelValues(kind, result)
}
}
RateLimitedTotal.WithLabelValues("auth_door")
for _, p := range OTPPurposes {
OTPLockoutsTotal.WithLabelValues(p)
}
}
// SyncServerGauge republishes felis_servers_total from a full snapshot of the // SyncServerGauge republishes felis_servers_total from a full snapshot of the
// fleet's per-server states. states holds one entry per MinecraftServer the // fleet's per-server states. states holds one entry per MinecraftServer the
// operator knows about (its desiredState). // operator knows about (its desiredState).
@@ -97,6 +133,8 @@ func Collectors() []prometheus.Collector {
ImageBuildFailuresTotal, ImageBuildFailuresTotal,
ReaperWorldsDeletedTotal, ReaperWorldsDeletedTotal,
OTPLockoutsTotal, OTPLockoutsTotal,
MailTotal,
RateLimitedTotal,
} }
} }
+23
View File
@@ -116,3 +116,26 @@ func TestCountersRecordExpectedValues(t *testing.T) {
t.Errorf("start_duration_seconds collected %d metrics, want 1 histogram", n) t.Errorf("start_duration_seconds collected %d metrics, want 1 histogram", n)
} }
} }
// The sign-in alerts use increase(), which needs a zero sample before the first
// event; every child they watch must be exposed before anything is counted.
func TestSignInSeriesStartAtZero(t *testing.T) {
want := map[string]int{
"felis_mail_total": 6,
"felis_rate_limited_total": 1,
"felis_auth_otp_lockouts_total": len(OTPPurposes),
}
for _, c := range []prometheus.Collector{MailTotal, RateLimitedTotal, OTPLockoutsTotal} {
reg := prometheus.NewRegistry()
reg.MustRegister(c)
mfs, err := reg.Gather()
if err != nil {
t.Fatalf("Gather: %v", err)
}
for _, mf := range mfs {
if n := len(mf.GetMetric()); n < want[mf.GetName()] {
t.Errorf("%s exposes %d children, want at least %d", mf.GetName(), n, want[mf.GetName()])
}
}
}
}
@@ -23,6 +23,8 @@
"otp_resend_cooldown": "Verification code requested too frequently, please try again later.", "otp_resend_cooldown": "Verification code requested too frequently, please try again later.",
"otp_locked": "Too many incorrect attempts, please request a new verification code.", "otp_locked": "Too many incorrect attempts, please request a new verification code.",
"otp_account_locked": "Too many wrong codes in 24 hours — email codes for this account are paused. Try again later or use a passkey.", "otp_account_locked": "Too many wrong codes in 24 hours — email codes for this account are paused. Try again later or use a passkey.",
"rate_limited": "Too many sign-in attempts from your network — wait a minute and try again.",
"mail_rate_limited": "This server is sending too much mail right now, so codes are paused. Try again shortly or use a passkey.",
"passkey_challenge_invalid": "Passkey challenge is invalid or expired, please try again.", "passkey_challenge_invalid": "Passkey challenge is invalid or expired, please try again.",
"invalid_attestation": "Could not verify this Passkey, please try again.", "invalid_attestation": "Could not verify this Passkey, please try again.",
"passkey_already_bound": "This Passkey is already bound to another account.", "passkey_already_bound": "This Passkey is already bound to another account.",
@@ -23,6 +23,8 @@
"otp_resend_cooldown": "验证码发送频繁,请稍后再试。", "otp_resend_cooldown": "验证码发送频繁,请稍后再试。",
"otp_locked": "验证码错误次数过多,请重新获取验证码。", "otp_locked": "验证码错误次数过多,请重新获取验证码。",
"otp_account_locked": "24 小时内验证码错误次数已达上限,此账号的邮箱验证码暂时停用,请稍后再试或改用 Passkey。", "otp_account_locked": "24 小时内验证码错误次数已达上限,此账号的邮箱验证码暂时停用,请稍后再试或改用 Passkey。",
"rate_limited": "登录请求过于频繁,请稍等一分钟再试。",
"mail_rate_limited": "服务器当前发信过多,验证码暂时发不出去,请稍后再试或改用 Passkey。",
"passkey_challenge_invalid": "验证挑战无效或已过期,请重试。", "passkey_challenge_invalid": "验证挑战无效或已过期,请重试。",
"invalid_attestation": "无法验证此 Passkey,请重试。", "invalid_attestation": "无法验证此 Passkey,请重试。",
"passkey_already_bound": "此 Passkey 已被其他账户绑定。", "passkey_already_bound": "此 Passkey 已被其他账户绑定。",
+6
View File
@@ -679,6 +679,12 @@ export function humanizeError(e: unknown): string {
return t("otp_locked"); return t("otp_locked");
case "otp_account_locked": case "otp_account_locked":
return t("otp_account_locked"); return t("otp_account_locked");
// Volumetric limits on the sign-in doors (internal/api/ratelimit.go): one
// network calling too fast, or the install-wide mail budget spent.
case "rate_limited":
return t("rate_limited");
case "mail_rate_limited":
return t("mail_rate_limited");
case "passkey_challenge_invalid": case "passkey_challenge_invalid":
return t("passkey_challenge_invalid"); return t("passkey_challenge_invalid");
case "invalid_attestation": case "invalid_attestation":