fix(auth): 公开登录门按来源限速并设全站发信上限,冷却表定期清理
This commit is contained in:
26 files changed
+1138
-61
No files matched your search
@@ -300,8 +300,20 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
|
||||
// for legitimate multi-tab / multi-server watching, while capping how many
|
||||
// upstream follow connections a single caller can tie up if their streams stall.
|
||||
MaxStreamsPerPrincipal: 16,
|
||||
// Public sign-in doors, per client address: a person signing in makes a
|
||||
// handful of calls, so 20 at once refilled at 20 a minute never bites a
|
||||
// real user and still turns a spray into a trickle. The client address
|
||||
// is the edge's header when the install names one (config.AuthConfig).
|
||||
AuthDoorLimit: api.RateLimit{Burst: 20, PerMinute: 20},
|
||||
ClientIPHeader: cfg.Auth.EffectiveClientIPHeader(),
|
||||
MailLimit: mailLimit(cfg.SMTP.MaxPerHour),
|
||||
}
|
||||
fmt.Fprintln(stderr, "felis api: external face fails closed (Access JWKS key function not configured)")
|
||||
if a.ClientIPHeader != "" {
|
||||
fmt.Fprintf(stderr, "felis api: sign-in rate limit keys on the %s header\n", a.ClientIPHeader)
|
||||
} else {
|
||||
fmt.Fprintln(stderr, "felis api: sign-in rate limit keys on the TCP peer ([auth] client_ip_header unset)")
|
||||
}
|
||||
|
||||
// Felis-nano: the multi-source hasJoined multiplexer. Mojang leads as the code-owned
|
||||
// identity anchor (正版优先); config can only append namespace-rewritten third-party
|
||||
@@ -546,3 +558,13 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket:
|
||||
// the hourly cap as the refill rate, with a quarter of it (at least 5) allowed
|
||||
// at once so a burst of real sign-ins is not queued behind the average.
|
||||
func mailLimit(perHour int) api.RateLimit {
|
||||
if perHour <= 0 {
|
||||
perHour = config.DefaultMailPerHour
|
||||
}
|
||||
return api.RateLimit{Burst: max(perHour/4, 5), PerMinute: float64(perHour) / 60}
|
||||
}
|
||||
@@ -32,7 +32,9 @@ func applyCloudflareEdge(ctx context.Context, result *cfsetup.Result, panelHost,
|
||||
if adminHost == "" {
|
||||
return fmt.Errorf("admin hostname is required")
|
||||
}
|
||||
if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud); err != nil {
|
||||
// cloudflared is the only way in once the NodePort is fenced, so the
|
||||
// visitor address it writes can key the sign-in rate limit.
|
||||
if err := writeConnectionConfig(panelHost, adminHost, result.AccessAud, "CF-Connecting-IP"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := applyFelisConfigSecret(ctx); err != nil {
|
||||
@@ -78,7 +80,8 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
|
||||
if adminHost == "" {
|
||||
return fmt.Errorf("admin hostname is required")
|
||||
}
|
||||
if err := writeConnectionConfig(panelHost, adminHost, ""); err != nil {
|
||||
// Caddy, nginx and Traefik all append the peer they saw to X-Forwarded-For.
|
||||
if err := writeConnectionConfig(panelHost, adminHost, "", "X-Forwarded-For"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := applyFelisConfigSecret(ctx); err != nil {
|
||||
@@ -93,16 +96,17 @@ func applyReverseProxy(ctx context.Context, panelHost, adminHost string) error {
|
||||
// writeConnectionConfig stamps the chosen hostnames (and optional Access audience)
|
||||
// into both the host and pod config files. An empty aud clears any prior
|
||||
// Cloudflare audience, which is correct when switching to a non-Access front.
|
||||
func writeConnectionConfig(panelHost, adminHost, aud string) error {
|
||||
// clientIPHeader is the header that front writes the visitor address into.
|
||||
func writeConnectionConfig(panelHost, adminHost, aud, clientIPHeader string) error {
|
||||
for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} {
|
||||
if err := updateAuthConfig(path, panelHost, adminHost, aud); err != nil {
|
||||
if err := updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateAuthConfig(path, panelHost, adminHost, aud string) error {
|
||||
func updateAuthConfig(path, panelHost, adminHost, aud, clientIPHeader string) error {
|
||||
cfg, err := config.Load(path)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -112,6 +116,7 @@ func updateAuthConfig(path, panelHost, adminHost, aud string) error {
|
||||
}
|
||||
cfg.Auth.AdminHostname = adminHost
|
||||
cfg.Auth.AccessJWTAud = aud
|
||||
cfg.Auth.ClientIPHeader = clientIPHeader
|
||||
return writeConfig(path, cfg)
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
#
|
||||
# felis_* series come from two processes:
|
||||
# - felis-operator pod :8080/metrics → felis_servers_total, felis_start_duration_seconds
|
||||
# - felis-api internal :8081/metrics → felis_image_build_failures_total
|
||||
# - felis-api internal :8081/metrics → felis_image_build_failures_total,
|
||||
# felis_mail_total, felis_rate_limited_total,
|
||||
# felis_auth_otp_lockouts_total
|
||||
# - node-exporter textfile collector → felis_db_backup_* (felis-db-backup.timer)
|
||||
# node_* / kube_* series come from node-exporter / kube-state-metrics.
|
||||
groups:
|
||||
@@ -94,3 +96,49 @@ groups:
|
||||
--collector.textfile.directory at the directory of
|
||||
FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
|
||||
(troubleshooting §16).
|
||||
- name: felis.auth.rules
|
||||
rules:
|
||||
- alert: FelisMailBudgetExhausted
|
||||
expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "the install-wide mail budget refused mail"
|
||||
description: >-
|
||||
felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
|
||||
spent, and every sign-in code is refused with 429 mail_rate_limited until
|
||||
it refills. Check felis_rate_limited_total for a flood before raising the
|
||||
budget (troubleshooting §17).
|
||||
- alert: FelisMailDeliveryFailing
|
||||
expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "the SMTP relay refused mail in the last 15m"
|
||||
description: >-
|
||||
felis_mail_total{result="failed"} increased: sign-in codes are not being
|
||||
delivered (502 mail_undeliverable). The relay's reason is in the
|
||||
felis-api log (troubleshooting §17).
|
||||
- alert: FelisSignInFlood
|
||||
expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "sign-in doors refusing over 10 requests a minute"
|
||||
description: >-
|
||||
The per-address sign-in limit has been refusing callers for 10 minutes.
|
||||
A script is hammering the auth doors; if real users report rate_limited
|
||||
at once instead, [auth] client_ip_header is missing and everyone shares
|
||||
the proxy's address (troubleshooting §17).
|
||||
- alert: FelisOTPAccountLocked
|
||||
expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "an account's email-code sign-in locked after 10 wrong codes"
|
||||
description: >-
|
||||
Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}).
|
||||
The audit log names the account (action auth.otp.locked); the owner was
|
||||
mailed. Unless they fumbled codes, someone is guessing at it
|
||||
(troubleshooting §17).
|
||||
@@ -151,3 +151,88 @@ tests:
|
||||
--collector.textfile.directory at the directory of
|
||||
FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
|
||||
(troubleshooting §16).
|
||||
- name: sign-in mail budget and relay
|
||||
interval: 1m
|
||||
input_series:
|
||||
# Created at zero on start; the budget refuses one mail at t=3m.
|
||||
- series: 'felis_mail_total{kind="otp",result="throttled",job="felis-api"}'
|
||||
values: '0 0 0 1x30'
|
||||
- series: 'felis_mail_total{kind="otp",result="failed",job="felis-api"}'
|
||||
values: '0x33'
|
||||
alert_rule_test:
|
||||
- eval_time: 2m
|
||||
alertname: FelisMailBudgetExhausted
|
||||
exp_alerts: []
|
||||
- eval_time: 5m
|
||||
alertname: FelisMailBudgetExhausted
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
summary: "the install-wide mail budget refused mail"
|
||||
description: >-
|
||||
felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
|
||||
spent, and every sign-in code is refused with 429 mail_rate_limited until
|
||||
it refills. Check felis_rate_limited_total for a flood before raising the
|
||||
budget (troubleshooting §17).
|
||||
- eval_time: 5m
|
||||
alertname: FelisMailDeliveryFailing
|
||||
exp_alerts: []
|
||||
- name: sign-in flood
|
||||
interval: 1m
|
||||
input_series:
|
||||
# 30 refusals a minute from t=0; a lone refused script at 2/min stays quiet.
|
||||
- series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}'
|
||||
values: '0+30x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 10m
|
||||
alertname: FelisSignInFlood
|
||||
exp_alerts: []
|
||||
- eval_time: 20m
|
||||
alertname: FelisSignInFlood
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
summary: "sign-in doors refusing over 10 requests a minute"
|
||||
description: >-
|
||||
The per-address sign-in limit has been refusing callers for 10 minutes.
|
||||
A script is hammering the auth doors; if real users report rate_limited
|
||||
at once instead, [auth] client_ip_header is missing and everyone shares
|
||||
the proxy's address (troubleshooting §17).
|
||||
- name: sign-in trickle stays quiet
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'felis_rate_limited_total{scope="auth_door",job="felis-api"}'
|
||||
values: '0+2x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 30m
|
||||
alertname: FelisSignInFlood
|
||||
exp_alerts: []
|
||||
- name: account email-code lock
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'felis_auth_otp_lockouts_total{purpose="login_email",job="felis-api"}'
|
||||
values: '0 0 1x90'
|
||||
- series: 'felis_auth_otp_lockouts_total{purpose="op_login",job="felis-api"}'
|
||||
values: '0x92'
|
||||
alert_rule_test:
|
||||
- eval_time: 1m
|
||||
alertname: FelisOTPAccountLocked
|
||||
exp_alerts: []
|
||||
- eval_time: 10m
|
||||
alertname: FelisOTPAccountLocked
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
purpose: login_email
|
||||
exp_annotations:
|
||||
summary: "an account's email-code sign-in locked after 10 wrong codes"
|
||||
description: >-
|
||||
Someone entered 10 wrong codes for one account within 24h (login_email).
|
||||
The audit log names the account (action auth.otp.locked); the owner was
|
||||
mailed. Unless they fumbled codes, someone is guessing at it
|
||||
(troubleshooting §17).
|
||||
- eval_time: 90m
|
||||
alertname: FelisOTPAccountLocked
|
||||
exp_alerts: []
|
||||
@@ -98,3 +98,49 @@ spec:
|
||||
--collector.textfile.directory at the directory of
|
||||
FELIS_DB_BACKUP_METRICS (default /var/lib/node_exporter/textfile_collector)
|
||||
(troubleshooting §16).
|
||||
- name: felis.auth.rules
|
||||
rules:
|
||||
- alert: FelisMailBudgetExhausted
|
||||
expr: sum(increase(felis_mail_total{result="throttled"}[15m])) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "the install-wide mail budget refused mail"
|
||||
description: >-
|
||||
felis_mail_total{result="throttled"} increased: [smtp] max_per_hour is
|
||||
spent, and every sign-in code is refused with 429 mail_rate_limited until
|
||||
it refills. Check felis_rate_limited_total for a flood before raising the
|
||||
budget (troubleshooting §17).
|
||||
- alert: FelisMailDeliveryFailing
|
||||
expr: sum(increase(felis_mail_total{result="failed"}[15m])) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "the SMTP relay refused mail in the last 15m"
|
||||
description: >-
|
||||
felis_mail_total{result="failed"} increased: sign-in codes are not being
|
||||
delivered (502 mail_undeliverable). The relay's reason is in the
|
||||
felis-api log (troubleshooting §17).
|
||||
- alert: FelisSignInFlood
|
||||
expr: sum(rate(felis_rate_limited_total{scope="auth_door"}[5m])) * 60 > 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "sign-in doors refusing over 10 requests a minute"
|
||||
description: >-
|
||||
The per-address sign-in limit has been refusing callers for 10 minutes.
|
||||
A script is hammering the auth doors; if real users report rate_limited
|
||||
at once instead, [auth] client_ip_header is missing and everyone shares
|
||||
the proxy's address (troubleshooting §17).
|
||||
- alert: FelisOTPAccountLocked
|
||||
expr: sum by (purpose) (increase(felis_auth_otp_lockouts_total[1h])) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "an account's email-code sign-in locked after 10 wrong codes"
|
||||
description: >-
|
||||
Someone entered 10 wrong codes for one account within 24h ({{ $labels.purpose }}).
|
||||
The audit log names the account (action auth.otp.locked); the owner was
|
||||
mailed. Unless they fumbled codes, someone is guessing at it
|
||||
(troubleshooting §17).
|
||||
+50
-12
@@ -147,6 +147,19 @@ components:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
RateLimited:
|
||||
description: >
|
||||
This client address called the public sign-in doors faster than the per-address
|
||||
limit allows (code rate_limited); Retry-After gives the seconds until the next
|
||||
call is admitted. The address is the visitor header the install's edge writes
|
||||
([auth] client_ip_header: CF-Connecting-IP behind the Cloudflare tunnel), else
|
||||
the TCP peer; IPv6 clients share one limit per /64.
|
||||
headers:
|
||||
Retry-After:
|
||||
schema: { type: integer }
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
AccessResult:
|
||||
description: The structured access mutation succeeded; the raw RCON reply is in output.
|
||||
content:
|
||||
@@ -1875,8 +1888,8 @@ paths:
|
||||
array. It never reveals staffness: methods are computed identically for every
|
||||
resolved account (no role branch), so a staff and a player address in the same
|
||||
credential state return byte-identical bodies. passkey is offered only when a
|
||||
verifier is wired. Sends no mail and mutates nothing; not rate-limited at the app
|
||||
layer (volumetric abuse is bounded at the edge). Gated on local_auth_enabled.
|
||||
verifier is wired. Sends no mail and mutates nothing; bounded by the per-address
|
||||
sign-in rate limit (429 rate_limited). Gated on local_auth_enabled.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: public
|
||||
security: []
|
||||
@@ -1918,6 +1931,8 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
$ref: '#/components/responses/RateLimited'
|
||||
|
||||
/api/v1/auth/passkey/login/begin:
|
||||
post:
|
||||
@@ -1973,7 +1988,9 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
description: A passkey login for this recipient was started too recently (otp_resend_cooldown).
|
||||
description: >-
|
||||
A passkey login for this recipient was started too recently (otp_resend_cooldown);
|
||||
or this client address called the sign-in doors too often (rate_limited, with Retry-After).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -2045,6 +2062,8 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
$ref: '#/components/responses/RateLimited'
|
||||
'503':
|
||||
description: No passkey verifier is wired on this deployment (passkey_unavailable).
|
||||
content:
|
||||
@@ -2066,9 +2085,9 @@ paths:
|
||||
userHandle inside the signed assertion at finish. The challenge cannot be
|
||||
user-keyed, so it is stashed under login_id in a non-user-keyed store and echoed
|
||||
back at finish. Mounted Public and gated on local_auth_enabled. There is no
|
||||
recipient or principal to key a per-caller cooldown on (that volumetric limiting
|
||||
is delegated to the edge), so the server-side brake is a hard global cap on live
|
||||
challenges (429 too_many_challenges). Inert for a credential until its owner
|
||||
recipient or principal to key a per-caller cooldown on, so one client is bounded
|
||||
by the per-address sign-in rate limit (429 rate_limited) and the table by a hard
|
||||
global cap on live challenges (429 too_many_challenges). Inert for a credential until its owner
|
||||
enrolls a resident passkey; email-OTP and username-first passkey remain the
|
||||
fallbacks, so no authenticator is ever locked out.
|
||||
x-felis-face: [external]
|
||||
@@ -2114,8 +2133,9 @@ paths:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
description: >-
|
||||
Too many discoverable logins are in flight server-wide; the global cap is hit
|
||||
(too_many_challenges). No per-recipient signal is leaked — the cap is global.
|
||||
Too many discoverable logins are in flight server-wide (too_many_challenges;
|
||||
the cap is global, so no per-recipient signal leaks); or this client address
|
||||
called the sign-in doors too often (rate_limited, with Retry-After).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -2192,6 +2212,8 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
$ref: '#/components/responses/RateLimited'
|
||||
'503':
|
||||
description: No passkey verifier is wired on this deployment (passkey_unavailable).
|
||||
content:
|
||||
@@ -2253,7 +2275,10 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
description: A code for this recipient was requested too recently (otp_resend_cooldown).
|
||||
description: >-
|
||||
A code for this recipient was requested too recently (otp_resend_cooldown);
|
||||
or this client address called the sign-in doors too often (rate_limited, with Retry-After);
|
||||
or the install-wide mail budget is spent (mail_rate_limited, with Retry-After).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -2318,6 +2343,8 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
$ref: '#/components/responses/RateLimited'
|
||||
|
||||
/api/v1/auth/op-login/start:
|
||||
post:
|
||||
@@ -2373,7 +2400,10 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
description: A code for this recipient was requested too recently (otp_resend_cooldown).
|
||||
description: >-
|
||||
A code for this recipient was requested too recently (otp_resend_cooldown);
|
||||
or this client address called the sign-in doors too often (rate_limited, with Retry-After);
|
||||
or the install-wide mail budget is spent (mail_rate_limited, with Retry-After).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -2467,6 +2497,8 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
$ref: '#/components/responses/RateLimited'
|
||||
|
||||
/api/v1/auth/setup/redeem:
|
||||
post:
|
||||
@@ -2524,6 +2556,8 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
$ref: '#/components/responses/RateLimited'
|
||||
|
||||
/api/v1/auth/setup/status:
|
||||
get:
|
||||
@@ -2640,6 +2674,8 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
$ref: '#/components/responses/RateLimited'
|
||||
|
||||
/api/v1/me:
|
||||
get:
|
||||
@@ -3778,7 +3814,8 @@ paths:
|
||||
description: >-
|
||||
Resend requested before the cooldown elapsed (otp_resend_cooldown); or the
|
||||
account spent its daily wrong-code budget (otp_account_locked, with
|
||||
Retry-After).
|
||||
Retry-After); or the install-wide mail budget is spent
|
||||
(mail_rate_limited, with Retry-After).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -4091,7 +4128,8 @@ paths:
|
||||
description: >-
|
||||
Resend requested before the cooldown elapsed (otp_resend_cooldown), or the
|
||||
account's daily wrong-code budget is spent (otp_account_locked, with
|
||||
Retry-After).
|
||||
Retry-After); or the install-wide mail budget is spent
|
||||
(mail_rate_limited, with Retry-After).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
|
||||
+79
-3
@@ -931,7 +931,9 @@ The series come from two processes:
|
||||
`felis_start_duration_seconds` (no Service; scrape pod-scoped, e.g. a
|
||||
PodMonitor targeting port `metrics`).
|
||||
- `felis-api` internal face `:8081/metrics` (Service `felis-api-internal`) —
|
||||
`felis_image_build_failures_total`. Unauthenticated like the probes;
|
||||
`felis_image_build_failures_total`, and the sign-in series of §17
|
||||
(`felis_mail_total`, `felis_rate_limited_total`,
|
||||
`felis_auth_otp_lockouts_total`). Unauthenticated like the probes;
|
||||
ClusterIP-only, and the external face never serves it.
|
||||
- `felis_reaper_worlds_deleted_total` is produced inside the one-shot reaper
|
||||
CronJob, which exits long before any scrape interval — without a pushgateway
|
||||
@@ -941,8 +943,10 @@ The series come from two processes:
|
||||
### Alert rules
|
||||
|
||||
`deploy/alerts/` ships ready-made rules: build failures, slow starts, node
|
||||
disk/memory thresholds, the kubelet `DiskPressure` condition, and control-plane
|
||||
database backup freshness (§16; needs node-exporter's textfile collector).
|
||||
disk/memory thresholds, the kubelet `DiskPressure` condition, control-plane
|
||||
database backup freshness (§16; needs node-exporter's textfile collector), and
|
||||
sign-in abuse: the mail budget, relay failures, throttled floods and account
|
||||
code locks (§17).
|
||||
|
||||
- Plain Prometheus: add `felis-alerts.yaml` to `rule_files`. Check and unit-test
|
||||
it standalone with `promtool check rules felis-alerts.yaml` and
|
||||
@@ -1167,6 +1171,75 @@ Skips the pre-migration snapshot (`migrate up -no-backup`). The installer warns
|
||||
loudly when it is set. Use it only when the snapshot cannot work and you have
|
||||
another backup, e.g. an external database newer than the host's `pg_dump`.
|
||||
|
||||
## 17. Sign-in refused with 429, mail budget, account code locks
|
||||
|
||||
The public sign-in doors (`/api/v1/auth/*` except logout and the op-login
|
||||
status poll) have three limits of their own. Each answers 429 with a
|
||||
`Retry-After` header and a distinct error code.
|
||||
|
||||
### `rate_limited`: one address called the doors too often
|
||||
|
||||
Each client address gets 20 calls at once, refilled at 20 a minute, shared
|
||||
across every door. A person signing in makes three or four calls, so this only
|
||||
bites scripts. IPv6 clients share one limit per /64. Refusals count in
|
||||
`felis_rate_limited_total{scope="auth_door"}`; `FelisSignInFlood` fires when
|
||||
more than 10 a minute are refused for 10 minutes.
|
||||
|
||||
The address comes from `[auth] client_ip_header`:
|
||||
|
||||
- Behind the Cloudflare tunnel it is `CF-Connecting-IP`. The edge setup writes
|
||||
it, and an install with an `access_jwt_aud` implies it. The header is
|
||||
trustworthy there because the same setup fences the panel NodePort to
|
||||
loopback, so every request reaching the API came through cloudflared.
|
||||
- Behind your own reverse proxy it is `X-Forwarded-For` (the rightmost entry,
|
||||
the one your proxy appended). Firewall the NodePort so only the proxy reaches
|
||||
it, or a direct caller can write any address it likes.
|
||||
- Unset, the TCP peer is used. Behind any proxy every visitor then shares the
|
||||
proxy's address and one limit, so **everyone gets `rate_limited` at once**.
|
||||
The `felis api` log says at start which it keys on (`sign-in rate limit keys
|
||||
on ...`). Set the header in `/etc/felis/felis.toml` and
|
||||
`/etc/felis/felis.pod.toml`, then `felis converge`.
|
||||
|
||||
### `mail_rate_limited`: the install-wide mail budget is spent
|
||||
|
||||
Every code and notice the API mails spends one token of a single budget,
|
||||
`[smtp] max_per_hour` (default 120; a quarter of it may go at once), so a flood
|
||||
cannot burn the relay's quota and get the sending account suspended. While it
|
||||
is spent, every address gets the same 429 and nothing reaches the relay.
|
||||
`felis_mail_total{result="throttled"}` counts refusals and
|
||||
`FelisMailBudgetExhausted` fires on the first one. Look at
|
||||
`felis_rate_limited_total` first: a flood shows there. If sign-ins are real,
|
||||
raise `max_per_hour` to what your relay allows.
|
||||
|
||||
`FelisMailDeliveryFailing` is the other half: the relay itself refused mail
|
||||
(`felis_mail_total{result="failed"}`, 502 `mail_undeliverable` to the caller).
|
||||
The relay's reason is in the `felis-api` log.
|
||||
|
||||
### `otp_account_locked`: ten wrong codes in 24 hours
|
||||
|
||||
Ten wrong email codes for one account within 24 hours, counted across every
|
||||
code it was sent, lock that account's email-code sign-in until 24 hours after
|
||||
the first miss. The public doors answer a locked account exactly like a wrong
|
||||
code, and the owner gets one mail saying so. Signed-in doors (email
|
||||
verification, migration step-up) answer 429 `otp_account_locked`. Passkey
|
||||
sign-in keeps working. Each lock is audited as `auth.otp.locked` and counted in
|
||||
`felis_auth_otp_lockouts_total{purpose}` (`FelisOTPAccountLocked`).
|
||||
|
||||
To lift a lock early once you have confirmed the owner locked themselves out:
|
||||
|
||||
```sh
|
||||
sudo -u postgres psql felis -c \
|
||||
"DELETE FROM otp_failure_windows WHERE user_id = (SELECT id FROM users WHERE username = '<name>');"
|
||||
```
|
||||
|
||||
### Optional: a Cloudflare rate limiting rule in front
|
||||
|
||||
The limits above live in the API, so they hold on any edge. Behind Cloudflare
|
||||
you can also stop floods before they reach the tunnel: Security → WAF → Rate
|
||||
limiting rules, match URI Path starts with `/api/v1/auth/` on the console and
|
||||
op.console hostnames, count by IP, 30 requests per 10 seconds, action Block
|
||||
for 10 seconds (the Free plan's limits).
|
||||
|
||||
---
|
||||
|
||||
## Quick reference: symptom → section
|
||||
@@ -1198,3 +1271,6 @@ another backup, e.g. an external database newer than the host's `pg_dump`.
|
||||
| `pre-migration backup failed, nothing applied` during an upgrade | §16 |
|
||||
| Undo a mistaken change / restore the control-plane database | §16 |
|
||||
| Host lost: rebuild from a database bundle | §16 |
|
||||
| Sign-in 429 `rate_limited` for everyone at once | §17 |
|
||||
| 429 `mail_rate_limited` / `FelisMailBudgetExhausted` | §17 |
|
||||
| Right code refused; `otp_account_locked` / `FelisOTPAccountLocked` | §17 |
|
||||
+67
-15
@@ -158,6 +158,16 @@ type API struct {
|
||||
// Consumed by handleHasJoined (handlers_hasjoined.go).
|
||||
AuthSources []AuthSource
|
||||
|
||||
// AuthDoorLimit bounds how often one client address may call the public
|
||||
// pre-session auth doors (ratelimit.go). MailLimit bounds all mail the API
|
||||
// sends, install-wide. Zero values disable them; cmd/felis wires both.
|
||||
AuthDoorLimit RateLimit
|
||||
MailLimit RateLimit
|
||||
// ClientIPHeader names the header the install's edge writes the client
|
||||
// address into (CF-Connecting-IP behind the Cloudflare tunnel,
|
||||
// X-Forwarded-For behind an operator proxy). Empty means the TCP peer.
|
||||
ClientIPHeader string
|
||||
|
||||
// Now is the clock, injectable for tests. Defaults to time.Now.
|
||||
Now func() time.Time
|
||||
|
||||
@@ -172,6 +182,11 @@ type API struct {
|
||||
|
||||
streamCapOnce sync.Once
|
||||
streamCap *streamLimiter
|
||||
|
||||
authDoorOnce sync.Once
|
||||
authDoorBuckets *bucketSet
|
||||
mailOnce sync.Once
|
||||
mailBuckets *bucketSet
|
||||
}
|
||||
|
||||
// panelURL returns the public player-console origin ("https://console.<root>"),
|
||||
@@ -286,6 +301,11 @@ type apiRoute struct {
|
||||
// whose EmailVerified is false is restricted to these routes only.
|
||||
SetupAllowed bool
|
||||
|
||||
// AuthDoor marks a public pre-session auth door: it is rate limited per
|
||||
// client address (throttleAuthDoor). The op-login status poll is left off,
|
||||
// since the browser calls it every few seconds while it waits.
|
||||
AuthDoor bool
|
||||
|
||||
h http.HandlerFunc
|
||||
}
|
||||
|
||||
@@ -381,21 +401,21 @@ func (a *API) externalAPIRoutes() []apiRoute {
|
||||
// counter-slice to the anti-enumeration doors — the ONE sanctioned place existence
|
||||
// is disclosed — but it never reveals staffness (methods computed with no role
|
||||
// branch, so a staff and a player address in the same state are indistinguishable).
|
||||
{Method: "POST", Pattern: "/api/v1/auth/options", Public: true, h: a.handleAuthOptions},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/setup/redeem", Public: true, h: a.handleSetupRedeem},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/options", Public: true, AuthDoor: true, h: a.handleAuthOptions},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/setup/redeem", Public: true, AuthDoor: true, h: a.handleSetupRedeem},
|
||||
{Method: "GET", Pattern: "/api/v1/auth/setup/status", SetupAllowed: true, h: a.handleSetupStatus},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/begin", Public: true, h: a.handlePasskeyLoginBegin},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/finish", Public: true, h: a.handlePasskeyLoginFinish},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/begin", Public: true, AuthDoor: true, h: a.handlePasskeyLoginBegin},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/finish", Public: true, AuthDoor: true, h: a.handlePasskeyLoginFinish},
|
||||
// Discoverable ("usernameless") passkey login (task #40): the from-zero sibling of the
|
||||
// email-first pair above — no identifier typed, the account is resolved from the
|
||||
// userHandle inside the signed assertion (handlers_passkey_discoverable.go).
|
||||
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/begin", Public: true, h: a.handlePasskeyLoginDiscoverableBegin},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/finish", Public: true, h: a.handlePasskeyLoginDiscoverableFinish},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/email/start", Public: true, h: a.handleLoginEmailStart},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/email/verify", Public: true, h: a.handleLoginEmailVerify},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/op-login/start", Public: true, h: a.handleOpLoginStart},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/begin", Public: true, AuthDoor: true, h: a.handlePasskeyLoginDiscoverableBegin},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/passkey/login/discoverable/finish", Public: true, AuthDoor: true, h: a.handlePasskeyLoginDiscoverableFinish},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/email/start", Public: true, AuthDoor: true, h: a.handleLoginEmailStart},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/email/verify", Public: true, AuthDoor: true, h: a.handleLoginEmailVerify},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/op-login/start", Public: true, AuthDoor: true, h: a.handleOpLoginStart},
|
||||
{Method: "GET", Pattern: "/api/v1/auth/op-login/status/{id}", Public: true, h: a.handleOpLoginStatus},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/op-login/finish", Public: true, h: a.handleOpLoginFinish},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/op-login/finish", Public: true, AuthDoor: true, h: a.handleOpLoginFinish},
|
||||
// Player-console bootstrap (console-tier access model): the account-less
|
||||
// player's door into console.<root_domain>. Public — like login there is no prior
|
||||
// principal — and session-minting, but the artifact it consumes is a one-time
|
||||
@@ -403,7 +423,7 @@ func (a *API) externalAPIRoutes() []apiRoute {
|
||||
// possession already proves a Minecraft identity. A code whose UUID belongs to
|
||||
// staff is refused (403) so this never yields an admin session; op.console stays
|
||||
// behind Zero Trust (handlers_onboard.go).
|
||||
{Method: "POST", Pattern: "/api/v1/auth/bind", Public: true, h: a.handleBindRedeem},
|
||||
{Method: "POST", Pattern: "/api/v1/auth/bind", Public: true, AuthDoor: true, h: a.handleBindRedeem},
|
||||
|
||||
// App-auth tier: operations on your own servers (spec §14).
|
||||
{Method: "POST", Pattern: "/api/v1/servers/{name}/wake", h: a.handleWake},
|
||||
@@ -614,7 +634,11 @@ func (a *API) buildFace(routes []apiRoute, guard func(http.Handler) http.Handler
|
||||
for _, rt := range routes {
|
||||
pattern := rt.Method + " " + rt.Pattern
|
||||
if rt.Public {
|
||||
mux.HandleFunc(pattern, rt.h)
|
||||
h := rt.h
|
||||
if rt.AuthDoor {
|
||||
h = a.throttleAuthDoor(h)
|
||||
}
|
||||
mux.HandleFunc(pattern, h)
|
||||
continue
|
||||
}
|
||||
h := rt.h
|
||||
@@ -727,10 +751,35 @@ func principalFromContext(ctx context.Context) *Principal {
|
||||
// reserve/release pair closes the intra-replica concurrent burst (the bug fixed in
|
||||
// #35); cross-replica bounding would need a shared store (out of scope for the
|
||||
// single-replica demo).
|
||||
//
|
||||
// Entries older than the longest window the limiter has been asked about can
|
||||
// no longer block anything, so checks sweep them out (at most once per
|
||||
// bucketSweepEvery). Without that, every distinct address typed into a public
|
||||
// door, whose neutral branch keeps its reservation, stayed in the map for the
|
||||
// life of the process.
|
||||
type cooldownLimiter struct {
|
||||
mu sync.Mutex
|
||||
now func() time.Time
|
||||
last map[string]time.Time
|
||||
maxWindow time.Duration
|
||||
swept time.Time
|
||||
}
|
||||
|
||||
// noteWindow widens the retention to window and sweeps stale entries when due.
|
||||
// The caller holds mu.
|
||||
func (c *cooldownLimiter) noteWindow(window time.Duration, now time.Time) {
|
||||
if window > c.maxWindow {
|
||||
c.maxWindow = window
|
||||
}
|
||||
if c.maxWindow <= 0 || now.Sub(c.swept) < bucketSweepEvery {
|
||||
return
|
||||
}
|
||||
c.swept = now
|
||||
for k, t := range c.last {
|
||||
if now.Sub(t) >= c.maxWindow {
|
||||
delete(c.last, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// allowed reports whether name may wake now WITHOUT recording the attempt. A
|
||||
@@ -745,7 +794,9 @@ func (c *cooldownLimiter) allowed(name string, window time.Duration) bool {
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if last, ok := c.last[name]; ok && c.now().Sub(last) < window {
|
||||
now := c.now()
|
||||
c.noteWindow(window, now)
|
||||
if last, ok := c.last[name]; ok && now.Sub(last) < window {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
@@ -778,10 +829,11 @@ func (c *cooldownLimiter) reserve(name string, window time.Duration) (time.Time,
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if last, ok := c.last[name]; ok && c.now().Sub(last) < window {
|
||||
t := c.now()
|
||||
c.noteWindow(window, t)
|
||||
if last, ok := c.last[name]; ok && t.Sub(last) < window {
|
||||
return time.Time{}, false
|
||||
}
|
||||
t := c.now()
|
||||
c.last[name] = t
|
||||
return t, true
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -130,10 +131,19 @@ type apiError struct {
|
||||
status int
|
||||
code string
|
||||
msg string
|
||||
// wait, when positive, is sent as Retry-After (whole seconds, rounded up).
|
||||
wait time.Duration
|
||||
}
|
||||
|
||||
func (e *apiError) Error() string { return e.msg }
|
||||
|
||||
// retryAfter returns a copy of e that tells the client when to retry.
|
||||
func (e *apiError) retryAfter(d time.Duration) *apiError {
|
||||
c := *e
|
||||
c.wait = d
|
||||
return &c
|
||||
}
|
||||
|
||||
// newError builds an apiError with a formatted message.
|
||||
func newError(status int, code, format string, a ...any) *apiError {
|
||||
return &apiError{status: status, code: code, msg: fmt.Sprintf(format, a...)}
|
||||
@@ -176,6 +186,9 @@ func writeError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
r.Method, r.URL.Path, requestIDFromContext(r.Context()), err)
|
||||
ae = newError(http.StatusInternalServerError, "internal", "internal error")
|
||||
}
|
||||
if ae.wait > 0 {
|
||||
w.Header().Set("Retry-After", strconv.FormatInt(int64((ae.wait+time.Second-1)/time.Second), 10))
|
||||
}
|
||||
body := map[string]any{
|
||||
"error": map[string]string{
|
||||
"code": ae.code,
|
||||
|
||||
@@ -24,12 +24,9 @@ import (
|
||||
// this separator).
|
||||
// - No principal. The throttle cannot key off a user id (there is none yet); it
|
||||
// keys off the typed recipient address, the same anti-bomb dimension the onboard
|
||||
// start uses. Per-source (client-IP) aggregate limiting is deliberately NOT done
|
||||
// here: cooldownLimiter is a one-per-window primitive, so keying it on client IP
|
||||
// would false-positive on shared egress (CGNAT / office NAT), and behind
|
||||
// Cloudflare RemoteAddr is the proxy anyway. The only real harm — bombing one
|
||||
// mailbox — is already bounded per recipient; volumetric per-source limiting
|
||||
// belongs at the edge.
|
||||
// start uses. Volume from one client is bounded separately by the per-address
|
||||
// token bucket every public auth door sits behind (throttleAuthDoor), and total
|
||||
// mail by the install-wide mail budget (ratelimit.go).
|
||||
// - Refuse staff. Like handleBindRedeem this public door provably never mints a
|
||||
// session for an admin identity: op.console stays behind Zero Trust (and its own
|
||||
// in-game approval gate). The refusal happens only AFTER a valid code is
|
||||
@@ -84,6 +81,13 @@ func (a *API) handleLoginEmailStart(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// The install-wide mail budget is checked before the address is resolved,
|
||||
// so while it is spent every address gets the same 429.
|
||||
if err := a.checkMailBudget(); err != nil {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Atomically reserve the per-recipient cooldown BEFORE any work, so a burst of
|
||||
// truly concurrent starts yields exactly one winner and each admitted send is one
|
||||
// real, non-idempotent email. The key is namespaced apart from the onboard door's
|
||||
|
||||
@@ -16,11 +16,10 @@ import (
|
||||
// address has an account precisely because THIS endpoint is the one sanctioned place
|
||||
// existence is revealed. An empty methods array means "no (verified) account". That
|
||||
// makes it a mass-enumeration surface by design — an accepted product decision, the
|
||||
// same one the email door's header records. It is bounded only at the edge: the
|
||||
// handler sends no mail and mutates nothing, so a per-recipient cooldown would merely
|
||||
// block a legitimate retry, and per-source (client-IP) limiting is the edge's job
|
||||
// (behind Cloudflare RemoteAddr is the proxy, and CGNAT would false-positive) — see
|
||||
// the handlers_auth_email.go header for the same reasoning.
|
||||
// same one the email door's header records. The handler sends no mail and mutates
|
||||
// nothing, so a per-recipient cooldown would merely block a legitimate retry; what
|
||||
// bounds enumeration is the per-client-address token bucket shared by every public
|
||||
// auth door (throttleAuthDoor in ratelimit.go).
|
||||
//
|
||||
// It never reveals STAFFNESS. Methods are computed by the SAME rule for every resolved
|
||||
// account — no role branch, no operator hint — so a staff email and a player email in
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/metrics"
|
||||
)
|
||||
|
||||
// Player email verification (spec §B2 onboarding). Forced web onboarding proves a
|
||||
@@ -250,12 +252,21 @@ func (a *API) handleEmailOTPVerify(w http.ResponseWriter, r *http.Request) {
|
||||
// deliverOTP hands the code to the configured Mailer, or — when none is wired (the
|
||||
// demo) — logs it server-side as a KNOWN-LIMITATION. The code is logged ONLY in the
|
||||
// no-mailer fallback and ONLY to the server log; it is never put in an HTTP response.
|
||||
//
|
||||
// Every real send spends one token of the install-wide mail budget (mailGate);
|
||||
// a spent budget is a 429 mail_rate_limited and nothing reaches the relay.
|
||||
func (a *API) deliverOTP(ctx context.Context, email, code string) error {
|
||||
if a.Mailer == nil {
|
||||
log.Printf("email-otp: no Mailer configured; code for %s is %s (KNOWN-LIMITATION: demo has no SMTP)", email, code)
|
||||
return nil
|
||||
}
|
||||
if ok, wait := a.mailGate().take(mailGateKey); !ok {
|
||||
metrics.MailTotal.WithLabelValues("otp", "throttled").Inc()
|
||||
log.Printf("api: OTP mail refused by the install-wide mail budget (request_id=%s)", requestIDFromContext(ctx))
|
||||
return errMailRateLimited(wait)
|
||||
}
|
||||
if err := a.Mailer.SendOTP(ctx, email, code); err != nil {
|
||||
metrics.MailTotal.WithLabelValues("otp", "failed").Inc()
|
||||
// Mapped here rather than at each of the four call sites, so every door that
|
||||
// mails a code answers the same way. A relay refusal is neither the caller's
|
||||
// fault nor a bug in Felis, and a bare 500 says neither — it reads as "the
|
||||
@@ -269,6 +280,7 @@ func (a *API) deliverOTP(ctx context.Context, email, code string) error {
|
||||
return newError(http.StatusBadGateway, "mail_undeliverable",
|
||||
"the mail relay refused this message; ask the server operator to check the SMTP settings")
|
||||
}
|
||||
metrics.MailTotal.WithLabelValues("otp", "sent").Inc()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -86,6 +86,13 @@ func (a *API) handleOpLoginStart(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// The install-wide mail budget is checked before the address is resolved,
|
||||
// so while it is spent every address gets the same 429.
|
||||
if err := a.checkMailBudget(); err != nil {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Per-recipient cooldown reserved BEFORE any work, identical to the console email
|
||||
// door: one winner per window, and the neutral (non-staff) branch keeps the
|
||||
// reservation too so probing an address is throttled exactly like a real send. The
|
||||
|
||||
@@ -21,10 +21,10 @@ import (
|
||||
//
|
||||
// Anti-abuse divergence from the email-first door: that door reserves a per-recipient cooldown
|
||||
// (a.otpLimiter) keyed on the typed email. A usernameless begin has no recipient OR principal to
|
||||
// key a fair per-caller limit on, so — matching the stance in handlers_auth_email.go (behind
|
||||
// Cloudflare RemoteAddr is the proxy; CGNAT false-positives) — volumetric per-source limiting is
|
||||
// left to the edge, and the server-side bound is a hard global cap on live challenges enforced
|
||||
// atomically in CreateDiscoverableChallenge (ErrTooManyDiscoverableChallenges → 429).
|
||||
// key a fair per-caller limit on, so one client is bounded by the per-address token bucket every
|
||||
// public auth door sits behind (throttleAuthDoor), and the table by a hard global cap on live
|
||||
// challenges enforced atomically in CreateDiscoverableChallenge (ErrTooManyDiscoverableChallenges
|
||||
// → 429).
|
||||
|
||||
// handlePasskeyLoginDiscoverableBegin starts a usernameless assertion ceremony (Public,
|
||||
// pre-session). It has no request body — the whole point is that the caller supplies no
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/metrics"
|
||||
@@ -35,14 +34,9 @@ var otpDoorName = map[string][2]string{
|
||||
|
||||
// writeOTPAccountLocked answers a signed-in door whose budget is spent.
|
||||
func writeOTPAccountLocked(w http.ResponseWriter, r *http.Request, until, now time.Time) {
|
||||
secs := int64(until.Sub(now).Round(time.Second) / time.Second)
|
||||
if secs < 1 {
|
||||
secs = 1
|
||||
}
|
||||
w.Header().Set("Retry-After", strconv.FormatInt(secs, 10))
|
||||
writeError(w, r, newError(http.StatusTooManyRequests, "otp_account_locked",
|
||||
"too many wrong codes on this account; email codes work again after %s",
|
||||
until.UTC().Format(time.RFC3339)))
|
||||
until.UTC().Format(time.RFC3339)).retryAfter(max(until.Sub(now), time.Second)))
|
||||
}
|
||||
|
||||
// noteOTPLock handles a redeem that met the account lock. Only the guess that
|
||||
@@ -83,10 +77,18 @@ func (a *API) noteOTPLock(r *http.Request, err error, userID, purpose string) {
|
||||
log.Printf("auth: no notice mailer; user %s was not told their %s is locked", userID, purpose)
|
||||
return
|
||||
}
|
||||
if ok, _ := a.mailGate().take(mailGateKey); !ok {
|
||||
metrics.MailTotal.WithLabelValues("notice", "throttled").Inc()
|
||||
log.Printf("auth: mail budget spent; user %s was not told their %s is locked", userID, purpose)
|
||||
return
|
||||
}
|
||||
subject, body := otpLockNotice(door, lock.Until)
|
||||
if err := sender.SendNotice(ctx, u.Email, subject, body); err != nil {
|
||||
metrics.MailTotal.WithLabelValues("notice", "failed").Inc()
|
||||
log.Printf("auth: otp lock notice to user %s failed: %v", userID, err)
|
||||
return
|
||||
}
|
||||
metrics.MailTotal.WithLabelValues("notice", "sent").Inc()
|
||||
}
|
||||
|
||||
// otpLockNotice renders the bilingual lock notice.
|
||||
|
||||
@@ -3,9 +3,12 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/metrics"
|
||||
)
|
||||
|
||||
// The per-code attempt cap resets on every resend; these pin the account-level
|
||||
@@ -181,3 +184,14 @@ func TestOTPLockNoticeNamesDoorAndTime(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The lockout alert sees a purpose only if metrics pre-creates its series.
|
||||
func TestOTPPurposesMatchMetricLabels(t *testing.T) {
|
||||
got := []string{otpPurposeOnboard, otpPurposeLogin, otpPurposeOpLogin, otpPurposeMigrate}
|
||||
want := slices.Clone(metrics.OTPPurposes)
|
||||
slices.Sort(got)
|
||||
slices.Sort(want)
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("otp purposes %v, metrics.OTPPurposes %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1218,9 +1218,8 @@ func (p *PGRepo) ConsumePasskeyChallengeByUser(ctx context.Context, userID, purp
|
||||
// exist, a new begin is refused (ErrTooManyDiscoverableChallenges → 429). The cap is generous —
|
||||
// a login challenge lives only passkeyChallengeTTL (5 min) and each row is ~1 KB — so real
|
||||
// concurrency never approaches it, while an abusive begin-flood is bounded to a few MB instead
|
||||
// of growing without limit. Volumetric per-IP limiting is the edge's job (handlers_auth_email.go):
|
||||
// behind Cloudflare RemoteAddr is the proxy, and a usernameless door has no recipient to key a
|
||||
// fair per-caller limit on.
|
||||
// of growing without limit. One client's volume is bounded before it gets here, by the
|
||||
// per-address token bucket in front of every public auth door (ratelimit.go).
|
||||
const maxLiveDiscoverableChallenges = 4096
|
||||
|
||||
// CreateDiscoverableChallenge stashes a discoverable-login ceremony under an opaque handle,
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/metrics"
|
||||
)
|
||||
|
||||
// Volumetric limits for the public auth doors and for outbound mail.
|
||||
//
|
||||
// The per-recipient OTP cooldown (otpLimiter) stops one mailbox being bombed,
|
||||
// and the per-account wrong-code budget stops one account being guessed. Neither
|
||||
// bounds a caller who sprays many addresses or many accounts, so two more limits
|
||||
// sit in front of them:
|
||||
//
|
||||
// - authDoorGate: a token bucket per client address over every pre-session
|
||||
// auth door (options, login, op-login, bind, setup redeem). The client
|
||||
// address comes from the edge's header only when the install says which
|
||||
// header its edge writes (ClientIPHeader); with Cloudflare that header is
|
||||
// CF-Connecting-IP, which is trustworthy because the edge setup fences the
|
||||
// panel NodePort to loopback, so every request reaching the origin came
|
||||
// through cloudflared. IPv6 clients are bucketed per /64, the unit one
|
||||
// subscriber is handed.
|
||||
// - mailGate: one install-wide bucket over every mail the API sends (codes
|
||||
// and lock notices), so no flood can burn the SMTP relay's quota and get
|
||||
// the sending account suspended. The public doors check it before they
|
||||
// resolve the address, so a spent budget answers every address alike.
|
||||
|
||||
// RateLimit is a token bucket: Burst calls at once, refilled at PerMinute. The
|
||||
// zero value disables the limit.
|
||||
type RateLimit struct {
|
||||
Burst int
|
||||
PerMinute float64
|
||||
}
|
||||
|
||||
func (l RateLimit) enabled() bool { return l.Burst > 0 && l.PerMinute > 0 }
|
||||
|
||||
// bucketSweepEvery is how often idle buckets are dropped; bucketMaxKeys bounds
|
||||
// the map between sweeps. Past the bound, new keys share one overflow bucket,
|
||||
// so a spray of fresh source addresses throttles itself instead of growing the
|
||||
// map.
|
||||
const (
|
||||
bucketSweepEvery = time.Minute
|
||||
bucketMaxKeys = 50_000
|
||||
bucketOverflow = "\x00overflow"
|
||||
)
|
||||
|
||||
type tokenBucket struct {
|
||||
tokens float64
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// bucketSet is a set of token buckets keyed by caller. A missing key is a full
|
||||
// bucket, so a bucket that has refilled completely carries no information and
|
||||
// is dropped by the sweep; memory is bounded by the keys active in the last
|
||||
// refill period.
|
||||
type bucketSet struct {
|
||||
mu sync.Mutex
|
||||
now func() time.Time
|
||||
limit RateLimit
|
||||
buckets map[string]*tokenBucket
|
||||
swept time.Time
|
||||
maxKeys int
|
||||
}
|
||||
|
||||
func newBucketSet(limit RateLimit, now func() time.Time) *bucketSet {
|
||||
return &bucketSet{now: now, limit: limit, buckets: map[string]*tokenBucket{}, maxKeys: bucketMaxKeys}
|
||||
}
|
||||
|
||||
// refill brings b up to now. The caller holds mu.
|
||||
func (s *bucketSet) refill(b *tokenBucket, now time.Time) {
|
||||
if elapsed := now.Sub(b.at); elapsed > 0 {
|
||||
b.tokens = math.Min(float64(s.limit.Burst), b.tokens+elapsed.Minutes()*s.limit.PerMinute)
|
||||
}
|
||||
b.at = now
|
||||
}
|
||||
|
||||
// sweep drops full buckets at most once per bucketSweepEvery, or at once when
|
||||
// force is set. The caller holds mu.
|
||||
func (s *bucketSet) sweep(now time.Time, force bool) {
|
||||
if !force && now.Sub(s.swept) < bucketSweepEvery {
|
||||
return
|
||||
}
|
||||
s.swept = now
|
||||
for k, b := range s.buckets {
|
||||
s.refill(b, now)
|
||||
if b.tokens >= float64(s.limit.Burst) {
|
||||
delete(s.buckets, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bucket returns key's bucket, creating a full one. The caller holds mu.
|
||||
func (s *bucketSet) bucket(key string, now time.Time) *tokenBucket {
|
||||
s.sweep(now, false)
|
||||
if b, ok := s.buckets[key]; ok {
|
||||
s.refill(b, now)
|
||||
return b
|
||||
}
|
||||
if len(s.buckets) >= s.maxKeys {
|
||||
s.sweep(now, true)
|
||||
if len(s.buckets) >= s.maxKeys {
|
||||
key = bucketOverflow
|
||||
if b, ok := s.buckets[key]; ok {
|
||||
s.refill(b, now)
|
||||
return b
|
||||
}
|
||||
}
|
||||
}
|
||||
b := &tokenBucket{tokens: float64(s.limit.Burst), at: now}
|
||||
s.buckets[key] = b
|
||||
return b
|
||||
}
|
||||
|
||||
// take spends one token from key's bucket. When none is left it reports how
|
||||
// long until one is. A disabled limit always admits.
|
||||
func (s *bucketSet) take(key string) (bool, time.Duration) {
|
||||
if s == nil || !s.limit.enabled() {
|
||||
return true, 0
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := s.now()
|
||||
b := s.bucket(key, now)
|
||||
if b.tokens >= 1 {
|
||||
b.tokens--
|
||||
return true, 0
|
||||
}
|
||||
return false, s.wait(b)
|
||||
}
|
||||
|
||||
// peek reports whether key's bucket holds a token, without spending it.
|
||||
func (s *bucketSet) peek(key string) (bool, time.Duration) {
|
||||
if s == nil || !s.limit.enabled() {
|
||||
return true, 0
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := s.now()
|
||||
b, ok := s.buckets[key]
|
||||
if !ok {
|
||||
return true, 0
|
||||
}
|
||||
s.refill(b, now)
|
||||
if b.tokens >= 1 {
|
||||
return true, 0
|
||||
}
|
||||
return false, s.wait(b)
|
||||
}
|
||||
|
||||
// wait is how long until b holds one token. The caller holds mu.
|
||||
func (s *bucketSet) wait(b *tokenBucket) time.Duration {
|
||||
missing := 1 - b.tokens
|
||||
return time.Duration(math.Ceil(missing / s.limit.PerMinute * float64(time.Minute)))
|
||||
}
|
||||
|
||||
func (a *API) authDoorGate() *bucketSet {
|
||||
a.authDoorOnce.Do(func() { a.authDoorBuckets = newBucketSet(a.AuthDoorLimit, a.now) })
|
||||
return a.authDoorBuckets
|
||||
}
|
||||
|
||||
func (a *API) mailGate() *bucketSet {
|
||||
a.mailOnce.Do(func() { a.mailBuckets = newBucketSet(a.MailLimit, a.now) })
|
||||
return a.mailBuckets
|
||||
}
|
||||
|
||||
// mailGateKey is the single install-wide mail bucket.
|
||||
const mailGateKey = "mail"
|
||||
|
||||
// throttleAuthDoor applies the per-source bucket to one public auth door.
|
||||
func (a *API) throttleAuthDoor(h http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
ok, wait := a.authDoorGate().take(sourceKey(a.clientIP(r)))
|
||||
if !ok {
|
||||
metrics.RateLimitedTotal.WithLabelValues("auth_door").Inc()
|
||||
writeError(w, r, newError(http.StatusTooManyRequests, "rate_limited",
|
||||
"too many sign-in requests from this network; try again shortly").retryAfter(wait))
|
||||
return
|
||||
}
|
||||
h(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// errMailRateLimited answers a door whose mail the install-wide budget refuses.
|
||||
func errMailRateLimited(wait time.Duration) *apiError {
|
||||
return newError(http.StatusTooManyRequests, "mail_rate_limited",
|
||||
"this server is sending too much mail right now; try again shortly").retryAfter(wait)
|
||||
}
|
||||
|
||||
// checkMailBudget is the public doors' pre-resolution check: it refuses every
|
||||
// address alike while the budget is spent, so the refusal says nothing about
|
||||
// whether the address has an account.
|
||||
func (a *API) checkMailBudget() error {
|
||||
if a.Mailer == nil {
|
||||
return nil
|
||||
}
|
||||
if ok, wait := a.mailGate().peek(mailGateKey); !ok {
|
||||
metrics.MailTotal.WithLabelValues("otp", "throttled").Inc()
|
||||
return errMailRateLimited(wait)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// clientIP is the caller's address: the edge's header when the install names
|
||||
// one and the request carries a parseable value, else the TCP peer. For
|
||||
// X-Forwarded-For the rightmost entry is used, the one the trusted proxy
|
||||
// appended itself.
|
||||
func (a *API) clientIP(r *http.Request) netip.Addr {
|
||||
if name := a.ClientIPHeader; name != "" {
|
||||
if v := r.Header.Get(name); v != "" {
|
||||
if strings.EqualFold(name, "X-Forwarded-For") {
|
||||
if i := strings.LastIndexByte(v, ','); i >= 0 {
|
||||
v = v[i+1:]
|
||||
}
|
||||
}
|
||||
if ip, err := netip.ParseAddr(strings.TrimSpace(v)); err == nil {
|
||||
return ip.Unmap()
|
||||
}
|
||||
}
|
||||
}
|
||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
host = r.RemoteAddr
|
||||
}
|
||||
ip, _ := netip.ParseAddr(host)
|
||||
return ip.Unmap()
|
||||
}
|
||||
|
||||
// sourceKey buckets an address: IPv4 per host, IPv6 per /64. An unparseable
|
||||
// address shares one key.
|
||||
func sourceKey(ip netip.Addr) string {
|
||||
switch {
|
||||
case !ip.IsValid():
|
||||
return "unknown"
|
||||
case ip.Is4():
|
||||
return ip.String()
|
||||
default:
|
||||
p, _ := ip.Prefix(64)
|
||||
return p.String()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// These pin the two volumetric limits in front of the public sign-in doors:
|
||||
// the per-client-address bucket (keyed on the edge's visitor header only when
|
||||
// the install names it) and the install-wide mail budget, which must refuse
|
||||
// every address alike so it never becomes an existence oracle. They also pin
|
||||
// that neither the buckets nor the OTP cooldown map grow without bound.
|
||||
|
||||
func TestBucketSetBurstRefillAndWait(t *testing.T) {
|
||||
clock := time.Unix(1_700_000_000, 0)
|
||||
s := newBucketSet(RateLimit{Burst: 3, PerMinute: 6}, func() time.Time { return clock })
|
||||
for i := 0; i < 3; i++ {
|
||||
if ok, _ := s.take("a"); !ok {
|
||||
t.Fatalf("take %d refused inside the burst", i+1)
|
||||
}
|
||||
}
|
||||
ok, wait := s.take("a")
|
||||
if ok || wait != 10*time.Second {
|
||||
t.Fatalf("4th take = %v wait %v, want refused with 10s wait (6/min)", ok, wait)
|
||||
}
|
||||
if ok, _ := s.take("b"); !ok {
|
||||
t.Fatal("another key shares a's bucket")
|
||||
}
|
||||
if ok, _ := s.peek("a"); ok {
|
||||
t.Fatal("peek admitted an empty bucket")
|
||||
}
|
||||
clock = clock.Add(10 * time.Second)
|
||||
if ok, _ := s.peek("a"); !ok {
|
||||
t.Fatal("peek refused after one token refilled")
|
||||
}
|
||||
if ok, _ := s.take("a"); !ok {
|
||||
t.Fatal("take refused after one token refilled (peek must not spend it)")
|
||||
}
|
||||
if ok, _ := s.take("a"); ok {
|
||||
t.Fatal("a second token appeared from nowhere")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketSetDropsIdleKeysAndCapsTheMap(t *testing.T) {
|
||||
clock := time.Unix(1_700_000_000, 0)
|
||||
s := newBucketSet(RateLimit{Burst: 2, PerMinute: 60}, func() time.Time { return clock })
|
||||
s.maxKeys = 100
|
||||
for i := 0; i < 100; i++ {
|
||||
s.take(fmt.Sprintf("10.0.0.%d", i))
|
||||
}
|
||||
// At the cap with every bucket still draining: fresh keys share the
|
||||
// overflow bucket instead of growing the map.
|
||||
s.take("fresh-1")
|
||||
s.take("fresh-2")
|
||||
if ok, _ := s.take("fresh-3"); ok {
|
||||
t.Fatal("overflow bucket admitted past its burst")
|
||||
}
|
||||
if n := len(s.buckets); n != 101 {
|
||||
t.Fatalf("map holds %d buckets, want 100 + overflow", n)
|
||||
}
|
||||
// Once they refill, the sweep drops them all.
|
||||
clock = clock.Add(2 * bucketSweepEvery)
|
||||
s.take("later")
|
||||
if n := len(s.buckets); n != 1 {
|
||||
t.Fatalf("after refill the map holds %d buckets, want only the new one", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisabledLimitAdmitsEverything(t *testing.T) {
|
||||
var nilSet *bucketSet
|
||||
if ok, _ := nilSet.take("x"); !ok {
|
||||
t.Fatal("nil set refused")
|
||||
}
|
||||
s := newBucketSet(RateLimit{}, time.Now)
|
||||
for i := 0; i < 1000; i++ {
|
||||
if ok, _ := s.take("x"); !ok {
|
||||
t.Fatal("zero limit refused")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientIPTrustsOnlyTheNamedHeader(t *testing.T) {
|
||||
req := func(remote string, h map[string]string) *http.Request {
|
||||
r := httptest.NewRequest("POST", "/", nil)
|
||||
r.RemoteAddr = remote
|
||||
for k, v := range h {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
return r
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
header string
|
||||
r *http.Request
|
||||
want string
|
||||
}{
|
||||
{"no header configured ignores CF-Connecting-IP", "", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "203.0.113.9"}), "10.42.0.1"},
|
||||
{"cloudflare header", "CF-Connecting-IP", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "203.0.113.9"}), "203.0.113.9"},
|
||||
{"cloudflare header absent falls back to peer", "CF-Connecting-IP", req("10.42.0.1:5000", nil), "10.42.0.1"},
|
||||
{"garbage header falls back to peer", "CF-Connecting-IP", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "not-an-ip"}), "10.42.0.1"},
|
||||
{"X-Forwarded-For takes the proxy-appended rightmost hop", "X-Forwarded-For", req("10.42.0.1:5000", map[string]string{"X-Forwarded-For": "1.1.1.1, 198.51.100.7"}), "198.51.100.7"},
|
||||
{"v4-mapped v6 is unmapped", "CF-Connecting-IP", req("10.42.0.1:5000", map[string]string{"CF-Connecting-IP": "::ffff:203.0.113.9"}), "203.0.113.9"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a := &API{ClientIPHeader: tc.header}
|
||||
if got := a.clientIP(tc.r).String(); got != tc.want {
|
||||
t.Fatalf("clientIP = %s, want %s", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceKeyGroupsIPv6By64(t *testing.T) {
|
||||
a := sourceKey(netip.MustParseAddr("2001:db8:1:2::1"))
|
||||
b := sourceKey(netip.MustParseAddr("2001:db8:1:2:ffff::9"))
|
||||
c := sourceKey(netip.MustParseAddr("2001:db8:1:3::1"))
|
||||
if a != b || a == c {
|
||||
t.Fatalf("keys %q %q %q: want one per /64", a, b, c)
|
||||
}
|
||||
if k := sourceKey(netip.MustParseAddr("203.0.113.9")); k != "203.0.113.9" {
|
||||
t.Fatalf("v4 key = %q", k)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthDoorsThrottlePerClientAddress(t *testing.T) {
|
||||
api, _, _ := seedLoginEmailAPI(t)
|
||||
api.AuthDoorLimit = RateLimit{Burst: 3, PerMinute: 3}
|
||||
api.ClientIPHeader = "CF-Connecting-IP"
|
||||
eh := api.ExternalHandler()
|
||||
from := func(ip string) map[string]string {
|
||||
return map[string]string{"Content-Type": "application/json", "CF-Connecting-IP": ip}
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
if w := do(eh, "POST", "/api/v1/auth/options", `{"email":"[email protected]"}`, from("203.0.113.9")); w.Code != http.StatusOK {
|
||||
t.Fatalf("call %d = %d (%s)", i+1, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
// The bucket spans every door: a different door from the same address is
|
||||
// refused too.
|
||||
w := do(eh, "POST", "/api/v1/auth/email/verify", `{"email":"[email protected]","code":"000000"}`, from("203.0.113.9"))
|
||||
if c, _ := errEnvelope(t, w); w.Code != http.StatusTooManyRequests || c != "rate_limited" {
|
||||
t.Fatalf("4th call = %d %s, want 429 rate_limited", w.Code, c)
|
||||
}
|
||||
if ra := w.Header().Get("Retry-After"); ra != "20" {
|
||||
t.Fatalf("Retry-After = %q, want 20 (3/min)", ra)
|
||||
}
|
||||
if w := do(eh, "POST", "/api/v1/auth/options", `{"email":"[email protected]"}`, from("198.51.100.7")); w.Code != http.StatusOK {
|
||||
t.Fatalf("another address was throttled: %d", w.Code)
|
||||
}
|
||||
// The op-login poll and logout are not doors: a waiting browser polls.
|
||||
for i := 0; i < 5; i++ {
|
||||
if w := do(eh, "GET", "/api/v1/auth/op-login/status/abc", "", from("203.0.113.9")); w.Code == http.StatusTooManyRequests {
|
||||
t.Fatal("op-login status poll was throttled")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthDoorsIgnoreVisitorHeaderUnlessConfigured(t *testing.T) {
|
||||
api, _, _ := seedLoginEmailAPI(t)
|
||||
api.AuthDoorLimit = RateLimit{Burst: 2, PerMinute: 2}
|
||||
eh := api.ExternalHandler()
|
||||
// Without client_ip_header a forged CF-Connecting-IP must not mint fresh
|
||||
// buckets: every call below comes from httptest's one peer address.
|
||||
for i := 0; i < 2; i++ {
|
||||
do(eh, "POST", "/api/v1/auth/options", `{"email":"[email protected]"}`,
|
||||
map[string]string{"Content-Type": "application/json", "CF-Connecting-IP": fmt.Sprintf("203.0.113.%d", i)})
|
||||
}
|
||||
w := do(eh, "POST", "/api/v1/auth/options", `{"email":"[email protected]"}`,
|
||||
map[string]string{"Content-Type": "application/json", "CF-Connecting-IP": "203.0.113.200"})
|
||||
if w.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("forged visitor header escaped the limit: %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMailBudgetRefusesEveryAddressAlike(t *testing.T) {
|
||||
api, repo, mailer := seedLoginEmailAPI(t)
|
||||
repo.staff["second"] = &StaffUser{ID: "u2", Username: "second", Email: "[email protected]", Role: "user", EmailVerified: true}
|
||||
api.MailLimit = RateLimit{Burst: 1, PerMinute: 1}
|
||||
clock := time.Unix(1_700_000_000, 0)
|
||||
api.Now = func() time.Time { return clock }
|
||||
eh := api.ExternalHandler()
|
||||
start := func(email string) *httptest.ResponseRecorder {
|
||||
return do(eh, "POST", "/api/v1/auth/email/start", `{"email":"`+email+`"}`, jsonHeader)
|
||||
}
|
||||
|
||||
if w := start("[email protected]"); w.Code != http.StatusAccepted || mailer.calls != 1 {
|
||||
t.Fatalf("first send = %d, %d mails", w.Code, mailer.calls)
|
||||
}
|
||||
// Budget spent: a real account and an unknown address get the same 429,
|
||||
// and nothing reaches the relay.
|
||||
for _, email := range []string{"[email protected]", "[email protected]"} {
|
||||
w := start(email)
|
||||
if c, _ := errEnvelope(t, w); w.Code != http.StatusTooManyRequests || c != "mail_rate_limited" || w.Header().Get("Retry-After") == "" {
|
||||
t.Fatalf("%s while the budget is spent = %d %s (Retry-After %q)", email, w.Code, c, w.Header().Get("Retry-After"))
|
||||
}
|
||||
}
|
||||
if mailer.calls != 1 {
|
||||
t.Fatalf("a spent budget still mailed: %d sends", mailer.calls)
|
||||
}
|
||||
// The refused starts did not burn their recipients' cooldowns.
|
||||
clock = clock.Add(time.Minute)
|
||||
if w := start("[email protected]"); w.Code != http.StatusAccepted || mailer.calls != 2 {
|
||||
t.Fatalf("after refill = %d, %d mails", w.Code, mailer.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignedInDoorMailBudget(t *testing.T) {
|
||||
repo := newFakeRepo()
|
||||
repo.staff["player"] = &StaffUser{ID: "u1", Username: "player"}
|
||||
api := newTestAPI(repo, newFakeCluster())
|
||||
api.External = staticExternal{p: &Principal{UserID: "u1", Email: "[email protected]", Role: "user"}}
|
||||
mailer := &captureMailer{}
|
||||
api.Mailer = mailer
|
||||
// One mail per two minutes, so stepping past the per-principal resend
|
||||
// cooldown leaves the mail budget still empty.
|
||||
api.MailLimit = RateLimit{Burst: 1, PerMinute: 0.5}
|
||||
clock := time.Unix(1_700_000_000, 0)
|
||||
api.Now = func() time.Time { return clock }
|
||||
eh := api.ExternalHandler()
|
||||
if w := do(eh, "POST", "/api/v1/account/email/start", `{"email":"[email protected]"}`, nil); w.Code != http.StatusAccepted {
|
||||
t.Fatalf("first = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
clock = clock.Add(otpResendCooldown + time.Second)
|
||||
w := do(eh, "POST", "/api/v1/account/email/start", `{"email":"[email protected]"}`, nil)
|
||||
if c, _ := errEnvelope(t, w); w.Code != http.StatusTooManyRequests || c != "mail_rate_limited" {
|
||||
t.Fatalf("second = %d %s, want 429 mail_rate_limited", w.Code, c)
|
||||
}
|
||||
if mailer.calls != 1 {
|
||||
t.Fatalf("mails = %d", mailer.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCooldownLimiterForgetsExpiredKeys(t *testing.T) {
|
||||
clock := time.Unix(1_700_000_000, 0)
|
||||
c := &cooldownLimiter{now: func() time.Time { return clock }, last: map[string]time.Time{}}
|
||||
for i := 0; i < 500; i++ {
|
||||
if _, ok := c.reserve(fmt.Sprintf("login:email:user%[email protected]", i), time.Minute); !ok {
|
||||
t.Fatalf("reserve %d refused", i)
|
||||
}
|
||||
}
|
||||
if _, ok := c.reserve("login:email:[email protected]", time.Minute); ok {
|
||||
t.Fatal("a live reservation was forgotten")
|
||||
}
|
||||
clock = clock.Add(time.Minute + bucketSweepEvery)
|
||||
c.reserve("login:email:[email protected]", time.Minute)
|
||||
if n := len(c.last); n != 1 {
|
||||
t.Fatalf("after every window ended the map holds %d keys, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMailLimitCountsNotices(t *testing.T) {
|
||||
// The lock notice spends the same budget as codes; with none left it is
|
||||
// skipped rather than sent.
|
||||
api, _, _ := seedLoginEmailAPI(t)
|
||||
mailer := ¬iceMailer{}
|
||||
api.Mailer = mailer
|
||||
api.MailLimit = RateLimit{Burst: 1, PerMinute: 1}
|
||||
api.mailGate().take(mailGateKey)
|
||||
r := httptest.NewRequest("POST", "/", strings.NewReader(""))
|
||||
api.noteOTPLock(r, &OTPAccountLockedError{Until: api.now().Add(time.Hour), JustLocked: true}, "u1", otpPurposeLogin)
|
||||
if len(mailer.notices) != 0 {
|
||||
t.Fatalf("notice sent past a spent budget: %q", mailer.notices)
|
||||
}
|
||||
}
|
||||
@@ -71,8 +71,17 @@ type SMTPConfig struct {
|
||||
// Username is the AUTH identity; empty means the relay needs no AUTH.
|
||||
Username string `toml:"username"`
|
||||
PasswordRef string `toml:"password_ref"`
|
||||
// MaxPerHour caps the mail the API sends install-wide (codes and notices),
|
||||
// so a flood cannot spend the relay's quota and get the account suspended.
|
||||
// 0 means DefaultMailPerHour. Size it to the relay's own limit.
|
||||
MaxPerHour int `toml:"max_per_hour"`
|
||||
}
|
||||
|
||||
// DefaultMailPerHour is the install-wide mail cap when smtp.max_per_hour is
|
||||
// unset: far above a community's normal sign-in mail, far below the daily
|
||||
// quota of common relays.
|
||||
const DefaultMailPerHour = 120
|
||||
|
||||
// ServerConfig is the [server] table.
|
||||
type ServerConfig struct {
|
||||
Listen string `toml:"listen"`
|
||||
@@ -106,6 +115,24 @@ type AuthConfig struct {
|
||||
AdminHostname string `toml:"admin_hostname"`
|
||||
PanelHostname string `toml:"panel_hostname"`
|
||||
AccessJWTAud string `toml:"access_jwt_aud"`
|
||||
// ClientIPHeader names the header the edge writes the visitor's address
|
||||
// into: CF-Connecting-IP behind the Cloudflare tunnel (the edge setup
|
||||
// writes it), X-Forwarded-For behind an operator's reverse proxy. The API
|
||||
// keys its per-client sign-in rate limit on it. Empty means the TCP peer,
|
||||
// except that an install with an Access audience (set only by the
|
||||
// Cloudflare edge setup) implies CF-Connecting-IP.
|
||||
ClientIPHeader string `toml:"client_ip_header"`
|
||||
}
|
||||
|
||||
// EffectiveClientIPHeader resolves ClientIPHeader with its Cloudflare default.
|
||||
func (a AuthConfig) EffectiveClientIPHeader() string {
|
||||
if a.ClientIPHeader != "" {
|
||||
return a.ClientIPHeader
|
||||
}
|
||||
if a.AccessJWTAud != "" {
|
||||
return "CF-Connecting-IP"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// K8sConfig is the [k8s] table.
|
||||
@@ -368,6 +395,12 @@ func (c *Config) Validate() error {
|
||||
return fmt.Errorf("config: [smtp] port %d must be 1-65535 (587 STARTTLS, 465 implicit TLS)", c.SMTP.Port)
|
||||
}
|
||||
}
|
||||
if c.SMTP.MaxPerHour < 0 {
|
||||
return fmt.Errorf("config: [smtp] max_per_hour %d must be positive (0 means the default %d)", c.SMTP.MaxPerHour, DefaultMailPerHour)
|
||||
}
|
||||
if h := c.Auth.ClientIPHeader; strings.ContainsAny(h, " :\t\r\n") {
|
||||
return fmt.Errorf("config: [auth] client_ip_header %q must be a bare header name such as CF-Connecting-IP or X-Forwarded-For", h)
|
||||
}
|
||||
return c.validateAuthSources()
|
||||
}
|
||||
|
||||
|
||||
@@ -574,3 +574,38 @@ host = "smtp.example.net"
|
||||
t.Fatal("expected error when [smtp] host is set without a from address")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClientIPHeaderAndMailCap pins the two knobs behind the sign-in rate
|
||||
// limits: the visitor-address header (explicit, or implied by an Access
|
||||
// audience that only the Cloudflare edge setup writes) and the mail cap.
|
||||
func TestClientIPHeaderAndMailCap(t *testing.T) {
|
||||
base := `
|
||||
[server]
|
||||
root_domain = "mc.example.net"
|
||||
[database]
|
||||
url = "postgres://felis@db/felis"
|
||||
`
|
||||
for _, tc := range []struct {
|
||||
name, auth, want string
|
||||
}{
|
||||
{"unset", "", ""},
|
||||
{"cloudflare audience implies CF-Connecting-IP", "access_jwt_aud = \"aud123\"\n", "CF-Connecting-IP"},
|
||||
{"explicit wins", "access_jwt_aud = \"aud123\"\nclient_ip_header = \"X-Forwarded-For\"\n", "X-Forwarded-For"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg, err := config.Load(writeTOML(t, base+"[auth]\n"+tc.auth))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if got := cfg.Auth.EffectiveClientIPHeader(); got != tc.want {
|
||||
t.Fatalf("EffectiveClientIPHeader = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := config.Load(writeTOML(t, base+"[auth]\nclient_ip_header = \"CF-Connecting-IP: 1.2.3.4\"\n")); err == nil {
|
||||
t.Fatal("a header line with a value was accepted as a header name")
|
||||
}
|
||||
if _, err := config.Load(writeTOML(t, base+"[smtp]\nhost = \"smtp.example.net\"\nfrom = \"[email protected]\"\nmax_per_hour = -1\n")); err == nil {
|
||||
t.Fatal("negative max_per_hour accepted")
|
||||
}
|
||||
}
|
||||
@@ -64,8 +64,44 @@ var (
|
||||
Name: "auth_otp_lockouts_total",
|
||||
Help: "Email-code doors locked after too many wrong codes, by purpose.",
|
||||
}, []string{"purpose"})
|
||||
|
||||
// MailTotal counts mail the API tried to send, by kind (otp, notice) and
|
||||
// result: sent, failed (the relay refused it) or throttled (the
|
||||
// install-wide mail budget refused it before it reached the relay).
|
||||
MailTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "mail_total",
|
||||
Help: "Mail the API tried to send, by kind and result (sent, failed, throttled).",
|
||||
}, []string{"kind", "result"})
|
||||
|
||||
// RateLimitedTotal counts requests refused by a volumetric limit, by scope
|
||||
// (auth_door: one client address calling the public sign-in doors too fast).
|
||||
RateLimitedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "rate_limited_total",
|
||||
Help: "Requests refused by a volumetric rate limit, by scope.",
|
||||
}, []string{"scope"})
|
||||
)
|
||||
|
||||
// OTPPurposes are the email-code doors OTPLockoutsTotal is labelled by.
|
||||
var OTPPurposes = []string{"onboard_email", "login_email", "op_login", "migrate_confirm"}
|
||||
|
||||
// The sign-in alerts watch these counters with increase(). A labelled child
|
||||
// that does not exist yet has no sample before its first event, so increase()
|
||||
// would miss exactly the first lockout or throttle; every child the alerts use
|
||||
// is created at zero up front.
|
||||
func init() {
|
||||
for _, kind := range []string{"otp", "notice"} {
|
||||
for _, result := range []string{"sent", "failed", "throttled"} {
|
||||
MailTotal.WithLabelValues(kind, result)
|
||||
}
|
||||
}
|
||||
RateLimitedTotal.WithLabelValues("auth_door")
|
||||
for _, p := range OTPPurposes {
|
||||
OTPLockoutsTotal.WithLabelValues(p)
|
||||
}
|
||||
}
|
||||
|
||||
// SyncServerGauge republishes felis_servers_total from a full snapshot of the
|
||||
// fleet's per-server states. states holds one entry per MinecraftServer the
|
||||
// operator knows about (its desiredState).
|
||||
@@ -97,6 +133,8 @@ func Collectors() []prometheus.Collector {
|
||||
ImageBuildFailuresTotal,
|
||||
ReaperWorldsDeletedTotal,
|
||||
OTPLockoutsTotal,
|
||||
MailTotal,
|
||||
RateLimitedTotal,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -116,3 +116,26 @@ func TestCountersRecordExpectedValues(t *testing.T) {
|
||||
t.Errorf("start_duration_seconds collected %d metrics, want 1 histogram", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The sign-in alerts use increase(), which needs a zero sample before the first
|
||||
// event; every child they watch must be exposed before anything is counted.
|
||||
func TestSignInSeriesStartAtZero(t *testing.T) {
|
||||
want := map[string]int{
|
||||
"felis_mail_total": 6,
|
||||
"felis_rate_limited_total": 1,
|
||||
"felis_auth_otp_lockouts_total": len(OTPPurposes),
|
||||
}
|
||||
for _, c := range []prometheus.Collector{MailTotal, RateLimitedTotal, OTPLockoutsTotal} {
|
||||
reg := prometheus.NewRegistry()
|
||||
reg.MustRegister(c)
|
||||
mfs, err := reg.Gather()
|
||||
if err != nil {
|
||||
t.Fatalf("Gather: %v", err)
|
||||
}
|
||||
for _, mf := range mfs {
|
||||
if n := len(mf.GetMetric()); n < want[mf.GetName()] {
|
||||
t.Errorf("%s exposes %d children, want at least %d", mf.GetName(), n, want[mf.GetName()])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,8 @@
|
||||
"otp_resend_cooldown": "Verification code requested too frequently, please try again later.",
|
||||
"otp_locked": "Too many incorrect attempts, please request a new verification code.",
|
||||
"otp_account_locked": "Too many wrong codes in 24 hours — email codes for this account are paused. Try again later or use a passkey.",
|
||||
"rate_limited": "Too many sign-in attempts from your network — wait a minute and try again.",
|
||||
"mail_rate_limited": "This server is sending too much mail right now, so codes are paused. Try again shortly or use a passkey.",
|
||||
"passkey_challenge_invalid": "Passkey challenge is invalid or expired, please try again.",
|
||||
"invalid_attestation": "Could not verify this Passkey, please try again.",
|
||||
"passkey_already_bound": "This Passkey is already bound to another account.",
|
||||
|
||||
@@ -23,6 +23,8 @@
|
||||
"otp_resend_cooldown": "验证码发送频繁,请稍后再试。",
|
||||
"otp_locked": "验证码错误次数过多,请重新获取验证码。",
|
||||
"otp_account_locked": "24 小时内验证码错误次数已达上限,此账号的邮箱验证码暂时停用,请稍后再试或改用 Passkey。",
|
||||
"rate_limited": "登录请求过于频繁,请稍等一分钟再试。",
|
||||
"mail_rate_limited": "服务器当前发信过多,验证码暂时发不出去,请稍后再试或改用 Passkey。",
|
||||
"passkey_challenge_invalid": "验证挑战无效或已过期,请重试。",
|
||||
"invalid_attestation": "无法验证此 Passkey,请重试。",
|
||||
"passkey_already_bound": "此 Passkey 已被其他账户绑定。",
|
||||
|
||||
@@ -679,6 +679,12 @@ export function humanizeError(e: unknown): string {
|
||||
return t("otp_locked");
|
||||
case "otp_account_locked":
|
||||
return t("otp_account_locked");
|
||||
// Volumetric limits on the sign-in doors (internal/api/ratelimit.go): one
|
||||
// network calling too fast, or the install-wide mail budget spent.
|
||||
case "rate_limited":
|
||||
return t("rate_limited");
|
||||
case "mail_rate_limited":
|
||||
return t("mail_rate_limited");
|
||||
case "passkey_challenge_invalid":
|
||||
return t("passkey_challenge_invalid");
|
||||
case "invalid_attestation":
|
||||
|
||||
Reference in new issue
Block a user