fix(submit): bound the untrusted upload lane — per-user caps + throttles (#75)

A logged-in user could file submissions without bound and stream a 1 GiB
context per submission. The only limits were the single-blob size cap and the
5 GiB uploads PVC (platform/workloads.go); nothing counted a user's rows or
bytes, so one account could fill the volume and every other user's upload
would start failing.

- Create: per-user pending_review cap (default 5) — the review queue cannot
  be parked full of one account's rows. Check-then-insert, documented soft.
- UploadContext: per-user stored-context budget (default 2 GiB) charged
  against the blob store's REAL sizes (new Blobs.Size on local/S3 stores), so
  the sum cannot drift from the volume; the write is capped at the remaining
  budget, so the excess is refused before it is persisted, and a re-upload is
  charged only for its new bytes.
- API: per-user create/upload throttles (30s/15s, cmd/felis-wired) on a
  dedicated cooldown keyspace, reserve→release so a failed attempt never
  burns the window and a burst collapses to one winner; ErrQuotaExceeded →
  403 submission_quota_exceeded (distinct from the 400 an oversize blob
  gets), 429 submission_cooldown for the throttles.
- Panel: zh/en copy for both codes; openapi documents 403/429 on the two
  user routes; pgint covers the pending-queue count.

Unit tests: submit package (cap, budget boundary/exact-fit/replacement,
oversize-vs-quota split) and api handlers (quota 403 both paths, throttle
429 + recovery + failure-release). go vet/go test/gofmt clean; panel
vitest 118 + typecheck green.
This commit is contained in:
Lemon-miaow committed 2026-09-24 10:14:42 +08:00
1 parent 3ed8bd7be9
commit ad4d256d8f
15 files changed
+577 -15

No files matched your search

+26
View File
@@ -123,6 +123,15 @@ type API struct {
// on the wake lever). Zero disables throttling.
WakeCooldown time.Duration
// SubmitCreateCooldown / SubmitUploadCooldown throttle the user-modpack
// submission lane per user: create bounds how quickly review-queue rows can
// appear, upload bounds how often a user may stream a (up to 1 GiB) build
// context. The keys are separate, so the lane's normal shape — create, then
// upload — is never blocked by its own throttle. Zero disables each lever
// (the same idiom as WakeCooldown); cmd/felis wires positive values.
SubmitCreateCooldown time.Duration
SubmitUploadCooldown time.Duration
// MaxRunningServers caps how many servers may be desired-Running cluster-wide
// (spec §9.1: the concurrency-上限 lever hanging on the same wake chokepoint as
// cooldown and autostartPolicy). Zero — the default — disables it: §9.2 wires
@@ -158,6 +167,9 @@ type API struct {
otpCooldownOnce sync.Once
otpCooldown *cooldownLimiter
submitCooldownOnce sync.Once
submitCooldown *cooldownLimiter
streamCapOnce sync.Once
streamCap *streamLimiter
}
@@ -204,6 +216,20 @@ func (a *API) otpLimiter() *cooldownLimiter {
return a.otpCooldown
}
// submitLimiter lazily builds a SEPARATE cooldown limiter for the user-modpack
// submission lane, so its throttles never share state with the wake or OTP
// keyspaces. One limiter backs both levers with prefixed keys (see the
// submissionCreateKey/UploadKey constants), so create and upload never contend
// with each other. Like the other cooldowns it is process-local; with multiple
// api replicas the effective spacing is per-replica, the same accepted
// KNOWN-LIMITATION the OTP resend throttle carries.
func (a *API) submitLimiter() *cooldownLimiter {
a.submitCooldownOnce.Do(func() {
a.submitCooldown = &cooldownLimiter{now: a.now, last: map[string]time.Time{}}
})
return a.submitCooldown
}
// streamGate lazily builds the per-principal SSE stream cap bound to
// MaxStreamsPerPrincipal. A zero cap yields a disabled limiter that admits every
// stream, so a deployment (or test) that leaves it unset pays nothing.
+59 -3
View File
@@ -59,6 +59,15 @@ type createSubmissionRequest struct {
DisplayName string `json:"display_name"`
}
// The submission lane's two cooldown keys, prefixed into the shared submit
// limiter's per-user keys. Create and upload are separate levers on purpose:
// creating a submission and then immediately uploading its context is the lane's
// normal shape, so one must never consume the other's window.
const (
submissionCreateKey = "create:"
submissionUploadKey = "upload:"
)
// rejectSubmissionRequest is the POST /submissions/{id}/reject body. A reason is
// required (the submit layer rejects an empty one with 400).
type rejectSubmissionRequest struct {
@@ -74,6 +83,26 @@ func (a *API) handleCreateSubmission(w http.ResponseWriter, r *http.Request) {
return
}
p := principalFromContext(r.Context())
// Reserve the per-user create cooldown BEFORE the store write. Unlike the
// wake lever's allowed→record (whose real gate is the running cap and whose
// effect is idempotent), a create is a non-idempotent row insertion with no
// other bound on its rate, so a burst of truly concurrent creates must yield
// exactly one winner per window. The deferred rollback frees the window
// whenever the create fails — a 400 typo, a spent quota, a store error — so
// only a row that was actually recorded consumes it.
lim := a.submitLimiter()
reservedAt, ok := lim.reserve(submissionCreateKey+p.UserID, a.SubmitCreateCooldown)
if !ok {
writeError(w, r, newError(http.StatusTooManyRequests, "submission_cooldown",
"a submission was created recently; wait a moment before creating another"))
return
}
committed := false
defer func() {
if !committed {
lim.release(submissionCreateKey+p.UserID, reservedAt)
}
}()
var body createSubmissionRequest
if err := decodeJSON(w, r, &body); err != nil {
writeError(w, r, err)
@@ -87,6 +116,7 @@ func (a *API) handleCreateSubmission(w http.ResponseWriter, r *http.Request) {
writeSubmitError(w, r, err)
return
}
committed = true
a.audit(r, p.Email, "submission.create", sub.ID)
writeJSON(w, http.StatusCreated, sub)
}
@@ -107,12 +137,34 @@ func (a *API) handleUploadSubmissionContext(w http.ResponseWriter, r *http.Reque
return
}
p := principalFromContext(r.Context())
// Reserve the per-user upload cooldown BEFORE streaming. The body is the
// expensive part (up to the 1 GiB blob cap), so without a reservation the
// throttle would never bound the resource it exists for: a caller could
// repeatedly start long uploads and abort them. Reserving also collapses the
// lane's parallel overshoot — a burst of concurrent uploads from one user
// yields exactly one admitted stream per replica. The rollback keeps a failed
// upload (aborted transfer, wrong format, spent quota) from burning the
// window, so a legit retry after a genuine failure is not punished.
lim := a.submitLimiter()
reservedAt, ok := lim.reserve(submissionUploadKey+p.UserID, a.SubmitUploadCooldown)
if !ok {
writeError(w, r, newError(http.StatusTooManyRequests, "submission_cooldown",
"an upload was accepted recently; wait a moment before uploading again"))
return
}
committed := false
defer func() {
if !committed {
lim.release(submissionUploadKey+p.UserID, reservedAt)
}
}()
id := r.PathValue("id")
sub, err := a.Submissions.UploadContext(r.Context(), id, p.UserID, r.Body)
if err != nil {
writeSubmitError(w, r, err)
return
}
committed = true
a.audit(r, p.Email, "submission.upload", sub.ID)
writeJSON(w, http.StatusOK, sub)
}
@@ -250,9 +302,10 @@ var errSubmissionsUnavailable = newError(http.StatusServiceUnavailable, "submiss
// writeSubmitError maps submit-package errors onto HTTP status codes. Only the
// business sentinels are client-facing: a validation failure is 400, a missing
// submission is 404, an already-reviewed submission is 409, and an unconfigured
// upload transport is 503 (the store this deployment set has no implemented
// transport — an honest "not available here", not a client error). Everything
// submission is 404, an already-reviewed submission is 409, a spent per-user
// allowance is 403 (the same status the server-resource quota answers with), and
// an unconfigured upload transport is 503 (the store this deployment set has no
// implemented transport — an honest "not available here", not a client error). Everything
// else — including a build.ErrInvalid raised by the pre-CAS build.Validate (a
// platform registry/context MISCONFIGURATION, never client input, since every
// build input is platform-derived) and a post-CAS Submit hand-off failure — is a
@@ -267,6 +320,9 @@ func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) {
case errors.Is(err, submit.ErrAlreadyReviewed):
writeError(w, r, newError(http.StatusConflict, "already_reviewed",
"submission has already been reviewed"))
case errors.Is(err, submit.ErrQuotaExceeded):
writeError(w, r, newError(http.StatusForbidden, "submission_quota_exceeded",
"submission quota reached"))
case errors.Is(err, submit.ErrBlobNotFound):
writeError(w, r, newError(http.StatusNotFound, "not_found", "no context uploaded for this submission"))
case errors.Is(err, submit.ErrUploadsUnavailable):
+110
View File
@@ -9,6 +9,7 @@ import (
"net/http"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/build"
"felis.lolicon.best/internal/submit"
@@ -567,3 +568,112 @@ func TestAdminSubmissionContextRoute(t *testing.T) {
}
})
}
// A spent per-user allowance is 403 submission_quota_exceeded on both the create
// and the upload path — distinctly NOT the 400 a malformed request gets, and not
// the 429 the cooldown answers with.
func TestSubmissionQuotaIs403(t *testing.T) {
t.Run("create", func(t *testing.T) {
fs := &fakeSubmissions{createErr: fmt.Errorf("%w: 5 submissions are already awaiting review", submit.ErrQuotaExceeded)}
w := do(appSubAPI(fs).ExternalHandler(), "POST", "/api/v1/me/submissions", `{"display_name":"Pack"}`, nil)
if w.Code != http.StatusForbidden {
t.Fatalf("code = %d, want 403 (%s)", w.Code, w.Body.String())
}
if got := decodeErr(t, w); got != "submission_quota_exceeded" {
t.Errorf("error code = %q, want submission_quota_exceeded", got)
}
})
t.Run("upload", func(t *testing.T) {
fs := &fakeSubmissions{uploadErr: fmt.Errorf("%w: exceeds your remaining storage allowance", submit.ErrQuotaExceeded)}
w := do(appSubAPI(fs).ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", nil)
if w.Code != http.StatusForbidden {
t.Fatalf("code = %d, want 403 (%s)", w.Code, w.Body.String())
}
if got := decodeErr(t, w); got != "submission_quota_exceeded" {
t.Errorf("error code = %q, want submission_quota_exceeded", got)
}
})
}
// The per-user create cooldown bounds review-queue growth: a second create in
// the same window is 429 submission_cooldown and never reaches the service; the
// window recovers afterwards.
func TestCreateSubmissionRateLimited(t *testing.T) {
fs := &fakeSubmissions{}
api := appSubAPI(fs)
clock := time.Unix(1_700_000_000, 0)
api.Now = func() time.Time { return clock }
api.SubmitCreateCooldown = time.Minute
eh := api.ExternalHandler()
if w := do(eh, "POST", "/api/v1/me/submissions", `{"display_name":"First"}`, nil); w.Code != http.StatusCreated {
t.Fatalf("first create: code = %d, want 201 (%s)", w.Code, w.Body.String())
}
w := do(eh, "POST", "/api/v1/me/submissions", `{"display_name":"Second"}`, nil)
if w.Code != http.StatusTooManyRequests || decodeErr(t, w) != "submission_cooldown" {
t.Fatalf("immediate second create: code = %d body %s, want 429 submission_cooldown", w.Code, w.Body.String())
}
// The gate sits before the body handling: even a malformed request is
// refused while the window is closed, so it cannot be used to probe.
if w := do(eh, "POST", "/api/v1/me/submissions", `{`, nil); w.Code != http.StatusTooManyRequests {
t.Fatalf("malformed create during cooldown: code = %d, want 429", w.Code)
}
clock = clock.Add(time.Minute + time.Second)
if w := do(eh, "POST", "/api/v1/me/submissions", `{"display_name":"Third"}`, nil); w.Code != http.StatusCreated {
t.Fatalf("post-cooldown create: code = %d, want 201 (%s)", w.Code, w.Body.String())
}
}
// A failed create frees the window: only a row that was actually recorded burns
// the cooldown, so a validation typo is not punished with a wait.
func TestCreateSubmissionFailureDoesNotBurnCooldown(t *testing.T) {
fs := &fakeSubmissions{createErr: fmt.Errorf("%w: display name is required", submit.ErrInvalid)}
api := appSubAPI(fs)
api.Now = func() time.Time { return time.Unix(1_700_000_000, 0) }
api.SubmitCreateCooldown = time.Minute
eh := api.ExternalHandler()
if w := do(eh, "POST", "/api/v1/me/submissions", `{"display_name":""}`, nil); w.Code != http.StatusBadRequest {
t.Fatalf("failed create: code = %d, want 400", w.Code)
}
fs.createErr = nil
if w := do(eh, "POST", "/api/v1/me/submissions", `{"display_name":"Fixed"}`, nil); w.Code != http.StatusCreated {
t.Fatalf("retry at the same instant: code = %d, want 201 (%s)", w.Code, w.Body.String())
}
}
// The per-user upload cooldown bounds context streaming: a second upload in the
// same window is 429 submission_cooldown, and a FAILED upload frees the window
// for an immediate retry.
func TestUploadSubmissionContextRateLimited(t *testing.T) {
fs := &fakeSubmissions{}
api := appSubAPI(fs)
clock := time.Unix(1_700_000_000, 0)
api.Now = func() time.Time { return clock }
api.SubmitUploadCooldown = time.Minute
eh := api.ExternalHandler()
body := "\x1f\x8b\x08\x00 the modpack bytes"
if w := do(eh, "POST", "/api/v1/me/submissions/sub-9/context", body, nil); w.Code != http.StatusOK {
t.Fatalf("first upload: code = %d, want 200 (%s)", w.Code, w.Body.String())
}
w := do(eh, "POST", "/api/v1/me/submissions/sub-9/context", body, nil)
if w.Code != http.StatusTooManyRequests || decodeErr(t, w) != "submission_cooldown" {
t.Fatalf("immediate second upload: code = %d body %s, want 429 submission_cooldown", w.Code, w.Body.String())
}
// A failed upload releases its reservation, so the user is not punished for
// a genuine failure (aborted transfer, spent quota) with a cooldown wait.
fs2 := &fakeSubmissions{uploadErr: submit.ErrUploadsUnavailable}
api2 := appSubAPI(fs2)
api2.Now = func() time.Time { return clock }
api2.SubmitUploadCooldown = time.Minute
eh2 := api2.ExternalHandler()
if w := do(eh2, "POST", "/api/v1/me/submissions/sub-9/context", body, nil); w.Code != http.StatusServiceUnavailable {
t.Fatalf("failed upload: code = %d, want 503", w.Code)
}
fs2.uploadErr = nil
if w := do(eh2, "POST", "/api/v1/me/submissions/sub-9/context", body, nil); w.Code != http.StatusOK {
t.Fatalf("retry at the same instant after failure: code = %d, want 200 (%s)", w.Code, w.Body.String())
}
}