feat(submit): 模组上传改为分片续传并显示进度，经 Cloudflare 边缘也能传满 1GiB (7bf81a09) · Commits · FelisMC / Felis · GitLab

cmd/felis/api.go

+33 −15

Changes for cmd/felis/api.go: 33 added lines, 15 removed lines.
Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@ import (
		"log/slog"
		"net/http"
		"os"
		"path/filepath"
		"regexp"
		"strings"
		"sync"
		@@ -229,6 +230,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		ContextBaseURL: internalAPIBaseURL(),
		Blobs: blobs,
		}
		if blobs != nil {
		submissions.Parts = &submit.PartStore{Dir: uploadPartsDir(contextBase)}
		}
		if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
		if n, err := parseByteSize(v); err != nil \|\| n <= 0 {
		fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
		@@ -237,7 +241,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		}
		}
		if n, err := contextMaxBytes(cfg); err != nil {
		fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 95Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
		fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 512Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
		} else {
		submissions.MaxContextBytes = n
		}
		@@ -700,9 +704,10 @@ func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
		}

		// reapRejectedContexts deletes, once an hour, the uploaded contexts of
		// submissions rejected more than submit.RejectedContextRetention ago. Without it a
		// rejected modpack keeps its bytes on the uploads store (and against its
		// submitter's budget) until an admin deletes the row.
		// submissions rejected more than submit.RejectedContextRetention ago, and the
		// chunked uploads left untouched for submit.StalePartRetention. Without it a
		// rejected modpack or an abandoned upload keeps its bytes on the uploads store
		// (and against its submitter's budget) until an admin deletes the row.
		func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
		t := time.NewTicker(time.Hour)
		defer t.Stop()
		@@ -714,6 +719,13 @@ func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writ
		if n > 0 {
		fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
		}
		n, err = m.ReapStaleParts(submit.StalePartRetention)
		if err != nil {
		fmt.Fprintf(stderr, "felis api: reap abandoned uploads: %v\n", err)
		}
		if n > 0 {
		fmt.Fprintf(stderr, "felis api: deleted %d abandoned chunked upload(s)\n", n)
		}
		select {
		case <-ctx.Done():
		return
		@@ -886,20 +898,26 @@ func startServerCache(ctx context.Context, cfg rest.Config, scheme runtime.Sch
		return c, inf.HasSynced, nil
		}

		// cloudflareContextMaxBytes is the per-upload cap behind the Cloudflare edge,
		// which refuses request bodies over 100 MB (the Free and Pro plan limit) with
		// its own 413 page before they reach the API. 95Mi leaves headroom under it, so
		// an oversized context meets the API's own JSON refusal instead.
		const cloudflareContextMaxBytes = "95Mi"
		// uploadPartsDir is where chunked uploads are staged: beside a local store's
		// contexts, so the room check and the budget see one disk and a staged upload
		// survives an API restart; for an s3:// store, on the uploads volume the
		// platform mounts either way, or the pod's /tmp when run by hand without it.
		func uploadPartsDir(contextBase string) string {
		if isLocalUploadsPath(contextBase) {
		return filepath.Join(strings.TrimPrefix(contextBase, "file://"), ".parts")
		}
		if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
		return filepath.Join(platform.UploadsLocalPath, ".parts")
		}
		return filepath.Join(os.TempDir(), "felis-upload-parts")
		}

		// contextMaxBytes resolves [registry] context_max_bytes, defaulting to
		// cloudflareContextMaxBytes behind the Cloudflare edge. 0 keeps the submit
		// package's own default (1 GiB).
		// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
		// package's own default (1 GiB). The Cloudflare edge refuses a single request
		// body over 100 MB, which the panel's chunked upload stays under, so the edge
		// does not lower the cap.
		func contextMaxBytes(cfg *config.Config) (int64, error) {
		v := cfg.Registry.ContextMaxBytes
		if v == "" && cfg.Auth.BehindCloudflare() {
		v = cloudflareContextMaxBytes
		}
		if v == "" {
		return 0, nil
		}

cmd/felis/context_max_test.go

+25 −5

Changes for cmd/felis/context_max_test.go: 25 added lines, 5 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"os"
		"path/filepath"
		"testing"

		"felis.lolicon.best/internal/config"
		)

		func TestContextMaxBytesFollowsTheEdge(t *testing.T) {
		func TestContextMaxBytes(t *testing.T) {
		cases := []struct {
		name string
		reg config.RegistryConfig
		@@ -15,10 +17,11 @@ func TestContextMaxBytesFollowsTheEdge(t *testing.T) {
		wantErr bool
		}{
		{name: "direct install keeps the package default", want: 0},
		{name: "access audience means the Cloudflare edge", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 99614720},
		{name: "CF-Connecting-IP header means the Cloudflare edge", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 99614720},
		{name: "an operator proxy keeps the package default", auth: config.AuthConfig{ClientIPHeader: "X-Forwarded-For"}, want: 0},
		{name: "explicit value wins over the edge default", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
		// The panel uploads in parts under the edge's 100 MB body limit, so the
		// Cloudflare edge keeps the full default.
		{name: "the Cloudflare edge keeps the package default", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 0},
		{name: "CF-Connecting-IP keeps the package default", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 0},
		{name: "explicit value behind the edge", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
		{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
		{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
		{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
		@@ -35,3 +38,20 @@ func TestContextMaxBytesFollowsTheEdge(t *testing.T) {
		})
		}
		}

		func TestUploadPartsDir(t *testing.T) {
		if got := uploadPartsDir("/var/lib/felis/uploads"); got != "/var/lib/felis/uploads/.parts" {
		t.Errorf("local store: parts dir = %q, want beside the contexts", got)
		}
		if got := uploadPartsDir("file:///srv/uploads"); got != "/srv/uploads/.parts" {
		t.Errorf("file:// store: parts dir = %q, want /srv/uploads/.parts", got)
		}
		// This machine has no /var/lib/felis/uploads mount, so an s3:// store falls
		// back to the temp dir.
		if _, err := os.Stat("/var/lib/felis/uploads"); err == nil {
		t.Skip("/var/lib/felis/uploads exists here")
		}
		if got := uploadPartsDir("s3://bucket/uploads"); got != filepath.Join(os.TempDir(), "felis-upload-parts") {
		t.Errorf("s3 store without the uploads mount: parts dir = %q", got)
		}
		}

docs/openapi.yaml

+156 −7

Changes for docs/openapi.yaml: 156 added lines, 7 removed lines.
Original line number	Diff line number	Diff line
		@@ -704,6 +704,22 @@ components:
		enabled: { type: boolean }
		added_at: { type: string, format: date-time }

		ContextUploadProgress:
		type: object
		required: [received, part_max_bytes, max_context_bytes]
		properties:
		received:
		type: integer
		format: int64
		description: Bytes staged so far; the next part starts here.
		part_max_bytes:
		type: integer
		format: int64
		description: The most one part may carry.
		max_context_bytes:
		type: integer
		format: int64
		description: The most the whole context may reach ([registry] context_max_bytes).
		Submission:
		type: object
		description: >-
		@@ -5448,9 +5464,11 @@ paths:
		security: [{ sessionCookie: [] }]
		summary: The per-upload build-context cap
		description: >-
		The effective [registry] context_max_bytes: 1 GiB by default, 95 MiB
		behind the Cloudflare edge (its proxy refuses bodies over 100 MB before
		they reach the API). The panel checks a file against it before upload.
		The effective [registry] context_max_bytes, 1 GiB by default. The
		panel checks a file against it before upload and sends the file through
		the chunked upload (/api/v1/me/submissions/{id}/context/upload), so the
		cap holds behind the Cloudflare edge too, whose proxy refuses a single
		body over 100 MB.
		responses:
		'200':
		description: The cap.
		@@ -5477,10 +5495,12 @@ paths:
		this endpoint cannot upload to or probe another user's submission. Only a
		pending_review submission accepts a context (409 otherwise); a wrong-format
		or oversize body is rejected with 400 (the per-upload cap is [registry]
		context_max_bytes: 1 GiB by default and 95 MiB behind the Cloudflare
		edge, whose proxy refuses bodies over 100 MB with its own HTML 413
		before they reach the API; GET /api/v1/me/submissions/limits reports
		the effective cap so a client can check a file before sending it), and an upload that would push the
		context_max_bytes, 1 GiB by default; GET /api/v1/me/submissions/limits
		reports it so a client can check a file before sending it). This
		request carries the whole context, so behind the Cloudflare edge, whose
		proxy refuses bodies over 100 MB with its own HTML 413 before they reach
		the API, a larger context goes through the chunked upload at
		/api/v1/me/submissions/{id}/context/upload instead. An upload that would push the
		caller past their per-user stored-context budget is refused with 403
		before the excess is persisted. Returns 503 when the deployment's context
		store has no implemented upload transport.
		@@ -5519,6 +5539,135 @@ paths:
		'503':
		$ref: '#/components/responses/ServiceUnavailable'

		/api/v1/me/submissions/{id}/context/upload:
		get:
		tags: [submissions]
		operationId: getContextUpload
		summary: Where your chunked context upload stands (the resume point).
		description: >-
		The chunked form of POST /api/v1/me/submissions/{id}/context, for a
		context larger than one request carries through the edge. received is
		how many bytes are staged: the next part starts there. A client reads it
		before the first part and again after a failed one. Nothing staged reads
		as 0. Same owner scoping as the single upload (404 for another user's
		submission, 409 once reviewed).
		x-felis-face: [external]
		x-felis-tier: app
		security: [{ sessionCookie: [] }]
		parameters:
		- { name: id, in: path, required: true, schema: { type: string } }
		responses:
		'200':
		description: The staged length and the limits a part and the whole must keep.
		content:
		application/json:
		schema: { $ref: '#/components/schemas/ContextUploadProgress' }
		'401':
		$ref: '#/components/responses/Unauthorized'
		'404':
		$ref: '#/components/responses/NotFound'
		'409':
		$ref: '#/components/responses/Conflict'
		'503':
		$ref: '#/components/responses/ServiceUnavailable'
		put:
		tags: [submissions]
		operationId: putContextUploadPart
		summary: Append one part of your chunked context upload.
		description: >-
		The body is the part's raw bytes, at most part_max_bytes (32 MiB).
		offset is where they start: 0 starts the upload over, and anything else
		must equal the staged length, or the answer is 409
		upload_offset_mismatch and the client reads GET for where to resume. The
		first part must open with the gzip magic (400). The staged total meets
		the same context cap (400) and storage budget (403) as a single upload.
		A part that breaks off is cut back off, so the staged bytes are always a
		prefix of the file. One request per upload at a time (409 upload_busy).
		Staged bytes untouched for 24 hours are deleted.
		x-felis-face: [external]
		x-felis-tier: app
		security: [{ sessionCookie: [] }]
		parameters:
		- { name: id, in: path, required: true, schema: { type: string } }
		- { name: offset, in: query, required: true, schema: { type: integer, format: int64, minimum: 0 } }
		requestBody:
		required: true
		content:
		application/octet-stream:
		schema: { type: string, format: binary }
		responses:
		'200':
		description: The part is staged; received is the new length.
		content:
		application/json:
		schema: { $ref: '#/components/schemas/ContextUploadProgress' }
		'400':
		$ref: '#/components/responses/BadRequest'
		'401':
		$ref: '#/components/responses/Unauthorized'
		'403':
		description: >-
		The staged total would exceed the caller's per-user stored-context
		budget (submission_quota_exceeded).
		'404':
		$ref: '#/components/responses/NotFound'
		'409':
		$ref: '#/components/responses/Conflict'
		'413':
		description: The part is larger than part_max_bytes (part_too_large).
		content:
		application/json:
		schema: { $ref: '#/components/schemas/Error' }
		'503':
		$ref: '#/components/responses/ServiceUnavailable'
		'507':
		description: The uploads store is full (uploads_full).
		content:
		application/json:
		schema: { $ref: '#/components/schemas/Error' }

		/api/v1/me/submissions/{id}/context/upload/complete:
		post:
		tags: [submissions]
		operationId: completeContextUpload
		summary: Store your staged chunked upload as the submission's build context.
		description: >-
		Runs every check of POST /api/v1/me/submissions/{id}/context on the
		staged bytes (format, cap, budget, room), records the digest the same
		way, and deletes the staged copy. Holds the same per-user upload
		cooldown (429) and writes the same submission.upload audit event.
		Nothing staged is 400. After a failure the staged bytes stay, for a
		retry.
		x-felis-face: [external]
		x-felis-tier: app
		security: [{ sessionCookie: [] }]
		parameters:
		- { name: id, in: path, required: true, schema: { type: string } }
		responses:
		'200':
		description: Context stored; the submission (unchanged) is returned.
		content:
		application/json:
		schema: { $ref: '#/components/schemas/Submission' }
		'400':
		$ref: '#/components/responses/BadRequest'
		'401':
		$ref: '#/components/responses/Unauthorized'
		'403':
		description: >-
		The context would exceed the caller's per-user stored-context budget
		(submission_quota_exceeded).
		'404':
		$ref: '#/components/responses/NotFound'
		'409':
		$ref: '#/components/responses/Conflict'
		'429':
		description: >-
		An upload was accepted within the per-user cooldown window
		(submission_cooldown).
		'503':
		$ref: '#/components/responses/ServiceUnavailable'

		/api/v1/me/submissions/{id}:
		delete:
		tags: [submissions]

docs/troubleshooting.md

+9 −4

Changes for docs/troubleshooting.md: 9 added lines, 4 removed lines.
Original line number	Diff line number	Diff line
		@@ -606,7 +606,7 @@ build_user_namespaces = "auto" # §8f: auto \| on \| off
		build_runtime_class = "" # §8f: e.g. "gvisor"
		max_concurrent_builds = 2 # §8f: 1-6; later builds queue
		user_uploads_max_bytes = "4Gi" # every user's uploaded contexts together; 507 uploads_full past it
		context_max_bytes = "95Mi" # one uploaded context; empty = 1Gi, or 95Mi behind Cloudflare (edge caps bodies at 100 MB)
		context_max_bytes = "1Gi" # one uploaded context; empty = 1Gi (sent in 32 MiB parts, so the Cloudflare edge's 100 MB body cap does not bind)
		```

		Put them in both `/etc/felis/felis.host.toml` (host-side CLI) and
		@@ -749,9 +749,14 @@ control namespace (or `--registry-namespace`):
		(`FELIS_UPLOADS_STORAGE`, 5Gi) and the world-archive PVC
		(`FELIS_BACKUP_STORAGE`, 10Gi) work the same way; re-running the installer
		keeps an existing claim's size and warns when the variable asks for another.
		One uploaded context is capped by `context_max_bytes` (1Gi, or 95Mi behind
		the Cloudflare edge, whose proxy answers its own 413 page for bodies over
		100 MB; the panel checks the file against it before uploading).
		One uploaded context is capped by `context_max_bytes` (1Gi; the panel checks
		the file against it before uploading). The panel sends a context in parts of
		at most 32 MiB, staged under `.parts/` on the uploads volume, so the
		Cloudflare edge, which answers its own 413 page for request bodies over
		100 MB, never sees a body that large; a dropped connection resumes from the
		staged length, and a staged upload untouched for 24 hours is deleted. The
		staged bytes count toward the budgets below. A script can still POST a whole
		context in one body, which the edge caps at 100 MB.
		Uploaded build contexts are bounded by `user_uploads_max_bytes` (4Gi for all
		users together, §8e), 2 GiB per user, and 10% free space on the volume; past
		any of them an upload answers `507 uploads_full` or `403 submission_quota_exceeded`. A

internal/api/api.go

+7 −0

Changes for internal/api/api.go: 7 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -616,6 +616,13 @@ func (a *API) externalAPIRoutes() []apiRoute {
		// App-tier and owner-scoped (the id must belong to the principal), exactly
		// like the create/list routes above.
		{Method: "POST", Pattern: "/api/v1/me/submissions/{id}/context", h: a.handleUploadSubmissionContext},
		// The chunked form of that upload, for a context larger than one request
		// carries through the edge (Cloudflare refuses bodies over 100 MB): GET
		// reports the staged length (the resume point), PUT ?offset= appends one
		// part, POST .../complete stores the staged whole. Same owner scoping.
		{Method: "GET", Pattern: "/api/v1/me/submissions/{id}/context/upload", h: a.handleContextUploadStatus},
		{Method: "PUT", Pattern: "/api/v1/me/submissions/{id}/context/upload", h: a.handleContextUploadPart},
		{Method: "POST", Pattern: "/api/v1/me/submissions/{id}/context/upload/complete", h: a.handleContextUploadComplete},
		// Withdraw the caller's OWN pending submission: the row and its uploaded
		// context are deleted, freeing the pending slot and storage budget. Same
		// owner-scoping as the upload route — a reviewed submission is frozen (409)