feat(submit): 模组上传改为分片续传并显示进度,经 Cloudflare 边缘也能传满 1GiB
This commit is contained in:
28 files changed
+2543
-154
No files matched your search
+33
-15
@@ -8,6 +8,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -229,6 +230,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
|
|||||||
ContextBaseURL: internalAPIBaseURL(),
|
ContextBaseURL: internalAPIBaseURL(),
|
||||||
Blobs: blobs,
|
Blobs: blobs,
|
||||||
}
|
}
|
||||||
|
if blobs != nil {
|
||||||
|
submissions.Parts = &submit.PartStore{Dir: uploadPartsDir(contextBase)}
|
||||||
|
}
|
||||||
if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
|
if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
|
||||||
if n, err := parseByteSize(v); err != nil || n <= 0 {
|
if n, err := parseByteSize(v); err != nil || n <= 0 {
|
||||||
fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
|
fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
|
||||||
@@ -237,7 +241,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if n, err := contextMaxBytes(cfg); err != nil {
|
if n, err := contextMaxBytes(cfg); err != nil {
|
||||||
fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 95Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
|
fmt.Fprintf(stderr, "felis api: [registry] context_max_bytes %q is not a positive size such as 512Mi; keeping the default\n", cfg.Registry.ContextMaxBytes)
|
||||||
} else {
|
} else {
|
||||||
submissions.MaxContextBytes = n
|
submissions.MaxContextBytes = n
|
||||||
}
|
}
|
||||||
@@ -700,9 +704,10 @@ func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// reapRejectedContexts deletes, once an hour, the uploaded contexts of
|
// reapRejectedContexts deletes, once an hour, the uploaded contexts of
|
||||||
// submissions rejected more than submit.RejectedContextRetention ago. Without it a
|
// submissions rejected more than submit.RejectedContextRetention ago, and the
|
||||||
// rejected modpack keeps its bytes on the uploads store (and against its
|
// chunked uploads left untouched for submit.StalePartRetention. Without it a
|
||||||
// submitter's budget) until an admin deletes the row.
|
// rejected modpack or an abandoned upload keeps its bytes on the uploads store
|
||||||
|
// (and against its submitter's budget) until an admin deletes the row.
|
||||||
func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
|
func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
|
||||||
t := time.NewTicker(time.Hour)
|
t := time.NewTicker(time.Hour)
|
||||||
defer t.Stop()
|
defer t.Stop()
|
||||||
@@ -714,6 +719,13 @@ func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writ
|
|||||||
if n > 0 {
|
if n > 0 {
|
||||||
fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
|
fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
|
||||||
}
|
}
|
||||||
|
n, err = m.ReapStaleParts(submit.StalePartRetention)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "felis api: reap abandoned uploads: %v\n", err)
|
||||||
|
}
|
||||||
|
if n > 0 {
|
||||||
|
fmt.Fprintf(stderr, "felis api: deleted %d abandoned chunked upload(s)\n", n)
|
||||||
|
}
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
@@ -886,20 +898,26 @@ func startServerCache(ctx context.Context, cfg *rest.Config, scheme *runtime.Sch
|
|||||||
return c, inf.HasSynced, nil
|
return c, inf.HasSynced, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// cloudflareContextMaxBytes is the per-upload cap behind the Cloudflare edge,
|
// uploadPartsDir is where chunked uploads are staged: beside a local store's
|
||||||
// which refuses request bodies over 100 MB (the Free and Pro plan limit) with
|
// contexts, so the room check and the budget see one disk and a staged upload
|
||||||
// its own 413 page before they reach the API. 95Mi leaves headroom under it, so
|
// survives an API restart; for an s3:// store, on the uploads volume the
|
||||||
// an oversized context meets the API's own JSON refusal instead.
|
// platform mounts either way, or the pod's /tmp when run by hand without it.
|
||||||
const cloudflareContextMaxBytes = "95Mi"
|
func uploadPartsDir(contextBase string) string {
|
||||||
|
if isLocalUploadsPath(contextBase) {
|
||||||
|
return filepath.Join(strings.TrimPrefix(contextBase, "file://"), ".parts")
|
||||||
|
}
|
||||||
|
if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
|
||||||
|
return filepath.Join(platform.UploadsLocalPath, ".parts")
|
||||||
|
}
|
||||||
|
return filepath.Join(os.TempDir(), "felis-upload-parts")
|
||||||
|
}
|
||||||
|
|
||||||
// contextMaxBytes resolves [registry] context_max_bytes, defaulting to
|
// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
|
||||||
// cloudflareContextMaxBytes behind the Cloudflare edge. 0 keeps the submit
|
// package's own default (1 GiB). The Cloudflare edge refuses a single request
|
||||||
// package's own default (1 GiB).
|
// body over 100 MB, which the panel's chunked upload stays under, so the edge
|
||||||
|
// does not lower the cap.
|
||||||
func contextMaxBytes(cfg *config.Config) (int64, error) {
|
func contextMaxBytes(cfg *config.Config) (int64, error) {
|
||||||
v := cfg.Registry.ContextMaxBytes
|
v := cfg.Registry.ContextMaxBytes
|
||||||
if v == "" && cfg.Auth.BehindCloudflare() {
|
|
||||||
v = cloudflareContextMaxBytes
|
|
||||||
}
|
|
||||||
if v == "" {
|
if v == "" {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,14 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"felis.lolicon.best/internal/config"
|
"felis.lolicon.best/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestContextMaxBytesFollowsTheEdge(t *testing.T) {
|
func TestContextMaxBytes(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
reg config.RegistryConfig
|
reg config.RegistryConfig
|
||||||
@@ -15,10 +17,11 @@ func TestContextMaxBytesFollowsTheEdge(t *testing.T) {
|
|||||||
wantErr bool
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{name: "direct install keeps the package default", want: 0},
|
{name: "direct install keeps the package default", want: 0},
|
||||||
{name: "access audience means the Cloudflare edge", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 99614720},
|
// The panel uploads in parts under the edge's 100 MB body limit, so the
|
||||||
{name: "CF-Connecting-IP header means the Cloudflare edge", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 99614720},
|
// Cloudflare edge keeps the full default.
|
||||||
{name: "an operator proxy keeps the package default", auth: config.AuthConfig{ClientIPHeader: "X-Forwarded-For"}, want: 0},
|
{name: "the Cloudflare edge keeps the package default", auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 0},
|
||||||
{name: "explicit value wins over the edge default", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
|
{name: "CF-Connecting-IP keeps the package default", auth: config.AuthConfig{ClientIPHeader: "cf-connecting-ip"}, want: 0},
|
||||||
|
{name: "explicit value behind the edge", reg: config.RegistryConfig{ContextMaxBytes: "50Mi"}, auth: config.AuthConfig{AccessJWTAud: "aud-1"}, want: 52428800},
|
||||||
{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
|
{name: "explicit value on a direct install", reg: config.RegistryConfig{ContextMaxBytes: "2Gi"}, want: 2147483648},
|
||||||
{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
|
{name: "garbage is refused", reg: config.RegistryConfig{ContextMaxBytes: "lots"}, wantErr: true},
|
||||||
{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
|
{name: "zero is refused", reg: config.RegistryConfig{ContextMaxBytes: "0"}, wantErr: true},
|
||||||
@@ -35,3 +38,20 @@ func TestContextMaxBytesFollowsTheEdge(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestUploadPartsDir(t *testing.T) {
|
||||||
|
if got := uploadPartsDir("/var/lib/felis/uploads"); got != "/var/lib/felis/uploads/.parts" {
|
||||||
|
t.Errorf("local store: parts dir = %q, want beside the contexts", got)
|
||||||
|
}
|
||||||
|
if got := uploadPartsDir("file:///srv/uploads"); got != "/srv/uploads/.parts" {
|
||||||
|
t.Errorf("file:// store: parts dir = %q, want /srv/uploads/.parts", got)
|
||||||
|
}
|
||||||
|
// This machine has no /var/lib/felis/uploads mount, so an s3:// store falls
|
||||||
|
// back to the temp dir.
|
||||||
|
if _, err := os.Stat("/var/lib/felis/uploads"); err == nil {
|
||||||
|
t.Skip("/var/lib/felis/uploads exists here")
|
||||||
|
}
|
||||||
|
if got := uploadPartsDir("s3://bucket/uploads"); got != filepath.Join(os.TempDir(), "felis-upload-parts") {
|
||||||
|
t.Errorf("s3 store without the uploads mount: parts dir = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
+156
-7
@@ -704,6 +704,22 @@ components:
|
|||||||
enabled: { type: boolean }
|
enabled: { type: boolean }
|
||||||
added_at: { type: string, format: date-time }
|
added_at: { type: string, format: date-time }
|
||||||
|
|
||||||
|
ContextUploadProgress:
|
||||||
|
type: object
|
||||||
|
required: [received, part_max_bytes, max_context_bytes]
|
||||||
|
properties:
|
||||||
|
received:
|
||||||
|
type: integer
|
||||||
|
format: int64
|
||||||
|
description: Bytes staged so far; the next part starts here.
|
||||||
|
part_max_bytes:
|
||||||
|
type: integer
|
||||||
|
format: int64
|
||||||
|
description: The most one part may carry.
|
||||||
|
max_context_bytes:
|
||||||
|
type: integer
|
||||||
|
format: int64
|
||||||
|
description: The most the whole context may reach ([registry] context_max_bytes).
|
||||||
Submission:
|
Submission:
|
||||||
type: object
|
type: object
|
||||||
description: >-
|
description: >-
|
||||||
@@ -5448,9 +5464,11 @@ paths:
|
|||||||
security: [{ sessionCookie: [] }]
|
security: [{ sessionCookie: [] }]
|
||||||
summary: The per-upload build-context cap
|
summary: The per-upload build-context cap
|
||||||
description: >-
|
description: >-
|
||||||
The effective [registry] context_max_bytes: 1 GiB by default, 95 MiB
|
The effective [registry] context_max_bytes, 1 GiB by default. The
|
||||||
behind the Cloudflare edge (its proxy refuses bodies over 100 MB before
|
panel checks a file against it before upload and sends the file through
|
||||||
they reach the API). The panel checks a file against it before upload.
|
the chunked upload (/api/v1/me/submissions/{id}/context/upload), so the
|
||||||
|
cap holds behind the Cloudflare edge too, whose proxy refuses a single
|
||||||
|
body over 100 MB.
|
||||||
responses:
|
responses:
|
||||||
'200':
|
'200':
|
||||||
description: The cap.
|
description: The cap.
|
||||||
@@ -5477,10 +5495,12 @@ paths:
|
|||||||
this endpoint cannot upload to or probe another user's submission. Only a
|
this endpoint cannot upload to or probe another user's submission. Only a
|
||||||
pending_review submission accepts a context (409 otherwise); a wrong-format
|
pending_review submission accepts a context (409 otherwise); a wrong-format
|
||||||
or oversize body is rejected with 400 (the per-upload cap is [registry]
|
or oversize body is rejected with 400 (the per-upload cap is [registry]
|
||||||
context_max_bytes: 1 GiB by default and 95 MiB behind the Cloudflare
|
context_max_bytes, 1 GiB by default; GET /api/v1/me/submissions/limits
|
||||||
edge, whose proxy refuses bodies over 100 MB with its own HTML 413
|
reports it so a client can check a file before sending it). This
|
||||||
before they reach the API; GET /api/v1/me/submissions/limits reports
|
request carries the whole context, so behind the Cloudflare edge, whose
|
||||||
the effective cap so a client can check a file before sending it), and an upload that would push the
|
proxy refuses bodies over 100 MB with its own HTML 413 before they reach
|
||||||
|
the API, a larger context goes through the chunked upload at
|
||||||
|
/api/v1/me/submissions/{id}/context/upload instead. An upload that would push the
|
||||||
caller past their per-user stored-context budget is refused with 403
|
caller past their per-user stored-context budget is refused with 403
|
||||||
before the excess is persisted. Returns 503 when the deployment's context
|
before the excess is persisted. Returns 503 when the deployment's context
|
||||||
store has no implemented upload transport.
|
store has no implemented upload transport.
|
||||||
@@ -5519,6 +5539,135 @@ paths:
|
|||||||
'503':
|
'503':
|
||||||
$ref: '#/components/responses/ServiceUnavailable'
|
$ref: '#/components/responses/ServiceUnavailable'
|
||||||
|
|
||||||
|
/api/v1/me/submissions/{id}/context/upload:
|
||||||
|
get:
|
||||||
|
tags: [submissions]
|
||||||
|
operationId: getContextUpload
|
||||||
|
summary: Where your chunked context upload stands (the resume point).
|
||||||
|
description: >-
|
||||||
|
The chunked form of POST /api/v1/me/submissions/{id}/context, for a
|
||||||
|
context larger than one request carries through the edge. received is
|
||||||
|
how many bytes are staged: the next part starts there. A client reads it
|
||||||
|
before the first part and again after a failed one. Nothing staged reads
|
||||||
|
as 0. Same owner scoping as the single upload (404 for another user's
|
||||||
|
submission, 409 once reviewed).
|
||||||
|
x-felis-face: [external]
|
||||||
|
x-felis-tier: app
|
||||||
|
security: [{ sessionCookie: [] }]
|
||||||
|
parameters:
|
||||||
|
- { name: id, in: path, required: true, schema: { type: string } }
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: The staged length and the limits a part and the whole must keep.
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: '#/components/schemas/ContextUploadProgress' }
|
||||||
|
'401':
|
||||||
|
$ref: '#/components/responses/Unauthorized'
|
||||||
|
'404':
|
||||||
|
$ref: '#/components/responses/NotFound'
|
||||||
|
'409':
|
||||||
|
$ref: '#/components/responses/Conflict'
|
||||||
|
'503':
|
||||||
|
$ref: '#/components/responses/ServiceUnavailable'
|
||||||
|
put:
|
||||||
|
tags: [submissions]
|
||||||
|
operationId: putContextUploadPart
|
||||||
|
summary: Append one part of your chunked context upload.
|
||||||
|
description: >-
|
||||||
|
The body is the part's raw bytes, at most part_max_bytes (32 MiB).
|
||||||
|
offset is where they start: 0 starts the upload over, and anything else
|
||||||
|
must equal the staged length, or the answer is 409
|
||||||
|
upload_offset_mismatch and the client reads GET for where to resume. The
|
||||||
|
first part must open with the gzip magic (400). The staged total meets
|
||||||
|
the same context cap (400) and storage budget (403) as a single upload.
|
||||||
|
A part that breaks off is cut back off, so the staged bytes are always a
|
||||||
|
prefix of the file. One request per upload at a time (409 upload_busy).
|
||||||
|
Staged bytes untouched for 24 hours are deleted.
|
||||||
|
x-felis-face: [external]
|
||||||
|
x-felis-tier: app
|
||||||
|
security: [{ sessionCookie: [] }]
|
||||||
|
parameters:
|
||||||
|
- { name: id, in: path, required: true, schema: { type: string } }
|
||||||
|
- { name: offset, in: query, required: true, schema: { type: integer, format: int64, minimum: 0 } }
|
||||||
|
requestBody:
|
||||||
|
required: true
|
||||||
|
content:
|
||||||
|
application/octet-stream:
|
||||||
|
schema: { type: string, format: binary }
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: The part is staged; received is the new length.
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: '#/components/schemas/ContextUploadProgress' }
|
||||||
|
'400':
|
||||||
|
$ref: '#/components/responses/BadRequest'
|
||||||
|
'401':
|
||||||
|
$ref: '#/components/responses/Unauthorized'
|
||||||
|
'403':
|
||||||
|
description: >-
|
||||||
|
The staged total would exceed the caller's per-user stored-context
|
||||||
|
budget (submission_quota_exceeded).
|
||||||
|
'404':
|
||||||
|
$ref: '#/components/responses/NotFound'
|
||||||
|
'409':
|
||||||
|
$ref: '#/components/responses/Conflict'
|
||||||
|
'413':
|
||||||
|
description: The part is larger than part_max_bytes (part_too_large).
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: '#/components/schemas/Error' }
|
||||||
|
'503':
|
||||||
|
$ref: '#/components/responses/ServiceUnavailable'
|
||||||
|
'507':
|
||||||
|
description: The uploads store is full (uploads_full).
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: '#/components/schemas/Error' }
|
||||||
|
|
||||||
|
/api/v1/me/submissions/{id}/context/upload/complete:
|
||||||
|
post:
|
||||||
|
tags: [submissions]
|
||||||
|
operationId: completeContextUpload
|
||||||
|
summary: Store your staged chunked upload as the submission's build context.
|
||||||
|
description: >-
|
||||||
|
Runs every check of POST /api/v1/me/submissions/{id}/context on the
|
||||||
|
staged bytes (format, cap, budget, room), records the digest the same
|
||||||
|
way, and deletes the staged copy. Holds the same per-user upload
|
||||||
|
cooldown (429) and writes the same submission.upload audit event.
|
||||||
|
Nothing staged is 400. After a failure the staged bytes stay, for a
|
||||||
|
retry.
|
||||||
|
x-felis-face: [external]
|
||||||
|
x-felis-tier: app
|
||||||
|
security: [{ sessionCookie: [] }]
|
||||||
|
parameters:
|
||||||
|
- { name: id, in: path, required: true, schema: { type: string } }
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: Context stored; the submission (unchanged) is returned.
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema: { $ref: '#/components/schemas/Submission' }
|
||||||
|
'400':
|
||||||
|
$ref: '#/components/responses/BadRequest'
|
||||||
|
'401':
|
||||||
|
$ref: '#/components/responses/Unauthorized'
|
||||||
|
'403':
|
||||||
|
description: >-
|
||||||
|
The context would exceed the caller's per-user stored-context budget
|
||||||
|
(submission_quota_exceeded).
|
||||||
|
'404':
|
||||||
|
$ref: '#/components/responses/NotFound'
|
||||||
|
'409':
|
||||||
|
$ref: '#/components/responses/Conflict'
|
||||||
|
'429':
|
||||||
|
description: >-
|
||||||
|
An upload was accepted within the per-user cooldown window
|
||||||
|
(submission_cooldown).
|
||||||
|
'503':
|
||||||
|
$ref: '#/components/responses/ServiceUnavailable'
|
||||||
|
|
||||||
/api/v1/me/submissions/{id}:
|
/api/v1/me/submissions/{id}:
|
||||||
delete:
|
delete:
|
||||||
tags: [submissions]
|
tags: [submissions]
|
||||||
|
|||||||
@@ -606,7 +606,7 @@ build_user_namespaces = "auto" # §8f: auto | on | off
|
|||||||
build_runtime_class = "" # §8f: e.g. "gvisor"
|
build_runtime_class = "" # §8f: e.g. "gvisor"
|
||||||
max_concurrent_builds = 2 # §8f: 1-6; later builds queue
|
max_concurrent_builds = 2 # §8f: 1-6; later builds queue
|
||||||
user_uploads_max_bytes = "4Gi" # every user's uploaded contexts together; 507 uploads_full past it
|
user_uploads_max_bytes = "4Gi" # every user's uploaded contexts together; 507 uploads_full past it
|
||||||
context_max_bytes = "95Mi" # one uploaded context; empty = 1Gi, or 95Mi behind Cloudflare (edge caps bodies at 100 MB)
|
context_max_bytes = "1Gi" # one uploaded context; empty = 1Gi (sent in 32 MiB parts, so the Cloudflare edge's 100 MB body cap does not bind)
|
||||||
```
|
```
|
||||||
|
|
||||||
Put them in **both** `/etc/felis/felis.host.toml` (host-side CLI) and
|
Put them in **both** `/etc/felis/felis.host.toml` (host-side CLI) and
|
||||||
@@ -749,9 +749,14 @@ control namespace (or `--registry-namespace`):
|
|||||||
(`FELIS_UPLOADS_STORAGE`, 5Gi) and the world-archive PVC
|
(`FELIS_UPLOADS_STORAGE`, 5Gi) and the world-archive PVC
|
||||||
(`FELIS_BACKUP_STORAGE`, 10Gi) work the same way; re-running the installer
|
(`FELIS_BACKUP_STORAGE`, 10Gi) work the same way; re-running the installer
|
||||||
keeps an existing claim's size and warns when the variable asks for another.
|
keeps an existing claim's size and warns when the variable asks for another.
|
||||||
One uploaded context is capped by `context_max_bytes` (1Gi, or 95Mi behind
|
One uploaded context is capped by `context_max_bytes` (1Gi; the panel checks
|
||||||
the Cloudflare edge, whose proxy answers its own 413 page for bodies over
|
the file against it before uploading). The panel sends a context in parts of
|
||||||
100 MB; the panel checks the file against it before uploading).
|
at most 32 MiB, staged under `.parts/` on the uploads volume, so the
|
||||||
|
Cloudflare edge, which answers its own 413 page for request bodies over
|
||||||
|
100 MB, never sees a body that large; a dropped connection resumes from the
|
||||||
|
staged length, and a staged upload untouched for 24 hours is deleted. The
|
||||||
|
staged bytes count toward the budgets below. A script can still POST a whole
|
||||||
|
context in one body, which the edge caps at 100 MB.
|
||||||
Uploaded build contexts are bounded by `user_uploads_max_bytes` (4Gi for all
|
Uploaded build contexts are bounded by `user_uploads_max_bytes` (4Gi for all
|
||||||
users together, §8e), 2 GiB per user, and 10% free space on the volume; past
|
users together, §8e), 2 GiB per user, and 10% free space on the volume; past
|
||||||
any of them an upload answers `507 uploads_full` or `403 submission_quota_exceeded`. A
|
any of them an upload answers `507 uploads_full` or `403 submission_quota_exceeded`. A
|
||||||
|
|||||||
@@ -616,6 +616,13 @@ func (a *API) externalAPIRoutes() []apiRoute {
|
|||||||
// App-tier and owner-scoped (the id must belong to the principal), exactly
|
// App-tier and owner-scoped (the id must belong to the principal), exactly
|
||||||
// like the create/list routes above.
|
// like the create/list routes above.
|
||||||
{Method: "POST", Pattern: "/api/v1/me/submissions/{id}/context", h: a.handleUploadSubmissionContext},
|
{Method: "POST", Pattern: "/api/v1/me/submissions/{id}/context", h: a.handleUploadSubmissionContext},
|
||||||
|
// The chunked form of that upload, for a context larger than one request
|
||||||
|
// carries through the edge (Cloudflare refuses bodies over 100 MB): GET
|
||||||
|
// reports the staged length (the resume point), PUT ?offset= appends one
|
||||||
|
// part, POST .../complete stores the staged whole. Same owner scoping.
|
||||||
|
{Method: "GET", Pattern: "/api/v1/me/submissions/{id}/context/upload", h: a.handleContextUploadStatus},
|
||||||
|
{Method: "PUT", Pattern: "/api/v1/me/submissions/{id}/context/upload", h: a.handleContextUploadPart},
|
||||||
|
{Method: "POST", Pattern: "/api/v1/me/submissions/{id}/context/upload/complete", h: a.handleContextUploadComplete},
|
||||||
// Withdraw the caller's OWN pending submission: the row and its uploaded
|
// Withdraw the caller's OWN pending submission: the row and its uploaded
|
||||||
// context are deleted, freeing the pending slot and storage budget. Same
|
// context are deleted, freeing the pending slot and storage budget. Same
|
||||||
// owner-scoping as the upload route — a reviewed submission is frozen (409)
|
// owner-scoping as the upload route — a reviewed submission is frozen (409)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"felis.lolicon.best/internal/build"
|
"felis.lolicon.best/internal/build"
|
||||||
|
"felis.lolicon.best/internal/submit"
|
||||||
"felis.lolicon.best/internal/updates"
|
"felis.lolicon.best/internal/updates"
|
||||||
"sigs.k8s.io/yaml"
|
"sigs.k8s.io/yaml"
|
||||||
)
|
)
|
||||||
@@ -34,26 +35,27 @@ func TestOpenAPISchemasMatchWireStructs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pairs := map[string]any{
|
pairs := map[string]any{
|
||||||
"ServerInfo": ServerInfo{},
|
"ServerInfo": ServerInfo{},
|
||||||
"FleetServer": fleetServerView{},
|
"FleetServer": fleetServerView{},
|
||||||
"MyServerView": MyServerView{},
|
"MyServerView": MyServerView{},
|
||||||
"BackupView": BackupView{},
|
"BackupView": BackupView{},
|
||||||
"Build": build.Build{},
|
"Build": build.Build{},
|
||||||
"Image": build.Image{},
|
"Image": build.Image{},
|
||||||
"BuildScan": buildScanView{},
|
"BuildScan": buildScanView{},
|
||||||
"ScanSummary": build.ScanSummary{},
|
"ScanSummary": build.ScanSummary{},
|
||||||
"ScanPolicy": build.ScanPolicy{},
|
"ScanPolicy": build.ScanPolicy{},
|
||||||
"ScanFinding": build.ScanFinding{},
|
"ScanFinding": build.ScanFinding{},
|
||||||
"Submission": submissionView{},
|
"Submission": submissionView{},
|
||||||
"UserView": UserView{},
|
"ContextUploadProgress": submit.UploadProgress{},
|
||||||
"UserDetail": UserDetail{},
|
"UserView": UserView{},
|
||||||
"QuotaView": QuotaView{},
|
"UserDetail": UserDetail{},
|
||||||
"SessionView": SessionView{},
|
"QuotaView": QuotaView{},
|
||||||
"PasskeyCredential": passkeyCredentialView{},
|
"SessionView": SessionView{},
|
||||||
"UpdateWindow": updateWindow{},
|
"PasskeyCredential": passkeyCredentialView{},
|
||||||
"DBBackupStatus": dbBackupView{},
|
"UpdateWindow": updateWindow{},
|
||||||
"UpdateReport": updateReportView{},
|
"DBBackupStatus": dbBackupView{},
|
||||||
"UpdateComponent": updates.ComponentStatus{},
|
"UpdateReport": updateReportView{},
|
||||||
|
"UpdateComponent": updates.ComponentStatus{},
|
||||||
}
|
}
|
||||||
for name, v := range pairs {
|
for name, v := range pairs {
|
||||||
s, ok := doc.Components.Schemas[name]
|
s, ok := doc.Components.Schemas[name]
|
||||||
|
|||||||
+108
-11
@@ -36,6 +36,14 @@ type SubmissionService interface {
|
|||||||
// submission at the platform-derived context ref. submittedBy is the principal,
|
// submission at the platform-derived context ref. submittedBy is the principal,
|
||||||
// never the body, so a user can only upload to a submission they own.
|
// never the body, so a user can only upload to a submission they own.
|
||||||
UploadContext(ctx context.Context, id, submittedBy string, r io.Reader) (*submit.Submission, error)
|
UploadContext(ctx context.Context, id, submittedBy string, r io.Reader) (*submit.Submission, error)
|
||||||
|
// UploadStatus, UploadPart and CompleteUpload are the chunked form of
|
||||||
|
// UploadContext, for a context larger than one request carries through the
|
||||||
|
// edge (Cloudflare refuses bodies over 100 MB): parts are staged in order, the
|
||||||
|
// staged length is the resume point, and completion stores the whole through
|
||||||
|
// UploadContext's checks. Same owner scoping.
|
||||||
|
UploadStatus(ctx context.Context, id, submittedBy string) (submit.UploadProgress, error)
|
||||||
|
UploadPart(ctx context.Context, id, submittedBy string, offset int64, r io.Reader) (submit.UploadProgress, error)
|
||||||
|
CompleteUpload(ctx context.Context, id, submittedBy string) (*submit.Submission, error)
|
||||||
// ListBy returns one page of one user's submissions, newest first (the "my
|
// ListBy returns one page of one user's submissions, newest first (the "my
|
||||||
// uploads" view). The scope is submittedBy, whatever opts says.
|
// uploads" view). The scope is submittedBy, whatever opts says.
|
||||||
ListBy(ctx context.Context, submittedBy string, opts submit.ListOpts) (submit.Page, error)
|
ListBy(ctx context.Context, submittedBy string, opts submit.ListOpts) (submit.Page, error)
|
||||||
@@ -173,26 +181,105 @@ func (a *API) handleUploadSubmissionContext(w http.ResponseWriter, r *http.Reque
|
|||||||
// yields exactly one admitted stream per replica. The rollback keeps a failed
|
// yields exactly one admitted stream per replica. The rollback keeps a failed
|
||||||
// upload (aborted transfer, wrong format, spent quota) from burning the
|
// upload (aborted transfer, wrong format, spent quota) from burning the
|
||||||
// window, so a legit retry after a genuine failure is not punished.
|
// window, so a legit retry after a genuine failure is not punished.
|
||||||
lim := a.submitLimiter()
|
commit, release, ok := a.reserveUpload(w, r, p.UserID)
|
||||||
reservedAt, ok := lim.reserve(submissionUploadKey+p.UserID, a.SubmitUploadCooldown)
|
|
||||||
if !ok {
|
if !ok {
|
||||||
writeError(w, r, newError(http.StatusTooManyRequests, "submission_cooldown",
|
|
||||||
"an upload was accepted recently; wait a moment before uploading again"))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
committed := false
|
defer release()
|
||||||
defer func() {
|
|
||||||
if !committed {
|
|
||||||
lim.release(submissionUploadKey+p.UserID, reservedAt)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
id := r.PathValue("id")
|
id := r.PathValue("id")
|
||||||
sub, err := a.Submissions.UploadContext(r.Context(), id, p.UserID, r.Body)
|
sub, err := a.Submissions.UploadContext(r.Context(), id, p.UserID, r.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeSubmitError(w, r, err)
|
writeSubmitError(w, r, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
committed = true
|
commit()
|
||||||
|
a.audit(r, "submission.upload", sub.ID)
|
||||||
|
writeJSON(w, http.StatusOK, sub)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reserveUpload claims the per-user upload cooldown for userID, answering 429
|
||||||
|
// when an upload landed within it. commit keeps the reservation; release, run
|
||||||
|
// deferred, gives it back unless commit ran.
|
||||||
|
func (a *API) reserveUpload(w http.ResponseWriter, r *http.Request, userID string) (commit, release func(), ok bool) {
|
||||||
|
lim := a.submitLimiter()
|
||||||
|
reservedAt, ok := lim.reserve(submissionUploadKey+userID, a.SubmitUploadCooldown)
|
||||||
|
if !ok {
|
||||||
|
writeError(w, r, newError(http.StatusTooManyRequests, "submission_cooldown",
|
||||||
|
"an upload was accepted recently; wait a moment before uploading again"))
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
committed := false
|
||||||
|
return func() { committed = true }, func() {
|
||||||
|
if !committed {
|
||||||
|
lim.release(submissionUploadKey+userID, reservedAt)
|
||||||
|
}
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleContextUploadStatus reports how far the caller's chunked upload of a
|
||||||
|
// submission has come (app-tier, owner-scoped like the single upload). The
|
||||||
|
// panel reads it before the first part and again after a failed one, and sends
|
||||||
|
// the next part from received.
|
||||||
|
func (a *API) handleContextUploadStatus(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.Submissions == nil {
|
||||||
|
writeError(w, r, errSubmissionsUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p := principalFromContext(r.Context())
|
||||||
|
prog, err := a.Submissions.UploadStatus(r.Context(), r.PathValue("id"), p.UserID)
|
||||||
|
if err != nil {
|
||||||
|
writeSubmitError(w, r, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, prog)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleContextUploadPart appends one part of the caller's chunked upload. The
|
||||||
|
// body is the raw bytes; ?offset= is where they start: 0 starts over, anything
|
||||||
|
// else must equal the staged length (409 upload_offset_mismatch otherwise). A
|
||||||
|
// part is small enough for any edge, so no cooldown applies here: the staged
|
||||||
|
// total is bounded by the context cap and the storage budget, and completion
|
||||||
|
// holds the cooldown.
|
||||||
|
func (a *API) handleContextUploadPart(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.Submissions == nil {
|
||||||
|
writeError(w, r, errSubmissionsUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
offset, err := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, r, newError(http.StatusBadRequest, "bad_request",
|
||||||
|
"offset must be the byte position the part starts at"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p := principalFromContext(r.Context())
|
||||||
|
prog, err := a.Submissions.UploadPart(r.Context(), r.PathValue("id"), p.UserID, offset, r.Body)
|
||||||
|
if err != nil {
|
||||||
|
writeSubmitError(w, r, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, prog)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleContextUploadComplete stores the caller's staged upload as the
|
||||||
|
// submission's context. It holds the per-user upload cooldown and is audited
|
||||||
|
// like the single upload, since this is where a context lands.
|
||||||
|
func (a *API) handleContextUploadComplete(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.Submissions == nil {
|
||||||
|
writeError(w, r, errSubmissionsUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p := principalFromContext(r.Context())
|
||||||
|
commit, release, ok := a.reserveUpload(w, r, p.UserID)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
sub, err := a.Submissions.CompleteUpload(r.Context(), r.PathValue("id"), p.UserID)
|
||||||
|
if err != nil {
|
||||||
|
writeSubmitError(w, r, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
commit()
|
||||||
a.audit(r, "submission.upload", sub.ID)
|
a.audit(r, "submission.upload", sub.ID)
|
||||||
writeJSON(w, http.StatusOK, sub)
|
writeJSON(w, http.StatusOK, sub)
|
||||||
}
|
}
|
||||||
@@ -431,6 +518,7 @@ var errSubmissionsUnavailable = newError(http.StatusServiceUnavailable, "submiss
|
|||||||
// server-side fault that collapses to 500 via writeError. The lane deliberately
|
// server-side fault that collapses to 500 via writeError. The lane deliberately
|
||||||
// does not surface those as 4xx: the client did nothing wrong.
|
// does not surface those as 4xx: the client did nothing wrong.
|
||||||
func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) {
|
func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) {
|
||||||
|
var mismatch *submit.OffsetMismatchError
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, submit.ErrInvalid):
|
case errors.Is(err, submit.ErrInvalid):
|
||||||
writeError(w, r, newError(http.StatusBadRequest, "bad_request", "%s", err.Error()))
|
writeError(w, r, newError(http.StatusBadRequest, "bad_request", "%s", err.Error()))
|
||||||
@@ -453,6 +541,15 @@ func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) {
|
|||||||
case errors.Is(err, submit.ErrUploadsUnavailable):
|
case errors.Is(err, submit.ErrUploadsUnavailable):
|
||||||
writeError(w, r, newError(http.StatusServiceUnavailable, "uploads_unavailable",
|
writeError(w, r, newError(http.StatusServiceUnavailable, "uploads_unavailable",
|
||||||
"modpack upload transport is not configured"))
|
"modpack upload transport is not configured"))
|
||||||
|
case errors.Is(err, submit.ErrUploadBusy):
|
||||||
|
writeError(w, r, newError(http.StatusConflict, "upload_busy",
|
||||||
|
"another request is still writing this upload; read where it stands and continue from there"))
|
||||||
|
case errors.As(err, &mismatch):
|
||||||
|
writeError(w, r, newError(http.StatusConflict, "upload_offset_mismatch",
|
||||||
|
"the upload holds %d bytes; send the part that starts there", mismatch.Received))
|
||||||
|
case errors.Is(err, submit.ErrPartTooLarge):
|
||||||
|
writeError(w, r, newError(http.StatusRequestEntityTooLarge, "part_too_large",
|
||||||
|
"the part is larger than part_max_bytes in the upload status"))
|
||||||
default:
|
default:
|
||||||
writeError(w, r, err)
|
writeError(w, r, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"felis.lolicon.best/internal/submit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestContextUploadPartForwardsOffsetBodyAndPrincipal(t *testing.T) {
|
||||||
|
fs := &fakeSubmissions{progress: submit.UploadProgress{Received: 8, PartMaxBytes: 33554432, MaxContextBytes: 1073741824}}
|
||||||
|
api := appSubAPI(fs)
|
||||||
|
w := do(api.ExternalHandler(), "PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=4", "abcd",
|
||||||
|
ctHeader("application/octet-stream"))
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("code = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if got := w.Body.String(); got != `{"received":8,"part_max_bytes":33554432,"max_context_bytes":1073741824}`+"\n" {
|
||||||
|
t.Fatalf("body = %s", got)
|
||||||
|
}
|
||||||
|
if fs.chunkID != "sub-9" || fs.chunkBy != "user-7" || fs.partOffset != 4 || fs.partBody != "abcd" {
|
||||||
|
t.Fatalf("forwarded id=%q by=%q offset=%d body=%q, want sub-9 user-7 4 abcd", fs.chunkID, fs.chunkBy, fs.partOffset, fs.partBody)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContextUploadPartNeedsAnOffset(t *testing.T) {
|
||||||
|
for _, target := range []string{
|
||||||
|
"/api/v1/me/submissions/sub-9/context/upload",
|
||||||
|
"/api/v1/me/submissions/sub-9/context/upload?offset=four",
|
||||||
|
} {
|
||||||
|
fs := &fakeSubmissions{}
|
||||||
|
w := do(appSubAPI(fs).ExternalHandler(), "PUT", target, "abcd", nil)
|
||||||
|
if w.Code != http.StatusBadRequest || decodeErr(t, w) != "bad_request" {
|
||||||
|
t.Fatalf("%s: code = %d (%s), want 400 bad_request", target, w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if fs.chunkID != "" {
|
||||||
|
t.Fatalf("%s: the part reached the lane", target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContextUploadStatusReportsTheStagedLength(t *testing.T) {
|
||||||
|
fs := &fakeSubmissions{progress: submit.UploadProgress{Received: 50331648, PartMaxBytes: 33554432, MaxContextBytes: 1073741824}}
|
||||||
|
w := do(appSubAPI(fs).ExternalHandler(), "GET", "/api/v1/me/submissions/sub-9/context/upload", "", nil)
|
||||||
|
if w.Code != http.StatusOK || w.Body.String() != `{"received":50331648,"part_max_bytes":33554432,"max_context_bytes":1073741824}`+"\n" {
|
||||||
|
t.Fatalf("status = %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if fs.chunkID != "sub-9" || fs.chunkBy != "user-7" {
|
||||||
|
t.Fatalf("asked for id=%q by=%q, want sub-9 user-7", fs.chunkID, fs.chunkBy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContextUploadErrors(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
code int
|
||||||
|
want string
|
||||||
|
msg string
|
||||||
|
}{
|
||||||
|
{"offset mismatch", &submit.OffsetMismatchError{Received: 12}, 409, "upload_offset_mismatch", "the upload holds 12 bytes"},
|
||||||
|
{"busy", submit.ErrUploadBusy, 409, "upload_busy", ""},
|
||||||
|
{"part too large", fmt.Errorf("submit: write upload part: %w", submit.ErrPartTooLarge), 413, "part_too_large", ""},
|
||||||
|
{"not owned", submit.ErrNotFound, 404, "not_found", ""},
|
||||||
|
{"no part store", submit.ErrUploadsUnavailable, 503, "uploads_unavailable", ""},
|
||||||
|
} {
|
||||||
|
fs := &fakeSubmissions{chunkErr: tc.err}
|
||||||
|
w := do(appSubAPI(fs).ExternalHandler(), "PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=12", "abcd", nil)
|
||||||
|
if w.Code != tc.code || decodeErr(t, w) != tc.want {
|
||||||
|
t.Errorf("%s: %d %s, want %d %s", tc.name, w.Code, w.Body.String(), tc.code, tc.want)
|
||||||
|
}
|
||||||
|
if tc.msg != "" && !strings.Contains(w.Body.String(), tc.msg) {
|
||||||
|
t.Errorf("%s: body %s does not say %q", tc.name, w.Body.String(), tc.msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Completion is where a context lands: it holds the per-user upload cooldown and
|
||||||
|
// writes the same audit event as the single upload, and a failed completion
|
||||||
|
// gives the cooldown back.
|
||||||
|
func TestContextUploadCompleteHoldsTheCooldownAndAudits(t *testing.T) {
|
||||||
|
repo := newFakeRepo()
|
||||||
|
fs := &fakeSubmissions{}
|
||||||
|
api := appSubAPI(fs)
|
||||||
|
api.Repo = repo
|
||||||
|
api.SubmitUploadCooldown = time.Minute
|
||||||
|
eh := api.ExternalHandler()
|
||||||
|
|
||||||
|
fs.completeErr = submit.ErrUploadsUnavailable
|
||||||
|
if w := do(eh, "POST", "/api/v1/me/submissions/sub-9/context/upload/complete", "", nil); w.Code != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("failed completion: code = %d, want 503 (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
fs.completeErr = nil
|
||||||
|
w := do(eh, "POST", "/api/v1/me/submissions/sub-9/context/upload/complete", "", nil)
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("completion right after a failed one: code = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if fs.chunkID != "sub-9" || fs.chunkBy != "user-7" {
|
||||||
|
t.Fatalf("completed id=%q by=%q, want sub-9 user-7", fs.chunkID, fs.chunkBy)
|
||||||
|
}
|
||||||
|
var audited int
|
||||||
|
for _, e := range repo.audits {
|
||||||
|
if e.Action == "submission.upload" {
|
||||||
|
audited++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if audited != 1 {
|
||||||
|
t.Fatalf("submission.upload audits = %d, want 1 (only the completion that stored): %+v", audited, repo.audits)
|
||||||
|
}
|
||||||
|
w = do(eh, "POST", "/api/v1/me/submissions/sub-9/context/upload/complete", "", nil)
|
||||||
|
if w.Code != http.StatusTooManyRequests || decodeErr(t, w) != "submission_cooldown" {
|
||||||
|
t.Fatalf("second completion in the window: %d %s, want 429 submission_cooldown", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
// A part never waits on the cooldown.
|
||||||
|
if w := do(eh, "PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=0", "\x1f\x8b", nil); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("part inside the cooldown: code = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContextUploadWithoutServiceIs503(t *testing.T) {
|
||||||
|
api := appSubAPI(&fakeSubmissions{})
|
||||||
|
api.Submissions = nil
|
||||||
|
eh := api.ExternalHandler()
|
||||||
|
for _, rq := range [][2]string{
|
||||||
|
{"GET", "/api/v1/me/submissions/sub-9/context/upload"},
|
||||||
|
{"PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=0"},
|
||||||
|
{"POST", "/api/v1/me/submissions/sub-9/context/upload/complete"},
|
||||||
|
} {
|
||||||
|
if w := do(eh, rq[0], rq[1], "", nil); w.Code != http.StatusServiceUnavailable {
|
||||||
|
t.Errorf("%s %s = %d, want 503", rq[0], rq[1], w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -55,6 +55,35 @@ type fakeSubmissions struct {
|
|||||||
|
|
||||||
approvedDigest string
|
approvedDigest string
|
||||||
openDigest string
|
openDigest string
|
||||||
|
|
||||||
|
// The chunked upload: what the handler forwarded, and canned outcomes.
|
||||||
|
chunkID string
|
||||||
|
chunkBy string
|
||||||
|
partOffset int64
|
||||||
|
partBody string
|
||||||
|
progress submit.UploadProgress
|
||||||
|
chunkErr error
|
||||||
|
completeErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeSubmissions) UploadStatus(_ context.Context, id, submittedBy string) (submit.UploadProgress, error) {
|
||||||
|
f.chunkID, f.chunkBy = id, submittedBy
|
||||||
|
return f.progress, f.chunkErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeSubmissions) UploadPart(_ context.Context, id, submittedBy string, offset int64, r io.Reader) (submit.UploadProgress, error) {
|
||||||
|
f.chunkID, f.chunkBy, f.partOffset = id, submittedBy, offset
|
||||||
|
b, _ := io.ReadAll(r)
|
||||||
|
f.partBody = string(b)
|
||||||
|
return f.progress, f.chunkErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeSubmissions) CompleteUpload(_ context.Context, id, submittedBy string) (*submit.Submission, error) {
|
||||||
|
f.chunkID, f.chunkBy = id, submittedBy
|
||||||
|
if f.completeErr != nil {
|
||||||
|
return nil, f.completeErr
|
||||||
|
}
|
||||||
|
return &submit.Submission{ID: id, SubmittedBy: submittedBy, Status: submit.StatusPendingReview}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeSubmissions) Create(_ context.Context, req submit.CreateRequest) (*submit.Submission, error) {
|
func (f *fakeSubmissions) Create(_ context.Context, req submit.CreateRequest) (*submit.Submission, error) {
|
||||||
|
|||||||
@@ -170,13 +170,6 @@ func (a AuthConfig) EffectiveClientIPHeader() string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// BehindCloudflare reports whether requests reach the API through the
|
|
||||||
// Cloudflare edge: an Access audience (set only by the edge setup) or
|
|
||||||
// CF-Connecting-IP as the client address header.
|
|
||||||
func (a AuthConfig) BehindCloudflare() bool {
|
|
||||||
return strings.EqualFold(a.EffectiveClientIPHeader(), "CF-Connecting-IP")
|
|
||||||
}
|
|
||||||
|
|
||||||
// K8sConfig is the [k8s] table.
|
// K8sConfig is the [k8s] table.
|
||||||
type K8sConfig struct {
|
type K8sConfig struct {
|
||||||
Namespace string `toml:"namespace"`
|
Namespace string `toml:"namespace"`
|
||||||
@@ -253,10 +246,10 @@ type RegistryConfig struct {
|
|||||||
// own; this bounds the sum, which on k3s local-path is the only bound, since
|
// own; this bounds the sum, which on k3s local-path is the only bound, since
|
||||||
// the uploads PVC's size is not enforced there. Empty keeps 4Gi.
|
// the uploads PVC's size is not enforced there. Empty keeps 4Gi.
|
||||||
UserUploadsMaxBytes string `toml:"user_uploads_max_bytes"`
|
UserUploadsMaxBytes string `toml:"user_uploads_max_bytes"`
|
||||||
// ContextMaxBytes caps one uploaded build context, as a quantity ("95Mi").
|
// ContextMaxBytes caps one uploaded build context, as a quantity ("512Mi").
|
||||||
// Empty keeps 1Gi, except behind the Cloudflare edge (see BehindCloudflare),
|
// Empty keeps 1Gi. The Cloudflare edge refuses a single request body over
|
||||||
// whose proxy refuses request bodies over 100 MB before they reach the API;
|
// 100 MB; the panel sends a context in 32 MiB parts
|
||||||
// there it keeps 95Mi, so the API's own 413 is what the uploader sees.
|
// (/api/v1/me/submissions/{id}/context/upload), so the cap holds behind it.
|
||||||
ContextMaxBytes string `toml:"context_max_bytes"`
|
ContextMaxBytes string `toml:"context_max_bytes"`
|
||||||
// S3 configures the object-store backend for user_uploads_context when it is an
|
// S3 configures the object-store backend for user_uploads_context when it is an
|
||||||
// s3:// base (the alternative to a local uploads path). It mirrors
|
// s3:// base (the alternative to a local uploads path). It mirrors
|
||||||
|
|||||||
@@ -58,16 +58,21 @@ const DefaultUploadsMinFree = 0.10
|
|||||||
// CheckRoom refuses an upload of up to need bytes that could push Base's
|
// CheckRoom refuses an upload of up to need bytes that could push Base's
|
||||||
// filesystem below its free floor.
|
// filesystem below its free floor.
|
||||||
func (s *LocalContextStore) CheckRoom(need int64) error {
|
func (s *LocalContextStore) CheckRoom(need int64) error {
|
||||||
|
return checkRoom(s.Base, need, s.MinFree)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkRoom refuses need more bytes under dir when they could push its filesystem
|
||||||
|
// below the minFree share (DefaultUploadsMinFree when 0), wrapping ErrUploadsFull.
|
||||||
|
func checkRoom(dir string, need int64, minFree float64) error {
|
||||||
var st syscall.Statfs_t
|
var st syscall.Statfs_t
|
||||||
if err := syscall.Statfs(s.Base, &st); err != nil {
|
if err := syscall.Statfs(dir, &st); err != nil {
|
||||||
return fmt.Errorf("submit: measure the uploads store %s: %w", s.Base, err)
|
return fmt.Errorf("submit: measure the uploads store %s: %w", dir, err)
|
||||||
}
|
}
|
||||||
bsize := uint64(st.Bsize) // uint32 on darwin
|
bsize := uint64(st.Bsize) // uint32 on darwin
|
||||||
total, avail := uint64(st.Blocks)*bsize, uint64(st.Bavail)*bsize
|
total, avail := uint64(st.Blocks)*bsize, uint64(st.Bavail)*bsize
|
||||||
if total == 0 {
|
if total == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
minFree := s.MinFree
|
|
||||||
if minFree <= 0 {
|
if minFree <= 0 {
|
||||||
minFree = DefaultUploadsMinFree
|
minFree = DefaultUploadsMinFree
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,238 @@
|
|||||||
|
package submit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultPartMaxBytes caps one part of a chunked upload. A single request body is
|
||||||
|
// bounded by whatever proxy fronts the API — the Cloudflare edge refuses bodies
|
||||||
|
// over 100 MB on the Free and Pro plans — so a large modpack arrives as a run of
|
||||||
|
// parts. 32 MiB stays well under that limit and keeps a retried part cheap.
|
||||||
|
const DefaultPartMaxBytes = 32 << 20
|
||||||
|
|
||||||
|
// StalePartRetention is how long a staged upload may sit untouched before the
|
||||||
|
// reaper deletes it: long enough to resume after a lost connection or a laptop
|
||||||
|
// lid, short enough that abandoned parts do not hold the uploads volume.
|
||||||
|
const StalePartRetention = 24 * time.Hour
|
||||||
|
|
||||||
|
// partSuffix names a staged upload on disk: {Dir}/{id}.part.
|
||||||
|
const partSuffix = ".part"
|
||||||
|
|
||||||
|
// ErrUploadBusy reports that another request is writing (or completing) the same
|
||||||
|
// staged upload. Parts go in order, one at a time; the API answers 409 and the
|
||||||
|
// client asks where the upload stands before it sends again.
|
||||||
|
var ErrUploadBusy = errors.New("submit: another request is writing this upload")
|
||||||
|
|
||||||
|
// ErrPartTooLarge reports a part longer than the part cap. The API answers 413.
|
||||||
|
var ErrPartTooLarge = errors.New("submit: an upload part exceeds the part size limit")
|
||||||
|
|
||||||
|
// OffsetMismatchError reports a part that does not start where the staged upload
|
||||||
|
// ends. Received is where it does end, so the client resumes from there.
|
||||||
|
type OffsetMismatchError struct {
|
||||||
|
Received int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *OffsetMismatchError) Error() string {
|
||||||
|
return fmt.Sprintf("submit: the upload holds %d bytes; send the part that starts there", e.Received)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PartStore stages a chunked upload until Manager.CompleteUpload hands the
|
||||||
|
// assembled bytes to Blobs. Parts are appended in order to {Dir}/{id}.part, and
|
||||||
|
// the file's length is the resume point: a client that lost its connection asks
|
||||||
|
// for it and carries on from there. The store serializes the requests for one id
|
||||||
|
// in process, which is enough for the single felis-api replica (its Deployment is
|
||||||
|
// Recreate, never two pods at once).
|
||||||
|
type PartStore struct {
|
||||||
|
// Dir holds the staged uploads. cmd/felis puts it on the uploads volume, so a
|
||||||
|
// staged upload survives an API restart and the room check sees the same disk.
|
||||||
|
Dir string
|
||||||
|
// MinFree is the share of Dir's filesystem a part must leave free; 0 uses
|
||||||
|
// DefaultUploadsMinFree.
|
||||||
|
MinFree float64
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
busy map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// hold claims id for one request; the returned func releases it. A second claim
|
||||||
|
// while the first is held fails with ErrUploadBusy.
|
||||||
|
func (s *PartStore) hold(id string) (func(), error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.busy[id] {
|
||||||
|
return nil, ErrUploadBusy
|
||||||
|
}
|
||||||
|
if s.busy == nil {
|
||||||
|
s.busy = map[string]bool{}
|
||||||
|
}
|
||||||
|
s.busy[id] = true
|
||||||
|
return func() {
|
||||||
|
s.mu.Lock()
|
||||||
|
delete(s.busy, id)
|
||||||
|
s.mu.Unlock()
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PartStore) path(id string) (string, error) {
|
||||||
|
if !idRE.MatchString(id) {
|
||||||
|
return "", fmt.Errorf("submit: invalid submission id %q", id)
|
||||||
|
}
|
||||||
|
return filepath.Join(s.Dir, id+partSuffix), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckRoom refuses a part of up to need bytes that could push Dir's filesystem
|
||||||
|
// below its free floor.
|
||||||
|
func (s *PartStore) CheckRoom(need int64) error {
|
||||||
|
if err := os.MkdirAll(s.Dir, 0o750); err != nil {
|
||||||
|
return fmt.Errorf("submit: mkdir upload parts dir: %w", err)
|
||||||
|
}
|
||||||
|
return checkRoom(s.Dir, need, s.MinFree)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size reports how many bytes are staged for id; nothing staged is (0, false, nil).
|
||||||
|
func (s *PartStore) Size(id string) (int64, bool, error) {
|
||||||
|
p, err := s.path(id)
|
||||||
|
if err != nil {
|
||||||
|
return 0, false, err
|
||||||
|
}
|
||||||
|
fi, err := os.Stat(p)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
return fi.Size(), true, nil
|
||||||
|
case os.IsNotExist(err):
|
||||||
|
return 0, false, nil
|
||||||
|
default:
|
||||||
|
return 0, false, fmt.Errorf("submit: stat staged upload: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Append writes r to id's staged upload at offset and returns the new length.
|
||||||
|
// Offset 0 starts the upload over; any other offset must equal the staged length,
|
||||||
|
// or the call fails with *OffsetMismatchError naming it. A part that fails to
|
||||||
|
// arrive whole is cut back off, so the staged bytes are always a prefix of what
|
||||||
|
// the client sent.
|
||||||
|
func (s *PartStore) Append(id string, offset int64, r io.Reader) (int64, error) {
|
||||||
|
p, err := s.path(id)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
release, err := s.hold(id)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
if err := os.MkdirAll(s.Dir, 0o750); err != nil {
|
||||||
|
return 0, fmt.Errorf("submit: mkdir upload parts dir: %w", err)
|
||||||
|
}
|
||||||
|
var f *os.File
|
||||||
|
if offset == 0 {
|
||||||
|
f, err = os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
|
||||||
|
} else {
|
||||||
|
f, err = os.OpenFile(p, os.O_WRONLY, 0)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return 0, &OffsetMismatchError{Received: 0}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("submit: open staged upload: %w", err)
|
||||||
|
}
|
||||||
|
fi, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
f.Close()
|
||||||
|
return 0, fmt.Errorf("submit: stat staged upload: %w", err)
|
||||||
|
}
|
||||||
|
if fi.Size() != offset {
|
||||||
|
f.Close()
|
||||||
|
return 0, &OffsetMismatchError{Received: fi.Size()}
|
||||||
|
}
|
||||||
|
if _, err := f.Seek(offset, io.SeekStart); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return 0, fmt.Errorf("submit: seek staged upload: %w", err)
|
||||||
|
}
|
||||||
|
n, err := io.Copy(f, r)
|
||||||
|
if err == nil {
|
||||||
|
err = f.Sync()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
f.Truncate(offset)
|
||||||
|
f.Close()
|
||||||
|
return 0, fmt.Errorf("submit: write upload part: %w", err)
|
||||||
|
}
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
return 0, fmt.Errorf("submit: close staged upload: %w", err)
|
||||||
|
}
|
||||||
|
return offset + n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// open returns id's staged upload for reading, or an ErrInvalid error when
|
||||||
|
// nothing is staged. The caller must already hold id.
|
||||||
|
func (s *PartStore) open(id string) (*os.File, error) {
|
||||||
|
p, err := s.path(id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
f, err := os.Open(p)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, invalidf("no upload in progress for this submission; send its parts first")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("submit: open staged upload: %w", err)
|
||||||
|
}
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes id's staged upload. Nothing staged is success.
|
||||||
|
func (s *PartStore) Delete(id string) error {
|
||||||
|
p, err := s.path(id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("submit: remove staged upload: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reap deletes every staged upload last written before cutoff, skipping one a
|
||||||
|
// request holds right now. It returns how many it deleted and carries on past
|
||||||
|
// one it cannot delete, reporting the first such error.
|
||||||
|
func (s *PartStore) Reap(cutoff time.Time) (int, error) {
|
||||||
|
entries, err := os.ReadDir(s.Dir)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("submit: list staged uploads: %w", err)
|
||||||
|
}
|
||||||
|
var reaped int
|
||||||
|
var firstErr error
|
||||||
|
for _, e := range entries {
|
||||||
|
id, ok := strings.CutSuffix(e.Name(), partSuffix)
|
||||||
|
if !ok || e.IsDir() || !idRE.MatchString(id) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fi, err := e.Info()
|
||||||
|
if err != nil || !fi.ModTime().Before(cutoff) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
release, err := s.hold(id)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
err = s.Delete(id)
|
||||||
|
release()
|
||||||
|
if err == nil {
|
||||||
|
reaped++
|
||||||
|
} else if firstErr == nil {
|
||||||
|
firstErr = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return reaped, firstErr
|
||||||
|
}
|
||||||
@@ -0,0 +1,338 @@
|
|||||||
|
package submit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newChunkedManager wires a Manager with an in-memory Blobs, a PartStore on a
|
||||||
|
// temp dir and 4-byte parts, and files one pending submission for user-1.
|
||||||
|
func newChunkedManager(t *testing.T) (*Manager, *fakeStore, *fakeBlobs, string) {
|
||||||
|
t.Helper()
|
||||||
|
m, st, _ := newManager()
|
||||||
|
fb := newFakeBlobs()
|
||||||
|
m.Blobs = fb
|
||||||
|
// A near-zero floor keeps the room check off this machine's own disk usage.
|
||||||
|
m.Parts = &PartStore{Dir: t.TempDir(), MinFree: 1e-9}
|
||||||
|
m.PartMaxBytes = 4
|
||||||
|
sub, err := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return m, st, fb, sub.ID
|
||||||
|
}
|
||||||
|
|
||||||
|
func sendPart(t *testing.T, m *Manager, id string, offset int64, part string) int64 {
|
||||||
|
t.Helper()
|
||||||
|
p, err := m.UploadPart(context.Background(), id, "user-1", offset, strings.NewReader(part))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("UploadPart(offset %d, %q): %v", offset, part, err)
|
||||||
|
}
|
||||||
|
return p.Received
|
||||||
|
}
|
||||||
|
|
||||||
|
func staged(t *testing.T, m *Manager, id string) int64 {
|
||||||
|
t.Helper()
|
||||||
|
p, err := m.UploadStatus(context.Background(), id, "user-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("UploadStatus: %v", err)
|
||||||
|
}
|
||||||
|
return p.Received
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkedUploadAssemblesAndStores(t *testing.T) {
|
||||||
|
m, _, fb, id := newChunkedManager(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
p, err := m.UploadStatus(ctx, id, "user-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p != (UploadProgress{Received: 0, PartMaxBytes: 4, MaxContextBytes: 1 << 30}) {
|
||||||
|
t.Fatalf("status before any part = %+v", p)
|
||||||
|
}
|
||||||
|
for _, step := range []struct {
|
||||||
|
offset int64
|
||||||
|
part string
|
||||||
|
want int64
|
||||||
|
}{
|
||||||
|
{0, "\x1f\x8b\x08\x00", 4},
|
||||||
|
{4, "abcd", 8},
|
||||||
|
{8, "efgh", 12},
|
||||||
|
{12, "ij", 14},
|
||||||
|
} {
|
||||||
|
if got := sendPart(t, m, id, step.offset, step.part); got != step.want {
|
||||||
|
t.Fatalf("after the part at %d: received %d, want %d", step.offset, got, step.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, ok := fb.stored[id]; ok {
|
||||||
|
t.Fatal("parts reached the blob store before the upload completed")
|
||||||
|
}
|
||||||
|
|
||||||
|
sub, err := m.CompleteUpload(ctx, id, "user-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CompleteUpload: %v", err)
|
||||||
|
}
|
||||||
|
if got := string(fb.stored[id]); got != "\x1f\x8b\x08\x00abcdefghij" {
|
||||||
|
t.Fatalf("stored %q, want the parts in order", got)
|
||||||
|
}
|
||||||
|
if sub.ContextSHA256 != "df86a051af3d8615de097952ac01bc094781053bcda61aaac848edb98068209d" {
|
||||||
|
t.Fatalf("recorded digest %s", sub.ContextSHA256)
|
||||||
|
}
|
||||||
|
if got := staged(t, m, id); got != 0 {
|
||||||
|
t.Fatalf("staged bytes after completion = %d, want the staged copy gone", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkedUploadResumesWhereTheStagedBytesEnd(t *testing.T) {
|
||||||
|
m, _, _, id := newChunkedManager(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
var mismatch *OffsetMismatchError
|
||||||
|
_, err := m.UploadPart(ctx, id, "user-1", 4, strings.NewReader("abcd"))
|
||||||
|
if !errors.As(err, &mismatch) || mismatch.Received != 0 {
|
||||||
|
t.Fatalf("a part past an empty upload = %v, want an offset mismatch at 0", err)
|
||||||
|
}
|
||||||
|
sendPart(t, m, id, 0, "\x1f\x8b\x08\x00")
|
||||||
|
_, err = m.UploadPart(ctx, id, "user-1", 8, strings.NewReader("efgh"))
|
||||||
|
if !errors.As(err, &mismatch) || mismatch.Received != 4 {
|
||||||
|
t.Fatalf("a part that skips ahead = %v, want an offset mismatch at 4", err)
|
||||||
|
}
|
||||||
|
if got := sendPart(t, m, id, 4, "abcd"); got != 8 {
|
||||||
|
t.Fatalf("the part at the staged length: received %d, want 8", got)
|
||||||
|
}
|
||||||
|
// Offset 0 starts over.
|
||||||
|
if got := sendPart(t, m, id, 0, "\x1f\x8b"); got != 2 {
|
||||||
|
t.Fatalf("restart at 0: received %d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type failingReader struct {
|
||||||
|
data string
|
||||||
|
done bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *failingReader) Read(p []byte) (int, error) {
|
||||||
|
if f.done {
|
||||||
|
return 0, errors.New("connection reset")
|
||||||
|
}
|
||||||
|
f.done = true
|
||||||
|
return copy(p, f.data), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkedUploadCutsBackAPartThatBrokeOff(t *testing.T) {
|
||||||
|
m, _, _, id := newChunkedManager(t)
|
||||||
|
sendPart(t, m, id, 0, "\x1f\x8b\x08\x00")
|
||||||
|
|
||||||
|
_, err := m.UploadPart(context.Background(), id, "user-1", 4, &failingReader{data: "ab"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a part that broke off was accepted")
|
||||||
|
}
|
||||||
|
if got := staged(t, m, id); got != 4 {
|
||||||
|
t.Fatalf("staged after a broken part = %d, want 4 (the half part cut back off)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkedUploadFirstPartMustBeGzip(t *testing.T) {
|
||||||
|
m, _, _, id := newChunkedManager(t)
|
||||||
|
_, err := m.UploadPart(context.Background(), id, "user-1", 0, strings.NewReader("PK\x03\x04"))
|
||||||
|
if !errors.Is(err, ErrInvalid) {
|
||||||
|
t.Fatalf("a zip first part = %v, want ErrInvalid", err)
|
||||||
|
}
|
||||||
|
if got := staged(t, m, id); got != 0 {
|
||||||
|
t.Fatalf("staged after a refused first part = %d, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkedUploadPartCap(t *testing.T) {
|
||||||
|
m, _, _, id := newChunkedManager(t)
|
||||||
|
sendPart(t, m, id, 0, "\x1f\x8b\x08\x00")
|
||||||
|
|
||||||
|
_, err := m.UploadPart(context.Background(), id, "user-1", 4, strings.NewReader("abcde"))
|
||||||
|
if !errors.Is(err, ErrPartTooLarge) {
|
||||||
|
t.Fatalf("a 5-byte part over a 4-byte cap = %v, want ErrPartTooLarge", err)
|
||||||
|
}
|
||||||
|
if got := staged(t, m, id); got != 4 {
|
||||||
|
t.Fatalf("staged after an oversize part = %d, want 4", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The staged total meets the same cap as a whole context, with the same error.
|
||||||
|
func TestChunkedUploadTotalCappedLikeAWholeContext(t *testing.T) {
|
||||||
|
m, _, _, id := newChunkedManager(t)
|
||||||
|
m.MaxContextBytes = 10
|
||||||
|
sendPart(t, m, id, 0, "\x1f\x8b\x08\x00")
|
||||||
|
sendPart(t, m, id, 4, "abcd")
|
||||||
|
|
||||||
|
_, err := m.UploadPart(context.Background(), id, "user-1", 8, strings.NewReader("efg"))
|
||||||
|
if !errors.Is(err, ErrInvalid) || errors.Is(err, ErrPartTooLarge) {
|
||||||
|
t.Fatalf("a part past the context cap = %v, want the context-too-large ErrInvalid", err)
|
||||||
|
}
|
||||||
|
if got := sendPart(t, m, id, 8, "ef"); got != 10 {
|
||||||
|
t.Fatalf("a part ending exactly at the cap: received %d, want 10", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Staged bytes count against the storage budget, so parts parked on several
|
||||||
|
// pending submissions cannot hold more than stored contexts could.
|
||||||
|
func TestChunkedUploadStagedBytesCountAgainstTheBudget(t *testing.T) {
|
||||||
|
m, _, fb, a := newChunkedManager(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
m.MaxStoredBytesPerUser = 10
|
||||||
|
m.PartMaxBytes = 16
|
||||||
|
sendPart(t, m, a, 0, "\x1f\x8b\x08\x00abcd")
|
||||||
|
|
||||||
|
b, err := m.Create(ctx, CreateRequest{DisplayName: "B", SubmittedBy: "user-1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := m.UploadContext(ctx, b.ID, "user-1", strings.NewReader("\x1f\x8b\x08")); !errors.Is(err, ErrQuotaExceeded) {
|
||||||
|
t.Fatalf("3 bytes beside 8 staged under a 10-byte budget = %v, want ErrQuotaExceeded", err)
|
||||||
|
}
|
||||||
|
if _, err := m.UploadContext(ctx, b.ID, "user-1", strings.NewReader("\x1f\x8b")); err != nil {
|
||||||
|
t.Fatalf("2 bytes beside 8 staged = %v, want accepted", err)
|
||||||
|
}
|
||||||
|
// And the other way round: B's stored 2 bytes bound what A may still stage.
|
||||||
|
if _, err := m.UploadPart(ctx, a, "user-1", 8, strings.NewReader("e")); !errors.Is(err, ErrQuotaExceeded) {
|
||||||
|
t.Fatalf("staging past the budget = %v, want ErrQuotaExceeded", err)
|
||||||
|
}
|
||||||
|
if _, ok := fb.stored[a]; ok {
|
||||||
|
t.Fatal("staging stored a blob")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkedUploadOwnerAndStatus(t *testing.T) {
|
||||||
|
m, st, _, id := newChunkedManager(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
sendPart(t, m, id, 0, "\x1f\x8b\x08\x00")
|
||||||
|
|
||||||
|
if _, err := m.UploadStatus(ctx, id, "user-2"); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("another user's status = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
if _, err := m.UploadPart(ctx, id, "user-2", 4, strings.NewReader("abcd")); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("another user's part = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
if _, err := m.CompleteUpload(ctx, id, "user-2"); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("another user's completion = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
st.subs[id].Status = StatusRejected
|
||||||
|
if _, err := m.UploadPart(ctx, id, "user-1", 4, strings.NewReader("abcd")); !errors.Is(err, ErrAlreadyReviewed) {
|
||||||
|
t.Fatalf("a part for a reviewed submission = %v, want ErrAlreadyReviewed", err)
|
||||||
|
}
|
||||||
|
if _, err := m.CompleteUpload(ctx, id, "user-1"); !errors.Is(err, ErrAlreadyReviewed) {
|
||||||
|
t.Fatalf("completing a reviewed submission = %v, want ErrAlreadyReviewed", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkedUploadWithoutPartStoreIsUnavailable(t *testing.T) {
|
||||||
|
m, _, _, id := newChunkedManager(t)
|
||||||
|
m.Parts = nil
|
||||||
|
if _, err := m.UploadStatus(context.Background(), id, "user-1"); !errors.Is(err, ErrUploadsUnavailable) {
|
||||||
|
t.Fatalf("status without a part store = %v, want ErrUploadsUnavailable", err)
|
||||||
|
}
|
||||||
|
if _, err := m.UploadPart(context.Background(), id, "user-1", 0, strings.NewReader("\x1f\x8b")); !errors.Is(err, ErrUploadsUnavailable) {
|
||||||
|
t.Fatalf("part without a part store = %v, want ErrUploadsUnavailable", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompleteUploadWithNothingStagedIsInvalid(t *testing.T) {
|
||||||
|
m, _, fb, id := newChunkedManager(t)
|
||||||
|
if _, err := m.CompleteUpload(context.Background(), id, "user-1"); !errors.Is(err, ErrInvalid) {
|
||||||
|
t.Fatalf("completing an empty upload = %v, want ErrInvalid", err)
|
||||||
|
}
|
||||||
|
if len(fb.stored) != 0 {
|
||||||
|
t.Fatal("an empty completion stored a blob")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompleteUploadFailureKeepsTheStagedBytes(t *testing.T) {
|
||||||
|
m, _, fb, id := newChunkedManager(t)
|
||||||
|
sendPart(t, m, id, 0, "\x1f\x8b\x08\x00")
|
||||||
|
sendPart(t, m, id, 4, "ab")
|
||||||
|
fb.putErr = errors.New("object store unreachable")
|
||||||
|
|
||||||
|
if _, err := m.CompleteUpload(context.Background(), id, "user-1"); err == nil {
|
||||||
|
t.Fatal("completion succeeded with the blob store down")
|
||||||
|
}
|
||||||
|
if got := staged(t, m, id); got != 6 {
|
||||||
|
t.Fatalf("staged after a failed completion = %d, want 6 kept for the retry", got)
|
||||||
|
}
|
||||||
|
fb.putErr = nil
|
||||||
|
if _, err := m.CompleteUpload(context.Background(), id, "user-1"); err != nil {
|
||||||
|
t.Fatalf("retried completion: %v", err)
|
||||||
|
}
|
||||||
|
if got := string(fb.stored[id]); got != "\x1f\x8b\x08\x00ab" {
|
||||||
|
t.Fatalf("stored %q after the retry", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkedUploadOneRequestAtATime(t *testing.T) {
|
||||||
|
m, _, _, id := newChunkedManager(t)
|
||||||
|
sendPart(t, m, id, 0, "\x1f\x8b\x08\x00")
|
||||||
|
release, err := m.Parts.hold(id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := m.UploadPart(context.Background(), id, "user-1", 4, strings.NewReader("ab")); !errors.Is(err, ErrUploadBusy) {
|
||||||
|
t.Fatalf("a part while another is writing = %v, want ErrUploadBusy", err)
|
||||||
|
}
|
||||||
|
if _, err := m.CompleteUpload(context.Background(), id, "user-1"); !errors.Is(err, ErrUploadBusy) {
|
||||||
|
t.Fatalf("completing while a part is writing = %v, want ErrUploadBusy", err)
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
if got := sendPart(t, m, id, 4, "ab"); got != 6 {
|
||||||
|
t.Fatalf("the part after release: received %d, want 6", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithdrawDeletesTheStagedUpload(t *testing.T) {
|
||||||
|
m, _, _, id := newChunkedManager(t)
|
||||||
|
sendPart(t, m, id, 0, "\x1f\x8b\x08\x00")
|
||||||
|
if _, err := m.Withdraw(context.Background(), id, "user-1"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(m.Parts.Dir, id+".part")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("staged upload after withdraw: stat err = %v, want it gone", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReapStaleParts(t *testing.T) {
|
||||||
|
m, _, _, oldID := newChunkedManager(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
fresh, err := m.Create(ctx, CreateRequest{DisplayName: "Fresh", SubmittedBy: "user-1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sendPart(t, m, oldID, 0, "\x1f\x8b\x08\x00")
|
||||||
|
sendPart(t, m, fresh.ID, 0, "\x1f\x8b\x08\x00")
|
||||||
|
if err := os.Chtimes(filepath.Join(m.Parts.Dir, oldID+".part"), testNow.Add(-25*time.Hour), testNow.Add(-25*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chtimes(filepath.Join(m.Parts.Dir, fresh.ID+".part"), testNow.Add(-23*time.Hour), testNow.Add(-23*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A stray file that is not a staged upload is left alone.
|
||||||
|
if err := os.WriteFile(filepath.Join(m.Parts.Dir, "notes"), []byte("x"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
os.Chtimes(filepath.Join(m.Parts.Dir, "notes"), testNow.Add(-48*time.Hour), testNow.Add(-48*time.Hour))
|
||||||
|
|
||||||
|
n, err := m.ReapStaleParts(StalePartRetention)
|
||||||
|
if err != nil || n != 1 {
|
||||||
|
t.Fatalf("ReapStaleParts = %d, %v; want 1", n, err)
|
||||||
|
}
|
||||||
|
if got := staged(t, m, oldID); got != 0 {
|
||||||
|
t.Fatalf("the 25-hour-old upload still holds %d bytes", got)
|
||||||
|
}
|
||||||
|
if got := staged(t, m, fresh.ID); got != 4 {
|
||||||
|
t.Fatalf("the 23-hour-old upload holds %d bytes, want 4 kept", got)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(m.Parts.Dir, "notes")); err != nil {
|
||||||
|
t.Fatalf("the reaper touched a file that is not a staged upload: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+192
-38
@@ -64,6 +64,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -379,6 +380,13 @@ type Manager struct {
|
|||||||
// MaxStoredBytesTotal overrides the budget for every user's stored contexts
|
// MaxStoredBytesTotal overrides the budget for every user's stored contexts
|
||||||
// together; 0 uses defaultMaxStoredBytesTotal.
|
// together; 0 uses defaultMaxStoredBytesTotal.
|
||||||
MaxStoredBytesTotal int64
|
MaxStoredBytesTotal int64
|
||||||
|
// Parts stages chunked uploads (UploadPart, CompleteUpload). Nil leaves only
|
||||||
|
// the single-request UploadContext, and the chunked calls return
|
||||||
|
// ErrUploadsUnavailable.
|
||||||
|
Parts *PartStore
|
||||||
|
// PartMaxBytes overrides the cap on one chunked-upload part; 0 uses
|
||||||
|
// DefaultPartMaxBytes.
|
||||||
|
PartMaxBytes int64
|
||||||
|
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
IDGen func() string
|
IDGen func() string
|
||||||
@@ -394,6 +402,13 @@ func (m *Manager) maxContextBytes() int64 {
|
|||||||
return defaultMaxContextBytes
|
return defaultMaxContextBytes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Manager) partMaxBytes() int64 {
|
||||||
|
if m.PartMaxBytes > 0 {
|
||||||
|
return m.PartMaxBytes
|
||||||
|
}
|
||||||
|
return DefaultPartMaxBytes
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Manager) maxPendingPerUser() int {
|
func (m *Manager) maxPendingPerUser() int {
|
||||||
if m.MaxPendingPerUser > 0 {
|
if m.MaxPendingPerUser > 0 {
|
||||||
return m.MaxPendingPerUser
|
return m.MaxPendingPerUser
|
||||||
@@ -575,18 +590,10 @@ func (m *Manager) UploadContext(ctx context.Context, id, submittedBy string, r i
|
|||||||
if m.Blobs == nil {
|
if m.Blobs == nil {
|
||||||
return nil, ErrUploadsUnavailable
|
return nil, ErrUploadsUnavailable
|
||||||
}
|
}
|
||||||
|
sub, err := m.ownPending(ctx, id, submittedBy)
|
||||||
sub, err := m.Store.GetSubmission(ctx, id)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if sub.SubmittedBy != submittedBy {
|
|
||||||
// Not the owner: invisible, so the endpoint cannot confirm the id exists.
|
|
||||||
return nil, ErrNotFound
|
|
||||||
}
|
|
||||||
if sub.Status != StatusPendingReview {
|
|
||||||
return nil, ErrAlreadyReviewed
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sniff the gzip magic before touching the store so a wrong-format upload fails
|
// Sniff the gzip magic before touching the store so a wrong-format upload fails
|
||||||
// fast, without persisting anything or reading the whole body.
|
// fast, without persisting anything or reading the whole body.
|
||||||
@@ -595,34 +602,10 @@ func (m *Manager) UploadContext(ctx context.Context, id, submittedBy string, r i
|
|||||||
return nil, invalidf("build context must be a gzip-compressed tarball (.tar.gz)")
|
return nil, invalidf("build context must be a gzip-compressed tarball (.tar.gz)")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Per-user storage budget: sum the bytes this user's OTHER submissions
|
limit, over, err := m.uploadLimit(ctx, submittedBy, id)
|
||||||
// already hold (excluding this id, whose blob a re-upload supersedes) and cap
|
|
||||||
// the write at whatever remains. cappedReader trips on the first byte past
|
|
||||||
// the limit, so the store never persists a blob that would exceed the budget
|
|
||||||
// (it removes its temp file on the copy error) and the failure surfaces as a
|
|
||||||
// 403, not a 500. The read-then-write pair is not atomic in this package: a
|
|
||||||
// burst that reaches two api replicas (or any direct caller of the Manager)
|
|
||||||
// can overshoot by up to one blob per interleaved upload — each write still
|
|
||||||
// bounded by the single-blob cap — while the API's per-user upload
|
|
||||||
// reservation collapses the single-replica case.
|
|
||||||
used, total, err := m.storedBytes(ctx, submittedBy, id)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
remaining, budgetErr := m.maxStoredBytesPerUser()-used, errStorageQuota
|
|
||||||
if left := m.maxStoredBytesTotal() - total; left < remaining {
|
|
||||||
remaining, budgetErr = left, fmt.Errorf("%w: every user's uploads together reached the %d-byte limit", ErrUploadsFull, m.maxStoredBytesTotal())
|
|
||||||
}
|
|
||||||
if remaining <= 0 {
|
|
||||||
return nil, budgetErr
|
|
||||||
}
|
|
||||||
limit, over := m.maxContextBytes(), errContextTooLarge
|
|
||||||
if remaining < limit {
|
|
||||||
// The budget binds before the single-blob cap: an upload tripping here is
|
|
||||||
// refused as a spent allowance (or a full store), never as a malformed
|
|
||||||
// request.
|
|
||||||
limit, over = remaining, budgetErr
|
|
||||||
}
|
|
||||||
// The upload's size is unknown until it ends, so the room check assumes the
|
// The upload's size is unknown until it ends, so the room check assumes the
|
||||||
// most it may write.
|
// most it may write.
|
||||||
if rc, ok := m.Blobs.(RoomChecker); ok {
|
if rc, ok := m.Blobs.(RoomChecker); ok {
|
||||||
@@ -648,12 +631,174 @@ func (m *Manager) UploadContext(ctx context.Context, id, submittedBy string, r i
|
|||||||
return sub, nil
|
return sub, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ownPending loads id for an upload by submittedBy: another user's submission is
|
||||||
|
// ErrNotFound (the endpoint cannot confirm the id exists), a reviewed one
|
||||||
|
// ErrAlreadyReviewed.
|
||||||
|
func (m *Manager) ownPending(ctx context.Context, id, submittedBy string) (*Submission, error) {
|
||||||
|
sub, err := m.Store.GetSubmission(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if sub.SubmittedBy != submittedBy {
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
if sub.Status != StatusPendingReview {
|
||||||
|
return nil, ErrAlreadyReviewed
|
||||||
|
}
|
||||||
|
return sub, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadLimit is how many bytes id's context may hold, and the error an upload
|
||||||
|
// past it fails with. Per-user storage budget: sum the bytes this user's OTHER
|
||||||
|
// submissions already hold (excluding this id, whose blob a re-upload
|
||||||
|
// supersedes) and cap the write at whatever remains. cappedReader trips on the
|
||||||
|
// first byte past the limit, so the store never persists a blob that would
|
||||||
|
// exceed the budget (it removes its temp file on the copy error) and the failure
|
||||||
|
// surfaces as a 403, not a 500. The read-then-write pair is not atomic in this
|
||||||
|
// package: a burst that reaches two api replicas (or any direct caller of the
|
||||||
|
// Manager) can overshoot by up to one blob per interleaved upload — each write
|
||||||
|
// still bounded by the single-blob cap — while the API's per-user upload
|
||||||
|
// reservation collapses the single-replica case.
|
||||||
|
func (m *Manager) uploadLimit(ctx context.Context, submittedBy, id string) (int64, error, error) {
|
||||||
|
used, total, err := m.storedBytes(ctx, submittedBy, id)
|
||||||
|
if err != nil {
|
||||||
|
return 0, nil, err
|
||||||
|
}
|
||||||
|
remaining, budgetErr := m.maxStoredBytesPerUser()-used, errStorageQuota
|
||||||
|
if left := m.maxStoredBytesTotal() - total; left < remaining {
|
||||||
|
remaining, budgetErr = left, fmt.Errorf("%w: every user's uploads together reached the %d-byte limit", ErrUploadsFull, m.maxStoredBytesTotal())
|
||||||
|
}
|
||||||
|
if remaining <= 0 {
|
||||||
|
return 0, nil, budgetErr
|
||||||
|
}
|
||||||
|
limit, over := m.maxContextBytes(), errContextTooLarge
|
||||||
|
if remaining < limit {
|
||||||
|
// The budget binds before the single-blob cap: an upload tripping here is
|
||||||
|
// refused as a spent allowance (or a full store), never as a malformed
|
||||||
|
// request.
|
||||||
|
limit, over = remaining, budgetErr
|
||||||
|
}
|
||||||
|
return limit, over, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadProgress is where a chunked upload stands: Received bytes are staged,
|
||||||
|
// the next part starts there and carries at most PartMaxBytes, and the whole
|
||||||
|
// context may reach MaxContextBytes.
|
||||||
|
type UploadProgress struct {
|
||||||
|
Received int64 `json:"received"`
|
||||||
|
PartMaxBytes int64 `json:"part_max_bytes"`
|
||||||
|
MaxContextBytes int64 `json:"max_context_bytes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Manager) chunked(ctx context.Context, id, submittedBy string) error {
|
||||||
|
if strings.TrimSpace(submittedBy) == "" {
|
||||||
|
return invalidf("submitter identity is required")
|
||||||
|
}
|
||||||
|
if m.Blobs == nil || m.Parts == nil {
|
||||||
|
return ErrUploadsUnavailable
|
||||||
|
}
|
||||||
|
_, err := m.ownPending(ctx, id, submittedBy)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadStatus reports how far the caller's chunked upload of id has come, so a
|
||||||
|
// client that lost its connection resumes where the staged bytes end. Nothing
|
||||||
|
// staged reads as Received 0.
|
||||||
|
func (m *Manager) UploadStatus(ctx context.Context, id, submittedBy string) (UploadProgress, error) {
|
||||||
|
if err := m.chunked(ctx, id, submittedBy); err != nil {
|
||||||
|
return UploadProgress{}, err
|
||||||
|
}
|
||||||
|
n, _, err := m.Parts.Size(id)
|
||||||
|
if err != nil {
|
||||||
|
return UploadProgress{}, err
|
||||||
|
}
|
||||||
|
return UploadProgress{Received: n, PartMaxBytes: m.partMaxBytes(), MaxContextBytes: m.maxContextBytes()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadPart appends one part of the caller's chunked upload of id, starting at
|
||||||
|
// offset: 0 starts over, anything else must equal what is staged
|
||||||
|
// (*OffsetMismatchError otherwise). The first part must open with the gzip
|
||||||
|
// magic. A part is capped at PartMaxBytes (ErrPartTooLarge), and the staged
|
||||||
|
// total at the same limit UploadContext applies to a whole context, with the
|
||||||
|
// same errors, so a chunked upload cannot stage more than a single request could
|
||||||
|
// store. Nothing reaches Blobs until CompleteUpload.
|
||||||
|
func (m *Manager) UploadPart(ctx context.Context, id, submittedBy string, offset int64, r io.Reader) (UploadProgress, error) {
|
||||||
|
if err := m.chunked(ctx, id, submittedBy); err != nil {
|
||||||
|
return UploadProgress{}, err
|
||||||
|
}
|
||||||
|
if offset < 0 {
|
||||||
|
return UploadProgress{}, invalidf("offset must not be negative")
|
||||||
|
}
|
||||||
|
br := bufio.NewReader(r)
|
||||||
|
if offset == 0 {
|
||||||
|
if magic, err := br.Peek(2); err != nil || magic[0] != 0x1f || magic[1] != 0x8b {
|
||||||
|
return UploadProgress{}, invalidf("build context must be a gzip-compressed tarball (.tar.gz)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
limit, over, err := m.uploadLimit(ctx, submittedBy, id)
|
||||||
|
if err != nil {
|
||||||
|
return UploadProgress{}, err
|
||||||
|
}
|
||||||
|
left, partOver := m.partMaxBytes(), error(ErrPartTooLarge)
|
||||||
|
if room := limit - offset; room <= left {
|
||||||
|
left, partOver = max(room, 0), over
|
||||||
|
}
|
||||||
|
if err := m.Parts.CheckRoom(left); err != nil {
|
||||||
|
return UploadProgress{}, err
|
||||||
|
}
|
||||||
|
n, err := m.Parts.Append(id, offset, &cappedReader{r: br, left: left, over: partOver})
|
||||||
|
if err != nil {
|
||||||
|
return UploadProgress{}, err
|
||||||
|
}
|
||||||
|
return UploadProgress{Received: n, PartMaxBytes: m.partMaxBytes(), MaxContextBytes: m.maxContextBytes()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompleteUpload stores the caller's staged upload of id as its build context,
|
||||||
|
// through UploadContext, so it meets every check a single-request upload does
|
||||||
|
// (format, size, budget, room) and records the digest the same way. The staged
|
||||||
|
// bytes are deleted once stored; after a failure they stay, for a retry or a
|
||||||
|
// fresh start at offset 0, until the reaper takes them.
|
||||||
|
func (m *Manager) CompleteUpload(ctx context.Context, id, submittedBy string) (*Submission, error) {
|
||||||
|
if err := m.chunked(ctx, id, submittedBy); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
release, err := m.Parts.hold(id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
f, err := m.Parts.open(id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sub, err := m.UploadContext(ctx, id, submittedBy, f)
|
||||||
|
f.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := m.Parts.Delete(id); err != nil {
|
||||||
|
// Stored and recorded; the leftover copy only waits for the reaper.
|
||||||
|
log.Printf("submit: %v", err)
|
||||||
|
}
|
||||||
|
return sub, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReapStaleParts deletes every staged upload untouched for longer than olderThan.
|
||||||
|
func (m *Manager) ReapStaleParts(olderThan time.Duration) (int, error) {
|
||||||
|
if m.Parts == nil {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
return m.Parts.Reap(m.now().Add(-olderThan))
|
||||||
|
}
|
||||||
|
|
||||||
// storedBytes sums the stored-blob sizes of submittedBy's submissions (user) and
|
// storedBytes sums the stored-blob sizes of submittedBy's submissions (user) and
|
||||||
// of everyone's (total), excluding excludeID — the submission a pending re-upload
|
// of everyone's (total), excluding excludeID — the submission a pending re-upload
|
||||||
// is about to replace, whose bytes must not be counted twice. Sizes are read from
|
// is about to replace, whose bytes must not be counted twice. Sizes are read from
|
||||||
// the blob store itself, the same source of truth uploads/approval consult, so
|
// the blob store itself, the same source of truth uploads/approval consult, so
|
||||||
// the sums cannot drift from what is actually occupying the volume (including
|
// the sums cannot drift from what is actually occupying the volume (including
|
||||||
// blobs uploaded before any budget existed).
|
// blobs uploaded before any budget existed). A staged chunked upload counts as
|
||||||
|
// well, so parts spread over several pending submissions cannot hold more than
|
||||||
|
// the budget allows.
|
||||||
func (m *Manager) storedBytes(ctx context.Context, submittedBy, excludeID string) (user, total int64, err error) {
|
func (m *Manager) storedBytes(ctx context.Context, submittedBy, excludeID string) (user, total int64, err error) {
|
||||||
subs, err := m.Store.ListSubmissions(ctx)
|
subs, err := m.Store.ListSubmissions(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -663,12 +808,16 @@ func (m *Manager) storedBytes(ctx context.Context, submittedBy, excludeID string
|
|||||||
if s.ID == excludeID {
|
if s.ID == excludeID {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
n, ok, err := m.Blobs.Size(ctx, s.ID)
|
n, _, err := m.Blobs.Size(ctx, s.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return 0, 0, err
|
||||||
}
|
}
|
||||||
if !ok {
|
if m.Parts != nil {
|
||||||
continue
|
staged, _, err := m.Parts.Size(s.ID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
n += staged
|
||||||
}
|
}
|
||||||
total += n
|
total += n
|
||||||
if s.SubmittedBy == submittedBy {
|
if s.SubmittedBy == submittedBy {
|
||||||
@@ -976,6 +1125,11 @@ func (m *Manager) Delete(ctx context.Context, id string) (*Submission, error) {
|
|||||||
// exactly what is left behind. A deployment with no upload transport (Blobs nil)
|
// exactly what is left behind. A deployment with no upload transport (Blobs nil)
|
||||||
// has no blobs to reap.
|
// has no blobs to reap.
|
||||||
func (m *Manager) deleteBlob(ctx context.Context, id string) error {
|
func (m *Manager) deleteBlob(ctx context.Context, id string) error {
|
||||||
|
if m.Parts != nil {
|
||||||
|
if err := m.Parts.Delete(id); err != nil {
|
||||||
|
return fmt.Errorf("submit: submission removed, but its unfinished upload could not be deleted: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
if m.Blobs == nil {
|
if m.Blobs == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
+96
-2
@@ -183,6 +183,12 @@ const DAY_MS = 86_400_000;
|
|||||||
// stand-in archive so the review page's download and digest check have bytes.
|
// stand-in archive so the review page's download and digest check have bytes.
|
||||||
const contextBlobs = new Map<string, Buffer>();
|
const contextBlobs = new Map<string, Buffer>();
|
||||||
|
|
||||||
|
// Staged chunked uploads by submission id, and the caps the mock reports. The
|
||||||
|
// part cap is small so a modest test file already goes up in several parts.
|
||||||
|
const stagedParts = new Map<string, Buffer>();
|
||||||
|
const MOCK_PART_MAX = 256 * 1024;
|
||||||
|
const MOCK_CONTEXT_MAX = 1024 * 1024 * 1024;
|
||||||
|
|
||||||
function contextBlob(id: string): Buffer {
|
function contextBlob(id: string): Buffer {
|
||||||
let blob = contextBlobs.get(id);
|
let blob = contextBlobs.get(id);
|
||||||
if (!blob) {
|
if (!blob) {
|
||||||
@@ -1850,11 +1856,99 @@ async function handleSubmissionRoute(ctx: SessionContext): Promise<boolean> {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// GET /api/v1/me/submissions/limits — the Cloudflare-edge default.
|
// GET /api/v1/me/submissions/limits — the 1 GiB default.
|
||||||
if (isMeSubmissions && is("GET", ctx) && ctx.parts.length === 5 && ctx.parts[4] === "limits") {
|
if (isMeSubmissions && is("GET", ctx) && ctx.parts.length === 5 && ctx.parts[4] === "limits") {
|
||||||
sendJSON(ctx.res, 200, { max_context_bytes: 95 * 1024 * 1024 });
|
sendJSON(ctx.res, 200, { max_context_bytes: MOCK_CONTEXT_MAX });
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DELETE /api/v1/me/submissions/{id} — withdraw a still-pending submission
|
||||||
|
// with its context and any staged upload.
|
||||||
|
if (isMeSubmissions && is("DELETE", ctx) && ctx.parts.length === 5) {
|
||||||
|
const id = ctx.parts[4];
|
||||||
|
const idx = ctx.state.submissions.findIndex((s) => s.id === id && s.submitted_by === ctx.account.email);
|
||||||
|
if (idx < 0) {
|
||||||
|
sendError(ctx.res, 404, "not_found", "submission not found");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const sub = ctx.state.submissions[idx];
|
||||||
|
if (sub.status !== "pending_review") {
|
||||||
|
sendError(ctx.res, 409, "already_reviewed", "submission has already been reviewed");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
ctx.state.submissions.splice(idx, 1);
|
||||||
|
contextBlobs.delete(id);
|
||||||
|
stagedParts.delete(id);
|
||||||
|
sendJSON(ctx.res, 200, sub);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The chunked upload: GET/PUT /api/v1/me/submissions/{id}/context/upload and
|
||||||
|
// POST .../upload/complete. Parts are staged in memory the way PartStore
|
||||||
|
// stages them on disk: the staged length is the resume point.
|
||||||
|
if (isMeSubmissions && ctx.parts[5] === "context" && ctx.parts[6] === "upload") {
|
||||||
|
const id = ctx.parts[4];
|
||||||
|
const sub = ctx.state.submissions.find((s) => s.id === id && s.submitted_by === ctx.account.email);
|
||||||
|
if (!sub) {
|
||||||
|
sendError(ctx.res, 404, "not_found", "submission not found");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (sub.status !== "pending_review") {
|
||||||
|
sendError(ctx.res, 409, "already_reviewed", "submission has already been reviewed");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const staged = stagedParts.get(id) ?? Buffer.alloc(0);
|
||||||
|
const progress = (received: number) => ({
|
||||||
|
received,
|
||||||
|
part_max_bytes: MOCK_PART_MAX,
|
||||||
|
max_context_bytes: MOCK_CONTEXT_MAX,
|
||||||
|
});
|
||||||
|
if (is("GET", ctx) && ctx.parts.length === 7) {
|
||||||
|
sendJSON(ctx.res, 200, progress(staged.length));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (is("PUT", ctx) && ctx.parts.length === 7) {
|
||||||
|
const raw = new URL(ctx.req.url ?? "/", "http://localhost").searchParams.get("offset");
|
||||||
|
const offset = raw === null || !/^\d+$/.test(raw) ? NaN : Number(raw);
|
||||||
|
if (Number.isNaN(offset)) {
|
||||||
|
sendError(ctx.res, 400, "bad_request", "offset must be the byte position the part starts at");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (offset !== 0 && offset !== staged.length) {
|
||||||
|
sendError(ctx.res, 409, "upload_offset_mismatch", `the upload holds ${staged.length} bytes; send the part that starts there`);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
for await (const chunk of ctx.req) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||||
|
const part = Buffer.concat(chunks);
|
||||||
|
if (part.length > MOCK_PART_MAX) {
|
||||||
|
sendError(ctx.res, 413, "part_too_large", "an upload part exceeds the part size limit");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (offset === 0 && (part[0] !== 0x1f || part[1] !== 0x8b)) {
|
||||||
|
sendError(ctx.res, 400, "bad_request", "context must be a gzip-compressed tarball");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const next = Buffer.concat([offset === 0 ? Buffer.alloc(0) : staged, part]);
|
||||||
|
stagedParts.set(id, next);
|
||||||
|
// A local network finishes a part at once; the pause lets the progress
|
||||||
|
// bar be seen and paused.
|
||||||
|
await new Promise((r) => setTimeout(r, 250));
|
||||||
|
sendJSON(ctx.res, 200, progress(next.length));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (is("POST", ctx) && ctx.parts.length === 8 && ctx.parts[7] === "complete") {
|
||||||
|
if (!stagedParts.has(id)) {
|
||||||
|
sendError(ctx.res, 400, "bad_request", "no upload in progress for this submission; send its parts first");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
contextBlobs.set(id, staged);
|
||||||
|
sub.context_sha256 = sha256Hex(staged);
|
||||||
|
stagedParts.delete(id);
|
||||||
|
sendJSON(ctx.res, 200, sub);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
// POST /api/v1/me/submissions
|
// POST /api/v1/me/submissions
|
||||||
if (isMeSubmissions && is("POST", ctx) && ctx.parts.length === 4) {
|
if (isMeSubmissions && is("POST", ctx) && ctx.parts.length === 4) {
|
||||||
const body = await readJSON<{ display_name?: string }>(ctx.req);
|
const body = await readJSON<{ display_name?: string }>(ctx.req);
|
||||||
|
|||||||
@@ -66,6 +66,9 @@
|
|||||||
"submission_cooldown": "Too many submission requests — try again shortly.",
|
"submission_cooldown": "Too many submission requests — try again shortly.",
|
||||||
"submissions_unavailable": "Submissions aren't available right now.",
|
"submissions_unavailable": "Submissions aren't available right now.",
|
||||||
"uploads_unavailable": "Uploads aren't available right now.",
|
"uploads_unavailable": "Uploads aren't available right now.",
|
||||||
|
"upload_busy": "Another upload of this submission is still running — wait a moment and try again.",
|
||||||
|
"upload_offset_mismatch": "The upload fell out of step with the server — try again to pick up where it stopped.",
|
||||||
|
"part_too_large": "A piece of the upload was larger than the server accepts.",
|
||||||
"backup_unavailable": "Backups aren't available right now.",
|
"backup_unavailable": "Backups aren't available right now.",
|
||||||
"account_retired": "This account has been retired — sign in with the account it was migrated to.",
|
"account_retired": "This account has been retired — sign in with the account it was migrated to.",
|
||||||
"no_migration": "There is no migration in progress.",
|
"no_migration": "There is no migration in progress.",
|
||||||
|
|||||||
@@ -40,9 +40,15 @@
|
|||||||
"field_context_sha256": "Context SHA-256",
|
"field_context_sha256": "Context SHA-256",
|
||||||
"field_context_sha256_hint": "The digest of the file the platform received; compare it with sha256sum of your local file. Approval binds exactly this content.",
|
"field_context_sha256_hint": "The digest of the file the platform received; compare it with sha256sum of your local file. Approval binds exactly this content.",
|
||||||
"clear_btn": "Clear",
|
"clear_btn": "Clear",
|
||||||
"file_hint": "Supports .tar.gz (max 1GB)",
|
"file_hint": "Supports .tar.gz, up to {{limit}}",
|
||||||
"withdraw_btn": "Withdraw",
|
"withdraw_btn": "Withdraw",
|
||||||
"withdraw_hint": "Withdrawing deletes this submission and its uploaded context, freeing your pending slot and storage budget.",
|
"withdraw_hint": "Withdrawing deletes this submission and its uploaded context, freeing your pending slot and storage budget.",
|
||||||
"withdraw_confirm": "Withdraw",
|
"withdraw_confirm": "Withdraw",
|
||||||
"withdraw_cancel": "Cancel"
|
"withdraw_cancel": "Cancel",
|
||||||
|
"upload_progress_label": "Upload progress",
|
||||||
|
"upload_progress": "{{sent}} of {{total}}",
|
||||||
|
"pause_btn": "Pause",
|
||||||
|
"resume_btn": "Continue upload",
|
||||||
|
"resume_hint": "“{{name}}” is saved and {{sent}} of its build context is on the server. Submit again to continue from there.",
|
||||||
|
"restart_hint": "“{{name}}” is saved. The file you picked uploads from the start."
|
||||||
}
|
}
|
||||||
@@ -66,6 +66,9 @@
|
|||||||
"submission_cooldown": "操作太频繁——请稍后再试。",
|
"submission_cooldown": "操作太频繁——请稍后再试。",
|
||||||
"submissions_unavailable": "提交流程当前不可用。",
|
"submissions_unavailable": "提交流程当前不可用。",
|
||||||
"uploads_unavailable": "上传功能当前不可用。",
|
"uploads_unavailable": "上传功能当前不可用。",
|
||||||
|
"upload_busy": "这个投稿的另一次上传仍在进行——请稍等片刻再试。",
|
||||||
|
"upload_offset_mismatch": "上传进度与服务器对不上——重试即可从中断处接着传。",
|
||||||
|
"part_too_large": "上传的某一片超过了服务器接受的大小。",
|
||||||
"backup_unavailable": "备份功能当前不可用。",
|
"backup_unavailable": "备份功能当前不可用。",
|
||||||
"account_retired": "该账户已退役——请使用迁移后的账户登录。",
|
"account_retired": "该账户已退役——请使用迁移后的账户登录。",
|
||||||
"no_migration": "当前没有进行中的迁移。",
|
"no_migration": "当前没有进行中的迁移。",
|
||||||
|
|||||||
@@ -40,9 +40,15 @@
|
|||||||
"field_context_sha256": "上下文 SHA-256",
|
"field_context_sha256": "上下文 SHA-256",
|
||||||
"field_context_sha256_hint": "平台收到的文件摘要,可用 sha256sum 与本地文件核对。审核通过的就是这份内容。",
|
"field_context_sha256_hint": "平台收到的文件摘要,可用 sha256sum 与本地文件核对。审核通过的就是这份内容。",
|
||||||
"clear_btn": "清除",
|
"clear_btn": "清除",
|
||||||
"file_hint": "支持 .tar.gz 格式 (最大 1GB)",
|
"file_hint": "支持 .tar.gz 格式,最大 {{limit}}",
|
||||||
"withdraw_btn": "撤回提交",
|
"withdraw_btn": "撤回提交",
|
||||||
"withdraw_hint": "撤回会删除该提交及其已上传的构建上下文,并释放你的待审核名额与存储配额。",
|
"withdraw_hint": "撤回会删除该提交及其已上传的构建上下文,并释放你的待审核名额与存储配额。",
|
||||||
"withdraw_confirm": "确认撤回",
|
"withdraw_confirm": "确认撤回",
|
||||||
"withdraw_cancel": "取消"
|
"withdraw_cancel": "取消",
|
||||||
|
"upload_progress_label": "上传进度",
|
||||||
|
"upload_progress": "{{sent}} / {{total}}",
|
||||||
|
"pause_btn": "暂停",
|
||||||
|
"resume_btn": "继续上传",
|
||||||
|
"resume_hint": "“{{name}}”已保存,构建上下文已有 {{sent}} 传到服务器。再次提交即可从这里接着传。",
|
||||||
|
"restart_hint": "“{{name}}”已保存。新选的文件会从头上传。"
|
||||||
}
|
}
|
||||||
@@ -1073,6 +1073,172 @@ describe("session and connection signals", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// A part of a chunked context upload goes by XMLHttpRequest for its upload
|
||||||
|
// progress; its outcomes must read exactly like a fetch call's.
|
||||||
|
class FakeXHR {
|
||||||
|
static last: FakeXHR | undefined;
|
||||||
|
method = "";
|
||||||
|
url = "";
|
||||||
|
withCredentials = false;
|
||||||
|
headers: Record<string, string> = {};
|
||||||
|
body: unknown = undefined;
|
||||||
|
status = 0;
|
||||||
|
statusText = "";
|
||||||
|
responseText = "";
|
||||||
|
aborted = false;
|
||||||
|
upload: { onprogress: ((e: { loaded: number }) => void) | null } = { onprogress: null };
|
||||||
|
onload: (() => void) | null = null;
|
||||||
|
onerror: (() => void) | null = null;
|
||||||
|
onabort: (() => void) | null = null;
|
||||||
|
constructor() {
|
||||||
|
FakeXHR.last = this;
|
||||||
|
}
|
||||||
|
open(method: string, url: string) {
|
||||||
|
this.method = method;
|
||||||
|
this.url = url;
|
||||||
|
}
|
||||||
|
setRequestHeader(k: string, v: string) {
|
||||||
|
this.headers[k] = v;
|
||||||
|
}
|
||||||
|
send(body: unknown) {
|
||||||
|
this.body = body;
|
||||||
|
}
|
||||||
|
abort() {
|
||||||
|
this.aborted = true;
|
||||||
|
this.onabort?.();
|
||||||
|
}
|
||||||
|
respond(status: number, body: string, statusText = "") {
|
||||||
|
this.status = status;
|
||||||
|
this.statusText = statusText;
|
||||||
|
this.responseText = body;
|
||||||
|
this.onload?.();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sentXHR(): Promise<FakeXHR> {
|
||||||
|
await vi.waitFor(() => expect(FakeXHR.last?.body).toBeDefined());
|
||||||
|
return FakeXHR.last as FakeXHR;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("chunked context upload", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
FakeXHR.last = undefined;
|
||||||
|
vi.stubGlobal("XMLHttpRequest", FakeXHR);
|
||||||
|
});
|
||||||
|
afterEach(() => vi.unstubAllGlobals());
|
||||||
|
|
||||||
|
const progress = { received: 12, part_max_bytes: 33554432, max_context_bytes: 1073741824 };
|
||||||
|
|
||||||
|
it("getContextUpload GETs where the staged upload stands", async () => {
|
||||||
|
const fetchSpy = fakeFetch({ received: 4, part_max_bytes: 33554432, max_context_bytes: 1073741824 });
|
||||||
|
vi.stubGlobal("fetch", fetchSpy);
|
||||||
|
expect(await api.getContextUpload("sub-3")).toEqual({ received: 4, part_max_bytes: 33554432, max_context_bytes: 1073741824 });
|
||||||
|
const [url, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||||
|
expect(String(url)).toBe("/me/submissions/sub-3/context/upload");
|
||||||
|
expect((opts as RequestInit).method).toBe("GET");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("completeContextUpload POSTs to store the staged upload", async () => {
|
||||||
|
const sub = { id: "sub-3", display_name: "new submission", status: "pending_review" };
|
||||||
|
const fetchSpy = fakeFetch(sub);
|
||||||
|
vi.stubGlobal("fetch", fetchSpy);
|
||||||
|
expect(await api.completeContextUpload("sub-3")).toEqual(sub);
|
||||||
|
const [url, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||||
|
expect(String(url)).toBe("/me/submissions/sub-3/context/upload/complete");
|
||||||
|
expect((opts as RequestInit).method).toBe("POST");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("putContextPart PUTs the part at its offset with the session cookie and reports progress", async () => {
|
||||||
|
const part = new Blob(["abcd"]);
|
||||||
|
const seen: number[] = [];
|
||||||
|
const done = api.putContextPart("sub-3", 8, part, { onProgress: (n) => seen.push(n) });
|
||||||
|
const xhr = await sentXHR();
|
||||||
|
expect(xhr.method).toBe("PUT");
|
||||||
|
expect(xhr.url).toBe("/me/submissions/sub-3/context/upload?offset=8");
|
||||||
|
expect(xhr.withCredentials).toBe(true);
|
||||||
|
expect(xhr.headers).toEqual({ "Content-Type": "application/octet-stream" });
|
||||||
|
expect(xhr.body).toBe(part);
|
||||||
|
xhr.upload.onprogress?.({ loaded: 3 });
|
||||||
|
xhr.respond(200, JSON.stringify(progress));
|
||||||
|
expect(await done).toEqual({ received: 12, part_max_bytes: 33554432, max_context_bytes: 1073741824 });
|
||||||
|
expect(seen).toEqual([3]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("putContextPart refuses a route value that is not one path segment", async () => {
|
||||||
|
await expect(api.putContextPart("..", 0, new Blob(["x"]))).rejects.toMatchObject({ code: "bad_path_param" });
|
||||||
|
expect(FakeXHR.last).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("an API refusal reads as its code and message", async () => {
|
||||||
|
const done = api.putContextPart("sub-3", 8, new Blob(["abcd"]));
|
||||||
|
(await sentXHR()).respond(
|
||||||
|
409,
|
||||||
|
JSON.stringify({ error: { code: "upload_offset_mismatch", message: "the upload holds 4 bytes; send the part that starts there" } }),
|
||||||
|
"Conflict",
|
||||||
|
);
|
||||||
|
await expect(done).rejects.toEqual({
|
||||||
|
status: 409,
|
||||||
|
code: "upload_offset_mismatch",
|
||||||
|
message: "the upload holds 4 bytes; send the part that starts there",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a tunnel page in place of the API reads as upstream_unavailable", async () => {
|
||||||
|
const done = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
|
||||||
|
(await sentXHR()).respond(524, "<html>A timeout occurred</html>", "");
|
||||||
|
await expect(done).rejects.toEqual({ status: 524, code: "upstream_unavailable", message: "" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a 2xx that is not JSON reads as upstream_unavailable", async () => {
|
||||||
|
const done = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
|
||||||
|
(await sentXHR()).respond(200, "<html>", "OK");
|
||||||
|
await expect(done).rejects.toEqual({ status: 200, code: "upstream_unavailable", message: "the response was not JSON" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("an expired session is announced like any protected call", async () => {
|
||||||
|
const target = new EventTarget();
|
||||||
|
vi.stubGlobal("window", target);
|
||||||
|
const seen: string[] = [];
|
||||||
|
target.addEventListener(SESSION_EXPIRED_EVENT, () => seen.push("expired"));
|
||||||
|
const done = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
|
||||||
|
(await sentXHR()).respond(401, JSON.stringify({ error: { code: "unauthorized", message: "sign in" } }));
|
||||||
|
await expect(done).rejects.toMatchObject({ status: 401, code: "unauthorized" });
|
||||||
|
expect(seen).toEqual(["expired"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("no response at all is a network error that marks the connection lost, until one gets through", async () => {
|
||||||
|
const lost = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
|
||||||
|
(await sentXHR()).onerror?.();
|
||||||
|
await expect(lost).rejects.toEqual({ status: 0, code: "network_error", message: "the upload did not reach the API" });
|
||||||
|
expect(isConnectionLost()).toBe(true);
|
||||||
|
|
||||||
|
FakeXHR.last = undefined;
|
||||||
|
const back = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
|
||||||
|
(await sentXHR()).respond(200, JSON.stringify(progress));
|
||||||
|
await back;
|
||||||
|
expect(isConnectionLost()).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("pausing aborts the request in flight", async () => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const done = api.putContextPart("sub-3", 0, new Blob(["abcd"]), { signal: controller.signal });
|
||||||
|
const xhr = await sentXHR();
|
||||||
|
controller.abort();
|
||||||
|
expect(xhr.aborted).toBe(true);
|
||||||
|
await expect(done).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a signal already paused sends nothing", async () => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
controller.abort();
|
||||||
|
await expect(
|
||||||
|
api.putContextPart("sub-3", 0, new Blob(["abcd"]), { signal: controller.signal }),
|
||||||
|
).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
expect(FakeXHR.last?.body).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// The API's generic codes carry an English developer message ("user not found",
|
// The API's generic codes carry an English developer message ("user not found",
|
||||||
// "invalid request"); the panel words them itself so a Chinese UI never shows it.
|
// "invalid request"); the panel words them itself so a Chinese UI never shows it.
|
||||||
describe("copy for the generic server codes", () => {
|
describe("copy for the generic server codes", () => {
|
||||||
|
|||||||
+105
-6
@@ -6,6 +6,7 @@ import type {
|
|||||||
BanlistResult,
|
BanlistResult,
|
||||||
Build,
|
Build,
|
||||||
BuildScan,
|
BuildScan,
|
||||||
|
ContextUploadProgress,
|
||||||
CreateServerRequest,
|
CreateServerRequest,
|
||||||
CreateUserRequest,
|
CreateUserRequest,
|
||||||
FleetServer,
|
FleetServer,
|
||||||
@@ -148,22 +149,34 @@ async function fetchOK(path: string, init: RequestInit): Promise<Response> {
|
|||||||
reportConnection(true);
|
reportConnection(true);
|
||||||
if (res.ok) return res;
|
if (res.ok) return res;
|
||||||
|
|
||||||
|
let text = "";
|
||||||
|
try {
|
||||||
|
text = await res.text();
|
||||||
|
} catch {
|
||||||
|
/* the body broke off; the status still says what happened */
|
||||||
|
}
|
||||||
|
throw failed(path, res.status, res.statusText, text);
|
||||||
|
}
|
||||||
|
|
||||||
|
// failed turns a non-2xx answer into the ApiError the callers see and announces
|
||||||
|
// the codes the app shell acts on.
|
||||||
|
function failed(path: string, status: number, statusText: string, text: string): ApiError {
|
||||||
let parsed: unknown = null;
|
let parsed: unknown = null;
|
||||||
try {
|
try {
|
||||||
parsed = JSON.parse(await res.text());
|
parsed = JSON.parse(text);
|
||||||
} catch {
|
} catch {
|
||||||
/* no body, or not JSON: an ingress or tunnel answered */
|
/* no body, or not JSON: an ingress or tunnel answered */
|
||||||
}
|
}
|
||||||
const err: ApiError = isApiError(parsed)
|
const err: ApiError = isApiError(parsed)
|
||||||
? { status: res.status, code: parsed.error.code, message: parsed.error.message }
|
? { status, code: parsed.error.code, message: parsed.error.message }
|
||||||
: {
|
: {
|
||||||
status: res.status,
|
status,
|
||||||
code: res.status >= 500 ? "upstream_unavailable" : "error",
|
code: status >= 500 ? "upstream_unavailable" : "error",
|
||||||
message: res.statusText,
|
message: statusText,
|
||||||
};
|
};
|
||||||
announceSetupRequired(err);
|
announceSetupRequired(err);
|
||||||
announceSessionExpired(err, path);
|
announceSessionExpired(err, path);
|
||||||
throw err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
// send returns a 2xx response's parsed JSON body (null for an empty one).
|
// send returns a 2xx response's parsed JSON body (null for an empty one).
|
||||||
@@ -206,6 +219,63 @@ function requestRaw<T>(
|
|||||||
return send<T>(path, { method, headers, body });
|
return send<T>(path, { method, headers, body });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sendWithProgress sends body by XMLHttpRequest, the one browser API that
|
||||||
|
// reports how much of a request body has gone out (fetch has no upload
|
||||||
|
// progress), and settles the way send does: the parsed 2xx body, or the same
|
||||||
|
// ApiError fetchOK would throw. onProgress gets the bytes of body sent so far;
|
||||||
|
// signal aborts the request with an AbortError.
|
||||||
|
async function sendWithProgress<T>(
|
||||||
|
method: string,
|
||||||
|
path: string,
|
||||||
|
body: Blob,
|
||||||
|
opts: { onProgress?: (sent: number) => void; signal?: AbortSignal } = {},
|
||||||
|
): Promise<T> {
|
||||||
|
const { apiBase } = await loadConfig();
|
||||||
|
const { onProgress, signal } = opts;
|
||||||
|
return new Promise<T>((resolve, reject) => {
|
||||||
|
const xhr = new XMLHttpRequest();
|
||||||
|
const onAbort = () => xhr.abort();
|
||||||
|
const settle = () => signal?.removeEventListener("abort", onAbort);
|
||||||
|
xhr.open(method, `${apiBase}${path}`);
|
||||||
|
xhr.withCredentials = true;
|
||||||
|
xhr.setRequestHeader("Content-Type", "application/octet-stream");
|
||||||
|
if (onProgress) xhr.upload.onprogress = (e) => onProgress(e.loaded);
|
||||||
|
xhr.onabort = () => {
|
||||||
|
settle();
|
||||||
|
reject(new DOMException("The upload was cancelled", "AbortError"));
|
||||||
|
};
|
||||||
|
xhr.onerror = () => {
|
||||||
|
settle();
|
||||||
|
reportConnection(false);
|
||||||
|
reject(networkError(new Error("the upload did not reach the API")));
|
||||||
|
};
|
||||||
|
xhr.onload = () => {
|
||||||
|
settle();
|
||||||
|
reportConnection(true);
|
||||||
|
if (xhr.status < 200 || xhr.status >= 300) {
|
||||||
|
reject(failed(path, xhr.status, xhr.statusText, xhr.responseText));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
resolve((xhr.responseText ? JSON.parse(xhr.responseText) : null) as T);
|
||||||
|
} catch {
|
||||||
|
const err: ApiError = {
|
||||||
|
status: xhr.status,
|
||||||
|
code: "upstream_unavailable",
|
||||||
|
message: "the response was not JSON",
|
||||||
|
};
|
||||||
|
reject(err);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if (signal?.aborted) {
|
||||||
|
reject(new DOMException("The upload was cancelled", "AbortError"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
signal?.addEventListener("abort", onAbort);
|
||||||
|
xhr.send(body);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// rejectingSync turns a synchronous throw inside an api method (urlPath refusing
|
// rejectingSync turns a synchronous throw inside an api method (urlPath refusing
|
||||||
// a segment) into a rejected promise, so every caller handles it the way it
|
// a segment) into a rejected promise, so every caller handles it the way it
|
||||||
// handles any failed call.
|
// handles any failed call.
|
||||||
@@ -746,6 +816,27 @@ export const api = rejectingSync({
|
|||||||
"Content-Type": "application/x-gzip",
|
"Content-Type": "application/x-gzip",
|
||||||
}),
|
}),
|
||||||
|
|
||||||
|
// A chunked context upload (lib/contextUpload.ts drives it): ask where the
|
||||||
|
// staged upload stands, send each part at its byte offset, then store it.
|
||||||
|
getContextUpload: (id: string) =>
|
||||||
|
request<ContextUploadProgress>("GET", urlPath`/me/submissions/${id}/context/upload`),
|
||||||
|
|
||||||
|
putContextPart: (
|
||||||
|
id: string,
|
||||||
|
offset: number,
|
||||||
|
part: Blob,
|
||||||
|
opts?: { onProgress?: (sent: number) => void; signal?: AbortSignal },
|
||||||
|
) =>
|
||||||
|
sendWithProgress<ContextUploadProgress>(
|
||||||
|
"PUT",
|
||||||
|
urlPath`/me/submissions/${id}/context/upload` + `?offset=${offset}`,
|
||||||
|
part,
|
||||||
|
opts,
|
||||||
|
),
|
||||||
|
|
||||||
|
completeContextUpload: (id: string) =>
|
||||||
|
request<Submission>("POST", urlPath`/me/submissions/${id}/context/upload/complete`),
|
||||||
|
|
||||||
// Retract the caller's own pending submission (and its uploaded context), which
|
// Retract the caller's own pending submission (and its uploaded context), which
|
||||||
// frees their pending slot and storage budget. Reviewed submissions are frozen.
|
// frees their pending slot and storage budget. Reviewed submissions are frozen.
|
||||||
withdrawSubmission: (id: string) => request<Submission>("DELETE", urlPath`/me/submissions/${id}`),
|
withdrawSubmission: (id: string) => request<Submission>("DELETE", urlPath`/me/submissions/${id}`),
|
||||||
@@ -1031,6 +1122,14 @@ export function humanizeError(e: unknown): string {
|
|||||||
return t("submissions_unavailable");
|
return t("submissions_unavailable");
|
||||||
case "uploads_unavailable":
|
case "uploads_unavailable":
|
||||||
return t("uploads_unavailable");
|
return t("uploads_unavailable");
|
||||||
|
// Chunked uploads: the client resumes on these by itself, so they surface
|
||||||
|
// only once its retries run out.
|
||||||
|
case "upload_busy":
|
||||||
|
return t("upload_busy");
|
||||||
|
case "upload_offset_mismatch":
|
||||||
|
return t("upload_offset_mismatch");
|
||||||
|
case "part_too_large":
|
||||||
|
return t("part_too_large");
|
||||||
case "backup_unavailable":
|
case "backup_unavailable":
|
||||||
return t("backup_unavailable");
|
return t("backup_unavailable");
|
||||||
// Account migration + the re-auth steps it depends on: every refusal below is
|
// Account migration + the re-auth steps it depends on: every refusal below is
|
||||||
|
|||||||
@@ -0,0 +1,264 @@
|
|||||||
|
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||||
|
|
||||||
|
const calls = vi.hoisted(() => ({
|
||||||
|
getContextUpload: vi.fn(),
|
||||||
|
putContextPart: vi.fn(),
|
||||||
|
completeContextUpload: vi.fn(),
|
||||||
|
}));
|
||||||
|
vi.mock("./api", () => ({ api: calls }));
|
||||||
|
|
||||||
|
const { uploadContext } = await import("./contextUpload");
|
||||||
|
|
||||||
|
// A 10-byte "file" and a server that takes parts of at most 4 bytes.
|
||||||
|
const FILE = new Blob(["0123456789"]);
|
||||||
|
const at = (received: number) => ({ received, part_max_bytes: 4, max_context_bytes: 1073741824 });
|
||||||
|
const dropped = { status: 0, code: "network_error", message: "Failed to fetch" };
|
||||||
|
const edgeTimeout = { status: 524, code: "upstream_unavailable", message: "" };
|
||||||
|
|
||||||
|
// A server that appends each part where it says and answers the new length.
|
||||||
|
function acceptParts() {
|
||||||
|
calls.putContextPart.mockImplementation(async (_id: string, offset: number, part: Blob) => at(offset + part.size));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sentParts(): Promise<Array<[number, string]>> {
|
||||||
|
return Promise.all(
|
||||||
|
calls.putContextPart.mock.calls.map(async ([, offset, part]) => [offset as number, await (part as Blob).text()] as [number, string]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const sleep = vi.fn(async (_ms: number, _signal?: AbortSignal) => undefined);
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
calls.getContextUpload.mockReset();
|
||||||
|
calls.putContextPart.mockReset();
|
||||||
|
calls.completeContextUpload.mockReset();
|
||||||
|
calls.completeContextUpload.mockResolvedValue({ id: "sub-1" });
|
||||||
|
sleep.mockClear();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("uploadContext", () => {
|
||||||
|
it("sends the file in parts no larger than the server's cap, then stores it", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(0));
|
||||||
|
acceptParts();
|
||||||
|
const progress: Array<[number, number]> = [];
|
||||||
|
await uploadContext("sub-1", FILE, { sleep, onProgress: (sent, total) => progress.push([sent, total]) });
|
||||||
|
expect(await sentParts()).toEqual([
|
||||||
|
[0, "0123"],
|
||||||
|
[4, "4567"],
|
||||||
|
[8, "89"],
|
||||||
|
]);
|
||||||
|
expect(calls.putContextPart.mock.calls.map((c) => c[0])).toEqual(["sub-1", "sub-1", "sub-1"]);
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledWith("sub-1");
|
||||||
|
expect(progress.at(-1)).toEqual([10, 10]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports the bytes of a part in flight on top of the parts already stored", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(0));
|
||||||
|
calls.putContextPart.mockImplementation(async (_id, offset: number, part: Blob, opts) => {
|
||||||
|
opts.onProgress(2);
|
||||||
|
return at(offset + part.size);
|
||||||
|
});
|
||||||
|
const progress: Array<[number, number, number]> = [];
|
||||||
|
await uploadContext("sub-1", FILE, {
|
||||||
|
sleep,
|
||||||
|
onProgress: (sent, total, stored) => progress.push([sent, total, stored]),
|
||||||
|
});
|
||||||
|
expect(progress).toEqual([
|
||||||
|
[0, 10, 0],
|
||||||
|
[2, 10, 0],
|
||||||
|
[4, 10, 4],
|
||||||
|
[6, 10, 4],
|
||||||
|
[8, 10, 8],
|
||||||
|
[10, 10, 8],
|
||||||
|
[10, 10, 10],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("starts a fresh file over even when the submission has parts staged", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(4));
|
||||||
|
acceptParts();
|
||||||
|
await uploadContext("sub-1", FILE, { sleep });
|
||||||
|
expect((await sentParts()).map(([o]) => o)).toEqual([0, 4, 8]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resumes the same file from where the staged bytes end", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(4));
|
||||||
|
acceptParts();
|
||||||
|
await uploadContext("sub-1", FILE, { sleep, resume: true });
|
||||||
|
expect(await sentParts()).toEqual([
|
||||||
|
[4, "4567"],
|
||||||
|
[8, "89"],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("goes straight to storing when every byte is already staged", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(10));
|
||||||
|
await uploadContext("sub-1", FILE, { sleep, resume: true });
|
||||||
|
expect(calls.putContextPart).not.toHaveBeenCalled();
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("starts over when more is staged than the file holds", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(12));
|
||||||
|
acceptParts();
|
||||||
|
await uploadContext("sub-1", FILE, { sleep, resume: true });
|
||||||
|
expect((await sentParts()).map(([o]) => o)).toEqual([0, 4, 8]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("after a dropped part asks where the upload stands and carries on from there", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValueOnce(at(4));
|
||||||
|
acceptParts();
|
||||||
|
calls.putContextPart.mockImplementationOnce(async (_id, offset: number, part: Blob) => at(offset + part.size));
|
||||||
|
calls.putContextPart.mockRejectedValueOnce(dropped);
|
||||||
|
await uploadContext("sub-1", FILE, { sleep });
|
||||||
|
expect((await sentParts()).map(([o]) => o)).toEqual([0, 4, 4, 8]);
|
||||||
|
expect(calls.getContextUpload).toHaveBeenCalledTimes(2);
|
||||||
|
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a fresh file whose first part dropped starts over at 0", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(2));
|
||||||
|
acceptParts();
|
||||||
|
calls.putContextPart.mockRejectedValueOnce(dropped);
|
||||||
|
await uploadContext("sub-1", FILE, { sleep });
|
||||||
|
expect((await sentParts()).map(([o]) => o)).toEqual([0, 0, 4, 8]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("follows the server when it says the upload holds a different length", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValueOnce(at(8));
|
||||||
|
acceptParts();
|
||||||
|
calls.putContextPart.mockImplementationOnce(async () => at(4));
|
||||||
|
calls.putContextPart.mockRejectedValueOnce({ status: 409, code: "upload_offset_mismatch", message: "" });
|
||||||
|
await uploadContext("sub-1", FILE, { sleep });
|
||||||
|
expect((await sentParts()).map(([o]) => o)).toEqual([0, 4, 8]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("gives up at once on a refusal the next attempt cannot outlast", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(0));
|
||||||
|
const refused = { status: 400, code: "bad_request", message: "context must be a gzip-compressed tarball" };
|
||||||
|
calls.putContextPart.mockRejectedValue(refused);
|
||||||
|
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(refused);
|
||||||
|
expect(calls.putContextPart).toHaveBeenCalledTimes(1);
|
||||||
|
expect(sleep).not.toHaveBeenCalled();
|
||||||
|
expect(calls.completeContextUpload).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("gives up after eight dropped attempts in a row, backing off up to 15 s", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(0));
|
||||||
|
calls.putContextPart.mockRejectedValue(dropped);
|
||||||
|
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(dropped);
|
||||||
|
expect(calls.putContextPart).toHaveBeenCalledTimes(8);
|
||||||
|
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000, 2000, 4000, 8000, 15000, 15000, 15000]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a part that gets through resets the count of dropped attempts", async () => {
|
||||||
|
calls.getContextUpload.mockImplementation(async () => at(calls.putContextPart.mock.calls.length >= 8 ? 4 : 0));
|
||||||
|
acceptParts();
|
||||||
|
for (let i = 0; i < 7; i++) calls.putContextPart.mockRejectedValueOnce(dropped);
|
||||||
|
calls.putContextPart.mockImplementationOnce(async () => at(4));
|
||||||
|
for (let i = 0; i < 7; i++) calls.putContextPart.mockRejectedValueOnce(dropped);
|
||||||
|
await uploadContext("sub-1", FILE, { sleep });
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("sends an empty file as one part, so the server gives the refusal", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(0));
|
||||||
|
const refused = { status: 400, code: "bad_request", message: "context must be a gzip-compressed tarball" };
|
||||||
|
calls.putContextPart.mockRejectedValue(refused);
|
||||||
|
await expect(uploadContext("sub-1", new Blob([]), { sleep })).rejects.toBe(refused);
|
||||||
|
expect(calls.putContextPart.mock.calls.map((c) => [c[1], (c[2] as Blob).size])).toEqual([[0, 0]]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stops on a pause without retrying", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(0));
|
||||||
|
const paused = new DOMException("The upload was cancelled", "AbortError");
|
||||||
|
calls.putContextPart.mockRejectedValue(paused);
|
||||||
|
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(paused);
|
||||||
|
expect(calls.putContextPart).toHaveBeenCalledTimes(1);
|
||||||
|
expect(sleep).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes the pause signal to every part", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(0));
|
||||||
|
acceptParts();
|
||||||
|
const controller = new AbortController();
|
||||||
|
await uploadContext("sub-1", FILE, { sleep, signal: controller.signal });
|
||||||
|
expect(calls.putContextPart.mock.calls.map((c) => c[3].signal)).toEqual([
|
||||||
|
controller.signal,
|
||||||
|
controller.signal,
|
||||||
|
controller.signal,
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("uploadContext completion", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
acceptParts();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats the staged upload vanishing after an edge timeout as stored", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValueOnce(at(0));
|
||||||
|
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout);
|
||||||
|
await uploadContext("sub-1", FILE, { sleep });
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
|
||||||
|
expect(calls.getContextUpload).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits out a store still in progress, then takes its answer", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValue(at(10));
|
||||||
|
calls.completeContextUpload
|
||||||
|
.mockRejectedValueOnce(edgeTimeout)
|
||||||
|
.mockRejectedValueOnce({ status: 409, code: "upload_busy", message: "" })
|
||||||
|
.mockResolvedValueOnce({ id: "sub-1" });
|
||||||
|
await uploadContext("sub-1", FILE, { sleep });
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(3);
|
||||||
|
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000, 2000]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("surfaces the real refusal when completing again answers with one", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValue(at(10));
|
||||||
|
const quota = { status: 403, code: "submission_quota_exceeded", message: "budget" };
|
||||||
|
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout).mockRejectedValueOnce(quota);
|
||||||
|
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(quota);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("surfaces a refusal to store at once", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValue(at(0));
|
||||||
|
const refused = { status: 429, code: "submission_cooldown", message: "" };
|
||||||
|
calls.completeContextUpload.mockRejectedValueOnce(refused);
|
||||||
|
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(refused);
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
|
||||||
|
expect(sleep).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports the timeout when the staged upload is cut short while storing", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValueOnce(at(6));
|
||||||
|
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout);
|
||||||
|
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(edgeTimeout);
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ends the wait when checking on the store is refused", async () => {
|
||||||
|
const gone = { status: 404, code: "not_found", message: "submission not found" };
|
||||||
|
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockRejectedValueOnce(gone);
|
||||||
|
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout);
|
||||||
|
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(gone);
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps waiting through a check that did not get through", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockRejectedValueOnce(dropped).mockResolvedValueOnce(at(0));
|
||||||
|
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout);
|
||||||
|
await uploadContext("sub-1", FILE, { sleep });
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
|
||||||
|
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000, 2000]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("gives up after forty checks on a store that never finishes", async () => {
|
||||||
|
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValue(at(10));
|
||||||
|
calls.completeContextUpload.mockRejectedValue(edgeTimeout);
|
||||||
|
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(edgeTimeout);
|
||||||
|
expect(calls.completeContextUpload).toHaveBeenCalledTimes(41);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
import { api } from "./api";
|
||||||
|
import type { ApiError } from "./types";
|
||||||
|
|
||||||
|
// uploadContext sends a submission's build context in parts. One request body is
|
||||||
|
// bounded by whatever proxy fronts the API (the Cloudflare edge refuses bodies
|
||||||
|
// over 100 MB), so the server stages parts of at most part_max_bytes, and the
|
||||||
|
// staged length is the resume point: after a dropped connection, a tunnel error
|
||||||
|
// or a busy answer, the loop asks the server where the upload stands and carries
|
||||||
|
// on from there. Completing stores the staged bytes as the context.
|
||||||
|
|
||||||
|
export interface UploadContextOptions {
|
||||||
|
/** Reports the bytes of file sent so far (stored plus the part in flight)
|
||||||
|
* and, as stored, the bytes the server has confirmed holding. */
|
||||||
|
onProgress?: (sent: number, total: number, stored: number) => void;
|
||||||
|
/** Cancels the upload; the staged parts stay for a later resume. */
|
||||||
|
signal?: AbortSignal;
|
||||||
|
/** Carry on from the parts this submission already staged, when they are a
|
||||||
|
* prefix of this same file (a retry after a failure). A fresh file starts
|
||||||
|
* over, which also discards whatever an earlier file left staged. */
|
||||||
|
resume?: boolean;
|
||||||
|
/** Test seam for the pause between attempts. */
|
||||||
|
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Consecutive failed attempts a part may take before the upload gives up. */
|
||||||
|
export const MAX_ATTEMPTS = 8;
|
||||||
|
/** Checks while the server finishes storing a large context (about 10 minutes). */
|
||||||
|
export const MAX_COMPLETE_WAITS = 40;
|
||||||
|
|
||||||
|
// A transient answer is one the next attempt can outlast: no response at all, a
|
||||||
|
// tunnel or ingress page in place of the API's (upstream_unavailable), or a 409
|
||||||
|
// that means "ask where the upload stands and send again".
|
||||||
|
export function isTransient(e: unknown): boolean {
|
||||||
|
const err = e as Partial<ApiError> | null;
|
||||||
|
if (!err || typeof err.code !== "string") return false;
|
||||||
|
return (
|
||||||
|
err.code === "network_error" ||
|
||||||
|
err.code === "upstream_unavailable" ||
|
||||||
|
err.code === "upload_busy" ||
|
||||||
|
err.code === "upload_offset_mismatch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** retryDelay is the pause before attempt n+1: 1s, 2s, 4s, 8s, then 15s. */
|
||||||
|
export function retryDelay(n: number): number {
|
||||||
|
return Math.min(1000 * 2 ** (n - 1), 15000);
|
||||||
|
}
|
||||||
|
|
||||||
|
function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
if (signal?.aborted) {
|
||||||
|
reject(new DOMException("The upload was cancelled", "AbortError"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
reject(new DOMException("The upload was cancelled", "AbortError"));
|
||||||
|
};
|
||||||
|
signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function uploadContext(id: string, file: Blob, opts: UploadContextOptions = {}): Promise<void> {
|
||||||
|
const { onProgress, signal } = opts;
|
||||||
|
const sleep = opts.sleep ?? wait;
|
||||||
|
const total = file.size;
|
||||||
|
|
||||||
|
// offset is where the next part starts; null means "ask the server first".
|
||||||
|
let offset: number | null = null;
|
||||||
|
let partMax = 0;
|
||||||
|
let fresh = !opts.resume;
|
||||||
|
let failures = 0;
|
||||||
|
for (;;) {
|
||||||
|
try {
|
||||||
|
if (offset === null) {
|
||||||
|
const at = await api.getContextUpload(id);
|
||||||
|
partMax = at.part_max_bytes;
|
||||||
|
// More staged than this file holds cannot be a prefix of it.
|
||||||
|
offset = fresh || at.received > total ? 0 : at.received;
|
||||||
|
onProgress?.(offset, total, offset);
|
||||||
|
}
|
||||||
|
if (offset < total || offset === 0) {
|
||||||
|
const start = offset;
|
||||||
|
const part = file.slice(start, Math.min(start + partMax, total));
|
||||||
|
const at = await api.putContextPart(id, start, part, {
|
||||||
|
signal,
|
||||||
|
onProgress: (sent) => onProgress?.(start + sent, total, start),
|
||||||
|
});
|
||||||
|
offset = at.received;
|
||||||
|
fresh = false;
|
||||||
|
failures = 0;
|
||||||
|
onProgress?.(offset, total, offset);
|
||||||
|
}
|
||||||
|
if (offset >= total) break;
|
||||||
|
} catch (e) {
|
||||||
|
// A pause (AbortError) is not transient, so it ends the upload here.
|
||||||
|
if (!isTransient(e) || ++failures >= MAX_ATTEMPTS) throw e;
|
||||||
|
await sleep(retryDelay(failures), signal);
|
||||||
|
offset = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await complete(id, total, sleep, signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
// complete stores the staged upload. Behind the edge a large context can take
|
||||||
|
// longer to store than the edge waits for an answer (it gives up after 100 s),
|
||||||
|
// and the server carries on regardless. So a transient failure here is followed
|
||||||
|
// by watching the staged upload: still all there means the store has not taken
|
||||||
|
// it yet (or failed and kept it), so completing again either waits (upload_busy)
|
||||||
|
// or answers with the real outcome; gone means it was stored.
|
||||||
|
async function complete(
|
||||||
|
id: string,
|
||||||
|
total: number,
|
||||||
|
sleep: (ms: number, signal?: AbortSignal) => Promise<void>,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<void> {
|
||||||
|
let lastErr: unknown;
|
||||||
|
for (let waits = 0; waits <= MAX_COMPLETE_WAITS; waits++) {
|
||||||
|
if (waits > 0) {
|
||||||
|
await sleep(retryDelay(waits), signal);
|
||||||
|
try {
|
||||||
|
const at = await api.getContextUpload(id);
|
||||||
|
if (at.received < total) {
|
||||||
|
if (at.received === 0) return;
|
||||||
|
throw lastErr;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
if (e === lastErr || !isTransient(e)) throw e;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await api.completeContextUpload(id);
|
||||||
|
return;
|
||||||
|
} catch (e) {
|
||||||
|
if (!isTransient(e)) throw e;
|
||||||
|
lastErr = e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastErr;
|
||||||
|
}
|
||||||
@@ -1856,7 +1856,7 @@ export interface paths {
|
|||||||
};
|
};
|
||||||
/**
|
/**
|
||||||
* The per-upload build-context cap
|
* The per-upload build-context cap
|
||||||
* @description The effective [registry] context_max_bytes: 1 GiB by default, 95 MiB behind the Cloudflare edge (its proxy refuses bodies over 100 MB before they reach the API). The panel checks a file against it before upload.
|
* @description The effective [registry] context_max_bytes, 1 GiB by default. The panel checks a file against it before upload and sends the file through the chunked upload (/api/v1/me/submissions/{id}/context/upload), so the cap holds behind the Cloudflare edge too, whose proxy refuses a single body over 100 MB.
|
||||||
*/
|
*/
|
||||||
get: operations["submissionLimits"];
|
get: operations["submissionLimits"];
|
||||||
put?: never;
|
put?: never;
|
||||||
@@ -1878,7 +1878,7 @@ export interface paths {
|
|||||||
put?: never;
|
put?: never;
|
||||||
/**
|
/**
|
||||||
* Upload the modpack build context for your own pending submission (user side; user-directed lane over §16).
|
* Upload the modpack build context for your own pending submission (user side; user-directed lane over §16).
|
||||||
* @description The request body IS the raw gzip build context (context.tar.gz) — not JSON, not multipart — streamed to the platform-derived, id-namespaced location Kaniko reads via --context. The submitter is taken from the principal; a submission the caller does not own is reported as 404, so this endpoint cannot upload to or probe another user's submission. Only a pending_review submission accepts a context (409 otherwise); a wrong-format or oversize body is rejected with 400 (the per-upload cap is [registry] context_max_bytes: 1 GiB by default and 95 MiB behind the Cloudflare edge, whose proxy refuses bodies over 100 MB with its own HTML 413 before they reach the API; GET /api/v1/me/submissions/limits reports the effective cap so a client can check a file before sending it), and an upload that would push the caller past their per-user stored-context budget is refused with 403 before the excess is persisted. Returns 503 when the deployment's context store has no implemented upload transport.
|
* @description The request body IS the raw gzip build context (context.tar.gz) — not JSON, not multipart — streamed to the platform-derived, id-namespaced location Kaniko reads via --context. The submitter is taken from the principal; a submission the caller does not own is reported as 404, so this endpoint cannot upload to or probe another user's submission. Only a pending_review submission accepts a context (409 otherwise); a wrong-format or oversize body is rejected with 400 (the per-upload cap is [registry] context_max_bytes, 1 GiB by default; GET /api/v1/me/submissions/limits reports it so a client can check a file before sending it). This request carries the whole context, so behind the Cloudflare edge, whose proxy refuses bodies over 100 MB with its own HTML 413 before they reach the API, a larger context goes through the chunked upload at /api/v1/me/submissions/{id}/context/upload instead. An upload that would push the caller past their per-user stored-context budget is refused with 403 before the excess is persisted. Returns 503 when the deployment's context store has no implemented upload transport.
|
||||||
*/
|
*/
|
||||||
post: operations["uploadSubmissionContext"];
|
post: operations["uploadSubmissionContext"];
|
||||||
delete?: never;
|
delete?: never;
|
||||||
@@ -1887,6 +1887,50 @@ export interface paths {
|
|||||||
patch?: never;
|
patch?: never;
|
||||||
trace?: never;
|
trace?: never;
|
||||||
};
|
};
|
||||||
|
"/api/v1/me/submissions/{id}/context/upload": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
/**
|
||||||
|
* Where your chunked context upload stands (the resume point).
|
||||||
|
* @description The chunked form of POST /api/v1/me/submissions/{id}/context, for a context larger than one request carries through the edge. received is how many bytes are staged: the next part starts there. A client reads it before the first part and again after a failed one. Nothing staged reads as 0. Same owner scoping as the single upload (404 for another user's submission, 409 once reviewed).
|
||||||
|
*/
|
||||||
|
get: operations["getContextUpload"];
|
||||||
|
/**
|
||||||
|
* Append one part of your chunked context upload.
|
||||||
|
* @description The body is the part's raw bytes, at most part_max_bytes (32 MiB). offset is where they start: 0 starts the upload over, and anything else must equal the staged length, or the answer is 409 upload_offset_mismatch and the client reads GET for where to resume. The first part must open with the gzip magic (400). The staged total meets the same context cap (400) and storage budget (403) as a single upload. A part that breaks off is cut back off, so the staged bytes are always a prefix of the file. One request per upload at a time (409 upload_busy). Staged bytes untouched for 24 hours are deleted.
|
||||||
|
*/
|
||||||
|
put: operations["putContextUploadPart"];
|
||||||
|
post?: never;
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
|
"/api/v1/me/submissions/{id}/context/upload/complete": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get?: never;
|
||||||
|
put?: never;
|
||||||
|
/**
|
||||||
|
* Store your staged chunked upload as the submission's build context.
|
||||||
|
* @description Runs every check of POST /api/v1/me/submissions/{id}/context on the staged bytes (format, cap, budget, room), records the digest the same way, and deletes the staged copy. Holds the same per-user upload cooldown (429) and writes the same submission.upload audit event. Nothing staged is 400. After a failure the staged bytes stay, for a retry.
|
||||||
|
*/
|
||||||
|
post: operations["completeContextUpload"];
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
"/api/v1/me/submissions/{id}": {
|
"/api/v1/me/submissions/{id}": {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: never;
|
query?: never;
|
||||||
@@ -2462,6 +2506,23 @@ export interface components {
|
|||||||
/** Format: date-time */
|
/** Format: date-time */
|
||||||
added_at: string;
|
added_at: string;
|
||||||
};
|
};
|
||||||
|
ContextUploadProgress: {
|
||||||
|
/**
|
||||||
|
* Format: int64
|
||||||
|
* @description Bytes staged so far; the next part starts here.
|
||||||
|
*/
|
||||||
|
received: number;
|
||||||
|
/**
|
||||||
|
* Format: int64
|
||||||
|
* @description The most one part may carry.
|
||||||
|
*/
|
||||||
|
part_max_bytes: number;
|
||||||
|
/**
|
||||||
|
* Format: int64
|
||||||
|
* @description The most the whole context may reach ([registry] context_max_bytes).
|
||||||
|
*/
|
||||||
|
max_context_bytes: number;
|
||||||
|
};
|
||||||
/** @description One user-submitted modpack in the approval lane (internal/submit Submission — a user-directed extension over the §16 build subsystem). The user supplies only display_name; submitted_by comes from the principal and context_ref/image_ref/build_id/reviewed_by are platform-controlled, never client input. */
|
/** @description One user-submitted modpack in the approval lane (internal/submit Submission — a user-directed extension over the §16 build subsystem). The user supplies only display_name; submitted_by comes from the principal and context_ref/image_ref/build_id/reviewed_by are platform-controlled, never client input. */
|
||||||
Submission: {
|
Submission: {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -7434,6 +7495,131 @@ export interface operations {
|
|||||||
503: components["responses"]["ServiceUnavailable"];
|
503: components["responses"]["ServiceUnavailable"];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
getContextUpload: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
id: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description The staged length and the limits a part and the whole must keep. */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["ContextUploadProgress"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
401: components["responses"]["Unauthorized"];
|
||||||
|
404: components["responses"]["NotFound"];
|
||||||
|
409: components["responses"]["Conflict"];
|
||||||
|
503: components["responses"]["ServiceUnavailable"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
putContextUploadPart: {
|
||||||
|
parameters: {
|
||||||
|
query: {
|
||||||
|
offset: number;
|
||||||
|
};
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
id: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody: {
|
||||||
|
content: {
|
||||||
|
"application/octet-stream": string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
responses: {
|
||||||
|
/** @description The part is staged; received is the new length. */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["ContextUploadProgress"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
400: components["responses"]["BadRequest"];
|
||||||
|
401: components["responses"]["Unauthorized"];
|
||||||
|
/** @description The staged total would exceed the caller's per-user stored-context budget (submission_quota_exceeded). */
|
||||||
|
403: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content?: never;
|
||||||
|
};
|
||||||
|
404: components["responses"]["NotFound"];
|
||||||
|
409: components["responses"]["Conflict"];
|
||||||
|
/** @description The part is larger than part_max_bytes (part_too_large). */
|
||||||
|
413: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["Error"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
503: components["responses"]["ServiceUnavailable"];
|
||||||
|
/** @description The uploads store is full (uploads_full). */
|
||||||
|
507: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["Error"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
completeContextUpload: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
id: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Context stored; the submission (unchanged) is returned. */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["Submission"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
400: components["responses"]["BadRequest"];
|
||||||
|
401: components["responses"]["Unauthorized"];
|
||||||
|
/** @description The context would exceed the caller's per-user stored-context budget (submission_quota_exceeded). */
|
||||||
|
403: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content?: never;
|
||||||
|
};
|
||||||
|
404: components["responses"]["NotFound"];
|
||||||
|
409: components["responses"]["Conflict"];
|
||||||
|
/** @description An upload was accepted within the per-user cooldown window (submission_cooldown). */
|
||||||
|
429: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content?: never;
|
||||||
|
};
|
||||||
|
503: components["responses"]["ServiceUnavailable"];
|
||||||
|
};
|
||||||
|
};
|
||||||
withdrawSubmission: {
|
withdrawSubmission: {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: never;
|
query?: never;
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ export type WireParity = [
|
|||||||
Holds<Parity<T.ScanFinding, S["ScanFinding"]>>,
|
Holds<Parity<T.ScanFinding, S["ScanFinding"]>>,
|
||||||
Holds<Parity<T.WhitelistImage, S["Image"]>>,
|
Holds<Parity<T.WhitelistImage, S["Image"]>>,
|
||||||
Holds<Parity<T.Submission, S["Submission"]>>,
|
Holds<Parity<T.Submission, S["Submission"]>>,
|
||||||
|
Holds<Parity<T.ContextUploadProgress, S["ContextUploadProgress"]>>,
|
||||||
Holds<Parity<T.UserView, S["UserView"]>>,
|
Holds<Parity<T.UserView, S["UserView"]>>,
|
||||||
Holds<Parity<T.UserDetail, S["UserDetail"]>>,
|
Holds<Parity<T.UserDetail, S["UserDetail"]>>,
|
||||||
Holds<Parity<T.QuotaView, S["QuotaView"]>>,
|
Holds<Parity<T.QuotaView, S["QuotaView"]>>,
|
||||||
|
|||||||
@@ -378,6 +378,15 @@ export interface Submission {
|
|||||||
context_sha256?: string;
|
context_sha256?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** ContextUploadProgress is where a chunked context upload stands: received is
|
||||||
|
* how many bytes the server holds (the next part starts there), part_max_bytes
|
||||||
|
* caps one part, max_context_bytes caps the whole context. */
|
||||||
|
export interface ContextUploadProgress {
|
||||||
|
received: number;
|
||||||
|
part_max_bytes: number;
|
||||||
|
max_context_bytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
/** SubmissionListParams picks one page of a submission list (server-side filter
|
/** SubmissionListParams picks one page of a submission list (server-side filter
|
||||||
* and paging; the scope is the endpoint, never a parameter). */
|
* and paging; the scope is the endpoint, never a parameter). */
|
||||||
export interface SubmissionListParams {
|
export interface SubmissionListParams {
|
||||||
|
|||||||
@@ -10,7 +10,10 @@ import type { SubmissionPage } from "@/lib/types";
|
|||||||
const calls = vi.hoisted(() => ({
|
const calls = vi.hoisted(() => ({
|
||||||
listMySubmissions: vi.fn(),
|
listMySubmissions: vi.fn(),
|
||||||
submissionLimits: vi.fn(),
|
submissionLimits: vi.fn(),
|
||||||
|
createSubmission: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
const upload = vi.hoisted(() => ({ uploadContext: vi.fn() }));
|
||||||
|
vi.mock("@/lib/contextUpload", () => upload);
|
||||||
vi.mock("@/lib/config", () => ({ loadConfig: () => Promise.resolve({}) }));
|
vi.mock("@/lib/config", () => ({ loadConfig: () => Promise.resolve({}) }));
|
||||||
vi.mock("@/lib/api", async (importOriginal) => {
|
vi.mock("@/lib/api", async (importOriginal) => {
|
||||||
const actual = await importOriginal<typeof import("@/lib/api")>();
|
const actual = await importOriginal<typeof import("@/lib/api")>();
|
||||||
@@ -34,6 +37,9 @@ const PAGE: SubmissionPage = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
|
upload.uploadContext.mockReset();
|
||||||
|
calls.createSubmission.mockReset();
|
||||||
|
calls.createSubmission.mockResolvedValue({ id: "sub-9", display_name: "Skyblock Pack", status: "pending_review" });
|
||||||
calls.listMySubmissions.mockReset();
|
calls.listMySubmissions.mockReset();
|
||||||
calls.submissionLimits.mockReset();
|
calls.submissionLimits.mockReset();
|
||||||
calls.submissionLimits.mockResolvedValue({ max_context_bytes: 99614720 });
|
calls.submissionLimits.mockResolvedValue({ max_context_bytes: 99614720 });
|
||||||
@@ -84,4 +90,138 @@ describe("MySubmissionsPage", () => {
|
|||||||
await userEvent.upload(await screen.findByLabelText(/Build Context/), pack);
|
await userEvent.upload(await screen.findByLabelText(/Build Context/), pack);
|
||||||
expect(await screen.findByText("The file is 120 MB; one upload may be at most 95 MB.")).toBeTruthy();
|
expect(await screen.findByText("The file is 120 MB; one upload may be at most 95 MB.")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("shows the server's cap under the drop zone", async () => {
|
||||||
|
render(
|
||||||
|
<MemoryRouter>
|
||||||
|
<MySubmissionsPage />
|
||||||
|
</MemoryRouter>,
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(calls.submissionLimits).toHaveBeenCalled());
|
||||||
|
await userEvent.click(await screen.findByRole("button", { name: "Submit New Modpack" }));
|
||||||
|
expect(await screen.findByText("Supports .tar.gz, up to 95 MB")).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("shows the upload's progress and closes once it is stored", async () => {
|
||||||
|
let finish: () => void = () => {};
|
||||||
|
upload.uploadContext.mockImplementation(
|
||||||
|
(_id: string, _file: File, opts: { onProgress: (sent: number, total: number, stored: number) => void }) =>
|
||||||
|
new Promise<void>((resolve) => {
|
||||||
|
opts.onProgress(1536, 4096, 1024);
|
||||||
|
finish = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const pack = await openFormWith("Skyblock Pack");
|
||||||
|
await userEvent.click(screen.getByRole("button", { name: "Submit" }));
|
||||||
|
|
||||||
|
const bar = await screen.findByRole("progressbar", { name: "Upload progress" });
|
||||||
|
expect(bar.getAttribute("aria-valuenow")).toBe("37");
|
||||||
|
expect(screen.getByText("1.5 KB of 4 KB")).toBeTruthy();
|
||||||
|
expect(screen.getByText("37%")).toBeTruthy();
|
||||||
|
expect(calls.createSubmission).toHaveBeenCalledWith("Skyblock Pack");
|
||||||
|
expect(upload.uploadContext.mock.calls[0][0]).toBe("sub-9");
|
||||||
|
expect(upload.uploadContext.mock.calls[0][1]).toBe(pack);
|
||||||
|
expect(upload.uploadContext.mock.calls[0][2].resume).toBe(false);
|
||||||
|
|
||||||
|
finish();
|
||||||
|
await vi.waitFor(() => expect(screen.queryByRole("dialog")).toBeNull());
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a submission whose upload failed and carries it on instead of creating another", async () => {
|
||||||
|
upload.uploadContext.mockImplementationOnce(
|
||||||
|
async (_id: string, _file: File, opts: { onProgress: (sent: number, total: number, stored: number) => void }) => {
|
||||||
|
opts.onProgress(2048, 4096, 2048);
|
||||||
|
opts.onProgress(3072, 4096, 2048);
|
||||||
|
throw { status: 0, code: "network_error", message: "the upload did not reach the API" };
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const pack = await openFormWith("Skyblock Pack");
|
||||||
|
await userEvent.click(screen.getByRole("button", { name: "Submit" }));
|
||||||
|
|
||||||
|
expect(
|
||||||
|
await screen.findByText(
|
||||||
|
"Can't reach Felis: the network is down, or your Cloudflare Access sign-in expired. Reload the page to sign in again.",
|
||||||
|
),
|
||||||
|
).toBeTruthy();
|
||||||
|
expect(
|
||||||
|
screen.getByText(
|
||||||
|
"“Skyblock Pack” is saved and 2 KB of its build context is on the server. Submit again to continue from there.",
|
||||||
|
),
|
||||||
|
).toBeTruthy();
|
||||||
|
expect((screen.getByLabelText(/Display Name/) as HTMLInputElement).disabled).toBe(true);
|
||||||
|
|
||||||
|
upload.uploadContext.mockResolvedValueOnce(undefined);
|
||||||
|
await userEvent.click(screen.getByRole("button", { name: "Continue upload" }));
|
||||||
|
await vi.waitFor(() => expect(screen.queryByRole("dialog")).toBeNull());
|
||||||
|
expect(calls.createSubmission).toHaveBeenCalledTimes(1);
|
||||||
|
expect(upload.uploadContext).toHaveBeenCalledTimes(2);
|
||||||
|
expect(upload.uploadContext.mock.calls[1][0]).toBe("sub-9");
|
||||||
|
expect(upload.uploadContext.mock.calls[1][1]).toBe(pack);
|
||||||
|
expect(upload.uploadContext.mock.calls[1][2].resume).toBe(true);
|
||||||
|
|
||||||
|
// Stored: the next submission starts from an empty form.
|
||||||
|
await userEvent.click(screen.getByRole("button", { name: "Submit New Modpack" }));
|
||||||
|
const name = (await screen.findByLabelText(/Display Name/)) as HTMLInputElement;
|
||||||
|
expect(name.value).toBe("");
|
||||||
|
expect(name.disabled).toBe(false);
|
||||||
|
expect(screen.queryByText(/is saved/)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uploads a newly picked file from the start to the same submission", async () => {
|
||||||
|
upload.uploadContext.mockRejectedValueOnce({ status: 0, code: "network_error", message: "x" });
|
||||||
|
await openFormWith("Skyblock Pack");
|
||||||
|
await userEvent.click(screen.getByRole("button", { name: "Submit" }));
|
||||||
|
await screen.findByRole("button", { name: "Continue upload" });
|
||||||
|
|
||||||
|
const other = new File(["\x1f\x8bother"], "other.tar.gz", { type: "application/gzip" });
|
||||||
|
await userEvent.upload(screen.getByLabelText(/Build Context/), other);
|
||||||
|
expect(screen.getByText("“Skyblock Pack” is saved. The file you picked uploads from the start.")).toBeTruthy();
|
||||||
|
upload.uploadContext.mockResolvedValueOnce(undefined);
|
||||||
|
await userEvent.click(screen.getByRole("button", { name: "Submit" }));
|
||||||
|
await vi.waitFor(() => expect(upload.uploadContext).toHaveBeenCalledTimes(2));
|
||||||
|
expect(calls.createSubmission).toHaveBeenCalledTimes(1);
|
||||||
|
expect(upload.uploadContext.mock.calls[1][0]).toBe("sub-9");
|
||||||
|
expect(upload.uploadContext.mock.calls[1][1]).toBe(other);
|
||||||
|
expect(upload.uploadContext.mock.calls[1][2].resume).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("Pause stops the upload without an error and offers to continue", async () => {
|
||||||
|
upload.uploadContext.mockImplementation(
|
||||||
|
(_id: string, _file: File, opts: { signal: AbortSignal; onProgress: (sent: number, total: number, stored: number) => void }) =>
|
||||||
|
new Promise<void>((_resolve, reject) => {
|
||||||
|
opts.onProgress(1536, 4096, 1024);
|
||||||
|
opts.signal.addEventListener("abort", () => reject(new DOMException("The upload was cancelled", "AbortError")));
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await openFormWith("Skyblock Pack");
|
||||||
|
await userEvent.click(screen.getByRole("button", { name: "Submit" }));
|
||||||
|
await screen.findByRole("progressbar", { name: "Upload progress" });
|
||||||
|
await userEvent.click(screen.getByRole("button", { name: "Pause" }));
|
||||||
|
|
||||||
|
expect(
|
||||||
|
await screen.findByText(
|
||||||
|
"“Skyblock Pack” is saved and 1 KB of its build context is on the server. Submit again to continue from there.",
|
||||||
|
),
|
||||||
|
).toBeTruthy();
|
||||||
|
expect(screen.queryByRole("alert")).toBeNull();
|
||||||
|
expect(screen.queryByRole("progressbar")).toBeNull();
|
||||||
|
expect(screen.getByRole("button", { name: "Continue upload" })).toBeTruthy();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// openFormWith opens the submit dialog, names the pack and picks a 4 KiB file.
|
||||||
|
async function openFormWith(name: string): Promise<File> {
|
||||||
|
render(
|
||||||
|
<MemoryRouter>
|
||||||
|
<MySubmissionsPage />
|
||||||
|
</MemoryRouter>,
|
||||||
|
);
|
||||||
|
await screen.findByText("Create Above and Beyond");
|
||||||
|
await vi.waitFor(() => expect(calls.submissionLimits).toHaveBeenCalled());
|
||||||
|
await userEvent.click(screen.getByRole("button", { name: "Submit New Modpack" }));
|
||||||
|
await userEvent.type(await screen.findByLabelText(/Display Name/), name);
|
||||||
|
const pack = new File(["\x1f\x8bpack"], "pack.tar.gz", { type: "application/gzip" });
|
||||||
|
Object.defineProperty(pack, "size", { value: 4096 });
|
||||||
|
await userEvent.upload(screen.getByLabelText(/Build Context/), pack);
|
||||||
|
return pack;
|
||||||
|
}
|
||||||
@@ -38,6 +38,7 @@ import { StatCard } from "@/components/StatCard";
|
|||||||
import { PageHeader } from "@/components/PageHeader";
|
import { PageHeader } from "@/components/PageHeader";
|
||||||
import { SubmissionStatusBadge } from "@/components/SubmissionStatusBadge";
|
import { SubmissionStatusBadge } from "@/components/SubmissionStatusBadge";
|
||||||
import { api, humanizeError } from "@/lib/api";
|
import { api, humanizeError } from "@/lib/api";
|
||||||
|
import { uploadContext } from "@/lib/contextUpload";
|
||||||
import { useAsync } from "@/lib/hooks";
|
import { useAsync } from "@/lib/hooks";
|
||||||
import { formatRelative, formatAbsolute } from "@/lib/format";
|
import { formatRelative, formatAbsolute } from "@/lib/format";
|
||||||
import type { BuildStatus, Submission, SubmissionStatus } from "@/lib/types";
|
import type { BuildStatus, Submission, SubmissionStatus } from "@/lib/types";
|
||||||
@@ -89,6 +90,13 @@ export function MySubmissionsPage() {
|
|||||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||||
const [submitStep, setSubmitStep] = useState<"create" | "upload" | null>(null);
|
const [submitStep, setSubmitStep] = useState<"create" | "upload" | null>(null);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [progress, setProgress] = useState<{ sent: number; total: number } | null>(null);
|
||||||
|
// A submission whose upload stopped (a failure, or Pause): submitting again
|
||||||
|
// sends to it, carrying on from its staged bytes when the file is the same,
|
||||||
|
// instead of creating a second pending submission.
|
||||||
|
const [pending, setPending] = useState<{ id: string; name: string; file: File; sent: number } | null>(null);
|
||||||
|
const sentRef = useRef(0);
|
||||||
|
const abortRef = useRef<AbortController | null>(null);
|
||||||
|
|
||||||
// Row action state: withdraw arms a row (trash → confirm/cancel) before it
|
// Row action state: withdraw arms a row (trash → confirm/cancel) before it
|
||||||
// fires, and a failure lands in actionError above the list.
|
// fires, and a failure lands in actionError above the list.
|
||||||
@@ -185,7 +193,7 @@ export function MySubmissionsPage() {
|
|||||||
setFile(null);
|
setFile(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// The server's cap (95 MiB behind the Cloudflare edge); 1 GiB until it answers.
|
// The server's cap; 1 GiB until it answers.
|
||||||
const maxBytes = contextLimit ?? DEFAULT_CONTEXT_LIMIT;
|
const maxBytes = contextLimit ?? DEFAULT_CONTEXT_LIMIT;
|
||||||
if (selectedFile.size > maxBytes) {
|
if (selectedFile.size > maxBytes) {
|
||||||
setError(t("error_file_size", { size: formatBytes(selectedFile.size), limit: formatBytes(maxBytes) }));
|
setError(t("error_file_size", { size: formatBytes(selectedFile.size), limit: formatBytes(maxBytes) }));
|
||||||
@@ -199,7 +207,7 @@ export function MySubmissionsPage() {
|
|||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
setError(null);
|
setError(null);
|
||||||
|
|
||||||
const trimmedName = displayName.trim();
|
const trimmedName = pending ? pending.name : displayName.trim();
|
||||||
if (!trimmedName) {
|
if (!trimmedName) {
|
||||||
setError(t("error_name_required"));
|
setError(t("error_name_required"));
|
||||||
return;
|
return;
|
||||||
@@ -210,16 +218,32 @@ export function MySubmissionsPage() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
setIsSubmitting(true);
|
setIsSubmitting(true);
|
||||||
setSubmitStep("create");
|
let target = pending;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// 1. Create the submission metadata
|
if (!target) {
|
||||||
const sub = await api.createSubmission(trimmedName);
|
setSubmitStep("create");
|
||||||
|
const sub = await api.createSubmission(trimmedName);
|
||||||
|
target = { id: sub.id, name: trimmedName, file, sent: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
// 2. Upload context file
|
// The context goes up in parts, so a large pack passes the edge's
|
||||||
|
// per-request cap and a dropped connection resumes instead of restarting.
|
||||||
setSubmitStep("upload");
|
setSubmitStep("upload");
|
||||||
await api.uploadSubmissionContext(sub.id, file);
|
sentRef.current = 0;
|
||||||
|
setProgress({ sent: 0, total: file.size });
|
||||||
|
const controller = new AbortController();
|
||||||
|
abortRef.current = controller;
|
||||||
|
await uploadContext(target.id, file, {
|
||||||
|
resume: pending !== null && pending.file === file,
|
||||||
|
signal: controller.signal,
|
||||||
|
onProgress: (sent, total, stored) => {
|
||||||
|
// A stopped part is lost; the retry resumes from what the server holds.
|
||||||
|
sentRef.current = stored;
|
||||||
|
setProgress({ sent, total });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
setPending(null);
|
||||||
setDisplayName("");
|
setDisplayName("");
|
||||||
setFile(null);
|
setFile(null);
|
||||||
if (fileInputRef.current) {
|
if (fileInputRef.current) {
|
||||||
@@ -228,13 +252,23 @@ export function MySubmissionsPage() {
|
|||||||
setDialogOpen(false);
|
setDialogOpen(false);
|
||||||
reload();
|
reload();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(humanizeError(err));
|
if (target) {
|
||||||
|
// The submission exists (and is in the list now); keep it for the retry.
|
||||||
|
setPending({ ...target, file, sent: sentRef.current });
|
||||||
|
if (!pending) reload();
|
||||||
|
}
|
||||||
|
const paused = err instanceof DOMException && err.name === "AbortError";
|
||||||
|
setError(paused ? null : humanizeError(err));
|
||||||
} finally {
|
} finally {
|
||||||
|
abortRef.current = null;
|
||||||
setIsSubmitting(false);
|
setIsSubmitting(false);
|
||||||
setSubmitStep(null);
|
setSubmitStep(null);
|
||||||
|
setProgress(null);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const percent = progress && progress.total > 0 ? Math.floor((progress.sent * 100) / progress.total) : 0;
|
||||||
|
|
||||||
// Withdraw retracts a still-pending submission and its uploaded context,
|
// Withdraw retracts a still-pending submission and its uploaded context,
|
||||||
// freeing the pending slot and the storage budget for a fresh submission.
|
// freeing the pending slot and the storage budget for a fresh submission.
|
||||||
async function handleWithdraw(id: string) {
|
async function handleWithdraw(id: string) {
|
||||||
@@ -262,9 +296,12 @@ export function MySubmissionsPage() {
|
|||||||
actions={
|
actions={
|
||||||
<Button
|
<Button
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
setError(null);
|
// A stopped upload stays in the form so Submit carries it on.
|
||||||
setDisplayName("");
|
if (!pending) {
|
||||||
setFile(null);
|
setError(null);
|
||||||
|
setDisplayName("");
|
||||||
|
setFile(null);
|
||||||
|
}
|
||||||
setDialogOpen(true);
|
setDialogOpen(true);
|
||||||
}}
|
}}
|
||||||
size="sm"
|
size="sm"
|
||||||
@@ -559,6 +596,14 @@ export function MySubmissionsPage() {
|
|||||||
{/* Error Banner */}
|
{/* Error Banner */}
|
||||||
{error && <MessageLine kind="error" message={error} compact />}
|
{error && <MessageLine kind="error" message={error} compact />}
|
||||||
|
|
||||||
|
{pending && !isSubmitting && (
|
||||||
|
<p role="status" className="rounded-md border border-border bg-muted/40 p-2.5 text-xs text-muted-foreground">
|
||||||
|
{file === pending.file
|
||||||
|
? t("resume_hint", { name: pending.name, sent: formatBytes(pending.sent) })
|
||||||
|
: t("restart_hint", { name: pending.name })}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
{/* Display Name Input */}
|
{/* Display Name Input */}
|
||||||
<div className="space-y-1.5">
|
<div className="space-y-1.5">
|
||||||
<Label htmlFor="displayName" className="text-xs font-semibold text-foreground">
|
<Label htmlFor="displayName" className="text-xs font-semibold text-foreground">
|
||||||
@@ -567,9 +612,9 @@ export function MySubmissionsPage() {
|
|||||||
<Input
|
<Input
|
||||||
id="displayName"
|
id="displayName"
|
||||||
placeholder={t("display_name_placeholder")}
|
placeholder={t("display_name_placeholder")}
|
||||||
value={displayName}
|
value={pending ? pending.name : displayName}
|
||||||
onChange={(e) => setDisplayName(e.target.value)}
|
onChange={(e) => setDisplayName(e.target.value)}
|
||||||
disabled={isSubmitting}
|
disabled={isSubmitting || !!pending}
|
||||||
required
|
required
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
@@ -633,25 +678,56 @@ export function MySubmissionsPage() {
|
|||||||
<>
|
<>
|
||||||
<Upload className="h-8 w-8 text-muted-foreground/80 mb-2" />
|
<Upload className="h-8 w-8 text-muted-foreground/80 mb-2" />
|
||||||
<p className="text-xs font-medium text-foreground">{t("file_drag_hint")}</p>
|
<p className="text-xs font-medium text-foreground">{t("file_drag_hint")}</p>
|
||||||
<p className="text-[10px] text-muted-foreground/70 mt-1">{t("file_hint")}</p>
|
<p className="text-[10px] text-muted-foreground/70 mt-1">
|
||||||
|
{t("file_hint", { limit: formatBytes(contextLimit ?? DEFAULT_CONTEXT_LIMIT, 0) })}
|
||||||
|
</p>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{progress && (
|
||||||
|
<div className="space-y-1.5 pt-1">
|
||||||
|
<div
|
||||||
|
role="progressbar"
|
||||||
|
aria-label={t("upload_progress_label")}
|
||||||
|
aria-valuemin={0}
|
||||||
|
aria-valuemax={100}
|
||||||
|
aria-valuenow={percent}
|
||||||
|
className="h-1.5 w-full overflow-hidden rounded-full bg-muted"
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className="h-full rounded-full bg-primary transition-[width] duration-300 ease-out"
|
||||||
|
style={{ width: `${percent}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<p className="flex justify-between text-[10px] text-muted-foreground tabular-nums">
|
||||||
|
<span>{t("upload_progress", { sent: formatBytes(progress.sent), total: formatBytes(progress.total) })}</span>
|
||||||
|
<span>{percent}%</span>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<DialogFooter className="gap-2 pt-2">
|
<DialogFooter className="gap-2 pt-2">
|
||||||
<Button
|
{submitStep === "upload" ? (
|
||||||
type="button"
|
// Pausing keeps the staged parts; Submit carries on from them.
|
||||||
variant="outline"
|
<Button type="button" variant="outline" onClick={() => abortRef.current?.abort()} className="text-xs">
|
||||||
onClick={() => setDialogOpen(false)}
|
{t("pause_btn")}
|
||||||
disabled={isSubmitting}
|
</Button>
|
||||||
className="text-xs"
|
) : (
|
||||||
>
|
<Button
|
||||||
{t("common:cancel")}
|
type="button"
|
||||||
</Button>
|
variant="outline"
|
||||||
|
onClick={() => setDialogOpen(false)}
|
||||||
|
disabled={isSubmitting}
|
||||||
|
className="text-xs"
|
||||||
|
>
|
||||||
|
{t("common:cancel")}
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
<Button
|
<Button
|
||||||
type="submit"
|
type="submit"
|
||||||
disabled={isSubmitting || !displayName.trim() || !file}
|
disabled={isSubmitting || !(pending ? pending.name : displayName.trim()) || !file}
|
||||||
className="text-xs"
|
className="text-xs"
|
||||||
>
|
>
|
||||||
{isSubmitting ? (
|
{isSubmitting ? (
|
||||||
@@ -662,7 +738,7 @@ export function MySubmissionsPage() {
|
|||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
<Upload className="mr-1.5 h-4 w-4" />
|
<Upload className="mr-1.5 h-4 w-4" />
|
||||||
{t("submit_btn")}
|
{pending && file === pending.file ? t("resume_btn") : t("submit_btn")}
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</Button>
|
</Button>
|
||||||
|
|||||||
Reference in new issue
Block a user