feat(submit): 模组上传改为分片续传并显示进度,经 Cloudflare 边缘也能传满 1GiB

This commit is contained in:
Lemon-miaow committed 2026-09-25 22:39:43 +08:00
1 parent 7f160e2feb
commit 7bf81a0921
28 files changed
+2543 -154

No files matched your search

+166
View File
@@ -1073,6 +1073,172 @@ describe("session and connection signals", () => {
});
});
// A part of a chunked context upload goes by XMLHttpRequest for its upload
// progress; its outcomes must read exactly like a fetch call's.
class FakeXHR {
static last: FakeXHR | undefined;
method = "";
url = "";
withCredentials = false;
headers: Record<string, string> = {};
body: unknown = undefined;
status = 0;
statusText = "";
responseText = "";
aborted = false;
upload: { onprogress: ((e: { loaded: number }) => void) | null } = { onprogress: null };
onload: (() => void) | null = null;
onerror: (() => void) | null = null;
onabort: (() => void) | null = null;
constructor() {
FakeXHR.last = this;
}
open(method: string, url: string) {
this.method = method;
this.url = url;
}
setRequestHeader(k: string, v: string) {
this.headers[k] = v;
}
send(body: unknown) {
this.body = body;
}
abort() {
this.aborted = true;
this.onabort?.();
}
respond(status: number, body: string, statusText = "") {
this.status = status;
this.statusText = statusText;
this.responseText = body;
this.onload?.();
}
}
async function sentXHR(): Promise<FakeXHR> {
await vi.waitFor(() => expect(FakeXHR.last?.body).toBeDefined());
return FakeXHR.last as FakeXHR;
}
describe("chunked context upload", () => {
beforeEach(() => {
vi.restoreAllMocks();
FakeXHR.last = undefined;
vi.stubGlobal("XMLHttpRequest", FakeXHR);
});
afterEach(() => vi.unstubAllGlobals());
const progress = { received: 12, part_max_bytes: 33554432, max_context_bytes: 1073741824 };
it("getContextUpload GETs where the staged upload stands", async () => {
const fetchSpy = fakeFetch({ received: 4, part_max_bytes: 33554432, max_context_bytes: 1073741824 });
vi.stubGlobal("fetch", fetchSpy);
expect(await api.getContextUpload("sub-3")).toEqual({ received: 4, part_max_bytes: 33554432, max_context_bytes: 1073741824 });
const [url, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
expect(String(url)).toBe("/me/submissions/sub-3/context/upload");
expect((opts as RequestInit).method).toBe("GET");
});
it("completeContextUpload POSTs to store the staged upload", async () => {
const sub = { id: "sub-3", display_name: "new submission", status: "pending_review" };
const fetchSpy = fakeFetch(sub);
vi.stubGlobal("fetch", fetchSpy);
expect(await api.completeContextUpload("sub-3")).toEqual(sub);
const [url, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
expect(String(url)).toBe("/me/submissions/sub-3/context/upload/complete");
expect((opts as RequestInit).method).toBe("POST");
});
it("putContextPart PUTs the part at its offset with the session cookie and reports progress", async () => {
const part = new Blob(["abcd"]);
const seen: number[] = [];
const done = api.putContextPart("sub-3", 8, part, { onProgress: (n) => seen.push(n) });
const xhr = await sentXHR();
expect(xhr.method).toBe("PUT");
expect(xhr.url).toBe("/me/submissions/sub-3/context/upload?offset=8");
expect(xhr.withCredentials).toBe(true);
expect(xhr.headers).toEqual({ "Content-Type": "application/octet-stream" });
expect(xhr.body).toBe(part);
xhr.upload.onprogress?.({ loaded: 3 });
xhr.respond(200, JSON.stringify(progress));
expect(await done).toEqual({ received: 12, part_max_bytes: 33554432, max_context_bytes: 1073741824 });
expect(seen).toEqual([3]);
});
it("putContextPart refuses a route value that is not one path segment", async () => {
await expect(api.putContextPart("..", 0, new Blob(["x"]))).rejects.toMatchObject({ code: "bad_path_param" });
expect(FakeXHR.last).toBeUndefined();
});
it("an API refusal reads as its code and message", async () => {
const done = api.putContextPart("sub-3", 8, new Blob(["abcd"]));
(await sentXHR()).respond(
409,
JSON.stringify({ error: { code: "upload_offset_mismatch", message: "the upload holds 4 bytes; send the part that starts there" } }),
"Conflict",
);
await expect(done).rejects.toEqual({
status: 409,
code: "upload_offset_mismatch",
message: "the upload holds 4 bytes; send the part that starts there",
});
});
it("a tunnel page in place of the API reads as upstream_unavailable", async () => {
const done = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
(await sentXHR()).respond(524, "<html>A timeout occurred</html>", "");
await expect(done).rejects.toEqual({ status: 524, code: "upstream_unavailable", message: "" });
});
it("a 2xx that is not JSON reads as upstream_unavailable", async () => {
const done = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
(await sentXHR()).respond(200, "<html>", "OK");
await expect(done).rejects.toEqual({ status: 200, code: "upstream_unavailable", message: "the response was not JSON" });
});
it("an expired session is announced like any protected call", async () => {
const target = new EventTarget();
vi.stubGlobal("window", target);
const seen: string[] = [];
target.addEventListener(SESSION_EXPIRED_EVENT, () => seen.push("expired"));
const done = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
(await sentXHR()).respond(401, JSON.stringify({ error: { code: "unauthorized", message: "sign in" } }));
await expect(done).rejects.toMatchObject({ status: 401, code: "unauthorized" });
expect(seen).toEqual(["expired"]);
});
it("no response at all is a network error that marks the connection lost, until one gets through", async () => {
const lost = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
(await sentXHR()).onerror?.();
await expect(lost).rejects.toEqual({ status: 0, code: "network_error", message: "the upload did not reach the API" });
expect(isConnectionLost()).toBe(true);
FakeXHR.last = undefined;
const back = api.putContextPart("sub-3", 0, new Blob(["abcd"]));
(await sentXHR()).respond(200, JSON.stringify(progress));
await back;
expect(isConnectionLost()).toBe(false);
});
it("pausing aborts the request in flight", async () => {
const controller = new AbortController();
const done = api.putContextPart("sub-3", 0, new Blob(["abcd"]), { signal: controller.signal });
const xhr = await sentXHR();
controller.abort();
expect(xhr.aborted).toBe(true);
await expect(done).rejects.toMatchObject({ name: "AbortError" });
});
it("a signal already paused sends nothing", async () => {
const controller = new AbortController();
controller.abort();
await expect(
api.putContextPart("sub-3", 0, new Blob(["abcd"]), { signal: controller.signal }),
).rejects.toMatchObject({ name: "AbortError" });
expect(FakeXHR.last?.body).toBeUndefined();
});
});
// The API's generic codes carry an English developer message ("user not found",
// "invalid request"); the panel words them itself so a Chinese UI never shows it.
describe("copy for the generic server codes", () => {
+105 -6
View File
@@ -6,6 +6,7 @@ import type {
BanlistResult,
Build,
BuildScan,
ContextUploadProgress,
CreateServerRequest,
CreateUserRequest,
FleetServer,
@@ -148,22 +149,34 @@ async function fetchOK(path: string, init: RequestInit): Promise<Response> {
reportConnection(true);
if (res.ok) return res;
let text = "";
try {
text = await res.text();
} catch {
/* the body broke off; the status still says what happened */
}
throw failed(path, res.status, res.statusText, text);
}
// failed turns a non-2xx answer into the ApiError the callers see and announces
// the codes the app shell acts on.
function failed(path: string, status: number, statusText: string, text: string): ApiError {
let parsed: unknown = null;
try {
parsed = JSON.parse(await res.text());
parsed = JSON.parse(text);
} catch {
/* no body, or not JSON: an ingress or tunnel answered */
}
const err: ApiError = isApiError(parsed)
? { status: res.status, code: parsed.error.code, message: parsed.error.message }
? { status, code: parsed.error.code, message: parsed.error.message }
: {
status: res.status,
code: res.status >= 500 ? "upstream_unavailable" : "error",
message: res.statusText,
status,
code: status >= 500 ? "upstream_unavailable" : "error",
message: statusText,
};
announceSetupRequired(err);
announceSessionExpired(err, path);
throw err;
return err;
}
// send returns a 2xx response's parsed JSON body (null for an empty one).
@@ -206,6 +219,63 @@ function requestRaw<T>(
return send<T>(path, { method, headers, body });
}
// sendWithProgress sends body by XMLHttpRequest, the one browser API that
// reports how much of a request body has gone out (fetch has no upload
// progress), and settles the way send does: the parsed 2xx body, or the same
// ApiError fetchOK would throw. onProgress gets the bytes of body sent so far;
// signal aborts the request with an AbortError.
async function sendWithProgress<T>(
method: string,
path: string,
body: Blob,
opts: { onProgress?: (sent: number) => void; signal?: AbortSignal } = {},
): Promise<T> {
const { apiBase } = await loadConfig();
const { onProgress, signal } = opts;
return new Promise<T>((resolve, reject) => {
const xhr = new XMLHttpRequest();
const onAbort = () => xhr.abort();
const settle = () => signal?.removeEventListener("abort", onAbort);
xhr.open(method, `${apiBase}${path}`);
xhr.withCredentials = true;
xhr.setRequestHeader("Content-Type", "application/octet-stream");
if (onProgress) xhr.upload.onprogress = (e) => onProgress(e.loaded);
xhr.onabort = () => {
settle();
reject(new DOMException("The upload was cancelled", "AbortError"));
};
xhr.onerror = () => {
settle();
reportConnection(false);
reject(networkError(new Error("the upload did not reach the API")));
};
xhr.onload = () => {
settle();
reportConnection(true);
if (xhr.status < 200 || xhr.status >= 300) {
reject(failed(path, xhr.status, xhr.statusText, xhr.responseText));
return;
}
try {
resolve((xhr.responseText ? JSON.parse(xhr.responseText) : null) as T);
} catch {
const err: ApiError = {
status: xhr.status,
code: "upstream_unavailable",
message: "the response was not JSON",
};
reject(err);
}
};
if (signal?.aborted) {
reject(new DOMException("The upload was cancelled", "AbortError"));
return;
}
signal?.addEventListener("abort", onAbort);
xhr.send(body);
});
}
// rejectingSync turns a synchronous throw inside an api method (urlPath refusing
// a segment) into a rejected promise, so every caller handles it the way it
// handles any failed call.
@@ -746,6 +816,27 @@ export const api = rejectingSync({
"Content-Type": "application/x-gzip",
}),
// A chunked context upload (lib/contextUpload.ts drives it): ask where the
// staged upload stands, send each part at its byte offset, then store it.
getContextUpload: (id: string) =>
request<ContextUploadProgress>("GET", urlPath`/me/submissions/${id}/context/upload`),
putContextPart: (
id: string,
offset: number,
part: Blob,
opts?: { onProgress?: (sent: number) => void; signal?: AbortSignal },
) =>
sendWithProgress<ContextUploadProgress>(
"PUT",
urlPath`/me/submissions/${id}/context/upload` + `?offset=${offset}`,
part,
opts,
),
completeContextUpload: (id: string) =>
request<Submission>("POST", urlPath`/me/submissions/${id}/context/upload/complete`),
// Retract the caller's own pending submission (and its uploaded context), which
// frees their pending slot and storage budget. Reviewed submissions are frozen.
withdrawSubmission: (id: string) => request<Submission>("DELETE", urlPath`/me/submissions/${id}`),
@@ -1031,6 +1122,14 @@ export function humanizeError(e: unknown): string {
return t("submissions_unavailable");
case "uploads_unavailable":
return t("uploads_unavailable");
// Chunked uploads: the client resumes on these by itself, so they surface
// only once its retries run out.
case "upload_busy":
return t("upload_busy");
case "upload_offset_mismatch":
return t("upload_offset_mismatch");
case "part_too_large":
return t("part_too_large");
case "backup_unavailable":
return t("backup_unavailable");
// Account migration + the re-auth steps it depends on: every refusal below is
+264
View File
@@ -0,0 +1,264 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
const calls = vi.hoisted(() => ({
getContextUpload: vi.fn(),
putContextPart: vi.fn(),
completeContextUpload: vi.fn(),
}));
vi.mock("./api", () => ({ api: calls }));
const { uploadContext } = await import("./contextUpload");
// A 10-byte "file" and a server that takes parts of at most 4 bytes.
const FILE = new Blob(["0123456789"]);
const at = (received: number) => ({ received, part_max_bytes: 4, max_context_bytes: 1073741824 });
const dropped = { status: 0, code: "network_error", message: "Failed to fetch" };
const edgeTimeout = { status: 524, code: "upstream_unavailable", message: "" };
// A server that appends each part where it says and answers the new length.
function acceptParts() {
calls.putContextPart.mockImplementation(async (_id: string, offset: number, part: Blob) => at(offset + part.size));
}
async function sentParts(): Promise<Array<[number, string]>> {
return Promise.all(
calls.putContextPart.mock.calls.map(async ([, offset, part]) => [offset as number, await (part as Blob).text()] as [number, string]),
);
}
const sleep = vi.fn(async (_ms: number, _signal?: AbortSignal) => undefined);
beforeEach(() => {
calls.getContextUpload.mockReset();
calls.putContextPart.mockReset();
calls.completeContextUpload.mockReset();
calls.completeContextUpload.mockResolvedValue({ id: "sub-1" });
sleep.mockClear();
});
describe("uploadContext", () => {
it("sends the file in parts no larger than the server's cap, then stores it", async () => {
calls.getContextUpload.mockResolvedValue(at(0));
acceptParts();
const progress: Array<[number, number]> = [];
await uploadContext("sub-1", FILE, { sleep, onProgress: (sent, total) => progress.push([sent, total]) });
expect(await sentParts()).toEqual([
[0, "0123"],
[4, "4567"],
[8, "89"],
]);
expect(calls.putContextPart.mock.calls.map((c) => c[0])).toEqual(["sub-1", "sub-1", "sub-1"]);
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
expect(calls.completeContextUpload).toHaveBeenCalledWith("sub-1");
expect(progress.at(-1)).toEqual([10, 10]);
});
it("reports the bytes of a part in flight on top of the parts already stored", async () => {
calls.getContextUpload.mockResolvedValue(at(0));
calls.putContextPart.mockImplementation(async (_id, offset: number, part: Blob, opts) => {
opts.onProgress(2);
return at(offset + part.size);
});
const progress: Array<[number, number, number]> = [];
await uploadContext("sub-1", FILE, {
sleep,
onProgress: (sent, total, stored) => progress.push([sent, total, stored]),
});
expect(progress).toEqual([
[0, 10, 0],
[2, 10, 0],
[4, 10, 4],
[6, 10, 4],
[8, 10, 8],
[10, 10, 8],
[10, 10, 10],
]);
});
it("starts a fresh file over even when the submission has parts staged", async () => {
calls.getContextUpload.mockResolvedValue(at(4));
acceptParts();
await uploadContext("sub-1", FILE, { sleep });
expect((await sentParts()).map(([o]) => o)).toEqual([0, 4, 8]);
});
it("resumes the same file from where the staged bytes end", async () => {
calls.getContextUpload.mockResolvedValue(at(4));
acceptParts();
await uploadContext("sub-1", FILE, { sleep, resume: true });
expect(await sentParts()).toEqual([
[4, "4567"],
[8, "89"],
]);
});
it("goes straight to storing when every byte is already staged", async () => {
calls.getContextUpload.mockResolvedValue(at(10));
await uploadContext("sub-1", FILE, { sleep, resume: true });
expect(calls.putContextPart).not.toHaveBeenCalled();
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
});
it("starts over when more is staged than the file holds", async () => {
calls.getContextUpload.mockResolvedValue(at(12));
acceptParts();
await uploadContext("sub-1", FILE, { sleep, resume: true });
expect((await sentParts()).map(([o]) => o)).toEqual([0, 4, 8]);
});
it("after a dropped part asks where the upload stands and carries on from there", async () => {
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValueOnce(at(4));
acceptParts();
calls.putContextPart.mockImplementationOnce(async (_id, offset: number, part: Blob) => at(offset + part.size));
calls.putContextPart.mockRejectedValueOnce(dropped);
await uploadContext("sub-1", FILE, { sleep });
expect((await sentParts()).map(([o]) => o)).toEqual([0, 4, 4, 8]);
expect(calls.getContextUpload).toHaveBeenCalledTimes(2);
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000]);
});
it("a fresh file whose first part dropped starts over at 0", async () => {
calls.getContextUpload.mockResolvedValue(at(2));
acceptParts();
calls.putContextPart.mockRejectedValueOnce(dropped);
await uploadContext("sub-1", FILE, { sleep });
expect((await sentParts()).map(([o]) => o)).toEqual([0, 0, 4, 8]);
});
it("follows the server when it says the upload holds a different length", async () => {
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValueOnce(at(8));
acceptParts();
calls.putContextPart.mockImplementationOnce(async () => at(4));
calls.putContextPart.mockRejectedValueOnce({ status: 409, code: "upload_offset_mismatch", message: "" });
await uploadContext("sub-1", FILE, { sleep });
expect((await sentParts()).map(([o]) => o)).toEqual([0, 4, 8]);
});
it("gives up at once on a refusal the next attempt cannot outlast", async () => {
calls.getContextUpload.mockResolvedValue(at(0));
const refused = { status: 400, code: "bad_request", message: "context must be a gzip-compressed tarball" };
calls.putContextPart.mockRejectedValue(refused);
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(refused);
expect(calls.putContextPart).toHaveBeenCalledTimes(1);
expect(sleep).not.toHaveBeenCalled();
expect(calls.completeContextUpload).not.toHaveBeenCalled();
});
it("gives up after eight dropped attempts in a row, backing off up to 15 s", async () => {
calls.getContextUpload.mockResolvedValue(at(0));
calls.putContextPart.mockRejectedValue(dropped);
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(dropped);
expect(calls.putContextPart).toHaveBeenCalledTimes(8);
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000, 2000, 4000, 8000, 15000, 15000, 15000]);
});
it("a part that gets through resets the count of dropped attempts", async () => {
calls.getContextUpload.mockImplementation(async () => at(calls.putContextPart.mock.calls.length >= 8 ? 4 : 0));
acceptParts();
for (let i = 0; i < 7; i++) calls.putContextPart.mockRejectedValueOnce(dropped);
calls.putContextPart.mockImplementationOnce(async () => at(4));
for (let i = 0; i < 7; i++) calls.putContextPart.mockRejectedValueOnce(dropped);
await uploadContext("sub-1", FILE, { sleep });
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
});
it("sends an empty file as one part, so the server gives the refusal", async () => {
calls.getContextUpload.mockResolvedValue(at(0));
const refused = { status: 400, code: "bad_request", message: "context must be a gzip-compressed tarball" };
calls.putContextPart.mockRejectedValue(refused);
await expect(uploadContext("sub-1", new Blob([]), { sleep })).rejects.toBe(refused);
expect(calls.putContextPart.mock.calls.map((c) => [c[1], (c[2] as Blob).size])).toEqual([[0, 0]]);
});
it("stops on a pause without retrying", async () => {
calls.getContextUpload.mockResolvedValue(at(0));
const paused = new DOMException("The upload was cancelled", "AbortError");
calls.putContextPart.mockRejectedValue(paused);
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(paused);
expect(calls.putContextPart).toHaveBeenCalledTimes(1);
expect(sleep).not.toHaveBeenCalled();
});
it("passes the pause signal to every part", async () => {
calls.getContextUpload.mockResolvedValue(at(0));
acceptParts();
const controller = new AbortController();
await uploadContext("sub-1", FILE, { sleep, signal: controller.signal });
expect(calls.putContextPart.mock.calls.map((c) => c[3].signal)).toEqual([
controller.signal,
controller.signal,
controller.signal,
]);
});
});
describe("uploadContext completion", () => {
beforeEach(() => {
acceptParts();
});
it("treats the staged upload vanishing after an edge timeout as stored", async () => {
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValueOnce(at(0));
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout);
await uploadContext("sub-1", FILE, { sleep });
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
expect(calls.getContextUpload).toHaveBeenCalledTimes(2);
});
it("waits out a store still in progress, then takes its answer", async () => {
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValue(at(10));
calls.completeContextUpload
.mockRejectedValueOnce(edgeTimeout)
.mockRejectedValueOnce({ status: 409, code: "upload_busy", message: "" })
.mockResolvedValueOnce({ id: "sub-1" });
await uploadContext("sub-1", FILE, { sleep });
expect(calls.completeContextUpload).toHaveBeenCalledTimes(3);
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000, 2000]);
});
it("surfaces the real refusal when completing again answers with one", async () => {
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValue(at(10));
const quota = { status: 403, code: "submission_quota_exceeded", message: "budget" };
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout).mockRejectedValueOnce(quota);
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(quota);
});
it("surfaces a refusal to store at once", async () => {
calls.getContextUpload.mockResolvedValue(at(0));
const refused = { status: 429, code: "submission_cooldown", message: "" };
calls.completeContextUpload.mockRejectedValueOnce(refused);
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(refused);
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
expect(sleep).not.toHaveBeenCalled();
});
it("reports the timeout when the staged upload is cut short while storing", async () => {
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValueOnce(at(6));
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout);
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(edgeTimeout);
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
});
it("ends the wait when checking on the store is refused", async () => {
const gone = { status: 404, code: "not_found", message: "submission not found" };
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockRejectedValueOnce(gone);
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout);
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(gone);
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
});
it("keeps waiting through a check that did not get through", async () => {
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockRejectedValueOnce(dropped).mockResolvedValueOnce(at(0));
calls.completeContextUpload.mockRejectedValueOnce(edgeTimeout);
await uploadContext("sub-1", FILE, { sleep });
expect(calls.completeContextUpload).toHaveBeenCalledTimes(1);
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000, 2000]);
});
it("gives up after forty checks on a store that never finishes", async () => {
calls.getContextUpload.mockResolvedValueOnce(at(0)).mockResolvedValue(at(10));
calls.completeContextUpload.mockRejectedValue(edgeTimeout);
await expect(uploadContext("sub-1", FILE, { sleep })).rejects.toBe(edgeTimeout);
expect(calls.completeContextUpload).toHaveBeenCalledTimes(41);
});
});
+145
View File
@@ -0,0 +1,145 @@
import { api } from "./api";
import type { ApiError } from "./types";
// uploadContext sends a submission's build context in parts. One request body is
// bounded by whatever proxy fronts the API (the Cloudflare edge refuses bodies
// over 100 MB), so the server stages parts of at most part_max_bytes, and the
// staged length is the resume point: after a dropped connection, a tunnel error
// or a busy answer, the loop asks the server where the upload stands and carries
// on from there. Completing stores the staged bytes as the context.
export interface UploadContextOptions {
/** Reports the bytes of file sent so far (stored plus the part in flight)
* and, as stored, the bytes the server has confirmed holding. */
onProgress?: (sent: number, total: number, stored: number) => void;
/** Cancels the upload; the staged parts stay for a later resume. */
signal?: AbortSignal;
/** Carry on from the parts this submission already staged, when they are a
* prefix of this same file (a retry after a failure). A fresh file starts
* over, which also discards whatever an earlier file left staged. */
resume?: boolean;
/** Test seam for the pause between attempts. */
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
}
/** Consecutive failed attempts a part may take before the upload gives up. */
export const MAX_ATTEMPTS = 8;
/** Checks while the server finishes storing a large context (about 10 minutes). */
export const MAX_COMPLETE_WAITS = 40;
// A transient answer is one the next attempt can outlast: no response at all, a
// tunnel or ingress page in place of the API's (upstream_unavailable), or a 409
// that means "ask where the upload stands and send again".
export function isTransient(e: unknown): boolean {
const err = e as Partial<ApiError> | null;
if (!err || typeof err.code !== "string") return false;
return (
err.code === "network_error" ||
err.code === "upstream_unavailable" ||
err.code === "upload_busy" ||
err.code === "upload_offset_mismatch"
);
}
/** retryDelay is the pause before attempt n+1: 1s, 2s, 4s, 8s, then 15s. */
export function retryDelay(n: number): number {
return Math.min(1000 * 2 ** (n - 1), 15000);
}
function wait(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(new DOMException("The upload was cancelled", "AbortError"));
return;
}
const timer = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
resolve();
}, ms);
const onAbort = () => {
clearTimeout(timer);
reject(new DOMException("The upload was cancelled", "AbortError"));
};
signal?.addEventListener("abort", onAbort, { once: true });
});
}
export async function uploadContext(id: string, file: Blob, opts: UploadContextOptions = {}): Promise<void> {
const { onProgress, signal } = opts;
const sleep = opts.sleep ?? wait;
const total = file.size;
// offset is where the next part starts; null means "ask the server first".
let offset: number | null = null;
let partMax = 0;
let fresh = !opts.resume;
let failures = 0;
for (;;) {
try {
if (offset === null) {
const at = await api.getContextUpload(id);
partMax = at.part_max_bytes;
// More staged than this file holds cannot be a prefix of it.
offset = fresh || at.received > total ? 0 : at.received;
onProgress?.(offset, total, offset);
}
if (offset < total || offset === 0) {
const start = offset;
const part = file.slice(start, Math.min(start + partMax, total));
const at = await api.putContextPart(id, start, part, {
signal,
onProgress: (sent) => onProgress?.(start + sent, total, start),
});
offset = at.received;
fresh = false;
failures = 0;
onProgress?.(offset, total, offset);
}
if (offset >= total) break;
} catch (e) {
// A pause (AbortError) is not transient, so it ends the upload here.
if (!isTransient(e) || ++failures >= MAX_ATTEMPTS) throw e;
await sleep(retryDelay(failures), signal);
offset = null;
}
}
await complete(id, total, sleep, signal);
}
// complete stores the staged upload. Behind the edge a large context can take
// longer to store than the edge waits for an answer (it gives up after 100 s),
// and the server carries on regardless. So a transient failure here is followed
// by watching the staged upload: still all there means the store has not taken
// it yet (or failed and kept it), so completing again either waits (upload_busy)
// or answers with the real outcome; gone means it was stored.
async function complete(
id: string,
total: number,
sleep: (ms: number, signal?: AbortSignal) => Promise<void>,
signal?: AbortSignal,
): Promise<void> {
let lastErr: unknown;
for (let waits = 0; waits <= MAX_COMPLETE_WAITS; waits++) {
if (waits > 0) {
await sleep(retryDelay(waits), signal);
try {
const at = await api.getContextUpload(id);
if (at.received < total) {
if (at.received === 0) return;
throw lastErr;
}
} catch (e) {
if (e === lastErr || !isTransient(e)) throw e;
continue;
}
}
try {
await api.completeContextUpload(id);
return;
} catch (e) {
if (!isTransient(e)) throw e;
lastErr = e;
}
}
throw lastErr;
}
+188 -2
View File
@@ -1856,7 +1856,7 @@ export interface paths {
};
/**
* The per-upload build-context cap
* @description The effective [registry] context_max_bytes: 1 GiB by default, 95 MiB behind the Cloudflare edge (its proxy refuses bodies over 100 MB before they reach the API). The panel checks a file against it before upload.
* @description The effective [registry] context_max_bytes, 1 GiB by default. The panel checks a file against it before upload and sends the file through the chunked upload (/api/v1/me/submissions/{id}/context/upload), so the cap holds behind the Cloudflare edge too, whose proxy refuses a single body over 100 MB.
*/
get: operations["submissionLimits"];
put?: never;
@@ -1878,7 +1878,7 @@ export interface paths {
put?: never;
/**
* Upload the modpack build context for your own pending submission (user side; user-directed lane over §16).
* @description The request body IS the raw gzip build context (context.tar.gz) — not JSON, not multipart — streamed to the platform-derived, id-namespaced location Kaniko reads via --context. The submitter is taken from the principal; a submission the caller does not own is reported as 404, so this endpoint cannot upload to or probe another user's submission. Only a pending_review submission accepts a context (409 otherwise); a wrong-format or oversize body is rejected with 400 (the per-upload cap is [registry] context_max_bytes: 1 GiB by default and 95 MiB behind the Cloudflare edge, whose proxy refuses bodies over 100 MB with its own HTML 413 before they reach the API; GET /api/v1/me/submissions/limits reports the effective cap so a client can check a file before sending it), and an upload that would push the caller past their per-user stored-context budget is refused with 403 before the excess is persisted. Returns 503 when the deployment's context store has no implemented upload transport.
* @description The request body IS the raw gzip build context (context.tar.gz) — not JSON, not multipart — streamed to the platform-derived, id-namespaced location Kaniko reads via --context. The submitter is taken from the principal; a submission the caller does not own is reported as 404, so this endpoint cannot upload to or probe another user's submission. Only a pending_review submission accepts a context (409 otherwise); a wrong-format or oversize body is rejected with 400 (the per-upload cap is [registry] context_max_bytes, 1 GiB by default; GET /api/v1/me/submissions/limits reports it so a client can check a file before sending it). This request carries the whole context, so behind the Cloudflare edge, whose proxy refuses bodies over 100 MB with its own HTML 413 before they reach the API, a larger context goes through the chunked upload at /api/v1/me/submissions/{id}/context/upload instead. An upload that would push the caller past their per-user stored-context budget is refused with 403 before the excess is persisted. Returns 503 when the deployment's context store has no implemented upload transport.
*/
post: operations["uploadSubmissionContext"];
delete?: never;
@@ -1887,6 +1887,50 @@ export interface paths {
patch?: never;
trace?: never;
};
"/api/v1/me/submissions/{id}/context/upload": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Where your chunked context upload stands (the resume point).
* @description The chunked form of POST /api/v1/me/submissions/{id}/context, for a context larger than one request carries through the edge. received is how many bytes are staged: the next part starts there. A client reads it before the first part and again after a failed one. Nothing staged reads as 0. Same owner scoping as the single upload (404 for another user's submission, 409 once reviewed).
*/
get: operations["getContextUpload"];
/**
* Append one part of your chunked context upload.
* @description The body is the part's raw bytes, at most part_max_bytes (32 MiB). offset is where they start: 0 starts the upload over, and anything else must equal the staged length, or the answer is 409 upload_offset_mismatch and the client reads GET for where to resume. The first part must open with the gzip magic (400). The staged total meets the same context cap (400) and storage budget (403) as a single upload. A part that breaks off is cut back off, so the staged bytes are always a prefix of the file. One request per upload at a time (409 upload_busy). Staged bytes untouched for 24 hours are deleted.
*/
put: operations["putContextUploadPart"];
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/me/submissions/{id}/context/upload/complete": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Store your staged chunked upload as the submission's build context.
* @description Runs every check of POST /api/v1/me/submissions/{id}/context on the staged bytes (format, cap, budget, room), records the digest the same way, and deletes the staged copy. Holds the same per-user upload cooldown (429) and writes the same submission.upload audit event. Nothing staged is 400. After a failure the staged bytes stay, for a retry.
*/
post: operations["completeContextUpload"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/me/submissions/{id}": {
parameters: {
query?: never;
@@ -2462,6 +2506,23 @@ export interface components {
/** Format: date-time */
added_at: string;
};
ContextUploadProgress: {
/**
* Format: int64
* @description Bytes staged so far; the next part starts here.
*/
received: number;
/**
* Format: int64
* @description The most one part may carry.
*/
part_max_bytes: number;
/**
* Format: int64
* @description The most the whole context may reach ([registry] context_max_bytes).
*/
max_context_bytes: number;
};
/** @description One user-submitted modpack in the approval lane (internal/submit Submission — a user-directed extension over the §16 build subsystem). The user supplies only display_name; submitted_by comes from the principal and context_ref/image_ref/build_id/reviewed_by are platform-controlled, never client input. */
Submission: {
id: string;
@@ -7434,6 +7495,131 @@ export interface operations {
503: components["responses"]["ServiceUnavailable"];
};
};
getContextUpload: {
parameters: {
query?: never;
header?: never;
path: {
id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description The staged length and the limits a part and the whole must keep. */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ContextUploadProgress"];
};
};
401: components["responses"]["Unauthorized"];
404: components["responses"]["NotFound"];
409: components["responses"]["Conflict"];
503: components["responses"]["ServiceUnavailable"];
};
};
putContextUploadPart: {
parameters: {
query: {
offset: number;
};
header?: never;
path: {
id: string;
};
cookie?: never;
};
requestBody: {
content: {
"application/octet-stream": string;
};
};
responses: {
/** @description The part is staged; received is the new length. */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ContextUploadProgress"];
};
};
400: components["responses"]["BadRequest"];
401: components["responses"]["Unauthorized"];
/** @description The staged total would exceed the caller's per-user stored-context budget (submission_quota_exceeded). */
403: {
headers: {
[name: string]: unknown;
};
content?: never;
};
404: components["responses"]["NotFound"];
409: components["responses"]["Conflict"];
/** @description The part is larger than part_max_bytes (part_too_large). */
413: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
503: components["responses"]["ServiceUnavailable"];
/** @description The uploads store is full (uploads_full). */
507: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
};
};
completeContextUpload: {
parameters: {
query?: never;
header?: never;
path: {
id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Context stored; the submission (unchanged) is returned. */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Submission"];
};
};
400: components["responses"]["BadRequest"];
401: components["responses"]["Unauthorized"];
/** @description The context would exceed the caller's per-user stored-context budget (submission_quota_exceeded). */
403: {
headers: {
[name: string]: unknown;
};
content?: never;
};
404: components["responses"]["NotFound"];
409: components["responses"]["Conflict"];
/** @description An upload was accepted within the per-user cooldown window (submission_cooldown). */
429: {
headers: {
[name: string]: unknown;
};
content?: never;
};
503: components["responses"]["ServiceUnavailable"];
};
};
withdrawSubmission: {
parameters: {
query?: never;
+1
View File
@@ -43,6 +43,7 @@ export type WireParity = [
Holds<Parity<T.ScanFinding, S["ScanFinding"]>>,
Holds<Parity<T.WhitelistImage, S["Image"]>>,
Holds<Parity<T.Submission, S["Submission"]>>,
Holds<Parity<T.ContextUploadProgress, S["ContextUploadProgress"]>>,
Holds<Parity<T.UserView, S["UserView"]>>,
Holds<Parity<T.UserDetail, S["UserDetail"]>>,
Holds<Parity<T.QuotaView, S["QuotaView"]>>,
+9
View File
@@ -378,6 +378,15 @@ export interface Submission {
context_sha256?: string;
}
/** ContextUploadProgress is where a chunked context upload stands: received is
* how many bytes the server holds (the next part starts there), part_max_bytes
* caps one part, max_context_bytes caps the whole context. */
export interface ContextUploadProgress {
received: number;
part_max_bytes: number;
max_context_bytes: number;
}
/** SubmissionListParams picks one page of a submission list (server-side filter
* and paging; the scope is the endpoint, never a parameter). */
export interface SubmissionListParams {