fix(submit): 已批准的投稿不再占上传者的存储额度,只计入存储总量
This commit is contained in:
6 files changed
+82
-21
No files matched your search
+5
-2
@@ -5764,7 +5764,9 @@ paths:
|
|||||||
description: >-
|
description: >-
|
||||||
The per-user submission allowance is spent — too many of the
|
The per-user submission allowance is spent — too many of the
|
||||||
caller's submissions are awaiting review, or their stored-upload
|
caller's submissions are awaiting review, or their stored-upload
|
||||||
budget is full (submission_quota_exceeded).
|
budget is full (submission_quota_exceeded). The budget counts
|
||||||
|
pending uploads and rejected ones until they are reaped, 7 days after
|
||||||
|
review; approved uploads leave it.
|
||||||
'429':
|
'429':
|
||||||
description: >-
|
description: >-
|
||||||
A submission was created within the per-user cooldown window
|
A submission was created within the per-user cooldown window
|
||||||
@@ -5890,7 +5892,8 @@ paths:
|
|||||||
'403':
|
'403':
|
||||||
description: >-
|
description: >-
|
||||||
The upload would exceed the caller's per-user stored-context budget
|
The upload would exceed the caller's per-user stored-context budget
|
||||||
(submission_quota_exceeded).
|
(submission_quota_exceeded), which counts their pending and rejected
|
||||||
|
uploads; approved ones leave it.
|
||||||
'404':
|
'404':
|
||||||
$ref: '#/components/responses/NotFound'
|
$ref: '#/components/responses/NotFound'
|
||||||
'409':
|
'409':
|
||||||
|
|||||||
+21
-15
@@ -155,10 +155,12 @@ const (
|
|||||||
// enough to stage a couple of packs, far short of a flood. Override per
|
// enough to stage a couple of packs, far short of a flood. Override per
|
||||||
// Manager via MaxPendingPerUser.
|
// Manager via MaxPendingPerUser.
|
||||||
defaultMaxPendingPerUser = 5
|
defaultMaxPendingPerUser = 5
|
||||||
// defaultMaxStoredBytesPerUser caps the total bytes one user's stored
|
// defaultMaxStoredBytesPerUser caps the total bytes one user's unapproved
|
||||||
// contexts may occupy on the uploads store. The uploads PVC renders at a
|
// contexts (pending, and rejected ones until ReapRejected takes them) may
|
||||||
// fixed 5Gi (platform/workloads.go); without a per-user budget one account
|
// occupy on the uploads store; approved ones count only toward the total,
|
||||||
// could fill it and every other user's upload would start failing. Two GiB
|
// since an admin chose to keep them. The uploads PVC renders at a fixed 5Gi
|
||||||
|
// (platform/workloads.go); without a per-user budget one account could fill
|
||||||
|
// it and every other user's upload would start failing. Two GiB
|
||||||
// leaves room for a couple of full-size modpacks (a single blob may be 1 GiB)
|
// leaves room for a couple of full-size modpacks (a single blob may be 1 GiB)
|
||||||
// while keeping a small user base from exhausting the volume; size the PVC
|
// while keeping a small user base from exhausting the volume; size the PVC
|
||||||
// above users × this budget before raising it. Override per Manager via
|
// above users × this budget before raising it. Override per Manager via
|
||||||
@@ -597,7 +599,7 @@ func (m *Manager) Create(ctx context.Context, req CreateRequest) (*Submission, e
|
|||||||
// has already consumed it, once rejected it is dead;
|
// has already consumed it, once rejected it is dead;
|
||||||
// - the body must be a gzip tarball (context.tar.gz) and is size-capped, so a
|
// - the body must be a gzip tarball (context.tar.gz) and is size-capped, so a
|
||||||
// wrong-format or oversize upload is rejected as a 400 without persisting;
|
// wrong-format or oversize upload is rejected as a 400 without persisting;
|
||||||
// - a user's stored contexts are budgeted (MaxStoredBytesPerUser): the write
|
// - a user's unapproved contexts are budgeted (MaxStoredBytesPerUser): the write
|
||||||
// is capped at the remaining budget, so an upload that would exceed it is
|
// is capped at the remaining budget, so an upload that would exceed it is
|
||||||
// refused as a spent allowance (403) before the excess is persisted;
|
// refused as a spent allowance (403) before the excess is persisted;
|
||||||
// - so are everyone's together (MaxStoredBytesTotal), and a local store checks
|
// - so are everyone's together (MaxStoredBytesTotal), and a local store checks
|
||||||
@@ -825,15 +827,19 @@ func (m *Manager) ReapStaleParts(olderThan time.Duration) (int, error) {
|
|||||||
return m.Parts.Reap(m.now().Add(-olderThan))
|
return m.Parts.Reap(m.now().Add(-olderThan))
|
||||||
}
|
}
|
||||||
|
|
||||||
// storedBytes sums the stored-blob sizes of submittedBy's submissions (user) and
|
// storedBytes sums the stored-blob sizes of submittedBy's unapproved submissions
|
||||||
// of everyone's (total), excluding excludeID — the submission a pending re-upload
|
// (user) and of everyone's submissions (total), excluding excludeID — the
|
||||||
// is about to replace, whose bytes must not be counted twice. Sizes are read from
|
// submission a pending re-upload is about to replace, whose bytes must not be
|
||||||
// the blob store itself, the same source of truth uploads/approval consult, so
|
// counted twice. An approved context leaves its uploader's budget: an admin chose
|
||||||
// the sums cannot drift from what is actually occupying the volume (including
|
// to keep it (it rebuilds the image after a registry loss), and the uploader can
|
||||||
// blobs uploaded before any budget existed). A staged chunked upload counts as
|
// neither withdraw nor replace it, so charging it would spend their allowance for
|
||||||
// well, so parts spread over several pending submissions cannot hold more than
|
// good. It still fills the store, so it stays in total. Sizes are read from the
|
||||||
// the budget allows. With fresh unset, a blob size read or written within
|
// blob store itself, the same source of truth uploads/approval consult, so the
|
||||||
// blobSizeTTL is used as it stands (blobSize). A size that cannot be read is
|
// sums cannot drift from what is actually occupying the volume (including blobs
|
||||||
|
// uploaded before any budget existed). A staged chunked upload counts as well, so
|
||||||
|
// parts spread over several pending submissions cannot hold more than the budget
|
||||||
|
// allows. With fresh unset, a blob size read or written within blobSizeTTL is
|
||||||
|
// used as it stands (blobSize). A size that cannot be read is
|
||||||
// ErrStoreUnavailable, for the caller to try again.
|
// ErrStoreUnavailable, for the caller to try again.
|
||||||
func (m *Manager) storedBytes(ctx context.Context, submittedBy, excludeID string, fresh bool) (user, total int64, err error) {
|
func (m *Manager) storedBytes(ctx context.Context, submittedBy, excludeID string, fresh bool) (user, total int64, err error) {
|
||||||
subs, err := m.Store.ListSubmissions(ctx)
|
subs, err := m.Store.ListSubmissions(ctx)
|
||||||
@@ -856,7 +862,7 @@ func (m *Manager) storedBytes(ctx context.Context, submittedBy, excludeID string
|
|||||||
n += staged
|
n += staged
|
||||||
}
|
}
|
||||||
total += n
|
total += n
|
||||||
if s.SubmittedBy == submittedBy {
|
if s.SubmittedBy == submittedBy && s.Status != StatusApproved {
|
||||||
user += n
|
user += n
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -887,6 +887,58 @@ func TestUploadContextStorageBudget(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An approved context stays on the store for rebuilds, out of its uploader's
|
||||||
|
// hands; it leaves their budget, so approval gives the allowance back. It still
|
||||||
|
// fills the store's total, and a rejected context keeps its uploader's budget
|
||||||
|
// until it is reaped.
|
||||||
|
func TestApprovedContextsLeaveTheUploadersBudget(t *testing.T) {
|
||||||
|
m, _, _ := newManager()
|
||||||
|
fb := newFakeBlobs()
|
||||||
|
m.Blobs = fb
|
||||||
|
m.MaxStoredBytesPerUser = 5 // one gzBody("x") of 5 bytes
|
||||||
|
m.MaxStoredBytesTotal = 10
|
||||||
|
ctx := context.Background()
|
||||||
|
create := func(user string) *Submission {
|
||||||
|
t.Helper()
|
||||||
|
sub, err := m.Create(ctx, CreateRequest{DisplayName: "P", SubmittedBy: user})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create for %s: %v", user, err)
|
||||||
|
}
|
||||||
|
return sub
|
||||||
|
}
|
||||||
|
upload := func(sub *Submission) (*Submission, error) {
|
||||||
|
return m.UploadContext(ctx, sub.ID, sub.SubmittedBy, strings.NewReader(gzBody("x")))
|
||||||
|
}
|
||||||
|
|
||||||
|
a, b := create("user-1"), create("user-1")
|
||||||
|
a, err := upload(a)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("upload A: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := upload(b); !errors.Is(err, ErrQuotaExceeded) {
|
||||||
|
t.Fatalf("upload B beside a pending A = %v, want ErrQuotaExceeded", err)
|
||||||
|
}
|
||||||
|
if _, err := m.Approve(ctx, a.ID, "[email protected]", a.ContextSHA256); err != nil {
|
||||||
|
t.Fatalf("approve A: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := upload(b); err != nil {
|
||||||
|
t.Fatalf("upload B beside an approved A = %v, want accepted", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A (approved) and B hold the store's 10 bytes between them.
|
||||||
|
if _, err := upload(create("user-2")); !errors.Is(err, ErrUploadsFull) {
|
||||||
|
t.Fatalf("upload into a store the approved A helps fill = %v, want ErrUploadsFull", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := m.Reject(ctx, b.ID, "[email protected]", "no"); err != nil {
|
||||||
|
t.Fatalf("reject B: %v", err)
|
||||||
|
}
|
||||||
|
m.MaxStoredBytesTotal = 100
|
||||||
|
if _, err := upload(create("user-1")); !errors.Is(err, ErrQuotaExceeded) {
|
||||||
|
t.Fatalf("upload beside a rejected B = %v, want ErrQuotaExceeded", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// A single oversize blob stays a 400 (ErrInvalid), distinct from the 403 the
|
// A single oversize blob stays a 400 (ErrInvalid), distinct from the 403 the
|
||||||
// per-user budget answers with — the two failure classes must not collapse.
|
// per-user budget answers with — the two failure classes must not collapse.
|
||||||
func TestUploadContextOversizeIsNotQuotaError(t *testing.T) {
|
func TestUploadContextOversizeIsNotQuotaError(t *testing.T) {
|
||||||
|
|||||||
@@ -69,7 +69,7 @@
|
|||||||
"build_logs_unavailable": "Build logs aren't available right now.",
|
"build_logs_unavailable": "Build logs aren't available right now.",
|
||||||
"already_reviewed": "This submission has already been reviewed.",
|
"already_reviewed": "This submission has already been reviewed.",
|
||||||
"context_changed": "The submitter uploaded the build context again after you reviewed it. Download and review the new upload before approving.",
|
"context_changed": "The submitter uploaded the build context again after you reviewed it. Download and review the new upload before approving.",
|
||||||
"submission_quota_exceeded": "Your submission quota is full: too many pending reviews, or your stored uploads are at the limit.",
|
"submission_quota_exceeded": "Your submission quota is full: too many pending reviews, or your pending and rejected uploads fill your storage allowance. Withdraw a pending one, or wait: a rejected upload frees its space 7 days after review, and approved ones don't count.",
|
||||||
"submission_cooldown": "Too many submission requests — try again shortly.",
|
"submission_cooldown": "Too many submission requests — try again shortly.",
|
||||||
"submissions_unavailable": "Submissions aren't available right now.",
|
"submissions_unavailable": "Submissions aren't available right now.",
|
||||||
"uploads_unavailable": "Uploads aren't available right now.",
|
"uploads_unavailable": "Uploads aren't available right now.",
|
||||||
|
|||||||
@@ -69,7 +69,7 @@
|
|||||||
"build_logs_unavailable": "构建日志暂时不可用。",
|
"build_logs_unavailable": "构建日志暂时不可用。",
|
||||||
"already_reviewed": "该提交已经审核过了。",
|
"already_reviewed": "该提交已经审核过了。",
|
||||||
"context_changed": "提交者在你审阅之后重新上传了构建上下文。请重新下载并审阅新的上传再通过。",
|
"context_changed": "提交者在你审阅之后重新上传了构建上下文。请重新下载并审阅新的上传再通过。",
|
||||||
"submission_quota_exceeded": "你的提交配额已满:待审核提交过多,或已存上传总量达到上限。",
|
"submission_quota_exceeded": "你的提交配额已满:待审核提交过多,或待审核和被拒绝的上传占满了存储额度。可以撤回一个待审核的提交;被拒绝的上传在审核 7 天后释放空间,已批准的不占额度。",
|
||||||
"submission_cooldown": "操作太频繁——请稍后再试。",
|
"submission_cooldown": "操作太频繁——请稍后再试。",
|
||||||
"submissions_unavailable": "提交流程当前不可用。",
|
"submissions_unavailable": "提交流程当前不可用。",
|
||||||
"uploads_unavailable": "上传功能当前不可用。",
|
"uploads_unavailable": "上传功能当前不可用。",
|
||||||
|
|||||||
@@ -7766,7 +7766,7 @@ export interface operations {
|
|||||||
};
|
};
|
||||||
400: components["responses"]["BadRequest"];
|
400: components["responses"]["BadRequest"];
|
||||||
401: components["responses"]["Unauthorized"];
|
401: components["responses"]["Unauthorized"];
|
||||||
/** @description The per-user submission allowance is spent — too many of the caller's submissions are awaiting review, or their stored-upload budget is full (submission_quota_exceeded). */
|
/** @description The per-user submission allowance is spent — too many of the caller's submissions are awaiting review, or their stored-upload budget is full (submission_quota_exceeded). The budget counts pending uploads and rejected ones until they are reaped, 7 days after review; approved uploads leave it. */
|
||||||
403: {
|
403: {
|
||||||
headers: {
|
headers: {
|
||||||
[name: string]: unknown;
|
[name: string]: unknown;
|
||||||
@@ -7833,7 +7833,7 @@ export interface operations {
|
|||||||
};
|
};
|
||||||
400: components["responses"]["BadRequest"];
|
400: components["responses"]["BadRequest"];
|
||||||
401: components["responses"]["Unauthorized"];
|
401: components["responses"]["Unauthorized"];
|
||||||
/** @description The upload would exceed the caller's per-user stored-context budget (submission_quota_exceeded). */
|
/** @description The upload would exceed the caller's per-user stored-context budget (submission_quota_exceeded), which counts their pending and rejected uploads; approved ones leave it. */
|
||||||
403: {
|
403: {
|
||||||
headers: {
|
headers: {
|
||||||
[name: string]: unknown;
|
[name: string]: unknown;
|
||||||
|
|||||||
Reference in new issue
Block a user