Loading cmd/felis/api.go +31 −0 Changes for cmd/felis/api.go: 31 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -213,6 +213,13 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { ContextBaseURL: internalAPIBaseURL(), Blobs: blobs, } if v := cfg.Registry.UserUploadsMaxBytes; v != "" { if n, err := parseByteSize(v); err != nil || n <= 0 { fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v) } else { submissions.MaxStoredBytesTotal = n } } // Restore subsystem (spec §7): the weak-SA restore Job mounts the target // world PVC + the backup PVC and runs `felis restore`. It needs deployment- Loading Loading @@ -404,6 +411,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { if pruner := registryPruner(cfg, builder.Store, a.Cluster, stderr); pruner != nil { go pruner.Loop(ctx, registryPruneInterval) } go reapRejectedContexts(ctx, submissions, stderr) select { case <-ctx.Done(): Loading Loading @@ -619,6 +627,29 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) { } } // reapRejectedContexts deletes, once an hour, the uploaded contexts of // submissions rejected more than submit.RejectedContextRetention ago. Without it a // rejected modpack keeps its bytes on the uploads store (and against its // submitter's budget) until an admin deletes the row. func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) { t := time.NewTicker(time.Hour) defer t.Stop() for { n, err := m.ReapRejected(ctx, submit.RejectedContextRetention) if err != nil { fmt.Fprintf(stderr, "felis api: reap rejected uploads: %v\n", err) } if n > 0 { fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n) } select { case <-ctx.Done(): return case <-t.C: } } } // registryPruneInterval spaces the registry pruner's runs. The registry-gc // sidecar sweeps once a day, so pruning more often only changes which sweep frees // a layer. Loading deploy/bootstrap.sh +1 −1 Changes for deploy/bootstrap.sh: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -2373,7 +2373,7 @@ persisted_registry_block() { out="$(awk ' /^[[:space:]]*\[/ { sect = $0; next } sect ~ /^[[:space:]]*\[registry\][[:space:]]*$/ && /^[[:space:]]*(kaniko_image|trivy_image|trivy_db_repository|trivy_java_db_repository|build_cpu_limit|build_mem_limit|build_disk_limit|build_user_namespaces|build_runtime_class|max_concurrent_builds|user_uploads_context)[[:space:]]*=/ { print } /^[[:space:]]*(kaniko_image|trivy_image|trivy_db_repository|trivy_java_db_repository|build_cpu_limit|build_mem_limit|build_disk_limit|build_user_namespaces|build_runtime_class|max_concurrent_builds|user_uploads_context|user_uploads_max_bytes)[[:space:]]*=/ { print } sect ~ /^[[:space:]]*\[registry\.s3\][[:space:]]*$/ && /^[[:space:]]*[A-Za-z_]+[[:space:]]*=/ { if (!s3hdr) { printf "[registry.s3]\n"; s3hdr = 1 } print Loading internal/api/submissions.go +6 −2 Changes for internal/api/submissions.go: 6 added lines, 2 removed lines. Original line number Diff line number Diff line Loading @@ -377,8 +377,9 @@ var errSubmissionsUnavailable = newError(http.StatusServiceUnavailable, "submiss // writeSubmitError maps submit-package errors onto HTTP status codes. Only the // business sentinels are client-facing: a validation failure is 400, a missing // submission is 404, an already-reviewed submission is 409, a spent per-user // allowance is 403 (the same status the server-resource quota answers with), and // an unconfigured upload transport is 503 (the store this deployment set has no // allowance is 403 (the same status the server-resource quota answers with), a // full uploads store (every user's uploads together at their cap, or the volume // short of free space) is 507, and an unconfigured upload transport is 503 (the store this deployment set has no // implemented transport — an honest "not available here", not a client error). Everything // else — including a build.ErrInvalid raised by the pre-CAS build.Validate (a // platform registry/context MISCONFIGURATION, never client input, since every Loading @@ -402,6 +403,9 @@ func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) { "submission quota reached")) case errors.Is(err, submit.ErrBlobNotFound): writeError(w, r, newError(http.StatusNotFound, "not_found", "no context uploaded for this submission")) case errors.Is(err, submit.ErrUploadsFull): writeError(w, r, newError(http.StatusInsufficientStorage, "uploads_full", "the uploads store is full; an admin has to delete reviewed submissions before new uploads fit")) case errors.Is(err, submit.ErrUploadsUnavailable): writeError(w, r, newError(http.StatusServiceUnavailable, "uploads_unavailable", "modpack upload transport is not configured")) Loading internal/config/config.go +5 −0 Changes for internal/config/config.go: 5 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -207,6 +207,11 @@ type RegistryConfig struct { // archive (§19 WorldArchiver) and a build context (§16) are different artifacts // with different lifecycles, so the two must not share a store binding. UserUploadsContext string `toml:"user_uploads_context"` // UserUploadsMaxBytes caps what every user's uploaded contexts may occupy // together, as a quantity ("4Gi"). Each user also has a 2 GiB budget of their // own; this bounds the sum, which on k3s local-path is the only bound, since // the uploads PVC's size is not enforced there. Empty keeps 4Gi. UserUploadsMaxBytes string `toml:"user_uploads_max_bytes"` // S3 configures the object-store backend for user_uploads_context when it is an // s3:// base (the alternative to a local uploads path). It mirrors // ArchiveS3Config: Endpoint + Region locate the store and the *Ref fields NAME Loading internal/submit/blobstore.go +34 −0 Changes for internal/submit/blobstore.go: 34 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "regexp" "syscall" ) // contextBlobName is the fixed object name of a submission's build context under Loading Loading @@ -43,6 +44,39 @@ type LocalContextStore struct { // Base is the directory (uploads PVC mount) submission contexts are written // under. Each submission gets its own {Base}/{id}/ subdirectory. Base string // MinFree is the share of Base's filesystem an upload must leave free; 0 uses // DefaultUploadsMinFree. MinFree float64 } // DefaultUploadsMinFree is the share of the uploads filesystem an upload must // leave free. On k3s local-path the uploads PVC is a directory on the node's // disk, beside the worlds and the database, and below about a tenth free the // kubelet starts evicting pods (the same floor backup.MinFreeAfter keeps). const DefaultUploadsMinFree = 0.10 // CheckRoom refuses an upload of up to need bytes that could push Base's // filesystem below its free floor. func (s *LocalContextStore) CheckRoom(need int64) error { var st syscall.Statfs_t if err := syscall.Statfs(s.Base, &st); err != nil { return fmt.Errorf("submit: measure the uploads store %s: %w", s.Base, err) } bsize := uint64(st.Bsize) // uint32 on darwin total, avail := uint64(st.Blocks)*bsize, uint64(st.Bavail)*bsize if total == 0 { return nil } minFree := s.MinFree if minFree <= 0 { minFree = DefaultUploadsMinFree } floor := uint64(float64(total) * minFree) if n := uint64(max(need, 0)); avail < n || avail-n < floor { return fmt.Errorf("%w: %d MiB free of %d MiB, and an upload of up to %d MiB would leave less than %.0f%% free", ErrUploadsFull, avail>>20, total>>20, n>>20, minFree*100) } return nil } // dir returns the per-submission directory, rejecting an id that could escape Loading Loading
cmd/felis/api.go +31 −0 Changes for cmd/felis/api.go: 31 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -213,6 +213,13 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { ContextBaseURL: internalAPIBaseURL(), Blobs: blobs, } if v := cfg.Registry.UserUploadsMaxBytes; v != "" { if n, err := parseByteSize(v); err != nil || n <= 0 { fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v) } else { submissions.MaxStoredBytesTotal = n } } // Restore subsystem (spec §7): the weak-SA restore Job mounts the target // world PVC + the backup PVC and runs `felis restore`. It needs deployment- Loading Loading @@ -404,6 +411,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { if pruner := registryPruner(cfg, builder.Store, a.Cluster, stderr); pruner != nil { go pruner.Loop(ctx, registryPruneInterval) } go reapRejectedContexts(ctx, submissions, stderr) select { case <-ctx.Done(): Loading Loading @@ -619,6 +627,29 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) { } } // reapRejectedContexts deletes, once an hour, the uploaded contexts of // submissions rejected more than submit.RejectedContextRetention ago. Without it a // rejected modpack keeps its bytes on the uploads store (and against its // submitter's budget) until an admin deletes the row. func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) { t := time.NewTicker(time.Hour) defer t.Stop() for { n, err := m.ReapRejected(ctx, submit.RejectedContextRetention) if err != nil { fmt.Fprintf(stderr, "felis api: reap rejected uploads: %v\n", err) } if n > 0 { fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n) } select { case <-ctx.Done(): return case <-t.C: } } } // registryPruneInterval spaces the registry pruner's runs. The registry-gc // sidecar sweeps once a day, so pruning more often only changes which sweep frees // a layer. Loading
deploy/bootstrap.sh +1 −1 Changes for deploy/bootstrap.sh: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -2373,7 +2373,7 @@ persisted_registry_block() { out="$(awk ' /^[[:space:]]*\[/ { sect = $0; next } sect ~ /^[[:space:]]*\[registry\][[:space:]]*$/ && /^[[:space:]]*(kaniko_image|trivy_image|trivy_db_repository|trivy_java_db_repository|build_cpu_limit|build_mem_limit|build_disk_limit|build_user_namespaces|build_runtime_class|max_concurrent_builds|user_uploads_context)[[:space:]]*=/ { print } /^[[:space:]]*(kaniko_image|trivy_image|trivy_db_repository|trivy_java_db_repository|build_cpu_limit|build_mem_limit|build_disk_limit|build_user_namespaces|build_runtime_class|max_concurrent_builds|user_uploads_context|user_uploads_max_bytes)[[:space:]]*=/ { print } sect ~ /^[[:space:]]*\[registry\.s3\][[:space:]]*$/ && /^[[:space:]]*[A-Za-z_]+[[:space:]]*=/ { if (!s3hdr) { printf "[registry.s3]\n"; s3hdr = 1 } print Loading
internal/api/submissions.go +6 −2 Changes for internal/api/submissions.go: 6 added lines, 2 removed lines. Original line number Diff line number Diff line Loading @@ -377,8 +377,9 @@ var errSubmissionsUnavailable = newError(http.StatusServiceUnavailable, "submiss // writeSubmitError maps submit-package errors onto HTTP status codes. Only the // business sentinels are client-facing: a validation failure is 400, a missing // submission is 404, an already-reviewed submission is 409, a spent per-user // allowance is 403 (the same status the server-resource quota answers with), and // an unconfigured upload transport is 503 (the store this deployment set has no // allowance is 403 (the same status the server-resource quota answers with), a // full uploads store (every user's uploads together at their cap, or the volume // short of free space) is 507, and an unconfigured upload transport is 503 (the store this deployment set has no // implemented transport — an honest "not available here", not a client error). Everything // else — including a build.ErrInvalid raised by the pre-CAS build.Validate (a // platform registry/context MISCONFIGURATION, never client input, since every Loading @@ -402,6 +403,9 @@ func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) { "submission quota reached")) case errors.Is(err, submit.ErrBlobNotFound): writeError(w, r, newError(http.StatusNotFound, "not_found", "no context uploaded for this submission")) case errors.Is(err, submit.ErrUploadsFull): writeError(w, r, newError(http.StatusInsufficientStorage, "uploads_full", "the uploads store is full; an admin has to delete reviewed submissions before new uploads fit")) case errors.Is(err, submit.ErrUploadsUnavailable): writeError(w, r, newError(http.StatusServiceUnavailable, "uploads_unavailable", "modpack upload transport is not configured")) Loading
internal/config/config.go +5 −0 Changes for internal/config/config.go: 5 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -207,6 +207,11 @@ type RegistryConfig struct { // archive (§19 WorldArchiver) and a build context (§16) are different artifacts // with different lifecycles, so the two must not share a store binding. UserUploadsContext string `toml:"user_uploads_context"` // UserUploadsMaxBytes caps what every user's uploaded contexts may occupy // together, as a quantity ("4Gi"). Each user also has a 2 GiB budget of their // own; this bounds the sum, which on k3s local-path is the only bound, since // the uploads PVC's size is not enforced there. Empty keeps 4Gi. UserUploadsMaxBytes string `toml:"user_uploads_max_bytes"` // S3 configures the object-store backend for user_uploads_context when it is an // s3:// base (the alternative to a local uploads path). It mirrors // ArchiveS3Config: Endpoint + Region locate the store and the *Ref fields NAME Loading
internal/submit/blobstore.go +34 −0 Changes for internal/submit/blobstore.go: 34 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "regexp" "syscall" ) // contextBlobName is the fixed object name of a submission's build context under Loading Loading @@ -43,6 +44,39 @@ type LocalContextStore struct { // Base is the directory (uploads PVC mount) submission contexts are written // under. Each submission gets its own {Base}/{id}/ subdirectory. Base string // MinFree is the share of Base's filesystem an upload must leave free; 0 uses // DefaultUploadsMinFree. MinFree float64 } // DefaultUploadsMinFree is the share of the uploads filesystem an upload must // leave free. On k3s local-path the uploads PVC is a directory on the node's // disk, beside the worlds and the database, and below about a tenth free the // kubelet starts evicting pods (the same floor backup.MinFreeAfter keeps). const DefaultUploadsMinFree = 0.10 // CheckRoom refuses an upload of up to need bytes that could push Base's // filesystem below its free floor. func (s *LocalContextStore) CheckRoom(need int64) error { var st syscall.Statfs_t if err := syscall.Statfs(s.Base, &st); err != nil { return fmt.Errorf("submit: measure the uploads store %s: %w", s.Base, err) } bsize := uint64(st.Bsize) // uint32 on darwin total, avail := uint64(st.Blocks)*bsize, uint64(st.Bavail)*bsize if total == 0 { return nil } minFree := s.MinFree if minFree <= 0 { minFree = DefaultUploadsMinFree } floor := uint64(float64(total) * minFree) if n := uint64(max(need, 0)); avail < n || avail-n < floor { return fmt.Errorf("%w: %d MiB free of %d MiB, and an upload of up to %d MiB would leave less than %.0f%% free", ErrUploadsFull, avail>>20, total>>20, n>>20, minFree*100) } return nil } // dir returns the per-submission directory, rejecting an id that could escape Loading