Unverified Commit 720ab81b authored by Lemon-miaow's avatar Lemon-miaow
Browse files

feat(submit): 上传存储全局上限、磁盘余量检查,被拒上下文 7 天后回收

parent 9baa0a10
Loading
Loading
Loading
Loading
+31 −0
Changes for cmd/felis/api.go: 31 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -213,6 +213,13 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		ContextBaseURL: internalAPIBaseURL(),
		Blobs:          blobs,
	}
	if v := cfg.Registry.UserUploadsMaxBytes; v != "" {
		if n, err := parseByteSize(v); err != nil || n <= 0 {
			fmt.Fprintf(stderr, "felis api: [registry] user_uploads_max_bytes %q is not a positive size such as 4Gi; keeping the default\n", v)
		} else {
			submissions.MaxStoredBytesTotal = n
		}
	}

	// Restore subsystem (spec §7): the weak-SA restore Job mounts the target
	// world PVC + the backup PVC and runs `felis restore`. It needs deployment-
@@ -404,6 +411,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
	if pruner := registryPruner(cfg, builder.Store, a.Cluster, stderr); pruner != nil {
		go pruner.Loop(ctx, registryPruneInterval)
	}
	go reapRejectedContexts(ctx, submissions, stderr)

	select {
	case <-ctx.Done():
@@ -619,6 +627,29 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) {
	}
}

// reapRejectedContexts deletes, once an hour, the uploaded contexts of
// submissions rejected more than submit.RejectedContextRetention ago. Without it a
// rejected modpack keeps its bytes on the uploads store (and against its
// submitter's budget) until an admin deletes the row.
func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writer) {
	t := time.NewTicker(time.Hour)
	defer t.Stop()
	for {
		n, err := m.ReapRejected(ctx, submit.RejectedContextRetention)
		if err != nil {
			fmt.Fprintf(stderr, "felis api: reap rejected uploads: %v\n", err)
		}
		if n > 0 {
			fmt.Fprintf(stderr, "felis api: deleted the uploaded contexts of %d rejected submission(s)\n", n)
		}
		select {
		case <-ctx.Done():
			return
		case <-t.C:
		}
	}
}

// registryPruneInterval spaces the registry pruner's runs. The registry-gc
// sidecar sweeps once a day, so pruning more often only changes which sweep frees
// a layer.
+1 −1
Changes for deploy/bootstrap.sh: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -2373,7 +2373,7 @@ persisted_registry_block() {
    out="$(awk '
      /^[[:space:]]*\[/ { sect = $0; next }
      sect ~ /^[[:space:]]*\[registry\][[:space:]]*$/ &&
        /^[[:space:]]*(kaniko_image|trivy_image|trivy_db_repository|trivy_java_db_repository|build_cpu_limit|build_mem_limit|build_disk_limit|build_user_namespaces|build_runtime_class|max_concurrent_builds|user_uploads_context)[[:space:]]*=/ { print }
        /^[[:space:]]*(kaniko_image|trivy_image|trivy_db_repository|trivy_java_db_repository|build_cpu_limit|build_mem_limit|build_disk_limit|build_user_namespaces|build_runtime_class|max_concurrent_builds|user_uploads_context|user_uploads_max_bytes)[[:space:]]*=/ { print }
      sect ~ /^[[:space:]]*\[registry\.s3\][[:space:]]*$/ && /^[[:space:]]*[A-Za-z_]+[[:space:]]*=/ {
        if (!s3hdr) { printf "[registry.s3]\n"; s3hdr = 1 }
        print
+6 −2
Changes for internal/api/submissions.go: 6 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -377,8 +377,9 @@ var errSubmissionsUnavailable = newError(http.StatusServiceUnavailable, "submiss
// writeSubmitError maps submit-package errors onto HTTP status codes. Only the
// business sentinels are client-facing: a validation failure is 400, a missing
// submission is 404, an already-reviewed submission is 409, a spent per-user
// allowance is 403 (the same status the server-resource quota answers with), and
// an unconfigured upload transport is 503 (the store this deployment set has no
// allowance is 403 (the same status the server-resource quota answers with), a
// full uploads store (every user's uploads together at their cap, or the volume
// short of free space) is 507, and an unconfigured upload transport is 503 (the store this deployment set has no
// implemented transport — an honest "not available here", not a client error). Everything
// else — including a build.ErrInvalid raised by the pre-CAS build.Validate (a
// platform registry/context MISCONFIGURATION, never client input, since every
@@ -402,6 +403,9 @@ func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) {
			"submission quota reached"))
	case errors.Is(err, submit.ErrBlobNotFound):
		writeError(w, r, newError(http.StatusNotFound, "not_found", "no context uploaded for this submission"))
	case errors.Is(err, submit.ErrUploadsFull):
		writeError(w, r, newError(http.StatusInsufficientStorage, "uploads_full",
			"the uploads store is full; an admin has to delete reviewed submissions before new uploads fit"))
	case errors.Is(err, submit.ErrUploadsUnavailable):
		writeError(w, r, newError(http.StatusServiceUnavailable, "uploads_unavailable",
			"modpack upload transport is not configured"))
+5 −0
Changes for internal/config/config.go: 5 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -207,6 +207,11 @@ type RegistryConfig struct {
	// archive (§19 WorldArchiver) and a build context (§16) are different artifacts
	// with different lifecycles, so the two must not share a store binding.
	UserUploadsContext string `toml:"user_uploads_context"`
	// UserUploadsMaxBytes caps what every user's uploaded contexts may occupy
	// together, as a quantity ("4Gi"). Each user also has a 2 GiB budget of their
	// own; this bounds the sum, which on k3s local-path is the only bound, since
	// the uploads PVC's size is not enforced there. Empty keeps 4Gi.
	UserUploadsMaxBytes string `toml:"user_uploads_max_bytes"`
	// S3 configures the object-store backend for user_uploads_context when it is an
	// s3:// base (the alternative to a local uploads path). It mirrors
	// ArchiveS3Config: Endpoint + Region locate the store and the *Ref fields NAME
+34 −0
Changes for internal/submit/blobstore.go: 34 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ import (
	"os"
	"path/filepath"
	"regexp"
	"syscall"
)

// contextBlobName is the fixed object name of a submission's build context under
@@ -43,6 +44,39 @@ type LocalContextStore struct {
	// Base is the directory (uploads PVC mount) submission contexts are written
	// under. Each submission gets its own {Base}/{id}/ subdirectory.
	Base string
	// MinFree is the share of Base's filesystem an upload must leave free; 0 uses
	// DefaultUploadsMinFree.
	MinFree float64
}

// DefaultUploadsMinFree is the share of the uploads filesystem an upload must
// leave free. On k3s local-path the uploads PVC is a directory on the node's
// disk, beside the worlds and the database, and below about a tenth free the
// kubelet starts evicting pods (the same floor backup.MinFreeAfter keeps).
const DefaultUploadsMinFree = 0.10

// CheckRoom refuses an upload of up to need bytes that could push Base's
// filesystem below its free floor.
func (s *LocalContextStore) CheckRoom(need int64) error {
	var st syscall.Statfs_t
	if err := syscall.Statfs(s.Base, &st); err != nil {
		return fmt.Errorf("submit: measure the uploads store %s: %w", s.Base, err)
	}
	bsize := uint64(st.Bsize) // uint32 on darwin
	total, avail := uint64(st.Blocks)*bsize, uint64(st.Bavail)*bsize
	if total == 0 {
		return nil
	}
	minFree := s.MinFree
	if minFree <= 0 {
		minFree = DefaultUploadsMinFree
	}
	floor := uint64(float64(total) * minFree)
	if n := uint64(max(need, 0)); avail < n || avail-n < floor {
		return fmt.Errorf("%w: %d MiB free of %d MiB, and an upload of up to %d MiB would leave less than %.0f%% free",
			ErrUploadsFull, avail>>20, total>>20, n>>20, minFree*100)
	}
	return nil
}

// dir returns the per-submission directory, rejecting an id that could escape
Loading