From 598f3d31f40a6ce6f1e7a4cec8622f90c84de38b Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Thu, 2 Jul 2026 22:04:21 +0800 Subject: [PATCH] feat(submit): local + S3 backends for modpack upload contexts, installer-selectable --- cmd/felis/api.go | 83 +++++- cmd/felis/breakglass.go | 4 + cmd/felis/tui_root.go | 44 ++- cmd/felis/tui_root_test.go | 105 ++++++- cmd/felis/tui_storage.go | 407 ++++++++++++++++++++++++++++ cmd/felis/tui_storage_apply.go | 133 +++++++++ cmd/felis/tui_summary.go | 8 +- docs/openapi.yaml | 41 +++ go.mod | 30 +- go.sum | 45 +++ internal/api/api.go | 5 + internal/api/submissions.go | 40 ++- internal/api/submissions_test.go | 98 +++++++ internal/config/config.go | 21 ++ internal/platform/workloads.go | 77 ++++++ internal/platform/workloads_test.go | 42 ++- internal/submit/blobstore.go | 115 ++++++++ internal/submit/blobstore_test.go | 94 +++++++ internal/submit/s3store.go | 217 +++++++++++++++ internal/submit/s3store_test.go | 224 +++++++++++++++ internal/submit/submit.go | 166 +++++++++++- internal/submit/submit_test.go | 212 +++++++++++++++ 22 files changed, 2177 insertions(+), 34 deletions(-) create mode 100644 cmd/felis/tui_storage.go create mode 100644 cmd/felis/tui_storage_apply.go create mode 100644 internal/submit/blobstore.go create mode 100644 internal/submit/blobstore_test.go create mode 100644 internal/submit/s3store.go create mode 100644 internal/submit/s3store_test.go diff --git a/cmd/felis/api.go b/cmd/felis/api.go index 7eb831d..e6334cd 100644 --- a/cmd/felis/api.go +++ b/cmd/felis/api.go @@ -7,7 +7,9 @@ import ( "io" "net/http" "os" + "regexp" goruntime "runtime" + "strings" "time" "felis.lolicon.best/internal/api" @@ -16,6 +18,7 @@ import ( "felis.lolicon.best/internal/config" "felis.lolicon.best/internal/panel" "felis.lolicon.best/internal/passkey" + "felis.lolicon.best/internal/platform" "felis.lolicon.best/internal/restore" "felis.lolicon.best/internal/store" "felis.lolicon.best/internal/submit" @@ -104,15 +107,43 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { // the SAME Trivy-gated Builder runs as for an admin's direct build. Registry // MUST match the Builder's RegistryURL (cfg.Registry.URL) — both are wired from // the one field here so the lane's pre-CAS validate and the Builder's Submit - // can never disagree about the push target. The blob upload transport that - // populates the derived context ref is deferred (INTEGRATION-ONLY): the - // create→approve→reject state machine is real Postgres truth, but a real - // Kaniko context pull needs that transport in place. + // can never disagree about the push target. + // + // The blob upload transport is selected by the shape of user_uploads_context — + // the two backends the setup wizard chooses between. A local path wires + // LocalContextStore (the mounted uploads PVC); an s3:// base wires + // S3ContextStore when its credentials resolve. Either way the store's target is + // derived from the SAME config field the context ref uses, so the blob lands + // exactly where Kaniko's --context points. Anything else — or an s3:// base with + // no credentials configured — leaves Blobs nil so POST + // /me/submissions/{id}/context returns 503, honest like the restore executor + // when its PVC is not supplied. (Letting the sandboxed Kaniko build Pod READ the + // context — PVC mount for local, creds+egress for S3 — is a separate deployment + // integration.) + contextBase := cfg.Registry.UserUploadsContext + var blobs submit.Blobs + switch { + case isLocalUploadsPath(contextBase): + // Normalize a file:// URL to the plain path ONCE and feed it to BOTH the + // derived ref (ContextStore) and the store (Base), so the recorded + // context_ref and the on-disk write location can never diverge. + contextBase = strings.TrimPrefix(contextBase, "file://") + blobs = &submit.LocalContextStore{Base: contextBase} + case strings.HasPrefix(strings.ToLower(contextBase), "s3://"): + if s3, err := newS3UploadsStore(cfg.Registry); err != nil { + fmt.Fprintf(stderr, "felis api: S3 user-uploads store not configured (%v) — modpack upload transport disabled (POST /api/v1/me/submissions/{id}/context returns 503)\n", err) + } else { + blobs = s3 + } + default: + fmt.Fprintf(stderr, "felis api: user-uploads context %q is neither a local path nor an s3:// base — modpack upload transport disabled (POST /api/v1/me/submissions/{id}/context returns 503)\n", contextBase) + } submissions := &submit.Manager{ Store: submit.NewPGStore(drv.DB()), Builds: builder, Registry: cfg.Registry.URL, - ContextStore: cfg.Registry.UserUploadsContext, + ContextStore: contextBase, + Blobs: blobs, } // Restore subsystem (spec §7): the weak-SA restore Job mounts the target @@ -280,6 +311,48 @@ func buildConfig(cfg *config.Config) build.Config { } } +// uploadsSchemeRE matches a leading URL scheme like "s3://" or "gs://". +var uploadsSchemeRE = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://`) + +// isLocalUploadsPath reports whether the user-uploads context base is a local +// filesystem path (a bare path or a file:// URL), i.e. one LocalContextStore can +// write to. An s3:// base routes to newS3UploadsStore instead; any other scheme +// has no implemented transport, so its uploads are left disabled (503). +func isLocalUploadsPath(base string) bool { + if strings.HasPrefix(base, "file://") { + return true + } + return !uploadsSchemeRE.MatchString(base) +} + +// newS3UploadsStore builds the S3 blob transport for an s3:// user_uploads_context. +// The bucket + key prefix come from the base itself; the endpoint/region come from +// [registry.s3]; and the credentials are read from the environment variables named +// by access_key_ref / secret_key_ref (defaulting to the fixed env names the +// felis-api Deployment injects from the felis-uploads-s3 Secret). Any missing piece +// is an error, so the caller leaves Blobs nil and the upload endpoint returns 503 +// rather than pretending it can persist a file. +func newS3UploadsStore(reg config.RegistryConfig) (submit.Blobs, error) { + accessRef, secretRef := reg.S3.AccessKeyRef, reg.S3.SecretKeyRef + if accessRef == "" { + accessRef = platform.UploadsS3AccessKeyEnv + } + if secretRef == "" { + secretRef = platform.UploadsS3SecretKeyEnv + } + accessKey, secretKey := os.Getenv(accessRef), os.Getenv(secretRef) + if accessKey == "" || secretKey == "" { + return nil, fmt.Errorf("credentials env %s/%s are empty", accessRef, secretRef) + } + return submit.NewS3ContextStore(submit.S3StoreConfig{ + Base: reg.UserUploadsContext, + Endpoint: reg.S3.Endpoint, + Region: reg.S3.Region, + AccessKey: accessKey, + SecretKey: secretKey, + }) +} + // restoreConfig projects felis.toml + the deployment-supplied image and backup // PVC onto the restore subsystem config (spec §7). The runtime identity, mount // roots, resource limits, and weak SA fall back to the restore package's diff --git a/cmd/felis/breakglass.go b/cmd/felis/breakglass.go index a99d9ae..531d71f 100644 --- a/cmd/felis/breakglass.go +++ b/cmd/felis/breakglass.go @@ -532,6 +532,10 @@ type breakGlassResult struct { panelHostname string reverseProxyGuide string + // storage backend outcome + storageMethod storageMethod + storageDetail string + // Cloudflare-specific edge detail (set only when connectMethod is Cloudflare) edgeConfigured bool edgeAud string diff --git a/cmd/felis/tui_root.go b/cmd/felis/tui_root.go index 4cc3706..9742f34 100644 --- a/cmd/felis/tui_root.go +++ b/cmd/felis/tui_root.go @@ -81,6 +81,11 @@ func goBack() tea.Cmd { return func() tea.Msg { return goBackMsg{} } } // connection chooser. type reconfigureConnectMsg struct{} +// reconfigureStorageMsg is sent from the summary/status screen to re-enter the +// storage chooser — the supported way to fix a mistyped S3 detail or switch +// backends after install, without hand-editing felis.toml and the Secret. +type reconfigureStorageMsg struct{} + // ---- rootModel: top-level session ---- type wizardStage int @@ -89,6 +94,7 @@ const ( stagePreflight wizardStage = iota stageOwner stageConnect + stageStorage stageSummary // stageMenu is the break-glass operation menu. It is appended last so the // setup-flow rail indices (Preflight…Done) are unshifted; the rail is suppressed @@ -101,7 +107,7 @@ const ( // and the post-install wizard owns cells 1–4. Defining it once keeps the two // programs' breadcrumbs identical so the rail reads as a single continuous bar // rather than restarting when the wizard takes over. -var setupRailSteps = []string{"Bootstrap", "Preflight", "Owner", "Connection", "Done"} +var setupRailSteps = []string{"Bootstrap", "Preflight", "Owner", "Connection", "Storage", "Done"} type rootModel struct { ctx context.Context @@ -113,6 +119,12 @@ type rootModel struct { // (read-only), or -1 when the live screen is in front. Driven by ←/→. reviewing int + // reconfiguringConnect is set while re-entering the connection chooser from the + // summary's "change connection" (or the re-run status screen). In that flow the + // storage backend is already configured, so completing the connection returns + // straight to the summary instead of forcing the operator back through storage. + reconfiguringConnect bool + width int height int @@ -229,13 +241,36 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case connectResultMsg: m.applyConnectResult(msg) + if m.reconfiguringConnect { + // Changing only the connection — storage is already set, so skip it. + m.reconfiguringConnect = false + return m.showSummary() + } + m.stage = stageStorage + return m.adopt(newStorageChooserModel(m.rootDomain, storageLocal, s3Inputs{})) + + case storageResultMsg: + m.result.storageMethod = msg.method + m.result.storageDetail = msg.detail return m.showSummary() + case storageBackMsg: + m.stage = stageStorage + return m.adopt(newStorageChooserModel(m.rootDomain, storageLocal, s3Inputs{})) + + case reconfigureStorageMsg: + // Fixing/switching storage after install: re-enter the chooser pre-selected on + // the current backend, with the non-secret S3 fields pre-filled. + method, prefill := currentStorageInputs() + m.stage = stageStorage + return m.adopt(newStorageChooserModel(m.rootDomain, method, prefill)) + case goBackMsg: m.stage = stageConnect return m.adopt(newConnectChooserModel(m.rootDomain, m.adminHost, m.panelHost)) case reconfigureConnectMsg: + m.reconfiguringConnect = true m.stage = stageConnect return m.adopt(newConnectChooserModel(m.rootDomain, m.adminHost, m.panelHost)) } @@ -335,6 +370,12 @@ func (m *rootModel) reviewBody(stage int) string { if m.result.panelURL != "" { b.WriteString(tuiLabel.Render("panel ") + m.result.panelURL) } + case stageStorage: + b.WriteString(tuiOK.Render("✓ Storage") + "\n") + b.WriteString(tuiLabel.Render("backend ") + storageMethodLabel(m.result.storageMethod) + "\n") + if m.result.storageDetail != "" { + b.WriteString(tuiHint.Render(m.result.storageDetail)) + } } b.WriteString("\n\n" + tuiHint.Render("read-only · ") + tuiLabel.Render("←/→") + tuiHint.Render(" walk steps · ") + tuiLabel.Render("esc") + tuiHint.Render(" back")) @@ -449,6 +490,7 @@ func (m *rootModel) showSummary() (tea.Model, tea.Cmd) { ownerUsername: m.result.username, ownerPassword: m.result.displayPassword, accessLabel: connectMethodLabel(m.result.connectMethod), + storageLabel: m.result.storageDetail, routedHosts: routed, localHint: m.result.connectMethod == connectLocal, }) diff --git a/cmd/felis/tui_root_test.go b/cmd/felis/tui_root_test.go index 70345bf..d6ab8c4 100644 --- a/cmd/felis/tui_root_test.go +++ b/cmd/felis/tui_root_test.go @@ -73,7 +73,7 @@ func TestRootSetupHappyPath(t *testing.T) { t.Fatalf("owner result not recorded: %+v", m.result) } - // Reverse-proxy chosen → Summary, with the connection recorded. + // Reverse-proxy chosen → Storage chooser, with the connection recorded. guide := "caddy config…" m = drive(t, m, connectResultMsg{ method: connectReverseProxy, @@ -81,12 +81,11 @@ func TestRootSetupHappyPath(t *testing.T) { adminHostname: "admin.felis.example.com", guide: guide, }) - if m.stage != stageSummary { - t.Fatalf("after connect, stage = %v, want stageSummary", m.stage) + if m.stage != stageStorage { + t.Fatalf("after connect, stage = %v, want stageStorage", m.stage) } - sum, ok := m.screen.(*summaryModel) - if !ok { - t.Fatalf("after connect, screen = %T, want *summaryModel", m.screen) + if _, ok := m.screen.(*storageChooserModel); !ok { + t.Fatalf("after connect, screen = %T, want *storageChooserModel", m.screen) } if !m.result.connectConfigured { t.Fatalf("connectConfigured not set") @@ -97,6 +96,22 @@ func TestRootSetupHappyPath(t *testing.T) { if m.result.reverseProxyGuide != guide { t.Fatalf("reverseProxyGuide = %q, want %q", m.result.reverseProxyGuide, guide) } + + // Storage chosen → Summary, with both the connection and storage recorded. + m = drive(t, m, storageResultMsg{method: storageS3, detail: "s3://bucket · minio:9000"}) + if m.stage != stageSummary { + t.Fatalf("after storage, stage = %v, want stageSummary", m.stage) + } + sum, ok := m.screen.(*summaryModel) + if !ok { + t.Fatalf("after storage, screen = %T, want *summaryModel", m.screen) + } + if m.result.storageMethod != storageS3 || m.result.storageDetail == "" { + t.Fatalf("storage result not recorded: %+v", m.result) + } + if sum.storageLabel != m.result.storageDetail { + t.Fatalf("summary storageLabel = %q, want %q", sum.storageLabel, m.result.storageDetail) + } if want := "https://panel.felis.example.com"; sum.panelURL != want { t.Fatalf("summary panelURL = %q, want %q", sum.panelURL, want) } @@ -113,6 +128,7 @@ func TestRootSetupLocalSummary(t *testing.T) { m = drive(t, m, preflightDoneMsg{}) m = drive(t, m, ownerResultMsg{username: "owner"}) m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.felis.example.com"}) + m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"}) sum, ok := m.screen.(*summaryModel) if !ok { @@ -127,6 +143,83 @@ func TestRootSetupLocalSummary(t *testing.T) { } } +// TestRootReconfigureConnectSkipsStorage locks the flow guard: from the finished +// summary, "change connection" re-enters only the connection chooser and returns +// straight to the summary — storage was already configured, so the operator is not +// dragged back through it, and the earlier storage recap is preserved. +func TestRootReconfigureConnectSkipsStorage(t *testing.T) { + m := newTestRoot(false, consoleModeSetup, "") + m = drive(t, m, preflightDoneMsg{}) + m = drive(t, m, ownerResultMsg{username: "owner", displayPassword: "hunter2"}) + m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.felis.example.com"}) + m = drive(t, m, storageResultMsg{method: storageS3, detail: "s3://bucket"}) + if _, ok := m.screen.(*summaryModel); !ok { + t.Fatalf("after first run, screen = %T, want *summaryModel", m.screen) + } + + // "change connection" re-enters the connection chooser. + m = drive(t, m, reconfigureConnectMsg{}) + if m.stage != stageConnect { + t.Fatalf("reconfigure stage = %v, want stageConnect", m.stage) + } + if _, ok := m.screen.(*connectChooserModel); !ok { + t.Fatalf("reconfigure screen = %T, want *connectChooserModel", m.screen) + } + + // Completing it returns straight to the summary — NOT the storage chooser — + // with the original storage recap intact. + m = drive(t, m, connectResultMsg{method: connectReverseProxy, panelHostname: "panel.felis.example.com", guide: "caddy…"}) + if m.stage != stageSummary { + t.Fatalf("after reconfigure connect, stage = %v, want stageSummary", m.stage) + } + sum, ok := m.screen.(*summaryModel) + if !ok { + t.Fatalf("after reconfigure connect, screen = %T, want *summaryModel", m.screen) + } + if sum.storageLabel != "s3://bucket" { + t.Fatalf("reconfigure summary storageLabel = %q, want preserved %q", sum.storageLabel, "s3://bucket") + } + if m.result.connectMethod != connectReverseProxy { + t.Fatalf("reconfigure did not update connectMethod: %v", m.result.connectMethod) + } +} + +// TestRootReconfigureStorageReEntersChooser locks the post-install "change storage" +// path: from the finished summary it re-enters the storage chooser (not the +// connection one) and returns to the summary carrying the new storage recap. +func TestRootReconfigureStorageReEntersChooser(t *testing.T) { + m := newTestRoot(false, consoleModeSetup, "") + m = drive(t, m, preflightDoneMsg{}) + m = drive(t, m, ownerResultMsg{username: "owner"}) + m = drive(t, m, connectResultMsg{method: connectLocal, panelHostname: "panel.felis.example.com"}) + m = drive(t, m, storageResultMsg{method: storageLocal, detail: "local disk · /var/lib/felis/uploads"}) + if _, ok := m.screen.(*summaryModel); !ok { + t.Fatalf("after first run, screen = %T, want *summaryModel", m.screen) + } + + // "change storage" re-enters the storage chooser. + m = drive(t, m, reconfigureStorageMsg{}) + if m.stage != stageStorage { + t.Fatalf("reconfigure-storage stage = %v, want stageStorage", m.stage) + } + if _, ok := m.screen.(*storageChooserModel); !ok { + t.Fatalf("reconfigure-storage screen = %T, want *storageChooserModel", m.screen) + } + + // Completing it returns to the summary with the updated storage recap. + m = drive(t, m, storageResultMsg{method: storageS3, detail: "s3://newbucket"}) + if m.stage != stageSummary { + t.Fatalf("after reconfigure-storage, stage = %v, want stageSummary", m.stage) + } + sum, ok := m.screen.(*summaryModel) + if !ok { + t.Fatalf("after reconfigure-storage, screen = %T, want *summaryModel", m.screen) + } + if sum.storageLabel != "s3://newbucket" { + t.Fatalf("summary storageLabel = %q, want updated %q", sum.storageLabel, "s3://newbucket") + } +} + func TestRootRerunLandsOnStatus(t *testing.T) { // adminExists at start of a setup run = re-run: preflight should skip straight // to the "manage in panel" status screen, never touching owner/connect. diff --git a/cmd/felis/tui_storage.go b/cmd/felis/tui_storage.go new file mode 100644 index 0000000..7af5fdb --- /dev/null +++ b/cmd/felis/tui_storage.go @@ -0,0 +1,407 @@ +package main + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + + "felis.lolicon.best/internal/platform" + + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/huh" +) + +// ---- Storage backends ---- + +type storageMethod int + +const ( + storageLocal storageMethod = iota + storageS3 +) + +func storageMethodLabel(m storageMethod) string { + if m == storageS3 { + return "Object storage (S3-compatible)" + } + return "Local disk (on this node)" +} + +// s3Inputs is the operator-entered coordinates for the S3 backend. Only endpoint, +// bucket and region reach felis.toml; the two keys go into a Kubernetes Secret. +type s3Inputs struct { + endpoint string + bucket string + region string + accessKey string + secretKey string +} + +// storageChooserModel presents the two build-context storage backends as peer +// choices. "Local" persists uploaded modpacks on a node-local PVC (nothing else to +// configure); "S3" points them at an S3-compatible bucket. Both are functional; S3 +// suits external object storage. It mirrors connectChooserModel's shape so the two +// mid-wizard forks read identically. +type storageChooserModel struct { + rootDomain string + + form *huh.Form + choice storageMethod + prefill s3Inputs // pre-filled non-secret fields when re-entering to change storage + width, height int +} + +// newStorageChooserModel opens the storage picker pre-selected on method and, for +// S3, carrying prefill's non-secret fields (endpoint/bucket/region) into the detail +// form. First-run callers pass (storageLocal, s3Inputs{}); the reconfigure path +// passes the backend already in felis.toml so an operator fixing a typo doesn't +// retype everything (credentials still must be re-entered — they live only in the +// Secret). +func newStorageChooserModel(rootDomain string, method storageMethod, prefill s3Inputs) *storageChooserModel { + m := &storageChooserModel{rootDomain: rootDomain, choice: method, prefill: prefill} + m.form = m.build() + return m +} + +func (m *storageChooserModel) build() *huh.Form { + return m.sized(newFelisForm(huh.NewGroup( + huh.NewSelect[storageMethod](). + Title("Where should uploaded modpacks be stored?"). + Description("Players submit modpacks for review; approved ones are built from here. You can change this later."). + Value(&m.choice). + Options( + huh.NewOption("Local disk · nothing else to set up", storageLocal), + huh.NewOption("Object storage · S3-compatible bucket", storageS3), + ), + // A dim footnote, subordinate to the picker — the connect-chooser pattern. + huh.NewNote().Description( + "⚠ Local keeps uploads on this node's disk — simplest, ideal for a single-node install. "+ + "Choose S3 for external object storage (AWS S3, MinIO, Cloudflare R2, …)."), + ))) +} + +func (m *storageChooserModel) sized(f *huh.Form) *huh.Form { + if m.width > 0 { + return f.WithWidth(m.width).WithHeight(m.height) + } + return f +} + +func (m *storageChooserModel) setSize(w, h int) { + m.width, m.height = w, h + if m.form != nil { + m.form = m.form.WithWidth(w).WithHeight(h) + } +} + +func (m *storageChooserModel) Init() tea.Cmd { return m.form.Init() } + +func (m *storageChooserModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + if key, ok := msg.(tea.KeyMsg); ok { + switch key.String() { + case "ctrl+c": + return m, tea.Quit + case "esc": + // Skipping is choosing local — the simplest backend, changeable later. + return newStorageModel(storageLocal, s3Inputs{}), nil + } + } + + form, cmd := m.form.Update(msg) + if f, ok := form.(*huh.Form); ok { + m.form = f + } + switch m.form.State { + case huh.StateCompleted: + return newStorageModel(m.choice, m.prefill), nil + case huh.StateAborted: + return m, tea.Quit + } + return m, cmd +} + +func (m *storageChooserModel) View() string { return m.form.View() } + +// arrowNavOK lets the root repurpose ←/→ to walk the step rail: the picker uses +// ↑/↓, so the horizontal arrows are free. +func (m *storageChooserModel) arrowNavOK() bool { return true } + +// ---- Storage apply: local applies immediately, S3 collects then applies ---- + +type storageStep int + +const ( + ssForm storageStep = iota // S3 only: collect endpoint/bucket/keys + ssWorking + ssDone + ssError +) + +type storageApplyMsg struct{ err error } + +// storageResultMsg is the method-agnostic outcome the root advances on, emitted +// once the chosen backend is written and the API has rolled. +type storageResultMsg struct { + method storageMethod + detail string +} + +// storageBackMsg returns from the storage sub-screen to the chooser. +type storageBackMsg struct{} + +func goBackStorage() tea.Cmd { return func() tea.Msg { return storageBackMsg{} } } + +// storageModel drives applying the chosen backend. Local has no form and applies +// straight away; S3 first collects its coordinates. Both share the working → done / +// error machine, mirroring reverseProxyModel. +type storageModel struct { + method storageMethod + step storageStep + form *huh.Form + sp spinner.Model + err error + in s3Inputs + + width, height int +} + +func newStorageModel(method storageMethod, in s3Inputs) *storageModel { + sp := spinner.New() + sp.Spinner = spinner.Dot + sp.Style = tuiLabel + + m := &storageModel{method: method, sp: sp, in: in} + if method == storageS3 { + m.step = ssForm + m.form = m.build() + } else { + m.step = ssWorking + } + return m +} + +func (m *storageModel) build() *huh.Form { + return m.sized(newFelisForm(huh.NewGroup( + huh.NewNote(). + Title("Object storage (S3-compatible)"). + Description("Enter your bucket and credentials. The keys go into a Kubernetes Secret; only the endpoint and bucket are written to felis.toml."), + huh.NewInput(). + Title("Endpoint"). + Description("Your S3 API host, e.g. s3.amazonaws.com or minio.example.com:9000 (prefix http:// for a plaintext dev store)."). + Value(&m.in.endpoint). + Validate(requiredStorageField("endpoint")), + huh.NewInput(). + Title("Bucket"). + Description("An existing bucket uploads are written to."). + Value(&m.in.bucket). + Validate(validateBucketName), + huh.NewInput(). + Title("Region"). + Description("Optional — leave blank for MinIO / R2."). + Value(&m.in.region), + huh.NewInput(). + Title("Access key ID"). + Value(&m.in.accessKey). + Validate(requiredStorageField("access key ID")), + huh.NewInput(). + Title("Secret access key"). + EchoMode(huh.EchoModePassword). + Value(&m.in.secretKey). + Validate(requiredStorageField("secret access key")), + ))) +} + +func (m *storageModel) sized(f *huh.Form) *huh.Form { + if m.width > 0 { + return f.WithWidth(m.width).WithHeight(m.height) + } + return f +} + +func (m *storageModel) setSize(w, h int) { + m.width, m.height = w, h + if m.form != nil { + m.form = m.form.WithWidth(w).WithHeight(h) + } +} + +func (m *storageModel) Init() tea.Cmd { + if m.method == storageS3 { + return m.form.Init() + } + // Local: the chooser already confirmed the choice, so apply immediately. + return tea.Batch(m.sp.Tick, m.apply()) +} + +func (m *storageModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case storageApplyMsg: + if msg.err != nil { + m.step, m.err = ssError, msg.err + return m, nil + } + m.step = ssDone + return m, nil + + case spinner.TickMsg: + if m.step == ssWorking { + var cmd tea.Cmd + m.sp, cmd = m.sp.Update(msg) + return m, cmd + } + return m, nil + + case tea.KeyMsg: + switch m.step { + case ssForm: + switch msg.String() { + case "ctrl+c": + return m, tea.Quit + case "esc": + return m, goBackStorage() + } + case ssDone: + switch msg.String() { + case "ctrl+c", "esc", "enter": + return m, m.emit() + } + return m, nil + case ssError: + switch msg.String() { + case "ctrl+c": + return m, tea.Quit + case "esc": + if m.method == storageS3 { + m.step, m.err = ssForm, nil + m.form = m.build() + return m, m.form.Init() + } + return m, goBackStorage() + case "enter": + m.step, m.err = ssWorking, nil + return m, tea.Batch(m.sp.Tick, m.apply()) + } + return m, nil + case ssWorking: + if msg.String() == "ctrl+c" { + return m, tea.Quit + } + return m, nil + } + } + + if m.step == ssForm && m.form != nil { + form, cmd := m.form.Update(msg) + if f, ok := form.(*huh.Form); ok { + m.form = f + } + switch m.form.State { + case huh.StateCompleted: + m.normalizeInputs() + m.step = ssWorking + return m, tea.Batch(m.sp.Tick, m.apply()) + case huh.StateAborted: + return m, goBackStorage() + } + return m, cmd + } + return m, nil +} + +func (m *storageModel) apply() tea.Cmd { + method, in := m.method, m.in + return func() tea.Msg { + return storageApplyMsg{err: applyStorageConfig(context.Background(), method, in)} + } +} + +func (m *storageModel) emit() tea.Cmd { + method, detail := m.method, storageDetail(m.method, m.in) + return func() tea.Msg { + return storageResultMsg{method: method, detail: detail} + } +} + +func (m *storageModel) normalizeInputs() { + m.in.endpoint = strings.TrimSpace(m.in.endpoint) + m.in.bucket = strings.TrimSpace(m.in.bucket) + m.in.region = strings.TrimSpace(m.in.region) + m.in.accessKey = strings.TrimSpace(m.in.accessKey) + m.in.secretKey = strings.TrimSpace(m.in.secretKey) +} + +func (m *storageModel) View() string { + switch m.step { + case ssWorking: + return " " + m.sp.View() + " " + tuiHint.Render("Saving storage settings and rolling the API…") + "\n" + case ssDone: + var b strings.Builder + if m.method == storageS3 { + b.WriteString(tuiSuccessBanner("Object storage configured.") + "\n\n") + b.WriteString(tuiInfo("Uploads → s3://"+m.in.bucket+" · "+m.in.endpoint) + "\n") + } else { + b.WriteString(tuiSuccessBanner("Local storage configured.") + "\n\n") + b.WriteString(tuiInfo("Uploads → "+platform.UploadsLocalPath+" on this node") + "\n") + } + b.WriteString("\n" + tuiAction("enter", "continue")) + return b.String() + case ssError: + var b strings.Builder + b.WriteString(tuiErrorBanner("Could not save storage settings.") + "\n\n") + if m.err != nil { + b.WriteString(tuiHint.Render(m.err.Error()) + "\n") + } + if m.method == storageS3 { + b.WriteString("\n" + tuiAction("enter", "retry", "esc", "edit")) + } else { + b.WriteString("\n" + tuiAction("enter", "retry", "esc", "back")) + } + return b.String() + default: + if m.form == nil { + return "" + } + return m.form.View() + } +} + +// arrowNavOK yields the horizontal arrows to the rail except while the S3 form is +// taking text input (where ←/→ move the cursor). +func (m *storageModel) arrowNavOK() bool { return m.step != ssForm } + +// storageDetail is the one-line backend recap shown on the summary and in review. +func storageDetail(method storageMethod, in s3Inputs) string { + if method == storageS3 { + return "s3://" + in.bucket + " · " + in.endpoint + } + return "local disk · " + platform.UploadsLocalPath +} + +// requiredStorageField rejects a blank value with a field-named message. +func requiredStorageField(name string) func(string) error { + return func(s string) error { + if strings.TrimSpace(s) == "" { + return fmt.Errorf("%s is required", name) + } + return nil + } +} + +// bucketNameRE is a pragmatic S3 bucket-name check: 3–63 chars, lowercase +// letters/digits/dots/hyphens, starting and ending alphanumeric. It catches typos +// without trying to encode every provider's exact rules. +var bucketNameRE = regexp.MustCompile(`^[a-z0-9][a-z0-9.\-]{1,61}[a-z0-9]$`) + +func validateBucketName(s string) error { + s = strings.TrimSpace(s) + if s == "" { + return errors.New("bucket is required") + } + if !bucketNameRE.MatchString(s) { + return errors.New("bucket must be 3–63 chars: lowercase letters, digits, dots or hyphens") + } + return nil +} diff --git a/cmd/felis/tui_storage_apply.go b/cmd/felis/tui_storage_apply.go new file mode 100644 index 0000000..d217d50 --- /dev/null +++ b/cmd/felis/tui_storage_apply.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "fmt" + "strings" + + "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/platform" + "felis.lolicon.best/internal/submit" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +// applyStorageConfig persists the operator's storage choice and rolls felis-api so +// it picks up the new backend. For local it stamps user_uploads_context at the +// uploads PVC mount; for S3 it stamps the s3:// base + the [registry.s3] endpoint +// and credential refs, and creates the felis-uploads-s3 Secret the deployment reads +// the keys from. It mirrors applyReverseProxy — the same write-config → +// apply-secret → roll chain, hardcoding the default "felis" namespace as the rest +// of the wizard does. +func applyStorageConfig(ctx context.Context, method storageMethod, in s3Inputs) error { + var uploadsCtx string + var s3cfg config.RegistryS3Config + if method == storageS3 { + // Preflight the coordinates BEFORE touching config, the Secret, or the + // deployment: a mistyped key, wrong endpoint, or missing bucket fails here at + // the keyboard instead of silently at the first real upload. Nothing has been + // written yet, so a failed check leaves the install untouched. + if err := submit.CheckS3Access(ctx, submit.S3StoreConfig{ + Base: "s3://" + in.bucket, + Endpoint: in.endpoint, + Region: in.region, + AccessKey: in.accessKey, + SecretKey: in.secretKey, + }); err != nil { + return err + } + uploadsCtx = "s3://" + in.bucket + s3cfg = config.RegistryS3Config{ + Endpoint: in.endpoint, + Region: in.region, + AccessKeyRef: platform.UploadsS3AccessKeyEnv, + SecretKeyRef: platform.UploadsS3SecretKeyEnv, + } + } else { + uploadsCtx = platform.UploadsLocalPath + } + + if err := writeStorageConfig(uploadsCtx, s3cfg); err != nil { + return err + } + // S3: land the credentials in their own Secret BEFORE the roll, so the optional + // env refs resolve on the fresh pod. Local needs no Secret. + if method == storageS3 { + if err := applyUploadsS3Secret(ctx, in.accessKey, in.secretKey); err != nil { + return err + } + } + if err := applyFelisConfigSecret(ctx); err != nil { + return err + } + if err := kubectl(ctx, "-n", "felis", "rollout", "restart", "deployment/felis-api"); err != nil { + return err + } + return kubectl(ctx, "-n", "felis", "rollout", "status", "deployment/felis-api", "--timeout=180s") +} + +// currentStorageInputs reads the storage backend already recorded in felis.toml so +// the reconfigure flow can pre-select the method and pre-fill the non-secret S3 +// fields (endpoint/bucket/region). Credentials live only in the felis-uploads-s3 +// Secret and are deliberately never read back — they must be re-entered to change. +// Any read error falls back to a blank local default rather than blocking reconfig. +func currentStorageInputs() (storageMethod, s3Inputs) { + cfg, err := config.Load(hostSetupConfigPath) + if err != nil { + return storageLocal, s3Inputs{} + } + base := cfg.Registry.UserUploadsContext + if !strings.HasPrefix(strings.ToLower(base), "s3://") { + return storageLocal, s3Inputs{} + } + bucket := base[len("s3://"):] + if i := strings.IndexByte(bucket, '/'); i >= 0 { + bucket = bucket[:i] + } + return storageS3, s3Inputs{ + endpoint: cfg.Registry.S3.Endpoint, + bucket: bucket, + region: cfg.Registry.S3.Region, + } +} + +// writeStorageConfig stamps the uploads backend into both the host and pod config +// files. The S3 subtable is set for S3 and cleared (zero value) for local, so +// switching backends never leaves stale coordinates behind. +func writeStorageConfig(uploadsCtx string, s3cfg config.RegistryS3Config) error { + for _, path := range []string{hostSetupConfigPath, podSetupConfigPath} { + cfg, err := config.Load(path) + if err != nil { + return err + } + cfg.Registry.UserUploadsContext = uploadsCtx + cfg.Registry.S3 = s3cfg + if err := writeConfig(path, cfg); err != nil { + return err + } + } + return nil +} + +// applyUploadsS3Secret creates (or replaces) the felis-uploads-s3 Secret the +// felis-api Deployment mounts the S3 credentials from. The Secret is rendered +// in-process and piped to `kubectl apply` — the keys are NEVER passed as +// command-line args, so they never appear in the host process table. +func applyUploadsS3Secret(ctx context.Context, accessKey, secretKey string) error { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: platform.UploadsS3SecretName, Namespace: "felis"}, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{ + platform.UploadsS3SecretAccessKey: accessKey, + platform.UploadsS3SecretSecretKey: secretKey, + }, + } + manifest, err := yaml.Marshal(secret) + if err != nil { + return fmt.Errorf("render uploads s3 secret: %w", err) + } + return kubectlWithInput(ctx, manifest, "apply", "-f", "-") +} diff --git a/cmd/felis/tui_summary.go b/cmd/felis/tui_summary.go index 9719c9c..d54019d 100644 --- a/cmd/felis/tui_summary.go +++ b/cmd/felis/tui_summary.go @@ -16,6 +16,7 @@ type summaryModel struct { ownerUsername string ownerPassword string // one-time; shown once accessLabel string + storageLabel string // build-context storage backend recap; empty to omit routedHosts []string alreadySetUp bool // re-run: Owner pre-existed localHint bool // show the self-signed-cert note @@ -32,6 +33,8 @@ func (m *summaryModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch key.String() { case "c", "C": return m, func() tea.Msg { return reconfigureConnectMsg{} } + case "s", "S": + return m, func() tea.Msg { return reconfigureStorageMsg{} } case "ctrl+c", "esc", "enter", "q": return m, tea.Quit } @@ -59,6 +62,9 @@ func (m *summaryModel) View() string { if m.accessLabel != "" { card.WriteString(tuiLabel.Render("access ") + m.accessLabel + "\n") } + if m.storageLabel != "" { + card.WriteString(tuiLabel.Render("storage ") + m.storageLabel + "\n") + } if len(m.routedHosts) > 0 { card.WriteString(tuiLabel.Render("routed ") + strings.Join(m.routedHosts, ", ") + "\n") } @@ -72,6 +78,6 @@ func (m *summaryModel) View() string { b.WriteString(tuiHint.Render(" The local certificate is self-signed; your browser may warn on first visit.") + "\n") } - b.WriteString("\n" + tuiAction("c", "change connection", "enter/esc", "exit")) + b.WriteString("\n" + tuiAction("c", "change connection", "s", "change storage", "enter/esc", "exit")) return b.String() } diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 6ec33ff..d8922d1 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -2139,6 +2139,47 @@ paths: '503': $ref: '#/components/responses/ServiceUnavailable' + /api/v1/me/submissions/{id}/context: + post: + tags: [submissions] + operationId: uploadSubmissionContext + summary: Upload the modpack build context for your own pending submission (user side; user-directed lane over §16). + description: >- + The request body IS the raw gzip build context (context.tar.gz) — not + JSON, not multipart — streamed to the platform-derived, id-namespaced + location Kaniko reads via --context. The submitter is taken from the + principal; a submission the caller does not own is reported as 404, so + this endpoint cannot upload to or probe another user's submission. Only a + pending_review submission accepts a context (409 otherwise); a wrong-format + or oversize body is rejected with 400. Returns 503 when the deployment's + context store has no implemented upload transport. + x-felis-face: [external] + x-felis-tier: app + security: [{ accessJWT: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/gzip: + schema: { type: string, format: binary } + responses: + '200': + description: Context stored; the submission (unchanged) is returned. + content: + application/json: + schema: { $ref: '#/components/schemas/Submission' } + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '404': + $ref: '#/components/responses/NotFound' + '409': + $ref: '#/components/responses/Conflict' + '503': + $ref: '#/components/responses/ServiceUnavailable' + # ------------------------------------------------------ external: admin ---- /api/v1/servers/{name}: patch: diff --git a/go.mod b/go.mod index 125c570..75c39bb 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module felis.lolicon.best go 1.26 require ( - github.com/BurntSushi/toml v1.4.0 + github.com/BurntSushi/toml v1.6.0 github.com/charmbracelet/bubbles v1.0.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/huh v1.0.0 @@ -28,14 +28,14 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/catppuccin/go v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/charmbracelet/colorprofile v0.4.1 // indirect - github.com/charmbracelet/x/ansi v0.11.6 // indirect + github.com/charmbracelet/colorprofile v0.4.3 // indirect + github.com/charmbracelet/x/ansi v0.11.7 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect - github.com/clipperhouse/displaywidth v0.9.0 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect - github.com/clipperhouse/uax29/v2 v2.5.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect @@ -52,7 +52,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -62,11 +62,17 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.2.1 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -79,10 +85,13 @@ require ( github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/tinylib/msgp v1.6.4 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/net v0.54.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect @@ -92,9 +101,10 @@ require ( golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.31.0 // indirect diff --git a/go.sum b/go.sum index 0b2f4e5..b8cf23a 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= @@ -20,12 +22,16 @@ github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlv github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk= github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk= +github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= +github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/huh v1.0.0 h1:wOnedH8G4qzJbmhftTqrpppyqHakl/zbbNdXIWJyIxw= github.com/charmbracelet/huh v1.0.0/go.mod h1:5YVc+SlZ1IhQALxRPpkGwwEKftN/+OlJlnJYlDRFqN4= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8= github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ= +github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= +github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= github.com/charmbracelet/x/conpty v0.1.0 h1:4zc8KaIcbiL4mghEON8D72agYtSeIgq8FSThSPQIb+U= @@ -44,10 +50,14 @@ github.com/charmbracelet/x/xpty v0.1.2 h1:Pqmu4TEJ8KeA9uSkISKMU3f+C1F6OGBn8ABuGl github.com/charmbracelet/x/xpty v0.1.2/go.mod h1:XK2Z0id5rtLWcpeNiMYBccNNBrP2IJnzHI0Lq13Xzq4= github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA= github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U= github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= @@ -101,6 +111,8 @@ github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYu github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba h1:qJEJcuLzH5KDR0gKc0zcktin6KSAwL7+jWKBYceddTc= @@ -128,6 +140,13 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -137,6 +156,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -145,6 +166,14 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.2.1 h1:PfBfwvKB/MmqyN8Vb1G9voWisaM9OrLv+WwOvMwS9Dw= +github.com/minio/minio-go/v7 v7.2.1/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -183,16 +212,23 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= @@ -203,6 +239,8 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= @@ -211,6 +249,9 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -262,6 +303,8 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -269,6 +312,8 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= +gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/internal/api/api.go b/internal/api/api.go index 178d9c1..5c8000d 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -369,6 +369,11 @@ func (a *API) externalAPIRoutes() []apiRoute { // session is the correct gate (the admin verdict lives below, behind adminOnly). {Method: "POST", Pattern: "/api/v1/me/submissions", h: a.handleCreateSubmission}, {Method: "GET", Pattern: "/api/v1/me/submissions", h: a.handleMySubmissions}, + // The blob upload for a submission the caller owns: the request body is the + // raw gzip build context, streamed to the derived, id-namespaced location. + // App-tier and owner-scoped (the id must belong to the principal), exactly + // like the create/list routes above. + {Method: "POST", Pattern: "/api/v1/me/submissions/{id}/context", h: a.handleUploadSubmissionContext}, // Admin (Zero-Trust) tier: create / mutate spec / image admission. These gate // on Principal.IsAdmin() inside the handler via the adminOnly wrapper, so the // boundary is exercised even where the body is a later-phase stub. diff --git a/internal/api/submissions.go b/internal/api/submissions.go index 6d3871e..9b8bb47 100644 --- a/internal/api/submissions.go +++ b/internal/api/submissions.go @@ -3,6 +3,7 @@ package api import ( "context" "errors" + "io" "net/http" "felis.lolicon.best/internal/submit" @@ -27,6 +28,10 @@ type SubmissionService interface { // Create records a pending_review submission. It starts NO build (the whole // point of the lane — nothing is built until an admin approves). Create(ctx context.Context, req submit.CreateRequest) (*submit.Submission, error) + // UploadContext stores the modpack blob for the caller's own pending + // submission at the platform-derived context ref. submittedBy is the principal, + // never the body, so a user can only upload to a submission they own. + UploadContext(ctx context.Context, id, submittedBy string, r io.Reader) (*submit.Submission, error) // ListBy returns one user's submissions, newest first (the "my uploads" view). ListBy(ctx context.Context, submittedBy string) ([]submit.Submission, error) // List returns every submission, newest first (the admin review queue). @@ -80,6 +85,32 @@ func (a *API) handleCreateSubmission(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusCreated, sub) } +// handleUploadSubmissionContext stores the caller's modpack blob as the build +// context for their own pending submission (app-tier). The request body IS the +// raw gzip tarball (context.tar.gz) — not JSON, not multipart — streamed straight +// to the transport; the submit layer sniffs the gzip magic and caps the size. The +// submitter is the authenticated principal, never the body, and a submission the +// caller does not own is reported as 404, so this endpoint cannot upload to — or +// probe the existence of — another user's submission. +// +// Uploading does not change the submission row (there is no "uploaded" column): +// the blob store is the presence source of truth, which admin approval consults. +func (a *API) handleUploadSubmissionContext(w http.ResponseWriter, r *http.Request) { + if a.Submissions == nil { + writeError(w, r, errSubmissionsUnavailable) + return + } + p := principalFromContext(r.Context()) + id := r.PathValue("id") + sub, err := a.Submissions.UploadContext(r.Context(), id, p.UserID, r.Body) + if err != nil { + writeSubmitError(w, r, err) + return + } + a.audit(r, p.Email, "submission.upload", sub.ID) + writeJSON(w, http.StatusOK, sub) +} + // handleMySubmissions lists the caller's own submissions (app-tier). It scopes // strictly to the principal's id; there is no parameter that could widen the // query to another user's uploads. @@ -162,8 +193,10 @@ var errSubmissionsUnavailable = newError(http.StatusServiceUnavailable, "submiss "modpack submission subsystem is not configured") // writeSubmitError maps submit-package errors onto HTTP status codes. Only the -// three business sentinels are client-facing: a validation failure is 400, a -// missing submission is 404, an already-reviewed submission is 409. Everything +// business sentinels are client-facing: a validation failure is 400, a missing +// submission is 404, an already-reviewed submission is 409, and an unconfigured +// upload transport is 503 (the store this deployment set has no implemented +// transport — an honest "not available here", not a client error). Everything // else — including a build.ErrInvalid raised by the pre-CAS build.Validate (a // platform registry/context MISCONFIGURATION, never client input, since every // build input is platform-derived) and a post-CAS Submit hand-off failure — is a @@ -178,6 +211,9 @@ func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) { case errors.Is(err, submit.ErrAlreadyReviewed): writeError(w, r, newError(http.StatusConflict, "already_reviewed", "submission has already been reviewed")) + case errors.Is(err, submit.ErrUploadsUnavailable): + writeError(w, r, newError(http.StatusServiceUnavailable, "uploads_unavailable", + "modpack upload transport is not configured")) default: writeError(w, r, err) } diff --git a/internal/api/submissions_test.go b/internal/api/submissions_test.go index d5b96f5..283315a 100644 --- a/internal/api/submissions_test.go +++ b/internal/api/submissions_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "testing" @@ -18,6 +19,10 @@ import ( type fakeSubmissions struct { created *submit.CreateRequest createErr error + uploadedID string + uploadedBy string + uploadedN int64 + uploadErr error listedBy string byResult []submit.Submission byErr error @@ -42,6 +47,16 @@ func (f *fakeSubmissions) Create(_ context.Context, req submit.CreateRequest) (* DisplayName: req.DisplayName, Status: submit.StatusPendingReview}, nil } +func (f *fakeSubmissions) UploadContext(_ context.Context, id, submittedBy string, r io.Reader) (*submit.Submission, error) { + f.uploadedID, f.uploadedBy = id, submittedBy + if f.uploadErr != nil { + return nil, f.uploadErr + } + n, _ := io.Copy(io.Discard, r) + f.uploadedN = n + return &submit.Submission{ID: id, SubmittedBy: submittedBy, Status: submit.StatusPendingReview}, nil +} + func (f *fakeSubmissions) ListBy(_ context.Context, submittedBy string) ([]submit.Submission, error) { f.listedBy = submittedBy return f.byResult, f.byErr @@ -135,6 +150,89 @@ func TestCreateSubmissionValidationIs400(t *testing.T) { } } +// The upload endpoint forwards the raw body to the transport and stamps the +// submitter from the principal, never the body — a user can only upload to a +// submission under their own identity. +func TestUploadSubmissionContextStreamsBody(t *testing.T) { + fs := &fakeSubmissions{} + api := appSubAPI(fs) + // A tiny gzip-magic-prefixed body stands in for a real context.tar.gz. + body := "\x1f\x8b\x08\x00 the modpack bytes" + w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", body, + map[string]string{"Content-Type": "application/gzip"}) + if w.Code != http.StatusOK { + t.Fatalf("code = %d, want 200 (%s)", w.Code, w.Body.String()) + } + if fs.uploadedID != "sub-9" { + t.Errorf("uploaded id = %q, want sub-9", fs.uploadedID) + } + if fs.uploadedBy != "user-7" { + t.Errorf("submitter = %q, want the principal id user-7", fs.uploadedBy) + } + if fs.uploadedN != int64(len(body)) { + t.Errorf("streamed %d bytes, want %d", fs.uploadedN, len(body)) + } +} + +// A submission the caller does not own reads back as 404 (the transport reports +// ErrNotFound), so the endpoint cannot probe another user's submission. +func TestUploadSubmissionContextNotOwnedIs404(t *testing.T) { + fs := &fakeSubmissions{uploadErr: submit.ErrNotFound} + api := appSubAPI(fs) + w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-x/context", "\x1f\x8bdata", nil) + if w.Code != http.StatusNotFound { + t.Fatalf("code = %d, want 404 (%s)", w.Code, w.Body.String()) + } +} + +// Uploading to an already-reviewed submission is a 409. +func TestUploadSubmissionContextAlreadyReviewedIs409(t *testing.T) { + fs := &fakeSubmissions{uploadErr: submit.ErrAlreadyReviewed} + api := appSubAPI(fs) + w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", nil) + if w.Code != http.StatusConflict { + t.Fatalf("code = %d, want 409 (%s)", w.Code, w.Body.String()) + } +} + +// A wrong-format / oversize body surfaces as 400 (the transport wraps ErrInvalid). +func TestUploadSubmissionContextBadFormatIs400(t *testing.T) { + fs := &fakeSubmissions{uploadErr: fmt.Errorf("%w: build context must be a gzip-compressed tarball (.tar.gz)", submit.ErrInvalid)} + api := appSubAPI(fs) + w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "not gzip", nil) + if w.Code != http.StatusBadRequest { + t.Fatalf("code = %d, want 400 (%s)", w.Code, w.Body.String()) + } + if got := decodeErr(t, w); got != "bad_request" { + t.Errorf("error code = %q, want bad_request", got) + } +} + +// When the deployment's store has no upload transport, the endpoint reports 503 +// (ErrUploadsUnavailable) — an honest "not available here", not a 500. +func TestUploadSubmissionContextNoTransportIs503(t *testing.T) { + fs := &fakeSubmissions{uploadErr: submit.ErrUploadsUnavailable} + api := appSubAPI(fs) + w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", nil) + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("code = %d, want 503 (%s)", w.Code, w.Body.String()) + } + if got := decodeErr(t, w); got != "uploads_unavailable" { + t.Errorf("error code = %q, want uploads_unavailable", got) + } +} + +// The upload route is app-tier: with no service configured it is 503, exactly +// like the other /me/submissions routes. +func TestUploadSubmissionContextWithoutServiceIs503(t *testing.T) { + app := appSubAPI(nil) + app.Submissions = nil + w := do(app.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", nil) + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("code = %d, want 503 (%s)", w.Code, w.Body.String()) + } +} + // The "my uploads" list scopes strictly to the principal's id — there is no // parameter that could widen it to another user's submissions. func TestMySubmissionsScopesToPrincipal(t *testing.T) { diff --git a/internal/config/config.go b/internal/config/config.go index e84560f..cb11dd4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -78,6 +78,27 @@ type RegistryConfig struct { // archive (§19 WorldArchiver) and a build context (§16) are different artifacts // with different lifecycles, so the two must not share a store binding. UserUploadsContext string `toml:"user_uploads_context"` + // S3 configures the object-store backend for user_uploads_context when it is an + // s3:// base (the alternative to a local uploads path). It mirrors + // ArchiveS3Config: Endpoint + Region locate the store and the *Ref fields NAME + // the environment variables felis-api reads the credentials from — never the + // secrets themselves, so no S3 key is ever written into felis.toml. The setup + // wizard injects those env vars into felis-api from a separate Secret + // (felis-uploads-s3). The bucket (and any key prefix) is taken from + // user_uploads_context itself, so it is not duplicated here. Empty for a + // local-storage install. + S3 RegistryS3Config `toml:"s3"` +} + +// RegistryS3Config is the [registry.s3] subtable: the object-store coordinates for +// a user_uploads_context that is an s3:// base. It deliberately reads like +// ArchiveS3Config (endpoint + credential refs) so the two S3 bindings are +// consistent, but omits Bucket because the s3:// base already carries it. +type RegistryS3Config struct { + Endpoint string `toml:"endpoint"` + Region string `toml:"region"` + AccessKeyRef string `toml:"access_key_ref"` + SecretKeyRef string `toml:"secret_key_ref"` } // ArchiveConfig is the [archive] table plus its [archive.s3] subtable (spec §19). diff --git a/internal/platform/workloads.go b/internal/platform/workloads.go index f7583d2..3d212e3 100644 --- a/internal/platform/workloads.go +++ b/internal/platform/workloads.go @@ -76,6 +76,32 @@ const ( registryVolume = "data" worldsVolume = "worlds" backupVolume = "backup" + uploadsVolume = "uploads" + + // uploads* wire the user-uploads build-context store into felis-api. The PVC + // backs a LOCAL user_uploads_context — durable across pod restarts and + // fsGroup-writable by the non-root pod (unlike a root-owned hostPath). It is + // always rendered but only written to when uploads are local. The S3 secret/env + // carry credentials for an s3:// user_uploads_context and are OPTIONAL, so a + // local-storage install (no such Secret) still starts. + uploadsPVCName = "felis-uploads" + uploadsStorageSize = "5Gi" + // UploadsLocalPath is the in-pod mount of the uploads PVC; a local + // user_uploads_context points here so the derived context ref and the on-disk + // write location agree. Exported so the setup wizard stamps it into felis.toml. + UploadsLocalPath = "/var/lib/felis/uploads" + // UploadsS3SecretName is the out-of-band Secret carrying the S3 credentials for + // an s3:// user_uploads_context. felis-api mounts its keys into env (optionally) + // and the setup wizard creates it. Like configSecretName it is NEVER rendered + // into the bundle — the credentials are the same red line. + UploadsS3SecretName = "felis-uploads-s3" + UploadsS3SecretAccessKey = "access_key_id" + UploadsS3SecretSecretKey = "secret_access_key" + // UploadsS3AccessKeyEnv / UploadsS3SecretKeyEnv are the env vars felis-api reads + // the S3 credentials from; registry.s3.access_key_ref / secret_key_ref default to + // these names. The deployment injects them from UploadsS3SecretName (optional). + UploadsS3AccessKeyEnv = "FELIS_UPLOADS_S3_ACCESS_KEY" + UploadsS3SecretKeyEnv = "FELIS_UPLOADS_S3_SECRET_KEY" // worldsMountPath is where the reaper CronJob mounts the worlds-root (read-only). // It is the default of `felis reaper --worlds-root`; the resolver then reads each @@ -129,6 +155,7 @@ func Workloads(p Params) []Object { registryDeployment(p), registryService(p), registryPVC(p), + uploadsPVC(p), } if reaperEnabled(p) { objs = append(objs, reaperCronJob(p)) @@ -159,6 +186,11 @@ func reaperEnabled(p Params) bool { // restore executor launches `felis restore` with the same image. FELIS_BACKUP_PVC // is rendered only when a backup PVC is named — otherwise the restore endpoint // degrades to 503 rather than enqueuing a Job that cannot mount its backup. +// +// The uploads PVC is mounted read-write at UploadsLocalPath for a local +// user_uploads_context, and the two optional S3 credential env vars +// (UploadsS3*Env, from the felis-uploads-s3 Secret) feed an s3:// one — the two +// storage backends the setup wizard chooses between. func APIDeployment(p Params) *appsv1.Deployment { p = p.withDefaults() @@ -177,6 +209,20 @@ func APIDeployment(p Params) *appsv1.Deployment { if p.BackupPVC != "" { env = append(env, corev1.EnvVar{Name: "FELIS_BACKUP_PVC", Value: p.BackupPVC}) } + // S3 credentials for an s3:// user_uploads_context, sourced from the + // felis-uploads-s3 Secret. Optional: a local-storage install has no such Secret, + // and marking these optional lets the pod start anyway (the store falls back to + // the uploads PVC). The setup wizard creates the Secret and rolls the API when + // the operator picks S3 storage. + optional := boolPtr(true) + env = append(env, + corev1.EnvVar{Name: UploadsS3AccessKeyEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: UploadsS3SecretName}, Key: UploadsS3SecretAccessKey, Optional: optional, + }}}, + corev1.EnvVar{Name: UploadsS3SecretKeyEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: UploadsS3SecretName}, Key: UploadsS3SecretSecretKey, Optional: optional, + }}}, + ) container := corev1.Container{ Name: ComponentAPI, @@ -199,6 +245,10 @@ func APIDeployment(p Params) *appsv1.Deployment { {Name: configVolume, MountPath: configMountPath, ReadOnly: true}, {Name: "tls", MountPath: apiTLSMountPath, ReadOnly: true}, {Name: tmpVolume, MountPath: "/tmp"}, + // Read-WRITE: local uploads land here (an s3:// store bypasses it). The + // hardened container root is read-only, so this PVC mount is where a local + // LocalContextStore can persist a submitted context. + {Name: uploadsVolume, MountPath: UploadsLocalPath}, }, Resources: controlPlaneResources(), SecurityContext: hardenedContainerSecurityContext(), @@ -218,6 +268,12 @@ func APIDeployment(p Params) *appsv1.Deployment { }, }, {Name: tmpVolume, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + { + Name: uploadsVolume, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: uploadsPVCName}, + }, + }, } return controlPlaneDeployment(p, SAAPI, container, volumes) @@ -509,6 +565,27 @@ func registryPVC(p Params) *corev1.PersistentVolumeClaim { } } +// uploadsPVC renders the felis-api user-uploads PVC (spec §16 build-context input +// domain). It backs a LOCAL user_uploads_context: felis-api mounts it read-write +// at UploadsLocalPath and LocalContextStore writes each submission's context there. +// It is always rendered (an s3:// install simply never writes to it) and carries +// control-plane labels so it reads as part of felis-api's storage. ReadWriteOnce is +// the fail-safe access mode: the single-node starter binds it to felis-api's node, +// and a future Kaniko-read integration mounts the same PVC on that node. +func uploadsPVC(p Params) *corev1.PersistentVolumeClaim { + p = p.withDefaults() + return &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "PersistentVolumeClaim"}, + ObjectMeta: metav1.ObjectMeta{Name: uploadsPVCName, Namespace: p.ControlNamespace, Labels: controlPlanePodLabels(ComponentAPI)}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(uploadsStorageSize)}, + }, + }, + } +} + // registryLabels are the registry's recommended labels. Note the absence of // part-of=felis-control-plane: that is what keeps the registry out of the RCON // NetworkPolicy peer's reach (asserted in workloads_test.go). diff --git a/internal/platform/workloads_test.go b/internal/platform/workloads_test.go index a036f7e..a302109 100644 --- a/internal/platform/workloads_test.go +++ b/internal/platform/workloads_test.go @@ -215,6 +215,44 @@ func TestAPIDeployment_Wiring(t *testing.T) { } } +// TestAPIDeployment_UploadsStorage pins both storage backends' wiring: the local +// uploads PVC mounted read-write, and the two S3 credential env vars sourced +// optionally from the felis-uploads-s3 Secret (so a local install still starts). +func TestAPIDeployment_UploadsStorage(t *testing.T) { + ps, c := podSpec(t, APIDeployment(testParams())) + + // Local backend: uploads PVC mounted read-WRITE at UploadsLocalPath. + vol := volumeByName(ps.Volumes, uploadsVolume) + if vol == nil || vol.PersistentVolumeClaim == nil || vol.PersistentVolumeClaim.ClaimName != uploadsPVCName { + t.Fatalf("uploads volume must mount PVC %q, got %#v", uploadsPVCName, vol) + } + if m := mountByName(c.VolumeMounts, uploadsVolume); m == nil || m.MountPath != UploadsLocalPath || m.ReadOnly { + t.Errorf("uploads mount = %#v, want read-write at %s", m, UploadsLocalPath) + } + + // S3 backend: both credential env vars come from the Secret (never literals) and + // are OPTIONAL, so a local install with no such Secret still starts. + for _, ev := range []struct{ name, key string }{ + {UploadsS3AccessKeyEnv, UploadsS3SecretAccessKey}, + {UploadsS3SecretKeyEnv, UploadsS3SecretSecretKey}, + } { + e := envVar(c.Env, ev.name) + if e == nil || e.ValueFrom == nil || e.ValueFrom.SecretKeyRef == nil { + t.Fatalf("%s must be sourced from a secretKeyRef", ev.name) + } + ref := e.ValueFrom.SecretKeyRef + if ref.Name != UploadsS3SecretName || ref.Key != ev.key { + t.Errorf("%s ref = %s/%s, want %s/%s", ev.name, ref.Name, ref.Key, UploadsS3SecretName, ev.key) + } + if ref.Optional == nil || !*ref.Optional { + t.Errorf("%s secretKeyRef must be optional (a local install has no such Secret)", ev.name) + } + if e.Value != "" { + t.Errorf("%s must not carry a literal value", ev.name) + } + } +} + func TestAPIService_NodePort(t *testing.T) { p := testParams() p.PanelNodePort = 30445 @@ -341,8 +379,8 @@ func TestRegistry_DeploymentServicePVC(t *testing.T) { // every one with TypeMeta (so its YAML header renders). func TestWorkloads_BundleContents(t *testing.T) { objs := Workloads(testParams()) - if len(objs) != 6 { - t.Fatalf("Workloads returned %d objects, want 6", len(objs)) + if len(objs) != 7 { + t.Fatalf("Workloads returned %d objects, want 7", len(objs)) } for _, o := range objs { gvk := o.GetObjectKind().GroupVersionKind() diff --git a/internal/submit/blobstore.go b/internal/submit/blobstore.go new file mode 100644 index 0000000..abb698d --- /dev/null +++ b/internal/submit/blobstore.go @@ -0,0 +1,115 @@ +package submit + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "regexp" +) + +// contextBlobName is the fixed object name of a submission's build context under +// its id-namespaced prefix. It is the single source of truth for both the +// derived context ref (deriveContextRef) and the on-disk write target +// (LocalContextStore), so the blob always lands exactly where Kaniko's +// --context points (build/jobspec.go). +const contextBlobName = "context.tar.gz" + +// idRE re-validates a submission id at the storage boundary. The Manager only +// ever passes ids it loaded from the Store (already the crypto-hex ids newID +// mints), but LocalContextStore is a standalone component that treats the id as +// untrusted path input: a lowercase-alphanumeric-with-dashes id can contain no +// path separator and no "..", so it can never escape Base. This is the same +// defense-in-depth stance as internal/backup's zip-slip guard. +var idRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,127}$`) + +// LocalContextStore is the filesystem-backed build-context blob store: it writes +// each submission's uploaded modpack to {Base}/{id}/context.tar.gz on a mounted +// PVC. It is one of two implemented Blobs backends — cmd/felis selects it when +// user_uploads_context is a local path and S3ContextStore when it is an s3:// base; +// a base that is neither (or an s3:// base with no credentials configured) leaves +// Manager.Blobs nil so the upload endpoint returns 503 rather than pretending to +// accept a file it cannot persist. +// +// Base MUST equal the Manager's ContextStore so the blob lands exactly where +// deriveContextRef points Kaniko's --context; cmd/felis wires both from the one +// config field (registry.user_uploads_context). +// +// INTEGRATION-ONLY seam (out of scope of the upload transport): persisting the +// blob is end-to-end only once the same uploads PVC is mounted into the Kaniko +// build Pod and Kaniko is told to read a local context (build/jobspec.go passes +// the ref straight into --context). The transport here makes the file durable at +// the derived location; wiring that path into the sandboxed build Job is a +// separate deployment integration, exactly like the restore executor's PVC mount. +type LocalContextStore struct { + // Base is the directory (uploads PVC mount) submission contexts are written + // under. Each submission gets its own {Base}/{id}/ subdirectory. + Base string +} + +// dir returns the per-submission directory, rejecting an id that could escape +// Base. Every path the store touches is rooted here. +func (s *LocalContextStore) dir(id string) (string, error) { + if !idRE.MatchString(id) { + return "", fmt.Errorf("submit: invalid submission id %q", id) + } + return filepath.Join(s.Base, id), nil +} + +// Put writes r to {Base}/{id}/context.tar.gz atomically: it streams into a temp +// file in the same directory and renames it over any previous upload only on a +// fully successful copy. So a failed, truncated, or oversize upload never +// replaces a good context and never leaves a half-written blob for Kaniko to +// read; a re-upload while the submission is still pending simply supersedes the +// previous one. It returns the number of bytes stored. +func (s *LocalContextStore) Put(_ context.Context, id string, r io.Reader) (int64, error) { + dir, err := s.dir(id) + if err != nil { + return 0, err + } + if err := os.MkdirAll(dir, 0o750); err != nil { + return 0, fmt.Errorf("submit: mkdir context dir: %w", err) + } + tmp, err := os.CreateTemp(dir, contextBlobName+".*.tmp") + if err != nil { + return 0, fmt.Errorf("submit: create temp context: %w", err) + } + tmpName := tmp.Name() + n, err := io.Copy(tmp, r) + if err != nil { + tmp.Close() + os.Remove(tmpName) + return 0, fmt.Errorf("submit: write context blob: %w", err) + } + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + return 0, fmt.Errorf("submit: close context blob: %w", err) + } + if err := os.Rename(tmpName, filepath.Join(dir, contextBlobName)); err != nil { + os.Remove(tmpName) + return 0, fmt.Errorf("submit: commit context blob: %w", err) + } + return n, nil +} + +// Exists reports whether a context blob has been stored for id. Approve consults +// it so a submission whose context was never uploaded is refused BEFORE the CAS, +// instead of being approved into a build Kaniko cannot pull. +func (s *LocalContextStore) Exists(_ context.Context, id string) (bool, error) { + dir, err := s.dir(id) + if err != nil { + return false, err + } + switch _, err := os.Stat(filepath.Join(dir, contextBlobName)); { + case err == nil: + return true, nil + case os.IsNotExist(err): + return false, nil + default: + return false, fmt.Errorf("submit: stat context blob: %w", err) + } +} + +// Compile-time proof that the filesystem store satisfies the Blobs transport. +var _ Blobs = (*LocalContextStore)(nil) diff --git a/internal/submit/blobstore_test.go b/internal/submit/blobstore_test.go new file mode 100644 index 0000000..c09feae --- /dev/null +++ b/internal/submit/blobstore_test.go @@ -0,0 +1,94 @@ +package submit + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLocalContextStorePutAndExists(t *testing.T) { + base := t.TempDir() + s := &LocalContextStore{Base: base} + ctx := context.Background() + + if ok, err := s.Exists(ctx, "sub-abc"); err != nil || ok { + t.Fatalf("Exists before Put = (%v, %v), want (false, nil)", ok, err) + } + + payload := "\x1f\x8b\x08\x00the modpack context" + n, err := s.Put(ctx, "sub-abc", strings.NewReader(payload)) + if err != nil { + t.Fatalf("Put: %v", err) + } + if n != int64(len(payload)) { + t.Fatalf("Put returned %d bytes, want %d", n, len(payload)) + } + + // The blob lands at exactly {base}/{id}/context.tar.gz — where deriveContextRef + // points Kaniko's --context. + dest := filepath.Join(base, "sub-abc", contextBlobName) + got, err := os.ReadFile(dest) + if err != nil { + t.Fatalf("read stored blob: %v", err) + } + if string(got) != payload { + t.Fatalf("stored %q, want %q", got, payload) + } + if ok, err := s.Exists(ctx, "sub-abc"); err != nil || !ok { + t.Fatalf("Exists after Put = (%v, %v), want (true, nil)", ok, err) + } + + // The write is atomic: no leftover temp files beside the committed blob. + entries, err := os.ReadDir(filepath.Join(base, "sub-abc")) + if err != nil { + t.Fatalf("read dir: %v", err) + } + if len(entries) != 1 || entries[0].Name() != contextBlobName { + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + t.Fatalf("dir entries = %v, want only %q (no temp files)", names, contextBlobName) + } +} + +func TestLocalContextStorePutOverwrites(t *testing.T) { + base := t.TempDir() + s := &LocalContextStore{Base: base} + ctx := context.Background() + + if _, err := s.Put(ctx, "sub-1", strings.NewReader("\x1f\x8bfirst")); err != nil { + t.Fatalf("first Put: %v", err) + } + if _, err := s.Put(ctx, "sub-1", strings.NewReader("\x1f\x8bsecond upload")); err != nil { + t.Fatalf("second Put: %v", err) + } + got, err := os.ReadFile(filepath.Join(base, "sub-1", contextBlobName)) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != "\x1f\x8bsecond upload" { + t.Fatalf("stored %q, want the second upload (a re-upload supersedes)", got) + } +} + +func TestLocalContextStoreRejectsUnsafeID(t *testing.T) { + base := t.TempDir() + s := &LocalContextStore{Base: base} + ctx := context.Background() + + for _, id := range []string{"../evil", "sub/../../etc", "SUB-UPPER", "has space", "", "a/b"} { + if _, err := s.Put(ctx, id, strings.NewReader("\x1f\x8bx")); err == nil { + t.Errorf("Put(%q) succeeded, want rejection", id) + } + if _, err := s.Exists(ctx, id); err == nil { + t.Errorf("Exists(%q) succeeded, want rejection", id) + } + } + // Nothing escaped the base directory. + if _, err := os.Stat(filepath.Join(filepath.Dir(base), "evil")); !os.IsNotExist(err) { + t.Fatal("an unsafe id wrote outside Base") + } +} diff --git a/internal/submit/s3store.go b/internal/submit/s3store.go new file mode 100644 index 0000000..372491c --- /dev/null +++ b/internal/submit/s3store.go @@ -0,0 +1,217 @@ +package submit + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "path" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// s3Client is the minimal object-store surface S3ContextStore needs. *minio.Client +// satisfies it, and a fake satisfies it in tests — so the store's key derivation +// and not-found handling are unit-verifiable without a live bucket. +type s3Client interface { + PutObject(ctx context.Context, bucket, object string, reader io.Reader, size int64, opts minio.PutObjectOptions) (minio.UploadInfo, error) + StatObject(ctx context.Context, bucket, object string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) +} + +// S3ContextStore is the object-store-backed build-context blob store: it writes +// each submission's uploaded modpack to {prefix}/{id}/context.tar.gz inside an S3 +// bucket. It is the second implemented Blobs backend (alongside LocalContextStore), +// selected by cmd/felis when user_uploads_context is an s3:// base. +// +// The bucket + key prefix are parsed from that same base (parseS3Base), so an +// object written here lands at exactly s3://{bucket}/{prefix}/{id}/context.tar.gz — +// the ref deriveContextRef records and Kaniko's native s3:// --context reads. +// Credentials are static V4 keys resolved by cmd/felis from the environment (the +// setup wizard injects them into felis-api from the felis-uploads-s3 Secret); they +// never touch felis.toml. +// +// Kaniko reading the S3 context at build time needs its own credentials + egress +// on the sandboxed build Job — a separate deployment integration, exactly like the +// LocalContextStore PVC mount. This transport only makes the upload durable at the +// derived location. +type S3ContextStore struct { + client s3Client + bucket string + prefix string // key prefix within the bucket; may be empty +} + +// S3StoreConfig is the resolved input for NewS3ContextStore. Base is the s3:// +// user_uploads_context (bucket + optional prefix are parsed from it, so the write +// path matches deriveContextRef); Endpoint may carry an http:// or https:// scheme +// (a bare host defaults to TLS); the keys come from the environment. +type S3StoreConfig struct { + Base string + Endpoint string + Region string + AccessKey string + SecretKey string +} + +// NewS3ContextStore builds a store backed by a real minio client. It fails fast +// when the base is malformed or the credentials are missing, so cmd/felis leaves +// Manager.Blobs nil (upload endpoint → 503) rather than wiring a store that cannot +// authenticate. +func NewS3ContextStore(cfg S3StoreConfig) (*S3ContextStore, error) { + bucket, prefix, err := parseS3Base(cfg.Base) + if err != nil { + return nil, err + } + if cfg.AccessKey == "" || cfg.SecretKey == "" { + return nil, errors.New("submit: s3 store requires credentials") + } + host, secure, err := splitS3Endpoint(cfg.Endpoint) + if err != nil { + return nil, fmt.Errorf("submit: s3 endpoint: %w", err) + } + client, err := minio.New(host, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: secure, + Region: cfg.Region, + }) + if err != nil { + return nil, fmt.Errorf("submit: s3 client: %w", err) + } + return &S3ContextStore{client: client, bucket: bucket, prefix: prefix}, nil +} + +// CheckS3Access verifies the S3 coordinates before they are committed to config: +// it builds a client from the entered endpoint/credentials and probes the bucket. +// It is the install-time preflight that turns a mistyped key, wrong endpoint, or +// missing bucket into an immediate, legible error at the keyboard instead of a 503 +// at the first real upload. +func CheckS3Access(ctx context.Context, cfg S3StoreConfig) error { + store, err := NewS3ContextStore(cfg) + if err != nil { + return err + } + return checkBucketAccess(ctx, store.client, store.bucket) +} + +// checkBucketAccess probes the bucket with the SAME object-level HEAD the upload +// path uses (StatObject on a key that will not exist), not a bucket-level +// HeadBucket. This matters: a least-privilege key scoped to object Put/Get may lack +// s3:ListBucket, so a HeadBucket would falsely reject a key that uploads fine. A +// NoSuchKey/absent result means the endpoint is reachable and the credentials are +// accepted — exactly the runtime dependency Exists() relies on. Split from +// CheckS3Access so the error mapping is unit-testable against a fake. +func checkBucketAccess(ctx context.Context, client s3Client, bucket string) error { + const probe = "felis-access-probe/does-not-exist" + if _, err := client.StatObject(ctx, bucket, probe, minio.StatObjectOptions{}); err != nil { + resp := minio.ToErrorResponse(err) + switch resp.Code { + case "NoSuchKey", "NotFound": + return nil // reachable + authorized; the probe object is simply absent + case "NoSuchBucket": + return fmt.Errorf("submit: bucket %q not found", bucket) + case "AccessDenied", "SignatureDoesNotMatch", "InvalidAccessKeyId": + return fmt.Errorf("submit: s3 credentials rejected: %w", err) + default: + // A bare 404 with no bucket-specific code = object absent in a live bucket. + if resp.StatusCode == http.StatusNotFound { + return nil + } + return fmt.Errorf("submit: cannot reach s3 (endpoint unreachable or credentials rejected): %w", err) + } + } + return nil // the probe object improbably exists — access clearly works +} + +// keyFor derives the object key for a submission, re-validating the id at the +// storage boundary (the same defense-in-depth as LocalContextStore: a validated id +// carries no path separator, so it cannot alter the key layout). +func (s *S3ContextStore) keyFor(id string) (string, error) { + if !idRE.MatchString(id) { + return "", fmt.Errorf("submit: invalid submission id %q", id) + } + return path.Join(s.prefix, id, contextBlobName), nil +} + +// Put streams r to the derived object key. Size is unknown (the Manager hands us a +// size-capped reader), so it is uploaded with size -1 (multipart). PutObject is +// atomic from a reader's perspective — a partial upload never becomes a readable +// object — so a failed or oversize upload never replaces a good context. It +// returns the number of bytes stored. +func (s *S3ContextStore) Put(ctx context.Context, id string, r io.Reader) (int64, error) { + key, err := s.keyFor(id) + if err != nil { + return 0, err + } + info, err := s.client.PutObject(ctx, s.bucket, key, r, -1, minio.PutObjectOptions{ContentType: "application/gzip"}) + if err != nil { + return 0, fmt.Errorf("submit: put context blob: %w", err) + } + return info.Size, nil +} + +// Exists reports whether a context blob has been stored for id. Approve consults +// it so a submission whose context was never uploaded is refused BEFORE the CAS. +func (s *S3ContextStore) Exists(ctx context.Context, id string) (bool, error) { + key, err := s.keyFor(id) + if err != nil { + return false, err + } + if _, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}); err != nil { + if isS3NotFound(err) { + return false, nil + } + return false, fmt.Errorf("submit: stat context blob: %w", err) + } + return true, nil +} + +// isS3NotFound recognizes the "object is absent" outcome across S3 +// implementations: a GET-shaped NoSuchKey code or a bare 404 from the HEAD that +// StatObject issues. +func isS3NotFound(err error) bool { + resp := minio.ToErrorResponse(err) + return resp.Code == "NoSuchKey" || resp.StatusCode == http.StatusNotFound +} + +// splitS3Endpoint separates a configured endpoint into the host[:port] minio.New +// wants and a TLS flag. A bare host defaults to TLS (the safe default); an +// explicit http:// opts out for a plaintext dev store. +func splitS3Endpoint(ep string) (host string, secure bool, err error) { + ep = strings.TrimSpace(ep) + switch { + case ep == "": + return "", false, errors.New("empty endpoint") + case strings.HasPrefix(ep, "https://"): + return strings.Trim(strings.TrimPrefix(ep, "https://"), "/"), true, nil + case strings.HasPrefix(ep, "http://"): + return strings.Trim(strings.TrimPrefix(ep, "http://"), "/"), false, nil + default: + return strings.Trim(ep, "/"), true, nil + } +} + +// parseS3Base splits an s3://bucket[/prefix] base into its bucket and key prefix. +// It is the single source of truth for how a user_uploads_context s3:// base maps +// onto object storage, kept beside the store so the write path and deriveContextRef +// can never disagree about where the blob lands. +func parseS3Base(base string) (bucket, prefix string, err error) { + rest := base + if i := strings.Index(strings.ToLower(rest), "://"); i >= 0 { + rest = rest[i+3:] + } + rest = strings.Trim(rest, "/") + if rest == "" { + return "", "", fmt.Errorf("submit: s3 base %q has no bucket", base) + } + parts := strings.SplitN(rest, "/", 2) + bucket = parts[0] + if len(parts) == 2 { + prefix = strings.Trim(parts[1], "/") + } + return bucket, prefix, nil +} + +// Compile-time proof that the object store satisfies the Blobs transport. +var _ Blobs = (*S3ContextStore)(nil) diff --git a/internal/submit/s3store_test.go b/internal/submit/s3store_test.go new file mode 100644 index 0000000..defb5a9 --- /dev/null +++ b/internal/submit/s3store_test.go @@ -0,0 +1,224 @@ +package submit + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/minio/minio-go/v7" +) + +// fakeS3 is an in-memory s3Client: it records puts and returns a NoSuchKey/404 +// error for a missing stat, so S3ContextStore's key derivation and not-found +// handling are exercised without a live bucket. +type fakeS3 struct { + objects map[string][]byte + putErr error + statErr error // when set, StatObject returns it (e.g. auth rejected / bucket missing) +} + +func (f *fakeS3) PutObject(_ context.Context, bucket, object string, r io.Reader, _ int64, _ minio.PutObjectOptions) (minio.UploadInfo, error) { + if f.putErr != nil { + return minio.UploadInfo{}, f.putErr + } + data, err := io.ReadAll(r) + if err != nil { + return minio.UploadInfo{}, err + } + if f.objects == nil { + f.objects = map[string][]byte{} + } + f.objects[bucket+"/"+object] = data + return minio.UploadInfo{Bucket: bucket, Key: object, Size: int64(len(data))}, nil +} + +func (f *fakeS3) StatObject(_ context.Context, bucket, object string, _ minio.StatObjectOptions) (minio.ObjectInfo, error) { + if f.statErr != nil { + return minio.ObjectInfo{}, f.statErr + } + if data, ok := f.objects[bucket+"/"+object]; ok { + return minio.ObjectInfo{Key: object, Size: int64(len(data))}, nil + } + return minio.ObjectInfo{}, minio.ErrorResponse{Code: "NoSuchKey", StatusCode: http.StatusNotFound} +} + +func TestCheckBucketAccess(t *testing.T) { + ctx := context.Background() + + // A reachable bucket where the probe object is simply absent (NoSuchKey) — the + // object-scoped key case that a bucket-level HeadBucket would wrongly reject. + if err := checkBucketAccess(ctx, &fakeS3{}, "b"); err != nil { + t.Fatalf("reachable bucket, probe absent = %v, want nil", err) + } + // A bare 404 (object absent, no bucket-specific code) is also success. + if err := checkBucketAccess(ctx, &fakeS3{statErr: minio.ErrorResponse{StatusCode: http.StatusNotFound}}, "b"); err != nil { + t.Fatalf("bare 404 = %v, want nil (object absent in a live bucket)", err) + } + // A missing bucket is a real error. + if err := checkBucketAccess(ctx, &fakeS3{statErr: minio.ErrorResponse{Code: "NoSuchBucket", StatusCode: http.StatusNotFound}}, "b"); err == nil { + t.Fatal("missing bucket = nil, want error") + } + // Rejected credentials are a real error. + if err := checkBucketAccess(ctx, &fakeS3{statErr: minio.ErrorResponse{Code: "AccessDenied", StatusCode: http.StatusForbidden}}, "b"); err == nil { + t.Fatal("rejected credentials = nil, want error") + } + // An unreachable endpoint (non-HTTP error) is a real error. + if err := checkBucketAccess(ctx, &fakeS3{statErr: errors.New("dial tcp: connection refused")}, "b"); err == nil { + t.Fatal("unreachable endpoint = nil, want error") + } +} + +func TestCheckS3AccessValidatesConfigFirst(t *testing.T) { + // A malformed base fails at construction, before any network probe is attempted. + if err := CheckS3Access(context.Background(), S3StoreConfig{Base: "s3://", Endpoint: "x", AccessKey: "a", SecretKey: "b"}); err == nil { + t.Fatal("CheckS3Access with no bucket succeeded, want error") + } +} + +func TestS3ContextStorePutAndExists(t *testing.T) { + fake := &fakeS3{} + s := &S3ContextStore{client: fake, bucket: "felis-uploads", prefix: "builds"} + ctx := context.Background() + + if ok, err := s.Exists(ctx, "sub-abc"); err != nil || ok { + t.Fatalf("Exists before Put = (%v, %v), want (false, nil)", ok, err) + } + + payload := "\x1f\x8b\x08\x00the modpack context" + n, err := s.Put(ctx, "sub-abc", strings.NewReader(payload)) + if err != nil { + t.Fatalf("Put: %v", err) + } + if n != int64(len(payload)) { + t.Fatalf("Put returned %d bytes, want %d", n, len(payload)) + } + + // The blob lands at exactly {prefix}/{id}/context.tar.gz — the key half of the + // s3://bucket/prefix/id/context.tar.gz ref deriveContextRef records. + wantKey := "felis-uploads/builds/sub-abc/" + contextBlobName + if got := string(fake.objects[wantKey]); got != payload { + t.Fatalf("object at %q = %q, want %q", wantKey, got, payload) + } + if ok, err := s.Exists(ctx, "sub-abc"); err != nil || !ok { + t.Fatalf("Exists after Put = (%v, %v), want (true, nil)", ok, err) + } +} + +func TestS3ContextStoreEmptyPrefix(t *testing.T) { + fake := &fakeS3{} + s := &S3ContextStore{client: fake, bucket: "b", prefix: ""} + if _, err := s.Put(context.Background(), "sub-1", strings.NewReader("\x1f\x8bx")); err != nil { + t.Fatalf("Put: %v", err) + } + // No prefix ⇒ the key is just {id}/context.tar.gz (no leading slash). + if _, ok := fake.objects["b/sub-1/"+contextBlobName]; !ok { + t.Fatalf("object not at expected key; got keys %v", s3KeysOf(fake.objects)) + } +} + +func TestS3ContextStoreRejectsUnsafeID(t *testing.T) { + fake := &fakeS3{} + s := &S3ContextStore{client: fake, bucket: "b", prefix: "p"} + ctx := context.Background() + + for _, id := range []string{"../evil", "sub/../../etc", "SUB-UPPER", "has space", "", "a/b"} { + if _, err := s.Put(ctx, id, strings.NewReader("\x1f\x8bx")); err == nil { + t.Errorf("Put(%q) succeeded, want rejection", id) + } + if _, err := s.Exists(ctx, id); err == nil { + t.Errorf("Exists(%q) succeeded, want rejection", id) + } + } + if len(fake.objects) != 0 { + t.Fatalf("an unsafe id wrote an object: %v", s3KeysOf(fake.objects)) + } +} + +func TestParseS3Base(t *testing.T) { + cases := []struct { + base string + bucket, prefix string + wantErr bool + }{ + {"s3://felis-user-uploads", "felis-user-uploads", "", false}, + {"s3://bucket/builds", "bucket", "builds", false}, + {"s3://bucket/a/b/c", "bucket", "a/b/c", false}, + {"S3://Bucket/", "Bucket", "", false}, + {"s3://bucket/pre/", "bucket", "pre", false}, + {"s3://", "", "", true}, + {"s3:///onlyslash", "", "", false}, // trims to "onlyslash" bucket + } + for _, c := range cases { + bucket, prefix, err := parseS3Base(c.base) + if (err != nil) != c.wantErr { + t.Errorf("parseS3Base(%q) err = %v, wantErr %v", c.base, err, c.wantErr) + continue + } + if err != nil { + continue + } + if c.base == "s3:///onlyslash" { + if bucket != "onlyslash" { + t.Errorf("parseS3Base(%q) bucket = %q, want onlyslash", c.base, bucket) + } + continue + } + if bucket != c.bucket || prefix != c.prefix { + t.Errorf("parseS3Base(%q) = (%q, %q), want (%q, %q)", c.base, bucket, prefix, c.bucket, c.prefix) + } + } +} + +func TestSplitS3Endpoint(t *testing.T) { + cases := []struct { + ep string + host string + secure bool + wantErr bool + }{ + {"https://s3.amazonaws.com", "s3.amazonaws.com", true, false}, + {"http://minio:9000", "minio:9000", false, false}, + {"minio.example.com:9000", "minio.example.com:9000", true, false}, + {"https://s3.example.com/", "s3.example.com", true, false}, + {"", "", false, true}, + } + for _, c := range cases { + host, secure, err := splitS3Endpoint(c.ep) + if (err != nil) != c.wantErr { + t.Errorf("splitS3Endpoint(%q) err = %v, wantErr %v", c.ep, err, c.wantErr) + continue + } + if err != nil { + continue + } + if host != c.host || secure != c.secure { + t.Errorf("splitS3Endpoint(%q) = (%q, %v), want (%q, %v)", c.ep, host, secure, c.host, c.secure) + } + } +} + +func TestNewS3ContextStoreValidation(t *testing.T) { + if _, err := NewS3ContextStore(S3StoreConfig{Base: "s3://", AccessKey: "a", SecretKey: "b", Endpoint: "x"}); err == nil { + t.Error("NewS3ContextStore with no bucket succeeded, want error") + } + if _, err := NewS3ContextStore(S3StoreConfig{Base: "s3://b", AccessKey: "", SecretKey: "", Endpoint: "x"}); err == nil { + t.Error("NewS3ContextStore with no credentials succeeded, want error") + } + if _, err := NewS3ContextStore(S3StoreConfig{Base: "s3://b", AccessKey: "a", SecretKey: "b", Endpoint: ""}); err == nil { + t.Error("NewS3ContextStore with no endpoint succeeded, want error") + } + if _, err := NewS3ContextStore(S3StoreConfig{Base: "s3://b/pre", AccessKey: "a", SecretKey: "b", Endpoint: "minio:9000"}); err != nil { + t.Errorf("NewS3ContextStore with valid config: %v", err) + } +} + +func s3KeysOf(m map[string][]byte) []string { + var out []string + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/submit/submit.go b/internal/submit/submit.go index 444be3f..a92bf69 100644 --- a/internal/submit/submit.go +++ b/internal/submit/submit.go @@ -55,11 +55,13 @@ package submit import ( + "bufio" "context" "crypto/rand" "encoding/hex" "errors" "fmt" + "io" "regexp" "strings" "time" @@ -76,13 +78,19 @@ const ( StatusRejected Status = "rejected" ) -// Sentinels. The API layer maps ErrInvalid→400, ErrNotFound→404 and -// ErrAlreadyReviewed→409; they are kept distinct from store/cluster failures so -// those surface as 500. +// Sentinels. The API layer maps ErrInvalid→400, ErrNotFound→404, +// ErrAlreadyReviewed→409 and ErrUploadsUnavailable→503; they are kept distinct +// from store/cluster failures so those surface as 500. var ( ErrInvalid = errors.New("submit: invalid request") ErrNotFound = errors.New("submit: submission not found") ErrAlreadyReviewed = errors.New("submit: submission already reviewed") + // ErrUploadsUnavailable means this deployment configured a context store with + // no implemented upload transport (a nil Manager.Blobs — e.g. an object-store + // base with no client wired). UploadContext returns it so the endpoint reports + // an honest 503, never a 500, exactly as the restore executor does when its + // integration is not wired. + ErrUploadsUnavailable = errors.New("submit: context upload transport not configured") ) // invalidf wraps ErrInvalid so every malformed-request case maps to one 400. @@ -90,9 +98,19 @@ func invalidf(format string, a ...any) error { return fmt.Errorf("%w: "+format, append([]any{ErrInvalid}, a...)...) } +// errContextTooLarge trips when an upload exceeds the size cap. It wraps +// ErrInvalid so an oversize upload maps to a 400; the storage layer's %w wrapping +// preserves that chain through to the API error mapper. +var errContextTooLarge = fmt.Errorf("%w: build context exceeds the maximum allowed size", ErrInvalid) + const ( maxDisplayName = 200 maxRejectReason = 1000 + // defaultMaxContextBytes caps an uploaded build-context blob. Modpack contexts + // (mods, configs, an occasional bundled world) are large, so the cap is + // generous; it bounds what one untrusted upload can write to the uploads PVC, + // not a tight quota. Override per-Manager via MaxContextBytes. + defaultMaxContextBytes = 1 << 30 // 1 GiB ) // displayNameRE constrains the user-supplied label to a calm, single-line set: @@ -156,6 +174,24 @@ type Builds interface { Submit(ctx context.Context, req build.Request) (*build.Build, error) } +// Blobs is the build-context blob transport the lane depends on to place a +// submitter's uploaded modpack at the platform-derived, id-namespaced location +// deriveContextRef points Kaniko at. It is the piece the package doc calls a +// "separate, deferred transport": creation only derives and records the ref, and +// the bytes behind it arrive through Put here. It is an interface so the Manager +// is unit-tested against an in-memory fake; the production implementation is the +// filesystem-backed LocalContextStore. +// +// Both methods key off the submission id, never a caller-supplied path, so the +// write target is as platform-pinned as the derived ref itself. Put stores (and +// atomically overwrites, while the submission is still pending) the blob; Exists +// reports whether one has been stored, so Approve can refuse to build a +// submission whose context was never uploaded. +type Blobs interface { + Put(ctx context.Context, id string, r io.Reader) (int64, error) + Exists(ctx context.Context, id string) (bool, error) +} + // Manager orchestrates the approval lane. It holds no mutable state; the clock // and id generator are injectable for hermetic tests. type Manager struct { @@ -171,15 +207,30 @@ type Manager struct { // field. The derived ref is {Registry}/user-uploads/{id}:latest. Registry string // ContextStore is the pinned Kaniko build-context base for user uploads, e.g. - // "s3://felis-user-uploads" (mirrors a configured object store). The derived - // context ref is {ContextStore}/{id}/context.tar.gz; the modpack blob is - // placed there by a separate upload transport (deferred — see package doc). + // "s3://felis-user-uploads" (an object store) or a local uploads PVC path. The + // derived context ref is {ContextStore}/{id}/context.tar.gz. ContextStore string + // Blobs is the upload transport that persists the modpack behind the derived + // context ref. When nil (a store with no implemented transport, e.g. an + // object-store base with no client), UploadContext returns ErrUploadsUnavailable + // so the endpoint reports 503. Its backing MUST match ContextStore so the blob + // lands exactly where the derived ref points. + Blobs Blobs + // MaxContextBytes overrides the uploaded-context size cap; 0 uses + // defaultMaxContextBytes. + MaxContextBytes int64 Now func() time.Time IDGen func() string } +func (m *Manager) maxContextBytes() int64 { + if m.MaxContextBytes > 0 { + return m.MaxContextBytes + } + return defaultMaxContextBytes +} + func (m *Manager) now() time.Time { if m.Now != nil { return m.Now() @@ -218,7 +269,7 @@ func (m *Manager) deriveImageRef(id string) string { // selects nothing that reaches Kaniko's --context argument; only the blob behind // this pinned, id-namespaced location (placed by the upload transport) varies. func (m *Manager) deriveContextRef(id string) string { - return fmt.Sprintf("%s/%s/context.tar.gz", strings.TrimRight(m.ContextStore, "/"), id) + return fmt.Sprintf("%s/%s/%s", strings.TrimRight(m.ContextStore, "/"), id, contextBlobName) } // auditDockerfile is the audit-archive Dockerfile recorded on the build row. It @@ -274,6 +325,91 @@ func (m *Manager) Create(ctx context.Context, req CreateRequest) (*Submission, e return s, nil } +// UploadContext stores the caller's uploaded modpack as the build context for +// their OWN pending submission — the blob transport the package doc calls +// deferred. It places the bytes at exactly deriveContextRef(id), the platform- +// pinned, id-namespaced location Kaniko reads via --context, so the submitter +// selects nothing that reaches the executor beyond the modpack itself. The row +// is not mutated (there is no "uploaded" column): the blob store is the source of +// truth for presence, which Approve consults via Blobs.Exists. +// +// The gates mirror the lane's trust model: +// - only the submitter may upload; another user's id is invisible (404, not +// 403) so this endpoint cannot probe other users' submissions; +// - the context is mutable ONLY while pending_review — once approved the build +// has already consumed it, once rejected it is dead; +// - the body must be a gzip tarball (context.tar.gz) and is size-capped, so a +// wrong-format or oversize upload is rejected as a 400 without persisting. +// +// A re-upload while still pending atomically supersedes the previous blob, so a +// user can fix their pack before an admin reviews it. +func (m *Manager) UploadContext(ctx context.Context, id, submittedBy string, r io.Reader) (*Submission, error) { + if strings.TrimSpace(submittedBy) == "" { + return nil, invalidf("submitter identity is required") + } + if m.Blobs == nil { + return nil, ErrUploadsUnavailable + } + + sub, err := m.Store.GetSubmission(ctx, id) + if err != nil { + return nil, err + } + if sub.SubmittedBy != submittedBy { + // Not the owner: invisible, so the endpoint cannot confirm the id exists. + return nil, ErrNotFound + } + if sub.Status != StatusPendingReview { + return nil, ErrAlreadyReviewed + } + + // Sniff the gzip magic before touching the store so a wrong-format upload fails + // fast, without persisting anything or reading the whole body. + br := bufio.NewReader(r) + if magic, err := br.Peek(2); err != nil || magic[0] != 0x1f || magic[1] != 0x8b { + return nil, invalidf("build context must be a gzip-compressed tarball (.tar.gz)") + } + + // Cap the size: cappedReader trips errContextTooLarge on the first byte past + // the limit, so the store never persists an oversize blob (it removes its temp + // file on the copy error) and the failure surfaces as a 400, not a 500. + if _, err := m.Blobs.Put(ctx, id, &cappedReader{r: br, left: m.maxContextBytes()}); err != nil { + return nil, err + } + return sub, nil +} + +// cappedReader passes through at most left bytes; the first byte beyond the limit +// trips errContextTooLarge. It reads one probe byte past the limit to tell an +// exactly-at-limit blob (accepted) from a larger one (rejected), so a stream of +// exactly the cap is never falsely rejected. +type cappedReader struct { + r io.Reader + left int64 +} + +func (c *cappedReader) Read(p []byte) (int, error) { + if c.left <= 0 { + // At the limit: peek one more byte. Any further data means too large; EOF + // means the blob was exactly the cap. + var probe [1]byte + n, err := c.r.Read(probe[:]) + if n > 0 { + return 0, errContextTooLarge + } + if err == nil { + return 0, io.EOF + } + return 0, err + } + if int64(len(p)) > c.left { + p = p[:c.left] + } + n, err := c.r.Read(p) + c.left -= int64(n) + return n, err +} + // Approve is the admin gate. It atomically claims the pending_review -> approved // transition (CAS) and ONLY the winner starts the build, so concurrent approvals // can never double-build. The build runs through the SAME gated Builder.Submit as @@ -314,6 +450,22 @@ func (m *Manager) Approve(ctx context.Context, id, reviewedBy string) (*Submissi return nil, ErrAlreadyReviewed } + // Refuse to approve a submission whose build context was never uploaded: the + // derived context ref would point Kaniko at nothing, failing the build after a + // committed CAS. This deterministic check runs BEFORE the CAS (like the + // build.Validate below), so a missing blob leaves the row pending, never + // stranded in approved. Skipped when no transport is wired (Blobs nil): the + // deferred/object-store case cannot be checked here and must not block approve. + if m.Blobs != nil { + ok, err := m.Blobs.Exists(ctx, id) + if err != nil { + return nil, err + } + if !ok { + return nil, invalidf("no build context has been uploaded for this submission") + } + } + imageRef := m.deriveImageRef(id) req := build.Request{ ImageRef: imageRef, diff --git a/internal/submit/submit_test.go b/internal/submit/submit_test.go index 01c3454..a4844cb 100644 --- a/internal/submit/submit_test.go +++ b/internal/submit/submit_test.go @@ -3,6 +3,7 @@ package submit import ( "context" "errors" + "io" "strings" "testing" "time" @@ -10,6 +11,45 @@ import ( "felis.lolicon.best/internal/build" ) +// gzBody returns a minimal gzip-magic-prefixed blob standing in for a real +// context.tar.gz: UploadContext only sniffs the first two bytes, so the payload +// after the magic is opaque. +func gzBody(payload string) string { return "\x1f\x8b\x08\x00" + payload } + +// fakeBlobs is an in-memory Blobs transport. It records what was stored so a test +// can assert the derived id was used, and can force Exists/Put outcomes. +type fakeBlobs struct { + stored map[string][]byte + putErr error + existsErr error + forceExists *bool // overrides the stored-map lookup for the approve-gate tests +} + +func newFakeBlobs() *fakeBlobs { return &fakeBlobs{stored: map[string][]byte{}} } + +func (f *fakeBlobs) Put(_ context.Context, id string, r io.Reader) (int64, error) { + if f.putErr != nil { + return 0, f.putErr + } + b, err := io.ReadAll(r) + if err != nil { + return 0, err // e.g. cappedReader tripping — persist nothing, mirror the real store + } + f.stored[id] = b + return int64(len(b)), nil +} + +func (f *fakeBlobs) Exists(_ context.Context, id string) (bool, error) { + if f.existsErr != nil { + return false, f.existsErr + } + if f.forceExists != nil { + return *f.forceExists, nil + } + _, ok := f.stored[id] + return ok, nil +} + // testNow is the frozen clock for hermetic assertions. var testNow = time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) @@ -452,3 +492,175 @@ func TestListBy(t *testing.T) { t.Fatalf("empty submitter err = %v, want ErrInvalid", err) } } + +func TestUploadContextStoresUnderDerivedID(t *testing.T) { + m, _, _ := newManager() + fb := newFakeBlobs() + m.Blobs = fb + seed, _ := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"}) + + payload := gzBody("the modpack bytes") + sub, err := m.UploadContext(context.Background(), seed.ID, "user-1", strings.NewReader(payload)) + if err != nil { + t.Fatalf("UploadContext: %v", err) + } + if sub.ID != seed.ID { + t.Fatalf("returned submission %q, want %q", sub.ID, seed.ID) + } + // The blob is stored under the submission id (the pinned, id-namespaced key), + // never a caller-supplied path. + got, ok := fb.stored[seed.ID] + if !ok { + t.Fatalf("nothing stored under id %q; stored keys: %v", seed.ID, keysOf(fb.stored)) + } + if string(got) != payload { + t.Fatalf("stored %q, want the uploaded bytes", got) + } +} + +func TestUploadContextRejectsNonGzip(t *testing.T) { + m, _, _ := newManager() + fb := newFakeBlobs() + m.Blobs = fb + seed, _ := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"}) + + _, err := m.UploadContext(context.Background(), seed.ID, "user-1", strings.NewReader("PK\x03\x04 a zip, not gzip")) + if !errors.Is(err, ErrInvalid) { + t.Fatalf("err = %v, want ErrInvalid", err) + } + if len(fb.stored) != 0 { + t.Fatal("a wrong-format upload must persist nothing") + } +} + +func TestUploadContextOversizeRejectedAndNotPersisted(t *testing.T) { + m, _, _ := newManager() + fb := newFakeBlobs() + m.Blobs = fb + m.MaxContextBytes = 8 // tiny cap + seed, _ := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"}) + + // gzBody's 4-byte magic + payload well over 8 bytes total. + _, err := m.UploadContext(context.Background(), seed.ID, "user-1", strings.NewReader(gzBody("this is far too large"))) + if !errors.Is(err, ErrInvalid) { + t.Fatalf("err = %v, want ErrInvalid (too large)", err) + } + if len(fb.stored) != 0 { + t.Fatal("an oversize upload must persist nothing") + } +} + +func TestUploadContextExactlyAtCapAccepted(t *testing.T) { + m, _, _ := newManager() + fb := newFakeBlobs() + m.Blobs = fb + body := gzBody("payload") // measure and cap at exactly this length + m.MaxContextBytes = int64(len(body)) + seed, _ := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"}) + + if _, err := m.UploadContext(context.Background(), seed.ID, "user-1", strings.NewReader(body)); err != nil { + t.Fatalf("a blob of exactly the cap must be accepted, got %v", err) + } + if string(fb.stored[seed.ID]) != body { + t.Fatalf("stored %q, want the full body (no truncation at the cap)", fb.stored[seed.ID]) + } +} + +func TestUploadContextWrongOwnerIsNotFound(t *testing.T) { + m, _, _ := newManager() + fb := newFakeBlobs() + m.Blobs = fb + seed, _ := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"}) + + // A different user uploading to user-1's submission sees 404, not 403: the id + // is invisible so it cannot be probed. + _, err := m.UploadContext(context.Background(), seed.ID, "user-2", strings.NewReader(gzBody("x"))) + if !errors.Is(err, ErrNotFound) { + t.Fatalf("err = %v, want ErrNotFound", err) + } + if len(fb.stored) != 0 { + t.Fatal("a non-owner upload must persist nothing") + } +} + +func TestUploadContextNotPendingIsAlreadyReviewed(t *testing.T) { + m, _, _ := newManager() + fb := newFakeBlobs() + m.Blobs = fb + seed, _ := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"}) + if _, err := m.Reject(context.Background(), seed.ID, "admin@x", "nope"); err != nil { + t.Fatalf("Reject: %v", err) + } + _, err := m.UploadContext(context.Background(), seed.ID, "user-1", strings.NewReader(gzBody("x"))) + if !errors.Is(err, ErrAlreadyReviewed) { + t.Fatalf("err = %v, want ErrAlreadyReviewed (context is frozen once reviewed)", err) + } +} + +func TestUploadContextUnknownSubmission(t *testing.T) { + m, _, _ := newManager() + m.Blobs = newFakeBlobs() + _, err := m.UploadContext(context.Background(), "sub-nope", "user-1", strings.NewReader(gzBody("x"))) + if !errors.Is(err, ErrNotFound) { + t.Fatalf("err = %v, want ErrNotFound", err) + } +} + +func TestUploadContextNoTransportUnavailable(t *testing.T) { + m, _, _ := newManager() // Blobs left nil + seed, _ := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"}) + _, err := m.UploadContext(context.Background(), seed.ID, "user-1", strings.NewReader(gzBody("x"))) + if !errors.Is(err, ErrUploadsUnavailable) { + t.Fatalf("err = %v, want ErrUploadsUnavailable", err) + } +} + +func TestApproveRefusesMissingContext(t *testing.T) { + // With a transport wired, approving a submission whose context was never + // uploaded fails BEFORE the CAS: the row stays pending and no build starts. + m, st, bl := newManager() + m.Blobs = newFakeBlobs() // empty → Exists=false + seed, _ := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"}) + + _, err := m.Approve(context.Background(), seed.ID, "admin@x") + if !errors.Is(err, ErrInvalid) { + t.Fatalf("err = %v, want ErrInvalid (no context uploaded)", err) + } + if got := st.subs[seed.ID]; got.Status != StatusPendingReview { + t.Fatalf("status = %q, want still pending_review (CAS not reached)", got.Status) + } + if bl.calls != 0 { + t.Fatalf("builds started = %d, want 0", bl.calls) + } +} + +func TestApproveProceedsWithUploadedContext(t *testing.T) { + // The end-to-end user path: create -> upload -> admin approve -> exactly one + // build through the SAME gated Builder. + m, _, bl := newManager() + m.Blobs = newFakeBlobs() + seed, _ := m.Create(context.Background(), CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"}) + if _, err := m.UploadContext(context.Background(), seed.ID, "user-1", strings.NewReader(gzBody("mods"))); err != nil { + t.Fatalf("UploadContext: %v", err) + } + + sub, err := m.Approve(context.Background(), seed.ID, "admin@x") + if err != nil { + t.Fatalf("Approve: %v", err) + } + if sub.Status != StatusApproved { + t.Fatalf("status = %q, want approved", sub.Status) + } + if bl.calls != 1 { + t.Fatalf("builds started = %d, want 1", bl.calls) + } +} + +// keysOf lists a map's keys for test diagnostics. +func keysOf(m map[string][]byte) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +}