feat(build): make the user-modpack build lane read its context (closes the last functional gap)

A submitted modpack was durable but unreadable: the uploads PVC cannot cross
namespaces (felis-api mounts it; Kaniko runs in felis-build) and the s3 lane
handed the sandboxed build Pod no credentials, so NO user build could ever
consume its context. The transport is now the API itself:

- submit: derived context refs become the internal-face URL
  /api/v1/internal/submissions/{id}/context (service-token gated), and Blobs
  gains Open (local + s3) with an ErrBlobNotFound sentinel for the route's 404.
- api: serves that route on the internal face only (openapi.yaml updated; the
  route-coverage test enforces it).
- build: an http(s) context renders a context-fetch initContainer (the felis
  image's new fetch-context entrypoint) that streams the blob with the
  namespace-local service-token Secret — never mounted into Kaniko — and
  extracts it under a zip-slip guard into a size-limited emptyDir that Kaniko
  reads read-only as --context=/context.
- platform/install: the api Deployment carries its own internal base URL; the
  build namespace gets the token Secret through the existing replica mechanism
  (bootstrap.sh + felis setup); the build egress lock opens exactly the control
  namespace on the internal port.
- cmd/felis: fetch-context entrypoint (registered, documented, unit-tested for
  escapes/symlinks/non-gzip).

Tests cover rendering, hardening, the s3/local Open paths, and the route's
404/503 mapping. Verified next on the real single-node cluster with Kaniko.
This commit is contained in:
Lemon-miaow committed 2026-09-22 22:45:09 +08:00
1 parent 0c8e29b05a
commit f79e5ebb5e
26 files changed
+1080 -72

No files matched your search

+3
View File
@@ -271,6 +271,9 @@ func (a *API) internalAPIRoutes() []apiRoute {
{Method: "GET", Pattern: "/readyz", Public: true, h: a.handleReadyz},
{Method: "GET", Pattern: "/api/v1/servers", h: a.handleListServers},
// The build Pod's context-fetch initContainer streams a submission's stored
// modpack through this route (build namespace cannot mount the uploads PVC).
{Method: "GET", Pattern: "/api/v1/internal/submissions/{id}/context", h: a.handleInternalSubmissionContext},
{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/ready", h: a.handleReady},
{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/join-event", h: a.handleJoinEvent},
// Domain-autostart (spec §9.1, §14): velocity drives the wake lever and polls
+34
View File
@@ -42,6 +42,11 @@ type SubmissionService interface {
// Reject is the admin's other verdict: pending_review -> rejected with a
// required reason; it starts no build.
Reject(ctx context.Context, id, reviewedBy, reason string) (*submit.Submission, error)
// OpenContext returns the stored build-context blob for the internal
// context-fetch route: the build Pod's initContainer cannot mount the uploads
// PVC across namespaces and holds no object-store credentials, so it streams
// the blob from the API over the service-token-gated internal face instead.
OpenContext(ctx context.Context, id string) (io.ReadCloser, error)
}
// createSubmissionRequest is the POST /me/submissions body. The user
@@ -211,6 +216,8 @@ func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) {
case errors.Is(err, submit.ErrAlreadyReviewed):
writeError(w, r, newError(http.StatusConflict, "already_reviewed",
"submission has already been reviewed"))
case errors.Is(err, submit.ErrBlobNotFound):
writeError(w, r, newError(http.StatusNotFound, "not_found", "no context uploaded for this submission"))
case errors.Is(err, submit.ErrUploadsUnavailable):
writeError(w, r, newError(http.StatusServiceUnavailable, "uploads_unavailable",
"modpack upload transport is not configured"))
@@ -219,5 +226,32 @@ func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) {
}
}
// handleInternalSubmissionContext streams a submission's stored build-context
// tarball to the build Pod's `felis fetch-context` initContainer. It lives on the
// internal face (service-token, no Zero Trust) because its only caller is
// in-cluster infrastructure: the build Job runs in the build namespace, where it
// can neither mount the uploads PVC nor hold object-store credentials, so the API
// — which wrote the blob — is the transport. The blob is served verbatim; the
// fetcher extracts it under a zip-slip guard, and Kaniko treats the result as
// hostile regardless (spec §16).
func (a *API) handleInternalSubmissionContext(w http.ResponseWriter, r *http.Request) {
if a.Submissions == nil {
writeError(w, r, errSubmissionsUnavailable)
return
}
rc, err := a.Submissions.OpenContext(r.Context(), r.PathValue("id"))
if err != nil {
writeSubmitError(w, r, err)
return
}
defer rc.Close()
w.Header().Set("Content-Type", "application/gzip")
if _, err := io.Copy(w, rc); err != nil {
// The status is already committed; the client sees a truncated stream and
// the fetch fails on size/extract, so there is nothing left to write here.
return
}
}
// Compile-time proof that the production Manager satisfies the API interface.
var _ SubmissionService = (*submit.Manager)(nil)
+57
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"net/http"
"strings"
"testing"
"felis.lolicon.best/internal/submit"
@@ -35,6 +36,9 @@ type fakeSubmissions struct {
rejectedBy string
rejectReas string
rejectErr error
openedID string
openBody string
openErr error
}
func (f *fakeSubmissions) Create(_ context.Context, req submit.CreateRequest) (*submit.Submission, error) {
@@ -82,6 +86,16 @@ func (f *fakeSubmissions) Reject(_ context.Context, id, reviewedBy, reason strin
return &submit.Submission{ID: id, Status: submit.StatusRejected, ReviewedBy: reviewedBy, RejectReason: reason}, nil
}
// openErr injects the OpenContext outcome; the body recorder lets the internal
// route test assert byte-exact streaming and the 404 mapping.
func (f *fakeSubmissions) OpenContext(_ context.Context, id string) (io.ReadCloser, error) {
f.openedID = id
if f.openErr != nil {
return nil, f.openErr
}
return io.NopCloser(strings.NewReader(f.openBody)), nil
}
// appSubAPI wires a submissions service behind an ordinary user principal (the
// app tier — /me/submissions). [email protected] / .test are deliberately not the
// deployment domain.
@@ -396,3 +410,46 @@ func TestSubmissionRoutesWithoutServiceAre503(t *testing.T) {
t.Fatalf("admin route: code = %d, want 503", w.Code)
}
}
// The internal context route is the build Pod's only read path to a submission's
// blob: it streams the bytes verbatim, and its error mapping distinguishes a
// missing blob (404) from an unwired transport (503).
func TestInternalSubmissionContextRoute(t *testing.T) {
newAPI := func(s SubmissionService) *API {
api := newTestAPI(newFakeRepo(), newFakeCluster())
api.Submissions = s
return api
}
t.Run("streams the blob", func(t *testing.T) {
fs := &fakeSubmissions{openBody: "\x1f\x8b\x08\x00blob"}
w := do(newAPI(fs).InternalHandler(), "GET", "/api/v1/internal/submissions/sub-7/context", "", nil)
if w.Code != http.StatusOK {
t.Fatalf("code = %d body %s", w.Code, w.Body.String())
}
if w.Body.String() != fs.openBody {
t.Fatalf("body = %q, want the stored blob %q", w.Body.String(), fs.openBody)
}
if fs.openedID != "sub-7" {
t.Fatalf("opened id = %q, want the path id", fs.openedID)
}
if ct := w.Header().Get("Content-Type"); ct != "application/gzip" {
t.Fatalf("content-type = %q, want application/gzip", ct)
}
})
t.Run("missing blob is 404", func(t *testing.T) {
fs := &fakeSubmissions{openErr: fmt.Errorf("%w: gone", submit.ErrBlobNotFound)}
w := do(newAPI(fs).InternalHandler(), "GET", "/api/v1/internal/submissions/sub-7/context", "", nil)
if w.Code != http.StatusNotFound {
t.Fatalf("code = %d, want 404", w.Code)
}
})
t.Run("unwired transport is 503", func(t *testing.T) {
w := do(newAPI(nil).InternalHandler(), "GET", "/api/v1/internal/submissions/sub-7/context", "", nil)
if w.Code != http.StatusServiceUnavailable {
t.Fatalf("code = %d, want 503", w.Code)
}
})
}
+6
View File
@@ -215,6 +215,11 @@ type Config struct {
// RegistryURL is the internal registry the build pushes to and Trivy scans
// (spec §17). Image refs are validated to be under it.
RegistryURL string
// FelisImage is the platform image whose `fetch-context` entrypoint streams a
// submission's context from the internal face into the build Pod. Required
// only when a build's ContextRef is an http(s) URL (the submit lane's derived
// shape); an install that never builds user submissions can leave it empty.
FelisImage string
// KanikoImage / TrivyImage are the executor images.
KanikoImage string
TrivyImage string
@@ -355,6 +360,7 @@ func (b *Builder) jobParams(bld *Build, cfg Config) JobParams {
Namespace: cfg.Namespace,
ServiceAccount: cfg.ServiceAccount,
RegistryURL: cfg.RegistryURL,
FelisImage: cfg.FelisImage,
KanikoImage: cfg.KanikoImage,
TrivyImage: cfg.TrivyImage,
Deadline: cfg.Deadline,
+127 -7
View File
@@ -2,8 +2,10 @@ package build
import (
"fmt"
"strings"
"time"
"felis.lolicon.best/internal/naming"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
@@ -32,8 +34,23 @@ const (
const (
ContainerKaniko = "kaniko"
ContainerTrivy = "trivy"
// ContainerFetch is the initContainer that pulls a submission's build context
// from the felis-api internal face and extracts it into the shared emptyDir.
// It exists only for an http(s) ContextRef (see BuildJob); a ref Kaniko can
// read natively (s3://) or a pre-mounted path renders no such container.
ContainerFetch = "context-fetch"
// contextVolume/contextMountPath carry a fetched build context: the fetch
// initContainer writes the extracted tree there, Kaniko reads it read-only.
contextVolume = "context"
contextMountPath = "/context"
)
// contextSizeLimit bounds the extracted (attacker-controlled) context tree so a
// tarball bomb wedges the build pod instead of the node's disk. The compressed
// upload is capped at 1 GiB by the submit lane; 4 GiB leaves expansion room.
var contextSizeLimit = resource.MustParse("4Gi")
// JobParams are the rendered inputs to a build Job. They are derived from a
// Build + Config by the Builder; jobspec is a pure function of them so the
// security-critical Job shape is unit-tested without a cluster.
@@ -44,11 +61,14 @@ type JobParams struct {
Namespace string
ServiceAccount string
RegistryURL string
KanikoImage string
TrivyImage string
Deadline time.Duration
CPULimit string
MemLimit string
// FelisImage runs the context-fetch initContainer (the felis binary's
// fetch-context entrypoint). Required when ContextRef is an http(s) URL.
FelisImage string
KanikoImage string
TrivyImage string
Deadline time.Duration
CPULimit string
MemLimit string
}
// BuildJobName is the deterministic Job name for a build id.
@@ -100,21 +120,75 @@ func BuildJob(p JobParams) (*batchv1.Job, error) {
Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}},
}
// The context Kaniko reads. An http(s) ref (the submit lane's derived ref: the
// API streams the blob on its internal face, because the build Pod can neither
// mount the control-plane uploads PVC across namespaces nor hold object-store
// credentials) is first fetched into a shared emptyDir; a ref Kaniko can read
// in place (s3://, or a path an installer pre-mounted) passes through untouched.
contextPath := p.ContextRef
initContainers := []corev1.Container{}
var kanikoMounts []corev1.VolumeMount
var podVolumes []corev1.Volume
if isHTTPContextRef(p.ContextRef) {
if p.FelisImage == "" {
return nil, fmt.Errorf("build: context ref %q needs FelisImage for the fetch initContainer", p.ContextRef)
}
contextPath = contextMountPath
fetch := corev1.Container{
Name: ContainerFetch,
Image: p.FelisImage,
Args: []string{
"fetch-context",
"--url=" + p.ContextRef,
"--out=" + contextMountPath,
},
// The internal face is service-token gated, and the token is read from a
// Secret the installer materializes in THIS namespace (secretKeyRef is
// namespace-local). It is mounted into this initContainer only: the Kaniko
// container executes the untrusted Dockerfile and must never hold it, and
// pod containers share neither environment nor PID namespace.
Env: []corev1.EnvVar{{
Name: "FELIS_SERVICE_TOKEN",
ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: naming.ServiceTokenSecretName},
Key: naming.ServiceTokenSecretKey,
}},
}},
VolumeMounts: []corev1.VolumeMount{{Name: contextVolume, MountPath: contextMountPath}},
Resources: corev1.ResourceRequirements{Limits: limits, Requests: limits},
SecurityContext: sec,
}
initContainers = append(initContainers, fetch)
kanikoMounts = []corev1.VolumeMount{{Name: contextVolume, MountPath: contextMountPath, ReadOnly: true}}
podVolumes = []corev1.Volume{{
Name: contextVolume,
VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{
// The extracted tree is attacker-controlled; bound it so a tarball
// bomb wedges THIS pod (admitted failure) instead of filling the
// node's disk. The compressed upload is capped at 1 GiB by the
// submit lane, and 4 GiB leaves room for a typical expansion.
SizeLimit: sizeLimitPtr(),
}},
}}
}
kaniko := corev1.Container{
Name: ContainerKaniko,
Image: p.KanikoImage,
Args: []string{
"--dockerfile=Dockerfile",
"--context=" + p.ContextRef,
"--context=" + contextPath,
"--destination=" + p.ImageRef,
// The internal registry is in-cluster only and may serve plain HTTP;
// it is never a public ingress (spec §17).
"--insecure",
"--skip-tls-verify",
},
VolumeMounts: kanikoMounts,
Resources: corev1.ResourceRequirements{Limits: limits, Requests: limits},
SecurityContext: sec,
}
initContainers = append(initContainers, kaniko)
trivy := corev1.Container{
Name: ContainerTrivy,
@@ -147,8 +221,9 @@ func BuildJob(p JobParams) (*batchv1.Job, error) {
RestartPolicy: corev1.RestartPolicyNever,
ServiceAccountName: p.ServiceAccount,
AutomountServiceAccountToken: boolPtr(false),
InitContainers: []corev1.Container{kaniko},
InitContainers: initContainers,
Containers: []corev1.Container{trivy},
Volumes: podVolumes,
},
},
},
@@ -156,11 +231,24 @@ func BuildJob(p JobParams) (*batchv1.Job, error) {
return job, nil
}
// isHTTPContextRef reports whether ref is an http(s) URL — the shape the submit
// lane derives when the API is the blob transport — i.e. a context only the
// fetch initContainer can turn into a local path for Kaniko.
func isHTTPContextRef(ref string) bool {
return strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://")
}
// NetPolParams parameterises the build-namespace egress lock.
type NetPolParams struct {
Namespace string
RegistryNamespace string
RegistryPort int32
// ControlNamespace and APIPort are where the felis-api internal face lives:
// the fetch initContainer's only egress besides DNS and the registry. Both
// defaults (felis, 8081) match platform.DefaultControlNamespace and the
// internal listener, so an unset Params is still the safe shape.
ControlNamespace string
APIPort int32
// PackageSourceCIDRs is an optional, explicit allowlist of external package
// mirrors (spec §16: egress 仅 registry + 包源). Empty means the most
// locked-down default — no internet egress at all (默认拒外网).
@@ -177,10 +265,19 @@ func BuildNetworkPolicy(p NetPolParams) *networkingv1.NetworkPolicy {
if port == 0 {
port = 5000
}
controlNS := p.ControlNamespace
if controlNS == "" {
controlNS = "felis"
}
apiPort := p.APIPort
if apiPort == 0 {
apiPort = 8081
}
dnsUDP := corev1.ProtocolUDP
dnsTCP := corev1.ProtocolTCP
dns53 := intstr.FromInt32(53)
regPort := intstr.FromInt32(port)
ctxPort := intstr.FromInt32(apiPort)
egress := []networkingv1.NetworkPolicyEgressRule{
// DNS resolution: port-restricted to 53, so this is not an open-internet
@@ -203,6 +300,21 @@ func BuildNetworkPolicy(p NetPolParams) *networkingv1.NetworkPolicy {
{Protocol: &dnsTCP, Port: &regPort},
},
},
// felis-api's internal face, where the fetch initContainer streams the
// submission's build context from. Without this rule the build Pod could
// not read the context and every user build would fail in its first init
// step — the default-deny here is exactly why the transport had to be
// planned, not assumed.
{
To: []networkingv1.NetworkPolicyPeer{{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"kubernetes.io/metadata.name": controlNS},
},
}},
Ports: []networkingv1.NetworkPolicyPort{
{Protocol: &dnsTCP, Port: &ctxPort},
},
},
}
// Explicit package-mirror CIDRs, when configured. No CIDR ⇒ no internet.
for _, cidr := range p.PackageSourceCIDRs {
@@ -281,4 +393,12 @@ func resourceLimits(cpu, mem string) (corev1.ResourceList, error) {
func boolPtr(b bool) *bool { return &b }
func int32Ptr(i int32) *int32 { return &i }
// sizeLimitPtr returns a copy of contextSizeLimit for a VolumeSource (the API
// object only ever gets serialized, but a shared pointer across rendered Jobs
// invites accidental aliasing).
func sizeLimitPtr() *resource.Quantity {
q := contextSizeLimit
return &q
}
func int64Ptr(i int64) *int64 { return &i }
+126
View File
@@ -177,6 +177,132 @@ func TestBuildNetworkPolicyIsDefaultDeny(t *testing.T) {
if !egressAllowsPort(np, 53) {
t.Error("egress must allow DNS (port 53)")
}
// The context fetch: build Pods stream submissions from the control
// namespace's internal face (defaults: felis + 8081).
if !egressAllowsNamespace(np, "felis") {
t.Error("egress must allow the control namespace (context fetch)")
}
if !egressAllowsPort(np, 8081) {
t.Error("egress must allow the internal face's port (8081)")
}
}
// An http(s) context ref — the submit lane's derived shape — must render the
// fetch initContainer ahead of Kaniko, with the service token mounted ONLY into
// that container, and hand Kaniko the extracted local directory.
func TestBuildJobFetchesHTTPContext(t *testing.T) {
p := sampleJobParams()
p.ContextRef = "http://felis-api-internal.felis.svc.cluster.local:8081/api/v1/internal/submissions/sub-abc/context"
p.FelisImage = "felis:test"
job, err := BuildJob(p)
if err != nil {
t.Fatalf("BuildJob: %v", err)
}
inits := job.Spec.Template.Spec.InitContainers
if len(inits) != 2 || inits[0].Name != ContainerFetch || inits[1].Name != ContainerKaniko {
t.Fatalf("initContainers = %v, want [%s %s]", initNames(inits), ContainerFetch, ContainerKaniko)
}
fetch, kaniko := inits[0], inits[1]
if fetch.Image != p.FelisImage {
t.Errorf("fetch image = %q, want the platform image %q", fetch.Image, p.FelisImage)
}
if !hasArg(fetch.Args, "fetch-context") || !hasArg(fetch.Args, "--url="+p.ContextRef) ||
!hasArg(fetch.Args, "--out="+contextMountPath) {
t.Errorf("fetch args = %v, want fetch-context --url=%s --out=%s", fetch.Args, p.ContextRef, contextMountPath)
}
// The token comes from the namespace-local Secret and is mounted into the
// fetch container only — never into Kaniko, which executes the untrusted
// Dockerfile.
var fetchToken *corev1.EnvVar
for i := range fetch.Env {
if fetch.Env[i].Name == "FELIS_SERVICE_TOKEN" {
fetchToken = &fetch.Env[i]
}
}
if fetchToken == nil || fetchToken.ValueFrom == nil || fetchToken.ValueFrom.SecretKeyRef == nil {
t.Fatalf("fetch container must read FELIS_SERVICE_TOKEN from a secretKeyRef, got %#v", fetchToken)
}
if fetchToken.Value != "" {
t.Error("fetch container must not carry a literal token")
}
if len(kaniko.Env) != 0 {
t.Errorf("kaniko must carry no env (especially no token), got %v", kaniko.Env)
}
if !hasArg(kaniko.Args, "--context="+contextMountPath) {
t.Errorf("kaniko context = %v, want the fetched local dir %s", kaniko.Args, contextMountPath)
}
// The shared emptyDir must exist, be bounded, and be read-only to Kaniko.
var ctxVol *corev1.Volume
for i := range job.Spec.Template.Spec.Volumes {
if job.Spec.Template.Spec.Volumes[i].Name == contextVolume {
ctxVol = &job.Spec.Template.Spec.Volumes[i]
}
}
if ctxVol == nil || ctxVol.EmptyDir == nil || ctxVol.EmptyDir.SizeLimit == nil {
t.Fatalf("context volume must be a size-limited emptyDir, got %#v", ctxVol)
}
mountedRO := false
for _, m := range kaniko.VolumeMounts {
if m.Name == contextVolume && m.MountPath == contextMountPath && m.ReadOnly {
mountedRO = true
}
}
if !mountedRO {
t.Errorf("kaniko must mount the context read-only at %s, got %v", contextMountPath, kaniko.VolumeMounts)
}
}
// Without the platform image the fetch initContainer cannot run, so rendering an
// http(s) context must fail loudly at Job-creation time, not with an ImagePull
// error at 3am.
func TestBuildJobHTTPContextNeedsFelisImage(t *testing.T) {
p := sampleJobParams()
p.ContextRef = "https://example.invalid/sub-abc/context"
if _, err := BuildJob(p); err == nil {
t.Fatal("http(s) context without FelisImage must fail to render")
}
}
// A ref Kaniko reads natively (or an installer pre-mounted) must NOT grow the
// fetch initContainer: the transport is for http(s) only.
func TestBuildJobNativeContextNeedsNoFetch(t *testing.T) {
p := sampleJobParams()
p.ContextRef = "s3://bucket/prefix/context.tar.gz"
job, err := BuildJob(p)
if err != nil {
t.Fatalf("BuildJob: %v", err)
}
if len(job.Spec.Template.Spec.InitContainers) != 1 || job.Spec.Template.Spec.InitContainers[0].Name != ContainerKaniko {
t.Errorf("a native ref must render just kaniko, got %v", initNames(job.Spec.Template.Spec.InitContainers))
}
if len(job.Spec.Template.Spec.Volumes) != 0 {
t.Errorf("a native ref must render no context volume, got %v", job.Spec.Template.Spec.Volumes)
}
}
func initNames(cs []corev1.Container) []string {
names := make([]string, 0, len(cs))
for _, c := range cs {
names = append(names, c.Name)
}
return names
}
// An explicit control namespace/port override must reach the egress rule (a
// renamed control namespace otherwise silently blocks every context fetch).
func TestBuildNetworkPolicyHonoursControlNamespaceOverride(t *testing.T) {
np := BuildNetworkPolicy(NetPolParams{
Namespace: "felis-build",
RegistryNamespace: "felis-system",
ControlNamespace: "control-plane",
APIPort: 9081,
})
if !egressAllowsNamespace(np, "control-plane") {
t.Error("egress must allow the overridden control namespace")
}
if !egressAllowsPort(np, 9081) {
t.Error("egress must allow the overridden api port")
}
}
// With no package-source CIDRs configured, there must be zero IPBlock egress —
+4
View File
@@ -56,6 +56,10 @@ const (
const (
ServiceTokenSecretName = "felis-service-token"
ServiceTokenSecretKey = "token"
// EnvAPIBaseURL carries the internal-face base URL (platform.InternalAPIBaseURL)
// into a pod: the login gate dials it, and the api reads it to derive the build
// contexts' fetch URLs, so both sides name the same address for the same face.
EnvAPIBaseURL = "FELIS_API_BASE_URL"
)
// ForwardingSecretName / ForwardingSecretKey name the Velocity modern player-info
+6 -3
View File
@@ -78,9 +78,12 @@ func Objects(p Params) []Object {
// Build-namespace egress lock (reused from internal/build; TypeMeta stamped).
buildNP := build.BuildNetworkPolicy(build.NetPolParams{
Namespace: p.BuildNamespace,
RegistryNamespace: p.RegistryNamespace,
RegistryPort: p.RegistryPort,
Namespace: p.BuildNamespace,
RegistryNamespace: p.RegistryNamespace,
RegistryPort: p.RegistryPort,
ControlNamespace: p.ControlNamespace,
// The internal face's port, single-sourced with the api Deployment below.
APIPort: apiInternalPort,
PackageSourceCIDRs: p.PackageSourceCIDRs,
})
buildNP.TypeMeta = metav1.TypeMeta{APIVersion: "networking.k8s.io/v1", Kind: "NetworkPolicy"}
+4
View File
@@ -274,6 +274,10 @@ func APIDeployment(p Params) *appsv1.Deployment {
},
},
{Name: "FELIS_IMAGE", Value: p.FelisImage},
// The api's own internal-face base URL, so it derives the submission
// context URLs that build Pods fetch through it. Same value the login gate
// is handed; one address for one face.
{Name: naming.EnvAPIBaseURL, Value: InternalAPIBaseURL(p.ControlNamespace)},
}
if p.BackupPVC != "" {
env = append(env, corev1.EnvVar{Name: "FELIS_BACKUP_PVC", Value: p.BackupPVC})
+5
View File
@@ -179,6 +179,11 @@ func TestAPIDeployment_Wiring(t *testing.T) {
if v := envValue(c.Env, "FELIS_IMAGE"); v != p.FelisImage {
t.Errorf("FELIS_IMAGE = %q, want %q", v, p.FelisImage)
}
// The internal-face base URL the api derives submission context-fetch URLs
// from — the same address the login gate is handed.
if v := envValue(c.Env, "FELIS_API_BASE_URL"); v != InternalAPIBaseURL(p.ControlNamespace) {
t.Errorf("FELIS_API_BASE_URL = %q, want %q", v, InternalAPIBaseURL(p.ControlNamespace))
}
// FELIS_SERVICE_TOKEN must come from a Secret, never a literal value.
tok := envVar(c.Env, "FELIS_SERVICE_TOKEN")
if tok == nil || tok.ValueFrom == nil || tok.ValueFrom.SecretKeyRef == nil {
+18
View File
@@ -111,5 +111,23 @@ func (s *LocalContextStore) Exists(_ context.Context, id string) (bool, error) {
}
}
// Open returns the stored context blob for id — the read side of the transport the
// build Pod's fetch initContainer uses. A missing blob is ErrBlobNotFound (404 on
// the route), never a bare os error, so the API keeps its status mapping.
func (s *LocalContextStore) Open(_ context.Context, id string) (io.ReadCloser, error) {
dir, err := s.dir(id)
if err != nil {
return nil, err
}
f, err := os.Open(filepath.Join(dir, contextBlobName))
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("%w: %v", ErrBlobNotFound, err)
}
return nil, fmt.Errorf("submit: open context blob: %w", err)
}
return f, nil
}
// Compile-time proof that the filesystem store satisfies the Blobs transport.
var _ Blobs = (*LocalContextStore)(nil)
+35
View File
@@ -2,6 +2,8 @@ package submit
import (
"context"
"errors"
"io"
"os"
"path/filepath"
"strings"
@@ -54,6 +56,39 @@ func TestLocalContextStorePutAndExists(t *testing.T) {
}
}
// Open is the internal context-fetch route's read path: it serves exactly the
// stored bytes, and a missing blob is ErrBlobNotFound (404), never a bare os error.
func TestLocalContextStoreOpen(t *testing.T) {
base := t.TempDir()
s := &LocalContextStore{Base: base}
ctx := context.Background()
if _, err := s.Open(ctx, "sub-gone"); !errors.Is(err, ErrBlobNotFound) {
t.Fatalf("Open of a missing blob = %v, want ErrBlobNotFound", err)
}
payload := "\x1f\x8b\x08\x00the modpack context"
if _, err := s.Put(ctx, "sub-abc", strings.NewReader(payload)); err != nil {
t.Fatalf("Put: %v", err)
}
rc, err := s.Open(ctx, "sub-abc")
if err != nil {
t.Fatalf("Open: %v", err)
}
defer rc.Close()
got, err := io.ReadAll(rc)
if err != nil {
t.Fatalf("read: %v", err)
}
if string(got) != payload {
t.Fatalf("Open served %q, want %q", got, payload)
}
// The same path guard as Put: an id that could escape Base is refused.
if _, err := s.Open(ctx, "../etc/passwd"); err == nil {
t.Fatal("Open must reject an unsafe id")
}
}
func TestLocalContextStorePutOverwrites(t *testing.T) {
base := t.TempDir()
s := &LocalContextStore{Base: base}
+58 -9
View File
@@ -14,11 +14,30 @@ import (
)
// s3Client is the minimal object-store surface S3ContextStore needs. *minio.Client
// satisfies it, and a fake satisfies it in tests — so the store's key derivation
// and not-found handling are unit-verifiable without a live bucket.
// satisfies it through minioStoreClient, and a fake satisfies it in tests — so the
// store's key derivation and not-found handling are unit-verifiable without a live
// bucket.
type s3Client interface {
PutObject(ctx context.Context, bucket, object string, reader io.Reader, size int64, opts minio.PutObjectOptions) (minio.UploadInfo, error)
StatObject(ctx context.Context, bucket, object string, opts minio.StatObjectOptions) (minio.ObjectInfo, error)
GetObject(ctx context.Context, bucket, object string, opts minio.GetObjectOptions) (s3Object, error)
}
// s3Object is the handle GetObject yields: a stream whose Stat performs the HEAD
// eagerly, so a missing object surfaces before the first byte is read.
type s3Object interface {
io.ReadCloser
Stat() (minio.ObjectInfo, error)
}
// minioStoreClient adapts *minio.Client to s3Client. The adapter exists because a
// method's return type cannot be narrowed by an interface: GetObject on the real
// client returns a concrete *minio.Object, which does not satisfy a method declared
// to return s3Object.
type minioStoreClient struct{ *minio.Client }
func (m minioStoreClient) GetObject(ctx context.Context, bucket, object string, opts minio.GetObjectOptions) (s3Object, error) {
return m.Client.GetObject(ctx, bucket, object, opts)
}
// S3ContextStore is the object-store-backed build-context blob store: it writes
@@ -27,16 +46,17 @@ type s3Client interface {
// selected by cmd/felis when user_uploads_context is an s3:// base.
//
// The bucket + key prefix are parsed from that same base (parseS3Base), so an
// object written here lands at exactly s3://{bucket}/{prefix}/{id}/context.tar.gz —
// the ref deriveContextRef records and Kaniko's native s3:// --context reads.
// object written here lands at exactly s3://{bucket}/{prefix}/{id}/context.tar.gz.
// Credentials are static V4 keys resolved by cmd/felis from the environment (the
// setup wizard injects them into felis-api from the felis-uploads-s3 Secret); they
// never touch felis.toml.
//
// Kaniko reading the S3 context at build time needs its own credentials + egress
// on the sandboxed build Job — a separate deployment integration, exactly like the
// LocalContextStore PVC mount. This transport only makes the upload durable at the
// derived location.
// The sandboxed build Job never needs S3 credentials of its own: the api reads the
// object back here (Open) and streams it over the internal face, which is the
// transport every in-cluster build uses (Manager.ContextBaseURL). Only a
// deployment that leaves ContextBaseURL empty would fall back to Kaniko reading
// s3:// natively — and such a deployment would still need to hand the build Pod
// credentials + egress itself.
type S3ContextStore struct {
client s3Client
bucket string
@@ -79,7 +99,7 @@ func NewS3ContextStore(cfg S3StoreConfig) (*S3ContextStore, error) {
if err != nil {
return nil, fmt.Errorf("submit: s3 client: %w", err)
}
return &S3ContextStore{client: client, bucket: bucket, prefix: prefix}, nil
return &S3ContextStore{client: minioStoreClient{client}, bucket: bucket, prefix: prefix}, nil
}
// CheckS3Access verifies the S3 coordinates before they are committed to config:
@@ -167,6 +187,35 @@ func (s *S3ContextStore) Exists(ctx context.Context, id string) (bool, error) {
return true, nil
}
// Open returns the stored context blob for id — the read side of the transport the
// build Pod's fetch initContainer uses. minio's GetObject returns only once the
// server answered with an object (it surfaces NoSuchKey up front), so a missing
// object maps to ErrBlobNotFound right here and the route answers 404.
func (s *S3ContextStore) Open(ctx context.Context, id string) (io.ReadCloser, error) {
key, err := s.keyFor(id)
if err != nil {
return nil, err
}
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
if isS3NotFound(err) {
return nil, fmt.Errorf("%w: %v", ErrBlobNotFound, err)
}
return nil, fmt.Errorf("submit: open context blob: %w", err)
}
// minio.Object is lazy: the first Read triggers the GET and is where a missing
// key actually surfaces, so stat it once here to translate that case eagerly
// (the caller can then trust the io.ReadCloser belongs to a real object).
if _, err := obj.Stat(); err != nil {
_ = obj.Close()
if isS3NotFound(err) {
return nil, fmt.Errorf("%w: %v", ErrBlobNotFound, err)
}
return nil, fmt.Errorf("submit: open context blob: %w", err)
}
return obj, nil
}
// isS3NotFound recognizes the "object is absent" outcome across S3
// implementations: a GET-shaped NoSuchKey code or a bare 404 from the HEAD that
// StatObject issues.
+63
View File
@@ -45,6 +45,41 @@ func (f *fakeS3) StatObject(_ context.Context, bucket, object string, _ minio.St
return minio.ObjectInfo{}, minio.ErrorResponse{Code: "NoSuchKey", StatusCode: http.StatusNotFound}
}
// fakeS3Object is the object handle fakeS3.GetObject yields: Stat mirrors
// StatObject's not-found behaviour, Read serves the stored bytes.
type fakeS3Object struct {
data []byte
err error
}
func (o *fakeS3Object) Read(p []byte) (int, error) {
if o.err != nil {
return 0, o.err
}
if len(o.data) == 0 {
return 0, io.EOF
}
n := copy(p, o.data)
o.data = o.data[n:]
return n, nil
}
func (o *fakeS3Object) Close() error { return nil }
func (o *fakeS3Object) Stat() (minio.ObjectInfo, error) {
if o.err != nil {
return minio.ObjectInfo{}, o.err
}
return minio.ObjectInfo{Size: int64(len(o.data))}, nil
}
func (f *fakeS3) GetObject(_ context.Context, bucket, object string, _ minio.GetObjectOptions) (s3Object, error) {
if data, ok := f.objects[bucket+"/"+object]; ok {
return &fakeS3Object{data: append([]byte(nil), data...)}, nil
}
return &fakeS3Object{err: minio.ErrorResponse{Code: "NoSuchKey", StatusCode: http.StatusNotFound}}, nil
}
func TestCheckBucketAccess(t *testing.T) {
ctx := context.Background()
@@ -107,6 +142,34 @@ func TestS3ContextStorePutAndExists(t *testing.T) {
}
}
// Open serves the stored object's bytes and maps a missing key to ErrBlobNotFound
// (the internal fetch route's 404), eagerly — before the caller reads a byte.
func TestS3ContextStoreOpen(t *testing.T) {
fake := &fakeS3{}
s := &S3ContextStore{client: fake, bucket: "felis-uploads", prefix: "builds"}
ctx := context.Background()
if _, err := s.Open(ctx, "sub-gone"); !errors.Is(err, ErrBlobNotFound) {
t.Fatalf("Open of a missing object = %v, want ErrBlobNotFound", err)
}
payload := "\x1f\x8b\x08\x00the modpack context"
if _, err := s.Put(ctx, "sub-abc", strings.NewReader(payload)); err != nil {
t.Fatalf("Put: %v", err)
}
rc, err := s.Open(ctx, "sub-abc")
if err != nil {
t.Fatalf("Open: %v", err)
}
defer rc.Close()
got, err := io.ReadAll(rc)
if err != nil {
t.Fatalf("read: %v", err)
}
if string(got) != payload {
t.Fatalf("Open served %q, want %q", got, payload)
}
}
func TestS3ContextStoreEmptyPrefix(t *testing.T) {
fake := &fakeS3{}
s := &S3ContextStore{client: fake, bucket: "b", prefix: ""}
+40 -6
View File
@@ -37,7 +37,8 @@
// chosen ref would let an untrusted origin point the build at an arbitrary
// source. By deriving it from the submission id the user selects nothing
// that reaches the executor — only the modpack blob behind the pinned,
// id-namespaced location (uploaded by a separate, deferred transport).
// id-namespaced location (uploaded through the Blobs transport, and served
// back to the build Pod over the service-token-gated internal face).
//
// Source of truth. submissions is a NEW Postgres business-truth domain, added to
// §1 invariant 2's enumeration (owner/claim/accounts/audit/quota/images/builds/
@@ -91,6 +92,10 @@ var (
// an honest 503, never a 500, exactly as the restore executor does when its
// integration is not wired.
ErrUploadsUnavailable = errors.New("submit: context upload transport not configured")
// ErrBlobNotFound reports that a submission has no stored context blob (or it
// was never uploaded). The internal context-fetch route maps it to 404, the
// same distinction Exists draws for Approve.
ErrBlobNotFound = errors.New("submit: context blob not found")
)
// invalidf wraps ErrInvalid so every malformed-request case maps to one 400.
@@ -176,11 +181,12 @@ type Builds interface {
// Blobs is the build-context blob transport the lane depends on to place a
// submitter's uploaded modpack at the platform-derived, id-namespaced location
// deriveContextRef points Kaniko at. It is the piece the package doc calls a
// "separate, deferred transport": creation only derives and records the ref, and
// the bytes behind it arrive through Put here. It is an interface so the Manager
// is unit-tested against an in-memory fake; the production implementation is the
// filesystem-backed LocalContextStore.
// deriveContextRef points Kaniko at. Creation only derives and records the ref;
// the bytes behind it arrive through Put here, and the build Pod reads them back
// through Open (the manager exposes it as OpenContext, which the API's internal
// context route serves). It is an interface so the Manager is unit-tested against
// an in-memory fake; the production implementations are LocalContextStore
// (filesystem) and S3ContextStore (object store).
//
// Both methods key off the submission id, never a caller-supplied path, so the
// write target is as platform-pinned as the derived ref itself. Put stores (and
@@ -190,6 +196,11 @@ type Builds interface {
type Blobs interface {
Put(ctx context.Context, id string, r io.Reader) (int64, error)
Exists(ctx context.Context, id string) (bool, error)
// Open returns the stored blob's bytes for the internal context-fetch route
// the build Pod's initContainer dials (cmd/felis fetch-context). It returns an
// error wrapping ErrBlobNotFound when no blob exists, so the route can answer
// 404 without leaking which ids do exist.
Open(ctx context.Context, id string) (io.ReadCloser, error)
}
// Manager orchestrates the approval lane. It holds no mutable state; the clock
@@ -210,6 +221,15 @@ type Manager struct {
// "s3://felis-user-uploads" (an object store) or a local uploads PVC path. The
// derived context ref is {ContextStore}/{id}/context.tar.gz.
ContextStore string
// ContextBaseURL, when set, is the platform's internal-face base URL
// (platform.InternalAPIBaseURL). It makes the derived context ref an HTTP URL
// on that face — {ContextBaseURL}/api/v1/internal/submissions/{id}/context —
// instead of a filesystem/object-store location: the build Pod cannot mount
// the uploads PVC (builds run in another namespace) and carries no object-store
// credentials, so the API streams the blob it stored at ContextStore over the
// service-token-gated internal face. Empty keeps the legacy ref shape for a
// deployment that predates the transport.
ContextBaseURL string
// Blobs is the upload transport that persists the modpack behind the derived
// context ref. When nil (a store with no implemented transport, e.g. an
// object-store base with no client), UploadContext returns ErrUploadsUnavailable
@@ -269,9 +289,23 @@ func (m *Manager) deriveImageRef(id string) string {
// selects nothing that reaches Kaniko's --context argument; only the blob behind
// this pinned, id-namespaced location (placed by the upload transport) varies.
func (m *Manager) deriveContextRef(id string) string {
if m.ContextBaseURL != "" {
return fmt.Sprintf("%s/api/v1/internal/submissions/%s/context", strings.TrimRight(m.ContextBaseURL, "/"), id)
}
return fmt.Sprintf("%s/%s/%s", strings.TrimRight(m.ContextStore, "/"), id, contextBlobName)
}
// OpenContext returns the stored build context for id — the read path behind the
// internal context-fetch route. It requires the upload transport (Blobs): with no
// transport there is no blob to read, so it reports ErrUploadsUnavailable, the
// same honest 503 the upload endpoint gives.
func (m *Manager) OpenContext(ctx context.Context, id string) (io.ReadCloser, error) {
if m.Blobs == nil {
return nil, ErrUploadsUnavailable
}
return m.Blobs.Open(ctx, id)
}
// auditDockerfile is the audit-archive Dockerfile recorded on the build row. It
// is NOT what Kaniko executes — Kaniko reads the real Dockerfile from inside the
// uploaded context (build/jobspec.go) — so this honestly documents the
+56
View File
@@ -1,8 +1,10 @@
package submit
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"strings"
"testing"
@@ -50,6 +52,14 @@ func (f *fakeBlobs) Exists(_ context.Context, id string) (bool, error) {
return ok, nil
}
func (f *fakeBlobs) Open(_ context.Context, id string) (io.ReadCloser, error) {
b, ok := f.stored[id]
if !ok {
return nil, fmt.Errorf("%w: no blob for %s", ErrBlobNotFound, id)
}
return io.NopCloser(bytes.NewReader(b)), nil
}
// testNow is the frozen clock for hermetic assertions.
var testNow = time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
@@ -217,6 +227,52 @@ func TestCreatePendingDoesNotBuild(t *testing.T) {
}
}
// With a ContextBaseURL the derived ref is the internal-face URL the build Pod's
// fetch initContainer dials — not a filesystem path it could never read across
// namespaces. OpenContext then serves whatever the Blobs transport stored.
func TestContextRefIsFetchURLAndOpenContextServesIt(t *testing.T) {
m, _, _ := newManager()
m.ContextBaseURL = "http://felis-api-internal.felis.svc.cluster.local:8081/"
m.Blobs = newFakeBlobs()
ctx := context.Background()
sub, err := m.Create(ctx, CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"})
if err != nil {
t.Fatalf("Create: %v", err)
}
want := "http://felis-api-internal.felis.svc.cluster.local:8081/api/v1/internal/submissions/sub-1/context"
if sub.ContextRef != want {
t.Fatalf("context_ref = %q, want the internal fetch URL %q", sub.ContextRef, want)
}
// Before any upload the read path reports not-found (the route's 404).
if _, err := m.OpenContext(ctx, sub.ID); !errors.Is(err, ErrBlobNotFound) {
t.Fatalf("OpenContext before upload = %v, want ErrBlobNotFound", err)
}
payload := "\x1f\x8b\x08\x00payload"
if _, err := m.UploadContext(ctx, sub.ID, "user-1", strings.NewReader(payload)); err != nil {
t.Fatalf("UploadContext: %v", err)
}
rc, err := m.OpenContext(ctx, sub.ID)
if err != nil {
t.Fatalf("OpenContext: %v", err)
}
defer rc.Close()
got, _ := io.ReadAll(rc)
if string(got) != payload {
t.Fatalf("OpenContext served %q, want %q", got, payload)
}
}
// No upload transport ⇒ no readable blob: the route reports the same 503 the
// upload endpoint does, rather than a misleading 404.
func TestOpenContextWithoutTransportIsUnavailable(t *testing.T) {
m, _, _ := newManager()
if _, err := m.OpenContext(context.Background(), "sub-1"); !errors.Is(err, ErrUploadsUnavailable) {
t.Fatalf("OpenContext with nil Blobs = %v, want ErrUploadsUnavailable", err)
}
}
func TestCreateValidation(t *testing.T) {
cases := []struct {
name string