feat(build): make the user-modpack build lane read its context (closes the last functional gap)

A submitted modpack was durable but unreadable: the uploads PVC cannot cross
namespaces (felis-api mounts it; Kaniko runs in felis-build) and the s3 lane
handed the sandboxed build Pod no credentials, so NO user build could ever
consume its context. The transport is now the API itself:

- submit: derived context refs become the internal-face URL
  /api/v1/internal/submissions/{id}/context (service-token gated), and Blobs
  gains Open (local + s3) with an ErrBlobNotFound sentinel for the route's 404.
- api: serves that route on the internal face only (openapi.yaml updated; the
  route-coverage test enforces it).
- build: an http(s) context renders a context-fetch initContainer (the felis
  image's new fetch-context entrypoint) that streams the blob with the
  namespace-local service-token Secret — never mounted into Kaniko — and
  extracts it under a zip-slip guard into a size-limited emptyDir that Kaniko
  reads read-only as --context=/context.
- platform/install: the api Deployment carries its own internal base URL; the
  build namespace gets the token Secret through the existing replica mechanism
  (bootstrap.sh + felis setup); the build egress lock opens exactly the control
  namespace on the internal port.
- cmd/felis: fetch-context entrypoint (registered, documented, unit-tested for
  escapes/symlinks/non-gzip).

Tests cover rendering, hardening, the s3/local Open paths, and the route's
404/503 mapping. Verified next on the real single-node cluster with Kaniko.
This commit is contained in:
Lemon-miaow committed 2026-09-22 22:45:09 +08:00
1 parent 0c8e29b05a
commit f79e5ebb5e
26 files changed
+1080 -72

No files matched your search

+36 -14
View File
@@ -18,6 +18,7 @@ import (
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/fileedit"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/panel"
"felis.lolicon.best/internal/passkey"
"felis.lolicon.best/internal/platform"
@@ -142,10 +143,14 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// build namespace and pushes to the internal registry. The build Pod never
// holds DB credentials — felis-api owns the PG store and admits scanned
// images, so the Builder is constructed here with both bindings.
buildCfg := buildConfig(cfg)
// The fetch initContainer runs THIS image's fetch-context entrypoint, so the
// build config carries the api's own image (the platform sets FELIS_IMAGE).
buildCfg.FelisImage = os.Getenv("FELIS_IMAGE")
builder := &build.Builder{
Store: build.NewPGStore(drv.DB()),
Jobs: build.NewK8sJobs(cl, buildConfig(cfg)),
Config: buildConfig(cfg),
Jobs: build.NewK8sJobs(cl, buildCfg),
Config: buildCfg,
}
// User-modpack approval lane (user-directed extension over §16; see
@@ -159,14 +164,19 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// The blob upload transport is selected by the shape of user_uploads_context —
// the two backends the setup wizard chooses between. A local path wires
// LocalContextStore (the mounted uploads PVC); an s3:// base wires
// S3ContextStore when its credentials resolve. Either way the store's target is
// derived from the SAME config field the context ref uses, so the blob lands
// exactly where Kaniko's --context points. Anything else — or an s3:// base with
// no credentials configured — leaves Blobs nil so POST
// S3ContextStore when its credentials resolve. Anything else — or an s3:// base
// with no credentials configured — leaves Blobs nil so POST
// /me/submissions/{id}/context returns 503, honest like the restore executor
// when its PVC is not supplied. (Letting the sandboxed Kaniko build Pod READ the
// context — PVC mount for local, creds+egress for S3 — is a separate deployment
// integration.)
// when its PVC is not supplied.
//
// Reading the blob back is the API's job, not Kaniko's: the build Pod runs in
// another namespace and can neither mount the uploads PVC (a PVC does not cross
// namespaces) nor hold object-store credentials, so ContextBaseURL makes the
// derived context ref an internal-face URL that the build Job's fetch
// initContainer streams (cmd/felis fetch-context). The platform renders this
// address into the api Deployment (felis API base URL env); the fallback keeps
// a hand-rolled deployment working under the platform's default control
// namespace.
contextBase := cfg.Registry.UserUploadsContext
var blobs submit.Blobs
switch {
@@ -186,11 +196,12 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "felis api: user-uploads context %q is neither a local path nor an s3:// base — modpack upload transport disabled (POST /api/v1/me/submissions/{id}/context returns 503)\n", contextBase)
}
submissions := &submit.Manager{
Store: submit.NewPGStore(drv.DB()),
Builds: builder,
Registry: cfg.Registry.URL,
ContextStore: contextBase,
Blobs: blobs,
Store: submit.NewPGStore(drv.DB()),
Builds: builder,
Registry: cfg.Registry.URL,
ContextStore: contextBase,
ContextBaseURL: internalAPIBaseURL(),
Blobs: blobs,
}
// Restore subsystem (spec §7): the weak-SA restore Job mounts the target
@@ -413,6 +424,17 @@ func buildConfig(cfg *config.Config) build.Config {
}
}
// internalAPIBaseURL resolves the platform's internal-face base URL: the address
// the platform rendered into this pod (felis API base URL env), or — for a
// hand-rolled deployment that set none — the platform default control namespace,
// the same fallback setup.go uses to hand the login gate its address.
func internalAPIBaseURL() string {
if base := os.Getenv(naming.EnvAPIBaseURL); base != "" {
return base
}
return platform.InternalAPIBaseURL(platform.DefaultControlNamespace)
}
// uploadsSchemeRE matches a leading URL scheme like "s3://" or "gs://".
var uploadsSchemeRE = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://`)
+150
View File
@@ -0,0 +1,150 @@
package main
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
)
// cmdFetchContext is the in-Pod entrypoint the build Job's context-fetch
// initContainer runs. It performs one read against the felis-api INTERNAL face —
// the blob the platform stored for a submission — and extracts it into the shared
// emptyDir the Kaniko container then builds from.
//
// Why this exists: the build Pod runs in the build namespace, where it can neither
// mount the control-plane uploads PVC (a PVC does not cross namespaces) nor hold
// object-store credentials, so the API that WROTE the blob is the transport. The
// route is service-token-gated; the token arrives through a namespace-local Secret
// mounted only into this initContainer, never into Kaniko's — so the untrusted
// Dockerfile's build steps have no credential to read (their containers share no
// environment, no PID namespace, and Kaniko itself mounts the context read-only).
//
// The extraction is deliberately paranoid: the tarball is attacker-controlled
// input, so absolute paths, ".." escapes, links, and special files are refused
// rather than sanitized. Kaniko treats the extracted tree as hostile regardless
// (spec §16), but the pod's own filesystem still must not be written outside the
// context directory it was given.
func cmdFetchContext(args []string, _, stderr io.Writer) int {
fs := flag.NewFlagSet("fetch-context", flag.ContinueOnError)
fs.SetOutput(stderr)
url := fs.String("url", "", "internal-face URL of the submission's build-context tarball")
out := fs.String("out", "/context", "directory to extract the build context into")
if err := fs.Parse(args); err != nil {
return 2
}
if *url == "" {
fmt.Fprintln(stderr, "felis fetch-context: --url is required")
return 2
}
token := os.Getenv("FELIS_SERVICE_TOKEN")
if token == "" {
fmt.Fprintln(stderr, "felis fetch-context: FELIS_SERVICE_TOKEN is empty — the internal face rejects anonymous reads")
return 2
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, *url, nil)
if err != nil {
fmt.Fprintf(stderr, "felis fetch-context: bad --url: %v\n", err)
return 2
}
req.Header.Set("Authorization", "Bearer "+token)
// No overall client timeout: a legitimate modpack context can be large and the
// Job's activeDeadlineSeconds is the real bound. The header timeout catches a
// wedged endpoint without capping a healthy download.
client := &http.Client{Transport: &http.Transport{ResponseHeaderTimeout: time.Minute}}
resp, err := client.Do(req)
if err != nil {
fmt.Fprintf(stderr, "felis fetch-context: GET failed: %v\n", err)
return 1
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
fmt.Fprintf(stderr, "felis fetch-context: %s\n", resp.Status)
return 1
}
if err := extractTarGz(resp.Body, *out); err != nil {
fmt.Fprintf(stderr, "felis fetch-context: %v\n", err)
return 1
}
return 0
}
// extractTarGz streams a gzip'd tarball into root, creating directories as
// needed. Every entry is vetted BEFORE anything is written: a path that is
// absolute or escapes root (via ".."), a link (symlink or hardlink), or any
// special file kind aborts the whole extraction. Refusing rather than skipping is
// deliberate — a context that needs one of those constructs is not a context this
// transport carries, and silently dropping entries would build from a corpus the
// submitter did not upload.
func extractTarGz(r io.Reader, root string) error {
if err := os.MkdirAll(root, 0o755); err != nil {
return fmt.Errorf("create context dir: %w", err)
}
zr, err := gzip.NewReader(r)
if err != nil {
return fmt.Errorf("context is not a valid gzip tarball: %w", err)
}
defer zr.Close()
tr := tar.NewReader(zr)
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return fmt.Errorf("read context tarball: %w", err)
}
name := filepath.Clean(hdr.Name)
if name == "." {
continue
}
// The zip-slip guard: reject, never rewrite. filepath.Clean collapses any
// "a/../../b", so these two checks are sufficient once Clean has run.
if filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) {
return fmt.Errorf("context entry %q escapes the context directory", hdr.Name)
}
target := filepath.Join(root, name)
switch hdr.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, 0o755); err != nil {
return fmt.Errorf("create %q: %w", name, err)
}
case tar.TypeReg, tar.TypeRegA:
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return fmt.Errorf("create parent of %q: %w", name, err)
}
mode := os.FileMode(0o644)
if hdr.FileInfo().Mode()&0o111 != 0 {
mode = 0o755 // preserve executability (entrypoint scripts), nothing else
}
f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
if err != nil {
return fmt.Errorf("create %q: %w", name, err)
}
if _, err := io.Copy(f, tr); err != nil {
_ = f.Close()
return fmt.Errorf("write %q: %w", name, err)
}
if err := f.Close(); err != nil {
return fmt.Errorf("close %q: %w", name, err)
}
default:
return fmt.Errorf("context entry %q has unsupported type %q (links and special files are refused)", hdr.Name, string(hdr.Typeflag))
}
}
}
+171
View File
@@ -0,0 +1,171 @@
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
type tarEntry struct {
name string
body string
mode int64
typ byte
linkname string
}
// tgzBody builds an in-memory .tar.gz from entries, preserving each entry's type
// and mode so the tests can exercise the guards with exactly the bytes an
// attacker could upload.
func tgzBody(t *testing.T, entries ...tarEntry) []byte {
t.Helper()
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for _, e := range entries {
typ := e.typ
if typ == 0 {
typ = tar.TypeReg
}
mode := e.mode
if mode == 0 {
mode = 0o644
}
hdr := &tar.Header{Name: e.name, Typeflag: typ, Mode: mode, Size: int64(len(e.body))}
if typ == tar.TypeSymlink {
hdr.Linkname = e.linkname
hdr.Size = 0
}
if err := tw.WriteHeader(hdr); err != nil {
t.Fatalf("write header %q: %v", e.name, err)
}
if hdr.Size > 0 {
if _, err := tw.Write([]byte(e.body)); err != nil {
t.Fatalf("write body %q: %v", e.name, err)
}
}
}
if err := tw.Close(); err != nil {
t.Fatalf("close tar: %v", err)
}
if err := zw.Close(); err != nil {
t.Fatalf("close gzip: %v", err)
}
return buf.Bytes()
}
// A normal context extracts with its tree intact, and the executable bit that
// modpack entrypoints rely on survives.
func TestExtractTarGzRoundTrip(t *testing.T) {
dir := t.TempDir()
body := tgzBody(t,
tarEntry{name: "Dockerfile", body: "FROM scratch\n"},
tarEntry{name: "mods/example.jar", body: "jar-bytes"},
tarEntry{name: "start.sh", body: "#!/bin/sh\n", mode: 0o755},
tarEntry{name: "mods/", typ: tar.TypeDir, mode: 0o755},
)
if err := extractTarGz(bytes.NewReader(body), dir); err != nil {
t.Fatalf("extract: %v", err)
}
for name, want := range map[string]string{
"Dockerfile": "FROM scratch\n",
"mods/example.jar": "jar-bytes",
} {
got, err := os.ReadFile(filepath.Join(dir, name))
if err != nil || string(got) != want {
t.Fatalf("%s = (%q, %v), want %q", name, got, err, want)
}
}
fi, err := os.Stat(filepath.Join(dir, "start.sh"))
if err != nil || fi.Mode()&0o111 == 0 {
t.Fatalf("entrypoint script lost its exec bit: %v (%v)", fi, err)
}
}
// The guards: "..", absolute paths, symlinks, and special files are refused whole
// — nothing escapes, and nothing is silently skipped.
func TestExtractTarGzRefusesEscapes(t *testing.T) {
cases := []struct {
name string
entries []tarEntry
}{
{"dotdot", []tarEntry{{name: "../outside", body: "x"}}},
{"nested dotdot", []tarEntry{{name: "a/../../outside", body: "x"}}},
{"absolute", []tarEntry{{name: "/etc/outside", body: "x"}}},
{"symlink", []tarEntry{{name: "link", typ: tar.TypeSymlink, linkname: "/etc"}}},
{"hardlink", []tarEntry{{name: "hard", typ: tar.TypeLink, linkname: "somewhere"}}},
{"device", []tarEntry{{name: "dev", typ: tar.TypeChar}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
if err := extractTarGz(bytes.NewReader(tgzBody(t, tc.entries...)), dir); err == nil {
t.Fatal("extract accepted a hostile entry, want an error")
}
// Nothing may have been written outside the target (or at all).
entries, _ := os.ReadDir(dir)
if len(entries) != 0 {
t.Fatalf("hostile archive left %d entries behind", len(entries))
}
})
}
}
// The command end to end: it dials the URL with the bearer token from the
// environment, and refuses to run without it (the internal face would 401
// anyway; failing at parse time is the honest earlier error).
func TestCmdFetchContextFetchAndExtract(t *testing.T) {
body := tgzBody(t, tarEntry{name: "Dockerfile", body: "FROM scratch\n"})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer test-token" {
w.WriteHeader(http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/gzip")
_, _ = w.Write(body)
}))
defer srv.Close()
dir := t.TempDir()
t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + dir}, io.Discard, io.Discard); code != 0 {
t.Fatalf("cmdFetchContext exit = %d, want 0", code)
}
if got, err := os.ReadFile(filepath.Join(dir, "Dockerfile")); err != nil || string(got) != "FROM scratch\n" {
t.Fatalf("extracted Dockerfile = (%q, %v)", got, err)
}
// No token: refuse before dialing.
t.Setenv("FELIS_SERVICE_TOKEN", "")
var stderr bytes.Buffer
if code := cmdFetchContext([]string{"--url=" + srv.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, &stderr); code != 2 {
t.Fatalf("missing token exit = %d, want 2 (stderr %q)", code, stderr.String())
}
// A non-200 answer (e.g. the route's 404 for a never-uploaded context) fails.
srv404 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer srv404.Close()
t.Setenv("FELIS_SERVICE_TOKEN", "test-token")
if code := cmdFetchContext([]string{"--url=" + srv404.URL + "/sub-1/context", "--out=" + t.TempDir()}, io.Discard, io.Discard); code != 1 {
t.Fatalf("404 exit = %d, want 1", code)
}
}
// A body that is not a gzip tarball must fail the extraction rather than produce
// an empty (or partial) context Kaniko would then try to build.
func TestExtractTarGzRejectsNonGzip(t *testing.T) {
dir := t.TempDir()
err := extractTarGz(strings.NewReader("not a tarball"), dir)
if err == nil || !strings.Contains(err.Error(), "gzip") {
t.Fatalf("err = %v, want a gzip complaint", err)
}
}
+2
View File
@@ -19,6 +19,7 @@ Commands:
restore Extract a world archive into a world volume (internal Job entrypoint)
backup Archive a world into the backup store and record it (internal Job entrypoint)
files List/read/write one file in a stopped server's world (internal Job entrypoint)
fetch-context Fetch and extract a submission's build context (internal Job entrypoint)
manifests Render the control-plane RBAC + NetworkPolicy install bundle as YAML
apply Create a MinecraftServer CRD (direct K8s write; use -f server.json)
setup Run host bootstrap + first-run setup console (TUI; requires root/sudo)
@@ -47,6 +48,7 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
"restore": cmdRestore,
"backup": cmdBackup,
"files": cmdFiles,
"fetch-context": cmdFetchContext,
"manifests": cmdManifests,
"apply": cmdApply,
"setup": cmdSetup,
+15 -3
View File
@@ -233,13 +233,25 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
// (the on-demand BACKUP Job runs in the minecraft namespace and mounts it to
// self-record its world_backups row; without the replica the Job's volume
// mount fails and every backup request strands in the cluster).
// An empty build_namespace means the build system's compiled-in default; the
// replica must target the namespace the Jobs actually run in.
buildNS := cfg.Registry.BuildNamespace
if buildNS == "" {
buildNS = platform.DefaultBuildNamespace
}
secretOutcomes := []systemServerOutcome{
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token"),
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns"),
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret"),
naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns"),
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
"felis-config", "felis.toml", "config"),
"felis-config", "felis.toml", "config", "minecraft ns"),
// The build namespace needs the same token: the build Job's fetch
// initContainer reads the submission context from the internal face. Best
// effort — a deployment that only installs the control plane simply never
// builds a user submission.
ensureSecretReplica(ctx, cl, controlNS, buildNS,
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "felis-build ns"),
}
outcomes := ensureSystemServers(ctx, cl, cfg.K8s.Namespace, cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, apiBaseURL, cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname))
outcomes = append(secretOutcomes, outcomes...)
+15 -14
View File
@@ -95,7 +95,7 @@ const felisLimboHealthPort int32 = 8080
// fail-safes to readiness-only, so a login pod that has the URL/domain but not yet
// the token is safe (it simply does not authenticate) rather than broken.
const (
envAPIBaseURL = "FELIS_API_BASE_URL"
envAPIBaseURL = naming.EnvAPIBaseURL
envRootDomain = "FELIS_ROOT_DOMAIN"
envPanelHostname = "FELIS_PANEL_HOSTNAME"
envLobbyServer = "FELIS_LOBBY_SERVER"
@@ -471,26 +471,27 @@ func phaseOrPending(p v1alpha1.Phase) string {
return string(p)
}
// ensureSecretReplica copies one Secret from the control namespace into the minecraft
// namespace so a backend pod can mount it via secretKeyRef. A secretKeyRef is
// namespace-local, but the backends run in the minecraft namespace while the sources
// of truth live beside the control plane — so without this replica the operator's
// injected secretKeyRef would dangle and wedge the pod in CreateContainerConfigError.
// ensureSecretReplica copies one Secret from the control namespace into a workload
// namespace (minecraft — or the build namespace, whose fetch initContainer reads the
// context from the felis-api internal face with the same token) so a pod can mount it
// via secretKeyRef. A secretKeyRef is namespace-local, but those workloads do not run
// beside the control plane — so without this replica the secretKeyRef would dangle and
// wedge the pod in CreateContainerConfigError.
//
// Two Secrets need it, for different reasons: the service token (login only — it
// authenticates the limbo plugin to the felis-api internal face) and the Velocity
// modern-forwarding secret (every backend — it is how a backend knows a login really
// came from the proxy, and so that the player's UUID is Mojang-verified rather than
// offline-derived).
// Two Secrets need it, for different reasons: the service token (the login limbo and
// the build Pod's context fetch — both authenticate to the felis-api internal face)
// and the Velocity modern-forwarding secret (every backend — it is how a backend knows
// a login really came from the proxy, and so that the player's UUID is Mojang-verified
// rather than offline-derived).
//
// It is create-if-absent: an existing replica is left untouched so a hand-rotated
// value in the minecraft namespace is never clobbered (to rotate, delete the replica
// value in the workload namespace is never clobbered (to rotate, delete the replica
// and re-run setup). Best-effort like the rest of the provisioner: a missing source or
// a create failure degrades to a reported outcome, never a hard setup failure. It
// copies only Type and Data — never labels/annotations/ownerRefs — so the replica
// carries no accidental GC owner or managed-by lineage.
func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label string) systemServerOutcome {
name := label + " (minecraft ns)"
func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label, where string) systemServerOutcome {
name := label + " (" + where + ")"
validate := func(secret *corev1.Secret, location, skipped string) systemServerOutcome {
if len(secret.Data[secretKey]) == 0 {
return systemServerOutcome{name: name, skipped: fmt.Sprintf(
+1 -1
View File
@@ -156,7 +156,7 @@ func TestEnsureSecretReplica(t *testing.T) {
}
replicate := func(cl client.Client, controlNS, mcNS string) systemServerOutcome {
return ensureSecretReplica(ctx, cl, controlNS, mcNS,
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token")
naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns")
}
t.Run("replicates when absent", func(t *testing.T) {