diff --git a/cmd/felis/api.go b/cmd/felis/api.go index ca7c3df..e32e8c4 100644 --- a/cmd/felis/api.go +++ b/cmd/felis/api.go @@ -119,9 +119,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { metrics.SetBuildInfo("api", resolvedVersion()) - token := os.Getenv("FELIS_SERVICE_TOKEN") - if token == "" { - fmt.Fprintln(stderr, "felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers") + internalAuth, err := internalCallerTokens(os.Getenv) + if err != nil { + fmt.Fprintf(stderr, "felis api: internal face tokens: %v\n", err) + return 1 + } + for _, ct := range naming.CallerTokens { + if internalAuth[api.Caller(ct.Caller)] == "" { + fmt.Fprintf(stderr, "felis api: warning: %s unset — the internal face turns the %s caller away\n", ct.APIEnv, ct.Caller) + } } // Email one-time codes go through the [smtp] relay when one is configured; the @@ -299,7 +305,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { // Build-log stream (spec §16) is scoped to the BUILD namespace — the same // value the Builder renders Jobs into — so it follows where build Pods run. BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace), - Internal: api.BearerTokenAuth{Token: token}, + Internal: internalAuth, Builder: builder, Images: imagePinner(cfg.Registry.URL), Restorer: restorer, @@ -800,3 +806,14 @@ func imagePinner(registry string) api.ImagePinner { } return imagepin.Resolver{Registry: registry} } + +// internalCallerTokens reads each internal caller's token from the env var the +// Deployment feeds it from (naming.CallerTokens). Two callers sharing a value +// would make the caller ambiguous, so that refuses to start. +func internalCallerTokens(getenv func(string) string) (api.CallerTokens, error) { + tokens := map[api.Caller]string{} + for _, ct := range naming.CallerTokens { + tokens[api.Caller(ct.Caller)] = strings.TrimSpace(getenv(ct.APIEnv)) + } + return api.NewCallerTokens(tokens) +} diff --git a/cmd/felis/api_tokens_test.go b/cmd/felis/api_tokens_test.go new file mode 100644 index 0000000..6f9d003 --- /dev/null +++ b/cmd/felis/api_tokens_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "net/http/httptest" + "strings" + "testing" + + "felis.lolicon.best/internal/api" +) + +// felis-api maps each env var onto the caller the Deployment feeds it for, so the +// build Job's token (FELIS_BUILD_TOKEN) authenticates as build and only as build. +func TestInternalCallerTokensReadEachCallersEnv(t *testing.T) { + env := map[string]string{ + "FELIS_SERVICE_TOKEN": "v-tok", + "FELIS_LIMBO_TOKEN": "l-tok", + "FELIS_BUILD_TOKEN": " b-tok\n", + "FELIS_OPS_TOKEN": "o-tok", + } + auth, err := internalCallerTokens(func(k string) string { return env[k] }) + if err != nil { + t.Fatal(err) + } + for tok, want := range map[string]api.Caller{"v-tok": api.CallerVelocity, "l-tok": api.CallerLimbo, "b-tok": api.CallerBuild, "o-tok": api.CallerOps} { + r := httptest.NewRequest("GET", "/", nil) + r.Header.Set("Authorization", "Bearer "+tok) + if got, err := auth.Authenticate(r); err != nil || got != want { + t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want) + } + } + + // An install where the build namespace still holds a copy of the proxy's token + // would let that copy act as the proxy; the api refuses to start on it. + env["FELIS_BUILD_TOKEN"] = "v-tok" + if _, err := internalCallerTokens(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "same value") { + t.Fatalf("shared token: err = %v, want a same-value refusal", err) + } +} diff --git a/cmd/felis/backupnow.go b/cmd/felis/backupnow.go index fb558e2..0b3539c 100644 --- a/cmd/felis/backupnow.go +++ b/cmd/felis/backupnow.go @@ -20,7 +20,7 @@ import ( // backupnow is the break-glass "back up a world now" op (§B4 "Sync"). Unlike halt — // which writes the CRD directly — a backup needs felis-api's deployment coordinates // (FELIS_IMAGE / FELIS_BACKUP_PVC) to render the one-shot backup Job, so the console -// cannot do it in-process. It POSTs the felis-api INTERNAL face (service-token auth) +// cannot do it in-process. It POSTs the felis-api INTERNAL face (ops-token auth) // while the API is alive, and the API renders the Job and audits the action. This file // is the pure core (no bubbletea); tui_backupnow.go is the terminal glue. @@ -33,7 +33,7 @@ type backupNowOutcome struct { // resolveInternalAPI reads the two things the on-node console needs to reach the // felis-api internal face: the felis-api-internal Service ClusterIP (the host's -// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the service +// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the ops // token. Both live in the control namespace. func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace string) (baseURL, token string, err error) { var svc corev1.Service @@ -45,13 +45,14 @@ func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace return "", "", fmt.Errorf("%s Service has no ClusterIP yet", platform.APIInternalServiceName) } + // The console's own token, which the api serves on the backup route alone. var sec corev1.Secret - if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.ServiceTokenSecretName}, &sec); err != nil { - return "", "", fmt.Errorf("get %s Secret: %w", naming.ServiceTokenSecretName, err) + if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.OpsTokenSecretName}, &sec); err != nil { + return "", "", fmt.Errorf("get %s Secret (re-run the installer to create it): %w", naming.OpsTokenSecretName, err) } token = string(sec.Data[naming.ServiceTokenSecretKey]) if token == "" { - return "", "", fmt.Errorf("secret %s has no %s key", naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey) + return "", "", fmt.Errorf("secret %s has no %s key", naming.OpsTokenSecretName, naming.ServiceTokenSecretKey) } return fmt.Sprintf("http://%s:%d", ip, platform.APIInternalPort), token, nil diff --git a/cmd/felis/backupnow_test.go b/cmd/felis/backupnow_test.go index 1594c2e..a3c51da 100644 --- a/cmd/felis/backupnow_test.go +++ b/cmd/felis/backupnow_test.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "felis.lolicon.best/internal/naming" "felis.lolicon.best/internal/platform" corev1 "k8s.io/api/core/v1" @@ -28,9 +27,15 @@ func internalAPIObjs(clusterIP, token string) []client.Object { ObjectMeta: metav1.ObjectMeta{Name: platform.APIInternalServiceName, Namespace: bgControlNS}, Spec: corev1.ServiceSpec{ClusterIP: clusterIP}, }, + // The console presents the ops token; the proxy's felis-service-token sits + // beside it and must not be the one picked up. &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: bgControlNS}, - Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)}, + ObjectMeta: metav1.ObjectMeta{Name: "felis-ops-token", Namespace: bgControlNS}, + Data: map[string][]byte{"token": []byte(token)}, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "felis-service-token", Namespace: bgControlNS}, + Data: map[string][]byte{"token": []byte("proxy-" + token)}, }, } } diff --git a/cmd/felis/rotatetoken.go b/cmd/felis/rotatetoken.go new file mode 100644 index 0000000..4a72034 --- /dev/null +++ b/cmd/felis/rotatetoken.go @@ -0,0 +1,309 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/platform" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// rotate-token replaces one internal caller's token (naming.CallerTokens): a new +// value goes into the installer's record, the control-namespace Secret and the +// replica the caller's pods mount, felis-api rolls so it accepts only the new +// value, and then the caller restarts so it presents it. Between the api's +// rollout and the caller's restart the caller is turned away with 401; for the +// login gate and the proxy that is the few seconds of a pod or unit restart. + +const ( + defaultSecretsEnvPath = "/etc/felis/secrets.env" + defaultLinkPropsPath = "/opt/felis/velocity/plugins/felis-link/felis-link.properties" + velocityUnit = "felis-velocity" +) + +// installerTokenKeys names each caller's token in the installer's secrets.env +// (deploy/bootstrap.sh load_or_make_secrets). A re-run of the installer applies +// these values to the Secrets, so a rotation that skipped the file would be +// undone by the next upgrade. +var installerTokenKeys = map[string]string{ + "velocity": "SERVICE_TOKEN", + "limbo": "LIMBO_TOKEN", + "build": "BUILD_TOKEN", + "ops": "OPS_TOKEN", +} + +type tokenRotator struct { + cl client.Client + controlNS string + minecraftNS string + buildNS string + // secretsEnv and linkProps are the installer's record and the proxy's + // felis-link.properties; a missing file is reported and skipped. + secretsEnv string + linkProps string + newToken func() (string, error) + // rollAPI restarts felis-api and waits for the rollout. + rollAPI func(ctx context.Context) error + // restartUnit restarts a systemd unit on this host. + restartUnit func(ctx context.Context, unit string) error + out io.Writer +} + +func cmdRotateToken(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("rotate-token", flag.ContinueOnError) + fs.SetOutput(stderr) + cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml") + secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value") + linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)") + fs.Usage = func() { + fmt.Fprintf(stderr, "Usage: felis rotate-token [flags] <%s>\n\n", strings.Join(callerNames(), "|")) + fmt.Fprintln(stderr, "Replaces one internal caller's token: the Secrets, felis-api, then the caller itself.") + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() != 1 { + fs.Usage() + return 2 + } + if _, ok := callerToken(fs.Arg(0)); !ok { + fmt.Fprintf(stderr, "felis rotate-token: unknown caller %q (one of %s)\n", fs.Arg(0), strings.Join(callerNames(), ", ")) + return 2 + } + if os.Geteuid() != 0 { + fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+fs.Arg(0)+")") + return 1 + } + cfg, err := config.Load(*cfgPath) + if err != nil { + fmt.Fprintf(stderr, "felis rotate-token: %v\n", err) + return 1 + } + cl, err := buildSystemServerClient() + if err != nil { + fmt.Fprintf(stderr, "felis rotate-token: %v\n", err) + return 1 + } + buildNS := cfg.Registry.BuildNamespace + if buildNS == "" { + buildNS = platform.DefaultBuildNamespace + } + r := tokenRotator{ + cl: cl, + controlNS: platform.DefaultControlNamespace, + minecraftNS: cfg.K8s.Namespace, + buildNS: buildNS, + secretsEnv: *secretsEnv, + linkProps: *linkProps, + newToken: randomToken, + rollAPI: func(ctx context.Context) error { + if err := kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "restart", "deployment/felis-api"); err != nil { + return err + } + return kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "status", "deployment/felis-api", "--timeout=180s") + }, + restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) }, + out: stdout, + } + if err := r.rotate(context.Background(), fs.Arg(0)); err != nil { + fmt.Fprintf(stderr, "felis rotate-token: %v\n", err) + return 1 + } + return 0 +} + +func callerNames() []string { + names := make([]string, 0, len(naming.CallerTokens)) + for _, ct := range naming.CallerTokens { + names = append(names, ct.Caller) + } + return names +} + +func callerToken(name string) (naming.CallerToken, bool) { + for _, ct := range naming.CallerTokens { + if ct.Caller == name { + return ct, true + } + } + return naming.CallerToken{}, false +} + +// randomToken is 32 random bytes in hex, the shape the installer generates. +func randomToken() (string, error) { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +func (r tokenRotator) rotate(ctx context.Context, caller string) error { + ct, ok := callerToken(caller) + if !ok { + return fmt.Errorf("unknown caller %q", caller) + } + tok, err := r.newToken() + if err != nil { + return fmt.Errorf("generate a token: %w", err) + } + + // The installer's record first: from here on, whatever fails, a re-run of the + // installer puts the new value everywhere. + switch err := setKeyValueLine(r.secretsEnv, installerTokenKeys[ct.Caller], "=", tok); { + case errors.Is(err, fs.ErrNotExist): + fmt.Fprintf(r.out, " - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv) + case err != nil: + return fmt.Errorf("record the new token in %s: %w", r.secretsEnv, err) + default: + fmt.Fprintf(r.out, " - %s: %s updated\n", r.secretsEnv, installerTokenKeys[ct.Caller]) + } + + namespaces := []string{r.controlNS} + replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica] + if replica != "" && replica != r.controlNS { + namespaces = append(namespaces, replica) + } + for _, ns := range namespaces { + if err := writeTokenSecret(ctx, r.cl, ns, ct.Secret, tok); err != nil { + return fmt.Errorf("write Secret %s/%s: %w", ns, ct.Secret, err) + } + fmt.Fprintf(r.out, " - Secret %s/%s: updated\n", ns, ct.Secret) + } + + hostProxy := false + if ct.Caller == "velocity" { + switch err := setKeyValueLine(r.linkProps, "service-token", "=", tok); { + case errors.Is(err, fs.ErrNotExist): + fmt.Fprintf(r.out, " - %s: not found; set service-token in your proxy's felis-link.properties to the value in Secret %s/%s and restart it\n", + r.linkProps, r.controlNS, ct.Secret) + case err != nil: + return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err) + default: + hostProxy = true + fmt.Fprintf(r.out, " - %s: service-token updated\n", r.linkProps) + } + } + + if err := r.rollAPI(ctx); err != nil { + return fmt.Errorf("roll felis-api: %w", err) + } + fmt.Fprintln(r.out, " - felis-api: rolled out, accepting only the new token") + + switch ct.Caller { + case "velocity": + if hostProxy { + if err := r.restartUnit(ctx, velocityUnit); err != nil { + return fmt.Errorf("restart %s: %w", velocityUnit, err) + } + fmt.Fprintf(r.out, " - %s: restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit) + } + case "limbo": + if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), + client.MatchingLabels{v1alpha1.LabelServer: naming.SystemLoginServer}); err != nil { + return fmt.Errorf("restart the login gate: %w", err) + } + fmt.Fprintln(r.out, " - login gate: pod restarted to read the new token") + case "build": + fmt.Fprintln(r.out, " - builds: the next build Job reads the new token; one fetching its context right now fails and can be submitted again") + case "ops": + fmt.Fprintln(r.out, " - felis backup-now reads the new token on its next run") + } + return nil +} + +// writeTokenSecret sets the token in a Secret, creating it when absent. +func writeTokenSecret(ctx context.Context, cl client.Client, namespace, name, token string) error { + var sec corev1.Secret + err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &sec) + if apierrors.IsNotFound(err) { + return cl.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)}, + }) + } + if err != nil { + return err + } + if sec.Data == nil { + sec.Data = map[string][]byte{} + } + sec.Data[naming.ServiceTokenSecretKey] = []byte(token) + return cl.Update(ctx, &sec) +} + +// setKeyValueLine rewrites the `keyvalue` line of a flat key/value file +// (secrets.env, a .properties file), appending one when the key is absent. The +// file is replaced atomically and keeps its mode and owner: felis-link.properties +// is root:felis-velocity 0640, and the proxy must still be able to read it. +func setKeyValueLine(path, key, sep, value string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n") + found := false + for i, ln := range lines { + k, _, ok := strings.Cut(ln, sep) + if ok && strings.TrimSpace(k) == key { + lines[i] = key + sep + value + found = true + } + } + if !found { + lines = append(lines, key+sep+value) + } + tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if err := tmp.Chmod(info.Mode().Perm()); err != nil { + tmp.Close() + return err + } + if st, ok := info.Sys().(*syscall.Stat_t); ok { + if err := tmp.Chown(int(st.Uid), int(st.Gid)); err != nil { + tmp.Close() + return err + } + } + if _, err := tmp.WriteString(strings.Join(lines, "\n") + "\n"); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} diff --git a/cmd/felis/rotatetoken_test.go b/cmd/felis/rotatetoken_test.go new file mode 100644 index 0000000..5adda38 --- /dev/null +++ b/cmd/felis/rotatetoken_test.go @@ -0,0 +1,283 @@ +package main + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type rotationRig struct { + r tokenRotator + cl client.Client + out *bytes.Buffer + events []string + dir string +} + +func tokenSecret(ns, name, val string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name}, + Data: map[string][]byte{"token": []byte(val)}, + } +} + +func serverPod(ns, name, server string) *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name, + Labels: map[string]string{"felis.lolicon.best/server": server}}} +} + +func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig { + t.Helper() + rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir()} + rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build() + rig.r = tokenRotator{ + cl: rig.cl, + controlNS: "felis", + minecraftNS: "minecraft", + buildNS: "felis-build", + secretsEnv: filepath.Join(rig.dir, "secrets.env"), + linkProps: filepath.Join(rig.dir, "felis-link.properties"), + newToken: func() (string, error) { return "NEWTOKEN", nil }, + rollAPI: func(context.Context) error { + rig.events = append(rig.events, "roll-api") + return nil + }, + restartUnit: func(_ context.Context, unit string) error { + rig.events = append(rig.events, "restart "+unit) + return nil + }, + out: rig.out, + } + return rig +} + +func (rig *rotationRig) secret(t *testing.T, ns, name string) string { + t.Helper() + var s corev1.Secret + if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil { + return "" + } + return string(s.Data["token"]) +} + +func writeTestFile(t *testing.T, path, body string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, mode); err != nil { + t.Fatal(err) + } +} + +func TestRotateLimboToken(t *testing.T) { + rig := newRotationRig(t, + tokenSecret("felis", "felis-limbo-token", "old"), + tokenSecret("minecraft", "felis-limbo-token", "old"), + tokenSecret("felis", "felis-service-token", "proxy"), + serverPod("minecraft", "login-0", "login"), + serverPod("minecraft", "survival-0", "survival"), + ) + writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600) + + // felis-api must roll only after both copies hold the new value, and the login + // pod must still be there then: restarting it earlier would have it present + // the new token to an api that does not know it yet. + rig.r.rollAPI = func(context.Context) error { + rig.events = append(rig.events, "roll-api") + if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" { + t.Errorf("api rolled while the control Secret held %q", got) + } + if got := rig.secret(t, "minecraft", "felis-limbo-token"); got != "NEWTOKEN" { + t.Errorf("api rolled while the minecraft replica held %q", got) + } + var pod corev1.Pod + if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil { + t.Errorf("the login pod was restarted before the api rolled") + } + return nil + } + if err := rig.r.rotate(context.Background(), "limbo"); err != nil { + t.Fatal(err) + } + + raw, _ := os.ReadFile(rig.r.secretsEnv) + if string(raw) != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" { + t.Errorf("secrets.env = %q", raw) + } + if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 { + t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm()) + } + if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" { + t.Errorf("the proxy's token changed to %q", got) + } + var pods corev1.PodList + if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil { + t.Fatal(err) + } + if len(pods.Items) != 1 || pods.Items[0].Name != "survival-0" { + t.Errorf("pods left = %v, want only survival-0 (the login pod restarted, user servers untouched)", pods.Items) + } + if strings.Join(rig.events, ",") != "roll-api" { + t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events) + } + if strings.Contains(rig.out.String(), "NEWTOKEN") { + t.Errorf("the new token was printed: %s", rig.out.String()) + } +} + +func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) { + rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old")) + writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600) + writeTestFile(t, rig.r.linkProps, "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n", 0o640) + + if err := rig.r.rotate(context.Background(), "velocity"); err != nil { + t.Fatal(err) + } + raw, _ := os.ReadFile(rig.r.linkProps) + if string(raw) != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" { + t.Errorf("felis-link.properties = %q", raw) + } + if info, _ := os.Stat(rig.r.linkProps); info.Mode().Perm() != 0o640 { + t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm()) + } + if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" { + t.Errorf("control Secret = %q, want NEWTOKEN", got) + } + // The proxy's token has no replica: it must not appear in a workload namespace. + if got := rig.secret(t, "minecraft", "felis-service-token"); got != "" { + t.Errorf("rotation copied the proxy token into minecraft (%q)", got) + } + if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" { + t.Errorf("events = %v, want the api roll then the proxy restart", rig.events) + } +} + +// An external proxy has no felis-link.properties here: the Secret still rotates, +// nothing is restarted on this host, and the output says where the value is +// without printing it. +func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) { + rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old")) + if err := rig.r.rotate(context.Background(), "velocity"); err != nil { + t.Fatal(err) + } + if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" { + t.Errorf("control Secret = %q, want NEWTOKEN", got) + } + if strings.Join(rig.events, ",") != "roll-api" { + t.Errorf("events = %v, want no proxy restart", rig.events) + } + out := rig.out.String() + if !strings.Contains(out, "felis/felis-service-token") || strings.Contains(out, "NEWTOKEN") { + t.Errorf("output should point at the Secret without the value: %s", out) + } +} + +func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) { + rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old")) + // An install from before per-caller tokens has no BUILD_TOKEN line yet. + writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600) + if err := rig.r.rotate(context.Background(), "build"); err != nil { + t.Fatal(err) + } + if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" { + t.Errorf("build replica = %q, want NEWTOKEN (created when absent)", got) + } + if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" { + t.Errorf("control Secret = %q, want NEWTOKEN", got) + } + raw, _ := os.ReadFile(rig.r.secretsEnv) + if string(raw) != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" { + t.Errorf("secrets.env = %q", raw) + } +} + +// A failed api rollout stops the rotation before the caller restarts: the login +// gate keeps running on the old value the old api pods still accept. +func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) { + rig := newRotationRig(t, + tokenSecret("felis", "felis-limbo-token", "old"), + serverPod("minecraft", "login-0", "login"), + ) + rig.r.rollAPI = func(context.Context) error { return errors.New("rollout timed out") } + err := rig.r.rotate(context.Background(), "limbo") + if err == nil || !strings.Contains(err.Error(), "rollout timed out") { + t.Fatalf("err = %v, want the rollout failure", err) + } + var pod corev1.Pod + if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil { + t.Error("the login pod was restarted although the api never rolled") + } +} + +func TestRotateRefusesAnUnknownCaller(t *testing.T) { + rig := newRotationRig(t) + writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600) + if err := rig.r.rotate(context.Background(), "admin"); err == nil { + t.Fatal("rotated a token for an unknown caller") + } + if raw, _ := os.ReadFile(rig.r.secretsEnv); string(raw) != "SERVICE_TOKEN=proxy\n" { + t.Errorf("secrets.env = %q, want it untouched", raw) + } + if len(rig.events) != 0 { + t.Errorf("events = %v, want nothing touched", rig.events) + } +} + +// The installer and rotate-token must agree on where each caller's token lives: +// rotate-token writes installerTokenKeys into secrets.env, and the installer +// applies those same keys to the Secrets on its next run. A key the installer +// does not read would be silently reverted by the next upgrade. +func TestInstallerProvisionsEveryCallerToken(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh")) + if err != nil { + t.Fatal(err) + } + script := string(raw) + for caller, key := range installerTokenKeys { + ct, ok := callerToken(caller) + if !ok { + t.Fatalf("installerTokenKeys names unknown caller %q", caller) + } + if !strings.Contains(script, key+`="${`+key+`:-$(openssl rand -hex 32)}"`) { + t.Errorf("bootstrap.sh does not generate %s", key) + } + if !strings.Contains(script, key+"=${"+key+"}\n") { + t.Errorf("bootstrap.sh does not persist %s to secrets.env", key) + } + apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`) + if !apply.MatchString(script) { + t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key) + } + } + // The replicas the installer applies straight into the workload namespaces, so an + // upgrade has them before the new operator and build Jobs reference them. + for _, line := range []string{ + `apply_literal_secret "$MINECRAFT_NS" felis-limbo-token token "$LIMBO_TOKEN"`, + `apply_literal_secret "$BUILD_NS" felis-build-token token "$BUILD_TOKEN"`, + `kube -n "$MINECRAFT_NS" delete secret felis-service-token --ignore-not-found`, + `kube -n "$BUILD_NS" delete secret felis-service-token --ignore-not-found`, + } { + if !strings.Contains(script, line) { + t.Errorf("bootstrap.sh lacks %s", line) + } + } + for _, ns := range []string{"MINECRAFT_NS", "BUILD_NS"} { + if strings.Contains(script, `apply_literal_secret "$`+ns+`" felis-service-token`) { + t.Errorf("bootstrap.sh still copies the proxy's token into %s", ns) + } + } + if len(installerTokenKeys) != len(callerNames()) { + t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames())) + } +} diff --git a/cmd/felis/run.go b/cmd/felis/run.go index 8c082bf..41d3fde 100644 --- a/cmd/felis/run.go +++ b/cmd/felis/run.go @@ -30,6 +30,7 @@ Commands: apply Create a MinecraftServer CRD (direct K8s write; use -f server.json) setup Run host bootstrap + first-run setup console (TUI; requires root/sudo) converge Fill in fields a newer desired spec added to already-installed system servers + rotate-token Replace one internal caller's token and restart what holds it (velocity|limbo|build|ops; requires root/sudo) watchdog Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer) version Print the build stamp of this binary update Report which platform components have updates available @@ -67,6 +68,7 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{ "apply": cmdApply, "setup": cmdSetup, "converge": cmdConverge, + "rotate-token": cmdRotateToken, "breakGlass": cmdBreakGlass, "bootstrap-assets": cmdBootstrapAssets, "init-forwarding": cmdInitForwarding, diff --git a/cmd/felis/setup.go b/cmd/felis/setup.go index edb66ec..ad5b25d 100644 --- a/cmd/felis/setup.go +++ b/cmd/felis/setup.go @@ -13,7 +13,6 @@ import ( "felis.lolicon.best/internal/api" "felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/config" - "felis.lolicon.best/internal/naming" "felis.lolicon.best/internal/platform" "felis.lolicon.best/internal/store" ) @@ -216,18 +215,18 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ "Re-run `sudo felis setup` on the control-plane host once the cluster is reachable", err) } // The login limbo authenticates to the felis-api INTERNAL face, so it needs the - // internal base URL, the root domain (to link players at the console), and the - // service token. The first two are plain env baked into the pod here; the token - // is a Secret the operator injects by reference — but a secretKeyRef is - // namespace-local, so first replicate the token Secret from the control namespace - // into the minecraft namespace where the login pod runs. The control namespace is + // internal base URL, the root domain (to link players at the console), and its + // own token (felis-limbo-token). The first two are plain env baked into the pod + // here; the token is a Secret the operator injects by reference — but a + // secretKeyRef is namespace-local, so first replicate the token Secret from the + // control namespace into the minecraft namespace where the login pod runs. The control namespace is // the platform default (there is no felis.toml override for it); a deployment that // renamed it must replicate the Secret by hand. controlNS := platform.DefaultControlNamespace apiBaseURL := platform.InternalAPIBaseURL(controlNS) // These Secrets must land in the minecraft namespace before the pods that - // mount them are created: the service token (login authenticates to felis-api - // with it), the Velocity forwarding secret (every backend verifies the proxy's + // mount them are created: the login gate's token (login authenticates to + // felis-api with it), the Velocity forwarding secret (every backend verifies the proxy's // signed handshake with it — without it the login gate would derive an OFFLINE // UUID and the Owner would bind the wrong Minecraft identity), and felis-config // (the on-demand BACKUP Job runs in the minecraft namespace and mounts it to @@ -239,31 +238,7 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ if buildNS == "" { buildNS = platform.DefaultBuildNamespace } - secretOutcomes := []systemServerOutcome{ - ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace, - naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false), - ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace, - naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false), - // refresh=true: felis-config is the rendered config, not a credential. The - // backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run - // must update it when the control plane's render has moved on (a stale copy - // e.g. keeps an old database URL after a credential rotation). - ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace, - "felis-config", "felis.toml", "config", "minecraft ns", true), - // The reaper's pre-reap warning emails authenticate with the same relay - // password felis-api uses; the reaper pod runs in the minecraft namespace, - // where a secretKeyRef resolves only against a local mirror. Skipped while - // the relay is not configured yet — the "configure email" screen refreshes - // both mirrors when it applies. - ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace, - "felis-smtp", "password", "smtp", "minecraft ns", false), - // The build namespace needs the same token: the build Job's fetch - // initContainer reads the submission context from the internal face. Best - // effort — a deployment that only installs the control plane simply never - // builds a user submission. - ensureSecretReplica(ctx, cl, controlNS, buildNS, - naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "felis-build ns", false), - } + secretOutcomes := provisionSecretReplicas(ctx, cl, controlNS, cfg.K8s.Namespace, buildNS) outcomes := ensureSystemServers(ctx, cl, cfg.K8s.Namespace, cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, apiBaseURL, cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname)) outcomes = append(secretOutcomes, outcomes...) fmt.Fprintln(out, "\nfelis setup: login/lobby system servers (always-on, reaper-exempt):") diff --git a/cmd/felis/systemservers.go b/cmd/felis/systemservers.go index 82db700..cf59f8c 100644 --- a/cmd/felis/systemservers.go +++ b/cmd/felis/systemservers.go @@ -588,15 +588,51 @@ func phaseOrPending(p v1alpha1.Phase) string { return string(p) } +// provisionSecretReplicas copies the Secrets workload pods mount from the control +// namespace into the namespaces those pods run in. The proxy's felis-service-token +// is not among them: it lives in the control namespace and on the host, and a copy +// anywhere else would open every game route to whoever reads that namespace. +func provisionSecretReplicas(ctx context.Context, cl client.Client, controlNS, minecraftNS, buildNS string) []systemServerOutcome { + return []systemServerOutcome{ + // refresh=true for both caller tokens: the control namespace holds the + // current value and `felis rotate-token` replaces it there, so a replica + // that differs is stale and the login gate would be turned away with it. + ensureSecretReplica(ctx, cl, controlNS, minecraftNS, + naming.LimboTokenSecretName, naming.ServiceTokenSecretKey, "limbo-token", "minecraft ns", true), + ensureSecretReplica(ctx, cl, controlNS, minecraftNS, + naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false), + // refresh=true: felis-config is the rendered config, not a credential. The + // backup/restore/fileedit Jobs and the reaper mount this copy, so a re-run + // must update it when the control plane's render has moved on (a stale copy + // e.g. keeps an old database URL after a credential rotation). + ensureSecretReplica(ctx, cl, controlNS, minecraftNS, + "felis-config", "felis.toml", "config", "minecraft ns", true), + // The reaper's pre-reap warning emails authenticate with the same relay + // password felis-api uses; the reaper pod runs in the minecraft namespace, + // where a secretKeyRef resolves only against a local mirror. Skipped while + // the relay is not configured yet — the "configure email" screen refreshes + // both mirrors when it applies. + ensureSecretReplica(ctx, cl, controlNS, minecraftNS, + "felis-smtp", "password", "smtp", "minecraft ns", false), + // The build namespace needs the build token: the build Job's fetch + // initContainer reads the submission context from the internal face, and + // that is all this token opens. Best effort — a deployment that only + // installs the control plane simply never builds a user submission. + ensureSecretReplica(ctx, cl, controlNS, buildNS, + naming.BuildTokenSecretName, naming.ServiceTokenSecretKey, "build-token", "felis-build ns", true), + } +} + // ensureSecretReplica copies one Secret from the control namespace into a workload // namespace (minecraft — or the build namespace, whose fetch initContainer reads the -// context from the felis-api internal face with the same token) so a pod can mount it +// context from the felis-api internal face with the build token) so a pod can mount it // via secretKeyRef. A secretKeyRef is namespace-local, but those workloads do not run // beside the control plane — so without this replica the secretKeyRef would dangle and // wedge the pod in CreateContainerConfigError. // -// Three Secrets need it, for different reasons: the service token (the login limbo and -// the build Pod's context fetch — both authenticate to the felis-api internal face), +// Several Secrets need it, for different reasons: the caller tokens of the login +// limbo and the build Pod's context fetch (both authenticate to the felis-api +// internal face, each with its own token), // the Velocity modern-forwarding secret (every backend — it is how a backend knows // a login really came from the proxy, and so that the player's UUID is Mojang-verified // rather than offline-derived), and the SMTP relay password (the reaper's pre-reap @@ -609,12 +645,14 @@ func phaseOrPending(p v1alpha1.Phase) string { // copies only Type and Data — never labels/annotations/ownerRefs — so the replica // carries no accidental GC owner or managed-by lineage. // -// refreshExisting switches the felis-config mirror to refresh-in-place: that Secret is -// a rendered config, never a hand-rotated credential, and the workload Jobs that mount -// it (backup/restore/fileedit) plus the reaper silently misbehave on a stale copy — -// e.g. after a database credential rotation the control plane moves on while every -// backup Job keeps failing auth. Credential Secrets keep the never-overwrite rule so a -// rotated value survives; to rotate those, delete the replica and re-run setup. +// refreshExisting switches a replica to refresh-in-place from the control namespace. +// The felis-config mirror uses it because that Secret is a rendered config and the +// workload Jobs that mount it (backup/restore/fileedit) plus the reaper silently +// misbehave on a stale copy — e.g. after a database credential rotation the control +// plane moves on while every backup Job keeps failing auth. The caller tokens use it +// because `felis rotate-token` replaces them in the control namespace, which makes a +// differing replica stale by definition. The forwarding and SMTP Secrets keep the +// never-overwrite rule. func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace, minecraftNamespace, secretName, secretKey, label, where string, refreshExisting bool) systemServerOutcome { name := label + " (" + where + ")" validate := func(secret *corev1.Secret, location, skipped string) systemServerOutcome { @@ -712,7 +750,7 @@ func ensureSecretReplica(ctx context.Context, cl client.Client, controlNamespace func requiredProvisioningError(outcomes []systemServerOutcome) error { required := map[string]struct{}{ - "service-token (minecraft ns)": {}, + "limbo-token (minecraft ns)": {}, "forwarding-secret (minecraft ns)": {}, naming.SystemLoginServer: {}, } diff --git a/cmd/felis/systemservers_test.go b/cmd/felis/systemservers_test.go index a52096b..080caa1 100644 --- a/cmd/felis/systemservers_test.go +++ b/cmd/felis/systemservers_test.go @@ -142,21 +142,21 @@ func TestLoginSystemServerEnv(t *testing.T) { // namespace (create-if-absent), so the operator's secretKeyRef on the backend pod // resolves. It must not overwrite an existing replica, and must degrade gracefully // when the source is missing or the namespaces coincide. Exercised here with the -// service token; setup runs it a second time for the Velocity forwarding secret. +// Velocity forwarding secret, which setup replicates in this never-overwrite mode. func TestEnsureSecretReplica(t *testing.T) { scheme := newSystemServerScheme(t) ctx := context.Background() srcSecret := func() *corev1.Secret { return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "felis"}, + ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "felis"}, Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte("s3cr3t")}, + Data: map[string][]byte{naming.ForwardingSecretKey: []byte("s3cr3t")}, } } replicate := func(cl client.Client, controlNS, mcNS string) systemServerOutcome { return ensureSecretReplica(ctx, cl, controlNS, mcNS, - naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey, "service-token", "minecraft ns", false) + naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns", false) } t.Run("replicates when absent", func(t *testing.T) { @@ -166,19 +166,19 @@ func TestEnsureSecretReplica(t *testing.T) { t.Fatalf("outcome = %+v, want created", out) } var replica corev1.Secret - if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil { + if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil { t.Fatalf("get replica: %v", err) } - if string(replica.Data[naming.ServiceTokenSecretKey]) != "s3cr3t" { - t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ServiceTokenSecretKey]) + if string(replica.Data[naming.ForwardingSecretKey]) != "s3cr3t" { + t.Errorf("replica token = %q, want s3cr3t", replica.Data[naming.ForwardingSecretKey]) } }) t.Run("does not overwrite existing replica", func(t *testing.T) { existing := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"}, + ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"}, Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte("rotated")}, + Data: map[string][]byte{naming.ForwardingSecretKey: []byte("rotated")}, } cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), existing).Build() out := replicate(cl, "felis", "minecraft") @@ -186,11 +186,11 @@ func TestEnsureSecretReplica(t *testing.T) { t.Fatalf("outcome = %+v, want skipped (not clobbered)", out) } var replica corev1.Secret - if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ServiceTokenSecretName}, &replica); err != nil { + if err := cl.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.ForwardingSecretName}, &replica); err != nil { t.Fatalf("get replica: %v", err) } - if string(replica.Data[naming.ServiceTokenSecretKey]) != "rotated" { - t.Error("existing replica was overwritten — a rotated token must survive") + if string(replica.Data[naming.ForwardingSecretKey]) != "rotated" { + t.Error("existing replica was overwritten — a hand-set value must survive") } }) @@ -204,22 +204,22 @@ func TestEnsureSecretReplica(t *testing.T) { t.Run("rejects a source with an empty required key", func(t *testing.T) { bad := srcSecret() - bad.Data[naming.ServiceTokenSecretKey] = nil + bad.Data[naming.ForwardingSecretKey] = nil cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build() out := replicate(cl, "felis", "minecraft") - if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) { + if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) { t.Fatalf("outcome = %+v, want unavailable required key", out) } }) t.Run("rejects an existing replica with an empty required key", func(t *testing.T) { bad := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: "minecraft"}, + ObjectMeta: metav1.ObjectMeta{Name: naming.ForwardingSecretName, Namespace: "minecraft"}, Data: map[string][]byte{}, } cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(srcSecret(), bad).Build() out := replicate(cl, "felis", "minecraft") - if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) { + if out.err != nil || out.created || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) { t.Fatalf("outcome = %+v, want unavailable existing replica", out) } }) @@ -245,7 +245,7 @@ func TestEnsureSecretReplica(t *testing.T) { bad.Data = nil cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(bad).Build() out := replicate(cl, "felis", "felis") - if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ServiceTokenSecretKey) { + if out.err != nil || out.available || !strings.Contains(out.skipped, naming.ForwardingSecretKey) { t.Fatalf("outcome = %+v, want unavailable required key", out) } }) @@ -334,7 +334,7 @@ func TestEnsureSecretReplicaRefresh(t *testing.T) { func TestRequiredProvisioningError(t *testing.T) { ready := []systemServerOutcome{ - {name: "service-token (minecraft ns)", available: true}, + {name: "limbo-token (minecraft ns)", available: true}, {name: "forwarding-secret (minecraft ns)", available: true}, {name: naming.SystemLoginServer, available: true}, {name: naming.SystemLobbyServer, skipped: "image not configured"}, @@ -349,6 +349,14 @@ func TestRequiredProvisioningError(t *testing.T) { t.Fatalf("missing forwarding secret = %v, want named error", err) } + // The login gate cannot reach felis-api without its token, so setup must not + // report success while that replica is missing. + noToken := append([]systemServerOutcome(nil), ready...) + noToken[0] = systemServerOutcome{name: "limbo-token (minecraft ns)", skipped: "source missing"} + if err := requiredProvisioningError(noToken); err == nil || !strings.Contains(err.Error(), "limbo-token") { + t.Fatalf("missing limbo token = %v, want named error", err) + } + failed := append([]systemServerOutcome(nil), ready...) failed[3] = systemServerOutcome{name: naming.SystemLobbyServer, err: context.DeadlineExceeded} if err := requiredProvisioningError(failed); err == nil || !strings.Contains(err.Error(), naming.SystemLobbyServer) { @@ -662,3 +670,62 @@ func TestEnsureSystemServersRefreshesDerivedEnv(t *testing.T) { } }) } + +// Setup's replicas carry each workload its own caller token and nothing more: the +// login gate gets felis-limbo-token in the minecraft namespace, the build Jobs get +// felis-build-token in the build namespace, a replica left stale by a rotation is +// brought up to date, and the proxy's felis-service-token is copied nowhere. +func TestProvisionSecretReplicasCarryCallerTokens(t *testing.T) { + scheme := newSystemServerScheme(t) + ctx := context.Background() + secret := func(ns, name, key, val string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{key: []byte(val)}, + } + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + secret("felis", "felis-service-token", "token", "proxy-tok"), + secret("felis", "felis-limbo-token", "token", "limbo-new"), + secret("felis", "felis-build-token", "token", "build-tok"), + secret("felis", "felis-ops-token", "token", "ops-tok"), + secret("felis", naming.ForwardingSecretName, naming.ForwardingSecretKey, "fwd"), + // What a rotation leaves behind before setup runs again. + secret("minecraft", "felis-limbo-token", "token", "limbo-old"), + ).Build() + + outcomes := provisionSecretReplicas(ctx, cl, "felis", "minecraft", "felis-build") + for _, o := range outcomes { + if o.err != nil { + t.Fatalf("%s: %v", o.name, o.err) + } + } + + read := func(ns, name string) (string, bool) { + var s corev1.Secret + if err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil { + return "", false + } + return string(s.Data["token"]), true + } + if got, _ := read("minecraft", "felis-limbo-token"); got != "limbo-new" { + t.Errorf("minecraft/felis-limbo-token = %q, want the rotated limbo-new", got) + } + if got, _ := read("felis-build", "felis-build-token"); got != "build-tok" { + t.Errorf("felis-build/felis-build-token = %q, want build-tok", got) + } + for _, ns := range []string{"minecraft", "felis-build"} { + for _, name := range []string{"felis-service-token", "felis-ops-token"} { + if _, ok := read(ns, name); ok { + t.Errorf("%s/%s was replicated; only the control namespace holds it", ns, name) + } + } + } + if _, ok := read("minecraft", "felis-build-token"); ok { + t.Error("the build token was copied into the minecraft namespace") + } + if _, ok := read("felis-build", "felis-limbo-token"); ok { + t.Error("the limbo token was copied into the build namespace") + } +} diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index c29e869..557e0eb 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -2819,7 +2819,15 @@ load_or_make_secrets() { ok "reusing persisted secrets from ${SECRETS_ENV}" fi DB_PASSWORD="${DB_PASSWORD:-$(openssl rand -hex 24)}" + # One felis-api internal token per caller (naming.CallerTokens), so each is scoped + # to its own routes and a leak is contained to that caller: SERVICE_TOKEN is the + # proxy's (felis-link.properties), LIMBO_TOKEN the login gate's, BUILD_TOKEN what a + # build Job fetches its context with, OPS_TOKEN what `felis backup-now` presents. + # `felis rotate-token ` rewrites the matching line here. SERVICE_TOKEN="${SERVICE_TOKEN:-$(openssl rand -hex 32)}" + LIMBO_TOKEN="${LIMBO_TOKEN:-$(openssl rand -hex 32)}" + BUILD_TOKEN="${BUILD_TOKEN:-$(openssl rand -hex 32)}" + OPS_TOKEN="${OPS_TOKEN:-$(openssl rand -hex 32)}" SESSION_SECRET="${SESSION_SECRET:-$(openssl rand -hex 32)}" # The Velocity modern-forwarding key. It is what makes a backend's UUID trustworthy: # the proxy does the Mojang handshake and HMACs the resulting profile with this key, @@ -2840,6 +2848,9 @@ load_or_make_secrets() { cat > "$SECRETS_ENV" < prints the size to render the claim diff --git a/deploy/limbo/README.md b/deploy/limbo/README.md index 011fac0..d3829e8 100644 --- a/deploy/limbo/README.md +++ b/deploy/limbo/README.md @@ -132,10 +132,13 @@ set them by hand: `spec.env` by `felis setup` (`cmd/felis` derives the internal API URL from the control namespace — the platform default `felis`; a renamed control namespace must be reflected by hand — and the root domain from `felis.toml`). -- `FELIS_SERVICE_TOKEN` is a **secret**, so it is never written into the CRD. `felis - setup` replicates the `felis-service-token` Secret from the control namespace into - the minecraft namespace, and the operator injects it into the `login` pod (only) - via a `secretKeyRef`, keyed off the reserved `login` name. Until the token is +- `FELIS_SERVICE_TOKEN` is a **secret**, so it is never written into the CRD. The + login gate has its own internal-API token, `felis-limbo-token`, which may only mint + link codes, poll link status and check the blacklist. The installer applies it into + the minecraft namespace (and `felis setup` refreshes that replica from the control + namespace), and the operator injects it into the `login` pod (only) as + `FELIS_SERVICE_TOKEN` via a `secretKeyRef`, keyed off the reserved `login` name. + `sudo felis rotate-token limbo` replaces it and restarts the pod. Until the token is present the plugin fail-safes to readiness-only, so the gate is never broken — it simply does not authenticate yet. - **Service:** the login pod dials `FELIS_API_BASE_URL`, which resolves to the diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e3bef91..a11655a 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -98,7 +98,13 @@ components: serviceToken: type: http scheme: bearer - description: Static service token presented by velocity / backend callers (internal face). + description: >- + Static per-caller token (internal face). Each machine holds its own — + velocity (felis-service-token), limbo (felis-limbo-token), build + (felis-build-token), ops (felis-ops-token) — and each operation lists the + callers it serves in x-felis-callers. A genuine token for a caller the + operation does not list is refused with 403 wrong_caller. `felis + rotate-token ` replaces one. accessJWT: type: apiKey in: header @@ -758,6 +764,7 @@ paths: summary: List all servers (velocity route table). x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] responses: '200': @@ -838,6 +845,7 @@ paths: summary: Backend readiness callback — the server reports it is accepting players. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] parameters: - { name: name, in: path, required: true, schema: { type: string } } @@ -861,6 +869,7 @@ paths: the internal face (service token, no Zero Trust). x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [build] security: [{ serviceToken: [] }] parameters: - { name: id, in: path, required: true, schema: { type: string } } @@ -884,6 +893,7 @@ paths: summary: Player-join event by online-mode UUID (activity tracking / idle reset). x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] parameters: - { name: name, in: path, required: true, schema: { type: string } } @@ -917,6 +927,7 @@ paths: server's autostartPolicy and the per-server wake cooldown. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] parameters: - { name: name, in: path, required: true, schema: { type: string } } @@ -968,6 +979,7 @@ paths: summary: Server status projection (velocity polls this after a wake). x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] parameters: - { name: name, in: path, required: true, schema: { type: string } } @@ -992,6 +1004,7 @@ paths: binding the unowned server to the player's linked account. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] parameters: - { name: name, in: path, required: true, schema: { type: string } } @@ -1035,6 +1048,7 @@ paths: summary: Lobby menu projection — status plus the ownership-derived `claimable`. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] parameters: - { name: name, in: path, required: true, schema: { type: string } } @@ -1067,6 +1081,7 @@ paths: description: Internal-only — the code is born from a UUID the web never holds. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity, limbo] security: [{ serviceToken: [] }] requestBody: required: true @@ -1124,6 +1139,7 @@ paths: UUID it already holds, so no identity detail crosses back. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity, limbo] security: [{ serviceToken: [] }] parameters: - { name: mc_uuid, in: path, required: true, schema: { type: string, format: uuid } } @@ -1154,6 +1170,7 @@ paths: web credentials — so this endpoint starts a flow, it does not move anything. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] requestBody: required: true @@ -1203,6 +1220,7 @@ paths: never the contested name, so the genuine Mojang player always passes. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] requestBody: required: true @@ -1248,6 +1266,7 @@ paths: different UUID — is never on the list and always passes. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity, limbo] security: [{ serviceToken: [] }] parameters: - { name: mc_uuid, in: path, required: true, schema: { type: string, format: uuid } } @@ -1277,6 +1296,7 @@ paths: is secret to the operator crew. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] responses: '200': @@ -1314,6 +1334,7 @@ paths: factor distinct from the mailbox. x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [velocity] security: [{ serviceToken: [] }] parameters: - { name: id, in: path, required: true, schema: { type: string } } @@ -1368,6 +1389,7 @@ paths: and the action is audited to "break-glass". x-felis-face: [internal] x-felis-tier: service + x-felis-callers: [ops] security: [{ serviceToken: [] }] parameters: - { name: name, in: path, required: true, schema: { type: string } } diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index cffc24b..f2ec637 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -364,8 +364,21 @@ is intended. [GO-TESTED for the session/QR-login logic.] ## 6. Internal API rejects Velocity / proxy callers (service-token) The internal face (`--internal-addr :8081`, routes under -`/api/v1/internal/...`) is **never** Zero-Trust; it authenticates a single -service token via `Authorization: Bearer `, compared in constant time. +`/api/v1/internal/...`) is **never** Zero-Trust; it authenticates a bearer token +(`Authorization: Bearer `, compared in constant time). Each internal caller +has its own token, and each route serves only the callers listed for it +(`x-felis-callers` in `docs/openapi.yaml`): + +| Caller | Secret (key `token`) | API env | Where the caller reads it | Routes | +|---|---|---|---|---| +| `velocity` (proxy felis-link) | `felis/felis-service-token` | `FELIS_SERVICE_TOKEN` | `service-token` in the host's `felis-link.properties` | server list, wake/claim/ready/status, join events, menu, op-login, migrate, reclaim, link codes, blacklist | +| `limbo` (login gate) | `felis/felis-limbo-token`, replica in `minecraft` | `FELIS_LIMBO_TOKEN` | login pod env `FELIS_SERVICE_TOKEN` | link codes, link status, blacklist | +| `build` (build Job fetch) | `felis/felis-build-token`, replica in `felis-build` | `FELIS_BUILD_TOKEN` | fetch initContainer env | submission build context | +| `ops` (`felis backup-now`) | `felis/felis-ops-token` | `FELIS_OPS_TOKEN` | read from the Secret on each run | break-glass backup | + +The installer generates all four into `/etc/felis/secrets.env` (`SERVICE_TOKEN`, +`LIMBO_TOKEN`, `BUILD_TOKEN`, `OPS_TOKEN`) and applies them on every run. The audit +log records internal actions with the source `internal:`. [GO-TESTED] In-cluster it is reached through the ClusterIP Service `felis-api-internal` (port 8081), which is separate from the external NodePort `felis-api` (443) precisely so @@ -378,24 +391,42 @@ break-glass console reaches it by resolving that Service's ClusterIP and dialing svc felis-api-internal` must show a ClusterIP with 8081; a bare `felis-api` name serves only 443 and every internal call would hang/refuse. -- **All internal calls 401** → the token is unset or wrong. The API reads env - `FELIS_SERVICE_TOKEN`. If unset, startup logs: +- **One caller's calls all 401** → its token is unset or differs from the api's + copy. The api logs one line per unset token at startup: ``` - felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers + felis api: warning: FELIS_LIMBO_TOKEN unset — the internal face turns the limbo caller away ``` - and wires an empty token, which rejects **everyone** (no bypass). [GO-TESTED - for the constant-time compare / empty-token rejection.] + An unset token never matches anything (no bypass). Compare the caller's value + with its Secret, e.g. for the proxy: -In-cluster, the token's source of truth is the Secret `felis-service-token` -(key `token`), injected as `FELIS_SERVICE_TOKEN` on the API Deployment. Fix: + ``` + kubectl -n felis get secret felis-service-token -o jsonpath='{.data.token}' | base64 -d + ``` -``` -kubectl get secret felis-service-token -o jsonpath='{.data.token}' | base64 -d -``` +- **`403 wrong_caller`** → the token is valid but belongs to a caller that route + does not serve, e.g. the build token calling a proxy route. Configure the caller + with its own token from the table above. -Ensure the proxy is configured with the identical value. +- **api pods stuck in `CreateContainerConfigError`** → one of the four Secrets is + missing in `felis`. Re-run the installer; it applies them before the bundle. + +- **Two tokens with the same value** → `felis api` refuses to start with + `the X and Y tokens are the same value; each caller needs its own`, since a shared + value would make the caller ambiguous. Rotate one of them. + +### Rotating a token + +`sudo felis rotate-token ` replaces one caller's token: +it writes the new value to `secrets.env` (so a later installer run keeps it), the +Secret and its replica, rolls felis-api so only the new value is accepted, then +restarts the caller — the `felis-velocity` unit when the proxy runs on this host, +or the login pod. Build Jobs and `felis backup-now` pick the new value up on their +next run. The old value stops working as soon as felis-api has rolled; the caller +is turned away for the few seconds until it restarts. For a proxy on another host, +the command leaves the host alone and tells you to copy the new value from the +Secret into that proxy's `felis-link.properties` and restart it. [GO-TESTED] --- diff --git a/internal/api/api.go b/internal/api/api.go index 22d11a7..1bd998e 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -14,6 +14,7 @@ package api import ( "context" + "fmt" "log" "log/slog" "net/http" @@ -352,6 +353,10 @@ type apiRoute struct { // since the browser calls it every few seconds while it waits. AuthDoor bool + // Callers lists the machines an internal-face route serves; every + // authenticated internal route names at least one, and external routes none. + Callers []Caller + h http.HandlerFunc } @@ -359,6 +364,14 @@ type apiRoute struct { // service-token auth, never Zero Trust. It carries both health probes and the // metrics scrape. func (a *API) internalAPIRoutes() []apiRoute { + // Who may call what (Caller). The proxy drives the game-facing routes; the + // login gate only checks a joining player's bar and link and mints their bind + // code; the build Job only reads the context of the submission it builds; the + // on-node console only asks for a break-glass backup. + proxy := []Caller{CallerVelocity} + gate := []Caller{CallerVelocity, CallerLimbo} + build := []Caller{CallerBuild} + ops := []Caller{CallerOps} return []apiRoute{ {Method: "GET", Pattern: "/healthz", Public: true, h: a.handleHealthz}, {Method: "GET", Pattern: "/readyz", Public: true, h: a.handleReadyz}, @@ -366,47 +379,47 @@ func (a *API) internalAPIRoutes() []apiRoute { // no token, internal-only so it is never exposed off-cluster. {Method: "GET", Pattern: "/metrics", Public: true, h: a.handleMetrics}, - {Method: "GET", Pattern: "/api/v1/servers", h: a.handleListServers}, + {Method: "GET", Pattern: "/api/v1/servers", Callers: proxy, h: a.handleListServers}, // The build Pod's context-fetch initContainer streams a submission's stored // modpack through this route (build namespace cannot mount the uploads PVC). - {Method: "GET", Pattern: "/api/v1/internal/submissions/{id}/context", h: a.handleInternalSubmissionContext}, - {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/ready", h: a.handleReady}, - {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/join-event", h: a.handleJoinEvent}, + {Method: "GET", Pattern: "/api/v1/internal/submissions/{id}/context", Callers: build, h: a.handleInternalSubmissionContext}, + {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/ready", Callers: proxy, h: a.handleReady}, + {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/join-event", Callers: proxy, h: a.handleJoinEvent}, // Domain-autostart (spec §9.1, §14): velocity drives the wake lever and polls // status with its service token, identifying the joining player by online-mode // UUID. These live on the internal face because velocity holds no web Principal; // the external face keeps its own Principal-gated wake/status for the panel. - {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/wake", h: a.handleInternalWake}, - {Method: "GET", Pattern: "/api/v1/internal/servers/{name}/status", h: a.handleStatus}, + {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/wake", Callers: proxy, h: a.handleInternalWake}, + {Method: "GET", Pattern: "/api/v1/internal/servers/{name}/status", Callers: proxy, h: a.handleStatus}, // Lobby `/menu` (spec §12): the felis-paper lobby is a pure UI face holding no // token, so velocity drives these on its behalf — claim by online-mode UUID // (the lobby's `Claim & Start`, separate from the autostartPolicy-gated wake) // and the menu projection that adds the ownership-derived `claimable` the §11 // list/status views never carry. - {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/claim", h: a.handleInternalClaim}, - {Method: "GET", Pattern: "/api/v1/internal/servers/{name}/menu", h: a.handleInternalMenuStatus}, + {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/claim", Callers: proxy, h: a.handleInternalClaim}, + {Method: "GET", Pattern: "/api/v1/internal/servers/{name}/menu", Callers: proxy, h: a.handleInternalMenuStatus}, // Account linking (spec §10): the in-game /link side mints a one-time code for a // verified UUID. Internal-only — the code is born from an online-mode UUID the // web never holds (account_link_codes has no user_id column). - {Method: "POST", Pattern: "/api/v1/internal/account/link/code", h: a.handleCreateLinkCode}, + {Method: "POST", Pattern: "/api/v1/internal/account/link/code", Callers: gate, h: a.handleCreateLinkCode}, // QR scan-to-login completion poll (spec §B3 player game-login). After the player // scans the QR-encoded code and the web verify writes the durable link, velocity // polls this for the UUID it minted against and admits on {linked:true}. Read-only // and keyed by the verified UUID (not the scanned code), so it consumes nothing // and is safe to poll repeatedly. - {Method: "GET", Pattern: "/api/v1/internal/account/link/status/{mc_uuid}", h: a.handleLinkStatus}, + {Method: "GET", Pattern: "/api/v1/internal/account/link/status/{mc_uuid}", Callers: gate, h: a.handleLinkStatus}, // Account migration (spec §B3 inherit), in-game side: /felis migrate puts the // account linked to the running player's verified UUID into migrate mode. Internal // only — the initiator is proven by online-mode auth, and the sensitive proof // (step-up) still happens web-side before anything transfers. - {Method: "POST", Pattern: "/api/v1/internal/account/migrate/start", h: a.handleMigrateStart}, + {Method: "POST", Pattern: "/api/v1/internal/account/migrate/start", Callers: proxy, h: a.handleMigrateStart}, // Username-collision reclaim (spec §B3): velocity records a Mojang-priority // reclaim (bar the squatter UUID + stash its data for 30 days) and gates the // limbo login by checking whether a connecting UUID was barred. Internal-only — // velocity holds a service token, and the bar is keyed by UUID so the genuine // Mojang player (same name, different UUID) always passes. - {Method: "POST", Pattern: "/api/v1/internal/player/reclaim", h: a.handleReclaimUsername}, - {Method: "GET", Pattern: "/api/v1/internal/player/blacklist/{mc_uuid}", h: a.handleCheckBlacklist}, + {Method: "POST", Pattern: "/api/v1/internal/player/reclaim", Callers: proxy, h: a.handleReclaimUsername}, + {Method: "GET", Pattern: "/api/v1/internal/player/blacklist/{mc_uuid}", Callers: gate, h: a.handleCheckBlacklist}, // Felis-nano multi-source session verifier, behind player game-login. Velocity is // pointed here with -Dmojang.sessionserver and issues the request itself; it speaks // the vanilla sessionserver protocol and carries no token, so this is Public. It @@ -419,13 +432,13 @@ func (a *API) internalAPIRoutes() []apiRoute { // /felis web op approve. Internal face carries the pending queue and the // approve action (service-token auth, no Principal); the public face carries // the start/status/finish the staff member's browser drives. - {Method: "GET", Pattern: "/api/v1/internal/op-login/pending", h: a.handleOpLoginPending}, - {Method: "POST", Pattern: "/api/v1/internal/op-login/{id}/approve", h: a.handleOpLoginApprove}, + {Method: "GET", Pattern: "/api/v1/internal/op-login/pending", Callers: proxy, h: a.handleOpLoginPending}, + {Method: "POST", Pattern: "/api/v1/internal/op-login/{id}/approve", Callers: proxy, h: a.handleOpLoginApprove}, // Break-glass backup (spec §B4 "Sync"): the on-node console POSTs here to // snapshot a stopped world while the API is alive. Service-token auth (no // Principal); the shared enqueueBackup tail enforces the RWO stopped-gate. - {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/backup", h: a.handleInternalBackup}, + {Method: "POST", Pattern: "/api/v1/internal/servers/{name}/backup", Callers: ops, h: a.handleInternalBackup}, } } @@ -705,6 +718,12 @@ func (a *API) buildFace(face string, routes []apiRoute, guard func(http.Handler) continue } h := rt.h + if (face == "internal") != (len(rt.Callers) > 0) { + panic(fmt.Sprintf("%s route %s %s: internal routes list their callers, external ones none", face, rt.Method, rt.Pattern)) + } + if len(rt.Callers) > 0 { + h = callersOnly(rt.Callers, h) + } if rt.Owner { h = a.ownerOnly(rt.h) } @@ -832,6 +851,7 @@ const ( ctxKeyRequestID ctxKey = iota ctxKeyPrincipal ctxKeyReqInfo + ctxKeyCaller ) func requestIDFromContext(ctx context.Context) string { @@ -849,6 +869,21 @@ func principalFromContext(ctx context.Context) *Principal { return nil } +// callerFromContext returns the internal-face caller, or "" off that face. +func callerFromContext(ctx context.Context) Caller { + c, _ := ctx.Value(ctxKeyCaller).(Caller) + return c +} + +// internalSource is the audit Source for an action taken on the internal face, +// naming the caller whose token asked for it ("internal:velocity"). +func internalSource(r *http.Request) string { + if c := callerFromContext(r.Context()); c != "" { + return "internal:" + string(c) + } + return "internal" +} + // ---- per-key cooldown (wake + OTP) ---- // cooldownLimiter is an in-memory per-key cooldown. It backs two throttles with diff --git a/internal/api/api_test.go b/internal/api/api_test.go index d8a269d..8eaac52 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -1752,9 +1752,15 @@ type staticExternal struct { func (s staticExternal) Authenticate(*http.Request) (*Principal, error) { return s.p, s.err } -type okInternal struct{} +// okInternal admits every request as one caller, the proxy unless set. +type okInternal struct{ caller Caller } -func (okInternal) Authenticate(*http.Request) error { return nil } +func (o okInternal) Authenticate(*http.Request) (Caller, error) { + if o.caller == "" { + return CallerVelocity, nil + } + return o.caller, nil +} // ---- helpers ---- @@ -1798,7 +1804,7 @@ func decodeErr(t *testing.T, w *httptest.ResponseRecorder) string { func TestInternalFaceRequiresServiceToken(t *testing.T) { api := newTestAPI(newFakeRepo(), newFakeCluster()) - api.Internal = BearerTokenAuth{Token: "s3cr3t"} + api.Internal = CallerTokens{CallerVelocity: "s3cr3t"} h := api.InternalHandler() // no token -> 401 @@ -1817,7 +1823,7 @@ func TestInternalFaceRequiresServiceToken(t *testing.T) { func TestHealthzIsUnauthenticated(t *testing.T) { api := newTestAPI(newFakeRepo(), newFakeCluster()) - api.Internal = BearerTokenAuth{Token: "s3cr3t"} + api.Internal = CallerTokens{CallerVelocity: "s3cr3t"} if w := do(api.InternalHandler(), "GET", "/healthz", "", nil); w.Code != http.StatusOK { t.Fatalf("healthz code = %d, want 200", w.Code) } @@ -1829,7 +1835,7 @@ func TestReadyzPingsDependencies(t *testing.T) { repo := newFakeRepo() cl := newFakeCluster() api := newTestAPI(repo, cl) - api.Internal = BearerTokenAuth{Token: "s3cr3t"} + api.Internal = CallerTokens{CallerVelocity: "s3cr3t"} // Both healthy. if w := do(api.InternalHandler(), "GET", "/readyz", "", nil); w.Code != http.StatusOK { diff --git a/internal/api/auth.go b/internal/api/auth.go index 2748f6e..08b7944 100644 --- a/internal/api/auth.go +++ b/internal/api/auth.go @@ -71,11 +71,33 @@ func (p *Principal) IsOwner() bool { return p != nil && p.Role == "owner" && p.ViaAdminAccess } -// InternalAuth authenticates the internal face (velocity / backend callbacks): -// a static service token presented as a Bearer credential. The internal face -// is never wrapped in Zero Trust (spec §1.8, §14 red line). +// Caller names the machine behind an internal-face token. Each caller holds a +// token of its own and each internal route lists the callers it serves +// (apiRoute.Callers), so a token copied out of one namespace opens only what +// that caller needs: the build Job's token reads a submission's context and +// nothing else, and only the proxy and the login gate can mint link codes. +type Caller string + +const ( + // CallerVelocity is the proxy's felis-link plugin (felis-service-token, + // written into felis-link.properties on the host). + CallerVelocity Caller = "velocity" + // CallerLimbo is the login gate's felis-limbo plugin (felis-limbo-token, + // injected into the login pod only). + CallerLimbo Caller = "limbo" + // CallerBuild is the build Job's context-fetch initContainer + // (felis-build-token in the build namespace). + CallerBuild Caller = "build" + // CallerOps is the on-node console, `felis backup-now` (felis-ops-token, + // control namespace only). + CallerOps Caller = "ops" +) + +// InternalAuth authenticates the internal face: a per-caller static token +// presented as a Bearer credential, answered with the caller it belongs to. The +// internal face is never wrapped in Zero Trust (spec §1.8, §14 red line). type InternalAuth interface { - Authenticate(r *http.Request) error + Authenticate(r *http.Request) (Caller, error) } // ExternalAuth authenticates the external face (people / panel) and returns the @@ -86,26 +108,45 @@ type ExternalAuth interface { Authenticate(r *http.Request) (*Principal, error) } -// BearerTokenAuth is the production InternalAuth: a constant-time comparison -// against the configured service token. A zero token fails closed so a -// misconfiguration can never silently disable internal-face auth. -type BearerTokenAuth struct { - Token string +// CallerTokens is the production InternalAuth: each caller's token, compared in +// constant time. A caller with no token cannot authenticate, so a missing +// Secret fails closed for that caller alone. +type CallerTokens map[Caller]string + +// NewCallerTokens refuses a set that would make the caller ambiguous: two +// callers sharing a value, which is also what an install whose callers all +// still hold the one old service token would look like. +func NewCallerTokens(tokens map[Caller]string) (CallerTokens, error) { + seen := map[string]Caller{} + for caller, tok := range tokens { + if tok == "" { + continue + } + if other, dup := seen[tok]; dup { + return nil, fmt.Errorf("the %s and %s tokens are the same value; each caller needs its own", other, caller) + } + seen[tok] = caller + } + return CallerTokens(tokens), nil } -// Authenticate checks the Authorization: Bearer header against the token. -func (b BearerTokenAuth) Authenticate(r *http.Request) error { - if b.Token == "" { - return fmt.Errorf("internal auth not configured") - } +// Authenticate matches the Authorization: Bearer header against every caller's +// token, comparing each so the time taken does not say which one matched. +func (c CallerTokens) Authenticate(r *http.Request) (Caller, error) { got := bearerToken(r) if got == "" { - return fmt.Errorf("missing bearer token") + return "", fmt.Errorf("missing bearer token") } - if subtle.ConstantTimeCompare([]byte(got), []byte(b.Token)) != 1 { - return fmt.Errorf("invalid service token") + var match Caller + for caller, tok := range c { + if tok != "" && subtle.ConstantTimeCompare([]byte(got), []byte(tok)) == 1 { + match = caller + } } - return nil + if match == "" { + return "", fmt.Errorf("invalid service token") + } + return match, nil } // AccessVerifier is the production ExternalAuth: it parses a Cloudflare Access diff --git a/internal/api/errors.go b/internal/api/errors.go index 7db33d1..09b6953 100644 --- a/internal/api/errors.go +++ b/internal/api/errors.go @@ -163,8 +163,10 @@ var ( // 503 "retry" instead of a 401 that reads as "log in again". errAuthUnavailable = newError(http.StatusServiceUnavailable, "auth_unavailable", "authentication is temporarily unavailable; retry shortly") - errForbidden = newError(http.StatusForbidden, "forbidden", "not permitted") - errBadRequest = newError(http.StatusBadRequest, "bad_request", "invalid request") + errForbidden = newError(http.StatusForbidden, "forbidden", "not permitted") + // errWrongCaller: a valid internal token for a caller this route does not serve. + errWrongCaller = newError(http.StatusForbidden, "wrong_caller", "this token's caller may not use this route") + errBadRequest = newError(http.StatusBadRequest, "bad_request", "invalid request") ) // writeJSON writes v as an indented JSON body with the given status. diff --git a/internal/api/handlers_account_migrate.go b/internal/api/handlers_account_migrate.go index d62ab91..17b75be 100644 --- a/internal/api/handlers_account_migrate.go +++ b/internal/api/handlers_account_migrate.go @@ -116,7 +116,7 @@ func (a *API) handleMigrateStart(w http.ResponseWriter, r *http.Request) { // Internal-face event: attribute to the in-game initiator, Source 'internal'. a.auditEntry(r, AuditEntry{ Actor: "mc:" + mcUUID, - Source: "internal", + Source: internalSource(r), Action: "account.migrate.start", }) writeJSON(w, http.StatusCreated, map[string]any{"started": true, "state": "initiated"}) diff --git a/internal/api/handlers_backup_now_test.go b/internal/api/handlers_backup_now_test.go index f70a00a..25bf86a 100644 --- a/internal/api/handlers_backup_now_test.go +++ b/internal/api/handlers_backup_now_test.go @@ -272,7 +272,7 @@ func TestBackupNow(t *testing.T) { // the stopped-gate / 503 / async-202 behaviour is proven there; here the focus is the // internal-face difference: no Principal (service-token auth), no owner gate — even a // server owned by someone else backs up (the on-node operator is trusted) — and the -// audit is attributed to "break-glass"/"internal", not an email/"external". +// audit is attributed to "break-glass"/"internal:ops", not an email/"external". func TestInternalBackup(t *testing.T) { mk := func() (*API, *fakeRepo, *fakeCluster, *fakeBackuper) { repo := newFakeRepo() @@ -282,6 +282,7 @@ func TestInternalBackup(t *testing.T) { Ready: false, DesiredState: string(v1alpha1.DesiredStopped)} backuper := &fakeBackuper{} api := newTestAPI(repo, cl) + api.Internal = okInternal{caller: CallerOps} api.Backuper = backuper return api, repo, cl, backuper } @@ -301,7 +302,7 @@ func TestInternalBackup(t *testing.T) { backuper.calls, backuper.gotName, backuper.gotFormerOwn) } if len(repo.audits) != 1 || repo.audits[0].Action != "backup.create" || - repo.audits[0].Actor != "break-glass" || repo.audits[0].Source != "internal" { + repo.audits[0].Actor != "break-glass" || repo.audits[0].Source != "internal:ops" { t.Fatalf("audit not attributed to break-glass/internal: %+v", repo.audits) } }) @@ -312,7 +313,7 @@ func TestInternalBackup(t *testing.T) { if w.Code != http.StatusAccepted { t.Fatalf("code = %d, want 202 (%s)", w.Code, w.Body.String()) } - if len(repo.audits) != 1 || repo.audits[0].Actor != "alice" || repo.audits[0].Source != "internal" { + if len(repo.audits) != 1 || repo.audits[0].Actor != "alice" || repo.audits[0].Source != "internal:ops" { t.Fatalf("audit actor should be the os_user, not break-glass: %+v", repo.audits) } }) diff --git a/internal/api/handlers_backups.go b/internal/api/handlers_backups.go index 65b7513..0d94c8f 100644 --- a/internal/api/handlers_backups.go +++ b/internal/api/handlers_backups.go @@ -375,7 +375,7 @@ func (a *API) handleInternalBackup(w http.ResponseWriter, r *http.Request) { return } - a.enqueueBackup(w, r, name, rec, actor, "internal") + a.enqueueBackup(w, r, name, rec, actor, internalSource(r)) } // enqueueBackup is the shared tail of both backup faces: the RWO stopped-gate, the diff --git a/internal/api/handlers_internal.go b/internal/api/handlers_internal.go index 132ad5c..7895da3 100644 --- a/internal/api/handlers_internal.go +++ b/internal/api/handlers_internal.go @@ -56,7 +56,7 @@ func (a *API) handleReady(w http.ResponseWriter, r *http.Request) { return } a.auditEntry(r, AuditEntry{ - Actor: "backend", Source: "internal", Action: "ready", ServerName: name, + Actor: "backend", Source: internalSource(r), Action: "ready", ServerName: name, }) w.WriteHeader(http.StatusNoContent) } @@ -167,7 +167,7 @@ func (a *API) handleInternalWake(w http.ResponseWriter, r *http.Request) { // untouched and the next join attempt is not also throttled. a.limiter().record(name) a.auditEntry(r, AuditEntry{ - Actor: "velocity", Source: "internal", Action: "wake", ServerName: name, + Actor: "velocity", Source: internalSource(r), Action: "wake", ServerName: name, }) writeJSON(w, http.StatusAccepted, map[string]any{ "name": name, "desiredState": "Running", @@ -259,7 +259,7 @@ func (a *API) handleInternalClaim(w http.ResponseWriter, r *http.Request) { } a.auditEntry(r, AuditEntry{ - Actor: "velocity", Source: "internal", Action: "claim", ServerName: name, + Actor: "velocity", Source: internalSource(r), Action: "claim", ServerName: name, }) writeJSON(w, http.StatusOK, map[string]any{"name": name, "claimed": true}) } diff --git a/internal/api/handlers_internal_menu_test.go b/internal/api/handlers_internal_menu_test.go index 52268ad..507a65c 100644 --- a/internal/api/handlers_internal_menu_test.go +++ b/internal/api/handlers_internal_menu_test.go @@ -211,7 +211,7 @@ func assertEq(t *testing.T, key string, got, want any) { func (f *fakeRepo) assertClaimAudit(t *testing.T, name string) { t.Helper() for _, e := range f.audits { - if e.Action == "claim" && e.ServerName == name && e.Actor == "velocity" && e.Source == "internal" { + if e.Action == "claim" && e.ServerName == name && e.Actor == "velocity" && e.Source == "internal:velocity" { return } } diff --git a/internal/api/handlers_op_login.go b/internal/api/handlers_op_login.go index 72ccfae..62c1eb3 100644 --- a/internal/api/handlers_op_login.go +++ b/internal/api/handlers_op_login.go @@ -430,7 +430,7 @@ func (a *API) handleOpLoginApprove(w http.ResponseWriter, r *http.Request) { } payload, _ := json.Marshal(map[string]string{"request_id": id, "approver_user_id": approverID}) a.auditEntry(r, AuditEntry{ - Actor: approver.Username, ActorUserID: approverID, Source: "internal", + Actor: approver.Username, ActorUserID: approverID, Source: internalSource(r), Action: "auth.op_login.approved", Payload: payload, }) writeJSON(w, http.StatusOK, map[string]any{"approved": true}) diff --git a/internal/api/handlers_player_reclaim.go b/internal/api/handlers_player_reclaim.go index 02eefc5..8405c48 100644 --- a/internal/api/handlers_player_reclaim.go +++ b/internal/api/handlers_player_reclaim.go @@ -99,7 +99,7 @@ func (a *API) handleReclaimUsername(w http.ResponseWriter, r *http.Request) { payload, _ := json.Marshal(map[string]string{ "username": req.Username, "squatter_uuid": req.SquatterUUID, "reason": "protected_admin"}) a.auditEntry(r, AuditEntry{ - Actor: "velocity", Source: "internal", Action: "player.reclaim.refused", Payload: payload, + Actor: "velocity", Source: internalSource(r), Action: "player.reclaim.refused", Payload: payload, }) writeError(w, r, newError(http.StatusConflict, "protected_admin", "that username belongs to a linked administrator on the login server and cannot be reclaimed")) @@ -126,7 +126,7 @@ func (a *API) handleReclaimUsername(w http.ResponseWriter, r *http.Request) { // internal since velocity, not a human, drives it. payload, _ := json.Marshal(map[string]string{"username": req.Username, "squatter_uuid": req.SquatterUUID}) a.auditEntry(r, AuditEntry{ - Actor: "velocity", Source: "internal", Action: "player.reclaim", Payload: payload, + Actor: "velocity", Source: internalSource(r), Action: "player.reclaim", Payload: payload, }) writeJSON(w, http.StatusOK, map[string]any{ "blacklisted": true, diff --git a/internal/api/handlers_player_reclaim_test.go b/internal/api/handlers_player_reclaim_test.go index 9b1a2e3..781a65b 100644 --- a/internal/api/handlers_player_reclaim_test.go +++ b/internal/api/handlers_player_reclaim_test.go @@ -130,7 +130,7 @@ func TestReclaimProtectsAdminOnYggdrasil(t *testing.T) { t.Fatalf("audits = %d, want 1 refusal row", len(repo.audits)) } a := repo.audits[0] - if a.Action != "player.reclaim.refused" || a.Actor != "velocity" || a.Source != "internal" { + if a.Action != "player.reclaim.refused" || a.Actor != "velocity" || a.Source != "internal:velocity" { t.Fatalf("audit = %+v, want player.reclaim.refused/velocity/internal", a) } var p map[string]string @@ -278,7 +278,7 @@ func TestReclaimAudited(t *testing.T) { t.Fatalf("audits = %d, want 1", len(repo.audits)) } a := repo.audits[0] - if a.Action != "player.reclaim" || a.Actor != "velocity" || a.Source != "internal" { + if a.Action != "player.reclaim" || a.Actor != "velocity" || a.Source != "internal:velocity" { t.Fatalf("audit = %+v, want player.reclaim/velocity/internal", a) } var p map[string]string diff --git a/internal/api/internal_callers_test.go b/internal/api/internal_callers_test.go new file mode 100644 index 0000000..249a1b6 --- /dev/null +++ b/internal/api/internal_callers_test.go @@ -0,0 +1,158 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func callerTokensForTest() CallerTokens { + return CallerTokens{ + CallerVelocity: "tok-velocity", + CallerLimbo: "tok-limbo", + CallerBuild: "tok-build", + CallerOps: "tok-ops", + } +} + +func bearer(tok string) map[string]string { + return map[string]string{"Authorization": "Bearer " + tok, "Content-Type": "application/json"} +} + +// Each token answers with its own caller, and nothing else gets in. +func TestCallerTokensNameTheCaller(t *testing.T) { + c := callerTokensForTest() + for tok, want := range map[string]Caller{ + "tok-velocity": CallerVelocity, + "tok-limbo": CallerLimbo, + "tok-build": CallerBuild, + "tok-ops": CallerOps, + } { + r := httptest.NewRequest("GET", "/", nil) + r.Header.Set("Authorization", "Bearer "+tok) + got, err := c.Authenticate(r) + if err != nil || got != want { + t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want) + } + } + for _, header := range []string{"", "Bearer tok-velocityX", "Bearer tok-veloci", "Basic tok-ops"} { + r := httptest.NewRequest("GET", "/", nil) + if header != "" { + r.Header.Set("Authorization", header) + } + if got, err := c.Authenticate(r); err == nil { + t.Errorf("%q authenticated as %q", header, got) + } + } + + // A caller whose Secret is missing has an empty token; that must not turn into + // "any empty-ish credential passes". + partial := CallerTokens{CallerVelocity: "tok-velocity", CallerBuild: ""} + r := httptest.NewRequest("GET", "/", nil) + r.Header.Set("Authorization", "Bearer ") + if got, err := partial.Authenticate(r); err == nil { + t.Fatalf("blank bearer authenticated as %q", got) + } +} + +func TestNewCallerTokensRefusesAmbiguousSets(t *testing.T) { + if _, err := NewCallerTokens(map[Caller]string{CallerVelocity: "a", CallerLimbo: "b", CallerBuild: "", CallerOps: "c"}); err != nil { + t.Fatalf("distinct tokens refused: %v", err) + } + _, err := NewCallerTokens(map[Caller]string{CallerVelocity: "same", CallerBuild: "same"}) + if err == nil || !strings.Contains(err.Error(), "same value") { + t.Fatalf("shared value: err = %v, want a same-value refusal", err) + } +} + +// The caller scopes the gap asked for, spelled out rather than read back from the +// route table: the build token reads a submission's context and nothing else, only +// the proxy and the login gate mint link codes, only the proxy approves an +// op-login, and only the on-node console asks for a break-glass backup. +func TestInternalRoutesServeOnlyTheirCallers(t *testing.T) { + a := newTestAPI(newFakeRepo(), newFakeCluster()) + a.Internal = callerTokensForTest() + h := a.InternalHandler() + + cases := []struct { + method, path string + allowed []Caller + }{ + {"GET", "/api/v1/servers", []Caller{CallerVelocity}}, + {"GET", "/api/v1/internal/submissions/sub-1/context", []Caller{CallerBuild}}, + {"POST", "/api/v1/internal/account/link/code", []Caller{CallerVelocity, CallerLimbo}}, + {"GET", "/api/v1/internal/account/link/status/00000000-0000-0000-0000-000000000001", []Caller{CallerVelocity, CallerLimbo}}, + {"GET", "/api/v1/internal/player/blacklist/00000000-0000-0000-0000-000000000001", []Caller{CallerVelocity, CallerLimbo}}, + {"POST", "/api/v1/internal/op-login/req-1/approve", []Caller{CallerVelocity}}, + {"GET", "/api/v1/internal/op-login/pending", []Caller{CallerVelocity}}, + {"POST", "/api/v1/internal/account/migrate/start", []Caller{CallerVelocity}}, + {"POST", "/api/v1/internal/player/reclaim", []Caller{CallerVelocity}}, + {"POST", "/api/v1/internal/servers/survival/wake", []Caller{CallerVelocity}}, + {"POST", "/api/v1/internal/servers/survival/claim", []Caller{CallerVelocity}}, + {"POST", "/api/v1/internal/servers/survival/backup", []Caller{CallerOps}}, + } + tokens := map[Caller]string{CallerVelocity: "tok-velocity", CallerLimbo: "tok-limbo", CallerBuild: "tok-build", CallerOps: "tok-ops"} + for _, tc := range cases { + for caller, tok := range tokens { + allowed := false + for _, c := range tc.allowed { + allowed = allowed || c == caller + } + w := do(h, tc.method, tc.path, "{}", bearer(tok)) + refused := w.Code == http.StatusForbidden && decodeErr(t, w) == "wrong_caller" + if allowed && (refused || w.Code == http.StatusUnauthorized) { + t.Errorf("%s %s as %s: refused (%d %s), want it served", tc.method, tc.path, caller, w.Code, w.Body.String()) + } + if !allowed && !refused { + t.Errorf("%s %s as %s: got %d %s, want 403 wrong_caller", tc.method, tc.path, caller, w.Code, w.Body.String()) + } + } + } +} + +// The audit names the caller whose token asked for the action. +func TestInternalAuditNamesTheCaller(t *testing.T) { + repo := newFakeRepo() + a := newTestAPI(repo, newFakeCluster()) + a.Internal = callerTokensForTest() + r := httptest.NewRequest("POST", "/", nil) + if got := internalSource(r); got != "internal" { + t.Fatalf("no caller: source = %q, want internal", got) + } + var seen string + probe := a.requireInternal(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = internalSource(r) + })) + r.Header.Set("Authorization", "Bearer tok-limbo") + probe.ServeHTTP(httptest.NewRecorder(), r) + if seen != "internal:limbo" { + t.Fatalf("source = %q, want internal:limbo", seen) + } +} + +// A route added to the internal table without saying who calls it would be open +// to every token; the face refuses to build instead. Callers on an external route +// would mean nothing, so that is refused too. +func TestBuildFaceRequiresCallersOnInternalRoutes(t *testing.T) { + a := newTestAPI(newFakeRepo(), newFakeCluster()) + noop := func(w http.ResponseWriter, r *http.Request) {} + pass := func(h http.Handler) http.Handler { return h } + mustPanic := func(name string, fn func()) { + t.Helper() + defer func() { + if recover() == nil { + t.Errorf("%s: built without complaint", name) + } + }() + fn() + } + mustPanic("internal route without callers", func() { + a.buildFace("internal", []apiRoute{{Method: "GET", Pattern: "/api/v1/internal/x", h: noop}}, pass) + }) + mustPanic("external route with callers", func() { + a.buildFace("external", []apiRoute{{Method: "GET", Pattern: "/api/v1/x", Callers: []Caller{CallerOps}, h: noop}}, pass) + }) + // Public internal routes (probes, hasJoined) carry no token and list no callers. + a.buildFace("internal", []apiRoute{{Method: "GET", Pattern: "/readyz", Public: true, h: noop}}, pass) +} diff --git a/internal/api/middleware.go b/internal/api/middleware.go index 91e1104..40fe083 100644 --- a/internal/api/middleware.go +++ b/internal/api/middleware.go @@ -210,18 +210,35 @@ func (b *deadlineBody) Read(p []byte) (int, error) { return n, err } -// requireInternal enforces service-token auth for the internal face. It never -// applies Zero Trust (spec §14 red line). +// requireInternal enforces service-token auth for the internal face and stashes +// the caller the token belongs to, which callersOnly checks against the route. +// It never applies Zero Trust (spec §14 red line). func (a *API) requireInternal(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := a.Internal.Authenticate(r); err != nil { + caller, err := a.Internal.Authenticate(r) + if err != nil { writeError(w, r, errUnauthorized) return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxKeyCaller, caller))) }) } +// callersOnly refuses an internal route to a caller it does not list: the token +// is genuine, it just belongs to a machine this route does not serve. +func callersOnly(callers []Caller, next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + caller := callerFromContext(r.Context()) + for _, c := range callers { + if c == caller { + next(w, r) + return + } + } + writeError(w, r, errWrongCaller) + } +} + // requireExternal enforces Access-JWT auth for the external face and stashes the // resolved Principal in the request context. func (a *API) requireExternal(next http.Handler) http.Handler { diff --git a/internal/api/openapi_test.go b/internal/api/openapi_test.go index d1fed99..e9c1e41 100644 --- a/internal/api/openapi_test.go +++ b/internal/api/openapi_test.go @@ -40,8 +40,9 @@ type oasDoc struct { } type oasOp struct { - Faces []string `json:"x-felis-face"` - Tier string `json:"x-felis-tier"` + Faces []string `json:"x-felis-face"` + Tier string `json:"x-felis-tier"` + Callers []string `json:"x-felis-callers"` } // oasFacet is the classification of one {method, path}: which face(s) serve it @@ -49,6 +50,8 @@ type oasOp struct { type oasFacet struct { faces map[string]bool tier string + // callers is the internal route's caller set, empty elsewhere. + callers map[string]bool } func TestOpenAPIMatchesServedRoutes(t *testing.T) { @@ -80,6 +83,10 @@ func TestOpenAPIMatchesServedRoutes(t *testing.T) { if s.tier != d.tier { t.Errorf("%s: x-felis-tier mismatch — served %q, documented %q", key, s.tier, d.tier) } + if !oasSameSet(s.callers, d.callers) { + t.Errorf("%s: x-felis-callers mismatch — served %v, documented %v", + key, oasSortedKeys(s.callers), oasSortedKeys(d.callers)) + } } } @@ -92,11 +99,14 @@ func oasServedFacets(t *testing.T) map[string]oasFacet { t.Helper() a := &API{} out := map[string]oasFacet{} - add := func(method, pattern, face, tier string) { + add := func(method, pattern, face, tier string, callers ...Caller) { key := method + " " + pattern f, ok := out[key] if !ok { - f = oasFacet{faces: map[string]bool{}} + f = oasFacet{faces: map[string]bool{}, callers: map[string]bool{}} + } + for _, c := range callers { + f.callers[string(c)] = true } f.faces[face] = true if f.tier != "" && f.tier != tier { @@ -110,7 +120,7 @@ func oasServedFacets(t *testing.T) map[string]oasFacet { if rt.Public { tier = "public" } - add(rt.Method, rt.Pattern, "internal", tier) + add(rt.Method, rt.Pattern, "internal", tier, rt.Callers...) } for _, rt := range a.externalAPIRoutes() { var tier string @@ -172,7 +182,11 @@ func oasDocumentedFacets(t *testing.T) map[string]oasFacet { if _, dup := out[key]; dup { t.Errorf("%s: documented more than once", key) } - out[key] = oasFacet{faces: faces, tier: op.Tier} + callers := map[string]bool{} + for _, c := range op.Callers { + callers[c] = true + } + out[key] = oasFacet{faces: faces, tier: op.Tier, callers: callers} } } return out diff --git a/internal/api/submissions_test.go b/internal/api/submissions_test.go index 67d1fb4..cfbed48 100644 --- a/internal/api/submissions_test.go +++ b/internal/api/submissions_test.go @@ -605,6 +605,7 @@ func TestSubmissionRoutesWithoutServiceAre503(t *testing.T) { func TestInternalSubmissionContextRoute(t *testing.T) { newAPI := func(s SubmissionService) *API { api := newTestAPI(newFakeRepo(), newFakeCluster()) + api.Internal = okInternal{caller: CallerBuild} api.Submissions = s return api } diff --git a/internal/build/jobspec.go b/internal/build/jobspec.go index 93154ce..12c2a28 100644 --- a/internal/build/jobspec.go +++ b/internal/build/jobspec.go @@ -291,15 +291,17 @@ func BuildJob(p JobParams) (*batchv1.Job, error) { Name: ContainerFetch, Image: p.FelisImage, Args: fetchArgs(p), - // The internal face is service-token gated, and the token is read from a - // Secret the installer materializes in THIS namespace (secretKeyRef is - // namespace-local). It is mounted into this initContainer only: the Kaniko + // The internal face is token gated, and the build caller's token + // (felis-build-token, which reads a submission's context and nothing + // else) is read from a Secret the installer materializes in THIS + // namespace (secretKeyRef is namespace-local). It is mounted into this + // initContainer only: the Kaniko // container executes the untrusted Dockerfile and must never hold it, and // pod containers share neither environment nor PID namespace. Env: []corev1.EnvVar{{ Name: "FELIS_SERVICE_TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: naming.ServiceTokenSecretName}, + LocalObjectReference: corev1.LocalObjectReference{Name: naming.BuildTokenSecretName}, Key: naming.ServiceTokenSecretKey, }}, }}, diff --git a/internal/build/jobspec_test.go b/internal/build/jobspec_test.go index d52e076..5694b9a 100644 --- a/internal/build/jobspec_test.go +++ b/internal/build/jobspec_test.go @@ -450,6 +450,11 @@ func TestBuildJobFetchesHTTPContext(t *testing.T) { if fetchToken.Value != "" { t.Error("fetch container must not carry a literal token") } + // The build token reads a submission's context and nothing else; the proxy's + // felis-service-token must never be copied into the build namespace. + if ref := fetchToken.ValueFrom.SecretKeyRef; ref.Name != "felis-build-token" || ref.Key != "token" { + t.Errorf("fetch token reads %s/%s, want felis-build-token/token", ref.Name, ref.Key) + } if len(kaniko.Env) != 0 { t.Errorf("kaniko must carry no env (especially no token), got %v", kaniko.Env) } diff --git a/internal/naming/naming.go b/internal/naming/naming.go index dfaa4f7..0bb6b7a 100644 --- a/internal/naming/naming.go +++ b/internal/naming/naming.go @@ -55,16 +55,23 @@ func IsSystemServer(name string) bool { return name == SystemLoginServer || name == SystemLobbyServer } -// ServiceTokenSecretName / ServiceTokenSecretKey name the internal-API bearer -// credential Secret (spec §7). They are one source of truth shared across -// subsystems: the platform renderer wires this Secret into the felis-api -// Deployment, and the operator injects it into the login system server's pod as -// FELIS_SERVICE_TOKEN via a secretKeyRef (never a literal). The Secret itself is -// provisioned out-of-band (deploy/bootstrap.sh) and, for the login gate, replicated -// into the minecraft namespace by `felis setup`; these constants only name it. +// ServiceTokenSecretName / ServiceTokenSecretKey name the proxy's internal-API +// bearer credential Secret (spec §7); CallerTokens lists it with the tokens the +// other internal callers hold. The Secrets are provisioned out-of-band +// (deploy/bootstrap.sh) and replicated by `felis setup` into the namespace whose +// pods mount them; these constants only name them. const ( ServiceTokenSecretName = "felis-service-token" ServiceTokenSecretKey = "token" + // LimboTokenSecretName is the login gate's token, replicated into the + // minecraft namespace; the operator injects it into the login pod only. + LimboTokenSecretName = "felis-limbo-token" + // BuildTokenSecretName is the build Job's token, replicated into the build + // namespace for the context-fetch initContainer. + BuildTokenSecretName = "felis-build-token" + // OpsTokenSecretName is the on-node console's token (`felis backup-now`); it + // stays in the control namespace. + OpsTokenSecretName = "felis-ops-token" // EnvAPIBaseURL carries the internal-face base URL (platform.InternalAPIBaseURL) // into a pod: the login gate dials it, and the api reads it to derive the build // contexts' fetch URLs, so both sides name the same address for the same face. @@ -209,3 +216,25 @@ func ValidateHostname(host, rootDomain string) error { } return nil } + +// CallerToken ties one internal-face caller (api.Caller) to the Secret holding +// its token, the env var felis-api reads that token from, and the namespace a +// replica of the Secret must reach for the caller's pods ("" when the caller +// runs outside the cluster or in the control namespace). +type CallerToken struct { + Caller string + Secret string + APIEnv string + // Replica names where the caller's pods run: "minecraft" or "build". + Replica string +} + +// CallerTokens is every internal caller, one token each. felis-api refuses to +// start when two share a value, so a token copied from one namespace opens only +// the routes that caller is listed on. +var CallerTokens = []CallerToken{ + {Caller: "velocity", Secret: ServiceTokenSecretName, APIEnv: "FELIS_SERVICE_TOKEN"}, + {Caller: "limbo", Secret: LimboTokenSecretName, APIEnv: "FELIS_LIMBO_TOKEN", Replica: "minecraft"}, + {Caller: "build", Secret: BuildTokenSecretName, APIEnv: "FELIS_BUILD_TOKEN", Replica: "build"}, + {Caller: "ops", Secret: OpsTokenSecretName, APIEnv: "FELIS_OPS_TOKEN"}, +} diff --git a/internal/operator/builders.go b/internal/operator/builders.go index 7bf2123..3e00488 100644 --- a/internal/operator/builders.go +++ b/internal/operator/builders.go @@ -323,21 +323,22 @@ func buildEnv(server *v1alpha1.MinecraftServer) []corev1.EnvVar { } // The login system server is the ONE workload that authenticates to the // felis-api internal face (its felis-limbo plugin mints bind codes and polls - // link status), so it — and only it — receives the service token. Injected + // link status), so it — and only it — receives a token: felis-limbo-token, + // which the api serves on those routes alone. Injected // from a Secret in this namespace, never inlined into the CRD (the same // discipline as RCON_PASSWORD above; the CRD's EnvVar type has no valueFrom // precisely so a user server cannot mount an arbitrary secret). Require both // the reserved name and the setup-owned system-role label: the label prevents // a legacy user server named "login" from receiving the token after upgrade. - // The Secret must exist in this (minecraft) namespace; `felis setup` replicates - // it there from the control namespace before creating this server. + // The Secret must exist in this (minecraft) namespace; the installer applies it + // there and `felis setup` replicates it from the control namespace. if server.Name == naming.SystemLoginServer && server.Labels[v1alpha1.LabelSystemRole] == naming.SystemLoginServer { env = append(env, corev1.EnvVar{ Name: envServiceToken, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: naming.ServiceTokenSecretName}, + LocalObjectReference: corev1.LocalObjectReference{Name: naming.LimboTokenSecretName}, Key: naming.ServiceTokenSecretKey, }, }, diff --git a/internal/operator/builders_internal_test.go b/internal/operator/builders_internal_test.go index c260737..f6dff23 100644 --- a/internal/operator/builders_internal_test.go +++ b/internal/operator/builders_internal_test.go @@ -227,9 +227,10 @@ func TestBuildStatefulSetAddsHealthPort(t *testing.T) { } } -// The login system server (and ONLY it) receives the service token, sourced from a -// Secret via secretKeyRef — never a literal — so its felis-limbo plugin can -// authenticate to the felis-api internal face. +// The login system server (and ONLY it) receives the login gate's own token, +// sourced from a Secret via secretKeyRef — never a literal — so its felis-limbo +// plugin can authenticate to the felis-api internal face as the limbo caller. It +// must not be the proxy's felis-service-token, which opens every game route. func TestBuildEnvInjectsServiceTokenForLogin(t *testing.T) { s := &v1alpha1.MinecraftServer{} s.Name = naming.SystemLoginServer @@ -245,8 +246,8 @@ func TestBuildEnvInjectsServiceTokenForLogin(t *testing.T) { t.Fatalf("%s must be sourced from a secretKeyRef", envServiceToken) } ref := tok.ValueFrom.SecretKeyRef - if ref.Name != naming.ServiceTokenSecretName || ref.Key != naming.ServiceTokenSecretKey { - t.Errorf("secretKeyRef = %s/%s, want %s/%s", ref.Name, ref.Key, naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey) + if ref.Name != "felis-limbo-token" || ref.Key != "token" { + t.Errorf("secretKeyRef = %s/%s, want felis-limbo-token/token", ref.Name, ref.Key) } } diff --git a/internal/platform/workloads.go b/internal/platform/workloads.go index e919e9e..11e3e14 100644 --- a/internal/platform/workloads.go +++ b/internal/platform/workloads.go @@ -50,20 +50,18 @@ import ( // kubernetes.io/hostname selector and PV node affinity) or the CronJob could // schedule on a node where the worlds-root is empty. const ( - // configSecretName / serviceTokenSecretName are referenced BY NAME and NEVER - // rendered into the bundle: felis.toml carries the database URL (a credential) - // and the service token is a credential, so writing either into a checked-in - // manifest is a hard red line. The deployment provisions both Secrets - // out-of-band before applying these workloads. + // configSecretName and the caller token Secrets (naming.CallerTokens) are + // referenced BY NAME and NEVER rendered into the bundle: felis.toml carries the + // database URL (a credential) and each token is a credential, so writing any of + // them into a checked-in manifest is a hard red line. The deployment provisions + // these Secrets out-of-band before applying these workloads. configSecretName = "felis-config" configSecretKey = "felis.toml" configMountPath = "/etc/felis" configFilePath = "/etc/felis/felis.toml" felisBinaryPath = "/usr/local/bin/felis" - // Single-sourced with the operator, which injects the same Secret into the - // login system server's pod (see internal/naming). - serviceTokenSecretName = naming.ServiceTokenSecretName - serviceTokenSecretKey = naming.ServiceTokenSecretKey + // The key every caller token Secret stores its value under (internal/naming). + serviceTokenSecretKey = naming.ServiceTokenSecretKey // Ports, single-sourced with the entrypoints (cmd/felis). The api external // port must match server.listen in felis.toml (default 0.0.0.0:8080); that @@ -323,8 +321,8 @@ func retentionEnabled(p Params) bool { // fence and is asserted in workloads_test.go. // // felis.toml is mounted read-only from a Secret (it carries the database URL, a -// credential, so it must never be a ConfigMap); FELIS_SERVICE_TOKEN comes from a -// second Secret by reference. FELIS_IMAGE is the felis image itself, so the +// credential, so it must never be a ConfigMap); each internal caller's token +// (naming.CallerTokens) comes from its own Secret by reference. FELIS_IMAGE is the felis image itself, so the // restore executor launches `felis restore` with the same image. FELIS_BACKUP_PVC // is rendered only when a backup PVC is named — otherwise the restore endpoint // degrades to 503 rather than enqueuing a Job that cannot mount its backup. @@ -336,22 +334,29 @@ func retentionEnabled(p Params) bool { func APIDeployment(p Params) *appsv1.Deployment { p = p.withDefaults() - env := []corev1.EnvVar{ - { - Name: "FELIS_SERVICE_TOKEN", + var env []corev1.EnvVar + // Every internal caller's token, each from its own Secret. Required: the + // installer applies all four before this Deployment, and a missing one should + // stall the rollout on the old pods rather than start an api that turns that + // caller away. + for _, ct := range naming.CallerTokens { + env = append(env, corev1.EnvVar{ + Name: ct.APIEnv, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: serviceTokenSecretName}, + LocalObjectReference: corev1.LocalObjectReference{Name: ct.Secret}, Key: serviceTokenSecretKey, }, }, - }, - {Name: "FELIS_IMAGE", Value: p.FelisImage}, + }) + } + env = append(env, + corev1.EnvVar{Name: "FELIS_IMAGE", Value: p.FelisImage}, // The api's own internal-face base URL, so it derives the submission // context URLs that build Pods fetch through it. Same value the login gate // is handed; one address for one face. - {Name: naming.EnvAPIBaseURL, Value: InternalAPIBaseURL(p.ControlNamespace)}, - } + corev1.EnvVar{Name: naming.EnvAPIBaseURL, Value: InternalAPIBaseURL(p.ControlNamespace)}, + ) if p.BackupPVC != "" { env = append(env, corev1.EnvVar{Name: "FELIS_BACKUP_PVC", Value: p.BackupPVC}) } diff --git a/internal/platform/workloads_test.go b/internal/platform/workloads_test.go index 1fdc67f..17ae3d5 100644 --- a/internal/platform/workloads_test.go +++ b/internal/platform/workloads_test.go @@ -233,13 +233,28 @@ func TestAPIDeployment_Wiring(t *testing.T) { if v := envValue(c.Env, "FELIS_API_BASE_URL"); v != InternalAPIBaseURL(p.ControlNamespace) { t.Errorf("FELIS_API_BASE_URL = %q, want %q", v, InternalAPIBaseURL(p.ControlNamespace)) } - // FELIS_SERVICE_TOKEN must come from a Secret, never a literal value. - tok := envVar(c.Env, "FELIS_SERVICE_TOKEN") - if tok == nil || tok.ValueFrom == nil || tok.ValueFrom.SecretKeyRef == nil { - t.Fatal("FELIS_SERVICE_TOKEN must be sourced from a secretKeyRef") - } - if tok.Value != "" { - t.Error("FELIS_SERVICE_TOKEN must not carry a literal value") + // Each internal caller's token comes from its own Secret, never a literal + // value, and none is optional: a missing Secret must hold the rollout back. + for env, secret := range map[string]string{ + "FELIS_SERVICE_TOKEN": "felis-service-token", + "FELIS_LIMBO_TOKEN": "felis-limbo-token", + "FELIS_BUILD_TOKEN": "felis-build-token", + "FELIS_OPS_TOKEN": "felis-ops-token", + } { + tok := envVar(c.Env, env) + if tok == nil || tok.ValueFrom == nil || tok.ValueFrom.SecretKeyRef == nil { + t.Fatalf("%s must be sourced from a secretKeyRef", env) + } + ref := tok.ValueFrom.SecretKeyRef + if ref.Name != secret || ref.Key != "token" { + t.Errorf("%s reads %s/%s, want %s/token", env, ref.Name, ref.Key, secret) + } + if ref.Optional != nil && *ref.Optional { + t.Errorf("%s is optional; the api must not start without it", env) + } + if tok.Value != "" { + t.Errorf("%s must not carry a literal value", env) + } } // felis.toml carries the DB URL, so its volume must be a Secret (NOT a diff --git a/plugins/README.md b/plugins/README.md index d2db098..8d57282 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -229,8 +229,11 @@ Drop the matching jar into the server/proxy mods or plugins directory, start once to generate `config/felis-link.properties` (or `plugins/felis-link/…` on Velocity), then set `api-base-url` and `service-token` — or provide `FELIS_API_BASE_URL` and `FELIS_SERVICE_TOKEN` in the environment, which take -precedence. The service token is the same one felis-api compares for its -internal endpoints; treat it as a secret. +precedence. The token is one of felis-api's per-caller internal tokens: the +Velocity proxy uses the `velocity` token (Secret `felis/felis-service-token`), the +login gate the `limbo` token (`felis-limbo-token`), and each serves only its own +routes (a token on another caller's route gets `403 wrong_caller`). Treat it as a +secret; `sudo felis rotate-token ` replaces it. On **Velocity**, also set `root-domain` (and optionally `lobby-server`) in the same file to turn on §11 routing, and make sure `online-mode=true` in