feat(api): 内部面 token 按调用方拆分为 velocity/limbo/build/ops... (a883c1fe) · Commits · FelisMC / Felis · GitLab

cmd/felis/api.go

+21 −4

Changes for cmd/felis/api.go: 21 added lines, 4 removed lines.
Original line number	Diff line number	Diff line
		@@ -119,9 +119,15 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {

		metrics.SetBuildInfo("api", resolvedVersion())

		token := os.Getenv("FELIS_SERVICE_TOKEN")
		if token == "" {
		fmt.Fprintln(stderr, "felis api: warning: FELIS_SERVICE_TOKEN unset — internal face will reject all callers")
		internalAuth, err := internalCallerTokens(os.Getenv)
		if err != nil {
		fmt.Fprintf(stderr, "felis api: internal face tokens: %v\n", err)
		return 1
		}
		for _, ct := range naming.CallerTokens {
		if internalAuth[api.Caller(ct.Caller)] == "" {
		fmt.Fprintf(stderr, "felis api: warning: %s unset — the internal face turns the %s caller away\n", ct.APIEnv, ct.Caller)
		}
		}

		// Email one-time codes go through the [smtp] relay when one is configured; the
		@@ -299,7 +305,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		// Build-log stream (spec §16) is scoped to the BUILD namespace — the same
		// value the Builder renders Jobs into — so it follows where build Pods run.
		BuildLogs: api.NewK8sBuildLogStreamer(clientset, cfg.Registry.BuildNamespace),
		Internal: api.BearerTokenAuth{Token: token},
		Internal: internalAuth,
		Builder: builder,
		Images: imagePinner(cfg.Registry.URL),
		Restorer: restorer,
		@@ -800,3 +806,14 @@ func imagePinner(registry string) api.ImagePinner {
		}
		return imagepin.Resolver{Registry: registry}
		}

		// internalCallerTokens reads each internal caller's token from the env var the
		// Deployment feeds it from (naming.CallerTokens). Two callers sharing a value
		// would make the caller ambiguous, so that refuses to start.
		func internalCallerTokens(getenv func(string) string) (api.CallerTokens, error) {
		tokens := map[api.Caller]string{}
		for _, ct := range naming.CallerTokens {
		tokens[api.Caller(ct.Caller)] = strings.TrimSpace(getenv(ct.APIEnv))
		}
		return api.NewCallerTokens(tokens)
		}

cmd/felis/api_tokens_test.go

0 → 100644

+38 −0

Changes for cmd/felis/api_tokens_test.go: 38 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"net/http/httptest"
		"strings"
		"testing"

		"felis.lolicon.best/internal/api"
		)

		// felis-api maps each env var onto the caller the Deployment feeds it for, so the
		// build Job's token (FELIS_BUILD_TOKEN) authenticates as build and only as build.
		func TestInternalCallerTokensReadEachCallersEnv(t *testing.T) {
		env := map[string]string{
		"FELIS_SERVICE_TOKEN": "v-tok",
		"FELIS_LIMBO_TOKEN": "l-tok",
		"FELIS_BUILD_TOKEN": " b-tok\n",
		"FELIS_OPS_TOKEN": "o-tok",
		}
		auth, err := internalCallerTokens(func(k string) string { return env[k] })
		if err != nil {
		t.Fatal(err)
		}
		for tok, want := range map[string]api.Caller{"v-tok": api.CallerVelocity, "l-tok": api.CallerLimbo, "b-tok": api.CallerBuild, "o-tok": api.CallerOps} {
		r := httptest.NewRequest("GET", "/", nil)
		r.Header.Set("Authorization", "Bearer "+tok)
		if got, err := auth.Authenticate(r); err != nil \|\| got != want {
		t.Errorf("%s: got (%q, %v), want %q", tok, got, err, want)
		}
		}

		// An install where the build namespace still holds a copy of the proxy's token
		// would let that copy act as the proxy; the api refuses to start on it.
		env["FELIS_BUILD_TOKEN"] = "v-tok"
		if _, err := internalCallerTokens(func(k string) string { return env[k] }); err == nil \|\| !strings.Contains(err.Error(), "same value") {
		t.Fatalf("shared token: err = %v, want a same-value refusal", err)
		}
		}

cmd/felis/backupnow.go

+6 −5

Changes for cmd/felis/backupnow.go: 6 added lines, 5 removed lines.
Original line number	Diff line number	Diff line
		@@ -20,7 +20,7 @@ import (
		// backupnow is the break-glass "back up a world now" op (§B4 "Sync"). Unlike halt —
		// which writes the CRD directly — a backup needs felis-api's deployment coordinates
		// (FELIS_IMAGE / FELIS_BACKUP_PVC) to render the one-shot backup Job, so the console
		// cannot do it in-process. It POSTs the felis-api INTERNAL face (service-token auth)
		// cannot do it in-process. It POSTs the felis-api INTERNAL face (ops-token auth)
		// while the API is alive, and the API renders the Job and audits the action. This file
		// is the pure core (no bubbletea); tui_backupnow.go is the terminal glue.

		@@ -33,7 +33,7 @@ type backupNowOutcome struct {

		// resolveInternalAPI reads the two things the on-node console needs to reach the
		// felis-api internal face: the felis-api-internal Service ClusterIP (the host's
		// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the service
		// resolver is not CoreDNS, so the cluster-DNS name is useless here) and the ops
		// token. Both live in the control namespace.
		func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace string) (baseURL, token string, err error) {
		var svc corev1.Service
		@@ -45,13 +45,14 @@ func resolveInternalAPI(ctx context.Context, cl client.Client, controlNamespace
		return "", "", fmt.Errorf("%s Service has no ClusterIP yet", platform.APIInternalServiceName)
		}

		// The console's own token, which the api serves on the backup route alone.
		var sec corev1.Secret
		if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.ServiceTokenSecretName}, &sec); err != nil {
		return "", "", fmt.Errorf("get %s Secret: %w", naming.ServiceTokenSecretName, err)
		if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNamespace, Name: naming.OpsTokenSecretName}, &sec); err != nil {
		return "", "", fmt.Errorf("get %s Secret (re-run the installer to create it): %w", naming.OpsTokenSecretName, err)
		}
		token = string(sec.Data[naming.ServiceTokenSecretKey])
		if token == "" {
		return "", "", fmt.Errorf("secret %s has no %s key", naming.ServiceTokenSecretName, naming.ServiceTokenSecretKey)
		return "", "", fmt.Errorf("secret %s has no %s key", naming.OpsTokenSecretName, naming.ServiceTokenSecretKey)
		}

		return fmt.Sprintf("http://%s:%d", ip, platform.APIInternalPort), token, nil

cmd/felis/backupnow_test.go

+8 −3

Changes for cmd/felis/backupnow_test.go: 8 added lines, 3 removed lines.
Original line number	Diff line number	Diff line
		@@ -11,7 +11,6 @@ import (
		"testing"
		"time"

		"felis.lolicon.best/internal/naming"
		"felis.lolicon.best/internal/platform"

		corev1 "k8s.io/api/core/v1"
		@@ -28,9 +27,15 @@ func internalAPIObjs(clusterIP, token string) []client.Object {
		ObjectMeta: metav1.ObjectMeta{Name: platform.APIInternalServiceName, Namespace: bgControlNS},
		Spec: corev1.ServiceSpec{ClusterIP: clusterIP},
		},
		// The console presents the ops token; the proxy's felis-service-token sits
		// beside it and must not be the one picked up.
		&corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{Name: naming.ServiceTokenSecretName, Namespace: bgControlNS},
		Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
		ObjectMeta: metav1.ObjectMeta{Name: "felis-ops-token", Namespace: bgControlNS},
		Data: map[string][]byte{"token": []byte(token)},
		},
		&corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{Name: "felis-service-token", Namespace: bgControlNS},
		Data: map[string][]byte{"token": []byte("proxy-" + token)},
		},
		}
		}

cmd/felis/rotatetoken.go

0 → 100644

+309 −0

Changes for cmd/felis/rotatetoken.go: 309 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"context"
		"crypto/rand"
		"encoding/hex"
		"errors"
		"flag"
		"fmt"
		"io"
		"io/fs"
		"os"
		"path/filepath"
		"strings"
		"syscall"

		"felis.lolicon.best/internal/apis/felis/v1alpha1"
		"felis.lolicon.best/internal/config"
		"felis.lolicon.best/internal/naming"
		"felis.lolicon.best/internal/platform"

		corev1 "k8s.io/api/core/v1"
		apierrors "k8s.io/apimachinery/pkg/api/errors"
		metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
		"sigs.k8s.io/controller-runtime/pkg/client"
		)

		// rotate-token replaces one internal caller's token (naming.CallerTokens): a new
		// value goes into the installer's record, the control-namespace Secret and the
		// replica the caller's pods mount, felis-api rolls so it accepts only the new
		// value, and then the caller restarts so it presents it. Between the api's
		// rollout and the caller's restart the caller is turned away with 401; for the
		// login gate and the proxy that is the few seconds of a pod or unit restart.

		const (
		defaultSecretsEnvPath = "/etc/felis/secrets.env"
		defaultLinkPropsPath = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
		velocityUnit = "felis-velocity"
		)

		// installerTokenKeys names each caller's token in the installer's secrets.env
		// (deploy/bootstrap.sh load_or_make_secrets). A re-run of the installer applies
		// these values to the Secrets, so a rotation that skipped the file would be
		// undone by the next upgrade.
		var installerTokenKeys = map[string]string{
		"velocity": "SERVICE_TOKEN",
		"limbo": "LIMBO_TOKEN",
		"build": "BUILD_TOKEN",
		"ops": "OPS_TOKEN",
		}

		type tokenRotator struct {
		cl client.Client
		controlNS string
		minecraftNS string
		buildNS string
		// secretsEnv and linkProps are the installer's record and the proxy's
		// felis-link.properties; a missing file is reported and skipped.
		secretsEnv string
		linkProps string
		newToken func() (string, error)
		// rollAPI restarts felis-api and waits for the rollout.
		rollAPI func(ctx context.Context) error
		// restartUnit restarts a systemd unit on this host.
		restartUnit func(ctx context.Context, unit string) error
		out io.Writer
		}

		func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
		fs := flag.NewFlagSet("rotate-token", flag.ContinueOnError)
		fs.SetOutput(stderr)
		cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
		secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
		linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
		fs.Usage = func() {
		fmt.Fprintf(stderr, "Usage: felis rotate-token [flags] <%s>\n\n", strings.Join(callerNames(), "\|"))
		fmt.Fprintln(stderr, "Replaces one internal caller's token: the Secrets, felis-api, then the caller itself.")
		fs.PrintDefaults()
		}
		if err := fs.Parse(args); err != nil {
		if errors.Is(err, flag.ErrHelp) {
		return 0
		}
		return 2
		}
		if fs.NArg() != 1 {
		fs.Usage()
		return 2
		}
		if _, ok := callerToken(fs.Arg(0)); !ok {
		fmt.Fprintf(stderr, "felis rotate-token: unknown caller %q (one of %s)\n", fs.Arg(0), strings.Join(callerNames(), ", "))
		return 2
		}
		if os.Geteuid() != 0 {
		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+fs.Arg(0)+")")
		return 1
		}
		cfg, err := config.Load(*cfgPath)
		if err != nil {
		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
		return 1
		}
		cl, err := buildSystemServerClient()
		if err != nil {
		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
		return 1
		}
		buildNS := cfg.Registry.BuildNamespace
		if buildNS == "" {
		buildNS = platform.DefaultBuildNamespace
		}
		r := tokenRotator{
		cl: cl,
		controlNS: platform.DefaultControlNamespace,
		minecraftNS: cfg.K8s.Namespace,
		buildNS: buildNS,
		secretsEnv: *secretsEnv,
		linkProps: *linkProps,
		newToken: randomToken,
		rollAPI: func(ctx context.Context) error {
		if err := kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "restart", "deployment/felis-api"); err != nil {
		return err
		}
		return kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "status", "deployment/felis-api", "--timeout=180s")
		},
		restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
		out: stdout,
		}
		if err := r.rotate(context.Background(), fs.Arg(0)); err != nil {
		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
		return 1
		}
		return 0
		}

		func callerNames() []string {
		names := make([]string, 0, len(naming.CallerTokens))
		for _, ct := range naming.CallerTokens {
		names = append(names, ct.Caller)
		}
		return names
		}

		func callerToken(name string) (naming.CallerToken, bool) {
		for _, ct := range naming.CallerTokens {
		if ct.Caller == name {
		return ct, true
		}
		}
		return naming.CallerToken{}, false
		}

		// randomToken is 32 random bytes in hex, the shape the installer generates.
		func randomToken() (string, error) {
		b := make([]byte, 32)
		if _, err := rand.Read(b); err != nil {
		return "", err
		}
		return hex.EncodeToString(b), nil
		}

		func (r tokenRotator) rotate(ctx context.Context, caller string) error {
		ct, ok := callerToken(caller)
		if !ok {
		return fmt.Errorf("unknown caller %q", caller)
		}
		tok, err := r.newToken()
		if err != nil {
		return fmt.Errorf("generate a token: %w", err)
		}

		// The installer's record first: from here on, whatever fails, a re-run of the
		// installer puts the new value everywhere.
		switch err := setKeyValueLine(r.secretsEnv, installerTokenKeys[ct.Caller], "=", tok); {
		case errors.Is(err, fs.ErrNotExist):
		fmt.Fprintf(r.out, " - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
		case err != nil:
		return fmt.Errorf("record the new token in %s: %w", r.secretsEnv, err)
		default:
		fmt.Fprintf(r.out, " - %s: %s updated\n", r.secretsEnv, installerTokenKeys[ct.Caller])
		}

		namespaces := []string{r.controlNS}
		replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
		if replica != "" && replica != r.controlNS {
		namespaces = append(namespaces, replica)
		}
		for _, ns := range namespaces {
		if err := writeTokenSecret(ctx, r.cl, ns, ct.Secret, tok); err != nil {
		return fmt.Errorf("write Secret %s/%s: %w", ns, ct.Secret, err)
		}
		fmt.Fprintf(r.out, " - Secret %s/%s: updated\n", ns, ct.Secret)
		}

		hostProxy := false
		if ct.Caller == "velocity" {
		switch err := setKeyValueLine(r.linkProps, "service-token", "=", tok); {
		case errors.Is(err, fs.ErrNotExist):
		fmt.Fprintf(r.out, " - %s: not found; set service-token in your proxy's felis-link.properties to the value in Secret %s/%s and restart it\n",
		r.linkProps, r.controlNS, ct.Secret)
		case err != nil:
		return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
		default:
		hostProxy = true
		fmt.Fprintf(r.out, " - %s: service-token updated\n", r.linkProps)
		}
		}

		if err := r.rollAPI(ctx); err != nil {
		return fmt.Errorf("roll felis-api: %w", err)
		}
		fmt.Fprintln(r.out, " - felis-api: rolled out, accepting only the new token")

		switch ct.Caller {
		case "velocity":
		if hostProxy {
		if err := r.restartUnit(ctx, velocityUnit); err != nil {
		return fmt.Errorf("restart %s: %w", velocityUnit, err)
		}
		fmt.Fprintf(r.out, " - %s: restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit)
		}
		case "limbo":
		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS),
		client.MatchingLabels{v1alpha1.LabelServer: naming.SystemLoginServer}); err != nil {
		return fmt.Errorf("restart the login gate: %w", err)
		}
		fmt.Fprintln(r.out, " - login gate: pod restarted to read the new token")
		case "build":
		fmt.Fprintln(r.out, " - builds: the next build Job reads the new token; one fetching its context right now fails and can be submitted again")
		case "ops":
		fmt.Fprintln(r.out, " - felis backup-now reads the new token on its next run")
		}
		return nil
		}

		// writeTokenSecret sets the token in a Secret, creating it when absent.
		func writeTokenSecret(ctx context.Context, cl client.Client, namespace, name, token string) error {
		var sec corev1.Secret
		err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &sec)
		if apierrors.IsNotFound(err) {
		return cl.Create(ctx, &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
		Type: corev1.SecretTypeOpaque,
		Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
		})
		}
		if err != nil {
		return err
		}
		if sec.Data == nil {
		sec.Data = map[string][]byte{}
		}
		sec.Data[naming.ServiceTokenSecretKey] = []byte(token)
		return cl.Update(ctx, &sec)
		}

		// setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file
		// (secrets.env, a .properties file), appending one when the key is absent. The
		// file is replaced atomically and keeps its mode and owner: felis-link.properties
		// is root:felis-velocity 0640, and the proxy must still be able to read it.
		func setKeyValueLine(path, key, sep, value string) error {
		info, err := os.Stat(path)
		if err != nil {
		return err
		}
		raw, err := os.ReadFile(path)
		if err != nil {
		return err
		}
		lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
		found := false
		for i, ln := range lines {
		k, _, ok := strings.Cut(ln, sep)
		if ok && strings.TrimSpace(k) == key {
		lines[i] = key + sep + value
		found = true
		}
		}
		if !found {
		lines = append(lines, key+sep+value)
		}
		tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
		if err != nil {
		return err
		}
		defer os.Remove(tmp.Name())
		if err := tmp.Chmod(info.Mode().Perm()); err != nil {
		tmp.Close()
		return err
		}
		if st, ok := info.Sys().(*syscall.Stat_t); ok {
		if err := tmp.Chown(int(st.Uid), int(st.Gid)); err != nil {
		tmp.Close()
		return err
		}
		}
		if _, err := tmp.WriteString(strings.Join(lines, "\n") + "\n"); err != nil {
		tmp.Close()
		return err
		}
		if err := tmp.Sync(); err != nil {
		tmp.Close()
		return err
		}
		if err := tmp.Close(); err != nil {
		return err
		}
		return os.Rename(tmp.Name(), path)
		}