diff --git a/cmd/felis/domain.go b/cmd/felis/domain.go index 88b2a62..a1417f8 100644 --- a/cmd/felis/domain.go +++ b/cmd/felis/domain.go @@ -235,7 +235,7 @@ func verifyTOMLEdit(orig, edited []byte, edits []tomlStringEdit) error { t[e.key] = e.value } if !reflect.DeepEqual(want, got) { - return errors.New("a line edit would change more than the domain keys (a multi-line value, or a quoted or dotted key?)") + return errors.New("a line edit would change more than the keys it sets (a multi-line value, or a quoted or dotted key?)") } return nil } @@ -591,9 +591,15 @@ type tomlTarget struct{ path, real string } // tomlTargets are the host and pod copies, and felis.toml when it is a file of // its own rather than the link to the host copy. func (h domainHost) tomlTargets() ([]tomlTarget, error) { + return tomlTargetsOf(h.paths.hostTOML, h.paths.podTOML, h.paths.defaultTOML) +} + +// tomlTargetsOf is the host copy, the pod copy, and def when it exists and is +// not a link to one of them. +func tomlTargetsOf(host, pod, def string) ([]tomlTarget, error) { var out []tomlTarget seen := map[string]bool{} - for i, p := range []string{h.paths.hostTOML, h.paths.podTOML, h.paths.defaultTOML} { + for i, p := range []string{host, pod, def} { real, err := filepath.EvalSymlinks(p) if errors.Is(err, fs.ErrNotExist) && i == 2 { continue diff --git a/cmd/felis/rotatetoken.go b/cmd/felis/rotatetoken.go index 9c3e44b..6bfb43a 100644 --- a/cmd/felis/rotatetoken.go +++ b/cmd/felis/rotatetoken.go @@ -2,40 +2,70 @@ package main import ( "context" + "crypto/hmac" + "crypto/pbkdf2" "crypto/rand" + "crypto/sha256" + "encoding/base64" "encoding/hex" "errors" "flag" "fmt" "io" "io/fs" + neturl "net/url" "os" + "os/exec" "path/filepath" + "strconv" "strings" "syscall" + "time" "felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/maintenance" "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/operator" "felis.lolicon.best/internal/platform" + "felis.lolicon.best/internal/registrygate" + "github.com/BurntSushi/toml" + "github.com/jackc/pgx/v5" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -// rotate-token replaces one internal caller's token (naming.CallerTokens): a new -// value goes into the installer's record, the control-namespace Secret and the -// replica the caller's pods mount, felis-api rolls so it accepts only the new -// value, and then the caller restarts so it presents it. Between the api's -// rollout and the caller's restart the caller is turned away with 401; for the -// login gate and the proxy that is the few seconds of a pod or unit restart. +// rotate-token replaces one credential the installer generated: an internal +// caller's token (naming.CallerTokens), the registry's write tokens, the +// Velocity forwarding secret or the database password. The new value goes into +// the installer's record first, so that whatever fails later a re-run of the +// installer puts it everywhere; then into the Secrets and files that carry it; +// then whatever read the old value at start restarts. Without -yes it prints +// what it would change and what that interrupts, and changes nothing. const ( defaultSecretsEnvPath = "/etc/felis/secrets.env" defaultLinkPropsPath = "/opt/felis/velocity/plugins/felis-link/felis-link.properties" + defaultForwardingPath = "/opt/felis/velocity/forwarding.secret" velocityUnit = "felis-velocity" + apiDeployment = "felis-api" + registryDeployment = "registry" + + kindRegistry = "registry" + kindForwarding = "forwarding" + kindDB = "db" + + // proxyReloadWait is how long the host proxy gets, once felis-api has rolled, + // to show it re-read its token: it reads the file again on its next call to + // felis-api, and it calls every 15 seconds. + proxyReloadWait = 60 * time.Second + + // apiRestartNote is in every plan: felis-api runs as one replica replaced in + // place, so its restart is a short outage of everything that talks to it. + apiRestartNote = "felis-api restarts (a single replica): the panel, sign-in and the proxy's calls are unavailable for the few seconds that takes" ) // installerTokenKeys names each caller's token in the installer's secrets.env @@ -49,21 +79,55 @@ var installerTokenKeys = map[string]string{ "ops": "OPS_TOKEN", } +// installerRegistryKeys names the registry principals' tokens in secrets.env, +// in registrygate.Principals order. +var installerRegistryKeys = map[string]string{ + registrygate.PrincipalPlatform: "REGISTRY_PLATFORM_TOKEN", + registrygate.PrincipalBuild: "REGISTRY_BUILD_TOKEN", + registrygate.PrincipalPrune: "REGISTRY_PRUNE_TOKEN", +} + +// installerForwardingKey and installerDBKey name the forwarding secret and the +// database password in secrets.env. +const ( + installerForwardingKey = "FORWARDING_SECRET" + installerDBKey = "DB_PASSWORD" +) + type tokenRotator struct { cl client.Client controlNS string minecraftNS string buildNS string - // secretsEnv and linkProps are the installer's record and the proxy's - // felis-link.properties; a missing file is reported and skipped. - secretsEnv string - linkProps string - newToken func() (string, error) - // rollAPI restarts felis-api and waits for the rollout. - rollAPI func(ctx context.Context) error + // secretsEnv, linkProps and forwardingFile are the installer's record and the + // host proxy's felis-link.properties and forwarding.secret; a missing file is + // reported and skipped. + secretsEnv string + linkProps string + forwardingFile string + // hostTOML, podTOML and defaultTOML are the config copies that carry the + // database URL (defaultTOML only when it is a file of its own). + hostTOML string + podTOML string + defaultTOML string + newToken func() (string, error) + // rollout restarts a control-namespace Deployment and waits for it. + rollout func(ctx context.Context, deployment string) error // restartUnit restarts a systemd unit on this host. restartUnit func(ctx context.Context, unit string) error - out io.Writer + // proxyLog is what the host proxy has logged since a moment, as far as it + // can be read. + proxyLog func(ctx context.Context, since time.Time) string + // alterRole stores a password verifier for a role of the database that runs + // as deployment ("namespace/name"); verifyDB connects with a URL. + alterRole func(ctx context.Context, deployment, role, verifier string) error + verifyDB func(ctx context.Context, url string) error + now func() time.Time + // reloadWait bounds the wait for the proxy to re-read its token, polled + // every pollEvery. + reloadWait time.Duration + pollEvery time.Duration + out io.Writer } func cmdRotateToken(args []string, stdout, stderr io.Writer) int { @@ -72,9 +136,12 @@ func cmdRotateToken(args []string, stdout, stderr io.Writer) int { cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml") secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value") linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)") + forwarding := fs.String("forwarding-secret", defaultForwardingPath, "the host proxy's forwarding secret file (forwarding only)") + yes := fs.Bool("yes", false, "rotate; without it the plan is printed and nothing changes") fs.Usage = func() { - fmt.Fprintf(stderr, "Usage: felis rotate-token [flags] <%s>\n\n", strings.Join(callerNames(), "|")) - fmt.Fprintln(stderr, "Replaces one internal caller's token: the Secrets, felis-api, then the caller itself.") + fmt.Fprintf(stderr, "Usage: felis rotate-token [-yes] [flags] <%s>\n\n", strings.Join(rotationKinds(), "|")) + fmt.Fprintln(stderr, "Replaces one generated credential: the installer's record, the Secrets and files that carry it, then what reads it.") + fmt.Fprintln(stderr, "Without -yes it prints what would change and what that interrupts.") fs.PrintDefaults() } if err := fs.Parse(args); err != nil { @@ -87,12 +154,13 @@ func cmdRotateToken(args []string, stdout, stderr io.Writer) int { fs.Usage() return 2 } - if _, ok := callerToken(fs.Arg(0)); !ok { - fmt.Fprintf(stderr, "felis rotate-token: unknown caller %q (one of %s)\n", fs.Arg(0), strings.Join(callerNames(), ", ")) + kind := fs.Arg(0) + if !knownRotation(kind) { + fmt.Fprintf(stderr, "felis rotate-token: unknown credential %q (one of %s)\n", kind, strings.Join(rotationKinds(), ", ")) return 2 } if os.Geteuid() != 0 { - fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+fs.Arg(0)+")") + fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+kind+")") return 1 } cfg, err := config.Load(*cfgPath) @@ -109,24 +177,43 @@ func cmdRotateToken(args []string, stdout, stderr io.Writer) int { if buildNS == "" { buildNS = platform.DefaultBuildNamespace } + controlNS := platform.DefaultControlNamespace r := tokenRotator{ - cl: cl, - controlNS: platform.DefaultControlNamespace, - minecraftNS: cfg.K8s.Namespace, - buildNS: buildNS, - secretsEnv: *secretsEnv, - linkProps: *linkProps, - newToken: randomToken, - rollAPI: func(ctx context.Context) error { - if err := kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "restart", "deployment/felis-api"); err != nil { + cl: cl, + controlNS: controlNS, + minecraftNS: cfg.K8s.Namespace, + buildNS: buildNS, + secretsEnv: *secretsEnv, + linkProps: *linkProps, + forwardingFile: *forwarding, + hostTOML: hostSetupConfigPath, + podTOML: podSetupConfigPath, + defaultTOML: defaultSetupConfigPath, + newToken: randomToken, + rollout: func(ctx context.Context, deployment string) error { + if err := kubectl(ctx, "-n", controlNS, "rollout", "restart", "deployment/"+deployment); err != nil { return err } - return kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "status", "deployment/felis-api", "--timeout=180s") + return kubectl(ctx, "-n", controlNS, "rollout", "status", "deployment/"+deployment, "--timeout=180s") }, restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) }, - out: stdout, + proxyLog: journalSince, + alterRole: alterRoleInPod, + verifyDB: func(ctx context.Context, url string) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + conn, err := pgx.Connect(ctx, url) + if err != nil { + return err + } + return conn.Close(ctx) + }, + now: time.Now, + reloadWait: proxyReloadWait, + pollEvery: 3 * time.Second, + out: stdout, } - if err := r.rotate(context.Background(), fs.Arg(0)); err != nil { + if err := r.rotate(context.Background(), kind, *yes); err != nil { fmt.Fprintf(stderr, "felis rotate-token: %v\n", err) return 1 } @@ -150,6 +237,21 @@ func callerToken(name string) (naming.CallerToken, bool) { return naming.CallerToken{}, false } +// rotationKinds is every credential rotate-token replaces: the callers' tokens +// and the installer's other generated secrets. +func rotationKinds() []string { + return append(callerNames(), kindRegistry, kindForwarding, kindDB) +} + +func knownRotation(kind string) bool { + for _, k := range rotationKinds() { + if k == kind { + return true + } + } + return false +} + // randomToken is 32 random bytes in hex, the shape the installer generates. func randomToken() (string, error) { b := make([]byte, 32) @@ -159,99 +261,527 @@ func randomToken() (string, error) { return hex.EncodeToString(b), nil } -func (r tokenRotator) rotate(ctx context.Context, caller string) error { - ct, ok := callerToken(caller) - if !ok { - return fmt.Errorf("unknown caller %q", caller) - } - tok, err := r.newToken() - if err != nil { - return fmt.Errorf("generate a token: %w", err) - } +// tokenFingerprint is how the proxy names the token it reloaded in its log +// (plugins/shared FileToken.fingerprint): the first twelve hex digits of its +// SHA-256, enough to tell tokens apart and useless for finding one. +func tokenFingerprint(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:])[:12] +} - // The installer's record first: from here on, whatever fails, a re-run of the - // installer puts the new value everywhere. - switch err := setKeyValueLine(r.secretsEnv, installerTokenKeys[ct.Caller], "=", tok); { +func (r tokenRotator) rotate(ctx context.Context, kind string, apply bool) error { + switch kind { + case kindRegistry: + return r.rotateRegistry(ctx, apply) + case kindForwarding: + return r.rotateForwarding(ctx, apply) + case kindDB: + return r.rotateDB(ctx, apply) + } + ct, ok := callerToken(kind) + if !ok { + return fmt.Errorf("unknown credential %q (one of %s)", kind, strings.Join(rotationKinds(), ", ")) + } + return r.rotateCaller(ctx, ct, apply) +} + +// confirm ends the plan: without apply it says nothing changed and how to go +// ahead, and reports false. +func (r tokenRotator) confirm(kind string, apply bool) bool { + if !apply { + fmt.Fprintf(r.out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes %s\n", kind) + return false + } + fmt.Fprintln(r.out, "\nRotating:") + return true +} + +// record writes new values into the installer's secrets.env. It comes first in +// every rotation: from then on, whatever fails, a re-run of the installer puts +// the new values everywhere. +func (r tokenRotator) record(kv ...[2]string) error { + keys := make([]string, len(kv)) + for i, p := range kv { + keys[i] = p[0] + } + switch err := setKeyValueLines(r.secretsEnv, "=", kv); { case errors.Is(err, fs.ErrNotExist): fmt.Fprintf(r.out, " - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv) case err != nil: - return fmt.Errorf("record the new token in %s: %w", r.secretsEnv, err) + return fmt.Errorf("record the new value in %s: %w", r.secretsEnv, err) default: - fmt.Fprintf(r.out, " - %s: %s updated\n", r.secretsEnv, installerTokenKeys[ct.Caller]) + fmt.Fprintf(r.out, " - %s: %s updated\n", r.secretsEnv, strings.Join(keys, ", ")) } + return nil +} +func (r tokenRotator) putSecret(ctx context.Context, ns, name string, data map[string][]byte) error { + if _, err := putSecretKeys(ctx, r.cl, ns, name, corev1.SecretTypeOpaque, data); err != nil { + return fmt.Errorf("write Secret %s/%s: %w", ns, name, err) + } + fmt.Fprintf(r.out, " - Secret %s/%s: updated\n", ns, name) + return nil +} + +func (r tokenRotator) rollAPI(ctx context.Context) error { + if err := r.rollout(ctx, apiDeployment); err != nil { + return fmt.Errorf("roll felis-api: %w", err) + } + fmt.Fprintln(r.out, " - felis-api: rolled out on the new value") + return nil +} + +// withMinecraft is the control namespace plus the minecraft namespace when that +// is a different one: where the Secrets game pods and Jobs mount are mirrored. +func (r tokenRotator) withMinecraft() []string { + if r.minecraftNS == "" || r.minecraftNS == r.controlNS { + return []string{r.controlNS} + } + return []string{r.controlNS, r.minecraftNS} +} + +func (r tokenRotator) rotateCaller(ctx context.Context, ct naming.CallerToken, apply bool) error { namespaces := []string{r.controlNS} replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica] if replica != "" && replica != r.controlNS { namespaces = append(namespaces, replica) } - for _, ns := range namespaces { - if err := writeTokenSecret(ctx, r.cl, ns, ct.Secret, tok); err != nil { - return fmt.Errorf("write Secret %s/%s: %w", ns, ct.Secret, err) - } - fmt.Fprintf(r.out, " - Secret %s/%s: updated\n", ns, ct.Secret) - } + key := installerTokenKeys[ct.Caller] - hostProxy := false - if ct.Caller == "velocity" { - switch err := setKeyValueLine(r.linkProps, "service-token", "=", tok); { - case errors.Is(err, fs.ErrNotExist): - fmt.Fprintf(r.out, " - %s: not found; set service-token in your proxy's felis-link.properties to the value in Secret %s/%s and restart it\n", - r.linkProps, r.controlNS, ct.Secret) - case err != nil: - return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err) - default: - hostProxy = true - fmt.Fprintf(r.out, " - %s: service-token updated\n", r.linkProps) - } + fmt.Fprintf(r.out, "felis rotate-token %s: a new internal token for the %s caller\n", ct.Caller, ct.Caller) + secrets := make([]string, len(namespaces)) + for i, ns := range namespaces { + secrets[i] = "Secret " + ns + "/" + ct.Secret } - - if err := r.rollAPI(ctx); err != nil { - return fmt.Errorf("roll felis-api: %w", err) + writes := append([]string{r.secretsEnv + " (" + key + ")"}, secrets...) + hostProxy := ct.Caller == "velocity" && fileExists(r.linkProps) + if hostProxy { + writes = append(writes, r.linkProps+" (service-token)") } - fmt.Fprintln(r.out, " - felis-api: rolled out, accepting only the new token") - + fmt.Fprintf(r.out, " - writes %s\n", strings.Join(writes, ", ")) + fmt.Fprintf(r.out, " - %s\n", apiRestartNote) switch ct.Caller { case "velocity": if hostProxy { - if err := r.restartUnit(ctx, velocityUnit); err != nil { - return fmt.Errorf("restart %s: %w", velocityUnit, err) - } - fmt.Fprintf(r.out, " - %s: restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit) + fmt.Fprintf(r.out, " - the proxy re-reads its token from the file and keeps its players; if it has not within %s of felis-api's restart, it is restarted, which disconnects everyone online\n", r.reloadWait) + } else { + fmt.Fprintf(r.out, " - no proxy on this host (%s): set service-token in your proxy's felis-link.properties to the value in Secret %s/%s afterwards; it re-reads the file without a restart\n", + r.linkProps, r.controlNS, ct.Secret) } case "limbo": - if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), - client.MatchingLabels{v1alpha1.LabelServer: naming.SystemLoginServer}); err != nil { + fmt.Fprintln(r.out, " - the login gate's pod restarts: a player signing in at that moment reconnects") + case "build": + fmt.Fprintln(r.out, " - a build fetching its context at that moment fails and can be submitted again") + case "ops": + fmt.Fprintln(r.out, " - felis backup-now presents the new token on its next run") + } + if !r.confirm(ct.Caller, apply) { + return nil + } + + tok, err := r.newToken() + if err != nil { + return fmt.Errorf("generate a token: %w", err) + } + if err := r.record([2]string{key, tok}); err != nil { + return err + } + for _, ns := range namespaces { + if err := r.putSecret(ctx, ns, ct.Secret, map[string][]byte{naming.ServiceTokenSecretKey: []byte(tok)}); err != nil { + return err + } + } + + // The proxy's file changes before felis-api rolls, so the file never falls + // behind the Secret; the proxy's log is read from just before the write, + // since it may pick the new token up before the rollout ends. + since := r.now() + if hostProxy { + if err := setProxyToken(r.linkProps, tok); err != nil { + return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err) + } + fmt.Fprintf(r.out, " - %s: service-token updated\n", r.linkProps) + } + + if err := r.rollAPI(ctx); err != nil { + return err + } + + switch ct.Caller { + case "velocity": + if !hostProxy { + break + } + if r.proxyReloaded(ctx, since, tokenFingerprint(tok)) { + fmt.Fprintf(r.out, " - %s: took the new token from its properties; players stayed connected\n", velocityUnit) + break + } + if err := r.restartUnit(ctx, velocityUnit); err != nil { + return fmt.Errorf("restart %s: %w", velocityUnit, err) + } + fmt.Fprintf(r.out, " - %s: had not taken the new token within %s, restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit, r.reloadWait) + case "limbo": + // Only the operator's server pods carry its managed-by label; a backup or + // restore Job's pod carries the server label too, and app.kubernetes.io ones. + if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), client.MatchingLabels{ + v1alpha1.LabelServer: naming.SystemLoginServer, + v1alpha1.LabelManagedBy: operator.ManagedByValue, + }); err != nil { return fmt.Errorf("restart the login gate: %w", err) } fmt.Fprintln(r.out, " - login gate: pod restarted to read the new token") case "build": - fmt.Fprintln(r.out, " - builds: the next build Job reads the new token; one fetching its context right now fails and can be submitted again") + fmt.Fprintln(r.out, " - builds: the next build Job reads the new token") case "ops": fmt.Fprintln(r.out, " - felis backup-now reads the new token on its next run") } return nil } -// writeTokenSecret sets the token in a Secret, creating it when absent. -func writeTokenSecret(ctx context.Context, cl client.Client, namespace, name, token string) error { - var sec corev1.Secret - err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &sec) - if apierrors.IsNotFound(err) { - return cl.Create(ctx, &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)}, - }) +// proxyReloaded waits up to reloadWait for the host proxy to log that it +// reloaded the token with this fingerprint (plugins/shared FileToken). +func (r tokenRotator) proxyReloaded(ctx context.Context, since time.Time, fingerprint string) bool { + want := "(fingerprint " + fingerprint + ")" + deadline := r.now().Add(r.reloadWait) + for { + if strings.Contains(r.proxyLog(ctx, since), want) { + return true + } + if !r.now().Before(deadline) { + return false + } + select { + case <-ctx.Done(): + return false + case <-time.After(r.pollEvery): + } } +} + +// journalSince is the proxy unit's journal from the second since falls in. A +// journal that cannot be read reads as one without the line, which ends in the +// restart a rotation made before the proxy could reload. +func journalSince(ctx context.Context, since time.Time) string { + out, _ := exec.CommandContext(ctx, "journalctl", "-u", velocityUnit, "--since", "@"+strconv.FormatInt(since.Unix(), 10), + "-o", "cat", "--no-pager", "-q").Output() + return string(out) +} + +// setProxyToken writes the proxy's token into felis-link.properties and puts +// the file's modification time back. The proxy re-reads its token by itself, +// while `felis domain check` and `felis domain set` read a file newer than the +// proxy's start as config it has not loaded (the installer's +// install_if_changed keeps the time for the same reason). +func setProxyToken(path, token string) error { + info, err := os.Stat(path) if err != nil { return err } - if sec.Data == nil { - sec.Data = map[string][]byte{} + if err := setKeyValueLine(path, "service-token", "=", token); err != nil { + return err } - sec.Data[naming.ServiceTokenSecretKey] = []byte(token) - return cl.Update(ctx, &sec) + return os.Chtimes(path, time.Time{}, info.ModTime()) +} + +func (r tokenRotator) rotateRegistry(ctx context.Context, apply bool) error { + keys := make([]string, len(registrygate.Principals)) + for i, p := range registrygate.Principals { + keys[i] = installerRegistryKeys[p] + } + fmt.Fprintf(r.out, "felis rotate-token %s: new write tokens for the image registry's principals (%s)\n", kindRegistry, strings.Join(registrygate.Principals, ", ")) + fmt.Fprintf(r.out, " - writes %s (%s), Secret %s/%s, Secret %s/%s\n", r.secretsEnv, strings.Join(keys, ", "), + r.controlNS, naming.RegistryAuthSecretName, r.buildNS, naming.RegistryPushSecretName) + fmt.Fprintln(r.out, " - the registry restarts to load them: a build pushing its image at that moment fails and can be submitted again, and an image pull in that moment retries") + fmt.Fprintf(r.out, " - %s (it presents the prune token)\n", apiRestartNote) + if !r.confirm(kindRegistry, apply) { + return nil + } + + tokens := map[string][]byte{} + kv := make([][2]string, 0, len(registrygate.Principals)) + for _, p := range registrygate.Principals { + tok, err := r.newToken() + if err != nil { + return fmt.Errorf("generate a token: %w", err) + } + tokens[p] = []byte(tok) + kv = append(kv, [2]string{installerRegistryKeys[p], tok}) + } + if err := r.record(kv...); err != nil { + return err + } + if err := r.putSecret(ctx, r.controlNS, naming.RegistryAuthSecretName, tokens); err != nil { + return err + } + if err := r.putSecret(ctx, r.buildNS, naming.RegistryPushSecretName, map[string][]byte{ + naming.RegistryPushUsernameKey: []byte(registrygate.PrincipalBuild), + naming.RegistryPushPasswordKey: tokens[registrygate.PrincipalBuild], + }); err != nil { + return err + } + if err := r.rollout(ctx, registryDeployment); err != nil { + return fmt.Errorf("restart the registry: %w", err) + } + fmt.Fprintln(r.out, " - registry: restarted on the new tokens") + return r.rollAPI(ctx) +} + +func (r tokenRotator) rotateForwarding(ctx context.Context, apply bool) error { + restart, held, err := r.gamePods(ctx) + if err != nil { + return err + } + hostProxy := fileExists(r.forwardingFile) + fmt.Fprintf(r.out, "felis rotate-token %s: a new Velocity forwarding secret, the key a server checks each player's identity with\n", kindForwarding) + secrets := []string{} + for _, ns := range r.withMinecraft() { + secrets = append(secrets, "Secret "+ns+"/"+naming.ForwardingSecretName) + } + writes := append([]string{r.secretsEnv + " (" + installerForwardingKey + ")"}, secrets...) + if hostProxy { + writes = append(writes, r.forwardingFile) + } + fmt.Fprintf(r.out, " - writes %s\n", strings.Join(writes, ", ")) + fmt.Fprintf(r.out, " - every running server restarts to read it (%d now), saving its world on the way down, and the proxy restarts: everyone online is disconnected and can rejoin once their server is back\n", len(restart)) + if len(held) > 0 { + fmt.Fprintf(r.out, " - left running, because a backup, restore or file write holds its world: %s. Players cannot join it until it restarts: stop and start it from the panel once that finishes\n", strings.Join(held, ", ")) + } + if !hostProxy { + fmt.Fprintf(r.out, " - no proxy on this host (%s): put the value in Secret %s/%s into your proxy's forwarding secret file and restart it\n", + r.forwardingFile, r.controlNS, naming.ForwardingSecretName) + } + if !r.confirm(kindForwarding, apply) { + return nil + } + + tok, err := r.newToken() + if err != nil { + return fmt.Errorf("generate a secret: %w", err) + } + if err := r.record([2]string{installerForwardingKey, tok}); err != nil { + return err + } + if hostProxy { + info, err := os.Stat(r.forwardingFile) + if err != nil { + return err + } + if err := replaceFileKeepingMode(r.forwardingFile, info, []byte(tok)); err != nil { + return fmt.Errorf("write %s: %w", r.forwardingFile, err) + } + fmt.Fprintf(r.out, " - %s: updated\n", r.forwardingFile) + } + for _, ns := range r.withMinecraft() { + if err := r.putSecret(ctx, ns, naming.ForwardingSecretName, map[string][]byte{naming.ForwardingSecretKey: []byte(tok)}); err != nil { + return err + } + } + // The servers go first: their pods read the Secret as they are recreated, + // and a player who rejoins through the restarted proxy meets a server on the + // new secret, or one still starting. + for i := range restart { + p := &restart[i] + if err := r.cl.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) { + return fmt.Errorf("restart %s: %w", p.Labels[v1alpha1.LabelServer], err) + } + } + fmt.Fprintf(r.out, " - servers: %d restarting on the new secret\n", len(restart)) + if hostProxy { + if err := r.restartUnit(ctx, velocityUnit); err != nil { + return fmt.Errorf("restart %s: %w", velocityUnit, err) + } + fmt.Fprintf(r.out, " - %s: restarted on the new secret\n", velocityUnit) + } + return nil +} + +// gamePods lists the running game server pods: those a rotation may restart, +// and the servers left alone because a backup, restore or file write holds +// their world (internal/maintenance), which a restart in the middle of would +// break. +func (r tokenRotator) gamePods(ctx context.Context) ([]corev1.Pod, []string, error) { + var pods corev1.PodList + // The operator's managed-by label is on its server pods alone (see rotateCaller). + if err := r.cl.List(ctx, &pods, client.InNamespace(r.minecraftNS), client.MatchingLabels{v1alpha1.LabelManagedBy: operator.ManagedByValue}); err != nil { + return nil, nil, fmt.Errorf("list the servers' pods: %w", err) + } + var jobs batchv1.JobList + if err := r.cl.List(ctx, &jobs, client.InNamespace(r.minecraftNS)); err != nil { + return nil, nil, fmt.Errorf("list the maintenance Jobs: %w", err) + } + var restart []corev1.Pod + var held []string + for _, p := range pods.Items { + if p.DeletionTimestamp != nil { + continue + } + server := p.Labels[v1alpha1.LabelServer] + var ms v1alpha1.MinecraftServer + if err := r.cl.Get(ctx, client.ObjectKey{Namespace: r.minecraftNS, Name: server}, &ms); client.IgnoreNotFound(err) != nil { + return nil, nil, fmt.Errorf("read server %s: %w", server, err) + } + if kind, ok := maintenance.Holder(server, ms.Annotations, jobs.Items, r.now()); ok { + held = append(held, server+" ("+kind+")") + continue + } + restart = append(restart, p) + } + return restart, held, nil +} + +func (r tokenRotator) rotateDB(ctx context.Context, apply bool) error { + cfg, err := config.Load(r.hostTOML) + if err != nil { + return err + } + if cfg.Database.Deployment == "" { + return fmt.Errorf("[database] deployment is unset in %s, so the database is not the installer's felis-postgres: change the role's password where it runs, then in [database] url of each config copy", r.hostTOML) + } + u, err := neturl.Parse(cfg.Database.URL) + if err != nil || u.User == nil || u.User.Username() == "" { + return fmt.Errorf("[database] url in %s names no role", r.hostTOML) + } + role := u.User.Username() + targets, err := tomlTargetsOf(r.hostTOML, r.podTOML, r.defaultTOML) + if err != nil { + return err + } + paths := make([]string, len(targets)) + for i, t := range targets { + paths[i] = t.path + } + secrets := []string{} + for _, ns := range r.withMinecraft() { + secrets = append(secrets, ns+"/"+platform.ConfigSecretName) + } + fmt.Fprintf(r.out, "felis rotate-token %s: a new password for the database role %q\n", kindDB, role) + fmt.Fprintf(r.out, " - writes %s (%s), the role in %s, [database] url in %s, Secret %s\n", + r.secretsEnv, installerDBKey, cfg.Database.Deployment, strings.Join(paths, " and "), strings.Join(secrets, " and ")) + fmt.Fprintf(r.out, " - %s\n", apiRestartNote) + fmt.Fprintln(r.out, " - a backup, restore or file Job that connects in the seconds between the password change and felis-api's restart fails and can be run again; the host's timers read the new config on their next run") + if !r.confirm(kindDB, apply) { + return nil + } + + password, err := r.newToken() + if err != nil { + return fmt.Errorf("generate a password: %w", err) + } + // Every config copy is edited in memory first, so one this cannot edit stops + // the rotation before the role's password changes. + edited := make([][]byte, len(targets)) + var hostURL string + var podConfig []byte + for i, t := range targets { + raw, err := os.ReadFile(t.real) + if err != nil { + return err + } + var doc struct { + Database struct { + URL string `toml:"url"` + } `toml:"database"` + } + if _, err := toml.Decode(string(raw), &doc); err != nil { + return fmt.Errorf("%s: %w", t.path, err) + } + next, err := withPassword(doc.Database.URL, password) + if err != nil { + return fmt.Errorf("%s: %w", t.path, err) + } + if edited[i], err = editTOMLStrings(raw, []tomlStringEdit{{"database", "url", next}}); err != nil { + return fmt.Errorf("%s: %w; set the password in its [database] url by hand", t.path, err) + } + switch t.path { + case r.hostTOML: + hostURL = next + case r.podTOML: + podConfig = edited[i] + } + } + if podConfig == nil { + return fmt.Errorf("%s resolves to the same file as %s; the pods reach the database at another address and need a copy of their own", r.podTOML, r.hostTOML) + } + + if err := r.record([2]string{installerDBKey, password}); err != nil { + return err + } + salt := make([]byte, 16) + if _, err := rand.Read(salt); err != nil { + return err + } + verifier, err := scramVerifier(password, salt, scramIterations) + if err != nil { + return err + } + if err := r.alterRole(ctx, cfg.Database.Deployment, role, verifier); err != nil { + return fmt.Errorf("set the role's password: %w", err) + } + if err := r.verifyDB(ctx, hostURL); err != nil { + return fmt.Errorf("the database does not accept the new password (%v); the config copies still hold the old one: run the installer again (sudo bash deploy/bootstrap.sh), which sets the password in %s everywhere", err, r.secretsEnv) + } + fmt.Fprintf(r.out, " - role %s: password changed, and the database accepts it\n", role) + for i, t := range targets { + info, err := os.Stat(t.real) + if err != nil { + return err + } + if err := replaceFileKeepingMode(t.real, info, edited[i]); err != nil { + return fmt.Errorf("write %s: %w", t.path, err) + } + fmt.Fprintf(r.out, " - %s: [database] url updated\n", t.path) + } + for _, ns := range r.withMinecraft() { + if err := r.putSecret(ctx, ns, platform.ConfigSecretName, map[string][]byte{platform.ConfigSecretKey: podConfig}); err != nil { + return err + } + } + return r.rollAPI(ctx) +} + +// withPassword is a database URL with its password replaced. +func withPassword(raw, password string) (string, error) { + u, err := neturl.Parse(raw) + if err != nil || u.User == nil || u.User.Username() == "" { + return "", errors.New("its [database] url names no role") + } + u.User = neturl.UserPassword(u.User.Username(), password) + return u.String(), nil +} + +// scramIterations is PostgreSQL's default scram_iterations. +const scramIterations = 4096 + +// scramVerifier is the SCRAM-SHA-256 verifier PostgreSQL stores for a password +// (RFC 5802 and RFC 7677, in the form libpq's PQencryptPasswordConn makes). +// ALTER ROLE stores a verifier as it is given, so the password itself never +// reaches the server, where a failing statement is logged with its text. +func scramVerifier(password string, salt []byte, iterations int) (string, error) { + salted, err := pbkdf2.Key(sha256.New, password, salt, iterations, sha256.Size) + if err != nil { + return "", err + } + mac := func(msg string) []byte { + h := hmac.New(sha256.New, salted) + h.Write([]byte(msg)) + return h.Sum(nil) + } + stored := sha256.Sum256(mac("Client Key")) + b64 := base64.StdEncoding.EncodeToString + return fmt.Sprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, b64(salt), b64(stored[:]), b64(mac("Server Key"))), nil +} + +// alterRoleInPod runs ALTER ROLE as the superuser inside the database's +// container, over its socket, with the statement on stdin. +func alterRoleInPod(ctx context.Context, deployment, role, verifier string) error { + ns, name, _ := strings.Cut(deployment, "/") + return kubectlWithInput(ctx, []byte(alterRoleSQL(role, verifier)), "-n", ns, "exec", "-i", "deploy/"+name, "-c", platform.PostgresContainer, "--", + "psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres") +} + +// alterRoleSQL sets role's password to a SCRAM verifier, which holds no quote. +func alterRoleSQL(role, verifier string) string { + return "ALTER ROLE \"" + strings.ReplaceAll(role, `"`, `""`) + "\" WITH PASSWORD '" + verifier + "';\n" } // setKeyValueLine rewrites the `keyvalue` line of a flat key/value file diff --git a/cmd/felis/rotatetoken_test.go b/cmd/felis/rotatetoken_test.go index 5adda38..85e0e43 100644 --- a/cmd/felis/rotatetoken_test.go +++ b/cmd/felis/rotatetoken_test.go @@ -3,13 +3,20 @@ package main import ( "bytes" "context" + "encoding/base64" "errors" + "fmt" "os" "path/filepath" "regexp" "strings" "testing" + "time" + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/maintenance" + + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -22,52 +29,111 @@ type rotationRig struct { out *bytes.Buffer events []string dir string + clock time.Time } func tokenSecret(ns, name, val string) *corev1.Secret { + return keySecret(ns, name, "token", val) +} + +func keySecret(ns, name, key, val string) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name}, - Data: map[string][]byte{"token": []byte(val)}, + Data: map[string][]byte{key: []byte(val)}, } } +// serverPod is a game server's pod as the operator labels it. func serverPod(ns, name, server string) *corev1.Pod { return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name, - Labels: map[string]string{"felis.lolicon.best/server": server}}} + Labels: map[string]string{ + "felis.lolicon.best/server": server, + "felis.lolicon.best/managed-by": "felis-operator", + "felis.lolicon.best/component": "server", + }}} +} + +// backupPod is a backup Job's pod as internal/backupjob labels it: it carries +// the server's label too, and no rotation may take it for the server's own. +func backupPod(ns, name, server string) *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name, + Labels: map[string]string{ + "felis.lolicon.best/server": server, + "app.kubernetes.io/managed-by": "felis-backup", + "app.kubernetes.io/component": "world-backup", + }}} } func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig { t.Helper() - rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir()} + rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir(), clock: time.Unix(1_800_000_000, 0)} rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build() rig.r = tokenRotator{ - cl: rig.cl, - controlNS: "felis", - minecraftNS: "minecraft", - buildNS: "felis-build", - secretsEnv: filepath.Join(rig.dir, "secrets.env"), - linkProps: filepath.Join(rig.dir, "felis-link.properties"), - newToken: func() (string, error) { return "NEWTOKEN", nil }, - rollAPI: func(context.Context) error { - rig.events = append(rig.events, "roll-api") + cl: rig.cl, + controlNS: "felis", + minecraftNS: "minecraft", + buildNS: "felis-build", + secretsEnv: filepath.Join(rig.dir, "secrets.env"), + linkProps: filepath.Join(rig.dir, "felis-link.properties"), + forwardingFile: filepath.Join(rig.dir, "forwarding.secret"), + hostTOML: filepath.Join(rig.dir, "felis.host.toml"), + podTOML: filepath.Join(rig.dir, "felis.pod.toml"), + defaultTOML: filepath.Join(rig.dir, "felis.toml"), + newToken: func() (string, error) { return "NEWTOKEN", nil }, + rollout: func(_ context.Context, deployment string) error { + rig.events = append(rig.events, "roll "+deployment) return nil }, restartUnit: func(_ context.Context, unit string) error { rig.events = append(rig.events, "restart "+unit) return nil }, - out: rig.out, + proxyLog: func(context.Context, time.Time) string { return "" }, + alterRole: func(context.Context, string, string, string) error { + rig.events = append(rig.events, "alter-role") + return nil + }, + verifyDB: func(context.Context, string) error { + rig.events = append(rig.events, "verify-db") + return nil + }, + // Every look at the clock moves it a second on, so a wait measured with it + // ends after a known number of looks. + now: func() time.Time { + rig.clock = rig.clock.Add(time.Second) + return rig.clock + }, + reloadWait: 5 * time.Second, + out: rig.out, } return rig } -func (rig *rotationRig) secret(t *testing.T, ns, name string) string { +func (rig *rotationRig) secretKey(t *testing.T, ns, name, key string) string { t.Helper() var s corev1.Secret if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil { return "" } - return string(s.Data["token"]) + return string(s.Data[key]) +} + +func (rig *rotationRig) secret(t *testing.T, ns, name string) string { + t.Helper() + return rig.secretKey(t, ns, name, "token") +} + +func (rig *rotationRig) pods(t *testing.T) string { + t.Helper() + var pods corev1.PodList + if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil { + t.Fatal(err) + } + names := make([]string, len(pods.Items)) + for i, p := range pods.Items { + names[i] = p.Name + } + return strings.Join(names, ",") } func writeTestFile(t *testing.T, path, body string, mode os.FileMode) { @@ -80,6 +146,15 @@ func writeTestFile(t *testing.T, path, body string, mode os.FileMode) { } } +func readTestFile(t *testing.T, path string) string { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + func TestRotateLimboToken(t *testing.T) { rig := newRotationRig(t, tokenSecret("felis", "felis-limbo-token", "old"), @@ -87,14 +162,15 @@ func TestRotateLimboToken(t *testing.T) { tokenSecret("felis", "felis-service-token", "proxy"), serverPod("minecraft", "login-0", "login"), serverPod("minecraft", "survival-0", "survival"), + backupPod("minecraft", "login-backup-x", "login"), ) writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600) // felis-api must roll only after both copies hold the new value, and the login // pod must still be there then: restarting it earlier would have it present // the new token to an api that does not know it yet. - rig.r.rollAPI = func(context.Context) error { - rig.events = append(rig.events, "roll-api") + rig.r.rollout = func(_ context.Context, deployment string) error { + rig.events = append(rig.events, "roll "+deployment) if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" { t.Errorf("api rolled while the control Secret held %q", got) } @@ -107,13 +183,12 @@ func TestRotateLimboToken(t *testing.T) { } return nil } - if err := rig.r.rotate(context.Background(), "limbo"); err != nil { + if err := rig.r.rotate(context.Background(), "limbo", true); err != nil { t.Fatal(err) } - raw, _ := os.ReadFile(rig.r.secretsEnv) - if string(raw) != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" { - t.Errorf("secrets.env = %q", raw) + if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" { + t.Errorf("secrets.env = %q", got) } if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 { t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm()) @@ -121,14 +196,12 @@ func TestRotateLimboToken(t *testing.T) { if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" { t.Errorf("the proxy's token changed to %q", got) } - var pods corev1.PodList - if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil { - t.Fatal(err) + // The login pod restarted; a user server and the backup Job's pod, which + // carries the login server's label too, are untouched. + if got := rig.pods(t); got != "login-backup-x,survival-0" { + t.Errorf("pods left = %s, want login-backup-x,survival-0", got) } - if len(pods.Items) != 1 || pods.Items[0].Name != "survival-0" { - t.Errorf("pods left = %v, want only survival-0 (the login pod restarted, user servers untouched)", pods.Items) - } - if strings.Join(rig.events, ",") != "roll-api" { + if strings.Join(rig.events, ",") != "roll felis-api" { t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events) } if strings.Contains(rig.out.String(), "NEWTOKEN") { @@ -136,21 +209,56 @@ func TestRotateLimboToken(t *testing.T) { } } -func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) { +const testLinkProps = "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n" + +func reloadLine(token string) string { + return "[12:00:00 INFO] [felis-link]: Felis: service-token reloaded from /x/felis-link.properties (fingerprint " + tokenFingerprint(token) + ")\n" +} + +// The host proxy re-reads its token: the rotation waits for it to say so and +// leaves it running. +func TestRotateVelocityTokenReloadsTheHostProxy(t *testing.T) { rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old")) writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600) - writeTestFile(t, rig.r.linkProps, "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n", 0o640) - - if err := rig.r.rotate(context.Background(), "velocity"); err != nil { + writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640) + written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + if err := os.Chtimes(rig.r.linkProps, written, written); err != nil { t.Fatal(err) } - raw, _ := os.ReadFile(rig.r.linkProps) - if string(raw) != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" { - t.Errorf("felis-link.properties = %q", raw) + + // The proxy logs its reload on its next call to felis-api, which may be + // while the api rolls: the log must be read from before that. + var reloadedAt time.Time + rig.r.rollout = func(_ context.Context, deployment string) error { + rig.events = append(rig.events, "roll "+deployment) + if got := readTestFile(t, rig.r.linkProps); !strings.Contains(got, "service-token=NEWTOKEN\n") { + t.Errorf("api rolled before the proxy's file held the new token: %q", got) + } + reloadedAt = rig.clock + return nil } - if info, _ := os.Stat(rig.r.linkProps); info.Mode().Perm() != 0o640 { + looks := 0 + rig.r.proxyLog = func(_ context.Context, since time.Time) string { + looks++ + log := reloadLine("old") // an earlier rotation's + if looks >= 3 && !since.After(reloadedAt) { + log += reloadLine("NEWTOKEN") + } + return log + } + if err := rig.r.rotate(context.Background(), "velocity", true); err != nil { + t.Fatal(err) + } + if got := readTestFile(t, rig.r.linkProps); got != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" { + t.Errorf("felis-link.properties = %q", got) + } + info, _ := os.Stat(rig.r.linkProps) + if info.Mode().Perm() != 0o640 { t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm()) } + if !info.ModTime().Equal(written) { + t.Errorf("properties mtime = %v, want it kept at %v (felis domain check would read the proxy as stale)", info.ModTime(), written) + } if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" { t.Errorf("control Secret = %q, want NEWTOKEN", got) } @@ -158,9 +266,49 @@ func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) { if got := rig.secret(t, "minecraft", "felis-service-token"); got != "" { t.Errorf("rotation copied the proxy token into minecraft (%q)", got) } - if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" { + if strings.Join(rig.events, ",") != "roll felis-api" { + t.Errorf("events = %v, want the api roll and no proxy restart", rig.events) + } + if looks != 3 { + t.Errorf("the log was read %d times, want 3 (until the line appeared)", looks) + } + if out := rig.out.String(); !strings.Contains(out, "felis-velocity: took the new token from its properties; players stayed connected") { + t.Errorf("output does not say the players stayed: %s", out) + } +} + +// A proxy that never logs the new fingerprint (an older plugin, a stuck proxy) +// is restarted once the wait is over. +func TestRotateVelocityTokenRestartsAProxyThatDoesNotReload(t *testing.T) { + rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old")) + writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600) + writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640) + looks := 0 + rig.r.proxyLog = func(context.Context, time.Time) string { + looks++ + return reloadLine("old") + } + if err := rig.r.rotate(context.Background(), "velocity", true); err != nil { + t.Fatal(err) + } + if strings.Join(rig.events, ",") != "roll felis-api,restart felis-velocity" { t.Errorf("events = %v, want the api roll then the proxy restart", rig.events) } + // reloadWait is 5s and each look moves the clock a second. + if looks != 5 { + t.Errorf("the log was read %d times, want 5", looks) + } + if out := rig.out.String(); !strings.Contains(out, "felis-velocity: had not taken the new token within 5s, restarted") { + t.Errorf("output does not explain the restart: %s", out) + } +} + +// The proxy and the Java plugin name a token by the same fingerprint +// (plugins/shared LinkConfigLoaderTest fileTokenFollowsTheFile). +func TestTokenFingerprintMatchesThePlugin(t *testing.T) { + if got := tokenFingerprint("new-token"); got != "348e9df2a42b" { + t.Errorf("tokenFingerprint(new-token) = %q, want 348e9df2a42b", got) + } } // An external proxy has no felis-link.properties here: the Secret still rotates, @@ -168,13 +316,13 @@ func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) { // without printing it. func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) { rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old")) - if err := rig.r.rotate(context.Background(), "velocity"); err != nil { + if err := rig.r.rotate(context.Background(), "velocity", true); err != nil { t.Fatal(err) } if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" { t.Errorf("control Secret = %q, want NEWTOKEN", got) } - if strings.Join(rig.events, ",") != "roll-api" { + if strings.Join(rig.events, ",") != "roll felis-api" { t.Errorf("events = %v, want no proxy restart", rig.events) } out := rig.out.String() @@ -187,7 +335,7 @@ func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) { rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old")) // An install from before per-caller tokens has no BUILD_TOKEN line yet. writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600) - if err := rig.r.rotate(context.Background(), "build"); err != nil { + if err := rig.r.rotate(context.Background(), "build", true); err != nil { t.Fatal(err) } if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" { @@ -196,9 +344,8 @@ func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) { if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" { t.Errorf("control Secret = %q, want NEWTOKEN", got) } - raw, _ := os.ReadFile(rig.r.secretsEnv) - if string(raw) != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" { - t.Errorf("secrets.env = %q", raw) + if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" { + t.Errorf("secrets.env = %q", got) } } @@ -209,31 +356,453 @@ func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) { tokenSecret("felis", "felis-limbo-token", "old"), serverPod("minecraft", "login-0", "login"), ) - rig.r.rollAPI = func(context.Context) error { return errors.New("rollout timed out") } - err := rig.r.rotate(context.Background(), "limbo") + rig.r.rollout = func(context.Context, string) error { return errors.New("rollout timed out") } + err := rig.r.rotate(context.Background(), "limbo", true) if err == nil || !strings.Contains(err.Error(), "rollout timed out") { t.Fatalf("err = %v, want the rollout failure", err) } - var pod corev1.Pod - if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil { - t.Error("the login pod was restarted although the api never rolled") + if got := rig.pods(t); got != "login-0" { + t.Errorf("pods left = %s: the login pod was restarted although the api never rolled", got) } } -func TestRotateRefusesAnUnknownCaller(t *testing.T) { +func TestRotateRefusesAnUnknownCredential(t *testing.T) { rig := newRotationRig(t) writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600) - if err := rig.r.rotate(context.Background(), "admin"); err == nil { - t.Fatal("rotated a token for an unknown caller") + if err := rig.r.rotate(context.Background(), "admin", true); err == nil { + t.Fatal("rotated an unknown credential") } - if raw, _ := os.ReadFile(rig.r.secretsEnv); string(raw) != "SERVICE_TOKEN=proxy\n" { - t.Errorf("secrets.env = %q, want it untouched", raw) + if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\n" { + t.Errorf("secrets.env = %q, want it untouched", got) } if len(rig.events) != 0 { t.Errorf("events = %v, want nothing touched", rig.events) } } +const ( + testHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:oldpw@127.0.0.1:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n" + testPodTOML = "[database]\n# The Service address.\nurl = \"postgres://felis:oldpw@felis-postgres.felis.svc:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n" +) + +// installedRig is a host as the installer leaves it, with every credential in +// place, for the plan test. +func installedRig(t *testing.T) *rotationRig { + t.Helper() + rig := newRotationRig(t, + tokenSecret("felis", "felis-service-token", "old"), + tokenSecret("felis", "felis-limbo-token", "old"), + tokenSecret("minecraft", "felis-limbo-token", "old"), + tokenSecret("felis", "felis-build-token", "old"), + tokenSecret("felis-build", "felis-build-token", "old"), + tokenSecret("felis", "felis-ops-token", "old"), + keySecret("felis", "felis-registry-auth", "platform", "old"), + keySecret("felis-build", "felis-registry-push", "password", "old"), + keySecret("felis", "felis-forwarding-secret", "secret", "old"), + keySecret("minecraft", "felis-forwarding-secret", "secret", "old"), + keySecret("felis", "felis-config", "felis.toml", testPodTOML), + keySecret("minecraft", "felis-config", "felis.toml", testPodTOML), + serverPod("minecraft", "login-0", "login"), + serverPod("minecraft", "survival-0", "survival"), + ) + writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=old\n", 0o600) + writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640) + writeTestFile(t, rig.r.forwardingFile, "old", 0o640) + writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600) + writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600) + return rig +} + +// Without -yes every rotation prints its plan and changes nothing. +func TestRotatePlanChangesNothing(t *testing.T) { + for _, kind := range rotationKinds() { + t.Run(kind, func(t *testing.T) { + rig := installedRig(t) + files := map[string]string{} + for _, p := range []string{rig.r.secretsEnv, rig.r.linkProps, rig.r.forwardingFile, rig.r.hostTOML, rig.r.podTOML} { + files[p] = readTestFile(t, p) + } + rig.r.newToken = func() (string, error) { + t.Error("a plan generated a token") + return "NEWTOKEN", nil + } + if err := rig.r.rotate(context.Background(), kind, false); err != nil { + t.Fatal(err) + } + for p, before := range files { + if got := readTestFile(t, p); got != before { + t.Errorf("%s changed to %q", filepath.Base(p), got) + } + } + for _, s := range [][3]string{ + {"felis", "felis-service-token", "token"}, {"felis", "felis-limbo-token", "token"}, + {"felis-build", "felis-build-token", "token"}, {"felis", "felis-ops-token", "token"}, + {"felis", "felis-registry-auth", "platform"}, {"felis-build", "felis-registry-push", "password"}, + {"minecraft", "felis-forwarding-secret", "secret"}, + } { + if got := rig.secretKey(t, s[0], s[1], s[2]); got != "old" { + t.Errorf("Secret %s/%s changed to %q", s[0], s[1], got) + } + } + if got := rig.secretKey(t, "minecraft", "felis-config", "felis.toml"); got != testPodTOML { + t.Errorf("felis-config changed to %q", got) + } + if len(rig.events) != 0 { + t.Errorf("events = %v, want none", rig.events) + } + if got := rig.pods(t); got != "login-0,survival-0" { + t.Errorf("pods left = %s, want all", got) + } + out := rig.out.String() + if !strings.HasSuffix(out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes "+kind+"\n") { + t.Errorf("the plan does not end with how to go ahead: %s", out) + } + // Each plan names the interruption it causes. + want := apiRestartNote + if kind == kindForwarding { + want = "everyone online is disconnected" + } + if !strings.Contains(out, want) { + t.Errorf("the plan does not say %q: %s", want, out) + } + }) + } +} + +func TestRotateRegistryTokens(t *testing.T) { + rig := newRotationRig(t, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: "felis-registry-auth"}, + Data: map[string][]byte{"platform": []byte("a"), "build": []byte("b"), "prune": []byte("c")}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis-build", Name: "felis-registry-push"}, + Data: map[string][]byte{"username": []byte("build"), "password": []byte("b")}}, + ) + writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=a\nREGISTRY_BUILD_TOKEN=b\nREGISTRY_PRUNE_TOKEN=c\n", 0o600) + n := 0 + rig.r.newToken = func() (string, error) { + n++ + return fmt.Sprintf("NEWTOKEN%d", n), nil + } + // The registry reads its tokens as it starts: it restarts after the Secret + // holds them, and felis-api (the prune token) after that. + rig.r.rollout = func(_ context.Context, deployment string) error { + rig.events = append(rig.events, "roll "+deployment) + if got := rig.secretKey(t, "felis", "felis-registry-auth", "prune"); got != "NEWTOKEN3" { + t.Errorf("%s rolled while the prune token was %q", deployment, got) + } + return nil + } + if err := rig.r.rotate(context.Background(), kindRegistry, true); err != nil { + t.Fatal(err) + } + if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=NEWTOKEN1\nREGISTRY_BUILD_TOKEN=NEWTOKEN2\nREGISTRY_PRUNE_TOKEN=NEWTOKEN3\n" { + t.Errorf("secrets.env = %q", got) + } + for key, want := range map[string]string{"platform": "NEWTOKEN1", "build": "NEWTOKEN2", "prune": "NEWTOKEN3"} { + if got := rig.secretKey(t, "felis", "felis-registry-auth", key); got != want { + t.Errorf("felis-registry-auth %s = %q, want %s", key, got, want) + } + } + if got := rig.secretKey(t, "felis-build", "felis-registry-push", "password"); got != "NEWTOKEN2" { + t.Errorf("the build Jobs' push password = %q, want the build token", got) + } + if got := rig.secretKey(t, "felis-build", "felis-registry-push", "username"); got != "build" { + t.Errorf("the build Jobs' push username = %q, want build", got) + } + if strings.Join(rig.events, ",") != "roll registry,roll felis-api" { + t.Errorf("events = %v, want the registry then felis-api", rig.events) + } + if strings.Contains(rig.out.String(), "NEWTOKEN") { + t.Errorf("a new token was printed: %s", rig.out.String()) + } +} + +func TestRotateForwardingSecret(t *testing.T) { + lock := time.Unix(1_800_000_000, 0) + rig := newRotationRig(t, + keySecret("felis", "felis-forwarding-secret", "secret", "old"), + keySecret("minecraft", "felis-forwarding-secret", "secret", "old"), + serverPod("minecraft", "survival-0", "survival"), + serverPod("minecraft", "lobby-0", "lobby"), + // creative is being backed up: a running Job holds its world. + serverPod("minecraft", "creative-0", "creative"), + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "creative-backup", + Labels: map[string]string{maintenance.LabelServer: "creative", maintenance.LabelManagedBy: "felis-backup"}}}, + // survival's last backup is over; its finished pod stays until the Job's TTL. + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival-backup", + Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: "felis-backup"}}, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}}, + backupPod("minecraft", "survival-backup-x", "survival"), + // A pod already on its way out is neither counted nor deleted again. + func() *corev1.Pod { + p := serverPod("minecraft", "lobby-old", "lobby") + p.DeletionTimestamp = &metav1.Time{Time: lock} + p.Finalizers = []string{"felis.lolicon.best/test"} + return p + }(), + // skyblock's restore has just been admitted, its Job not created yet. + serverPod("minecraft", "skyblock-0", "skyblock"), + &v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "skyblock", + Annotations: map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lock)}}}, + &v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival"}}, + ) + writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nFORWARDING_SECRET=old\n", 0o600) + writeTestFile(t, rig.r.forwardingFile, "old", 0o640) + // The proxy restarts after the servers were told to: a player who rejoins + // must not meet a server still on the old secret. + rig.r.restartUnit = func(_ context.Context, unit string) error { + rig.events = append(rig.events, "restart "+unit) + if got := rig.pods(t); strings.Contains(got, "survival-0") { + t.Errorf("the proxy restarted before the servers (pods %s)", got) + } + if got := readTestFile(t, rig.r.forwardingFile); got != "NEWTOKEN" { + t.Errorf("the proxy restarted on forwarding.secret %q", got) + } + return nil + } + if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil { + t.Fatal(err) + } + if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nFORWARDING_SECRET=NEWTOKEN\n" { + t.Errorf("secrets.env = %q", got) + } + if info, _ := os.Stat(rig.r.forwardingFile); info.Mode().Perm() != 0o640 { + t.Errorf("forwarding.secret mode = %v, want 0640", info.Mode().Perm()) + } + for _, ns := range []string{"felis", "minecraft"} { + if got := rig.secretKey(t, ns, "felis-forwarding-secret", "secret"); got != "NEWTOKEN" { + t.Errorf("Secret %s/felis-forwarding-secret = %q, want NEWTOKEN", ns, got) + } + } + if got := rig.pods(t); got != "creative-0,lobby-old,skyblock-0,survival-backup-x" { + t.Errorf("pods left = %s, want the held servers, the terminating pod and the backup's pod", got) + } + if strings.Join(rig.events, ",") != "restart felis-velocity" { + t.Errorf("events = %v, want only the proxy restart (felis-api does not hold this secret)", rig.events) + } + out := rig.out.String() + for _, want := range []string{ + "every running server restarts to read it (2 now)", + "left running, because a backup, restore or file write holds its world: creative (backup), skyblock (restore).", + " - servers: 2 restarting on the new secret\n", + } { + if !strings.Contains(out, want) { + t.Errorf("output lacks %q: %s", want, out) + } + } + if strings.Contains(out, "NEWTOKEN") { + t.Errorf("the new secret was printed: %s", out) + } +} + +func TestRotateForwardingSecretForAnExternalProxy(t *testing.T) { + rig := newRotationRig(t, keySecret("felis", "felis-forwarding-secret", "secret", "old")) + if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil { + t.Fatal(err) + } + if got := rig.secretKey(t, "felis", "felis-forwarding-secret", "secret"); got != "NEWTOKEN" { + t.Errorf("control Secret = %q, want NEWTOKEN", got) + } + if len(rig.events) != 0 { + t.Errorf("events = %v, want no proxy restart on this host", rig.events) + } + if out := rig.out.String(); !strings.Contains(out, "put the value in Secret felis/felis-forwarding-secret into your proxy's forwarding secret file") { + t.Errorf("output does not say where the value is: %s", out) + } +} + +// dbRig is an installed host for the database rotation: felis.toml links to the +// host copy, as the installer makes it. +func dbRig(t *testing.T) *rotationRig { + t.Helper() + rig := newRotationRig(t, + keySecret("felis", "felis-config", "felis.toml", testPodTOML), + keySecret("minecraft", "felis-config", "felis.toml", testPodTOML), + ) + writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n", 0o600) + writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600) + writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600) + if err := os.Symlink("felis.host.toml", rig.r.defaultTOML); err != nil { + t.Fatal(err) + } + return rig +} + +const ( + wantHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:NEWTOKEN@127.0.0.1:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n" + wantPodTOML = "[database]\n# The Service address.\nurl = \"postgres://felis:NEWTOKEN@felis-postgres.felis.svc:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n" +) + +func TestRotateDatabasePassword(t *testing.T) { + rig := dbRig(t) + rig.r.alterRole = func(_ context.Context, deployment, role, verifier string) error { + rig.events = append(rig.events, "alter-role") + if deployment != "felis/felis-postgres" || role != "felis" { + t.Errorf("alterRole(%q, %q), want felis/felis-postgres and felis", deployment, role) + } + if !strings.Contains(readTestFile(t, rig.r.secretsEnv), "DB_PASSWORD=NEWTOKEN\n") { + t.Error("the role changed before secrets.env recorded the new password") + } + // The verifier is the new password's, under the salt it carries. + m := regexp.MustCompile(`^SCRAM-SHA-256\$4096:([^$]+)\$`).FindStringSubmatch(verifier) + if m == nil { + t.Fatalf("verifier %q is not a SCRAM-SHA-256 verifier", verifier) + } + salt, err := base64.StdEncoding.DecodeString(m[1]) + if err != nil || len(salt) != 16 { + t.Fatalf("verifier salt %q: %v", m[1], err) + } + if want, _ := scramVerifier("NEWTOKEN", salt, 4096); verifier != want { + t.Errorf("verifier = %q, want %q", verifier, want) + } + return nil + } + // The configs change only once the database accepts the new password. + rig.r.verifyDB = func(_ context.Context, url string) error { + rig.events = append(rig.events, "verify-db") + if url != "postgres://felis:NEWTOKEN@127.0.0.1:30432/felis?sslmode=disable" { + t.Errorf("verified with %q, want the host URL with the new password", url) + } + if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML { + t.Errorf("the host config changed before the database accepted the password: %q", got) + } + return nil + } + rig.r.rollout = func(_ context.Context, deployment string) error { + rig.events = append(rig.events, "roll "+deployment) + if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != wantPodTOML { + t.Errorf("felis-api rolled on felis-config %q", got) + } + return nil + } + if err := rig.r.rotate(context.Background(), kindDB, true); err != nil { + t.Fatal(err) + } + if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=NEWTOKEN\nSERVICE_TOKEN=s\n" { + t.Errorf("secrets.env = %q", got) + } + if got := readTestFile(t, rig.r.hostTOML); got != wantHostTOML { + t.Errorf("host config = %q", got) + } + if got := readTestFile(t, rig.r.podTOML); got != wantPodTOML { + t.Errorf("pod config = %q", got) + } + if info, err := os.Lstat(rig.r.defaultTOML); err != nil || info.Mode()&os.ModeSymlink == 0 { + t.Errorf("felis.toml is no longer the link to the host copy (%v)", err) + } + for _, ns := range []string{"felis", "minecraft"} { + if got := rig.secretKey(t, ns, "felis-config", "felis.toml"); got != wantPodTOML { + t.Errorf("Secret %s/felis-config = %q", ns, got) + } + } + if strings.Join(rig.events, ",") != "alter-role,verify-db,roll felis-api" { + t.Errorf("events = %v", rig.events) + } + if strings.Contains(rig.out.String(), "NEWTOKEN") { + t.Errorf("the new password was printed: %s", rig.out.String()) + } +} + +// A password the database does not accept leaves the configs on the old one +// and felis-api running: the installer's record, already new, is the way back. +func TestRotateDatabasePasswordStopsWhenTheDatabaseRefuses(t *testing.T) { + rig := dbRig(t) + rig.r.verifyDB = func(context.Context, string) error { + rig.events = append(rig.events, "verify-db") + return errors.New("password authentication failed") + } + err := rig.r.rotate(context.Background(), kindDB, true) + if err == nil || !strings.Contains(err.Error(), "password authentication failed") || !strings.Contains(err.Error(), "run the installer again") { + t.Fatalf("err = %v, want the refusal and the way back", err) + } + if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML { + t.Errorf("host config = %q, want it unchanged", got) + } + if got := readTestFile(t, rig.r.podTOML); got != testPodTOML { + t.Errorf("pod config = %q, want it unchanged", got) + } + if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != testPodTOML { + t.Errorf("felis-config = %q, want it unchanged", got) + } + if strings.Join(rig.events, ",") != "alter-role,verify-db" { + t.Errorf("events = %v, want no api roll", rig.events) + } +} + +// Everything that can refuse does so before the installer's record or the +// role change; what the host config alone shows is refused by the plan too. +func TestRotateDatabasePasswordRefusesEarly(t *testing.T) { + for _, tc := range []struct { + name string + apply bool + setup func(t *testing.T, rig *rotationRig) + want string + }{ + {"a database the installer does not run", false, func(t *testing.T, rig *rotationRig) { + writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "deployment = \"felis/felis-postgres\"\n", "", 1), 0o600) + }, "[database] deployment is unset"}, + {"a database url without a role", false, func(t *testing.T, rig *rotationRig) { + writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "felis:oldpw@", "", 1), 0o600) + }, "names no role"}, + {"a config copy the line editor cannot edit", true, func(t *testing.T, rig *rotationRig) { + writeTestFile(t, rig.r.podTOML, "[database]\nurl = \"\"\"\npostgres://felis:oldpw@felis-postgres.felis.svc:5432/felis\"\"\"\n", 0o600) + }, "set the password in its [database] url by hand"}, + {"a pod copy that is the host copy", true, func(t *testing.T, rig *rotationRig) { + if err := os.Remove(rig.r.podTOML); err != nil { + t.Fatal(err) + } + if err := os.Symlink("felis.host.toml", rig.r.podTOML); err != nil { + t.Fatal(err) + } + }, "resolves to the same file as"}, + } { + t.Run(tc.name, func(t *testing.T) { + rig := dbRig(t) + tc.setup(t, rig) + err := rig.r.rotate(context.Background(), kindDB, tc.apply) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n" { + t.Errorf("secrets.env = %q, want it untouched", got) + } + if len(rig.events) != 0 { + t.Errorf("events = %v, want nothing done", rig.events) + } + }) + } +} + +// The RFC 7677 example (user "user", password "pencil"), with the stored and +// server keys computed by openssl's PBKDF2 and HMAC rather than this code. +func TestScramVerifier(t *testing.T) { + salt, _ := base64.StdEncoding.DecodeString("W22ZaJ0SNY7soEsUEjb6gQ==") + got, err := scramVerifier("pencil", salt, 4096) + if err != nil { + t.Fatal(err) + } + if want := "SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==$WG5d8oPm3OtcPnkdi4Uo7BkeZkBFzpcXkuLmtbsT4qY=:wfPLwcE6nTWhTAmQ7tl2KeoiWGPlZqQxSrmfPwDl2dU="; got != want { + t.Errorf("scramVerifier = %q\nwant %q", got, want) + } +} + +func TestAlterRoleSQL(t *testing.T) { + if got := alterRoleSQL(`fe"lis`, "SCRAM-SHA-256$4096:c2FsdA==$a:b"); got != "ALTER ROLE \"fe\"\"lis\" WITH PASSWORD 'SCRAM-SHA-256$4096:c2FsdA==$a:b';\n" { + t.Errorf("alterRoleSQL = %q", got) + } +} + +// installerSecretKeys is every secrets.env key a rotation writes. +func installerSecretKeys() map[string]bool { + keys := map[string]bool{installerForwardingKey: true, installerDBKey: true} + for _, k := range installerTokenKeys { + keys[k] = true + } + for _, k := range installerRegistryKeys { + keys[k] = true + } + return keys +} + // The installer and rotate-token must agree on where each caller's token lives: // rotate-token writes installerTokenKeys into secrets.env, and the installer // applies those same keys to the Secrets on its next run. A key the installer @@ -249,12 +818,6 @@ func TestInstallerProvisionsEveryCallerToken(t *testing.T) { if !ok { t.Fatalf("installerTokenKeys names unknown caller %q", caller) } - if !strings.Contains(script, key+`="${`+key+`:-$(openssl rand -hex 32)}"`) { - t.Errorf("bootstrap.sh does not generate %s", key) - } - if !strings.Contains(script, key+"=${"+key+"}\n") { - t.Errorf("bootstrap.sh does not persist %s to secrets.env", key) - } apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`) if !apply.MatchString(script) { t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key) @@ -281,3 +844,40 @@ func TestInstallerProvisionsEveryCallerToken(t *testing.T) { t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames())) } } + +// Every value the installer keeps in secrets.env has a rotation that rewrites +// that very key, and every key a rotation writes is one the installer +// generates, keeps and so re-applies on its next run. +func TestInstallerSecretsAreAllRotatable(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh")) + if err != nil { + t.Fatal(err) + } + script := string(raw) + m := regexp.MustCompile(`(?s)write_file_atomic "\$SECRETS_ENV" 0600 <` rewrites the matching line here. + # `felis rotate-token ` replaces one of them, and `felis rotate-token + # registry|forwarding|db` the other values persisted below: every line of secrets.env + # has a rotation that rewrites it (cmd/felis TestInstallerSecretsAreAllRotatable). SERVICE_TOKEN="${SERVICE_TOKEN:-$(openssl rand -hex 32)}" LIMBO_TOKEN="${LIMBO_TOKEN:-$(openssl rand -hex 32)}" BUILD_TOKEN="${BUILD_TOKEN:-$(openssl rand -hex 32)}" OPS_TOKEN="${OPS_TOKEN:-$(openssl rand -hex 32)}" - SESSION_SECRET="${SESSION_SECRET:-$(openssl rand -hex 32)}" # The Velocity modern-forwarding key. It is what makes a backend's UUID trustworthy: # the proxy does the Mojang handshake and HMACs the resulting profile with this key, # and a backend that cannot verify it would fall back to an offline UUID derived from @@ -4213,7 +4221,6 @@ SERVICE_TOKEN=${SERVICE_TOKEN} LIMBO_TOKEN=${LIMBO_TOKEN} BUILD_TOKEN=${BUILD_TOKEN} OPS_TOKEN=${OPS_TOKEN} -SESSION_SECRET=${SESSION_SECRET} FORWARDING_SECRET=${FORWARDING_SECRET} REGISTRY_PLATFORM_TOKEN=${REGISTRY_PLATFORM_TOKEN} REGISTRY_BUILD_TOKEN=${REGISTRY_BUILD_TOKEN} diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index 9708b73..e97cf55 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -1643,7 +1643,7 @@ run_atomic() { # script; under the installer's shell options, its temp files rem mode_of() { ls -l "$1" | cut -c1-10; } printf 'A=1\nB=2\n' > "$wadir/in.new" -printf 'DB_PASSWORD=new\nSESSION_SECRET=new\n' > "$wadir/in.replace" +printf 'DB_PASSWORD=new\nSERVICE_TOKEN=new\n' > "$wadir/in.replace" out="$(run_atomic "umask 000; write_file_atomic '$wadir/new.env' 0600 < '$wadir/in.new'; echo done")" expect "an atomic write finishes" "done" "$out" expect "an atomic write holds all of its input" "$(printf 'A=1\nB=2')" "$(cat "$wadir/new.env")" @@ -1651,11 +1651,11 @@ expect "an atomic write is private under a permissive umask" "-rw-------" "$(mod run_atomic "write_file_atomic '$wadir/new.env' 0640 < '$wadir/in.new'" >/dev/null expect "an atomic write sets the mode it is given" "-rw-r-----" "$(mode_of "$wadir/new.env")" -printf 'DB_PASSWORD=old-and-whole\nSESSION_SECRET=kept\n' > "$wadir/old.env" +printf 'DB_PASSWORD=old-and-whole\nSERVICE_TOKEN=kept\n' > "$wadir/old.env" out="$(run_atomic "cat() { head -c 7; return 1; }; write_file_atomic '$wadir/old.env' 0600 < '$wadir/in.replace'; echo survived")" expect "a write that fails halfway dies" "DIE: could not write $wadir/old.env; it is left as it was" "$out" expect "a write that fails halfway leaves the old file whole" \ - "$(printf 'DB_PASSWORD=old-and-whole\nSESSION_SECRET=kept')" "$(cat "$wadir/old.env")" + "$(printf 'DB_PASSWORD=old-and-whole\nSERVICE_TOKEN=kept')" "$(cat "$wadir/old.env")" out="$(run_atomic "sync() { return 1; }; write_file_atomic '$wadir/old.env' 0600 < '$wadir/in.replace'")" expect "content that did not reach the disk does not replace the old file" "DIE: could not write $wadir/old.env" "$out" expect "the old file survives a failed sync" "DB_PASSWORD=old-and-whole" "$(cat "$wadir/old.env")" @@ -2675,6 +2675,20 @@ expect "a new heap size restarts the proxy" "SYSTEMCTL restart felis-velocity" " expect "the unit carries the new ceiling" "java -Xms512M -Xmx3G " "$(cat "$vdir/unit")" run_velocity_service 1 384M >/dev/null expect "a ceiling below 512M is also the initial heap" "java -Xms384M -Xmx384M " "$(cat "$vdir/unit")" +# `felis rotate-token velocity` rewrites service-token, which the plugin re-reads by itself: +# that line alone changing leaves the proxy running, and any other change restarts it. +props="$vdir/v/plugins/felis-link/felis-link.properties" +printf 'api-base-url=http://a\nservice-token=one\n' > "$props" +expect "new proxy properties restart the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1 384M)" +printf 'api-base-url=http://a\nservice-token=two\n' > "$props" +out="$(run_velocity_service 1 384M)" +case "$out" in + *"SYSTEMCTL restart"*) echo "FAIL a new service-token alone restarted the proxy"; fails=$((fails + 1)) ;; + *"felis-velocity unchanged; left running"*) echo "PASS a new service-token alone leaves the proxy running" ;; + *) echo "FAIL install_velocity_service died on a new service-token: $out"; fails=$((fails + 1)) ;; +esac +printf 'api-base-url=http://b\nservice-token=two\n' > "$props" +expect "another change to the proxy properties restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1 384M)" rm -rf "$vdir" ssblock="$(awk '/^restart_existing_system_servers\(\) \{/,/^}/' "$BS")" diff --git a/deploy/limbo/README.md b/deploy/limbo/README.md index 18c05ea..39a38f5 100644 --- a/deploy/limbo/README.md +++ b/deploy/limbo/README.md @@ -144,7 +144,7 @@ set them by hand: the minecraft namespace (and `felis setup` refreshes that replica from the control namespace), and the operator injects it into the `login` pod (only) as `FELIS_SERVICE_TOKEN` via a `secretKeyRef`, keyed off the reserved `login` name. - `sudo felis rotate-token limbo` replaces it and restarts the pod. Until the token is + `sudo felis rotate-token -yes limbo` replaces it and restarts the pod. Until the token is present the plugin fail-safes to readiness-only, so the gate is never broken — it simply does not authenticate yet. - **Service:** the login pod dials `FELIS_API_BASE_URL`, which resolves to the diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e489fcd..f6673ef 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -143,7 +143,7 @@ components: (felis-build-token), ops (felis-ops-token) — and each operation lists the callers it serves in x-felis-callers. A genuine token for a caller the operation does not list is refused with 403 wrong_caller. `felis - rotate-token ` replaces one. + rotate-token -yes ` replaces one. sessionCookie: type: apiKey in: cookie diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 6bab44f..5775f2e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -439,15 +439,47 @@ token on the wire, which is one more reason the proxy belongs on the node (§1 o ### Rotating a token -`sudo felis rotate-token ` replaces one caller's token: -it writes the new value to `secrets.env` (so a later installer run keeps it), the -Secret and its replica, rolls felis-api so only the new value is accepted, then -restarts the caller — the `felis-velocity` unit when the proxy runs on this host, -or the login pod. Build Jobs and `felis backup-now` pick the new value up on their -next run. The old value stops working as soon as felis-api has rolled; the caller -is turned away for the few seconds until it restarts. For a proxy on another host, -the command leaves the host alone and tells you to copy the new value from the -Secret into that proxy's `felis-link.properties` and restart it. [GO-TESTED] +`sudo felis rotate-token ` replaces one credential the installer generated. +Without `-yes` it only prints what it would write and what that interrupts; +`sudo felis rotate-token -yes ` rotates. Every rotation writes the new value +into `/etc/felis/secrets.env` first, so whatever fails after that, a re-run of the +installer (`sudo bash deploy/bootstrap.sh`) puts the value everywhere. [GO-TESTED] + +| kind | replaces | interrupts | +|---|---|---| +| `velocity` | the proxy's token: Secret `felis-service-token`, `service-token` in the host proxy's `felis-link.properties` | felis-api restarts (one replica: the panel, sign-in and the proxy's calls stop for a few seconds). The proxy re-reads its file and keeps its players | +| `limbo` | the login gate's token, `felis-limbo-token` and its minecraft replica | felis-api restarts, then the login pod | +| `build` | the build Jobs' token, `felis-build-token` and its build-namespace replica | felis-api restarts; a build fetching its context at that moment fails and can be submitted again | +| `ops` | `felis backup-now`'s token, `felis-ops-token` | felis-api restarts | +| `registry` | the registry's `platform`, `build` and `prune` write tokens: Secret `felis-registry-auth`, the build Jobs' `felis-registry-push` | the registry restarts (a push at that moment fails, a pull retries), then felis-api | +| `forwarding` | the Velocity forwarding secret: `/opt/felis/velocity/forwarding.secret`, Secret `felis-forwarding-secret` in both namespaces | every running server restarts (saving its world) and the proxy restarts: everyone online is disconnected | +| `db` | the database role's password: the role in felis-postgres, `[database] url` in `felis.host.toml` and `felis.pod.toml`, Secret `felis-config` in both namespaces | felis-api restarts; a backup, restore or file Job that connects in those seconds fails and can be run again | + +- **velocity** — the rotation waits for the proxy's log line + `Felis: service-token reloaded from … (fingerprint <12 hex digits>)`. A proxy + that has not logged it within 60 s of felis-api's restart (a plugin from before + this release) is restarted, which disconnects everyone online. The file keeps + its modification time, so `felis domain check` does not read the proxy as + stale. A proxy on another host is left alone: set `service-token` in its + `felis-link.properties` to the value in Secret `felis/felis-service-token`, and + it takes it within a few seconds. +- **forwarding** — a server whose world a backup, restore or file write holds is + left running and named in the plan and the output; players cannot join it until + it restarts, so stop and start it from the panel once that finishes. The next + installer run restarts the proxy once more (its record of what the proxy was + started from predates the rotation); an upgrade restarts it for the new plugin + jar anyway. A proxy on another host needs the value in Secret + `felis/felis-forwarding-secret` in its forwarding secret file, and a restart. +- **db** — the password reaches PostgreSQL as a SCRAM-SHA-256 verifier, never as + text a failed statement could log. The config copies change only after a + connection with the new password succeeds; when it does not, the command stops + and says so, and the installer re-run sets the password from `secrets.env` in + the role and every copy. A database the installer does not run (`[database] + deployment` unset) is refused: change its password where it runs, then in each + copy's `[database] url`. `felis.pod.toml` must be a file of its own, since the + pods reach the database at another address. +- The old value stops working as felis-api (or the registry) restarts; a caller + still presenting it is turned away until it has the new one. --- @@ -867,9 +899,9 @@ control namespace (or `--registry-namespace`): container, via `felis-registry-push` in `felis-build`) may write anything outside `felis/` and `mirror/` and may never delete. A missing Secret leaves the registry read-only rather than down. The tokens persist in `/etc/felis/secrets.env`; - rotating one means editing it there and re-running the installer, then - `kubectl -n felis rollout restart deployment/registry` (the gate reads its tokens - at start). + `sudo felis rotate-token -yes registry` replaces all three and restarts the + registry (the gate reads its tokens at start) and felis-api, which presents the + `prune` token (§6 "Rotating a token"). - **Who can connect:** `felis-registry-ingress` admits only the `felis-build` namespace to the registry pod. Node-local traffic (containerd pulls, the installer's pushes through the hostPort) is always allowed by Kubernetes; game @@ -1916,7 +1948,7 @@ runs changed: | Component | Restarted when | |---|---| -| `felis-velocity` (the proxy) | its unit, the JRE, `velocity.jar`, `velocity.toml`, the forwarding secret, the felis-link settings or a plugin jar changed, or it was not running. The fingerprint lives in `/etc/felis/velocity.fingerprint`; delete it to force a restart. | +| `felis-velocity` (the proxy) | its unit, the JRE, `velocity.jar`, `velocity.toml`, the forwarding secret, the felis-link settings (all but `service-token`, which the plugin re-reads) or a plugin jar changed, or it was not running. The fingerprint lives in `/etc/felis/velocity.fingerprint`; delete it to force a restart. | | login and lobby pods | the rebuilt limbo or lobby image has a new image ID (`/etc/felis/system-server-images`). The installer then pins that server's `spec.image` to the digest its tag names now (`felis pin-images --system login\|lobby`) and the operator rolls the pod onto it, each on its own; with the registry unreachable it recreates the pod instead. The installer turns off BuildKit's default provenance attestation (`BUILDX_NO_DEFAULT_ATTESTATIONS=1`): it records the build time, which would give every rebuild a new ID. | | felis-postgres | the release moved `POSTGRES_IMAGE` or changed the pod; a few seconds without the API. A rerun that changes neither leaves it running. | | felis-api, felis-operator, the registry pod (its gate and GC containers run the felis binary) | the image tag changed (an upgrade), or a same-version rerun rebuilt it. | @@ -2078,7 +2110,7 @@ along). One bundle is `felis-db--