fix(rotate-token): 加确认步骤、velocity 热加载免踢人，补齐 registry/forwarding/db 轮换

cmd/felis/domain.go

@@ -235,7 +235,7 @@ func verifyTOMLEdit(orig, edited []byte, edits []tomlStringEdit) error {
 		t[e.key] = e.value
 	}
 	if !reflect.DeepEqual(want, got) {
-		return errors.New("a line edit would change more than the domain keys (a multi-line value, or a quoted or dotted key?)")
+		return errors.New("a line edit would change more than the keys it sets (a multi-line value, or a quoted or dotted key?)")
 	}
 	return nil
 }
@@ -591,9 +591,15 @@ type tomlTarget struct{ path, real string }
 // tomlTargets are the host and pod copies, and felis.toml when it is a file of
 // its own rather than the link to the host copy.
 func (h domainHost) tomlTargets() ([]tomlTarget, error) {
+	return tomlTargetsOf(h.paths.hostTOML, h.paths.podTOML, h.paths.defaultTOML)
+}
+
+// tomlTargetsOf is the host copy, the pod copy, and def when it exists and is
+// not a link to one of them.
+func tomlTargetsOf(host, pod, def string) ([]tomlTarget, error) {
 	var out []tomlTarget
 	seen := map[string]bool{}
-	for i, p := range []string{h.paths.hostTOML, h.paths.podTOML, h.paths.defaultTOML} {
+	for i, p := range []string{host, pod, def} {
 		real, err := filepath.EvalSymlinks(p)
 		if errors.Is(err, fs.ErrNotExist) && i == 2 {
 			continue

cmd/felis/rotatetoken.go

@@ -2,40 +2,70 @@ package main
 
 import (
 	"context"
+	"crypto/hmac"
+	"crypto/pbkdf2"
 	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"io/fs"
+	neturl "net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/maintenance"
 	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/operator"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/registrygate"
 
+	"github.com/BurntSushi/toml"
+	"github.com/jackc/pgx/v5"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-// rotate-token replaces one internal caller's token (naming.CallerTokens): a new
-// value goes into the installer's record, the control-namespace Secret and the
-// replica the caller's pods mount, felis-api rolls so it accepts only the new
-// value, and then the caller restarts so it presents it. Between the api's
-// rollout and the caller's restart the caller is turned away with 401; for the
-// login gate and the proxy that is the few seconds of a pod or unit restart.
+// rotate-token replaces one credential the installer generated: an internal
+// caller's token (naming.CallerTokens), the registry's write tokens, the
+// Velocity forwarding secret or the database password. The new value goes into
+// the installer's record first, so that whatever fails later a re-run of the
+// installer puts it everywhere; then into the Secrets and files that carry it;
+// then whatever read the old value at start restarts. Without -yes it prints
+// what it would change and what that interrupts, and changes nothing.
 
 const (
 	defaultSecretsEnvPath = "/etc/felis/secrets.env"
 	defaultLinkPropsPath  = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
+	defaultForwardingPath = "/opt/felis/velocity/forwarding.secret"
 	velocityUnit          = "felis-velocity"
+	apiDeployment         = "felis-api"
+	registryDeployment    = "registry"
+
+	kindRegistry   = "registry"
+	kindForwarding = "forwarding"
+	kindDB         = "db"
+
+	// proxyReloadWait is how long the host proxy gets, once felis-api has rolled,
+	// to show it re-read its token: it reads the file again on its next call to
+	// felis-api, and it calls every 15 seconds.
+	proxyReloadWait = 60 * time.Second
+
+	// apiRestartNote is in every plan: felis-api runs as one replica replaced in
+	// place, so its restart is a short outage of everything that talks to it.
+	apiRestartNote = "felis-api restarts (a single replica): the panel, sign-in and the proxy's calls are unavailable for the few seconds that takes"
 )
 
 // installerTokenKeys names each caller's token in the installer's secrets.env
@@ -49,21 +79,55 @@ var installerTokenKeys = map[string]string{
 	"ops":      "OPS_TOKEN",
 }
 
+// installerRegistryKeys names the registry principals' tokens in secrets.env,
+// in registrygate.Principals order.
+var installerRegistryKeys = map[string]string{
+	registrygate.PrincipalPlatform: "REGISTRY_PLATFORM_TOKEN",
+	registrygate.PrincipalBuild:    "REGISTRY_BUILD_TOKEN",
+	registrygate.PrincipalPrune:    "REGISTRY_PRUNE_TOKEN",
+}
+
+// installerForwardingKey and installerDBKey name the forwarding secret and the
+// database password in secrets.env.
+const (
+	installerForwardingKey = "FORWARDING_SECRET"
+	installerDBKey         = "DB_PASSWORD"
+)
+
 type tokenRotator struct {
 	cl          client.Client
 	controlNS   string
 	minecraftNS string
 	buildNS     string
-	// secretsEnv and linkProps are the installer's record and the proxy's
-	// felis-link.properties; a missing file is reported and skipped.
-	secretsEnv string
-	linkProps  string
-	newToken   func() (string, error)
-	// rollAPI restarts felis-api and waits for the rollout.
-	rollAPI func(ctx context.Context) error
+	// secretsEnv, linkProps and forwardingFile are the installer's record and the
+	// host proxy's felis-link.properties and forwarding.secret; a missing file is
+	// reported and skipped.
+	secretsEnv     string
+	linkProps      string
+	forwardingFile string
+	// hostTOML, podTOML and defaultTOML are the config copies that carry the
+	// database URL (defaultTOML only when it is a file of its own).
+	hostTOML    string
+	podTOML     string
+	defaultTOML string
+	newToken    func() (string, error)
+	// rollout restarts a control-namespace Deployment and waits for it.
+	rollout func(ctx context.Context, deployment string) error
 	// restartUnit restarts a systemd unit on this host.
 	restartUnit func(ctx context.Context, unit string) error
-	out         io.Writer
+	// proxyLog is what the host proxy has logged since a moment, as far as it
+	// can be read.
+	proxyLog func(ctx context.Context, since time.Time) string
+	// alterRole stores a password verifier for a role of the database that runs
+	// as deployment ("namespace/name"); verifyDB connects with a URL.
+	alterRole func(ctx context.Context, deployment, role, verifier string) error
+	verifyDB  func(ctx context.Context, url string) error
+	now       func() time.Time
+	// reloadWait bounds the wait for the proxy to re-read its token, polled
+	// every pollEvery.
+	reloadWait time.Duration
+	pollEvery  time.Duration
+	out        io.Writer
 }
 
 func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
@@ -72,9 +136,12 @@ func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
 	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
 	secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
 	linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
+	forwarding := fs.String("forwarding-secret", defaultForwardingPath, "the host proxy's forwarding secret file (forwarding only)")
+	yes := fs.Bool("yes", false, "rotate; without it the plan is printed and nothing changes")
 	fs.Usage = func() {
-		fmt.Fprintf(stderr, "Usage: felis rotate-token [flags] <%s>\n\n", strings.Join(callerNames(), "|"))
-		fmt.Fprintln(stderr, "Replaces one internal caller's token: the Secrets, felis-api, then the caller itself.")
+		fmt.Fprintf(stderr, "Usage: felis rotate-token [-yes] [flags] <%s>\n\n", strings.Join(rotationKinds(), "|"))
+		fmt.Fprintln(stderr, "Replaces one generated credential: the installer's record, the Secrets and files that carry it, then what reads it.")
+		fmt.Fprintln(stderr, "Without -yes it prints what would change and what that interrupts.")
 		fs.PrintDefaults()
 	}
 	if err := fs.Parse(args); err != nil {
@@ -87,12 +154,13 @@ func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
 		fs.Usage()
 		return 2
 	}
-	if _, ok := callerToken(fs.Arg(0)); !ok {
-		fmt.Fprintf(stderr, "felis rotate-token: unknown caller %q (one of %s)\n", fs.Arg(0), strings.Join(callerNames(), ", "))
+	kind := fs.Arg(0)
+	if !knownRotation(kind) {
+		fmt.Fprintf(stderr, "felis rotate-token: unknown credential %q (one of %s)\n", kind, strings.Join(rotationKinds(), ", "))
 		return 2
 	}
 	if os.Geteuid() != 0 {
-		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+fs.Arg(0)+")")
+		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+kind+")")
 		return 1
 	}
 	cfg, err := config.Load(*cfgPath)
@@ -109,24 +177,43 @@ func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
 	if buildNS == "" {
 		buildNS = platform.DefaultBuildNamespace
 	}
+	controlNS := platform.DefaultControlNamespace
 	r := tokenRotator{
-		cl:          cl,
-		controlNS:   platform.DefaultControlNamespace,
-		minecraftNS: cfg.K8s.Namespace,
-		buildNS:     buildNS,
-		secretsEnv:  *secretsEnv,
-		linkProps:   *linkProps,
-		newToken:    randomToken,
-		rollAPI: func(ctx context.Context) error {
-			if err := kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "restart", "deployment/felis-api"); err != nil {
+		cl:             cl,
+		controlNS:      controlNS,
+		minecraftNS:    cfg.K8s.Namespace,
+		buildNS:        buildNS,
+		secretsEnv:     *secretsEnv,
+		linkProps:      *linkProps,
+		forwardingFile: *forwarding,
+		hostTOML:       hostSetupConfigPath,
+		podTOML:        podSetupConfigPath,
+		defaultTOML:    defaultSetupConfigPath,
+		newToken:       randomToken,
+		rollout: func(ctx context.Context, deployment string) error {
+			if err := kubectl(ctx, "-n", controlNS, "rollout", "restart", "deployment/"+deployment); err != nil {
 				return err
 			}
-			return kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "status", "deployment/felis-api", "--timeout=180s")
+			return kubectl(ctx, "-n", controlNS, "rollout", "status", "deployment/"+deployment, "--timeout=180s")
 		},
 		restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
-		out:         stdout,
+		proxyLog:    journalSince,
+		alterRole:   alterRoleInPod,
+		verifyDB: func(ctx context.Context, url string) error {
+			ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+			defer cancel()
+			conn, err := pgx.Connect(ctx, url)
+			if err != nil {
+				return err
+			}
+			return conn.Close(ctx)
+		},
+		now:        time.Now,
+		reloadWait: proxyReloadWait,
+		pollEvery:  3 * time.Second,
+		out:        stdout,
 	}
-	if err := r.rotate(context.Background(), fs.Arg(0)); err != nil {
+	if err := r.rotate(context.Background(), kind, *yes); err != nil {
 		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
 		return 1
 	}
@@ -150,6 +237,21 @@ func callerToken(name string) (naming.CallerToken, bool) {
 	return naming.CallerToken{}, false
 }
 
+// rotationKinds is every credential rotate-token replaces: the callers' tokens
+// and the installer's other generated secrets.
+func rotationKinds() []string {
+	return append(callerNames(), kindRegistry, kindForwarding, kindDB)
+}
+
+func knownRotation(kind string) bool {
+	for _, k := range rotationKinds() {
+		if k == kind {
+			return true
+		}
+	}
+	return false
+}
+
 // randomToken is 32 random bytes in hex, the shape the installer generates.
 func randomToken() (string, error) {
 	b := make([]byte, 32)
@@ -159,99 +261,527 @@ func randomToken() (string, error) {
 	return hex.EncodeToString(b), nil
 }
 
-func (r tokenRotator) rotate(ctx context.Context, caller string) error {
-	ct, ok := callerToken(caller)
-	if !ok {
-		return fmt.Errorf("unknown caller %q", caller)
-	}
-	tok, err := r.newToken()
-	if err != nil {
-		return fmt.Errorf("generate a token: %w", err)
-	}
+// tokenFingerprint is how the proxy names the token it reloaded in its log
+// (plugins/shared FileToken.fingerprint): the first twelve hex digits of its
+// SHA-256, enough to tell tokens apart and useless for finding one.
+func tokenFingerprint(token string) string {
+	sum := sha256.Sum256([]byte(token))
+	return hex.EncodeToString(sum[:])[:12]
+}
 
-	// The installer's record first: from here on, whatever fails, a re-run of the
-	// installer puts the new value everywhere.
-	switch err := setKeyValueLine(r.secretsEnv, installerTokenKeys[ct.Caller], "=", tok); {
+func (r tokenRotator) rotate(ctx context.Context, kind string, apply bool) error {
+	switch kind {
+	case kindRegistry:
+		return r.rotateRegistry(ctx, apply)
+	case kindForwarding:
+		return r.rotateForwarding(ctx, apply)
+	case kindDB:
+		return r.rotateDB(ctx, apply)
+	}
+	ct, ok := callerToken(kind)
+	if !ok {
+		return fmt.Errorf("unknown credential %q (one of %s)", kind, strings.Join(rotationKinds(), ", "))
+	}
+	return r.rotateCaller(ctx, ct, apply)
+}
+
+// confirm ends the plan: without apply it says nothing changed and how to go
+// ahead, and reports false.
+func (r tokenRotator) confirm(kind string, apply bool) bool {
+	if !apply {
+		fmt.Fprintf(r.out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes %s\n", kind)
+		return false
+	}
+	fmt.Fprintln(r.out, "\nRotating:")
+	return true
+}
+
+// record writes new values into the installer's secrets.env. It comes first in
+// every rotation: from then on, whatever fails, a re-run of the installer puts
+// the new values everywhere.
+func (r tokenRotator) record(kv ...[2]string) error {
+	keys := make([]string, len(kv))
+	for i, p := range kv {
+		keys[i] = p[0]
+	}
+	switch err := setKeyValueLines(r.secretsEnv, "=", kv); {
 	case errors.Is(err, fs.ErrNotExist):
 		fmt.Fprintf(r.out, "  - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
 	case err != nil:
-		return fmt.Errorf("record the new token in %s: %w", r.secretsEnv, err)
+		return fmt.Errorf("record the new value in %s: %w", r.secretsEnv, err)
 	default:
-		fmt.Fprintf(r.out, "  - %s: %s updated\n", r.secretsEnv, installerTokenKeys[ct.Caller])
+		fmt.Fprintf(r.out, "  - %s: %s updated\n", r.secretsEnv, strings.Join(keys, ", "))
 	}
+	return nil
+}
 
+func (r tokenRotator) putSecret(ctx context.Context, ns, name string, data map[string][]byte) error {
+	if _, err := putSecretKeys(ctx, r.cl, ns, name, corev1.SecretTypeOpaque, data); err != nil {
+		return fmt.Errorf("write Secret %s/%s: %w", ns, name, err)
+	}
+	fmt.Fprintf(r.out, "  - Secret %s/%s: updated\n", ns, name)
+	return nil
+}
+
+func (r tokenRotator) rollAPI(ctx context.Context) error {
+	if err := r.rollout(ctx, apiDeployment); err != nil {
+		return fmt.Errorf("roll felis-api: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - felis-api: rolled out on the new value")
+	return nil
+}
+
+// withMinecraft is the control namespace plus the minecraft namespace when that
+// is a different one: where the Secrets game pods and Jobs mount are mirrored.
+func (r tokenRotator) withMinecraft() []string {
+	if r.minecraftNS == "" || r.minecraftNS == r.controlNS {
+		return []string{r.controlNS}
+	}
+	return []string{r.controlNS, r.minecraftNS}
+}
+
+func (r tokenRotator) rotateCaller(ctx context.Context, ct naming.CallerToken, apply bool) error {
 	namespaces := []string{r.controlNS}
 	replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
 	if replica != "" && replica != r.controlNS {
 		namespaces = append(namespaces, replica)
 	}
-	for _, ns := range namespaces {
-		if err := writeTokenSecret(ctx, r.cl, ns, ct.Secret, tok); err != nil {
-			return fmt.Errorf("write Secret %s/%s: %w", ns, ct.Secret, err)
-		}
-		fmt.Fprintf(r.out, "  - Secret %s/%s: updated\n", ns, ct.Secret)
-	}
+	key := installerTokenKeys[ct.Caller]
 
-	hostProxy := false
-	if ct.Caller == "velocity" {
-		switch err := setKeyValueLine(r.linkProps, "service-token", "=", tok); {
-		case errors.Is(err, fs.ErrNotExist):
-			fmt.Fprintf(r.out, "  - %s: not found; set service-token in your proxy's felis-link.properties to the value in Secret %s/%s and restart it\n",
-				r.linkProps, r.controlNS, ct.Secret)
-		case err != nil:
-			return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
-		default:
-			hostProxy = true
-			fmt.Fprintf(r.out, "  - %s: service-token updated\n", r.linkProps)
-		}
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new internal token for the %s caller\n", ct.Caller, ct.Caller)
+	secrets := make([]string, len(namespaces))
+	for i, ns := range namespaces {
+		secrets[i] = "Secret " + ns + "/" + ct.Secret
 	}
-
-	if err := r.rollAPI(ctx); err != nil {
-		return fmt.Errorf("roll felis-api: %w", err)
+	writes := append([]string{r.secretsEnv + " (" + key + ")"}, secrets...)
+	hostProxy := ct.Caller == "velocity" && fileExists(r.linkProps)
+	if hostProxy {
+		writes = append(writes, r.linkProps+" (service-token)")
 	}
-	fmt.Fprintln(r.out, "  - felis-api: rolled out, accepting only the new token")
-
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
 	switch ct.Caller {
 	case "velocity":
 		if hostProxy {
-			if err := r.restartUnit(ctx, velocityUnit); err != nil {
-				return fmt.Errorf("restart %s: %w", velocityUnit, err)
-			}
-			fmt.Fprintf(r.out, "  - %s: restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit)
+			fmt.Fprintf(r.out, "  - the proxy re-reads its token from the file and keeps its players; if it has not within %s of felis-api's restart, it is restarted, which disconnects everyone online\n", r.reloadWait)
+		} else {
+			fmt.Fprintf(r.out, "  - no proxy on this host (%s): set service-token in your proxy's felis-link.properties to the value in Secret %s/%s afterwards; it re-reads the file without a restart\n",
+				r.linkProps, r.controlNS, ct.Secret)
 		}
 	case "limbo":
-		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS),
-			client.MatchingLabels{v1alpha1.LabelServer: naming.SystemLoginServer}); err != nil {
+		fmt.Fprintln(r.out, "  - the login gate's pod restarts: a player signing in at that moment reconnects")
+	case "build":
+		fmt.Fprintln(r.out, "  - a build fetching its context at that moment fails and can be submitted again")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now presents the new token on its next run")
+	}
+	if !r.confirm(ct.Caller, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a token: %w", err)
+	}
+	if err := r.record([2]string{key, tok}); err != nil {
+		return err
+	}
+	for _, ns := range namespaces {
+		if err := r.putSecret(ctx, ns, ct.Secret, map[string][]byte{naming.ServiceTokenSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+
+	// The proxy's file changes before felis-api rolls, so the file never falls
+	// behind the Secret; the proxy's log is read from just before the write,
+	// since it may pick the new token up before the rollout ends.
+	since := r.now()
+	if hostProxy {
+		if err := setProxyToken(r.linkProps, tok); err != nil {
+			return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: service-token updated\n", r.linkProps)
+	}
+
+	if err := r.rollAPI(ctx); err != nil {
+		return err
+	}
+
+	switch ct.Caller {
+	case "velocity":
+		if !hostProxy {
+			break
+		}
+		if r.proxyReloaded(ctx, since, tokenFingerprint(tok)) {
+			fmt.Fprintf(r.out, "  - %s: took the new token from its properties; players stayed connected\n", velocityUnit)
+			break
+		}
+		if err := r.restartUnit(ctx, velocityUnit); err != nil {
+			return fmt.Errorf("restart %s: %w", velocityUnit, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: had not taken the new token within %s, restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit, r.reloadWait)
+	case "limbo":
+		// Only the operator's server pods carry its managed-by label; a backup or
+		// restore Job's pod carries the server label too, and app.kubernetes.io ones.
+		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), client.MatchingLabels{
+			v1alpha1.LabelServer:    naming.SystemLoginServer,
+			v1alpha1.LabelManagedBy: operator.ManagedByValue,
+		}); err != nil {
 			return fmt.Errorf("restart the login gate: %w", err)
 		}
 		fmt.Fprintln(r.out, "  - login gate: pod restarted to read the new token")
 	case "build":
-		fmt.Fprintln(r.out, "  - builds: the next build Job reads the new token; one fetching its context right now fails and can be submitted again")
+		fmt.Fprintln(r.out, "  - builds: the next build Job reads the new token")
 	case "ops":
 		fmt.Fprintln(r.out, "  - felis backup-now reads the new token on its next run")
 	}
 	return nil
 }
 
-// writeTokenSecret sets the token in a Secret, creating it when absent.
-func writeTokenSecret(ctx context.Context, cl client.Client, namespace, name, token string) error {
-	var sec corev1.Secret
-	err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &sec)
-	if apierrors.IsNotFound(err) {
-		return cl.Create(ctx, &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
-			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
-		})
+// proxyReloaded waits up to reloadWait for the host proxy to log that it
+// reloaded the token with this fingerprint (plugins/shared FileToken).
+func (r tokenRotator) proxyReloaded(ctx context.Context, since time.Time, fingerprint string) bool {
+	want := "(fingerprint " + fingerprint + ")"
+	deadline := r.now().Add(r.reloadWait)
+	for {
+		if strings.Contains(r.proxyLog(ctx, since), want) {
+			return true
+		}
+		if !r.now().Before(deadline) {
+			return false
+		}
+		select {
+		case <-ctx.Done():
+			return false
+		case <-time.After(r.pollEvery):
+		}
 	}
+}
+
+// journalSince is the proxy unit's journal from the second since falls in. A
+// journal that cannot be read reads as one without the line, which ends in the
+// restart a rotation made before the proxy could reload.
+func journalSince(ctx context.Context, since time.Time) string {
+	out, _ := exec.CommandContext(ctx, "journalctl", "-u", velocityUnit, "--since", "@"+strconv.FormatInt(since.Unix(), 10),
+		"-o", "cat", "--no-pager", "-q").Output()
+	return string(out)
+}
+
+// setProxyToken writes the proxy's token into felis-link.properties and puts
+// the file's modification time back. The proxy re-reads its token by itself,
+// while `felis domain check` and `felis domain set` read a file newer than the
+// proxy's start as config it has not loaded (the installer's
+// install_if_changed keeps the time for the same reason).
+func setProxyToken(path, token string) error {
+	info, err := os.Stat(path)
 	if err != nil {
 		return err
 	}
-	if sec.Data == nil {
-		sec.Data = map[string][]byte{}
+	if err := setKeyValueLine(path, "service-token", "=", token); err != nil {
+		return err
 	}
-	sec.Data[naming.ServiceTokenSecretKey] = []byte(token)
-	return cl.Update(ctx, &sec)
+	return os.Chtimes(path, time.Time{}, info.ModTime())
+}
+
+func (r tokenRotator) rotateRegistry(ctx context.Context, apply bool) error {
+	keys := make([]string, len(registrygate.Principals))
+	for i, p := range registrygate.Principals {
+		keys[i] = installerRegistryKeys[p]
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: new write tokens for the image registry's principals (%s)\n", kindRegistry, strings.Join(registrygate.Principals, ", "))
+	fmt.Fprintf(r.out, "  - writes %s (%s), Secret %s/%s, Secret %s/%s\n", r.secretsEnv, strings.Join(keys, ", "),
+		r.controlNS, naming.RegistryAuthSecretName, r.buildNS, naming.RegistryPushSecretName)
+	fmt.Fprintln(r.out, "  - the registry restarts to load them: a build pushing its image at that moment fails and can be submitted again, and an image pull in that moment retries")
+	fmt.Fprintf(r.out, "  - %s (it presents the prune token)\n", apiRestartNote)
+	if !r.confirm(kindRegistry, apply) {
+		return nil
+	}
+
+	tokens := map[string][]byte{}
+	kv := make([][2]string, 0, len(registrygate.Principals))
+	for _, p := range registrygate.Principals {
+		tok, err := r.newToken()
+		if err != nil {
+			return fmt.Errorf("generate a token: %w", err)
+		}
+		tokens[p] = []byte(tok)
+		kv = append(kv, [2]string{installerRegistryKeys[p], tok})
+	}
+	if err := r.record(kv...); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.controlNS, naming.RegistryAuthSecretName, tokens); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.buildNS, naming.RegistryPushSecretName, map[string][]byte{
+		naming.RegistryPushUsernameKey: []byte(registrygate.PrincipalBuild),
+		naming.RegistryPushPasswordKey: tokens[registrygate.PrincipalBuild],
+	}); err != nil {
+		return err
+	}
+	if err := r.rollout(ctx, registryDeployment); err != nil {
+		return fmt.Errorf("restart the registry: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - registry: restarted on the new tokens")
+	return r.rollAPI(ctx)
+}
+
+func (r tokenRotator) rotateForwarding(ctx context.Context, apply bool) error {
+	restart, held, err := r.gamePods(ctx)
+	if err != nil {
+		return err
+	}
+	hostProxy := fileExists(r.forwardingFile)
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new Velocity forwarding secret, the key a server checks each player's identity with\n", kindForwarding)
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, "Secret "+ns+"/"+naming.ForwardingSecretName)
+	}
+	writes := append([]string{r.secretsEnv + " (" + installerForwardingKey + ")"}, secrets...)
+	if hostProxy {
+		writes = append(writes, r.forwardingFile)
+	}
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - every running server restarts to read it (%d now), saving its world on the way down, and the proxy restarts: everyone online is disconnected and can rejoin once their server is back\n", len(restart))
+	if len(held) > 0 {
+		fmt.Fprintf(r.out, "  - left running, because a backup, restore or file write holds its world: %s. Players cannot join it until it restarts: stop and start it from the panel once that finishes\n", strings.Join(held, ", "))
+	}
+	if !hostProxy {
+		fmt.Fprintf(r.out, "  - no proxy on this host (%s): put the value in Secret %s/%s into your proxy's forwarding secret file and restart it\n",
+			r.forwardingFile, r.controlNS, naming.ForwardingSecretName)
+	}
+	if !r.confirm(kindForwarding, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a secret: %w", err)
+	}
+	if err := r.record([2]string{installerForwardingKey, tok}); err != nil {
+		return err
+	}
+	if hostProxy {
+		info, err := os.Stat(r.forwardingFile)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(r.forwardingFile, info, []byte(tok)); err != nil {
+			return fmt.Errorf("write %s: %w", r.forwardingFile, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: updated\n", r.forwardingFile)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, naming.ForwardingSecretName, map[string][]byte{naming.ForwardingSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+	// The servers go first: their pods read the Secret as they are recreated,
+	// and a player who rejoins through the restarted proxy meets a server on the
+	// new secret, or one still starting.
+	for i := range restart {
+		p := &restart[i]
+		if err := r.cl.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) {
+			return fmt.Errorf("restart %s: %w", p.Labels[v1alpha1.LabelServer], err)
+		}
+	}
+	fmt.Fprintf(r.out, "  - servers: %d restarting on the new secret\n", len(restart))
+	if hostProxy {
+		if err := r.restartUnit(ctx, velocityUnit); err != nil {
+			return fmt.Errorf("restart %s: %w", velocityUnit, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: restarted on the new secret\n", velocityUnit)
+	}
+	return nil
+}
+
+// gamePods lists the running game server pods: those a rotation may restart,
+// and the servers left alone because a backup, restore or file write holds
+// their world (internal/maintenance), which a restart in the middle of would
+// break.
+func (r tokenRotator) gamePods(ctx context.Context) ([]corev1.Pod, []string, error) {
+	var pods corev1.PodList
+	// The operator's managed-by label is on its server pods alone (see rotateCaller).
+	if err := r.cl.List(ctx, &pods, client.InNamespace(r.minecraftNS), client.MatchingLabels{v1alpha1.LabelManagedBy: operator.ManagedByValue}); err != nil {
+		return nil, nil, fmt.Errorf("list the servers' pods: %w", err)
+	}
+	var jobs batchv1.JobList
+	if err := r.cl.List(ctx, &jobs, client.InNamespace(r.minecraftNS)); err != nil {
+		return nil, nil, fmt.Errorf("list the maintenance Jobs: %w", err)
+	}
+	var restart []corev1.Pod
+	var held []string
+	for _, p := range pods.Items {
+		if p.DeletionTimestamp != nil {
+			continue
+		}
+		server := p.Labels[v1alpha1.LabelServer]
+		var ms v1alpha1.MinecraftServer
+		if err := r.cl.Get(ctx, client.ObjectKey{Namespace: r.minecraftNS, Name: server}, &ms); client.IgnoreNotFound(err) != nil {
+			return nil, nil, fmt.Errorf("read server %s: %w", server, err)
+		}
+		if kind, ok := maintenance.Holder(server, ms.Annotations, jobs.Items, r.now()); ok {
+			held = append(held, server+" ("+kind+")")
+			continue
+		}
+		restart = append(restart, p)
+	}
+	return restart, held, nil
+}
+
+func (r tokenRotator) rotateDB(ctx context.Context, apply bool) error {
+	cfg, err := config.Load(r.hostTOML)
+	if err != nil {
+		return err
+	}
+	if cfg.Database.Deployment == "" {
+		return fmt.Errorf("[database] deployment is unset in %s, so the database is not the installer's felis-postgres: change the role's password where it runs, then in [database] url of each config copy", r.hostTOML)
+	}
+	u, err := neturl.Parse(cfg.Database.URL)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return fmt.Errorf("[database] url in %s names no role", r.hostTOML)
+	}
+	role := u.User.Username()
+	targets, err := tomlTargetsOf(r.hostTOML, r.podTOML, r.defaultTOML)
+	if err != nil {
+		return err
+	}
+	paths := make([]string, len(targets))
+	for i, t := range targets {
+		paths[i] = t.path
+	}
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, ns+"/"+platform.ConfigSecretName)
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new password for the database role %q\n", kindDB, role)
+	fmt.Fprintf(r.out, "  - writes %s (%s), the role in %s, [database] url in %s, Secret %s\n",
+		r.secretsEnv, installerDBKey, cfg.Database.Deployment, strings.Join(paths, " and "), strings.Join(secrets, " and "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
+	fmt.Fprintln(r.out, "  - a backup, restore or file Job that connects in the seconds between the password change and felis-api's restart fails and can be run again; the host's timers read the new config on their next run")
+	if !r.confirm(kindDB, apply) {
+		return nil
+	}
+
+	password, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a password: %w", err)
+	}
+	// Every config copy is edited in memory first, so one this cannot edit stops
+	// the rotation before the role's password changes.
+	edited := make([][]byte, len(targets))
+	var hostURL string
+	var podConfig []byte
+	for i, t := range targets {
+		raw, err := os.ReadFile(t.real)
+		if err != nil {
+			return err
+		}
+		var doc struct {
+			Database struct {
+				URL string `toml:"url"`
+			} `toml:"database"`
+		}
+		if _, err := toml.Decode(string(raw), &doc); err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		next, err := withPassword(doc.Database.URL, password)
+		if err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		if edited[i], err = editTOMLStrings(raw, []tomlStringEdit{{"database", "url", next}}); err != nil {
+			return fmt.Errorf("%s: %w; set the password in its [database] url by hand", t.path, err)
+		}
+		switch t.path {
+		case r.hostTOML:
+			hostURL = next
+		case r.podTOML:
+			podConfig = edited[i]
+		}
+	}
+	if podConfig == nil {
+		return fmt.Errorf("%s resolves to the same file as %s; the pods reach the database at another address and need a copy of their own", r.podTOML, r.hostTOML)
+	}
+
+	if err := r.record([2]string{installerDBKey, password}); err != nil {
+		return err
+	}
+	salt := make([]byte, 16)
+	if _, err := rand.Read(salt); err != nil {
+		return err
+	}
+	verifier, err := scramVerifier(password, salt, scramIterations)
+	if err != nil {
+		return err
+	}
+	if err := r.alterRole(ctx, cfg.Database.Deployment, role, verifier); err != nil {
+		return fmt.Errorf("set the role's password: %w", err)
+	}
+	if err := r.verifyDB(ctx, hostURL); err != nil {
+		return fmt.Errorf("the database does not accept the new password (%v); the config copies still hold the old one: run the installer again (sudo bash deploy/bootstrap.sh), which sets the password in %s everywhere", err, r.secretsEnv)
+	}
+	fmt.Fprintf(r.out, "  - role %s: password changed, and the database accepts it\n", role)
+	for i, t := range targets {
+		info, err := os.Stat(t.real)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(t.real, info, edited[i]); err != nil {
+			return fmt.Errorf("write %s: %w", t.path, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: [database] url updated\n", t.path)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, platform.ConfigSecretName, map[string][]byte{platform.ConfigSecretKey: podConfig}); err != nil {
+			return err
+		}
+	}
+	return r.rollAPI(ctx)
+}
+
+// withPassword is a database URL with its password replaced.
+func withPassword(raw, password string) (string, error) {
+	u, err := neturl.Parse(raw)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return "", errors.New("its [database] url names no role")
+	}
+	u.User = neturl.UserPassword(u.User.Username(), password)
+	return u.String(), nil
+}
+
+// scramIterations is PostgreSQL's default scram_iterations.
+const scramIterations = 4096
+
+// scramVerifier is the SCRAM-SHA-256 verifier PostgreSQL stores for a password
+// (RFC 5802 and RFC 7677, in the form libpq's PQencryptPasswordConn makes).
+// ALTER ROLE stores a verifier as it is given, so the password itself never
+// reaches the server, where a failing statement is logged with its text.
+func scramVerifier(password string, salt []byte, iterations int) (string, error) {
+	salted, err := pbkdf2.Key(sha256.New, password, salt, iterations, sha256.Size)
+	if err != nil {
+		return "", err
+	}
+	mac := func(msg string) []byte {
+		h := hmac.New(sha256.New, salted)
+		h.Write([]byte(msg))
+		return h.Sum(nil)
+	}
+	stored := sha256.Sum256(mac("Client Key"))
+	b64 := base64.StdEncoding.EncodeToString
+	return fmt.Sprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, b64(salt), b64(stored[:]), b64(mac("Server Key"))), nil
+}
+
+// alterRoleInPod runs ALTER ROLE as the superuser inside the database's
+// container, over its socket, with the statement on stdin.
+func alterRoleInPod(ctx context.Context, deployment, role, verifier string) error {
+	ns, name, _ := strings.Cut(deployment, "/")
+	return kubectlWithInput(ctx, []byte(alterRoleSQL(role, verifier)), "-n", ns, "exec", "-i", "deploy/"+name, "-c", platform.PostgresContainer, "--",
+		"psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres")
+}
+
+// alterRoleSQL sets role's password to a SCRAM verifier, which holds no quote.
+func alterRoleSQL(role, verifier string) string {
+	return "ALTER ROLE \"" + strings.ReplaceAll(role, `"`, `""`) + "\" WITH PASSWORD '" + verifier + "';\n"
 }
 
 // setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file

cmd/felis/rotatetoken_test.go

@@ -3,13 +3,20 @@ package main
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/maintenance"
+
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -22,52 +29,111 @@ type rotationRig struct {
 	out    *bytes.Buffer
 	events []string
 	dir    string
+	clock  time.Time
 }
 
 func tokenSecret(ns, name, val string) *corev1.Secret {
+	return keySecret(ns, name, "token", val)
+}
+
+func keySecret(ns, name, key, val string) *corev1.Secret {
 	return &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name},
-		Data:       map[string][]byte{"token": []byte(val)},
+		Data:       map[string][]byte{key: []byte(val)},
 	}
 }
 
+// serverPod is a game server's pod as the operator labels it.
 func serverPod(ns, name, server string) *corev1.Pod {
 	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
-		Labels: map[string]string{"felis.lolicon.best/server": server}}}
+		Labels: map[string]string{
+			"felis.lolicon.best/server":     server,
+			"felis.lolicon.best/managed-by": "felis-operator",
+			"felis.lolicon.best/component":  "server",
+		}}}
+}
+
+// backupPod is a backup Job's pod as internal/backupjob labels it: it carries
+// the server's label too, and no rotation may take it for the server's own.
+func backupPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{
+			"felis.lolicon.best/server":    server,
+			"app.kubernetes.io/managed-by": "felis-backup",
+			"app.kubernetes.io/component":  "world-backup",
+		}}}
 }
 
 func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
 	t.Helper()
-	rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir()}
+	rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir(), clock: time.Unix(1_800_000_000, 0)}
 	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
 	rig.r = tokenRotator{
-		cl:          rig.cl,
-		controlNS:   "felis",
-		minecraftNS: "minecraft",
-		buildNS:     "felis-build",
-		secretsEnv:  filepath.Join(rig.dir, "secrets.env"),
-		linkProps:   filepath.Join(rig.dir, "felis-link.properties"),
-		newToken:    func() (string, error) { return "NEWTOKEN", nil },
-		rollAPI: func(context.Context) error {
-			rig.events = append(rig.events, "roll-api")
+		cl:             rig.cl,
+		controlNS:      "felis",
+		minecraftNS:    "minecraft",
+		buildNS:        "felis-build",
+		secretsEnv:     filepath.Join(rig.dir, "secrets.env"),
+		linkProps:      filepath.Join(rig.dir, "felis-link.properties"),
+		forwardingFile: filepath.Join(rig.dir, "forwarding.secret"),
+		hostTOML:       filepath.Join(rig.dir, "felis.host.toml"),
+		podTOML:        filepath.Join(rig.dir, "felis.pod.toml"),
+		defaultTOML:    filepath.Join(rig.dir, "felis.toml"),
+		newToken:       func() (string, error) { return "NEWTOKEN", nil },
+		rollout: func(_ context.Context, deployment string) error {
+			rig.events = append(rig.events, "roll "+deployment)
 			return nil
 		},
 		restartUnit: func(_ context.Context, unit string) error {
 			rig.events = append(rig.events, "restart "+unit)
 			return nil
 		},
-		out: rig.out,
+		proxyLog: func(context.Context, time.Time) string { return "" },
+		alterRole: func(context.Context, string, string, string) error {
+			rig.events = append(rig.events, "alter-role")
+			return nil
+		},
+		verifyDB: func(context.Context, string) error {
+			rig.events = append(rig.events, "verify-db")
+			return nil
+		},
+		// Every look at the clock moves it a second on, so a wait measured with it
+		// ends after a known number of looks.
+		now: func() time.Time {
+			rig.clock = rig.clock.Add(time.Second)
+			return rig.clock
+		},
+		reloadWait: 5 * time.Second,
+		out:        rig.out,
 	}
 	return rig
 }
 
-func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
+func (rig *rotationRig) secretKey(t *testing.T, ns, name, key string) string {
 	t.Helper()
 	var s corev1.Secret
 	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
 		return "<missing>"
 	}
-	return string(s.Data["token"])
+	return string(s.Data[key])
+}
+
+func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
+	t.Helper()
+	return rig.secretKey(t, ns, name, "token")
+}
+
+func (rig *rotationRig) pods(t *testing.T) string {
+	t.Helper()
+	var pods corev1.PodList
+	if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
+		t.Fatal(err)
+	}
+	names := make([]string, len(pods.Items))
+	for i, p := range pods.Items {
+		names[i] = p.Name
+	}
+	return strings.Join(names, ",")
 }
 
 func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
@@ -80,6 +146,15 @@ func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
 	}
 }
 
+func readTestFile(t *testing.T, path string) string {
+	t.Helper()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(raw)
+}
+
 func TestRotateLimboToken(t *testing.T) {
 	rig := newRotationRig(t,
 		tokenSecret("felis", "felis-limbo-token", "old"),
@@ -87,14 +162,15 @@ func TestRotateLimboToken(t *testing.T) {
 		tokenSecret("felis", "felis-service-token", "proxy"),
 		serverPod("minecraft", "login-0", "login"),
 		serverPod("minecraft", "survival-0", "survival"),
+		backupPod("minecraft", "login-backup-x", "login"),
 	)
 	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600)
 
 	// felis-api must roll only after both copies hold the new value, and the login
 	// pod must still be there then: restarting it earlier would have it present
 	// the new token to an api that does not know it yet.
-	rig.r.rollAPI = func(context.Context) error {
-		rig.events = append(rig.events, "roll-api")
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
 		if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" {
 			t.Errorf("api rolled while the control Secret held %q", got)
 		}
@@ -107,13 +183,12 @@ func TestRotateLimboToken(t *testing.T) {
 		}
 		return nil
 	}
-	if err := rig.r.rotate(context.Background(), "limbo"); err != nil {
+	if err := rig.r.rotate(context.Background(), "limbo", true); err != nil {
 		t.Fatal(err)
 	}
 
-	raw, _ := os.ReadFile(rig.r.secretsEnv)
-	if string(raw) != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
-		t.Errorf("secrets.env = %q", raw)
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
+		t.Errorf("secrets.env = %q", got)
 	}
 	if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 {
 		t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm())
@@ -121,14 +196,12 @@ func TestRotateLimboToken(t *testing.T) {
 	if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" {
 		t.Errorf("the proxy's token changed to %q", got)
 	}
-	var pods corev1.PodList
-	if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
-		t.Fatal(err)
+	// The login pod restarted; a user server and the backup Job's pod, which
+	// carries the login server's label too, are untouched.
+	if got := rig.pods(t); got != "login-backup-x,survival-0" {
+		t.Errorf("pods left = %s, want login-backup-x,survival-0", got)
 	}
-	if len(pods.Items) != 1 || pods.Items[0].Name != "survival-0" {
-		t.Errorf("pods left = %v, want only survival-0 (the login pod restarted, user servers untouched)", pods.Items)
-	}
-	if strings.Join(rig.events, ",") != "roll-api" {
+	if strings.Join(rig.events, ",") != "roll felis-api" {
 		t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events)
 	}
 	if strings.Contains(rig.out.String(), "NEWTOKEN") {
@@ -136,21 +209,56 @@ func TestRotateLimboToken(t *testing.T) {
 	}
 }
 
-func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) {
+const testLinkProps = "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n"
+
+func reloadLine(token string) string {
+	return "[12:00:00 INFO] [felis-link]: Felis: service-token reloaded from /x/felis-link.properties (fingerprint " + tokenFingerprint(token) + ")\n"
+}
+
+// The host proxy re-reads its token: the rotation waits for it to say so and
+// leaves it running.
+func TestRotateVelocityTokenReloadsTheHostProxy(t *testing.T) {
 	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
 	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
-	writeTestFile(t, rig.r.linkProps, "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n", 0o640)
-
-	if err := rig.r.rotate(context.Background(), "velocity"); err != nil {
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	if err := os.Chtimes(rig.r.linkProps, written, written); err != nil {
 		t.Fatal(err)
 	}
-	raw, _ := os.ReadFile(rig.r.linkProps)
-	if string(raw) != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
-		t.Errorf("felis-link.properties = %q", raw)
+
+	// The proxy logs its reload on its next call to felis-api, which may be
+	// while the api rolls: the log must be read from before that.
+	var reloadedAt time.Time
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := readTestFile(t, rig.r.linkProps); !strings.Contains(got, "service-token=NEWTOKEN\n") {
+			t.Errorf("api rolled before the proxy's file held the new token: %q", got)
+		}
+		reloadedAt = rig.clock
+		return nil
 	}
-	if info, _ := os.Stat(rig.r.linkProps); info.Mode().Perm() != 0o640 {
+	looks := 0
+	rig.r.proxyLog = func(_ context.Context, since time.Time) string {
+		looks++
+		log := reloadLine("old") // an earlier rotation's
+		if looks >= 3 && !since.After(reloadedAt) {
+			log += reloadLine("NEWTOKEN")
+		}
+		return log
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.linkProps); got != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
+		t.Errorf("felis-link.properties = %q", got)
+	}
+	info, _ := os.Stat(rig.r.linkProps)
+	if info.Mode().Perm() != 0o640 {
 		t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm())
 	}
+	if !info.ModTime().Equal(written) {
+		t.Errorf("properties mtime = %v, want it kept at %v (felis domain check would read the proxy as stale)", info.ModTime(), written)
+	}
 	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
 		t.Errorf("control Secret = %q, want NEWTOKEN", got)
 	}
@@ -158,9 +266,49 @@ func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) {
 	if got := rig.secret(t, "minecraft", "felis-service-token"); got != "<missing>" {
 		t.Errorf("rotation copied the proxy token into minecraft (%q)", got)
 	}
-	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want the api roll and no proxy restart", rig.events)
+	}
+	if looks != 3 {
+		t.Errorf("the log was read %d times, want 3 (until the line appeared)", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: took the new token from its properties; players stayed connected") {
+		t.Errorf("output does not say the players stayed: %s", out)
+	}
+}
+
+// A proxy that never logs the new fingerprint (an older plugin, a stuck proxy)
+// is restarted once the wait is over.
+func TestRotateVelocityTokenRestartsAProxyThatDoesNotReload(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	looks := 0
+	rig.r.proxyLog = func(context.Context, time.Time) string {
+		looks++
+		return reloadLine("old")
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api,restart felis-velocity" {
 		t.Errorf("events = %v, want the api roll then the proxy restart", rig.events)
 	}
+	// reloadWait is 5s and each look moves the clock a second.
+	if looks != 5 {
+		t.Errorf("the log was read %d times, want 5", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: had not taken the new token within 5s, restarted") {
+		t.Errorf("output does not explain the restart: %s", out)
+	}
+}
+
+// The proxy and the Java plugin name a token by the same fingerprint
+// (plugins/shared LinkConfigLoaderTest fileTokenFollowsTheFile).
+func TestTokenFingerprintMatchesThePlugin(t *testing.T) {
+	if got := tokenFingerprint("new-token"); got != "348e9df2a42b" {
+		t.Errorf("tokenFingerprint(new-token) = %q, want 348e9df2a42b", got)
+	}
 }
 
 // An external proxy has no felis-link.properties here: the Secret still rotates,
@@ -168,13 +316,13 @@ func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) {
 // without printing it.
 func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) {
 	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
-	if err := rig.r.rotate(context.Background(), "velocity"); err != nil {
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
 		t.Fatal(err)
 	}
 	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
 		t.Errorf("control Secret = %q, want NEWTOKEN", got)
 	}
-	if strings.Join(rig.events, ",") != "roll-api" {
+	if strings.Join(rig.events, ",") != "roll felis-api" {
 		t.Errorf("events = %v, want no proxy restart", rig.events)
 	}
 	out := rig.out.String()
@@ -187,7 +335,7 @@ func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
 	rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old"))
 	// An install from before per-caller tokens has no BUILD_TOKEN line yet.
 	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600)
-	if err := rig.r.rotate(context.Background(), "build"); err != nil {
+	if err := rig.r.rotate(context.Background(), "build", true); err != nil {
 		t.Fatal(err)
 	}
 	if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" {
@@ -196,9 +344,8 @@ func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
 	if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" {
 		t.Errorf("control Secret = %q, want NEWTOKEN", got)
 	}
-	raw, _ := os.ReadFile(rig.r.secretsEnv)
-	if string(raw) != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
-		t.Errorf("secrets.env = %q", raw)
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
 	}
 }
 
@@ -209,31 +356,453 @@ func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) {
 		tokenSecret("felis", "felis-limbo-token", "old"),
 		serverPod("minecraft", "login-0", "login"),
 	)
-	rig.r.rollAPI = func(context.Context) error { return errors.New("rollout timed out") }
-	err := rig.r.rotate(context.Background(), "limbo")
+	rig.r.rollout = func(context.Context, string) error { return errors.New("rollout timed out") }
+	err := rig.r.rotate(context.Background(), "limbo", true)
 	if err == nil || !strings.Contains(err.Error(), "rollout timed out") {
 		t.Fatalf("err = %v, want the rollout failure", err)
 	}
-	var pod corev1.Pod
-	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
-		t.Error("the login pod was restarted although the api never rolled")
+	if got := rig.pods(t); got != "login-0" {
+		t.Errorf("pods left = %s: the login pod was restarted although the api never rolled", got)
 	}
 }
 
-func TestRotateRefusesAnUnknownCaller(t *testing.T) {
+func TestRotateRefusesAnUnknownCredential(t *testing.T) {
 	rig := newRotationRig(t)
 	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600)
-	if err := rig.r.rotate(context.Background(), "admin"); err == nil {
-		t.Fatal("rotated a token for an unknown caller")
+	if err := rig.r.rotate(context.Background(), "admin", true); err == nil {
+		t.Fatal("rotated an unknown credential")
 	}
-	if raw, _ := os.ReadFile(rig.r.secretsEnv); string(raw) != "SERVICE_TOKEN=proxy\n" {
-		t.Errorf("secrets.env = %q, want it untouched", raw)
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\n" {
+		t.Errorf("secrets.env = %q, want it untouched", got)
 	}
 	if len(rig.events) != 0 {
 		t.Errorf("events = %v, want nothing touched", rig.events)
 	}
 }
 
+const (
+	testHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	testPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+// installedRig is a host as the installer leaves it, with every credential in
+// place, for the plan test.
+func installedRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-service-token", "old"),
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-build-token", "old"),
+		tokenSecret("felis-build", "felis-build-token", "old"),
+		tokenSecret("felis", "felis-ops-token", "old"),
+		keySecret("felis", "felis-registry-auth", "platform", "old"),
+		keySecret("felis-build", "felis-registry-push", "password", "old"),
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	return rig
+}
+
+// Without -yes every rotation prints its plan and changes nothing.
+func TestRotatePlanChangesNothing(t *testing.T) {
+	for _, kind := range rotationKinds() {
+		t.Run(kind, func(t *testing.T) {
+			rig := installedRig(t)
+			files := map[string]string{}
+			for _, p := range []string{rig.r.secretsEnv, rig.r.linkProps, rig.r.forwardingFile, rig.r.hostTOML, rig.r.podTOML} {
+				files[p] = readTestFile(t, p)
+			}
+			rig.r.newToken = func() (string, error) {
+				t.Error("a plan generated a token")
+				return "NEWTOKEN", nil
+			}
+			if err := rig.r.rotate(context.Background(), kind, false); err != nil {
+				t.Fatal(err)
+			}
+			for p, before := range files {
+				if got := readTestFile(t, p); got != before {
+					t.Errorf("%s changed to %q", filepath.Base(p), got)
+				}
+			}
+			for _, s := range [][3]string{
+				{"felis", "felis-service-token", "token"}, {"felis", "felis-limbo-token", "token"},
+				{"felis-build", "felis-build-token", "token"}, {"felis", "felis-ops-token", "token"},
+				{"felis", "felis-registry-auth", "platform"}, {"felis-build", "felis-registry-push", "password"},
+				{"minecraft", "felis-forwarding-secret", "secret"},
+			} {
+				if got := rig.secretKey(t, s[0], s[1], s[2]); got != "old" {
+					t.Errorf("Secret %s/%s changed to %q", s[0], s[1], got)
+				}
+			}
+			if got := rig.secretKey(t, "minecraft", "felis-config", "felis.toml"); got != testPodTOML {
+				t.Errorf("felis-config changed to %q", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want none", rig.events)
+			}
+			if got := rig.pods(t); got != "login-0,survival-0" {
+				t.Errorf("pods left = %s, want all", got)
+			}
+			out := rig.out.String()
+			if !strings.HasSuffix(out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes "+kind+"\n") {
+				t.Errorf("the plan does not end with how to go ahead: %s", out)
+			}
+			// Each plan names the interruption it causes.
+			want := apiRestartNote
+			if kind == kindForwarding {
+				want = "everyone online is disconnected"
+			}
+			if !strings.Contains(out, want) {
+				t.Errorf("the plan does not say %q: %s", want, out)
+			}
+		})
+	}
+}
+
+func TestRotateRegistryTokens(t *testing.T) {
+	rig := newRotationRig(t,
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: "felis-registry-auth"},
+			Data: map[string][]byte{"platform": []byte("a"), "build": []byte("b"), "prune": []byte("c")}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis-build", Name: "felis-registry-push"},
+			Data: map[string][]byte{"username": []byte("build"), "password": []byte("b")}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=a\nREGISTRY_BUILD_TOKEN=b\nREGISTRY_PRUNE_TOKEN=c\n", 0o600)
+	n := 0
+	rig.r.newToken = func() (string, error) {
+		n++
+		return fmt.Sprintf("NEWTOKEN%d", n), nil
+	}
+	// The registry reads its tokens as it starts: it restarts after the Secret
+	// holds them, and felis-api (the prune token) after that.
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", "prune"); got != "NEWTOKEN3" {
+			t.Errorf("%s rolled while the prune token was %q", deployment, got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindRegistry, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=NEWTOKEN1\nREGISTRY_BUILD_TOKEN=NEWTOKEN2\nREGISTRY_PRUNE_TOKEN=NEWTOKEN3\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	for key, want := range map[string]string{"platform": "NEWTOKEN1", "build": "NEWTOKEN2", "prune": "NEWTOKEN3"} {
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", key); got != want {
+			t.Errorf("felis-registry-auth %s = %q, want %s", key, got, want)
+		}
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "password"); got != "NEWTOKEN2" {
+		t.Errorf("the build Jobs' push password = %q, want the build token", got)
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "username"); got != "build" {
+		t.Errorf("the build Jobs' push username = %q, want build", got)
+	}
+	if strings.Join(rig.events, ",") != "roll registry,roll felis-api" {
+		t.Errorf("events = %v, want the registry then felis-api", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("a new token was printed: %s", rig.out.String())
+	}
+}
+
+func TestRotateForwardingSecret(t *testing.T) {
+	lock := time.Unix(1_800_000_000, 0)
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		serverPod("minecraft", "survival-0", "survival"),
+		serverPod("minecraft", "lobby-0", "lobby"),
+		// creative is being backed up: a running Job holds its world.
+		serverPod("minecraft", "creative-0", "creative"),
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "creative-backup",
+			Labels: map[string]string{maintenance.LabelServer: "creative", maintenance.LabelManagedBy: "felis-backup"}}},
+		// survival's last backup is over; its finished pod stays until the Job's TTL.
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival-backup",
+			Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: "felis-backup"}},
+			Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}},
+		backupPod("minecraft", "survival-backup-x", "survival"),
+		// A pod already on its way out is neither counted nor deleted again.
+		func() *corev1.Pod {
+			p := serverPod("minecraft", "lobby-old", "lobby")
+			p.DeletionTimestamp = &metav1.Time{Time: lock}
+			p.Finalizers = []string{"felis.lolicon.best/test"}
+			return p
+		}(),
+		// skyblock's restore has just been admitted, its Job not created yet.
+		serverPod("minecraft", "skyblock-0", "skyblock"),
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "skyblock",
+			Annotations: map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lock)}}},
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival"}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nFORWARDING_SECRET=old\n", 0o600)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	// The proxy restarts after the servers were told to: a player who rejoins
+	// must not meet a server still on the old secret.
+	rig.r.restartUnit = func(_ context.Context, unit string) error {
+		rig.events = append(rig.events, "restart "+unit)
+		if got := rig.pods(t); strings.Contains(got, "survival-0") {
+			t.Errorf("the proxy restarted before the servers (pods %s)", got)
+		}
+		if got := readTestFile(t, rig.r.forwardingFile); got != "NEWTOKEN" {
+			t.Errorf("the proxy restarted on forwarding.secret %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nFORWARDING_SECRET=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if info, _ := os.Stat(rig.r.forwardingFile); info.Mode().Perm() != 0o640 {
+		t.Errorf("forwarding.secret mode = %v, want 0640", info.Mode().Perm())
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+			t.Errorf("Secret %s/felis-forwarding-secret = %q, want NEWTOKEN", ns, got)
+		}
+	}
+	if got := rig.pods(t); got != "creative-0,lobby-old,skyblock-0,survival-backup-x" {
+		t.Errorf("pods left = %s, want the held servers, the terminating pod and the backup's pod", got)
+	}
+	if strings.Join(rig.events, ",") != "restart felis-velocity" {
+		t.Errorf("events = %v, want only the proxy restart (felis-api does not hold this secret)", rig.events)
+	}
+	out := rig.out.String()
+	for _, want := range []string{
+		"every running server restarts to read it (2 now)",
+		"left running, because a backup, restore or file write holds its world: creative (backup), skyblock (restore).",
+		"  - servers: 2 restarting on the new secret\n",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output lacks %q: %s", want, out)
+		}
+	}
+	if strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("the new secret was printed: %s", out)
+	}
+}
+
+func TestRotateForwardingSecretForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, keySecret("felis", "felis-forwarding-secret", "secret", "old"))
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secretKey(t, "felis", "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want no proxy restart on this host", rig.events)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "put the value in Secret felis/felis-forwarding-secret into your proxy's forwarding secret file") {
+		t.Errorf("output does not say where the value is: %s", out)
+	}
+}
+
+// dbRig is an installed host for the database rotation: felis.toml links to the
+// host copy, as the installer makes it.
+func dbRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n", 0o600)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	if err := os.Symlink("felis.host.toml", rig.r.defaultTOML); err != nil {
+		t.Fatal(err)
+	}
+	return rig
+}
+
+const (
+	wantHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	wantPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+func TestRotateDatabasePassword(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.alterRole = func(_ context.Context, deployment, role, verifier string) error {
+		rig.events = append(rig.events, "alter-role")
+		if deployment != "felis/felis-postgres" || role != "felis" {
+			t.Errorf("alterRole(%q, %q), want felis/felis-postgres and felis", deployment, role)
+		}
+		if !strings.Contains(readTestFile(t, rig.r.secretsEnv), "DB_PASSWORD=NEWTOKEN\n") {
+			t.Error("the role changed before secrets.env recorded the new password")
+		}
+		// The verifier is the new password's, under the salt it carries.
+		m := regexp.MustCompile(`^SCRAM-SHA-256\$4096:([^$]+)\$`).FindStringSubmatch(verifier)
+		if m == nil {
+			t.Fatalf("verifier %q is not a SCRAM-SHA-256 verifier", verifier)
+		}
+		salt, err := base64.StdEncoding.DecodeString(m[1])
+		if err != nil || len(salt) != 16 {
+			t.Fatalf("verifier salt %q: %v", m[1], err)
+		}
+		if want, _ := scramVerifier("NEWTOKEN", salt, 4096); verifier != want {
+			t.Errorf("verifier = %q, want %q", verifier, want)
+		}
+		return nil
+	}
+	// The configs change only once the database accepts the new password.
+	rig.r.verifyDB = func(_ context.Context, url string) error {
+		rig.events = append(rig.events, "verify-db")
+		if url != "postgres://felis:[email protected]:30432/felis?sslmode=disable" {
+			t.Errorf("verified with %q, want the host URL with the new password", url)
+		}
+		if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+			t.Errorf("the host config changed before the database accepted the password: %q", got)
+		}
+		return nil
+	}
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("felis-api rolled on felis-config %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindDB, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=NEWTOKEN\nSERVICE_TOKEN=s\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != wantHostTOML {
+		t.Errorf("host config = %q", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != wantPodTOML {
+		t.Errorf("pod config = %q", got)
+	}
+	if info, err := os.Lstat(rig.r.defaultTOML); err != nil || info.Mode()&os.ModeSymlink == 0 {
+		t.Errorf("felis.toml is no longer the link to the host copy (%v)", err)
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("Secret %s/felis-config = %q", ns, got)
+		}
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db,roll felis-api" {
+		t.Errorf("events = %v", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new password was printed: %s", rig.out.String())
+	}
+}
+
+// A password the database does not accept leaves the configs on the old one
+// and felis-api running: the installer's record, already new, is the way back.
+func TestRotateDatabasePasswordStopsWhenTheDatabaseRefuses(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.verifyDB = func(context.Context, string) error {
+		rig.events = append(rig.events, "verify-db")
+		return errors.New("password authentication failed")
+	}
+	err := rig.r.rotate(context.Background(), kindDB, true)
+	if err == nil || !strings.Contains(err.Error(), "password authentication failed") || !strings.Contains(err.Error(), "run the installer again") {
+		t.Fatalf("err = %v, want the refusal and the way back", err)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+		t.Errorf("host config = %q, want it unchanged", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != testPodTOML {
+		t.Errorf("pod config = %q, want it unchanged", got)
+	}
+	if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != testPodTOML {
+		t.Errorf("felis-config = %q, want it unchanged", got)
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db" {
+		t.Errorf("events = %v, want no api roll", rig.events)
+	}
+}
+
+// Everything that can refuse does so before the installer's record or the
+// role change; what the host config alone shows is refused by the plan too.
+func TestRotateDatabasePasswordRefusesEarly(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		apply bool
+		setup func(t *testing.T, rig *rotationRig)
+		want  string
+	}{
+		{"a database the installer does not run", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "deployment = \"felis/felis-postgres\"\n", "", 1), 0o600)
+		}, "[database] deployment is unset"},
+		{"a database url without a role", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "felis:oldpw@", "", 1), 0o600)
+		}, "names no role"},
+		{"a config copy the line editor cannot edit", true, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.podTOML, "[database]\nurl = \"\"\"\npostgres://felis:[email protected]:5432/felis\"\"\"\n", 0o600)
+		}, "set the password in its [database] url by hand"},
+		{"a pod copy that is the host copy", true, func(t *testing.T, rig *rotationRig) {
+			if err := os.Remove(rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.Symlink("felis.host.toml", rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+		}, "resolves to the same file as"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			rig := dbRig(t)
+			tc.setup(t, rig)
+			err := rig.r.rotate(context.Background(), kindDB, tc.apply)
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err = %v, want %q", err, tc.want)
+			}
+			if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n" {
+				t.Errorf("secrets.env = %q, want it untouched", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want nothing done", rig.events)
+			}
+		})
+	}
+}
+
+// The RFC 7677 example (user "user", password "pencil"), with the stored and
+// server keys computed by openssl's PBKDF2 and HMAC rather than this code.
+func TestScramVerifier(t *testing.T) {
+	salt, _ := base64.StdEncoding.DecodeString("W22ZaJ0SNY7soEsUEjb6gQ==")
+	got, err := scramVerifier("pencil", salt, 4096)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := "SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==$WG5d8oPm3OtcPnkdi4Uo7BkeZkBFzpcXkuLmtbsT4qY=:wfPLwcE6nTWhTAmQ7tl2KeoiWGPlZqQxSrmfPwDl2dU="; got != want {
+		t.Errorf("scramVerifier = %q\nwant            %q", got, want)
+	}
+}
+
+func TestAlterRoleSQL(t *testing.T) {
+	if got := alterRoleSQL(`fe"lis`, "SCRAM-SHA-256$4096:c2FsdA==$a:b"); got != "ALTER ROLE \"fe\"\"lis\" WITH PASSWORD 'SCRAM-SHA-256$4096:c2FsdA==$a:b';\n" {
+		t.Errorf("alterRoleSQL = %q", got)
+	}
+}
+
+// installerSecretKeys is every secrets.env key a rotation writes.
+func installerSecretKeys() map[string]bool {
+	keys := map[string]bool{installerForwardingKey: true, installerDBKey: true}
+	for _, k := range installerTokenKeys {
+		keys[k] = true
+	}
+	for _, k := range installerRegistryKeys {
+		keys[k] = true
+	}
+	return keys
+}
+
 // The installer and rotate-token must agree on where each caller's token lives:
 // rotate-token writes installerTokenKeys into secrets.env, and the installer
 // applies those same keys to the Secrets on its next run. A key the installer
@@ -249,12 +818,6 @@ func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
 		if !ok {
 			t.Fatalf("installerTokenKeys names unknown caller %q", caller)
 		}
-		if !strings.Contains(script, key+`="${`+key+`:-$(openssl rand -hex 32)}"`) {
-			t.Errorf("bootstrap.sh does not generate %s", key)
-		}
-		if !strings.Contains(script, key+"=${"+key+"}\n") {
-			t.Errorf("bootstrap.sh does not persist %s to secrets.env", key)
-		}
 		apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`)
 		if !apply.MatchString(script) {
 			t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key)
@@ -281,3 +844,40 @@ func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
 		t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames()))
 	}
 }
+
+// Every value the installer keeps in secrets.env has a rotation that rewrites
+// that very key, and every key a rotation writes is one the installer
+// generates, keeps and so re-applies on its next run.
+func TestInstallerSecretsAreAllRotatable(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	m := regexp.MustCompile(`(?s)write_file_atomic "\$SECRETS_ENV" 0600 <<EOF\n(.*?)\nEOF\n`).FindStringSubmatch(script)
+	if m == nil {
+		t.Fatal("bootstrap.sh: no secrets.env heredoc found")
+	}
+	persisted := map[string]bool{}
+	for _, line := range strings.Split(m[1], "\n") {
+		key, value, _ := strings.Cut(line, "=")
+		if value != "${"+key+"}" {
+			t.Errorf("secrets.env line %q is not KEY=${KEY}", line)
+		}
+		persisted[key] = true
+	}
+	rotated := installerSecretKeys()
+	for key := range persisted {
+		if !rotated[key] {
+			t.Errorf("secrets.env keeps %s, which no rotation replaces", key)
+		}
+	}
+	for key := range rotated {
+		if !persisted[key] {
+			t.Errorf("a rotation writes %s, which the installer does not keep in secrets.env", key)
+		}
+		if !regexp.MustCompile(`\n  ` + key + `="\$\{` + key + `:-\$\(openssl rand -hex [0-9]+\)\}"\n`).MatchString(script) {
+			t.Errorf("bootstrap.sh does not generate %s when secrets.env lacks it", key)
+		}
+	}
+}

cmd/felis/run.go

@@ -31,7 +31,7 @@ Commands:
   apply             Create a MinecraftServer CRD (direct K8s write; use -f server.json)
   setup             Run host bootstrap + first-run setup console (TUI; requires root/sudo)
   converge          Fill in fields a newer desired spec added to already-installed system servers
-  rotate-token      Replace one internal caller's token and restart what holds it (velocity|limbo|build|ops; requires root/sudo)
+  rotate-token      Replace a generated credential and restart what reads it (velocity|limbo|build|ops|registry|forwarding|db; prints the plan, -yes applies; requires root/sudo)
   domain            Move the install to a new root domain on every surface that carries it, or check each one (set|check; requires root/sudo)
   watchdog          Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
   version           Print the build stamp of this binary