fix(rotate-token): 加确认步骤、velocity 热加载免踢人，补齐 registry/forwarding/db 轮换

cmd/felis/domain.go

@@ -235,7 +235,7 @@ func verifyTOMLEdit(orig, edited []byte, edits []tomlStringEdit) error {
 		t[e.key] = e.value
 	}
 	if !reflect.DeepEqual(want, got) {
-		return errors.New("a line edit would change more than the domain keys (a multi-line value, or a quoted or dotted key?)")
+		return errors.New("a line edit would change more than the keys it sets (a multi-line value, or a quoted or dotted key?)")
 	}
 	return nil
 }
@@ -591,9 +591,15 @@ type tomlTarget struct{ path, real string }
 // tomlTargets are the host and pod copies, and felis.toml when it is a file of
 // its own rather than the link to the host copy.
 func (h domainHost) tomlTargets() ([]tomlTarget, error) {
+	return tomlTargetsOf(h.paths.hostTOML, h.paths.podTOML, h.paths.defaultTOML)
+}
+
+// tomlTargetsOf is the host copy, the pod copy, and def when it exists and is
+// not a link to one of them.
+func tomlTargetsOf(host, pod, def string) ([]tomlTarget, error) {
 	var out []tomlTarget
 	seen := map[string]bool{}
-	for i, p := range []string{h.paths.hostTOML, h.paths.podTOML, h.paths.defaultTOML} {
+	for i, p := range []string{host, pod, def} {
 		real, err := filepath.EvalSymlinks(p)
 		if errors.Is(err, fs.ErrNotExist) && i == 2 {
 			continue

cmd/felis/rotatetoken.go

@@ -2,40 +2,70 @@ package main
 
 import (
 	"context"
+	"crypto/hmac"
+	"crypto/pbkdf2"
 	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"io/fs"
+	neturl "net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/maintenance"
 	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/operator"
 	"felis.lolicon.best/internal/platform"
+	"felis.lolicon.best/internal/registrygate"
 
+	"github.com/BurntSushi/toml"
+	"github.com/jackc/pgx/v5"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-// rotate-token replaces one internal caller's token (naming.CallerTokens): a new
-// value goes into the installer's record, the control-namespace Secret and the
-// replica the caller's pods mount, felis-api rolls so it accepts only the new
-// value, and then the caller restarts so it presents it. Between the api's
-// rollout and the caller's restart the caller is turned away with 401; for the
-// login gate and the proxy that is the few seconds of a pod or unit restart.
+// rotate-token replaces one credential the installer generated: an internal
+// caller's token (naming.CallerTokens), the registry's write tokens, the
+// Velocity forwarding secret or the database password. The new value goes into
+// the installer's record first, so that whatever fails later a re-run of the
+// installer puts it everywhere; then into the Secrets and files that carry it;
+// then whatever read the old value at start restarts. Without -yes it prints
+// what it would change and what that interrupts, and changes nothing.
 
 const (
 	defaultSecretsEnvPath = "/etc/felis/secrets.env"
 	defaultLinkPropsPath  = "/opt/felis/velocity/plugins/felis-link/felis-link.properties"
+	defaultForwardingPath = "/opt/felis/velocity/forwarding.secret"
 	velocityUnit          = "felis-velocity"
+	apiDeployment         = "felis-api"
+	registryDeployment    = "registry"
+
+	kindRegistry   = "registry"
+	kindForwarding = "forwarding"
+	kindDB         = "db"
+
+	// proxyReloadWait is how long the host proxy gets, once felis-api has rolled,
+	// to show it re-read its token: it reads the file again on its next call to
+	// felis-api, and it calls every 15 seconds.
+	proxyReloadWait = 60 * time.Second
+
+	// apiRestartNote is in every plan: felis-api runs as one replica replaced in
+	// place, so its restart is a short outage of everything that talks to it.
+	apiRestartNote = "felis-api restarts (a single replica): the panel, sign-in and the proxy's calls are unavailable for the few seconds that takes"
 )
 
 // installerTokenKeys names each caller's token in the installer's secrets.env
@@ -49,20 +79,54 @@ var installerTokenKeys = map[string]string{
 	"ops":      "OPS_TOKEN",
 }
 
+// installerRegistryKeys names the registry principals' tokens in secrets.env,
+// in registrygate.Principals order.
+var installerRegistryKeys = map[string]string{
+	registrygate.PrincipalPlatform: "REGISTRY_PLATFORM_TOKEN",
+	registrygate.PrincipalBuild:    "REGISTRY_BUILD_TOKEN",
+	registrygate.PrincipalPrune:    "REGISTRY_PRUNE_TOKEN",
+}
+
+// installerForwardingKey and installerDBKey name the forwarding secret and the
+// database password in secrets.env.
+const (
+	installerForwardingKey = "FORWARDING_SECRET"
+	installerDBKey         = "DB_PASSWORD"
+)
+
 type tokenRotator struct {
 	cl          client.Client
 	controlNS   string
 	minecraftNS string
 	buildNS     string
-	// secretsEnv and linkProps are the installer's record and the proxy's
-	// felis-link.properties; a missing file is reported and skipped.
+	// secretsEnv, linkProps and forwardingFile are the installer's record and the
+	// host proxy's felis-link.properties and forwarding.secret; a missing file is
+	// reported and skipped.
 	secretsEnv     string
 	linkProps      string
+	forwardingFile string
+	// hostTOML, podTOML and defaultTOML are the config copies that carry the
+	// database URL (defaultTOML only when it is a file of its own).
+	hostTOML    string
+	podTOML     string
+	defaultTOML string
 	newToken    func() (string, error)
-	// rollAPI restarts felis-api and waits for the rollout.
-	rollAPI func(ctx context.Context) error
+	// rollout restarts a control-namespace Deployment and waits for it.
+	rollout func(ctx context.Context, deployment string) error
 	// restartUnit restarts a systemd unit on this host.
 	restartUnit func(ctx context.Context, unit string) error
+	// proxyLog is what the host proxy has logged since a moment, as far as it
+	// can be read.
+	proxyLog func(ctx context.Context, since time.Time) string
+	// alterRole stores a password verifier for a role of the database that runs
+	// as deployment ("namespace/name"); verifyDB connects with a URL.
+	alterRole func(ctx context.Context, deployment, role, verifier string) error
+	verifyDB  func(ctx context.Context, url string) error
+	now       func() time.Time
+	// reloadWait bounds the wait for the proxy to re-read its token, polled
+	// every pollEvery.
+	reloadWait time.Duration
+	pollEvery  time.Duration
 	out        io.Writer
 }
 
@@ -72,9 +136,12 @@ func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
 	cfgPath := fs.String("config", defaultSetupConfigPath, "path to felis.toml")
 	secretsEnv := fs.String("secrets-env", defaultSecretsEnvPath, "the installer's secrets file, updated so a re-run keeps the new value")
 	linkProps := fs.String("link-properties", defaultLinkPropsPath, "the host proxy's felis-link.properties (velocity only)")
+	forwarding := fs.String("forwarding-secret", defaultForwardingPath, "the host proxy's forwarding secret file (forwarding only)")
+	yes := fs.Bool("yes", false, "rotate; without it the plan is printed and nothing changes")
 	fs.Usage = func() {
-		fmt.Fprintf(stderr, "Usage: felis rotate-token [flags] <%s>\n\n", strings.Join(callerNames(), "|"))
-		fmt.Fprintln(stderr, "Replaces one internal caller's token: the Secrets, felis-api, then the caller itself.")
+		fmt.Fprintf(stderr, "Usage: felis rotate-token [-yes] [flags] <%s>\n\n", strings.Join(rotationKinds(), "|"))
+		fmt.Fprintln(stderr, "Replaces one generated credential: the installer's record, the Secrets and files that carry it, then what reads it.")
+		fmt.Fprintln(stderr, "Without -yes it prints what would change and what that interrupts.")
 		fs.PrintDefaults()
 	}
 	if err := fs.Parse(args); err != nil {
@@ -87,12 +154,13 @@ func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
 		fs.Usage()
 		return 2
 	}
-	if _, ok := callerToken(fs.Arg(0)); !ok {
-		fmt.Fprintf(stderr, "felis rotate-token: unknown caller %q (one of %s)\n", fs.Arg(0), strings.Join(callerNames(), ", "))
+	kind := fs.Arg(0)
+	if !knownRotation(kind) {
+		fmt.Fprintf(stderr, "felis rotate-token: unknown credential %q (one of %s)\n", kind, strings.Join(rotationKinds(), ", "))
 		return 2
 	}
 	if os.Geteuid() != 0 {
-		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+fs.Arg(0)+")")
+		fmt.Fprintln(stderr, "felis rotate-token: refused — rotating writes the cluster Secrets and the installer's secrets file, so it must run as root (try: sudo felis rotate-token "+kind+")")
 		return 1
 	}
 	cfg, err := config.Load(*cfgPath)
@@ -109,24 +177,43 @@ func cmdRotateToken(args []string, stdout, stderr io.Writer) int {
 	if buildNS == "" {
 		buildNS = platform.DefaultBuildNamespace
 	}
+	controlNS := platform.DefaultControlNamespace
 	r := tokenRotator{
 		cl:             cl,
-		controlNS:   platform.DefaultControlNamespace,
+		controlNS:      controlNS,
 		minecraftNS:    cfg.K8s.Namespace,
 		buildNS:        buildNS,
 		secretsEnv:     *secretsEnv,
 		linkProps:      *linkProps,
+		forwardingFile: *forwarding,
+		hostTOML:       hostSetupConfigPath,
+		podTOML:        podSetupConfigPath,
+		defaultTOML:    defaultSetupConfigPath,
 		newToken:       randomToken,
-		rollAPI: func(ctx context.Context) error {
-			if err := kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "restart", "deployment/felis-api"); err != nil {
+		rollout: func(ctx context.Context, deployment string) error {
+			if err := kubectl(ctx, "-n", controlNS, "rollout", "restart", "deployment/"+deployment); err != nil {
 				return err
 			}
-			return kubectl(ctx, "-n", platform.DefaultControlNamespace, "rollout", "status", "deployment/felis-api", "--timeout=180s")
+			return kubectl(ctx, "-n", controlNS, "rollout", "status", "deployment/"+deployment, "--timeout=180s")
 		},
 		restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) },
+		proxyLog:    journalSince,
+		alterRole:   alterRoleInPod,
+		verifyDB: func(ctx context.Context, url string) error {
+			ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+			defer cancel()
+			conn, err := pgx.Connect(ctx, url)
+			if err != nil {
+				return err
+			}
+			return conn.Close(ctx)
+		},
+		now:        time.Now,
+		reloadWait: proxyReloadWait,
+		pollEvery:  3 * time.Second,
 		out:        stdout,
 	}
-	if err := r.rotate(context.Background(), fs.Arg(0)); err != nil {
+	if err := r.rotate(context.Background(), kind, *yes); err != nil {
 		fmt.Fprintf(stderr, "felis rotate-token: %v\n", err)
 		return 1
 	}
@@ -150,6 +237,21 @@ func callerToken(name string) (naming.CallerToken, bool) {
 	return naming.CallerToken{}, false
 }
 
+// rotationKinds is every credential rotate-token replaces: the callers' tokens
+// and the installer's other generated secrets.
+func rotationKinds() []string {
+	return append(callerNames(), kindRegistry, kindForwarding, kindDB)
+}
+
+func knownRotation(kind string) bool {
+	for _, k := range rotationKinds() {
+		if k == kind {
+			return true
+		}
+	}
+	return false
+}
+
 // randomToken is 32 random bytes in hex, the shape the installer generates.
 func randomToken() (string, error) {
 	b := make([]byte, 32)
@@ -159,99 +261,527 @@ func randomToken() (string, error) {
 	return hex.EncodeToString(b), nil
 }
 
-func (r tokenRotator) rotate(ctx context.Context, caller string) error {
-	ct, ok := callerToken(caller)
-	if !ok {
-		return fmt.Errorf("unknown caller %q", caller)
-	}
-	tok, err := r.newToken()
-	if err != nil {
-		return fmt.Errorf("generate a token: %w", err)
-	}
+// tokenFingerprint is how the proxy names the token it reloaded in its log
+// (plugins/shared FileToken.fingerprint): the first twelve hex digits of its
+// SHA-256, enough to tell tokens apart and useless for finding one.
+func tokenFingerprint(token string) string {
+	sum := sha256.Sum256([]byte(token))
+	return hex.EncodeToString(sum[:])[:12]
+}
 
-	// The installer's record first: from here on, whatever fails, a re-run of the
-	// installer puts the new value everywhere.
-	switch err := setKeyValueLine(r.secretsEnv, installerTokenKeys[ct.Caller], "=", tok); {
+func (r tokenRotator) rotate(ctx context.Context, kind string, apply bool) error {
+	switch kind {
+	case kindRegistry:
+		return r.rotateRegistry(ctx, apply)
+	case kindForwarding:
+		return r.rotateForwarding(ctx, apply)
+	case kindDB:
+		return r.rotateDB(ctx, apply)
+	}
+	ct, ok := callerToken(kind)
+	if !ok {
+		return fmt.Errorf("unknown credential %q (one of %s)", kind, strings.Join(rotationKinds(), ", "))
+	}
+	return r.rotateCaller(ctx, ct, apply)
+}
+
+// confirm ends the plan: without apply it says nothing changed and how to go
+// ahead, and reports false.
+func (r tokenRotator) confirm(kind string, apply bool) bool {
+	if !apply {
+		fmt.Fprintf(r.out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes %s\n", kind)
+		return false
+	}
+	fmt.Fprintln(r.out, "\nRotating:")
+	return true
+}
+
+// record writes new values into the installer's secrets.env. It comes first in
+// every rotation: from then on, whatever fails, a re-run of the installer puts
+// the new values everywhere.
+func (r tokenRotator) record(kv ...[2]string) error {
+	keys := make([]string, len(kv))
+	for i, p := range kv {
+		keys[i] = p[0]
+	}
+	switch err := setKeyValueLines(r.secretsEnv, "=", kv); {
 	case errors.Is(err, fs.ErrNotExist):
 		fmt.Fprintf(r.out, "  - %s: not found, skipped (this host was not installed by deploy/bootstrap.sh)\n", r.secretsEnv)
 	case err != nil:
-		return fmt.Errorf("record the new token in %s: %w", r.secretsEnv, err)
+		return fmt.Errorf("record the new value in %s: %w", r.secretsEnv, err)
 	default:
-		fmt.Fprintf(r.out, "  - %s: %s updated\n", r.secretsEnv, installerTokenKeys[ct.Caller])
+		fmt.Fprintf(r.out, "  - %s: %s updated\n", r.secretsEnv, strings.Join(keys, ", "))
 	}
+	return nil
+}
 
+func (r tokenRotator) putSecret(ctx context.Context, ns, name string, data map[string][]byte) error {
+	if _, err := putSecretKeys(ctx, r.cl, ns, name, corev1.SecretTypeOpaque, data); err != nil {
+		return fmt.Errorf("write Secret %s/%s: %w", ns, name, err)
+	}
+	fmt.Fprintf(r.out, "  - Secret %s/%s: updated\n", ns, name)
+	return nil
+}
+
+func (r tokenRotator) rollAPI(ctx context.Context) error {
+	if err := r.rollout(ctx, apiDeployment); err != nil {
+		return fmt.Errorf("roll felis-api: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - felis-api: rolled out on the new value")
+	return nil
+}
+
+// withMinecraft is the control namespace plus the minecraft namespace when that
+// is a different one: where the Secrets game pods and Jobs mount are mirrored.
+func (r tokenRotator) withMinecraft() []string {
+	if r.minecraftNS == "" || r.minecraftNS == r.controlNS {
+		return []string{r.controlNS}
+	}
+	return []string{r.controlNS, r.minecraftNS}
+}
+
+func (r tokenRotator) rotateCaller(ctx context.Context, ct naming.CallerToken, apply bool) error {
 	namespaces := []string{r.controlNS}
 	replica := map[string]string{"minecraft": r.minecraftNS, "build": r.buildNS}[ct.Replica]
 	if replica != "" && replica != r.controlNS {
 		namespaces = append(namespaces, replica)
 	}
-	for _, ns := range namespaces {
-		if err := writeTokenSecret(ctx, r.cl, ns, ct.Secret, tok); err != nil {
-			return fmt.Errorf("write Secret %s/%s: %w", ns, ct.Secret, err)
-		}
-		fmt.Fprintf(r.out, "  - Secret %s/%s: updated\n", ns, ct.Secret)
-	}
+	key := installerTokenKeys[ct.Caller]
 
-	hostProxy := false
-	if ct.Caller == "velocity" {
-		switch err := setKeyValueLine(r.linkProps, "service-token", "=", tok); {
-		case errors.Is(err, fs.ErrNotExist):
-			fmt.Fprintf(r.out, "  - %s: not found; set service-token in your proxy's felis-link.properties to the value in Secret %s/%s and restart it\n",
-				r.linkProps, r.controlNS, ct.Secret)
-		case err != nil:
-			return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
-		default:
-			hostProxy = true
-			fmt.Fprintf(r.out, "  - %s: service-token updated\n", r.linkProps)
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new internal token for the %s caller\n", ct.Caller, ct.Caller)
+	secrets := make([]string, len(namespaces))
+	for i, ns := range namespaces {
+		secrets[i] = "Secret " + ns + "/" + ct.Secret
 	}
+	writes := append([]string{r.secretsEnv + " (" + key + ")"}, secrets...)
+	hostProxy := ct.Caller == "velocity" && fileExists(r.linkProps)
+	if hostProxy {
+		writes = append(writes, r.linkProps+" (service-token)")
 	}
-
-	if err := r.rollAPI(ctx); err != nil {
-		return fmt.Errorf("roll felis-api: %w", err)
-	}
-	fmt.Fprintln(r.out, "  - felis-api: rolled out, accepting only the new token")
-
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
 	switch ct.Caller {
 	case "velocity":
 		if hostProxy {
+			fmt.Fprintf(r.out, "  - the proxy re-reads its token from the file and keeps its players; if it has not within %s of felis-api's restart, it is restarted, which disconnects everyone online\n", r.reloadWait)
+		} else {
+			fmt.Fprintf(r.out, "  - no proxy on this host (%s): set service-token in your proxy's felis-link.properties to the value in Secret %s/%s afterwards; it re-reads the file without a restart\n",
+				r.linkProps, r.controlNS, ct.Secret)
+		}
+	case "limbo":
+		fmt.Fprintln(r.out, "  - the login gate's pod restarts: a player signing in at that moment reconnects")
+	case "build":
+		fmt.Fprintln(r.out, "  - a build fetching its context at that moment fails and can be submitted again")
+	case "ops":
+		fmt.Fprintln(r.out, "  - felis backup-now presents the new token on its next run")
+	}
+	if !r.confirm(ct.Caller, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a token: %w", err)
+	}
+	if err := r.record([2]string{key, tok}); err != nil {
+		return err
+	}
+	for _, ns := range namespaces {
+		if err := r.putSecret(ctx, ns, ct.Secret, map[string][]byte{naming.ServiceTokenSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+
+	// The proxy's file changes before felis-api rolls, so the file never falls
+	// behind the Secret; the proxy's log is read from just before the write,
+	// since it may pick the new token up before the rollout ends.
+	since := r.now()
+	if hostProxy {
+		if err := setProxyToken(r.linkProps, tok); err != nil {
+			return fmt.Errorf("write the proxy's token into %s: %w", r.linkProps, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: service-token updated\n", r.linkProps)
+	}
+
+	if err := r.rollAPI(ctx); err != nil {
+		return err
+	}
+
+	switch ct.Caller {
+	case "velocity":
+		if !hostProxy {
+			break
+		}
+		if r.proxyReloaded(ctx, since, tokenFingerprint(tok)) {
+			fmt.Fprintf(r.out, "  - %s: took the new token from its properties; players stayed connected\n", velocityUnit)
+			break
+		}
 		if err := r.restartUnit(ctx, velocityUnit); err != nil {
 			return fmt.Errorf("restart %s: %w", velocityUnit, err)
 		}
-			fmt.Fprintf(r.out, "  - %s: restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit)
-		}
+		fmt.Fprintf(r.out, "  - %s: had not taken the new token within %s, restarted (players on the proxy were disconnected and can rejoin)\n", velocityUnit, r.reloadWait)
 	case "limbo":
-		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS),
-			client.MatchingLabels{v1alpha1.LabelServer: naming.SystemLoginServer}); err != nil {
+		// Only the operator's server pods carry its managed-by label; a backup or
+		// restore Job's pod carries the server label too, and app.kubernetes.io ones.
+		if err := r.cl.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(r.minecraftNS), client.MatchingLabels{
+			v1alpha1.LabelServer:    naming.SystemLoginServer,
+			v1alpha1.LabelManagedBy: operator.ManagedByValue,
+		}); err != nil {
 			return fmt.Errorf("restart the login gate: %w", err)
 		}
 		fmt.Fprintln(r.out, "  - login gate: pod restarted to read the new token")
 	case "build":
-		fmt.Fprintln(r.out, "  - builds: the next build Job reads the new token; one fetching its context right now fails and can be submitted again")
+		fmt.Fprintln(r.out, "  - builds: the next build Job reads the new token")
 	case "ops":
 		fmt.Fprintln(r.out, "  - felis backup-now reads the new token on its next run")
 	}
 	return nil
 }
 
-// writeTokenSecret sets the token in a Secret, creating it when absent.
-func writeTokenSecret(ctx context.Context, cl client.Client, namespace, name, token string) error {
-	var sec corev1.Secret
-	err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &sec)
-	if apierrors.IsNotFound(err) {
-		return cl.Create(ctx, &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
-			Type:       corev1.SecretTypeOpaque,
-			Data:       map[string][]byte{naming.ServiceTokenSecretKey: []byte(token)},
-		})
+// proxyReloaded waits up to reloadWait for the host proxy to log that it
+// reloaded the token with this fingerprint (plugins/shared FileToken).
+func (r tokenRotator) proxyReloaded(ctx context.Context, since time.Time, fingerprint string) bool {
+	want := "(fingerprint " + fingerprint + ")"
+	deadline := r.now().Add(r.reloadWait)
+	for {
+		if strings.Contains(r.proxyLog(ctx, since), want) {
+			return true
 		}
+		if !r.now().Before(deadline) {
+			return false
+		}
+		select {
+		case <-ctx.Done():
+			return false
+		case <-time.After(r.pollEvery):
+		}
+	}
+}
+
+// journalSince is the proxy unit's journal from the second since falls in. A
+// journal that cannot be read reads as one without the line, which ends in the
+// restart a rotation made before the proxy could reload.
+func journalSince(ctx context.Context, since time.Time) string {
+	out, _ := exec.CommandContext(ctx, "journalctl", "-u", velocityUnit, "--since", "@"+strconv.FormatInt(since.Unix(), 10),
+		"-o", "cat", "--no-pager", "-q").Output()
+	return string(out)
+}
+
+// setProxyToken writes the proxy's token into felis-link.properties and puts
+// the file's modification time back. The proxy re-reads its token by itself,
+// while `felis domain check` and `felis domain set` read a file newer than the
+// proxy's start as config it has not loaded (the installer's
+// install_if_changed keeps the time for the same reason).
+func setProxyToken(path, token string) error {
+	info, err := os.Stat(path)
 	if err != nil {
 		return err
 	}
-	if sec.Data == nil {
-		sec.Data = map[string][]byte{}
+	if err := setKeyValueLine(path, "service-token", "=", token); err != nil {
+		return err
 	}
-	sec.Data[naming.ServiceTokenSecretKey] = []byte(token)
-	return cl.Update(ctx, &sec)
+	return os.Chtimes(path, time.Time{}, info.ModTime())
+}
+
+func (r tokenRotator) rotateRegistry(ctx context.Context, apply bool) error {
+	keys := make([]string, len(registrygate.Principals))
+	for i, p := range registrygate.Principals {
+		keys[i] = installerRegistryKeys[p]
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: new write tokens for the image registry's principals (%s)\n", kindRegistry, strings.Join(registrygate.Principals, ", "))
+	fmt.Fprintf(r.out, "  - writes %s (%s), Secret %s/%s, Secret %s/%s\n", r.secretsEnv, strings.Join(keys, ", "),
+		r.controlNS, naming.RegistryAuthSecretName, r.buildNS, naming.RegistryPushSecretName)
+	fmt.Fprintln(r.out, "  - the registry restarts to load them: a build pushing its image at that moment fails and can be submitted again, and an image pull in that moment retries")
+	fmt.Fprintf(r.out, "  - %s (it presents the prune token)\n", apiRestartNote)
+	if !r.confirm(kindRegistry, apply) {
+		return nil
+	}
+
+	tokens := map[string][]byte{}
+	kv := make([][2]string, 0, len(registrygate.Principals))
+	for _, p := range registrygate.Principals {
+		tok, err := r.newToken()
+		if err != nil {
+			return fmt.Errorf("generate a token: %w", err)
+		}
+		tokens[p] = []byte(tok)
+		kv = append(kv, [2]string{installerRegistryKeys[p], tok})
+	}
+	if err := r.record(kv...); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.controlNS, naming.RegistryAuthSecretName, tokens); err != nil {
+		return err
+	}
+	if err := r.putSecret(ctx, r.buildNS, naming.RegistryPushSecretName, map[string][]byte{
+		naming.RegistryPushUsernameKey: []byte(registrygate.PrincipalBuild),
+		naming.RegistryPushPasswordKey: tokens[registrygate.PrincipalBuild],
+	}); err != nil {
+		return err
+	}
+	if err := r.rollout(ctx, registryDeployment); err != nil {
+		return fmt.Errorf("restart the registry: %w", err)
+	}
+	fmt.Fprintln(r.out, "  - registry: restarted on the new tokens")
+	return r.rollAPI(ctx)
+}
+
+func (r tokenRotator) rotateForwarding(ctx context.Context, apply bool) error {
+	restart, held, err := r.gamePods(ctx)
+	if err != nil {
+		return err
+	}
+	hostProxy := fileExists(r.forwardingFile)
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new Velocity forwarding secret, the key a server checks each player's identity with\n", kindForwarding)
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, "Secret "+ns+"/"+naming.ForwardingSecretName)
+	}
+	writes := append([]string{r.secretsEnv + " (" + installerForwardingKey + ")"}, secrets...)
+	if hostProxy {
+		writes = append(writes, r.forwardingFile)
+	}
+	fmt.Fprintf(r.out, "  - writes %s\n", strings.Join(writes, ", "))
+	fmt.Fprintf(r.out, "  - every running server restarts to read it (%d now), saving its world on the way down, and the proxy restarts: everyone online is disconnected and can rejoin once their server is back\n", len(restart))
+	if len(held) > 0 {
+		fmt.Fprintf(r.out, "  - left running, because a backup, restore or file write holds its world: %s. Players cannot join it until it restarts: stop and start it from the panel once that finishes\n", strings.Join(held, ", "))
+	}
+	if !hostProxy {
+		fmt.Fprintf(r.out, "  - no proxy on this host (%s): put the value in Secret %s/%s into your proxy's forwarding secret file and restart it\n",
+			r.forwardingFile, r.controlNS, naming.ForwardingSecretName)
+	}
+	if !r.confirm(kindForwarding, apply) {
+		return nil
+	}
+
+	tok, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a secret: %w", err)
+	}
+	if err := r.record([2]string{installerForwardingKey, tok}); err != nil {
+		return err
+	}
+	if hostProxy {
+		info, err := os.Stat(r.forwardingFile)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(r.forwardingFile, info, []byte(tok)); err != nil {
+			return fmt.Errorf("write %s: %w", r.forwardingFile, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: updated\n", r.forwardingFile)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, naming.ForwardingSecretName, map[string][]byte{naming.ForwardingSecretKey: []byte(tok)}); err != nil {
+			return err
+		}
+	}
+	// The servers go first: their pods read the Secret as they are recreated,
+	// and a player who rejoins through the restarted proxy meets a server on the
+	// new secret, or one still starting.
+	for i := range restart {
+		p := &restart[i]
+		if err := r.cl.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) {
+			return fmt.Errorf("restart %s: %w", p.Labels[v1alpha1.LabelServer], err)
+		}
+	}
+	fmt.Fprintf(r.out, "  - servers: %d restarting on the new secret\n", len(restart))
+	if hostProxy {
+		if err := r.restartUnit(ctx, velocityUnit); err != nil {
+			return fmt.Errorf("restart %s: %w", velocityUnit, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: restarted on the new secret\n", velocityUnit)
+	}
+	return nil
+}
+
+// gamePods lists the running game server pods: those a rotation may restart,
+// and the servers left alone because a backup, restore or file write holds
+// their world (internal/maintenance), which a restart in the middle of would
+// break.
+func (r tokenRotator) gamePods(ctx context.Context) ([]corev1.Pod, []string, error) {
+	var pods corev1.PodList
+	// The operator's managed-by label is on its server pods alone (see rotateCaller).
+	if err := r.cl.List(ctx, &pods, client.InNamespace(r.minecraftNS), client.MatchingLabels{v1alpha1.LabelManagedBy: operator.ManagedByValue}); err != nil {
+		return nil, nil, fmt.Errorf("list the servers' pods: %w", err)
+	}
+	var jobs batchv1.JobList
+	if err := r.cl.List(ctx, &jobs, client.InNamespace(r.minecraftNS)); err != nil {
+		return nil, nil, fmt.Errorf("list the maintenance Jobs: %w", err)
+	}
+	var restart []corev1.Pod
+	var held []string
+	for _, p := range pods.Items {
+		if p.DeletionTimestamp != nil {
+			continue
+		}
+		server := p.Labels[v1alpha1.LabelServer]
+		var ms v1alpha1.MinecraftServer
+		if err := r.cl.Get(ctx, client.ObjectKey{Namespace: r.minecraftNS, Name: server}, &ms); client.IgnoreNotFound(err) != nil {
+			return nil, nil, fmt.Errorf("read server %s: %w", server, err)
+		}
+		if kind, ok := maintenance.Holder(server, ms.Annotations, jobs.Items, r.now()); ok {
+			held = append(held, server+" ("+kind+")")
+			continue
+		}
+		restart = append(restart, p)
+	}
+	return restart, held, nil
+}
+
+func (r tokenRotator) rotateDB(ctx context.Context, apply bool) error {
+	cfg, err := config.Load(r.hostTOML)
+	if err != nil {
+		return err
+	}
+	if cfg.Database.Deployment == "" {
+		return fmt.Errorf("[database] deployment is unset in %s, so the database is not the installer's felis-postgres: change the role's password where it runs, then in [database] url of each config copy", r.hostTOML)
+	}
+	u, err := neturl.Parse(cfg.Database.URL)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return fmt.Errorf("[database] url in %s names no role", r.hostTOML)
+	}
+	role := u.User.Username()
+	targets, err := tomlTargetsOf(r.hostTOML, r.podTOML, r.defaultTOML)
+	if err != nil {
+		return err
+	}
+	paths := make([]string, len(targets))
+	for i, t := range targets {
+		paths[i] = t.path
+	}
+	secrets := []string{}
+	for _, ns := range r.withMinecraft() {
+		secrets = append(secrets, ns+"/"+platform.ConfigSecretName)
+	}
+	fmt.Fprintf(r.out, "felis rotate-token %s: a new password for the database role %q\n", kindDB, role)
+	fmt.Fprintf(r.out, "  - writes %s (%s), the role in %s, [database] url in %s, Secret %s\n",
+		r.secretsEnv, installerDBKey, cfg.Database.Deployment, strings.Join(paths, " and "), strings.Join(secrets, " and "))
+	fmt.Fprintf(r.out, "  - %s\n", apiRestartNote)
+	fmt.Fprintln(r.out, "  - a backup, restore or file Job that connects in the seconds between the password change and felis-api's restart fails and can be run again; the host's timers read the new config on their next run")
+	if !r.confirm(kindDB, apply) {
+		return nil
+	}
+
+	password, err := r.newToken()
+	if err != nil {
+		return fmt.Errorf("generate a password: %w", err)
+	}
+	// Every config copy is edited in memory first, so one this cannot edit stops
+	// the rotation before the role's password changes.
+	edited := make([][]byte, len(targets))
+	var hostURL string
+	var podConfig []byte
+	for i, t := range targets {
+		raw, err := os.ReadFile(t.real)
+		if err != nil {
+			return err
+		}
+		var doc struct {
+			Database struct {
+				URL string `toml:"url"`
+			} `toml:"database"`
+		}
+		if _, err := toml.Decode(string(raw), &doc); err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		next, err := withPassword(doc.Database.URL, password)
+		if err != nil {
+			return fmt.Errorf("%s: %w", t.path, err)
+		}
+		if edited[i], err = editTOMLStrings(raw, []tomlStringEdit{{"database", "url", next}}); err != nil {
+			return fmt.Errorf("%s: %w; set the password in its [database] url by hand", t.path, err)
+		}
+		switch t.path {
+		case r.hostTOML:
+			hostURL = next
+		case r.podTOML:
+			podConfig = edited[i]
+		}
+	}
+	if podConfig == nil {
+		return fmt.Errorf("%s resolves to the same file as %s; the pods reach the database at another address and need a copy of their own", r.podTOML, r.hostTOML)
+	}
+
+	if err := r.record([2]string{installerDBKey, password}); err != nil {
+		return err
+	}
+	salt := make([]byte, 16)
+	if _, err := rand.Read(salt); err != nil {
+		return err
+	}
+	verifier, err := scramVerifier(password, salt, scramIterations)
+	if err != nil {
+		return err
+	}
+	if err := r.alterRole(ctx, cfg.Database.Deployment, role, verifier); err != nil {
+		return fmt.Errorf("set the role's password: %w", err)
+	}
+	if err := r.verifyDB(ctx, hostURL); err != nil {
+		return fmt.Errorf("the database does not accept the new password (%v); the config copies still hold the old one: run the installer again (sudo bash deploy/bootstrap.sh), which sets the password in %s everywhere", err, r.secretsEnv)
+	}
+	fmt.Fprintf(r.out, "  - role %s: password changed, and the database accepts it\n", role)
+	for i, t := range targets {
+		info, err := os.Stat(t.real)
+		if err != nil {
+			return err
+		}
+		if err := replaceFileKeepingMode(t.real, info, edited[i]); err != nil {
+			return fmt.Errorf("write %s: %w", t.path, err)
+		}
+		fmt.Fprintf(r.out, "  - %s: [database] url updated\n", t.path)
+	}
+	for _, ns := range r.withMinecraft() {
+		if err := r.putSecret(ctx, ns, platform.ConfigSecretName, map[string][]byte{platform.ConfigSecretKey: podConfig}); err != nil {
+			return err
+		}
+	}
+	return r.rollAPI(ctx)
+}
+
+// withPassword is a database URL with its password replaced.
+func withPassword(raw, password string) (string, error) {
+	u, err := neturl.Parse(raw)
+	if err != nil || u.User == nil || u.User.Username() == "" {
+		return "", errors.New("its [database] url names no role")
+	}
+	u.User = neturl.UserPassword(u.User.Username(), password)
+	return u.String(), nil
+}
+
+// scramIterations is PostgreSQL's default scram_iterations.
+const scramIterations = 4096
+
+// scramVerifier is the SCRAM-SHA-256 verifier PostgreSQL stores for a password
+// (RFC 5802 and RFC 7677, in the form libpq's PQencryptPasswordConn makes).
+// ALTER ROLE stores a verifier as it is given, so the password itself never
+// reaches the server, where a failing statement is logged with its text.
+func scramVerifier(password string, salt []byte, iterations int) (string, error) {
+	salted, err := pbkdf2.Key(sha256.New, password, salt, iterations, sha256.Size)
+	if err != nil {
+		return "", err
+	}
+	mac := func(msg string) []byte {
+		h := hmac.New(sha256.New, salted)
+		h.Write([]byte(msg))
+		return h.Sum(nil)
+	}
+	stored := sha256.Sum256(mac("Client Key"))
+	b64 := base64.StdEncoding.EncodeToString
+	return fmt.Sprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, b64(salt), b64(stored[:]), b64(mac("Server Key"))), nil
+}
+
+// alterRoleInPod runs ALTER ROLE as the superuser inside the database's
+// container, over its socket, with the statement on stdin.
+func alterRoleInPod(ctx context.Context, deployment, role, verifier string) error {
+	ns, name, _ := strings.Cut(deployment, "/")
+	return kubectlWithInput(ctx, []byte(alterRoleSQL(role, verifier)), "-n", ns, "exec", "-i", "deploy/"+name, "-c", platform.PostgresContainer, "--",
+		"psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres")
+}
+
+// alterRoleSQL sets role's password to a SCRAM verifier, which holds no quote.
+func alterRoleSQL(role, verifier string) string {
+	return "ALTER ROLE \"" + strings.ReplaceAll(role, `"`, `""`) + "\" WITH PASSWORD '" + verifier + "';\n"
 }
 
 // setKeyValueLine rewrites the `key<sep>value` line of a flat key/value file

cmd/felis/rotatetoken_test.go

@@ -3,13 +3,20 @@ package main
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/maintenance"
+
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -22,23 +29,44 @@ type rotationRig struct {
 	out    *bytes.Buffer
 	events []string
 	dir    string
+	clock  time.Time
 }
 
 func tokenSecret(ns, name, val string) *corev1.Secret {
+	return keySecret(ns, name, "token", val)
+}
+
+func keySecret(ns, name, key, val string) *corev1.Secret {
 	return &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name},
-		Data:       map[string][]byte{"token": []byte(val)},
+		Data:       map[string][]byte{key: []byte(val)},
 	}
 }
 
+// serverPod is a game server's pod as the operator labels it.
 func serverPod(ns, name, server string) *corev1.Pod {
 	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
-		Labels: map[string]string{"felis.lolicon.best/server": server}}}
+		Labels: map[string]string{
+			"felis.lolicon.best/server":     server,
+			"felis.lolicon.best/managed-by": "felis-operator",
+			"felis.lolicon.best/component":  "server",
+		}}}
+}
+
+// backupPod is a backup Job's pod as internal/backupjob labels it: it carries
+// the server's label too, and no rotation may take it for the server's own.
+func backupPod(ns, name, server string) *corev1.Pod {
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name,
+		Labels: map[string]string{
+			"felis.lolicon.best/server":    server,
+			"app.kubernetes.io/managed-by": "felis-backup",
+			"app.kubernetes.io/component":  "world-backup",
+		}}}
 }
 
 func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
 	t.Helper()
-	rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir()}
+	rig := &rotationRig{out: &bytes.Buffer{}, dir: t.TempDir(), clock: time.Unix(1_800_000_000, 0)}
 	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
 	rig.r = tokenRotator{
 		cl:             rig.cl,
@@ -47,27 +75,65 @@ func newRotationRig(t *testing.T, objs ...client.Object) *rotationRig {
 		buildNS:        "felis-build",
 		secretsEnv:     filepath.Join(rig.dir, "secrets.env"),
 		linkProps:      filepath.Join(rig.dir, "felis-link.properties"),
+		forwardingFile: filepath.Join(rig.dir, "forwarding.secret"),
+		hostTOML:       filepath.Join(rig.dir, "felis.host.toml"),
+		podTOML:        filepath.Join(rig.dir, "felis.pod.toml"),
+		defaultTOML:    filepath.Join(rig.dir, "felis.toml"),
 		newToken:       func() (string, error) { return "NEWTOKEN", nil },
-		rollAPI: func(context.Context) error {
-			rig.events = append(rig.events, "roll-api")
+		rollout: func(_ context.Context, deployment string) error {
+			rig.events = append(rig.events, "roll "+deployment)
 			return nil
 		},
 		restartUnit: func(_ context.Context, unit string) error {
 			rig.events = append(rig.events, "restart "+unit)
 			return nil
 		},
+		proxyLog: func(context.Context, time.Time) string { return "" },
+		alterRole: func(context.Context, string, string, string) error {
+			rig.events = append(rig.events, "alter-role")
+			return nil
+		},
+		verifyDB: func(context.Context, string) error {
+			rig.events = append(rig.events, "verify-db")
+			return nil
+		},
+		// Every look at the clock moves it a second on, so a wait measured with it
+		// ends after a known number of looks.
+		now: func() time.Time {
+			rig.clock = rig.clock.Add(time.Second)
+			return rig.clock
+		},
+		reloadWait: 5 * time.Second,
 		out:        rig.out,
 	}
 	return rig
 }
 
-func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
+func (rig *rotationRig) secretKey(t *testing.T, ns, name, key string) string {
 	t.Helper()
 	var s corev1.Secret
 	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
 		return "<missing>"
 	}
-	return string(s.Data["token"])
+	return string(s.Data[key])
+}
+
+func (rig *rotationRig) secret(t *testing.T, ns, name string) string {
+	t.Helper()
+	return rig.secretKey(t, ns, name, "token")
+}
+
+func (rig *rotationRig) pods(t *testing.T) string {
+	t.Helper()
+	var pods corev1.PodList
+	if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
+		t.Fatal(err)
+	}
+	names := make([]string, len(pods.Items))
+	for i, p := range pods.Items {
+		names[i] = p.Name
+	}
+	return strings.Join(names, ",")
 }
 
 func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
@@ -80,6 +146,15 @@ func writeTestFile(t *testing.T, path, body string, mode os.FileMode) {
 	}
 }
 
+func readTestFile(t *testing.T, path string) string {
+	t.Helper()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(raw)
+}
+
 func TestRotateLimboToken(t *testing.T) {
 	rig := newRotationRig(t,
 		tokenSecret("felis", "felis-limbo-token", "old"),
@@ -87,14 +162,15 @@ func TestRotateLimboToken(t *testing.T) {
 		tokenSecret("felis", "felis-service-token", "proxy"),
 		serverPod("minecraft", "login-0", "login"),
 		serverPod("minecraft", "survival-0", "survival"),
+		backupPod("minecraft", "login-backup-x", "login"),
 	)
 	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=old\nOPS_TOKEN=ops\n", 0o600)
 
 	// felis-api must roll only after both copies hold the new value, and the login
 	// pod must still be there then: restarting it earlier would have it present
 	// the new token to an api that does not know it yet.
-	rig.r.rollAPI = func(context.Context) error {
-		rig.events = append(rig.events, "roll-api")
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
 		if got := rig.secret(t, "felis", "felis-limbo-token"); got != "NEWTOKEN" {
 			t.Errorf("api rolled while the control Secret held %q", got)
 		}
@@ -107,13 +183,12 @@ func TestRotateLimboToken(t *testing.T) {
 		}
 		return nil
 	}
-	if err := rig.r.rotate(context.Background(), "limbo"); err != nil {
+	if err := rig.r.rotate(context.Background(), "limbo", true); err != nil {
 		t.Fatal(err)
 	}
 
-	raw, _ := os.ReadFile(rig.r.secretsEnv)
-	if string(raw) != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
-		t.Errorf("secrets.env = %q", raw)
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nSERVICE_TOKEN=proxy\nLIMBO_TOKEN=NEWTOKEN\nOPS_TOKEN=ops\n" {
+		t.Errorf("secrets.env = %q", got)
 	}
 	if info, _ := os.Stat(rig.r.secretsEnv); info.Mode().Perm() != 0o600 {
 		t.Errorf("secrets.env mode = %v, want 0600", info.Mode().Perm())
@@ -121,14 +196,12 @@ func TestRotateLimboToken(t *testing.T) {
 	if got := rig.secret(t, "felis", "felis-service-token"); got != "proxy" {
 		t.Errorf("the proxy's token changed to %q", got)
 	}
-	var pods corev1.PodList
-	if err := rig.cl.List(context.Background(), &pods, client.InNamespace("minecraft")); err != nil {
-		t.Fatal(err)
+	// The login pod restarted; a user server and the backup Job's pod, which
+	// carries the login server's label too, are untouched.
+	if got := rig.pods(t); got != "login-backup-x,survival-0" {
+		t.Errorf("pods left = %s, want login-backup-x,survival-0", got)
 	}
-	if len(pods.Items) != 1 || pods.Items[0].Name != "survival-0" {
-		t.Errorf("pods left = %v, want only survival-0 (the login pod restarted, user servers untouched)", pods.Items)
-	}
-	if strings.Join(rig.events, ",") != "roll-api" {
+	if strings.Join(rig.events, ",") != "roll felis-api" {
 		t.Errorf("events = %v, want only the api roll (no unit restart for limbo)", rig.events)
 	}
 	if strings.Contains(rig.out.String(), "NEWTOKEN") {
@@ -136,21 +209,56 @@ func TestRotateLimboToken(t *testing.T) {
 	}
 }
 
-func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) {
+const testLinkProps = "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n"
+
+func reloadLine(token string) string {
+	return "[12:00:00 INFO] [felis-link]: Felis: service-token reloaded from /x/felis-link.properties (fingerprint " + tokenFingerprint(token) + ")\n"
+}
+
+// The host proxy re-reads its token: the rotation waits for it to say so and
+// leaves it running.
+func TestRotateVelocityTokenReloadsTheHostProxy(t *testing.T) {
 	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
 	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
-	writeTestFile(t, rig.r.linkProps, "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=old\nroot-domain=example.com\n", 0o640)
-
-	if err := rig.r.rotate(context.Background(), "velocity"); err != nil {
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	if err := os.Chtimes(rig.r.linkProps, written, written); err != nil {
 		t.Fatal(err)
 	}
-	raw, _ := os.ReadFile(rig.r.linkProps)
-	if string(raw) != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
-		t.Errorf("felis-link.properties = %q", raw)
+
+	// The proxy logs its reload on its next call to felis-api, which may be
+	// while the api rolls: the log must be read from before that.
+	var reloadedAt time.Time
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := readTestFile(t, rig.r.linkProps); !strings.Contains(got, "service-token=NEWTOKEN\n") {
+			t.Errorf("api rolled before the proxy's file held the new token: %q", got)
 		}
-	if info, _ := os.Stat(rig.r.linkProps); info.Mode().Perm() != 0o640 {
+		reloadedAt = rig.clock
+		return nil
+	}
+	looks := 0
+	rig.r.proxyLog = func(_ context.Context, since time.Time) string {
+		looks++
+		log := reloadLine("old") // an earlier rotation's
+		if looks >= 3 && !since.After(reloadedAt) {
+			log += reloadLine("NEWTOKEN")
+		}
+		return log
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.linkProps); got != "# Generated\napi-base-url=http://10.0.0.1:8081\nservice-token=NEWTOKEN\nroot-domain=example.com\n" {
+		t.Errorf("felis-link.properties = %q", got)
+	}
+	info, _ := os.Stat(rig.r.linkProps)
+	if info.Mode().Perm() != 0o640 {
 		t.Errorf("properties mode = %v, want 0640 (the proxy's group must still read it)", info.Mode().Perm())
 	}
+	if !info.ModTime().Equal(written) {
+		t.Errorf("properties mtime = %v, want it kept at %v (felis domain check would read the proxy as stale)", info.ModTime(), written)
+	}
 	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
 		t.Errorf("control Secret = %q, want NEWTOKEN", got)
 	}
@@ -158,9 +266,49 @@ func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) {
 	if got := rig.secret(t, "minecraft", "felis-service-token"); got != "<missing>" {
 		t.Errorf("rotation copied the proxy token into minecraft (%q)", got)
 	}
-	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
+	if strings.Join(rig.events, ",") != "roll felis-api" {
+		t.Errorf("events = %v, want the api roll and no proxy restart", rig.events)
+	}
+	if looks != 3 {
+		t.Errorf("the log was read %d times, want 3 (until the line appeared)", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: took the new token from its properties; players stayed connected") {
+		t.Errorf("output does not say the players stayed: %s", out)
+	}
+}
+
+// A proxy that never logs the new fingerprint (an older plugin, a stuck proxy)
+// is restarted once the wait is over.
+func TestRotateVelocityTokenRestartsAProxyThatDoesNotReload(t *testing.T) {
+	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
+	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	looks := 0
+	rig.r.proxyLog = func(context.Context, time.Time) string {
+		looks++
+		return reloadLine("old")
+	}
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(rig.events, ",") != "roll felis-api,restart felis-velocity" {
 		t.Errorf("events = %v, want the api roll then the proxy restart", rig.events)
 	}
+	// reloadWait is 5s and each look moves the clock a second.
+	if looks != 5 {
+		t.Errorf("the log was read %d times, want 5", looks)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "felis-velocity: had not taken the new token within 5s, restarted") {
+		t.Errorf("output does not explain the restart: %s", out)
+	}
+}
+
+// The proxy and the Java plugin name a token by the same fingerprint
+// (plugins/shared LinkConfigLoaderTest fileTokenFollowsTheFile).
+func TestTokenFingerprintMatchesThePlugin(t *testing.T) {
+	if got := tokenFingerprint("new-token"); got != "348e9df2a42b" {
+		t.Errorf("tokenFingerprint(new-token) = %q, want 348e9df2a42b", got)
+	}
 }
 
 // An external proxy has no felis-link.properties here: the Secret still rotates,
@@ -168,13 +316,13 @@ func TestRotateVelocityTokenOnTheHostProxy(t *testing.T) {
 // without printing it.
 func TestRotateVelocityTokenForAnExternalProxy(t *testing.T) {
 	rig := newRotationRig(t, tokenSecret("felis", "felis-service-token", "old"))
-	if err := rig.r.rotate(context.Background(), "velocity"); err != nil {
+	if err := rig.r.rotate(context.Background(), "velocity", true); err != nil {
 		t.Fatal(err)
 	}
 	if got := rig.secret(t, "felis", "felis-service-token"); got != "NEWTOKEN" {
 		t.Errorf("control Secret = %q, want NEWTOKEN", got)
 	}
-	if strings.Join(rig.events, ",") != "roll-api" {
+	if strings.Join(rig.events, ",") != "roll felis-api" {
 		t.Errorf("events = %v, want no proxy restart", rig.events)
 	}
 	out := rig.out.String()
@@ -187,7 +335,7 @@ func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
 	rig := newRotationRig(t, tokenSecret("felis", "felis-build-token", "old"))
 	// An install from before per-caller tokens has no BUILD_TOKEN line yet.
 	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy", 0o600)
-	if err := rig.r.rotate(context.Background(), "build"); err != nil {
+	if err := rig.r.rotate(context.Background(), "build", true); err != nil {
 		t.Fatal(err)
 	}
 	if got := rig.secret(t, "felis-build", "felis-build-token"); got != "NEWTOKEN" {
@@ -196,9 +344,8 @@ func TestRotateBuildTokenReachesTheBuildNamespace(t *testing.T) {
 	if got := rig.secret(t, "felis", "felis-build-token"); got != "NEWTOKEN" {
 		t.Errorf("control Secret = %q, want NEWTOKEN", got)
 	}
-	raw, _ := os.ReadFile(rig.r.secretsEnv)
-	if string(raw) != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
-		t.Errorf("secrets.env = %q", raw)
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\nBUILD_TOKEN=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
 	}
 }
 
@@ -209,31 +356,453 @@ func TestRotateStopsWhenTheAPIDoesNotRoll(t *testing.T) {
 		tokenSecret("felis", "felis-limbo-token", "old"),
 		serverPod("minecraft", "login-0", "login"),
 	)
-	rig.r.rollAPI = func(context.Context) error { return errors.New("rollout timed out") }
-	err := rig.r.rotate(context.Background(), "limbo")
+	rig.r.rollout = func(context.Context, string) error { return errors.New("rollout timed out") }
+	err := rig.r.rotate(context.Background(), "limbo", true)
 	if err == nil || !strings.Contains(err.Error(), "rollout timed out") {
 		t.Fatalf("err = %v, want the rollout failure", err)
 	}
-	var pod corev1.Pod
-	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
-		t.Error("the login pod was restarted although the api never rolled")
+	if got := rig.pods(t); got != "login-0" {
+		t.Errorf("pods left = %s: the login pod was restarted although the api never rolled", got)
 	}
 }
 
-func TestRotateRefusesAnUnknownCaller(t *testing.T) {
+func TestRotateRefusesAnUnknownCredential(t *testing.T) {
 	rig := newRotationRig(t)
 	writeTestFile(t, rig.r.secretsEnv, "SERVICE_TOKEN=proxy\n", 0o600)
-	if err := rig.r.rotate(context.Background(), "admin"); err == nil {
-		t.Fatal("rotated a token for an unknown caller")
+	if err := rig.r.rotate(context.Background(), "admin", true); err == nil {
+		t.Fatal("rotated an unknown credential")
 	}
-	if raw, _ := os.ReadFile(rig.r.secretsEnv); string(raw) != "SERVICE_TOKEN=proxy\n" {
-		t.Errorf("secrets.env = %q, want it untouched", raw)
+	if got := readTestFile(t, rig.r.secretsEnv); got != "SERVICE_TOKEN=proxy\n" {
+		t.Errorf("secrets.env = %q, want it untouched", got)
 	}
 	if len(rig.events) != 0 {
 		t.Errorf("events = %v, want nothing touched", rig.events)
 	}
 }
 
+const (
+	testHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	testPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+// installedRig is a host as the installer leaves it, with every credential in
+// place, for the plan test.
+func installedRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		tokenSecret("felis", "felis-service-token", "old"),
+		tokenSecret("felis", "felis-limbo-token", "old"),
+		tokenSecret("minecraft", "felis-limbo-token", "old"),
+		tokenSecret("felis", "felis-build-token", "old"),
+		tokenSecret("felis-build", "felis-build-token", "old"),
+		tokenSecret("felis", "felis-ops-token", "old"),
+		keySecret("felis", "felis-registry-auth", "platform", "old"),
+		keySecret("felis-build", "felis-registry-push", "password", "old"),
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+		serverPod("minecraft", "login-0", "login"),
+		serverPod("minecraft", "survival-0", "survival"),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=old\n", 0o600)
+	writeTestFile(t, rig.r.linkProps, testLinkProps, 0o640)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	return rig
+}
+
+// Without -yes every rotation prints its plan and changes nothing.
+func TestRotatePlanChangesNothing(t *testing.T) {
+	for _, kind := range rotationKinds() {
+		t.Run(kind, func(t *testing.T) {
+			rig := installedRig(t)
+			files := map[string]string{}
+			for _, p := range []string{rig.r.secretsEnv, rig.r.linkProps, rig.r.forwardingFile, rig.r.hostTOML, rig.r.podTOML} {
+				files[p] = readTestFile(t, p)
+			}
+			rig.r.newToken = func() (string, error) {
+				t.Error("a plan generated a token")
+				return "NEWTOKEN", nil
+			}
+			if err := rig.r.rotate(context.Background(), kind, false); err != nil {
+				t.Fatal(err)
+			}
+			for p, before := range files {
+				if got := readTestFile(t, p); got != before {
+					t.Errorf("%s changed to %q", filepath.Base(p), got)
+				}
+			}
+			for _, s := range [][3]string{
+				{"felis", "felis-service-token", "token"}, {"felis", "felis-limbo-token", "token"},
+				{"felis-build", "felis-build-token", "token"}, {"felis", "felis-ops-token", "token"},
+				{"felis", "felis-registry-auth", "platform"}, {"felis-build", "felis-registry-push", "password"},
+				{"minecraft", "felis-forwarding-secret", "secret"},
+			} {
+				if got := rig.secretKey(t, s[0], s[1], s[2]); got != "old" {
+					t.Errorf("Secret %s/%s changed to %q", s[0], s[1], got)
+				}
+			}
+			if got := rig.secretKey(t, "minecraft", "felis-config", "felis.toml"); got != testPodTOML {
+				t.Errorf("felis-config changed to %q", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want none", rig.events)
+			}
+			if got := rig.pods(t); got != "login-0,survival-0" {
+				t.Errorf("pods left = %s, want all", got)
+			}
+			out := rig.out.String()
+			if !strings.HasSuffix(out, "\nNothing was changed. To rotate: sudo felis rotate-token -yes "+kind+"\n") {
+				t.Errorf("the plan does not end with how to go ahead: %s", out)
+			}
+			// Each plan names the interruption it causes.
+			want := apiRestartNote
+			if kind == kindForwarding {
+				want = "everyone online is disconnected"
+			}
+			if !strings.Contains(out, want) {
+				t.Errorf("the plan does not say %q: %s", want, out)
+			}
+		})
+	}
+}
+
+func TestRotateRegistryTokens(t *testing.T) {
+	rig := newRotationRig(t,
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: "felis-registry-auth"},
+			Data: map[string][]byte{"platform": []byte("a"), "build": []byte("b"), "prune": []byte("c")}},
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis-build", Name: "felis-registry-push"},
+			Data: map[string][]byte{"username": []byte("build"), "password": []byte("b")}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=a\nREGISTRY_BUILD_TOKEN=b\nREGISTRY_PRUNE_TOKEN=c\n", 0o600)
+	n := 0
+	rig.r.newToken = func() (string, error) {
+		n++
+		return fmt.Sprintf("NEWTOKEN%d", n), nil
+	}
+	// The registry reads its tokens as it starts: it restarts after the Secret
+	// holds them, and felis-api (the prune token) after that.
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", "prune"); got != "NEWTOKEN3" {
+			t.Errorf("%s rolled while the prune token was %q", deployment, got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindRegistry, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nREGISTRY_PLATFORM_TOKEN=NEWTOKEN1\nREGISTRY_BUILD_TOKEN=NEWTOKEN2\nREGISTRY_PRUNE_TOKEN=NEWTOKEN3\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	for key, want := range map[string]string{"platform": "NEWTOKEN1", "build": "NEWTOKEN2", "prune": "NEWTOKEN3"} {
+		if got := rig.secretKey(t, "felis", "felis-registry-auth", key); got != want {
+			t.Errorf("felis-registry-auth %s = %q, want %s", key, got, want)
+		}
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "password"); got != "NEWTOKEN2" {
+		t.Errorf("the build Jobs' push password = %q, want the build token", got)
+	}
+	if got := rig.secretKey(t, "felis-build", "felis-registry-push", "username"); got != "build" {
+		t.Errorf("the build Jobs' push username = %q, want build", got)
+	}
+	if strings.Join(rig.events, ",") != "roll registry,roll felis-api" {
+		t.Errorf("events = %v, want the registry then felis-api", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("a new token was printed: %s", rig.out.String())
+	}
+}
+
+func TestRotateForwardingSecret(t *testing.T) {
+	lock := time.Unix(1_800_000_000, 0)
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-forwarding-secret", "secret", "old"),
+		keySecret("minecraft", "felis-forwarding-secret", "secret", "old"),
+		serverPod("minecraft", "survival-0", "survival"),
+		serverPod("minecraft", "lobby-0", "lobby"),
+		// creative is being backed up: a running Job holds its world.
+		serverPod("minecraft", "creative-0", "creative"),
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "creative-backup",
+			Labels: map[string]string{maintenance.LabelServer: "creative", maintenance.LabelManagedBy: "felis-backup"}}},
+		// survival's last backup is over; its finished pod stays until the Job's TTL.
+		&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival-backup",
+			Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: "felis-backup"}},
+			Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}},
+		backupPod("minecraft", "survival-backup-x", "survival"),
+		// A pod already on its way out is neither counted nor deleted again.
+		func() *corev1.Pod {
+			p := serverPod("minecraft", "lobby-old", "lobby")
+			p.DeletionTimestamp = &metav1.Time{Time: lock}
+			p.Finalizers = []string{"felis.lolicon.best/test"}
+			return p
+		}(),
+		// skyblock's restore has just been admitted, its Job not created yet.
+		serverPod("minecraft", "skyblock-0", "skyblock"),
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "skyblock",
+			Annotations: map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lock)}}},
+		&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "survival"}},
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=db\nFORWARDING_SECRET=old\n", 0o600)
+	writeTestFile(t, rig.r.forwardingFile, "old", 0o640)
+	// The proxy restarts after the servers were told to: a player who rejoins
+	// must not meet a server still on the old secret.
+	rig.r.restartUnit = func(_ context.Context, unit string) error {
+		rig.events = append(rig.events, "restart "+unit)
+		if got := rig.pods(t); strings.Contains(got, "survival-0") {
+			t.Errorf("the proxy restarted before the servers (pods %s)", got)
+		}
+		if got := readTestFile(t, rig.r.forwardingFile); got != "NEWTOKEN" {
+			t.Errorf("the proxy restarted on forwarding.secret %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=db\nFORWARDING_SECRET=NEWTOKEN\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if info, _ := os.Stat(rig.r.forwardingFile); info.Mode().Perm() != 0o640 {
+		t.Errorf("forwarding.secret mode = %v, want 0640", info.Mode().Perm())
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+			t.Errorf("Secret %s/felis-forwarding-secret = %q, want NEWTOKEN", ns, got)
+		}
+	}
+	if got := rig.pods(t); got != "creative-0,lobby-old,skyblock-0,survival-backup-x" {
+		t.Errorf("pods left = %s, want the held servers, the terminating pod and the backup's pod", got)
+	}
+	if strings.Join(rig.events, ",") != "restart felis-velocity" {
+		t.Errorf("events = %v, want only the proxy restart (felis-api does not hold this secret)", rig.events)
+	}
+	out := rig.out.String()
+	for _, want := range []string{
+		"every running server restarts to read it (2 now)",
+		"left running, because a backup, restore or file write holds its world: creative (backup), skyblock (restore).",
+		"  - servers: 2 restarting on the new secret\n",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output lacks %q: %s", want, out)
+		}
+	}
+	if strings.Contains(out, "NEWTOKEN") {
+		t.Errorf("the new secret was printed: %s", out)
+	}
+}
+
+func TestRotateForwardingSecretForAnExternalProxy(t *testing.T) {
+	rig := newRotationRig(t, keySecret("felis", "felis-forwarding-secret", "secret", "old"))
+	if err := rig.r.rotate(context.Background(), kindForwarding, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := rig.secretKey(t, "felis", "felis-forwarding-secret", "secret"); got != "NEWTOKEN" {
+		t.Errorf("control Secret = %q, want NEWTOKEN", got)
+	}
+	if len(rig.events) != 0 {
+		t.Errorf("events = %v, want no proxy restart on this host", rig.events)
+	}
+	if out := rig.out.String(); !strings.Contains(out, "put the value in Secret felis/felis-forwarding-secret into your proxy's forwarding secret file") {
+		t.Errorf("output does not say where the value is: %s", out)
+	}
+}
+
+// dbRig is an installed host for the database rotation: felis.toml links to the
+// host copy, as the installer makes it.
+func dbRig(t *testing.T) *rotationRig {
+	t.Helper()
+	rig := newRotationRig(t,
+		keySecret("felis", "felis-config", "felis.toml", testPodTOML),
+		keySecret("minecraft", "felis-config", "felis.toml", testPodTOML),
+	)
+	writeTestFile(t, rig.r.secretsEnv, "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n", 0o600)
+	writeTestFile(t, rig.r.hostTOML, testHostTOML, 0o600)
+	writeTestFile(t, rig.r.podTOML, testPodTOML, 0o600)
+	if err := os.Symlink("felis.host.toml", rig.r.defaultTOML); err != nil {
+		t.Fatal(err)
+	}
+	return rig
+}
+
+const (
+	wantHostTOML = "# Written by the installer.\n[database]\nurl = \"postgres://felis:[email protected]:30432/felis?sslmode=disable\"\ndeployment = \"felis/felis-postgres\"\n\n[server]\nroot_domain = \"example.com\"\n"
+	wantPodTOML  = "[database]\n# The Service address.\nurl = \"postgres://felis:[email protected]:5432/felis?sslmode=disable\"\n\n[server]\nroot_domain = \"example.com\"\n"
+)
+
+func TestRotateDatabasePassword(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.alterRole = func(_ context.Context, deployment, role, verifier string) error {
+		rig.events = append(rig.events, "alter-role")
+		if deployment != "felis/felis-postgres" || role != "felis" {
+			t.Errorf("alterRole(%q, %q), want felis/felis-postgres and felis", deployment, role)
+		}
+		if !strings.Contains(readTestFile(t, rig.r.secretsEnv), "DB_PASSWORD=NEWTOKEN\n") {
+			t.Error("the role changed before secrets.env recorded the new password")
+		}
+		// The verifier is the new password's, under the salt it carries.
+		m := regexp.MustCompile(`^SCRAM-SHA-256\$4096:([^$]+)\$`).FindStringSubmatch(verifier)
+		if m == nil {
+			t.Fatalf("verifier %q is not a SCRAM-SHA-256 verifier", verifier)
+		}
+		salt, err := base64.StdEncoding.DecodeString(m[1])
+		if err != nil || len(salt) != 16 {
+			t.Fatalf("verifier salt %q: %v", m[1], err)
+		}
+		if want, _ := scramVerifier("NEWTOKEN", salt, 4096); verifier != want {
+			t.Errorf("verifier = %q, want %q", verifier, want)
+		}
+		return nil
+	}
+	// The configs change only once the database accepts the new password.
+	rig.r.verifyDB = func(_ context.Context, url string) error {
+		rig.events = append(rig.events, "verify-db")
+		if url != "postgres://felis:[email protected]:30432/felis?sslmode=disable" {
+			t.Errorf("verified with %q, want the host URL with the new password", url)
+		}
+		if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+			t.Errorf("the host config changed before the database accepted the password: %q", got)
+		}
+		return nil
+	}
+	rig.r.rollout = func(_ context.Context, deployment string) error {
+		rig.events = append(rig.events, "roll "+deployment)
+		if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("felis-api rolled on felis-config %q", got)
+		}
+		return nil
+	}
+	if err := rig.r.rotate(context.Background(), kindDB, true); err != nil {
+		t.Fatal(err)
+	}
+	if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=NEWTOKEN\nSERVICE_TOKEN=s\n" {
+		t.Errorf("secrets.env = %q", got)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != wantHostTOML {
+		t.Errorf("host config = %q", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != wantPodTOML {
+		t.Errorf("pod config = %q", got)
+	}
+	if info, err := os.Lstat(rig.r.defaultTOML); err != nil || info.Mode()&os.ModeSymlink == 0 {
+		t.Errorf("felis.toml is no longer the link to the host copy (%v)", err)
+	}
+	for _, ns := range []string{"felis", "minecraft"} {
+		if got := rig.secretKey(t, ns, "felis-config", "felis.toml"); got != wantPodTOML {
+			t.Errorf("Secret %s/felis-config = %q", ns, got)
+		}
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db,roll felis-api" {
+		t.Errorf("events = %v", rig.events)
+	}
+	if strings.Contains(rig.out.String(), "NEWTOKEN") {
+		t.Errorf("the new password was printed: %s", rig.out.String())
+	}
+}
+
+// A password the database does not accept leaves the configs on the old one
+// and felis-api running: the installer's record, already new, is the way back.
+func TestRotateDatabasePasswordStopsWhenTheDatabaseRefuses(t *testing.T) {
+	rig := dbRig(t)
+	rig.r.verifyDB = func(context.Context, string) error {
+		rig.events = append(rig.events, "verify-db")
+		return errors.New("password authentication failed")
+	}
+	err := rig.r.rotate(context.Background(), kindDB, true)
+	if err == nil || !strings.Contains(err.Error(), "password authentication failed") || !strings.Contains(err.Error(), "run the installer again") {
+		t.Fatalf("err = %v, want the refusal and the way back", err)
+	}
+	if got := readTestFile(t, rig.r.hostTOML); got != testHostTOML {
+		t.Errorf("host config = %q, want it unchanged", got)
+	}
+	if got := readTestFile(t, rig.r.podTOML); got != testPodTOML {
+		t.Errorf("pod config = %q, want it unchanged", got)
+	}
+	if got := rig.secretKey(t, "felis", "felis-config", "felis.toml"); got != testPodTOML {
+		t.Errorf("felis-config = %q, want it unchanged", got)
+	}
+	if strings.Join(rig.events, ",") != "alter-role,verify-db" {
+		t.Errorf("events = %v, want no api roll", rig.events)
+	}
+}
+
+// Everything that can refuse does so before the installer's record or the
+// role change; what the host config alone shows is refused by the plan too.
+func TestRotateDatabasePasswordRefusesEarly(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		apply bool
+		setup func(t *testing.T, rig *rotationRig)
+		want  string
+	}{
+		{"a database the installer does not run", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "deployment = \"felis/felis-postgres\"\n", "", 1), 0o600)
+		}, "[database] deployment is unset"},
+		{"a database url without a role", false, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.hostTOML, strings.Replace(testHostTOML, "felis:oldpw@", "", 1), 0o600)
+		}, "names no role"},
+		{"a config copy the line editor cannot edit", true, func(t *testing.T, rig *rotationRig) {
+			writeTestFile(t, rig.r.podTOML, "[database]\nurl = \"\"\"\npostgres://felis:[email protected]:5432/felis\"\"\"\n", 0o600)
+		}, "set the password in its [database] url by hand"},
+		{"a pod copy that is the host copy", true, func(t *testing.T, rig *rotationRig) {
+			if err := os.Remove(rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.Symlink("felis.host.toml", rig.r.podTOML); err != nil {
+				t.Fatal(err)
+			}
+		}, "resolves to the same file as"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			rig := dbRig(t)
+			tc.setup(t, rig)
+			err := rig.r.rotate(context.Background(), kindDB, tc.apply)
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err = %v, want %q", err, tc.want)
+			}
+			if got := readTestFile(t, rig.r.secretsEnv); got != "DB_PASSWORD=oldpw\nSERVICE_TOKEN=s\n" {
+				t.Errorf("secrets.env = %q, want it untouched", got)
+			}
+			if len(rig.events) != 0 {
+				t.Errorf("events = %v, want nothing done", rig.events)
+			}
+		})
+	}
+}
+
+// The RFC 7677 example (user "user", password "pencil"), with the stored and
+// server keys computed by openssl's PBKDF2 and HMAC rather than this code.
+func TestScramVerifier(t *testing.T) {
+	salt, _ := base64.StdEncoding.DecodeString("W22ZaJ0SNY7soEsUEjb6gQ==")
+	got, err := scramVerifier("pencil", salt, 4096)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := "SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==$WG5d8oPm3OtcPnkdi4Uo7BkeZkBFzpcXkuLmtbsT4qY=:wfPLwcE6nTWhTAmQ7tl2KeoiWGPlZqQxSrmfPwDl2dU="; got != want {
+		t.Errorf("scramVerifier = %q\nwant            %q", got, want)
+	}
+}
+
+func TestAlterRoleSQL(t *testing.T) {
+	if got := alterRoleSQL(`fe"lis`, "SCRAM-SHA-256$4096:c2FsdA==$a:b"); got != "ALTER ROLE \"fe\"\"lis\" WITH PASSWORD 'SCRAM-SHA-256$4096:c2FsdA==$a:b';\n" {
+		t.Errorf("alterRoleSQL = %q", got)
+	}
+}
+
+// installerSecretKeys is every secrets.env key a rotation writes.
+func installerSecretKeys() map[string]bool {
+	keys := map[string]bool{installerForwardingKey: true, installerDBKey: true}
+	for _, k := range installerTokenKeys {
+		keys[k] = true
+	}
+	for _, k := range installerRegistryKeys {
+		keys[k] = true
+	}
+	return keys
+}
+
 // The installer and rotate-token must agree on where each caller's token lives:
 // rotate-token writes installerTokenKeys into secrets.env, and the installer
 // applies those same keys to the Secrets on its next run. A key the installer
@@ -249,12 +818,6 @@ func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
 		if !ok {
 			t.Fatalf("installerTokenKeys names unknown caller %q", caller)
 		}
-		if !strings.Contains(script, key+`="${`+key+`:-$(openssl rand -hex 32)}"`) {
-			t.Errorf("bootstrap.sh does not generate %s", key)
-		}
-		if !strings.Contains(script, key+"=${"+key+"}\n") {
-			t.Errorf("bootstrap.sh does not persist %s to secrets.env", key)
-		}
 		apply := regexp.MustCompile(`apply_literal_secret "\$CONTROL_NS" ` + regexp.QuoteMeta(ct.Secret) + ` token "\$` + key + `"`)
 		if !apply.MatchString(script) {
 			t.Errorf("bootstrap.sh does not apply %s from %s in the control namespace", ct.Secret, key)
@@ -281,3 +844,40 @@ func TestInstallerProvisionsEveryCallerToken(t *testing.T) {
 		t.Errorf("installerTokenKeys covers %d callers, naming.CallerTokens lists %d", len(installerTokenKeys), len(callerNames()))
 	}
 }
+
+// Every value the installer keeps in secrets.env has a rotation that rewrites
+// that very key, and every key a rotation writes is one the installer
+// generates, keeps and so re-applies on its next run.
+func TestInstallerSecretsAreAllRotatable(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "deploy", "bootstrap.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	script := string(raw)
+	m := regexp.MustCompile(`(?s)write_file_atomic "\$SECRETS_ENV" 0600 <<EOF\n(.*?)\nEOF\n`).FindStringSubmatch(script)
+	if m == nil {
+		t.Fatal("bootstrap.sh: no secrets.env heredoc found")
+	}
+	persisted := map[string]bool{}
+	for _, line := range strings.Split(m[1], "\n") {
+		key, value, _ := strings.Cut(line, "=")
+		if value != "${"+key+"}" {
+			t.Errorf("secrets.env line %q is not KEY=${KEY}", line)
+		}
+		persisted[key] = true
+	}
+	rotated := installerSecretKeys()
+	for key := range persisted {
+		if !rotated[key] {
+			t.Errorf("secrets.env keeps %s, which no rotation replaces", key)
+		}
+	}
+	for key := range rotated {
+		if !persisted[key] {
+			t.Errorf("a rotation writes %s, which the installer does not keep in secrets.env", key)
+		}
+		if !regexp.MustCompile(`\n  ` + key + `="\$\{` + key + `:-\$\(openssl rand -hex [0-9]+\)\}"\n`).MatchString(script) {
+			t.Errorf("bootstrap.sh does not generate %s when secrets.env lacks it", key)
+		}
+	}
+}

cmd/felis/run.go

@@ -31,7 +31,7 @@ Commands:
   apply             Create a MinecraftServer CRD (direct K8s write; use -f server.json)
   setup             Run host bootstrap + first-run setup console (TUI; requires root/sudo)
   converge          Fill in fields a newer desired spec added to already-installed system servers
-  rotate-token      Replace one internal caller's token and restart what holds it (velocity|limbo|build|ops; requires root/sudo)
+  rotate-token      Replace a generated credential and restart what reads it (velocity|limbo|build|ops|registry|forwarding|db; prints the plan, -yes applies; requires root/sudo)
   domain            Move the install to a new root domain on every surface that carries it, or check each one (set|check; requires root/sudo)
   watchdog          Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
   version           Print the build stamp of this binary

deploy/bootstrap.sh

@@ -3859,13 +3859,20 @@ EOF
 # velocity_fingerprint hashes what the proxy process runs: its unit (JVM flags and system
 # properties), the JRE, the jars and the files the installer writes for it. The Via config
 # and whatever else plugins write at runtime stay out; Via rewrites its config on every load.
+# So does the service-token line of felis-link.properties: the plugin re-reads it on its own
+# (`felis rotate-token velocity` counts on that), and a restart for it would only disconnect
+# every player.
 velocity_fingerprint() {
-  local f
+  local f sum
   for f in "$VELOCITY_SERVICE" "${JRE_DIR}/release" "${VELOCITY_DIR}/velocity.jar" \
       "${VELOCITY_DIR}/velocity.toml" "${VELOCITY_DIR}/forwarding.secret" \
       "${VELOCITY_DIR}/plugins/felis-link/felis-link.properties" "${VELOCITY_DIR}"/plugins/*.jar; do
     [ -f "$f" ] || continue
-    printf '%s %s\n' "$(sha256sum <"$f" | cut -d' ' -f1)" "$f"
+    case "$f" in
+      */felis-link.properties) sum="$({ grep -v '^service-token=' "$f" || true; } | sha256sum | cut -d' ' -f1)" ;;
+      *) sum="$(sha256sum <"$f" | cut -d' ' -f1)" ;;
+    esac
+    printf '%s %s\n' "$sum" "$f"
   done | sha256sum | cut -d' ' -f1
 }
 
@@ -4187,12 +4194,13 @@ load_or_make_secrets() {
   # to its own routes and a leak is contained to that caller: SERVICE_TOKEN is the
   # proxy's (felis-link.properties), LIMBO_TOKEN the login gate's, BUILD_TOKEN what a
   # build Job fetches its context with, OPS_TOKEN what `felis backup-now` presents.
-  # `felis rotate-token <caller>` rewrites the matching line here.
+  # `felis rotate-token <caller>` replaces one of them, and `felis rotate-token
+  # registry|forwarding|db` the other values persisted below: every line of secrets.env
+  # has a rotation that rewrites it (cmd/felis TestInstallerSecretsAreAllRotatable).
   SERVICE_TOKEN="${SERVICE_TOKEN:-$(openssl rand -hex 32)}"
   LIMBO_TOKEN="${LIMBO_TOKEN:-$(openssl rand -hex 32)}"
   BUILD_TOKEN="${BUILD_TOKEN:-$(openssl rand -hex 32)}"
   OPS_TOKEN="${OPS_TOKEN:-$(openssl rand -hex 32)}"
-  SESSION_SECRET="${SESSION_SECRET:-$(openssl rand -hex 32)}"
   # The Velocity modern-forwarding key. It is what makes a backend's UUID trustworthy:
   # the proxy does the Mojang handshake and HMACs the resulting profile with this key,
   # and a backend that cannot verify it would fall back to an offline UUID derived from
@@ -4213,7 +4221,6 @@ SERVICE_TOKEN=${SERVICE_TOKEN}
 LIMBO_TOKEN=${LIMBO_TOKEN}
 BUILD_TOKEN=${BUILD_TOKEN}
 OPS_TOKEN=${OPS_TOKEN}
-SESSION_SECRET=${SESSION_SECRET}
 FORWARDING_SECRET=${FORWARDING_SECRET}
 REGISTRY_PLATFORM_TOKEN=${REGISTRY_PLATFORM_TOKEN}
 REGISTRY_BUILD_TOKEN=${REGISTRY_BUILD_TOKEN}

deploy/bootstrap_test.sh

@@ -1643,7 +1643,7 @@ run_atomic() { # script; under the installer's shell options, its temp files rem
 mode_of() { ls -l "$1" | cut -c1-10; }
 
 printf 'A=1\nB=2\n' > "$wadir/in.new"
-printf 'DB_PASSWORD=new\nSESSION_SECRET=new\n' > "$wadir/in.replace"
+printf 'DB_PASSWORD=new\nSERVICE_TOKEN=new\n' > "$wadir/in.replace"
 out="$(run_atomic "umask 000; write_file_atomic '$wadir/new.env' 0600 < '$wadir/in.new'; echo done")"
 expect "an atomic write finishes" "done" "$out"
 expect "an atomic write holds all of its input" "$(printf 'A=1\nB=2')" "$(cat "$wadir/new.env")"
@@ -1651,11 +1651,11 @@ expect "an atomic write is private under a permissive umask" "-rw-------" "$(mod
 run_atomic "write_file_atomic '$wadir/new.env' 0640 < '$wadir/in.new'" >/dev/null
 expect "an atomic write sets the mode it is given" "-rw-r-----" "$(mode_of "$wadir/new.env")"
 
-printf 'DB_PASSWORD=old-and-whole\nSESSION_SECRET=kept\n' > "$wadir/old.env"
+printf 'DB_PASSWORD=old-and-whole\nSERVICE_TOKEN=kept\n' > "$wadir/old.env"
 out="$(run_atomic "cat() { head -c 7; return 1; }; write_file_atomic '$wadir/old.env' 0600 < '$wadir/in.replace'; echo survived")"
 expect "a write that fails halfway dies" "DIE: could not write $wadir/old.env; it is left as it was" "$out"
 expect "a write that fails halfway leaves the old file whole" \
-  "$(printf 'DB_PASSWORD=old-and-whole\nSESSION_SECRET=kept')" "$(cat "$wadir/old.env")"
+  "$(printf 'DB_PASSWORD=old-and-whole\nSERVICE_TOKEN=kept')" "$(cat "$wadir/old.env")"
 out="$(run_atomic "sync() { return 1; }; write_file_atomic '$wadir/old.env' 0600 < '$wadir/in.replace'")"
 expect "content that did not reach the disk does not replace the old file" "DIE: could not write $wadir/old.env" "$out"
 expect "the old file survives a failed sync" "DB_PASSWORD=old-and-whole" "$(cat "$wadir/old.env")"
@@ -2675,6 +2675,20 @@ expect "a new heap size restarts the proxy" "SYSTEMCTL restart felis-velocity" "
 expect "the unit carries the new ceiling" "java -Xms512M -Xmx3G " "$(cat "$vdir/unit")"
 run_velocity_service 1 384M >/dev/null
 expect "a ceiling below 512M is also the initial heap" "java -Xms384M -Xmx384M " "$(cat "$vdir/unit")"
+# `felis rotate-token velocity` rewrites service-token, which the plugin re-reads by itself:
+# that line alone changing leaves the proxy running, and any other change restarts it.
+props="$vdir/v/plugins/felis-link/felis-link.properties"
+printf 'api-base-url=http://a\nservice-token=one\n' > "$props"
+expect "new proxy properties restart the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1 384M)"
+printf 'api-base-url=http://a\nservice-token=two\n' > "$props"
+out="$(run_velocity_service 1 384M)"
+case "$out" in
+  *"SYSTEMCTL restart"*) echo "FAIL a new service-token alone restarted the proxy"; fails=$((fails + 1)) ;;
+  *"felis-velocity unchanged; left running"*) echo "PASS a new service-token alone leaves the proxy running" ;;
+  *) echo "FAIL install_velocity_service died on a new service-token: $out"; fails=$((fails + 1)) ;;
+esac
+printf 'api-base-url=http://b\nservice-token=two\n' > "$props"
+expect "another change to the proxy properties restarts the proxy" "SYSTEMCTL restart felis-velocity" "$(run_velocity_service 1 384M)"
 rm -rf "$vdir"
 
 ssblock="$(awk '/^restart_existing_system_servers\(\) \{/,/^}/' "$BS")"

deploy/limbo/README.md

@@ -144,7 +144,7 @@ set them by hand:
   the minecraft namespace (and `felis setup` refreshes that replica from the control
   namespace), and the operator injects it into the `login` pod (only) as
   `FELIS_SERVICE_TOKEN` via a `secretKeyRef`, keyed off the reserved `login` name.
-  `sudo felis rotate-token limbo` replaces it and restarts the pod. Until the token is
+  `sudo felis rotate-token -yes limbo` replaces it and restarts the pod. Until the token is
   present the plugin fail-safes to readiness-only, so the gate is never broken — it
   simply does not authenticate yet.
 - **Service:** the login pod dials `FELIS_API_BASE_URL`, which resolves to the

docs/openapi.yaml

@@ -143,7 +143,7 @@ components:
         (felis-build-token), ops (felis-ops-token) — and each operation lists the
         callers it serves in x-felis-callers. A genuine token for a caller the
         operation does not list is refused with 403 wrong_caller. `felis
-        rotate-token <caller>` replaces one.
+        rotate-token -yes <caller>` replaces one.
     sessionCookie:
       type: apiKey
       in: cookie

docs/troubleshooting.md

@@ -439,15 +439,47 @@ token on the wire, which is one more reason the proxy belongs on the node (§1 o
 
 ### Rotating a token
 
-`sudo felis rotate-token <velocity|limbo|build|ops>` replaces one caller's token:
-it writes the new value to `secrets.env` (so a later installer run keeps it), the
-Secret and its replica, rolls felis-api so only the new value is accepted, then
-restarts the caller — the `felis-velocity` unit when the proxy runs on this host,
-or the login pod. Build Jobs and `felis backup-now` pick the new value up on their
-next run. The old value stops working as soon as felis-api has rolled; the caller
-is turned away for the few seconds until it restarts. For a proxy on another host,
-the command leaves the host alone and tells you to copy the new value from the
-Secret into that proxy's `felis-link.properties` and restart it. [GO-TESTED]
+`sudo felis rotate-token <kind>` replaces one credential the installer generated.
+Without `-yes` it only prints what it would write and what that interrupts;
+`sudo felis rotate-token -yes <kind>` rotates. Every rotation writes the new value
+into `/etc/felis/secrets.env` first, so whatever fails after that, a re-run of the
+installer (`sudo bash deploy/bootstrap.sh`) puts the value everywhere. [GO-TESTED]
+
+| kind | replaces | interrupts |
+|---|---|---|
+| `velocity` | the proxy's token: Secret `felis-service-token`, `service-token` in the host proxy's `felis-link.properties` | felis-api restarts (one replica: the panel, sign-in and the proxy's calls stop for a few seconds). The proxy re-reads its file and keeps its players |
+| `limbo` | the login gate's token, `felis-limbo-token` and its minecraft replica | felis-api restarts, then the login pod |
+| `build` | the build Jobs' token, `felis-build-token` and its build-namespace replica | felis-api restarts; a build fetching its context at that moment fails and can be submitted again |
+| `ops` | `felis backup-now`'s token, `felis-ops-token` | felis-api restarts |
+| `registry` | the registry's `platform`, `build` and `prune` write tokens: Secret `felis-registry-auth`, the build Jobs' `felis-registry-push` | the registry restarts (a push at that moment fails, a pull retries), then felis-api |
+| `forwarding` | the Velocity forwarding secret: `/opt/felis/velocity/forwarding.secret`, Secret `felis-forwarding-secret` in both namespaces | every running server restarts (saving its world) and the proxy restarts: everyone online is disconnected |
+| `db` | the database role's password: the role in felis-postgres, `[database] url` in `felis.host.toml` and `felis.pod.toml`, Secret `felis-config` in both namespaces | felis-api restarts; a backup, restore or file Job that connects in those seconds fails and can be run again |
+
+- **velocity** — the rotation waits for the proxy's log line
+  `Felis: service-token reloaded from … (fingerprint <12 hex digits>)`. A proxy
+  that has not logged it within 60 s of felis-api's restart (a plugin from before
+  this release) is restarted, which disconnects everyone online. The file keeps
+  its modification time, so `felis domain check` does not read the proxy as
+  stale. A proxy on another host is left alone: set `service-token` in its
+  `felis-link.properties` to the value in Secret `felis/felis-service-token`, and
+  it takes it within a few seconds.
+- **forwarding** — a server whose world a backup, restore or file write holds is
+  left running and named in the plan and the output; players cannot join it until
+  it restarts, so stop and start it from the panel once that finishes. The next
+  installer run restarts the proxy once more (its record of what the proxy was
+  started from predates the rotation); an upgrade restarts it for the new plugin
+  jar anyway. A proxy on another host needs the value in Secret
+  `felis/felis-forwarding-secret` in its forwarding secret file, and a restart.
+- **db** — the password reaches PostgreSQL as a SCRAM-SHA-256 verifier, never as
+  text a failed statement could log. The config copies change only after a
+  connection with the new password succeeds; when it does not, the command stops
+  and says so, and the installer re-run sets the password from `secrets.env` in
+  the role and every copy. A database the installer does not run (`[database]
+  deployment` unset) is refused: change its password where it runs, then in each
+  copy's `[database] url`. `felis.pod.toml` must be a file of its own, since the
+  pods reach the database at another address.
+- The old value stops working as felis-api (or the registry) restarts; a caller
+  still presenting it is turned away until it has the new one.
 
 ---
 
@@ -867,9 +899,9 @@ control namespace (or `--registry-namespace`):
   container, via `felis-registry-push` in `felis-build`) may write anything outside
   `felis/` and `mirror/` and may never delete. A missing Secret leaves the registry
   read-only rather than down. The tokens persist in `/etc/felis/secrets.env`;
-  rotating one means editing it there and re-running the installer, then
-  `kubectl -n felis rollout restart deployment/registry` (the gate reads its tokens
-  at start).
+  `sudo felis rotate-token -yes registry` replaces all three and restarts the
+  registry (the gate reads its tokens at start) and felis-api, which presents the
+  `prune` token (§6 "Rotating a token").
 - **Who can connect:** `felis-registry-ingress` admits only the `felis-build`
   namespace to the registry pod. Node-local traffic (containerd pulls, the
   installer's pushes through the hostPort) is always allowed by Kubernetes; game
@@ -1916,7 +1948,7 @@ runs changed:
 
 | Component | Restarted when |
 |---|---|
-| `felis-velocity` (the proxy) | its unit, the JRE, `velocity.jar`, `velocity.toml`, the forwarding secret, the felis-link settings or a plugin jar changed, or it was not running. The fingerprint lives in `/etc/felis/velocity.fingerprint`; delete it to force a restart. |
+| `felis-velocity` (the proxy) | its unit, the JRE, `velocity.jar`, `velocity.toml`, the forwarding secret, the felis-link settings (all but `service-token`, which the plugin re-reads) or a plugin jar changed, or it was not running. The fingerprint lives in `/etc/felis/velocity.fingerprint`; delete it to force a restart. |
 | login and lobby pods | the rebuilt limbo or lobby image has a new image ID (`/etc/felis/system-server-images`). The installer then pins that server's `spec.image` to the digest its tag names now (`felis pin-images --system login\|lobby`) and the operator rolls the pod onto it, each on its own; with the registry unreachable it recreates the pod instead. The installer turns off BuildKit's default provenance attestation (`BUILDX_NO_DEFAULT_ATTESTATIONS=1`): it records the build time, which would give every rebuild a new ID. |
 | felis-postgres | the release moved `POSTGRES_IMAGE` or changed the pod; a few seconds without the API. A rerun that changes neither leaves it running. |
 | felis-api, felis-operator, the registry pod (its gate and GC containers run the felis binary) | the image tag changed (an upgrade), or a same-version rerun rebuilt it. |
@@ -2078,7 +2110,7 @@ along). One bundle is `felis-db-<UTC stamp>-<label>.tar`:
 |---|---|
 | `MANIFEST.json` | version, schema version, `pg_dump --version`, sha256 of every member |
 | `db.dump` | `pg_dump --format=custom` of the `felis` database |
-| `state/etc/felis/...` | every file in `/etc/felis`: `secrets.env` (DB password, session/forwarding secrets, registry tokens), `felis.host.toml`, `felis.pod.toml`, the `felis.toml` symlink, `offsite.env` (bucket credentials and encryption key), `smtp-password`, `uploads-s3-access-key` and `uploads-s3-secret-key` (the mail relay password and the uploads bucket keys `felis setup` took), the panel TLS pair, and the installer's own markers (`system-server-images`, `velocity.fingerprint`). `bootstrap.done` is left out on purpose |
+| `state/etc/felis/...` | every file in `/etc/felis`: `secrets.env` (DB password, forwarding secret, caller and registry tokens), `felis.host.toml`, `felis.pod.toml`, the `felis.toml` symlink, `offsite.env` (bucket credentials and encryption key), `smtp-password`, `uploads-s3-access-key` and `uploads-s3-secret-key` (the mail relay password and the uploads bucket keys `felis setup` took), the panel TLS pair, and the installer's own markers (`system-server-images`, `velocity.fingerprint`). `bootstrap.done` is left out on purpose |
 | `k8s/minecraftservers.json` | every MinecraftServer, status and server-side metadata stripped, ready for `kubectl apply`. The export is tried 3 times, 10 s apart; when the cluster still does not answer, the bundle is written without it and the manifest records why (next section) |
 
 next to a `.sha256` sidecar in `sha256sum` format. **A bundle contains the
@@ -2327,7 +2359,7 @@ host yourself, plus the off-site encryption key if the copy is in the bucket.
    written with a warning. Bundles from releases before the counts were
    recorded show `not recorded`.
 2. Put the old host's state in place **before** installing, so the installer
-   reuses the same DB password, session secret, forwarding secret, the mail
+   reuses the same DB password, caller tokens, forwarding secret, the mail
    relay password and uploads bucket keys `felis setup` took, and the
    `[offsite]` bucket with its credentials and key (`offsite.env`):
 

plugins/README.md

@@ -74,7 +74,8 @@ carry works, but nothing installs them for you.
 > the `limbo` token: it opens the link-code, link-status and blacklist routes and nothing
 > else. The `velocity` token also approves op-logins and wakes or claims servers for any
 > player; it stays on the proxy host. `sudo felis rotate-token limbo` replaces a leaked
-> token (the login gate restarts onto the new value; copy it to the mod by hand).
+> token (the login gate restarts onto the new value; copy it into the mod's
+> `felis-link.properties` by hand; the mod takes it on its next call, without a restart).
 
 ## Whose identity each path trusts
 
@@ -379,7 +380,10 @@ login gate the `limbo` token (`felis-limbo-token`), and each serves only its own
 routes (a token on another caller's route gets `403 wrong_caller`). A loader mod
 takes the `limbo` token, on a standalone online-mode server only (see the warning
 under the module table). Treat it as a secret; `sudo felis rotate-token <caller>`
-replaces it.
+replaces it. A token read from this file follows the file: the proxy or mod
+presents a new `service-token` from its next call to felis-api (it re-reads the
+file at most once a second), logs `service-token reloaded from … (fingerprint …)`, and keeps its
+players. A token from `FELIS_SERVICE_TOKEN` is fixed until the process restarts.
 
 On **Velocity**, also set `root-domain` (and optionally `lobby-server`) in the
 same file to turn on §11 routing, and make sure `online-mode=true` in

plugins/shared/src/main/java/best/lolicon/felis/link/FileToken.java

@@ -0,0 +1,86 @@
+package best.lolicon.felis.link;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HexFormat;
+import java.util.Properties;
+import java.util.function.Consumer;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
+
+/**
+ * FileToken is a service token read from felis-link.properties that follows the
+ * file while the process runs. {@code felis rotate-token velocity} rewrites the
+ * file (a rename, so it is never seen half-written) and waits for the notice this
+ * gives; the proxy presents the new token from its next felis-api call and keeps
+ * every player connected, where a restart would have dropped them all.
+ *
+ * <p>The file is read again at most once every {@link #RECHECK_NANOS} ns, on the
+ * calls themselves. A file that cannot be read, or holds no token, leaves the
+ * current token in place: an operator's half-finished edit never blanks a token
+ * that works.
+ */
+final class FileToken implements Supplier<String> {
+    static final long RECHECK_NANOS = 1_000_000_000L;
+
+    private final Path file;
+    private final String key;
+    private final LongSupplier clock;
+    private final Consumer<String> notice;
+    private String current;
+    private long checkedAt;
+
+    FileToken(Path file, String key, String initial, LongSupplier clock, Consumer<String> notice) {
+        this.file = file;
+        this.key = key;
+        this.current = initial;
+        this.clock = clock;
+        this.notice = notice;
+        this.checkedAt = clock.getAsLong();
+    }
+
+    @Override
+    public synchronized String get() {
+        long now = clock.getAsLong();
+        if (now - checkedAt < RECHECK_NANOS) {
+            return current;
+        }
+        checkedAt = now;
+        String read = read();
+        if (read != null && !read.equals(current)) {
+            current = read;
+            notice.accept("service-token reloaded from " + file + " (fingerprint " + fingerprint(read) + ")");
+        }
+        return current;
+    }
+
+    private String read() {
+        Properties props = new Properties();
+        try (InputStream in = Files.newInputStream(file)) {
+            props.load(in);
+        } catch (IOException | IllegalArgumentException e) {
+            return null;
+        }
+        String v = props.getProperty(key);
+        return v == null || v.trim().isEmpty() ? null : v;
+    }
+
+    /**
+     * fingerprint names a token in a log line without giving it away: the first 12
+     * hex digits of its SHA-256, which {@code felis rotate-token} computes the same
+     * way to recognise the reload it is waiting for.
+     */
+    static String fingerprint(String token) {
+        try {
+            byte[] sum = MessageDigest.getInstance("SHA-256").digest(token.getBytes(StandardCharsets.UTF_8));
+            return HexFormat.of().formatHex(sum).substring(0, 12);
+        } catch (NoSuchAlgorithmException e) {
+            throw new IllegalStateException("SHA-256 is missing from this JVM", e);
+        }
+    }
+}

plugins/shared/src/main/java/best/lolicon/felis/link/LinkConfig.java

@@ -2,14 +2,16 @@ package best.lolicon.felis.link;
 
 import java.time.Duration;
 import java.util.Objects;
+import java.util.function.Supplier;
 
 /**
- * LinkConfig is the immutable configuration a {@link LinkClient} needs to reach
- * the felis-api internal face. Both the base URL and the service token are
- * deployment inputs — operator config or a Secret-injected environment variable —
- * and are <em>never</em> compiled in. Keeping them out of source is what lets the
- * tree stay domain- and credential-free; each platform's config loader is
- * responsible for sourcing them.
+ * LinkConfig is the configuration a {@link LinkClient} needs to reach the
+ * felis-api internal face. It is fixed once built, except for a service token
+ * read from felis-link.properties, which follows that file ({@link FileToken}).
+ * Both the base URL and the service token are deployment inputs — operator config
+ * or a Secret-injected environment variable — and are <em>never</em> compiled in.
+ * Keeping them out of source is what lets the tree stay domain- and
+ * credential-free; each platform's config loader is responsible for sourcing them.
  *
  * <p>Two timeouts bound each call: {@code connectTimeout} for opening the TCP
  * connection and {@code requestTimeout} for the whole exchange once it is open. A
@@ -18,11 +20,17 @@ import java.util.Objects;
  */
 public final class LinkConfig {
     private final String apiBaseUrl;
-    private final String serviceToken;
+    private final Supplier<String> serviceToken;
     private final Duration connectTimeout;
     private final Duration requestTimeout;
 
     public LinkConfig(String apiBaseUrl, String serviceToken, Duration connectTimeout, Duration requestTimeout) {
+        this(apiBaseUrl, fixed(serviceToken), connectTimeout, requestTimeout);
+    }
+
+    // LinkConfig with a token that may change while the process runs (FileToken);
+    // it must hold one from the start.
+    LinkConfig(String apiBaseUrl, Supplier<String> serviceToken, Duration connectTimeout, Duration requestTimeout) {
         this.apiBaseUrl = stripTrailingSlash(Objects.requireNonNull(apiBaseUrl, "apiBaseUrl"));
         this.serviceToken = Objects.requireNonNull(serviceToken, "serviceToken");
         this.connectTimeout = Objects.requireNonNull(connectTimeout, "connectTimeout");
@@ -30,7 +38,7 @@ public final class LinkConfig {
         if (this.apiBaseUrl.isEmpty()) {
             throw new IllegalArgumentException("apiBaseUrl is empty");
         }
-        if (this.serviceToken.isEmpty()) {
+        if (this.serviceToken.get().isEmpty()) {
             throw new IllegalArgumentException("serviceToken is empty");
         }
         if (this.connectTimeout.isZero() || this.connectTimeout.isNegative()) {
@@ -49,8 +57,9 @@ public final class LinkConfig {
         return apiBaseUrl;
     }
 
+    /** serviceToken is the token to present on this call; FileToken may have replaced it since the last. */
     public String serviceToken() {
-        return serviceToken;
+        return serviceToken.get();
     }
 
     public Duration connectTimeout() {
@@ -61,6 +70,11 @@ public final class LinkConfig {
         return requestTimeout;
     }
 
+    private static Supplier<String> fixed(String token) {
+        Objects.requireNonNull(token, "serviceToken");
+        return () -> token;
+    }
+
     private static String stripTrailingSlash(String u) {
         String t = u.trim();
         while (t.endsWith("/")) {

plugins/shared/src/main/java/best/lolicon/felis/link/LinkConfigLoader.java

@@ -8,6 +8,8 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
 import java.util.Properties;
+import java.util.function.Consumer;
+import java.util.function.LongSupplier;
 import java.util.function.UnaryOperator;
 
 /**
@@ -18,7 +20,9 @@ import java.util.function.UnaryOperator;
  * compiled in — this loader is the single seam each loader's entrypoint calls, so
  * the source tree stays domain- and credential-free. On first run it writes a
  * commented template and then reports the values as missing, so an operator gets
- * a file to fill in rather than a silent half-configured plugin.
+ * a file to fill in rather than a silent half-configured plugin. A token from the
+ * file follows the file while the process runs ({@link FileToken}), so
+ * {@code felis rotate-token velocity} replaces it without restarting the proxy.
  *
  * <p>The two call timeouts ({@code connect-timeout-seconds},
  * {@code request-timeout-seconds}, or {@code FELIS_API_CONNECT_TIMEOUT_SECONDS} /
@@ -51,11 +55,26 @@ public final class LinkConfigLoader {
      *     timeout is not a whole number of seconds in range.
      */
     public static LinkConfig load(Path propertiesFile) throws IOException {
-        return load(propertiesFile, System::getenv);
+        return load(propertiesFile, msg -> { });
+    }
+
+    /**
+     * load, with {@code notice} told when a token from the file is replaced while
+     * the process runs ({@link FileToken}). A token from the environment is fixed
+     * for the life of the process.
+     */
+    public static LinkConfig load(Path propertiesFile, Consumer<String> notice) throws IOException {
+        return load(propertiesFile, System::getenv, notice, System::nanoTime);
     }
 
-    // load with the environment passed in, so the precedence rules are testable.
     static LinkConfig load(Path propertiesFile, UnaryOperator<String> env) throws IOException {
+        return load(propertiesFile, env, msg -> { }, System::nanoTime);
+    }
+
+    // load with the environment and the clock passed in, so the precedence rules
+    // and the file token's re-reads are testable.
+    static LinkConfig load(Path propertiesFile, UnaryOperator<String> env, Consumer<String> notice,
+                           LongSupplier clock) throws IOException {
         Properties props = new Properties();
         if (Files.exists(propertiesFile)) {
             try (InputStream in = Files.newInputStream(propertiesFile)) {
@@ -76,6 +95,9 @@ public final class LinkConfigLoader {
                 firstNonBlank(env.apply(ENV_CONNECT_TIMEOUT), props.getProperty(KEY_CONNECT_TIMEOUT)));
         Duration request = seconds(KEY_REQUEST_TIMEOUT, ENV_REQUEST_TIMEOUT,
                 firstNonBlank(env.apply(ENV_REQUEST_TIMEOUT), props.getProperty(KEY_REQUEST_TIMEOUT)));
+        if (isBlank(env.apply(ENV_TOKEN))) {
+            return new LinkConfig(url, new FileToken(propertiesFile, KEY_TOKEN, token, clock, notice), connect, request);
+        }
         return new LinkConfig(url, token, connect, request);
     }
 

plugins/shared/test/best/lolicon/felis/link/LinkConfigLoaderTest.java

@@ -5,8 +5,10 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
@@ -15,7 +17,9 @@ import java.util.stream.Stream;
  * the environment wins, both required values must come from somewhere, a first run
  * leaves a template and still refuses to start, and the call timeouts default to
  * 10 s, take the operator's value, and refuse anything that is not a whole number
- * of seconds from 1 to 120.
+ * of seconds from 1 to 120. A token from the file follows the file (at most one
+ * look a second, a notice naming its fingerprint on each change, a blank or missing
+ * file ignored); one from the environment stays fixed.
  *
  * <p>Run: {@code javac -d <out> shared/src/main/java/best/lolicon/felis/link/*.java
  * shared/test/best/lolicon/felis/link/LinkConfigLoaderTest.java && java -cp <out>
@@ -35,6 +39,8 @@ public final class LinkConfigLoaderTest {
             firstRunWritesATemplateAndRefuses();
             timeoutsComeFromFileOrEnvironment();
             badTimeoutsAreRefused();
+            fileTokenFollowsTheFile();
+            environmentTokenIsFixed();
         } finally {
             try (Stream<Path> walk = Files.walk(dir)) {
                 walk.sorted(Comparator.reverseOrder()).forEach(p -> p.toFile().delete());
@@ -124,6 +130,55 @@ public final class LinkConfigLoaderTest {
         assertContains("env bad names the variable", e.getMessage(), "FELIS_API_REQUEST_TIMEOUT_SECONDS");
     }
 
+    // `felis rotate-token velocity` rewrites the file and waits for the notice
+    // naming the new token's fingerprint; the proxy presents the new token from
+    // then on, without a restart.
+    private static void fileTokenFollowsTheFile() throws IOException {
+        Path f = write("rotate.properties", "api-base-url=http://x:8081\nservice-token=old-token\n");
+        long[] now = {5_000_000_000L};
+        List<String> notices = new ArrayList<>();
+        LinkConfig c = LinkConfigLoader.load(f, env(), notices::add, () -> now[0]);
+        assertEq("initial file token", "old-token", c.serviceToken());
+
+        write("rotate.properties", "api-base-url=http://x:8081\nservice-token=new-token\n");
+        now[0] += FileToken.RECHECK_NANOS - 1;
+        assertEq("within a second of the last look the file is not read", "old-token", c.serviceToken());
+        now[0] += 1;
+        assertEq("a second on, the rewritten token is presented", "new-token", c.serviceToken());
+        assertEq("one notice for one change", 1, notices.size());
+        write("rotate.properties", "api-base-url=http://x:8081\nservice-token=third-token\n");
+        now[0] += FileToken.RECHECK_NANOS - 1;
+        assertEq("the second counts from the last look", "new-token", c.serviceToken());
+        write("rotate.properties", "api-base-url=http://x:8081\nservice-token=new-token\n");
+        assertEq("the notice names the file and the fingerprint",
+                "service-token reloaded from " + f + " (fingerprint 348e9df2a42b)", notices.get(0));
+        now[0] += FileToken.RECHECK_NANOS;
+        c.serviceToken();
+        assertEq("an unchanged file gives no notice", 1, notices.size());
+
+        // A file mid-edit, or gone, keeps the token that works.
+        write("rotate.properties", "api-base-url=http://x:8081\nservice-token=  \n");
+        now[0] += FileToken.RECHECK_NANOS;
+        assertEq("a blank token in the file is ignored", "new-token", c.serviceToken());
+        Files.delete(f);
+        now[0] += FileToken.RECHECK_NANOS;
+        assertEq("a missing file is ignored", "new-token", c.serviceToken());
+        assertEq("neither gave a notice", 1, notices.size());
+    }
+
+    // The login gate's token comes from its pod's environment, which only a
+    // restart changes: the file does not override it later.
+    private static void environmentTokenIsFixed() throws IOException {
+        Path f = write("env-fixed.properties", "api-base-url=http://x:8081\nservice-token=file-token\n");
+        long[] now = {0};
+        List<String> notices = new ArrayList<>();
+        LinkConfig c = LinkConfigLoader.load(f, env("FELIS_SERVICE_TOKEN", "env-token"), notices::add, () -> now[0]);
+        write("env-fixed.properties", "api-base-url=http://x:8081\nservice-token=other-token\n");
+        now[0] += 10 * FileToken.RECHECK_NANOS;
+        assertEq("env token stays", "env-token", c.serviceToken());
+        assertEq("env token gives no notice", 0, notices.size());
+    }
+
     // ---- harness ----
 
     private static java.util.function.UnaryOperator<String> env(String... kv) {

plugins/velocity/src/main/java/best/lolicon/felis/velocity/FelisVelocityConfig.java

@@ -9,6 +9,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Locale;
 import java.util.Properties;
+import java.util.function.Consumer;
 
 /**
  * FelisVelocityConfig extends the shared link config with the inputs only the full
@@ -62,8 +63,9 @@ final class FelisVelocityConfig {
         this.adminHostname = adminHostname;
     }
 
-    static FelisVelocityConfig load(Path file) throws IOException {
-        LinkConfig link = LinkConfigLoader.load(file); // url + token (required) + template + validate
+    // load reads the file; notice hears of a service token replaced in it later.
+    static FelisVelocityConfig load(Path file, Consumer<String> notice) throws IOException {
+        LinkConfig link = LinkConfigLoader.load(file, notice); // url + token (required) + template + validate
         Properties props = new Properties();
         if (Files.exists(file)) {
             try (InputStream in = Files.newInputStream(file)) {

plugins/velocity/src/main/java/best/lolicon/felis/velocity/FelisVelocityPlugin.java

@@ -131,7 +131,8 @@ public final class FelisVelocityPlugin {
     @Subscribe
     public void onProxyInitialize(ProxyInitializeEvent event) {
         try {
-            this.config = FelisVelocityConfig.load(dataDirectory.resolve("felis-link.properties"));
+            this.config = FelisVelocityConfig.load(dataDirectory.resolve("felis-link.properties"),
+                    msg -> logger.info("Felis: {}", msg));
         } catch (Exception e) {
             logger.error("Felis disabled: {}", e.getMessage());
             return;