fix(operator): 有玩家在线时停服先在游戏里预告 30 秒，期间改回运行会取消

deploy/crd/felis.lolicon.best_minecraftservers.yaml

@@ -433,6 +433,14 @@ spec:
                   because the two endpoints fall in different reconcile passes.
                 format: date-time
                 type: string
+              stopNoticeAt:
+                description: |-
+                  StopNoticeAt is when the operator told the players on a server that it is
+                  about to stop (desiredState flipped to Stopped with players online). The stop
+                  itself waits until StopNoticeWindow has passed since then; the stamp is cleared
+                  once the server is scaled down, or when desiredState goes back to Running first.
+                format: date-time
+                type: string
             type: object
         type: object
     served: true

docs/troubleshooting.md

@@ -284,8 +284,19 @@ panel shows `Stopped` means the game pod is still terminating (its shutdown save
 can take a while); retry once `kubectl -n minecraft get pods -l
 felis.lolicon.best/server=<name>` shows nothing.
 
+A stop that leaves the server `Running` for about 30 s is the players' warning.
+When desiredState turns `Stopped` and the RCON tally does not say the server is
+empty, the operator sends everyone online a yellow chat line (`tellraw @a`),
+stamps `status.stopNoticeAt`, records a `StopNotice` Event, and scales down
+once `StopNoticeWindow` (30 s) has passed, with a last "stopping now" line
+before the save. Starting the server again inside the window calls the stop
+off, and the players are told so. An empty server, RCON off, or a probe or
+broadcast that fails stops at once. Idle auto-stop only fires on an empty
+server, so it never waits.
+
 [GO-TESTED: `internal/maintenance`, `k8scluster_maintenance_test.go`,
-`handlers_maintenance_test.go`, operator `maintenance_test.go`.]
+`handlers_maintenance_test.go`, operator `maintenance_test.go`,
+`stopnotice_test.go`.]
 
 ---
 

internal/apis/felis/v1alpha1/minecraftserver_types.go

@@ -342,6 +342,11 @@ type MinecraftServerStatus struct {
 	// or the server stops, so the empty-duration counter starts fresh each time
 	// the server becomes unoccupied.
 	EmptySince *metav1.Time `json:"emptySince,omitempty"`
+	// StopNoticeAt is when the operator told the players on a server that it is
+	// about to stop (desiredState flipped to Stopped with players online). The stop
+	// itself waits until StopNoticeWindow has passed since then; the stamp is cleared
+	// once the server is scaled down, or when desiredState goes back to Running first.
+	StopNoticeAt *metav1.Time `json:"stopNoticeAt,omitempty"`
 	// AutoRestarts counts how often the operator recreated the pod of a start
 	// that timed out (at most MaxAutoRestarts, with a doubling backoff); reaching Ready or
 	// stopping resets it.

internal/apis/felis/v1alpha1/zz_generated.deepcopy.go

@@ -121,6 +121,9 @@ func (in *MinecraftServerStatus) DeepCopyInto(out *MinecraftServerStatus) {
 	if in.EmptySince != nil {
 		out.EmptySince = in.EmptySince.DeepCopy()
 	}
+	if in.StopNoticeAt != nil {
+		out.StopNoticeAt = in.StopNoticeAt.DeepCopy()
+	}
 	if in.LastAutoRestartAt != nil {
 		out.LastAutoRestartAt = in.LastAutoRestartAt.DeepCopy()
 	}

internal/operator/autorestart_test.go

@@ -122,6 +122,8 @@ func (p switchProber) Probe(context.Context, string, string) (operator.PlayerCou
 
 func (switchProber) Save(context.Context, string, string) error { return nil }
 
+func (switchProber) Broadcast(context.Context, string, string, string) error { return nil }
+
 // A Running server keeps its phase and endpoint through two missed probes,
 // re-probing every 10s; the third consecutive miss degrades it to Starting,
 // and any success in between starts the count over.

internal/operator/prober.go

@@ -2,9 +2,13 @@ package operator
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"regexp"
 	"strconv"
+	"strings"
 	"time"
+	"unicode/utf16"
 
 	"felis.lolicon.best/internal/rcon"
 )
@@ -31,9 +35,13 @@ type PlayerCount struct {
 // Save flushes the world to disk (`save-all flush`) and returns once the server
 // has answered, i.e. once the save is done. The reconciler runs it right before
 // scaling a server to zero (spec §7).
+//
+// Broadcast shows text in every online player's chat (`tellraw @a`). The reconciler
+// uses it to warn players before a stop.
 type Prober interface {
 	Probe(ctx context.Context, addr, password string) (PlayerCount, error)
 	Save(ctx context.Context, addr, password string) error
+	Broadcast(ctx context.Context, addr, password, text string) error
 }
 
 // RconProber is the production Prober: a successful Dial (TCP connect + auth)
@@ -110,6 +118,45 @@ func (p RconProber) Save(ctx context.Context, addr, password string) error {
 	return err
 }
 
+// Broadcast runs `tellraw @a` with text as one yellow chat line. tellraw takes a
+// JSON text component on every loader and version Felis runs, and unlike `say` it
+// shows the text without a "[Server]" or "[Rcon]" prefix.
+func (p RconProber) Broadcast(ctx context.Context, addr, password, text string) error {
+	timeout := boundTimeout(ctx, p.Timeout, 5*time.Second)
+	conn, err := rcon.Dial(addr, password, timeout)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {
+		return err
+	}
+	_, err = conn.Execute(tellrawCommand(text))
+	return err
+}
+
+// tellrawCommand is the `tellraw @a` command that shows text to everyone online.
+// Everything past ASCII is written as a JSON \u escape, so the command survives
+// an RCON implementation that does not read its payload as UTF-8; the game's JSON
+// parser turns the escapes back into the text.
+func tellrawCommand(text string) string {
+	component, _ := json.Marshal(map[string]string{"text": text, "color": "yellow"})
+	var b strings.Builder
+	b.WriteString("tellraw @a ")
+	for _, r := range string(component) {
+		switch {
+		case r < 0x80:
+			b.WriteRune(r)
+		case r > 0xFFFF:
+			hi, lo := utf16.EncodeRune(r)
+			fmt.Fprintf(&b, `\u%04x\u%04x`, hi, lo)
+		default:
+			fmt.Fprintf(&b, `\u%04x`, r)
+		}
+	}
+	return b.String()
+}
+
 // listReplyPatterns match the `list` replies of the loaders Felis runs, tried in
 // order against the reply with § color codes stripped:
 //

internal/operator/prober_internal_test.go

@@ -3,8 +3,10 @@ package operator
 import (
 	"context"
 	"encoding/binary"
+	"encoding/json"
 	"io"
 	"net"
+	"strings"
 	"testing"
 	"time"
 )
@@ -159,3 +161,36 @@ func TestRconProberSaveTimesOut(t *testing.T) {
 		t.Errorf("Save took %v, want it bounded by SaveTimeout", elapsed)
 	}
 }
+
+// The broadcast is plain ASCII on the wire and decodes back to the exact text, CJK
+// and a character outside the BMP included.
+func TestTellrawCommand(t *testing.T) {
+	text := `[Felis] 服务器将在 30 秒后关闭 / "stops" 🙂`
+	cmd := tellrawCommand(text)
+	want := `tellraw @a {"color":"yellow","text":"[Felis] \u670d\u52a1\u5668\u5c06\u5728 30 \u79d2\u540e\u5173\u95ed / \"stops\" \ud83d\ude42"}`
+	if cmd != want {
+		t.Errorf("command =\n%s\nwant\n%s", cmd, want)
+	}
+	for i := 0; i < len(cmd); i++ {
+		if cmd[i] >= 0x80 {
+			t.Fatalf("byte %d of %q is not ASCII", i, cmd)
+		}
+	}
+	var component map[string]string
+	if err := json.Unmarshal([]byte(strings.TrimPrefix(cmd, "tellraw @a ")), &component); err != nil {
+		t.Fatalf("component does not parse: %v", err)
+	}
+	if component["text"] != text {
+		t.Errorf("decoded text = %q, want %q", component["text"], text)
+	}
+}
+
+func TestRconProberBroadcastSendsTellraw(t *testing.T) {
+	addr, cmds := serveFakeRcon(t, 0)
+	if err := (RconProber{}).Broadcast(context.Background(), addr, "pw", "关闭 / stop"); err != nil {
+		t.Fatalf("Broadcast: %v", err)
+	}
+	if got, want := <-cmds, tellrawCommand("关闭 / stop"); got != want {
+		t.Errorf("command = %q, want %q", got, want)
+	}
+}

internal/operator/reconciler.go

@@ -277,6 +277,11 @@ func (r *Reconciler) event(server *v1alpha1.MinecraftServer, eventType, reason,
 }
 
 func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.MinecraftServer) (ctrl.Result, error) {
+	if server.Status.StopNoticeAt != nil {
+		if err := r.callOffStop(ctx, server); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
 	if kind, held, err := r.maintenanceHold(ctx, server); err != nil {
 		return ctrl.Result{}, err
 	} else if held {
@@ -495,7 +500,11 @@ func (r *Reconciler) reconcileStopped(ctx context.Context, server *v1alpha1.Mine
 	// which sends the server SIGTERM and so its own shutdown save within the
 	// grace period.
 	if sts.Spec.Replicas == nil || *sts.Spec.Replicas != 0 {
+		if wait, err := r.noticeStop(ctx, server, &sts); err != nil || wait > 0 {
+			return ctrl.Result{RequeueAfter: wait}, err
+		}
 		r.saveBeforeStop(ctx, server, &sts)
+		server.Status.StopNoticeAt = nil
 		zero := int32(0)
 		sts.Spec.Replicas = &zero
 		if err := r.Update(ctx, &sts); err != nil {
@@ -515,6 +524,86 @@ func (r *Reconciler) reconcileStopped(ctx context.Context, server *v1alpha1.Mine
 	return ctrl.Result{}, r.patchStatus(ctx, server)
 }
 
+// StopNoticeWindow is how long the players on a server get between the warning
+// and the stop: long enough to finish a fight or get off a boat, short enough that
+// the owner's stop (and the backup or restore waiting behind it) is not held up.
+const StopNoticeWindow = 30 * time.Second
+
+// The lines players see around a stop. The operator does not know a player's
+// language, so each carries both.
+const (
+	stopNoticeText    = "[Felis] 服务器将在 30 秒后关闭，请尽快找安全的地方 / This server stops in 30 seconds"
+	stopNowText       = "[Felis] 正在保存世界并关闭服务器 / Saving the world and stopping now"
+	stopCalledOffText = "[Felis] 关闭已取消 / The stop was called off"
+)
+
+// noticeStop warns the players on a server that is about to be scaled down, and
+// returns how long the stop must still wait for them. The first pass tells everyone
+// online and stamps StopNoticeAt; the stop goes ahead once StopNoticeWindow has
+// passed since, with a last line as it does.
+//
+// There is no wait when nobody can be told or nobody is there: RCON off, no ready
+// pod, a probe or broadcast that fails, or a tally that says zero players. A tally
+// the server did not give (an unfamiliar `list` reply) still gets the warning,
+// since players may well be on it. Idle auto-stop therefore never waits: it only
+// fires on an empty server.
+func (r *Reconciler) noticeStop(ctx context.Context, server *v1alpha1.MinecraftServer, sts *appsv1.StatefulSet) (time.Duration, error) {
+	if at := server.Status.StopNoticeAt; at != nil {
+		if wait := at.Time.Add(StopNoticeWindow).Sub(r.now().Time); wait > 0 {
+			return wait, nil
+		}
+		r.broadcast(ctx, server, stopNowText)
+		return 0, nil
+	}
+	if !server.Spec.Rcon.Enabled || sts.Status.ReadyReplicas == 0 {
+		return 0, nil
+	}
+	password, err := r.rconPassword(ctx, server)
+	if err != nil {
+		return 0, nil
+	}
+	addr := rconAddress(server)
+	players, err := r.Prober.Probe(ctx, addr, password)
+	if err != nil || (players.Known && players.Online == 0) {
+		return 0, nil
+	}
+	if err := r.Prober.Broadcast(ctx, addr, password, stopNoticeText); err != nil {
+		ctrl.LoggerFrom(ctx).Info("could not warn players before the stop; stopping now", "error", err.Error())
+		return 0, nil
+	}
+	now := r.now()
+	server.Status.StopNoticeAt = &now
+	if err := r.patchStatus(ctx, server); err != nil {
+		return 0, err
+	}
+	r.event(server, corev1.EventTypeNormal, "StopNotice",
+		fmt.Sprintf("players warned; stopping in %s", StopNoticeWindow))
+	return StopNoticeWindow, nil
+}
+
+// callOffStop answers a desiredState that went back to Running inside the notice
+// window: the players who were warned hear that the stop is off.
+func (r *Reconciler) callOffStop(ctx context.Context, server *v1alpha1.MinecraftServer) error {
+	r.broadcast(ctx, server, stopCalledOffText)
+	server.Status.StopNoticeAt = nil
+	return r.patchStatus(ctx, server)
+}
+
+// broadcast shows text to everyone on server, best-effort: a server that cannot be
+// reached has nobody listening either.
+func (r *Reconciler) broadcast(ctx context.Context, server *v1alpha1.MinecraftServer, text string) {
+	if !server.Spec.Rcon.Enabled {
+		return
+	}
+	password, err := r.rconPassword(ctx, server)
+	if err != nil {
+		return
+	}
+	if err := r.Prober.Broadcast(ctx, rconAddress(server), password, text); err != nil {
+		ctrl.LoggerFrom(ctx).Info("player broadcast failed", "error", err.Error())
+	}
+}
+
 // saveBeforeStop runs `save-all flush` on a server that is about to be scaled to
 // zero. The SIGTERM that follows makes the server save again on its way out, but
 // that save races the grace period: a large world killed mid-save rolls back to

internal/operator/reconciler_test.go

@@ -31,6 +31,9 @@ type fakeProber struct {
 	saveErr error
 	// saves, when set, records each Save's address.
 	saves *[]string
+	// broadcasts, when set, records each Broadcast's text.
+	broadcasts   *[]string
+	broadcastErr error
 }
 
 func (f fakeProber) Probe(context.Context, string, string) (operator.PlayerCount, error) {
@@ -44,6 +47,13 @@ func (f fakeProber) Save(_ context.Context, addr, _ string) error {
 	return f.saveErr
 }
 
+func (f fakeProber) Broadcast(_ context.Context, _, _, text string) error {
+	if f.broadcasts != nil {
+		*f.broadcasts = append(*f.broadcasts, text)
+	}
+	return f.broadcastErr
+}
+
 func newScheme(t *testing.T) *runtime.Scheme {
 	t.Helper()
 	scheme := runtime.NewScheme()
@@ -316,7 +326,7 @@ func TestReconcileStopped_NoWorkloadIsStopped(t *testing.T) {
 }
 
 func TestReconcileStopped_ScalesRunningWorkloadDown(t *testing.T) {
-	r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret())
+	r, c := newReconciler(t, fakeProber{players: operator.PlayerCount{Online: 0, Max: 20, Known: true}}, runningServer(), rconSecret())
 
 	reconcile(t, r, "survival")
 	markPodReady(t, c, "survival")
@@ -353,7 +363,7 @@ func stopRunningServer(t *testing.T, r *operator.Reconciler, c client.Client) {
 
 func TestReconcileStopped_SavesBeforeScalingDown(t *testing.T) {
 	var saves []string
-	r, c := newReconciler(t, fakeProber{saves: &saves}, runningServer(), rconSecret())
+	r, c := newReconciler(t, fakeProber{saves: &saves, players: operator.PlayerCount{Known: true}}, runningServer(), rconSecret())
 
 	stopRunningServer(t, r, c)
 
@@ -374,7 +384,7 @@ func TestReconcileStopped_SavesBeforeScalingDown(t *testing.T) {
 
 func TestReconcileStopped_FailedSaveStillStops(t *testing.T) {
 	var saves []string
-	prober := fakeProber{saves: &saves, saveErr: errors.New("i/o timeout")}
+	prober := fakeProber{saves: &saves, saveErr: errors.New("i/o timeout"), players: operator.PlayerCount{Known: true}}
 	r, c := newReconciler(t, prober, runningServer(), rconSecret())
 
 	stopRunningServer(t, r, c)
@@ -1149,7 +1159,7 @@ func TestFelisUpgradeLeavesARunningServerAlone(t *testing.T) {
 // TestFelisUpgradeReachesAServerOnItsNextStart: a stopped server's next start
 // writes the whole template, the new felis image included.
 func TestFelisUpgradeReachesAServerOnItsNextStart(t *testing.T) {
-	r, c := newReconciler(t, fakeProber{players: operator.PlayerCount{Online: 1, Max: 20, Known: true}}, runningServer(), rconSecret())
+	r, c := newReconciler(t, fakeProber{players: operator.PlayerCount{Online: 0, Max: 20, Known: true}}, runningServer(), rconSecret())
 	r.FelisImage = felisV1
 	stopRunningServer(t, r, c)
 	markPodTerminated(t, c, "survival")

internal/operator/stopnotice_test.go

@@ -0,0 +1,185 @@
+package operator_test
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/operator"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// runThenStop takes survival to Running with a ready pod, flips it to Stopped and
+// returns the result of the first Stopped pass.
+func runThenStop(t *testing.T, r *operator.Reconciler, c client.Client) time.Duration {
+	t.Helper()
+	reconcile(t, r, "survival")
+	markPodReady(t, c, "survival")
+	reconcile(t, r, "survival")
+	setDesired(t, c, v1alpha1.DesiredStopped)
+	return reconcile(t, r, "survival").RequeueAfter
+}
+
+func setDesired(t *testing.T, c client.Client, desired v1alpha1.DesiredState) {
+	t.Helper()
+	server := getServer(t, c, "survival")
+	server.Spec.DesiredState = desired
+	if err := c.Update(context.Background(), server); err != nil {
+		t.Fatalf("set desiredState %s: %v", desired, err)
+	}
+}
+
+func at(offset time.Duration) func() metav1.Time {
+	return func() metav1.Time { return metav1.NewTime(fixedNow().Add(offset)) }
+}
+
+func replicas(t *testing.T, c client.Client) int32 {
+	t.Helper()
+	sts := getSTS(t, c, "survival")
+	if sts.Spec.Replicas == nil {
+		return 1
+	}
+	return *sts.Spec.Replicas
+}
+
+// A stop with players on the server warns them, holds the pod for the window, then
+// saves and scales down with a last line.
+func TestStopNotice_WarnsPlayersThenStops(t *testing.T) {
+	var saves, said []string
+	prober := fakeProber{players: operator.PlayerCount{Online: 3, Max: 20, Known: true}, saves: &saves, broadcasts: &said}
+	r, c := newReconciler(t, prober, runningServer(), rconSecret())
+
+	if wait := runThenStop(t, r, c); wait != operator.StopNoticeWindow {
+		t.Fatalf("requeue = %v, want the %v window", wait, operator.StopNoticeWindow)
+	}
+	if len(said) != 1 || !strings.Contains(said[0], "30 秒") || !strings.Contains(said[0], "30 seconds") {
+		t.Fatalf("broadcasts = %q, want the 30-second warning in both languages", said)
+	}
+	server := getServer(t, c, "survival")
+	if server.Status.StopNoticeAt == nil || !server.Status.StopNoticeAt.Equal(ptrTime(fixedNow())) {
+		t.Errorf("stopNoticeAt = %v, want %v", server.Status.StopNoticeAt, fixedNow())
+	}
+	if server.Status.Phase != v1alpha1.PhaseRunning || !server.Status.Ready {
+		t.Errorf("phase = %s ready=%v, want the server still Running while players are warned", server.Status.Phase, server.Status.Ready)
+	}
+	if n := replicas(t, c); n != 1 || len(saves) != 0 {
+		t.Fatalf("replicas = %d saves = %d, want the pod left alone during the window", n, len(saves))
+	}
+
+	// A pass inside the window waits out the rest and says nothing new.
+	r.Now = at(10 * time.Second)
+	if wait := reconcile(t, r, "survival").RequeueAfter; wait != 20*time.Second {
+		t.Errorf("requeue 10s in = %v, want 20s", wait)
+	}
+	if len(said) != 1 || len(saves) != 0 || replicas(t, c) != 1 {
+		t.Fatalf("10s in: broadcasts=%q saves=%d replicas=%d, want nothing new", said, len(saves), replicas(t, c))
+	}
+
+	r.Now = at(operator.StopNoticeWindow)
+	reconcile(t, r, "survival")
+	if len(said) != 2 || !strings.Contains(said[1], "正在保存") {
+		t.Errorf("broadcasts = %q, want the stopping-now line last", said)
+	}
+	if len(saves) != 1 || replicas(t, c) != 0 {
+		t.Fatalf("saves = %d replicas = %d, want one save and the scale-down once the window is over", len(saves), replicas(t, c))
+	}
+	server = getServer(t, c, "survival")
+	if server.Status.StopNoticeAt != nil || server.Status.Phase != v1alpha1.PhaseStopping {
+		t.Errorf("stopNoticeAt = %v phase = %s, want the stamp cleared and Stopping", server.Status.StopNoticeAt, server.Status.Phase)
+	}
+
+	reconcile(t, r, "survival")
+	if len(said) != 2 || len(saves) != 1 {
+		t.Errorf("after the scale-down: broadcasts=%q saves=%d, want no more", said, len(saves))
+	}
+}
+
+// Nobody to warn, or no way to warn them: the stop goes ahead on the first pass.
+// A tally the server did not give still counts as maybe-populated.
+func TestStopNotice_OnlyWhenSomeoneMayBeWarned(t *testing.T) {
+	cases := []struct {
+		name       string
+		prober     fakeProber
+		noRcon     bool
+		wantNotice bool
+		wantSaid   int
+	}{
+		{name: "empty server", prober: fakeProber{players: operator.PlayerCount{Online: 0, Max: 20, Known: true}}},
+		{name: "probe fails", prober: fakeProber{err: errors.New("connection refused")}},
+		{name: "broadcast fails", prober: fakeProber{players: operator.PlayerCount{Online: 2, Max: 20, Known: true}, broadcastErr: errors.New("i/o timeout")}, wantSaid: 1},
+		{name: "rcon disabled", prober: fakeProber{players: operator.PlayerCount{Online: 2, Max: 20, Known: true}}, noRcon: true},
+		{name: "unfamiliar list reply", prober: fakeProber{}, wantNotice: true, wantSaid: 1},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			var saves, said []string
+			tc.prober.saves, tc.prober.broadcasts = &saves, &said
+			s := runningServer()
+			if tc.noRcon {
+				// The Secret and its reference stay, as they do when RCON is switched off.
+				s.Spec.Rcon.Enabled = false
+			}
+			r, c := newReconciler(t, tc.prober, s, rconSecret())
+
+			wait := runThenStop(t, r, c)
+			if len(said) != tc.wantSaid {
+				t.Errorf("broadcasts = %q, want %d", said, tc.wantSaid)
+			}
+			stamped := getServer(t, c, "survival").Status.StopNoticeAt != nil
+			if tc.wantNotice {
+				if wait != operator.StopNoticeWindow || replicas(t, c) != 1 || !stamped {
+					t.Errorf("requeue=%v replicas=%d stamped=%v, want the notice window", wait, replicas(t, c), stamped)
+				}
+				return
+			}
+			if replicas(t, c) != 0 || stamped {
+				t.Errorf("replicas=%d stamped=%v, want an immediate stop", replicas(t, c), stamped)
+			}
+		})
+	}
+}
+
+// Starting the server again inside the window calls the stop off: the warned
+// players hear so, the pod stays, and a later stop warns afresh.
+func TestStopNotice_CalledOffByStart(t *testing.T) {
+	var saves, said []string
+	prober := fakeProber{players: operator.PlayerCount{Online: 3, Max: 20, Known: true}, saves: &saves, broadcasts: &said}
+	r, c := newReconciler(t, prober, runningServer(), rconSecret())
+	runThenStop(t, r, c)
+
+	r.Now = at(12 * time.Second)
+	setDesired(t, c, v1alpha1.DesiredRunning)
+	reconcile(t, r, "survival")
+	if len(said) != 2 || !strings.Contains(said[1], "取消") {
+		t.Errorf("broadcasts = %q, want the called-off line", said)
+	}
+	server := getServer(t, c, "survival")
+	if server.Status.StopNoticeAt != nil {
+		t.Errorf("stopNoticeAt = %v, want it cleared", server.Status.StopNoticeAt)
+	}
+	if replicas(t, c) != 1 || len(saves) != 0 || server.Status.Phase != v1alpha1.PhaseRunning {
+		t.Fatalf("replicas=%d saves=%d phase=%s, want the server left running", replicas(t, c), len(saves), server.Status.Phase)
+	}
+
+	// A Running pass after that says nothing more.
+	reconcile(t, r, "survival")
+	if len(said) != 2 {
+		t.Errorf("broadcasts = %q, want no repeat of the called-off line", said)
+	}
+
+	r.Now = at(40 * time.Second)
+	setDesired(t, c, v1alpha1.DesiredStopped)
+	if wait := reconcile(t, r, "survival").RequeueAfter; wait != operator.StopNoticeWindow {
+		t.Errorf("requeue = %v, want a full window for the new stop", wait)
+	}
+	if len(said) != 3 || !strings.Contains(said[2], "30 秒") {
+		t.Errorf("broadcasts = %q, want a fresh warning", said)
+	}
+	if got := getServer(t, c, "survival").Status.StopNoticeAt; got == nil || !got.Equal(ptrTime(at(40*time.Second)())) {
+		t.Errorf("stopNoticeAt = %v, want the new stop's time", got)
+	}
+}

panel/src/i18n/resources/en-US/backups.json

@@ -52,7 +52,7 @@
   "cancel": "Cancel",
   "restore_started_short": "Restore started",
   "restore_snapshot_started_short": "Snapshot, then restore",
-  "stopping": "Stopping the server…",
+  "stopping": "Stopping the server (players online get a 30-second warning first)…",
   "restoring": "Restoring…",
   "stop_timeout": "The server did not stop in time. Close this window and try again shortly.",
   "corrupt_badge": "Corrupt",

panel/src/i18n/resources/en-US/servers.json

@@ -217,8 +217,8 @@
   "idle_stop_seconds": "{{count}} s",
   "idle_stop_minutes": "{{count}} min",
   "idle_stop_hours": "{{count}} h",
-  "stop_confirm_players_one": "1 player is online and will be disconnected. Stop anyway?",
+  "stop_confirm_players_one": "1 player is online. They get a 30-second warning in game, then are disconnected. Stop anyway?",
-  "stop_confirm_players_other": "{{count}} players are online and will be disconnected. Stop anyway?",
+  "stop_confirm_players_other": "{{count}} players are online. They get a 30-second warning in game, then are disconnected. Stop anyway?",
   "stop_confirm_unknown": "The player count can't be read, so players may be online. Stop anyway?",
   "retire_card_title": "Give up server",
   "retire_card_title_admin": "Give up or delete server",

panel/src/i18n/resources/zh-CN/backups.json

@@ -52,7 +52,7 @@
   "cancel": "取消",
   "restore_started_short": "已启动恢复",
   "restore_snapshot_started_short": "快照中，随后恢复",
-  "stopping": "正在停止服务器……",
+  "stopping": "正在停止服务器（有玩家在线时先在游戏里预告 30 秒）……",
   "restoring": "正在恢复备份……",
   "stop_timeout": "服务器停止超时。请关闭此窗口，稍后重试。",
   "corrupt_badge": "已损坏",

panel/src/i18n/resources/zh-CN/servers.json

@@ -217,7 +217,7 @@
   "idle_stop_seconds": "{{count}} 秒",
   "idle_stop_minutes": "{{count}} 分钟",
   "idle_stop_hours": "{{count}} 小时",
-  "stop_confirm_players": "{{count}} 名玩家在线，停服会断开他们。确定停止？",
+  "stop_confirm_players": "{{count}} 名玩家在线。停服会先在游戏里提醒他们，30 秒后断开。确定停止？",
   "stop_confirm_unknown": "读不到在线人数，可能有玩家在线。确定停止？",
   "retire_card_title": "放弃服务器",
   "retire_card_title_admin": "放弃或删除服务器",

panel/src/pages/ServerBackups.test.tsx

@@ -1,6 +1,6 @@
 // @vitest-environment jsdom
 import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-import { render, screen } from "@testing-library/react";
+import { act, fireEvent, render, screen } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { MemoryRouter, Route, Routes } from "react-router-dom";
 import i18next from "i18next";
@@ -12,6 +12,8 @@ const calls = vi.hoisted(() => ({
   myServers: vi.fn(),
   serverJobs: vi.fn(),
   listBackups: vi.fn(),
+  stop: vi.fn(),
+  restoreBackup: vi.fn(),
 }));
 vi.mock("@/lib/tier", () => ({
   useTier: () => ({
@@ -53,6 +55,7 @@ beforeEach(() => {
   }));
 });
 afterEach(() => {
+  vi.useRealTimers();
   vi.restoreAllMocks();
   return i18next.changeLanguage("en-US");
 });
@@ -111,4 +114,43 @@ describe("ServerBackups", () => {
     expect(await screen.findByText("Latest backup")).toBeTruthy();
     expect(screen.queryByText(/^Page \d+ of/)).toBeNull();
   });
+
+  // Restoring a running server stops it first and restores only once it is down.
+  // With players online the operator warns them for 30 s before the stop, and the
+  // saves follow, so the wait must outlast that.
+  async function restoreRunningServer(stopsAfterMs: number) {
+    let stoppedAt = Infinity;
+    calls.status.mockImplementation(async () => {
+      const since = Date.now() - stoppedAt;
+      // Running through the warning, Stopping while the pod saves and goes.
+      const phase = since >= stopsAfterMs ? "Stopped" : since >= 30_000 ? "Stopping" : "Running";
+      return { name: "survival", displayName: "Survival", phase };
+    });
+    calls.stop.mockImplementation(async () => {
+      stoppedAt = Date.now();
+    });
+    calls.restoreBackup.mockResolvedValue({ safety_snapshot: true });
+    calls.listBackups.mockResolvedValue({ backups: [backup("bk-1", 3)], total: 1 });
+    renderPage();
+    await userEvent.click(await screen.findByRole("button", { name: "Restore" }));
+    const confirm = await screen.findByRole("button", { name: "Confirm restore" });
+    vi.useFakeTimers();
+    await act(async () => {
+      fireEvent.click(confirm);
+    });
+    expect(calls.stop).toHaveBeenCalledWith("survival");
+    await act(() => vi.advanceTimersByTimeAsync(130_000));
+  }
+
+  it("waits out a stop held for the players' 30-second warning, then restores", async () => {
+    await restoreRunningServer(75_000);
+    expect(calls.restoreBackup).toHaveBeenCalledWith("survival", "bk-1", true);
+    expect(screen.queryByText(i18next.t("backups:stop_timeout"))).toBeNull();
+  });
+
+  it("gives up without restoring when the server never goes down", async () => {
+    await restoreRunningServer(Infinity);
+    expect(calls.restoreBackup).not.toHaveBeenCalled();
+    expect(screen.getByText(i18next.t("backups:stop_timeout"))).toBeTruthy();
+  });
 });

panel/src/pages/ServerBackups.tsx

@@ -283,13 +283,16 @@ function ChainNote({ job }: { job: ServerJob }) {
  *  server plus consciously confirming a player-kicking, world-overwriting act.
  *
  *  The stop-and-wait is a bounded async loop (not an effect): after api.stop it polls
- *  status until Stopped is observed, giving up after ~60s with a retryable timeout — so
+ *  status until Stopped is observed, giving up after ~2 min with a retryable timeout — so
  *  restore only fires once the volume is provably free. While the chain runs the dialog
  *  is locked (no ✕, no dismiss) so a mid-flight close can't strand it. A 202 is
  *  terminal: the dialog closes and the row shows a "restore started" note in place of
  *  the button, so a second restore Job can't race the first. */
 const POLL_MS = 2500;
-const MAX_POLLS = 24; // ~60s ceiling before we stop waiting for Stopped
+// ~2 min before we stop waiting for Stopped: with players online the operator warns
+// them in game and holds the stop 30 s, then the pre-stop save and the pod's own
+// shutdown save follow.
+const MAX_POLLS = 48;
 const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 
 function RestoreControls({