diff --git a/deploy/crd/felis.lolicon.best_minecraftservers.yaml b/deploy/crd/felis.lolicon.best_minecraftservers.yaml index 4d34df9..92756c5 100644 --- a/deploy/crd/felis.lolicon.best_minecraftservers.yaml +++ b/deploy/crd/felis.lolicon.best_minecraftservers.yaml @@ -433,6 +433,14 @@ spec: because the two endpoints fall in different reconcile passes. format: date-time type: string + stopNoticeAt: + description: |- + StopNoticeAt is when the operator told the players on a server that it is + about to stop (desiredState flipped to Stopped with players online). The stop + itself waits until StopNoticeWindow has passed since then; the stamp is cleared + once the server is scaled down, or when desiredState goes back to Running first. + format: date-time + type: string type: object type: object served: true diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index cb9a777..88cd4f3 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -284,8 +284,19 @@ panel shows `Stopped` means the game pod is still terminating (its shutdown save can take a while); retry once `kubectl -n minecraft get pods -l felis.lolicon.best/server=` shows nothing. +A stop that leaves the server `Running` for about 30 s is the players' warning. +When desiredState turns `Stopped` and the RCON tally does not say the server is +empty, the operator sends everyone online a yellow chat line (`tellraw @a`), +stamps `status.stopNoticeAt`, records a `StopNotice` Event, and scales down +once `StopNoticeWindow` (30 s) has passed, with a last "stopping now" line +before the save. Starting the server again inside the window calls the stop +off, and the players are told so. An empty server, RCON off, or a probe or +broadcast that fails stops at once. Idle auto-stop only fires on an empty +server, so it never waits. + [GO-TESTED: `internal/maintenance`, `k8scluster_maintenance_test.go`, -`handlers_maintenance_test.go`, operator `maintenance_test.go`.] +`handlers_maintenance_test.go`, operator `maintenance_test.go`, +`stopnotice_test.go`.] --- diff --git a/internal/apis/felis/v1alpha1/minecraftserver_types.go b/internal/apis/felis/v1alpha1/minecraftserver_types.go index ba8f9dc..d254159 100644 --- a/internal/apis/felis/v1alpha1/minecraftserver_types.go +++ b/internal/apis/felis/v1alpha1/minecraftserver_types.go @@ -342,6 +342,11 @@ type MinecraftServerStatus struct { // or the server stops, so the empty-duration counter starts fresh each time // the server becomes unoccupied. EmptySince *metav1.Time `json:"emptySince,omitempty"` + // StopNoticeAt is when the operator told the players on a server that it is + // about to stop (desiredState flipped to Stopped with players online). The stop + // itself waits until StopNoticeWindow has passed since then; the stamp is cleared + // once the server is scaled down, or when desiredState goes back to Running first. + StopNoticeAt *metav1.Time `json:"stopNoticeAt,omitempty"` // AutoRestarts counts how often the operator recreated the pod of a start // that timed out (at most MaxAutoRestarts, with a doubling backoff); reaching Ready or // stopping resets it. diff --git a/internal/apis/felis/v1alpha1/zz_generated.deepcopy.go b/internal/apis/felis/v1alpha1/zz_generated.deepcopy.go index a125a81..b7eed08 100644 --- a/internal/apis/felis/v1alpha1/zz_generated.deepcopy.go +++ b/internal/apis/felis/v1alpha1/zz_generated.deepcopy.go @@ -121,6 +121,9 @@ func (in *MinecraftServerStatus) DeepCopyInto(out *MinecraftServerStatus) { if in.EmptySince != nil { out.EmptySince = in.EmptySince.DeepCopy() } + if in.StopNoticeAt != nil { + out.StopNoticeAt = in.StopNoticeAt.DeepCopy() + } if in.LastAutoRestartAt != nil { out.LastAutoRestartAt = in.LastAutoRestartAt.DeepCopy() } diff --git a/internal/operator/autorestart_test.go b/internal/operator/autorestart_test.go index b7e491e..4c3a591 100644 --- a/internal/operator/autorestart_test.go +++ b/internal/operator/autorestart_test.go @@ -122,6 +122,8 @@ func (p switchProber) Probe(context.Context, string, string) (operator.PlayerCou func (switchProber) Save(context.Context, string, string) error { return nil } +func (switchProber) Broadcast(context.Context, string, string, string) error { return nil } + // A Running server keeps its phase and endpoint through two missed probes, // re-probing every 10s; the third consecutive miss degrades it to Starting, // and any success in between starts the count over. diff --git a/internal/operator/prober.go b/internal/operator/prober.go index 78c4fb3..4197066 100644 --- a/internal/operator/prober.go +++ b/internal/operator/prober.go @@ -2,9 +2,13 @@ package operator import ( "context" + "encoding/json" + "fmt" "regexp" "strconv" + "strings" "time" + "unicode/utf16" "felis.lolicon.best/internal/rcon" ) @@ -31,9 +35,13 @@ type PlayerCount struct { // Save flushes the world to disk (`save-all flush`) and returns once the server // has answered, i.e. once the save is done. The reconciler runs it right before // scaling a server to zero (spec §7). +// +// Broadcast shows text in every online player's chat (`tellraw @a`). The reconciler +// uses it to warn players before a stop. type Prober interface { Probe(ctx context.Context, addr, password string) (PlayerCount, error) Save(ctx context.Context, addr, password string) error + Broadcast(ctx context.Context, addr, password, text string) error } // RconProber is the production Prober: a successful Dial (TCP connect + auth) @@ -110,6 +118,45 @@ func (p RconProber) Save(ctx context.Context, addr, password string) error { return err } +// Broadcast runs `tellraw @a` with text as one yellow chat line. tellraw takes a +// JSON text component on every loader and version Felis runs, and unlike `say` it +// shows the text without a "[Server]" or "[Rcon]" prefix. +func (p RconProber) Broadcast(ctx context.Context, addr, password, text string) error { + timeout := boundTimeout(ctx, p.Timeout, 5*time.Second) + conn, err := rcon.Dial(addr, password, timeout) + if err != nil { + return err + } + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil { + return err + } + _, err = conn.Execute(tellrawCommand(text)) + return err +} + +// tellrawCommand is the `tellraw @a` command that shows text to everyone online. +// Everything past ASCII is written as a JSON \u escape, so the command survives +// an RCON implementation that does not read its payload as UTF-8; the game's JSON +// parser turns the escapes back into the text. +func tellrawCommand(text string) string { + component, _ := json.Marshal(map[string]string{"text": text, "color": "yellow"}) + var b strings.Builder + b.WriteString("tellraw @a ") + for _, r := range string(component) { + switch { + case r < 0x80: + b.WriteRune(r) + case r > 0xFFFF: + hi, lo := utf16.EncodeRune(r) + fmt.Fprintf(&b, `\u%04x\u%04x`, hi, lo) + default: + fmt.Fprintf(&b, `\u%04x`, r) + } + } + return b.String() +} + // listReplyPatterns match the `list` replies of the loaders Felis runs, tried in // order against the reply with § color codes stripped: // diff --git a/internal/operator/prober_internal_test.go b/internal/operator/prober_internal_test.go index e2c3548..496cf86 100644 --- a/internal/operator/prober_internal_test.go +++ b/internal/operator/prober_internal_test.go @@ -3,8 +3,10 @@ package operator import ( "context" "encoding/binary" + "encoding/json" "io" "net" + "strings" "testing" "time" ) @@ -159,3 +161,36 @@ func TestRconProberSaveTimesOut(t *testing.T) { t.Errorf("Save took %v, want it bounded by SaveTimeout", elapsed) } } + +// The broadcast is plain ASCII on the wire and decodes back to the exact text, CJK +// and a character outside the BMP included. +func TestTellrawCommand(t *testing.T) { + text := `[Felis] 服务器将在 30 秒后关闭 / "stops" 🙂` + cmd := tellrawCommand(text) + want := `tellraw @a {"color":"yellow","text":"[Felis] \u670d\u52a1\u5668\u5c06\u5728 30 \u79d2\u540e\u5173\u95ed / \"stops\" \ud83d\ude42"}` + if cmd != want { + t.Errorf("command =\n%s\nwant\n%s", cmd, want) + } + for i := 0; i < len(cmd); i++ { + if cmd[i] >= 0x80 { + t.Fatalf("byte %d of %q is not ASCII", i, cmd) + } + } + var component map[string]string + if err := json.Unmarshal([]byte(strings.TrimPrefix(cmd, "tellraw @a ")), &component); err != nil { + t.Fatalf("component does not parse: %v", err) + } + if component["text"] != text { + t.Errorf("decoded text = %q, want %q", component["text"], text) + } +} + +func TestRconProberBroadcastSendsTellraw(t *testing.T) { + addr, cmds := serveFakeRcon(t, 0) + if err := (RconProber{}).Broadcast(context.Background(), addr, "pw", "关闭 / stop"); err != nil { + t.Fatalf("Broadcast: %v", err) + } + if got, want := <-cmds, tellrawCommand("关闭 / stop"); got != want { + t.Errorf("command = %q, want %q", got, want) + } +} diff --git a/internal/operator/reconciler.go b/internal/operator/reconciler.go index 2ab8d33..4708396 100644 --- a/internal/operator/reconciler.go +++ b/internal/operator/reconciler.go @@ -277,6 +277,11 @@ func (r *Reconciler) event(server *v1alpha1.MinecraftServer, eventType, reason, } func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.MinecraftServer) (ctrl.Result, error) { + if server.Status.StopNoticeAt != nil { + if err := r.callOffStop(ctx, server); err != nil { + return ctrl.Result{}, err + } + } if kind, held, err := r.maintenanceHold(ctx, server); err != nil { return ctrl.Result{}, err } else if held { @@ -495,7 +500,11 @@ func (r *Reconciler) reconcileStopped(ctx context.Context, server *v1alpha1.Mine // which sends the server SIGTERM and so its own shutdown save within the // grace period. if sts.Spec.Replicas == nil || *sts.Spec.Replicas != 0 { + if wait, err := r.noticeStop(ctx, server, &sts); err != nil || wait > 0 { + return ctrl.Result{RequeueAfter: wait}, err + } r.saveBeforeStop(ctx, server, &sts) + server.Status.StopNoticeAt = nil zero := int32(0) sts.Spec.Replicas = &zero if err := r.Update(ctx, &sts); err != nil { @@ -515,6 +524,86 @@ func (r *Reconciler) reconcileStopped(ctx context.Context, server *v1alpha1.Mine return ctrl.Result{}, r.patchStatus(ctx, server) } +// StopNoticeWindow is how long the players on a server get between the warning +// and the stop: long enough to finish a fight or get off a boat, short enough that +// the owner's stop (and the backup or restore waiting behind it) is not held up. +const StopNoticeWindow = 30 * time.Second + +// The lines players see around a stop. The operator does not know a player's +// language, so each carries both. +const ( + stopNoticeText = "[Felis] 服务器将在 30 秒后关闭,请尽快找安全的地方 / This server stops in 30 seconds" + stopNowText = "[Felis] 正在保存世界并关闭服务器 / Saving the world and stopping now" + stopCalledOffText = "[Felis] 关闭已取消 / The stop was called off" +) + +// noticeStop warns the players on a server that is about to be scaled down, and +// returns how long the stop must still wait for them. The first pass tells everyone +// online and stamps StopNoticeAt; the stop goes ahead once StopNoticeWindow has +// passed since, with a last line as it does. +// +// There is no wait when nobody can be told or nobody is there: RCON off, no ready +// pod, a probe or broadcast that fails, or a tally that says zero players. A tally +// the server did not give (an unfamiliar `list` reply) still gets the warning, +// since players may well be on it. Idle auto-stop therefore never waits: it only +// fires on an empty server. +func (r *Reconciler) noticeStop(ctx context.Context, server *v1alpha1.MinecraftServer, sts *appsv1.StatefulSet) (time.Duration, error) { + if at := server.Status.StopNoticeAt; at != nil { + if wait := at.Time.Add(StopNoticeWindow).Sub(r.now().Time); wait > 0 { + return wait, nil + } + r.broadcast(ctx, server, stopNowText) + return 0, nil + } + if !server.Spec.Rcon.Enabled || sts.Status.ReadyReplicas == 0 { + return 0, nil + } + password, err := r.rconPassword(ctx, server) + if err != nil { + return 0, nil + } + addr := rconAddress(server) + players, err := r.Prober.Probe(ctx, addr, password) + if err != nil || (players.Known && players.Online == 0) { + return 0, nil + } + if err := r.Prober.Broadcast(ctx, addr, password, stopNoticeText); err != nil { + ctrl.LoggerFrom(ctx).Info("could not warn players before the stop; stopping now", "error", err.Error()) + return 0, nil + } + now := r.now() + server.Status.StopNoticeAt = &now + if err := r.patchStatus(ctx, server); err != nil { + return 0, err + } + r.event(server, corev1.EventTypeNormal, "StopNotice", + fmt.Sprintf("players warned; stopping in %s", StopNoticeWindow)) + return StopNoticeWindow, nil +} + +// callOffStop answers a desiredState that went back to Running inside the notice +// window: the players who were warned hear that the stop is off. +func (r *Reconciler) callOffStop(ctx context.Context, server *v1alpha1.MinecraftServer) error { + r.broadcast(ctx, server, stopCalledOffText) + server.Status.StopNoticeAt = nil + return r.patchStatus(ctx, server) +} + +// broadcast shows text to everyone on server, best-effort: a server that cannot be +// reached has nobody listening either. +func (r *Reconciler) broadcast(ctx context.Context, server *v1alpha1.MinecraftServer, text string) { + if !server.Spec.Rcon.Enabled { + return + } + password, err := r.rconPassword(ctx, server) + if err != nil { + return + } + if err := r.Prober.Broadcast(ctx, rconAddress(server), password, text); err != nil { + ctrl.LoggerFrom(ctx).Info("player broadcast failed", "error", err.Error()) + } +} + // saveBeforeStop runs `save-all flush` on a server that is about to be scaled to // zero. The SIGTERM that follows makes the server save again on its way out, but // that save races the grace period: a large world killed mid-save rolls back to diff --git a/internal/operator/reconciler_test.go b/internal/operator/reconciler_test.go index ce1952a..59f0267 100644 --- a/internal/operator/reconciler_test.go +++ b/internal/operator/reconciler_test.go @@ -31,6 +31,9 @@ type fakeProber struct { saveErr error // saves, when set, records each Save's address. saves *[]string + // broadcasts, when set, records each Broadcast's text. + broadcasts *[]string + broadcastErr error } func (f fakeProber) Probe(context.Context, string, string) (operator.PlayerCount, error) { @@ -44,6 +47,13 @@ func (f fakeProber) Save(_ context.Context, addr, _ string) error { return f.saveErr } +func (f fakeProber) Broadcast(_ context.Context, _, _, text string) error { + if f.broadcasts != nil { + *f.broadcasts = append(*f.broadcasts, text) + } + return f.broadcastErr +} + func newScheme(t *testing.T) *runtime.Scheme { t.Helper() scheme := runtime.NewScheme() @@ -316,7 +326,7 @@ func TestReconcileStopped_NoWorkloadIsStopped(t *testing.T) { } func TestReconcileStopped_ScalesRunningWorkloadDown(t *testing.T) { - r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret()) + r, c := newReconciler(t, fakeProber{players: operator.PlayerCount{Online: 0, Max: 20, Known: true}}, runningServer(), rconSecret()) reconcile(t, r, "survival") markPodReady(t, c, "survival") @@ -353,7 +363,7 @@ func stopRunningServer(t *testing.T, r *operator.Reconciler, c client.Client) { func TestReconcileStopped_SavesBeforeScalingDown(t *testing.T) { var saves []string - r, c := newReconciler(t, fakeProber{saves: &saves}, runningServer(), rconSecret()) + r, c := newReconciler(t, fakeProber{saves: &saves, players: operator.PlayerCount{Known: true}}, runningServer(), rconSecret()) stopRunningServer(t, r, c) @@ -374,7 +384,7 @@ func TestReconcileStopped_SavesBeforeScalingDown(t *testing.T) { func TestReconcileStopped_FailedSaveStillStops(t *testing.T) { var saves []string - prober := fakeProber{saves: &saves, saveErr: errors.New("i/o timeout")} + prober := fakeProber{saves: &saves, saveErr: errors.New("i/o timeout"), players: operator.PlayerCount{Known: true}} r, c := newReconciler(t, prober, runningServer(), rconSecret()) stopRunningServer(t, r, c) @@ -1149,7 +1159,7 @@ func TestFelisUpgradeLeavesARunningServerAlone(t *testing.T) { // TestFelisUpgradeReachesAServerOnItsNextStart: a stopped server's next start // writes the whole template, the new felis image included. func TestFelisUpgradeReachesAServerOnItsNextStart(t *testing.T) { - r, c := newReconciler(t, fakeProber{players: operator.PlayerCount{Online: 1, Max: 20, Known: true}}, runningServer(), rconSecret()) + r, c := newReconciler(t, fakeProber{players: operator.PlayerCount{Online: 0, Max: 20, Known: true}}, runningServer(), rconSecret()) r.FelisImage = felisV1 stopRunningServer(t, r, c) markPodTerminated(t, c, "survival") diff --git a/internal/operator/stopnotice_test.go b/internal/operator/stopnotice_test.go new file mode 100644 index 0000000..6c8e2ee --- /dev/null +++ b/internal/operator/stopnotice_test.go @@ -0,0 +1,185 @@ +package operator_test + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/operator" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// runThenStop takes survival to Running with a ready pod, flips it to Stopped and +// returns the result of the first Stopped pass. +func runThenStop(t *testing.T, r *operator.Reconciler, c client.Client) time.Duration { + t.Helper() + reconcile(t, r, "survival") + markPodReady(t, c, "survival") + reconcile(t, r, "survival") + setDesired(t, c, v1alpha1.DesiredStopped) + return reconcile(t, r, "survival").RequeueAfter +} + +func setDesired(t *testing.T, c client.Client, desired v1alpha1.DesiredState) { + t.Helper() + server := getServer(t, c, "survival") + server.Spec.DesiredState = desired + if err := c.Update(context.Background(), server); err != nil { + t.Fatalf("set desiredState %s: %v", desired, err) + } +} + +func at(offset time.Duration) func() metav1.Time { + return func() metav1.Time { return metav1.NewTime(fixedNow().Add(offset)) } +} + +func replicas(t *testing.T, c client.Client) int32 { + t.Helper() + sts := getSTS(t, c, "survival") + if sts.Spec.Replicas == nil { + return 1 + } + return *sts.Spec.Replicas +} + +// A stop with players on the server warns them, holds the pod for the window, then +// saves and scales down with a last line. +func TestStopNotice_WarnsPlayersThenStops(t *testing.T) { + var saves, said []string + prober := fakeProber{players: operator.PlayerCount{Online: 3, Max: 20, Known: true}, saves: &saves, broadcasts: &said} + r, c := newReconciler(t, prober, runningServer(), rconSecret()) + + if wait := runThenStop(t, r, c); wait != operator.StopNoticeWindow { + t.Fatalf("requeue = %v, want the %v window", wait, operator.StopNoticeWindow) + } + if len(said) != 1 || !strings.Contains(said[0], "30 秒") || !strings.Contains(said[0], "30 seconds") { + t.Fatalf("broadcasts = %q, want the 30-second warning in both languages", said) + } + server := getServer(t, c, "survival") + if server.Status.StopNoticeAt == nil || !server.Status.StopNoticeAt.Equal(ptrTime(fixedNow())) { + t.Errorf("stopNoticeAt = %v, want %v", server.Status.StopNoticeAt, fixedNow()) + } + if server.Status.Phase != v1alpha1.PhaseRunning || !server.Status.Ready { + t.Errorf("phase = %s ready=%v, want the server still Running while players are warned", server.Status.Phase, server.Status.Ready) + } + if n := replicas(t, c); n != 1 || len(saves) != 0 { + t.Fatalf("replicas = %d saves = %d, want the pod left alone during the window", n, len(saves)) + } + + // A pass inside the window waits out the rest and says nothing new. + r.Now = at(10 * time.Second) + if wait := reconcile(t, r, "survival").RequeueAfter; wait != 20*time.Second { + t.Errorf("requeue 10s in = %v, want 20s", wait) + } + if len(said) != 1 || len(saves) != 0 || replicas(t, c) != 1 { + t.Fatalf("10s in: broadcasts=%q saves=%d replicas=%d, want nothing new", said, len(saves), replicas(t, c)) + } + + r.Now = at(operator.StopNoticeWindow) + reconcile(t, r, "survival") + if len(said) != 2 || !strings.Contains(said[1], "正在保存") { + t.Errorf("broadcasts = %q, want the stopping-now line last", said) + } + if len(saves) != 1 || replicas(t, c) != 0 { + t.Fatalf("saves = %d replicas = %d, want one save and the scale-down once the window is over", len(saves), replicas(t, c)) + } + server = getServer(t, c, "survival") + if server.Status.StopNoticeAt != nil || server.Status.Phase != v1alpha1.PhaseStopping { + t.Errorf("stopNoticeAt = %v phase = %s, want the stamp cleared and Stopping", server.Status.StopNoticeAt, server.Status.Phase) + } + + reconcile(t, r, "survival") + if len(said) != 2 || len(saves) != 1 { + t.Errorf("after the scale-down: broadcasts=%q saves=%d, want no more", said, len(saves)) + } +} + +// Nobody to warn, or no way to warn them: the stop goes ahead on the first pass. +// A tally the server did not give still counts as maybe-populated. +func TestStopNotice_OnlyWhenSomeoneMayBeWarned(t *testing.T) { + cases := []struct { + name string + prober fakeProber + noRcon bool + wantNotice bool + wantSaid int + }{ + {name: "empty server", prober: fakeProber{players: operator.PlayerCount{Online: 0, Max: 20, Known: true}}}, + {name: "probe fails", prober: fakeProber{err: errors.New("connection refused")}}, + {name: "broadcast fails", prober: fakeProber{players: operator.PlayerCount{Online: 2, Max: 20, Known: true}, broadcastErr: errors.New("i/o timeout")}, wantSaid: 1}, + {name: "rcon disabled", prober: fakeProber{players: operator.PlayerCount{Online: 2, Max: 20, Known: true}}, noRcon: true}, + {name: "unfamiliar list reply", prober: fakeProber{}, wantNotice: true, wantSaid: 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var saves, said []string + tc.prober.saves, tc.prober.broadcasts = &saves, &said + s := runningServer() + if tc.noRcon { + // The Secret and its reference stay, as they do when RCON is switched off. + s.Spec.Rcon.Enabled = false + } + r, c := newReconciler(t, tc.prober, s, rconSecret()) + + wait := runThenStop(t, r, c) + if len(said) != tc.wantSaid { + t.Errorf("broadcasts = %q, want %d", said, tc.wantSaid) + } + stamped := getServer(t, c, "survival").Status.StopNoticeAt != nil + if tc.wantNotice { + if wait != operator.StopNoticeWindow || replicas(t, c) != 1 || !stamped { + t.Errorf("requeue=%v replicas=%d stamped=%v, want the notice window", wait, replicas(t, c), stamped) + } + return + } + if replicas(t, c) != 0 || stamped { + t.Errorf("replicas=%d stamped=%v, want an immediate stop", replicas(t, c), stamped) + } + }) + } +} + +// Starting the server again inside the window calls the stop off: the warned +// players hear so, the pod stays, and a later stop warns afresh. +func TestStopNotice_CalledOffByStart(t *testing.T) { + var saves, said []string + prober := fakeProber{players: operator.PlayerCount{Online: 3, Max: 20, Known: true}, saves: &saves, broadcasts: &said} + r, c := newReconciler(t, prober, runningServer(), rconSecret()) + runThenStop(t, r, c) + + r.Now = at(12 * time.Second) + setDesired(t, c, v1alpha1.DesiredRunning) + reconcile(t, r, "survival") + if len(said) != 2 || !strings.Contains(said[1], "取消") { + t.Errorf("broadcasts = %q, want the called-off line", said) + } + server := getServer(t, c, "survival") + if server.Status.StopNoticeAt != nil { + t.Errorf("stopNoticeAt = %v, want it cleared", server.Status.StopNoticeAt) + } + if replicas(t, c) != 1 || len(saves) != 0 || server.Status.Phase != v1alpha1.PhaseRunning { + t.Fatalf("replicas=%d saves=%d phase=%s, want the server left running", replicas(t, c), len(saves), server.Status.Phase) + } + + // A Running pass after that says nothing more. + reconcile(t, r, "survival") + if len(said) != 2 { + t.Errorf("broadcasts = %q, want no repeat of the called-off line", said) + } + + r.Now = at(40 * time.Second) + setDesired(t, c, v1alpha1.DesiredStopped) + if wait := reconcile(t, r, "survival").RequeueAfter; wait != operator.StopNoticeWindow { + t.Errorf("requeue = %v, want a full window for the new stop", wait) + } + if len(said) != 3 || !strings.Contains(said[2], "30 秒") { + t.Errorf("broadcasts = %q, want a fresh warning", said) + } + if got := getServer(t, c, "survival").Status.StopNoticeAt; got == nil || !got.Equal(ptrTime(at(40*time.Second)())) { + t.Errorf("stopNoticeAt = %v, want the new stop's time", got) + } +} diff --git a/panel/src/i18n/resources/en-US/backups.json b/panel/src/i18n/resources/en-US/backups.json index b50959b..5166a3e 100644 --- a/panel/src/i18n/resources/en-US/backups.json +++ b/panel/src/i18n/resources/en-US/backups.json @@ -52,7 +52,7 @@ "cancel": "Cancel", "restore_started_short": "Restore started", "restore_snapshot_started_short": "Snapshot, then restore", - "stopping": "Stopping the server…", + "stopping": "Stopping the server (players online get a 30-second warning first)…", "restoring": "Restoring…", "stop_timeout": "The server did not stop in time. Close this window and try again shortly.", "corrupt_badge": "Corrupt", diff --git a/panel/src/i18n/resources/en-US/servers.json b/panel/src/i18n/resources/en-US/servers.json index 0e0163d..15230c3 100644 --- a/panel/src/i18n/resources/en-US/servers.json +++ b/panel/src/i18n/resources/en-US/servers.json @@ -217,8 +217,8 @@ "idle_stop_seconds": "{{count}} s", "idle_stop_minutes": "{{count}} min", "idle_stop_hours": "{{count}} h", - "stop_confirm_players_one": "1 player is online and will be disconnected. Stop anyway?", - "stop_confirm_players_other": "{{count}} players are online and will be disconnected. Stop anyway?", + "stop_confirm_players_one": "1 player is online. They get a 30-second warning in game, then are disconnected. Stop anyway?", + "stop_confirm_players_other": "{{count}} players are online. They get a 30-second warning in game, then are disconnected. Stop anyway?", "stop_confirm_unknown": "The player count can't be read, so players may be online. Stop anyway?", "retire_card_title": "Give up server", "retire_card_title_admin": "Give up or delete server", diff --git a/panel/src/i18n/resources/zh-CN/backups.json b/panel/src/i18n/resources/zh-CN/backups.json index 297734e..803fcb0 100644 --- a/panel/src/i18n/resources/zh-CN/backups.json +++ b/panel/src/i18n/resources/zh-CN/backups.json @@ -52,7 +52,7 @@ "cancel": "取消", "restore_started_short": "已启动恢复", "restore_snapshot_started_short": "快照中,随后恢复", - "stopping": "正在停止服务器……", + "stopping": "正在停止服务器(有玩家在线时先在游戏里预告 30 秒)……", "restoring": "正在恢复备份……", "stop_timeout": "服务器停止超时。请关闭此窗口,稍后重试。", "corrupt_badge": "已损坏", diff --git a/panel/src/i18n/resources/zh-CN/servers.json b/panel/src/i18n/resources/zh-CN/servers.json index 0da53a9..5d4b39b 100644 --- a/panel/src/i18n/resources/zh-CN/servers.json +++ b/panel/src/i18n/resources/zh-CN/servers.json @@ -217,7 +217,7 @@ "idle_stop_seconds": "{{count}} 秒", "idle_stop_minutes": "{{count}} 分钟", "idle_stop_hours": "{{count}} 小时", - "stop_confirm_players": "{{count}} 名玩家在线,停服会断开他们。确定停止?", + "stop_confirm_players": "{{count}} 名玩家在线。停服会先在游戏里提醒他们,30 秒后断开。确定停止?", "stop_confirm_unknown": "读不到在线人数,可能有玩家在线。确定停止?", "retire_card_title": "放弃服务器", "retire_card_title_admin": "放弃或删除服务器", diff --git a/panel/src/pages/ServerBackups.test.tsx b/panel/src/pages/ServerBackups.test.tsx index a01cff8..cb0db8c 100644 --- a/panel/src/pages/ServerBackups.test.tsx +++ b/panel/src/pages/ServerBackups.test.tsx @@ -1,6 +1,6 @@ // @vitest-environment jsdom import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { render, screen } from "@testing-library/react"; +import { act, fireEvent, render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { MemoryRouter, Route, Routes } from "react-router-dom"; import i18next from "i18next"; @@ -12,6 +12,8 @@ const calls = vi.hoisted(() => ({ myServers: vi.fn(), serverJobs: vi.fn(), listBackups: vi.fn(), + stop: vi.fn(), + restoreBackup: vi.fn(), })); vi.mock("@/lib/tier", () => ({ useTier: () => ({ @@ -53,6 +55,7 @@ beforeEach(() => { })); }); afterEach(() => { + vi.useRealTimers(); vi.restoreAllMocks(); return i18next.changeLanguage("en-US"); }); @@ -111,4 +114,43 @@ describe("ServerBackups", () => { expect(await screen.findByText("Latest backup")).toBeTruthy(); expect(screen.queryByText(/^Page \d+ of/)).toBeNull(); }); + + // Restoring a running server stops it first and restores only once it is down. + // With players online the operator warns them for 30 s before the stop, and the + // saves follow, so the wait must outlast that. + async function restoreRunningServer(stopsAfterMs: number) { + let stoppedAt = Infinity; + calls.status.mockImplementation(async () => { + const since = Date.now() - stoppedAt; + // Running through the warning, Stopping while the pod saves and goes. + const phase = since >= stopsAfterMs ? "Stopped" : since >= 30_000 ? "Stopping" : "Running"; + return { name: "survival", displayName: "Survival", phase }; + }); + calls.stop.mockImplementation(async () => { + stoppedAt = Date.now(); + }); + calls.restoreBackup.mockResolvedValue({ safety_snapshot: true }); + calls.listBackups.mockResolvedValue({ backups: [backup("bk-1", 3)], total: 1 }); + renderPage(); + await userEvent.click(await screen.findByRole("button", { name: "Restore" })); + const confirm = await screen.findByRole("button", { name: "Confirm restore" }); + vi.useFakeTimers(); + await act(async () => { + fireEvent.click(confirm); + }); + expect(calls.stop).toHaveBeenCalledWith("survival"); + await act(() => vi.advanceTimersByTimeAsync(130_000)); + } + + it("waits out a stop held for the players' 30-second warning, then restores", async () => { + await restoreRunningServer(75_000); + expect(calls.restoreBackup).toHaveBeenCalledWith("survival", "bk-1", true); + expect(screen.queryByText(i18next.t("backups:stop_timeout"))).toBeNull(); + }); + + it("gives up without restoring when the server never goes down", async () => { + await restoreRunningServer(Infinity); + expect(calls.restoreBackup).not.toHaveBeenCalled(); + expect(screen.getByText(i18next.t("backups:stop_timeout"))).toBeTruthy(); + }); }); diff --git a/panel/src/pages/ServerBackups.tsx b/panel/src/pages/ServerBackups.tsx index 99fd558..8c51dad 100644 --- a/panel/src/pages/ServerBackups.tsx +++ b/panel/src/pages/ServerBackups.tsx @@ -283,13 +283,16 @@ function ChainNote({ job }: { job: ServerJob }) { * server plus consciously confirming a player-kicking, world-overwriting act. * * The stop-and-wait is a bounded async loop (not an effect): after api.stop it polls - * status until Stopped is observed, giving up after ~60s with a retryable timeout — so + * status until Stopped is observed, giving up after ~2 min with a retryable timeout — so * restore only fires once the volume is provably free. While the chain runs the dialog * is locked (no ✕, no dismiss) so a mid-flight close can't strand it. A 202 is * terminal: the dialog closes and the row shows a "restore started" note in place of * the button, so a second restore Job can't race the first. */ const POLL_MS = 2500; -const MAX_POLLS = 24; // ~60s ceiling before we stop waiting for Stopped +// ~2 min before we stop waiting for Stopped: with players online the operator warns +// them in game and holds the stop 30 s, then the pre-stop save and the pod's own +// shutdown save follow. +const MAX_POLLS = 48; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); function RestoreControls({