fix(operator): 服务器停服或删除时丢掉内存里的 RCON 失败计数

This commit is contained in:
Lemon-miaow committed 2026-09-27 13:34:42 +08:00
1 parent c272e2ea24
commit 0d485992c6
3 files changed
+73 -12

No files matched your search

+43
View File
@@ -164,6 +164,49 @@ func TestRunningServerDegradesOnlyAfterThreeMisses(t *testing.T) {
} }
} }
// A miss count lives in the operator's memory, so it has to go with the run it
// counts: a server meant to stop and a server deleted leave none behind.
func TestProbeMissCountsGoWithTheServer(t *testing.T) {
missOnce := func(t *testing.T) (*operator.Reconciler, client.Client) {
t.Helper()
down := false
r, c := newReconciler(t, switchProber{down: &down}, runningServer(), rconSecret())
reconcile(t, r, "survival")
markPodReady(t, c, "survival")
reconcile(t, r, "survival")
down = true
reconcile(t, r, "survival")
if got := r.ProbeMissesTracked(); got != 1 {
t.Fatalf("after a miss: %d servers tracked, want 1", got)
}
return r, c
}
t.Run("stopped", func(t *testing.T) {
r, c := missOnce(t)
s := getServer(t, c, "survival")
s.Spec.DesiredState = v1alpha1.DesiredStopped
if err := c.Update(context.Background(), s); err != nil {
t.Fatalf("set desiredState=Stopped: %v", err)
}
reconcile(t, r, "survival")
if got := r.ProbeMissesTracked(); got != 0 {
t.Fatalf("after the stop: %d servers tracked, want 0", got)
}
})
t.Run("deleted", func(t *testing.T) {
r, c := missOnce(t)
if err := c.Delete(context.Background(), getServer(t, c, "survival")); err != nil {
t.Fatalf("delete server: %v", err)
}
reconcile(t, r, "survival")
if got := r.ProbeMissesTracked(); got != 0 {
t.Fatalf("after the delete: %d servers tracked, want 0", got)
}
})
}
// A Failed server wakes when its next auto-restart falls due, and every 5m // A Failed server wakes when its next auto-restart falls due, and every 5m
// once the attempts are spent, instead of every 2s. // once the attempts are spent, instead of every 2s.
func TestFailedServerRequeuesWhenTheRetryIsDue(t *testing.T) { func TestFailedServerRequeuesWhenTheRetryIsDue(t *testing.T) {
+9
View File
@@ -0,0 +1,9 @@
package operator
// ProbeMissesTracked is how many servers the reconciler holds a probe-miss count
// for, so the external tests can see a count go with its server.
func (r *Reconciler) ProbeMissesTracked() int {
r.probeMu.Lock()
defer r.probeMu.Unlock()
return len(r.probeFailures)
}
+21 -12
View File
@@ -143,10 +143,13 @@ type Reconciler struct {
Watch *ReconcileWatch Watch *ReconcileWatch
// probeFailures counts consecutive failed RCON probes of a Running server, // probeFailures counts consecutive failed RCON probes of a Running server,
// by UID. In memory: an operator restart forgets them, which only delays a // by namespaced name. In memory: an operator restart forgets them, which only
// degrade by a few probes. // delays a degrade by a few probes. A count goes when a probe succeeds, when
// the server is meant to stop and when it is deleted. A later server of the
// same name inherits nothing that matters: it reaches Running only through a
// probe that succeeds, and that clears the count.
probeMu sync.Mutex probeMu sync.Mutex
probeFailures map[types.UID]int probeFailures map[types.NamespacedName]int
} }
func (r *Reconciler) now() metav1.Time { func (r *Reconciler) now() metav1.Time {
@@ -191,7 +194,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
var server v1alpha1.MinecraftServer var server v1alpha1.MinecraftServer
if err := r.Get(ctx, req.NamespacedName, &server); err != nil { if err := r.Get(ctx, req.NamespacedName, &server); err != nil {
// Deletion is handled by owner references on the children. // Deletion is handled by owner references on the children; the probe-miss
// count is the one part of a server kept in memory.
if apierrors.IsNotFound(err) {
r.clearProbeFailures(req.NamespacedName)
}
return ctrl.Result{}, client.IgnoreNotFound(err) return ctrl.Result{}, client.IgnoreNotFound(err)
} }
@@ -208,6 +215,8 @@ func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
var res ctrl.Result var res ctrl.Result
var err error var err error
if desired == v1alpha1.DesiredStopped { if desired == v1alpha1.DesiredStopped {
// Misses count only against a Running server; the next run starts afresh.
r.clearProbeFailures(req.NamespacedName)
res, err = r.reconcileStopped(ctx, &server) res, err = r.reconcileStopped(ctx, &server)
} else { } else {
res, err = r.reconcileRunning(ctx, &server) res, err = r.reconcileRunning(ctx, &server)
@@ -388,7 +397,7 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
if err != nil { if err != nil {
// One missed probe of a Running server is noise (a lag spike, a save); // One missed probe of a Running server is noise (a lag spike, a save);
// it keeps its status and endpoint until the misses run consecutive. // it keeps its status and endpoint until the misses run consecutive.
if server.Status.Phase == v1alpha1.PhaseRunning && r.noteProbeFailure(server) < runningProbeMissesBeforeDegrade { if server.Status.Phase == v1alpha1.PhaseRunning && r.noteProbeFailure(client.ObjectKeyFromObject(server)) < runningProbeMissesBeforeDegrade {
return ctrl.Result{RequeueAfter: requeueProbeRetry}, nil return ctrl.Result{RequeueAfter: requeueProbeRetry}, nil
} }
r.markStarting(server, "RconNotReachable", err.Error()) r.markStarting(server, "RconNotReachable", err.Error())
@@ -403,7 +412,7 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
} }
return ctrl.Result{RequeueAfter: r.startingRequeue(server, readinessTimeout(server))}, nil return ctrl.Result{RequeueAfter: r.startingRequeue(server, readinessTimeout(server))}, nil
} }
r.clearProbeFailures(server) r.clearProbeFailures(client.ObjectKeyFromObject(server))
players = pc players = pc
// A tally that could not be read pauses idle auto-stop (below) instead of // A tally that could not be read pauses idle auto-stop (below) instead of
// counting as an empty server; the condition says so, so a server that // counting as an empty server; the condition says so, so a server that
@@ -1060,20 +1069,20 @@ func (r *Reconciler) startingRequeue(server *v1alpha1.MinecraftServer, timeout t
return max(wait, requeueStarting) return max(wait, requeueStarting)
} }
func (r *Reconciler) noteProbeFailure(server *v1alpha1.MinecraftServer) int { func (r *Reconciler) noteProbeFailure(key types.NamespacedName) int {
r.probeMu.Lock() r.probeMu.Lock()
defer r.probeMu.Unlock() defer r.probeMu.Unlock()
if r.probeFailures == nil { if r.probeFailures == nil {
r.probeFailures = map[types.UID]int{} r.probeFailures = map[types.NamespacedName]int{}
} }
r.probeFailures[server.UID]++ r.probeFailures[key]++
return r.probeFailures[server.UID] return r.probeFailures[key]
} }
func (r *Reconciler) clearProbeFailures(server *v1alpha1.MinecraftServer) { func (r *Reconciler) clearProbeFailures(key types.NamespacedName) {
r.probeMu.Lock() r.probeMu.Lock()
defer r.probeMu.Unlock() defer r.probeMu.Unlock()
delete(r.probeFailures, server.UID) delete(r.probeFailures, key)
} }
// recoverFailedStart retries a start that timed out. Once a backoff past the // recoverFailedStart retries a start that timed out. Once a backoff past the