fix(operator): 服务器停服或删除时丢掉内存里的 RCON 失败计数
This commit is contained in:
3 files changed
+73
-12
No files matched your search
@@ -164,6 +164,49 @@ func TestRunningServerDegradesOnlyAfterThreeMisses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A miss count lives in the operator's memory, so it has to go with the run it
|
||||||
|
// counts: a server meant to stop and a server deleted leave none behind.
|
||||||
|
func TestProbeMissCountsGoWithTheServer(t *testing.T) {
|
||||||
|
missOnce := func(t *testing.T) (*operator.Reconciler, client.Client) {
|
||||||
|
t.Helper()
|
||||||
|
down := false
|
||||||
|
r, c := newReconciler(t, switchProber{down: &down}, runningServer(), rconSecret())
|
||||||
|
reconcile(t, r, "survival")
|
||||||
|
markPodReady(t, c, "survival")
|
||||||
|
reconcile(t, r, "survival")
|
||||||
|
down = true
|
||||||
|
reconcile(t, r, "survival")
|
||||||
|
if got := r.ProbeMissesTracked(); got != 1 {
|
||||||
|
t.Fatalf("after a miss: %d servers tracked, want 1", got)
|
||||||
|
}
|
||||||
|
return r, c
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("stopped", func(t *testing.T) {
|
||||||
|
r, c := missOnce(t)
|
||||||
|
s := getServer(t, c, "survival")
|
||||||
|
s.Spec.DesiredState = v1alpha1.DesiredStopped
|
||||||
|
if err := c.Update(context.Background(), s); err != nil {
|
||||||
|
t.Fatalf("set desiredState=Stopped: %v", err)
|
||||||
|
}
|
||||||
|
reconcile(t, r, "survival")
|
||||||
|
if got := r.ProbeMissesTracked(); got != 0 {
|
||||||
|
t.Fatalf("after the stop: %d servers tracked, want 0", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("deleted", func(t *testing.T) {
|
||||||
|
r, c := missOnce(t)
|
||||||
|
if err := c.Delete(context.Background(), getServer(t, c, "survival")); err != nil {
|
||||||
|
t.Fatalf("delete server: %v", err)
|
||||||
|
}
|
||||||
|
reconcile(t, r, "survival")
|
||||||
|
if got := r.ProbeMissesTracked(); got != 0 {
|
||||||
|
t.Fatalf("after the delete: %d servers tracked, want 0", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// A Failed server wakes when its next auto-restart falls due, and every 5m
|
// A Failed server wakes when its next auto-restart falls due, and every 5m
|
||||||
// once the attempts are spent, instead of every 2s.
|
// once the attempts are spent, instead of every 2s.
|
||||||
func TestFailedServerRequeuesWhenTheRetryIsDue(t *testing.T) {
|
func TestFailedServerRequeuesWhenTheRetryIsDue(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package operator
|
||||||
|
|
||||||
|
// ProbeMissesTracked is how many servers the reconciler holds a probe-miss count
|
||||||
|
// for, so the external tests can see a count go with its server.
|
||||||
|
func (r *Reconciler) ProbeMissesTracked() int {
|
||||||
|
r.probeMu.Lock()
|
||||||
|
defer r.probeMu.Unlock()
|
||||||
|
return len(r.probeFailures)
|
||||||
|
}
|
||||||
@@ -143,10 +143,13 @@ type Reconciler struct {
|
|||||||
Watch *ReconcileWatch
|
Watch *ReconcileWatch
|
||||||
|
|
||||||
// probeFailures counts consecutive failed RCON probes of a Running server,
|
// probeFailures counts consecutive failed RCON probes of a Running server,
|
||||||
// by UID. In memory: an operator restart forgets them, which only delays a
|
// by namespaced name. In memory: an operator restart forgets them, which only
|
||||||
// degrade by a few probes.
|
// delays a degrade by a few probes. A count goes when a probe succeeds, when
|
||||||
|
// the server is meant to stop and when it is deleted. A later server of the
|
||||||
|
// same name inherits nothing that matters: it reaches Running only through a
|
||||||
|
// probe that succeeds, and that clears the count.
|
||||||
probeMu sync.Mutex
|
probeMu sync.Mutex
|
||||||
probeFailures map[types.UID]int
|
probeFailures map[types.NamespacedName]int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) now() metav1.Time {
|
func (r *Reconciler) now() metav1.Time {
|
||||||
@@ -191,7 +194,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
|
|||||||
func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||||
var server v1alpha1.MinecraftServer
|
var server v1alpha1.MinecraftServer
|
||||||
if err := r.Get(ctx, req.NamespacedName, &server); err != nil {
|
if err := r.Get(ctx, req.NamespacedName, &server); err != nil {
|
||||||
// Deletion is handled by owner references on the children.
|
// Deletion is handled by owner references on the children; the probe-miss
|
||||||
|
// count is the one part of a server kept in memory.
|
||||||
|
if apierrors.IsNotFound(err) {
|
||||||
|
r.clearProbeFailures(req.NamespacedName)
|
||||||
|
}
|
||||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,6 +215,8 @@ func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
|
|||||||
var res ctrl.Result
|
var res ctrl.Result
|
||||||
var err error
|
var err error
|
||||||
if desired == v1alpha1.DesiredStopped {
|
if desired == v1alpha1.DesiredStopped {
|
||||||
|
// Misses count only against a Running server; the next run starts afresh.
|
||||||
|
r.clearProbeFailures(req.NamespacedName)
|
||||||
res, err = r.reconcileStopped(ctx, &server)
|
res, err = r.reconcileStopped(ctx, &server)
|
||||||
} else {
|
} else {
|
||||||
res, err = r.reconcileRunning(ctx, &server)
|
res, err = r.reconcileRunning(ctx, &server)
|
||||||
@@ -388,7 +397,7 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
// One missed probe of a Running server is noise (a lag spike, a save);
|
// One missed probe of a Running server is noise (a lag spike, a save);
|
||||||
// it keeps its status and endpoint until the misses run consecutive.
|
// it keeps its status and endpoint until the misses run consecutive.
|
||||||
if server.Status.Phase == v1alpha1.PhaseRunning && r.noteProbeFailure(server) < runningProbeMissesBeforeDegrade {
|
if server.Status.Phase == v1alpha1.PhaseRunning && r.noteProbeFailure(client.ObjectKeyFromObject(server)) < runningProbeMissesBeforeDegrade {
|
||||||
return ctrl.Result{RequeueAfter: requeueProbeRetry}, nil
|
return ctrl.Result{RequeueAfter: requeueProbeRetry}, nil
|
||||||
}
|
}
|
||||||
r.markStarting(server, "RconNotReachable", err.Error())
|
r.markStarting(server, "RconNotReachable", err.Error())
|
||||||
@@ -403,7 +412,7 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
|
|||||||
}
|
}
|
||||||
return ctrl.Result{RequeueAfter: r.startingRequeue(server, readinessTimeout(server))}, nil
|
return ctrl.Result{RequeueAfter: r.startingRequeue(server, readinessTimeout(server))}, nil
|
||||||
}
|
}
|
||||||
r.clearProbeFailures(server)
|
r.clearProbeFailures(client.ObjectKeyFromObject(server))
|
||||||
players = pc
|
players = pc
|
||||||
// A tally that could not be read pauses idle auto-stop (below) instead of
|
// A tally that could not be read pauses idle auto-stop (below) instead of
|
||||||
// counting as an empty server; the condition says so, so a server that
|
// counting as an empty server; the condition says so, so a server that
|
||||||
@@ -1060,20 +1069,20 @@ func (r *Reconciler) startingRequeue(server *v1alpha1.MinecraftServer, timeout t
|
|||||||
return max(wait, requeueStarting)
|
return max(wait, requeueStarting)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) noteProbeFailure(server *v1alpha1.MinecraftServer) int {
|
func (r *Reconciler) noteProbeFailure(key types.NamespacedName) int {
|
||||||
r.probeMu.Lock()
|
r.probeMu.Lock()
|
||||||
defer r.probeMu.Unlock()
|
defer r.probeMu.Unlock()
|
||||||
if r.probeFailures == nil {
|
if r.probeFailures == nil {
|
||||||
r.probeFailures = map[types.UID]int{}
|
r.probeFailures = map[types.NamespacedName]int{}
|
||||||
}
|
}
|
||||||
r.probeFailures[server.UID]++
|
r.probeFailures[key]++
|
||||||
return r.probeFailures[server.UID]
|
return r.probeFailures[key]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) clearProbeFailures(server *v1alpha1.MinecraftServer) {
|
func (r *Reconciler) clearProbeFailures(key types.NamespacedName) {
|
||||||
r.probeMu.Lock()
|
r.probeMu.Lock()
|
||||||
defer r.probeMu.Unlock()
|
defer r.probeMu.Unlock()
|
||||||
delete(r.probeFailures, server.UID)
|
delete(r.probeFailures, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// recoverFailedStart retries a start that timed out. Once a backoff past the
|
// recoverFailedStart retries a start that timed out. Once a backoff past the
|
||||||
|
|||||||
Reference in new issue
Block a user