fix(operator): enforce startup and readiness timeouts (§5, §8)
Previously a server whose pod was ready but RCON probe kept failing would stay in Starting phase forever. The CRD defines TimeoutSeconds and ReadinessTimeoutSeconds but the reconciler never checked them. - PodNotReady path: if the pod stays not-ready past timeoutSeconds (default 300s), transition to Failed - RCON unreachable path: if RCON stays unreachable past readinessTimeoutSeconds (default 300s), transition to Failed - Helper methods startupTimedOut/readinessTimedOut compare StartRequestedAt against the respective timeout, falling back to 300s defaults when unset - 2 new tests: ReadinessTimeoutConvertsToFailed, StartupTimeoutConvertsToFailed Fixes the scenario where a broken backend (bad jar, crash-looping process) would permanently occupy a Starting server slot.
This commit is contained in:
2 files changed
+119
-3
No files matched your search
@@ -25,9 +25,11 @@ import (
|
||||
|
||||
// Requeue cadences for the transient phases.
|
||||
const (
|
||||
requeueStarting = 5 * time.Second
|
||||
requeueStopping = 5 * time.Second
|
||||
requeueSecret = 10 * time.Second
|
||||
requeueStarting = 5 * time.Second
|
||||
requeueStopping = 5 * time.Second
|
||||
requeueSecret = 10 * time.Second
|
||||
defaultTimeoutSeconds = 300
|
||||
defaultReadinessTimeoutSec = 300
|
||||
)
|
||||
|
||||
// Reconciler reconciles a MinecraftServer with its managed children.
|
||||
@@ -102,6 +104,9 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
|
||||
// The pod must first pass its tcpSocket readiness (readyReplicas >= 1).
|
||||
if current.Status.ReadyReplicas < 1 {
|
||||
r.markStarting(server, "PodNotReady", "waiting for pod TCP readiness")
|
||||
if r.startupTimedOut(server) {
|
||||
r.markFailed(server, "StartupTimeout", "pod did not become ready within startup timeout")
|
||||
}
|
||||
if err := r.patchStatus(ctx, server); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
@@ -123,6 +128,9 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
|
||||
pc, err := r.Prober.Probe(ctx, rconAddress(server), password)
|
||||
if err != nil {
|
||||
r.markStarting(server, "RconNotReachable", err.Error())
|
||||
if r.readinessTimedOut(server) {
|
||||
r.markFailed(server, "ReadinessTimeout", "RCON probe did not succeed within readiness timeout")
|
||||
}
|
||||
if perr := r.patchStatus(ctx, server); perr != nil {
|
||||
return ctrl.Result{}, perr
|
||||
}
|
||||
@@ -346,3 +354,25 @@ func (r *Reconciler) setCondition(server *v1alpha1.MinecraftServer, condType str
|
||||
LastTransitionTime: r.now(),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Reconciler) startupTimedOut(server *v1alpha1.MinecraftServer) bool {
|
||||
if server.Status.StartRequestedAt == nil {
|
||||
return false
|
||||
}
|
||||
timeout := time.Duration(server.Spec.Startup.TimeoutSeconds) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = defaultTimeoutSeconds * time.Second
|
||||
}
|
||||
return r.now().Time.Sub(server.Status.StartRequestedAt.Time) >= timeout
|
||||
}
|
||||
|
||||
func (r *Reconciler) readinessTimedOut(server *v1alpha1.MinecraftServer) bool {
|
||||
if server.Status.StartRequestedAt == nil {
|
||||
return false
|
||||
}
|
||||
timeout := time.Duration(server.Spec.Startup.ReadinessTimeoutSeconds) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = defaultReadinessTimeoutSec * time.Second
|
||||
}
|
||||
return r.now().Time.Sub(server.Status.StartRequestedAt.Time) >= timeout
|
||||
}
|
||||
Reference in new issue
Block a user