fix(operator): enforce startup and readiness timeouts (§5, §8)

Previously a server whose pod was ready but RCON probe kept failing
would stay in Starting phase forever. The CRD defines TimeoutSeconds
and ReadinessTimeoutSeconds but the reconciler never checked them.

- PodNotReady path: if the pod stays not-ready past timeoutSeconds
  (default 300s), transition to Failed
- RCON unreachable path: if RCON stays unreachable past
  readinessTimeoutSeconds (default 300s), transition to Failed
- Helper methods startupTimedOut/readinessTimedOut compare
  StartRequestedAt against the respective timeout, falling back to
  300s defaults when unset
- 2 new tests: ReadinessTimeoutConvertsToFailed, StartupTimeoutConvertsToFailed

Fixes the scenario where a broken backend (bad jar, crash-looping
process) would permanently occupy a Starting server slot.
This commit is contained in:
Lemon-miaow committed 2026-07-05 16:22:08 +08:00
1 parent 9e1df12975
commit 7f7e459746
2 files changed
+119 -3

No files matched your search

+33 -3
View File
@@ -25,9 +25,11 @@ import (
// Requeue cadences for the transient phases.
const (
requeueStarting = 5 * time.Second
requeueStopping = 5 * time.Second
requeueSecret = 10 * time.Second
requeueStarting = 5 * time.Second
requeueStopping = 5 * time.Second
requeueSecret = 10 * time.Second
defaultTimeoutSeconds = 300
defaultReadinessTimeoutSec = 300
)
// Reconciler reconciles a MinecraftServer with its managed children.
@@ -102,6 +104,9 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
// The pod must first pass its tcpSocket readiness (readyReplicas >= 1).
if current.Status.ReadyReplicas < 1 {
r.markStarting(server, "PodNotReady", "waiting for pod TCP readiness")
if r.startupTimedOut(server) {
r.markFailed(server, "StartupTimeout", "pod did not become ready within startup timeout")
}
if err := r.patchStatus(ctx, server); err != nil {
return ctrl.Result{}, err
}
@@ -123,6 +128,9 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
pc, err := r.Prober.Probe(ctx, rconAddress(server), password)
if err != nil {
r.markStarting(server, "RconNotReachable", err.Error())
if r.readinessTimedOut(server) {
r.markFailed(server, "ReadinessTimeout", "RCON probe did not succeed within readiness timeout")
}
if perr := r.patchStatus(ctx, server); perr != nil {
return ctrl.Result{}, perr
}
@@ -346,3 +354,25 @@ func (r *Reconciler) setCondition(server *v1alpha1.MinecraftServer, condType str
LastTransitionTime: r.now(),
})
}
func (r *Reconciler) startupTimedOut(server *v1alpha1.MinecraftServer) bool {
if server.Status.StartRequestedAt == nil {
return false
}
timeout := time.Duration(server.Spec.Startup.TimeoutSeconds) * time.Second
if timeout <= 0 {
timeout = defaultTimeoutSeconds * time.Second
}
return r.now().Time.Sub(server.Status.StartRequestedAt.Time) >= timeout
}
func (r *Reconciler) readinessTimedOut(server *v1alpha1.MinecraftServer) bool {
if server.Status.StartRequestedAt == nil {
return false
}
timeout := time.Duration(server.Spec.Startup.ReadinessTimeoutSeconds) * time.Second
if timeout <= 0 {
timeout = defaultReadinessTimeoutSec * time.Second
}
return r.now().Time.Sub(server.Status.StartRequestedAt.Time) >= timeout
}