fix(felis): 启动失败的服可在面板重试或停止,玩家入服时直接告知启动失败
This commit is contained in:
28 files changed
+775
-75
No files matched your search
@@ -195,6 +195,11 @@ func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
|
||||
desired = v1alpha1.DesiredStopped
|
||||
}
|
||||
prevPhase, prevRestarts := server.Status.Phase, server.Status.AutoRestarts
|
||||
if _, ok := server.Annotations[v1alpha1.AnnotationStartRetry]; ok {
|
||||
if err := r.takeStartRetry(ctx, &server, desired); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
var res ctrl.Result
|
||||
var err error
|
||||
if desired == v1alpha1.DesiredStopped {
|
||||
@@ -233,6 +238,33 @@ func (r *Reconciler) recordTransition(ctx context.Context, server *v1alpha1.Mine
|
||||
r.event(server, eventType, reason, fmt.Sprintf("%s → %s: %s", from, phase, msg))
|
||||
}
|
||||
|
||||
// takeStartRetry answers felis-api's AnnotationStartRetry: a server still Failed
|
||||
// and meant to run starts over as if freshly woken — pod recreated, restart
|
||||
// budget back to zero, a new start anchor — and in any other state the request
|
||||
// is stale and only removed. The fresh status is written before the request is
|
||||
// removed, so a pass that fails in between leaves the request to be taken again
|
||||
// (at the cost of one more pod recreate), never a removed request with the old
|
||||
// spent budget still in place.
|
||||
func (r *Reconciler) takeStartRetry(ctx context.Context, server *v1alpha1.MinecraftServer, desired v1alpha1.DesiredState) error {
|
||||
if desired == v1alpha1.DesiredRunning && server.Status.Phase == v1alpha1.PhaseFailed {
|
||||
pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: server.Name + "-0", Namespace: server.Namespace}}
|
||||
if err := r.Delete(ctx, pod); client.IgnoreNotFound(err) != nil {
|
||||
return err
|
||||
}
|
||||
server.Status.AutoRestarts = 0
|
||||
server.Status.StartRequestedAt = nil
|
||||
r.markStarting(server, "StartRetried", "start requested again after it failed; recreated the pod")
|
||||
if err := r.patchStatus(ctx, server); err != nil {
|
||||
return err
|
||||
}
|
||||
log.FromContext(ctx).Info("start retried")
|
||||
r.event(server, corev1.EventTypeNormal, "StartRetried", "start requested again after it failed; recreated the pod")
|
||||
}
|
||||
patch := client.MergeFrom(server.DeepCopy())
|
||||
delete(server.Annotations, v1alpha1.AnnotationStartRetry)
|
||||
return r.Patch(ctx, server, patch)
|
||||
}
|
||||
|
||||
func (r *Reconciler) event(server *v1alpha1.MinecraftServer, eventType, reason, msg string) {
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Event(server, eventType, reason, msg)
|
||||
@@ -807,6 +839,7 @@ func (r *Reconciler) markFailed(server *v1alpha1.MinecraftServer, reason, msg st
|
||||
server.Status.Phase = v1alpha1.PhaseFailed
|
||||
server.Status.Ready = false
|
||||
server.Status.ObservedGeneration = server.Generation
|
||||
server.Status.LiveMotd = server.Spec.Motd.Failed
|
||||
r.setCondition(server, v1alpha1.ConditionReady, metav1.ConditionFalse, reason, msg)
|
||||
r.setCondition(server, v1alpha1.ConditionProvisioned, metav1.ConditionFalse, reason, msg)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
package operator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||
)
|
||||
|
||||
// requestStartRetry is felis-api's RetryStart as the operator sees it.
|
||||
func requestStartRetry(t *testing.T, c client.Client) {
|
||||
t.Helper()
|
||||
s := getServer(t, c, "survival")
|
||||
patch := client.MergeFrom(s.DeepCopy())
|
||||
if s.Annotations == nil {
|
||||
s.Annotations = map[string]string{}
|
||||
}
|
||||
s.Annotations[v1alpha1.AnnotationStartRetry] = "2026-07-01T12:00:00Z"
|
||||
if err := c.Patch(context.Background(), s, patch); err != nil {
|
||||
t.Fatalf("annotate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func podPresent(t *testing.T, c client.Client) bool {
|
||||
t.Helper()
|
||||
err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival-0"}, &corev1.Pod{})
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("get pod: %v", err)
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Once the automatic restarts are spent, a retry request starts the server over
|
||||
// with the whole budget back: the pod is recreated, the start re-anchored, and
|
||||
// the next timeout is retried automatically again.
|
||||
func TestStartRetryRevivesAServerThatGaveUp(t *testing.T) {
|
||||
srv := runningServer()
|
||||
srv.Spec.Startup.TimeoutSeconds = 30
|
||||
srv.Spec.Motd.Failed = "broken"
|
||||
r, c := newReconciler(t, fakeProber{}, srv, rconSecret(), gamePod())
|
||||
base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
|
||||
clock := base
|
||||
r.Now = func() metav1.Time { return metav1.NewTime(clock) }
|
||||
at := func(offset time.Duration) *v1alpha1.MinecraftServer {
|
||||
clock = base.Add(offset)
|
||||
reconcile(t, r, "survival")
|
||||
return getServer(t, c, "survival")
|
||||
}
|
||||
recreatePod := func() {
|
||||
if err := c.Create(context.Background(), gamePod()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Spend the three automatic restarts (autorestart_test.go pins the schedule).
|
||||
at(0)
|
||||
for _, due := range []time.Duration{90 * time.Second, 240 * time.Second, 510 * time.Second} {
|
||||
at(due)
|
||||
recreatePod()
|
||||
}
|
||||
s := at(time.Hour)
|
||||
if !v1alpha1.StartGaveUp(&s.Status) {
|
||||
t.Fatalf("setup: not given up: phase=%s restarts=%d", s.Status.Phase, s.Status.AutoRestarts)
|
||||
}
|
||||
if s.Status.LiveMotd != "broken" {
|
||||
t.Fatalf("Failed advertises liveMotd %q, want the failed MOTD", s.Status.LiveMotd)
|
||||
}
|
||||
|
||||
requestStartRetry(t, c)
|
||||
s = at(2 * time.Hour)
|
||||
if _, left := s.Annotations[v1alpha1.AnnotationStartRetry]; left {
|
||||
t.Fatal("the retry request was left on the server")
|
||||
}
|
||||
if podPresent(t, c) {
|
||||
t.Fatal("the retry kept the failed pod")
|
||||
}
|
||||
if s.Status.Phase != v1alpha1.PhaseStarting || s.Status.AutoRestarts != 0 || v1alpha1.StartGaveUp(&s.Status) {
|
||||
t.Fatalf("after retry: phase=%s restarts=%d gaveUp=%v", s.Status.Phase, s.Status.AutoRestarts, v1alpha1.StartGaveUp(&s.Status))
|
||||
}
|
||||
if s.Status.StartRequestedAt == nil || !s.Status.StartRequestedAt.Time.Equal(base.Add(2*time.Hour)) {
|
||||
t.Fatalf("start not re-anchored at the retry: %v", s.Status.StartRequestedAt)
|
||||
}
|
||||
recreatePod()
|
||||
|
||||
// The budget is really back: this start times out and is retried on its own.
|
||||
if s := at(2*time.Hour + 89*time.Second); s.Status.Phase != v1alpha1.PhaseFailed || v1alpha1.StartGaveUp(&s.Status) {
|
||||
t.Fatalf("retried start timing out: phase=%s gaveUp=%v, want Failed with retries left", s.Status.Phase, v1alpha1.StartGaveUp(&s.Status))
|
||||
}
|
||||
if s := at(2*time.Hour + 90*time.Second); s.Status.AutoRestarts != 1 || podPresent(t, c) {
|
||||
t.Fatalf("retried start: restarts=%d pod=%v, want the first automatic restart", s.Status.AutoRestarts, podPresent(t, c))
|
||||
}
|
||||
}
|
||||
|
||||
// A request that finds the server anywhere but Failed is stale (the start
|
||||
// already recovered, or someone stopped it): it is removed and nothing else.
|
||||
func TestStaleStartRetryIsOnlyRemoved(t *testing.T) {
|
||||
srv := runningServer()
|
||||
srv.Annotations = map[string]string{v1alpha1.AnnotationStartRetry: "2026-07-01T12:00:00Z"}
|
||||
anchor := metav1.NewTime(fixedNow().Add(-10 * time.Second))
|
||||
srv.Status = v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStarting, AutoRestarts: 2, StartRequestedAt: &anchor}
|
||||
r, c := newReconciler(t, fakeProber{}, srv, rconSecret(), gamePod())
|
||||
|
||||
reconcile(t, r, "survival")
|
||||
s := getServer(t, c, "survival")
|
||||
if _, left := s.Annotations[v1alpha1.AnnotationStartRetry]; left {
|
||||
t.Fatal("the stale retry request was left on the server")
|
||||
}
|
||||
if !podPresent(t, c) || s.Status.AutoRestarts != 2 || !s.Status.StartRequestedAt.Equal(&anchor) {
|
||||
t.Fatalf("a stale request touched the start: pod=%v restarts=%d anchor=%v",
|
||||
podPresent(t, c), s.Status.AutoRestarts, s.Status.StartRequestedAt)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user