fix(felis): 启动失败的服可在面板重试或停止,玩家入服时直接告知启动失败

This commit is contained in:
Lemon-miaow committed 2026-09-26 22:08:29 +08:00
1 parent c1025274e1
commit 22d1e834ad
28 files changed
+775 -75

No files matched your search

+33
View File
@@ -195,6 +195,11 @@ func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
desired = v1alpha1.DesiredStopped
}
prevPhase, prevRestarts := server.Status.Phase, server.Status.AutoRestarts
if _, ok := server.Annotations[v1alpha1.AnnotationStartRetry]; ok {
if err := r.takeStartRetry(ctx, &server, desired); err != nil {
return ctrl.Result{}, err
}
}
var res ctrl.Result
var err error
if desired == v1alpha1.DesiredStopped {
@@ -233,6 +238,33 @@ func (r *Reconciler) recordTransition(ctx context.Context, server *v1alpha1.Mine
r.event(server, eventType, reason, fmt.Sprintf("%s → %s: %s", from, phase, msg))
}
// takeStartRetry answers felis-api's AnnotationStartRetry: a server still Failed
// and meant to run starts over as if freshly woken — pod recreated, restart
// budget back to zero, a new start anchor — and in any other state the request
// is stale and only removed. The fresh status is written before the request is
// removed, so a pass that fails in between leaves the request to be taken again
// (at the cost of one more pod recreate), never a removed request with the old
// spent budget still in place.
func (r *Reconciler) takeStartRetry(ctx context.Context, server *v1alpha1.MinecraftServer, desired v1alpha1.DesiredState) error {
if desired == v1alpha1.DesiredRunning && server.Status.Phase == v1alpha1.PhaseFailed {
pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: server.Name + "-0", Namespace: server.Namespace}}
if err := r.Delete(ctx, pod); client.IgnoreNotFound(err) != nil {
return err
}
server.Status.AutoRestarts = 0
server.Status.StartRequestedAt = nil
r.markStarting(server, "StartRetried", "start requested again after it failed; recreated the pod")
if err := r.patchStatus(ctx, server); err != nil {
return err
}
log.FromContext(ctx).Info("start retried")
r.event(server, corev1.EventTypeNormal, "StartRetried", "start requested again after it failed; recreated the pod")
}
patch := client.MergeFrom(server.DeepCopy())
delete(server.Annotations, v1alpha1.AnnotationStartRetry)
return r.Patch(ctx, server, patch)
}
func (r *Reconciler) event(server *v1alpha1.MinecraftServer, eventType, reason, msg string) {
if r.Recorder != nil {
r.Recorder.Event(server, eventType, reason, msg)
@@ -807,6 +839,7 @@ func (r *Reconciler) markFailed(server *v1alpha1.MinecraftServer, reason, msg st
server.Status.Phase = v1alpha1.PhaseFailed
server.Status.Ready = false
server.Status.ObservedGeneration = server.Generation
server.Status.LiveMotd = server.Spec.Motd.Failed
r.setCondition(server, v1alpha1.ConditionReady, metav1.ConditionFalse, reason, msg)
r.setCondition(server, v1alpha1.ConditionProvisioned, metav1.ConditionFalse, reason, msg)
}
+119
View File
@@ -0,0 +1,119 @@
package operator_test
import (
"context"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
)
// requestStartRetry is felis-api's RetryStart as the operator sees it.
func requestStartRetry(t *testing.T, c client.Client) {
t.Helper()
s := getServer(t, c, "survival")
patch := client.MergeFrom(s.DeepCopy())
if s.Annotations == nil {
s.Annotations = map[string]string{}
}
s.Annotations[v1alpha1.AnnotationStartRetry] = "2026-07-01T12:00:00Z"
if err := c.Patch(context.Background(), s, patch); err != nil {
t.Fatalf("annotate: %v", err)
}
}
func podPresent(t *testing.T, c client.Client) bool {
t.Helper()
err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival-0"}, &corev1.Pod{})
if err != nil && !apierrors.IsNotFound(err) {
t.Fatalf("get pod: %v", err)
}
return err == nil
}
// Once the automatic restarts are spent, a retry request starts the server over
// with the whole budget back: the pod is recreated, the start re-anchored, and
// the next timeout is retried automatically again.
func TestStartRetryRevivesAServerThatGaveUp(t *testing.T) {
srv := runningServer()
srv.Spec.Startup.TimeoutSeconds = 30
srv.Spec.Motd.Failed = "broken"
r, c := newReconciler(t, fakeProber{}, srv, rconSecret(), gamePod())
base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
clock := base
r.Now = func() metav1.Time { return metav1.NewTime(clock) }
at := func(offset time.Duration) *v1alpha1.MinecraftServer {
clock = base.Add(offset)
reconcile(t, r, "survival")
return getServer(t, c, "survival")
}
recreatePod := func() {
if err := c.Create(context.Background(), gamePod()); err != nil {
t.Fatal(err)
}
}
// Spend the three automatic restarts (autorestart_test.go pins the schedule).
at(0)
for _, due := range []time.Duration{90 * time.Second, 240 * time.Second, 510 * time.Second} {
at(due)
recreatePod()
}
s := at(time.Hour)
if !v1alpha1.StartGaveUp(&s.Status) {
t.Fatalf("setup: not given up: phase=%s restarts=%d", s.Status.Phase, s.Status.AutoRestarts)
}
if s.Status.LiveMotd != "broken" {
t.Fatalf("Failed advertises liveMotd %q, want the failed MOTD", s.Status.LiveMotd)
}
requestStartRetry(t, c)
s = at(2 * time.Hour)
if _, left := s.Annotations[v1alpha1.AnnotationStartRetry]; left {
t.Fatal("the retry request was left on the server")
}
if podPresent(t, c) {
t.Fatal("the retry kept the failed pod")
}
if s.Status.Phase != v1alpha1.PhaseStarting || s.Status.AutoRestarts != 0 || v1alpha1.StartGaveUp(&s.Status) {
t.Fatalf("after retry: phase=%s restarts=%d gaveUp=%v", s.Status.Phase, s.Status.AutoRestarts, v1alpha1.StartGaveUp(&s.Status))
}
if s.Status.StartRequestedAt == nil || !s.Status.StartRequestedAt.Time.Equal(base.Add(2*time.Hour)) {
t.Fatalf("start not re-anchored at the retry: %v", s.Status.StartRequestedAt)
}
recreatePod()
// The budget is really back: this start times out and is retried on its own.
if s := at(2*time.Hour + 89*time.Second); s.Status.Phase != v1alpha1.PhaseFailed || v1alpha1.StartGaveUp(&s.Status) {
t.Fatalf("retried start timing out: phase=%s gaveUp=%v, want Failed with retries left", s.Status.Phase, v1alpha1.StartGaveUp(&s.Status))
}
if s := at(2*time.Hour + 90*time.Second); s.Status.AutoRestarts != 1 || podPresent(t, c) {
t.Fatalf("retried start: restarts=%d pod=%v, want the first automatic restart", s.Status.AutoRestarts, podPresent(t, c))
}
}
// A request that finds the server anywhere but Failed is stale (the start
// already recovered, or someone stopped it): it is removed and nothing else.
func TestStaleStartRetryIsOnlyRemoved(t *testing.T) {
srv := runningServer()
srv.Annotations = map[string]string{v1alpha1.AnnotationStartRetry: "2026-07-01T12:00:00Z"}
anchor := metav1.NewTime(fixedNow().Add(-10 * time.Second))
srv.Status = v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStarting, AutoRestarts: 2, StartRequestedAt: &anchor}
r, c := newReconciler(t, fakeProber{}, srv, rconSecret(), gamePod())
reconcile(t, r, "survival")
s := getServer(t, c, "survival")
if _, left := s.Annotations[v1alpha1.AnnotationStartRetry]; left {
t.Fatal("the stale retry request was left on the server")
}
if !podPresent(t, c) || s.Status.AutoRestarts != 2 || !s.Status.StartRequestedAt.Equal(&anchor) {
t.Fatalf("a stale request touched the start: pod=%v restarts=%d anchor=%v",
podPresent(t, c), s.Status.AutoRestarts, s.Status.StartRequestedAt)
}
}