feat(operator): 游戏容器加 startup/liveness 探针，超时启动按 1/2/4 分钟退避重建 Pod 至多 3... (6ec1b272) · Commits · FelisMC / Felis · GitLab

deploy/crd/felis.lolicon.best_minecraftservers.yaml

+12 −0

Changes for deploy/crd/felis.lolicon.best_minecraftservers.yaml: 12 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -285,6 +285,13 @@ spec:
		status:
		description: MinecraftServerStatus is the observed state (spec §4 status.*).
		properties:
		autoRestarts:
		description: \|-
		AutoRestarts counts how often the operator recreated the pod of a start
		that timed out (at most 3, with a doubling backoff); reaching Ready or
		stopping resets it.
		format: int32
		type: integer
		conditions:
		description: Conditions are the standard metav1 conditions (Ready,
		RconReached, ...).
		@@ -361,6 +368,11 @@ spec:
		description: Mode is "direct" or "fallback".
		type: string
		type: object
		lastAutoRestartAt:
		description: LastAutoRestartAt is when the operator last recreated
		the pod.
		format: date-time
		type: string
		liveMotd:
		description: LiveMotd is the MOTD currently advertised for the active
		phase.

internal/apis/felis/v1alpha1/minecraftserver_types.go

+6 −0

Changes for internal/apis/felis/v1alpha1/minecraftserver_types.go: 6 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -277,6 +277,12 @@ type MinecraftServerStatus struct {
		// or the server stops, so the empty-duration counter starts fresh each time
		// the server becomes unoccupied.
		EmptySince *metav1.Time `json:"emptySince,omitempty"`
		// AutoRestarts counts how often the operator recreated the pod of a start
		// that timed out (at most 3, with a doubling backoff); reaching Ready or
		// stopping resets it.
		AutoRestarts int32 `json:"autoRestarts,omitempty"`
		// LastAutoRestartAt is when the operator last recreated the pod.
		LastAutoRestartAt *metav1.Time `json:"lastAutoRestartAt,omitempty"`
		// ObservedGeneration is the spec generation this status reflects.
		ObservedGeneration int64 `json:"observedGeneration,omitempty"`
		// Conditions are the standard metav1 conditions (Ready, RconReached, ...).

internal/apis/felis/v1alpha1/zz_generated.deepcopy.go

+3 −0

Changes for internal/apis/felis/v1alpha1/zz_generated.deepcopy.go: 3 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -121,6 +121,9 @@ func (in MinecraftServerStatus) DeepCopyInto(out MinecraftServerStatus) {
		if in.EmptySince != nil {
		out.EmptySince = in.EmptySince.DeepCopy()
		}
		if in.LastAutoRestartAt != nil {
		out.LastAutoRestartAt = in.LastAutoRestartAt.DeepCopy()
		}
		if in.Conditions != nil {
		l := make([]metav1.Condition, len(in.Conditions))
		for i := range in.Conditions {

internal/operator/autorestart_test.go

0 → 100644

+184 −0

Changes for internal/operator/autorestart_test.go: 184 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package operator_test

		import (
		"context"
		"errors"
		"testing"
		"time"

		corev1 "k8s.io/api/core/v1"
		apierrors "k8s.io/apimachinery/pkg/api/errors"
		metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
		"k8s.io/apimachinery/pkg/types"

		"felis.lolicon.best/internal/apis/felis/v1alpha1"
		"felis.lolicon.best/internal/operator"
		)

		func gamePod() *corev1.Pod {
		return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "survival-0", Namespace: "minecraft"}}
		}

		// A start that timed out is retried by recreating its pod after a doubling
		// backoff (1m, 2m, 4m past the timeout), three times, then left Failed.
		func TestTimedOutStartRecreatesThePodWithBackoff(t *testing.T) {
		srv := runningServer()
		srv.Spec.Startup.TimeoutSeconds = 30
		r, c := newReconciler(t, fakeProber{}, srv, rconSecret(), gamePod())
		base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
		clock := base
		r.Now = func() metav1.Time { return metav1.NewTime(clock) }
		podExists := func() bool {
		err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival-0"}, &corev1.Pod{})
		if err != nil && !apierrors.IsNotFound(err) {
		t.Fatalf("get pod: %v", err)
		}
		return err == nil
		}
		at := func(offset time.Duration) *v1alpha1.MinecraftServer {
		clock = base.Add(offset)
		reconcile(t, r, "survival")
		return getServer(t, c, "survival")
		}

		at(0) // anchors the start at base
		// Each attempt: [anchor, the instant just before it is due, the due instant].
		attempts := [][3]time.Duration{
		{0, 89 * time.Second, 90 * time.Second}, // 30s timeout + 1m
		{90 * time.Second, 239 * time.Second, 240 * time.Second}, // + 30s + 2m
		{240 * time.Second, 509 * time.Second, 510 * time.Second}, // + 30s + 4m
		}
		for i, a := range attempts {
		if s := at(a[1]); s.Status.Phase != v1alpha1.PhaseFailed \|\| !podExists() \|\| s.Status.AutoRestarts != int32(i) {
		t.Fatalf("attempt %d before due: phase=%s pod=%v restarts=%d", i+1, s.Status.Phase, podExists(), s.Status.AutoRestarts)
		}
		s := at(a[2])
		if podExists() {
		t.Fatalf("attempt %d: pod survived the due instant", i+1)
		}
		if s.Status.AutoRestarts != int32(i+1) \|\| s.Status.Phase != v1alpha1.PhaseStarting {
		t.Fatalf("attempt %d: restarts=%d phase=%s", i+1, s.Status.AutoRestarts, s.Status.Phase)
		}
		if s.Status.LastAutoRestartAt == nil \|\| !s.Status.LastAutoRestartAt.Time.Equal(base.Add(a[2])) {
		t.Fatalf("attempt %d: lastAutoRestartAt=%v", i+1, s.Status.LastAutoRestartAt)
		}
		if s.Status.StartRequestedAt == nil \|\| !s.Status.StartRequestedAt.Time.Equal(base.Add(a[2])) {
		t.Fatalf("attempt %d: start not re-anchored: %v", i+1, s.Status.StartRequestedAt)
		}
		if err := c.Create(context.Background(), gamePod()); err != nil { // the StatefulSet's replacement
		t.Fatal(err)
		}
		}

		// Attempts spent: long after the next timeout the pod stays and so does Failed.
		s := at(time.Hour)
		if s.Status.Phase != v1alpha1.PhaseFailed \|\| s.Status.AutoRestarts != 3 \|\| !podExists() {
		t.Fatalf("after three attempts: phase=%s restarts=%d pod=%v", s.Status.Phase, s.Status.AutoRestarts, podExists())
		}
		}

		// An RCON channel that never answers on a TCP-ready pod gets the same retry.
		func TestReadinessTimeoutRecreatesThePod(t *testing.T) {
		srv := runningServer()
		srv.Spec.Startup.ReadinessTimeoutSeconds = 30
		r, c := newReconciler(t, fakeProber{err: errors.New("connection refused")}, srv, rconSecret(), gamePod())
		base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
		clock := base
		r.Now = func() metav1.Time { return metav1.NewTime(clock) }

		reconcile(t, r, "survival")
		markPodReady(t, c, "survival")
		clock = base.Add(89 * time.Second)
		reconcile(t, r, "survival")
		if s := getServer(t, c, "survival"); s.Status.Phase != v1alpha1.PhaseFailed \|\| s.Status.AutoRestarts != 0 {
		t.Fatalf("before due: phase=%s restarts=%d", s.Status.Phase, s.Status.AutoRestarts)
		}
		clock = base.Add(90 * time.Second)
		reconcile(t, r, "survival")
		s := getServer(t, c, "survival")
		err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival-0"}, &corev1.Pod{})
		if !apierrors.IsNotFound(err) \|\| s.Status.AutoRestarts != 1 \|\| s.Status.Phase != v1alpha1.PhaseStarting {
		t.Fatalf("due: pod err=%v restarts=%d phase=%s", err, s.Status.AutoRestarts, s.Status.Phase)
		}
		}

		// switchProber fails while *down is true, so one server can miss and recover.
		type switchProber struct{ down *bool }

		func (p switchProber) Probe(context.Context, string, string) (operator.PlayerCount, error) {
		if *p.down {
		return operator.PlayerCount{}, errors.New("i/o timeout")
		}
		return operator.PlayerCount{Online: 1, Max: 20}, nil
		}

		func (switchProber) Save(context.Context, string, string) error { return nil }

		// A Running server keeps its phase and endpoint through two missed probes,
		// re-probing every 10s; the third consecutive miss degrades it to Starting,
		// and any success in between starts the count over.
		func TestRunningServerDegradesOnlyAfterThreeMisses(t *testing.T) {
		down := false
		r, c := newReconciler(t, switchProber{down: &down}, runningServer(), rconSecret())
		reconcile(t, r, "survival")
		markPodReady(t, c, "survival")
		if res := reconcile(t, r, "survival"); res.RequeueAfter != time.Minute {
		t.Fatalf("healthy Running requeue = %v, want 1m", res.RequeueAfter)
		}
		stillRunning := func(label string) {
		t.Helper()
		s := getServer(t, c, "survival")
		if s.Status.Phase != v1alpha1.PhaseRunning \|\| !s.Status.Ready \|\| s.Status.Endpoint.Address != "10.43.0.42:25565" {
		t.Fatalf("%s: phase=%s ready=%v endpoint=%q", label, s.Status.Phase, s.Status.Ready, s.Status.Endpoint.Address)
		}
		}
		stillRunning("ready")

		down = true
		for i := 1; i <= 2; i++ {
		if res := reconcile(t, r, "survival"); res.RequeueAfter != 10*time.Second {
		t.Fatalf("miss %d requeue = %v, want 10s", i, res.RequeueAfter)
		}
		stillRunning("after a miss")
		}
		down = false
		reconcile(t, r, "survival") // a success clears the two misses
		down = true
		reconcile(t, r, "survival")
		reconcile(t, r, "survival")
		stillRunning("two misses after a recovery")

		reconcile(t, r, "survival") // third in a row
		if s := getServer(t, c, "survival"); s.Status.Phase != v1alpha1.PhaseStarting \|\| s.Status.Ready {
		t.Fatalf("third miss: phase=%s ready=%v, want Starting not-ready", s.Status.Phase, s.Status.Ready)
		}
		}

		// A Failed server wakes when its next auto-restart falls due, and every 5m
		// once the attempts are spent, instead of every 2s.
		func TestFailedServerRequeuesWhenTheRetryIsDue(t *testing.T) {
		srv := runningServer()
		srv.Spec.Startup.TimeoutSeconds = 30
		r, c := newReconciler(t, fakeProber{}, srv, rconSecret(), gamePod())
		base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
		clock := base
		r.Now = func() metav1.Time { return metav1.NewTime(clock) }

		if res := reconcile(t, r, "survival"); res.RequeueAfter != 2*time.Second {
		t.Fatalf("Starting requeue = %v, want 2s", res.RequeueAfter)
		}
		clock = base.Add(40 * time.Second) // timed out at 30s, first retry due at 90s
		if res := reconcile(t, r, "survival"); res.RequeueAfter != 50*time.Second {
		t.Fatalf("Failed requeue = %v, want 50s", res.RequeueAfter)
		}

		s := getServer(t, c, "survival")
		s.Status.AutoRestarts = 3
		if err := c.Status().Update(context.Background(), s); err != nil {
		t.Fatal(err)
		}
		clock = base.Add(41 * time.Second)
		if res := reconcile(t, r, "survival"); res.RequeueAfter != 5*time.Minute {
		t.Fatalf("spent requeue = %v, want 5m", res.RequeueAfter)
		}
		}

internal/operator/builders.go

+38 −9

Changes for internal/operator/builders.go: 38 added lines, 9 removed lines.
Original line number	Diff line number	Diff line
		package operator

		import (
		"time"

		"fmt"
		"strconv"

		@@ -165,25 +167,50 @@ func servicePorts(server *v1alpha1.MinecraftServer) []corev1.ServicePort {
		// RCON-less loader's own "started" signal — not the mere fact that the game
		// socket is bound — gates readiness. Timings are identical across both modes.
		func readinessProbe(server v1alpha1.MinecraftServer) corev1.Probe {
		probe := &corev1.Probe{
		return &corev1.Probe{
		ProbeHandler: healthHandler(server),
		InitialDelaySeconds: 20,
		PeriodSeconds: 10,
		FailureThreshold: 6,
		}
		}

		// startupProbe holds liveness off until the server first answers. Its budget
		// outlasts the operator's startup timeout plus the first auto-restart backoff,
		// so a slow first world generation is the operator's to judge (a counted,
		// bounded pod restart) and never a kubelet restart loop.
		func startupProbe(server v1alpha1.MinecraftServer) corev1.Probe {
		const period = 10
		budget := startupTimeout(server) + autoRestartBaseBackoff
		return &corev1.Probe{
		ProbeHandler: healthHandler(server),
		PeriodSeconds: period,
		FailureThreshold: int32((budget+periodtime.Second-1)/(periodtime.Second)) + 1,
		}
		}

		// livenessProbe restarts a server that stopped answering for two minutes: long
		// enough to ride out a world save or a lag spike.
		func livenessProbe(server v1alpha1.MinecraftServer) corev1.Probe {
		return &corev1.Probe{
		ProbeHandler: healthHandler(server),
		PeriodSeconds: 20,
		TimeoutSeconds: 5,
		FailureThreshold: 6,
		}
		}

		// healthHandler is a plain TCP check on the game port, or an HTTP GET on the
		// loader's health endpoint when StartupSpec.HealthHTTPPort is set.
		func healthHandler(server *v1alpha1.MinecraftServer) corev1.ProbeHandler {
		if hp := server.Spec.Startup.HealthHTTPPort; hp > 0 {
		path := server.Spec.Startup.HealthHTTPPath
		if path == "" {
		path = "/healthz"
		}
		probe.ProbeHandler = corev1.ProbeHandler{
		HTTPGet: &corev1.HTTPGetAction{Path: path, Port: intstr.FromInt32(hp)},
		}
		return probe
		}
		probe.ProbeHandler = corev1.ProbeHandler{
		TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(GamePort)},
		return corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: path, Port: intstr.FromInt32(hp)}}
		}
		return probe
		return corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(GamePort)}}
		}

		// buildStatefulSet renders the workload for replicas in {0,1}. Its half of
		@@ -217,6 +244,8 @@ func buildStatefulSet(server *v1alpha1.MinecraftServer, replicas int32, felisIma
		// StartupSpec.HealthHTTPPort) that reports true readiness — used below when
		// set.
		ReadinessProbe: readinessProbe(server),
		StartupProbe: startupProbe(server),
		LivenessProbe: livenessProbe(server),
		// The server runs untrusted plugins, so it keeps no capability and can never
		// regain one. The root filesystem stays writable: an arbitrary Paper image
		// may unpack its runtime or write temp files outside /data.