fix(api): wake 与回档/备份/改文件按服务器互斥

internal/operator/maintenance_test.go

@@ -0,0 +1,108 @@
+package operator_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/maintenance"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func maintenanceJob(managedBy string, finished bool) *batchv1.Job {
+	j := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "restore-survival", Namespace: "minecraft",
+		Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: managedBy}}}
+	if finished {
+		j.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
+	}
+	return j
+}
+
+// A desiredState flipped to Running behind felis-api's back (kubectl, a script)
+// while a restore Job is unpacking into the world must not get a game pod.
+func TestReconcileRunning_HeldDuringMaintenance(t *testing.T) {
+	r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret(), maintenanceJob("felis-restore", false))
+	r.Jobs = c
+
+	res := reconcile(t, r, "survival")
+	if res.RequeueAfter <= 0 {
+		t.Fatal("a held server must requeue to notice the Job finishing")
+	}
+	var sts appsv1.StatefulSet
+	err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &sts)
+	if !apierrors.IsNotFound(err) {
+		t.Fatalf("StatefulSet created while the restore runs (err=%v)", err)
+	}
+	s := getServer(t, c, "survival")
+	cond := meta.FindStatusCondition(s.Status.Conditions, v1alpha1.ConditionReady)
+	if cond == nil || cond.Reason != "MaintenanceInProgress" {
+		t.Fatalf("Ready condition = %+v, want reason MaintenanceInProgress", cond)
+	}
+	if s.Status.StartRequestedAt != nil {
+		t.Fatal("the hold must not start the startup-timeout clock")
+	}
+
+	// The Job finishes: the next pass starts the server.
+	var j batchv1.Job
+	if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "restore-survival"}, &j); err != nil {
+		t.Fatalf("get job: %v", err)
+	}
+	j.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
+	if err := c.Status().Update(context.Background(), &j); err != nil {
+		if err := c.Update(context.Background(), &j); err != nil {
+			t.Fatalf("finish job: %v", err)
+		}
+	}
+	reconcile(t, r, "survival")
+	if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &sts); err != nil {
+		t.Fatalf("StatefulSet not created after the restore finished: %v", err)
+	}
+}
+
+func TestReconcileRunning_FreshLockHolds(t *testing.T) {
+	ms := runningServer()
+	ms.Annotations = map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindBackup, fixedNow().Add(-5*time.Second))}
+	r, c := newReconciler(t, fakeProber{}, ms, rconSecret())
+	r.Jobs = c
+	reconcile(t, r, "survival")
+	var sts appsv1.StatefulSet
+	if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &sts); !apierrors.IsNotFound(err) {
+		t.Fatalf("StatefulSet created under a fresh maintenance lock (err=%v)", err)
+	}
+}
+
+func TestReconcileRunning_NotHeldByReadsOrFinishedJobs(t *testing.T) {
+	read := maintenanceJob("felis-files", false)
+	read.Name = "files-survival-read"
+	read.Labels[maintenance.LabelFilesMode] = "read"
+	r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret(), read, maintenanceJob("felis-backup", true))
+	r.Jobs = c
+	reconcile(t, r, "survival")
+	var sts appsv1.StatefulSet
+	if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &sts); err != nil {
+		t.Fatalf("StatefulSet not created: %v", err)
+	}
+}
+
+// A server that is already up is never held: its pod is the one on the volume,
+// and holding would only stall its readiness bookkeeping.
+func TestReconcileRunning_RunningServerNotHeld(t *testing.T) {
+	r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret())
+	reconcile(t, r, "survival") // creates the StatefulSet at replicas 1
+	if err := c.Create(context.Background(), maintenanceJob("felis-restore", false)); err != nil {
+		t.Fatalf("create job: %v", err)
+	}
+	r.Jobs = c
+	reconcile(t, r, "survival")
+	cond := meta.FindStatusCondition(getServer(t, c, "survival").Status.Conditions, v1alpha1.ConditionReady)
+	if cond != nil && cond.Reason == "MaintenanceInProgress" {
+		t.Fatal("a scaled-up server was held")
+	}
+}

internal/operator/reconciler.go

@@ -13,8 +13,10 @@ import (
 	"time"
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/maintenance"
 	"felis.lolicon.best/internal/metrics"
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -37,6 +39,7 @@ const (
 	requeueStopping            = 5 * time.Second
 	requeueSecret              = 10 * time.Second
 	requeueIdleProbe           = 30 * time.Second
+	requeueMaintenance         = 5 * time.Second
 	defaultTimeoutSeconds      = 300
 	defaultReadinessTimeoutSec = 300
 )
@@ -69,6 +72,10 @@ type Reconciler struct {
 	// Now is injectable for deterministic timestamps in tests; defaults to
 	// metav1.Now.
 	Now func() metav1.Time
+	// Jobs reads the minecraft namespace's Jobs for the world-volume lock
+	// (internal/maintenance). It is the manager's uncached API reader, so the
+	// operator needs jobs:list and no Job informer. Nil skips the check.
+	Jobs client.Reader
 }
 
 func (r *Reconciler) now() metav1.Time {
@@ -111,6 +118,20 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 }
 
 func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.MinecraftServer) (ctrl.Result, error) {
+	if kind, held, err := r.maintenanceHold(ctx, server); err != nil {
+		return ctrl.Result{}, err
+	} else if held {
+		// Leave phase and the start anchor alone: nothing is starting yet, and a
+		// long restore must not be charged to the startup timeout.
+		server.Status.ObservedGeneration = server.Generation
+		r.setCondition(server, v1alpha1.ConditionReady, metav1.ConditionFalse, "MaintenanceInProgress",
+			"waiting for the "+kind+" on this server's world to finish before starting")
+		if err := r.patchStatus(ctx, server); err != nil {
+			return ctrl.Result{}, err
+		}
+		return ctrl.Result{RequeueAfter: requeueMaintenance}, nil
+	}
+
 	endpointAddress, err := r.ensureServices(ctx, server)
 	if err != nil {
 		return ctrl.Result{}, err
@@ -283,6 +304,35 @@ func (r *Reconciler) reconcileStopped(ctx context.Context, server *v1alpha1.Mine
 	return ctrl.Result{}, r.patchStatus(ctx, server)
 }
 
+// maintenanceHold reports whether a restore, backup or file write holds the
+// server's world volume (internal/maintenance) while its pod is about to be
+// created. felis-api already refuses a wake in that state; this is the same rule
+// for a desiredState flipped by anything else (kubectl, a script), since a game
+// pod scheduled beside a restore Job boots on a half-extracted world. A server
+// whose StatefulSet is already scaled up is never held: its pod exists, and
+// stopping it here would only lose the players on it.
+func (r *Reconciler) maintenanceHold(ctx context.Context, server *v1alpha1.MinecraftServer) (string, bool, error) {
+	if r.Jobs == nil {
+		return "", false, nil
+	}
+	var sts appsv1.StatefulSet
+	err := r.Get(ctx, types.NamespacedName{Namespace: server.Namespace, Name: server.Name}, &sts)
+	switch {
+	case apierrors.IsNotFound(err):
+	case err != nil:
+		return "", false, err
+	case sts.Spec.Replicas == nil || *sts.Spec.Replicas > 0 || sts.Status.Replicas > 0:
+		return "", false, nil
+	}
+	var jobs batchv1.JobList
+	if err := r.Jobs.List(ctx, &jobs, client.InNamespace(server.Namespace),
+		client.MatchingLabels{maintenance.LabelServer: server.Name}); err != nil {
+		return "", false, err
+	}
+	kind, held := maintenance.Holder(server.Name, server.Annotations, jobs.Items, r.now().Time)
+	return kind, held, nil
+}
+
 func (r *Reconciler) ensureServices(ctx context.Context, server *v1alpha1.MinecraftServer) (string, error) {
 	endpointAddress := ""
 	for _, svc := range []*corev1.Service{buildHeadlessService(server), buildClientService(server)} {