feat(operator): 状态迁移、超时、自动重建、空闲停机与 RCON Secret 创建写 Event 和结构化日志,RBAC 增加 events create/patch

This commit is contained in:
Lemon-miaow committed 2026-09-25 19:32:25 +08:00
1 parent a977e229ca
commit b7d4275ca9
5 files changed
+204 -7

No files matched your search

+1
View File
@@ -121,6 +121,7 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
// Uncached too: RCON Secrets are read by name, so the Role grants
// secrets:get without the list/watch an informer would need.
Secrets: mgr.GetAPIReader(),
Recorder: mgr.GetEventRecorderFor("felis-operator"),
Watch: watch,
}
if err := r.SetupWithManager(mgr); err != nil {
+132
View File
@@ -0,0 +1,132 @@
package operator_test
import (
"context"
"errors"
"slices"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/operator"
)
// drain returns the Events recorded since the last drain.
func drain(rec *record.FakeRecorder) []string {
var out []string
for {
select {
case e := <-rec.Events:
out = append(out, e)
default:
return out
}
}
}
func expectEvents(t *testing.T, rec *record.FakeRecorder, step string, want ...string) {
t.Helper()
if got := drain(rec); !slices.Equal(got, want) {
t.Fatalf("%s: events = %q, want %q", step, got, want)
}
}
// Each phase change lands one Event, and a pass that changes nothing lands none.
func TestReconcilerRecordsThePhaseTimeline(t *testing.T) {
down := false
r, c := newReconciler(t, switchProber{down: &down}, runningServer())
rec := record.NewFakeRecorder(32)
r.Recorder = rec
reconcile(t, r, "survival")
expectEvents(t, rec, "first pass",
"Normal RconSecretCreated created the RCON password Secret survival-rcon",
"Normal PodNotReady New → Starting: waiting for pod TCP readiness")
reconcile(t, r, "survival")
expectEvents(t, rec, "still starting")
markPodReady(t, c, "survival")
reconcile(t, r, "survival")
expectEvents(t, rec, "ready", "Normal RconReached Starting → Running: server is accepting RCON")
reconcile(t, r, "survival")
expectEvents(t, rec, "steady")
down = true
for range 3 {
reconcile(t, r, "survival")
}
expectEvents(t, rec, "degraded", "Warning RconNotReachable Running → Starting: i/o timeout")
}
// A timed-out start and the pod recreation that retries it are Warnings.
func TestTimeoutAndAutoRestartAreWarnings(t *testing.T) {
srv := runningServer()
srv.Spec.Startup.TimeoutSeconds = 30
r, _ := newReconciler(t, fakeProber{}, srv, rconSecret(), gamePod())
rec := record.NewFakeRecorder(32)
r.Recorder = rec
base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
clock := base
r.Now = func() metav1.Time { return metav1.NewTime(clock) }
reconcile(t, r, "survival")
drain(rec)
clock = base.Add(30 * time.Second)
reconcile(t, r, "survival")
expectEvents(t, rec, "timeout",
"Warning StartupTimeout Starting → Failed: pod did not become ready within startup timeout")
clock = base.Add(90 * time.Second)
reconcile(t, r, "survival")
expectEvents(t, rec, "retry",
"Warning AutoRestart Failed → Starting: start timed out; recreated the pod (attempt 1 of 3)")
}
// Idle auto-stop says why it stopped the server.
func TestIdleStopRecordsAnEvent(t *testing.T) {
srv := runningServer()
srv.Spec.Idle = v1alpha1.IdleSpec{AutoStopEnabled: true, EmptySecondsBeforeStop: 60}
r, c := newReconciler(t, fakeProber{players: operator.PlayerCount{Online: 0, Max: 20, Known: true}}, srv, rconSecret())
rec := record.NewFakeRecorder(32)
r.Recorder = rec
base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
clock := base
r.Now = func() metav1.Time { return metav1.NewTime(clock) }
reconcile(t, r, "survival")
markPodReady(t, c, "survival")
reconcile(t, r, "survival")
drain(rec)
clock = base.Add(61 * time.Second)
reconcile(t, r, "survival")
expectEvents(t, rec, "idle", "Normal IdleStop no players online for 60s; set desiredState to Stopped")
}
// A pass that fails records nothing, even though it changed the phase in memory.
func TestFailedPassRecordsNoEvent(t *testing.T) {
r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret())
rec := record.NewFakeRecorder(32)
r.Recorder = rec
r.Client = failStatus{c}
req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "minecraft", Name: "survival"}}
if _, err := r.Reconcile(context.Background(), req); err == nil {
t.Fatal("want the status write error")
}
expectEvents(t, rec, "failed write")
}
// failStatus refuses every status write.
type failStatus struct{ client.Client }
func (f failStatus) Status() client.SubResourceWriter { return failingWriter{f.Client.Status()} }
type failingWriter struct{ client.SubResourceWriter }
func (failingWriter) Update(context.Context, client.Object, ...client.SubResourceUpdateOption) error {
return errors.New("status write refused")
}
+52 -2
View File
@@ -26,10 +26,12 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
)
// Requeue cadences for the transient phases.
@@ -128,6 +130,10 @@ type Reconciler struct {
// mirror with the database URL among them) and grow with them. Nil falls back
// to the embedded client.
Secrets client.Reader
// Recorder puts an Event on the MinecraftServer at each phase change, pod
// recreation, idle stop and RCON Secret provision, so `kubectl describe`
// shows the timeline the status alone overwrites. Nil records none.
Recorder record.EventRecorder
// Watch records the passes in flight for the liveness probe. Nil skips it.
Watch *ReconcileWatch
@@ -188,10 +194,49 @@ func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
if desired == "" {
desired = v1alpha1.DesiredStopped
}
prevPhase, prevRestarts := server.Status.Phase, server.Status.AutoRestarts
var res ctrl.Result
var err error
if desired == v1alpha1.DesiredStopped {
return r.reconcileStopped(ctx, &server)
res, err = r.reconcileStopped(ctx, &server)
} else {
res, err = r.reconcileRunning(ctx, &server)
}
if err == nil {
r.recordTransition(ctx, &server, prevPhase, prevRestarts)
}
return res, err
}
// recordTransition logs and records a pass that moved the server to another
// phase, with the Ready condition's reason and message. Failing, a recreated
// pod and a Running server falling back to Starting are Warnings.
func (r *Reconciler) recordTransition(ctx context.Context, server *v1alpha1.MinecraftServer, prevPhase v1alpha1.Phase, prevRestarts int32) {
phase := server.Status.Phase
if phase == prevPhase {
return
}
reason, msg := string(phase), ""
if c := meta.FindStatusCondition(server.Status.Conditions, v1alpha1.ConditionReady); c != nil {
reason, msg = c.Reason, c.Message
}
eventType := corev1.EventTypeNormal
if phase == v1alpha1.PhaseFailed || server.Status.AutoRestarts > prevRestarts ||
(prevPhase == v1alpha1.PhaseRunning && phase == v1alpha1.PhaseStarting) {
eventType = corev1.EventTypeWarning
}
from := string(prevPhase)
if from == "" {
from = "New"
}
log.FromContext(ctx).Info("phase changed", "from", from, "to", phase, "reason", reason, "message", msg)
r.event(server, eventType, reason, fmt.Sprintf("%s → %s: %s", from, phase, msg))
}
func (r *Reconciler) event(server *v1alpha1.MinecraftServer, eventType, reason, msg string) {
if r.Recorder != nil {
r.Recorder.Event(server, eventType, reason, msg)
}
return r.reconcileRunning(ctx, &server)
}
func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.MinecraftServer) (ctrl.Result, error) {
@@ -343,6 +388,9 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
if err := r.Patch(ctx, server, patch); err != nil {
return ctrl.Result{}, err
}
msg := fmt.Sprintf("no players online for %ds; set desiredState to Stopped", server.Spec.Idle.EmptySecondsBeforeStop)
log.FromContext(ctx).Info("idle stop", "emptySeconds", server.Spec.Idle.EmptySecondsBeforeStop)
r.event(server, corev1.EventTypeNormal, "IdleStop", msg)
return ctrl.Result{}, nil
}
} else if server.Status.EmptySince != nil {
@@ -573,6 +621,8 @@ func (r *Reconciler) ensureRconSecret(ctx context.Context, server *v1alpha1.Mine
}
return "", err
}
log.FromContext(ctx).Info("provisioned the RCON password Secret", "secret", ref.Name)
r.event(server, corev1.EventTypeNormal, "RconSecretCreated", "created the RCON password Secret "+ref.Name)
return rconStamp([]byte(password)), nil
}
+7 -2
View File
@@ -155,8 +155,9 @@ func APIBuildRole(p Params) *rbacv1.Role {
// the operator checks that no restore/backup/file-write Job holds its world
// (internal/maintenance). Pods are delete-only: a start that timed out is retried
// by deleting its pod for the StatefulSet to recreate (bounded, three attempts;
// internal/operator.recoverFailedStart). It never touches PVCs, Events, or
// finalizers, so none appear here.
// internal/operator.recoverFailedStart). Events are create/patch only, for the
// timeline it records on each server. It never touches PVCs or finalizers, so
// none appear here.
func OperatorRole(p Params) *rbacv1.Role {
p = p.withDefaults()
return role(p.MinecraftNamespace, "felis-operator", ComponentOperator, []rbacv1.PolicyRule{
@@ -177,6 +178,10 @@ func OperatorRole(p Params) *rbacv1.Role {
// delete only, through the direct client: no read of pods is needed to
// remove the one named <server>-0.
rule([]string{groupCore}, []string{"pods"}, []string{"delete"}),
// The Events the reconciler records on MinecraftServers (phase changes,
// pod recreation, idle stop): the recorder creates one and patches its
// count when the same Event repeats.
rule([]string{groupCore}, []string{"events"}, []string{"create", "patch"}),
})
}
+12 -3
View File
@@ -189,9 +189,18 @@ func TestOperatorRole_ScopeExact(t *testing.T) {
}
}
// Hard exclusions.
for _, res := range []string{"persistentvolumeclaims", "events"} {
if grantsResource(op, groupCore, res) {
t.Errorf("operator must NOT touch core/%s", res)
if grantsResource(op, groupCore, "persistentvolumeclaims") {
t.Error("operator must NOT touch core/persistentvolumeclaims")
}
// Events: the recorder creates one and patches its count on a repeat.
for _, v := range []string{"create", "patch"} {
if !hasRule(op, groupCore, "events", v) {
t.Errorf("operator must have events:%s (the server timeline)", v)
}
}
for _, v := range []string{"get", "list", "watch", "update", "delete", "deletecollection", "*"} {
if hasRule(op, groupCore, "events", v) {
t.Errorf("operator events rule must be create+patch only, found %s", v)
}
}
// RCON Secrets are read by name through the uncached reader and created once;