feat(operator): 状态迁移、超时、自动重建、空闲停机与 RCON Secret 创建写 Event 和结构化日志,RBAC 增加 events create/patch
This commit is contained in:
5 files changed
+206
-9
No files matched your search
@@ -120,8 +120,9 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
|
||||
Jobs: mgr.GetAPIReader(),
|
||||
// Uncached too: RCON Secrets are read by name, so the Role grants
|
||||
// secrets:get without the list/watch an informer would need.
|
||||
Secrets: mgr.GetAPIReader(),
|
||||
Watch: watch,
|
||||
Secrets: mgr.GetAPIReader(),
|
||||
Recorder: mgr.GetEventRecorderFor("felis-operator"),
|
||||
Watch: watch,
|
||||
}
|
||||
if err := r.SetupWithManager(mgr); err != nil {
|
||||
fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
package operator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/record"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||
"felis.lolicon.best/internal/operator"
|
||||
)
|
||||
|
||||
// drain returns the Events recorded since the last drain.
|
||||
func drain(rec *record.FakeRecorder) []string {
|
||||
var out []string
|
||||
for {
|
||||
select {
|
||||
case e := <-rec.Events:
|
||||
out = append(out, e)
|
||||
default:
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func expectEvents(t *testing.T, rec *record.FakeRecorder, step string, want ...string) {
|
||||
t.Helper()
|
||||
if got := drain(rec); !slices.Equal(got, want) {
|
||||
t.Fatalf("%s: events = %q, want %q", step, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Each phase change lands one Event, and a pass that changes nothing lands none.
|
||||
func TestReconcilerRecordsThePhaseTimeline(t *testing.T) {
|
||||
down := false
|
||||
r, c := newReconciler(t, switchProber{down: &down}, runningServer())
|
||||
rec := record.NewFakeRecorder(32)
|
||||
r.Recorder = rec
|
||||
|
||||
reconcile(t, r, "survival")
|
||||
expectEvents(t, rec, "first pass",
|
||||
"Normal RconSecretCreated created the RCON password Secret survival-rcon",
|
||||
"Normal PodNotReady New → Starting: waiting for pod TCP readiness")
|
||||
reconcile(t, r, "survival")
|
||||
expectEvents(t, rec, "still starting")
|
||||
|
||||
markPodReady(t, c, "survival")
|
||||
reconcile(t, r, "survival")
|
||||
expectEvents(t, rec, "ready", "Normal RconReached Starting → Running: server is accepting RCON")
|
||||
reconcile(t, r, "survival")
|
||||
expectEvents(t, rec, "steady")
|
||||
|
||||
down = true
|
||||
for range 3 {
|
||||
reconcile(t, r, "survival")
|
||||
}
|
||||
expectEvents(t, rec, "degraded", "Warning RconNotReachable Running → Starting: i/o timeout")
|
||||
}
|
||||
|
||||
// A timed-out start and the pod recreation that retries it are Warnings.
|
||||
func TestTimeoutAndAutoRestartAreWarnings(t *testing.T) {
|
||||
srv := runningServer()
|
||||
srv.Spec.Startup.TimeoutSeconds = 30
|
||||
r, _ := newReconciler(t, fakeProber{}, srv, rconSecret(), gamePod())
|
||||
rec := record.NewFakeRecorder(32)
|
||||
r.Recorder = rec
|
||||
base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
|
||||
clock := base
|
||||
r.Now = func() metav1.Time { return metav1.NewTime(clock) }
|
||||
|
||||
reconcile(t, r, "survival")
|
||||
drain(rec)
|
||||
clock = base.Add(30 * time.Second)
|
||||
reconcile(t, r, "survival")
|
||||
expectEvents(t, rec, "timeout",
|
||||
"Warning StartupTimeout Starting → Failed: pod did not become ready within startup timeout")
|
||||
clock = base.Add(90 * time.Second)
|
||||
reconcile(t, r, "survival")
|
||||
expectEvents(t, rec, "retry",
|
||||
"Warning AutoRestart Failed → Starting: start timed out; recreated the pod (attempt 1 of 3)")
|
||||
}
|
||||
|
||||
// Idle auto-stop says why it stopped the server.
|
||||
func TestIdleStopRecordsAnEvent(t *testing.T) {
|
||||
srv := runningServer()
|
||||
srv.Spec.Idle = v1alpha1.IdleSpec{AutoStopEnabled: true, EmptySecondsBeforeStop: 60}
|
||||
r, c := newReconciler(t, fakeProber{players: operator.PlayerCount{Online: 0, Max: 20, Known: true}}, srv, rconSecret())
|
||||
rec := record.NewFakeRecorder(32)
|
||||
r.Recorder = rec
|
||||
base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
|
||||
clock := base
|
||||
r.Now = func() metav1.Time { return metav1.NewTime(clock) }
|
||||
|
||||
reconcile(t, r, "survival")
|
||||
markPodReady(t, c, "survival")
|
||||
reconcile(t, r, "survival")
|
||||
drain(rec)
|
||||
clock = base.Add(61 * time.Second)
|
||||
reconcile(t, r, "survival")
|
||||
expectEvents(t, rec, "idle", "Normal IdleStop no players online for 60s; set desiredState to Stopped")
|
||||
}
|
||||
|
||||
// A pass that fails records nothing, even though it changed the phase in memory.
|
||||
func TestFailedPassRecordsNoEvent(t *testing.T) {
|
||||
r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret())
|
||||
rec := record.NewFakeRecorder(32)
|
||||
r.Recorder = rec
|
||||
r.Client = failStatus{c}
|
||||
req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "minecraft", Name: "survival"}}
|
||||
if _, err := r.Reconcile(context.Background(), req); err == nil {
|
||||
t.Fatal("want the status write error")
|
||||
}
|
||||
expectEvents(t, rec, "failed write")
|
||||
}
|
||||
|
||||
// failStatus refuses every status write.
|
||||
type failStatus struct{ client.Client }
|
||||
|
||||
func (f failStatus) Status() client.SubResourceWriter { return failingWriter{f.Client.Status()} }
|
||||
|
||||
type failingWriter struct{ client.SubResourceWriter }
|
||||
|
||||
func (failingWriter) Update(context.Context, client.Object, ...client.SubResourceUpdateOption) error {
|
||||
return errors.New("status write refused")
|
||||
}
|
||||
@@ -26,10 +26,12 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/record"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
// Requeue cadences for the transient phases.
|
||||
@@ -128,6 +130,10 @@ type Reconciler struct {
|
||||
// mirror with the database URL among them) and grow with them. Nil falls back
|
||||
// to the embedded client.
|
||||
Secrets client.Reader
|
||||
// Recorder puts an Event on the MinecraftServer at each phase change, pod
|
||||
// recreation, idle stop and RCON Secret provision, so `kubectl describe`
|
||||
// shows the timeline the status alone overwrites. Nil records none.
|
||||
Recorder record.EventRecorder
|
||||
// Watch records the passes in flight for the liveness probe. Nil skips it.
|
||||
Watch *ReconcileWatch
|
||||
|
||||
@@ -188,10 +194,49 @@ func (r *Reconciler) reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
|
||||
if desired == "" {
|
||||
desired = v1alpha1.DesiredStopped
|
||||
}
|
||||
prevPhase, prevRestarts := server.Status.Phase, server.Status.AutoRestarts
|
||||
var res ctrl.Result
|
||||
var err error
|
||||
if desired == v1alpha1.DesiredStopped {
|
||||
return r.reconcileStopped(ctx, &server)
|
||||
res, err = r.reconcileStopped(ctx, &server)
|
||||
} else {
|
||||
res, err = r.reconcileRunning(ctx, &server)
|
||||
}
|
||||
if err == nil {
|
||||
r.recordTransition(ctx, &server, prevPhase, prevRestarts)
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
// recordTransition logs and records a pass that moved the server to another
|
||||
// phase, with the Ready condition's reason and message. Failing, a recreated
|
||||
// pod and a Running server falling back to Starting are Warnings.
|
||||
func (r *Reconciler) recordTransition(ctx context.Context, server *v1alpha1.MinecraftServer, prevPhase v1alpha1.Phase, prevRestarts int32) {
|
||||
phase := server.Status.Phase
|
||||
if phase == prevPhase {
|
||||
return
|
||||
}
|
||||
reason, msg := string(phase), ""
|
||||
if c := meta.FindStatusCondition(server.Status.Conditions, v1alpha1.ConditionReady); c != nil {
|
||||
reason, msg = c.Reason, c.Message
|
||||
}
|
||||
eventType := corev1.EventTypeNormal
|
||||
if phase == v1alpha1.PhaseFailed || server.Status.AutoRestarts > prevRestarts ||
|
||||
(prevPhase == v1alpha1.PhaseRunning && phase == v1alpha1.PhaseStarting) {
|
||||
eventType = corev1.EventTypeWarning
|
||||
}
|
||||
from := string(prevPhase)
|
||||
if from == "" {
|
||||
from = "New"
|
||||
}
|
||||
log.FromContext(ctx).Info("phase changed", "from", from, "to", phase, "reason", reason, "message", msg)
|
||||
r.event(server, eventType, reason, fmt.Sprintf("%s → %s: %s", from, phase, msg))
|
||||
}
|
||||
|
||||
func (r *Reconciler) event(server *v1alpha1.MinecraftServer, eventType, reason, msg string) {
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Event(server, eventType, reason, msg)
|
||||
}
|
||||
return r.reconcileRunning(ctx, &server)
|
||||
}
|
||||
|
||||
func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.MinecraftServer) (ctrl.Result, error) {
|
||||
@@ -343,6 +388,9 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
|
||||
if err := r.Patch(ctx, server, patch); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
msg := fmt.Sprintf("no players online for %ds; set desiredState to Stopped", server.Spec.Idle.EmptySecondsBeforeStop)
|
||||
log.FromContext(ctx).Info("idle stop", "emptySeconds", server.Spec.Idle.EmptySecondsBeforeStop)
|
||||
r.event(server, corev1.EventTypeNormal, "IdleStop", msg)
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
} else if server.Status.EmptySince != nil {
|
||||
@@ -573,6 +621,8 @@ func (r *Reconciler) ensureRconSecret(ctx context.Context, server *v1alpha1.Mine
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
log.FromContext(ctx).Info("provisioned the RCON password Secret", "secret", ref.Name)
|
||||
r.event(server, corev1.EventTypeNormal, "RconSecretCreated", "created the RCON password Secret "+ref.Name)
|
||||
return rconStamp([]byte(password)), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -155,8 +155,9 @@ func APIBuildRole(p Params) *rbacv1.Role {
|
||||
// the operator checks that no restore/backup/file-write Job holds its world
|
||||
// (internal/maintenance). Pods are delete-only: a start that timed out is retried
|
||||
// by deleting its pod for the StatefulSet to recreate (bounded, three attempts;
|
||||
// internal/operator.recoverFailedStart). It never touches PVCs, Events, or
|
||||
// finalizers, so none appear here.
|
||||
// internal/operator.recoverFailedStart). Events are create/patch only, for the
|
||||
// timeline it records on each server. It never touches PVCs or finalizers, so
|
||||
// none appear here.
|
||||
func OperatorRole(p Params) *rbacv1.Role {
|
||||
p = p.withDefaults()
|
||||
return role(p.MinecraftNamespace, "felis-operator", ComponentOperator, []rbacv1.PolicyRule{
|
||||
@@ -177,6 +178,10 @@ func OperatorRole(p Params) *rbacv1.Role {
|
||||
// delete only, through the direct client: no read of pods is needed to
|
||||
// remove the one named <server>-0.
|
||||
rule([]string{groupCore}, []string{"pods"}, []string{"delete"}),
|
||||
// The Events the reconciler records on MinecraftServers (phase changes,
|
||||
// pod recreation, idle stop): the recorder creates one and patches its
|
||||
// count when the same Event repeats.
|
||||
rule([]string{groupCore}, []string{"events"}, []string{"create", "patch"}),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -189,9 +189,18 @@ func TestOperatorRole_ScopeExact(t *testing.T) {
|
||||
}
|
||||
}
|
||||
// Hard exclusions.
|
||||
for _, res := range []string{"persistentvolumeclaims", "events"} {
|
||||
if grantsResource(op, groupCore, res) {
|
||||
t.Errorf("operator must NOT touch core/%s", res)
|
||||
if grantsResource(op, groupCore, "persistentvolumeclaims") {
|
||||
t.Error("operator must NOT touch core/persistentvolumeclaims")
|
||||
}
|
||||
// Events: the recorder creates one and patches its count on a repeat.
|
||||
for _, v := range []string{"create", "patch"} {
|
||||
if !hasRule(op, groupCore, "events", v) {
|
||||
t.Errorf("operator must have events:%s (the server timeline)", v)
|
||||
}
|
||||
}
|
||||
for _, v := range []string{"get", "list", "watch", "update", "delete", "deletecollection", "*"} {
|
||||
if hasRule(op, groupCore, "events", v) {
|
||||
t.Errorf("operator events rule must be create+patch only, found %s", v)
|
||||
}
|
||||
}
|
||||
// RCON Secrets are read by name through the uncached reader and created once;
|
||||
|
||||
Reference in new issue
Block a user