fix(platform): control plane runs system-cluster-critical, so eviction refuses it (#8)

Following the first shield attempt (custom class, value 1e6) a live drill
showed the limit: kubelet evicted the game pods and then the api,
operator and registry anyway — evicting them was never what reclaimed
the disk — and with the images containerd-only, the GC stage left
everything in ImagePullBackOff. A custom class cannot be raised past 1e9
(the API caps user-defined values), while kubelet's eviction refusal
needs >= 2e9, so the control plane now uses the built-in
system-cluster-critical.

Re-drilled: disk filled to 1.7G free -> login/lobby evicted, and kubelet
logged "cannot evict a critical pod" for felis-api/operator/registry,
which stayed Running throughout. Recovery facts now in troubleshooting
13b: the DiskPressure condition lingers ~5m after space is freed
(--eviction-pressure-transition-period), and game images GC'd while
their pods were evicted need the documented re-import (verified: 25s to
Running).
This commit is contained in:
Lemon-miaow committed 2026-09-22 21:55:54 +08:00
1 parent fe310743a2
commit 0a2d654e68
5 files changed
+72 -77

No files matched your search

+14 -31
View File
@@ -6,7 +6,6 @@ import (
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
schedulingv1 "k8s.io/api/scheduling/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)
@@ -460,8 +459,8 @@ func TestRegistry_DeploymentServicePVC(t *testing.T) {
// internal Service must be present or the login pod's felis-api:8081 path is dead.
func TestWorkloads_BundleContents(t *testing.T) {
objs := Workloads(testParams())
if len(objs) != 9 {
t.Fatalf("Workloads returned %d objects, want 9", len(objs))
if len(objs) != 8 {
t.Fatalf("Workloads returned %d objects, want 8", len(objs))
}
var haveInternalSvc bool
for _, o := range objs {
@@ -479,24 +478,20 @@ func TestWorkloads_BundleContents(t *testing.T) {
}
// TestControlPlanePriorityClass pins the node-pressure eviction shield: every
// control-plane pod template (api/operator/reaper/registry) references the one
// PriorityClass, which outranks the default-0 game pods by eviction order while
// never preempting them, and is the bundle's only cluster-scoped object. Without
// this, a full disk evicts the api alongside the game pods and (no reachable
// registry on an air-gapped box) recovery needs a human re-importing images.
// control-plane pod template (api/operator/reaper/registry) runs under the
// BUILT-IN system-cluster-critical class (value 2e9), at which kubelet's
// eviction manager refuses to evict the pod. A live drill showed the whole
// cascade with plain ordering: disk pressure evicted the game pods and then the
// control plane, whose images (air-gapped, containerd-only) were GC'd →
// ImagePullBackOff plus a manual re-import. User-defined classes are capped at
// 1e9 (API-enforced), so the built-in class is the only way to reach the
// critical threshold.
func TestControlPlanePriorityClass(t *testing.T) {
objs := Workloads(reaperParams()) // include the reaper CronJob's pod template
var class *schedulingv1.PriorityClass
clusterScoped := 0
deployments, cronJobs := 0, 0
for _, o := range objs {
if o.GetNamespace() == "" {
clusterScoped++
}
switch v := o.(type) {
case *schedulingv1.PriorityClass:
class = v
case *appsv1.Deployment:
deployments++
if v.Spec.Template.Spec.PriorityClassName != controlPlanePriorityName {
@@ -509,25 +504,13 @@ func TestControlPlanePriorityClass(t *testing.T) {
}
}
}
if class == nil {
t.Fatal("Workloads must render the control-plane PriorityClass")
}
if class.Name != controlPlanePriorityName || class.Value != controlPlanePriorityValue {
t.Errorf("class = %s/%d, want %s/%d", class.Name, class.Value, controlPlanePriorityName, controlPlanePriorityValue)
}
if class.PreemptionPolicy == nil || *class.PreemptionPolicy != corev1.PreemptNever {
t.Errorf("class preemptionPolicy = %v, want Never (an eviction shield, never a lever against running game servers)", class.PreemptionPolicy)
}
if clusterScoped != 1 {
t.Errorf("bundle has %d cluster-scoped objects, want exactly the PriorityClass", clusterScoped)
}
if deployments != 3 || cronJobs != 1 {
t.Errorf("scanned %d deployments / %d cronjobs, want 3 / 1 — a pod template escaped the class check", deployments, cronJobs)
}
// The class must outrank the default game-pod priority (0); equality would make
// the eviction order nondeterministic between the api and a full game server.
if controlPlanePriorityValue <= 0 {
t.Errorf("control-plane priority %d must exceed the default 0 game-pod priority", controlPlanePriorityValue)
// The name must be the built-in critical class: any custom class is capped at
// 1e9 by the API server and would be evictable.
if controlPlanePriorityName != "system-cluster-critical" {
t.Errorf("control-plane priority class = %q; only the built-in critical classes reach the 2e9 eviction-refusal threshold", controlPlanePriorityName)
}
}