fix(platform): control plane runs system-cluster-critical, so eviction refuses it (#8)
Following the first shield attempt (custom class, value 1e6) a live drill showed the limit: kubelet evicted the game pods and then the api, operator and registry anyway — evicting them was never what reclaimed the disk — and with the images containerd-only, the GC stage left everything in ImagePullBackOff. A custom class cannot be raised past 1e9 (the API caps user-defined values), while kubelet's eviction refusal needs >= 2e9, so the control plane now uses the built-in system-cluster-critical. Re-drilled: disk filled to 1.7G free -> login/lobby evicted, and kubelet logged "cannot evict a critical pod" for felis-api/operator/registry, which stayed Running throughout. Recovery facts now in troubleshooting 13b: the DiskPressure condition lingers ~5m after space is freed (--eviction-pressure-transition-period), and game images GC'd while their pods were evicted need the documented re-import (verified: 25s to Running).
This commit is contained in:
5 files changed
+72
-77
No files matched your search
@@ -6,7 +6,6 @@ import (
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
schedulingv1 "k8s.io/api/scheduling/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
)
|
||||
@@ -460,8 +459,8 @@ func TestRegistry_DeploymentServicePVC(t *testing.T) {
|
||||
// internal Service must be present or the login pod's felis-api:8081 path is dead.
|
||||
func TestWorkloads_BundleContents(t *testing.T) {
|
||||
objs := Workloads(testParams())
|
||||
if len(objs) != 9 {
|
||||
t.Fatalf("Workloads returned %d objects, want 9", len(objs))
|
||||
if len(objs) != 8 {
|
||||
t.Fatalf("Workloads returned %d objects, want 8", len(objs))
|
||||
}
|
||||
var haveInternalSvc bool
|
||||
for _, o := range objs {
|
||||
@@ -479,24 +478,20 @@ func TestWorkloads_BundleContents(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestControlPlanePriorityClass pins the node-pressure eviction shield: every
|
||||
// control-plane pod template (api/operator/reaper/registry) references the one
|
||||
// PriorityClass, which outranks the default-0 game pods by eviction order while
|
||||
// never preempting them, and is the bundle's only cluster-scoped object. Without
|
||||
// this, a full disk evicts the api alongside the game pods and (no reachable
|
||||
// registry on an air-gapped box) recovery needs a human re-importing images.
|
||||
// control-plane pod template (api/operator/reaper/registry) runs under the
|
||||
// BUILT-IN system-cluster-critical class (value 2e9), at which kubelet's
|
||||
// eviction manager refuses to evict the pod. A live drill showed the whole
|
||||
// cascade with plain ordering: disk pressure evicted the game pods and then the
|
||||
// control plane, whose images (air-gapped, containerd-only) were GC'd →
|
||||
// ImagePullBackOff plus a manual re-import. User-defined classes are capped at
|
||||
// 1e9 (API-enforced), so the built-in class is the only way to reach the
|
||||
// critical threshold.
|
||||
func TestControlPlanePriorityClass(t *testing.T) {
|
||||
objs := Workloads(reaperParams()) // include the reaper CronJob's pod template
|
||||
|
||||
var class *schedulingv1.PriorityClass
|
||||
clusterScoped := 0
|
||||
deployments, cronJobs := 0, 0
|
||||
for _, o := range objs {
|
||||
if o.GetNamespace() == "" {
|
||||
clusterScoped++
|
||||
}
|
||||
switch v := o.(type) {
|
||||
case *schedulingv1.PriorityClass:
|
||||
class = v
|
||||
case *appsv1.Deployment:
|
||||
deployments++
|
||||
if v.Spec.Template.Spec.PriorityClassName != controlPlanePriorityName {
|
||||
@@ -509,25 +504,13 @@ func TestControlPlanePriorityClass(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if class == nil {
|
||||
t.Fatal("Workloads must render the control-plane PriorityClass")
|
||||
}
|
||||
if class.Name != controlPlanePriorityName || class.Value != controlPlanePriorityValue {
|
||||
t.Errorf("class = %s/%d, want %s/%d", class.Name, class.Value, controlPlanePriorityName, controlPlanePriorityValue)
|
||||
}
|
||||
if class.PreemptionPolicy == nil || *class.PreemptionPolicy != corev1.PreemptNever {
|
||||
t.Errorf("class preemptionPolicy = %v, want Never (an eviction shield, never a lever against running game servers)", class.PreemptionPolicy)
|
||||
}
|
||||
if clusterScoped != 1 {
|
||||
t.Errorf("bundle has %d cluster-scoped objects, want exactly the PriorityClass", clusterScoped)
|
||||
}
|
||||
if deployments != 3 || cronJobs != 1 {
|
||||
t.Errorf("scanned %d deployments / %d cronjobs, want 3 / 1 — a pod template escaped the class check", deployments, cronJobs)
|
||||
}
|
||||
// The class must outrank the default game-pod priority (0); equality would make
|
||||
// the eviction order nondeterministic between the api and a full game server.
|
||||
if controlPlanePriorityValue <= 0 {
|
||||
t.Errorf("control-plane priority %d must exceed the default 0 game-pod priority", controlPlanePriorityValue)
|
||||
// The name must be the built-in critical class: any custom class is capped at
|
||||
// 1e9 by the API server and would be evictable.
|
||||
if controlPlanePriorityName != "system-cluster-critical" {
|
||||
t.Errorf("control-plane priority class = %q; only the built-in critical classes reach the 2e9 eviction-refusal threshold", controlPlanePriorityName)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user