fix(platform): give the control plane a PriorityClass eviction shield (#8)

A full disk made kubelet's node-pressure eviction pick control-plane pods
alongside game pods (both priority 0), and with the images existing only
in the node's containerd (air-gapped), losing the api meant a manual
image re-import. Every control-plane pod template (api/operator/reaper/
registry) now names the bundle's cluster-scoped felis-control-plane
PriorityClass: value 1,000,000, preemptionPolicy Never — eviction order
only, never preempting a running game server. The image-GC half is not
code-fixable on an air-gapped box; troubleshooting gains 13b with the
recovery path (re-run the installer to rebuild imports, or docker save |
k3s ctr images import - for one image).
This commit is contained in:
Lemon-miaow committed 2026-09-22 21:08:56 +08:00
1 parent 2b87a5a13b
commit fe310743a2
5 files changed
+138 -7

No files matched your search

+3 -1
View File
@@ -34,7 +34,9 @@ type Object interface {
// Deployments and the in-cluster registry (Deployment + Service + PVC), which
// make the SAs and NetworkPolicy peers refer to something real, plus the
// world-archive PVC that backs backup/restore when a backup PVC is named (see
// workloads.go).
// workloads.go). Workloads also carries the one cluster-scoped object, the
// felis-control-plane PriorityClass (node-pressure eviction shield — a
// PriorityClass is not namespaced by design); everything else is namespaced.
// The reaper CronJob is also part of Workloads, rendered only when the retention
// storage topology is supplied (WorldsHostPath + BackupPVC + ArchiveLocalPath —
// workloads.go documents the gate and the shape-asserted hostPath caveat). The
+40 -2
View File
@@ -7,6 +7,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
schedulingv1 "k8s.io/api/scheduling/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
@@ -176,11 +177,14 @@ func InternalAPIBaseURL(controlNamespace string) string {
// felis-operator Deployment, and the in-cluster registry (Deployment + Service +
// PVC), the world-archive PVC when p.BackupPVC names it (it backs the
// backup/restore Jobs and the reaper), plus the reaper CronJob when
// reaperEnabled(p). FelisImage is required — `felis manifests` enforces it
// (fail-loud), so a rendered bundle always names a concrete image.
// reaperEnabled(p). Every pod template carries the felis-control-plane
// PriorityClass (controlPlanePriorityClass below), the node-pressure eviction
// shield. FelisImage is required — `felis manifests` enforces it (fail-loud), so
// a rendered bundle always names a concrete image.
func Workloads(p Params) []Object {
p = p.withDefaults()
objs := []Object{
controlPlanePriorityClass(),
APIDeployment(p),
apiService(p),
apiInternalService(p),
@@ -199,6 +203,37 @@ func Workloads(p Params) []Object {
return objs
}
// controlPlanePriorityName is the PriorityClass every control-plane workload runs
// under (api, operator, reaper, registry). Node-pressure eviction (a full disk
// being the realistic case on a game box) removes pods in ASCENDING priority, and
// a classless pod is priority 0 — the same as the game servers, which are exactly
// the pods a burst of joins has just filled the node with. A full disk then takes
// the api/operator down too, and recovery needs a human: with no reachable
// registry on an air-gapped box the images are gone, so the fix is a re-run of the
// installer to re-import them. The class is an eviction shield, not a preemption
// lever: PreemptionPolicy=Never, so a busy node never loses a running game server
// merely to schedule the api. Value 1,000,000 sits above every game pod (0) and
// far below the kubelet's system-reserved classes (2,000,000,000).
const (
controlPlanePriorityName = "felis-control-plane"
controlPlanePriorityValue int32 = 1_000_000
controlPlanePriorityReason = "Felis control plane (api/operator/reaper/registry) survives node-pressure eviction before game servers"
)
// controlPlanePriorityClass renders the cluster-scoped PriorityClass the
// control-plane pod templates reference (the ONE cluster-scoped object in the
// bundle; a PriorityClass is not namespaced by design).
func controlPlanePriorityClass() *schedulingv1.PriorityClass {
never := corev1.PreemptNever
return &schedulingv1.PriorityClass{
TypeMeta: metav1.TypeMeta{APIVersion: "scheduling.k8s.io/v1", Kind: "PriorityClass"},
ObjectMeta: metav1.ObjectMeta{Name: controlPlanePriorityName},
Value: controlPlanePriorityValue,
PreemptionPolicy: &never,
Description: controlPlanePriorityReason,
}
}
// reaperEnabled reports whether the retention CronJob should render. It needs all
// three storage coordinates: WorldsHostPath (where worlds live, mounted to read
// them), BackupPVC (where archives are written) and ArchiveLocalPath (the mount
@@ -509,6 +544,7 @@ func reaperCronJob(p Params) *batchv1.CronJob {
ObjectMeta: metav1.ObjectMeta{Labels: labels},
Spec: corev1.PodSpec{
ServiceAccountName: SAReaper,
PriorityClassName: controlPlanePriorityName,
RestartPolicy: corev1.RestartPolicyNever,
SecurityContext: hardenedPodSecurityContext(),
Containers: []corev1.Container{container},
@@ -544,6 +580,7 @@ func controlPlaneDeployment(p Params, sa string, container corev1.Container, vol
ObjectMeta: metav1.ObjectMeta{Labels: labels},
Spec: corev1.PodSpec{
ServiceAccountName: sa,
PriorityClassName: controlPlanePriorityName,
SecurityContext: hardenedPodSecurityContext(),
Containers: []corev1.Container{container},
Volumes: volumes,
@@ -591,6 +628,7 @@ func registryDeployment(p Params) *appsv1.Deployment {
ObjectMeta: metav1.ObjectMeta{Labels: labels},
Spec: corev1.PodSpec{
AutomountServiceAccountToken: boolPtr(false),
PriorityClassName: controlPlanePriorityName,
SecurityContext: hardenedPodSecurityContext(),
Containers: []corev1.Container{container},
Volumes: []corev1.Volume{
+56 -2
View File
@@ -6,6 +6,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
schedulingv1 "k8s.io/api/scheduling/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)
@@ -459,8 +460,8 @@ func TestRegistry_DeploymentServicePVC(t *testing.T) {
// internal Service must be present or the login pod's felis-api:8081 path is dead.
func TestWorkloads_BundleContents(t *testing.T) {
objs := Workloads(testParams())
if len(objs) != 8 {
t.Fatalf("Workloads returned %d objects, want 8", len(objs))
if len(objs) != 9 {
t.Fatalf("Workloads returned %d objects, want 9", len(objs))
}
var haveInternalSvc bool
for _, o := range objs {
@@ -477,6 +478,59 @@ func TestWorkloads_BundleContents(t *testing.T) {
}
}
// TestControlPlanePriorityClass pins the node-pressure eviction shield: every
// control-plane pod template (api/operator/reaper/registry) references the one
// PriorityClass, which outranks the default-0 game pods by eviction order while
// never preempting them, and is the bundle's only cluster-scoped object. Without
// this, a full disk evicts the api alongside the game pods and (no reachable
// registry on an air-gapped box) recovery needs a human re-importing images.
func TestControlPlanePriorityClass(t *testing.T) {
objs := Workloads(reaperParams()) // include the reaper CronJob's pod template
var class *schedulingv1.PriorityClass
clusterScoped := 0
deployments, cronJobs := 0, 0
for _, o := range objs {
if o.GetNamespace() == "" {
clusterScoped++
}
switch v := o.(type) {
case *schedulingv1.PriorityClass:
class = v
case *appsv1.Deployment:
deployments++
if v.Spec.Template.Spec.PriorityClassName != controlPlanePriorityName {
t.Errorf("deployment %s pod priority = %q, want %q", v.Name, v.Spec.Template.Spec.PriorityClassName, controlPlanePriorityName)
}
case *batchv1.CronJob:
cronJobs++
if v.Spec.JobTemplate.Spec.Template.Spec.PriorityClassName != controlPlanePriorityName {
t.Errorf("CronJob %s pod priority = %q, want %q", v.Name, v.Spec.JobTemplate.Spec.Template.Spec.PriorityClassName, controlPlanePriorityName)
}
}
}
if class == nil {
t.Fatal("Workloads must render the control-plane PriorityClass")
}
if class.Name != controlPlanePriorityName || class.Value != controlPlanePriorityValue {
t.Errorf("class = %s/%d, want %s/%d", class.Name, class.Value, controlPlanePriorityName, controlPlanePriorityValue)
}
if class.PreemptionPolicy == nil || *class.PreemptionPolicy != corev1.PreemptNever {
t.Errorf("class preemptionPolicy = %v, want Never (an eviction shield, never a lever against running game servers)", class.PreemptionPolicy)
}
if clusterScoped != 1 {
t.Errorf("bundle has %d cluster-scoped objects, want exactly the PriorityClass", clusterScoped)
}
if deployments != 3 || cronJobs != 1 {
t.Errorf("scanned %d deployments / %d cronjobs, want 3 / 1 — a pod template escaped the class check", deployments, cronJobs)
}
// The class must outrank the default game-pod priority (0); equality would make
// the eviction order nondeterministic between the api and a full game server.
if controlPlanePriorityValue <= 0 {
t.Errorf("control-plane priority %d must exceed the default 0 game-pod priority", controlPlanePriorityValue)
}
}
// cronPodSpec returns the single container and pod template of a CronJob's Job
// template, failing if the shape is not a single-container pod (the reaper's shape).
func cronPodSpec(t *testing.T, cj *batchv1.CronJob) (corev1.PodSpec, corev1.Container) {