fix(platform): give the control plane a PriorityClass eviction shield (#8)
A full disk made kubelet's node-pressure eviction pick control-plane pods alongside game pods (both priority 0), and with the images existing only in the node's containerd (air-gapped), losing the api meant a manual image re-import. Every control-plane pod template (api/operator/reaper/ registry) now names the bundle's cluster-scoped felis-control-plane PriorityClass: value 1,000,000, preemptionPolicy Never — eviction order only, never preempting a running game server. The image-GC half is not code-fixable on an air-gapped box; troubleshooting gains 13b with the recovery path (re-run the installer to rebuild imports, or docker save | k3s ctr images import - for one image).
This commit is contained in:
5 files changed
+138
-7
No files matched your search
@@ -34,7 +34,9 @@ type Object interface {
|
||||
// Deployments and the in-cluster registry (Deployment + Service + PVC), which
|
||||
// make the SAs and NetworkPolicy peers refer to something real, plus the
|
||||
// world-archive PVC that backs backup/restore when a backup PVC is named (see
|
||||
// workloads.go).
|
||||
// workloads.go). Workloads also carries the one cluster-scoped object, the
|
||||
// felis-control-plane PriorityClass (node-pressure eviction shield — a
|
||||
// PriorityClass is not namespaced by design); everything else is namespaced.
|
||||
// The reaper CronJob is also part of Workloads, rendered only when the retention
|
||||
// storage topology is supplied (WorldsHostPath + BackupPVC + ArchiveLocalPath —
|
||||
// workloads.go documents the gate and the shape-asserted hostPath caveat). The
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
schedulingv1 "k8s.io/api/scheduling/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
@@ -176,11 +177,14 @@ func InternalAPIBaseURL(controlNamespace string) string {
|
||||
// felis-operator Deployment, and the in-cluster registry (Deployment + Service +
|
||||
// PVC), the world-archive PVC when p.BackupPVC names it (it backs the
|
||||
// backup/restore Jobs and the reaper), plus the reaper CronJob when
|
||||
// reaperEnabled(p). FelisImage is required — `felis manifests` enforces it
|
||||
// (fail-loud), so a rendered bundle always names a concrete image.
|
||||
// reaperEnabled(p). Every pod template carries the felis-control-plane
|
||||
// PriorityClass (controlPlanePriorityClass below), the node-pressure eviction
|
||||
// shield. FelisImage is required — `felis manifests` enforces it (fail-loud), so
|
||||
// a rendered bundle always names a concrete image.
|
||||
func Workloads(p Params) []Object {
|
||||
p = p.withDefaults()
|
||||
objs := []Object{
|
||||
controlPlanePriorityClass(),
|
||||
APIDeployment(p),
|
||||
apiService(p),
|
||||
apiInternalService(p),
|
||||
@@ -199,6 +203,37 @@ func Workloads(p Params) []Object {
|
||||
return objs
|
||||
}
|
||||
|
||||
// controlPlanePriorityName is the PriorityClass every control-plane workload runs
|
||||
// under (api, operator, reaper, registry). Node-pressure eviction (a full disk
|
||||
// being the realistic case on a game box) removes pods in ASCENDING priority, and
|
||||
// a classless pod is priority 0 — the same as the game servers, which are exactly
|
||||
// the pods a burst of joins has just filled the node with. A full disk then takes
|
||||
// the api/operator down too, and recovery needs a human: with no reachable
|
||||
// registry on an air-gapped box the images are gone, so the fix is a re-run of the
|
||||
// installer to re-import them. The class is an eviction shield, not a preemption
|
||||
// lever: PreemptionPolicy=Never, so a busy node never loses a running game server
|
||||
// merely to schedule the api. Value 1,000,000 sits above every game pod (0) and
|
||||
// far below the kubelet's system-reserved classes (2,000,000,000).
|
||||
const (
|
||||
controlPlanePriorityName = "felis-control-plane"
|
||||
controlPlanePriorityValue int32 = 1_000_000
|
||||
controlPlanePriorityReason = "Felis control plane (api/operator/reaper/registry) survives node-pressure eviction before game servers"
|
||||
)
|
||||
|
||||
// controlPlanePriorityClass renders the cluster-scoped PriorityClass the
|
||||
// control-plane pod templates reference (the ONE cluster-scoped object in the
|
||||
// bundle; a PriorityClass is not namespaced by design).
|
||||
func controlPlanePriorityClass() *schedulingv1.PriorityClass {
|
||||
never := corev1.PreemptNever
|
||||
return &schedulingv1.PriorityClass{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "scheduling.k8s.io/v1", Kind: "PriorityClass"},
|
||||
ObjectMeta: metav1.ObjectMeta{Name: controlPlanePriorityName},
|
||||
Value: controlPlanePriorityValue,
|
||||
PreemptionPolicy: &never,
|
||||
Description: controlPlanePriorityReason,
|
||||
}
|
||||
}
|
||||
|
||||
// reaperEnabled reports whether the retention CronJob should render. It needs all
|
||||
// three storage coordinates: WorldsHostPath (where worlds live, mounted to read
|
||||
// them), BackupPVC (where archives are written) and ArchiveLocalPath (the mount
|
||||
@@ -509,6 +544,7 @@ func reaperCronJob(p Params) *batchv1.CronJob {
|
||||
ObjectMeta: metav1.ObjectMeta{Labels: labels},
|
||||
Spec: corev1.PodSpec{
|
||||
ServiceAccountName: SAReaper,
|
||||
PriorityClassName: controlPlanePriorityName,
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
SecurityContext: hardenedPodSecurityContext(),
|
||||
Containers: []corev1.Container{container},
|
||||
@@ -544,6 +580,7 @@ func controlPlaneDeployment(p Params, sa string, container corev1.Container, vol
|
||||
ObjectMeta: metav1.ObjectMeta{Labels: labels},
|
||||
Spec: corev1.PodSpec{
|
||||
ServiceAccountName: sa,
|
||||
PriorityClassName: controlPlanePriorityName,
|
||||
SecurityContext: hardenedPodSecurityContext(),
|
||||
Containers: []corev1.Container{container},
|
||||
Volumes: volumes,
|
||||
@@ -591,6 +628,7 @@ func registryDeployment(p Params) *appsv1.Deployment {
|
||||
ObjectMeta: metav1.ObjectMeta{Labels: labels},
|
||||
Spec: corev1.PodSpec{
|
||||
AutomountServiceAccountToken: boolPtr(false),
|
||||
PriorityClassName: controlPlanePriorityName,
|
||||
SecurityContext: hardenedPodSecurityContext(),
|
||||
Containers: []corev1.Container{container},
|
||||
Volumes: []corev1.Volume{
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
schedulingv1 "k8s.io/api/scheduling/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
)
|
||||
@@ -459,8 +460,8 @@ func TestRegistry_DeploymentServicePVC(t *testing.T) {
|
||||
// internal Service must be present or the login pod's felis-api:8081 path is dead.
|
||||
func TestWorkloads_BundleContents(t *testing.T) {
|
||||
objs := Workloads(testParams())
|
||||
if len(objs) != 8 {
|
||||
t.Fatalf("Workloads returned %d objects, want 8", len(objs))
|
||||
if len(objs) != 9 {
|
||||
t.Fatalf("Workloads returned %d objects, want 9", len(objs))
|
||||
}
|
||||
var haveInternalSvc bool
|
||||
for _, o := range objs {
|
||||
@@ -477,6 +478,59 @@ func TestWorkloads_BundleContents(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestControlPlanePriorityClass pins the node-pressure eviction shield: every
|
||||
// control-plane pod template (api/operator/reaper/registry) references the one
|
||||
// PriorityClass, which outranks the default-0 game pods by eviction order while
|
||||
// never preempting them, and is the bundle's only cluster-scoped object. Without
|
||||
// this, a full disk evicts the api alongside the game pods and (no reachable
|
||||
// registry on an air-gapped box) recovery needs a human re-importing images.
|
||||
func TestControlPlanePriorityClass(t *testing.T) {
|
||||
objs := Workloads(reaperParams()) // include the reaper CronJob's pod template
|
||||
|
||||
var class *schedulingv1.PriorityClass
|
||||
clusterScoped := 0
|
||||
deployments, cronJobs := 0, 0
|
||||
for _, o := range objs {
|
||||
if o.GetNamespace() == "" {
|
||||
clusterScoped++
|
||||
}
|
||||
switch v := o.(type) {
|
||||
case *schedulingv1.PriorityClass:
|
||||
class = v
|
||||
case *appsv1.Deployment:
|
||||
deployments++
|
||||
if v.Spec.Template.Spec.PriorityClassName != controlPlanePriorityName {
|
||||
t.Errorf("deployment %s pod priority = %q, want %q", v.Name, v.Spec.Template.Spec.PriorityClassName, controlPlanePriorityName)
|
||||
}
|
||||
case *batchv1.CronJob:
|
||||
cronJobs++
|
||||
if v.Spec.JobTemplate.Spec.Template.Spec.PriorityClassName != controlPlanePriorityName {
|
||||
t.Errorf("CronJob %s pod priority = %q, want %q", v.Name, v.Spec.JobTemplate.Spec.Template.Spec.PriorityClassName, controlPlanePriorityName)
|
||||
}
|
||||
}
|
||||
}
|
||||
if class == nil {
|
||||
t.Fatal("Workloads must render the control-plane PriorityClass")
|
||||
}
|
||||
if class.Name != controlPlanePriorityName || class.Value != controlPlanePriorityValue {
|
||||
t.Errorf("class = %s/%d, want %s/%d", class.Name, class.Value, controlPlanePriorityName, controlPlanePriorityValue)
|
||||
}
|
||||
if class.PreemptionPolicy == nil || *class.PreemptionPolicy != corev1.PreemptNever {
|
||||
t.Errorf("class preemptionPolicy = %v, want Never (an eviction shield, never a lever against running game servers)", class.PreemptionPolicy)
|
||||
}
|
||||
if clusterScoped != 1 {
|
||||
t.Errorf("bundle has %d cluster-scoped objects, want exactly the PriorityClass", clusterScoped)
|
||||
}
|
||||
if deployments != 3 || cronJobs != 1 {
|
||||
t.Errorf("scanned %d deployments / %d cronjobs, want 3 / 1 — a pod template escaped the class check", deployments, cronJobs)
|
||||
}
|
||||
// The class must outrank the default game-pod priority (0); equality would make
|
||||
// the eviction order nondeterministic between the api and a full game server.
|
||||
if controlPlanePriorityValue <= 0 {
|
||||
t.Errorf("control-plane priority %d must exceed the default 0 game-pod priority", controlPlanePriorityValue)
|
||||
}
|
||||
}
|
||||
|
||||
// cronPodSpec returns the single container and pod template of a CronJob's Job
|
||||
// template, failing if the shape is not a single-container pod (the reaper's shape).
|
||||
func cronPodSpec(t *testing.T, cj *batchv1.CronJob) (corev1.PodSpec, corev1.Container) {
|
||||
|
||||
Reference in new issue
Block a user