From a9b275abbbd691eec9068c4a0137d78d20363cf9 Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Wed, 23 Sep 2026 18:55:23 +0800 Subject: [PATCH] =?UTF-8?q?fix(platform):=20registry=20OOM=20(audit=20#46)?= =?UTF-8?q?=20+=20loopback=20hostPort=20=E2=80=94=20the=20node-side=20pull?= =?UTF-8?q?=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes to the registry Deployment, both prerequisite to GC-durable images: - Dedicated resource template: the control plane's 256Mi memory limit was a live-bite bug (#46) — pushing a 475MB layer OOM-killed the registry mid-upload (dmesg oom-kill, oom_score_adj 989) and the push failed; the same push completes in 2s with 2Gi. Registry limits are now 1 CPU / 2Gi. - The container port carries hostPort 127.0.0.1:5000. Node containerd cannot reach the Service VIP (live stack: "Empty reply"), so the node-side pull path is a registries.yaml mirror rewriting registry..svc:5000 onto http://127.0.0.1:5000, which lands on this hostPort. Loopback-only keeps the plain-HTTP registry off every other interface. Tests pin both: exactly one port with hostIP 127.0.0.1, and a memory limit >= 2Gi (exceeding the control-plane template) with the #46 evidence cited. --- internal/platform/workloads.go | 44 +++++++++++++++++++++++++++-- internal/platform/workloads_test.go | 15 ++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/internal/platform/workloads.go b/internal/platform/workloads.go index 05757f9..fb5b903 100644 --- a/internal/platform/workloads.go +++ b/internal/platform/workloads.go @@ -77,6 +77,13 @@ const ( registryName = "registry" registryDataPath = "/var/lib/registry" registryStorageSize = "10Gi" + // registryLoopbackHost is the interface the registry's hostPort binds. Node-level + // containerd is the only client that needs it: it cannot reach the registry Service + // VIP (the live stack answered "Empty reply" to it), so the node's registries.yaml + // mirror rewrites registry..svc: onto http://127.0.0.1: and the + // pull lands here. Loopback-only is deliberate — the registry serves plain HTTP + // and must never be reachable off the node. + registryLoopbackHost = "127.0.0.1" configVolume = "config" tmpVolume = "tmp" @@ -671,6 +678,13 @@ func controlPlaneDeployment(p Params, sa string, container corev1.Container, vol // target real. The registry never calls the K8s API, so its token auto-mount is // disabled (matching the weak build/restore SA hygiene), and REGISTRY_HTTP_ADDR // pins its listen port to the Service port instead of trusting the image default. +// +// The container port also carries a loopback hostPort (registryLoopbackHost): it is +// the node-side pull path. The node's containerd cannot dial the Service VIP, so +// deploy/bootstrap.sh writes a registries.yaml mirror rewriting +// registry..svc: onto http://127.0.0.1:, and that request +// arrives at this hostPort — which is what lets kubelet re-pull a garbage-collected +// platform image without an operator re-import. func registryDeployment(p Params) *appsv1.Deployment { p = p.withDefaults() labels := registryLabels() @@ -682,7 +696,12 @@ func registryDeployment(p Params) *appsv1.Deployment { {Name: "REGISTRY_HTTP_ADDR", Value: fmt.Sprintf(":%d", p.RegistryPort)}, }, Ports: []corev1.ContainerPort{ - {Name: registryName, ContainerPort: p.RegistryPort, Protocol: corev1.ProtocolTCP}, + { + Name: registryName, ContainerPort: p.RegistryPort, Protocol: corev1.ProtocolTCP, + // The node-side pull path: containerd's mirror rewrites the Service + // name onto 127.0.0.1: (see the doc comment above). + HostPort: p.RegistryPort, HostIP: registryLoopbackHost, + }, }, VolumeMounts: []corev1.VolumeMount{ {Name: registryVolume, MountPath: registryDataPath}, @@ -710,7 +729,7 @@ func registryDeployment(p Params) *appsv1.Deployment { TimeoutSeconds: 3, FailureThreshold: 3, }, - Resources: controlPlaneResources(), + Resources: registryResources(), SecurityContext: hardenedContainerSecurityContext(), } @@ -854,6 +873,27 @@ func controlPlaneResources() corev1.ResourceRequirements { } } +// registryResources sizes the registry for what it actually does: it is the pull +// source for every image the node runs and the push target of every build, so its +// limits are not the control plane's. The memory limit is LOAD-BEARING, not +// tuning: a live drill pushing a 475MB layer into a 256Mi registry (the old +// control-plane template) OOM-killed the registry mid-upload — dmesg oom-kill, +// oom_score_adj 989, the push failed — and the identical push completed in +// 2 seconds once the limit was 2Gi. Large layers (Kaniko-built modpacks, the game +// images) are exactly that case, so 2Gi is the floor this renderer stands on. +func registryResources() corev1.ResourceRequirements { + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + } +} + // hardenedPodSecurityContext is the pod-level hardening shared by the // control-plane workloads (api, operator, registry — the world-touching reaper // uses reaperPodSecurityContext instead): run as a fixed non-root uid/gid with a diff --git a/internal/platform/workloads_test.go b/internal/platform/workloads_test.go index 40ad8e0..b3056ab 100644 --- a/internal/platform/workloads_test.go +++ b/internal/platform/workloads_test.go @@ -425,6 +425,21 @@ func TestRegistry_DeploymentServicePVC(t *testing.T) { if v := envValue(c.Env, "REGISTRY_HTTP_ADDR"); v != ":5000" { t.Errorf("REGISTRY_HTTP_ADDR = %q, want :5000", v) } + // The node-side pull path: exactly one container port, mirrored by a LOOPBACK + // hostPort. Node containerd cannot dial the Service VIP, so its registries.yaml + // mirror rewrites the Service name onto 127.0.0.1:; nothing else may be + // exposed (the registry serves plain HTTP). + if len(c.Ports) != 1 { + t.Fatalf("registry container ports = %+v, want exactly 1", c.Ports) + } + if p0 := c.Ports[0]; p0.ContainerPort != p.RegistryPort || p0.HostPort != p.RegistryPort || p0.HostIP != registryLoopbackHost { + t.Errorf("registry port = %+v, want container/host port %d bound to %s", p0, p.RegistryPort, registryLoopbackHost) + } + // The registry's limits are deliberately NOT the control-plane template's: audit + // #46 caught the registry OOM-killed mid-upload at 256Mi on a real 475MB-layer push. + if mem := c.Resources.Limits[corev1.ResourceMemory]; mem.Value() < 2*1024*1024*1024 { + t.Errorf("registry memory limit = %s, want >= 2Gi (audit #46: 256Mi OOM-killed on a 475MB-layer push)", mem.String()) + } // Registry never calls the K8s API ⇒ no auto-mounted token. if ps.AutomountServiceAccountToken == nil || *ps.AutomountServiceAccountToken { t.Error("registry pod must set automountServiceAccountToken=false")