Unverified Commit a9b275ab authored by Lemon-miaow's avatar Lemon-miaow
Browse files

fix(platform): registry OOM (audit #46) + loopback hostPort — the node-side pull path 

Two changes to the registry Deployment, both prerequisite to GC-durable images:

- Dedicated resource template: the control plane's 256Mi memory limit was a
  live-bite bug (#46) — pushing a 475MB layer OOM-killed the registry
  mid-upload (dmesg oom-kill, oom_score_adj 989) and the push failed; the
  same push completes in 2s with 2Gi. Registry limits are now 1 CPU / 2Gi.

- The container port carries hostPort 127.0.0.1:5000. Node containerd cannot
  reach the Service VIP (live stack: "Empty reply"), so the node-side pull
  path is a registries.yaml mirror rewriting registry.<ns>.svc:5000 onto
  http://127.0.0.1:5000, which lands on this hostPort. Loopback-only keeps
  the plain-HTTP registry off every other interface.

Tests pin both: exactly one port with hostIP 127.0.0.1, and a memory limit
>= 2Gi (exceeding the control-plane template) with the #46 evidence cited.
parent 92c06ac8

internal/platform/workloads.go

+42 −2
Changes for internal/platform/workloads.go: 42 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -77,6 +77,13 @@ const ( 
	registryName        = "registry"

	registryDataPath    = "/var/lib/registry"

	registryStorageSize = "10Gi"

	// registryLoopbackHost is the interface the registry's hostPort binds. Node-level

	// containerd is the only client that needs it: it cannot reach the registry Service

	// VIP (the live stack answered "Empty reply" to it), so the node's registries.yaml

	// mirror rewrites registry.<ns>.svc:<port> onto http://127.0.0.1:<port> and the

	// pull lands here. Loopback-only is deliberate — the registry serves plain HTTP

	// and must never be reachable off the node.

	registryLoopbackHost = "127.0.0.1"



	configVolume   = "config"

	tmpVolume      = "tmp"
@@ -671,6 +678,13 @@ func controlPlaneDeployment(p Params, sa string, container corev1.Container, vol 
// target real. The registry never calls the K8s API, so its token auto-mount is

// disabled (matching the weak build/restore SA hygiene), and REGISTRY_HTTP_ADDR

// pins its listen port to the Service port instead of trusting the image default.

//

// The container port also carries a loopback hostPort (registryLoopbackHost): it is

// the node-side pull path. The node's containerd cannot dial the Service VIP, so

// deploy/bootstrap.sh writes a registries.yaml mirror rewriting

// registry.<registry-ns>.svc:<port> onto http://127.0.0.1:<port>, and that request

// arrives at this hostPort — which is what lets kubelet re-pull a garbage-collected

// platform image without an operator re-import.

func registryDeployment(p Params) *appsv1.Deployment {

	p = p.withDefaults()

	labels := registryLabels()
@@ -682,7 +696,12 @@ func registryDeployment(p Params) *appsv1.Deployment { 
			{Name: "REGISTRY_HTTP_ADDR", Value: fmt.Sprintf(":%d", p.RegistryPort)},

		},

		Ports: []corev1.ContainerPort{

			{Name: registryName, ContainerPort: p.RegistryPort, Protocol: corev1.ProtocolTCP},

			{

				Name: registryName, ContainerPort: p.RegistryPort, Protocol: corev1.ProtocolTCP,

				// The node-side pull path: containerd's mirror rewrites the Service

				// name onto 127.0.0.1:<port> (see the doc comment above).

				HostPort: p.RegistryPort, HostIP: registryLoopbackHost,

			},

		},

		VolumeMounts: []corev1.VolumeMount{

			{Name: registryVolume, MountPath: registryDataPath},
@@ -710,7 +729,7 @@ func registryDeployment(p Params) *appsv1.Deployment { 
			TimeoutSeconds:      3,

			FailureThreshold:    3,

		},

		Resources:       controlPlaneResources(),

		Resources:       registryResources(),

		SecurityContext: hardenedContainerSecurityContext(),

	}
@@ -854,6 +873,27 @@ func controlPlaneResources() corev1.ResourceRequirements { 
	}

}



// registryResources sizes the registry for what it actually does: it is the pull

// source for every image the node runs and the push target of every build, so its

// limits are not the control plane's. The memory limit is LOAD-BEARING, not

// tuning: a live drill pushing a 475MB layer into a 256Mi registry (the old

// control-plane template) OOM-killed the registry mid-upload — dmesg oom-kill,

// oom_score_adj 989, the push failed — and the identical push completed in

// 2 seconds once the limit was 2Gi. Large layers (Kaniko-built modpacks, the game

// images) are exactly that case, so 2Gi is the floor this renderer stands on.

func registryResources() corev1.ResourceRequirements {

	return corev1.ResourceRequirements{

		Requests: corev1.ResourceList{

			corev1.ResourceCPU:    resource.MustParse("50m"),

			corev1.ResourceMemory: resource.MustParse("64Mi"),

		},

		Limits: corev1.ResourceList{

			corev1.ResourceCPU:    resource.MustParse("1"),

			corev1.ResourceMemory: resource.MustParse("2Gi"),

		},

	}

}



// hardenedPodSecurityContext is the pod-level hardening shared by the

// control-plane workloads (api, operator, registry — the world-touching reaper

// uses reaperPodSecurityContext instead): run as a fixed non-root uid/gid with a

internal/platform/workloads_test.go

+15 −0
Changes for internal/platform/workloads_test.go: 15 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -425,6 +425,21 @@ func TestRegistry_DeploymentServicePVC(t *testing.T) { 
	if v := envValue(c.Env, "REGISTRY_HTTP_ADDR"); v != ":5000" {

		t.Errorf("REGISTRY_HTTP_ADDR = %q, want :5000", v)

	}

	// The node-side pull path: exactly one container port, mirrored by a LOOPBACK

	// hostPort. Node containerd cannot dial the Service VIP, so its registries.yaml

	// mirror rewrites the Service name onto 127.0.0.1:<port>; nothing else may be

	// exposed (the registry serves plain HTTP).

	if len(c.Ports) != 1 {

		t.Fatalf("registry container ports = %+v, want exactly 1", c.Ports)

	}

	if p0 := c.Ports[0]; p0.ContainerPort != p.RegistryPort || p0.HostPort != p.RegistryPort || p0.HostIP != registryLoopbackHost {

		t.Errorf("registry port = %+v, want container/host port %d bound to %s", p0, p.RegistryPort, registryLoopbackHost)

	}

	// The registry's limits are deliberately NOT the control-plane template's: audit

	// #46 caught the registry OOM-killed mid-upload at 256Mi on a real 475MB-layer push.

	if mem := c.Resources.Limits[corev1.ResourceMemory]; mem.Value() < 2*1024*1024*1024 {

		t.Errorf("registry memory limit = %s, want >= 2Gi (audit #46: 256Mi OOM-killed on a 475MB-layer push)", mem.String())

	}

	// Registry never calls the K8s API ⇒ no auto-mounted token.

	if ps.AutomountServiceAccountToken == nil || *ps.AutomountServiceAccountToken {

		t.Error("registry pod must set automountServiceAccountToken=false")