feat(operator): secure system server workloads
This commit is contained in:
10 files changed
+191
-59
No files matched your search
@@ -30,9 +30,9 @@ func (m *multiFlag) Set(v string) error {
|
|||||||
// multi-document YAML stream on stdout, ready for `kubectl apply -f -`.
|
// multi-document YAML stream on stdout, ready for `kubectl apply -f -`.
|
||||||
//
|
//
|
||||||
// It is a pure renderer: it never contacts a cluster and holds no credentials.
|
// It is a pure renderer: it never contacts a cluster and holds no credentials.
|
||||||
// --velocity-cidr is REQUIRED because the game NetworkPolicy fails closed without
|
// --velocity-cidr records the proxy host addresses allowed by the game NetworkPolicy.
|
||||||
// it; emitting a bundle whose 25565 ingress admitted no one would silently break
|
// Kubernetes permits resident-node traffic regardless, but remote proxy deployments
|
||||||
// the server, so the generator refuses rather than guess.
|
// need an explicit CIDR, so the renderer refuses to guess.
|
||||||
func cmdManifests(args []string, stdout, stderr io.Writer) int {
|
func cmdManifests(args []string, stdout, stderr io.Writer) int {
|
||||||
fs := flag.NewFlagSet("manifests", flag.ContinueOnError)
|
fs := flag.NewFlagSet("manifests", flag.ContinueOnError)
|
||||||
fs.SetOutput(stderr)
|
fs.SetOutput(stderr)
|
||||||
@@ -48,18 +48,18 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
|
|||||||
worldsHostPath := fs.String("worlds-host-path", "", "node directory under which each world PVC is visible as <path>/<pvc>; enables the reaper CronJob (requires --backup-pvc and --archive-local-path)")
|
worldsHostPath := fs.String("worlds-host-path", "", "node directory under which each world PVC is visible as <path>/<pvc>; enables the reaper CronJob (requires --backup-pvc and --archive-local-path)")
|
||||||
archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
|
archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
|
||||||
var velocityCIDRs multiFlag
|
var velocityCIDRs multiFlag
|
||||||
fs.Var(&velocityCIDRs, "velocity-cidr", "CIDR of an off-cluster Velocity proxy host allowed to reach game port 25565 (repeatable, REQUIRED)")
|
fs.Var(&velocityCIDRs, "velocity-cidr", "CIDR of a Velocity proxy host allowed to reach game port 25565 (repeatable, REQUIRED)")
|
||||||
var packageCIDRs multiFlag
|
var packageCIDRs multiFlag
|
||||||
fs.Var(&packageCIDRs, "package-cidr", "CIDR of a package mirror build Pods may reach (repeatable; default none = no internet egress)")
|
fs.Var(&packageCIDRs, "package-cidr", "CIDR of a package mirror build Pods may reach (repeatable; default none = no internet egress)")
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
|
||||||
// --velocity-cidr is mandatory: the game policy is fail-closed, so omitting it
|
// Keep proxy placement explicit. This matters for remote proxies and documents
|
||||||
// would render a server nobody can reach. Fail loudly at generation time.
|
// the expected source even when Velocity runs on the resident node.
|
||||||
if len(velocityCIDRs) == 0 {
|
if len(velocityCIDRs) == 0 {
|
||||||
fmt.Fprintln(stderr, "felis manifests: at least one --velocity-cidr is required "+
|
fmt.Fprintln(stderr, "felis manifests: at least one --velocity-cidr is required "+
|
||||||
"(the game NetworkPolicy fails closed without it; pass the Velocity proxy host CIDR, e.g. --velocity-cidr 10.0.0.5/32)")
|
"(pass the Velocity proxy host CIDR, e.g. --velocity-cidr 10.0.0.5/32)")
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ const (
|
|||||||
LabelManagedBy = GroupName + "/managed-by"
|
LabelManagedBy = GroupName + "/managed-by"
|
||||||
// LabelComponent distinguishes the workload role (server, rcon, ...).
|
// LabelComponent distinguishes the workload role (server, rcon, ...).
|
||||||
LabelComponent = GroupName + "/component"
|
LabelComponent = GroupName + "/component"
|
||||||
|
// LabelSystemRole identifies setup-owned system servers. Its value is the
|
||||||
|
// reserved role name (for example, "login" or "lobby").
|
||||||
|
LabelSystemRole = GroupName + "/system-role"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DesiredState is the operator-facing intent toggle (spec §4 spec.desiredState).
|
// DesiredState is the operator-facing intent toggle (spec §4 spec.desiredState).
|
||||||
@@ -121,7 +124,7 @@ type MinecraftServerSpec struct {
|
|||||||
// Jar is the server jar path/name inside the image, if the entrypoint
|
// Jar is the server jar path/name inside the image, if the entrypoint
|
||||||
// needs it explicitly.
|
// needs it explicitly.
|
||||||
Jar string `json:"jar,omitempty"`
|
Jar string `json:"jar,omitempty"`
|
||||||
// JavaMemory is the heap sizing passed as -Xmx/-Xms (e.g. "4G").
|
// JavaMemory is the maximum heap sizing passed as -Xmx (e.g. "4G").
|
||||||
JavaMemory string `json:"javaMemory,omitempty"`
|
JavaMemory string `json:"javaMemory,omitempty"`
|
||||||
// JavaFlags are additional JVM flags (e.g. Aikar's flags).
|
// JavaFlags are additional JVM flags (e.g. Aikar's flags).
|
||||||
JavaFlags []string `json:"javaFlags,omitempty"`
|
JavaFlags []string `json:"javaFlags,omitempty"`
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ var (
|
|||||||
// reserved subdomains/server names that users may not claim: proxy/lobby and
|
// reserved subdomains/server names that users may not claim: proxy/lobby and
|
||||||
// the platform's own faces.
|
// the platform's own faces.
|
||||||
var reserved = map[string]struct{}{
|
var reserved = map[string]struct{}{
|
||||||
|
"login": {},
|
||||||
"lobby": {},
|
"lobby": {},
|
||||||
"admin": {},
|
"admin": {},
|
||||||
"panel": {},
|
"panel": {},
|
||||||
@@ -56,6 +57,31 @@ const (
|
|||||||
ServiceTokenSecretKey = "token"
|
ServiceTokenSecretKey = "token"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ForwardingSecretName / ForwardingSecretKey name the Velocity modern player-info
|
||||||
|
// forwarding secret — the shared HMAC key the proxy signs each login handshake with
|
||||||
|
// and every backend verifies. It is what makes a backend's idea of "who is this
|
||||||
|
// player" trustworthy: with modern forwarding on, the UUID arrives inside the signed
|
||||||
|
// forwarding payload rather than being derived offline from the username, which is
|
||||||
|
// the whole basis of the Owner bind (the Owner IS a Minecraft account, claimed by
|
||||||
|
// joining the login gate). Legacy/BungeeCord forwarding carries no secret at all and
|
||||||
|
// fails OPEN — anyone who can reach a backend directly can assert any UUID — so Felis
|
||||||
|
// mandates modern (spec §20).
|
||||||
|
//
|
||||||
|
// Unlike the service token this is NOT login-only: Velocity's forwarding mode is a
|
||||||
|
// single proxy-wide setting, so once it is "modern" EVERY backend must speak it or it
|
||||||
|
// rejects the proxy's logins outright. The secret authenticates the PROXY to the
|
||||||
|
// backend; every backend verifies it before accepting the forwarded identity. The
|
||||||
|
// NetworkPolicy narrows game-port reachability to declared Velocity CIDRs for non-node
|
||||||
|
// traffic, but Kubernetes always permits traffic from a pod's resident node, so the
|
||||||
|
// policy is defense in depth and never replaces HMAC verification.
|
||||||
|
//
|
||||||
|
// Provisioned out-of-band (deploy/bootstrap.sh, the same run that writes Velocity's
|
||||||
|
// forwarding.secret) and replicated into the minecraft namespace by `felis setup`.
|
||||||
|
const (
|
||||||
|
ForwardingSecretName = "felis-forwarding-secret"
|
||||||
|
ForwardingSecretKey = "secret"
|
||||||
|
)
|
||||||
|
|
||||||
// ValidateServerName checks the §22 name rule and reservation list.
|
// ValidateServerName checks the §22 name rule and reservation list.
|
||||||
func ValidateServerName(name string) error {
|
func ValidateServerName(name string) error {
|
||||||
if !serverNameRE.MatchString(name) {
|
if !serverNameRE.MatchString(name) {
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ func TestValidateServerName(t *testing.T) {
|
|||||||
{"has space", false}, // illegal char
|
{"has space", false}, // illegal char
|
||||||
{"-leading", false}, // leading hyphen
|
{"-leading", false}, // leading hyphen
|
||||||
{"trailing-", false}, // trailing hyphen
|
{"trailing-", false}, // trailing hyphen
|
||||||
|
{"login", false}, // reserved system server
|
||||||
{"lobby", false}, // reserved
|
{"lobby", false}, // reserved
|
||||||
{"admin", false}, // reserved
|
{"admin", false}, // reserved
|
||||||
{"api", false}, // reserved
|
{"api", false}, // reserved
|
||||||
@@ -45,9 +46,9 @@ func TestValidateSystemServerName(t *testing.T) {
|
|||||||
name string
|
name string
|
||||||
ok bool
|
ok bool
|
||||||
}{
|
}{
|
||||||
{"login", true}, // reserved, but a legal system service
|
{"login", true}, // reserved, but a legal system service
|
||||||
{"lobby", true}, // reserved, but a legal system service
|
{"lobby", true}, // reserved, but a legal system service
|
||||||
{"admin", true}, // reserved names are allowed on this path
|
{"admin", true}, // reserved names are allowed on this path
|
||||||
{"survival", true},
|
{"survival", true},
|
||||||
{"ab", false}, // still too short
|
{"ab", false}, // still too short
|
||||||
{"Login", false}, // still case-sensitive
|
{"Login", false}, // still case-sensitive
|
||||||
@@ -64,13 +65,15 @@ func TestValidateSystemServerName(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The two paths must genuinely differ on reserved names: user path refuses
|
// The two paths must genuinely differ on system names: the user path
|
||||||
// "lobby", system path accepts it.
|
// refuses them while the system path accepts them.
|
||||||
if naming.ValidateServerName("lobby") == nil {
|
for _, name := range []string{"login", "lobby"} {
|
||||||
t.Error("ValidateServerName(lobby) accepted; reserved name must be refused for users")
|
if naming.ValidateServerName(name) == nil {
|
||||||
}
|
t.Errorf("ValidateServerName(%s) accepted; reserved name must be refused for users", name)
|
||||||
if naming.ValidateSystemServerName("lobby") != nil {
|
}
|
||||||
t.Error("ValidateSystemServerName(lobby) refused; system path must accept it")
|
if naming.ValidateSystemServerName(name) != nil {
|
||||||
|
t.Errorf("ValidateSystemServerName(%s) refused; system path must accept it", name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,11 @@ import (
|
|||||||
// system server (see buildEnv), sourced from a Secret, never a literal.
|
// system server (see buildEnv), sourced from a Secret, never a literal.
|
||||||
const envServiceToken = "FELIS_SERVICE_TOKEN"
|
const envServiceToken = "FELIS_SERVICE_TOKEN"
|
||||||
|
|
||||||
|
// envForwardingSecret is the environment variable a backend reads the Velocity
|
||||||
|
// modern-forwarding secret from. Unlike the service token it goes to EVERY backend
|
||||||
|
// (see buildEnv), because Velocity's forwarding mode is proxy-wide.
|
||||||
|
const envForwardingSecret = "FELIS_FORWARDING_SECRET"
|
||||||
|
|
||||||
// Workload constants shared by the builders.
|
// Workload constants shared by the builders.
|
||||||
const (
|
const (
|
||||||
// GamePort is the Minecraft TCP port the proxy and readiness probe target.
|
// GamePort is the Minecraft TCP port the proxy and readiness probe target.
|
||||||
@@ -86,11 +91,6 @@ func rconAddress(server *v1alpha1.MinecraftServer) string {
|
|||||||
return fmt.Sprintf("%s.%s.svc.cluster.local:%d", server.Name, server.Namespace, rconPort(server))
|
return fmt.Sprintf("%s.%s.svc.cluster.local:%d", server.Name, server.Namespace, rconPort(server))
|
||||||
}
|
}
|
||||||
|
|
||||||
// gameAddress is the in-cluster game endpoint advertised when Running.
|
|
||||||
func gameAddress(server *v1alpha1.MinecraftServer) string {
|
|
||||||
return fmt.Sprintf("%s.%s.svc.cluster.local:%d", server.Name, server.Namespace, GamePort)
|
|
||||||
}
|
|
||||||
|
|
||||||
// preStopScript is the operator-injected graceful-shutdown sequence (spec §7):
|
// preStopScript is the operator-injected graceful-shutdown sequence (spec §7):
|
||||||
// flush the world, then stop the server, both over RCON. It relies on rcon-cli
|
// flush the world, then stop the server, both over RCON. It relies on rcon-cli
|
||||||
// being present in the Felis base image and reading the RCON_* env injected
|
// being present in the Felis base image and reading the RCON_* env injected
|
||||||
@@ -309,12 +309,13 @@ func buildEnv(server *v1alpha1.MinecraftServer) []corev1.EnvVar {
|
|||||||
// link status), so it — and only it — receives the service token. Injected
|
// link status), so it — and only it — receives the service token. Injected
|
||||||
// from a Secret in this namespace, never inlined into the CRD (the same
|
// from a Secret in this namespace, never inlined into the CRD (the same
|
||||||
// discipline as RCON_PASSWORD above; the CRD's EnvVar type has no valueFrom
|
// discipline as RCON_PASSWORD above; the CRD's EnvVar type has no valueFrom
|
||||||
// precisely so a user server cannot mount an arbitrary secret). Keyed off the
|
// precisely so a user server cannot mount an arbitrary secret). Require both
|
||||||
// reserved "login" name, which naming.ValidateServerName forbids any user
|
// the reserved name and the setup-owned system-role label: the label prevents
|
||||||
// server from claiming — so this can never leak the token into a user's pod.
|
// a legacy user server named "login" from receiving the token after upgrade.
|
||||||
// The Secret must exist in this (minecraft) namespace; `felis setup` replicates
|
// The Secret must exist in this (minecraft) namespace; `felis setup` replicates
|
||||||
// it there from the control namespace before creating this server.
|
// it there from the control namespace before creating this server.
|
||||||
if server.Name == naming.SystemLoginServer {
|
if server.Name == naming.SystemLoginServer &&
|
||||||
|
server.Labels[v1alpha1.LabelSystemRole] == naming.SystemLoginServer {
|
||||||
env = append(env, corev1.EnvVar{
|
env = append(env, corev1.EnvVar{
|
||||||
Name: envServiceToken,
|
Name: envServiceToken,
|
||||||
ValueFrom: &corev1.EnvVarSource{
|
ValueFrom: &corev1.EnvVarSource{
|
||||||
@@ -325,6 +326,30 @@ func buildEnv(server *v1alpha1.MinecraftServer) []corev1.EnvVar {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
// The Velocity modern-forwarding secret goes to EVERY backend, system and user
|
||||||
|
// alike — not because user servers are trusted, but because Velocity's forwarding
|
||||||
|
// mode is one proxy-wide setting: with it on, a backend that cannot verify the
|
||||||
|
// signed handshake rejects every login the proxy sends it. Withholding the secret
|
||||||
|
// from user servers would not harden them, it would simply make them unjoinable.
|
||||||
|
// It is the backend's proof that a login really came from the proxy (and so that
|
||||||
|
// the player's UUID is Mojang-verified, not offline-derived) — the pod-level fence
|
||||||
|
// against bypassing the proxy is the NetworkPolicy, not this value's secrecy.
|
||||||
|
//
|
||||||
|
// A user server is built from an operator-typed Dockerfile, so Felis cannot make it
|
||||||
|
// consume this; the two images Felis does build (deploy/limbo, deploy/lobby) read it
|
||||||
|
// in their entrypoints and refuse to start without it. Optional so a cluster whose
|
||||||
|
// proxy is not in modern mode — no Secret provisioned — still schedules its pods
|
||||||
|
// instead of wedging them all in CreateContainerConfigError.
|
||||||
|
env = append(env, corev1.EnvVar{
|
||||||
|
Name: envForwardingSecret,
|
||||||
|
ValueFrom: &corev1.EnvVarSource{
|
||||||
|
SecretKeyRef: &corev1.SecretKeySelector{
|
||||||
|
LocalObjectReference: corev1.LocalObjectReference{Name: naming.ForwardingSecretName},
|
||||||
|
Key: naming.ForwardingSecretKey,
|
||||||
|
Optional: boolPtr(true),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
return env
|
return env
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -83,6 +83,7 @@ func TestBuildStatefulSetAddsHealthPort(t *testing.T) {
|
|||||||
func TestBuildEnvInjectsServiceTokenForLogin(t *testing.T) {
|
func TestBuildEnvInjectsServiceTokenForLogin(t *testing.T) {
|
||||||
s := &v1alpha1.MinecraftServer{}
|
s := &v1alpha1.MinecraftServer{}
|
||||||
s.Name = naming.SystemLoginServer
|
s.Name = naming.SystemLoginServer
|
||||||
|
s.Labels = map[string]string{v1alpha1.LabelSystemRole: naming.SystemLoginServer}
|
||||||
tok := findEnv(buildEnv(s), envServiceToken)
|
tok := findEnv(buildEnv(s), envServiceToken)
|
||||||
if tok == nil {
|
if tok == nil {
|
||||||
t.Fatalf("%s not injected for the login server", envServiceToken)
|
t.Fatalf("%s not injected for the login server", envServiceToken)
|
||||||
@@ -99,15 +100,60 @@ func TestBuildEnvInjectsServiceTokenForLogin(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A user server (any non-login name) must NOT receive the service token — the
|
// A user server must NOT receive the service token. This includes a legacy,
|
||||||
// reserved-name gate is what stops the internal credential leaking into a player's
|
// unlabeled server named "login" that predates the reserved-name rule.
|
||||||
// pod. naming.ValidateServerName forbids users from ever claiming "login".
|
|
||||||
func TestBuildEnvWithholdsServiceTokenFromUserServers(t *testing.T) {
|
func TestBuildEnvWithholdsServiceTokenFromUserServers(t *testing.T) {
|
||||||
for _, name := range []string{"survival", "creative", naming.SystemLobbyServer} {
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
labels map[string]string
|
||||||
|
}{
|
||||||
|
{name: "survival"},
|
||||||
|
{name: "creative"},
|
||||||
|
{name: naming.SystemLobbyServer, labels: map[string]string{v1alpha1.LabelSystemRole: naming.SystemLobbyServer}},
|
||||||
|
{name: naming.SystemLoginServer},
|
||||||
|
{name: naming.SystemLoginServer, labels: map[string]string{v1alpha1.LabelSystemRole: naming.SystemLobbyServer}},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
s := &v1alpha1.MinecraftServer{}
|
s := &v1alpha1.MinecraftServer{}
|
||||||
s.Name = name
|
s.Name = tt.name
|
||||||
|
s.Labels = tt.labels
|
||||||
if tok := findEnv(buildEnv(s), envServiceToken); tok != nil {
|
if tok := findEnv(buildEnv(s), envServiceToken); tok != nil {
|
||||||
t.Errorf("%s: service token leaked into a non-login server", name)
|
t.Errorf("%s labels=%v: service token leaked into a non-system login server", tt.name, tt.labels)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every backend receives the Velocity modern-forwarding secret — system and user alike.
|
||||||
|
// This is the opposite rule from the service token, and deliberately so: Velocity's
|
||||||
|
// forwarding mode is proxy-wide, so a backend without the secret cannot verify the
|
||||||
|
// signed handshake and rejects every login the proxy sends it. It is also what makes a
|
||||||
|
// backend's view of a player's UUID trustworthy (Mojang-verified via the signed payload,
|
||||||
|
// not offline-derived from the username) — the premise the Owner bind rests on.
|
||||||
|
// Sourced from a Secret, never a literal, and optional so a cluster whose proxy is not
|
||||||
|
// in modern mode still schedules its pods.
|
||||||
|
func TestBuildEnvInjectsForwardingSecretIntoEveryBackend(t *testing.T) {
|
||||||
|
for _, name := range []string{naming.SystemLoginServer, naming.SystemLobbyServer, "survival"} {
|
||||||
|
s := &v1alpha1.MinecraftServer{}
|
||||||
|
s.Name = name
|
||||||
|
fwd := findEnv(buildEnv(s), envForwardingSecret)
|
||||||
|
if fwd == nil {
|
||||||
|
t.Errorf("%s: %s not injected — the backend would reject every proxied login", name, envForwardingSecret)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if fwd.Value != "" {
|
||||||
|
t.Errorf("%s: %s carries a literal value %q — it must be a secretKeyRef", name, envForwardingSecret, fwd.Value)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if fwd.ValueFrom == nil || fwd.ValueFrom.SecretKeyRef == nil {
|
||||||
|
t.Errorf("%s: %s must be sourced from a secretKeyRef", name, envForwardingSecret)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ref := fwd.ValueFrom.SecretKeyRef
|
||||||
|
if ref.Name != naming.ForwardingSecretName || ref.Key != naming.ForwardingSecretKey {
|
||||||
|
t.Errorf("%s: secretKeyRef = %s/%s, want %s/%s", name, ref.Name, ref.Key, naming.ForwardingSecretName, naming.ForwardingSecretKey)
|
||||||
|
}
|
||||||
|
if ref.Optional == nil || !*ref.Optional {
|
||||||
|
t.Errorf("%s: secretKeyRef must be optional, or a cluster without the Secret wedges every pod in CreateContainerConfigError", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -79,7 +79,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.MinecraftServer) (ctrl.Result, error) {
|
func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.MinecraftServer) (ctrl.Result, error) {
|
||||||
if err := r.ensureServices(ctx, server); err != nil {
|
endpointAddress, err := r.ensureServices(ctx, server)
|
||||||
|
if err != nil {
|
||||||
return ctrl.Result{}, err
|
return ctrl.Result{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,6 +113,13 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
|
|||||||
}
|
}
|
||||||
return ctrl.Result{RequeueAfter: requeueStarting}, nil
|
return ctrl.Result{RequeueAfter: requeueStarting}, nil
|
||||||
}
|
}
|
||||||
|
if endpointAddress == "" {
|
||||||
|
r.markStarting(server, "ServiceAddressPending", "waiting for the client Service ClusterIP")
|
||||||
|
if err := r.patchStatus(ctx, server); err != nil {
|
||||||
|
return ctrl.Result{}, err
|
||||||
|
}
|
||||||
|
return ctrl.Result{RequeueAfter: requeueStarting}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Then the operator gates true readiness on an RCON probe (spec §5), which
|
// Then the operator gates true readiness on an RCON probe (spec §5), which
|
||||||
// also samples the current player tally for Status.Players.
|
// also samples the current player tally for Status.Players.
|
||||||
@@ -162,7 +170,7 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
r.markRunningReady(server, players)
|
r.markRunningReady(server, players, endpointAddress)
|
||||||
return ctrl.Result{}, r.patchStatus(ctx, server)
|
return ctrl.Result{}, r.patchStatus(ctx, server)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -198,16 +206,26 @@ func (r *Reconciler) reconcileStopped(ctx context.Context, server *v1alpha1.Mine
|
|||||||
return ctrl.Result{}, r.patchStatus(ctx, server)
|
return ctrl.Result{}, r.patchStatus(ctx, server)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) ensureServices(ctx context.Context, server *v1alpha1.MinecraftServer) error {
|
func (r *Reconciler) ensureServices(ctx context.Context, server *v1alpha1.MinecraftServer) (string, error) {
|
||||||
|
endpointAddress := ""
|
||||||
for _, svc := range []*corev1.Service{buildHeadlessService(server), buildClientService(server)} {
|
for _, svc := range []*corev1.Service{buildHeadlessService(server), buildClientService(server)} {
|
||||||
if err := controllerutil.SetControllerReference(server, svc, r.Scheme); err != nil {
|
if err := controllerutil.SetControllerReference(server, svc, r.Scheme); err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
if err := r.applyService(ctx, svc); err != nil {
|
if err := r.applyService(ctx, svc); err != nil {
|
||||||
return err
|
return "", err
|
||||||
|
}
|
||||||
|
if svc.Name == server.Name {
|
||||||
|
var current corev1.Service
|
||||||
|
if err := r.Get(ctx, client.ObjectKeyFromObject(svc), ¤t); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if current.Spec.ClusterIP != "" && current.Spec.ClusterIP != corev1.ClusterIPNone {
|
||||||
|
endpointAddress = fmt.Sprintf("%s:%d", current.Spec.ClusterIP, GamePort)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return endpointAddress, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) rconPassword(ctx context.Context, server *v1alpha1.MinecraftServer) (string, error) {
|
func (r *Reconciler) rconPassword(ctx context.Context, server *v1alpha1.MinecraftServer) (string, error) {
|
||||||
@@ -287,7 +305,7 @@ func (r *Reconciler) markStarting(server *v1alpha1.MinecraftServer, reason, msg
|
|||||||
r.setCondition(server, v1alpha1.ConditionRconReached, metav1.ConditionFalse, reason, msg)
|
r.setCondition(server, v1alpha1.ConditionRconReached, metav1.ConditionFalse, reason, msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) markRunningReady(server *v1alpha1.MinecraftServer, players PlayerCount) {
|
func (r *Reconciler) markRunningReady(server *v1alpha1.MinecraftServer, players PlayerCount, endpointAddress string) {
|
||||||
server.Status.Phase = v1alpha1.PhaseRunning
|
server.Status.Phase = v1alpha1.PhaseRunning
|
||||||
server.Status.Ready = true
|
server.Status.Ready = true
|
||||||
server.Status.ObservedGeneration = server.Generation
|
server.Status.ObservedGeneration = server.Generation
|
||||||
@@ -305,7 +323,7 @@ func (r *Reconciler) markRunningReady(server *v1alpha1.MinecraftServer, players
|
|||||||
metrics.StartDurationSeconds.Observe(t.Sub(server.Status.StartRequestedAt.Time).Seconds())
|
metrics.StartDurationSeconds.Observe(t.Sub(server.Status.StartRequestedAt.Time).Seconds())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
server.Status.Endpoint = v1alpha1.EndpointStatus{Mode: v1alpha1.EndpointDirect, Address: gameAddress(server)}
|
server.Status.Endpoint = v1alpha1.EndpointStatus{Mode: v1alpha1.EndpointDirect, Address: endpointAddress}
|
||||||
server.Status.LiveMotd = server.Spec.Motd.Running
|
server.Status.LiveMotd = server.Spec.Motd.Running
|
||||||
r.setCondition(server, v1alpha1.ConditionRconReached, metav1.ConditionTrue, "Probed", "RCON probe succeeded")
|
r.setCondition(server, v1alpha1.ConditionRconReached, metav1.ConditionTrue, "Probed", "RCON probe succeeded")
|
||||||
r.setCondition(server, v1alpha1.ConditionReady, metav1.ConditionTrue, "RconReached", "server is accepting RCON")
|
r.setCondition(server, v1alpha1.ConditionReady, metav1.ConditionTrue, "RconReached", "server is accepting RCON")
|
||||||
|
|||||||
@@ -120,6 +120,15 @@ func getSTS(t *testing.T, c client.Client, name string) *appsv1.StatefulSet {
|
|||||||
// markPodReady simulates the kubelet flipping the StatefulSet to ready.
|
// markPodReady simulates the kubelet flipping the StatefulSet to ready.
|
||||||
func markPodReady(t *testing.T, c client.Client, name string) {
|
func markPodReady(t *testing.T, c client.Client, name string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
var svc corev1.Service
|
||||||
|
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: name}, &svc); err != nil {
|
||||||
|
t.Fatalf("get client service: %v", err)
|
||||||
|
}
|
||||||
|
svc.Spec.ClusterIP = "10.43.0.42"
|
||||||
|
svc.Spec.ClusterIPs = []string{"10.43.0.42"}
|
||||||
|
if err := c.Update(context.Background(), &svc); err != nil {
|
||||||
|
t.Fatalf("assign client service ClusterIP: %v", err)
|
||||||
|
}
|
||||||
sts := getSTS(t, c, name)
|
sts := getSTS(t, c, name)
|
||||||
sts.Status.Replicas = 1
|
sts.Status.Replicas = 1
|
||||||
sts.Status.ReadyReplicas = 1
|
sts.Status.ReadyReplicas = 1
|
||||||
@@ -233,6 +242,9 @@ func TestReconcileRunning_RconProbeGatesReadiness(t *testing.T) {
|
|||||||
if server.Status.Endpoint.Mode != v1alpha1.EndpointDirect {
|
if server.Status.Endpoint.Mode != v1alpha1.EndpointDirect {
|
||||||
t.Errorf("endpoint mode = %s, want direct", server.Status.Endpoint.Mode)
|
t.Errorf("endpoint mode = %s, want direct", server.Status.Endpoint.Mode)
|
||||||
}
|
}
|
||||||
|
if server.Status.Endpoint.Address != "10.43.0.42:25565" {
|
||||||
|
t.Errorf("endpoint address = %q, want client Service ClusterIP", server.Status.Endpoint.Address)
|
||||||
|
}
|
||||||
if !isConditionTrue(server, v1alpha1.ConditionReady) {
|
if !isConditionTrue(server, v1alpha1.ConditionReady) {
|
||||||
t.Error("Ready condition should be True")
|
t.Error("Ready condition should be True")
|
||||||
}
|
}
|
||||||
@@ -621,9 +633,9 @@ func TestReconcileRunning_StartDurationObservedOnce(t *testing.T) {
|
|||||||
if err := c.Update(context.Background(), stopped); err != nil {
|
if err := c.Update(context.Background(), stopped); err != nil {
|
||||||
t.Fatalf("set desiredState=Stopped: %v", err)
|
t.Fatalf("set desiredState=Stopped: %v", err)
|
||||||
}
|
}
|
||||||
reconcile(t, r, "survival") // scales spec to 0; pods still terminating
|
reconcile(t, r, "survival") // scales spec to 0; pods still terminating
|
||||||
markPodTerminated(t, c, "survival") // pods finish draining
|
markPodTerminated(t, c, "survival") // pods finish draining
|
||||||
reconcile(t, r, "survival") // reaches Stopped, clears startRequestedAt
|
reconcile(t, r, "survival") // reaches Stopped, clears startRequestedAt
|
||||||
if s := getServer(t, c, "survival"); s.Status.StartRequestedAt != nil {
|
if s := getServer(t, c, "survival"); s.Status.StartRequestedAt != nil {
|
||||||
t.Errorf("startRequestedAt = %v after Stop, want nil so the next start re-anchors", s.Status.StartRequestedAt)
|
t.Errorf("startRequestedAt = %v after Stop, want nil so the next start re-anchors", s.Status.StartRequestedAt)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -77,12 +77,12 @@ type Params struct {
|
|||||||
// BuildNamespace is where image-build Jobs run under the weak felis-build SA,
|
// BuildNamespace is where image-build Jobs run under the weak felis-build SA,
|
||||||
// with the egress-locked NetworkPolicy.
|
// with the egress-locked NetworkPolicy.
|
||||||
BuildNamespace string
|
BuildNamespace string
|
||||||
// VelocityCIDRs are the off-cluster Velocity proxy source addresses permitted
|
// VelocityCIDRs are the Velocity proxy source addresses permitted to reach
|
||||||
// to reach server game ports (25565). Velocity runs on a separate macvlan host
|
// server game ports (25565) as ipBlock peers. The bootstrap proxy runs on the
|
||||||
// (spec §20), NOT a Kubernetes node, so this is an ipBlock allowlist and can
|
// k3s node; other deployments may use a separate host. Kubernetes always permits
|
||||||
// never be a podSelector. It has NO default: an empty list renders a
|
// resident-node traffic independently of NetworkPolicy, so this list constrains
|
||||||
// fail-closed game policy that admits no one (never an accidental allow-all),
|
// non-node sources. It has NO default, and the `felis manifests` generator
|
||||||
// and the `felis manifests` generator refuses to emit a bundle without it.
|
// refuses to emit a bundle without an explicit proxy placement.
|
||||||
VelocityCIDRs []string
|
VelocityCIDRs []string
|
||||||
// RegistryNamespace / RegistryPort locate the in-cluster image registry the
|
// RegistryNamespace / RegistryPort locate the in-cluster image registry the
|
||||||
// build egress policy may reach (spec §16). RegistryNamespace defaults to the
|
// build egress policy may reach (spec §16). RegistryNamespace defaults to the
|
||||||
|
|||||||
@@ -95,16 +95,15 @@ func allowRConFromControlPlane(p Params) *networkingv1.NetworkPolicy {
|
|||||||
return np
|
return np
|
||||||
}
|
}
|
||||||
|
|
||||||
// allowGameFromVelocity opens 25565 on server pods to the off-cluster Velocity
|
// allowGameFromVelocity opens 25565 on server pods to Velocity proxy host(s) by
|
||||||
// proxy host(s) by ipBlock. Velocity is NOT a K8s pod (spec §20: it runs on a
|
// ipBlock. The supported proxy runs outside the pod network (on the k3s node or a
|
||||||
// separate macvlan host), so the peer can only be an ipBlock — never a
|
// separate host), so the peer is an ipBlock rather than a podSelector. Kubernetes
|
||||||
// podSelector.
|
// always permits resident-node traffic; these rules constrain other sources.
|
||||||
//
|
//
|
||||||
// Fail-closed: with no VelocityCIDRs the policy carries NO ingress rule (deny all
|
// With no VelocityCIDRs the policy carries NO ingress rule, never an empty-From
|
||||||
// game ingress), never an empty-From rule, which K8s would read as allow-all. The
|
// rule (which K8s would read as allow-all). That denies non-node game traffic;
|
||||||
// `felis manifests` generator additionally refuses an empty velocity list, so the
|
// resident-node traffic remains outside NetworkPolicy's blocking capability. The
|
||||||
// rendered bundle is always either correctly scoped or absent — never accidentally
|
// manifest generator still refuses an empty list so remote proxies fail loudly.
|
||||||
// open.
|
|
||||||
func allowGameFromVelocity(p Params) *networkingv1.NetworkPolicy {
|
func allowGameFromVelocity(p Params) *networkingv1.NetworkPolicy {
|
||||||
tcp := corev1.ProtocolTCP
|
tcp := corev1.ProtocolTCP
|
||||||
port := intstr.FromInt32(gamePort)
|
port := intstr.FromInt32(gamePort)
|
||||||
|
|||||||
Reference in new issue
Block a user