feat(distributed): 支持单主控多节点部署和停服迁移

复用现有 k3s 调度和 Job 生命周期,增加 worker 接入与批准、受保护节点身份、归档传输、持久迁移锁及活动 PVC 切换;同步管理员 API、CLI、面板和隔离规则。分布式模式默认关闭,保持单机兼容。

验证:Go 全量测试与 vet;面板 874 个测试、lint/build;Linux VM 安装器测试、清单服务端 dry-run、网络命名空间防火墙实测。A/B/C 三机 WireGuard、Velocity 和迁移验收仍待完成。
This commit is contained in:
Lemon-miaow committed 2026-10-01 19:48:37 +08:00
1 parent 26e817f2c9
commit adf563ffe3
77 files changed
+5224 -73

No files matched your search

+5
View File
@@ -72,6 +72,11 @@ func Objects(p Params) []Object {
objs = append(objs, rb)
}
if p.Distributed {
objs = append(objs, DistributedRBAC(p)...)
objs = append(objs, ArchiveNetworkPolicies(p)...)
}
// Weak Job SAs. They come from the build/restore packages (single source of
// truth for AutomountServiceAccountToken=false), which set ObjectMeta but not
// TypeMeta — stamp it so the YAML header is present. The restore Job runs in
+82
View File
@@ -0,0 +1,82 @@
package platform
import (
"fmt"
"felis.lolicon.best/internal/archivetransfer"
"felis.lolicon.best/internal/distributed"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const ArchiveName = "felis-archive"
const ArchiveSecret = "felis-archive-key"
const ArchivePort int32 = 8090
func archiveKeyEnv() corev1.EnvVar {
return corev1.EnvVar{Name: archivetransfer.KeyEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: ArchiveSecret}, Key: "key"}}}
}
func distributedEnv(p Params) []corev1.EnvVar {
return []corev1.EnvVar{{Name: "FELIS_DISTRIBUTED", Value: "true"}, {Name: "FELIS_CONTROLLER_NODE", Value: p.ControllerNode}, {Name: "FELIS_ARCHIVE_URL", Value: fmt.Sprintf("http://%s.%s.svc:%d", ArchiveName, p.MinecraftNamespace, ArchivePort)}, {Name: "FELIS_EGRESS_PROBE", Value: p.EgressProbe}, archiveKeyEnv()}
}
func ArchiveDeployment(p Params) *appsv1.Deployment {
labels := map[string]string{"app.kubernetes.io/name": ArchiveName}
container := corev1.Container{Name: ArchiveName, Image: p.FelisImage, Command: []string{felisBinaryPath, "archive-serve"}, Args: []string{"--root", p.ArchiveLocalPath}, Env: []corev1.EnvVar{archiveKeyEnv()}, Ports: []corev1.ContainerPort{{Name: "archive", ContainerPort: ArchivePort}}, Resources: controlPlaneResources(), SecurityContext: &corev1.SecurityContext{RunAsUser: int64Ptr(0), RunAsNonRoot: boolPtr(false), AllowPrivilegeEscalation: boolPtr(false), Privileged: boolPtr(false), ReadOnlyRootFilesystem: boolPtr(true), Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, VolumeMounts: []corev1.VolumeMount{{Name: "archives", MountPath: p.ArchiveLocalPath}}}
container.ReadinessProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromString("archive")}}}
container.LivenessProbe = container.ReadinessProbe.DeepCopy()
return &appsv1.Deployment{TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, ObjectMeta: metav1.ObjectMeta{Name: ArchiveName, Namespace: p.MinecraftNamespace, Labels: labels}, Spec: appsv1.DeploymentSpec{Replicas: int32Ptr(1), Strategy: appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType}, Selector: &metav1.LabelSelector{MatchLabels: labels}, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{NodeSelector: controllerSelector(p), AutomountServiceAccountToken: boolPtr(false), SecurityContext: &corev1.PodSecurityContext{SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{container}, Volumes: []corev1.Volume{{Name: "archives", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: p.BackupPVC}}}}}}}}
}
func ArchiveService(p Params) *corev1.Service {
return &corev1.Service{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, ObjectMeta: metav1.ObjectMeta{Name: ArchiveName, Namespace: p.MinecraftNamespace}, Spec: corev1.ServiceSpec{Selector: map[string]string{"app.kubernetes.io/name": ArchiveName}, Ports: []corev1.ServicePort{{Name: "archive", Port: ArchivePort, TargetPort: intstr.FromInt32(ArchivePort)}}}}
}
// Cluster grants are read-only and bound only to A's API/operator/reaper identities.
func DistributedRBAC(p Params) []Object {
var out []Object
for _, entry := range []struct {
name, ns string
rules []rbacv1.PolicyRule
}{
{SAAPI, p.ControlNamespace, []rbacv1.PolicyRule{rule([]string{groupCore}, []string{"nodes"}, []string{"get", "list"}), rule([]string{groupCore}, []string{"persistentvolumes"}, []string{"get"})}},
{SAOperator, p.ControlNamespace, []rbacv1.PolicyRule{rule([]string{groupCore}, []string{"nodes"}, []string{"get"})}},
{SAReaper, p.MinecraftNamespace, []rbacv1.PolicyRule{rule([]string{groupCore}, []string{"nodes"}, []string{"get"})}},
} {
name := entry.name + "-" + p.MinecraftNamespace + "-nodes"
out = append(out, &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole"}, ObjectMeta: metav1.ObjectMeta{Name: name}, Rules: entry.rules}, &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRoleBinding"}, ObjectMeta: metav1.ObjectMeta{Name: name}, RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: name}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: entry.name, Namespace: entry.ns}}})
}
return out
}
func ArchiveNetworkPolicies(p Params) []Object {
tcp, udp := corev1.ProtocolTCP, corev1.ProtocolUDP
archive := metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": ArchiveName}}
job := metav1.LabelSelector{MatchLabels: map[string]string{distributed.LabelTransfer: "true"}}
control := networkingv1.NetworkPolicyPeer{NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": p.ControlNamespace}}, PodSelector: &metav1.LabelSelector{MatchLabels: controlPlanePodLabels(ComponentAPI)}}
reaper := networkingv1.NetworkPolicyPeer{PodSelector: &metav1.LabelSelector{MatchLabels: controlPlanePodLabels(ComponentReaper)}}
peers := []networkingv1.NetworkPolicyPeer{control, reaper, {PodSelector: &job}}
ingress := &networkingv1.NetworkPolicy{TypeMeta: metav1.TypeMeta{APIVersion: "networking.k8s.io/v1", Kind: "NetworkPolicy"}, ObjectMeta: metav1.ObjectMeta{Name: "felis-archive", Namespace: p.MinecraftNamespace}, Spec: networkingv1.NetworkPolicySpec{PodSelector: archive, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress}, Ingress: []networkingv1.NetworkPolicyIngressRule{{From: peers, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: portPtr(ArchivePort)}}}}}}
egress := &networkingv1.NetworkPolicy{TypeMeta: metav1.TypeMeta{APIVersion: "networking.k8s.io/v1", Kind: "NetworkPolicy"}, ObjectMeta: metav1.ObjectMeta{Name: "felis-archive-jobs", Namespace: p.MinecraftNamespace}, Spec: networkingv1.NetworkPolicySpec{PodSelector: job, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, Egress: []networkingv1.NetworkPolicyEgressRule{
{To: []networkingv1.NetworkPolicyPeer{{PodSelector: &archive}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: portPtr(ArchivePort)}}},
{To: []networkingv1.NetworkPolicyPeer{{NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"}}, PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"k8s-app": "kube-dns"}}}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &udp, Port: portPtr(53)}, {Protocol: &tcp, Port: portPtr(53)}}},
}}}
// Files/export maintenance Pods may send results only to A's existing upload receiver.
// Transfer Pods are excluded so grants remain limited to the archive service.
maintenance := &networkingv1.NetworkPolicy{TypeMeta: metav1.TypeMeta{APIVersion: "networking.k8s.io/v1", Kind: "NetworkPolicy"}, ObjectMeta: metav1.ObjectMeta{Name: "felis-maintenance-egress", Namespace: p.MinecraftNamespace}, Spec: networkingv1.NetworkPolicySpec{
PodSelector: metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
{Key: "app.kubernetes.io/managed-by", Operator: metav1.LabelSelectorOpIn, Values: []string{"felis-files", "felis-export", "felis-restore", "felis-backup"}},
{Key: distributed.LabelTransfer, Operator: metav1.LabelSelectorOpDoesNotExist},
}}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress},
Egress: []networkingv1.NetworkPolicyEgressRule{
{To: []networkingv1.NetworkPolicyPeer{control}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: portPtr(8081)}}},
egress.Spec.Egress[1],
},
}}
return []Object{ingress, egress, maintenance}
}
func portPtr(p int32) *intstr.IntOrString { v := intstr.FromInt32(p); return &v }
+61
View File
@@ -0,0 +1,61 @@
package platform
import (
"testing"
"felis.lolicon.best/internal/archivetransfer"
"felis.lolicon.best/internal/placement"
appsv1 "k8s.io/api/apps/v1"
rbacv1 "k8s.io/api/rbac/v1"
)
func TestDistributedControllerPinningAndArchivePowers(t *testing.T) {
p := testParams()
p.Distributed = true
p.ControllerNode = "a"
p.EgressProbe = "felis-api.felis.svc:443"
p.BackupPVC = "felis-backups"
p.ArchiveLocalPath = "/backups"
p.VelocityCIDRs = []string{"192.0.2.1/32"}
if err := p.Validate(); err != nil {
t.Fatal(err)
}
found := false
for _, obj := range Objects(p) {
if d, ok := obj.(*appsv1.Deployment); ok {
if d.Spec.Template.Spec.NodeSelector[placement.LabelIdentity] != "a" {
t.Fatalf("controller workload %s may run on worker", d.Name)
}
if d.Name == ArchiveName {
found = true
s := d.Spec.Template.Spec
if s.AutomountServiceAccountToken == nil || *s.AutomountServiceAccountToken || s.ServiceAccountName != "" || len(s.Volumes) != 1 || s.Volumes[0].PersistentVolumeClaim.ClaimName != p.BackupPVC {
t.Fatal("archive has extra powers")
}
for _, c := range s.Containers {
for _, e := range c.Env {
if e.Name != archivetransfer.KeyEnv {
t.Fatal("archive received controller config")
}
}
}
}
}
if r, ok := obj.(*rbacv1.ClusterRole); ok {
for _, rule := range r.Rules {
for _, verb := range rule.Verbs {
if verb != "get" && verb != "list" {
t.Fatal("distributed cluster grant writes", r.Name, verb)
}
}
}
}
}
if !found {
t.Fatal("archive service missing")
}
p.VelocityCIDRs = []string{"10.0.0.0/8"}
if p.Validate() == nil {
t.Fatal("broad Velocity source accepted")
}
}
+24
View File
@@ -2,6 +2,7 @@ package platform
import (
"fmt"
"net"
"k8s.io/apimachinery/pkg/api/resource"
)
@@ -89,6 +90,10 @@ const (
// Params parameterises the install bundle. Namespaces and the registry location
// have safe defaults; VelocityCIDRs has none — see the field comment.
type Params struct {
Distributed bool
ControllerNode string
EgressProbe string
RegistryNodeCIDRs []string
// ControlNamespace is where felis-api/operator run; their SAs live here and the
// RoleBindings' subjects reference them here, even though the Roles they bind to
// live in the minecraft (and build) namespaces. The reaper alone runs — CronJob
@@ -212,6 +217,25 @@ type Params struct {
// Validate reports a Params the renderer cannot turn into objects.
func (p Params) Validate() error {
if p.Distributed && (p.ControllerNode == "" || p.EgressProbe == "" || p.BackupPVC == "" || p.ArchiveLocalPath == "") {
return fmt.Errorf("distributed mode requires controller node, egress probe, backup PVC and archive path")
}
if p.Distributed && len(p.ServerEgressAllowCIDRs) > 0 {
return fmt.Errorf("distributed mode does not permit private game egress exceptions")
}
if p.Distributed {
for _, cidr := range append(append([]string{}, p.VelocityCIDRs...), p.RegistryNodeCIDRs...) {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
return fmt.Errorf("invalid node source %q", cidr)
}
ones, bits := network.Mask.Size()
if ones != bits {
return fmt.Errorf("distributed node sources must be exact /32 or /128 addresses: %q", cidr)
}
}
}
for _, q := range []struct{ name, v string }{
{"registry storage", p.RegistryStorage},
{"uploads storage", p.UploadsStorage},
+3
View File
@@ -266,6 +266,9 @@ func RegistryIngressPolicy(p Params) *networkingv1.NetworkPolicy {
Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &port}},
}},
)
for _, cidr := range p.RegistryNodeCIDRs {
np.Spec.Ingress[0].From = append(np.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
}
return np
}
+1
View File
@@ -206,6 +206,7 @@ func postgresDeployment(p Params) *appsv1.Deployment {
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels},
Spec: corev1.PodSpec{
NodeSelector: controllerSelector(p),
AutomountServiceAccountToken: boolPtr(false),
EnableServiceLinks: boolPtr(false),
PriorityClassName: controlPlanePriorityName,
+13 -5
View File
@@ -19,7 +19,7 @@ var groupFelis = v1alpha1.GroupName // "felis.lolicon.best"
// RBAC is the control-plane authorization bundle: one SA per identity and the
// namespaced Roles + RoleBindings that grant each exactly the verbs its code path
// exercises. There is deliberately no ClusterRole or ClusterRoleBinding anywhere.
// exercises. DistributedRBAC adds read-only node/PV grants when distributed mode is enabled.
type RBAC struct {
ServiceAccounts []*corev1.ServiceAccount
Roles []*rbacv1.Role
@@ -100,7 +100,7 @@ func ControlPlaneRBAC(p Params) RBAC {
// than its logs).
func APIMinecraftRole(p Params) *rbacv1.Role {
p = p.withDefaults()
return role(p.MinecraftNamespace, "felis-api", ComponentAPI, []rbacv1.PolicyRule{
rules := []rbacv1.PolicyRule{
rule([]string{groupFelis}, []string{"minecraftservers"}, []string{"get", "list", "watch", "create", "patch"}),
rule([]string{groupCore}, []string{"secrets"}, []string{"get"}),
// get-only: WorldVolumeExists does a single direct Get of the world PVC;
@@ -116,7 +116,11 @@ func APIMinecraftRole(p Params) *rbacv1.Role {
// the verbs stay tight — list on pods, get on pods/log, and nothing else.
rule([]string{groupCore}, []string{"pods"}, []string{"list"}),
rule([]string{groupCore}, []string{"pods/log"}, []string{"get"}),
})
}
if p.Distributed {
rules = append(rules, rule([]string{groupCore}, []string{"persistentvolumeclaims"}, []string{"create", "list"}), rule([]string{groupApps}, []string{"statefulsets"}, []string{"get", "delete"}))
}
return role(p.MinecraftNamespace, "felis-api", ComponentAPI, rules)
}
// APIBuildRole grants felis-api the build-Job lifecycle in the build namespace
@@ -218,12 +222,16 @@ func OperatorRole(p Params) *rbacv1.Role {
// request and never deletes a CR itself.
func ReaperRole(p Params) *rbacv1.Role {
p = p.withDefaults()
return role(p.MinecraftNamespace, "felis-reaper", ComponentReaper, []rbacv1.PolicyRule{
rules := []rbacv1.PolicyRule{
rule([]string{groupFelis}, []string{"minecraftservers"}, []string{"get", "patch", "delete"}),
rule([]string{groupCore}, []string{"persistentvolumeclaims"}, []string{"get", "delete"}),
rule([]string{groupCore}, []string{"pods"}, []string{"list"}),
rule([]string{groupBatch}, []string{"jobs"}, []string{"list"}),
})
}
if p.Distributed {
rules = append(rules, rule([]string{groupBatch}, []string{"jobs"}, []string{"create", "get"}), rule([]string{groupCore}, []string{"persistentvolumeclaims"}, []string{"list"}))
}
return role(p.MinecraftNamespace, "felis-reaper", ComponentReaper, rules)
}
// controlPlaneServiceAccount renders a control-plane SA. Unlike the weak
+32 -3
View File
@@ -7,6 +7,7 @@ import (
"time"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/placement"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
@@ -273,11 +274,16 @@ func Workloads(p Params) []Object {
objs = append(objs, backupPVC(p))
}
switch {
case p.Distributed && retentionEnabled(p):
objs = append(objs, reaperCronJob(p))
case reaperEnabled(p):
objs = append(objs, worldsRootPV(p), worldsRootPVC(p), reaperCronJob(p))
case retentionEnabled(p):
objs = append(objs, reaperCronJob(p))
}
if p.Distributed {
objs = append(objs, ArchiveDeployment(p), ArchiveService(p))
}
return objs
}
@@ -315,7 +321,7 @@ const controlPlanePriorityName = "system-cluster-critical"
// together so a partial configuration fails loudly rather than silently dropping
// retention here.
func reaperEnabled(p Params) bool {
return p.WorldsHostPath != "" && retentionEnabled(p)
return (p.WorldsHostPath != "" || p.Distributed) && retentionEnabled(p)
}
// retentionEnabled reports whether the archive store can be looked after: the
@@ -716,6 +722,11 @@ func reaperCronJob(p Params) *batchv1.CronJob {
})
}
if p.Distributed {
container.Args = []string{"--config", configFilePath}
container.Env = append(container.Env, distributedEnv(p)...)
container.Env = append(container.Env, corev1.EnvVar{Name: "FELIS_IMAGE", Value: p.FelisImage})
}
return &batchv1.CronJob{
TypeMeta: metav1.TypeMeta{APIVersion: "batch/v1", Kind: "CronJob"},
// The CronJob lives in the MINECRAFT namespace: a Pod can only mount PVCs
@@ -844,10 +855,12 @@ func reaperPodSpec(p Params, container corev1.Container, volumes []corev1.Volume
Containers: []corev1.Container{container},
Volumes: volumes,
}
if p.ReaperNode != "" {
if p.ControllerNode != "" {
spec.NodeSelector = controllerSelector(p)
} else if p.ReaperNode != "" {
spec.NodeSelector = map[string]string{"kubernetes.io/hostname": p.ReaperNode}
}
if p.WorldsHostPath != "" {
if p.WorldsHostPath != "" || p.Distributed {
spec.ServiceAccountName = SAReaper
} else {
spec.ServiceAccountName = "default"
@@ -868,6 +881,13 @@ func reaperPodSpec(p Params, container corev1.Container, volumes []corev1.Volume
// Recreate guarantees the old pod is gone before the new one starts.
func controlPlaneDeployment(p Params, sa string, container corev1.Container, volumes []corev1.Volume) *appsv1.Deployment {
labels := controlPlanePodLabels(container.Name)
if p.Distributed {
if sa == SAOperator {
container.Env = append(container.Env, corev1.EnvVar{Name: "FELIS_DISTRIBUTED", Value: "true"}, corev1.EnvVar{Name: "FELIS_CONTROLLER_NODE", Value: p.ControllerNode}, corev1.EnvVar{Name: "FELIS_EGRESS_PROBE", Value: p.EgressProbe})
} else {
container.Env = append(container.Env, distributedEnv(p)...)
}
}
return &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"},
ObjectMeta: metav1.ObjectMeta{Name: sa, Namespace: p.ControlNamespace, Labels: labels},
@@ -878,6 +898,7 @@ func controlPlaneDeployment(p Params, sa string, container corev1.Container, vol
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels},
Spec: corev1.PodSpec{
NodeSelector: controllerSelector(p),
ServiceAccountName: sa,
PriorityClassName: controlPlanePriorityName,
SecurityContext: hardenedPodSecurityContext(),
@@ -1003,6 +1024,7 @@ func registryDeployment(p Params) *appsv1.Deployment {
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels},
Spec: corev1.PodSpec{
NodeSelector: controllerSelector(p),
AutomountServiceAccountToken: boolPtr(false),
PriorityClassName: controlPlanePriorityName,
SecurityContext: hardenedPodSecurityContext(),
@@ -1373,3 +1395,10 @@ func hardenedContainerSecurityContext() *corev1.SecurityContext {
func boolPtr(b bool) *bool { return &b }
func int32Ptr(i int32) *int32 { return &i }
func int64Ptr(i int64) *int64 { return &i }
func controllerSelector(p Params) map[string]string {
if p.ControllerNode == "" {
return nil
}
return map[string]string{placement.LabelIdentity: p.ControllerNode, placement.LabelRole: placement.RoleController}
}