feat(distributed): 支持单主控多节点部署和停服迁移

复用现有 k3s 调度和 Job 生命周期，增加 worker 接入与批准、受保护节点身份、归档传输、持久迁移锁及活动 PVC 切换；同步管理员 API、CLI、面板和隔离规则。分布式模式默认关闭，保持单机兼容。

验证：Go 全量测试与 vet；面板 874 个测试、lint/build；Linux VM 安装器测试、清单服务端 dry-run、网络命名空间防火墙实测。A/B/C 三机 WireGuard、Velocity 和迁移验收仍待完成。

cmd/felis/api.go

@@ -20,6 +20,7 @@ import (
 	"felis.lolicon.best/internal/backupjob"
 	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
 	"felis.lolicon.best/internal/fileedit"
 	"felis.lolicon.best/internal/imagepin"
 	"felis.lolicon.best/internal/mail"
@@ -27,6 +28,7 @@ import (
 	"felis.lolicon.best/internal/naming"
 	"felis.lolicon.best/internal/panel"
 	"felis.lolicon.best/internal/passkey"
+	"felis.lolicon.best/internal/placement"
 	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/reaper"
 	"felis.lolicon.best/internal/registryprune"
@@ -273,9 +275,23 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// store at load, so by this point cfg.Archive.Store is guaranteed tarLocal.)
 	var restorer api.Restorer
 	felisImage, backupPVC := os.Getenv("FELIS_IMAGE"), os.Getenv("FELIS_BACKUP_PVC")
+	worldResolver := placement.Resolve(cl, cfg.K8s.Namespace)
+	distribution, err := distributionManager(cl, cfg, felisImage)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if distribution != nil {
+		worldResolver = distribution.Resolve
+	}
 	if felisImage != "" && backupPVC != "" {
 		rcfg := restoreConfig(cfg, felisImage, backupPVC)
-		restorer = &restore.Restorer{Jobs: restore.NewK8sJobs(cl), Config: rcfg}
+		rcfg.ResolveWorld = worldResolver
+		var jobs restore.Jobs = restore.NewK8sJobs(cl)
+		if distribution != nil {
+			jobs = distribution
+		}
+		restorer = &restore.Restorer{Jobs: jobs, Config: rcfg}
 	} else {
 		fmt.Fprintln(stderr, "felis api: restore executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — restore endpoint returns 503")
 	}
@@ -288,7 +304,13 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// endpoint honestly returns 503.
 	var backuper api.Backuper
 	if felisImage != "" && backupPVC != "" {
-		backuper = &backupjob.Backuper{Jobs: backupjob.NewK8sJobs(cl), Config: backupConfig(cfg, felisImage, backupPVC)}
+		bcfg := backupConfig(cfg, felisImage, backupPVC)
+		bcfg.ResolveWorld = worldResolver
+		var jobs backupjob.Jobs = backupjob.NewK8sJobs(cl)
+		if distribution != nil {
+			jobs = distribution
+		}
+		backuper = &backupjob.Backuper{Jobs: jobs, Config: bcfg}
 	} else {
 		fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503")
 	}
@@ -299,7 +321,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	// is wired under the restore gate; otherwise the export routes return 503.
 	var exporter api.Exporter
 	if felisImage != "" && backupPVC != "" {
-		exporter = worldexport.New(clientset, exportConfig(cfg, felisImage, backupPVC))
+		ecfg := exportConfig(cfg, felisImage, backupPVC)
+		ecfg.ResolveWorld = worldResolver
+		exporter = worldexport.New(clientset, ecfg)
 	} else {
 		fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503")
 	}
@@ -319,9 +343,11 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	var files api.FileEditor
 	var fileStage *fileedit.Stage
 	if felisImage != "" {
+		fcfg := fileEditConfig(cfg, felisImage)
+		fcfg.ResolveWorld = worldResolver
 		files = &fileedit.Editor{
 			Runner: fileedit.NewK8sRunner(clientset),
-			Config: fileEditConfig(cfg, felisImage),
+			Config: fcfg,
 		}
 		fileStage = &fileedit.Stage{Dir: fileStagingDir()}
 		if err := fileStage.Sweep(); err != nil {
@@ -355,7 +381,21 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err)
 		return 1
 	}
-	cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced)
+	cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced).WithDistributed(distribution != nil, os.Getenv("FELIS_CONTROLLER_NODE"))
+	if distribution != nil {
+		distribution.Record = func(ctx context.Context, b distributed.Backup) error {
+			keep, retention, ok := backupPolicy(b.Reason, rcfg)
+			if !ok {
+				return fmt.Errorf("unknown backup reason %s", b.Reason)
+			}
+			st := reaper.NewPGStore(drv.DB())
+			if err := st.InsertBackup(ctx, reaper.BackupRecord{ID: "bk-" + b.ID, ServerName: b.Server, FormerOwner: b.Owner, BackupRef: b.Receipt.Ref, SizeBytes: b.Receipt.Size, SHA256: b.Receipt.SHA256, Reason: b.Reason, ExpiresAt: time.Now().Add(retention)}); err != nil {
+				return err
+			}
+			pruneBackups(ctx, st, distribution.Archive, b.Server, b.Owner, b.Reason, keep, b.Protect, stdout, stderr)
+			return nil
+		}
+	}
 	jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace)
 	a := &api.API{
 		Repo:    repo,
@@ -455,7 +495,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain,
 		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname),
 		defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname),
-		cfg.Velocity.GamePort, resolvedVersion())
+		cfg.Velocity.GamePort, resolvedVersion(), distribution != nil)
 	internalSrv := newAPIServer(*internalAddr, a.InternalHandler())
 	externalSrv := newAPIServer(cfg.Server.Listen, externalHandler)
 
@@ -499,6 +539,10 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 	}
 	go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
 
+	if distribution != nil {
+		a.Distribution = distribution
+		go reconcileDistribution(ctx, distribution, stderr)
+	}
 	servers := []*http.Server{internalSrv, externalSrv}
 	if httpsSrv != nil {
 		servers = append(servers, httpsSrv)

cmd/felis/archive_serve.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	"felis.lolicon.best/internal/archivetransfer"
+)
+
+func cmdArchiveServe(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("archive-serve", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	root := fs.String("root", "/backups", "archive PVC mount (must match archive.local_path)")
+	addr := fs.String("listen", ":8090", "private archive listener")
+	limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum compressed archive bytes")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	key := os.Getenv(archivetransfer.KeyEnv)
+	if len(key) < 32 || *limit <= 0 {
+		fmt.Fprintln(stderr, "archive-serve: signing key and positive size limit required")
+		return 2
+	}
+	if err := os.MkdirAll(*root, 0750); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	limitHeapToCgroup()
+	transport := &archivetransfer.Server{Root: *root, Key: key, Limit: *limit}
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		ticker := time.NewTicker(time.Hour)
+		defer ticker.Stop()
+		for {
+			if err := transport.Sweep(time.Now()); err != nil {
+				fmt.Fprintln(stderr, "archive journal cleanup:", err)
+			}
+			select {
+			case <-done:
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+	srv := &http.Server{Addr: *addr, Handler: transport, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Hour, WriteTimeout: 2 * time.Hour, MaxHeaderBytes: 16 << 10}
+	fmt.Fprintln(stdout, "archive transport listening", *addr)
+	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	return 0
+}

cmd/felis/backup.go

@@ -168,7 +168,9 @@ func backupPolicy(reason string, rcfg reaper.Config) (keep int, retention time.D
 // previous owner's. protect is never removed: it is the backup a chained restore
 // is about to extract. The new backup is already recorded; a removal that fails
 // is reported and retried after the next backup.
-func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) {
+func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver interface {
+	Delete(context.Context, backup.ArchiveRef) error
+}, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) {
 	excess, err := st.ExcessBackups(ctx, server, owner, reason, keep, protect)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)

cmd/felis/distributed.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"felis.lolicon.best/internal/archivetransfer"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/placement"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func distributionManager(cl client.Client, cfg *config.Config, image string) (*distributed.Manager, error) {
+	if os.Getenv("FELIS_DISTRIBUTED") != "true" {
+		return nil, nil
+	}
+	controller, url, key := os.Getenv("FELIS_CONTROLLER_NODE"), os.Getenv("FELIS_ARCHIVE_URL"), os.Getenv(archivetransfer.KeyEnv)
+	if controller == "" || url == "" || len(key) < 32 || image == "" || cfg.Archive.Store != "tarLocal" {
+		return nil, fmt.Errorf("distributed mode requires controller identity, archive service/key, Felis image and tarLocal")
+	}
+	return &distributed.Manager{Client: cl, Namespace: cfg.K8s.Namespace, Image: image, Controller: controller, Archive: archivetransfer.Client{URL: url, Root: cfg.Archive.LocalPath, Key: key}, Resolve: placement.Resolve(cl, cfg.K8s.Namespace, controller)}, nil
+}
+
+func reconcileDistribution(ctx context.Context, m *distributed.Manager, stderr io.Writer) {
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+	for {
+		if err := m.SettleBackups(ctx); err != nil {
+			fmt.Fprintln(stderr, "distributed backup:", err)
+		}
+		if err := m.ReconcileMigrations(ctx); err != nil {
+			fmt.Fprintln(stderr, "migration:", err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}

cmd/felis/egressgate.go

@@ -37,6 +37,7 @@ func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	probe := fs.String("probe", "", "host:port the pod's NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")
+	positive := fs.String("positive-probe", "", "allowed host:port that must remain reachable during denial checks")
 	wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the gate gives up")
 	failOpen := fs.Bool("fail-open", false, "when --wait runs out, warn and let the pod go on instead of refusing it")
 	if err := fs.Parse(args); err != nil {
@@ -53,6 +54,18 @@ func cmdEgressGate(args []string, stdout, stderr io.Writer) int {
 
 	start := time.Now()
 	for {
+		if *positive != "" {
+			allowed, err := net.DialTimeout("tcp", *positive, egressDialTimeout)
+			if err != nil {
+				if time.Since(start) >= *wait {
+					fmt.Fprintln(stderr, "felis egress-gate: positive probe unavailable; refusing to start", err)
+					return 1
+				}
+				time.Sleep(egressPollInterval)
+				continue
+			}
+			allowed.Close()
+		}
 		conn, err := net.DialTimeout("tcp", *probe, egressDialTimeout)
 		if err != nil {
 			fmt.Fprintf(stdout, "felis egress-gate: %s is unreachable after %s (%v); the egress lock is in effect\n",

cmd/felis/egressgate_test.go

@@ -15,6 +15,20 @@ func shrinkEgressGate(t *testing.T) {
 	t.Cleanup(func() { egressDialTimeout, egressPollInterval = dial, poll })
 }
 
+func TestEgressGateRequiresPositiveReachability(t *testing.T) {
+	shrinkEgressGate(t)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	closed := ln.Addr().String()
+	ln.Close()
+	var out, errb bytes.Buffer
+	if code := cmdEgressGate([]string{"--positive-probe", closed, "--probe", closed, "--wait", "20ms"}, &out, &errb); code != 1 {
+		t.Fatal("unavailable probes allowed startup", code)
+	}
+}
+
 // The gate holds while the probe answers and lets the pod go on once the policy
 // lands, which the test plays by closing the listener.
 func TestEgressGateWaitsForTheLock(t *testing.T) {

cmd/felis/export.go

@@ -60,6 +60,7 @@ func cmdExport(args []string, stdout, stderr io.Writer) int {
 	sum := fs.String("sha256", "", "backup only: the sha256 recorded when the archive was written; a mismatch fails the export before its end is sent")
 	worldsRoot := fs.String("worlds-root", "/world", "world and files: mount path of the world PVC")
 	path := fs.String("path", "", "files only: the file or folder to send, relative to the world root")
+	rawArchive := fs.Bool("archive-raw", false, "internal archive transfer: preserve the complete world")
 	dir := fs.Bool("dir", false, "files only: the path is a folder, sent as a zip")
 	if err := fs.Parse(args); err != nil {
 		return 2
@@ -82,7 +83,11 @@ func cmdExport(args []string, stdout, stderr io.Writer) int {
 		}
 		err = exportBackup(ctx, *target, token, *ref, *backupRoot, *sum, stdout)
 	case worldexport.ModeWorld:
-		err = exportWorld(ctx, *target, token, *worldsRoot, stdout)
+		if *rawArchive {
+			err = streamExport(ctx, *target, token, archiveType, -1, func(w io.Writer) error { _, _, err := backup.WriteTarGz(ctx, w, *worldsRoot, nil); return err })
+		} else {
+			err = exportWorld(ctx, *target, token, *worldsRoot, stdout)
+		}
 	case worldexport.ModeFiles:
 		if *path == "" {
 			fmt.Fprintln(stderr, "felis export: --path is required for files")

cmd/felis/export_test.go

@@ -526,7 +526,7 @@ func TestCmdExportWiring(t *testing.T) {
 	cfg := &config.Config{}
 	cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups"
 	want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"}
-	if got := exportConfig(cfg, "felis:1", "felis-backups"); got != want {
+	if got := exportConfig(cfg, "felis:1", "felis-backups"); !reflect.DeepEqual(got, want) {
 		t.Fatalf("exportConfig = %+v, want %+v", got, want)
 	}
 }

cmd/felis/manifests.go

@@ -43,6 +43,11 @@ func (m *multiFlag) Set(v string) error {
 func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("manifests", flag.ContinueOnError)
 	fs.SetOutput(stderr)
+	distributed := fs.Bool("distributed", false, "enable approved workers and archive transport")
+	controller := fs.String("controller-node", "", "protected controller identity for A")
+	probe := fs.String("egress-probe", "", "reachable controller host:port denied to game Pods")
+	var registryNodes multiFlag
+	fs.Var(&registryNodes, "registry-node-cidr", "exact node pull source for the registry (repeatable)")
 	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace the control plane (api/operator/reaper) runs in")
 	minecraftNS := fs.String("minecraft-namespace", platform.DefaultMinecraftNamespace, "namespace MinecraftServer workloads run in")
 	buildNS := fs.String("build-namespace", platform.DefaultBuildNamespace, "namespace image-build Jobs run in")
@@ -75,6 +80,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	case "":
 	case "postgres":
 		return renderManifests(stdout, stderr, platform.PostgresObjects(platform.Params{
+			ControllerNode:     *controller,
 			ControlNamespace:   *controlNS,
 			MinecraftNamespace: *minecraftNS,
 			PostgresImage:      *postgresImage,
@@ -115,7 +121,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	// The node pin exists only for the reaper's hostPath: naming a node without the
 	// worlds root would be silently dropped (no CronJob renders), so fail loud like
 	// the storage-trio check below.
-	if *reaperNode != "" && *worldsHostPath == "" {
+	if *reaperNode != "" && *worldsHostPath == "" && !*distributed {
 		fmt.Fprintln(stderr, "felis manifests: --reaper-node requires --worlds-host-path "+
 			"(it pins the reaper CronJob, which renders only with the retention storage trio)")
 		return 2
@@ -156,7 +162,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 			"writes under /var/lib/rancher/k3s/storage). Any other provisioner needs its volumes exposed as "+
 			"<path>/<pvc>, or each candidate's archive fails and the world is preserved;\n"+
 			"  - %s.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, pin)
-	} else {
+	} else if !*distributed {
 		switch {
 		case *archiveLocalPath != "" && *backupPVC == "":
 			fmt.Fprintln(stderr, "felis manifests: --archive-local-path names where the backup PVC is mounted, "+
@@ -176,6 +182,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	}
 
 	params := platform.Params{
+		Distributed: *distributed, ControllerNode: *controller, EgressProbe: *probe, RegistryNodeCIDRs: registryNodes,
 		ControlNamespace:   *controlNS,
 		MinecraftNamespace: *minecraftNS,
 		BuildNamespace:     *buildNS,

cmd/felis/node_approve.go

@@ -0,0 +1,466 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/archivetransfer"
+	"felis.lolicon.best/internal/operator"
+	"felis.lolicon.best/internal/placement"
+	"felis.lolicon.best/internal/platform"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func approveNode(ctx context.Context, cl client.Client, cs kubernetes.Interface, ns, controlNS, name, image, remote string, stdout io.Writer) error {
+	var n corev1.Node
+	if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil {
+		return err
+	}
+	_, controlPlane := n.Labels["node-role.kubernetes.io/control-plane"]
+	if !placement.Online(&n) || controlPlane || n.Labels[placement.LabelRole] == placement.RoleController {
+		return fmt.Errorf("candidate must be an online agent")
+	}
+	var nodes corev1.NodeList
+	if err := cl.List(ctx, &nodes); err != nil {
+		return err
+	}
+	var controller *corev1.Node
+	var peers []string
+	var denied []string
+	for i := range nodes.Items {
+		node := &nodes.Items[i]
+		if node.Labels[placement.LabelRole] == placement.RoleController {
+			if controller != nil {
+				return fmt.Errorf("multiple controllers found")
+			}
+			controller = node
+		}
+		for _, addr := range node.Status.Addresses {
+			if addr.Type == corev1.NodeInternalIP || addr.Type == corev1.NodeExternalIP {
+				cidr, err := exactCIDR(addr.Address)
+				if err != nil {
+					return err
+				}
+				peers = append(peers, cidr)
+				for _, p := range []string{"6443", "10250", "5000", "30443"} {
+					denied = append(denied, net.JoinHostPort(addr.Address, p))
+				}
+			}
+		}
+	}
+	if controller == nil || controller.Name == n.Name || controller.Status.NodeInfo.Architecture != n.Status.NodeInfo.Architecture {
+		return fmt.Errorf("candidate architecture must match the sole controller")
+	}
+	if n.Status.NodeInfo.KubeletVersion != controller.Status.NodeInfo.KubeletVersion {
+		return fmt.Errorf("candidate k3s version must match A")
+	}
+	var apiSvc, registrySvc, archiveSvc, kubernetesSvc corev1.Service
+	for _, svc := range []struct {
+		obj      *corev1.Service
+		ns, name string
+	}{{&kubernetesSvc, "default", "kubernetes"}, {&apiSvc, controlNS, platform.SAAPI}, {&registrySvc, controlNS, "registry"}, {&archiveSvc, ns, platform.ArchiveName}} {
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: svc.ns, Name: svc.name}, svc.obj); err != nil {
+			return err
+		}
+	}
+	if len(apiSvc.Spec.Ports) == 0 || apiSvc.Spec.Ports[0].NodePort == 0 {
+		return fmt.Errorf("controller API NodePort is absent")
+	}
+	// A's exact interface addresses let the probe observe DNAT/SNAT before the production policy is adjusted.
+	var aCIDRs []string
+	if addrs, err := net.InterfaceAddrs(); err == nil {
+		for _, a := range addrs {
+			ip, _, err := net.ParseCIDR(a.String())
+			if err == nil && !ip.IsLoopback() {
+				cidr, _ := exactCIDR(ip.String())
+				aCIDRs = append(aCIDRs, cidr)
+			}
+		}
+	}
+	if len(aCIDRs) == 0 {
+		return fmt.Errorf("cannot determine A's exact interface addresses")
+	}
+	onController := false
+	for _, addr := range controller.Status.Addresses {
+		cidr, err := exactCIDR(addr.Address)
+		if err != nil {
+			continue
+		}
+		for _, local := range aCIDRs {
+			if cidr == local {
+				onController = true
+			}
+		}
+	}
+	if !onController || !placement.Online(controller) {
+		return fmt.Errorf("run admission on the online controller A")
+	}
+	// Quarantine first. Approval is the final write, after all checks have succeeded.
+	before := n.DeepCopy()
+	if n.Labels == nil {
+		n.Labels = map[string]string{}
+	}
+	delete(n.Labels, placement.LabelApproved)
+	found := false
+	for _, t := range n.Spec.Taints {
+		if t.Key == "felis.lolicon.best/unapproved" {
+			found = true
+		}
+	}
+	if !found {
+		n.Spec.Taints = append(n.Spec.Taints, corev1.Taint{Key: "felis.lolicon.best/unapproved", Value: "true", Effect: corev1.TaintEffectNoSchedule})
+	}
+	if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil {
+		return err
+	}
+	if err := denyNodeAddresses(ctx, cl, ns, nodes.Items); err != nil {
+		return err
+	}
+	// The host's Service dial may be masqueraded to its bridge gateway. Permit exact host addresses only.
+	pullSources := append([]string{}, peers...)
+	if ip, network, err := net.ParseCIDR(n.Spec.PodCIDR); err == nil && ip.To4() != nil {
+		v := append(net.IP(nil), network.IP.To4()...)
+		pullSources = append(pullSources, v.String()+"/32")
+		v[3]++
+		pullSources = append(pullSources, v.String()+"/32")
+	}
+	var registryNP networkingv1.NetworkPolicy
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNS, Name: "felis-registry-ingress"}, &registryNP); err != nil {
+		return err
+	}
+	if len(registryNP.Spec.Ingress) == 0 {
+		return fmt.Errorf("registry ingress policy is not configured")
+	}
+	prev := registryNP.DeepCopy()
+	for _, cidr := range pullSources {
+		registryNP.Spec.Ingress[0].From = append(registryNP.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
+	}
+	if err := cl.Patch(ctx, &registryNP, client.MergeFrom(prev)); err != nil {
+		return err
+	}
+	ctrlIP := ""
+	for _, a := range controller.Status.Addresses {
+		if a.Type == corev1.NodeInternalIP {
+			ctrlIP = a.Address
+			break
+		}
+	}
+	if ctrlIP == "" {
+		return fmt.Errorf("controller has no address")
+	}
+	script := "set -euo pipefail\numask 077\n" +
+		"systemctl is-active --quiet k3s-agent\n! systemctl is-active --quiet k3s\ntest ! -f /etc/rancher/k3s/k3s.yaml\n" +
+		"ip -d link show flannel-wg | grep -q wireguard\n" +
+		"/usr/local/bin/felis node firewall --peers " + shellQuote(strings.Join(peers, ",")) + " --controller-ip " + shellQuote(ctrlIP) + " --api-service-ip " + shellQuote(kubernetesSvc.Spec.ClusterIP) + " --node-port " + strconv.Itoa(int(apiSvc.Spec.Ports[0].NodePort)) + " --namespace " + shellQuote(ns) + " --control-namespace " + shellQuote(controlNS) + "\n" +
+		"kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig\n" +
+		"/usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" get node " + shellQuote(name) + " -o name >/dev/null\n" +
+		"proof=$(mktemp); trap 'rm -f \"$proof\"' EXIT\n" +
+		"if /usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" label node " + shellQuote(name) + " felis.node-restriction.kubernetes.io/probe=controller --overwrite 2>\"$proof\"; then echo 'NodeRestriction failed' >&2;exit 1;fi\ngrep -qi forbidden \"$proof\"\n" +
+		"image=" + shellQuote(image) + "\nif [ -n \"$(/usr/local/bin/k3s crictl images -q \"$image\")\" ]; then /usr/local/bin/k3s crictl rmi \"$image\" >/dev/null; fi\ntest -z \"$(/usr/local/bin/k3s crictl images -q \"$image\")\"\n/usr/local/bin/k3s crictl pull \"$image\" >/dev/null\n"
+	cmd := exec.CommandContext(ctx, "ssh", "--", remote, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi")
+	cmd.Stdin = strings.NewReader(script)
+	cmd.Stdout = stdout
+	cmd.Stderr = stdout
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("worker host, NodeRestriction or uncached registry pull check failed: %w", err)
+	}
+	id := "felis-probe-" + archivetransfer.ID()[:12]
+	var objects []client.Object
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		for i := len(objects) - 1; i >= 0; i-- {
+			cl.Delete(cleanup, objects[i])
+		}
+	}()
+	create := func(o client.Object) error {
+		if err := cl.Create(ctx, o); err != nil {
+			return err
+		}
+		objects = append(objects, o)
+		return nil
+	}
+	var endpoints []string
+	var echoPods []*corev1.Pod
+	for i := range nodes.Items {
+		node := &nodes.Items[i]
+		if !placement.Online(node) || (node.Name != name && node.Name != controller.Name && node.Labels[placement.LabelApproved] != "true") {
+			continue
+		}
+		echoName := fmt.Sprintf("%s-%d", id, i)
+		p := probePod(echoName, ns, node.Name, image, []string{"--listen", ":25565"}, true)
+		p.Labels["felis.lolicon.best/probe-echo"] = id
+		if err := create(p); err != nil {
+			return err
+		}
+		echoPods = append(echoPods, p)
+		svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: echoName, Namespace: ns}, Spec: corev1.ServiceSpec{Selector: map[string]string{"felis.lolicon.best/probe-name": echoName}, Ports: []corev1.ServicePort{{Port: 25565, TargetPort: intstr.FromInt32(25565)}}}}
+		if err := create(svc); err != nil {
+			return err
+		}
+		endpoints = append(endpoints, net.JoinHostPort(svc.Spec.ClusterIP, "25565"))
+	}
+	tcp := corev1.ProtocolTCP
+	port := intstr.FromInt32(25565)
+	policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id, Namespace: ns}, Spec: networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-echo": id}}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}}}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &port}}}}}}
+	for _, cidr := range aCIDRs {
+		policy.Spec.Ingress[0].From = append(policy.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
+	}
+	if err := create(policy); err != nil {
+		return err
+	}
+	for _, p := range echoPods {
+		if err := waitProbePod(ctx, cl, p); err != nil {
+			return err
+		}
+	}
+	open := append([]string{}, endpoints...)
+	open = append(open, net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), net.JoinHostPort(registrySvc.Spec.ClusterIP, "5000"), net.JoinHostPort(archiveSvc.Spec.ClusterIP, "8090"))
+	// Positive probes need temporary, narrowly scoped ingress grants where the
+	// production fence admits only the controller or archive-transfer jobs.
+	for _, svc := range []*corev1.Service{&apiSvc, &registrySvc, &archiveSvc} {
+		if len(svc.Spec.Selector) == 0 || len(svc.Spec.Ports) == 0 {
+			return fmt.Errorf("probe Service %s has no backend", svc.Name)
+		}
+		targetPort := svc.Spec.Ports[0].TargetPort
+		if targetPort.IntVal == 0 && targetPort.StrVal == "" {
+			targetPort = intstr.FromInt32(svc.Spec.Ports[0].Port)
+		}
+		allow := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id + "-" + svc.Name, Namespace: svc.Namespace}, Spec: networkingv1.NetworkPolicySpec{
+			PodSelector: metav1.LabelSelector{MatchLabels: svc.Spec.Selector}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
+			Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{
+				NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ns}},
+				PodSelector:       &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}},
+			}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &targetPort}}}},
+		}}
+		if err := create(allow); err != nil {
+			return err
+		}
+	}
+	positive := probeJob(id+"-positive", ns, name, image, probeArgs("--open", open), false)
+	positive.Spec.Template.Labels["felis.lolicon.best/probe-source"] = id
+	if err := create(positive); err != nil {
+		return err
+	}
+	if err := waitProbeJob(ctx, cl, positive); err != nil {
+		return err
+	}
+	denied = append(denied, open...)
+	denied = append(denied, "169.254.169.254:80", "169.254.170.2:80")
+	negative := probeJob(id+"-negative", ns, name, image, probeArgs("--closed", denied), true)
+	negative.Spec.Template.Spec.InitContainers = []corev1.Container{{Name: "egress-gate", Image: image, Command: []string{"/usr/local/bin/felis", "egress-gate", "--probe", net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), "--wait", "2m"}, SecurityContext: negative.Spec.Template.Spec.Containers[0].SecurityContext}}
+	if err := create(negative); err != nil {
+		return err
+	}
+	if err := waitProbeJob(ctx, cl, negative); err != nil {
+		return err
+	}
+	// Dial from Velocity's host namespace and record the address actually observed inside each backend.
+	for _, endpoint := range endpoints {
+		c, err := net.DialTimeout("tcp", endpoint, 5*time.Second)
+		if err != nil {
+			return fmt.Errorf("Velocity host cannot dial backend Service %s: %w", endpoint, err)
+		}
+		c.Close()
+	}
+	var observed []string
+	time.Sleep(500 * time.Millisecond)
+	for _, p := range echoPods {
+		foundA := false
+		stream, err := cs.CoreV1().Pods(ns).GetLogs(p.Name, &corev1.PodLogOptions{}).Stream(ctx)
+		if err != nil {
+			return err
+		}
+		scan := bufio.NewScanner(stream)
+		for scan.Scan() {
+			line := scan.Text()
+			if strings.HasPrefix(line, "felis-probe-source ") {
+				host, _, err := net.SplitHostPort(strings.TrimPrefix(line, "felis-probe-source "))
+				if err == nil {
+					cidr, _ := exactCIDR(host)
+					for _, a := range aCIDRs {
+						if cidr == a {
+							observed = append(observed, cidr)
+							foundA = true
+						}
+					}
+				}
+			}
+		}
+		if !foundA {
+			stream.Close()
+			return fmt.Errorf("backend %s did not observe A as an exact source", p.Name)
+		}
+		err = scan.Err()
+		stream.Close()
+		if err != nil {
+			return err
+		}
+	}
+	if len(observed) < len(echoPods) {
+		return fmt.Errorf("backend observed a source outside A's exact interfaces")
+	}
+	var game networkingv1.NetworkPolicy
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-allow-game-from-velocity"}, &game); err != nil {
+		return err
+	}
+	if len(game.Spec.Ingress) == 0 {
+		return fmt.Errorf("Velocity ingress policy is not configured")
+	}
+	prevGame := game.DeepCopy()
+	for _, cidr := range observed {
+		exists := false
+		for _, peer := range game.Spec.Ingress[0].From {
+			if peer.IPBlock != nil && peer.IPBlock.CIDR == cidr {
+				exists = true
+				break
+			}
+		}
+		if exists {
+			continue
+		}
+		game.Spec.Ingress[0].From = append(game.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}})
+	}
+	if err := cl.Patch(ctx, &game, client.MergeFrom(prevGame)); err != nil {
+		return err
+	}
+	// Read fresh resourceVersion so concurrent node health writes cannot be overwritten.
+	if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil {
+		return err
+	}
+	if !placement.Online(&n) {
+		return fmt.Errorf("worker became offline during validation")
+	}
+	before = n.DeepCopy()
+	n.Labels[placement.LabelIdentity] = name
+	n.Labels[placement.LabelRole] = placement.RoleWorker
+	n.Labels[placement.LabelApproved] = "true"
+	taints := n.Spec.Taints[:0]
+	for _, t := range n.Spec.Taints {
+		if t.Key != "felis.lolicon.best/unapproved" {
+			taints = append(taints, t)
+		}
+	}
+	n.Spec.Taints = taints
+	if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil {
+		return err
+	}
+	fmt.Fprintln(stdout, "approved worker", name, "Velocity sources", strings.Join(observed, ","))
+	return nil
+}
+
+func probeArgs(flag string, addresses []string) []string {
+	var args []string
+	for _, a := range addresses {
+		args = append(args, flag, a)
+	}
+	return args
+}
+func probePod(name, ns, node, image string, args []string, game bool) *corev1.Pod {
+	no, yes := false, true
+	uid := int64(1000)
+	labels := map[string]string{"felis.lolicon.best/probe-name": name}
+	if game {
+		labels[v1alpha1.LabelManagedBy] = operator.ManagedByValue
+		labels[v1alpha1.LabelComponent] = operator.ComponentValue
+		labels[v1alpha1.LabelServer] = name
+	}
+	return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, Spec: corev1.PodSpec{NodeName: node, RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &no, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &yes, RunAsUser: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "probe", Image: image, ImagePullPolicy: corev1.PullAlways, Command: []string{"/usr/local/bin/felis", "node-probe"}, Args: args, Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi"), corev1.ResourceCPU: resource.MustParse("200m")}}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &no, ReadOnlyRootFilesystem: &yes, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}}}}}
+}
+func probeJob(name, ns, node, image string, args []string, game bool) *batchv1.Job {
+	p := probePod(name, ns, node, image, args, game)
+	zero := int32(0)
+	deadline := int64(600)
+	return &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, Spec: batchv1.JobSpec{BackoffLimit: &zero, ActiveDeadlineSeconds: &deadline, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: p.Labels}, Spec: p.Spec}}}
+}
+func waitProbePod(ctx context.Context, cl client.Client, p *corev1.Pod) error {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+	for {
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(p), p); err != nil {
+			return err
+		}
+		for _, c := range p.Status.Conditions {
+			if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue {
+				return nil
+			}
+		}
+		if p.Status.Phase == corev1.PodFailed {
+			return fmt.Errorf("probe Pod %s failed", p.Name)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-t.C:
+		}
+	}
+}
+func waitProbeJob(ctx context.Context, cl client.Client, j *batchv1.Job) error {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+	for {
+		if err := cl.Get(ctx, client.ObjectKeyFromObject(j), j); err != nil {
+			return err
+		}
+		for _, c := range j.Status.Conditions {
+			if c.Status != corev1.ConditionTrue {
+				continue
+			}
+			if c.Type == batchv1.JobComplete {
+				return nil
+			}
+			if c.Type == batchv1.JobFailed {
+				return fmt.Errorf("admission probe Job %s failed; inspect its Pod log", j.Name)
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-t.C:
+		}
+	}
+}
+
+func denyNodeAddresses(ctx context.Context, cl client.Client, ns string, nodes []corev1.Node) error {
+	var np networkingv1.NetworkPolicy
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-server-egress"}, &np); err != nil {
+		return err
+	}
+	before := np.DeepCopy()
+	for _, n := range nodes {
+		for _, a := range n.Status.Addresses {
+			if a.Type != corev1.NodeInternalIP && a.Type != corev1.NodeExternalIP {
+				continue
+			}
+			cidr, err := exactCIDR(a.Address)
+			if err != nil {
+				return err
+			}
+			for i := range np.Spec.Egress {
+				for k := range np.Spec.Egress[i].To {
+					block := np.Spec.Egress[i].To[k].IPBlock
+					if block != nil && ((strings.Contains(cidr, ":") && block.CIDR == "::/0") || (!strings.Contains(cidr, ":") && block.CIDR == "0.0.0.0/0")) {
+						block.Except = append(block.Except, cidr)
+					}
+				}
+			}
+		}
+	}
+	return cl.Patch(ctx, &np, client.MergeFrom(before))
+}

cmd/felis/node_firewall.go

@@ -0,0 +1,187 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+func nodeFirewall(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("node firewall", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	peers := fs.String("peers", "", "comma-separated exact node IP CIDRs")
+	controller := fs.String("controller-ip", "", "controller address (optionally :6443)")
+	main := fs.Bool("controller", false, "A hosts the public API NodePort")
+	pod := fs.String("pod-cidr", "10.42.0.0/16", "cluster Pod CIDR")
+	dryRun := fs.Bool("dry-run", false, "print firewall scripts without installing")
+	controlNS := fs.String("control-namespace", "felis", "controller namespace")
+	minecraftNS := fs.String("namespace", "minecraft", "game namespace")
+	apiIP := fs.String("api-service-ip", "10.43.0.1", "Kubernetes API Service IP")
+	port := fs.Int("node-port", 30443, "API NodePort to block on workers before DNAT")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if host, _, err := net.SplitHostPort(*controller); err == nil {
+		*controller = host
+	}
+	if net.ParseIP(*apiIP) == nil || net.ParseIP(*controller) == nil || *port < 30000 || *port > 32767 {
+		fmt.Fprintln(stderr, "node firewall: controller IP and NodePort required")
+		return 2
+	}
+	if _, _, err := net.ParseCIDR(*pod); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+	var v4, v6 []string
+	for _, p := range strings.Split(*peers, ",") {
+		ip, n, err := net.ParseCIDR(p)
+		if err != nil {
+			fmt.Fprintln(stderr, "node firewall: invalid peer CIDR")
+			return 2
+		}
+		ones, bits := n.Mask.Size()
+		if ones != bits {
+			fmt.Fprintln(stderr, "node firewall: only exact peer addresses accepted")
+			return 2
+		}
+		if ip.To4() != nil {
+			v4 = append(v4, n.String())
+		} else {
+			v6 = append(v6, n.String())
+		}
+	}
+	script := "#!/bin/bash\nset -euo pipefail\n"
+	for _, f := range []struct {
+		bin      string
+		peers    []string
+		metadata string
+	}{{"iptables", v4, "169.254.0.0/16"}, {"ip6tables", v6, "fe80::/10"}} {
+		if f.bin == "ip6tables" && len(f.peers) == 0 {
+			continue
+		}
+		b := f.bin + " -w 10"
+		script += b + " -N FELIS-HOST 2>/dev/null || true\n" + b + " -F FELIS-HOST\n"
+		script += b + " -N FELIS-FORWARD 2>/dev/null || true\n" + b + " -F FELIS-FORWARD\n"
+		script += b + " -t raw -N FELIS-NODEPORT 2>/dev/null || true\n" + b + " -t raw -F FELIS-NODEPORT\n"
+		for _, c := range []struct{ table, parent, chain string }{{"filter", "INPUT", "FELIS-HOST"}, {"filter", "FORWARD", "FELIS-FORWARD"}, {"raw", "PREROUTING", "FELIS-NODEPORT"}} {
+			script += "while " + b + " -t " + c.table + " -D " + c.parent + " -j " + c.chain + " 2>/dev/null; do :; done\n" + b + " -t " + c.table + " -I " + c.parent + " 1 -j " + c.chain + "\n"
+		}
+		script += b + " -A FELIS-HOST -i lo -j RETURN\n"
+		if *main {
+			script += b + " -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -A FELIS-HOST -j FELIS-CONTROL\n"
+			script += b + " -t raw -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -t raw -A FELIS-NODEPORT -j FELIS-CONTROL\n"
+		}
+		script += b + " -A FELIS-HOST -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n" + b + " -A FELIS-FORWARD -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n"
+		family := 4
+		if f.bin == "ip6tables" {
+			family = 6
+		}
+		podIP, _, _ := net.ParseCIDR(*pod)
+		podFamily := 6
+		if podIP.To4() != nil {
+			podFamily = 4
+		}
+		if family == podFamily {
+			script += b + " -A FELIS-HOST -s " + *pod + " -j DROP\n"
+			// raw precedes DNAT and kube-router's filter ACCEPT rules. Block new
+			// host connections while preserving established Velocity/RCON replies.
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --syn -j DROP\n"
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p udp -j DROP\n"
+			if (net.ParseIP(*apiIP).To4() != nil) == (family == 4) {
+				script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + *apiIP + " -p tcp --dport 443 --syn -j DROP\n"
+			}
+
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n"
+			script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + f.metadata + " -j DROP\n"
+			script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + f.metadata + " -j DROP\n"
+			for _, p := range f.peers {
+				script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + p + " -j DROP\n"
+				script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p tcp --syn -j DROP\n"
+				script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p udp -j DROP\n"
+			}
+		}
+		if !*main {
+			script += b + " -t raw -A FELIS-NODEPORT -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n"
+		}
+		for _, p := range f.peers {
+			script += b + " -A FELIS-HOST -s " + p + " -p udp --dport 51820:51821 -j RETURN\n"
+		}
+		script += b + " -A FELIS-HOST -p udp --dport 51820:51821 -j DROP\n"
+		ctrlIP := net.ParseIP(*controller)
+		ctrlFamily := 6
+		if ctrlIP.To4() != nil {
+			ctrlFamily = 4
+		}
+		if *main {
+			for _, p := range f.peers {
+				script += b + " -A FELIS-HOST -s " + p + " -p tcp --dport 6443 -j RETURN\n"
+			}
+		}
+		if ctrlFamily == family {
+			script += b + " -A FELIS-HOST -s " + ctrlIP.String() + " -p tcp --dport 10250 -j RETURN\n"
+		}
+		script += b + " -A FELIS-HOST -p tcp -m multiport --dports 6443,6444,10250,10255,2379,2380,5000,5001,15432 -j DROP\n"
+	}
+	if *dryRun {
+		fmt.Fprint(stdout, script)
+		if *main {
+			fmt.Fprint(stdout, controlFirewallScript(*controlNS, *minecraftNS))
+		}
+		return 0
+	}
+	if err := os.MkdirAll("/etc/felis", 0700); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err := os.WriteFile("/etc/felis/node-firewall.sh", []byte(script), 0700); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	unit := "[Unit]\nDescription=Felis host and NodePort isolation\nAfter=network-online.target firewalld.service ufw.service\nBefore=k3s.service k3s-agent.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/node-firewall.sh\nRemainAfterExit=yes\n[Install]\nWantedBy=multi-user.target\n"
+	if err := os.WriteFile("/etc/systemd/system/felis-node-firewall.service", []byte(unit), 0644); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if *main {
+		refresh := controlFirewallScript(*controlNS, *minecraftNS)
+		if err := os.WriteFile("/etc/felis/control-firewall.sh", []byte(refresh), 0700); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		svc := "[Unit]\nDescription=Refresh exact controller Pod access to the apiserver\nAfter=k3s.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/control-firewall.sh\n"
+		timer := "[Unit]\nDescription=Track trusted controller Pods after rescheduling\n[Timer]\nOnBootSec=5s\nOnUnitActiveSec=5s\n[Install]\nWantedBy=timers.target\n"
+		if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.service", []byte(svc), 0644); err != nil {
+			return 1
+		}
+		if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.timer", []byte(timer), 0644); err != nil {
+			return 1
+		}
+	}
+	for _, cmd := range []*exec.Cmd{exec.Command("systemctl", "daemon-reload"), exec.Command("systemctl", "enable", "felis-node-firewall.service"), exec.Command("bash", "/etc/felis/node-firewall.sh")} {
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		if err := cmd.Run(); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	if *main {
+		cmd := exec.Command("systemctl", "enable", "--now", "felis-control-firewall.timer")
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		if err := cmd.Run(); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	return 0
+}
+
+func controlFirewallScript(controlNS, minecraftNS string) string {
+	return "#!/bin/bash\nset -euo pipefail\niptables -w 10 -F FELIS-CONTROL\niptables -w 10 -t raw -F FELIS-CONTROL\n/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{\" \"}{.spec.serviceAccountName}{\" \"}{.status.podIP}{\"\\n\"}{end}' | while read -r ns sa ip; do\ncase \"$ns/$sa\" in " + shellQuote(controlNS+"/felis-api") + "|" + shellQuote(controlNS+"/felis-operator") + "|" + shellQuote(minecraftNS+"/felis-reaper") + "|kube-system/*) ;; *) continue;; esac\n[[ $ip =~ ^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$ ]] || continue\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p tcp --dport 6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p tcp -m multiport --dports 443,6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\ndone\n"
+}

cmd/felis/node_firewall_test.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"bytes"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestDistributedFirewallPathsAndSyntax(t *testing.T) {
+	for _, controller := range []bool{false, true} {
+		args := []string{"--dry-run", "--peers", "192.0.2.1/32,192.0.2.2/32", "--controller-ip", "192.0.2.1"}
+		if controller {
+			args = append(args, "--controller")
+		}
+		var out, err bytes.Buffer
+		if code := nodeFirewall(args, &out, &err); code != 0 {
+			t.Fatal(code, err.String())
+		}
+		script := out.String()
+		for _, must := range []string{"-I INPUT 1 -j FELIS-HOST", "-I FORWARD 1 -j FELIS-FORWARD", "-t raw -I PREROUTING 1 -j FELIS-NODEPORT", "-A FELIS-HOST -s 10.42.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --dport 30443 -j DROP", "-d 169.254.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --syn -j DROP", "-d 10.43.0.1 -p tcp --dport 443 --syn -j DROP"} {
+			if !strings.Contains(script, must) {
+				t.Fatal("missing protection", must)
+			}
+		}
+		if !controller && strings.Contains(script, " -p tcp --dport 6443 -j RETURN") {
+			t.Fatal("worker exposed apiserver")
+		}
+		check := exec.Command("bash", "-n")
+		check.Stdin = strings.NewReader(script)
+		if result, e := check.CombinedOutput(); e != nil {
+			t.Fatalf("invalid firewall script: %s %v", result, e)
+		}
+	}
+	var out, err bytes.Buffer
+	if code := nodeFirewall([]string{"--dry-run", "--peers", "10.0.0.0/8", "--controller-ip", "10.0.0.1"}, &out, &err); code != 2 {
+		t.Fatal("broad node range accepted")
+	}
+}

cmd/felis/node_probe.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"time"
+)
+
+// A trusted probe runs with the same server labels and security context before node admission.
+func cmdNodeProbe(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("node-probe", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	listen := fs.String("listen", "", "listen and print observed source addresses (admission only)")
+	var open, closed multiFlag
+	fs.Var(&open, "open", "required reachable host:port")
+	fs.Var(&closed, "closed", "required blocked host:port")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if *listen != "" {
+		l, err := net.Listen("tcp", *listen)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		defer l.Close()
+		for {
+			c, err := l.Accept()
+			if err != nil {
+				return 1
+			}
+			fmt.Fprintln(stdout, "felis-probe-source", c.RemoteAddr().String())
+			c.Write([]byte("felis-probe\n"))
+			c.Close()
+		}
+	}
+	time.Sleep(3 * time.Second)
+	for _, check := range []struct {
+		addresses []string
+		wantOpen  bool
+	}{{open, true}, {closed, false}} {
+		for _, addr := range check.addresses {
+			c, err := net.DialTimeout("tcp", addr, 2*time.Second)
+			if c != nil {
+				c.Close()
+			}
+			if (err == nil) != check.wantOpen {
+				fmt.Fprintf(stderr, "node-probe: %s open=%t, expected %t\n", addr, err == nil, check.wantOpen)
+				return 1
+			}
+		}
+	}
+	return 0
+}

cmd/felis/nodes.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	felis "felis.lolicon.best"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/platform"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func cmdNode(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprintln(stderr, "felis node: list | token | join | approve | firewall")
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "felis node requires root/sudo")
+		return 1
+	}
+	if args[0] == "firewall" {
+		return nodeFirewall(args[1:], stdout, stderr)
+	}
+	fs := flag.NewFlagSet("node "+args[0], flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	name := fs.String("name", "", "stable worker node name")
+	kubeconfig := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local administrator kubeconfig")
+	ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
+	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "controller namespace")
+	image := fs.String("image", "", "Felis image to re-pull and use for admission probes")
+	remote := fs.String("ssh-target", "", "SSH target of the trusted worker (normal SSH host verification applies)")
+	out := fs.String("out", "", "token output file; never printed to stdout")
+	ttl := fs.Duration("ttl", 10*time.Minute, "bootstrap token lifetime (maximum 1h)")
+	server := fs.String("server", "", "A's https://address:6443 endpoint for join")
+	tokenFile := fs.String("token-file", "", "CA-pinned bootstrap token file for join")
+	registry := fs.String("registry-ip", "", "registry ClusterIP for join")
+	peers := fs.String("peers", "", "comma-separated exact peer CIDRs for host firewall")
+	external := fs.String("external-ip", "", "this worker's fixed external IP")
+	if err := fs.Parse(args[1:]); err != nil {
+		return 2
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer cancel()
+	switch args[0] {
+	case "token":
+		if *name == "" || *out == "" || *ttl <= 0 || *ttl > time.Hour {
+			fmt.Fprintln(stderr, "node token: name, output file and lifetime <=1h required")
+			return 2
+		}
+		cmd := exec.CommandContext(ctx, "k3s", "token", "create", "--ttl", ttl.String(), "--description", "Felis worker "+*name)
+		cmd.Stderr = stderr
+		raw, err := cmd.Output()
+		if err != nil {
+			fmt.Fprintln(stderr, "node token: creation failed:", err)
+			return 1
+		}
+		f, err := os.OpenFile(*out, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		_, err = f.Write(raw)
+		if err == nil {
+			err = f.Sync()
+		}
+		f.Close()
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		fmt.Fprintln(stdout, "limited bootstrap token written to", *out)
+		return 0
+	case "join":
+		exe, err := os.Executable()
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		cmd := exec.CommandContext(ctx, "bash", "-s")
+		cmd.Stdin = strings.NewReader(felis.BootstrapScript())
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		cmd.Env = append(os.Environ(), "FELIS_INSTALL_MODE=worker", "FELIS_BOOTSTRAP_FROM_TUI=1", "FELIS_BOOTSTRAP_BINARY="+exe, "FELIS_NODE_NAME="+*name, "FELIS_SERVER_URL="+*server, "FELIS_BOOTSTRAP_TOKEN_FILE="+*tokenFile, "FELIS_REGISTRY_CLUSTER_IP="+*registry, "FELIS_PEER_CIDRS="+*peers, "FELIS_NODE_EXTERNAL_IP="+*external)
+		if err = cmd.Run(); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		return 0
+	case "list", "approve":
+	default:
+		fmt.Fprintln(stderr, "unknown node operation")
+		return 2
+	}
+	cfg, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	scheme := runtime.NewScheme()
+	clientgoscheme.AddToScheme(scheme)
+	v1alpha1.AddToScheme(scheme)
+	cl, err := client.New(cfg, client.Options{Scheme: scheme})
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	m := &distributed.Manager{Client: cl, Namespace: *ns}
+	if args[0] == "list" {
+		nodes, err := m.Nodes(ctx)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		json.NewEncoder(stdout).Encode(nodes)
+		return 0
+	}
+	if *name == "" || *image == "" || *remote == "" || strings.HasPrefix(*remote, "-") {
+		fmt.Fprintln(stderr, "node approve requires name, image and SSH target")
+		return 2
+	}
+	cs, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err = approveNode(ctx, cl, cs, *ns, *controlNS, *name, *image, *remote, stdout); err != nil {
+		fmt.Fprintln(stderr, "node remains quarantined:", err)
+		return 1
+	}
+	return 0
+}
+
+func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" }
+func exactCIDR(ip string) (string, error) {
+	parsed := net.ParseIP(ip)
+	if parsed == nil {
+		return "", fmt.Errorf("invalid node address %q", ip)
+	}
+	if parsed.To4() != nil {
+		return parsed.String() + "/32", nil
+	}
+	return parsed.String() + "/128", nil
+}

cmd/felis/operator.go

@@ -114,7 +114,10 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		// The operator's own image, for the forwarding-config initContainer it
 		// injects into user servers. The Deployment passes it as FELIS_IMAGE (see
 		// platform.OperatorDeployment); absent, that injection is simply skipped.
-		FelisImage: os.Getenv("FELIS_IMAGE"),
+		FelisImage:     os.Getenv("FELIS_IMAGE"),
+		Nodes:          nil,
+		EgressProbe:    os.Getenv("FELIS_EGRESS_PROBE"),
+		ControllerNode: os.Getenv("FELIS_CONTROLLER_NODE"),
 		// Uncached: the maintenance-lock check lists Jobs only when a server is
 		// about to start, which does not justify a namespace-wide Job informer.
 		Jobs: mgr.GetAPIReader(),
@@ -127,6 +130,9 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
 		Recorder: mgr.GetEventRecorderFor("felis-operator"),
 		Watch:    watch,
 	}
+	if os.Getenv("FELIS_DISTRIBUTED") == "true" {
+		r.Nodes = mgr.GetAPIReader()
+	}
 	if err := r.SetupWithManager(mgr); err != nil {
 		fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
 		return 1

cmd/felis/reaper.go

@@ -16,6 +16,7 @@ import (
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/backup"
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
 	"felis.lolicon.best/internal/platform"
 	"felis.lolicon.best/internal/reaper"
 	corev1 "k8s.io/api/core/v1"
@@ -78,6 +79,19 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 		return 1
 	}
 
+	if os.Getenv("FELIS_DISTRIBUTED") == "true" && !*retentionOnly {
+		m, err := distributionManager(cl, cfg, os.Getenv("FELIS_IMAGE"))
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		local, ok := archiver.(*backup.TarLocal)
+		if !ok {
+			fmt.Fprintln(stderr, "remote reaper requires tarLocal")
+			return 1
+		}
+		archiver = &distributed.RemoteArchiver{TarLocal: local, Manager: m}
+	}
 	drv, err := openPodStore(ctx, cfg.Database.URL, "reaper", stderr)
 	if err != nil {
 		fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
@@ -94,7 +108,7 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintln(stderr, "felis reaper: retention only — no worlds root is configured, so idle worlds are neither archived nor released")
 		return reportReaperRun(r.RunRetention(ctx), stdout, stderr)
 	}
-	r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace)
+	r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace).WithDistributed(os.Getenv("FELIS_DISTRIBUTED") == "true")
 
 	// Pre-reap warnings go out by email when [smtp] is configured (the same
 	// relay and password_ref convention felis-api uses); without it the channel

cmd/felis/restore.go

@@ -1,14 +1,18 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io"
+	"os"
+	"os/signal"
 	"path/filepath"
 	"strings"
+	"syscall"
 
+	"felis.lolicon.best/internal/archivetransfer"
 	"felis.lolicon.best/internal/backup"
-	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 // cmdRestore is the in-Pod entrypoint the restore Job runs. internal/restore
@@ -26,6 +30,9 @@ import (
 func cmdRestore(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("restore", flag.ContinueOnError)
 	fs.SetOutput(stderr)
+	source := fs.String("source-url", "", "one-use archive download URL")
+	sum := fs.String("sha256", "", "required digest for remote archive")
+	limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum download size")
 	server := fs.String("server", "", "server name being restored (for logging)")
 	ref := fs.String("ref", "", "absolute path to the archive on the backup mount")
 	store := fs.String("archive-store", "tarLocal", "archive backend (only tarLocal is implemented)")
@@ -35,6 +42,22 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int {
 		return 2
 	}
 
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+	if *source != "" {
+		dir, err := os.MkdirTemp("/tmp", "felis-restore-")
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		defer os.RemoveAll(dir)
+		*backupRoot = dir
+		*ref = filepath.Join(dir, "world.tar.gz")
+		if err := archivetransfer.Fetch(ctx, *source, os.Getenv(archivetransfer.TokenEnv), *ref, *sum, *limit); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
 	if *ref == "" {
 		fmt.Fprintln(stderr, "felis restore: --ref is required")
 		return 2
@@ -62,11 +85,16 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int {
 		},
 	}
 
-	ctx := ctrl.SetupSignalHandler()
 	if err := archiver.Restore(ctx, backup.ArchiveRef(*ref), *server); err != nil {
 		fmt.Fprintf(stderr, "felis restore: %v\n", err)
 		return 1
 	}
+	if *source != "" {
+		if err := backup.VerifyRestored(ctx, *ref, *worldsRoot); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
 	fmt.Fprintf(stdout, "felis restore: server=%s restored from %s into %s\n", *server, *ref, *worldsRoot)
 	return 0
 }

cmd/felis/run.go

@@ -28,6 +28,10 @@ Commands:
   scan-gate         Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
   push-image        Push a scanned image tarball to the registry (internal Job entrypoint)
   mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
+  server-migrate    Move a stopped world between approved nodes (start|status|retry; requires root)
+  node              Join and approve trusted daemon nodes (list|token|join|approve|firewall; requires root)
+  node-probe        Verify reachability and observed sources (internal admission probe)
+  archive-serve     Serve scoped one-use archive transfers on the controller (internal entrypoint)
   registry-gate     Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
   manifests         Render the control-plane RBAC + NetworkPolicy install bundle as YAML
   apply             Create a MinecraftServer CRD (direct K8s write; use -f server.json)
@@ -74,6 +78,10 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
 	"push-image":         cmdPushImage,
 	"mirror-build-tools": cmdMirrorBuildTools,
 	"registry-gate":      cmdRegistryGate,
+	"archive-serve":      cmdArchiveServe,
+	"node":               cmdNode,
+	"server-migrate":     cmdServerMigrate,
+	"node-probe":         cmdNodeProbe,
 	"manifests":          cmdManifests,
 	"apply":              cmdApply,
 	"setup":              cmdSetup,

cmd/felis/server_migrate.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/distributed"
+	"felis.lolicon.best/internal/placement"
+	"felis.lolicon.best/internal/platform"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func cmdServerMigrate(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprintln(stderr, "server-migrate: start | status | retry (separate from database migrate)")
+		return 2
+	}
+	fs := flag.NewFlagSet("server-migrate "+args[0], flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	name := fs.String("name", "", "stopped user server")
+	target := fs.String("target-node", "", "approved target worker")
+	id := fs.String("id", "", "operation id (required for retry)")
+	kube := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local kubeconfig")
+	ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
+	cfgPath := fs.String("config", "/var/lib/felis/felis.host.toml", "host config used to record the current owner on the safety backup")
+	if err := fs.Parse(args[1:]); err != nil {
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "server-migrate requires root/sudo")
+		return 1
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	cfg, err := clientcmd.BuildConfigFromFlags("", *kube)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	scheme := runtime.NewScheme()
+	clientgoscheme.AddToScheme(scheme)
+	v1alpha1.AddToScheme(scheme)
+	cl, err := client.New(cfg, client.Options{Scheme: scheme})
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	m := &distributed.Manager{Client: cl, Namespace: *ns, Resolve: placement.Resolve(cl, *ns)}
+	var nodes corev1.NodeList
+	if err = cl.List(ctx, &nodes); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	for _, n := range nodes.Items {
+		if n.Labels[placement.LabelRole] == placement.RoleController {
+			m.Controller = n.Name
+		}
+	}
+	if m.Controller == "" {
+		fmt.Fprintln(stderr, "distributed controller identity is not configured")
+		return 1
+	}
+	m.Resolve = placement.Resolve(cl, *ns, m.Controller)
+	var op distributed.Operation
+	switch args[0] {
+	case "start":
+		host, err := config.Load(*cfgPath)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		drv, err := openPodStore(ctx, host.Database.URL, "migration", stderr)
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		defer drv.Close()
+		var owner sql.NullString
+		if err = drv.DB().QueryRowContext(ctx, "SELECT owner_id FROM servers WHERE name=$1 AND deleted_at IS NULL AND retire_requested_at IS NULL", *name).Scan(&owner); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		op, err = m.BeginMigration(ctx, *name, *target, owner.String)
+	case "status":
+		op, err = m.Migration(ctx, *name, *id)
+	case "retry":
+		if *id == "" {
+			fmt.Fprintln(stderr, "retry requires --id")
+			return 2
+		}
+		op, err = m.RetryMigration(ctx, *name, *id)
+	default:
+		fmt.Fprintln(stderr, "unknown migration operation")
+		return 2
+	}
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	json.NewEncoder(stdout).Encode(op)
+	return 0
+}