feat: manage distributed node operations from Owner panel

cmd/felis/api.go

@@ -26,6 +26,7 @@ import (
 	"felis.lolicon.best/internal/mail"
 	"felis.lolicon.best/internal/metrics"
 	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/nodecontrol"
 	"felis.lolicon.best/internal/panel"
 	"felis.lolicon.best/internal/passkey"
 	"felis.lolicon.best/internal/placement"
@@ -549,6 +550,10 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
 		a.Distribution = distribution
 		go reconcileDistribution(ctx, distribution, stderr)
 	}
+	if socket := os.Getenv("FELIS_NODE_CONTROL_SOCKET"); socket != "" {
+		a.NodeControl = nodecontrol.NewClient(socket)
+		cluster.NodeMaintenanceGuard = api.NodeMaintenanceGuard(a.NodeControl)
+	}
 	servers := []*http.Server{internalSrv, externalSrv}
 	if httpsSrv != nil {
 		servers = append(servers, httpsSrv)

cmd/felis/manifests.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"path/filepath"
 	"strings"
 
 	"felis.lolicon.best/internal/platform"
@@ -48,6 +49,8 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 	probe := fs.String("egress-probe", "", "reachable controller host:port denied to game Pods")
 	var registryNodes multiFlag
 	fs.Var(&registryNodes, "registry-node-cidr", "exact node pull source for the registry (repeatable)")
+	socket := fs.String("node-control-socket", "", "host node-control Unix socket (optional, API only)")
+	nodeControlNode := fs.String("node-control-node", "", "controller hostname hosting the socket")
 	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace the control plane (api/operator/reaper) runs in")
 	minecraftNS := fs.String("minecraft-namespace", platform.DefaultMinecraftNamespace, "namespace MinecraftServer workloads run in")
 	buildNS := fs.String("build-namespace", platform.DefaultBuildNamespace, "namespace image-build Jobs run in")
@@ -181,7 +184,12 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
 		}
 	}
 
+	if *socket != "" && (!filepath.IsAbs(*socket) || *nodeControlNode == "") {
+		fmt.Fprintln(stderr, "node-control requires an absolute socket and its controller hostname")
+		return 2
+	}
 	params := platform.Params{
+		NodeControlSocket: *socket, NodeControlNode: *nodeControlNode,
 		Distributed: *distributed, ControllerNode: *controller, EgressProbe: *probe, RegistryNodeCIDRs: registryNodes,
 		ControlNamespace:   *controlNS,
 		MinecraftNamespace: *minecraftNS,

cmd/felis/node_approve.go

@@ -173,7 +173,7 @@ func approveNode(ctx context.Context, cl client.Client, cs kubernetes.Interface,
 		"proof=$(mktemp); trap 'rm -f \"$proof\"' EXIT\n" +
 		"if /usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" label node " + shellQuote(name) + " felis.node-restriction.kubernetes.io/probe=controller --overwrite 2>\"$proof\"; then echo 'NodeRestriction failed' >&2;exit 1;fi\ngrep -qi forbidden \"$proof\"\n" +
 		"image=" + shellQuote(image) + "\nif [ -n \"$(/usr/local/bin/k3s crictl images -q \"$image\")\" ]; then /usr/local/bin/k3s crictl rmi \"$image\" >/dev/null; fi\ntest -z \"$(/usr/local/bin/k3s crictl images -q \"$image\")\"\n/usr/local/bin/k3s crictl pull \"$image\" >/dev/null\n"
-	cmd := exec.CommandContext(ctx, "ssh", "--", remote, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi")
+	cmd := exec.CommandContext(ctx, "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=10", "--", remote, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi")
 	cmd.Stdin = strings.NewReader(script)
 	cmd.Stdout = stdout
 	cmd.Stderr = stdout

cmd/felis/node_control.go

@@ -0,0 +1,581 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strings"
+	"syscall"
+	"time"
+
+	felis "felis.lolicon.best"
+	"felis.lolicon.best/internal/apis/felis/v1alpha1"
+	"felis.lolicon.best/internal/nodecontrol"
+	"felis.lolicon.best/internal/placement"
+	"felis.lolicon.best/internal/platform"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+func cmdNodeControl(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("node-control", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	socket := fs.String("socket", nodecontrol.Socket, "local API-only Unix socket")
+	dir := fs.String("state", "/var/lib/felis/node-control", "root-owned persistent task state")
+	kubeconfig := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "controller kubeconfig")
+	config := fs.String("config", "/etc/felis/felis.host.toml", "host configuration")
+	ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace")
+	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "control namespace")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	if os.Geteuid() != 0 {
+		fmt.Fprintln(stderr, "node-control requires root")
+		return 1
+	}
+	cfg, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	cs, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	exe, err := os.Executable()
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer cancel()
+	execute := nodeExecutor{cs: cs, binary: exe, kubeconfig: *kubeconfig, config: *config, namespace: *ns, controlNamespace: *controlNS, stateDir: *dir}
+	socketDir := filepath.Dir(*socket)
+	if err = os.MkdirAll(socketDir, 0750); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err = os.Chown(socketDir, 0, 65532); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	if err = os.Chmod(socketDir, 0750); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+
+	if existing, err := os.Lstat(*socket); err == nil {
+		if existing.Mode()&os.ModeSocket == 0 {
+			fmt.Fprintln(stderr, "refusing to replace non-socket path")
+			return 1
+		}
+		conn, err := net.DialTimeout("unix", *socket, time.Second)
+		if err == nil {
+			conn.Close()
+			fmt.Fprintln(stderr, "node-control is already running")
+			return 1
+		}
+		if err = os.Remove(*socket); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	listener, err := net.Listen("unix", *socket)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	defer listener.Close()
+	if err = os.Chown(*socket, 0, 65532); err == nil {
+		err = os.Chmod(*socket, 0660)
+	}
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	// /run is recreated at boot; label both the directory and the new socket inode.
+	if os.Getenv("FELIS_NODE_CONTROL_SELINUX") == "1" {
+		if err := exec.Command("chcon", "-R", "-t", "felis_node_control_socket_t", socketDir).Run(); err != nil {
+			fmt.Fprintln(stderr, "node-control socket labeling failed:", err)
+			return 1
+		}
+	}
+	manager, err := nodecontrol.Open(ctx, *dir, execute.run)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	server := &http.Server{Handler: manager.Handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second}
+	go func() { <-ctx.Done(); server.Close() }()
+	fmt.Fprintln(stdout, "node-control listening on", *socket)
+	err = server.Serve(listener)
+	cancel()
+	manager.Wait()
+	if err != nil && !errors.Is(err, http.ErrServerClosed) {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	return 0
+}
+
+type nodeExecutor struct {
+	cs                                                                kubernetes.Interface
+	binary, kubeconfig, config, namespace, controlNamespace, stateDir string
+}
+
+func (e nodeExecutor) run(ctx context.Context, r nodecontrol.Request, stage func(string) error, out io.Writer) (resultErr error) {
+	// Host-wide changes and cache-removing admission probes are serialized by the manager.
+	if r.Action == "approve" {
+		node, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		if node.Labels[placement.LabelRole] != placement.RoleWorker {
+			return errors.New("only worker nodes can be approved")
+		}
+		patch := []byte(`{"spec":{"unschedulable":true}}`)
+		if !node.Spec.Unschedulable {
+			patch = []byte(`{"spec":{"unschedulable":true},"metadata":{"annotations":{"felis.lolicon.best/node-control-cordon":"true"}}}`)
+		}
+		if _, err = e.cs.CoreV1().Nodes().Patch(ctx, r.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
+			return err
+		}
+	}
+	if err := e.requireStopped(ctx, r); err != nil {
+		return err
+	}
+	nodes, err := e.cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	controller, peers, err := nodeController(nodes.Items, r.ExternalIP, r.Peers)
+	if err != nil {
+		return err
+	}
+	deployment, err := e.cs.AppsV1().Deployments(e.controlNamespace).Get(ctx, platform.SAAPI, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if len(deployment.Spec.Template.Spec.Containers) == 0 {
+		return errors.New("Felis API image is unavailable")
+	}
+	image := deployment.Spec.Template.Spec.Containers[0].Image
+	if r.Action == "enable" {
+		if err := stage("pause_operator"); err != nil {
+			return err
+		}
+		scale, err := e.cs.AppsV1().Deployments(e.controlNamespace).GetScale(ctx, platform.SAOperator, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		replicas := scale.Spec.Replicas
+		if replicas < 1 {
+			replicas = 1
+		}
+		scale.Spec.Replicas = 0
+		if _, err = e.cs.AppsV1().Deployments(e.controlNamespace).UpdateScale(ctx, platform.SAOperator, scale, metav1.UpdateOptions{}); err != nil {
+			return err
+		}
+		defer func() {
+			cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+			defer cancel()
+			latest, err := e.cs.AppsV1().Deployments(e.controlNamespace).GetScale(cleanup, platform.SAOperator, metav1.GetOptions{})
+			if err == nil {
+				latest.Spec.Replicas = replicas
+				_, err = e.cs.AppsV1().Deployments(e.controlNamespace).UpdateScale(cleanup, platform.SAOperator, latest, metav1.UpdateOptions{})
+			}
+			if err != nil {
+				fmt.Fprintln(out, "Operator restoration failed:", err)
+				resultErr = fmt.Errorf("operator restoration failed: %w", err)
+			}
+		}()
+		// Drain the old reconciler before the second stopped-state check.
+		for {
+			pods, err := e.cs.CoreV1().Pods(e.controlNamespace).List(ctx, metav1.ListOptions{LabelSelector: platform.LabelComponent + "=" + platform.ComponentOperator})
+			if err != nil {
+				return err
+			}
+			if len(pods.Items) == 0 {
+				break
+			}
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(time.Second):
+			}
+		}
+		if err = e.requireStopped(ctx, r); err != nil {
+			return err
+		}
+		return e.enable(ctx, r, controller, peers, stage, out)
+	}
+	if controller.Labels[placement.LabelRole] != placement.RoleController {
+		return errors.New("distributed mode is not configured on the controller")
+	}
+	if r.Name == controller.Name {
+		return errors.New("worker name must differ from the controller")
+	}
+	// SSH trust and keys are configured on A; no password, key or bootstrap token crosses the API.
+	if err := stage("ssh_check"); err != nil {
+		return err
+	}
+	arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[runtime.GOARCH]
+	if arch == "" {
+		return errors.New("unsupported host architecture")
+	}
+	check := "set -eu\n[ \"$(uname -s)\" = Linux ] || { echo 'worker requires Linux' >&2; exit 1; }\n[ \"$(uname -m)\" = " + shellQuote(arch) + " ] || { echo 'worker architecture differs from controller' >&2; exit 1; }\n"
+	if r.Action == "join" {
+		check += "[ ! -e /etc/rancher/k3s/k3s.yaml ] && [ ! -e /var/lib/rancher/k3s/agent ] || { echo 'k3s is already installed; enrollment will not overwrite an existing node' >&2; exit 1; }\n"
+		check += "ip -o address show | awk '{print $4}' | cut -d/ -f1 | grep -Fx -- " + shellQuote(r.ExternalIP) + " >/dev/null || { echo 'fixed node IP is not assigned to the worker' >&2; exit 1; }\n"
+	}
+
+	if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi", strings.NewReader(check), out); err != nil {
+		return fmt.Errorf("worker preflight or SSH connection failed: %w", err)
+	}
+	if r.Action == "join" {
+		for _, n := range nodes.Items {
+			if n.Name == r.Name {
+				return errors.New("node already exists; use approval instead of reinstalling it")
+			}
+		}
+		if err = e.join(ctx, r, controller, peers, stage, out); err != nil {
+			return err
+		}
+	}
+	if err = stage("approval"); err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, e.binary, "node", "approve", "--name", r.Name, "--ssh-target", r.SSHTarget, "--image", image, "--kubeconfig", e.kubeconfig, "--namespace", e.namespace, "--control-namespace", e.controlNamespace)
+	cmd.Stdout = out
+	cmd.Stderr = out
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("node approval failed; node remains quarantined: %w", err)
+	}
+	// Admission succeeded; release only the task's scheduling quarantine.
+	admitted, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if admitted.Annotations["felis.lolicon.best/node-control-cordon"] == "true" {
+		if _, err = e.cs.CoreV1().Nodes().Patch(ctx, r.Name, types.MergePatchType, []byte(`{"spec":{"unschedulable":false},"metadata":{"annotations":{"felis.lolicon.best/node-control-cordon":null}}}`), metav1.PatchOptions{}); err != nil {
+			return fmt.Errorf("node approved but scheduling remains disabled: %w", err)
+		}
+	}
+	return stage("complete")
+}
+func (e nodeExecutor) requireStopped(ctx context.Context, r nodecontrol.Request) error {
+	// Even pre-existing worker names may contain worlds. Never run installer/admission on an occupied worker.
+	ss, err := e.cs.AppsV1().StatefulSets(e.namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, s := range ss.Items {
+		if r.Action != "enable" && s.Spec.Template.Spec.NodeSelector[placement.LabelIdentity] != r.Name {
+			continue
+		}
+		if s.Spec.Replicas != nil && *s.Spec.Replicas > 0 {
+			return fmt.Errorf("StatefulSet %s still requests %d replicas; stop the server before changing nodes", s.Name, *s.Spec.Replicas)
+		}
+	}
+	pods, err := e.cs.CoreV1().Pods(e.namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, p := range pods.Items {
+		if r.Action != "enable" && p.Spec.NodeName != r.Name && p.Spec.NodeSelector[placement.LabelIdentity] != r.Name {
+			continue
+		}
+		if p.Status.Phase != corev1.PodSucceeded && p.Status.Phase != corev1.PodFailed {
+			return fmt.Errorf("Pod %s has not exited (phase %s); wait for game and maintenance workloads to stop", p.Name, p.Status.Phase)
+		}
+	}
+	// Prevent a pending wake intent racing admission or an installation.
+	raw, err := e.cs.CoreV1().RESTClient().Get().AbsPath("/apis/" + v1alpha1.GroupVersion.Group + "/" + v1alpha1.GroupVersion.Version + "/namespaces/" + e.namespace + "/minecraftservers").DoRaw(ctx)
+	if err != nil {
+		return err
+	}
+	var servers v1alpha1.MinecraftServerList
+	if err = json.Unmarshal(raw, &servers); err != nil {
+		return err
+	}
+	for _, s := range servers.Items {
+		if r.Action != "enable" && s.Spec.NodeName != r.Name && s.Status.NodeName != r.Name {
+			continue
+		}
+		if s.Spec.DesiredState != "" && string(s.Spec.DesiredState) != "Stopped" {
+			return fmt.Errorf("server %s must have stopped intent", s.Name)
+		}
+	}
+	return nil
+}
+
+func nodeController(nodes []corev1.Node, ip string, extra []string) (*corev1.Node, []string, error) {
+	var controller *corev1.Node
+	peers := append([]string{}, extra...)
+	for i := range nodes {
+		n := &nodes[i]
+		_, controlPlane := n.Labels["node-role.kubernetes.io/control-plane"]
+		if controlPlane || n.Labels[placement.LabelRole] == placement.RoleController {
+			if controller != nil {
+				return nil, nil, errors.New("exactly one controller is required")
+			}
+			controller = n
+		}
+		for _, a := range n.Status.Addresses {
+			if a.Type == corev1.NodeExternalIP || a.Type == corev1.NodeInternalIP {
+				cidr, err := exactCIDR(a.Address)
+				if err != nil {
+					return nil, nil, err
+				}
+				peers = append(peers, cidr)
+			}
+		}
+	}
+	if controller == nil || !placement.Online(controller) {
+		return nil, nil, errors.New("the sole controller must be online")
+	}
+	if ip != "" {
+		cidr, err := exactCIDR(ip)
+		if err != nil {
+			return nil, nil, err
+		}
+		peers = append(peers, cidr)
+	}
+	slices.Sort(peers)
+	peers = slices.Compact(peers)
+	return controller, peers, nil
+}
+func controllerIP(n *corev1.Node) string {
+	for _, kind := range []corev1.NodeAddressType{corev1.NodeExternalIP, corev1.NodeInternalIP} {
+		for _, a := range n.Status.Addresses {
+			if a.Type == kind {
+				return a.Address
+			}
+		}
+	}
+	return ""
+}
+func (e nodeExecutor) ssh(ctx context.Context, target, command string, in io.Reader, out io.Writer) error {
+	cmd := exec.CommandContext(ctx, "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=10", "--", target, command)
+	cmd.Stdin = in
+	cmd.Stdout = out
+	cmd.Stderr = out
+	return cmd.Run()
+}
+func (e nodeExecutor) join(ctx context.Context, r nodecontrol.Request, controller *corev1.Node, peers []string, stage func(string) error, out io.Writer) error {
+	registry, err := e.cs.CoreV1().Services(e.controlNamespace).Get(ctx, "registry", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if err = stage("firewall"); err != nil {
+		return err
+	}
+	// All peers must be updated before the new agent is admitted. Existing worker SSH targets must be configured by stable node name.
+	list, err := e.cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, n := range list.Items {
+		if n.Name == controller.Name || n.Name == r.Name {
+			continue
+		}
+		if n.Labels[placement.LabelRole] != placement.RoleWorker {
+			return fmt.Errorf("node %s has an unknown role", n.Name)
+		}
+		script := shellQuote(e.binary) + " node firewall --peers " + shellQuote(strings.Join(peers, ",")) + " --controller-ip " + shellQuote(controllerIP(controller))
+		script += " --namespace " + shellQuote(e.namespace) + " --control-namespace " + shellQuote(e.controlNamespace)
+		if err = e.ssh(ctx, n.Name, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n "+script+"; fi", nil, out); err != nil {
+			return fmt.Errorf("update peer firewall on %s: %w", n.Name, err)
+		}
+	}
+	cmd := exec.CommandContext(ctx, e.binary, "node", "firewall", "--controller", "--controller-ip", controllerIP(controller), "--peers", strings.Join(peers, ","), "--namespace", e.namespace, "--control-namespace", e.controlNamespace)
+	cmd.Stdout = out
+	cmd.Stderr = out
+	if err = cmd.Run(); err != nil {
+		return err
+	}
+	if err = stage("bootstrap_token"); err != nil {
+		return err
+	}
+	temp, err := os.MkdirTemp("", "felis-node-join-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(temp)
+	tokenPath := filepath.Join(temp, "bootstrap")
+	tokenCmd := exec.CommandContext(ctx, e.binary, "node", "token", "--name", r.Name, "--ttl", "1h", "--out", tokenPath)
+	tokenCmd.Stdout = out
+	tokenCmd.Stderr = out
+	if err = tokenCmd.Run(); err != nil {
+		return err
+	}
+	// Revoke even on failure. The worker exchanges bootstrap credentials for its node identity.
+	token, err := os.ReadFile(tokenPath)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		exec.CommandContext(cleanup, "k3s", "token", "delete", bootstrapTokenID(string(token))).Run()
+	}()
+	remoteDir := "/var/tmp/felis-node-" + filepath.Base(temp)
+	if err = stage("transfer"); err != nil {
+		return err
+	}
+	script := "set -eu; umask 077; mkdir " + shellQuote(remoteDir) + "; cat > " + shellQuote(remoteDir+"/bootstrap")
+	rootCommand := "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi"
+	defer func() {
+		cleanup, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		e.ssh(cleanup, r.SSHTarget, rootCommand, strings.NewReader("rm -rf -- "+shellQuote(remoteDir)), io.Discard)
+	}()
+	// stdin carries the token; it is never in command arguments, task records or logs.
+	if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n sh -c "+shellQuote(script)+"; fi", strings.NewReader(string(token)), out); err != nil {
+		return err
+	}
+
+	binary, err := os.Open(e.binary)
+	if err != nil {
+		return err
+	}
+	defer binary.Close()
+	script = "set -eu; cat > " + shellQuote(remoteDir+"/felis") + "; chmod 0700 " + shellQuote(remoteDir+"/felis")
+	if err = e.ssh(ctx, r.SSHTarget, "if [ \"$(id -u)\" = 0 ]; then "+script+"; else sudo -n sh -c "+shellQuote(script)+"; fi", binary, out); err != nil {
+		return err
+	}
+	if err = stage("install_worker"); err != nil {
+		return err
+	}
+	script = "set -eu\nexport FELIS_K3S_VERSION=" + shellQuote(controller.Status.NodeInfo.KubeletVersion) + "\n" + shellQuote(remoteDir+"/felis") + " node join --name " + shellQuote(r.Name) + " --server " + shellQuote("https://"+net.JoinHostPort(controllerIP(controller), "6443")) + " --external-ip " + shellQuote(r.ExternalIP) + " --token-file " + shellQuote(remoteDir+"/bootstrap") + " --registry-ip " + shellQuote(registry.Spec.ClusterIP) + " --peers " + shellQuote(strings.Join(peers, ",")) + "\n"
+	if err = e.ssh(ctx, r.SSHTarget, rootCommand, strings.NewReader(script), out); err != nil {
+		return fmt.Errorf("worker installation failed: %w", err)
+	}
+	if err = stage("wait_ready"); err != nil {
+		return err
+	}
+	deadline := time.NewTimer(5 * time.Minute)
+	defer deadline.Stop()
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+	for {
+		n, err := e.cs.CoreV1().Nodes().Get(ctx, r.Name, metav1.GetOptions{})
+		if err == nil && placement.Online(n) {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-deadline.C:
+			return errors.New("worker did not become Ready within five minutes")
+		case <-ticker.C:
+		}
+	}
+}
+func (e nodeExecutor) enable(ctx context.Context, r nodecontrol.Request, controller *corev1.Node, peers []string, stage func(string) error, out io.Writer) error {
+	if controllerIP(controller) != r.ExternalIP {
+		return errors.New("controller IP must match the registered fixed node address")
+	}
+	if len(peers) == 0 {
+		return errors.New("peer addresses are required")
+	}
+	if err := stage("database_backup"); err != nil {
+		return err
+	}
+	backup := exec.CommandContext(ctx, e.binary, "db", "backup", "-config", e.config, "-dir", "/var/lib/felis/db-backups", "-label", "pre-migrate")
+	backup.Stdout = out
+	backup.Stderr = out
+	if err := backup.Run(); err != nil {
+		return err
+	}
+	if err := stage("cluster_backup"); err != nil {
+		return err
+	}
+	if err := e.backupCluster(ctx, out); err != nil {
+		return err
+	}
+	if err := stage("configure_controller"); err != nil {
+		return err
+	}
+	patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s","%s":"controller"}}}`, placement.LabelIdentity, controller.Name, placement.LabelRole)
+	if _, err := e.cs.CoreV1().Nodes().Patch(ctx, controller.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil {
+		return err
+	}
+	for _, name := range []string{platform.SAAPI, platform.SAOperator, platform.PostgresName, "registry"} {
+		body := fmt.Sprintf(`{"spec":{"template":{"spec":{"nodeSelector":{"%s":"%s"}}}}}`, placement.LabelIdentity, controller.Name)
+		if _, err := e.cs.AppsV1().Deployments(e.controlNamespace).Patch(ctx, name, types.MergePatchType, []byte(body), metav1.PatchOptions{}); err != nil {
+			return err
+		}
+	}
+	if err := stage("install_controller"); err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, "bash", "-s")
+	cmd.Stdin = strings.NewReader(felis.BootstrapScript())
+	cmd.Stdout = out
+	cmd.Stderr = out
+	cmd.Env = append(os.Environ(), "FELIS_DISTRIBUTED=1", "FELIS_NODE_EXTERNAL_IP="+r.ExternalIP, "FELIS_PEER_CIDRS="+strings.Join(peers, ","), "FELIS_BOOTSTRAP_FROM_TUI=1", "FELIS_BOOTSTRAP_BINARY="+e.binary, "FELIS_NO_SETUP=1", "FELIS_NODE_CONTROL_TASK=1", "FELIS_INSTALL_MODE=full")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("controller installation failed; retain the database backup: %w", err)
+	}
+	return stage("complete")
+}
+
+func bootstrapTokenID(token string) string {
+	token = strings.TrimSpace(token)
+	if _, short, ok := strings.Cut(token, "::"); ok {
+		token = short
+	}
+	id, _, _ := strings.Cut(token, ".")
+	return id
+}
+
+func (e nodeExecutor) backupCluster(ctx context.Context, out io.Writer) (err error) {
+	dir := filepath.Join(filepath.Dir(e.stateDir), "cluster-backups")
+	if err = os.MkdirAll(dir, 0700); err != nil {
+		return err
+	}
+	path := filepath.Join(dir, "pre-distributed-"+time.Now().UTC().Format("20060102T150405.000000000Z")+".tar")
+	stop := exec.CommandContext(ctx, "systemctl", "stop", "k3s")
+	stop.Stdout = out
+	stop.Stderr = out
+	if err = stop.Run(); err != nil {
+		return err
+	}
+	defer func() {
+		restart, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		defer cancel()
+		cmd := exec.CommandContext(restart, "systemctl", "start", "k3s")
+		cmd.Stdout = out
+		cmd.Stderr = out
+		if restartErr := cmd.Run(); restartErr != nil {
+			err = fmt.Errorf("k3s restart failed after snapshot: %w", restartErr)
+		}
+	}()
+	cmd := exec.CommandContext(ctx, "tar", "-cf", path, "-C", "/var/lib/rancher/k3s/server", "db", "token", "tls")
+	cmd.Stdout = out
+	cmd.Stderr = out
+	if err = cmd.Run(); err != nil {
+		return fmt.Errorf("cluster snapshot failed: %w", err)
+	}
+	if err = os.Chmod(path, 0600); err != nil {
+		return err
+	}
+	fmt.Fprintln(out, "Cluster snapshot:", path)
+	return nil
+}

cmd/felis/node_control_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"felis.lolicon.best/internal/nodecontrol"
+	"felis.lolicon.best/internal/platform"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func TestNodeControlTopologyAndBootstrapID(t *testing.T) {
+	node := corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "controller", Labels: map[string]string{"node-role.kubernetes.io/control-plane": "true"}}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.0.2.1"}}}}
+	controller, peers, err := nodeController([]corev1.Node{node}, "192.0.2.2", []string{"192.0.2.1/32"})
+	if err != nil || controller.Name != "controller" || strings.Join(peers, ",") != "192.0.2.1/32,192.0.2.2/32" {
+		t.Fatal(controller, peers, err)
+	}
+	duplicate := node
+	duplicate.Name = "second"
+	if _, _, err = nodeController([]corev1.Node{node, duplicate}, "", nil); err == nil {
+		t.Fatal("multiple controllers accepted")
+	}
+	for _, token := range []string{"abcdef.0123456789abcdef", "K10hash::abcdef.0123456789abcdef\n"} {
+		if bootstrapTokenID(token) != "abcdef" {
+			t.Fatal("token secret passed to revocation")
+		}
+	}
+}
+func TestNodeControlRejectsActiveWorkloadsBeforeHostCommands(t *testing.T) {
+	replicas := int32(1)
+	set := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"}, Spec: appsv1.StatefulSetSpec{Replicas: &replicas}}
+	executor := nodeExecutor{cs: fake.NewSimpleClientset(set), namespace: platform.DefaultMinecraftNamespace}
+	if err := executor.requireStopped(context.Background(), nodecontrol.Request{Action: "enable"}); err == nil {
+		t.Fatal("host changes permitted with active worlds")
+	}
+}

cmd/felis/run.go

@@ -30,6 +30,7 @@ Commands:
   mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
   server-migrate    Move a stopped world between approved nodes (start|status|retry; requires root)
   node              Join and approve trusted daemon nodes (list|token|join|approve|firewall; requires root)
+  node-control      Run the host node-task service on an API-only Unix socket (requires root)
   node-probe        Verify reachability and observed sources (internal admission probe)
   archive-serve     Serve scoped one-use archive transfers on the controller (internal entrypoint)
   registry-gate     Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
@@ -80,6 +81,7 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
 	"registry-gate":      cmdRegistryGate,
 	"archive-serve":      cmdArchiveServe,
 	"node":               cmdNode,
+	"node-control":       cmdNodeControl,
 	"server-migrate":     cmdServerMigrate,
 	"node-probe":         cmdNodeProbe,
 	"manifests":          cmdManifests,

deploy/bootstrap.sh

@@ -5217,6 +5217,56 @@ EOF
 # The daily database backup. The first run happens now, so a broken pipeline (pg_dump
 # missing, directory unwritable) shows up in this install rather than in the first
 # restore someone needs.
+# The API mounts only this Unix socket, never the host kubeconfig or SSH keys.
+install_node_control_service() {
+  local selinux_environment=""
+  if command -v selinuxenabled >/dev/null 2>&1 && selinuxenabled; then
+    command -v semodule >/dev/null 2>&1 || die "node control on SELinux requires semodule"
+    cat > "${STATE_DIR}/felis-node-control.cil" <<'EOF_NODE_SELINUX'
+(type felis_node_control_socket_t)
+(typeattributeset file_type (felis_node_control_socket_t))
+(typeattributeset non_auth_file_type (felis_node_control_socket_t))
+(allow container_t felis_node_control_socket_t (dir (search getattr open read)))
+(allow container_t felis_node_control_socket_t (sock_file (write open read getattr)))
+(allow container_t unconfined_service_t (unix_stream_socket (connectto)))
+EOF_NODE_SELINUX
+    semodule -i "${STATE_DIR}/felis-node-control.cil" || die "could not install the node-control SELinux policy"
+    selinux_environment="Environment=FELIS_NODE_CONTROL_SELINUX=1"
+  fi
+  install -d -o root -g 65532 -m 0750 /run/felis-node-control
+  cat > /etc/tmpfiles.d/felis-node-control.conf <<'EOF_NODE_TMP'
+d /run/felis-node-control 0750 root 65532 -
+EOF_NODE_TMP
+  if [ -n "$selinux_environment" ] && command -v semanage >/dev/null 2>&1; then
+    semanage fcontext -a -t felis_node_control_socket_t '/run/felis-node-control(/.*)?' 2>/dev/null \
+      || semanage fcontext -m -t felis_node_control_socket_t '/run/felis-node-control(/.*)?'
+  fi
+  if [ -n "$selinux_environment" ] && command -v chcon >/dev/null 2>&1; then
+    chcon -R -t felis_node_control_socket_t /run/felis-node-control 2>/dev/null || true
+  fi
+  cat > /etc/systemd/system/felis-node-control.service <<EOF_NODE_UNIT
+[Unit]
+Description=Felis host node operations
+After=network-online.target k3s.service
+Wants=network-online.target
+[Service]
+Type=simple
+ExecStart=${HOST_BIN} node-control --config ${STATE_DIR}/felis.host.toml --state /var/lib/felis/node-control --namespace ${MINECRAFT_NS} --control-namespace ${CONTROL_NS}
+${selinux_environment}
+Restart=on-failure
+RestartSec=5
+UMask=0077
+[Install]
+WantedBy=multi-user.target
+EOF_NODE_UNIT
+  systemctl daemon-reload
+  systemctl enable felis-node-control.service
+  # A conversion task runs inside this service; restarting it would terminate the installer.
+  if [ "${FELIS_NODE_CONTROL_TASK:-0}" != 1 ]; then
+    systemctl restart felis-node-control.service
+  fi
+}
+
 install_db_backup_timer() {
   install -d -m 0700 "$FELIS_DB_BACKUP_DIR"
   cat > "$DB_BACKUP_SERVICE" <<EOF
@@ -5406,6 +5456,7 @@ deploy_bundle() {
     --postgres-image "$POSTGRES_IMAGE"
     --panel-node-port "$FELIS_PANEL_NODEPORT"
     --velocity-cidr "${NODE_IP}/32"
+    --node-control-socket /run/felis-node-control/control.sock --node-control-node "$(k3s_node_name)"
   )
   if [ "${DISTRIBUTED:-0}" = 1 ]; then
     manifest_args+=(--distributed --controller-node "$controller" --egress-probe "felis-api.${CONTROL_NS}.svc:443")
@@ -6351,6 +6402,7 @@ main() {
   # move, the database is the host PostgreSQL an earlier release installed.
   migrate_host_postgres
   run_migrations
+  install_node_control_service
   deploy_bundle
   # AFTER deploy_bundle: the registry the built images are mirrored into is part
   # of that bundle.

docs/distributed.md

@@ -2,6 +2,20 @@
 
 分布式模式默认关闭。A 运行唯一 Felis API/operator、k3s server、PostgreSQL、Registry、归档服务和系统服；Velocity 继续使用 A 的 systemd 服务。B、C 等节点只运行 k3s-agent/containerd、游戏 Pod 和 A 创建的维护 Job。节点必须与 A 同架构、同 k3s 版本，宿主机由管理员信任并维护。
 
+## 面板管理
+
+Owner 可在「平台设置 → 节点操作」中启用分布式部署、通过 SSH 接入 worker、批准已接入的 worker，以及查询或重试失败任务。服务器的节点选择和停止后的世界迁移继续使用服务器管理页面。以下 CLI 流程保留为维护及恢复入口。
+
+完整安装会安装 `felis-node-control.service`。面板 API 保持非 root 身份，仅挂载该服务的只读 Unix socket 目录；宿主机服务只接受固定的节点操作，不提供通用命令执行。SELinux 主机为专用 socket 目录与文件配置独立类型；API 通过文件权限及独占挂载访问该服务。HTTP 接口仅限 Owner，并沿用敏感操作的身份复核与审计要求。SSH 私钥、bootstrap token 和宿主机 kubeconfig 不返回浏览器。
+
+首次接入前，在 A 的 root SSH 配置中准备免交互认证并核验主机指纹。SSH 账户须为 root 或具有 `sudo -n` 权限；已接入 worker 的 SSH 别名须与节点名称一致，以便增加节点时更新所有对等防火墙。系统不会接受未知主机指纹，也不会重新安装已经加入集群的主机。安装中断后，若 worker 已加入集群，应改用批准操作。
+
+启用部署前须停止全部游戏及维护任务。服务暂停 operator 后再次检查停止状态，备份数据库，并停止 k3s 保存 `db`、`token`、`tls` 快照后重新启动 k3s；快照保存在 `/var/lib/felis/cluster-backups`。随后复用当前二进制的安装器配置 WireGuard、主控身份与平台组件。期间面板可能暂时无法访问，宿主机任务继续执行。worker 批准复用 CLI 的镜像拉取、网络隔离及节点身份验收；失败时不解除隔离。手动 cordon 的节点不会自动解除 cordon。
+
+任务状态及日志持久化在 `/var/lib/felis/node-control`，由 root 保护。操作执行超时上限为 45 分钟，中断后执行必要恢复动作；保留最近 100 条记录；单项日志文件限制为 4 MiB，面板读取最近 64 KiB。服务重启会将未完成任务标记为失败，恢复前应核对主机状态，不自动重复执行安装。配置了主机服务的 API 在节点操作期间阻止启动、维护及迁移；主机服务不可用时这些操作返回明确错误，停止入口仍保留。
+
+主机服务异常时，宿主机管理员可执行 `systemctl status felis-node-control`、`journalctl -u felis-node-control` 和 `systemctl restart felis-node-control`。首次升级至包含该功能的版本，需要通过完整安装流程安装服务及 socket 挂载；仅替换网页资源不能启用宿主机操作。
+
 ## 先准备 A
 
 在维护窗口停服，使用现有数据库备份流程备份 PostgreSQL，并离线保存 k3s 状态、server token 和当前安装配置。SQLite k3s 的状态目录是 `/var/lib/rancher/k3s/server/db`；使用其他 datastore 时按对应备份流程操作。不要把这些文件复制到 worker。

docs/openapi.yaml

@@ -565,6 +565,35 @@ components:
         attempt: { type: integer }
         error: { type: string }
 
+    NodeControlRequest:
+      type: object
+      required: [action, confirmMaintenance]
+      additionalProperties: false
+      properties:
+        action: { type: string, enum: [enable, join, approve] }
+        name: { type: string, description: Stable worker node name; required for join and approve. }
+        sshTarget: { type: string, description: Verified root SSH alias or user@host; required for workers. }
+        externalIP: { type: string, description: Fixed node IP; required for enable and join. }
+        peers:
+          type: array
+          maxItems: 100
+          items: { type: string }
+          description: Additional exact /32 or /128 peer addresses.
+        confirmMaintenance: { type: boolean, const: true }
+    NodeControlTask:
+      type: object
+      required: [id, request, actor, state, stage, startedAt]
+      properties:
+        id: { type: string }
+        request: { $ref: '#/components/schemas/NodeControlRequest' }
+        actor: { type: string }
+        state: { type: string, enum: [running, succeeded, failed] }
+        stage: { type: string }
+        startedAt: { type: string, format: date-time }
+        finishedAt: { type: string, format: date-time }
+        error: { type: string }
+        log: { type: string, description: Most recent 64 KiB of host execution output; absent in task lists. }
+
     ServerInfo:
       type: object
       description: Status projection of one server (internal/api/cluster.go ServerInfo).
@@ -4331,6 +4360,100 @@ paths:
         '401':
           $ref: '#/components/responses/Unauthorized'
 
+  /api/v1/settings/node-control:
+    get:
+      operationId: nodeTasks
+      summary: Read host node-service availability and persistent task history (Owner).
+      tags: [account]
+      x-felis-face: [external]
+      x-felis-tier: owner
+      security: [{ sessionCookie: [] }]
+      responses:
+        '200':
+          description: Availability and most recent task records; no credentials or logs.
+          content:
+            application/json:
+              schema:
+                type: object
+                required: [available, tasks]
+                properties:
+                  available: { type: boolean }
+                  tasks: { type: array, items: { $ref: '#/components/schemas/NodeControlTask' } }
+        '400': { $ref: '#/components/responses/BadRequest' }
+        '401': { $ref: '#/components/responses/Unauthorized' }
+        '403': { $ref: '#/components/responses/Forbidden' }
+        '409': { description: Another node task is running or this task cannot be retried. }
+        '503': { description: Host node execution service is unavailable. }
+  /api/v1/settings/node-control/tasks:
+    post:
+      operationId: startNodeTask
+      summary: Execute fixed host node operation (Owner, fresh reauthentication).
+      description: Tasks are serialized, audited and persisted on the controller. The host verifies stopped workloads and SSH trust before enrollment or admission. Bootstrap credentials never cross the external API. Server starts fail closed during node operations and when the configured host service cannot report its state.
+      tags: [account]
+      x-felis-face: [external]
+      x-felis-tier: owner
+      security: [{ sessionCookie: [] }]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: { $ref: '#/components/schemas/NodeControlRequest' }
+      responses:
+        '202':
+          description: Persistent task created.
+          content:
+            application/json:
+              schema: { $ref: '#/components/schemas/NodeControlTask' }
+        '400': { $ref: '#/components/responses/BadRequest' }
+        '401': { $ref: '#/components/responses/Unauthorized' }
+        '403': { $ref: '#/components/responses/Forbidden' }
+        '409': { description: Another node task is running or this task cannot be retried. }
+        '503': { description: Host node execution service is unavailable. }
+  /api/v1/settings/node-control/tasks/{id}:
+    get:
+      operationId: nodeTask
+      summary: Read task stage, terminal error and bounded execution log (Owner).
+      tags: [account]
+      x-felis-face: [external]
+      x-felis-tier: owner
+      security: [{ sessionCookie: [] }]
+      parameters:
+        - { name: id, in: path, required: true, schema: { type: string } }
+      responses:
+        '200':
+          description: Current task and recent output.
+          content:
+            application/json:
+              schema: { $ref: '#/components/schemas/NodeControlTask' }
+        '404': { $ref: '#/components/responses/NotFound' }
+        '400': { $ref: '#/components/responses/BadRequest' }
+        '401': { $ref: '#/components/responses/Unauthorized' }
+        '403': { $ref: '#/components/responses/Forbidden' }
+        '409': { description: Another node task is running or this task cannot be retried. }
+        '503': { description: Host node execution service is unavailable. }
+  /api/v1/settings/node-control/tasks/{id}/retry:
+    post:
+      operationId: retryNodeTask
+      summary: Create a new attempt from a failed task (Owner, fresh reauthentication).
+      tags: [account]
+      x-felis-face: [external]
+      x-felis-tier: owner
+      security: [{ sessionCookie: [] }]
+      parameters:
+        - { name: id, in: path, required: true, schema: { type: string } }
+      responses:
+        '202':
+          description: New task created; original task is retained.
+          content:
+            application/json:
+              schema: { $ref: '#/components/schemas/NodeControlTask' }
+        '404': { $ref: '#/components/responses/NotFound' }
+        '400': { $ref: '#/components/responses/BadRequest' }
+        '401': { $ref: '#/components/responses/Unauthorized' }
+        '403': { $ref: '#/components/responses/Forbidden' }
+        '409': { description: Another node task is running or this task cannot be retried. }
+        '503': { description: Host node execution service is unavailable. }
+
   /api/v1/settings/wake-policy:
     get:
       tags: [account]

internal/api/api.go

@@ -32,6 +32,7 @@ import (
 // API holds the dependencies shared by every handler.
 type API struct {
 	Distribution Distribution
+	NodeControl  NodeControl
 	Repo         Repo
 	Cluster      Cluster
 	Internal     InternalAuth
@@ -650,6 +651,10 @@ func (a *API) externalAPIRoutes() []apiRoute {
 		// Staff can designate their own game identity after panel setup. Players
 		// retain the in-game proof flow above.
 		{Method: "GET", Pattern: "/api/v1/account/link/sources", Admin: true, h: a.handleLinkSources},
+		{Method: "GET", Pattern: "/api/v1/settings/node-control", Owner: true, Admin: true, h: a.handleNodeTasks},
+		{Method: "POST", Pattern: "/api/v1/settings/node-control/tasks", Owner: true, Admin: true, h: a.handleStartNodeTask},
+		{Method: "GET", Pattern: "/api/v1/settings/node-control/tasks/{id}", Owner: true, Admin: true, h: a.handleNodeTask},
+		{Method: "POST", Pattern: "/api/v1/settings/node-control/tasks/{id}/retry", Owner: true, Admin: true, h: a.handleRetryNodeTask},
 		{Method: "GET", Pattern: "/api/v1/settings/wake-policy", Owner: true, Admin: true, h: a.handleGetWakePolicy},
 		{Method: "PUT", Pattern: "/api/v1/settings/wake-policy", Owner: true, Admin: true, h: a.handleSetWakePolicy},
 		{Method: "GET", Pattern: "/api/v1/settings/auth-sources", Owner: true, Admin: true, h: a.handleGetAuthSources},

internal/api/handlers_distributed.go

@@ -39,6 +39,13 @@ func (a *API) handleNodes(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *API) handleMigration(w http.ResponseWriter, r *http.Request) {
+	if a.NodeControl != nil {
+		if err := NodeMaintenanceGuard(a.NodeControl)(r.Context()); err != nil {
+			writeError(w, r, err)
+			return
+		}
+	}
+
 	if !a.distributedReady(w, r) {
 		return
 	}
@@ -88,6 +95,13 @@ func (a *API) handleMigrationStatus(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *API) handleMigrationRetry(w http.ResponseWriter, r *http.Request) {
+	if a.NodeControl != nil {
+		if err := NodeMaintenanceGuard(a.NodeControl)(r.Context()); err != nil {
+			writeError(w, r, err)
+			return
+		}
+	}
+
 	if !a.distributedReady(w, r) {
 		return
 	}

internal/api/handlers_node_control.go

@@ -0,0 +1,115 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"felis.lolicon.best/internal/nodecontrol"
+)
+
+type NodeControl interface {
+	List(context.Context) ([]nodecontrol.Task, error)
+	Get(context.Context, string) (nodecontrol.Task, error)
+	Start(context.Context, nodecontrol.Request, string) (nodecontrol.Task, error)
+}
+
+func (a *API) handleNodeTasks(w http.ResponseWriter, r *http.Request) {
+	if a.NodeControl == nil {
+		writeJSON(w, 200, map[string]any{"available": false, "tasks": []nodecontrol.Task{}})
+		return
+	}
+	tasks, err := a.NodeControl.List(r.Context())
+	if err != nil {
+		a.writeNodeControlError(w, r, err)
+		return
+	}
+	writeJSON(w, 200, map[string]any{"available": true, "tasks": tasks})
+}
+func (a *API) handleNodeTask(w http.ResponseWriter, r *http.Request) {
+	if !a.nodeControlReady(w, r) {
+		return
+	}
+	task, err := a.NodeControl.Get(r.Context(), r.PathValue("id"))
+	if err != nil {
+		a.writeNodeControlError(w, r, err)
+		return
+	}
+	writeJSON(w, 200, task)
+}
+func (a *API) handleStartNodeTask(w http.ResponseWriter, r *http.Request) {
+	if !a.requireReauth(w, r, principalFromContext(r.Context())) || !a.nodeControlReady(w, r) {
+		return
+	}
+	if err := requireJSONContentType(r); err != nil {
+		writeError(w, r, err)
+		return
+	}
+	var req nodecontrol.Request
+	if err := decodeJSON(w, r, &req); err != nil {
+		writeError(w, r, err)
+		return
+	}
+	a.startNodeTask(w, r, req)
+}
+func (a *API) handleRetryNodeTask(w http.ResponseWriter, r *http.Request) {
+	if !a.requireReauth(w, r, principalFromContext(r.Context())) || !a.nodeControlReady(w, r) {
+		return
+	}
+	task, err := a.NodeControl.Get(r.Context(), r.PathValue("id"))
+	if err != nil {
+		a.writeNodeControlError(w, r, err)
+		return
+	}
+	if task.State != "failed" {
+		writeError(w, r, newError(409, "conflict", "only failed node tasks can be retried"))
+		return
+	}
+	a.startNodeTask(w, r, task.Request)
+}
+func (a *API) startNodeTask(w http.ResponseWriter, r *http.Request, req nodecontrol.Request) {
+	if err := req.Validate(); err != nil {
+		writeError(w, r, newError(400, "bad_request", "%v", err))
+		return
+	}
+	task, err := a.NodeControl.Start(r.Context(), req, principalFromContext(r.Context()).UserID)
+	if err != nil {
+		a.writeNodeControlError(w, r, err)
+		return
+	}
+	a.audit(r, "platform.node."+req.Action, task.ID)
+	writeJSON(w, http.StatusAccepted, task)
+}
+func (a *API) nodeControlReady(w http.ResponseWriter, r *http.Request) bool {
+	if a.NodeControl == nil {
+		a.writeNodeControlError(w, r, errors.New("node-control not configured"))
+		return false
+	}
+	return true
+}
+func (a *API) writeNodeControlError(w http.ResponseWriter, r *http.Request, err error) {
+	switch {
+	case errors.Is(err, nodecontrol.ErrBusy):
+		writeError(w, r, newError(409, "node_operation_busy", "a node operation is already running"))
+	case errors.Is(err, nodecontrol.ErrNotFound):
+		writeError(w, r, newError(404, "not_found", "node task not found"))
+	default:
+		writeError(w, r, newError(503, "node_control_unavailable", "host node service is unavailable; inspect felis-node-control.service"))
+	}
+}
+
+// Fail closed for starts when the configured host service cannot report its maintenance state.
+func NodeMaintenanceGuard(control NodeControl) func(context.Context) error {
+	return func(ctx context.Context) error {
+		tasks, err := control.List(ctx)
+		if err != nil {
+			return newError(503, "node_control_unavailable", "host node service is unavailable")
+		}
+		for _, task := range tasks {
+			if task.State == "running" {
+				return newError(409, "node_operation_busy", "node maintenance is in progress")
+			}
+		}
+		return nil
+	}
+}

internal/api/handlers_node_control_test.go

@@ -0,0 +1,122 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"felis.lolicon.best/internal/nodecontrol"
+)
+
+type fakeNodeControl struct {
+	tasks   []nodecontrol.Task
+	calls   int
+	err     error
+	request nodecontrol.Request
+	actor   string
+}
+
+func (f *fakeNodeControl) List(context.Context) ([]nodecontrol.Task, error) { return f.tasks, f.err }
+func (f *fakeNodeControl) Get(context.Context, string) (nodecontrol.Task, error) {
+	if f.err != nil {
+		return nodecontrol.Task{}, f.err
+	}
+	if len(f.tasks) == 0 {
+		return nodecontrol.Task{}, nodecontrol.ErrNotFound
+	}
+	return f.tasks[0], nil
+}
+func (f *fakeNodeControl) Start(_ context.Context, r nodecontrol.Request, actor string) (nodecontrol.Task, error) {
+	f.calls++
+	f.request = r
+	f.actor = actor
+	return nodecontrol.Task{ID: "task", Request: r, State: "running"}, f.err
+}
+
+const nodeTaskPath = "/api/v1/settings/node-control/tasks"
+const nodeTaskBody = `{"action":"approve","name":"worker-01","sshTarget":"worker-01","confirmMaintenance":true}`
+
+func TestNodeControlOwnerBoundaryAndRetries(t *testing.T) {
+	for _, role := range []string{"user", "admin", "owner"} {
+		a := newTestAPI(newFakeRepo(), newFakeCluster())
+		a.External = staticExternal{p: &Principal{UserID: "actor", Role: role, ViaAdminAccess: role != "user"}}
+		control := &fakeNodeControl{tasks: []nodecontrol.Task{{ID: "task", State: "failed", Request: nodecontrol.Request{Action: "approve", Name: "worker-01", SSHTarget: "worker-01", ConfirmMaintenance: true}}}}
+		a.NodeControl = control
+		for _, tc := range []struct {
+			method, path, body string
+			want               int
+		}{{"GET", "/api/v1/settings/node-control", "", 200}, {"GET", nodeTaskPath + "/task", "", 200}, {"POST", nodeTaskPath, nodeTaskBody, 202}, {"POST", nodeTaskPath + "/task/retry", "", 202}} {
+			w := do(a.ExternalHandler(), tc.method, tc.path, tc.body, jsonHeader)
+			want := tc.want
+			if role != "owner" {
+				want = 403
+			}
+			if w.Code != want {
+				t.Fatalf("%s %s: %d %s", role, tc.path, w.Code, w.Body.String())
+			}
+		}
+		if role != "owner" && control.calls != 0 {
+			t.Fatal("nonowner reached root operations")
+		}
+		if role == "owner" && (control.actor != "actor" || control.request.Name != "worker-01") {
+			t.Fatal("actor/scope not preserved")
+		}
+	}
+}
+func TestNodeControlErrorsAndStartGuard(t *testing.T) {
+	a := newTestAPI(newFakeRepo(), newFakeCluster())
+	a.External = staticExternal{p: &Principal{UserID: "owner", Role: "owner", ViaAdminAccess: true}}
+	control := &fakeNodeControl{}
+	a.NodeControl = control
+	w := do(a.ExternalHandler(), "POST", nodeTaskPath, `{"action":"approve","name":"x","sshTarget":"-exec","confirmMaintenance":true}`, jsonHeader)
+	if w.Code != 400 || control.calls != 0 {
+		t.Fatal(w.Code, w.Body.String())
+	}
+	for _, tc := range []struct {
+		err  error
+		code int
+	}{{nodecontrol.ErrBusy, 409}, {errors.New("private host detail"), 503}} {
+		control.err = tc.err
+		w := do(a.ExternalHandler(), "POST", nodeTaskPath, nodeTaskBody, jsonHeader)
+		if w.Code != tc.code {
+			t.Fatal(w.Code, w.Body.String())
+		}
+	}
+	guard := NodeMaintenanceGuard(control)
+	if guard(context.Background()) == nil {
+		t.Fatal("host failure permitted start")
+	}
+	control.err = nil
+	control.tasks = []nodecontrol.Task{{State: "running"}}
+	if guard(context.Background()) == nil {
+		t.Fatal("maintenance permitted start")
+	}
+	control.tasks[0].State = "failed"
+	if err := guard(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	control.tasks[0].State = "succeeded"
+	w = do(a.ExternalHandler(), "POST", nodeTaskPath+"/task/retry", "", jsonHeader)
+	if w.Code != 409 {
+		t.Fatal(w.Code, w.Body.String())
+	}
+}
+
+func TestNodeControlRequiresFreshOwnerProof(t *testing.T) {
+	repo := newFakeRepo()
+	a := newTestAPI(repo, newFakeCluster())
+	p := &Principal{UserID: "owner", Role: "owner", ViaAdminAccess: true, ViaSession: true}
+	repo.passkeyCreds["owner-key"] = PasskeyCredential{ID: "owner-key", UserID: "owner", UserVerified: true}
+	a.External = staticExternal{p: p}
+	control := &fakeNodeControl{}
+	a.NodeControl = control
+	w := do(a.ExternalHandler(), "POST", nodeTaskPath, nodeTaskBody, jsonHeader)
+	if w.Code != 403 || decodeErr(t, w) != "reauth_required" || control.calls != 0 {
+		t.Fatal(w.Code, w.Body.String())
+	}
+	p.ReauthAt = a.now()
+	w = do(a.ExternalHandler(), "POST", nodeTaskPath, nodeTaskBody, jsonHeader)
+	if w.Code != 202 || control.calls != 1 {
+		t.Fatal(w.Code, w.Body.String())
+	}
+}

internal/api/k8scluster.go

@@ -36,6 +36,8 @@ import (
 // the direct client c, so a write never works from a copy the watch has not caught
 // up with yet.
 type K8sCluster struct {
+	// NodeMaintenanceGuard checks host maintenance for all manual and scheduled starts.
+	NodeMaintenanceGuard func(context.Context) error
 	distributed          bool
 	controller           string
 	c                    client.Client
@@ -308,6 +310,11 @@ func (k *K8sCluster) RestartServer(ctx context.Context, name string) error {
 }
 
 func (k *K8sCluster) startWith(ctx context.Context, name, annotation string) error {
+	if k.NodeMaintenanceGuard != nil {
+		if err := k.NodeMaintenanceGuard(ctx); err != nil {
+			return err
+		}
+	}
 	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
 		var ms v1alpha1.MinecraftServer
 		if err := k.getServer(ctx, name, &ms); err != nil {
@@ -350,6 +357,11 @@ func (k *K8sCluster) startWith(ctx context.Context, name, annotation string) err
 			}
 			ms.Annotations[annotation] = k.clock().UTC().Format(time.RFC3339Nano)
 		}
+		if k.NodeMaintenanceGuard != nil {
+			if err := k.NodeMaintenanceGuard(ctx); err != nil {
+				return err
+			}
+		}
 		return k.c.Patch(ctx, &ms, patch)
 	})
 }
@@ -362,6 +374,12 @@ func (k *K8sCluster) startWith(ctx context.Context, name, annotation string) err
 // checked; the caller creates its Job and then calls ReleaseMaintenance, after
 // which the Job itself is the lock.
 func (k *K8sCluster) AcquireMaintenance(ctx context.Context, name, kind string) error {
+	if k.NodeMaintenanceGuard != nil {
+		if err := k.NodeMaintenanceGuard(ctx); err != nil {
+			return err
+		}
+	}
+
 	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
 		var ms v1alpha1.MinecraftServer
 		if err := k.getServer(ctx, name, &ms); err != nil {

internal/api/k8scluster_test.go

@@ -459,3 +459,26 @@ func TestResourcePatchesKeepWhatTheyLeaveOut(t *testing.T) {
 		t.Fatalf("cleared cpu: limits %v, want the CPU limit gone and memory 8Gi kept", after.Resources.Limits)
 	}
 }
+
+func TestHostNodeMaintenanceBlocksStartsAndWorldJobs(t *testing.T) {
+	scheme := runtime.NewScheme()
+	v1alpha1.AddToScheme(scheme)
+	server := &v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"}, Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped}, Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped}}
+	c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(server).Build()
+	k := NewK8sCluster(c, server.Namespace)
+	blocked := newError(409, "node_operation_busy", "host task running")
+	k.NodeMaintenanceGuard = func(context.Context) error { return blocked }
+	ctx := context.Background()
+	for _, operation := range []func() error{func() error { return k.SetDesiredState(ctx, server.Name, v1alpha1.DesiredRunning) }, func() error { return k.RetryStart(ctx, server.Name) }, func() error { return k.AcquireMaintenance(ctx, server.Name, maintenance.KindBackup) }} {
+		if err := operation(); err != blocked {
+			t.Fatal("host maintenance bypassed", err)
+		}
+	}
+	if err := k.SetDesiredState(ctx, server.Name, v1alpha1.DesiredStopped); err != nil {
+		t.Fatal("stop entry blocked", err)
+	}
+	var got v1alpha1.MinecraftServer
+	if err := c.Get(ctx, types.NamespacedName{Namespace: server.Namespace, Name: server.Name}, &got); err != nil || got.Spec.DesiredState != v1alpha1.DesiredStopped || len(got.Annotations) != 0 {
+		t.Fatal("state changed during host maintenance", got, err)
+	}
+}

internal/nodecontrol/control.go

@@ -0,0 +1,447 @@
+// Package nodecontrol exposes fixed host operations over a local Unix socket.
+package nodecontrol
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+const Socket = "/run/felis-node-control/control.sock"
+const MaxLog = 64 << 10
+
+var ErrBusy = errors.New("a node operation is already running")
+var ErrNotFound = errors.New("node operation not found")
+var namePattern = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`)
+var targetPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.@:-]{0,252}$`)
+var idPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
+
+func validNodeName(name string) bool {
+	if len(name) > 63 {
+		return false
+	}
+	for _, label := range strings.Split(name, ".") {
+		if !namePattern.MatchString(label) {
+			return false
+		}
+	}
+	return true
+}
+
+type Request struct {
+	Action             string   `json:"action"`
+	Name               string   `json:"name,omitempty"`
+	SSHTarget          string   `json:"sshTarget,omitempty"`
+	ExternalIP         string   `json:"externalIP,omitempty"`
+	Peers              []string `json:"peers,omitempty"`
+	ConfirmMaintenance bool     `json:"confirmMaintenance"`
+}
+
+func (r Request) Validate() error {
+	if !r.ConfirmMaintenance {
+		return errors.New("maintenance impact confirmation is required")
+	}
+	switch r.Action {
+	case "join", "approve":
+		if !validNodeName(r.Name) || !targetPattern.MatchString(r.SSHTarget) {
+			return errors.New("invalid worker name or SSH target")
+		}
+	case "enable":
+	default:
+		return errors.New("unsupported node operation")
+	}
+	if r.Action != "approve" {
+		if a, err := netip.ParseAddr(r.ExternalIP); err != nil || a.IsUnspecified() || a.IsLoopback() || a.IsMulticast() || a.Zone() != "" {
+			return errors.New("a fixed node IP is required")
+		}
+	}
+	for _, peer := range r.Peers {
+		p, err := netip.ParsePrefix(peer)
+		if err != nil || p.Bits() != p.Addr().BitLen() || p.Addr().IsLoopback() || p.Addr().IsMulticast() || p.Addr().IsUnspecified() {
+			return errors.New("peer addresses must use exact /32 or /128 prefixes")
+		}
+	}
+	if len(r.Peers) > 100 {
+		return errors.New("too many peer addresses")
+	}
+	return nil
+}
+
+type Task struct {
+	ID         string     `json:"id"`
+	Request    Request    `json:"request"`
+	Actor      string     `json:"actor"`
+	State      string     `json:"state"`
+	Stage      string     `json:"stage"`
+	StartedAt  time.Time  `json:"startedAt"`
+	FinishedAt *time.Time `json:"finishedAt,omitempty"`
+	Error      string     `json:"error,omitempty"`
+	Log        string     `json:"log,omitempty"`
+}
+
+type Executor func(context.Context, Request, func(string) error, io.Writer) error
+
+type Manager struct {
+	dir     string
+	execute Executor
+	mu      sync.Mutex
+	tasks   map[string]Task
+	active  string
+	ctx     context.Context
+	wg      sync.WaitGroup
+}
+
+func Open(ctx context.Context, dir string, execute Executor) (*Manager, error) {
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return nil, err
+	}
+	m := &Manager{dir: dir, execute: execute, tasks: map[string]Task{}, ctx: ctx}
+	paths, err := filepath.Glob(filepath.Join(dir, "*.json"))
+	if err != nil {
+		return nil, err
+	}
+	for _, path := range paths {
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			return nil, err
+		}
+		var t Task
+		if err = json.Unmarshal(raw, &t); err != nil {
+			return nil, fmt.Errorf("read node task: %w", err)
+		}
+		if !idPattern.MatchString(t.ID) || filepath.Base(path) != t.ID+".json" {
+			return nil, errors.New("invalid stored task ID")
+		}
+		if t.State == "running" {
+			t.State, t.Error = "failed", "Host execution service restarted. Verify the host state before retrying."
+			now := time.Now().UTC()
+			t.FinishedAt = &now
+			if err = m.persist(t); err != nil {
+				return nil, err
+			}
+		}
+		m.tasks[t.ID] = t
+	}
+	return m, nil
+}
+func (m *Manager) persist(t Task) error {
+	t.Log = ""
+	raw, err := json.Marshal(t)
+	if err != nil {
+		return err
+	}
+	path := filepath.Join(m.dir, t.ID+".json")
+	f, err := os.OpenFile(path+".tmp", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+	if _, err = f.Write(raw); err == nil {
+		err = f.Sync()
+	}
+	closeErr := f.Close()
+	if err != nil {
+		return err
+	}
+	if closeErr != nil {
+		return closeErr
+	}
+	if err = os.Rename(path+".tmp", path); err != nil {
+		return err
+	}
+	d, err := os.Open(m.dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	return d.Sync()
+}
+func (m *Manager) List() []Task {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	out := make([]Task, 0, len(m.tasks))
+	for _, t := range m.tasks {
+		out = append(out, t)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].StartedAt.After(out[j].StartedAt) })
+	if len(out) > 100 {
+		out = out[:100]
+	}
+	return out
+}
+func (m *Manager) Get(id string) (Task, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	t, ok := m.tasks[id]
+	if !ok {
+		return Task{}, ErrNotFound
+	}
+	f, err := os.Open(filepath.Join(m.dir, id+".log"))
+	if errors.Is(err, os.ErrNotExist) {
+		return t, nil
+	}
+	if err != nil {
+		return t, err
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return t, err
+	}
+	if info.Size() > MaxLog {
+		if _, err = f.Seek(-MaxLog, io.SeekEnd); err != nil {
+			return t, err
+		}
+	}
+	raw, err := io.ReadAll(f)
+	t.Log = string(raw)
+	return t, err
+}
+func (m *Manager) Start(r Request, actor string) (Task, error) {
+	if err := r.Validate(); err != nil {
+		return Task{}, err
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.active != "" {
+		return Task{}, ErrBusy
+	}
+	if err := m.ctx.Err(); err != nil {
+		return Task{}, err
+	}
+	var id [16]byte
+	if _, err := rand.Read(id[:]); err != nil {
+		return Task{}, err
+	}
+	t := Task{ID: hex.EncodeToString(id[:]), Request: r, Actor: actor, State: "running", Stage: "preflight", StartedAt: time.Now().UTC()}
+	if err := m.persist(t); err != nil {
+		return Task{}, err
+	}
+	m.tasks[t.ID] = t
+	m.active = t.ID
+	m.wg.Add(1)
+	go m.run(t)
+	return t, nil
+}
+func (m *Manager) run(t Task) {
+	defer m.wg.Done()
+	ctx, cancel := context.WithTimeout(m.ctx, 45*time.Minute)
+	defer cancel()
+	log, err := os.OpenFile(filepath.Join(m.dir, t.ID+".log"), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0600)
+	if err == nil {
+		stage := func(s string) error {
+			m.mu.Lock()
+			defer m.mu.Unlock()
+			t.Stage = s
+			if err := m.persist(t); err != nil {
+				return err
+			}
+			m.tasks[t.ID] = t
+			_, err := fmt.Fprintln(log, "[felis]", s)
+			return err
+		}
+		err = m.execute(ctx, t.Request, stage, &boundedLog{file: log})
+		if ctx.Err() != nil {
+			err = ctx.Err()
+		}
+		if closeErr := log.Close(); err == nil {
+			err = closeErr
+		}
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	now := time.Now().UTC()
+	t.FinishedAt = &now
+	t.State = "succeeded"
+	if err != nil {
+		t.State = "failed"
+		t.Error = err.Error()
+		if errors.Is(err, context.DeadlineExceeded) {
+			t.Error = "Host operation exceeded the 45-minute execution limit. Verify the host state before retrying."
+		}
+		if errors.Is(err, context.Canceled) {
+			t.Error = "Host operation was interrupted. Verify the host state before retrying."
+		}
+	}
+	if persistErr := m.persist(t); persistErr != nil {
+		t.State = "failed"
+		t.Error = "Task result persistence failed: " + persistErr.Error()
+	}
+	m.tasks[t.ID] = t
+	m.active = ""
+	for len(m.tasks) > 100 {
+		oldest := ""
+		for id, candidate := range m.tasks {
+			if id == t.ID {
+				continue
+			}
+			if oldest == "" || candidate.StartedAt.Before(m.tasks[oldest].StartedAt) {
+				oldest = id
+			}
+		}
+		if oldest != "" {
+			if err := os.Remove(filepath.Join(m.dir, oldest+".json")); err == nil {
+				os.Remove(filepath.Join(m.dir, oldest+".log"))
+				delete(m.tasks, oldest)
+			} else {
+				break
+			}
+		}
+	}
+}
+func (m *Manager) Wait() { m.wg.Wait() }
+
+// Handler is reachable only through the root-owned socket mounted into felis-api.
+func (m *Manager) Handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /tasks", func(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(m.List()) })
+	mux.HandleFunc("GET /tasks/{id}", func(w http.ResponseWriter, r *http.Request) {
+		t, err := m.Get(r.PathValue("id"))
+		if err != nil {
+			http.Error(w, err.Error(), 404)
+			return
+		}
+		json.NewEncoder(w).Encode(t)
+	})
+	mux.HandleFunc("POST /tasks", func(w http.ResponseWriter, r *http.Request) {
+		var body struct {
+			Request Request `json:"request"`
+			Actor   string  `json:"actor"`
+		}
+		dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 8192))
+		dec.DisallowUnknownFields()
+		if err := dec.Decode(&body); err != nil {
+			http.Error(w, "invalid task request", 400)
+			return
+		}
+		if err := dec.Decode(&struct{}{}); err != io.EOF {
+			http.Error(w, "invalid trailing task data", 400)
+			return
+		}
+		if err := body.Request.Validate(); err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		t, err := m.Start(body.Request, body.Actor)
+		if err != nil {
+			code := 500
+			if errors.Is(err, ErrBusy) {
+				code = 409
+			}
+			http.Error(w, err.Error(), code)
+			return
+		}
+		w.WriteHeader(http.StatusAccepted)
+		json.NewEncoder(w).Encode(t)
+	})
+	return mux
+}
+
+type Client struct{ http *http.Client }
+
+func NewClient(socket string) *Client {
+	return &Client{http: &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, "unix", socket)
+	}}}}
+}
+func (c *Client) call(ctx context.Context, method, path string, body any, out any) error {
+	var reader io.Reader
+	if body != nil {
+		raw, err := json.Marshal(body)
+		if err != nil {
+			return err
+		}
+		reader = strings.NewReader(string(raw))
+	}
+	req, err := http.NewRequestWithContext(ctx, method, "http://node-control"+path, reader)
+	if err != nil {
+		return err
+	}
+	res, err := c.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode == 409 {
+		return ErrBusy
+	}
+	if res.StatusCode == 404 {
+		return ErrNotFound
+	}
+	if res.StatusCode >= 400 {
+		return fmt.Errorf("host service returned HTTP %d", res.StatusCode)
+	}
+	return json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(out)
+}
+func (c *Client) List(ctx context.Context) ([]Task, error) {
+	var out []Task
+	err := c.call(ctx, "GET", "/tasks", nil, &out)
+	return out, err
+}
+func (c *Client) Get(ctx context.Context, id string) (Task, error) {
+	if !idPattern.MatchString(id) {
+		return Task{}, ErrNotFound
+	}
+	var out Task
+	err := c.call(ctx, "GET", "/tasks/"+id, nil, &out)
+	return out, err
+}
+func (c *Client) Start(ctx context.Context, r Request, actor string) (Task, error) {
+	var out Task
+	err := c.call(ctx, "POST", "/tasks", map[string]any{"request": r, "actor": actor}, &out)
+	return out, err
+}
+
+// Bound disk use while retaining the most recent output of long builds.
+type boundedLog struct {
+	file *os.File
+	mu   sync.Mutex
+}
+
+func (w *boundedLog) Write(p []byte) (int, error) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	const limit = 4 << 20
+	info, err := w.file.Stat()
+	if err != nil {
+		return 0, err
+	}
+	if info.Size()+int64(len(p)) > limit {
+		n := int64(MaxLog)
+		if info.Size() < n {
+			n = info.Size()
+		}
+		tail := make([]byte, n)
+		if _, err = w.file.ReadAt(tail, info.Size()-n); err != nil {
+			return 0, err
+		}
+		if err = w.file.Truncate(0); err != nil {
+			return 0, err
+		}
+		if _, err = w.file.Write(tail); err != nil {
+			return 0, err
+		}
+	}
+	original := len(p)
+	if len(p) > limit {
+		p = p[len(p)-limit:]
+	}
+	_, err = w.file.Write(p)
+	if err != nil {
+		return 0, err
+	}
+	return original, nil
+}

internal/nodecontrol/control_test.go

@@ -0,0 +1,192 @@
+package nodecontrol
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func validRequest() Request {
+	return Request{Action: "join", Name: "worker-01", SSHTarget: "[email protected]", ExternalIP: "192.0.2.10", ConfirmMaintenance: true}
+}
+func TestRequestRejectsUnsafeInputs(t *testing.T) {
+	for _, change := range []func(*Request){func(r *Request) { r.ConfirmMaintenance = false }, func(r *Request) { r.SSHTarget = "-oProxyCommand=sh" }, func(r *Request) { r.SSHTarget = "root@host;id" }, func(r *Request) { r.Name = "../../escape" }, func(r *Request) { r.ExternalIP = "127.0.0.1" }, func(r *Request) { r.Peers = []string{"10.0.0.0/8"} }, func(r *Request) { r.Action = "exec" }} {
+		r := validRequest()
+		change(&r)
+		if r.Validate() == nil {
+			t.Fatalf("accepted %+v", r)
+		}
+	}
+	if err := validRequest().Validate(); err != nil {
+		t.Fatal(err)
+	}
+}
+func TestNodeNames(t *testing.T) {
+	for _, name := range []string{"worker-01", "worker.localdomain"} {
+		r := validRequest()
+		r.Name = name
+		if err := r.Validate(); err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+	}
+	for _, name := range []string{"worker..localdomain", "worker.-domain", "worker.", strings.Repeat("a", 64)} {
+		r := validRequest()
+		r.Name = name
+		if r.Validate() == nil {
+			t.Fatalf("accepted %s", name)
+		}
+	}
+}
+
+func TestPersistentSerializedTasksAndLogs(t *testing.T) {
+	dir := t.TempDir()
+	entered := make(chan struct{})
+	finish := make(chan struct{})
+	manager, err := Open(context.Background(), dir, func(ctx context.Context, r Request, stage func(string) error, out io.Writer) error {
+		if err := stage("approval"); err != nil {
+			return err
+		}
+		io.WriteString(out, "probe failed\n")
+		close(entered)
+		<-finish
+		return errors.New("quarantine retained")
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	task, err := manager.Start(validRequest(), "owner")
+	if err != nil {
+		t.Fatal(err)
+	}
+	<-entered
+	if _, err = manager.Start(validRequest(), "owner"); !errors.Is(err, ErrBusy) {
+		t.Fatal("concurrent task", err)
+	}
+	live, err := manager.Get(task.ID)
+	if err != nil || live.Stage != "approval" || !strings.Contains(live.Log, "probe failed") {
+		t.Fatal(live, err)
+	}
+	close(finish)
+	manager.Wait()
+	reopened, err := Open(context.Background(), dir, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := reopened.Get(task.ID)
+	if err != nil || got.State != "failed" || got.Error != "quarantine retained" || got.FinishedAt == nil || got.Actor != "owner" {
+		t.Fatal(got, err)
+	}
+	if reopened.List()[0].Log != "" {
+		t.Fatal("task list leaked logs")
+	}
+}
+func TestInterruptedTaskIsNotReportedRunning(t *testing.T) {
+	dir := t.TempDir()
+	task := Task{ID: strings.Repeat("a", 32), Request: validRequest(), State: "running", StartedAt: time.Now()}
+	raw, _ := json.Marshal(task)
+	if err := os.WriteFile(filepath.Join(dir, task.ID+".json"), raw, 0600); err != nil {
+		t.Fatal(err)
+	}
+	manager, err := Open(context.Background(), dir, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := manager.Get(task.ID)
+	if err != nil || got.State != "failed" || got.FinishedAt == nil || !strings.Contains(got.Error, "restarted") {
+		t.Fatal(got, err)
+	}
+}
+func TestLocalClientAndBoundedOutput(t *testing.T) {
+	dir := t.TempDir()
+	manager, err := Open(context.Background(), dir, func(ctx context.Context, r Request, stage func(string) error, out io.Writer) error {
+		for range 90 {
+			if _, err := io.WriteString(out, strings.Repeat("x", MaxLog)); err != nil {
+				return err
+			}
+		}
+		io.WriteString(out, "last output")
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Avoid macOS' short Unix socket path limit under t.TempDir.
+	sock, err := os.CreateTemp("/tmp", "felis-node-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := sock.Name()
+	sock.Close()
+	os.Remove(path)
+	defer os.Remove(path)
+	listener, err := net.Listen("unix", path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	server := &http.Server{Handler: manager.Handler()}
+	go server.Serve(listener)
+	defer server.Close()
+	client := NewClient(path)
+	task, err := client.Start(context.Background(), validRequest(), "owner")
+	if err != nil {
+		t.Fatal(err)
+	}
+	manager.Wait()
+	got, err := client.Get(context.Background(), task.ID)
+	if err != nil || got.State != "succeeded" || len(got.Log) > MaxLog || !strings.HasSuffix(got.Log, "last output") {
+		t.Fatal("task/log", got.State, len(got.Log), err)
+	}
+	stat, err := os.Stat(filepath.Join(dir, task.ID+".log"))
+	if err != nil || stat.Size() > 4<<20 {
+		t.Fatal("unbounded file", stat, err)
+	}
+	if _, err := client.Get(context.Background(), "../secret"); !errors.Is(err, ErrNotFound) {
+		t.Fatal(err)
+	}
+}
+func TestCancellationFinishesTask(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	entered := make(chan struct{})
+	manager, err := Open(ctx, t.TempDir(), func(ctx context.Context, r Request, stage func(string) error, out io.Writer) error {
+		close(entered)
+		<-ctx.Done()
+		return ctx.Err()
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	task, err := manager.Start(validRequest(), "owner")
+	if err != nil {
+		t.Fatal(err)
+	}
+	<-entered
+	cancel()
+	manager.Wait()
+	got, err := manager.Get(task.ID)
+	if err != nil || got.State != "failed" {
+		t.Fatal(got, err)
+	}
+}
+
+// Optional live check under the API container's UID and SELinux domain.
+func TestHostSocketConnectivity(t *testing.T) {
+	socket := os.Getenv("FELIS_TEST_NODE_CONTROL_SOCKET")
+	if socket == "" {
+		t.Skip("live host socket not supplied")
+	}
+	tasks, err := NewClient(socket).List(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tasks == nil {
+		t.Fatal("host returned no task list")
+	}
+}

internal/platform/distributed_test.go

@@ -59,3 +59,43 @@ func TestDistributedControllerPinningAndArchivePowers(t *testing.T) {
 		t.Fatal("broad Velocity source accepted")
 	}
 }
+
+func TestNodeControlSocketIsAPIOnly(t *testing.T) {
+	p := testParams()
+	p.NodeControlSocket = "/run/felis-node-control/control.sock"
+	p.NodeControlNode = "controller"
+	for _, object := range Objects(p) {
+		d, ok := object.(*appsv1.Deployment)
+		if !ok {
+			continue
+		}
+		for _, volume := range d.Spec.Template.Spec.Volumes {
+			if volume.Name == "node-control" && d.Name != SAAPI {
+				t.Fatal("host socket leaked to", d.Name)
+			}
+		}
+		if d.Name != SAAPI {
+			continue
+		}
+		if d.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"] != "controller" {
+			t.Fatal("API can run away from socket")
+		}
+		found := false
+		for _, container := range d.Spec.Template.Spec.Containers {
+			for _, mount := range container.VolumeMounts {
+				if mount.Name == "node-control" {
+					found = true
+					if !mount.ReadOnly {
+						t.Fatal("host directory writable")
+					}
+				}
+			}
+			if d.Spec.Template.Spec.SecurityContext.RunAsUser == nil || *d.Spec.Template.Spec.SecurityContext.RunAsUser == 0 {
+				t.Fatal("API elevated")
+			}
+		}
+		if !found {
+			t.Fatal("API has no node socket")
+		}
+	}
+}

internal/platform/identities.go

@@ -90,6 +90,8 @@ const (
 // Params parameterises the install bundle. Namespaces and the registry location
 // have safe defaults; VelocityCIDRs has none — see the field comment.
 type Params struct {
+	NodeControlSocket string
+	NodeControlNode   string
 	Distributed       bool
 	ControllerNode    string
 	EgressProbe       string

internal/platform/workloads.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"path/filepath"
 	"time"
 
 	"felis.lolicon.best/internal/naming"
@@ -480,7 +481,18 @@ func APIDeployment(p Params) *appsv1.Deployment {
 		},
 	}
 
-	return controlPlaneDeployment(p, SAAPI, container, volumes)
+	if p.NodeControlSocket != "" {
+		directory := filepath.Dir(p.NodeControlSocket)
+		hostType := corev1.HostPathDirectory
+		volumes = append(volumes, corev1.Volume{Name: "node-control", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: directory, Type: &hostType}}})
+		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: "node-control", MountPath: directory, ReadOnly: true})
+		container.Env = append(container.Env, corev1.EnvVar{Name: "FELIS_NODE_CONTROL_SOCKET", Value: p.NodeControlSocket})
+	}
+	deployment := controlPlaneDeployment(p, SAAPI, container, volumes)
+	if p.NodeControlNode != "" && p.ControllerNode == "" {
+		deployment.Spec.Template.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": p.NodeControlNode}
+	}
+	return deployment
 }
 
 // apiService exposes the built-in HTTPS panel/API origin as a stable NodePort.

panel/dev/mockApi.ts

@@ -1019,6 +1019,10 @@ async function handlePublic(ctx: RequestContext): Promise<boolean> {
 
 async function handleSession(ctx: SessionContext): Promise<boolean> {
   switch (route(ctx)) {
+    case "GET settings/node-control":
+      if (!isOwner(ctx.account.role)) sendError(ctx.res, 403, "forbidden", "Owner account required");
+      else sendJSON(ctx.res, 200, { available: false, tasks: [] });
+      return true;
     case "GET settings/wake-policy":
     case "PUT settings/wake-policy": {
       if (!isOwner(ctx.account.role)) {

panel/e2e/startup-platform.smoke.spec.ts

@@ -84,3 +84,32 @@ test("Owner emergency stop stays secondary, confirms name, and distinguishes acc
   const links = await ownerNav.evaluateAll((elements) => elements.map((el) => el.getAttribute("href")));
   expect(links.at(-1)).toBe("/admin/platform");
 });
+
+test("Owner executes node management and receives stage, failure logs and retry", async ({ page, signIn }) => {
+  await signIn("owner");
+  let task: Record<string, unknown> = {};
+  let submitted: Record<string, unknown> | null = null;
+  await page.route("**/api/v1/settings/node-control", (route) => route.fulfill({ json: { available: true, tasks: task.id ? [task] : [] } }));
+  await page.route("**/api/v1/settings/node-control/tasks", async (route) => {
+    submitted = route.request().postDataJSON();
+    task = { id: "test-task", request: submitted, actor: "owner", state: "running", stage: "database_backup", startedAt: new Date().toISOString(), log: "[felis] database_backup" };
+    await route.fulfill({ status: 202, json: task });
+  });
+  await page.route("**/api/v1/settings/node-control/tasks/test-task", (route) => route.fulfill({ json: task }));
+  await page.route("**/api/v1/settings/node-control/tasks/test-task/retry", (route) => { task = { ...task, state: "running", error: undefined }; return route.fulfill({ status: 202, json: task }); });
+  await page.goto("/admin/platform");
+  await page.getByLabel(t("admin:node_control_ip")).fill("192.0.2.10");
+  const submit = page.getByRole("button", { name: t("admin:node_control_submit"), exact: true });
+  await expect(submit).toBeDisabled();
+  await page.getByRole("switch").click();
+  await submit.click();
+  expect(submitted).toMatchObject({ action: "enable", externalIP: "192.0.2.10", confirmMaintenance: true });
+  await expect(page.getByText(t("admin:node_control_stage_database_backup"), { exact: true })).toBeVisible();
+  task = { ...task, state: "failed", error: "Database backup failed", log: "[felis] pg_dump failed: connection refused" };
+  await page.getByRole("button", { name: t("admin:node_control_reload"), exact: true }).click();
+  await expect(page.getByText("Database backup failed", { exact: true })).toBeVisible();
+  await expect(page.getByLabel(t("admin:node_control_log"))).toContainText("pg_dump failed: connection refused");
+  await page.getByRole("button", { name: t("admin:node_control_retry"), exact: true }).click();
+  await expect(page.getByText(t("admin:node_control_state_running"), { exact: true })).toBeVisible();
+  await expectFitsScreen(page);
+});

panel/src/components/NodeControlPanel.test.tsx

@@ -0,0 +1,63 @@
+// @vitest-environment jsdom
+import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import userEvent from "@testing-library/user-event";
+import { NodeControlPanel } from "./NodeControlPanel";
+import type { NodeControlTask } from "@/lib/types";
+const calls = vi.hoisted(() => ({ nodeTasks: vi.fn(), nodeTask: vi.fn(), startNodeTask: vi.fn(), retryNodeTask: vi.fn() }));
+vi.mock("@/lib/api", async (original) => ({ ...await original<typeof import("@/lib/api")>(), api: calls }));
+const failed: NodeControlTask = { id: "task-1", request: { action: "approve", name: "worker-01", sshTarget: "worker-01", confirmMaintenance: true }, actor: "owner", state: "failed", stage: "approval", startedAt: "2026-10-06T10:00:00Z", error: "Network isolation probe failed", log: "probe: controller port rejected" };
+beforeEach(() => {
+  vi.clearAllMocks(); calls.nodeTasks.mockResolvedValue({ available: true, tasks: [] }); calls.nodeTask.mockResolvedValue(failed);
+});
+describe("host node management", () => {
+  it("keeps enrollment form visible and disabled while availability is unknown", async () => {
+    let resolve!: (value: { available: boolean; tasks: [] }) => void;
+    calls.nodeTasks.mockReturnValue(new Promise((r) => { resolve = r; }));
+    render(<NodeControlPanel distributed />);
+    expect(screen.getByLabelText("Node name")).toHaveProperty("disabled", true);
+    expect(screen.getByRole("button", { name: "Execute operation" })).toHaveProperty("disabled", true);
+    await act(async () => resolve({ available: true, tasks: [] }));
+    expect(screen.getByLabelText("Node name")).toHaveProperty("disabled", false);
+  });
+  it("updates the default action after the deployment mode loads", async () => {
+    const view = render(<NodeControlPanel distributed={null} />);
+    await waitFor(() => expect(calls.nodeTasks).toHaveBeenCalled());
+    expect(screen.queryByLabelText("Node name")).toBeNull();
+    view.rerender(<NodeControlPanel distributed />);
+    await waitFor(() => expect(screen.getByLabelText("Node name")).toHaveProperty("disabled", false));
+    expect(screen.getByRole("combobox", { name: "Operation" }).textContent).toContain("Enroll and approve worker");
+  });
+  it("requires maintenance acknowledgement and submits structured enrollment without credentials", async () => {
+    const running = { ...failed, state: "running", request: { ...failed.request, action: "join" } };
+    calls.startNodeTask.mockResolvedValue(running); calls.nodeTask.mockResolvedValue(running);
+    render(<NodeControlPanel distributed />);
+    await waitFor(() => expect(screen.getByLabelText("Node name")).toHaveProperty("disabled", false));
+    fireEvent.change(screen.getByLabelText("Node name"), { target: { value: "worker-01" } });
+    fireEvent.change(screen.getByLabelText("SSH target"), { target: { value: "[email protected]" } });
+    fireEvent.change(screen.getByLabelText("Fixed node IP"), { target: { value: "192.0.2.10" } });
+    expect(screen.getByRole("button", { name: "Execute operation" })).toHaveProperty("disabled", true);
+    await userEvent.click(screen.getByRole("switch"));
+    await userEvent.click(screen.getByRole("button", { name: "Execute operation" }));
+    expect(calls.startNodeTask).toHaveBeenCalledWith({ action: "join", name: "worker-01", sshTarget: "[email protected]", externalIP: "192.0.2.10", peers: [], confirmMaintenance: true });
+    await screen.findByRole("region", { name: "Task progress" });
+    expect(screen.getByRole("button", { name: "Execute operation" })).toHaveProperty("disabled", true);
+  });
+  it("restores failed tasks with concrete errors and logs and retries as a new attempt", async () => {
+    calls.nodeTasks.mockResolvedValue({ available: true, tasks: [failed] });
+    calls.retryNodeTask.mockResolvedValue({ ...failed, id: "task-2", state: "running" });
+    render(<NodeControlPanel distributed />);
+    expect(await screen.findByText("Network isolation probe failed")).toBeTruthy();
+    expect(screen.getByLabelText("Execution log").textContent).toContain("probe: controller port rejected");
+    await userEvent.click(screen.getByRole("button", { name: "Retry operation" }));
+    expect(calls.retryNodeTask).toHaveBeenCalledWith("task-1");
+    await waitFor(() => expect(calls.nodeTask).toHaveBeenCalledWith("task-2"));
+  });
+  it("reports a host connection failure and disables changes", async () => {
+    calls.nodeTasks.mockRejectedValue({ status: 503, code: "node_control_unavailable" });
+    render(<NodeControlPanel distributed={false} />);
+    expect(await screen.findByRole("alert")).toBeTruthy();
+    expect(screen.getByRole("button", { name: "Execute operation" })).toHaveProperty("disabled", true);
+    expect(calls.startNodeTask).not.toHaveBeenCalled();
+  });
+});

panel/src/components/NodeControlPanel.tsx

@@ -0,0 +1,90 @@
+import { useEffect, useState } from "react";
+import { Loader2, Play, RefreshCw } from "lucide-react";
+import { useTranslation } from "react-i18next";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Switch } from "@/components/ui/switch";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import { MessageLine } from "@/components/MessageLine";
+import { isReauthCancelled, useReauth } from "@/components/ReauthDialog";
+import { api, humanizeError } from "@/lib/api";
+import { useAsync, usePolling } from "@/lib/hooks";
+import type { NodeControlRequest, NodeControlTask } from "@/lib/types";
+
+export function NodeControlPanel({ distributed }: { distributed: boolean | null }) {
+  const { t } = useTranslation("admin");
+  const reauth = useReauth();
+  const tasks = useAsync(api.nodeTasks, [], { coalesce: true });
+  const [id, setId] = useState<string | null>(null);
+  const [action, setAction] = useState<NodeControlRequest["action"]>(distributed ? "join" : "enable");
+  const [name, setName] = useState("");
+  const [ssh, setSSH] = useState("");
+  const [ip, setIP] = useState("");
+  const [peers, setPeers] = useState("");
+  const [confirmed, setConfirmed] = useState(false);
+  const [submitting, setSubmitting] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const progress = useAsync(() => id ? api.nodeTask(id) : Promise.resolve(null), [id], { coalesce: true });
+  const task = progress.data;
+  const running = tasks.data?.tasks.some((entry) => entry.state === "running") || task?.state === "running";
+  const busy = submitting || !!running;
+  const available = tasks.data?.available === true && !tasks.error && distributed !== null;
+  useEffect(() => { if (distributed !== null) { setAction(distributed ? "join" : "enable"); setConfirmed(false); } }, [distributed]);
+  useEffect(() => { if (!id && tasks.data?.tasks[0]) setId(tasks.data.tasks[0].id); }, [id, tasks.data]);
+  usePolling(() => { tasks.reload(); if (id) progress.reload(); }, running || progress.error || tasks.error ? 3000 : 10000);
+  async function run(retry = false) {
+    if (!available || busy || (!retry && !confirmed)) return;
+    setSubmitting(true); setError(null);
+    try {
+      const result: NodeControlTask = await reauth.guard(() => retry && id ? api.retryNodeTask(id) : api.startNodeTask({
+        action, ...(action !== "enable" ? { name: name.trim(), sshTarget: ssh.trim() } : {}),
+        ...(action !== "approve" ? { externalIP: ip.trim(), peers: peers.split(/[\s,]+/).filter(Boolean) } : {}),
+        confirmMaintenance: confirmed,
+      }));
+      setId(result.id); setConfirmed(false); tasks.reload();
+    } catch (e) { if (!isReauthCancelled(e)) setError(humanizeError(e)); }
+    finally { setSubmitting(false); }
+  }
+  const valid = confirmed && (action === "enable" || !!name.trim() && !!ssh.trim()) && (action === "approve" || !!ip.trim());
+  return <Card>
+    <CardHeader className="flex-row items-center justify-between gap-3"><CardTitle>{t("node_control_title")}</CardTitle><Button variant="outline" size="sm" disabled={tasks.loading} onClick={() => { tasks.reload(); progress.reload(); }}><RefreshCw />{t("node_control_reload")}</Button></CardHeader>
+    <CardContent className="space-y-5">
+      <p className="text-sm leading-relaxed text-muted-foreground">{t("node_control_description")}</p>
+      {tasks.loading && !tasks.data && <p role="status" className="flex items-center gap-2 text-sm text-muted-foreground"><Loader2 className="h-4 w-4 animate-spin" />{t("node_control_loading")}</p>}
+      {tasks.error != null && <MessageLine kind="error" message={humanizeError(tasks.error)} />}
+      {tasks.data?.available === false && <p role="status" className="text-sm text-muted-foreground">{t("node_control_unavailable")}</p>}
+      {running && task?.state !== "running" && <div className="flex flex-wrap items-center gap-3"><p role="status" className="text-sm text-muted-foreground">{t("node_control_other_running")}</p><Button variant="outline" size="sm" onClick={() => { const active = tasks.data?.tasks.find((entry) => entry.state === "running"); if (active) setId(active.id); }}>{t("node_control_show_running")}</Button></div>}
+      <div className="grid gap-5 md:grid-cols-2">
+        <div className="space-y-2"><Label htmlFor="node-action">{t("node_control_action")}</Label><Select value={action} disabled={!available || busy} onValueChange={(value) => { setAction(value as NodeControlRequest["action"]); setConfirmed(false); }}><SelectTrigger id="node-action"><SelectValue /></SelectTrigger><SelectContent>{!distributed && <SelectItem value="enable">{t("node_control_enable")}</SelectItem>}<SelectItem value="join" disabled={!distributed}>{t("node_control_join")}</SelectItem><SelectItem value="approve" disabled={!distributed}>{t("node_control_approve")}</SelectItem></SelectContent></Select></div>
+        {action !== "enable" && <>
+          <div className="space-y-2"><Label htmlFor="node-name">{t("node_control_name")}</Label><Input id="node-name" value={name} disabled={!available || busy} onChange={(e) => setName(e.target.value)} placeholder="worker-01" /></div>
+          <div className="space-y-2"><Label htmlFor="node-ssh">{t("node_control_ssh")}</Label><Input id="node-ssh" value={ssh} disabled={!available || busy} onChange={(e) => setSSH(e.target.value)} placeholder="[email protected]" /><p className="text-xs text-muted-foreground">{t("node_control_ssh_hint")}</p></div>
+        </>}
+        {action !== "approve" && <>
+          <div className="space-y-2"><Label htmlFor="node-ip">{t("node_control_ip")}</Label><Input id="node-ip" value={ip} disabled={!available || busy} onChange={(e) => setIP(e.target.value)} placeholder="192.168.1.20" /></div>
+          <div className="space-y-2 md:col-span-2"><Label htmlFor="node-peers">{t("node_control_peers")}</Label><Input id="node-peers" value={peers} disabled={!available || busy} onChange={(e) => setPeers(e.target.value)} placeholder="192.168.1.21/32, 192.168.1.22/32" /><p className="text-xs text-muted-foreground">{t("node_control_peers_hint")}</p></div>
+        </>}
+      </div>
+      <div className="flex items-start gap-3 rounded-lg border border-border p-4"><Switch id="node-confirm" checked={confirmed} disabled={!available || busy} onCheckedChange={setConfirmed} /><Label htmlFor="node-confirm" className="text-sm leading-relaxed">{t(action === "enable" ? "node_control_confirm_enable" : "node_control_confirm_worker")}</Label></div>
+      {error && <MessageLine kind="error" message={error} />}
+      <Button disabled={!available || busy || !valid} onClick={() => void run()}>{submitting ? <Loader2 className="animate-spin" /> : <Play />}{t("node_control_submit")}</Button>
+      <details className="rounded-lg border border-border p-4"><summary className="cursor-pointer text-sm font-medium">{t("node_control_prerequisites")}</summary><p className="mt-3 text-sm leading-relaxed text-muted-foreground">{t("node_control_prerequisites_body")}</p><Button asChild variant="outline" size="sm" className="mt-3"><a href="https://github.com/FelisMC/Felis/blob/main/docs/distributed.md" target="_blank" rel="noreferrer">{t("platform_distribution_runbook")}</a></Button></details>
+      {!!tasks.data?.tasks.length && <div className="space-y-2"><Label htmlFor="node-task">{t("node_control_history")}</Label><Select value={id ?? undefined} onValueChange={setId}><SelectTrigger id="node-task"><SelectValue /></SelectTrigger><SelectContent>{tasks.data.tasks.map((entry) => <SelectItem key={entry.id} value={entry.id}>{t(`node_control_${entry.request.action}`)} · {entry.request.name || t("node_control_controller")} · {new Date(entry.startedAt).toLocaleString()} · {t(`node_control_state_${entry.state}`)}</SelectItem>)}</SelectContent></Select></div>}
+      {progress.error != null && <MessageLine kind="error" message={t("node_control_connection_lost", { error: humanizeError(progress.error) })} />}
+      {id && progress.loading && !task && <p role="status" className="text-sm text-muted-foreground">{t("node_control_loading")}</p>}
+      {task && <section aria-label={t("node_control_progress")} className="space-y-3 rounded-lg border border-border p-4">
+        <div className="flex flex-wrap items-center gap-3"><Badge variant={task.state === "failed" ? "destructive" : task.state === "succeeded" ? "default" : "muted"}>{t(`node_control_state_${task.state === "running" && (progress.error || tasks.error) ? "unknown" : task.state}`)}</Badge><span className="text-sm">{t(`node_control_stage_${task.stage}`, { defaultValue: task.stage })}</span></div>
+        <p className="text-xs text-muted-foreground">{t("node_control_task_id")}: <code>{task.id}</code></p>
+        <p className="text-xs text-muted-foreground">{t("node_control_started", { time: new Date(task.startedAt).toLocaleString() })} · {t("node_control_timeout")}</p>
+        {task.error && <MessageLine kind="error" message={task.error} />}
+        <pre aria-label={t("node_control_log")} className="max-h-80 overflow-auto rounded-md bg-muted p-3 font-mono text-xs leading-relaxed whitespace-pre-wrap break-all">{task.log || t("node_control_log_pending")}</pre>
+        {task.state === "failed" && <><p className="text-sm text-muted-foreground">{t("node_control_retry_hint")}</p><Button variant="outline" disabled={!available || busy} onClick={() => void run(true)}><RefreshCw />{t("node_control_retry")}</Button></>}
+        {task.state === "succeeded" && task.request.action === "enable" && <Button variant="outline" onClick={() => window.location.reload()}>{t("node_control_refresh_deployment")}</Button>}
+      </section>}
+    </CardContent>
+    {reauth.dialog}
+  </Card>;
+}

panel/src/i18n/resources/en-US/admin.json

@@ -335,5 +335,57 @@
   "platform_distribution_approve": "Approve the node from the controller. The command verifies identity, network isolation, image pulling, and cross-node connectivity. Failed checks preserve isolation. Approval removes the specified cached image and must run before the node hosts game workloads.",
   "platform_distribution_runbook": "Deployment and acceptance runbook",
   "platform_distribution_management": "Online approved workers can be selected when creating a server. Existing servers must be fully stopped before migration. Successful migration leaves the server stopped and retains its source world volume.",
-  "platform_distribution_servers": "Manage server placement and migration"
+  "platform_distribution_servers": "Manage server placement and migration",
+  "node_control_title": "Node operations",
+  "node_control_description": "Enable distributed deployment, enroll workers and run admission checks. Tasks execute on the controller host; panel disconnection does not terminate a task.",
+  "node_control_loading": "Loading host task status.",
+  "node_control_unavailable": "Host node service is not installed. Update the controller installation to enable panel management.",
+  "node_control_action": "Operation",
+  "node_control_enable": "Enable distributed deployment",
+  "node_control_join": "Enroll and approve worker",
+  "node_control_approve": "Approve existing worker",
+  "node_control_name": "Node name",
+  "node_control_ssh": "SSH target",
+  "node_control_ssh_hint": "Use an alias from the controller root SSH configuration or user@host. Noninteractive authentication and verified host fingerprints are required.",
+  "node_control_ip": "Fixed node IP",
+  "node_control_peers": "Additional peer addresses (optional)",
+  "node_control_peers_hint": "Enter exact /32 or /128 addresses separated by commas. Existing nodes and the new node are included automatically.",
+  "node_control_confirm_enable": "All game servers (including Login and Lobby) and maintenance tasks are stopped. Database and cluster backups and controller network changes are authorized. Panel access may be temporarily interrupted.",
+  "node_control_confirm_worker": "Game and maintenance workloads on the target node are stopped. Node installation, peer firewall updates, image pulls and network isolation checks are authorized.",
+  "node_control_submit": "Execute operation",
+  "node_control_prerequisites": "Environment and SSH requirements",
+  "node_control_prerequisites_body": "Controller and workers require matching architecture and k3s versions and fixed addresses. SSH accounts require root or sudo -n access. Configure keys and verify host fingerprints on the controller host before first use; unknown fingerprints are rejected. Existing worker SSH aliases must match node names for peer firewall updates. Enrollment requires a host without k3s. Nodes already joined after an interrupted enrollment must use “Approve existing worker”. Failed admission retains quarantine and does not start game servers.",
+  "node_control_history": "Task history",
+  "node_control_controller": "Controller",
+  "node_control_state_running": "Running",
+  "node_control_state_succeeded": "Completed",
+  "node_control_state_failed": "Failed",
+  "node_control_connection_lost": "Task status is temporarily unavailable: {{error}}. Polling resumes when connectivity is restored; the last confirmed status remains visible.",
+  "node_control_progress": "Task progress",
+  "node_control_task_id": "Task ID",
+  "node_control_log": "Execution log",
+  "node_control_log_pending": "No execution output yet.",
+  "node_control_retry_hint": "Verify current host state and resolve the failure before retrying. Retry creates a new task and repeats checks. Already enrolled workers require approval instead of enrollment.",
+  "node_control_retry": "Retry operation",
+  "node_control_refresh_deployment": "Refresh deployment status",
+  "node_control_stage_preflight": "Checking prerequisites",
+  "node_control_stage_pause_operator": "Pausing operator reconciliation",
+  "node_control_stage_database_backup": "Backing up database",
+  "node_control_stage_cluster_backup": "Backing up cluster state",
+  "node_control_stage_configure_controller": "Configuring controller identity",
+  "node_control_stage_install_controller": "Deploying controller components and network",
+  "node_control_stage_ssh_check": "Checking SSH and host environment",
+  "node_control_stage_firewall": "Updating peer firewalls",
+  "node_control_stage_bootstrap_token": "Creating temporary enrollment token",
+  "node_control_stage_transfer": "Transferring installation files",
+  "node_control_stage_install_worker": "Installing worker components",
+  "node_control_stage_wait_ready": "Waiting for node readiness",
+  "node_control_stage_approval": "Running node admission checks",
+  "node_control_stage_complete": "Operation completed",
+  "node_control_reload": "Refresh node tasks",
+  "node_control_state_unknown": "State unconfirmed",
+  "node_control_started": "Started: {{time}}",
+  "node_control_timeout": "Operation timeout: 45 minutes",
+  "node_control_other_running": "Another node task is running; new node operations are temporarily blocked.",
+  "node_control_show_running": "View running task"
 }

panel/src/i18n/resources/en-US/errors.json

@@ -145,5 +145,7 @@
   "auth_sources_unavailable": "Authentication source management is unavailable.",
   "auth_sources_changed": "Authentication sources changed. Discard your edits and reload before saving.",
   "auth_source_tag_locked": "Saved source IDs cannot be renamed or removed. Disable the source instead.",
-  "distributed_unavailable": "Distributed deployment is not configured. Configure the controller and worker nodes using the deployment runbook first."
+  "distributed_unavailable": "Distributed deployment is not configured. Configure the controller and worker nodes using the deployment runbook first.",
+  "node_operation_busy": "Node maintenance is running. Server starts and additional node tasks are temporarily blocked.",
+  "node_control_unavailable": "Host node service is unavailable. Inspect felis-node-control.service on the controller."
 }

panel/src/i18n/resources/zh-CN/admin.json

@@ -331,5 +331,57 @@
   "platform_distribution_approve": "在主控批准节点。该命令验证节点身份、网络隔离、镜像拉取和跨节点连通性；任何检查失败均保留隔离状态。批准检查会删除指定缓存镜像，须在该节点尚无游戏任务时执行。",
   "platform_distribution_runbook": "查看完整部署与验收流程",
   "platform_distribution_management": "在线且已批准的 worker 可在创建服务器时选择。已有服务器须完全停止后才能迁移；迁移完成后仍保持停止，源世界卷保留。",
-  "platform_distribution_servers": "管理服务器节点与迁移"
+  "platform_distribution_servers": "管理服务器节点与迁移",
+  "node_control_title": "节点操作",
+  "node_control_description": "启用分布式部署、接入 worker 和执行节点批准。任务在主控宿主机执行；面板连接中断不代表任务终止。",
+  "node_control_loading": "正在读取主机任务状态。",
+  "node_control_unavailable": "主机节点执行服务尚未安装。更新主控安装后可启用面板管理。",
+  "node_control_action": "操作类型",
+  "node_control_enable": "启用分布式部署",
+  "node_control_join": "接入并批准 worker",
+  "node_control_approve": "批准已有 worker",
+  "node_control_name": "节点名称",
+  "node_control_ssh": "SSH 目标",
+  "node_control_ssh_hint": "使用主控 root SSH 配置中的别名或 user@host。要求免交互认证及已核验的主机指纹。",
+  "node_control_ip": "节点固定 IP",
+  "node_control_peers": "额外对等节点地址（选填）",
+  "node_control_peers_hint": "填写精确的 /32 或 /128 地址，以逗号分隔。系统自动包含现有节点及本次接入节点的地址。",
+  "node_control_confirm_enable": "已停止全部游戏服务器（含登录空间与正式大厅）及维护任务，确认允许备份数据库与集群状态，并变更主控网络配置。期间面板可能暂时不可访问。",
+  "node_control_confirm_worker": "已停止目标节点上的游戏与维护任务。确认允许安装节点组件、更新对等防火墙并执行镜像拉取及网络隔离验收。",
+  "node_control_submit": "执行操作",
+  "node_control_prerequisites": "环境与 SSH 要求",
+  "node_control_prerequisites_body": "主控与 worker 需使用相同架构及 k3s 版本，节点使用固定地址。SSH 账户须为 root 或具备 sudo -n 权限。首次使用前，应在主控宿主机配置密钥并核验各主机指纹；系统不会自动接受未知指纹。已有 worker 的 SSH 别名应与节点名称一致，以便更新对等防火墙。接入仅用于尚未安装 k3s 的主机；中断后已加入集群的节点应使用“批准已有 worker”。批准失败时保留隔离状态，不自动启动游戏服务器。",
+  "node_control_history": "任务记录",
+  "node_control_controller": "主控",
+  "node_control_state_running": "执行中",
+  "node_control_state_succeeded": "已完成",
+  "node_control_state_failed": "执行失败",
+  "node_control_connection_lost": "任务状态暂时无法读取：{{error}}。连接恢复后自动继续查询；当前展示最后一次已确认的状态。",
+  "node_control_progress": "任务进度",
+  "node_control_task_id": "任务 ID",
+  "node_control_log": "执行日志",
+  "node_control_log_pending": "尚未产生执行日志。",
+  "node_control_retry_hint": "重试前需核对主机当前状态并修复失败原因。重试创建新任务并重新执行检查；已接入的 worker 应改用批准操作。",
+  "node_control_retry": "重试操作",
+  "node_control_refresh_deployment": "刷新部署状态",
+  "node_control_stage_preflight": "检查环境",
+  "node_control_stage_pause_operator": "暂停控制器调谐",
+  "node_control_stage_database_backup": "备份数据库",
+  "node_control_stage_cluster_backup": "备份集群状态",
+  "node_control_stage_configure_controller": "配置主控身份",
+  "node_control_stage_install_controller": "部署主控组件与网络",
+  "node_control_stage_ssh_check": "检查 SSH 与主机环境",
+  "node_control_stage_firewall": "更新对等防火墙",
+  "node_control_stage_bootstrap_token": "创建限时接入令牌",
+  "node_control_stage_transfer": "传输节点安装文件",
+  "node_control_stage_install_worker": "安装 worker 组件",
+  "node_control_stage_wait_ready": "等待节点就绪",
+  "node_control_stage_approval": "执行节点验收",
+  "node_control_stage_complete": "操作完成",
+  "node_control_reload": "刷新节点任务",
+  "node_control_state_unknown": "状态未确认",
+  "node_control_started": "开始时间：{{time}}",
+  "node_control_timeout": "操作超时上限为 45 分钟",
+  "node_control_other_running": "另一项节点任务正在执行，当前暂不允许新的节点操作。",
+  "node_control_show_running": "查看执行中任务"
 }

panel/src/i18n/resources/zh-CN/errors.json

@@ -145,5 +145,7 @@
   "auth_sources_unavailable": "认证源管理暂不可用。",
   "auth_sources_changed": "认证源已被其他操作修改。请放弃当前更改并重新读取后再保存。",
   "auth_source_tag_locked": "已保存的认证源标识不能改名或移除，请使用停用。",
-  "distributed_unavailable": "分布式部署未配置。请先按部署流程配置主控与 worker 节点。"
+  "distributed_unavailable": "分布式部署未配置。请先按部署流程配置主控与 worker 节点。",
+  "node_operation_busy": "节点维护正在执行，暂不允许启动服务器或创建另一项节点任务。",
+  "node_control_unavailable": "主机节点执行服务不可用。请检查主控上的 felis-node-control.service。"
 }

panel/src/lib/api.ts

@@ -1,4 +1,6 @@
 import type {
+  NodeControlRequest,
+  NodeControlTask,
   AccessResult,
   ExecutionNode,
   WorldMigration,
@@ -610,6 +612,10 @@ export const api = rejectingSync({
     URL.revokeObjectURL(url);
   },
 
+  nodeTasks: () => request<{ available: boolean; tasks: NodeControlTask[] }>("GET", "/settings/node-control"),
+  nodeTask: (id: string) => request<NodeControlTask>("GET", `/settings/node-control/tasks/${encodeURIComponent(id)}`),
+  startNodeTask: (body: NodeControlRequest) => request<NodeControlTask>("POST", "/settings/node-control/tasks", body),
+  retryNodeTask: (id: string) => request<NodeControlTask>("POST", `/settings/node-control/tasks/${encodeURIComponent(id)}/retry`),
   nodes: () => request<{ nodes: ExecutionNode[] }>("GET", "/nodes").then((r) => r.nodes),
   migration: (name: string) => request<WorldMigration>("GET", `/servers/${encodeURIComponent(name)}/migrations`),
   migrateServer: (name: string, targetNode: string) =>
@@ -1253,6 +1259,10 @@ export function humanizeError(e: unknown): string {
   switch (err.code) {
     // Session doors (spec §B): every passwordless door 403s this when local
     // sessions are disabled on a Zero-Trust-only deployment.
+    case "node_operation_busy":
+      return t("node_operation_busy");
+    case "node_control_unavailable":
+      return t("node_control_unavailable");
     case "distributed_unavailable":
       return t("distributed_unavailable");
     case "local_auth_disabled":

panel/src/lib/openapi.gen.ts

@@ -1263,6 +1263,77 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/api/v1/settings/node-control": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /** Read host node-service availability and persistent task history (Owner). */
+        get: operations["nodeTasks"];
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+    "/api/v1/settings/node-control/tasks": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /**
+         * Execute fixed host node operation (Owner, fresh reauthentication).
+         * @description Tasks are serialized, audited and persisted on the controller. The host verifies stopped workloads and SSH trust before enrollment or admission. Bootstrap credentials never cross the external API. Server starts fail closed during node operations and when the configured host service cannot report its state.
+         */
+        post: operations["startNodeTask"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+    "/api/v1/settings/node-control/tasks/{id}": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /** Read task stage, terminal error and bounded execution log (Owner). */
+        get: operations["nodeTask"];
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+    "/api/v1/settings/node-control/tasks/{id}/retry": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** Create a new attempt from a failed task (Owner, fresh reauthentication). */
+        post: operations["retryNodeTask"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/api/v1/settings/wake-policy": {
         parameters: {
             query?: never;
@@ -3128,6 +3199,35 @@ export interface components {
             attempt: number;
             error?: string;
         };
+        NodeControlRequest: {
+            /** @enum {string} */
+            action: "enable" | "join" | "approve";
+            /** @description Stable worker node name; required for join and approve. */
+            name?: string;
+            /** @description Verified root SSH alias or user@host; required for workers. */
+            sshTarget?: string;
+            /** @description Fixed node IP; required for enable and join. */
+            externalIP?: string;
+            /** @description Additional exact /32 or /128 peer addresses. */
+            peers?: string[];
+            /** @constant */
+            confirmMaintenance: true;
+        };
+        NodeControlTask: {
+            id: string;
+            request: components["schemas"]["NodeControlRequest"];
+            actor: string;
+            /** @enum {string} */
+            state: "running" | "succeeded" | "failed";
+            stage: string;
+            /** Format: date-time */
+            startedAt: string;
+            /** Format: date-time */
+            finishedAt?: string;
+            error?: string;
+            /** @description Most recent 64 KiB of host execution output; absent in task lists. */
+            log?: string;
+        };
         /** @description Status projection of one server (internal/api/cluster.go ServerInfo). */
         ServerInfo: {
             /** @description Private runtime and startup diagnostics for authorized server managers. */
@@ -6825,6 +6925,167 @@ export interface operations {
             401: components["responses"]["Unauthorized"];
         };
     };
+    nodeTasks: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Availability and most recent task records; no credentials or logs. */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": {
+                        available: boolean;
+                        tasks: components["schemas"]["NodeControlTask"][];
+                    };
+                };
+            };
+            400: components["responses"]["BadRequest"];
+            401: components["responses"]["Unauthorized"];
+            403: components["responses"]["Forbidden"];
+            /** @description Another node task is running or this task cannot be retried. */
+            409: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content?: never;
+            };
+            /** @description Host node execution service is unavailable. */
+            503: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content?: never;
+            };
+        };
+    };
+    startNodeTask: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["NodeControlRequest"];
+            };
+        };
+        responses: {
+            /** @description Persistent task created. */
+            202: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["NodeControlTask"];
+                };
+            };
+            400: components["responses"]["BadRequest"];
+            401: components["responses"]["Unauthorized"];
+            403: components["responses"]["Forbidden"];
+            /** @description Another node task is running or this task cannot be retried. */
+            409: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content?: never;
+            };
+            /** @description Host node execution service is unavailable. */
+            503: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content?: never;
+            };
+        };
+    };
+    nodeTask: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                id: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Current task and recent output. */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["NodeControlTask"];
+                };
+            };
+            400: components["responses"]["BadRequest"];
+            401: components["responses"]["Unauthorized"];
+            403: components["responses"]["Forbidden"];
+            404: components["responses"]["NotFound"];
+            /** @description Another node task is running or this task cannot be retried. */
+            409: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content?: never;
+            };
+            /** @description Host node execution service is unavailable. */
+            503: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content?: never;
+            };
+        };
+    };
+    retryNodeTask: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                id: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description New task created; original task is retained. */
+            202: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["NodeControlTask"];
+                };
+            };
+            400: components["responses"]["BadRequest"];
+            401: components["responses"]["Unauthorized"];
+            403: components["responses"]["Forbidden"];
+            404: components["responses"]["NotFound"];
+            /** @description Another node task is running or this task cannot be retried. */
+            409: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content?: never;
+            };
+            /** @description Host node execution service is unavailable. */
+            503: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content?: never;
+            };
+        };
+    };
     getWakePolicy: {
         parameters: {
             query?: never;

panel/src/lib/types.ts

@@ -776,3 +776,23 @@ export interface WakePolicySettings {
   revision: string;
   managed: boolean;
 }
+
+export interface NodeControlRequest {
+  action: "enable" | "join" | "approve";
+  name?: string;
+  sshTarget?: string;
+  externalIP?: string;
+  peers?: string[];
+  confirmMaintenance: boolean;
+}
+export interface NodeControlTask {
+  id: string;
+  request: NodeControlRequest;
+  actor: string;
+  state: "running" | "succeeded" | "failed";
+  stage: string;
+  startedAt: string;
+  finishedAt?: string;
+  error?: string;
+  log?: string;
+}

panel/src/pages/admin/PlatformSettingsPage.test.tsx

@@ -5,7 +5,7 @@ import userEvent from "@testing-library/user-event";
 import { MemoryRouter } from "react-router-dom";
 import { PlatformSettingsPage } from "./PlatformSettingsPage";
 import type { WakePolicySettings } from "@/lib/types";
-const calls = vi.hoisted(() => ({ getWakePolicy: vi.fn(), setWakePolicy: vi.fn(), nodes: vi.fn() }));
+const calls = vi.hoisted(() => ({ getWakePolicy: vi.fn(), setWakePolicy: vi.fn(), nodes: vi.fn(), nodeTasks: vi.fn(), nodeTask: vi.fn(), startNodeTask: vi.fn(), retryNodeTask: vi.fn() }));
 vi.mock("@/lib/api", async (original) => ({ ...await original<typeof import("@/lib/api")>(), api: calls }));
 const runtime = vi.hoisted(() => ({ distributed: false, fallback: false, apiBase: "/api/v1", rootDomain: "example.test" }));
 vi.mock("@/lib/hooks", async (original) => ({ ...await original<typeof import("@/lib/hooks")>(), useConfig: () => runtime }));
@@ -18,6 +18,7 @@ beforeEach(() => {
   runtime.distributed = false;
   runtime.fallback = false;
   calls.nodes.mockResolvedValue([]);
+  calls.nodeTasks.mockResolvedValue({ available: false, tasks: [] });
   calls.getWakePolicy.mockResolvedValue(policy);
   calls.setWakePolicy.mockImplementation(async (body) => ({ ...body, revision: "saved", managed: true }));
 });
@@ -28,7 +29,7 @@ describe("platform policy", () => {
     page();
     expect(limit().disabled).toBe(true);
     expect(limit().value).toBe("");
-    expect(screen.getByRole("status").textContent).toContain("Loading the saved policy");
+    expect(screen.getAllByRole("status").some((node) => node.textContent?.includes("Loading the saved policy"))).toBe(true);
     await act(async () => resolve(policy));
     expect(limit().disabled).toBe(false);
     expect(limit().value).toBe("0");
@@ -57,9 +58,8 @@ describe("distributed deployment entry", () => {
     page();
     await waitFor(() => expect(limit().disabled).toBe(false));
     expect(screen.getByText("Single-node mode")).toBeTruthy();
-    expect(screen.getByText(/cannot be switched live from the panel/)).toBeTruthy();
-    expect(screen.getByText("Connect and approve worker nodes")).toBeTruthy();
-    expect(screen.getByRole("link", { name: "Deployment and acceptance runbook" }).getAttribute("href")).toContain("docs/distributed.md");
+    expect(screen.getByText("Node operations")).toBeTruthy();
+    expect(await screen.findByText(/Host node service is not installed/)).toBeTruthy();
     expect(calls.nodes).not.toHaveBeenCalled();
   });
 

panel/src/pages/admin/PlatformSettingsPage.tsx

@@ -2,6 +2,7 @@ import { Link } from "react-router-dom";
 import { useEffect, useState } from "react";
 import { Loader2, RefreshCw, Save, Settings } from "lucide-react";
 import { useTranslation } from "react-i18next";
+import { NodeControlPanel } from "@/components/NodeControlPanel";
 import { DistributedNodes } from "@/components/DistributedNodes";
 import { Badge } from "@/components/ui/badge";
 import { PageHeader } from "@/components/PageHeader";
@@ -75,21 +76,11 @@ export function PlatformSettingsPage() {
           {t(!config || config.fallback ? "platform_distribution_unknown" : config.distributed ? "platform_distribution_enabled" : "platform_distribution_disabled")}
         </Badge>
         <p className="text-sm leading-relaxed text-muted-foreground">{t("platform_distribution_architecture")}</p>
-        {config && !config.fallback && !config.distributed && <p className="text-sm leading-relaxed">{t("platform_distribution_enable_hint")}</p>}
-        <details className="rounded-lg border border-border p-4">
-          <summary className="cursor-pointer text-sm font-medium">{t("platform_distribution_join")}</summary>
-          <ol className="mt-4 list-decimal space-y-3 pl-5 text-sm leading-relaxed text-muted-foreground">
-            <li>{t("platform_distribution_prepare")}</li>
-            <li>{t("platform_distribution_token")}<pre className="mt-2 overflow-x-auto rounded-md bg-muted p-3 text-xs">{"felis node token --name <worker-name> --ttl 10m --out /root/worker.bootstrap"}</pre></li>
-            <li>{t("platform_distribution_worker")}</li>
-            <li>{t("platform_distribution_approve")}<pre className="mt-2 overflow-x-auto rounded-md bg-muted p-3 text-xs">{"felis node approve --name <worker-name> --ssh-target <worker-ssh-alias> --image <felis-image>\nfelis node list"}</pre></li>
-          </ol>
-          <Button asChild variant="outline" size="sm" className="mt-4"><a href="https://github.com/FelisMC/Felis/blob/main/docs/distributed.md" target="_blank" rel="noreferrer">{t("platform_distribution_runbook")}</a></Button>
-        </details>
         <p className="text-sm leading-relaxed text-muted-foreground">{t("platform_distribution_management")}</p>
         <Button asChild variant="outline" size="sm"><Link to="/servers">{t("platform_distribution_servers")}</Link></Button>
       </CardContent>
     </Card>
+    <NodeControlPanel distributed={!config || config.fallback ? null : !!config.distributed} />
     {config?.distributed && !config.fallback && <DistributedNodes />}
     {reauth.dialog}
   </div>;