feat(distributed): 支持单主控多节点部署和停服迁移

复用现有 k3s 调度和 Job 生命周期，增加 worker 接入与批准、受保护节点身份、归档传输、持久迁移锁及活动 PVC 切换；同步管理员 API、CLI、面板和隔离规则。分布式模式默认关闭，保持单机兼容。验证：Go 全量测试与 vet；面板 874 个测试、lint/build；Linux VM 安装器测试、清单服务端 dry-run、网络命名空间防火墙实测。A/B/C 三机 WireGuard、Velocity 和迁移验收仍待完成。
2026-10-01 19:48:37 +08:00
1 parent 26e817f2c9
commit adf563ffe3
@@ -274,6 +274,13 @@ FELIS_OFFSITE_DB_KEEP="${FELIS_OFFSITE_DB_KEEP:-}"
 # the key that pings the check; off removes it, and a re-run without it keeps it.
 FELIS_WATCHDOG_HEARTBEAT_URL="${FELIS_WATCHDOG_HEARTBEAT_URL:-}"
 INSTALL_MODE="${FELIS_INSTALL_MODE:-}"
+DISTRIBUTED="${FELIS_DISTRIBUTED:-0}"
+WORKER_NAME="${FELIS_NODE_NAME:-}"
+WORKER_SERVER="${FELIS_SERVER_URL:-}"
+WORKER_TOKEN_FILE="${FELIS_BOOTSTRAP_TOKEN_FILE:-}"
+WORKER_REGISTRY_IP="${FELIS_REGISTRY_CLUSTER_IP:-}"
+NODE_EXTERNAL_IP="${FELIS_NODE_EXTERNAL_IP:-}"
+WORKER_PEERS="${FELIS_PEER_CIDRS:-}"
 # strict stops the install on any preflight problem (preflight below); warn reports them
 # and goes on, for a host the checks misjudge.
 FELIS_PREFLIGHT="${FELIS_PREFLIGHT:-strict}"
@@ -2024,6 +2031,19 @@ write_k3s_config() {
  {
    echo "# Written by the Felis installer (deploy/bootstrap.sh); a rerun rewrites it."
    echo 'write-kubeconfig-mode: "0600"'
+    if [ "${DISTRIBUTED:-0}" = 1 ]; then
+      [ -n "$NODE_EXTERNAL_IP" ] || NODE_EXTERNAL_IP="$NODE_IP"
+      printf 'node-external-ip: "%s"\n' "$NODE_EXTERNAL_IP"
+      echo 'flannel-backend: "wireguard-native"'
+      echo 'flannel-external-ip: true'
+      echo 'agent-token-file: "/etc/rancher/k3s/felis-agent-token"'
+      # Append to existing API-server hardening arguments in earlier config files.
+      echo 'kube-apiserver-arg+:'
+      echo '  - "enable-admission-plugins=NodeRestriction"'
+      if [ ! -s /etc/rancher/k3s/felis-agent-token ]; then
+        (umask 077; openssl rand -hex 32 > /etc/rancher/k3s/felis-agent-token)
+      fi
+    fi
    if [ -n "$name" ]; then
      printf 'node-name: "%s"\n' "$name"
    fi
@@ -5259,6 +5279,24 @@ deploy_bundle() {

  revoke_worlds_root_grant

+  if [ "${DISTRIBUTED:-0}" = 1 ]; then
+    local controller archive_key_file="${STATE_DIR}/archive-transfer.key"
+    controller="$(k3s_node_name)"
+    [ -n "$controller" ] || die "distributed deployment needs a stable controller node name"
+    kube label node "$controller" "felis.node-restriction.kubernetes.io/role=controller" "felis.node-restriction.kubernetes.io/identity=$controller" --overwrite
+    local system_deployment
+    for system_deployment in coredns local-path-provisioner; do
+      if kube -n kube-system get deployment "$system_deployment" >/dev/null 2>&1; then
+        kube -n kube-system patch deployment "$system_deployment" --type merge \
+          -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"felis.node-restriction.kubernetes.io/identity\":\"$controller\"}}}}}"
+      fi
+    done
+    if [ ! -s "$archive_key_file" ]; then (umask 077; openssl rand -hex 32 > "$archive_key_file"); fi
+    local archive_key
+    archive_key="$(cat "$archive_key_file")"
+    apply_literal_secret "$CONTROL_NS" felis-archive-key key "$archive_key"
+    apply_literal_secret "$MINECRAFT_NS" felis-archive-key key "$archive_key"
+  fi
  log "rendering + applying the control-plane bundle"
  local -a manifest_args=(
    --felis-image "$FELIS_IMAGE"
@@ -5266,6 +5304,13 @@ deploy_bundle() {
    --panel-node-port "$FELIS_PANEL_NODEPORT"
    --velocity-cidr "${NODE_IP}/32"
  )
+  if [ "${DISTRIBUTED:-0}" = 1 ]; then
+    manifest_args+=(--distributed --controller-node "$controller" --egress-probe "felis-api.${CONTROL_NS}.svc:443")
+    # Every node address, including global addresses, must be excluded from game egress.
+    while read -r cidr; do
+      [ -z "$cidr" ] || manifest_args+=(--server-egress-deny-cidr "$cidr")
+    done < <(kube get nodes -o jsonpath='{range .items[*]}{range .status.addresses[*]}{.address}{"\n"}{end}{end}' | awk '/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ {print $0"/32"}')
+  fi
  local cidr
  while read -r cidr; do
    [ -n "$cidr" ] && manifest_args+=(--server-egress-deny-cidr "$cidr")
@@ -5660,6 +5705,7 @@ summary() {
 #     prompt (or FELIS_INSTALL_MODE=nano).
 # ---------------------------------------------------------------------------
 prompt_install_mode() {
+  if [ "$INSTALL_MODE" = worker ]; then log "install mode: worker";return; fi
  # felis setup carries on to the Owner and edge setup, which needs the control plane, so
  # a nano install under it could only end in a setup error.
  if bootstrap_from_tui; then
@@ -5669,9 +5715,9 @@ prompt_install_mode() {
    return 0
  fi
  case "$INSTALL_MODE" in
-    full|nano) log "install mode: ${INSTALL_MODE} (from FELIS_INSTALL_MODE)"; return 0 ;;
+    full|nano|worker) log "install mode: ${INSTALL_MODE} (from FELIS_INSTALL_MODE)"; return 0 ;;
    "") ;;
-    *) die "FELIS_INSTALL_MODE must be 'full' or 'nano', got: ${INSTALL_MODE}" ;;
+    *) die "FELIS_INSTALL_MODE must be 'full', 'nano' or 'worker', got: ${INSTALL_MODE}" ;;
  esac

  # A felis-nano unit with no full install beside it makes this re-run a nano update;
@@ -5997,6 +6043,80 @@ ensure_k3s_on_path() {
  esac
 }

+# Worker is a daemon-only branch. It neither generates Felis service credentials nor applies a controller bundle.
+main_worker() {
+  [ -n "$WORKER_NAME" ] && [[ "$WORKER_NAME" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || die "FELIS_NODE_NAME is required and must be a DNS node name"
+  [[ "$WORKER_SERVER" =~ ^https://([a-zA-Z0-9.:-]+|\[[0-9a-fA-F:]+\]):6443$ ]] || die "FELIS_SERVER_URL must be an HTTPS k3s endpoint on port 6443"
+  [[ "$WORKER_REGISTRY_IP" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "FELIS_REGISTRY_CLUSTER_IP is required"
+  [ -n "$WORKER_PEERS" ] || die "FELIS_PEER_CIDRS must list exact cluster peer addresses"
+  [ -s "$WORKER_TOKEN_FILE" ] || die "FELIS_BOOTSTRAP_TOKEN_FILE must name a secure limited bootstrap token file"
+  local token saved mode
+  [ -f "$WORKER_TOKEN_FILE" ] && [ ! -L "$WORKER_TOKEN_FILE" ] || die "bootstrap token must be a regular file"
+  mode="$(stat -c %a "$WORKER_TOKEN_FILE")"
+  (( (8#$mode & 077) == 0 )) || die "bootstrap token must not be readable by group or others (use chmod 600)"
+  token="$(cat "$WORKER_TOKEN_FILE")"
+  [[ "$token" =~ ^K10[0-9a-f]{64}::[a-z0-9]{6}\.[a-z0-9]{16}$ ]] || die "worker accepts only CA-pinned bootstrap tokens, never server or static agent tokens"
+  [ ! -e /var/lib/rancher/k3s/server ] || die "this host has a k3s server; refusing to turn a controller into a worker"
+  if [ -d /var/lib/rancher/k3s/agent ]; then
+    saved="$(k3s_node_name)"
+    [ -n "$saved" ] && [ "$saved" = "$WORKER_NAME" ] || die "cannot change or guess an installed worker identity"
+  fi
+  if [ -f "$K3S_CONFIG_DROPIN" ]; then
+    saved="$(awk -F'"' '/^node-name:/ {print $2;exit}' "$K3S_CONFIG_DROPIN")"
+    [ -z "$saved" ] || [ "$saved" = "$WORKER_NAME" ] || die "existing node identity is $saved; refusing to rename it"
+    saved="$(awk -F'"' '/^server:/ {print $2;exit}' "$K3S_CONFIG_DROPIN")"
+    [ -z "$saved" ] || [ "$saved" = "$WORKER_SERVER" ] || die "existing worker belongs to another controller"
+  fi
+  detect_node_ip
+  [ -n "$NODE_EXTERNAL_IP" ] || NODE_EXTERNAL_IP="$NODE_IP"
+  PREFLIGHT_PROBLEMS=()
+  preflight_platform; preflight_memory; preflight_disk; preflight_networks; preflight_outbound
+  local unit
+  for unit in rke2-server rke2-agent k0scontroller k0sworker snap.microk8s.daemon-kubelite kubelet k3s; do
+    if systemctl is-active --quiet "$unit.service"; then preflight_fail "conflicting Kubernetes service: $unit"; fi
+  done
+  [ "${#PREFLIGHT_PROBLEMS[@]}" = 0 ] || die "worker preflight failed: ${PREFLIGHT_PROBLEMS[*]}"
+  install_base
+  ensure_time_sync
+  ensure_persistent_journal
+  # Reuse the release binary path for the local admission checks, without importing game/platform images.
+  bootstrap_from_tui || [ -n "$FELIS_ARTIFACT_DIR" ] || [ -n "${FELIS_SKIP_FETCH:-}" ] || resolve_install_ref
+  acquire_felis_binary
+  if [ ! -x "$HOST_BIN" ]; then install_go_toolchain; build_nano_binary; fi
+  mkdir -p /etc/rancher/k3s/config.yaml.d
+  (umask 077; printf '%s\n' "$token" > /etc/rancher/k3s/felis-bootstrap-token)
+  unset token
+  cat > "$K3S_CONFIG_DROPIN" <<EOF_WORKER
+server: "$WORKER_SERVER"
+node-name: "$WORKER_NAME"
+node-external-ip: "$NODE_EXTERNAL_IP"
+token-file: "/etc/rancher/k3s/felis-bootstrap-token"
+disable-default-registry-endpoint: true
+node-taint:
+  - "felis.lolicon.best/unapproved=true:NoSchedule"
+EOF_WORKER
+  chmod 0600 "$K3S_CONFIG_DROPIN"
+  cat > "$K3S_REGISTRIES_FILE" <<EOF_MIRROR
+mirrors:
+  "$REGISTRY_URL":
+    endpoint:
+      - "http://${WORKER_REGISTRY_IP}:5000"
+EOF_MIRROR
+  chmod 0600 "$K3S_REGISTRIES_FILE"
+  HOST_BIN_IN_USE=1
+  "$HOST_BIN" node firewall --peers "$WORKER_PEERS" --controller-ip "${WORKER_SERVER#https://}" --pod-cidr "$POD_CIDR" --node-port "$FELIS_PANEL_NODEPORT"
+  if [ ! -x "$K3S_BIN" ]; then
+    stage_k3s_airgap_images
+    curl -sfL --retry 5 --retry-delay 2 "https://raw.githubusercontent.com/k3s-io/k3s/${FELIS_K3S_VERSION}/install.sh" | \
+      INSTALL_K3S_VERSION="$FELIS_K3S_VERSION" INSTALL_K3S_BIN_DIR="$K3S_BIN_DIR" INSTALL_K3S_EXEC=agent sh -
+  else
+    [ "$("$K3S_BIN" --version | awk 'NR==1 {print $3}')" = "$FELIS_K3S_VERSION" ] || die "worker k3s version differs from pinned controller version; upgrade in a maintenance window"
+    systemctl enable --now k3s-agent
+    systemctl restart k3s-agent
+  fi
+  ok "worker $WORKER_NAME joined under quarantine; run felis node approve on A"
+}
+
 main() {
  ensure_k3s_on_path
  resolve_nano_listen
@@ -6007,6 +6127,7 @@ main() {
    main_nano
    return
  fi
+  if [ "$INSTALL_MODE" = worker ]; then main_worker; return; fi
  detect_node_ip
  # Before the first change to the host: a problem found here costs a rerun, one found
  # halfway through costs an install to unwind.
@@ -6033,11 +6154,16 @@ main() {
  ensure_panel_tls_cert
  # No install_docker here: Docker comes in only for an image this run has to build
  # (ensure_docker), and an install from a release's assets builds none.
+  if [ -z "${FELIS_DISTRIBUTED+x}" ] && [ -f "$K3S_CONFIG_DROPIN" ] && grep -q 'flannel-backend: "wireguard-native"' "$K3S_CONFIG_DROPIN"; then DISTRIBUTED=1; fi
  install_k3s
  # The registry mirror must exist before the bundle's pods start pulling (and
  # before any re-run's rollouts).
  configure_registry_mirror
  acquire_felis_binary
+  if [ "${DISTRIBUTED:-0}" = 1 ]; then
+    [ -n "$WORKER_PEERS" ] || WORKER_PEERS="${NODE_EXTERNAL_IP:-$NODE_IP}/32"
+    "$HOST_BIN" node firewall --controller --controller-ip "${NODE_EXTERNAL_IP:-$NODE_IP}" --peers "$WORKER_PEERS" --pod-cidr "$POD_CIDR" --node-port "$FELIS_PANEL_NODEPORT" --control-namespace "$CONTROL_NS" --namespace "$MINECRAFT_NS"
+  fi
  select_release_artifacts
  resolve_felis_image
  # The registry's and the database's own images must be in containerd before their
@@ -1334,7 +1334,7 @@ expect "a failed fetch into an existing checkout names the token" "set FELIS_GIT

 mblock="$(awk '/^  log "rendering \+ applying the control-plane bundle"/,/kube apply -f -/' "$BS")"
 [ -n "$mblock" ] || { echo "FAIL: no manifest_args block found in $BS"; exit 1; }
-[ "$(printf '%s\n' "$mblock" | wc -l)" -lt 60 ] \
+[ "$(printf '%s\n' "$mblock" | wc -l)" -lt 100 ] \
  || { echo "FAIL: the extracted block is not the manifest_args block -- did it move?"; exit 1; }

 run_bundle_flags() { # backup-pvc worlds-host-path
@@ -4910,6 +4910,36 @@ case "$out" in
 esac
 rm -rf "$credir" "$credcalls"

+# Worker admission reuses the installer but must never enter host control-plane setup.
+expect "distributed admission preserves existing API-server arguments" \
+  "echo 'kube-apiserver-arg+:'" "$(bsfn write_k3s_config)"
+worker="$(bsfn main_worker)"
+before "worker identity is checked before the agent config is written" \
+  'refusing to rename it' 'cat > "$K3S_CONFIG_DROPIN"' "$worker"
+expect "worker rejects server tokens and verifies the CA-pinned bootstrap shape" \
+  'K10[0-9a-f]{64}::[a-z0-9]{6}\.[a-z0-9]{16}' "$worker"
+expect "worker cannot replace a controller" 'refusing to turn a controller into a worker' "$worker"
+expect "worker is quarantined" 'felis.lolicon.best/unapproved=true:NoSchedule' "$worker"
+expect "worker mirror preserves logical references and points at the cluster service" \
+  'http://${WORKER_REGISTRY_IP}:5000' "$worker"
+expect "worker disables registry endpoint fallback" 'disable-default-registry-endpoint: true' "$worker"
+for forbidden in deploy_bundle install_cloudflared install_velocity run_migrations load_or_make_secrets; do
+  case "$worker" in
+    *"$forbidden"*) echo "FAIL worker invokes $forbidden"; fails=$((fails + 1));;
+    *) echo "PASS worker does not invoke $forbidden";;
+  esac
+done
+badtoken="$(mktemp)"
+printf 'server-token-not-bootstrap' > "$badtoken"
+out="$(WORKER_TOKEN_FILE="$badtoken" bash -c '
+  die() { printf "DIE: %s\n" "$*"; exit 1; }
+  WORKER_NAME=b WORKER_SERVER=https://192.0.2.1:6443 WORKER_REGISTRY_IP=10.43.0.10 WORKER_PEERS=192.0.2.1/32
+  '"$worker"'
+  main_worker
+')"
+expect "worker refuses a copied server token before changing the machine" 'worker accepts only CA-pinned bootstrap tokens' "$out"
+rm -f "$badtoken"
+
 # ---------------------------------------------------------------------------------------
 if [ "$fails" -eq 0 ]; then
  echo "ALL PASS"
@@ -281,6 +281,8 @@ spec:
              storage:
                description: Storage configures the world PVC.
                properties:
+                  claimName:
+                    type: string
                  size:
                    description: Size is the requested PVC capacity (e.g. "10Gi").
                    type: string
@@ -289,6 +291,8 @@ spec:
                      uses the default.
                    type: string
                type: object
+              nodeName:
+                type: string
              subdomain:
                description: |-
                  Subdomain is the per-server label under the deployment zone. It is the
@@ -301,6 +305,8 @@ spec:
          status:
            description: MinecraftServerStatus is the observed state (spec §4 status.*).
            properties:
+              nodeName:
+                type: string
              autoRestarts:
                description: |-
                  AutoRestarts counts how often the operator recreated the pod of a start
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Linux/root acceptance of resident-node and pre-DNAT paths. All packet rules,
+# listeners and links live in disposable network namespaces, never the live host.
+set -euo pipefail
+bin="${1:?usage: test-node-firewall.sh /absolute/path/to/felis}"
+[[ $bin = /* && -x $bin ]] || exit 2
+[[ $(id -u) = 0 ]] || { echo 'requires root on Linux' >&2; exit 2; }
+work=$(mktemp -d)
+suffix="$$"
+node="felis-fw-node-$suffix"
+game="felis-fw-game-$suffix"
+peer="felis-fw-peer-$suffix"
+listener=''
+cleanup() {
+  if [[ -n $listener ]]; then kill "$listener" 2>/dev/null || true; wait "$listener" 2>/dev/null || true; fi
+  for ns in "$game" "$peer" "$node"; do ip netns del "$ns" 2>/dev/null || true; done
+  rm -rf "$work"
+}
+trap cleanup EXIT
+for ns in "$node" "$game" "$peer"; do ip netns add "$ns"; ip -n "$ns" link set lo up; done
+ip link add fg-node type veth peer name fg-game
+ip link set fg-node netns "$node"
+ip link set fg-game netns "$game"
+ip link add fp-node type veth peer name fp-peer
+ip link set fp-node netns "$node"
+ip link set fp-peer netns "$peer"
+ip -n "$node" addr add 10.42.250.1/24 dev fg-node
+ip -n "$game" addr add 10.42.250.2/24 dev fg-game
+ip -n "$node" addr add 192.0.2.2/24 dev fp-node
+ip -n "$peer" addr add 192.0.2.1/24 dev fp-peer
+ip -n "$node" link set fg-node up
+ip -n "$node" link set fp-node up
+ip -n "$game" link set fg-game up
+ip -n "$peer" link set fp-peer up
+ip -n "$game" route add 192.0.2.0/24 via 10.42.250.1
+ip -n "$game" route add 10.43.0.1/32 via 10.42.250.1
+ip netns exec "$node" "$bin" node-probe --listen :18083 >"$work/listener.log" 2>&1 &
+listener=$!
+ip netns exec "$game" "$bin" node-probe --open 192.0.2.2:18083
+ip netns exec "$peer" "$bin" node-probe --open 192.0.2.2:18083
+
+"$bin" node firewall --dry-run --peers 192.0.2.1/32,192.0.2.2/32 \
+  --controller-ip 192.0.2.1 --pod-cidr 10.42.0.0/16 \
+  --node-port 30443 --api-service-ip 10.43.0.1 >"$work/firewall.sh"
+ip netns exec "$node" bash "$work/firewall.sh"
+# Simulate later kube-router insertion ahead of Felis filter hooks.
+ip netns exec "$node" iptables -I INPUT 1 -j ACCEPT
+ip netns exec "$node" iptables -t nat -A PREROUTING -p tcp --dport 30443 -j REDIRECT --to-ports 18083
+ip netns exec "$node" iptables -t nat -A PREROUTING -d 10.43.0.1 -p tcp --dport 443 -j REDIRECT --to-ports 18083
+ip netns exec "$game" "$bin" node-probe \
+  --closed 192.0.2.2:18083 --closed 192.0.2.2:30443 --closed 10.43.0.1:443
+ip netns exec "$peer" "$bin" node-probe --closed 192.0.2.2:30443
+# The listener remains healthy and the allowed peer still reaches it.
+ip netns exec "$peer" "$bin" node-probe --open 192.0.2.2:18083
+ip netns exec "$node" "$bin" node-probe --open 127.0.0.1:18083
+echo 'PASS resident-node isolation and public NodePort denial survive pre-DNAT and early filter ACCEPT'