From adf563ffe3a283ce16d1db418187503261a5324b Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Thu, 1 Oct 2026 19:48:37 +0800 Subject: [PATCH] =?UTF-8?q?feat(distributed):=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=8D=95=E4=B8=BB=E6=8E=A7=E5=A4=9A=E8=8A=82=E7=82=B9=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E5=92=8C=E5=81=9C=E6=9C=8D=E8=BF=81=E7=A7=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复用现有 k3s 调度和 Job 生命周期，增加 worker 接入与批准、受保护节点身份、归档传输、持久迁移锁及活动 PVC 切换；同步管理员 API、CLI、面板和隔离规则。分布式模式默认关闭，保持单机兼容。 验证：Go 全量测试与 vet；面板 874 个测试、lint/build；Linux VM 安装器测试、清单服务端 dry-run、网络命名空间防火墙实测。A/B/C 三机 WireGuard、Velocity 和迁移验收仍待完成。 --- cmd/felis/api.go | 56 +- cmd/felis/archive_serve.go | 58 ++ cmd/felis/backup.go | 4 +- cmd/felis/distributed.go | 44 ++ cmd/felis/egressgate.go | 13 + cmd/felis/egressgate_test.go | 14 + cmd/felis/export.go | 7 +- cmd/felis/export_test.go | 2 +- cmd/felis/manifests.go | 11 +- cmd/felis/node_approve.go | 466 ++++++++++++++ cmd/felis/node_firewall.go | 187 ++++++ cmd/felis/node_firewall_test.go | 39 ++ cmd/felis/node_probe.go | 56 ++ cmd/felis/nodes.go | 157 +++++ cmd/felis/operator.go | 8 +- cmd/felis/reaper.go | 16 +- cmd/felis/restore.go | 32 +- cmd/felis/run.go | 8 + cmd/felis/server_migrate.go | 114 ++++ deploy/bootstrap.sh | 130 +++- deploy/bootstrap_test.sh | 32 +- .../felis.lolicon.best_minecraftservers.yaml | 6 + deploy/test-node-firewall.sh | 56 ++ docs/distributed.md | 109 ++++ docs/openapi.yaml | 199 ++++++ docs/operations.md | 2 +- internal/api/api.go | 14 +- internal/api/cluster.go | 2 + internal/api/handlers_distributed.go | 122 ++++ internal/api/handlers_distributed_test.go | 77 +++ internal/api/handlers_user.go | 11 + internal/api/k8scluster.go | 68 ++- internal/api/k8scluster_test.go | 34 ++ internal/api/k8sjobstatus.go | 3 + .../felis/v1alpha1/minecraftserver_types.go | 15 + internal/archivetransfer/transfer.go | 573 ++++++++++++++++++ internal/archivetransfer/transfer_test.go | 228 +++++++ internal/backup/verify_restored.go | 69 +++ internal/backupjob/backup.go | 26 +- internal/distributed/manager.go | 278 +++++++++ internal/distributed/migration.go | 521 ++++++++++++++++ internal/distributed/migration_test.go | 352 +++++++++++ internal/fileedit/editor.go | 17 + internal/fileedit/jobspec.go | 12 +- internal/maintenance/maintenance.go | 6 +- internal/operator/builders.go | 19 +- internal/operator/builders_internal_test.go | 30 + internal/operator/reconciler.go | 50 +- internal/panel/panel.go | 6 +- internal/placement/placement.go | 84 +++ internal/platform/bundle.go | 5 + internal/platform/distributed.go | 82 +++ internal/platform/distributed_test.go | 61 ++ internal/platform/identities.go | 24 + internal/platform/netpol.go | 3 + internal/platform/postgres.go | 1 + internal/platform/rbac.go | 18 +- internal/platform/workloads.go | 35 +- internal/reaper/k8scluster.go | 17 +- internal/reaper/pgstore.go | 2 +- internal/reaper/reaper.go | 11 + internal/restore/jobspec.go | 48 +- internal/restore/restore.go | 12 +- internal/worldexport/jobspec.go | 12 +- internal/worldexport/worldexport.go | 14 +- .../components/CreateServerDialog.test.tsx | 24 +- panel/src/components/CreateServerDialog.tsx | 16 + .../src/components/DistributedNodes.test.tsx | 44 ++ panel/src/components/DistributedNodes.tsx | 87 +++ panel/src/i18n/resources/en-US/servers.json | 19 +- panel/src/i18n/resources/zh-CN/servers.json | 19 +- panel/src/lib/api.ts | 9 + panel/src/lib/config.test.ts | 1 + panel/src/lib/config.ts | 2 + panel/src/lib/openapi.gen.ts | 253 ++++++++ panel/src/lib/types.ts | 24 + panel/src/pages/servers/ServersPage.tsx | 11 + 77 files changed, 5224 insertions(+), 73 deletions(-) create mode 100644 cmd/felis/archive_serve.go create mode 100644 cmd/felis/distributed.go create mode 100644 cmd/felis/node_approve.go create mode 100644 cmd/felis/node_firewall.go create mode 100644 cmd/felis/node_firewall_test.go create mode 100644 cmd/felis/node_probe.go create mode 100644 cmd/felis/nodes.go create mode 100644 cmd/felis/server_migrate.go create mode 100755 deploy/test-node-firewall.sh create mode 100644 docs/distributed.md create mode 100644 internal/api/handlers_distributed.go create mode 100644 internal/api/handlers_distributed_test.go create mode 100644 internal/archivetransfer/transfer.go create mode 100644 internal/archivetransfer/transfer_test.go create mode 100644 internal/backup/verify_restored.go create mode 100644 internal/distributed/manager.go create mode 100644 internal/distributed/migration.go create mode 100644 internal/distributed/migration_test.go create mode 100644 internal/placement/placement.go create mode 100644 internal/platform/distributed.go create mode 100644 internal/platform/distributed_test.go create mode 100644 panel/src/components/DistributedNodes.test.tsx create mode 100644 panel/src/components/DistributedNodes.tsx diff --git a/cmd/felis/api.go b/cmd/felis/api.go index a010d4a..860dfd6 100644 --- a/cmd/felis/api.go +++ b/cmd/felis/api.go @@ -20,6 +20,7 @@ import ( "felis.lolicon.best/internal/backupjob" "felis.lolicon.best/internal/build" "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/distributed" "felis.lolicon.best/internal/fileedit" "felis.lolicon.best/internal/imagepin" "felis.lolicon.best/internal/mail" @@ -27,6 +28,7 @@ import ( "felis.lolicon.best/internal/naming" "felis.lolicon.best/internal/panel" "felis.lolicon.best/internal/passkey" + "felis.lolicon.best/internal/placement" "felis.lolicon.best/internal/platform" "felis.lolicon.best/internal/reaper" "felis.lolicon.best/internal/registryprune" @@ -273,9 +275,23 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { // store at load, so by this point cfg.Archive.Store is guaranteed tarLocal.) var restorer api.Restorer felisImage, backupPVC := os.Getenv("FELIS_IMAGE"), os.Getenv("FELIS_BACKUP_PVC") + worldResolver := placement.Resolve(cl, cfg.K8s.Namespace) + distribution, err := distributionManager(cl, cfg, felisImage) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if distribution != nil { + worldResolver = distribution.Resolve + } if felisImage != "" && backupPVC != "" { rcfg := restoreConfig(cfg, felisImage, backupPVC) - restorer = &restore.Restorer{Jobs: restore.NewK8sJobs(cl), Config: rcfg} + rcfg.ResolveWorld = worldResolver + var jobs restore.Jobs = restore.NewK8sJobs(cl) + if distribution != nil { + jobs = distribution + } + restorer = &restore.Restorer{Jobs: jobs, Config: rcfg} } else { fmt.Fprintln(stderr, "felis api: restore executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — restore endpoint returns 503") } @@ -288,7 +304,13 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { // endpoint honestly returns 503. var backuper api.Backuper if felisImage != "" && backupPVC != "" { - backuper = &backupjob.Backuper{Jobs: backupjob.NewK8sJobs(cl), Config: backupConfig(cfg, felisImage, backupPVC)} + bcfg := backupConfig(cfg, felisImage, backupPVC) + bcfg.ResolveWorld = worldResolver + var jobs backupjob.Jobs = backupjob.NewK8sJobs(cl) + if distribution != nil { + jobs = distribution + } + backuper = &backupjob.Backuper{Jobs: jobs, Config: bcfg} } else { fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503") } @@ -299,7 +321,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { // is wired under the restore gate; otherwise the export routes return 503. var exporter api.Exporter if felisImage != "" && backupPVC != "" { - exporter = worldexport.New(clientset, exportConfig(cfg, felisImage, backupPVC)) + ecfg := exportConfig(cfg, felisImage, backupPVC) + ecfg.ResolveWorld = worldResolver + exporter = worldexport.New(clientset, ecfg) } else { fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503") } @@ -319,9 +343,11 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { var files api.FileEditor var fileStage *fileedit.Stage if felisImage != "" { + fcfg := fileEditConfig(cfg, felisImage) + fcfg.ResolveWorld = worldResolver files = &fileedit.Editor{ Runner: fileedit.NewK8sRunner(clientset), - Config: fileEditConfig(cfg, felisImage), + Config: fcfg, } fileStage = &fileedit.Stage{Dir: fileStagingDir()} if err := fileStage.Sweep(); err != nil { @@ -355,7 +381,21 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err) return 1 } - cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced) + cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced).WithDistributed(distribution != nil, os.Getenv("FELIS_CONTROLLER_NODE")) + if distribution != nil { + distribution.Record = func(ctx context.Context, b distributed.Backup) error { + keep, retention, ok := backupPolicy(b.Reason, rcfg) + if !ok { + return fmt.Errorf("unknown backup reason %s", b.Reason) + } + st := reaper.NewPGStore(drv.DB()) + if err := st.InsertBackup(ctx, reaper.BackupRecord{ID: "bk-" + b.ID, ServerName: b.Server, FormerOwner: b.Owner, BackupRef: b.Receipt.Ref, SizeBytes: b.Receipt.Size, SHA256: b.Receipt.SHA256, Reason: b.Reason, ExpiresAt: time.Now().Add(retention)}); err != nil { + return err + } + pruneBackups(ctx, st, distribution.Archive, b.Server, b.Owner, b.Reason, keep, b.Protect, stdout, stderr) + return nil + } + } jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace) a := &api.API{ Repo: repo, @@ -455,7 +495,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain, defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname), defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname), - cfg.Velocity.GamePort, resolvedVersion()) + cfg.Velocity.GamePort, resolvedVersion(), distribution != nil) internalSrv := newAPIServer(*internalAddr, a.InternalHandler()) externalSrv := newAPIServer(cfg.Server.Listen, externalHandler) @@ -499,6 +539,10 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { } go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default()) + if distribution != nil { + a.Distribution = distribution + go reconcileDistribution(ctx, distribution, stderr) + } servers := []*http.Server{internalSrv, externalSrv} if httpsSrv != nil { servers = append(servers, httpsSrv) diff --git a/cmd/felis/archive_serve.go b/cmd/felis/archive_serve.go new file mode 100644 index 0000000..eda9667 --- /dev/null +++ b/cmd/felis/archive_serve.go @@ -0,0 +1,58 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "time" + + "felis.lolicon.best/internal/archivetransfer" +) + +func cmdArchiveServe(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("archive-serve", flag.ContinueOnError) + fs.SetOutput(stderr) + root := fs.String("root", "/backups", "archive PVC mount (must match archive.local_path)") + addr := fs.String("listen", ":8090", "private archive listener") + limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum compressed archive bytes") + if err := fs.Parse(args); err != nil { + return 2 + } + key := os.Getenv(archivetransfer.KeyEnv) + if len(key) < 32 || *limit <= 0 { + fmt.Fprintln(stderr, "archive-serve: signing key and positive size limit required") + return 2 + } + if err := os.MkdirAll(*root, 0750); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + limitHeapToCgroup() + transport := &archivetransfer.Server{Root: *root, Key: key, Limit: *limit} + done := make(chan struct{}) + defer close(done) + go func() { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + if err := transport.Sweep(time.Now()); err != nil { + fmt.Fprintln(stderr, "archive journal cleanup:", err) + } + select { + case <-done: + return + case <-ticker.C: + } + } + }() + srv := &http.Server{Addr: *addr, Handler: transport, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Hour, WriteTimeout: 2 * time.Hour, MaxHeaderBytes: 16 << 10} + fmt.Fprintln(stdout, "archive transport listening", *addr) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + fmt.Fprintln(stderr, err) + return 1 + } + return 0 +} diff --git a/cmd/felis/backup.go b/cmd/felis/backup.go index 8c784d1..ff819cb 100644 --- a/cmd/felis/backup.go +++ b/cmd/felis/backup.go @@ -168,7 +168,9 @@ func backupPolicy(reason string, rcfg reaper.Config) (keep int, retention time.D // previous owner's. protect is never removed: it is the backup a chained restore // is about to extract. The new backup is already recorded; a removal that fails // is reported and retried after the next backup. -func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) { +func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver interface { + Delete(context.Context, backup.ArchiveRef) error +}, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) { excess, err := st.ExcessBackups(ctx, server, owner, reason, keep, protect) if err != nil { fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err) diff --git a/cmd/felis/distributed.go b/cmd/felis/distributed.go new file mode 100644 index 0000000..5d9ceaf --- /dev/null +++ b/cmd/felis/distributed.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "time" + + "felis.lolicon.best/internal/archivetransfer" + "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/distributed" + "felis.lolicon.best/internal/placement" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func distributionManager(cl client.Client, cfg *config.Config, image string) (*distributed.Manager, error) { + if os.Getenv("FELIS_DISTRIBUTED") != "true" { + return nil, nil + } + controller, url, key := os.Getenv("FELIS_CONTROLLER_NODE"), os.Getenv("FELIS_ARCHIVE_URL"), os.Getenv(archivetransfer.KeyEnv) + if controller == "" || url == "" || len(key) < 32 || image == "" || cfg.Archive.Store != "tarLocal" { + return nil, fmt.Errorf("distributed mode requires controller identity, archive service/key, Felis image and tarLocal") + } + return &distributed.Manager{Client: cl, Namespace: cfg.K8s.Namespace, Image: image, Controller: controller, Archive: archivetransfer.Client{URL: url, Root: cfg.Archive.LocalPath, Key: key}, Resolve: placement.Resolve(cl, cfg.K8s.Namespace, controller)}, nil +} + +func reconcileDistribution(ctx context.Context, m *distributed.Manager, stderr io.Writer) { + ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() + for { + if err := m.SettleBackups(ctx); err != nil { + fmt.Fprintln(stderr, "distributed backup:", err) + } + if err := m.ReconcileMigrations(ctx); err != nil { + fmt.Fprintln(stderr, "migration:", err) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} diff --git a/cmd/felis/egressgate.go b/cmd/felis/egressgate.go index b80b2e1..a6c670b 100644 --- a/cmd/felis/egressgate.go +++ b/cmd/felis/egressgate.go @@ -37,6 +37,7 @@ func cmdEgressGate(args []string, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError) fs.SetOutput(stderr) probe := fs.String("probe", "", "host:port the pod's NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)") + positive := fs.String("positive-probe", "", "allowed host:port that must remain reachable during denial checks") wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the gate gives up") failOpen := fs.Bool("fail-open", false, "when --wait runs out, warn and let the pod go on instead of refusing it") if err := fs.Parse(args); err != nil { @@ -53,6 +54,18 @@ func cmdEgressGate(args []string, stdout, stderr io.Writer) int { start := time.Now() for { + if *positive != "" { + allowed, err := net.DialTimeout("tcp", *positive, egressDialTimeout) + if err != nil { + if time.Since(start) >= *wait { + fmt.Fprintln(stderr, "felis egress-gate: positive probe unavailable; refusing to start", err) + return 1 + } + time.Sleep(egressPollInterval) + continue + } + allowed.Close() + } conn, err := net.DialTimeout("tcp", *probe, egressDialTimeout) if err != nil { fmt.Fprintf(stdout, "felis egress-gate: %s is unreachable after %s (%v); the egress lock is in effect\n", diff --git a/cmd/felis/egressgate_test.go b/cmd/felis/egressgate_test.go index cbcd61c..0a64300 100644 --- a/cmd/felis/egressgate_test.go +++ b/cmd/felis/egressgate_test.go @@ -15,6 +15,20 @@ func shrinkEgressGate(t *testing.T) { t.Cleanup(func() { egressDialTimeout, egressPollInterval = dial, poll }) } +func TestEgressGateRequiresPositiveReachability(t *testing.T) { + shrinkEgressGate(t) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + closed := ln.Addr().String() + ln.Close() + var out, errb bytes.Buffer + if code := cmdEgressGate([]string{"--positive-probe", closed, "--probe", closed, "--wait", "20ms"}, &out, &errb); code != 1 { + t.Fatal("unavailable probes allowed startup", code) + } +} + // The gate holds while the probe answers and lets the pod go on once the policy // lands, which the test plays by closing the listener. func TestEgressGateWaitsForTheLock(t *testing.T) { diff --git a/cmd/felis/export.go b/cmd/felis/export.go index ab6fb3f..d1cdccf 100644 --- a/cmd/felis/export.go +++ b/cmd/felis/export.go @@ -60,6 +60,7 @@ func cmdExport(args []string, stdout, stderr io.Writer) int { sum := fs.String("sha256", "", "backup only: the sha256 recorded when the archive was written; a mismatch fails the export before its end is sent") worldsRoot := fs.String("worlds-root", "/world", "world and files: mount path of the world PVC") path := fs.String("path", "", "files only: the file or folder to send, relative to the world root") + rawArchive := fs.Bool("archive-raw", false, "internal archive transfer: preserve the complete world") dir := fs.Bool("dir", false, "files only: the path is a folder, sent as a zip") if err := fs.Parse(args); err != nil { return 2 @@ -82,7 +83,11 @@ func cmdExport(args []string, stdout, stderr io.Writer) int { } err = exportBackup(ctx, *target, token, *ref, *backupRoot, *sum, stdout) case worldexport.ModeWorld: - err = exportWorld(ctx, *target, token, *worldsRoot, stdout) + if *rawArchive { + err = streamExport(ctx, *target, token, archiveType, -1, func(w io.Writer) error { _, _, err := backup.WriteTarGz(ctx, w, *worldsRoot, nil); return err }) + } else { + err = exportWorld(ctx, *target, token, *worldsRoot, stdout) + } case worldexport.ModeFiles: if *path == "" { fmt.Fprintln(stderr, "felis export: --path is required for files") diff --git a/cmd/felis/export_test.go b/cmd/felis/export_test.go index e25cb55..89405ab 100644 --- a/cmd/felis/export_test.go +++ b/cmd/felis/export_test.go @@ -526,7 +526,7 @@ func TestCmdExportWiring(t *testing.T) { cfg := &config.Config{} cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups" want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"} - if got := exportConfig(cfg, "felis:1", "felis-backups"); got != want { + if got := exportConfig(cfg, "felis:1", "felis-backups"); !reflect.DeepEqual(got, want) { t.Fatalf("exportConfig = %+v, want %+v", got, want) } } diff --git a/cmd/felis/manifests.go b/cmd/felis/manifests.go index f31f5cb..764287c 100644 --- a/cmd/felis/manifests.go +++ b/cmd/felis/manifests.go @@ -43,6 +43,11 @@ func (m *multiFlag) Set(v string) error { func cmdManifests(args []string, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("manifests", flag.ContinueOnError) fs.SetOutput(stderr) + distributed := fs.Bool("distributed", false, "enable approved workers and archive transport") + controller := fs.String("controller-node", "", "protected controller identity for A") + probe := fs.String("egress-probe", "", "reachable controller host:port denied to game Pods") + var registryNodes multiFlag + fs.Var(®istryNodes, "registry-node-cidr", "exact node pull source for the registry (repeatable)") controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace the control plane (api/operator/reaper) runs in") minecraftNS := fs.String("minecraft-namespace", platform.DefaultMinecraftNamespace, "namespace MinecraftServer workloads run in") buildNS := fs.String("build-namespace", platform.DefaultBuildNamespace, "namespace image-build Jobs run in") @@ -75,6 +80,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int { case "": case "postgres": return renderManifests(stdout, stderr, platform.PostgresObjects(platform.Params{ + ControllerNode: *controller, ControlNamespace: *controlNS, MinecraftNamespace: *minecraftNS, PostgresImage: *postgresImage, @@ -115,7 +121,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int { // The node pin exists only for the reaper's hostPath: naming a node without the // worlds root would be silently dropped (no CronJob renders), so fail loud like // the storage-trio check below. - if *reaperNode != "" && *worldsHostPath == "" { + if *reaperNode != "" && *worldsHostPath == "" && !*distributed { fmt.Fprintln(stderr, "felis manifests: --reaper-node requires --worlds-host-path "+ "(it pins the reaper CronJob, which renders only with the retention storage trio)") return 2 @@ -156,7 +162,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int { "writes under /var/lib/rancher/k3s/storage). Any other provisioner needs its volumes exposed as "+ "/, or each candidate's archive fails and the world is preserved;\n"+ " - %s.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath, pin) - } else { + } else if !*distributed { switch { case *archiveLocalPath != "" && *backupPVC == "": fmt.Fprintln(stderr, "felis manifests: --archive-local-path names where the backup PVC is mounted, "+ @@ -176,6 +182,7 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int { } params := platform.Params{ + Distributed: *distributed, ControllerNode: *controller, EgressProbe: *probe, RegistryNodeCIDRs: registryNodes, ControlNamespace: *controlNS, MinecraftNamespace: *minecraftNS, BuildNamespace: *buildNS, diff --git a/cmd/felis/node_approve.go b/cmd/felis/node_approve.go new file mode 100644 index 0000000..acbf1e5 --- /dev/null +++ b/cmd/felis/node_approve.go @@ -0,0 +1,466 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "os/exec" + "strconv" + "strings" + "time" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/archivetransfer" + "felis.lolicon.best/internal/operator" + "felis.lolicon.best/internal/placement" + "felis.lolicon.best/internal/platform" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func approveNode(ctx context.Context, cl client.Client, cs kubernetes.Interface, ns, controlNS, name, image, remote string, stdout io.Writer) error { + var n corev1.Node + if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil { + return err + } + _, controlPlane := n.Labels["node-role.kubernetes.io/control-plane"] + if !placement.Online(&n) || controlPlane || n.Labels[placement.LabelRole] == placement.RoleController { + return fmt.Errorf("candidate must be an online agent") + } + var nodes corev1.NodeList + if err := cl.List(ctx, &nodes); err != nil { + return err + } + var controller *corev1.Node + var peers []string + var denied []string + for i := range nodes.Items { + node := &nodes.Items[i] + if node.Labels[placement.LabelRole] == placement.RoleController { + if controller != nil { + return fmt.Errorf("multiple controllers found") + } + controller = node + } + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP || addr.Type == corev1.NodeExternalIP { + cidr, err := exactCIDR(addr.Address) + if err != nil { + return err + } + peers = append(peers, cidr) + for _, p := range []string{"6443", "10250", "5000", "30443"} { + denied = append(denied, net.JoinHostPort(addr.Address, p)) + } + } + } + } + if controller == nil || controller.Name == n.Name || controller.Status.NodeInfo.Architecture != n.Status.NodeInfo.Architecture { + return fmt.Errorf("candidate architecture must match the sole controller") + } + if n.Status.NodeInfo.KubeletVersion != controller.Status.NodeInfo.KubeletVersion { + return fmt.Errorf("candidate k3s version must match A") + } + var apiSvc, registrySvc, archiveSvc, kubernetesSvc corev1.Service + for _, svc := range []struct { + obj *corev1.Service + ns, name string + }{{&kubernetesSvc, "default", "kubernetes"}, {&apiSvc, controlNS, platform.SAAPI}, {®istrySvc, controlNS, "registry"}, {&archiveSvc, ns, platform.ArchiveName}} { + if err := cl.Get(ctx, types.NamespacedName{Namespace: svc.ns, Name: svc.name}, svc.obj); err != nil { + return err + } + } + if len(apiSvc.Spec.Ports) == 0 || apiSvc.Spec.Ports[0].NodePort == 0 { + return fmt.Errorf("controller API NodePort is absent") + } + // A's exact interface addresses let the probe observe DNAT/SNAT before the production policy is adjusted. + var aCIDRs []string + if addrs, err := net.InterfaceAddrs(); err == nil { + for _, a := range addrs { + ip, _, err := net.ParseCIDR(a.String()) + if err == nil && !ip.IsLoopback() { + cidr, _ := exactCIDR(ip.String()) + aCIDRs = append(aCIDRs, cidr) + } + } + } + if len(aCIDRs) == 0 { + return fmt.Errorf("cannot determine A's exact interface addresses") + } + onController := false + for _, addr := range controller.Status.Addresses { + cidr, err := exactCIDR(addr.Address) + if err != nil { + continue + } + for _, local := range aCIDRs { + if cidr == local { + onController = true + } + } + } + if !onController || !placement.Online(controller) { + return fmt.Errorf("run admission on the online controller A") + } + // Quarantine first. Approval is the final write, after all checks have succeeded. + before := n.DeepCopy() + if n.Labels == nil { + n.Labels = map[string]string{} + } + delete(n.Labels, placement.LabelApproved) + found := false + for _, t := range n.Spec.Taints { + if t.Key == "felis.lolicon.best/unapproved" { + found = true + } + } + if !found { + n.Spec.Taints = append(n.Spec.Taints, corev1.Taint{Key: "felis.lolicon.best/unapproved", Value: "true", Effect: corev1.TaintEffectNoSchedule}) + } + if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + if err := denyNodeAddresses(ctx, cl, ns, nodes.Items); err != nil { + return err + } + // The host's Service dial may be masqueraded to its bridge gateway. Permit exact host addresses only. + pullSources := append([]string{}, peers...) + if ip, network, err := net.ParseCIDR(n.Spec.PodCIDR); err == nil && ip.To4() != nil { + v := append(net.IP(nil), network.IP.To4()...) + pullSources = append(pullSources, v.String()+"/32") + v[3]++ + pullSources = append(pullSources, v.String()+"/32") + } + var registryNP networkingv1.NetworkPolicy + if err := cl.Get(ctx, types.NamespacedName{Namespace: controlNS, Name: "felis-registry-ingress"}, ®istryNP); err != nil { + return err + } + if len(registryNP.Spec.Ingress) == 0 { + return fmt.Errorf("registry ingress policy is not configured") + } + prev := registryNP.DeepCopy() + for _, cidr := range pullSources { + registryNP.Spec.Ingress[0].From = append(registryNP.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}}) + } + if err := cl.Patch(ctx, ®istryNP, client.MergeFrom(prev)); err != nil { + return err + } + ctrlIP := "" + for _, a := range controller.Status.Addresses { + if a.Type == corev1.NodeInternalIP { + ctrlIP = a.Address + break + } + } + if ctrlIP == "" { + return fmt.Errorf("controller has no address") + } + script := "set -euo pipefail\numask 077\n" + + "systemctl is-active --quiet k3s-agent\n! systemctl is-active --quiet k3s\ntest ! -f /etc/rancher/k3s/k3s.yaml\n" + + "ip -d link show flannel-wg | grep -q wireguard\n" + + "/usr/local/bin/felis node firewall --peers " + shellQuote(strings.Join(peers, ",")) + " --controller-ip " + shellQuote(ctrlIP) + " --api-service-ip " + shellQuote(kubernetesSvc.Spec.ClusterIP) + " --node-port " + strconv.Itoa(int(apiSvc.Spec.Ports[0].NodePort)) + " --namespace " + shellQuote(ns) + " --control-namespace " + shellQuote(controlNS) + "\n" + + "kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig\n" + + "/usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" get node " + shellQuote(name) + " -o name >/dev/null\n" + + "proof=$(mktemp); trap 'rm -f \"$proof\"' EXIT\n" + + "if /usr/local/bin/k3s kubectl --kubeconfig \"$kubeconfig\" label node " + shellQuote(name) + " felis.node-restriction.kubernetes.io/probe=controller --overwrite 2>\"$proof\"; then echo 'NodeRestriction failed' >&2;exit 1;fi\ngrep -qi forbidden \"$proof\"\n" + + "image=" + shellQuote(image) + "\nif [ -n \"$(/usr/local/bin/k3s crictl images -q \"$image\")\" ]; then /usr/local/bin/k3s crictl rmi \"$image\" >/dev/null; fi\ntest -z \"$(/usr/local/bin/k3s crictl images -q \"$image\")\"\n/usr/local/bin/k3s crictl pull \"$image\" >/dev/null\n" + cmd := exec.CommandContext(ctx, "ssh", "--", remote, "if [ \"$(id -u)\" = 0 ]; then bash -s; else sudo -n bash -s; fi") + cmd.Stdin = strings.NewReader(script) + cmd.Stdout = stdout + cmd.Stderr = stdout + if err := cmd.Run(); err != nil { + return fmt.Errorf("worker host, NodeRestriction or uncached registry pull check failed: %w", err) + } + id := "felis-probe-" + archivetransfer.ID()[:12] + var objects []client.Object + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for i := len(objects) - 1; i >= 0; i-- { + cl.Delete(cleanup, objects[i]) + } + }() + create := func(o client.Object) error { + if err := cl.Create(ctx, o); err != nil { + return err + } + objects = append(objects, o) + return nil + } + var endpoints []string + var echoPods []*corev1.Pod + for i := range nodes.Items { + node := &nodes.Items[i] + if !placement.Online(node) || (node.Name != name && node.Name != controller.Name && node.Labels[placement.LabelApproved] != "true") { + continue + } + echoName := fmt.Sprintf("%s-%d", id, i) + p := probePod(echoName, ns, node.Name, image, []string{"--listen", ":25565"}, true) + p.Labels["felis.lolicon.best/probe-echo"] = id + if err := create(p); err != nil { + return err + } + echoPods = append(echoPods, p) + svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: echoName, Namespace: ns}, Spec: corev1.ServiceSpec{Selector: map[string]string{"felis.lolicon.best/probe-name": echoName}, Ports: []corev1.ServicePort{{Port: 25565, TargetPort: intstr.FromInt32(25565)}}}} + if err := create(svc); err != nil { + return err + } + endpoints = append(endpoints, net.JoinHostPort(svc.Spec.ClusterIP, "25565")) + } + tcp := corev1.ProtocolTCP + port := intstr.FromInt32(25565) + policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id, Namespace: ns}, Spec: networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-echo": id}}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}}}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &port}}}}}} + for _, cidr := range aCIDRs { + policy.Spec.Ingress[0].From = append(policy.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}}) + } + if err := create(policy); err != nil { + return err + } + for _, p := range echoPods { + if err := waitProbePod(ctx, cl, p); err != nil { + return err + } + } + open := append([]string{}, endpoints...) + open = append(open, net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), net.JoinHostPort(registrySvc.Spec.ClusterIP, "5000"), net.JoinHostPort(archiveSvc.Spec.ClusterIP, "8090")) + // Positive probes need temporary, narrowly scoped ingress grants where the + // production fence admits only the controller or archive-transfer jobs. + for _, svc := range []*corev1.Service{&apiSvc, ®istrySvc, &archiveSvc} { + if len(svc.Spec.Selector) == 0 || len(svc.Spec.Ports) == 0 { + return fmt.Errorf("probe Service %s has no backend", svc.Name) + } + targetPort := svc.Spec.Ports[0].TargetPort + if targetPort.IntVal == 0 && targetPort.StrVal == "" { + targetPort = intstr.FromInt32(svc.Spec.Ports[0].Port) + } + allow := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: id + "-" + svc.Name, Namespace: svc.Namespace}, Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: svc.Spec.Selector}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + Ingress: []networkingv1.NetworkPolicyIngressRule{{From: []networkingv1.NetworkPolicyPeer{{ + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ns}}, + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"felis.lolicon.best/probe-source": id}}, + }}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &targetPort}}}}, + }} + if err := create(allow); err != nil { + return err + } + } + positive := probeJob(id+"-positive", ns, name, image, probeArgs("--open", open), false) + positive.Spec.Template.Labels["felis.lolicon.best/probe-source"] = id + if err := create(positive); err != nil { + return err + } + if err := waitProbeJob(ctx, cl, positive); err != nil { + return err + } + denied = append(denied, open...) + denied = append(denied, "169.254.169.254:80", "169.254.170.2:80") + negative := probeJob(id+"-negative", ns, name, image, probeArgs("--closed", denied), true) + negative.Spec.Template.Spec.InitContainers = []corev1.Container{{Name: "egress-gate", Image: image, Command: []string{"/usr/local/bin/felis", "egress-gate", "--probe", net.JoinHostPort(apiSvc.Spec.ClusterIP, "443"), "--wait", "2m"}, SecurityContext: negative.Spec.Template.Spec.Containers[0].SecurityContext}} + if err := create(negative); err != nil { + return err + } + if err := waitProbeJob(ctx, cl, negative); err != nil { + return err + } + // Dial from Velocity's host namespace and record the address actually observed inside each backend. + for _, endpoint := range endpoints { + c, err := net.DialTimeout("tcp", endpoint, 5*time.Second) + if err != nil { + return fmt.Errorf("Velocity host cannot dial backend Service %s: %w", endpoint, err) + } + c.Close() + } + var observed []string + time.Sleep(500 * time.Millisecond) + for _, p := range echoPods { + foundA := false + stream, err := cs.CoreV1().Pods(ns).GetLogs(p.Name, &corev1.PodLogOptions{}).Stream(ctx) + if err != nil { + return err + } + scan := bufio.NewScanner(stream) + for scan.Scan() { + line := scan.Text() + if strings.HasPrefix(line, "felis-probe-source ") { + host, _, err := net.SplitHostPort(strings.TrimPrefix(line, "felis-probe-source ")) + if err == nil { + cidr, _ := exactCIDR(host) + for _, a := range aCIDRs { + if cidr == a { + observed = append(observed, cidr) + foundA = true + } + } + } + } + } + if !foundA { + stream.Close() + return fmt.Errorf("backend %s did not observe A as an exact source", p.Name) + } + err = scan.Err() + stream.Close() + if err != nil { + return err + } + } + if len(observed) < len(echoPods) { + return fmt.Errorf("backend observed a source outside A's exact interfaces") + } + var game networkingv1.NetworkPolicy + if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-allow-game-from-velocity"}, &game); err != nil { + return err + } + if len(game.Spec.Ingress) == 0 { + return fmt.Errorf("Velocity ingress policy is not configured") + } + prevGame := game.DeepCopy() + for _, cidr := range observed { + exists := false + for _, peer := range game.Spec.Ingress[0].From { + if peer.IPBlock != nil && peer.IPBlock.CIDR == cidr { + exists = true + break + } + } + if exists { + continue + } + game.Spec.Ingress[0].From = append(game.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}}) + } + if err := cl.Patch(ctx, &game, client.MergeFrom(prevGame)); err != nil { + return err + } + // Read fresh resourceVersion so concurrent node health writes cannot be overwritten. + if err := cl.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil { + return err + } + if !placement.Online(&n) { + return fmt.Errorf("worker became offline during validation") + } + before = n.DeepCopy() + n.Labels[placement.LabelIdentity] = name + n.Labels[placement.LabelRole] = placement.RoleWorker + n.Labels[placement.LabelApproved] = "true" + taints := n.Spec.Taints[:0] + for _, t := range n.Spec.Taints { + if t.Key != "felis.lolicon.best/unapproved" { + taints = append(taints, t) + } + } + n.Spec.Taints = taints + if err := cl.Patch(ctx, &n, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + fmt.Fprintln(stdout, "approved worker", name, "Velocity sources", strings.Join(observed, ",")) + return nil +} + +func probeArgs(flag string, addresses []string) []string { + var args []string + for _, a := range addresses { + args = append(args, flag, a) + } + return args +} +func probePod(name, ns, node, image string, args []string, game bool) *corev1.Pod { + no, yes := false, true + uid := int64(1000) + labels := map[string]string{"felis.lolicon.best/probe-name": name} + if game { + labels[v1alpha1.LabelManagedBy] = operator.ManagedByValue + labels[v1alpha1.LabelComponent] = operator.ComponentValue + labels[v1alpha1.LabelServer] = name + } + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, Spec: corev1.PodSpec{NodeName: node, RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &no, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &yes, RunAsUser: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "probe", Image: image, ImagePullPolicy: corev1.PullAlways, Command: []string{"/usr/local/bin/felis", "node-probe"}, Args: args, Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi"), corev1.ResourceCPU: resource.MustParse("200m")}}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &no, ReadOnlyRootFilesystem: &yes, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}}}}} +} +func probeJob(name, ns, node, image string, args []string, game bool) *batchv1.Job { + p := probePod(name, ns, node, image, args, game) + zero := int32(0) + deadline := int64(600) + return &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, Spec: batchv1.JobSpec{BackoffLimit: &zero, ActiveDeadlineSeconds: &deadline, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: p.Labels}, Spec: p.Spec}}} +} +func waitProbePod(ctx context.Context, cl client.Client, p *corev1.Pod) error { + t := time.NewTicker(time.Second) + defer t.Stop() + for { + if err := cl.Get(ctx, client.ObjectKeyFromObject(p), p); err != nil { + return err + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return nil + } + } + if p.Status.Phase == corev1.PodFailed { + return fmt.Errorf("probe Pod %s failed", p.Name) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + } +} +func waitProbeJob(ctx context.Context, cl client.Client, j *batchv1.Job) error { + t := time.NewTicker(time.Second) + defer t.Stop() + for { + if err := cl.Get(ctx, client.ObjectKeyFromObject(j), j); err != nil { + return err + } + for _, c := range j.Status.Conditions { + if c.Status != corev1.ConditionTrue { + continue + } + if c.Type == batchv1.JobComplete { + return nil + } + if c.Type == batchv1.JobFailed { + return fmt.Errorf("admission probe Job %s failed; inspect its Pod log", j.Name) + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + } +} + +func denyNodeAddresses(ctx context.Context, cl client.Client, ns string, nodes []corev1.Node) error { + var np networkingv1.NetworkPolicy + if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: "felis-server-egress"}, &np); err != nil { + return err + } + before := np.DeepCopy() + for _, n := range nodes { + for _, a := range n.Status.Addresses { + if a.Type != corev1.NodeInternalIP && a.Type != corev1.NodeExternalIP { + continue + } + cidr, err := exactCIDR(a.Address) + if err != nil { + return err + } + for i := range np.Spec.Egress { + for k := range np.Spec.Egress[i].To { + block := np.Spec.Egress[i].To[k].IPBlock + if block != nil && ((strings.Contains(cidr, ":") && block.CIDR == "::/0") || (!strings.Contains(cidr, ":") && block.CIDR == "0.0.0.0/0")) { + block.Except = append(block.Except, cidr) + } + } + } + } + } + return cl.Patch(ctx, &np, client.MergeFrom(before)) +} diff --git a/cmd/felis/node_firewall.go b/cmd/felis/node_firewall.go new file mode 100644 index 0000000..f46b089 --- /dev/null +++ b/cmd/felis/node_firewall.go @@ -0,0 +1,187 @@ +package main + +import ( + "flag" + "fmt" + "io" + "net" + "os" + "os/exec" + "strconv" + "strings" +) + +func nodeFirewall(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("node firewall", flag.ContinueOnError) + fs.SetOutput(stderr) + peers := fs.String("peers", "", "comma-separated exact node IP CIDRs") + controller := fs.String("controller-ip", "", "controller address (optionally :6443)") + main := fs.Bool("controller", false, "A hosts the public API NodePort") + pod := fs.String("pod-cidr", "10.42.0.0/16", "cluster Pod CIDR") + dryRun := fs.Bool("dry-run", false, "print firewall scripts without installing") + controlNS := fs.String("control-namespace", "felis", "controller namespace") + minecraftNS := fs.String("namespace", "minecraft", "game namespace") + apiIP := fs.String("api-service-ip", "10.43.0.1", "Kubernetes API Service IP") + port := fs.Int("node-port", 30443, "API NodePort to block on workers before DNAT") + if err := fs.Parse(args); err != nil { + return 2 + } + if host, _, err := net.SplitHostPort(*controller); err == nil { + *controller = host + } + if net.ParseIP(*apiIP) == nil || net.ParseIP(*controller) == nil || *port < 30000 || *port > 32767 { + fmt.Fprintln(stderr, "node firewall: controller IP and NodePort required") + return 2 + } + if _, _, err := net.ParseCIDR(*pod); err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + var v4, v6 []string + for _, p := range strings.Split(*peers, ",") { + ip, n, err := net.ParseCIDR(p) + if err != nil { + fmt.Fprintln(stderr, "node firewall: invalid peer CIDR") + return 2 + } + ones, bits := n.Mask.Size() + if ones != bits { + fmt.Fprintln(stderr, "node firewall: only exact peer addresses accepted") + return 2 + } + if ip.To4() != nil { + v4 = append(v4, n.String()) + } else { + v6 = append(v6, n.String()) + } + } + script := "#!/bin/bash\nset -euo pipefail\n" + for _, f := range []struct { + bin string + peers []string + metadata string + }{{"iptables", v4, "169.254.0.0/16"}, {"ip6tables", v6, "fe80::/10"}} { + if f.bin == "ip6tables" && len(f.peers) == 0 { + continue + } + b := f.bin + " -w 10" + script += b + " -N FELIS-HOST 2>/dev/null || true\n" + b + " -F FELIS-HOST\n" + script += b + " -N FELIS-FORWARD 2>/dev/null || true\n" + b + " -F FELIS-FORWARD\n" + script += b + " -t raw -N FELIS-NODEPORT 2>/dev/null || true\n" + b + " -t raw -F FELIS-NODEPORT\n" + for _, c := range []struct{ table, parent, chain string }{{"filter", "INPUT", "FELIS-HOST"}, {"filter", "FORWARD", "FELIS-FORWARD"}, {"raw", "PREROUTING", "FELIS-NODEPORT"}} { + script += "while " + b + " -t " + c.table + " -D " + c.parent + " -j " + c.chain + " 2>/dev/null; do :; done\n" + b + " -t " + c.table + " -I " + c.parent + " 1 -j " + c.chain + "\n" + } + script += b + " -A FELIS-HOST -i lo -j RETURN\n" + if *main { + script += b + " -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -A FELIS-HOST -j FELIS-CONTROL\n" + script += b + " -t raw -N FELIS-CONTROL 2>/dev/null || true\n" + b + " -t raw -A FELIS-NODEPORT -j FELIS-CONTROL\n" + } + script += b + " -A FELIS-HOST -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n" + b + " -A FELIS-FORWARD -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n" + family := 4 + if f.bin == "ip6tables" { + family = 6 + } + podIP, _, _ := net.ParseCIDR(*pod) + podFamily := 6 + if podIP.To4() != nil { + podFamily = 4 + } + if family == podFamily { + script += b + " -A FELIS-HOST -s " + *pod + " -j DROP\n" + // raw precedes DNAT and kube-router's filter ACCEPT rules. Block new + // host connections while preserving established Velocity/RCON replies. + script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --syn -j DROP\n" + script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p udp -j DROP\n" + if (net.ParseIP(*apiIP).To4() != nil) == (family == 4) { + script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + *apiIP + " -p tcp --dport 443 --syn -j DROP\n" + } + + script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n" + script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + f.metadata + " -j DROP\n" + script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + f.metadata + " -j DROP\n" + for _, p := range f.peers { + script += b + " -A FELIS-FORWARD -s " + *pod + " -d " + p + " -j DROP\n" + script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p tcp --syn -j DROP\n" + script += b + " -t raw -A FELIS-NODEPORT -s " + *pod + " -d " + p + " -p udp -j DROP\n" + } + } + if !*main { + script += b + " -t raw -A FELIS-NODEPORT -m addrtype --dst-type LOCAL -p tcp --dport " + strconv.Itoa(*port) + " -j DROP\n" + } + for _, p := range f.peers { + script += b + " -A FELIS-HOST -s " + p + " -p udp --dport 51820:51821 -j RETURN\n" + } + script += b + " -A FELIS-HOST -p udp --dport 51820:51821 -j DROP\n" + ctrlIP := net.ParseIP(*controller) + ctrlFamily := 6 + if ctrlIP.To4() != nil { + ctrlFamily = 4 + } + if *main { + for _, p := range f.peers { + script += b + " -A FELIS-HOST -s " + p + " -p tcp --dport 6443 -j RETURN\n" + } + } + if ctrlFamily == family { + script += b + " -A FELIS-HOST -s " + ctrlIP.String() + " -p tcp --dport 10250 -j RETURN\n" + } + script += b + " -A FELIS-HOST -p tcp -m multiport --dports 6443,6444,10250,10255,2379,2380,5000,5001,15432 -j DROP\n" + } + if *dryRun { + fmt.Fprint(stdout, script) + if *main { + fmt.Fprint(stdout, controlFirewallScript(*controlNS, *minecraftNS)) + } + return 0 + } + if err := os.MkdirAll("/etc/felis", 0700); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if err := os.WriteFile("/etc/felis/node-firewall.sh", []byte(script), 0700); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + unit := "[Unit]\nDescription=Felis host and NodePort isolation\nAfter=network-online.target firewalld.service ufw.service\nBefore=k3s.service k3s-agent.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/node-firewall.sh\nRemainAfterExit=yes\n[Install]\nWantedBy=multi-user.target\n" + if err := os.WriteFile("/etc/systemd/system/felis-node-firewall.service", []byte(unit), 0644); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if *main { + refresh := controlFirewallScript(*controlNS, *minecraftNS) + if err := os.WriteFile("/etc/felis/control-firewall.sh", []byte(refresh), 0700); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + svc := "[Unit]\nDescription=Refresh exact controller Pod access to the apiserver\nAfter=k3s.service\n[Service]\nType=oneshot\nExecStart=/etc/felis/control-firewall.sh\n" + timer := "[Unit]\nDescription=Track trusted controller Pods after rescheduling\n[Timer]\nOnBootSec=5s\nOnUnitActiveSec=5s\n[Install]\nWantedBy=timers.target\n" + if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.service", []byte(svc), 0644); err != nil { + return 1 + } + if err := os.WriteFile("/etc/systemd/system/felis-control-firewall.timer", []byte(timer), 0644); err != nil { + return 1 + } + } + for _, cmd := range []*exec.Cmd{exec.Command("systemctl", "daemon-reload"), exec.Command("systemctl", "enable", "felis-node-firewall.service"), exec.Command("bash", "/etc/felis/node-firewall.sh")} { + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + } + if *main { + cmd := exec.Command("systemctl", "enable", "--now", "felis-control-firewall.timer") + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + } + return 0 +} + +func controlFirewallScript(controlNS, minecraftNS string) string { + return "#!/bin/bash\nset -euo pipefail\niptables -w 10 -F FELIS-CONTROL\niptables -w 10 -t raw -F FELIS-CONTROL\n/usr/local/bin/k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{\" \"}{.spec.serviceAccountName}{\" \"}{.status.podIP}{\"\\n\"}{end}' | while read -r ns sa ip; do\ncase \"$ns/$sa\" in " + shellQuote(controlNS+"/felis-api") + "|" + shellQuote(controlNS+"/felis-operator") + "|" + shellQuote(minecraftNS+"/felis-reaper") + "|kube-system/*) ;; *) continue;; esac\n[[ $ip =~ ^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$ ]] || continue\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p tcp --dport 6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p tcp -m multiport --dports 443,6443 -j ACCEPT\niptables -w 10 -t raw -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\niptables -w 10 -A FELIS-CONTROL -s \"$ip/32\" -p udp --dport 53 -j ACCEPT\ndone\n" +} diff --git a/cmd/felis/node_firewall_test.go b/cmd/felis/node_firewall_test.go new file mode 100644 index 0000000..d4aec6e --- /dev/null +++ b/cmd/felis/node_firewall_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "bytes" + "os/exec" + "strings" + "testing" +) + +func TestDistributedFirewallPathsAndSyntax(t *testing.T) { + for _, controller := range []bool{false, true} { + args := []string{"--dry-run", "--peers", "192.0.2.1/32,192.0.2.2/32", "--controller-ip", "192.0.2.1"} + if controller { + args = append(args, "--controller") + } + var out, err bytes.Buffer + if code := nodeFirewall(args, &out, &err); code != 0 { + t.Fatal(code, err.String()) + } + script := out.String() + for _, must := range []string{"-I INPUT 1 -j FELIS-HOST", "-I FORWARD 1 -j FELIS-FORWARD", "-t raw -I PREROUTING 1 -j FELIS-NODEPORT", "-A FELIS-HOST -s 10.42.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --dport 30443 -j DROP", "-d 169.254.0.0/16 -j DROP", "--dst-type LOCAL -p tcp --syn -j DROP", "-d 10.43.0.1 -p tcp --dport 443 --syn -j DROP"} { + if !strings.Contains(script, must) { + t.Fatal("missing protection", must) + } + } + if !controller && strings.Contains(script, " -p tcp --dport 6443 -j RETURN") { + t.Fatal("worker exposed apiserver") + } + check := exec.Command("bash", "-n") + check.Stdin = strings.NewReader(script) + if result, e := check.CombinedOutput(); e != nil { + t.Fatalf("invalid firewall script: %s %v", result, e) + } + } + var out, err bytes.Buffer + if code := nodeFirewall([]string{"--dry-run", "--peers", "10.0.0.0/8", "--controller-ip", "10.0.0.1"}, &out, &err); code != 2 { + t.Fatal("broad node range accepted") + } +} diff --git a/cmd/felis/node_probe.go b/cmd/felis/node_probe.go new file mode 100644 index 0000000..007e60d --- /dev/null +++ b/cmd/felis/node_probe.go @@ -0,0 +1,56 @@ +package main + +import ( + "flag" + "fmt" + "io" + "net" + "time" +) + +// A trusted probe runs with the same server labels and security context before node admission. +func cmdNodeProbe(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("node-probe", flag.ContinueOnError) + fs.SetOutput(stderr) + listen := fs.String("listen", "", "listen and print observed source addresses (admission only)") + var open, closed multiFlag + fs.Var(&open, "open", "required reachable host:port") + fs.Var(&closed, "closed", "required blocked host:port") + if err := fs.Parse(args); err != nil { + return 2 + } + if *listen != "" { + l, err := net.Listen("tcp", *listen) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + defer l.Close() + for { + c, err := l.Accept() + if err != nil { + return 1 + } + fmt.Fprintln(stdout, "felis-probe-source", c.RemoteAddr().String()) + c.Write([]byte("felis-probe\n")) + c.Close() + } + } + time.Sleep(3 * time.Second) + for _, check := range []struct { + addresses []string + wantOpen bool + }{{open, true}, {closed, false}} { + for _, addr := range check.addresses { + c, err := net.DialTimeout("tcp", addr, 2*time.Second) + if c != nil { + c.Close() + } + if (err == nil) != check.wantOpen { + fmt.Fprintf(stderr, "node-probe: %s open=%t, expected %t\n", addr, err == nil, check.wantOpen) + return 1 + } + } + } + return 0 +} diff --git a/cmd/felis/nodes.go b/cmd/felis/nodes.go new file mode 100644 index 0000000..5fadc23 --- /dev/null +++ b/cmd/felis/nodes.go @@ -0,0 +1,157 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net" + "os" + "os/exec" + "strings" + "time" + + felis "felis.lolicon.best" + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/distributed" + "felis.lolicon.best/internal/platform" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func cmdNode(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprintln(stderr, "felis node: list | token | join | approve | firewall") + return 2 + } + if os.Geteuid() != 0 { + fmt.Fprintln(stderr, "felis node requires root/sudo") + return 1 + } + if args[0] == "firewall" { + return nodeFirewall(args[1:], stdout, stderr) + } + fs := flag.NewFlagSet("node "+args[0], flag.ContinueOnError) + fs.SetOutput(stderr) + name := fs.String("name", "", "stable worker node name") + kubeconfig := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local administrator kubeconfig") + ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace") + controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "controller namespace") + image := fs.String("image", "", "Felis image to re-pull and use for admission probes") + remote := fs.String("ssh-target", "", "SSH target of the trusted worker (normal SSH host verification applies)") + out := fs.String("out", "", "token output file; never printed to stdout") + ttl := fs.Duration("ttl", 10*time.Minute, "bootstrap token lifetime (maximum 1h)") + server := fs.String("server", "", "A's https://address:6443 endpoint for join") + tokenFile := fs.String("token-file", "", "CA-pinned bootstrap token file for join") + registry := fs.String("registry-ip", "", "registry ClusterIP for join") + peers := fs.String("peers", "", "comma-separated exact peer CIDRs for host firewall") + external := fs.String("external-ip", "", "this worker's fixed external IP") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + switch args[0] { + case "token": + if *name == "" || *out == "" || *ttl <= 0 || *ttl > time.Hour { + fmt.Fprintln(stderr, "node token: name, output file and lifetime <=1h required") + return 2 + } + cmd := exec.CommandContext(ctx, "k3s", "token", "create", "--ttl", ttl.String(), "--description", "Felis worker "+*name) + cmd.Stderr = stderr + raw, err := cmd.Output() + if err != nil { + fmt.Fprintln(stderr, "node token: creation failed:", err) + return 1 + } + f, err := os.OpenFile(*out, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + _, err = f.Write(raw) + if err == nil { + err = f.Sync() + } + f.Close() + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintln(stdout, "limited bootstrap token written to", *out) + return 0 + case "join": + exe, err := os.Executable() + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + cmd := exec.CommandContext(ctx, "bash", "-s") + cmd.Stdin = strings.NewReader(felis.BootstrapScript()) + cmd.Stdout = stdout + cmd.Stderr = stderr + cmd.Env = append(os.Environ(), "FELIS_INSTALL_MODE=worker", "FELIS_BOOTSTRAP_FROM_TUI=1", "FELIS_BOOTSTRAP_BINARY="+exe, "FELIS_NODE_NAME="+*name, "FELIS_SERVER_URL="+*server, "FELIS_BOOTSTRAP_TOKEN_FILE="+*tokenFile, "FELIS_REGISTRY_CLUSTER_IP="+*registry, "FELIS_PEER_CIDRS="+*peers, "FELIS_NODE_EXTERNAL_IP="+*external) + if err = cmd.Run(); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + return 0 + case "list", "approve": + default: + fmt.Fprintln(stderr, "unknown node operation") + return 2 + } + cfg, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + scheme := runtime.NewScheme() + clientgoscheme.AddToScheme(scheme) + v1alpha1.AddToScheme(scheme) + cl, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + m := &distributed.Manager{Client: cl, Namespace: *ns} + if args[0] == "list" { + nodes, err := m.Nodes(ctx) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + json.NewEncoder(stdout).Encode(nodes) + return 0 + } + if *name == "" || *image == "" || *remote == "" || strings.HasPrefix(*remote, "-") { + fmt.Fprintln(stderr, "node approve requires name, image and SSH target") + return 2 + } + cs, err := kubernetes.NewForConfig(cfg) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if err = approveNode(ctx, cl, cs, *ns, *controlNS, *name, *image, *remote, stdout); err != nil { + fmt.Fprintln(stderr, "node remains quarantined:", err) + return 1 + } + return 0 +} + +func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" } +func exactCIDR(ip string) (string, error) { + parsed := net.ParseIP(ip) + if parsed == nil { + return "", fmt.Errorf("invalid node address %q", ip) + } + if parsed.To4() != nil { + return parsed.String() + "/32", nil + } + return parsed.String() + "/128", nil +} diff --git a/cmd/felis/operator.go b/cmd/felis/operator.go index 1f96868..59b4cad 100644 --- a/cmd/felis/operator.go +++ b/cmd/felis/operator.go @@ -114,7 +114,10 @@ func cmdOperator(args []string, _, stderr io.Writer) int { // The operator's own image, for the forwarding-config initContainer it // injects into user servers. The Deployment passes it as FELIS_IMAGE (see // platform.OperatorDeployment); absent, that injection is simply skipped. - FelisImage: os.Getenv("FELIS_IMAGE"), + FelisImage: os.Getenv("FELIS_IMAGE"), + Nodes: nil, + EgressProbe: os.Getenv("FELIS_EGRESS_PROBE"), + ControllerNode: os.Getenv("FELIS_CONTROLLER_NODE"), // Uncached: the maintenance-lock check lists Jobs only when a server is // about to start, which does not justify a namespace-wide Job informer. Jobs: mgr.GetAPIReader(), @@ -127,6 +130,9 @@ func cmdOperator(args []string, _, stderr io.Writer) int { Recorder: mgr.GetEventRecorderFor("felis-operator"), Watch: watch, } + if os.Getenv("FELIS_DISTRIBUTED") == "true" { + r.Nodes = mgr.GetAPIReader() + } if err := r.SetupWithManager(mgr); err != nil { fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err) return 1 diff --git a/cmd/felis/reaper.go b/cmd/felis/reaper.go index 4126694..7c135e7 100644 --- a/cmd/felis/reaper.go +++ b/cmd/felis/reaper.go @@ -16,6 +16,7 @@ import ( "felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/backup" "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/distributed" "felis.lolicon.best/internal/platform" "felis.lolicon.best/internal/reaper" corev1 "k8s.io/api/core/v1" @@ -78,6 +79,19 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int { return 1 } + if os.Getenv("FELIS_DISTRIBUTED") == "true" && !*retentionOnly { + m, err := distributionManager(cl, cfg, os.Getenv("FELIS_IMAGE")) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + local, ok := archiver.(*backup.TarLocal) + if !ok { + fmt.Fprintln(stderr, "remote reaper requires tarLocal") + return 1 + } + archiver = &distributed.RemoteArchiver{TarLocal: local, Manager: m} + } drv, err := openPodStore(ctx, cfg.Database.URL, "reaper", stderr) if err != nil { fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err) @@ -94,7 +108,7 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int { fmt.Fprintln(stderr, "felis reaper: retention only — no worlds root is configured, so idle worlds are neither archived nor released") return reportReaperRun(r.RunRetention(ctx), stdout, stderr) } - r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace) + r.Cluster = reaper.NewK8sCluster(cl, cfg.K8s.Namespace).WithDistributed(os.Getenv("FELIS_DISTRIBUTED") == "true") // Pre-reap warnings go out by email when [smtp] is configured (the same // relay and password_ref convention felis-api uses); without it the channel diff --git a/cmd/felis/restore.go b/cmd/felis/restore.go index 2e21fca..f331fe1 100644 --- a/cmd/felis/restore.go +++ b/cmd/felis/restore.go @@ -1,14 +1,18 @@ package main import ( + "context" "flag" "fmt" "io" + "os" + "os/signal" "path/filepath" "strings" + "syscall" + "felis.lolicon.best/internal/archivetransfer" "felis.lolicon.best/internal/backup" - ctrl "sigs.k8s.io/controller-runtime" ) // cmdRestore is the in-Pod entrypoint the restore Job runs. internal/restore @@ -26,6 +30,9 @@ import ( func cmdRestore(args []string, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("restore", flag.ContinueOnError) fs.SetOutput(stderr) + source := fs.String("source-url", "", "one-use archive download URL") + sum := fs.String("sha256", "", "required digest for remote archive") + limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum download size") server := fs.String("server", "", "server name being restored (for logging)") ref := fs.String("ref", "", "absolute path to the archive on the backup mount") store := fs.String("archive-store", "tarLocal", "archive backend (only tarLocal is implemented)") @@ -35,6 +42,22 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int { return 2 } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if *source != "" { + dir, err := os.MkdirTemp("/tmp", "felis-restore-") + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + defer os.RemoveAll(dir) + *backupRoot = dir + *ref = filepath.Join(dir, "world.tar.gz") + if err := archivetransfer.Fetch(ctx, *source, os.Getenv(archivetransfer.TokenEnv), *ref, *sum, *limit); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + } if *ref == "" { fmt.Fprintln(stderr, "felis restore: --ref is required") return 2 @@ -62,11 +85,16 @@ func cmdRestore(args []string, stdout, stderr io.Writer) int { }, } - ctx := ctrl.SetupSignalHandler() if err := archiver.Restore(ctx, backup.ArchiveRef(*ref), *server); err != nil { fmt.Fprintf(stderr, "felis restore: %v\n", err) return 1 } + if *source != "" { + if err := backup.VerifyRestored(ctx, *ref, *worldsRoot); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + } fmt.Fprintf(stdout, "felis restore: server=%s restored from %s into %s\n", *server, *ref, *worldsRoot) return 0 } diff --git a/cmd/felis/run.go b/cmd/felis/run.go index 7052dd8..ce8748e 100644 --- a/cmd/felis/run.go +++ b/cmd/felis/run.go @@ -28,6 +28,10 @@ Commands: scan-gate Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint) push-image Push a scanned image tarball to the registry (internal Job entrypoint) mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer) + server-migrate Move a stopped world between approved nodes (start|status|retry; requires root) + node Join and approve trusted daemon nodes (list|token|join|approve|firewall; requires root) + node-probe Verify reachability and observed sources (internal admission probe) + archive-serve Serve scoped one-use archive transfers on the controller (internal entrypoint) registry-gate Authorize registry writes in front of registry:2 (internal sidecar entrypoint) manifests Render the control-plane RBAC + NetworkPolicy install bundle as YAML apply Create a MinecraftServer CRD (direct K8s write; use -f server.json) @@ -74,6 +78,10 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{ "push-image": cmdPushImage, "mirror-build-tools": cmdMirrorBuildTools, "registry-gate": cmdRegistryGate, + "archive-serve": cmdArchiveServe, + "node": cmdNode, + "server-migrate": cmdServerMigrate, + "node-probe": cmdNodeProbe, "manifests": cmdManifests, "apply": cmdApply, "setup": cmdSetup, diff --git a/cmd/felis/server_migrate.go b/cmd/felis/server_migrate.go new file mode 100644 index 0000000..a9329ad --- /dev/null +++ b/cmd/felis/server_migrate.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "time" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/distributed" + "felis.lolicon.best/internal/placement" + "felis.lolicon.best/internal/platform" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func cmdServerMigrate(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprintln(stderr, "server-migrate: start | status | retry (separate from database migrate)") + return 2 + } + fs := flag.NewFlagSet("server-migrate "+args[0], flag.ContinueOnError) + fs.SetOutput(stderr) + name := fs.String("name", "", "stopped user server") + target := fs.String("target-node", "", "approved target worker") + id := fs.String("id", "", "operation id (required for retry)") + kube := fs.String("kubeconfig", "/etc/rancher/k3s/k3s.yaml", "A's local kubeconfig") + ns := fs.String("namespace", platform.DefaultMinecraftNamespace, "world namespace") + cfgPath := fs.String("config", "/var/lib/felis/felis.host.toml", "host config used to record the current owner on the safety backup") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + if os.Geteuid() != 0 { + fmt.Fprintln(stderr, "server-migrate requires root/sudo") + return 1 + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cfg, err := clientcmd.BuildConfigFromFlags("", *kube) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + scheme := runtime.NewScheme() + clientgoscheme.AddToScheme(scheme) + v1alpha1.AddToScheme(scheme) + cl, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + m := &distributed.Manager{Client: cl, Namespace: *ns, Resolve: placement.Resolve(cl, *ns)} + var nodes corev1.NodeList + if err = cl.List(ctx, &nodes); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + for _, n := range nodes.Items { + if n.Labels[placement.LabelRole] == placement.RoleController { + m.Controller = n.Name + } + } + if m.Controller == "" { + fmt.Fprintln(stderr, "distributed controller identity is not configured") + return 1 + } + m.Resolve = placement.Resolve(cl, *ns, m.Controller) + var op distributed.Operation + switch args[0] { + case "start": + host, err := config.Load(*cfgPath) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + drv, err := openPodStore(ctx, host.Database.URL, "migration", stderr) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + defer drv.Close() + var owner sql.NullString + if err = drv.DB().QueryRowContext(ctx, "SELECT owner_id FROM servers WHERE name=$1 AND deleted_at IS NULL AND retire_requested_at IS NULL", *name).Scan(&owner); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + op, err = m.BeginMigration(ctx, *name, *target, owner.String) + case "status": + op, err = m.Migration(ctx, *name, *id) + case "retry": + if *id == "" { + fmt.Fprintln(stderr, "retry requires --id") + return 2 + } + op, err = m.RetryMigration(ctx, *name, *id) + default: + fmt.Fprintln(stderr, "unknown migration operation") + return 2 + } + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + json.NewEncoder(stdout).Encode(op) + return 0 +} diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 6153575..78f3c76 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -274,6 +274,13 @@ FELIS_OFFSITE_DB_KEEP="${FELIS_OFFSITE_DB_KEEP:-}" # the key that pings the check; off removes it, and a re-run without it keeps it. FELIS_WATCHDOG_HEARTBEAT_URL="${FELIS_WATCHDOG_HEARTBEAT_URL:-}" INSTALL_MODE="${FELIS_INSTALL_MODE:-}" +DISTRIBUTED="${FELIS_DISTRIBUTED:-0}" +WORKER_NAME="${FELIS_NODE_NAME:-}" +WORKER_SERVER="${FELIS_SERVER_URL:-}" +WORKER_TOKEN_FILE="${FELIS_BOOTSTRAP_TOKEN_FILE:-}" +WORKER_REGISTRY_IP="${FELIS_REGISTRY_CLUSTER_IP:-}" +NODE_EXTERNAL_IP="${FELIS_NODE_EXTERNAL_IP:-}" +WORKER_PEERS="${FELIS_PEER_CIDRS:-}" # strict stops the install on any preflight problem (preflight below); warn reports them # and goes on, for a host the checks misjudge. FELIS_PREFLIGHT="${FELIS_PREFLIGHT:-strict}" @@ -2024,6 +2031,19 @@ write_k3s_config() { { echo "# Written by the Felis installer (deploy/bootstrap.sh); a rerun rewrites it." echo 'write-kubeconfig-mode: "0600"' + if [ "${DISTRIBUTED:-0}" = 1 ]; then + [ -n "$NODE_EXTERNAL_IP" ] || NODE_EXTERNAL_IP="$NODE_IP" + printf 'node-external-ip: "%s"\n' "$NODE_EXTERNAL_IP" + echo 'flannel-backend: "wireguard-native"' + echo 'flannel-external-ip: true' + echo 'agent-token-file: "/etc/rancher/k3s/felis-agent-token"' + # Append to existing API-server hardening arguments in earlier config files. + echo 'kube-apiserver-arg+:' + echo ' - "enable-admission-plugins=NodeRestriction"' + if [ ! -s /etc/rancher/k3s/felis-agent-token ]; then + (umask 077; openssl rand -hex 32 > /etc/rancher/k3s/felis-agent-token) + fi + fi if [ -n "$name" ]; then printf 'node-name: "%s"\n' "$name" fi @@ -5259,6 +5279,24 @@ deploy_bundle() { revoke_worlds_root_grant + if [ "${DISTRIBUTED:-0}" = 1 ]; then + local controller archive_key_file="${STATE_DIR}/archive-transfer.key" + controller="$(k3s_node_name)" + [ -n "$controller" ] || die "distributed deployment needs a stable controller node name" + kube label node "$controller" "felis.node-restriction.kubernetes.io/role=controller" "felis.node-restriction.kubernetes.io/identity=$controller" --overwrite + local system_deployment + for system_deployment in coredns local-path-provisioner; do + if kube -n kube-system get deployment "$system_deployment" >/dev/null 2>&1; then + kube -n kube-system patch deployment "$system_deployment" --type merge \ + -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"felis.node-restriction.kubernetes.io/identity\":\"$controller\"}}}}}" + fi + done + if [ ! -s "$archive_key_file" ]; then (umask 077; openssl rand -hex 32 > "$archive_key_file"); fi + local archive_key + archive_key="$(cat "$archive_key_file")" + apply_literal_secret "$CONTROL_NS" felis-archive-key key "$archive_key" + apply_literal_secret "$MINECRAFT_NS" felis-archive-key key "$archive_key" + fi log "rendering + applying the control-plane bundle" local -a manifest_args=( --felis-image "$FELIS_IMAGE" @@ -5266,6 +5304,13 @@ deploy_bundle() { --panel-node-port "$FELIS_PANEL_NODEPORT" --velocity-cidr "${NODE_IP}/32" ) + if [ "${DISTRIBUTED:-0}" = 1 ]; then + manifest_args+=(--distributed --controller-node "$controller" --egress-probe "felis-api.${CONTROL_NS}.svc:443") + # Every node address, including global addresses, must be excluded from game egress. + while read -r cidr; do + [ -z "$cidr" ] || manifest_args+=(--server-egress-deny-cidr "$cidr") + done < <(kube get nodes -o jsonpath='{range .items[*]}{range .status.addresses[*]}{.address}{"\n"}{end}{end}' | awk '/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ {print $0"/32"}') + fi local cidr while read -r cidr; do [ -n "$cidr" ] && manifest_args+=(--server-egress-deny-cidr "$cidr") @@ -5660,6 +5705,7 @@ summary() { # prompt (or FELIS_INSTALL_MODE=nano). # --------------------------------------------------------------------------- prompt_install_mode() { + if [ "$INSTALL_MODE" = worker ]; then log "install mode: worker";return; fi # felis setup carries on to the Owner and edge setup, which needs the control plane, so # a nano install under it could only end in a setup error. if bootstrap_from_tui; then @@ -5669,9 +5715,9 @@ prompt_install_mode() { return 0 fi case "$INSTALL_MODE" in - full|nano) log "install mode: ${INSTALL_MODE} (from FELIS_INSTALL_MODE)"; return 0 ;; + full|nano|worker) log "install mode: ${INSTALL_MODE} (from FELIS_INSTALL_MODE)"; return 0 ;; "") ;; - *) die "FELIS_INSTALL_MODE must be 'full' or 'nano', got: ${INSTALL_MODE}" ;; + *) die "FELIS_INSTALL_MODE must be 'full', 'nano' or 'worker', got: ${INSTALL_MODE}" ;; esac # A felis-nano unit with no full install beside it makes this re-run a nano update; @@ -5997,6 +6043,80 @@ ensure_k3s_on_path() { esac } +# Worker is a daemon-only branch. It neither generates Felis service credentials nor applies a controller bundle. +main_worker() { + [ -n "$WORKER_NAME" ] && [[ "$WORKER_NAME" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || die "FELIS_NODE_NAME is required and must be a DNS node name" + [[ "$WORKER_SERVER" =~ ^https://([a-zA-Z0-9.:-]+|\[[0-9a-fA-F:]+\]):6443$ ]] || die "FELIS_SERVER_URL must be an HTTPS k3s endpoint on port 6443" + [[ "$WORKER_REGISTRY_IP" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "FELIS_REGISTRY_CLUSTER_IP is required" + [ -n "$WORKER_PEERS" ] || die "FELIS_PEER_CIDRS must list exact cluster peer addresses" + [ -s "$WORKER_TOKEN_FILE" ] || die "FELIS_BOOTSTRAP_TOKEN_FILE must name a secure limited bootstrap token file" + local token saved mode + [ -f "$WORKER_TOKEN_FILE" ] && [ ! -L "$WORKER_TOKEN_FILE" ] || die "bootstrap token must be a regular file" + mode="$(stat -c %a "$WORKER_TOKEN_FILE")" + (( (8#$mode & 077) == 0 )) || die "bootstrap token must not be readable by group or others (use chmod 600)" + token="$(cat "$WORKER_TOKEN_FILE")" + [[ "$token" =~ ^K10[0-9a-f]{64}::[a-z0-9]{6}\.[a-z0-9]{16}$ ]] || die "worker accepts only CA-pinned bootstrap tokens, never server or static agent tokens" + [ ! -e /var/lib/rancher/k3s/server ] || die "this host has a k3s server; refusing to turn a controller into a worker" + if [ -d /var/lib/rancher/k3s/agent ]; then + saved="$(k3s_node_name)" + [ -n "$saved" ] && [ "$saved" = "$WORKER_NAME" ] || die "cannot change or guess an installed worker identity" + fi + if [ -f "$K3S_CONFIG_DROPIN" ]; then + saved="$(awk -F'"' '/^node-name:/ {print $2;exit}' "$K3S_CONFIG_DROPIN")" + [ -z "$saved" ] || [ "$saved" = "$WORKER_NAME" ] || die "existing node identity is $saved; refusing to rename it" + saved="$(awk -F'"' '/^server:/ {print $2;exit}' "$K3S_CONFIG_DROPIN")" + [ -z "$saved" ] || [ "$saved" = "$WORKER_SERVER" ] || die "existing worker belongs to another controller" + fi + detect_node_ip + [ -n "$NODE_EXTERNAL_IP" ] || NODE_EXTERNAL_IP="$NODE_IP" + PREFLIGHT_PROBLEMS=() + preflight_platform; preflight_memory; preflight_disk; preflight_networks; preflight_outbound + local unit + for unit in rke2-server rke2-agent k0scontroller k0sworker snap.microk8s.daemon-kubelite kubelet k3s; do + if systemctl is-active --quiet "$unit.service"; then preflight_fail "conflicting Kubernetes service: $unit"; fi + done + [ "${#PREFLIGHT_PROBLEMS[@]}" = 0 ] || die "worker preflight failed: ${PREFLIGHT_PROBLEMS[*]}" + install_base + ensure_time_sync + ensure_persistent_journal + # Reuse the release binary path for the local admission checks, without importing game/platform images. + bootstrap_from_tui || [ -n "$FELIS_ARTIFACT_DIR" ] || [ -n "${FELIS_SKIP_FETCH:-}" ] || resolve_install_ref + acquire_felis_binary + if [ ! -x "$HOST_BIN" ]; then install_go_toolchain; build_nano_binary; fi + mkdir -p /etc/rancher/k3s/config.yaml.d + (umask 077; printf '%s\n' "$token" > /etc/rancher/k3s/felis-bootstrap-token) + unset token + cat > "$K3S_CONFIG_DROPIN" < "$K3S_REGISTRIES_FILE" < "$K3S_CONFIG_DROPIN"' "$worker" +expect "worker rejects server tokens and verifies the CA-pinned bootstrap shape" \ + 'K10[0-9a-f]{64}::[a-z0-9]{6}\.[a-z0-9]{16}' "$worker" +expect "worker cannot replace a controller" 'refusing to turn a controller into a worker' "$worker" +expect "worker is quarantined" 'felis.lolicon.best/unapproved=true:NoSchedule' "$worker" +expect "worker mirror preserves logical references and points at the cluster service" \ + 'http://${WORKER_REGISTRY_IP}:5000' "$worker" +expect "worker disables registry endpoint fallback" 'disable-default-registry-endpoint: true' "$worker" +for forbidden in deploy_bundle install_cloudflared install_velocity run_migrations load_or_make_secrets; do + case "$worker" in + *"$forbidden"*) echo "FAIL worker invokes $forbidden"; fails=$((fails + 1));; + *) echo "PASS worker does not invoke $forbidden";; + esac +done +badtoken="$(mktemp)" +printf 'server-token-not-bootstrap' > "$badtoken" +out="$(WORKER_TOKEN_FILE="$badtoken" bash -c ' + die() { printf "DIE: %s\n" "$*"; exit 1; } + WORKER_NAME=b WORKER_SERVER=https://192.0.2.1:6443 WORKER_REGISTRY_IP=10.43.0.10 WORKER_PEERS=192.0.2.1/32 + '"$worker"' + main_worker +')" +expect "worker refuses a copied server token before changing the machine" 'worker accepts only CA-pinned bootstrap tokens' "$out" +rm -f "$badtoken" + # --------------------------------------------------------------------------------------- if [ "$fails" -eq 0 ]; then echo "ALL PASS" diff --git a/deploy/crd/felis.lolicon.best_minecraftservers.yaml b/deploy/crd/felis.lolicon.best_minecraftservers.yaml index c4672e4..89e5727 100644 --- a/deploy/crd/felis.lolicon.best_minecraftservers.yaml +++ b/deploy/crd/felis.lolicon.best_minecraftservers.yaml @@ -281,6 +281,8 @@ spec: storage: description: Storage configures the world PVC. properties: + claimName: + type: string size: description: Size is the requested PVC capacity (e.g. "10Gi"). type: string @@ -289,6 +291,8 @@ spec: uses the default. type: string type: object + nodeName: + type: string subdomain: description: |- Subdomain is the per-server label under the deployment zone. It is the @@ -301,6 +305,8 @@ spec: status: description: MinecraftServerStatus is the observed state (spec §4 status.*). properties: + nodeName: + type: string autoRestarts: description: |- AutoRestarts counts how often the operator recreated the pod of a start diff --git a/deploy/test-node-firewall.sh b/deploy/test-node-firewall.sh new file mode 100755 index 0000000..a12cf18 --- /dev/null +++ b/deploy/test-node-firewall.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Linux/root acceptance of resident-node and pre-DNAT paths. All packet rules, +# listeners and links live in disposable network namespaces, never the live host. +set -euo pipefail +bin="${1:?usage: test-node-firewall.sh /absolute/path/to/felis}" +[[ $bin = /* && -x $bin ]] || exit 2 +[[ $(id -u) = 0 ]] || { echo 'requires root on Linux' >&2; exit 2; } +work=$(mktemp -d) +suffix="$$" +node="felis-fw-node-$suffix" +game="felis-fw-game-$suffix" +peer="felis-fw-peer-$suffix" +listener='' +cleanup() { + if [[ -n $listener ]]; then kill "$listener" 2>/dev/null || true; wait "$listener" 2>/dev/null || true; fi + for ns in "$game" "$peer" "$node"; do ip netns del "$ns" 2>/dev/null || true; done + rm -rf "$work" +} +trap cleanup EXIT +for ns in "$node" "$game" "$peer"; do ip netns add "$ns"; ip -n "$ns" link set lo up; done +ip link add fg-node type veth peer name fg-game +ip link set fg-node netns "$node" +ip link set fg-game netns "$game" +ip link add fp-node type veth peer name fp-peer +ip link set fp-node netns "$node" +ip link set fp-peer netns "$peer" +ip -n "$node" addr add 10.42.250.1/24 dev fg-node +ip -n "$game" addr add 10.42.250.2/24 dev fg-game +ip -n "$node" addr add 192.0.2.2/24 dev fp-node +ip -n "$peer" addr add 192.0.2.1/24 dev fp-peer +ip -n "$node" link set fg-node up +ip -n "$node" link set fp-node up +ip -n "$game" link set fg-game up +ip -n "$peer" link set fp-peer up +ip -n "$game" route add 192.0.2.0/24 via 10.42.250.1 +ip -n "$game" route add 10.43.0.1/32 via 10.42.250.1 +ip netns exec "$node" "$bin" node-probe --listen :18083 >"$work/listener.log" 2>&1 & +listener=$! +ip netns exec "$game" "$bin" node-probe --open 192.0.2.2:18083 +ip netns exec "$peer" "$bin" node-probe --open 192.0.2.2:18083 + +"$bin" node firewall --dry-run --peers 192.0.2.1/32,192.0.2.2/32 \ + --controller-ip 192.0.2.1 --pod-cidr 10.42.0.0/16 \ + --node-port 30443 --api-service-ip 10.43.0.1 >"$work/firewall.sh" +ip netns exec "$node" bash "$work/firewall.sh" +# Simulate later kube-router insertion ahead of Felis filter hooks. +ip netns exec "$node" iptables -I INPUT 1 -j ACCEPT +ip netns exec "$node" iptables -t nat -A PREROUTING -p tcp --dport 30443 -j REDIRECT --to-ports 18083 +ip netns exec "$node" iptables -t nat -A PREROUTING -d 10.43.0.1 -p tcp --dport 443 -j REDIRECT --to-ports 18083 +ip netns exec "$game" "$bin" node-probe \ + --closed 192.0.2.2:18083 --closed 192.0.2.2:30443 --closed 10.43.0.1:443 +ip netns exec "$peer" "$bin" node-probe --closed 192.0.2.2:30443 +# The listener remains healthy and the allowed peer still reaches it. +ip netns exec "$peer" "$bin" node-probe --open 192.0.2.2:18083 +ip netns exec "$node" "$bin" node-probe --open 127.0.0.1:18083 +echo 'PASS resident-node isolation and public NodePort denial survive pre-DNAT and early filter ACCEPT' diff --git a/docs/distributed.md b/docs/distributed.md new file mode 100644 index 0000000..a0df0a8 --- /dev/null +++ b/docs/distributed.md @@ -0,0 +1,109 @@ +# A 主控与多机 worker + +分布式模式默认关闭。A 运行唯一 Felis API/operator、k3s server、PostgreSQL、Registry、归档服务和系统服；Velocity 继续使用 A 的 systemd 服务。B、C 等节点只运行 k3s-agent/containerd、游戏 Pod 和 A 创建的维护 Job。节点必须与 A 同架构、同 k3s 版本，宿主机由管理员信任并维护。 + +## 先准备 A + +在维护窗口停服，使用现有数据库备份流程备份 PostgreSQL，并离线保存 k3s 状态、server token 和当前安装配置。SQLite k3s 的状态目录是 `/var/lib/rancher/k3s/server/db`；使用其他 datastore 时按对应备份流程操作。不要把这些文件复制到 worker。 + +记录 A 的现有节点名称，后续安装必须沿用。先把当前控制工作负载固定到 A，再启用 WireGuard；已有游戏 PVC 不迁移。 + +```bash +# 在 A，以 root 执行；替换节点名和所有固定节点地址。 +A_NODE=existing-node-name +PEERS=192.0.2.10/32,192.0.2.11/32,192.0.2.12/32 +k3s kubectl label node "$A_NODE" \ + felis.node-restriction.kubernetes.io/identity="$A_NODE" \ + felis.node-restriction.kubernetes.io/role=controller --overwrite + +# 按实际部署名称执行，均使用受保护身份标签。 +for d in felis-api felis-operator felis-postgres registry; do + k3s kubectl -n felis patch deployment "$d" --type merge \ + -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"felis.node-restriction.kubernetes.io/identity\":\"$A_NODE\"}}}}}" +done +``` + +用包含此功能的 Felis 安装器在 A 重跑安装：`FELIS_DISTRIBUTED=1`、`FELIS_NODE_EXTERNAL_IP=`、`FELIS_PEER_CIDRS="$PEERS"`，保留现有安装参数。该步骤会启用 `wireguard-native`、`flannel-external-ip`、NodeRestriction 和独立 agent token，并安装归档服务、最小 RBAC 和宿主机隔离规则。WireGuard 更换需要停服维护窗口。已有 worker 的对等地址列表也必须提前更新。 + +直接生成部署清单时，增加： + +```bash +felis manifests --felis-image <现有逻辑镜像引用> \ + --distributed --controller-node "$A_NODE" \ + --egress-probe felis-api.felis.svc:443 \ + --velocity-cidr \ + --archive-local-path <原archive.local_path> --backup-pvc felis-backups +``` + +其他已有参数继续保留。安装器也把 CoreDNS、local-path-provisioner 固定在 A；手动部署时同样给这些 Deployment 设置 A 的受保护节点选择器。手动部署须把同一随机密钥保存到 `felis` 和 `minecraft` 命名空间的 `felis-archive-key` Secret（字段 `key`，至少 32 字符）；只有 API、Reaper 和归档服务获得密钥，operator 不获得。归档 PVC、Registry 和数据库均留在 A。 + +## 接入 B，再接入 C + +先在所有现有节点执行 `felis node firewall --peers "$PEERS" --controller-ip ` 更新完整对等地址列表；A 增加 `--controller`。只允许精确 `/32` 或 `/128` 地址，不能用整个节点/Pod 网段充当 Velocity 或 Registry 来源。固定公网节点之间允许 WireGuard UDP 51820–51821，k3s 6443 只允许已知对等节点。 + +```bash +# A：每台 worker 独立创建，默认 10 分钟；输出文件 root-only，令牌不打印。 +felis node token --name b --ttl 10m --out /root/b.bootstrap +k3s kubectl -n felis get svc registry +# 通过可信 SSH/SCP 把该文件和同版本 felis 二进制交给 B。 + +# B：不需要 server token、管理员 kubeconfig、数据库或 Registry 写入凭据。 +felis node join --name b --server https://:6443 \ + --external-ip --token-file /root/b.bootstrap \ + --registry-ip --peers "$PEERS" + +# A：SSH 使用既有主机密钥校验；目标账号须能 sudo -n。 +felis node approve --name b --ssh-target \ + --image +felis node list +``` + +安装 worker 不安装数据库、Velocity、API 或 operator，不删除本地卷或 node-password；重复安装会拒绝改名或更换集群。mirror 使用 Registry ClusterIP，关闭默认镜像源回退，保留原镜像引用。 + +批准前 worker 带 `NoSchedule` 隔离 taint，且没有受保护的 approved 标签。批准命令检查架构、版本、节点在线状态、WireGuard、宿主机隔离、kubelet 修改受保护标签被拒绝，删除指定镜像后执行真实 `crictl pull`。随后创建临时探针，检查跨节点 Service、控制服务的正向可达性和游戏标签 Pod 的拒绝路径。A 从宿主机连接每个测试 Service，读取后端实际观察到的源地址，只将属于 A 的精确地址写入游戏策略。任何检查失败都保留隔离状态。批准过程中会删除指定缓存镜像，须在该节点尚无游戏任务时执行。 + +节点批准后，管理员可在面板创建服务器时选择它，或在创建请求中传 `nodeName`。普通服主不能选节点或指定 PVC。失联节点拒绝新任务；operator 撤销该服务器 Service 的后端和 Ready 状态，Velocity 进入原有 fallback 流程，不换机。 + +## 停服迁移 + +先在面板停服并等待 `Stopped` 和游戏 Pod 退出。打开“停服迁移”，选择在线且已批准的目标 worker。面板从 CR 的持久化记录读取最新操作，刷新页面或重启 A 后仍可查进度。 + +```bash +# A 的 root 运维入口，区别于数据库 felis migrate： +felis server-migrate start --name survival --target-node c +felis server-migrate status --name survival +felis server-migrate retry --name survival --id <操作ID> +``` + +管理员 API： + +| 方法 | 路径 | 用途 | +| --- | --- | --- | +| GET | `/api/v1/nodes` | 执行节点列表 | +| POST | `/api/v1/servers/{name}/migrations` | `{ "targetNode": "c" }`，返回操作及 ID | +| GET | `/api/v1/servers/{name}/migrations` | 最新迁移记录 | +| GET | `/api/v1/servers/{name}/migrations/{id}` | 当前操作进度 | +| POST | `/api/v1/servers/{name}/migrations/{id}/retry` | 重试失败阶段 | + +阶段为 `backing_up` → `restoring` → `switching` → `succeeded`。锁是 `migration@<开始时间>`，不按临时维护锁的两分钟规则过期。失败保存阶段与原因，保持锁和停服状态；相同目标的重复请求返回已有操作，其他迁移被拒绝。恢复 Job 校验下载 SHA-256，恢复后逐文件读回校验；成功后才在一次乐观锁 CR 更新中切换节点、活动 PVC 和进度。停服 StatefulSet 会重建以使用新的 PVC，CR、Service、ClusterIP、域名及归属保持不变。 + +迁移成功仍不自动启动。检查目标世界后手动启动。记录中的 `sourcePVC` 保留，不被回收流程顺带删除；确认不再需要后由管理员显式清理。失败不要手动删除迁移注解或锁；排除源节点失联、目标磁盘不足、传输/校验错误后使用重试。已提交的切换不会自动回退到源世界。 + +## 隔离与验收 + +游戏进程沿用非 root、禁止提权、drop ALL、无 SA token 和宿主机命名空间的限制。维护 Job 只挂一个世界 PVC：传输 Job 仅访问归档服务和 DNS，文件/导出 Job 仅额外访问既有 API 传输入口。归档服务无数据库配置和 Kubernetes 身份；只有完整归档原子落盘后 A 才记录成功，现有 tarLocal 路径、保留与 offsite 流程继续使用。 + +宿主机 INPUT/FORWARD 规则封闭 resident-node 路径，raw PREROUTING 在 DNAT 前封闭 worker NodePort；A 的受信控制 Pod 精确地址可以访问 apiserver。raw 规则也封闭游戏 Pod 向宿主机发起的新连接，避免 kube-router 的提前 ACCEPT 绕过 filter 规则；Velocity/RCON 的已建立连接回复保留。规则由 systemd 安装，控制 Pod 地址定期更新。节点地址、防火墙或 CNI 配置变动后，先停服并重新执行批准检查，再运行不可信代码。游戏 egress gate 同时检查允许的 DNS TCP 路径和拒绝路径，分布式模式超时拒绝启动。 + +必须在 A/B/C 三台 Linux 机器完成上线验收，不能用单机单元测试代替： + +- Velocity `ClusterIP:25565` 跨节点连接及实际源地址、RCON、休眠唤醒、缓存清除后镜像拉取。 +- B 上备份恢复，B→C 迁移，确认 Service IP/归属不变、源 PVC 保留、目标仍停服。 +- 迁移期间唤醒、文件写入、导出、回收及重复请求互斥；A 重启继续协调。 +- 源节点失联、传输中断、目标/归档磁盘满和读回校验失败，确认源世界可恢复。 +- 游戏 Pod 对各服、主控、Registry、宿主机端口、kubelet、元数据的请求均拒绝；各拒绝目标在受信正向探针中确实可达。 +- 过期、重放、跨服和错误操作的归档令牌拒绝；kubelet 伪造受保护标签拒绝。 + +当前本地验证使用一台已有 Felis 的 ARM64 CentOS Stream 9 VM，通过临时测试程序验证归档和迁移逻辑；`deploy/test-node-firewall.sh` 在独立网络命名空间中实测 resident-node、NodePort DNAT 和提前 ACCEPT 的防护，不改变其现有集群网络。三机网络验收仍是上线前必要步骤。A 继续是控制面和公网入口单点，首版没有主控 HA 或自动故障迁移。 + +相关上游说明：[k3s 跨公网组网](https://docs.k3s.io/networking/distributed-multicloud)、[限时 bootstrap token](https://docs.k3s.io/cli/token)、[NodeRestriction 标签](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-isolationrestriction)、[NetworkPolicy 的节点边界](https://kubernetes.io/docs/concepts/services-networking/network-policies/)。 diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 3241043..b78d185 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -468,11 +468,47 @@ components: description: MinecraftServer lifecycle phase (internal/apis/felis/v1alpha1). enum: [Unknown, Stopped, Starting, Running, Stopping, Failed] + ExecutionNode: + type: object + required: [name, role, ready, approved, addresses, architecture] + properties: + name: { type: string } + role: { type: string } + ready: { type: boolean } + approved: { type: boolean } + addresses: { type: array, items: { type: string } } + architecture: { type: string } + WorldMigration: + type: object + required: [id, server, state, stage, sourceNode, targetNode, sourcePVC, targetPVC, backup, started, updated, switched, attempt] + properties: + id: { type: string } + server: { type: string } + state: { type: string, enum: [backing_up, restoring, switching, succeeded, failed] } + stage: { type: string } + sourceNode: { type: string } + targetNode: { type: string } + sourcePVC: { type: string } + targetPVC: { type: string } + backup: + type: object + required: [ref, size, sha256] + properties: + ref: { type: string } + size: { type: integer, format: int64 } + sha256: { type: string } + started: { type: string, format: date-time } + updated: { type: string, format: date-time } + switched: { type: boolean } + attempt: { type: integer } + error: { type: string } + ServerInfo: type: object description: Status projection of one server (internal/api/cluster.go ServerInfo). required: [name, subdomain, phase, ready, playersOnline, playersMax, idleStopSeconds] properties: + nodeName: { type: string, description: Execution node; legacy servers report the observed node. } name: { type: string } subdomain: { type: string } phase: { $ref: '#/components/schemas/Phase' } @@ -1106,6 +1142,168 @@ components: on. Absent otherwise. paths: + /api/v1/nodes: + get: + operationId: executionNodes + tags: [admin-servers] + summary: List execution nodes (administrator). + x-felis-face: [external] + x-felis-tier: admin + security: [{ sessionCookie: [] }] + responses: + '200': + description: Accepted operation or current state. + content: + application/json: + schema: + type: object + required: [nodes] + properties: + nodes: { type: array, items: { $ref: '#/components/schemas/ExecutionNode' } } + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + $ref: '#/components/responses/Conflict' + '503': + $ref: '#/components/responses/ServiceUnavailable' + + /api/v1/servers/{name}/migrations: + get: + operationId: latestWorldMigrationStatus + tags: [admin-servers] + summary: Read durable migration progress. + x-felis-face: [external] + x-felis-tier: admin + security: [{ sessionCookie: [] }] + parameters: + - { name: name, in: path, required: true, schema: { type: string } } + responses: + '200': + description: Accepted operation or current state. + content: + application/json: + schema: + $ref: '#/components/schemas/WorldMigration' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + $ref: '#/components/responses/Conflict' + '503': + $ref: '#/components/responses/ServiceUnavailable' + + post: + operationId: startWorldMigration + tags: [admin-servers] + summary: Migrate an already stopped world; persistent lock survives controller restart. + x-felis-face: [external] + x-felis-tier: admin + security: [{ sessionCookie: [] }] + parameters: + - { name: name, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: false + required: [targetNode] + properties: + targetNode: { type: string } + responses: + '202': + description: Accepted operation or current state. + content: + application/json: + schema: + $ref: '#/components/schemas/WorldMigration' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + $ref: '#/components/responses/Conflict' + '503': + $ref: '#/components/responses/ServiceUnavailable' + + /api/v1/servers/{name}/migrations/{id}: + get: + operationId: worldMigrationStatus + tags: [admin-servers] + summary: Read durable migration progress. + x-felis-face: [external] + x-felis-tier: admin + security: [{ sessionCookie: [] }] + parameters: + - { name: name, in: path, required: true, schema: { type: string } } + - { name: id, in: path, required: true, schema: { type: string } } + responses: + '200': + description: Accepted operation or current state. + content: + application/json: + schema: + $ref: '#/components/schemas/WorldMigration' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + $ref: '#/components/responses/Conflict' + '503': + $ref: '#/components/responses/ServiceUnavailable' + + /api/v1/servers/{name}/migrations/{id}/retry: + post: + operationId: retryWorldMigration + tags: [admin-servers] + summary: Retry a failed migration, retaining the source and stopped state. + x-felis-face: [external] + x-felis-tier: admin + security: [{ sessionCookie: [] }] + parameters: + - { name: name, in: path, required: true, schema: { type: string } } + - { name: id, in: path, required: true, schema: { type: string } } + responses: + '202': + description: Accepted operation or current state. + content: + application/json: + schema: + $ref: '#/components/schemas/WorldMigration' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + $ref: '#/components/responses/Conflict' + '503': + $ref: '#/components/responses/ServiceUnavailable' + # ----------------------------------------------------------------- health --- /healthz: get: @@ -1272,6 +1470,7 @@ paths: type: object required: [name, subdomain] properties: + nodeName: { type: string, description: Required approved worker in distributed mode; administrator only. } name: { type: string } subdomain: { type: string } displayName: diff --git a/docs/operations.md b/docs/operations.md index 5d395d6..a9d9a41 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -11,7 +11,7 @@ Evidence tags follow troubleshooting.md: **[VM-VERIFIED]** was run on a real hos ## 1. Supported hosts -`deploy/bootstrap.sh` provisions a single node. It needs systemd, root, and one of the +`deploy/bootstrap.sh` defaults to a single node. For the opt-in A controller / worker deployment, see [distributed.md](distributed.md). It needs systemd, root, and one of the package managers below; everything else (k3s, the JRE, cloudflared, and Docker when an image has to be built on the host; see "Where the binary and the images come from" below) it installs. diff --git a/internal/api/api.go b/internal/api/api.go index ccc79f6..b6a7574 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -29,10 +29,11 @@ import ( // API holds the dependencies shared by every handler. type API struct { - Repo Repo - Cluster Cluster - Internal InternalAuth - External ExternalAuth + Distribution Distribution + Repo Repo + Cluster Cluster + Internal InternalAuth + External ExternalAuth // Builder is the image build subsystem (spec §16). It is optional: when nil // the /images routes report 503 rather than 404, so the admin boundary is @@ -721,6 +722,11 @@ func (a *API) externalAPIRoutes() []apiRoute { // Zero-Trust path, unlike the app-tier /me/servers. A path distinct from the // internal velocity GET /api/v1/servers on purpose: the parity test forbids one // {method, path} from carrying both the service and admin tiers. + {Method: "GET", Pattern: "/api/v1/nodes", Admin: true, h: a.handleNodes}, + {Method: "GET", Pattern: "/api/v1/servers/{name}/migrations", Admin: true, h: a.handleMigrationStatus}, + {Method: "POST", Pattern: "/api/v1/servers/{name}/migrations", Admin: true, h: a.handleMigration}, + {Method: "GET", Pattern: "/api/v1/servers/{name}/migrations/{id}", Admin: true, h: a.handleMigrationStatus}, + {Method: "POST", Pattern: "/api/v1/servers/{name}/migrations/{id}/retry", Admin: true, h: a.handleMigrationRetry}, {Method: "GET", Pattern: "/api/v1/fleet", Admin: true, h: a.handleFleet}, // Image build + whitelist (spec §16, §15). Every route is admin-tier: a build // is build-time RCE against the cluster, so submission requires the admin diff --git a/internal/api/cluster.go b/internal/api/cluster.go index a09f6c0..4f57852 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -13,6 +13,7 @@ import ( // patch (spec §7 PATCH /servers/{name}) — never the business-layer fields, which // live in Postgres (spec §22). type ServerInfo struct { + NodeName string `json:"nodeName,omitempty"` Name string `json:"name"` Subdomain string `json:"subdomain"` Phase string `json:"phase"` @@ -64,6 +65,7 @@ type ServerInfo struct { // no free-form YAML path — every field is a typed, validated value. A created // server starts DesiredState=Stopped and unowned (claimed later, spec §9.3). type CreateServerInput struct { + NodeName string Name string Subdomain string DisplayName string diff --git a/internal/api/handlers_distributed.go b/internal/api/handlers_distributed.go new file mode 100644 index 0000000..08eb7c7 --- /dev/null +++ b/internal/api/handlers_distributed.go @@ -0,0 +1,122 @@ +package api + +import ( + "context" + "errors" + "net/http" + + "felis.lolicon.best/internal/distributed" + "felis.lolicon.best/internal/naming" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +type Distribution interface { + Nodes(context.Context) ([]distributed.Node, error) + ValidateNode(context.Context, string) error + BeginMigration(context.Context, string, string, string) (distributed.Operation, error) + Migration(context.Context, string, string) (distributed.Operation, error) + RetryMigration(context.Context, string, string) (distributed.Operation, error) +} + +func (a *API) distributedReady(w http.ResponseWriter, r *http.Request) bool { + if a.Distribution == nil { + writeError(w, r, newError(503, "distributed_unavailable", "distributed deployment is not configured")) + return false + } + return true +} + +func (a *API) handleNodes(w http.ResponseWriter, r *http.Request) { + if !a.distributedReady(w, r) { + return + } + nodes, err := a.Distribution.Nodes(r.Context()) + if err != nil { + writeError(w, r, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"nodes": nodes}) +} + +func (a *API) handleMigration(w http.ResponseWriter, r *http.Request) { + if !a.distributedReady(w, r) { + return + } + server := r.PathValue("name") + if err := naming.ValidateServerName(server); err != nil { + writeError(w, r, newError(400, "bad_name", "%v", err)) + return + } + rec, err := a.Repo.ServerByName(r.Context(), server) + if err != nil { + a.writeLookupError(w, r, err) + return + } + if rec.Retire != nil { + writeError(w, r, errServerRetiring) + return + } + var body struct { + TargetNode string `json:"targetNode"` + } + if err := decodeJSON(w, r, &body); err != nil { + writeError(w, r, err) + return + } + if err := a.Distribution.ValidateNode(r.Context(), body.TargetNode); err != nil { + writeError(w, r, newError(400, "bad_node", "%v", err)) + return + } + op, err := a.Distribution.BeginMigration(r.Context(), server, body.TargetNode, rec.OwnerID) + if err != nil { + a.writeMigrationError(w, r, err) + return + } + writeJSON(w, http.StatusAccepted, op) +} + +func (a *API) handleMigrationStatus(w http.ResponseWriter, r *http.Request) { + if !a.distributedReady(w, r) { + return + } + op, err := a.Distribution.Migration(r.Context(), r.PathValue("name"), r.PathValue("id")) + if err != nil { + a.writeMigrationError(w, r, err) + return + } + writeJSON(w, http.StatusOK, op) +} + +func (a *API) handleMigrationRetry(w http.ResponseWriter, r *http.Request) { + if !a.distributedReady(w, r) { + return + } + rec, err := a.Repo.ServerByName(r.Context(), r.PathValue("name")) + if err != nil { + a.writeLookupError(w, r, err) + return + } + if rec.Retire != nil { + writeError(w, r, errServerRetiring) + return + } + op, err := a.Distribution.RetryMigration(r.Context(), r.PathValue("name"), r.PathValue("id")) + if err != nil { + a.writeMigrationError(w, r, err) + return + } + writeJSON(w, http.StatusAccepted, op) +} + +func (a *API) writeMigrationError(w http.ResponseWriter, r *http.Request, err error) { + switch { + case errors.Is(err, distributed.ErrBusy): + writeError(w, r, newError(409, "migration_busy", "%v", err)) + case errors.Is(err, distributed.ErrNotFound), apierrors.IsNotFound(err): + writeError(w, r, newError(404, "not_found", "%v", err)) + case apierrors.IsConflict(err): + writeError(w, r, newError(409, "conflict", "retry after refreshing migration status")) + default: + writeError(w, r, err) + } +} diff --git a/internal/api/handlers_distributed_test.go b/internal/api/handlers_distributed_test.go new file mode 100644 index 0000000..e6b8494 --- /dev/null +++ b/internal/api/handlers_distributed_test.go @@ -0,0 +1,77 @@ +package api + +import ( + "context" + "errors" + "testing" + + "felis.lolicon.best/internal/distributed" +) + +type fakeDistribution struct { + calls int + target, owner string +} + +func (d *fakeDistribution) Nodes(context.Context) ([]distributed.Node, error) { + d.calls++ + return []distributed.Node{{Name: "b", Ready: true, Approved: true, Addresses: []string{}}}, nil +} +func (d *fakeDistribution) ValidateNode(_ context.Context, name string) error { + if name != "b" { + return errors.New("not approved") + } + return nil +} +func (d *fakeDistribution) BeginMigration(_ context.Context, name, target, owner string) (distributed.Operation, error) { + d.calls++ + d.target = target + d.owner = owner + return distributed.Operation{ID: "op", Server: name, State: "backing_up"}, nil +} +func (d *fakeDistribution) Migration(context.Context, string, string) (distributed.Operation, error) { + d.calls++ + return distributed.Operation{ID: "op", State: "failed"}, nil +} +func (d *fakeDistribution) RetryMigration(context.Context, string, string) (distributed.Operation, error) { + d.calls++ + return distributed.Operation{ID: "op", State: "restoring"}, nil +} + +func TestDistributedRoutesAreAdministratorOnly(t *testing.T) { + for _, role := range []string{"user", "admin"} { + t.Run(role, func(t *testing.T) { + repo := newFakeRepo() + repo.byName["survival"] = &ServerRecord{Name: "survival", OwnerID: "owner"} + a := newTestAPI(repo, newFakeCluster()) + a.External = staticExternal{p: &Principal{UserID: "owner", Role: role, ViaAdminAccess: role == "admin"}} + d := &fakeDistribution{} + a.Distribution = d + for _, tc := range []struct { + method, path, body string + status int + }{ + {"GET", "/api/v1/nodes", "", 200}, + {"POST", "/api/v1/servers/survival/migrations", `{"targetNode":"b"}`, 202}, + {"GET", "/api/v1/servers/survival/migrations", "", 200}, + {"GET", "/api/v1/servers/survival/migrations/op", "", 200}, + {"POST", "/api/v1/servers/survival/migrations/op/retry", "", 202}, + } { + w := do(a.ExternalHandler(), tc.method, tc.path, tc.body, jsonHeader) + want := tc.status + if role == "user" { + want = 403 + } + if w.Code != want { + t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String()) + } + } + if role == "user" && d.calls != 0 { + t.Fatal("owner reached cluster-wide operations") + } + if role == "admin" && (d.target != "b" || d.owner != "owner") { + t.Fatal("wrong migration scope") + } + }) + } +} diff --git a/internal/api/handlers_user.go b/internal/api/handlers_user.go index ba8e990..e6169b1 100644 --- a/internal/api/handlers_user.go +++ b/internal/api/handlers_user.go @@ -406,6 +406,7 @@ type fleetServerView struct { // value and decodeJSON rejects unknown fields, so a caller can never smuggle // free-form YAML or raw CRD fields through this endpoint. type createServerRequest struct { + NodeName string `json:"nodeName,omitempty"` Name string `json:"name"` Subdomain string `json:"subdomain"` DisplayName string `json:"displayName,omitempty"` @@ -445,6 +446,15 @@ func (a *API) handleCreateServer(w http.ResponseWriter, r *http.Request) { return } + if a.Distribution != nil { + if err := a.Distribution.ValidateNode(r.Context(), body.NodeName); err != nil { + writeError(w, r, newError(400, "bad_node", "select an approved worker: %v", err)) + return + } + } else if body.NodeName != "" { + writeError(w, r, newError(400, "bad_node", "node selection requires distributed deployment")) + return + } // Server name and subdomain both obey the §22 portability rule and the // reservation list. if err := naming.ValidateServerName(body.Name); err != nil { @@ -572,6 +582,7 @@ func (a *API) handleCreateServer(w http.ResponseWriter, r *http.Request) { } in := CreateServerInput{ + NodeName: body.NodeName, Name: body.Name, Subdomain: body.Subdomain, DisplayName: displayName, diff --git a/internal/api/k8scluster.go b/internal/api/k8scluster.go index a4c0590..c1dc071 100644 --- a/internal/api/k8scluster.go +++ b/internal/api/k8scluster.go @@ -3,11 +3,13 @@ package api import ( "context" "errors" + "strings" "time" "felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/maintenance" "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/placement" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -32,8 +34,10 @@ import ( // the direct client c, so a write never works from a copy the watch has not caught // up with yet. type K8sCluster struct { - c client.Client - namespace string + distributed bool + controller string + c client.Client + namespace string // servers serves the fleet-wide reads; nil means c. servers client.Reader // synced reports whether servers has its first full list; nil means no cache. @@ -101,15 +105,28 @@ func (k *K8sCluster) GetServer(ctx context.Context, name string) (*ServerInfo, e // and restore Jobs mount), so existence here is exactly existence at Job mount // time. NotFound is (false, nil): the caller refuses with a specific 409. func (k *K8sCluster) WorldVolumeExists(ctx context.Context, name string) (bool, error) { - var pvc corev1.PersistentVolumeClaim - err := k.c.Get(ctx, types.NamespacedName{Namespace: k.namespace, Name: naming.WorldPVCName(name)}, &pvc) - if apierrors.IsNotFound(err) { - return false, nil - } - if err != nil { + var ms v1alpha1.MinecraftServer + err := k.getServer(ctx, name, &ms) + claim := naming.WorldPVCName(name) + if err == nil { + claim = ms.WorldPVC() + } else if !errors.Is(err, ErrNotFound) { return false, err } - return true, nil + var pvc corev1.PersistentVolumeClaim + if err := k.c.Get(ctx, types.NamespacedName{Namespace: k.namespace, Name: claim}, &pvc); err == nil { + return true, nil + } else if !apierrors.IsNotFound(err) { + return false, err + } + if ms.Name == "" { + var retained corev1.PersistentVolumeClaimList + if err := k.c.List(ctx, &retained, client.InNamespace(k.namespace), client.MatchingLabels{v1alpha1.LabelServer: name}); err != nil { + return false, err + } + return len(retained.Items) > 0, nil + } + return false, nil } // PodImages lists the image of every container and init container of every pod @@ -183,6 +200,7 @@ func (k *K8sCluster) CreateServer(ctx context.Context, in CreateServerInput) err Namespace: k.namespace, }, Spec: v1alpha1.MinecraftServerSpec{ + NodeName: in.NodeName, Subdomain: in.Subdomain, DisplayName: in.DisplayName, Image: in.Image, @@ -262,6 +280,22 @@ func (k *K8sCluster) startWith(ctx context.Context, name string, retryFailed boo if err := k.getServer(ctx, name, &ms); err != nil { return err } + node := ms.Spec.NodeName + if node == "" { + node = ms.Status.NodeName + } + if node == "" { + node = k.controller + } + if k.distributed && node != "" { + var n corev1.Node + if err := k.c.Get(ctx, types.NamespacedName{Name: node}, &n); err != nil { + return err + } + if !placement.Admitted(&n, k.controller) || n.Spec.Unschedulable { + return newError(503, "node_unavailable", "execution node is offline") + } + } kind, held, err := k.maintenanceHolder(ctx, &ms) if err != nil { return err @@ -335,6 +369,9 @@ func (k *K8sCluster) ReleaseMaintenance(ctx context.Context, name string) error } return err } + if value := ms.Annotations[maintenance.Annotation]; strings.HasPrefix(value, maintenance.KindMigration+"@") { + return nil + } if _, ok := ms.Annotations[maintenance.Annotation]; !ok { return nil } @@ -445,8 +482,13 @@ func serverInfo(ms *v1alpha1.MinecraftServer) *ServerInfo { memStr = limit.String() } + node := ms.Spec.NodeName + if node == "" { + node = ms.Status.NodeName + } return &ServerInfo{ Name: ms.Name, + NodeName: node, Subdomain: ms.Spec.Subdomain, Phase: string(ms.Status.Phase), Ready: ms.Status.Ready, @@ -483,3 +525,11 @@ func idleStopSeconds(ms *v1alpha1.MinecraftServer) int32 { } return ms.Spec.Idle.EmptySecondsBeforeStop } + +func (k *K8sCluster) WithDistributed(enabled bool, controller ...string) *K8sCluster { + k.distributed = enabled + if len(controller) > 0 { + k.controller = controller[0] + } + return k +} diff --git a/internal/api/k8scluster_test.go b/internal/api/k8scluster_test.go index 2e08206..7718617 100644 --- a/internal/api/k8scluster_test.go +++ b/internal/api/k8scluster_test.go @@ -7,9 +7,12 @@ import ( "sort" "strings" "testing" + "time" "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/maintenance" "felis.lolicon.best/internal/naming" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -19,6 +22,37 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +func TestPersistentMigrationBlocksWakeAndWorldOperations(t *testing.T) { + scheme := runtime.NewScheme() + v1alpha1.AddToScheme(scheme) + corev1.AddToScheme(scheme) + batchv1.AddToScheme(scheme) + s := &v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft", Annotations: map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindMigration, time.Now().Add(-24*time.Hour))}}, Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped}, Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped}} + s.Spec.Storage.ClaimName = "world-survival-migrated" + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(s, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: s.WorldPVC(), Namespace: s.Namespace}}).Build() + k := NewK8sCluster(c, s.Namespace) + ctx := context.Background() + if exists, err := k.WorldVolumeExists(ctx, s.Name); err != nil || !exists { + t.Fatal("active volume ignored", err) + } + if err := k.SetDesiredState(ctx, s.Name, v1alpha1.DesiredRunning); !errors.Is(err, ErrMaintenanceInProgress) { + t.Fatal("migration admitted wake", err) + } + for _, kind := range []string{maintenance.KindBackup, maintenance.KindFileWrite, maintenance.KindReap} { + if err := k.AcquireMaintenance(ctx, s.Name, kind); !errors.Is(err, ErrMaintenanceInProgress) { + t.Fatal("migration admitted", kind, err) + } + } + if err := k.ReleaseMaintenance(ctx, s.Name); err != nil { + t.Fatal(err) + } + var current v1alpha1.MinecraftServer + c.Get(ctx, types.NamespacedName{Namespace: s.Namespace, Name: s.Name}, ¤t) + if current.Annotations[maintenance.Annotation] == "" { + t.Fatal("normal release cleared persistent migration lock") + } +} + // K8sCluster is documented as integration-tested against a live cluster rather // than covered by the hermetic suite, and for most of it that is the right call — // merge-patch semantics are not worth faking. This one test departs from it diff --git a/internal/api/k8sjobstatus.go b/internal/api/k8sjobstatus.go index cbc15be..c7bd2ed 100644 --- a/internal/api/k8sjobstatus.go +++ b/internal/api/k8sjobstatus.go @@ -202,6 +202,9 @@ func (k *K8sJobStatus) PendingRestoreChains(ctx context.Context) ([]RestoreChain snapshot = ChainSnapshotFailed } } + if j.Labels["felis.lolicon.best/archive-pending"] == "true" && snapshot == ChainSnapshotSucceeded { + snapshot = ChainSnapshotRunning + } out = append(out, RestoreChain{ Job: j.Name, Server: j.Labels[jobServerLabel], diff --git a/internal/apis/felis/v1alpha1/minecraftserver_types.go b/internal/apis/felis/v1alpha1/minecraftserver_types.go index 749562a..3046eb0 100644 --- a/internal/apis/felis/v1alpha1/minecraftserver_types.go +++ b/internal/apis/felis/v1alpha1/minecraftserver_types.go @@ -1,6 +1,7 @@ package v1alpha1 import ( + "felis.lolicon.best/internal/naming" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -159,6 +160,9 @@ type MinecraftServerSpec struct { // ReaperExempt opts this server out of the world reaper entirely (spec §18). ReaperExempt bool `json:"reaperExempt,omitempty"` + // NodeName is an administrator-approved execution node; empty preserves legacy placement. + NodeName string `json:"nodeName,omitempty"` + // DesiredState toggles the server up or down (default Stopped). // +kubebuilder:validation:Enum=Running;Stopped DesiredState DesiredState `json:"desiredState,omitempty"` @@ -245,6 +249,8 @@ type SecretKeyRef struct { // StorageSpec configures the world PVC (spec §4 spec.storage). type StorageSpec struct { + // ClaimName is managed internally by stopped-world migration, never by a public spec patch. + ClaimName string `json:"claimName,omitempty"` // Size is the requested PVC capacity (e.g. "10Gi"). Size string `json:"size,omitempty"` // StorageClassName selects the StorageClass; empty uses the default. @@ -317,6 +323,7 @@ type IdleSpec struct { // MinecraftServerStatus is the observed state (spec §4 status.*). type MinecraftServerStatus struct { + NodeName string `json:"nodeName,omitempty"` // Phase is the coarse lifecycle phase. Phase Phase `json:"phase,omitempty"` // Ready is true only after a successful RCON probe (loader-agnostic; a @@ -377,3 +384,11 @@ type PlayersStatus struct { Online int32 `json:"online"` Max int32 `json:"max"` } + +// WorldPVC resolves the active world, including servers created before migration support. +func (s *MinecraftServer) WorldPVC() string { + if s.Spec.Storage.ClaimName != "" { + return s.Spec.Storage.ClaimName + } + return naming.WorldPVCName(s.Name) +} diff --git a/internal/archivetransfer/transfer.go b/internal/archivetransfer/transfer.go new file mode 100644 index 0000000..5d90ed0 --- /dev/null +++ b/internal/archivetransfer/transfer.go @@ -0,0 +1,573 @@ +// Package archivetransfer serves archive bytes on A. It has no Kubernetes or database access. +package archivetransfer + +import ( + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "syscall" + "time" + + "felis.lolicon.best/internal/backup" + "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/worldexport" +) + +const TokenEnv = "FELIS_ARCHIVE_TOKEN" +const KeyEnv = "FELIS_ARCHIVE_KEY" +const LabelPending = "felis.lolicon.best/archive-pending" +const Annotation = "felis.lolicon.best/archive-transfer" +const DefaultLimit int64 = 100 << 30 + +var idRE = regexp.MustCompile(`^[a-f0-9]{32}$`) + +// Ticket names exactly one operation, server and archive. Only A holds the signing key. +type Ticket struct { + ID string `json:"id"` + Server string `json:"server"` + Method string `json:"method"` + Ref string `json:"ref"` + SHA256 string `json:"sha256,omitempty"` + Limit int64 `json:"limit"` + Expires time.Time `json:"expires"` +} + +type Receipt struct { + Ref string `json:"ref"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} + +type Client struct { + URL, Root, Key string + Limit int64 +} + +func ID() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + panic(err) + } + return hex.EncodeToString(b[:]) +} + +func (c Client) Issue(server, method, ref, sum string, ttl time.Duration) (Ticket, string, string, error) { + if len(c.Key) < 32 || c.URL == "" || ttl <= 0 || ttl > 24*time.Hour { + return Ticket{}, "", "", errors.New("archive transport is not configured or ticket lifetime is invalid") + } + id := ID() + if method == http.MethodPut { + ref = filepath.Join(c.Root, fmt.Sprintf("%s-%d.tar.gz", server, time.Now().UnixNano())) + } + limit := c.Limit + if limit <= 0 { + limit = DefaultLimit + } + t := Ticket{ID: id, Server: server, Method: method, Ref: ref, SHA256: sum, Limit: limit, Expires: time.Now().Add(ttl)} + if err := validate(t, c.Root, limit); err != nil { + return Ticket{}, "", "", err + } + raw, _ := json.Marshal(t) + payload := base64.RawURLEncoding.EncodeToString(raw) + mac := hmac.New(sha256.New, []byte(c.Key)) + mac.Write([]byte(payload)) + token := payload + "." + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) + return t, strings.TrimRight(c.URL, "/") + "/transfers/" + id, token, nil +} + +func validate(t Ticket, root string, limit int64) error { + if !idRE.MatchString(t.ID) || t.Limit <= 0 || t.Limit > limit { + return errors.New("invalid transfer bounds") + } + if err := naming.ValidateSystemServerName(t.Server); err != nil { + return err + } + if t.Method != http.MethodPut && t.Method != http.MethodGet { + return errors.New("invalid transfer operation") + } + if !filepath.IsAbs(root) || filepath.Dir(t.Ref) != filepath.Clean(root) || !strings.HasSuffix(t.Ref, ".tar.gz") { + return errors.New("archive must be directly inside the archive root") + } + if !strings.HasPrefix(filepath.Base(t.Ref), t.Server+"-") { + return errors.New("archive belongs to another server") + } + if t.SHA256 != "" { + b, err := hex.DecodeString(t.SHA256) + if err != nil || len(b) != sha256.Size { + return errors.New("invalid SHA-256") + } + } + return nil +} + +// Server journals consumption before IO, so process restarts cannot enable replay. +// A failure consumes the ticket too: the controller issues a fresh ticket on retry. +type Server struct { + Root, Key string + Limit int64 + mu sync.Mutex + freeBytes func() (int64, error) +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" && r.Method == http.MethodGet { + w.WriteHeader(http.StatusNoContent) + return + } + if r.URL.Path == "/archives" && r.Method == http.MethodDelete { + if len(s.Key) < 32 || !hmac.Equal([]byte(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")), []byte(s.Key)) { + http.Error(w, "unauthorized", 401) + return + } + var q struct { + Ref string `json:"ref"` + } + if json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&q) != nil || filepath.Dir(q.Ref) != filepath.Clean(s.Root) { + http.Error(w, "bad ref", 400) + return + } + if err := os.Remove(q.Ref); err != nil && !errors.Is(err, os.ErrNotExist) { + http.Error(w, "archive delete failed", 500) + return + } + if err := syncDir(s.Root); err != nil { + http.Error(w, "archive delete commit failed", 500) + return + } + w.WriteHeader(204) + return + } + if r.URL.Path == "/inspect" && r.Method == http.MethodPost { + if len(s.Key) < 32 || !hmac.Equal([]byte(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")), []byte(s.Key)) { + http.Error(w, "unauthorized", 401) + return + } + var q struct { + Ref string `json:"ref"` + } + if json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&q) != nil || filepath.Dir(q.Ref) != filepath.Clean(s.Root) { + http.Error(w, "bad archive ref", 400) + return + } + local := &backup.TarLocal{BackupRoot: s.Root} + sum, err := local.Verify(r.Context(), backup.ArchiveRef(q.Ref), "") + if err != nil { + http.Error(w, "archive is absent or corrupt", 422) + return + } + st, err := os.Stat(q.Ref) + if err != nil { + http.Error(w, "archive absent", 404) + return + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(Receipt{Ref: q.Ref, SHA256: sum, Size: st.Size()}) + return + } + if strings.HasPrefix(r.URL.Path, "/receipts/") && r.Method == http.MethodGet { + if len(s.Key) < 32 || !hmac.Equal([]byte(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")), []byte(s.Key)) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + id := strings.TrimPrefix(r.URL.Path, "/receipts/") + if !idRE.MatchString(id) { + http.NotFound(w, r) + return + } + rec, err := s.receipt(r.Context(), id) + if err != nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(rec) + return + } + token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + payload, signature, ok := strings.Cut(token, ".") + mac := hmac.New(sha256.New, []byte(s.Key)) + mac.Write([]byte(payload)) + sig, err := base64.RawURLEncoding.DecodeString(signature) + if !ok || err != nil || len(s.Key) < 32 || !hmac.Equal(sig, mac.Sum(nil)) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + raw, err := base64.RawURLEncoding.DecodeString(payload) + var t Ticket + limit := s.Limit + if limit <= 0 { + limit = DefaultLimit + } + if err != nil || json.Unmarshal(raw, &t) != nil || validate(t, s.Root, limit) != nil || time.Now().After(t.Expires) || t.Expires.After(time.Now().Add(24*time.Hour)) || t.Method != r.Method || r.URL.Path != "/transfers/"+t.ID { + http.Error(w, "invalid or expired transfer", http.StatusForbidden) + return + } + if err := s.consume(t); err != nil { + if errors.Is(err, os.ErrExist) { + http.Error(w, "transfer already consumed", http.StatusConflict) + } else { + http.Error(w, "cannot journal transfer", http.StatusInsufficientStorage) + } + return + } + if t.Method == http.MethodGet { + s.download(w, r, t) + return + } + // Serialize uploads and disk checks to preserve a free-space reserve. + s.mu.Lock() + defer s.mu.Unlock() + if err := s.upload(r, t); err != nil { + http.Error(w, "archive upload failed: "+err.Error(), http.StatusUnprocessableEntity) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func syncDir(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} + +func (s *Server) consume(t Ticket) error { + id := t.ID + dir := filepath.Join(s.Root, ".transfers") + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + f, err := os.OpenFile(filepath.Join(dir, id+".used"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + raw, _ := json.Marshal(t) + _, err = f.Write(raw) + if err == nil { + err = f.Sync() + } + cerr := f.Close() + if err != nil { + return err + } + if cerr != nil { + return cerr + } + return syncDir(dir) +} + +func (s *Server) upload(r *http.Request, t Ticket) error { + available, err := s.availableBytes() + if err != nil { + return err + } + bound := min(t.Limit, available) + if bound <= 0 { + return backup.ErrNoRoom + } + tmp := filepath.Join(s.Root, "."+filepath.Base(t.Ref)+".partial") + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer os.Remove(tmp) + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(f, hash), io.LimitReader(r.Body, bound+1)) + if err == nil && n > bound { + err = errors.New("archive exceeds size or disk limit") + } + digest := "sha-256=:" + base64.StdEncoding.EncodeToString(hash.Sum(nil)) + ":" + if err == nil && r.Trailer.Get(worldexport.DigestTrailer) != digest { + err = errors.New("SHA-256 trailer is absent or incorrect") + } + sum := hex.EncodeToString(hash.Sum(nil)) + if err == nil && t.SHA256 != "" && t.SHA256 != sum { + err = errors.New("SHA-256 does not match ticket") + } + if err == nil { + err = f.Sync() + } + cerr := f.Close() + if err == nil { + err = cerr + } + if err != nil { + return err + } + local := &backup.TarLocal{BackupRoot: s.Root} + if _, err = local.Verify(r.Context(), backup.ArchiveRef(tmp), sum); err != nil { + return err + } + // Never overwrite a committed archive, including one whose receipt was interrupted. + if err = os.Link(tmp, t.Ref); err != nil { + return err + } + if err = os.Remove(tmp); err != nil { + return err + } + if err = syncDir(s.Root); err != nil { + return err + } + rec := Receipt{Ref: t.Ref, Size: n, SHA256: sum} + raw, _ := json.Marshal(rec) + journal := filepath.Join(s.Root, ".transfers", t.ID+".json") + f, err = os.OpenFile(journal, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + _, err = f.Write(raw) + if err == nil { + err = f.Sync() + } + cerr = f.Close() + if err == nil { + err = cerr + } + if err != nil { + return err + } + return syncDir(filepath.Dir(journal)) +} + +func (s *Server) download(w http.ResponseWriter, r *http.Request, t Ticket) { + root, err := os.OpenRoot(s.Root) + if err != nil { + http.Error(w, "archive unavailable", 503) + return + } + defer root.Close() + f, err := root.Open(filepath.Base(t.Ref)) + if err != nil { + http.NotFound(w, r) + return + } + defer f.Close() + st, err := f.Stat() + if err != nil || !st.Mode().IsRegular() || st.Size() > t.Limit { + http.Error(w, "archive exceeds transfer bounds", 413) + return + } + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Length", fmt.Sprint(st.Size())) + io.Copy(w, f) +} + +func (c Client) Receipt(ctx context.Context, id string) (Receipt, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(c.URL, "/")+"/receipts/"+id, nil) + if err != nil { + return Receipt{}, err + } + req.Header.Set("Authorization", "Bearer "+c.Key) + hc := &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, err := hc.Do(req) + if err != nil { + return Receipt{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return Receipt{}, fmt.Errorf("archive receipt: %s", resp.Status) + } + var rec Receipt + err = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&rec) + return rec, err +} + +// Fetch stages only the authorized archive and verifies its hash before extraction. +func Fetch(ctx context.Context, url, token, dest, want string, limit int64) error { + if want == "" { + return errors.New("download requires recorded SHA-256") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + hc := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, err := hc.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return fmt.Errorf("archive download: %s", resp.Status) + } + f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(f, hash), io.LimitReader(resp.Body, limit+1)) + if err == nil && (n > limit || hex.EncodeToString(hash.Sum(nil)) != want) { + err = errors.New("archive size or SHA-256 mismatch") + } + if err == nil { + err = f.Sync() + } + cerr := f.Close() + if err == nil { + err = cerr + } + if err != nil { + os.Remove(dest) + } + return err +} + +func (c Client) Inspect(ctx context.Context, ref string) (Receipt, error) { + raw, _ := json.Marshal(map[string]string{"ref": ref}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.URL, "/")+"/inspect", strings.NewReader(string(raw))) + if err != nil { + return Receipt{}, err + } + req.Header.Set("Authorization", "Bearer "+c.Key) + hc := &http.Client{Timeout: 30 * time.Minute, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, err := hc.Do(req) + if err != nil { + return Receipt{}, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return Receipt{}, fmt.Errorf("archive inspect: %s", resp.Status) + } + var rec Receipt + err = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&rec) + return rec, err +} + +func (c Client) Delete(ctx context.Context, ref backup.ArchiveRef) error { + raw, _ := json.Marshal(map[string]string{"ref": string(ref)}) + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, strings.TrimRight(c.URL, "/")+"/archives", strings.NewReader(string(raw))) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+c.Key) + hc := &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, err := hc.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != 204 && resp.StatusCode != 404 { + return fmt.Errorf("archive delete: %s", resp.Status) + } + return nil +} + +func (s *Server) receipt(ctx context.Context, id string) (Receipt, error) { + s.mu.Lock() + defer s.mu.Unlock() + path := filepath.Join(s.Root, ".transfers", id+".json") + var rec Receipt + if raw, err := os.ReadFile(path); err == nil && json.Unmarshal(raw, &rec) == nil { + return rec, nil + } + raw, err := os.ReadFile(filepath.Join(s.Root, ".transfers", id+".used")) + if err != nil { + return rec, err + } + var t Ticket + if err = json.Unmarshal(raw, &t); err != nil { + return rec, err + } + if t.Method != http.MethodPut { + return rec, errors.New("not an upload") + } + sum, err := (&backup.TarLocal{BackupRoot: s.Root}).Verify(ctx, backup.ArchiveRef(t.Ref), t.SHA256) + if err != nil { + return rec, err + } + st, err := os.Stat(t.Ref) + if err != nil || st.Size() > t.Limit { + return rec, errors.New("invalid committed archive") + } + rec = Receipt{Ref: t.Ref, Size: st.Size(), SHA256: sum} + raw, _ = json.Marshal(rec) + tmp := path + ".partial" + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return rec, err + } + _, err = f.Write(raw) + if err == nil { + err = f.Sync() + } + cerr := f.Close() + if err == nil { + err = cerr + } + if err != nil { + return rec, err + } + if err = os.Rename(tmp, path); err != nil { + return rec, err + } + return rec, syncDir(filepath.Dir(path)) +} + +func (s *Server) availableBytes() (int64, error) { + if s.freeBytes != nil { + return s.freeBytes() + } + var st syscall.Statfs_t + if err := syscall.Statfs(s.Root, &st); err != nil { + return 0, err + } + return int64(st.Bavail)*int64(st.Bsize) - int64(float64(st.Blocks)*float64(st.Bsize)*backup.MinFreeAfter), nil +} + +// Sweep keeps receipts for committed archives across arbitrarily long A outages. +// Expired failed transfers and downloads need no replay journal: signature expiry +// still rejects them. The extra day leaves no overlap with an active upload. +func (s *Server) Sweep(now time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + dir := filepath.Join(s.Root, ".transfers") + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".used") { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + return err + } + var ticket Ticket + if json.Unmarshal(raw, &ticket) != nil || now.Before(ticket.Expires.Add(24*time.Hour)) { + continue + } + if ticket.Method == http.MethodPut { + if _, err := os.Stat(ticket.Ref); err == nil { + continue + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + } + for _, name := range []string{e.Name(), ticket.ID + ".json", ticket.ID + ".json.partial"} { + if err := os.Remove(filepath.Join(dir, name)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + } + return syncDir(dir) +} diff --git a/internal/archivetransfer/transfer_test.go b/internal/archivetransfer/transfer_test.go new file mode 100644 index 0000000..3433260 --- /dev/null +++ b/internal/archivetransfer/transfer_test.go @@ -0,0 +1,228 @@ +package archivetransfer + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "felis.lolicon.best/internal/backup" + "felis.lolicon.best/internal/worldexport" +) + +func archiveBytes(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{Name: "world/level.dat", Mode: 0644, Size: 5}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte("world")); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func upload(t *testing.T, url, token string, body []byte, digest bool) int { + t.Helper() + r, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + r.Header.Set("Authorization", "Bearer "+token) + if digest { + sum := sha256.Sum256(body) + r.ContentLength = -1 + r.Trailer = http.Header{worldexport.DigestTrailer: {"sha-256=:" + base64.StdEncoding.EncodeToString(sum[:]) + ":"}} + } + resp, err := http.DefaultClient.Do(r) + if err != nil { + t.Fatal(err) + } + response, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 400 { + t.Log(string(response)) + } + resp.Body.Close() + return resp.StatusCode +} + +func TestDurableTransferAndRestore(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + key := strings.Repeat("k", 32) + s := &Server{Root: root, Key: key, Limit: 1 << 20, freeBytes: func() (int64, error) { return 1 << 30, nil }} + ts := httptest.NewServer(s) + defer ts.Close() + c := Client{Root: root, Key: key, URL: ts.URL, Limit: 1 << 20} + ticket, url, token, err := c.Issue("alice", "PUT", "", "", time.Minute) + if err != nil { + t.Fatal(err) + } + body := archiveBytes(t) + if code := upload(t, url, token, body, true); code != 204 { + t.Fatalf("upload %d", code) + } + if code := upload(t, url, token, body, true); code != 409 { + t.Fatalf("replay %d", code) + } + rec, err := c.Receipt(ctx, ticket.ID) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(body) + if rec.Size != int64(len(body)) || rec.SHA256 != hex.EncodeToString(sum[:]) { + t.Fatalf("receipt %+v", rec) + } + // Simulate A losing the response and crashing between archive commit and receipt write. + if err := os.Remove(filepath.Join(root, ".transfers", ticket.ID+".json")); err != nil { + t.Fatal(err) + } + restarted := &Server{Root: root, Key: key, Limit: 1 << 20, freeBytes: func() (int64, error) { return 1 << 30, nil }} + if recovered, err := restarted.receipt(ctx, ticket.ID); err != nil || recovered != rec { + t.Fatalf("recovery %+v: %v", recovered, err) + } + if _, _, _, err := c.Issue("bob", "GET", rec.Ref, rec.SHA256, time.Minute); err == nil { + t.Fatal("cross-server download ticket accepted") + } + _, getURL, getToken, err := c.Issue("alice", "GET", rec.Ref, rec.SHA256, time.Minute) + if err != nil { + t.Fatal(err) + } + dest := filepath.Join(t.TempDir(), "download.tar.gz") + if err := Fetch(ctx, getURL, getToken, dest, rec.SHA256, 1<<20); err != nil { + t.Fatal(err) + } + if err := Fetch(ctx, getURL, getToken, filepath.Join(t.TempDir(), "replay"), rec.SHA256, 1<<20); err == nil { + t.Fatal("download replay accepted") + } + world := t.TempDir() + a := &backup.TarLocal{BackupRoot: filepath.Dir(dest), Resolve: func(string) (string, error) { return world, nil }} + if err := a.Restore(ctx, backup.ArchiveRef(dest), "alice"); err != nil { + t.Fatal(err) + } + if err := backup.VerifyRestored(ctx, dest, world); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(world, "world", "level.dat"), []byte("wrong"), 0644); err != nil { + t.Fatal(err) + } + if err := backup.VerifyRestored(ctx, dest, world); err == nil { + t.Fatal("read-back corruption accepted") + } +} + +func TestRejectedTransfersNeverCommit(t *testing.T) { + for _, tc := range []struct { + name string + limit int64 + body []byte + digest bool + }{ + {"missing checksum", 1 << 20, archiveBytes(t), false}, + {"invalid tar", 1 << 20, []byte("corrupt archive"), true}, + {"size limit", 10, archiveBytes(t), true}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + key := strings.Repeat("k", 32) + ts := httptest.NewServer(&Server{Root: root, Key: key, Limit: tc.limit, freeBytes: func() (int64, error) { return 1 << 30, nil }}) + defer ts.Close() + c := Client{Root: root, Key: key, URL: ts.URL, Limit: tc.limit} + ticket, url, token, err := c.Issue("alice", "PUT", "", "", time.Minute) + if err != nil { + t.Fatal(err) + } + if code := upload(t, url, token, tc.body, tc.digest); code != 422 { + t.Fatalf("status %d", code) + } + if _, err := os.Stat(ticket.Ref); !os.IsNotExist(err) { + t.Fatalf("archive committed: %v", err) + } + if _, err := c.Receipt(context.Background(), ticket.ID); err == nil { + t.Fatal("failed upload has receipt") + } + if code := upload(t, url, token, tc.body, true); code != 409 { + t.Fatalf("failed upload replay %d", code) + } + }) + } +} + +func signTicket(t Ticket, key string) string { + raw, _ := json.Marshal(t) + payload := base64.RawURLEncoding.EncodeToString(raw) + h := hmac.New(sha256.New, []byte(key)) + h.Write([]byte(payload)) + return payload + "." + base64.RawURLEncoding.EncodeToString(h.Sum(nil)) +} + +func TestExpiredWrongMethodAndPath(t *testing.T) { + root := t.TempDir() + key := strings.Repeat("k", 32) + s := &Server{Root: root, Key: key} + c := Client{Root: root, Key: key, URL: "http://archive"} + ticket, _, token, err := c.Issue("alice", "PUT", "", "", time.Minute) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ method, path, token string }{ + {"GET", "/transfers/" + ticket.ID, token}, + {"PUT", "/transfers/" + ID(), token}, + {"PUT", "/transfers/" + ticket.ID, token + "broken"}, + } { + r := httptest.NewRequest(tc.method, tc.path, nil) + r.Header.Set("Authorization", "Bearer "+tc.token) + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + if w.Code != 403 && w.Code != 401 { + t.Fatalf("scope accepted: %d", w.Code) + } + } + ticket.Expires = time.Now().Add(-time.Minute) + r := httptest.NewRequest("PUT", "/transfers/"+ticket.ID, nil) + r.Header.Set("Authorization", "Bearer "+signTicket(ticket, key)) + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + if w.Code != 403 { + t.Fatalf("expired %d", w.Code) + } +} + +func TestFullDiskDoesNotCommit(t *testing.T) { + root := t.TempDir() + key := strings.Repeat("k", 32) + ts := httptest.NewServer(&Server{Root: root, Key: key, freeBytes: func() (int64, error) { return 0, nil }}) + defer ts.Close() + c := Client{Root: root, Key: key, URL: ts.URL} + ticket, url, token, err := c.Issue("alice", "PUT", "", "", time.Minute) + if err != nil { + t.Fatal(err) + } + if code := upload(t, url, token, archiveBytes(t), true); code != 422 { + t.Fatal("disk full accepted", code) + } + if _, err := os.Stat(ticket.Ref); !os.IsNotExist(err) { + t.Fatal("full disk committed archive", err) + } +} diff --git a/internal/backup/verify_restored.go b/internal/backup/verify_restored.go new file mode 100644 index 0000000..d0c8742 --- /dev/null +++ b/internal/backup/verify_restored.go @@ -0,0 +1,69 @@ +package backup + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "os" +) + +// VerifyRestored reads target files back and compares them with every regular archive entry. +// OpenRoot keeps a malicious path or existing symlink inside the world volume. +func VerifyRestored(ctx context.Context, ref, world string) error { + root, err := os.OpenRoot(world) + if err != nil { + return err + } + defer root.Close() + f, err := os.Open(ref) + if err != nil { + return err + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + return err + } + defer gz.Close() + tr := tar.NewReader(gz) + for { + if err := ctx.Err(); err != nil { + return err + } + h, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + if h.Typeflag != tar.TypeReg && h.Typeflag != tar.TypeRegA { + continue + } + dst, err := root.Open(h.Name) + if err != nil { + return err + } + st, err := dst.Stat() + if err != nil || !st.Mode().IsRegular() || st.Size() != h.Size { + dst.Close() + return fmt.Errorf("restored entry %q has the wrong type or size", h.Name) + } + sourceHash, targetHash := sha256.New(), sha256.New() + _, err = io.Copy(sourceHash, tr) + if err == nil { + _, err = io.Copy(targetHash, dst) + } + dst.Close() + if err != nil { + return err + } + if string(sourceHash.Sum(nil)) != string(targetHash.Sum(nil)) { + return fmt.Errorf("restored entry %q failed SHA-256 read-back", h.Name) + } + } +} diff --git a/internal/backupjob/backup.go b/internal/backupjob/backup.go index fdfba32..95930be 100644 --- a/internal/backupjob/backup.go +++ b/internal/backupjob/backup.go @@ -35,6 +35,7 @@ import ( "time" "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/placement" ) // ErrAlreadyExists is returned by a Jobs implementation when a backup Job for a @@ -57,6 +58,7 @@ type Jobs interface { // the caller leaves the API's Backuper nil so the endpoint reports 503 rather than // enqueuing a Job that cannot run. type Config struct { + ResolveWorld placement.Resolver // Namespace is where the world PVCs live and the backup Job runs (the minecraft // namespace), co-located with the world it snapshots. Namespace string @@ -178,7 +180,15 @@ type Backuper struct { // land on different nodes the RWO attach fails one cleanly. Add single-flight-on- // running only if a real double-tap storm ever shows up. func (b *Backuper) Backup(ctx context.Context, serverName, formerOwner string) error { - if err := b.Jobs.CreateBackupJob(ctx, b.jobParams(serverName, formerOwner)); err != nil { + p := b.jobParams(serverName, formerOwner) + if b.Config.ResolveWorld != nil { + w, err := b.Config.ResolveWorld(ctx, serverName) + if err != nil { + return err + } + p.WorldPVC = w.Claim + } + if err := b.Jobs.CreateBackupJob(ctx, p); err != nil { if errors.Is(err, ErrAlreadyExists) { return nil // suffix collision — treat as enqueued } @@ -195,6 +205,13 @@ func (b *Backuper) Backup(ctx context.Context, serverName, formerOwner string) e // the world volume as a restore until then (internal/maintenance). func (b *Backuper) BackupThenRestore(ctx context.Context, serverName, formerOwner, backupID, backupRef string) error { p := b.jobParams(serverName, formerOwner) + if b.Config.ResolveWorld != nil { + w, err := b.Config.ResolveWorld(ctx, serverName) + if err != nil { + return err + } + p.WorldPVC = w.Claim + } p.RestoreRef, p.RestoreBackupID = backupRef, backupID if err := b.Jobs.CreateBackupJob(ctx, p); err != nil { if errors.Is(err, ErrAlreadyExists) { @@ -210,6 +227,13 @@ func (b *Backuper) BackupThenRestore(ctx context.Context, serverName, formerOwne // [archive] scheduled_keep, so the owner's own backups keep their count. func (b *Backuper) BackupScheduled(ctx context.Context, serverName, formerOwner string) error { p := b.jobParams(serverName, formerOwner) + if b.Config.ResolveWorld != nil { + w, err := b.Config.ResolveWorld(ctx, serverName) + if err != nil { + return err + } + p.WorldPVC = w.Claim + } p.Scheduled = true if err := b.Jobs.CreateBackupJob(ctx, p); err != nil { if errors.Is(err, ErrAlreadyExists) { diff --git a/internal/distributed/manager.go b/internal/distributed/manager.go new file mode 100644 index 0000000..fc48e45 --- /dev/null +++ b/internal/distributed/manager.go @@ -0,0 +1,278 @@ +// Package distributed extends the existing Job executors with archive transport and stopped migration. +package distributed + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/archivetransfer" + "felis.lolicon.best/internal/backup" + "felis.lolicon.best/internal/backupjob" + "felis.lolicon.best/internal/maintenance" + "felis.lolicon.best/internal/placement" + "felis.lolicon.best/internal/restore" + "felis.lolicon.best/internal/worldexport" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const LabelTransfer = "felis.lolicon.best/archive-job" + +type Backup struct { + ID string `json:"id"` + Server string `json:"server"` + Owner string `json:"owner"` + Reason string `json:"reason"` + Protect string `json:"protect,omitempty"` + Receipt archivetransfer.Receipt `json:"receipt"` +} + +type pendingBackup struct { + Ticket archivetransfer.Ticket + Owner, Reason, Protect string +} + +type Manager struct { + Client client.Client + Namespace, Image, Controller string + Archive archivetransfer.Client + Resolve placement.Resolver + // Record is idempotent by transfer ID and runs on A after durable archive commit. + Record func(context.Context, Backup) error +} + +func (m *Manager) CreateBackupJob(ctx context.Context, p backupjob.JobParams) error { + world, err := m.Resolve(ctx, p.Server) + if err != nil { + return err + } + job, t, err := m.uploadJob(p.Server, world.Claim, world.Node, p.JobName) + if err != nil { + return err + } + reason := "manual" + if p.Scheduled { + reason = backupjob.ReasonScheduled + } + if p.RestoreRef != "" { + reason = backupjob.ReasonPreRestore + } + meta := pendingBackup{Ticket: t, Owner: p.FormerOwner, Reason: reason, Protect: p.RestoreBackupID} + raw, _ := json.Marshal(meta) + job.Annotations = map[string]string{archivetransfer.Annotation: string(raw)} + job.Labels[maintenance.LabelManagedBy] = "felis-backup" + job.Labels[archivetransfer.LabelPending] = "true" + if p.RestoreRef != "" { + job.Labels[maintenance.LabelThenRestore] = maintenance.ThenRestorePending + job.Annotations[maintenance.AnnotationRestoreRef] = p.RestoreRef + job.Annotations[maintenance.AnnotationRestoreBackupID] = p.RestoreBackupID + } + job.Spec.TTLSecondsAfterFinished = nil + err = m.Client.Create(ctx, job) + if apierrors.IsAlreadyExists(err) { + return backupjob.ErrAlreadyExists + } + return err +} + +func (m *Manager) uploadJob(server, claim, node, name string) (*batchv1.Job, archivetransfer.Ticket, error) { + t, url, token, err := m.Archive.Issue(server, http.MethodPut, "", "", 2*time.Hour) + if err != nil { + return nil, t, err + } + job, err := worldexport.ExportJob(worldexport.JobParams{Server: server, ID: t.ID[:16], Mode: worldexport.ModeWorld, WorldPVC: claim, TargetURL: url, Token: token, Namespace: m.Namespace, ServiceAccount: "felis-restore", Image: m.Image, WorldsRoot: "/world", Deadline: 2 * time.Hour}) + if err != nil { + return nil, t, err + } + job.Name = name + job.Labels[LabelTransfer] = "true" + job.Spec.Template.Labels[LabelTransfer] = "true" + job.Spec.Template.Spec.Containers[0].Args = append(job.Spec.Template.Spec.Containers[0].Args, "--archive-raw") + m.pin(&job.Spec.Template.Spec, node) + return job, t, nil +} + +func (m *Manager) pin(p *corev1.PodSpec, node string) { + if node == "" { + node = m.Controller + } + p.NodeSelector = map[string]string{placement.LabelIdentity: node} +} + +func (m *Manager) restoreParams(ctx context.Context, p restore.JobParams) (restore.JobParams, error) { + rec, err := m.Archive.Inspect(ctx, p.BackupRef) + if err != nil { + return p, err + } + _, url, token, err := m.Archive.Issue(p.Server, http.MethodGet, p.BackupRef, rec.SHA256, 2*time.Hour) + if err != nil { + return p, err + } + p.SourceURL, p.Token, p.SHA256, p.MaxBytes = url, token, rec.SHA256, m.Archive.Limit + if p.MaxBytes <= 0 { + p.MaxBytes = archivetransfer.DefaultLimit + } + p.BackupPVC = "" + return p, nil +} + +func (m *Manager) CreateRestoreJob(ctx context.Context, p restore.JobParams) error { + world, err := m.Resolve(ctx, p.Server) + if err != nil { + return err + } + p.WorldPVC = world.Claim + p, err = m.restoreParams(ctx, p) + if err != nil { + return err + } + // Preserve the existing deterministic-name conflict and finished-Job retry rules. + return restore.NewK8sJobs(&pinnedClient{Client: m.Client, node: world.Node, manager: m}).CreateRestoreJob(ctx, p) +} + +type pinnedClient struct { + client.Client + node string + manager *Manager +} + +func (c *pinnedClient) Create(ctx context.Context, o client.Object, opts ...client.CreateOption) error { + if j, ok := o.(*batchv1.Job); ok { + c.manager.pin(&j.Spec.Template.Spec, c.node) + j.Labels[LabelTransfer] = "true" + j.Spec.Template.Labels[LabelTransfer] = "true" + } + return c.Client.Create(ctx, o, opts...) +} + +// SettleBackups is retried after A restarts. Jobs remain durable until the row is recorded. +func (m *Manager) SettleBackups(ctx context.Context) error { + var list batchv1.JobList + if err := m.Client.List(ctx, &list, client.InNamespace(m.Namespace), client.MatchingLabels{archivetransfer.LabelPending: "true"}); err != nil { + return err + } + var errs []error + for i := range list.Items { + j := &list.Items[i] + var p pendingBackup + if err := json.Unmarshal([]byte(j.Annotations[archivetransfer.Annotation]), &p); err != nil { + errs = append(errs, err) + continue + } + rec, err := m.Archive.Receipt(ctx, p.Ticket.ID) + if err != nil { + if !maintenance.JobFinished(j) { + continue + } + // A terminal upload failure holds nothing, but must never start its restore chain. + if jobSucceeded(j) { + errs = append(errs, fmt.Errorf("backup %s awaits durable receipt: %w", j.Name, err)) + continue + } + } else { + if rec.Ref != p.Ticket.Ref || rec.SHA256 == "" || rec.Size <= 0 || rec.Size > p.Ticket.Limit { + errs = append(errs, fmt.Errorf("invalid receipt for %s", j.Name)) + continue + } + if m.Record == nil { + errs = append(errs, errors.New("backup recorder unavailable")) + continue + } + if err := m.Record(ctx, Backup{ID: p.Ticket.ID, Server: p.Ticket.Server, Owner: p.Owner, Reason: p.Reason, Protect: p.Protect, Receipt: rec}); err != nil { + errs = append(errs, err) + continue + } + } + before := j.DeepCopy() + delete(j.Labels, archivetransfer.LabelPending) + ttl := int32(600) + j.Spec.TTLSecondsAfterFinished = &ttl + if err := m.Client.Patch(ctx, j, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func jobSucceeded(j *batchv1.Job) bool { + for _, c := range j.Status.Conditions { + if c.Type == batchv1.JobComplete && c.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +// RemoteArchiver lets the existing reaper make every decision on A and snapshot only one remote PVC. +// Local verification, retention and offsite continue to use the same tarLocal paths. +type RemoteArchiver struct { + *backup.TarLocal + Manager *Manager +} + +// A failed migration may wait for operator intervention longer than normal +// backup retention. Keep its safety archive until the persistent lock releases. +func (a *RemoteArchiver) Delete(ctx context.Context, ref backup.ArchiveRef) error { + var servers v1alpha1.MinecraftServerList + if err := a.Manager.Client.List(ctx, &servers, client.InNamespace(a.Manager.Namespace)); err != nil { + return err + } + for i := range servers.Items { + op, err := readOperation(&servers.Items[i]) + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + if err == nil && op.State != "succeeded" && op.Backup.Ref == string(ref) { + return fmt.Errorf("%w: migration retains its safety archive", ErrBusy) + } + } + return a.TarLocal.Delete(ctx, ref) +} + +func (a *RemoteArchiver) Archive(ctx context.Context, server, pvc string) (backup.Archived, error) { + m := a.Manager + world, err := m.Resolve(ctx, server) + if err != nil { + return backup.Archived{}, err + } + if world.Claim != pvc { + return backup.Archived{}, errors.New("active PVC changed") + } + id := archivetransfer.ID() + name := "reap-" + id[:16] + j, t, err := m.uploadJob(server, pvc, world.Node, name) + if err != nil { + return backup.Archived{}, err + } + if err = m.Client.Create(ctx, j); err != nil { + return backup.Archived{}, err + } + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + rec, err := m.Archive.Receipt(ctx, t.ID) + if err == nil { + return backup.Archived{Ref: backup.ArchiveRef(rec.Ref), Size: rec.Size, SHA256: rec.SHA256}, nil + } + var job batchv1.Job + if err = m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: name}, &job); err != nil { + return backup.Archived{}, err + } + if maintenance.JobFinished(&job) && !jobSucceeded(&job) { + return backup.Archived{}, errors.New("remote archive Job failed") + } + select { + case <-ctx.Done(): + return backup.Archived{}, ctx.Err() + case <-ticker.C: + } + } +} diff --git a/internal/distributed/migration.go b/internal/distributed/migration.go new file mode 100644 index 0000000..da7ffce --- /dev/null +++ b/internal/distributed/migration.go @@ -0,0 +1,521 @@ +package distributed + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "time" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/archivetransfer" + "felis.lolicon.best/internal/maintenance" + "felis.lolicon.best/internal/placement" + "felis.lolicon.best/internal/restore" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const MigrationAnnotation = "felis.lolicon.best/migration" + +// Admission errors keep HTTP policy in the API layer. +var ErrBusy = errors.New("server must be fully stopped with no maintenance operation") +var ErrNotFound = errors.New("migration not found") + +type Node struct { + Name string `json:"name"` + Role string `json:"role"` + Ready bool `json:"ready"` + Approved bool `json:"approved"` + Addresses []string `json:"addresses"` + Architecture string `json:"architecture"` +} + +func (m *Manager) Nodes(ctx context.Context) ([]Node, error) { + var list corev1.NodeList + if err := m.Client.List(ctx, &list); err != nil { + return nil, err + } + out := make([]Node, 0, len(list.Items)) + for _, n := range list.Items { + info := Node{Name: n.Name, Role: n.Labels[placement.LabelRole], Ready: placement.Online(&n), Approved: n.Labels[placement.LabelApproved] == "true" && !n.Spec.Unschedulable, Addresses: []string{}, Architecture: n.Status.NodeInfo.Architecture} + for _, a := range n.Status.Addresses { + if a.Type == corev1.NodeInternalIP || a.Type == corev1.NodeExternalIP { + info.Addresses = append(info.Addresses, a.Address) + } + } + out = append(out, info) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, nil +} + +func (m *Manager) ValidateNode(ctx context.Context, name string) error { + return placement.Worker(ctx, m.Client, name) +} + +// Operation is persisted on the CR together with its non-expiring maintenance lock. +// SourcePVC is retained even after success, and retry never changes the committed active world. +type Operation struct { + ID string `json:"id"` + Server string `json:"server"` + State string `json:"state"` + Stage string `json:"stage"` + SourceNode string `json:"sourceNode"` + TargetNode string `json:"targetNode"` + SourcePVC string `json:"sourcePVC"` + TargetPVC string `json:"targetPVC"` + Backup archivetransfer.Receipt `json:"backup"` + Owner string `json:"-"` + Started time.Time `json:"started"` + Updated time.Time `json:"updated"` + Error string `json:"error,omitempty"` + Switched bool `json:"switched"` + Attempt int `json:"attempt"` +} + +// owner travels in persistence, but never on the public operation view. +type persistedOperation struct { + Operation + OwnerID string `json:"ownerId"` +} + +func readOperation(s *v1alpha1.MinecraftServer) (Operation, error) { + var stored persistedOperation + if s.Annotations[MigrationAnnotation] == "" { + return Operation{}, ErrNotFound + } + if err := json.Unmarshal([]byte(s.Annotations[MigrationAnnotation]), &stored); err != nil { + return Operation{}, err + } + stored.Operation.Owner = stored.OwnerID + return stored.Operation, nil +} + +func (m *Manager) Migration(ctx context.Context, server, id string) (Operation, error) { + var s v1alpha1.MinecraftServer + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: server}, &s); err != nil { + return Operation{}, err + } + op, err := readOperation(&s) + if err == nil && id != "" && op.ID != id { + return Operation{}, ErrNotFound + } + return op, err +} + +func (m *Manager) quiet(ctx context.Context, s *v1alpha1.MinecraftServer, own ...string) error { + if s.Spec.DesiredState != v1alpha1.DesiredStopped || s.Status.Phase != v1alpha1.PhaseStopped || s.Status.Ready { + return ErrBusy + } + var pods corev1.PodList + if err := m.Client.List(ctx, &pods, client.InNamespace(m.Namespace), client.MatchingLabels{v1alpha1.LabelServer: s.Name}); err != nil { + return err + } + // Even terminal maintenance Pods must have exited before a new attempt writes its PVC. + for _, p := range pods.Items { + allowed := len(own) > 0 && p.Labels[MigrationAnnotation] == own[0] + if p.Labels[v1alpha1.LabelComponent] == "server" || (!allowed && p.Status.Phase != corev1.PodSucceeded && p.Status.Phase != corev1.PodFailed) { + return ErrBusy + } + } + return nil +} + +func (m *Manager) BeginMigration(ctx context.Context, server, target, owner string) (Operation, error) { + if err := m.ValidateNode(ctx, target); err != nil { + return Operation{}, err + } + var op Operation + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var s v1alpha1.MinecraftServer + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: server}, &s); err != nil { + return err + } + if previous, err := readOperation(&s); err == nil && previous.State != "succeeded" { + if previous.TargetNode == target { + op = previous + return nil + } + return ErrBusy + } else if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + if s.Spec.ReaperExempt || s.Labels[v1alpha1.LabelSystemRole] != "" { + return ErrBusy + } + if err := m.quiet(ctx, &s); err != nil { + return err + } + var jobs batchv1.JobList + if err := m.Client.List(ctx, &jobs, client.InNamespace(m.Namespace), client.MatchingLabels{maintenance.LabelServer: server}); err != nil { + return err + } + if _, held := maintenance.Holder(server, s.Annotations, jobs.Items, time.Now()); held { + return ErrBusy + } + world, err := m.Resolve(ctx, server) + if err != nil { + return err + } + source := world.Node + if source == "" { + source = m.Controller + } + if source == target { + return errors.New("source and target nodes are identical") + } + // Bound local-path worlds have a physical node; reject a guessed or mismatched source. + var pvc corev1.PersistentVolumeClaim + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: world.Claim}, &pvc); err != nil { + return err + } + if pvc.Spec.VolumeName == "" { + return errors.New("source world is not bound") + } + if err := m.volumeOnNode(ctx, pvc.Spec.VolumeName, source); err != nil { + return err + } + id := archivetransfer.ID() + now := time.Now().UTC() + op = Operation{ID: id, Server: server, State: "backing_up", Stage: "backing_up", SourceNode: source, TargetNode: target, SourcePVC: world.Claim, TargetPVC: "world-" + server + "-m" + id[:12], Owner: owner, Started: now, Updated: now} + return m.save(ctx, &s, op, true) + }) + return op, err +} + +func (m *Manager) volumeOnNode(ctx context.Context, volume, node string) error { + var pv corev1.PersistentVolume + if err := m.Client.Get(ctx, types.NamespacedName{Name: volume}, &pv); err != nil { + return err + } + if pv.Spec.NodeAffinity == nil || pv.Spec.NodeAffinity.Required == nil { + return errors.New("migration requires a node-local volume with node affinity") + } + var n corev1.Node + if err := m.Client.Get(ctx, types.NamespacedName{Name: node}, &n); err != nil { + return err + } + for _, term := range pv.Spec.NodeAffinity.Required.NodeSelectorTerms { + if len(term.MatchFields) > 0 { + continue + } + matched := len(term.MatchExpressions) > 0 + for _, e := range term.MatchExpressions { + if e.Operator != corev1.NodeSelectorOpIn { + matched = false + break + } + found := false + for _, v := range e.Values { + if n.Labels[e.Key] == v { + found = true + } + } + if !found { + matched = false + break + } + } + if matched { + return nil + } + } + return errors.New("world volume is not on the recorded execution node") +} + +func (m *Manager) save(ctx context.Context, s *v1alpha1.MinecraftServer, op Operation, lock bool) error { + before := s.DeepCopy() + op.Updated = time.Now().UTC() + raw, _ := json.Marshal(persistedOperation{Operation: op, OwnerID: op.Owner}) + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + s.Annotations[MigrationAnnotation] = string(raw) + if lock { + s.Annotations[maintenance.Annotation] = maintenance.LockValue(maintenance.KindMigration, op.Started) + } else { + delete(s.Annotations, maintenance.Annotation) + } + return m.Client.Patch(ctx, s, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) +} + +func (m *Manager) RetryMigration(ctx context.Context, server, id string) (Operation, error) { + var op Operation + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var s v1alpha1.MinecraftServer + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: server}, &s); err != nil { + return err + } + var err error + op, err = readOperation(&s) + if err != nil { + return err + } + if op.ID != id { + return ErrNotFound + } + if op.State != "failed" { + return ErrBusy + } + if err := m.quiet(ctx, &s); err != nil { + return err + } + if err := m.ValidateNode(ctx, op.TargetNode); err != nil { + return err + } + op.State = op.Stage + op.Error = "" + op.Attempt++ + return m.save(ctx, &s, op, true) + }) + return op, err +} + +func (m *Manager) ReconcileMigrations(ctx context.Context) error { + var servers v1alpha1.MinecraftServerList + if err := m.Client.List(ctx, &servers, client.InNamespace(m.Namespace)); err != nil { + return err + } + var errs []error + for i := range servers.Items { + s := &servers.Items[i] + op, err := readOperation(s) + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + errs = append(errs, err) + continue + } + if op.State == "succeeded" { + if err := m.expireMigrationJobs(ctx, op.ID); err != nil { + errs = append(errs, err) + } + continue + } + if op.State == "failed" { + continue + } + err = m.advance(ctx, s, &op) + if err != nil { + if apierrors.IsConflict(err) { + continue + } + op.Stage = op.State + op.State = "failed" + op.Error = err.Error() + if saveErr := m.save(ctx, s, op, true); saveErr != nil { + errs = append(errs, saveErr) + } + } + } + return errors.Join(errs...) +} + +func (m *Manager) advance(ctx context.Context, s *v1alpha1.MinecraftServer, op *Operation) error { + if err := m.quiet(ctx, s, op.ID); err != nil { + return err + } + if s.Annotations[maintenance.Annotation] != maintenance.LockValue(maintenance.KindMigration, op.Started) { + return errors.New("persistent migration lock was changed") + } + if err := m.ValidateNode(ctx, op.TargetNode); err != nil { + return err + } + if !op.Switched && (s.WorldPVC() != op.SourcePVC || (s.Spec.NodeName != "" && s.Spec.NodeName != op.SourceNode)) { + return errors.New("source placement changed during migration") + } + var source corev1.Node + if !op.Switched { + if err := m.Client.Get(ctx, types.NamespacedName{Name: op.SourceNode}, &source); err != nil { + return err + } + if !placement.Online(&source) { + return errors.New("source node is offline") + } + } + name := fmt.Sprintf("migration-%s-%d", op.ID[:16], op.Attempt) + switch op.State { + case "backing_up": + var j batchv1.Job + err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: name + "-backup"}, &j) + if apierrors.IsNotFound(err) { + job, t, err := m.uploadJob(op.Server, op.SourcePVC, op.SourceNode, name+"-backup") + if err != nil { + return err + } + raw, _ := json.Marshal(t) + job.Annotations = map[string]string{archivetransfer.Annotation: string(raw)} + job.Labels[MigrationAnnotation] = op.ID + job.Spec.Template.Labels[MigrationAnnotation] = op.ID + job.Spec.TTLSecondsAfterFinished = nil + return m.Client.Create(ctx, job) + } + if err != nil { + return err + } + var t archivetransfer.Ticket + if err = json.Unmarshal([]byte(j.Annotations[archivetransfer.Annotation]), &t); err != nil { + return err + } + rec, err := m.Archive.Receipt(ctx, t.ID) + if err != nil { + if maintenance.JobFinished(&j) { + return fmt.Errorf("migration backup has no durable receipt: %w", err) + } + return nil + } + if rec.Ref != t.Ref || rec.SHA256 == "" { + return errors.New("migration backup receipt mismatch") + } + if m.Record == nil { + return errors.New("backup recorder unavailable") + } + if err = m.Record(ctx, Backup{ID: t.ID, Server: op.Server, Owner: op.Owner, Reason: "pre_restore", Receipt: rec}); err != nil { + return err + } + op.Backup = rec + op.State = "restoring" + op.Stage = op.State + return m.save(ctx, s, *op, true) + case "restoring": + if err := m.ensureTarget(ctx, s, op); err != nil { + return err + } + var j batchv1.Job + err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: name + "-restore"}, &j) + if apierrors.IsNotFound(err) { + p := restore.JobParams{Server: op.Server, WorldPVC: op.TargetPVC, BackupRef: op.Backup.Ref, ArchiveStore: "tarLocal", Namespace: m.Namespace, ServiceAccount: "felis-restore", Image: m.Image, WorldsRoot: "/world", BackupRoot: m.Archive.Root, Deadline: 2 * time.Hour} + p, err = m.restoreParams(ctx, p) + if err != nil { + return err + } + if p.SHA256 != op.Backup.SHA256 { + return errors.New("migration archive digest changed") + } + job, err := restore.RestoreJob(p) + if err != nil { + return err + } + job.Name = name + "-restore" + m.pin(&job.Spec.Template.Spec, op.TargetNode) + job.Labels[LabelTransfer] = "true" + job.Spec.Template.Labels[LabelTransfer] = "true" + job.Labels[MigrationAnnotation] = op.ID + job.Spec.Template.Labels[MigrationAnnotation] = op.ID + job.Spec.TTLSecondsAfterFinished = nil + return m.Client.Create(ctx, job) + } + if err != nil { + return err + } + if !maintenance.JobFinished(&j) { + return nil + } + if !jobSucceeded(&j) { + return errors.New("target restore or read-back verification failed") + } + // Job Complete alone is not enough for a RWO handoff: all its processes must be gone. + if err := m.quiet(ctx, s); err != nil { + return nil + } + var pvc corev1.PersistentVolumeClaim + if err = m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: op.TargetPVC}, &pvc); err != nil { + return err + } + if err = m.volumeOnNode(ctx, pvc.Spec.VolumeName, op.TargetNode); err != nil { + return err + } + op.State = "switching" + op.Stage = op.State + return m.save(ctx, s, *op, true) + case "switching": + var sts appsv1.StatefulSet + err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: s.Name}, &sts) + if err == nil { + if sts.Spec.Replicas != nil && *sts.Spec.Replicas != 0 { + return ErrBusy + } + if sts.Status.Replicas != 0 { + return nil + } + policy := metav1.DeletePropagationForeground + return m.Client.Delete(ctx, &sts, &client.DeleteOptions{PropagationPolicy: &policy, Preconditions: &metav1.Preconditions{UID: &sts.UID, ResourceVersion: &sts.ResourceVersion}}) + } + if !apierrors.IsNotFound(err) { + return err + } + // One optimistic CR write commits node, claim and progress. No failure can roll this back. + target := s.DeepCopy() + committed := *op + target.Spec.NodeName = op.TargetNode + target.Spec.Storage.ClaimName = op.TargetPVC + committed.Switched = true + committed.State = "succeeded" + committed.Stage = "succeeded" + committed.Updated = time.Now().UTC() + raw, _ := json.Marshal(persistedOperation{Operation: committed, OwnerID: op.Owner}) + target.Annotations[MigrationAnnotation] = string(raw) + delete(target.Annotations, maintenance.Annotation) + if err := m.Client.Patch(ctx, target, client.MergeFromWithOptions(s.DeepCopy(), client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + *s, *op = *target, committed + return nil + default: + return fmt.Errorf("unknown migration stage %q", op.State) + } +} + +func (m *Manager) ensureTarget(ctx context.Context, s *v1alpha1.MinecraftServer, op *Operation) error { + var pvc corev1.PersistentVolumeClaim + err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: op.TargetPVC}, &pvc) + if err == nil { + if pvc.Labels[MigrationAnnotation] != op.ID { + return errors.New("target PVC identity mismatch") + } + return nil + } + if !apierrors.IsNotFound(err) { + return err + } + var source corev1.PersistentVolumeClaim + if err = m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: op.SourcePVC}, &source); err != nil { + return err + } + size := source.Spec.Resources.Requests[corev1.ResourceStorage] + if size.IsZero() { + size = resource.MustParse("8Gi") + } + pvc = corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: op.TargetPVC, Namespace: m.Namespace, Labels: map[string]string{maintenance.LabelServer: s.Name, MigrationAnnotation: op.ID}}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, StorageClassName: source.Spec.StorageClassName, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: size}}}} + return m.Client.Create(ctx, &pvc) +} + +func (m *Manager) expireMigrationJobs(ctx context.Context, id string) error { + var jobs batchv1.JobList + if err := m.Client.List(ctx, &jobs, client.InNamespace(m.Namespace), client.MatchingLabels{MigrationAnnotation: id}); err != nil { + return err + } + for i := range jobs.Items { + j := &jobs.Items[i] + if j.Spec.TTLSecondsAfterFinished != nil || !maintenance.JobFinished(j) { + continue + } + before := j.DeepCopy() + ttl := int32(600) + j.Spec.TTLSecondsAfterFinished = &ttl + if err := m.Client.Patch(ctx, j, client.MergeFrom(before)); err != nil { + return err + } + } + return nil +} diff --git a/internal/distributed/migration_test.go b/internal/distributed/migration_test.go new file mode 100644 index 0000000..f4c4dc6 --- /dev/null +++ b/internal/distributed/migration_test.go @@ -0,0 +1,352 @@ +package distributed + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/archivetransfer" + "felis.lolicon.best/internal/backup" + "felis.lolicon.best/internal/backupjob" + "felis.lolicon.best/internal/maintenance" + "felis.lolicon.best/internal/placement" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func fixture(t *testing.T) (*Manager, context.Context) { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, batchv1.AddToScheme, appsv1.AddToScheme, v1alpha1.AddToScheme} { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + node := func(name string) *corev1.Node { + return &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{placement.LabelIdentity: name, placement.LabelRole: placement.RoleWorker, placement.LabelApproved: "true", "kubernetes.io/hostname": name}}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}} + } + s := &v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "alice", Namespace: "minecraft"}, Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped, NodeName: "b"}, Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped}} + s.Spec.Storage.Size = "1Gi" + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: s.WorldPVC(), Namespace: "minecraft"}, Spec: corev1.PersistentVolumeClaimSpec{VolumeName: "source", Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}}}} + zero := int32(0) + cl := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(s, &batchv1.Job{}).WithObjects(node("b"), node("c"), s, pvc, localPV("source", "b"), &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alice", Namespace: "minecraft"}, Spec: appsv1.StatefulSetSpec{Replicas: &zero}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "alice", Namespace: "minecraft"}, Spec: corev1.ServiceSpec{ClusterIP: "10.43.0.80"}}).Build() + var receipt archivetransfer.Receipt + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/inspect" && r.Method == "POST" || strings.HasPrefix(r.URL.Path, "/receipts/") { + var jobs batchv1.JobList + cl.List(r.Context(), &jobs) + for _, j := range jobs.Items { + var ticket archivetransfer.Ticket + if json.Unmarshal([]byte(j.Annotations[archivetransfer.Annotation]), &ticket) == nil && ticket.Ref != "" { + receipt = archivetransfer.Receipt{Ref: ticket.Ref, SHA256: strings.Repeat("a", 64), Size: 50} + } + } + if receipt.Ref == "" { + http.NotFound(w, r) + return + } + json.NewEncoder(w).Encode(receipt) + return + } + http.NotFound(w, r) + })) + t.Cleanup(server.Close) + m := &Manager{Client: cl, Namespace: "minecraft", Controller: "a", Image: "registry.local/felis:1", Archive: archivetransfer.Client{Root: "/backups", URL: server.URL, Key: strings.Repeat("k", 32)}, Record: func(context.Context, Backup) error { return nil }} + m.Resolve = placement.Resolve(cl, "minecraft") + return m, context.Background() +} + +func localPV(name, node string) *corev1.PersistentVolume { + return &corev1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: corev1.PersistentVolumeSpec{NodeAffinity: &corev1.VolumeNodeAffinity{Required: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "kubernetes.io/hostname", Operator: corev1.NodeSelectorOpIn, Values: []string{node}}}}}}}}} +} + +func getServer(t *testing.T, m *Manager, ctx context.Context) *v1alpha1.MinecraftServer { + t.Helper() + var s v1alpha1.MinecraftServer + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: "alice"}, &s); err != nil { + t.Fatal(err) + } + return &s +} + +func TestMigrationSurvivesRestartAndKeepsIdentity(t *testing.T) { + m, ctx := fixture(t) + op, err := m.BeginMigration(ctx, "alice", "c", "owner") + if err != nil { + t.Fatal(err) + } + if again, err := m.BeginMigration(ctx, "alice", "c", "owner"); err != nil || again.ID != op.ID { + t.Fatalf("duplicate %+v: %v", again, err) + } + s := getServer(t, m, ctx) + if kind, held := maintenance.Holder(s.Name, s.Annotations, nil, time.Now().Add(365*24*time.Hour)); !held || kind != maintenance.KindMigration { + t.Fatal("migration lock expired") + } + if _, err := m.BeginMigration(ctx, "alice", "b", "owner"); !errors.Is(err, ErrBusy) { + t.Fatalf("competing migration: %v", err) + } + // A restart reconstructs the coordinator solely from persisted CR and Job state. + restarted := *m + m = &restarted + for i := 0; i < 3; i++ { + if err := m.ReconcileMigrations(ctx); err != nil { + t.Fatal(err) + } + } + current, err := m.Migration(ctx, "alice", op.ID) + if err != nil || current.State != "restoring" { + t.Fatalf("progress %+v: %v", current, err) + } + var pvc corev1.PersistentVolumeClaim + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: op.TargetPVC}, &pvc); err != nil { + t.Fatal(err) + } + pvc.Spec.VolumeName = "target" + if err := m.Client.Update(ctx, &pvc); err != nil { + t.Fatal(err) + } + if err := m.Client.Create(ctx, localPV("target", "c")); err != nil { + t.Fatal(err) + } + var jobs batchv1.JobList + if err := m.Client.List(ctx, &jobs); err != nil { + t.Fatal(err) + } + for i := range jobs.Items { + j := &jobs.Items[i] + assertJobIsolation(t, j) + j.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := m.Client.Status().Update(ctx, j); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 3; i++ { + if err := m.ReconcileMigrations(ctx); err != nil { + t.Fatal(err) + } + } + s = getServer(t, m, ctx) + if s.Spec.NodeName != "c" || s.WorldPVC() != op.TargetPVC || s.Spec.DesiredState != v1alpha1.DesiredStopped || s.Status.Ready { + t.Fatalf("unsafe switch %+v", s) + } + if _, held := maintenance.Holder(s.Name, s.Annotations, nil, time.Now()); held { + t.Fatal("success did not release lock") + } + var svc corev1.Service + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: "alice"}, &svc); err != nil || svc.Spec.ClusterIP != "10.43.0.80" { + t.Fatalf("service changed: %v", err) + } + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: op.SourcePVC}, &pvc); err != nil { + t.Fatal("source PVC removed", err) + } + var sts appsv1.StatefulSet + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: "alice"}, &sts); !apierrors.IsNotFound(err) { + t.Fatal("stopped immutable StatefulSet not removed", err) + } +} + +func assertJobIsolation(t *testing.T, j *batchv1.Job) { + t.Helper() + p := j.Spec.Template.Spec + if p.AutomountServiceAccountToken == nil || *p.AutomountServiceAccountToken || p.HostNetwork || p.HostPID || p.HostIPC { + t.Fatal("maintenance has host credentials/namespaces") + } + claims := 0 + for _, v := range p.Volumes { + if v.Secret != nil || v.HostPath != nil { + t.Fatal("maintenance carries secret or host mount") + } + if v.PersistentVolumeClaim != nil { + claims++ + } + } + if claims != 1 { + t.Fatalf("cross-node job mounts %d PVCs", claims) + } + for _, c := range p.Containers { + if c.SecurityContext.AllowPrivilegeEscalation == nil || *c.SecurityContext.AllowPrivilegeEscalation { + t.Fatal("maintenance can escalate") + } + for _, e := range c.Env { + if strings.Contains(e.Name, "DATABASE") || e.Name == archivetransfer.KeyEnv { + t.Fatal("full credentials leaked") + } + } + } +} + +func TestLostSourceFailsClosedAndRetryKeepsSource(t *testing.T) { + m, ctx := fixture(t) + op, err := m.BeginMigration(ctx, "alice", "c", "owner") + if err != nil { + t.Fatal(err) + } + var n corev1.Node + m.Client.Get(ctx, types.NamespacedName{Name: "b"}, &n) + n.Status.Conditions = nil + if err := m.Client.Status().Update(ctx, &n); err != nil { + t.Fatal(err) + } + if err := m.ReconcileMigrations(ctx); err != nil { + t.Fatal(err) + } + failed, err := m.Migration(ctx, "alice", op.ID) + if err != nil || failed.State != "failed" { + t.Fatalf("offline source %+v: %v", failed, err) + } + s := getServer(t, m, ctx) + if s.WorldPVC() != op.SourcePVC || s.Spec.NodeName != "b" { + t.Fatal("failed migration switched placement") + } + if _, held := maintenance.Holder(s.Name, s.Annotations, nil, time.Now().Add(time.Hour)); !held { + t.Fatal("failed migration lost lock") + } + if _, err := m.RetryMigration(ctx, "alice", op.ID); err != nil { + t.Fatal(err) + } + if err := m.ReconcileMigrations(ctx); err != nil { + t.Fatal(err) + } + if current, _ := m.Migration(ctx, "alice", op.ID); current.State != "failed" { + t.Fatal("retry started tasks on offline source") + } +} + +func TestMigrationRejectsLiveProcessAndUnapprovedTarget(t *testing.T) { + m, ctx := fixture(t) + if err := m.Client.Create(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "alice-0", Namespace: m.Namespace, Labels: map[string]string{v1alpha1.LabelServer: "alice", v1alpha1.LabelComponent: "server"}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}); err != nil { + t.Fatal(err) + } + if _, err := m.BeginMigration(ctx, "alice", "c", "owner"); !errors.Is(err, ErrBusy) { + t.Fatal("live source accepted", err) + } + var n corev1.Node + m.Client.Get(ctx, types.NamespacedName{Name: "c"}, &n) + delete(n.Labels, placement.LabelApproved) + m.Client.Update(ctx, &n) + if err := m.ValidateNode(ctx, "c"); err == nil { + t.Fatal("unapproved worker accepted") + } +} + +func TestMigrationFailureRetainsSourceAndCanRetry(t *testing.T) { + for _, stage := range []string{"restoring", "switching"} { + t.Run(stage, func(t *testing.T) { + m, ctx := fixture(t) + op, err := m.BeginMigration(ctx, "alice", "c", "owner") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + if err := m.ReconcileMigrations(ctx); err != nil { + t.Fatal(err) + } + } + var jobs batchv1.JobList + if err := m.Client.List(ctx, &jobs); err != nil { + t.Fatal(err) + } + for i := range jobs.Items { + j := &jobs.Items[i] + condition := batchv1.JobComplete + if stage == "restoring" && strings.HasSuffix(j.Name, "-restore") { + condition = batchv1.JobFailed + } + j.Status.Conditions = []batchv1.JobCondition{{Type: condition, Status: corev1.ConditionTrue}} + if err := m.Client.Status().Update(ctx, j); err != nil { + t.Fatal(err) + } + } + if stage == "switching" { + var pvc corev1.PersistentVolumeClaim + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: op.TargetPVC}, &pvc); err != nil { + t.Fatal(err) + } + pvc.Spec.VolumeName = "target" + if err := m.Client.Update(ctx, &pvc); err != nil { + t.Fatal(err) + } + if err := m.Client.Create(ctx, localPV("target", "c")); err != nil { + t.Fatal(err) + } + fail := true + m.Client = interceptor.NewClient(m.Client.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if s, ok := obj.(*v1alpha1.MinecraftServer); ok && s.Spec.NodeName == "c" && fail { + fail = false + return errors.New("commit interrupted") + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + } + for i := 0; i < 3; i++ { + if err := m.ReconcileMigrations(ctx); err != nil { + t.Fatal(err) + } + } + failed, err := m.Migration(ctx, "alice", op.ID) + if err != nil || failed.State != "failed" || failed.Stage != stage || failed.Switched || failed.Error == "" { + t.Fatalf("failure %+v: %v", failed, err) + } + s := getServer(t, m, ctx) + if s.WorldPVC() != op.SourcePVC || s.Spec.NodeName != "b" || s.Spec.DesiredState != v1alpha1.DesiredStopped { + t.Fatal("failed operation changed active source") + } + if _, held := maintenance.Holder(s.Name, s.Annotations, nil, time.Now().Add(30*24*time.Hour)); !held { + t.Fatal("failed operation released lock") + } + archiver := &RemoteArchiver{TarLocal: &backup.TarLocal{BackupRoot: t.TempDir()}, Manager: m} + if err := archiver.Delete(ctx, backup.ArchiveRef(failed.Backup.Ref)); !errors.Is(err, ErrBusy) { + t.Fatalf("failed migration lost its safety archive: %v", err) + } + if retry, err := m.RetryMigration(ctx, "alice", op.ID); err != nil || retry.State != stage || retry.Attempt != 1 { + t.Fatalf("retry %+v: %v", retry, err) + } + if err := m.ReconcileMigrations(ctx); err != nil { + t.Fatal(err) + } + if stage == "switching" { + if done, _ := m.Migration(ctx, "alice", op.ID); done.State != "succeeded" || !done.Switched { + t.Fatalf("commit retry %+v", done) + } + } + }) + } +} + +func TestBackupResolvesActiveClaimAndPreservesConflictContract(t *testing.T) { + m, ctx := fixture(t) + p := backupjob.JobParams{Server: "alice", WorldPVC: "stale-claim", JobName: "manual-backup"} + if err := m.CreateBackupJob(ctx, p); err != nil { + t.Fatal(err) + } + var j batchv1.Job + if err := m.Client.Get(ctx, types.NamespacedName{Namespace: m.Namespace, Name: p.JobName}, &j); err != nil { + t.Fatal(err) + } + assertJobIsolation(t, &j) + for _, v := range j.Spec.Template.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName != getServer(t, m, ctx).WorldPVC() { + t.Fatal("backup used a stale world claim") + } + } + if err := m.CreateBackupJob(ctx, p); !errors.Is(err, backupjob.ErrAlreadyExists) { + t.Fatalf("duplicate backup: %v", err) + } +} diff --git a/internal/fileedit/editor.go b/internal/fileedit/editor.go index adde9cd..467085f 100644 --- a/internal/fileedit/editor.go +++ b/internal/fileedit/editor.go @@ -52,6 +52,7 @@ import ( "time" "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/placement" ) // Errors the Editor returns, which internal/api maps onto HTTP status codes @@ -106,6 +107,7 @@ type Runner interface { // deployment-specific, and when it is empty cmd/felis leaves the API's FileEditor // nil so the endpoints report 503 rather than creating a Job that cannot run. type Config struct { + ResolveWorld placement.Resolver // Namespace is where the world PVCs live and the Job runs (the minecraft // namespace), co-located with the world it edits. Namespace string @@ -338,6 +340,14 @@ func (e *Editor) run(ctx context.Context, server string, p JobParams) (Result, e return Result{}, err } + if cfg.ResolveWorld != nil { + world, err := cfg.ResolveWorld(ctx, server) + if err != nil { + return Result{}, err + } + p.WorldPVC, p.NodeName = world.Claim, world.Node + } + // Bound the wait here rather than trusting the caller's context: this is an HTTP // handler's goroutine and the Pod it waits on may never become ready (an // unschedulable node, an unpullable image). The Job's own activeDeadlineSeconds @@ -438,6 +448,13 @@ func (e *Editor) start(ctx context.Context, server string, p JobParams) (OpState if err != nil { return OpState{}, err } + if cfg.ResolveWorld != nil { + world, err := cfg.ResolveWorld(ctx, server) + if err != nil { + return OpState{}, err + } + p.WorldPVC, p.NodeName = world.Claim, world.Node + } p.Async = true p.Deadline, p.TTLAfterFinished, p.CPULimit = cfg.AsyncDeadline, cfg.AsyncTTL, cfg.AsyncCPULimit if err := e.Runner.Start(ctx, p); err != nil { diff --git a/internal/fileedit/jobspec.go b/internal/fileedit/jobspec.go index ed1794e..891e244 100644 --- a/internal/fileedit/jobspec.go +++ b/internal/fileedit/jobspec.go @@ -5,6 +5,7 @@ import ( "strconv" "time" + "felis.lolicon.best/internal/placement" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -43,7 +44,8 @@ const ( // operation + Config by the Editor. jobspec is a pure function of them so the // security-critical Job shape is unit-tested without a cluster. type JobParams struct { - Server string + NodeName string + Server string // OpID is the per-invocation identifier that both names the Job and labels its // Pod. See Editor.run for why every invocation gets a fresh one. OpID string @@ -273,6 +275,7 @@ func FilesJob(p JobParams) (*batchv1.Job, error) { ObjectMeta: metav1.ObjectMeta{Labels: filesLabels(p)}, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, + NodeSelector: jobNodeSelector(p.NodeName), ServiceAccountName: p.ServiceAccount, AutomountServiceAccountToken: boolPtr(false), SecurityContext: filesPodSecurityContext(p), @@ -356,3 +359,10 @@ func filesAnnotations(p JobParams) map[string]string { } return map[string]string{AnnotationPath: p.Path} } + +func jobNodeSelector(name string) map[string]string { + if name == "" { + return nil + } + return map[string]string{placement.LabelIdentity: name} +} diff --git a/internal/maintenance/maintenance.go b/internal/maintenance/maintenance.go index 396c9aa..7d01ab1 100644 --- a/internal/maintenance/maintenance.go +++ b/internal/maintenance/maintenance.go @@ -90,6 +90,7 @@ const ( // Kinds of holder. const ( + KindMigration = "migration" KindRestore = "restore" KindBackup = "backup" KindFileWrite = "file-write" @@ -191,6 +192,9 @@ func Holder(server string, annotations map[string]string, jobs []batchv1.Job, no if j.Labels[LabelServer] != server { continue } + if j.Labels["felis.lolicon.best/archive-pending"] == "true" { + return KindBackup, true + } if RestorePending(j) { return KindRestore, true } @@ -202,7 +206,7 @@ func Holder(server string, annotations map[string]string, jobs []batchv1.Job, no } } if v, ok := annotations[Annotation]; ok { - if kind, at, ok := parseLock(v); ok && now.Sub(at) < Grace && at.Sub(now) < Grace { + if kind, at, ok := parseLock(v); ok && (kind == KindMigration || (now.Sub(at) < Grace && at.Sub(now) < Grace)) { return kind, true } } diff --git a/internal/operator/builders.go b/internal/operator/builders.go index 3abe929..7f61b02 100644 --- a/internal/operator/builders.go +++ b/internal/operator/builders.go @@ -8,6 +8,7 @@ import ( "felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/placement" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -217,7 +218,7 @@ func healthHandler(server *v1alpha1.MinecraftServer) corev1.ProbeHandler { // graceful shutdown is terminationGracePeriodSeconds, the time the server gets to // save on SIGTERM; the reconciler flushes the world over RCON before it scales to // zero (saveBeforeStop). -func buildStatefulSet(server *v1alpha1.MinecraftServer, replicas int32, felisImage string) (*appsv1.StatefulSet, error) { +func buildStatefulSet(server *v1alpha1.MinecraftServer, replicas int32, felisImage string, gateProbe ...string) (*appsv1.StatefulSet, error) { storageSize := server.Spec.Storage.Size if storageSize == "" { storageSize = defaultStorageSize @@ -278,7 +279,14 @@ func buildStatefulSet(server *v1alpha1.MinecraftServer, replicas int32, felisIma if server.Labels[v1alpha1.LabelSystemRole] == "" { initContainers = append(initContainers, forwardingInitContainer(felisImage)) } - initContainers = append(initContainers, egressGateInitContainer(felisImage)) + gate := egressGateInitContainer(felisImage) + if server.Spec.NodeName != "" || (len(gateProbe) > 0 && gateProbe[0] != "") { + gate.Command = append(gate.Command[:len(gate.Command)-1], "--positive-probe", "kube-dns.kube-system.svc:53") + if len(gateProbe) > 0 && gateProbe[0] != "" { + gate.Command = append(gate.Command, "--probe", gateProbe[0]) + } + } + initContainers = append(initContainers, gate) } grace := graceSeconds(server) @@ -334,6 +342,13 @@ func buildStatefulSet(server *v1alpha1.MinecraftServer, replicas int32, felisIma }, }, } + if server.Spec.Storage.ClaimName != "" { + sts.Spec.VolumeClaimTemplates = nil + sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, corev1.Volume{Name: dataVolumeName, VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: server.WorldPVC()}}}) + } + if server.Spec.NodeName != "" { + sts.Spec.Template.Spec.NodeSelector = map[string]string{placement.LabelIdentity: server.Spec.NodeName} + } return sts, nil } diff --git a/internal/operator/builders_internal_test.go b/internal/operator/builders_internal_test.go index 5cee7da..322c8f4 100644 --- a/internal/operator/builders_internal_test.go +++ b/internal/operator/builders_internal_test.go @@ -6,9 +6,39 @@ import ( "felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/placement" corev1 "k8s.io/api/core/v1" ) +func TestMigratedWorldPlacementAndFailClosedGate(t *testing.T) { + s := &v1alpha1.MinecraftServer{} + s.Spec.NodeName = "c" + s.Spec.Storage.ClaimName = "world-alice-migrated" + sts, err := buildStatefulSet(s, 1, "felis:test", "felis-api.felis.svc:443") + if err != nil { + t.Fatal(err) + } + if len(sts.Spec.VolumeClaimTemplates) != 0 || sts.Spec.Template.Spec.NodeSelector[placement.LabelIdentity] != "c" { + t.Fatal("migration rebuilt or moved the wrong world") + } + found := false + for _, v := range sts.Spec.Template.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == s.Spec.Storage.ClaimName { + found = true + } + } + if !found { + t.Fatal("active PVC not mounted") + } + for _, c := range sts.Spec.Template.Spec.InitContainers { + if c.Name == "egress-gate" { + if slices.Contains(c.Command, "--fail-open") || !slices.Contains(c.Command, "--positive-probe") { + t.Fatal("distributed gate can fail open") + } + } + } +} + // findEnv returns the env var with the given name, or nil. func findEnv(env []corev1.EnvVar, name string) *corev1.EnvVar { for i := range env { diff --git a/internal/operator/reconciler.go b/internal/operator/reconciler.go index cafa21e..046eb65 100644 --- a/internal/operator/reconciler.go +++ b/internal/operator/reconciler.go @@ -18,6 +18,7 @@ import ( "felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/maintenance" "felis.lolicon.best/internal/metrics" + "felis.lolicon.best/internal/placement" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -116,6 +117,9 @@ func podTemplateStamp(tmpl *corev1.PodTemplateSpec, felisImage string) (string, // Reconciler reconciles a MinecraftServer with its managed children. type Reconciler struct { + Nodes client.Reader + EgressProbe string + ControllerNode string client.Client Scheme *runtime.Scheme // Prober gates readiness on RCON reachability. @@ -312,6 +316,46 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine return ctrl.Result{RequeueAfter: requeueMaintenance}, nil } + if r.Nodes != nil { + node := server.Spec.NodeName + var pod corev1.Pod + if err := r.podReader().Get(ctx, types.NamespacedName{Namespace: server.Namespace, Name: server.Name + "-0"}, &pod); err == nil { + server.Status.NodeName = pod.Spec.NodeName + if node == "" { + node = pod.Spec.NodeName + } + } else if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + if node == "" { + node = r.ControllerNode + } + if node != "" { + var n corev1.Node + err := r.Nodes.Get(ctx, types.NamespacedName{Name: node}, &n) + if err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err != nil || !placement.Admitted(&n, r.ControllerNode) { + var svc corev1.Service + if err := r.Get(ctx, types.NamespacedName{Namespace: server.Namespace, Name: server.Name}, &svc); err == nil { + if svc.Spec.Selector == nil { + svc.Spec.Selector = map[string]string{} + } + svc.Spec.Selector["felis.lolicon.best/node-online"] = "true" + if err := r.Update(ctx, &svc); err != nil { + return ctrl.Result{}, err + } + } else if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + server.Status.Ready = false + server.Status.Endpoint = v1alpha1.EndpointStatus{Mode: v1alpha1.EndpointFallback} + r.setCondition(server, v1alpha1.ConditionReady, metav1.ConditionFalse, "NodeUnavailable", "execution node is offline; automatic relocation is disabled") + return ctrl.Result{RequeueAfter: 10 * time.Second}, r.patchStatus(ctx, server) + } + } + } endpointAddress, err := r.ensureServices(ctx, server) if err != nil { return ctrl.Result{}, err @@ -330,7 +374,11 @@ func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.Mine return ctrl.Result{RequeueAfter: requeueSecret}, nil } - desired, err := buildStatefulSet(server, 1, r.FelisImage) + placed := server.DeepCopy() + if placed.Spec.NodeName == "" { + placed.Spec.NodeName = r.ControllerNode + } + desired, err := buildStatefulSet(placed, 1, r.FelisImage, r.EgressProbe) if err != nil { // A malformed spec (e.g. bad storage quantity) is terminal until edited. r.markFailed(server, "InvalidSpec", err.Error()) diff --git a/internal/panel/panel.go b/internal/panel/panel.go index 820b223..aca63a6 100644 --- a/internal/panel/panel.go +++ b/internal/panel/panel.go @@ -20,6 +20,7 @@ import ( var static embed.FS type runtimeConfig struct { + Distributed bool `json:"distributed,omitempty"` APIBase string `json:"apiBase"` RootDomain string `json:"rootDomain"` PanelHostname string `json:"panelHostname,omitempty"` @@ -112,13 +113,14 @@ func parseBuildVersion(raw string) buildInfo { // right surface (player console vs SysAdmin console) without a rebuild. gamePort // is the public Minecraft port ([velocity] game_port), which the SPA appends to // the server addresses players copy; 0 or 25565 leaves them bare. -func Handler(api http.Handler, rootDomain, panelHost, adminHost string, gamePort int, version string) http.Handler { +func Handler(api http.Handler, rootDomain, panelHost, adminHost string, gamePort int, version string, distributed ...bool) http.Handler { files, err := fs.Sub(static, "static") if err != nil { panic(err) } return &handler{ api: api, + distributed: len(distributed) > 0 && distributed[0], rootDomain: rootDomain, panelHostname: panelHost, adminHostname: adminHost, @@ -143,6 +145,7 @@ func publicGamePort(p int) int { } type handler struct { + distributed bool api http.Handler rootDomain string panelHostname string @@ -223,6 +226,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-store") _ = json.NewEncoder(w).Encode(runtimeConfig{ APIBase: "/api/v1", + Distributed: h.distributed, RootDomain: h.rootDomain, PanelHostname: h.panelHostname, AdminHostname: h.adminHostname, diff --git a/internal/placement/placement.go b/internal/placement/placement.go new file mode 100644 index 0000000..7aef42c --- /dev/null +++ b/internal/placement/placement.go @@ -0,0 +1,84 @@ +// Package placement resolves worlds and checks the protected node admission labels. +package placement + +import ( + "context" + "fmt" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + LabelRole = "felis.node-restriction.kubernetes.io/role" + LabelIdentity = "felis.node-restriction.kubernetes.io/identity" + LabelApproved = "felis.node-restriction.kubernetes.io/approved" + RoleController = "controller" + RoleWorker = "worker" +) + +type World struct{ Claim, Node string } +type Resolver func(context.Context, string) (World, error) + +func Online(n *corev1.Node) bool { + for _, c := range n.Status.Conditions { + if c.Type == corev1.NodeReady { + return c.Status == corev1.ConditionTrue && n.DeletionTimestamp == nil + } + } + return false +} + +// Admitted accepts only A's protected controller identity or an approved worker. +func Admitted(n *corev1.Node, controller string) bool { + if !Online(n) || n.Labels[LabelIdentity] != n.Name { + return false + } + if n.Name == controller { + return n.Labels[LabelRole] == RoleController + } + return n.Labels[LabelRole] == RoleWorker && n.Labels[LabelApproved] == "true" +} + +func Worker(ctx context.Context, r client.Reader, name string) error { + var n corev1.Node + if err := r.Get(ctx, types.NamespacedName{Name: name}, &n); err != nil { + return err + } + if !Admitted(&n, "") || n.Spec.Unschedulable { + return fmt.Errorf("node %q is not an online, approved worker", name) + } + return nil +} + +func Resolve(r client.Reader, namespace string, controller ...string) Resolver { + return func(ctx context.Context, name string) (World, error) { + var s v1alpha1.MinecraftServer + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &s); err != nil { + return World{}, err + } + node := s.Spec.NodeName + if node == "" { + node = s.Status.NodeName + } + if node == "" && len(controller) > 0 { + node = controller[0] + } + if node != "" { + var n corev1.Node + if err := r.Get(ctx, types.NamespacedName{Name: node}, &n); err != nil { + return World{}, err + } + a := "" + if len(controller) > 0 { + a = controller[0] + } + if !Admitted(&n, a) { + return World{}, fmt.Errorf("node %q is offline", node) + } + } + return World{Claim: s.WorldPVC(), Node: node}, nil + } +} diff --git a/internal/platform/bundle.go b/internal/platform/bundle.go index f8c891a..284e959 100644 --- a/internal/platform/bundle.go +++ b/internal/platform/bundle.go @@ -72,6 +72,11 @@ func Objects(p Params) []Object { objs = append(objs, rb) } + if p.Distributed { + objs = append(objs, DistributedRBAC(p)...) + objs = append(objs, ArchiveNetworkPolicies(p)...) + } + // Weak Job SAs. They come from the build/restore packages (single source of // truth for AutomountServiceAccountToken=false), which set ObjectMeta but not // TypeMeta — stamp it so the YAML header is present. The restore Job runs in diff --git a/internal/platform/distributed.go b/internal/platform/distributed.go new file mode 100644 index 0000000..9c6ed90 --- /dev/null +++ b/internal/platform/distributed.go @@ -0,0 +1,82 @@ +package platform + +import ( + "fmt" + + "felis.lolicon.best/internal/archivetransfer" + "felis.lolicon.best/internal/distributed" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ArchiveName = "felis-archive" +const ArchiveSecret = "felis-archive-key" +const ArchivePort int32 = 8090 + +func archiveKeyEnv() corev1.EnvVar { + return corev1.EnvVar{Name: archivetransfer.KeyEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: ArchiveSecret}, Key: "key"}}} +} + +func distributedEnv(p Params) []corev1.EnvVar { + return []corev1.EnvVar{{Name: "FELIS_DISTRIBUTED", Value: "true"}, {Name: "FELIS_CONTROLLER_NODE", Value: p.ControllerNode}, {Name: "FELIS_ARCHIVE_URL", Value: fmt.Sprintf("http://%s.%s.svc:%d", ArchiveName, p.MinecraftNamespace, ArchivePort)}, {Name: "FELIS_EGRESS_PROBE", Value: p.EgressProbe}, archiveKeyEnv()} +} + +func ArchiveDeployment(p Params) *appsv1.Deployment { + labels := map[string]string{"app.kubernetes.io/name": ArchiveName} + container := corev1.Container{Name: ArchiveName, Image: p.FelisImage, Command: []string{felisBinaryPath, "archive-serve"}, Args: []string{"--root", p.ArchiveLocalPath}, Env: []corev1.EnvVar{archiveKeyEnv()}, Ports: []corev1.ContainerPort{{Name: "archive", ContainerPort: ArchivePort}}, Resources: controlPlaneResources(), SecurityContext: &corev1.SecurityContext{RunAsUser: int64Ptr(0), RunAsNonRoot: boolPtr(false), AllowPrivilegeEscalation: boolPtr(false), Privileged: boolPtr(false), ReadOnlyRootFilesystem: boolPtr(true), Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, VolumeMounts: []corev1.VolumeMount{{Name: "archives", MountPath: p.ArchiveLocalPath}}} + container.ReadinessProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromString("archive")}}} + container.LivenessProbe = container.ReadinessProbe.DeepCopy() + return &appsv1.Deployment{TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, ObjectMeta: metav1.ObjectMeta{Name: ArchiveName, Namespace: p.MinecraftNamespace, Labels: labels}, Spec: appsv1.DeploymentSpec{Replicas: int32Ptr(1), Strategy: appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType}, Selector: &metav1.LabelSelector{MatchLabels: labels}, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{NodeSelector: controllerSelector(p), AutomountServiceAccountToken: boolPtr(false), SecurityContext: &corev1.PodSecurityContext{SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{container}, Volumes: []corev1.Volume{{Name: "archives", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: p.BackupPVC}}}}}}}} +} +func ArchiveService(p Params) *corev1.Service { + return &corev1.Service{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, ObjectMeta: metav1.ObjectMeta{Name: ArchiveName, Namespace: p.MinecraftNamespace}, Spec: corev1.ServiceSpec{Selector: map[string]string{"app.kubernetes.io/name": ArchiveName}, Ports: []corev1.ServicePort{{Name: "archive", Port: ArchivePort, TargetPort: intstr.FromInt32(ArchivePort)}}}} +} + +// Cluster grants are read-only and bound only to A's API/operator/reaper identities. +func DistributedRBAC(p Params) []Object { + var out []Object + for _, entry := range []struct { + name, ns string + rules []rbacv1.PolicyRule + }{ + {SAAPI, p.ControlNamespace, []rbacv1.PolicyRule{rule([]string{groupCore}, []string{"nodes"}, []string{"get", "list"}), rule([]string{groupCore}, []string{"persistentvolumes"}, []string{"get"})}}, + {SAOperator, p.ControlNamespace, []rbacv1.PolicyRule{rule([]string{groupCore}, []string{"nodes"}, []string{"get"})}}, + {SAReaper, p.MinecraftNamespace, []rbacv1.PolicyRule{rule([]string{groupCore}, []string{"nodes"}, []string{"get"})}}, + } { + name := entry.name + "-" + p.MinecraftNamespace + "-nodes" + out = append(out, &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole"}, ObjectMeta: metav1.ObjectMeta{Name: name}, Rules: entry.rules}, &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRoleBinding"}, ObjectMeta: metav1.ObjectMeta{Name: name}, RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: name}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: entry.name, Namespace: entry.ns}}}) + } + return out +} + +func ArchiveNetworkPolicies(p Params) []Object { + tcp, udp := corev1.ProtocolTCP, corev1.ProtocolUDP + archive := metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": ArchiveName}} + job := metav1.LabelSelector{MatchLabels: map[string]string{distributed.LabelTransfer: "true"}} + control := networkingv1.NetworkPolicyPeer{NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": p.ControlNamespace}}, PodSelector: &metav1.LabelSelector{MatchLabels: controlPlanePodLabels(ComponentAPI)}} + reaper := networkingv1.NetworkPolicyPeer{PodSelector: &metav1.LabelSelector{MatchLabels: controlPlanePodLabels(ComponentReaper)}} + peers := []networkingv1.NetworkPolicyPeer{control, reaper, {PodSelector: &job}} + ingress := &networkingv1.NetworkPolicy{TypeMeta: metav1.TypeMeta{APIVersion: "networking.k8s.io/v1", Kind: "NetworkPolicy"}, ObjectMeta: metav1.ObjectMeta{Name: "felis-archive", Namespace: p.MinecraftNamespace}, Spec: networkingv1.NetworkPolicySpec{PodSelector: archive, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress}, Ingress: []networkingv1.NetworkPolicyIngressRule{{From: peers, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: portPtr(ArchivePort)}}}}}} + egress := &networkingv1.NetworkPolicy{TypeMeta: metav1.TypeMeta{APIVersion: "networking.k8s.io/v1", Kind: "NetworkPolicy"}, ObjectMeta: metav1.ObjectMeta{Name: "felis-archive-jobs", Namespace: p.MinecraftNamespace}, Spec: networkingv1.NetworkPolicySpec{PodSelector: job, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, Egress: []networkingv1.NetworkPolicyEgressRule{ + {To: []networkingv1.NetworkPolicyPeer{{PodSelector: &archive}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: portPtr(ArchivePort)}}}, + {To: []networkingv1.NetworkPolicyPeer{{NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"}}, PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"k8s-app": "kube-dns"}}}}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &udp, Port: portPtr(53)}, {Protocol: &tcp, Port: portPtr(53)}}}, + }}} + // Files/export maintenance Pods may send results only to A's existing upload receiver. + // Transfer Pods are excluded so grants remain limited to the archive service. + maintenance := &networkingv1.NetworkPolicy{TypeMeta: metav1.TypeMeta{APIVersion: "networking.k8s.io/v1", Kind: "NetworkPolicy"}, ObjectMeta: metav1.ObjectMeta{Name: "felis-maintenance-egress", Namespace: p.MinecraftNamespace}, Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "app.kubernetes.io/managed-by", Operator: metav1.LabelSelectorOpIn, Values: []string{"felis-files", "felis-export", "felis-restore", "felis-backup"}}, + {Key: distributed.LabelTransfer, Operator: metav1.LabelSelectorOpDoesNotExist}, + }}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, + Egress: []networkingv1.NetworkPolicyEgressRule{ + {To: []networkingv1.NetworkPolicyPeer{control}, Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: portPtr(8081)}}}, + egress.Spec.Egress[1], + }, + }} + return []Object{ingress, egress, maintenance} +} +func portPtr(p int32) *intstr.IntOrString { v := intstr.FromInt32(p); return &v } diff --git a/internal/platform/distributed_test.go b/internal/platform/distributed_test.go new file mode 100644 index 0000000..e96351e --- /dev/null +++ b/internal/platform/distributed_test.go @@ -0,0 +1,61 @@ +package platform + +import ( + "testing" + + "felis.lolicon.best/internal/archivetransfer" + "felis.lolicon.best/internal/placement" + appsv1 "k8s.io/api/apps/v1" + rbacv1 "k8s.io/api/rbac/v1" +) + +func TestDistributedControllerPinningAndArchivePowers(t *testing.T) { + p := testParams() + p.Distributed = true + p.ControllerNode = "a" + p.EgressProbe = "felis-api.felis.svc:443" + p.BackupPVC = "felis-backups" + p.ArchiveLocalPath = "/backups" + p.VelocityCIDRs = []string{"192.0.2.1/32"} + if err := p.Validate(); err != nil { + t.Fatal(err) + } + found := false + for _, obj := range Objects(p) { + if d, ok := obj.(*appsv1.Deployment); ok { + if d.Spec.Template.Spec.NodeSelector[placement.LabelIdentity] != "a" { + t.Fatalf("controller workload %s may run on worker", d.Name) + } + if d.Name == ArchiveName { + found = true + s := d.Spec.Template.Spec + if s.AutomountServiceAccountToken == nil || *s.AutomountServiceAccountToken || s.ServiceAccountName != "" || len(s.Volumes) != 1 || s.Volumes[0].PersistentVolumeClaim.ClaimName != p.BackupPVC { + t.Fatal("archive has extra powers") + } + for _, c := range s.Containers { + for _, e := range c.Env { + if e.Name != archivetransfer.KeyEnv { + t.Fatal("archive received controller config") + } + } + } + } + } + if r, ok := obj.(*rbacv1.ClusterRole); ok { + for _, rule := range r.Rules { + for _, verb := range rule.Verbs { + if verb != "get" && verb != "list" { + t.Fatal("distributed cluster grant writes", r.Name, verb) + } + } + } + } + } + if !found { + t.Fatal("archive service missing") + } + p.VelocityCIDRs = []string{"10.0.0.0/8"} + if p.Validate() == nil { + t.Fatal("broad Velocity source accepted") + } +} diff --git a/internal/platform/identities.go b/internal/platform/identities.go index aed75ec..d85e8e6 100644 --- a/internal/platform/identities.go +++ b/internal/platform/identities.go @@ -2,6 +2,7 @@ package platform import ( "fmt" + "net" "k8s.io/apimachinery/pkg/api/resource" ) @@ -89,6 +90,10 @@ const ( // Params parameterises the install bundle. Namespaces and the registry location // have safe defaults; VelocityCIDRs has none — see the field comment. type Params struct { + Distributed bool + ControllerNode string + EgressProbe string + RegistryNodeCIDRs []string // ControlNamespace is where felis-api/operator run; their SAs live here and the // RoleBindings' subjects reference them here, even though the Roles they bind to // live in the minecraft (and build) namespaces. The reaper alone runs — CronJob @@ -212,6 +217,25 @@ type Params struct { // Validate reports a Params the renderer cannot turn into objects. func (p Params) Validate() error { + if p.Distributed && (p.ControllerNode == "" || p.EgressProbe == "" || p.BackupPVC == "" || p.ArchiveLocalPath == "") { + return fmt.Errorf("distributed mode requires controller node, egress probe, backup PVC and archive path") + } + if p.Distributed && len(p.ServerEgressAllowCIDRs) > 0 { + return fmt.Errorf("distributed mode does not permit private game egress exceptions") + } + + if p.Distributed { + for _, cidr := range append(append([]string{}, p.VelocityCIDRs...), p.RegistryNodeCIDRs...) { + _, network, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("invalid node source %q", cidr) + } + ones, bits := network.Mask.Size() + if ones != bits { + return fmt.Errorf("distributed node sources must be exact /32 or /128 addresses: %q", cidr) + } + } + } for _, q := range []struct{ name, v string }{ {"registry storage", p.RegistryStorage}, {"uploads storage", p.UploadsStorage}, diff --git a/internal/platform/netpol.go b/internal/platform/netpol.go index 48b7846..8e4d29a 100644 --- a/internal/platform/netpol.go +++ b/internal/platform/netpol.go @@ -266,6 +266,9 @@ func RegistryIngressPolicy(p Params) *networkingv1.NetworkPolicy { Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: &port}}, }}, ) + for _, cidr := range p.RegistryNodeCIDRs { + np.Spec.Ingress[0].From = append(np.Spec.Ingress[0].From, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}}) + } return np } diff --git a/internal/platform/postgres.go b/internal/platform/postgres.go index 28ea727..f06c426 100644 --- a/internal/platform/postgres.go +++ b/internal/platform/postgres.go @@ -206,6 +206,7 @@ func postgresDeployment(p Params) *appsv1.Deployment { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ + NodeSelector: controllerSelector(p), AutomountServiceAccountToken: boolPtr(false), EnableServiceLinks: boolPtr(false), PriorityClassName: controlPlanePriorityName, diff --git a/internal/platform/rbac.go b/internal/platform/rbac.go index 741aecd..ab0ca82 100644 --- a/internal/platform/rbac.go +++ b/internal/platform/rbac.go @@ -19,7 +19,7 @@ var groupFelis = v1alpha1.GroupName // "felis.lolicon.best" // RBAC is the control-plane authorization bundle: one SA per identity and the // namespaced Roles + RoleBindings that grant each exactly the verbs its code path -// exercises. There is deliberately no ClusterRole or ClusterRoleBinding anywhere. +// exercises. DistributedRBAC adds read-only node/PV grants when distributed mode is enabled. type RBAC struct { ServiceAccounts []*corev1.ServiceAccount Roles []*rbacv1.Role @@ -100,7 +100,7 @@ func ControlPlaneRBAC(p Params) RBAC { // than its logs). func APIMinecraftRole(p Params) *rbacv1.Role { p = p.withDefaults() - return role(p.MinecraftNamespace, "felis-api", ComponentAPI, []rbacv1.PolicyRule{ + rules := []rbacv1.PolicyRule{ rule([]string{groupFelis}, []string{"minecraftservers"}, []string{"get", "list", "watch", "create", "patch"}), rule([]string{groupCore}, []string{"secrets"}, []string{"get"}), // get-only: WorldVolumeExists does a single direct Get of the world PVC; @@ -116,7 +116,11 @@ func APIMinecraftRole(p Params) *rbacv1.Role { // the verbs stay tight — list on pods, get on pods/log, and nothing else. rule([]string{groupCore}, []string{"pods"}, []string{"list"}), rule([]string{groupCore}, []string{"pods/log"}, []string{"get"}), - }) + } + if p.Distributed { + rules = append(rules, rule([]string{groupCore}, []string{"persistentvolumeclaims"}, []string{"create", "list"}), rule([]string{groupApps}, []string{"statefulsets"}, []string{"get", "delete"})) + } + return role(p.MinecraftNamespace, "felis-api", ComponentAPI, rules) } // APIBuildRole grants felis-api the build-Job lifecycle in the build namespace @@ -218,12 +222,16 @@ func OperatorRole(p Params) *rbacv1.Role { // request and never deletes a CR itself. func ReaperRole(p Params) *rbacv1.Role { p = p.withDefaults() - return role(p.MinecraftNamespace, "felis-reaper", ComponentReaper, []rbacv1.PolicyRule{ + rules := []rbacv1.PolicyRule{ rule([]string{groupFelis}, []string{"minecraftservers"}, []string{"get", "patch", "delete"}), rule([]string{groupCore}, []string{"persistentvolumeclaims"}, []string{"get", "delete"}), rule([]string{groupCore}, []string{"pods"}, []string{"list"}), rule([]string{groupBatch}, []string{"jobs"}, []string{"list"}), - }) + } + if p.Distributed { + rules = append(rules, rule([]string{groupBatch}, []string{"jobs"}, []string{"create", "get"}), rule([]string{groupCore}, []string{"persistentvolumeclaims"}, []string{"list"})) + } + return role(p.MinecraftNamespace, "felis-reaper", ComponentReaper, rules) } // controlPlaneServiceAccount renders a control-plane SA. Unlike the weak diff --git a/internal/platform/workloads.go b/internal/platform/workloads.go index 2c29cfb..00cef09 100644 --- a/internal/platform/workloads.go +++ b/internal/platform/workloads.go @@ -7,6 +7,7 @@ import ( "time" "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/placement" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -273,11 +274,16 @@ func Workloads(p Params) []Object { objs = append(objs, backupPVC(p)) } switch { + case p.Distributed && retentionEnabled(p): + objs = append(objs, reaperCronJob(p)) case reaperEnabled(p): objs = append(objs, worldsRootPV(p), worldsRootPVC(p), reaperCronJob(p)) case retentionEnabled(p): objs = append(objs, reaperCronJob(p)) } + if p.Distributed { + objs = append(objs, ArchiveDeployment(p), ArchiveService(p)) + } return objs } @@ -315,7 +321,7 @@ const controlPlanePriorityName = "system-cluster-critical" // together so a partial configuration fails loudly rather than silently dropping // retention here. func reaperEnabled(p Params) bool { - return p.WorldsHostPath != "" && retentionEnabled(p) + return (p.WorldsHostPath != "" || p.Distributed) && retentionEnabled(p) } // retentionEnabled reports whether the archive store can be looked after: the @@ -716,6 +722,11 @@ func reaperCronJob(p Params) *batchv1.CronJob { }) } + if p.Distributed { + container.Args = []string{"--config", configFilePath} + container.Env = append(container.Env, distributedEnv(p)...) + container.Env = append(container.Env, corev1.EnvVar{Name: "FELIS_IMAGE", Value: p.FelisImage}) + } return &batchv1.CronJob{ TypeMeta: metav1.TypeMeta{APIVersion: "batch/v1", Kind: "CronJob"}, // The CronJob lives in the MINECRAFT namespace: a Pod can only mount PVCs @@ -844,10 +855,12 @@ func reaperPodSpec(p Params, container corev1.Container, volumes []corev1.Volume Containers: []corev1.Container{container}, Volumes: volumes, } - if p.ReaperNode != "" { + if p.ControllerNode != "" { + spec.NodeSelector = controllerSelector(p) + } else if p.ReaperNode != "" { spec.NodeSelector = map[string]string{"kubernetes.io/hostname": p.ReaperNode} } - if p.WorldsHostPath != "" { + if p.WorldsHostPath != "" || p.Distributed { spec.ServiceAccountName = SAReaper } else { spec.ServiceAccountName = "default" @@ -868,6 +881,13 @@ func reaperPodSpec(p Params, container corev1.Container, volumes []corev1.Volume // Recreate guarantees the old pod is gone before the new one starts. func controlPlaneDeployment(p Params, sa string, container corev1.Container, volumes []corev1.Volume) *appsv1.Deployment { labels := controlPlanePodLabels(container.Name) + if p.Distributed { + if sa == SAOperator { + container.Env = append(container.Env, corev1.EnvVar{Name: "FELIS_DISTRIBUTED", Value: "true"}, corev1.EnvVar{Name: "FELIS_CONTROLLER_NODE", Value: p.ControllerNode}, corev1.EnvVar{Name: "FELIS_EGRESS_PROBE", Value: p.EgressProbe}) + } else { + container.Env = append(container.Env, distributedEnv(p)...) + } + } return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, ObjectMeta: metav1.ObjectMeta{Name: sa, Namespace: p.ControlNamespace, Labels: labels}, @@ -878,6 +898,7 @@ func controlPlaneDeployment(p Params, sa string, container corev1.Container, vol Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ + NodeSelector: controllerSelector(p), ServiceAccountName: sa, PriorityClassName: controlPlanePriorityName, SecurityContext: hardenedPodSecurityContext(), @@ -1003,6 +1024,7 @@ func registryDeployment(p Params) *appsv1.Deployment { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ + NodeSelector: controllerSelector(p), AutomountServiceAccountToken: boolPtr(false), PriorityClassName: controlPlanePriorityName, SecurityContext: hardenedPodSecurityContext(), @@ -1373,3 +1395,10 @@ func hardenedContainerSecurityContext() *corev1.SecurityContext { func boolPtr(b bool) *bool { return &b } func int32Ptr(i int32) *int32 { return &i } func int64Ptr(i int64) *int64 { return &i } + +func controllerSelector(p Params) map[string]string { + if p.ControllerNode == "" { + return nil + } + return map[string]string{placement.LabelIdentity: p.ControllerNode, placement.LabelRole: placement.RoleController} +} diff --git a/internal/reaper/k8scluster.go b/internal/reaper/k8scluster.go index 9397e95..73e59b7 100644 --- a/internal/reaper/k8scluster.go +++ b/internal/reaper/k8scluster.go @@ -41,7 +41,8 @@ type K8sCluster struct { now func() time.Time // beat is how often a held lock is rewritten; maintenance.Grace/4 unless a // test shortens it. - beat time.Duration + beat time.Duration + distributed bool } // NewK8sCluster builds a Cluster over c, scoped to namespace. @@ -49,6 +50,18 @@ func NewK8sCluster(c client.Client, namespace string) *K8sCluster { return &K8sCluster{c: c, namespace: namespace} } +func (k *K8sCluster) WithDistributed(enabled bool) *K8sCluster { k.distributed = enabled; return k } +func (k *K8sCluster) RetainedWorlds(ctx context.Context, name string) (bool, error) { + if !k.distributed { + return false, nil + } + var pvcs corev1.PersistentVolumeClaimList + if err := k.c.List(ctx, &pvcs, client.InNamespace(k.namespace), client.MatchingLabels{maintenance.LabelServer: name}); err != nil { + return false, err + } + return len(pvcs.Items) > 0, nil +} + func (k *K8sCluster) clock() time.Time { if k.now != nil { return k.now() @@ -61,7 +74,7 @@ func (k *K8sCluster) Inspect(ctx context.Context, name string) (ServerCRD, error if err := k.get(ctx, name, &ms); err != nil { return ServerCRD{}, err } - return ServerCRD{Exempt: ms.Spec.ReaperExempt, PVC: WorldPVCName(name), UID: string(ms.UID)}, nil + return ServerCRD{Exempt: ms.Spec.ReaperExempt, PVC: ms.WorldPVC(), UID: string(ms.UID)}, nil } // HoldWorld implements Cluster. The lock is the same Annotation felis-api diff --git a/internal/reaper/pgstore.go b/internal/reaper/pgstore.go index 1cb8707..cc1fe56 100644 --- a/internal/reaper/pgstore.go +++ b/internal/reaper/pgstore.go @@ -76,7 +76,7 @@ func (s *PGStore) FreshBackup(ctx context.Context, server string, since time.Tim func (s *PGStore) InsertBackup(ctx context.Context, rec BackupRecord) error { const q = `INSERT INTO world_backups (id, server_name, former_owner, backup_ref, size_bytes, reason, status, created_at, expires_at, sha256, skipped_entries) - VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, 'present', now(), $7, NULLIF($8, ''), $9)` + VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, 'present', now(), $7, NULLIF($8, ''), $9) ON CONFLICT (id) DO NOTHING` _, err := s.db.ExecContext(ctx, q, rec.ID, rec.ServerName, rec.FormerOwner, rec.BackupRef, rec.SizeBytes, rec.Reason, rec.ExpiresAt, rec.SHA256, rec.SkippedEntries) diff --git a/internal/reaper/reaper.go b/internal/reaper/reaper.go index 7d9f7c5..5c1581b 100644 --- a/internal/reaper/reaper.go +++ b/internal/reaper/reaper.go @@ -734,6 +734,17 @@ func (r *Reaper) retire(ctx context.Context, now time.Time, c Candidate, crd Ser // MinecraftServer the reaper cannot hold it still to archive it: an operator // takes it from there, and the run reports the server until then. func (r *Reaper) forgetServer(ctx context.Context, now time.Time, c Candidate, sum *Summary) error { + if cluster, ok := r.Cluster.(interface { + RetainedWorlds(context.Context, string) (bool, error) + }); ok { + retained, err := cluster.RetainedWorlds(ctx, c.Name) + if err != nil { + return err + } + if retained { + return fmt.Errorf("retained world volumes require explicit administrator cleanup") + } + } pvc := WorldPVCName(c.Name) exists, err := r.Cluster.WorldExists(ctx, pvc) if err != nil { diff --git a/internal/restore/jobspec.go b/internal/restore/jobspec.go index 5b31646..d960a1e 100644 --- a/internal/restore/jobspec.go +++ b/internal/restore/jobspec.go @@ -1,7 +1,9 @@ package restore import ( + "felis.lolicon.best/internal/archivetransfer" "fmt" + "strconv" "time" batchv1 "k8s.io/api/batch/v1" @@ -34,22 +36,24 @@ const ( // archive ref + Config by the Restorer. jobspec is a pure function of them so // the security-critical Job shape is unit-tested without a cluster. type JobParams struct { - Server string - WorldPVC string - BackupPVC string - BackupRef string - ArchiveStore string - Namespace string - ServiceAccount string - Image string - BackupRoot string - WorldsRoot string - Deadline time.Duration - CPULimit string - MemLimit string - RunAsUser int64 - RunAsGroup int64 - FSGroup int64 + SourceURL, Token, SHA256 string + MaxBytes int64 + Server string + WorldPVC string + BackupPVC string + BackupRef string + ArchiveStore string + Namespace string + ServiceAccount string + Image string + BackupRoot string + WorldsRoot string + Deadline time.Duration + CPULimit string + MemLimit string + RunAsUser int64 + RunAsGroup int64 + FSGroup int64 TTLAfterFinished time.Duration } @@ -92,7 +96,7 @@ func RestoreJob(p JobParams) (*batchv1.Job, error) { if p.Image == "" { return nil, fmt.Errorf("restore: image is empty") } - if p.WorldPVC == "" || p.BackupPVC == "" { + if p.WorldPVC == "" || (p.BackupPVC == "" && p.SourceURL == "") { return nil, fmt.Errorf("restore: world and backup PVC names are required") } limits, err := resourceLimits(p.CPULimit, p.MemLimit) @@ -192,6 +196,16 @@ func RestoreJob(p JobParams) (*batchv1.Job, error) { }, }, } + if p.SourceURL != "" { + if p.Token == "" || p.SHA256 == "" || p.MaxBytes <= 0 { + return nil, fmt.Errorf("restore: remote archive credentials and bounds required") + } + c := &job.Spec.Template.Spec.Containers[0] + c.Args = append(c.Args, "--source-url", p.SourceURL, "--sha256", p.SHA256, "--max-bytes", strconv.FormatInt(p.MaxBytes, 10)) + c.Env = []corev1.EnvVar{{Name: archivetransfer.TokenEnv, Value: p.Token}} + c.VolumeMounts[1] = corev1.VolumeMount{Name: "tmp", MountPath: "/tmp"} + job.Spec.Template.Spec.Volumes[1] = corev1.Volume{Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}} + } return job, nil } diff --git a/internal/restore/restore.go b/internal/restore/restore.go index 860c89b..ad30dbe 100644 --- a/internal/restore/restore.go +++ b/internal/restore/restore.go @@ -32,6 +32,7 @@ import ( "time" "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/placement" ) // ErrAlreadyExists is returned by a Jobs implementation when a restore Job for a @@ -75,6 +76,7 @@ type Jobs interface { // either is empty the caller leaves the API's Restorer nil so the endpoint // reports 503 rather than enqueuing a Job that cannot run. type Config struct { + ResolveWorld placement.Resolver // Namespace is where the world PVCs live and the restore Job runs (the // minecraft namespace). The Job is intentionally co-located with the world it // restores; it never runs in the felis control-plane namespace. @@ -199,7 +201,15 @@ type Restorer struct { // keeps the handler's 202 honest in both directions — not a 500 for a genuine // duplicate, and not a false "restoring" for a retry after a failure. func (r *Restorer) Restore(ctx context.Context, serverName, backupRef string) error { - if err := r.Jobs.CreateRestoreJob(ctx, r.jobParams(serverName, backupRef)); err != nil { + p := r.jobParams(serverName, backupRef) + if r.Config.ResolveWorld != nil { + w, err := r.Config.ResolveWorld(ctx, serverName) + if err != nil { + return err + } + p.WorldPVC = w.Claim + } + if err := r.Jobs.CreateRestoreJob(ctx, p); err != nil { if errors.Is(err, ErrAlreadyExists) { return nil // already enqueued — idempotent } diff --git a/internal/worldexport/jobspec.go b/internal/worldexport/jobspec.go index f4fef3b..f1861fe 100644 --- a/internal/worldexport/jobspec.go +++ b/internal/worldexport/jobspec.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "felis.lolicon.best/internal/placement" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -57,7 +58,8 @@ const ( // JobParams are the rendered inputs to an export Job. ExportJob is a pure // function of them, so the Job shape is unit-tested without a cluster. type JobParams struct { - Server string + NodeName string + Server string // ID names this export: it is the tail of the Job name and of the internal // upload path, so two exports of one server never collide. ID string @@ -227,6 +229,7 @@ func ExportJob(p JobParams) (*batchv1.Job, error) { ObjectMeta: metav1.ObjectMeta{Labels: exportLabels(p)}, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, + NodeSelector: jobNodeSelector(p.NodeName), ServiceAccountName: p.ServiceAccount, AutomountServiceAccountToken: boolPtr(false), SecurityContext: sc, @@ -269,3 +272,10 @@ func resourceLimits(cpu, mem string) (corev1.ResourceList, error) { func boolPtr(b bool) *bool { return &b } func int32Ptr(i int32) *int32 { return &i } func int64Ptr(i int64) *int64 { return &i } + +func jobNodeSelector(name string) map[string]string { + if name == "" { + return nil + } + return map[string]string{placement.LabelIdentity: name} +} diff --git a/internal/worldexport/worldexport.go b/internal/worldexport/worldexport.go index d207e2d..646460a 100644 --- a/internal/worldexport/worldexport.go +++ b/internal/worldexport/worldexport.go @@ -20,6 +20,7 @@ import ( "time" "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/placement" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -47,6 +48,7 @@ type Request struct { // cmd/felis leaves the API's Exporter nil when either is missing, and the // routes answer 503. type Config struct { + ResolveWorld placement.Resolver Namespace string ServiceAccount string Image string @@ -112,9 +114,19 @@ func New(cs kubernetes.Interface, cfg Config) *Exporter { // Start creates the export Job for r and returns its name. func (e *Exporter) Start(ctx context.Context, r Request) (string, error) { c := e.cfg + claim := naming.WorldPVCName(r.Server) + node := "" + if c.ResolveWorld != nil && r.Mode != ModeBackup { + world, err := c.ResolveWorld(ctx, r.Server) + if err != nil { + return "", err + } + claim, node = world.Claim, world.Node + } job, err := ExportJob(JobParams{ Server: r.Server, ID: r.ID, Mode: r.Mode, - WorldPVC: naming.WorldPVCName(r.Server), BackupPVC: c.BackupPVC, BackupRef: r.BackupRef, + NodeName: node, + WorldPVC: claim, BackupPVC: c.BackupPVC, BackupRef: r.BackupRef, BackupSHA256: r.BackupSHA256, Path: r.Path, Dir: r.Dir, TargetURL: r.TargetURL, Token: r.Token, Namespace: c.Namespace, ServiceAccount: c.ServiceAccount, Image: c.Image, diff --git a/panel/src/components/CreateServerDialog.test.tsx b/panel/src/components/CreateServerDialog.test.tsx index 3a15f90..cd61eb0 100644 --- a/panel/src/components/CreateServerDialog.test.tsx +++ b/panel/src/components/CreateServerDialog.test.tsx @@ -6,7 +6,7 @@ import { CreateServerDialog } from "./CreateServerDialog"; import { humanizeError } from "@/lib/api"; import type { CreateServerRequest, WhitelistImage } from "@/lib/types"; -const calls = vi.hoisted(() => ({ listImages: vi.fn(), createServer: vi.fn() })); +const calls = vi.hoisted(() => ({ listImages: vi.fn(), createServer: vi.fn(), nodes: vi.fn() })); vi.mock("@/lib/api", async (importActual) => { const actual = await importActual(); return { ...actual, api: { ...actual.api, ...calls } }; @@ -258,3 +258,25 @@ describe("CreateServerDialog on reopening", () => { expect(screen.queryByRole("alert")).toBeNull(); }); }); + + +describe("distributed placement", () => { + it("requires an online approved worker and sends its name", async () => { + calls.nodes.mockResolvedValue([ + {name:"b",role:"worker",ready:true,approved:true}, + {name:"c",role:"worker",ready:false,approved:true}, + {name:"a",role:"controller",ready:true,approved:true}, + ]); + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button",{name:"New server"})); + await fillValid(user); + expect(create().disabled).toBe(true); + await user.click(screen.getByRole("combobox",{name:"Execution node"})); + expect(screen.queryByRole("option",{name:"c"})).toBeNull(); + expect(screen.queryByRole("option",{name:"a"})).toBeNull(); + await user.click(await screen.findByRole("option",{name:"b"})); + await user.click(create()); + expect(sent().nodeName).toBe("b"); + }); +}); diff --git a/panel/src/components/CreateServerDialog.tsx b/panel/src/components/CreateServerDialog.tsx index 33d2da8..a9bfaa3 100644 --- a/panel/src/components/CreateServerDialog.tsx +++ b/panel/src/components/CreateServerDialog.tsx @@ -56,6 +56,9 @@ export function CreateServerDialog({ cfg, onCreated }: Props) { // or retired since the page loaded shows up or drops out without a reload. const images = useAsync(() => (open ? api.listImages() : Promise.resolve(null)), [open]); + const nodes = useAsync(() => (open && cfg.distributed ? api.nodes() : Promise.resolve(null)), [open, cfg.distributed]); + const workers = (nodes.data ?? []).filter((n) => n.role === "worker" && n.ready && n.approved); + const [form, setForm] = useState({ name: "", subdomain: "", @@ -89,6 +92,7 @@ export function CreateServerDialog({ cfg, onCreated }: Props) { nameIssue === null && subdomainIssue === null && !!image && + (!cfg.distributed || workers.some((n) => n.name === form.nodeName)) && !!form.memory && !!form.storage && !submitting; @@ -216,6 +220,18 @@ export function CreateServerDialog({ cfg, onCreated }: Props) { )} + {cfg.distributed && ( +
+ + + {!!nodes.error && } + {!nodes.loading && !nodes.error && workers.length === 0 &&

{t("node_none")}

} +
+ )} +
diff --git a/panel/src/components/DistributedNodes.test.tsx b/panel/src/components/DistributedNodes.test.tsx new file mode 100644 index 0000000..ddd6800 --- /dev/null +++ b/panel/src/components/DistributedNodes.test.tsx @@ -0,0 +1,44 @@ +// @vitest-environment jsdom +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { MigrationDialog } from "./DistributedNodes"; + +const calls = vi.hoisted(() => ({ nodes: vi.fn(), migration: vi.fn(), retryMigration: vi.fn(), migrateServer: vi.fn() })); +vi.mock("@/lib/api", async (importActual) => { + const actual = await importActual(); + return { ...actual, api: { ...actual.api, ...calls } }; +}); + +beforeEach(() => { + vi.clearAllMocks(); + calls.nodes.mockResolvedValue([{ name: "c", role: "worker", ready: true, approved: true }]); + calls.migration.mockResolvedValue({ id: "persisted-op", state: "failed", sourceNode: "b", targetNode: "c", error: "source node is offline" }); + calls.retryMigration.mockResolvedValue({ id: "persisted-op", state: "backing_up" }); +}); + +describe("durable migration", () => { + it("loads the persisted operation on opening and retries its ID", async () => { + const user = userEvent.setup(); + const changed = vi.fn(); + render(); + await user.click(screen.getByRole("button", { name: "Migrate world" })); + await screen.findByText("source node is offline"); + expect(calls.migration).toHaveBeenCalledWith("survival"); + expect(screen.getByText("persisted-op")).toBeTruthy(); + await user.click(screen.getByRole("button", { name: "Try again" })); + await waitFor(() => expect(calls.retryMigration).toHaveBeenCalledWith("survival", "persisted-op")); + expect(changed).toHaveBeenCalled(); + expect(calls.migrateServer).not.toHaveBeenCalled(); + }); + + it("keeps retry disabled until the server is stopped", async () => { + const user = userEvent.setup(); + render( {}} />); + await user.click(screen.getByRole("button", { name: "Migrate world" })); + const retry = await screen.findByRole("button", { name: "Try again" }); + expect((retry as HTMLButtonElement).disabled).toBe(true); + await user.click(retry); + expect(calls.retryMigration).not.toHaveBeenCalled(); + }); +}); diff --git a/panel/src/components/DistributedNodes.tsx b/panel/src/components/DistributedNodes.tsx new file mode 100644 index 0000000..0e9ac2f --- /dev/null +++ b/panel/src/components/DistributedNodes.tsx @@ -0,0 +1,87 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { api, humanizeError } from "@/lib/api"; +import { useAsync, usePolling } from "@/lib/hooks"; +import { Button } from "@/components/ui/button"; +import { InlineError } from "@/components/MessageLine"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, DialogTrigger } from "@/components/ui/dialog"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Label } from "@/components/ui/label"; + +export function DistributedNodes() { + const { t } = useTranslation("servers"); + const nodes = useAsync(api.nodes); + usePolling(nodes.reload, 10_000); + return ( +
+

{t("nodes")}

+ {!!nodes.error && } +
    {nodes.data?.map((n) => ( +
  • + {n.name}{n.role || t("node_pending")} + {n.ready ? t("node_online") : t("node_offline")} + {n.approved ? t("node_approved") : t("node_pending")} + {n.architecture} · {n.addresses.join(", ")} +
    • + ))}
+
+ ); +} + +export function MigrationDialog({ name, nodeName, stopped, onChanged }: { + name: string; nodeName?: string; stopped: boolean; onChanged: () => void; +}) { + const { t } = useTranslation("servers"); + const [open, setOpen] = useState(false); + const [target, setTarget] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const nodes = useAsync(() => open ? api.nodes() : Promise.resolve(null), [open]); + const progress = useAsync(async () => { + if (!open) return null; + try { return await api.migration(name); } + catch (e) { if ((e as { status?: number }).status === 404) return null; throw e; } + }, [open, name]); + usePolling(() => { if (open) progress.reload(); }, 3_000); + const op = progress.data; + const active = !!op && op.state !== "succeeded" && op.state !== "failed"; + const workers = (nodes.data ?? []).filter((n) => n.ready && n.approved && n.role === "worker" && n.name !== nodeName); + async function run(retry: boolean) { + setBusy(true); setError(null); + try { + if (retry && op) await api.retryMigration(name, op.id); + else await api.migrateServer(name, target); + progress.reload(); onChanged(); + } catch (e) { setError(humanizeError(e)); } + finally { setBusy(false); } + } + return ( + + + + {t("migration")}{t("migration_hint")} + {nodeName &&

{t("node")}: {nodeName}

} + {op &&
+

{op.sourceNode} → {op.targetNode}

+

{t(`migration_state_${op.state}`, { defaultValue: op.state })}

+ {op.id} + {op.error && } +
} + {!!progress.error && } + {!!nodes.error && } + + + {!stopped &&

{t("migration_stop")}

} + + {op?.state === "failed" ? ( + + ) : ( + + )} + + + ); +} diff --git a/panel/src/i18n/resources/en-US/servers.json b/panel/src/i18n/resources/en-US/servers.json index 8d79989..c800757 100644 --- a/panel/src/i18n/resources/en-US/servers.json +++ b/panel/src/i18n/resources/en-US/servers.json @@ -251,5 +251,22 @@ "retiring_cancel_admin_only": "An administrator asked for the deletion, so only an administrator can cancel it.", "retiring_badge_release": "Given up", "retiring_badge_delete": "Deleting", - "retiring_badge_hint": "Given up or being deleted; handled at the next daily reclaim run, and it can't be started until then." + "retiring_badge_hint": "Given up or being deleted; handled at the next daily reclaim run, and it can't be started until then.", + "nodes": "Execution nodes", + "node": "Execution node", + "node_choose": "Choose an approved node", + "node_none": "No online approved worker nodes", + "node_online": "Online", + "node_offline": "Offline", + "node_approved": "Approved", + "node_pending": "Pending approval", + "migration": "Migrate world", + "migration_hint": "The server stays stopped after migration. The source volume is retained; check the world before starting it.", + "migration_stop": "Stop the server and wait for the game process to exit first.", + "migration_start": "Start migration", + "migration_state_backing_up": "Archiving source world", + "migration_state_restoring": "Restoring and verifying target world", + "migration_state_switching": "Switching active volume", + "migration_state_succeeded": "Migration complete; start manually after inspection", + "migration_state_failed": "Migration failed; server remains stopped" } diff --git a/panel/src/i18n/resources/zh-CN/servers.json b/panel/src/i18n/resources/zh-CN/servers.json index 6027da4..6b0a465 100644 --- a/panel/src/i18n/resources/zh-CN/servers.json +++ b/panel/src/i18n/resources/zh-CN/servers.json @@ -250,5 +250,22 @@ "retiring_cancel_admin_only": "删除由管理员提出，只有管理员可以撤销。", "retiring_badge_release": "等待回收", "retiring_badge_delete": "等待删除", - "retiring_badge_hint": "已放弃或正在删除，下一次每日回收时处理；在此之前不能启动。" + "retiring_badge_hint": "已放弃或正在删除，下一次每日回收时处理；在此之前不能启动。", + "nodes": "执行节点", + "node": "执行位置", + "node_choose": "选择已批准的节点", + "node_none": "没有在线且已批准的执行节点", + "node_online": "在线", + "node_offline": "失联", + "node_approved": "已批准", + "node_pending": "待批准", + "migration": "停服迁移", + "migration_hint": "迁移完成后仍保持停服。源卷保留，请检查后手动启动。", + "migration_stop": "请先停服并等待游戏进程退出。", + "migration_start": "开始迁移", + "migration_state_backing_up": "正在归档源卷", + "migration_state_restoring": "正在恢复并校验目标卷", + "migration_state_switching": "正在切换活动卷", + "migration_state_succeeded": "迁移完成，等待手动启动", + "migration_state_failed": "迁移失败，服务器保持停服" } diff --git a/panel/src/lib/api.ts b/panel/src/lib/api.ts index 0d0bb5b..50f56da 100644 --- a/panel/src/lib/api.ts +++ b/panel/src/lib/api.ts @@ -1,5 +1,7 @@ import type { AccessResult, + ExecutionNode, + WorldMigration, AllowlistEntry, ApiError, AutostartPolicy, @@ -594,6 +596,13 @@ export const api = rejectingSync({ URL.revokeObjectURL(url); }, + nodes: () => request<{ nodes: ExecutionNode[] }>("GET", "/nodes").then((r) => r.nodes), + migration: (name: string) => request("GET", `/servers/${encodeURIComponent(name)}/migrations`), + migrateServer: (name: string, targetNode: string) => + request("POST", `/servers/${encodeURIComponent(name)}/migrations`, { targetNode }), + retryMigration: (name: string, id: string) => + request("POST", `/servers/${encodeURIComponent(name)}/migrations/${encodeURIComponent(id)}/retry`), + createServer: (req: CreateServerRequest) => request<{ name: string; subdomain: string; desiredState: string }>( "POST", diff --git a/panel/src/lib/config.test.ts b/panel/src/lib/config.test.ts index d6e5f88..3a7c535 100644 --- a/panel/src/lib/config.test.ts +++ b/panel/src/lib/config.test.ts @@ -33,6 +33,7 @@ describe("loadConfig", () => { const cfg = await loadConfig(); expect(cfg).toEqual({ apiBase: "/api/v1", + distributed: false, rootDomain: "mc.example", panelHostname: undefined, adminHostname: "op.console.mc.example", diff --git a/panel/src/lib/config.ts b/panel/src/lib/config.ts index d1e15ec..ae7cc31 100644 --- a/panel/src/lib/config.ts +++ b/panel/src/lib/config.ts @@ -16,6 +16,7 @@ export interface BuildInfo { } export interface RuntimeConfig { + distributed?: boolean; apiBase: string; rootDomain: string; /** Player-console hostname (console.), absent when unconfigured. */ @@ -79,6 +80,7 @@ export async function loadConfig(): Promise { } cached = { apiBase: raw.apiBase ?? FALLBACK.apiBase, + distributed: raw.distributed === true, rootDomain: raw.rootDomain, panelHostname: raw.panelHostname, adminHostname: raw.adminHostname, diff --git a/panel/src/lib/openapi.gen.ts b/panel/src/lib/openapi.gen.ts index c5fffc3..a5ad16e 100644 --- a/panel/src/lib/openapi.gen.ts +++ b/panel/src/lib/openapi.gen.ts @@ -4,6 +4,75 @@ */ export interface paths { + "/api/v1/nodes": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List execution nodes (administrator). */ + get: operations["executionNodes"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/servers/{name}/migrations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Read durable migration progress. */ + get: operations["latestWorldMigrationStatus"]; + put?: never; + /** Migrate an already stopped world; persistent lock survives controller restart. */ + post: operations["startWorldMigration"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/servers/{name}/migrations/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Read durable migration progress. */ + get: operations["worldMigrationStatus"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/servers/{name}/migrations/{id}/retry": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Retry a failed migration, retaining the source and stopped state. */ + post: operations["retryWorldMigration"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/healthz": { parameters: { query?: never; @@ -2810,8 +2879,42 @@ export interface components { * @enum {string} */ Phase: "Unknown" | "Stopped" | "Starting" | "Running" | "Stopping" | "Failed"; + ExecutionNode: { + name: string; + role: string; + ready: boolean; + approved: boolean; + addresses: string[]; + architecture: string; + }; + WorldMigration: { + id: string; + server: string; + /** @enum {string} */ + state: "backing_up" | "restoring" | "switching" | "succeeded" | "failed"; + stage: string; + sourceNode: string; + targetNode: string; + sourcePVC: string; + targetPVC: string; + backup: { + ref: string; + /** Format: int64 */ + size: number; + sha256: string; + }; + /** Format: date-time */ + started: string; + /** Format: date-time */ + updated: string; + switched: boolean; + attempt: number; + error?: string; + }; /** @description Status projection of one server (internal/api/cluster.go ServerInfo). */ ServerInfo: { + /** @description Execution node; legacy servers report the observed node. */ + nodeName?: string; name: string; subdomain: string; phase: components["schemas"]["Phase"]; @@ -3561,6 +3664,154 @@ export interface components { } export type $defs = Record; export interface operations { + executionNodes: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Accepted operation or current state. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + nodes: components["schemas"]["ExecutionNode"][]; + }; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 503: components["responses"]["ServiceUnavailable"]; + }; + }; + latestWorldMigrationStatus: { + parameters: { + query?: never; + header?: never; + path: { + name: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Accepted operation or current state. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorldMigration"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 503: components["responses"]["ServiceUnavailable"]; + }; + }; + startWorldMigration: { + parameters: { + query?: never; + header?: never; + path: { + name: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + targetNode: string; + }; + }; + }; + responses: { + /** @description Accepted operation or current state. */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorldMigration"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 503: components["responses"]["ServiceUnavailable"]; + }; + }; + worldMigrationStatus: { + parameters: { + query?: never; + header?: never; + path: { + name: string; + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Accepted operation or current state. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorldMigration"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 503: components["responses"]["ServiceUnavailable"]; + }; + }; + retryWorldMigration: { + parameters: { + query?: never; + header?: never; + path: { + name: string; + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Accepted operation or current state. */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorldMigration"]; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 409: components["responses"]["Conflict"]; + 503: components["responses"]["ServiceUnavailable"]; + }; + }; healthz: { parameters: { query?: never; @@ -3721,6 +3972,8 @@ export interface operations { requestBody: { content: { "application/json": { + /** @description Required approved worker in distributed mode; administrator only. */ + nodeName?: string; name: string; subdomain: string; /** @description Trimmed; at most 64 characters, all visible ones or spaces (400 bad_display_name otherwise). */ diff --git a/panel/src/lib/types.ts b/panel/src/lib/types.ts index 949e41a..00ee8c3 100644 --- a/panel/src/lib/types.ts +++ b/panel/src/lib/types.ts @@ -59,6 +59,7 @@ export interface MyServerView { * A caller who does not own the server gets the public subset, so everything * past the counts may be absent. */ export interface ServerStatus { + nodeName?: string; name: string; subdomain: string; phase: Phase; @@ -340,6 +341,7 @@ export interface WhitelistImage { /** CreateServerRequest is the §15 structured form — the ONLY create path. */ export interface CreateServerRequest { + nodeName?: string; name: string; subdomain: string; displayName?: string; @@ -716,3 +718,25 @@ export interface SessionView { /** On the holder's own list only: the session this request came in on. */ current?: boolean; } + +export interface ExecutionNode { + name: string; + role: string; + ready: boolean; + approved: boolean; + addresses: string[]; + architecture: string; +} +export interface WorldMigration { + id: string; + server: string; + state: string; + stage: string; + sourceNode: string; + targetNode: string; + sourcePVC: string; + targetPVC: string; + error?: string; + switched: boolean; + attempt: number; +} diff --git a/panel/src/pages/servers/ServersPage.tsx b/panel/src/pages/servers/ServersPage.tsx index a687602..8c9c21a 100644 --- a/panel/src/pages/servers/ServersPage.tsx +++ b/panel/src/pages/servers/ServersPage.tsx @@ -34,6 +34,7 @@ import { PhaseBadge, PHASE_KEY, PHASE_COLOR, shownPhase, startFailure } from "@/ import { PowerButton } from "@/components/PowerButton"; import { Loading, ErrorState, EmptyState, RefreshError } from "@/components/States"; import { Pagination } from "@/components/Pagination"; +import { DistributedNodes, MigrationDialog } from "@/components/DistributedNodes"; import { CreateServerDialog } from "@/components/CreateServerDialog"; import { StatCard } from "@/components/StatCard"; import { CopyAddress } from "@/components/CopyAddress"; @@ -106,6 +107,7 @@ function compareBy(key: SortKey): (a: UnifiedServer, b: UnifiedServer) => number } interface UnifiedServer { + nodeName?: string; name: string; subdomain: string; /** The owner-chosen label; the list leads with it and keeps the name beside. */ @@ -163,6 +165,7 @@ export function ServersPage() { if (isAdmin) { return (data as FleetServer[]).map((s) => ({ name: s.name, + nodeName: s.nodeName, displayName: s.displayName, subdomain: s.subdomain, phase: s.phase, @@ -276,6 +279,8 @@ export function ServersPage() { className="mb-6" /> + {isAdmin && cfg?.distributed && } + {showInitialLoading ? ( ) : showInitialError ? ( @@ -508,6 +513,7 @@ function ServerActions({ className?: string; }) { const { t: ts } = useTranslation("servers"); + const cfg = useConfig(); const [busy, setBusy] = useState(null); const [confirmOpen, setConfirmOpen] = useState(false); const [error, setError] = useState(null); @@ -543,6 +549,7 @@ function ServerActions({ return (
+ {isAdmin && cfg?.distributed && } {claimable && ( <>
; +} + +function EndpointAddress({ server }: { server: UnifiedServer }) { const { t } = useTranslation("ops"); const endpoint = server.ready && server.endpointAddress ? server.endpointAddress : null; if (!endpoint) {