Unverified Commit adf563ff authored by Lemon-miaow's avatar Lemon-miaow
Browse files

feat(distributed): 支持单主控多节点部署和停服迁移 

复用现有 k3s 调度和 Job 生命周期，增加 worker 接入与批准、受保护节点身份、归档传输、持久迁移锁及活动 PVC 切换；同步管理员 API、CLI、面板和隔离规则。分布式模式默认关闭，保持单机兼容。

验证：Go 全量测试与 vet；面板 874 个测试、lint/build；Linux VM 安装器测试、清单服务端 dry-run、网络命名空间防火墙实测。A/B/C 三机 WireGuard、Velocity 和迁移验收仍待完成。
parent 26e817f2

cmd/felis/api.go

+50 −6
Changes for cmd/felis/api.go: 50 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ import ( 
	"felis.lolicon.best/internal/backupjob"

	"felis.lolicon.best/internal/build"

	"felis.lolicon.best/internal/config"

	"felis.lolicon.best/internal/distributed"

	"felis.lolicon.best/internal/fileedit"

	"felis.lolicon.best/internal/imagepin"

	"felis.lolicon.best/internal/mail"
@@ -27,6 +28,7 @@ import ( 
	"felis.lolicon.best/internal/naming"

	"felis.lolicon.best/internal/panel"

	"felis.lolicon.best/internal/passkey"

	"felis.lolicon.best/internal/placement"

	"felis.lolicon.best/internal/platform"

	"felis.lolicon.best/internal/reaper"

	"felis.lolicon.best/internal/registryprune"
@@ -273,9 +275,23 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { 
	// store at load, so by this point cfg.Archive.Store is guaranteed tarLocal.)

	var restorer api.Restorer

	felisImage, backupPVC := os.Getenv("FELIS_IMAGE"), os.Getenv("FELIS_BACKUP_PVC")

	worldResolver := placement.Resolve(cl, cfg.K8s.Namespace)

	distribution, err := distributionManager(cl, cfg, felisImage)

	if err != nil {

		fmt.Fprintln(stderr, err)

		return 1

	}

	if distribution != nil {

		worldResolver = distribution.Resolve

	}

	if felisImage != "" && backupPVC != "" {

		rcfg := restoreConfig(cfg, felisImage, backupPVC)

		restorer = &restore.Restorer{Jobs: restore.NewK8sJobs(cl), Config: rcfg}

		rcfg.ResolveWorld = worldResolver

		var jobs restore.Jobs = restore.NewK8sJobs(cl)

		if distribution != nil {

			jobs = distribution

		}

		restorer = &restore.Restorer{Jobs: jobs, Config: rcfg}

	} else {

		fmt.Fprintln(stderr, "felis api: restore executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — restore endpoint returns 503")

	}
@@ -288,7 +304,13 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { 
	// endpoint honestly returns 503.

	var backuper api.Backuper

	if felisImage != "" && backupPVC != "" {

		backuper = &backupjob.Backuper{Jobs: backupjob.NewK8sJobs(cl), Config: backupConfig(cfg, felisImage, backupPVC)}

		bcfg := backupConfig(cfg, felisImage, backupPVC)

		bcfg.ResolveWorld = worldResolver

		var jobs backupjob.Jobs = backupjob.NewK8sJobs(cl)

		if distribution != nil {

			jobs = distribution

		}

		backuper = &backupjob.Backuper{Jobs: jobs, Config: bcfg}

	} else {

		fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503")

	}
@@ -299,7 +321,9 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { 
	// is wired under the restore gate; otherwise the export routes return 503.

	var exporter api.Exporter

	if felisImage != "" && backupPVC != "" {

		exporter = worldexport.New(clientset, exportConfig(cfg, felisImage, backupPVC))

		ecfg := exportConfig(cfg, felisImage, backupPVC)

		ecfg.ResolveWorld = worldResolver

		exporter = worldexport.New(clientset, ecfg)

	} else {

		fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503")

	}
@@ -319,9 +343,11 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { 
	var files api.FileEditor

	var fileStage *fileedit.Stage

	if felisImage != "" {

		fcfg := fileEditConfig(cfg, felisImage)

		fcfg.ResolveWorld = worldResolver

		files = &fileedit.Editor{

			Runner: fileedit.NewK8sRunner(clientset),

			Config: fileEditConfig(cfg, felisImage),

			Config: fcfg,

		}

		fileStage = &fileedit.Stage{Dir: fileStagingDir()}

		if err := fileStage.Sweep(); err != nil {
@@ -355,7 +381,21 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { 
		fmt.Fprintf(stderr, "felis api: MinecraftServer cache: %v\n", err)

		return 1

	}

	cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced)

	cluster := api.NewK8sCluster(cl, cfg.K8s.Namespace).WithServerCache(serverCache, serversSynced).WithDistributed(distribution != nil, os.Getenv("FELIS_CONTROLLER_NODE"))

	if distribution != nil {

		distribution.Record = func(ctx context.Context, b distributed.Backup) error {

			keep, retention, ok := backupPolicy(b.Reason, rcfg)

			if !ok {

				return fmt.Errorf("unknown backup reason %s", b.Reason)

			}

			st := reaper.NewPGStore(drv.DB())

			if err := st.InsertBackup(ctx, reaper.BackupRecord{ID: "bk-" + b.ID, ServerName: b.Server, FormerOwner: b.Owner, BackupRef: b.Receipt.Ref, SizeBytes: b.Receipt.Size, SHA256: b.Receipt.SHA256, Reason: b.Reason, ExpiresAt: time.Now().Add(retention)}); err != nil {

				return err

			}

			pruneBackups(ctx, st, distribution.Archive, b.Server, b.Owner, b.Reason, keep, b.Protect, stdout, stderr)

			return nil

		}

	}

	jobStatus := api.NewK8sJobStatus(cl, cfg.K8s.Namespace)

	a := &api.API{

		Repo:    repo,
@@ -455,7 +495,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { 
	externalHandler := panel.Handler(a.ExternalHandler(), cfg.Server.RootDomain,

		defaultPanelHostname(cfg.Server.RootDomain, cfg.Auth.PanelHostname),

		defaultAdminHostname(cfg.Server.RootDomain, cfg.Auth.AdminHostname),

		cfg.Velocity.GamePort, resolvedVersion())

		cfg.Velocity.GamePort, resolvedVersion(), distribution != nil)

	internalSrv := newAPIServer(*internalAddr, a.InternalHandler())

	externalSrv := newAPIServer(cfg.Server.Listen, externalHandler)
@@ -499,6 +539,10 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { 
	}

	go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())



	if distribution != nil {

		a.Distribution = distribution

		go reconcileDistribution(ctx, distribution, stderr)

	}

	servers := []*http.Server{internalSrv, externalSrv}

	if httpsSrv != nil {

		servers = append(servers, httpsSrv)

cmd/felis/archive_serve.go

0 → 100644
+58 −0
Changes for cmd/felis/archive_serve.go: 58 added lines, 0 removed lines.
Original line number Diff line number Diff line 
package main



import (

	"errors"

	"flag"

	"fmt"

	"io"

	"net/http"

	"os"

	"time"



	"felis.lolicon.best/internal/archivetransfer"

)



func cmdArchiveServe(args []string, stdout, stderr io.Writer) int {

	fs := flag.NewFlagSet("archive-serve", flag.ContinueOnError)

	fs.SetOutput(stderr)

	root := fs.String("root", "/backups", "archive PVC mount (must match archive.local_path)")

	addr := fs.String("listen", ":8090", "private archive listener")

	limit := fs.Int64("max-bytes", archivetransfer.DefaultLimit, "maximum compressed archive bytes")

	if err := fs.Parse(args); err != nil {

		return 2

	}

	key := os.Getenv(archivetransfer.KeyEnv)

	if len(key) < 32 || *limit <= 0 {

		fmt.Fprintln(stderr, "archive-serve: signing key and positive size limit required")

		return 2

	}

	if err := os.MkdirAll(*root, 0750); err != nil {

		fmt.Fprintln(stderr, err)

		return 1

	}

	limitHeapToCgroup()

	transport := &archivetransfer.Server{Root: *root, Key: key, Limit: *limit}

	done := make(chan struct{})

	defer close(done)

	go func() {

		ticker := time.NewTicker(time.Hour)

		defer ticker.Stop()

		for {

			if err := transport.Sweep(time.Now()); err != nil {

				fmt.Fprintln(stderr, "archive journal cleanup:", err)

			}

			select {

			case <-done:

				return

			case <-ticker.C:

			}

		}

	}()

	srv := &http.Server{Addr: *addr, Handler: transport, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Hour, WriteTimeout: 2 * time.Hour, MaxHeaderBytes: 16 << 10}

	fmt.Fprintln(stdout, "archive transport listening", *addr)

	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {

		fmt.Fprintln(stderr, err)

		return 1

	}

	return 0

}

cmd/felis/backup.go

+3 −1
Changes for cmd/felis/backup.go: 3 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -168,7 +168,9 @@ func backupPolicy(reason string, rcfg reaper.Config) (keep int, retention time.D 
// previous owner's. protect is never removed: it is the backup a chained restore

// is about to extract. The new backup is already recorded; a removal that fails

// is reported and retried after the next backup.

func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver backup.WorldArchiver, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) {

func pruneBackups(ctx context.Context, st *reaper.PGStore, archiver interface {

	Delete(context.Context, backup.ArchiveRef) error

}, server, owner, reason string, keep int, protect string, stdout, stderr io.Writer) {

	excess, err := st.ExcessBackups(ctx, server, owner, reason, keep, protect)

	if err != nil {

		fmt.Fprintf(stderr, "felis backup: list older backups of %s: %v\n", server, err)

cmd/felis/distributed.go

0 → 100644
+44 −0
Changes for cmd/felis/distributed.go: 44 added lines, 0 removed lines.
Original line number Diff line number Diff line 
package main



import (

	"context"

	"fmt"

	"io"

	"os"

	"time"



	"felis.lolicon.best/internal/archivetransfer"

	"felis.lolicon.best/internal/config"

	"felis.lolicon.best/internal/distributed"

	"felis.lolicon.best/internal/placement"

	"sigs.k8s.io/controller-runtime/pkg/client"

)



func distributionManager(cl client.Client, cfg *config.Config, image string) (*distributed.Manager, error) {

	if os.Getenv("FELIS_DISTRIBUTED") != "true" {

		return nil, nil

	}

	controller, url, key := os.Getenv("FELIS_CONTROLLER_NODE"), os.Getenv("FELIS_ARCHIVE_URL"), os.Getenv(archivetransfer.KeyEnv)

	if controller == "" || url == "" || len(key) < 32 || image == "" || cfg.Archive.Store != "tarLocal" {

		return nil, fmt.Errorf("distributed mode requires controller identity, archive service/key, Felis image and tarLocal")

	}

	return &distributed.Manager{Client: cl, Namespace: cfg.K8s.Namespace, Image: image, Controller: controller, Archive: archivetransfer.Client{URL: url, Root: cfg.Archive.LocalPath, Key: key}, Resolve: placement.Resolve(cl, cfg.K8s.Namespace, controller)}, nil

}



func reconcileDistribution(ctx context.Context, m *distributed.Manager, stderr io.Writer) {

	ticker := time.NewTicker(3 * time.Second)

	defer ticker.Stop()

	for {

		if err := m.SettleBackups(ctx); err != nil {

			fmt.Fprintln(stderr, "distributed backup:", err)

		}

		if err := m.ReconcileMigrations(ctx); err != nil {

			fmt.Fprintln(stderr, "migration:", err)

		}

		select {

		case <-ctx.Done():

			return

		case <-ticker.C:

		}

	}

}

cmd/felis/egressgate.go

+13 −0
Changes for cmd/felis/egressgate.go: 13 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ func cmdEgressGate(args []string, stdout, stderr io.Writer) int { 
	fs := flag.NewFlagSet("egress-gate", flag.ContinueOnError)

	fs.SetOutput(stderr)

	probe := fs.String("probe", "", "host:port the pod's NetworkPolicy denies (default: the Kubernetes API Service from KUBERNETES_SERVICE_HOST/PORT)")

	positive := fs.String("positive-probe", "", "allowed host:port that must remain reachable during denial checks")

	wait := fs.Duration("wait", 2*time.Minute, "how long the probe may keep answering before the gate gives up")

	failOpen := fs.Bool("fail-open", false, "when --wait runs out, warn and let the pod go on instead of refusing it")

	if err := fs.Parse(args); err != nil {
@@ -53,6 +54,18 @@ func cmdEgressGate(args []string, stdout, stderr io.Writer) int { 


	start := time.Now()

	for {

		if *positive != "" {

			allowed, err := net.DialTimeout("tcp", *positive, egressDialTimeout)

			if err != nil {

				if time.Since(start) >= *wait {

					fmt.Fprintln(stderr, "felis egress-gate: positive probe unavailable; refusing to start", err)

					return 1

				}

				time.Sleep(egressPollInterval)

				continue

			}

			allowed.Close()

		}

		conn, err := net.DialTimeout("tcp", *probe, egressDialTimeout)

		if err != nil {

			fmt.Fprintf(stdout, "felis egress-gate: %s is unreachable after %s (%v); the egress lock is in effect\n",