feat(backups): 主人和管理员可下载单个备份、导出停服世界，字节经一次性票据从导出 Job 流式转给浏览器，备份按 sha256 核对 (5dffadb4) · Commits · FelisMC / Felis · GitLab

cmd/felis/api.go

+24 −0

Changes for cmd/felis/api.go: 24 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@ import (
		"felis.lolicon.best/internal/restore"
		"felis.lolicon.best/internal/retention"
		"felis.lolicon.best/internal/submit"
		"felis.lolicon.best/internal/worldexport"
		"k8s.io/apimachinery/pkg/runtime"
		utilruntime "k8s.io/apimachinery/pkg/util/runtime"
		"k8s.io/client-go/kubernetes"
		@@ -292,6 +293,17 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503")
		}

		// World export: a weak-SA Job mounts the world PVC, or the backup PVC, read-only
		// and PUTs the archive to the internal face, which streams it on to the owner's
		// browser (internal/worldexport). A backup export mounts the backup PVC, so it
		// is wired under the restore gate; otherwise the export routes return 503.
		var exporter api.Exporter
		if felisImage != "" && backupPVC != "" {
		exporter = worldexport.New(clientset, exportConfig(cfg, felisImage, backupPVC))
		} else {
		fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503")
		}

		// Server file editor: a weak-SA Job mounts ONLY the target world PVC and runs
		// `felis files`, printing its result for felis-api to read back through
		// pods/log (see internal/fileedit). It needs FELIS_IMAGE but — unlike restore
		@@ -368,6 +380,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		// namespace, where the internal face is reachable like it is for the login
		// gate.
		InternalBaseURL: internalAPIBaseURL(),
		Exporter: exporter,
		Submissions: submissions,
		Mailer: mailer,
		Schedules: repo,
		@@ -687,6 +700,17 @@ func backupConfig(cfg *config.Config, image, backupPVC string) backupjob.Config
		}
		}

		// exportConfig builds the world export executor's config. BackupRoot mirrors
		// restoreConfig: the stored refs are absolute paths under [archive] local_path.
		func exportConfig(cfg *config.Config, image, backupPVC string) worldexport.Config {
		return worldexport.Config{
		Namespace: cfg.K8s.Namespace,
		Image: image,
		BackupPVC: backupPVC,
		BackupRoot: cfg.Archive.LocalPath,
		}
		}

		// fileEditConfig builds the file editor's config from felis.toml plus the
		// deployment-supplied image. It is the shortest of the three: the editor mounts
		// only the world PVC, so it needs no archive coordinates at all, and everything

cmd/felis/export.go

0 → 100644

+153 −0

Changes for cmd/felis/export.go: 153 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"context"
		"encoding/json"
		"flag"
		"fmt"
		"io"
		"net/http"
		"os"
		"os/signal"
		"syscall"
		"time"

		"felis.lolicon.best/internal/backup"
		"felis.lolicon.best/internal/worldexport"
		)

		// cmdExport is the in-Pod entrypoint the export Job runs. internal/worldexport
		// renders a Pod whose command is `/usr/local/bin/felis export`. It archives the
		// mounted world (or opens one archive on the mounted backup store), PUTs the
		// tar.gz to felis-api's internal face, and exits once felis-api says the
		// owner's browser got all of it. It is NOT a user-facing command and is never
		// invoked by hand.
		//
		// Like cmdRestore it holds no database credentials and never calls config.Load:
		// felis-api made every decision (who may download what, that the server is
		// stopped, which archive) before the Job existed. Its input is the flags below
		// plus the one-time upload token in the environment, which opens this one
		// export and nothing else.
		//
		// Exit status: 0 once felis-api answers 204 (the download completed), 1 when
		// the archive could not be read or handed over, or felis-api refused it (the
		// browser never came, left early, or the backup failed its digest check), 2 on
		// bad flags. The last stderr line reaches the export's status and the jobs list.
		func cmdExport(args []string, stdout, stderr io.Writer) int {
		fs := flag.NewFlagSet("export", flag.ContinueOnError)
		fs.SetOutput(stderr)
		mode := fs.String("mode", "", "what to export: world or backup")
		server := fs.String("server", "", "server name being exported (for logging)")
		target := fs.String("target-url", "", "felis-api URL to PUT the archive to")
		ref := fs.String("ref", "", "backup only: absolute path to the archive on the backup mount")
		backupRoot := fs.String("backup-root", "/backups", "backup only: mount path of the backup PVC (the ref must resolve under it)")
		worldsRoot := fs.String("worlds-root", "/world", "world only: mount path of the world PVC to archive")
		if err := fs.Parse(args); err != nil {
		return 2
		}
		token := os.Getenv(worldexport.TokenEnv)
		if *target == "" \|\| token == "" {
		fmt.Fprintf(stderr, "felis export: --target-url and %s are required\n", worldexport.TokenEnv)
		return 2
		}

		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
		defer stop()
		var err error
		switch *mode {
		case worldexport.ModeBackup:
		if *ref == "" {
		fmt.Fprintln(stderr, "felis export: --ref is required for a backup")
		return 2
		}
		err = exportBackup(ctx, target, token, ref, *backupRoot)
		case worldexport.ModeWorld:
		err = exportWorld(ctx, target, token, worldsRoot, stdout)
		default:
		fmt.Fprintf(stderr, "felis export: --mode must be %s or %s\n", worldexport.ModeWorld, worldexport.ModeBackup)
		return 2
		}
		if err != nil {
		fmt.Fprintf(stderr, "felis export: %v\n", err)
		return 1
		}
		fmt.Fprintf(stdout, "felis export: server=%s mode=%s downloaded\n", server, mode)
		return 0
		}

		// exportBackup hands over one stored archive as it is, with its length, so the
		// browser shows real progress and felis-api can check its recorded digest.
		func exportBackup(ctx context.Context, target, token, ref, root string) error {
		// Defense in depth, as in cmdRestore: the ref comes from felis-api, but this
		// process opens it, so it confirms the ref stays on the backup mount.
		if !refWithinRoot(ref, root) {
		return fmt.Errorf("ref %q is not under backup root %q", ref, root)
		}
		f, err := os.Open(ref)
		if err != nil {
		return err
		}
		defer f.Close()
		st, err := f.Stat()
		if err != nil {
		return err
		}
		return putExport(ctx, target, token, f, st.Size())
		}

		// exportWorld archives the world straight into the request body: nothing is
		// staged, so a world bigger than the Pod's memory or any scratch disk exports
		// the same. A read error mid-way aborts the chunked body, and felis-api cuts
		// the browser's download off rather than end it.
		func exportWorld(ctx context.Context, target, token, root string, stdout io.Writer) error {
		pr, pw := io.Pipe()
		skippedc := make(chan []string, 1)
		go func() {
		skipped, err := backup.WriteTarGz(ctx, pw, root)
		pw.CloseWithError(err)
		skippedc <- skipped
		}()
		err := putExport(ctx, target, token, pr, -1)
		pr.CloseWithError(io.ErrClosedPipe) // stops the archiver if the PUT ended first
		if skipped := <-skippedc; len(skipped) > 0 {
		fmt.Fprintf(stdout, "felis export: left out %d entries a tar cannot hold (symbolic links, devices, sockets)\n", len(skipped))
		}
		return err
		}

		// putExport PUTs the archive to felis-api. There is no retry: the token opens
		// the export once, so a second attempt could only be refused. Redirects are
		// refused because the request carries the token and the internal face never
		// redirects. felis-api answers only after the whole download, which the Job's
		// activeDeadlineSeconds bounds, so the header timeout is a backstop for a
		// wedged endpoint and not the real limit.
		func putExport(ctx context.Context, target, token string, body io.Reader, size int64) error {
		req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body)
		if err != nil {
		return err
		}
		req.ContentLength = size
		req.Header.Set("Authorization", "Bearer "+token)
		req.Header.Set("Content-Type", "application/gzip")
		client := &http.Client{
		Transport: &http.Transport{ResponseHeaderTimeout: 10 * time.Minute},
		CheckRedirect: func(http.Request, []http.Request) error { return http.ErrUseLastResponse },
		}
		resp, err := client.Do(req)
		if err != nil {
		return err
		}
		defer resp.Body.Close()
		if resp.StatusCode == http.StatusNoContent {
		return nil
		}
		var e struct {
		Error struct {
		Message string `json:"message"`
		} `json:"error"`
		}
		if json.NewDecoder(io.LimitReader(resp.Body, 4<<10)).Decode(&e) == nil && e.Error.Message != "" {
		return fmt.Errorf("felis-api answered %s: %s", resp.Status, e.Error.Message)
		}
		return fmt.Errorf("felis-api answered %s", resp.Status)
		}

cmd/felis/export_test.go

0 → 100644

+243 −0

Changes for cmd/felis/export_test.go: 243 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"archive/tar"
		"bytes"
		"compress/gzip"
		"io"
		"net/http"
		"net/http/httptest"
		"os"
		"path/filepath"
		"strconv"
		"strings"
		"sync/atomic"
		"testing"
		"time"

		"felis.lolicon.best/internal/config"
		"felis.lolicon.best/internal/worldexport"
		)

		// exportReceiver stands in for felis-api's internal upload route: it records
		// the PUT it gets (or the error reading it ended on) and answers with reply.
		type exportReceiver struct {
		srv *httptest.Server
		hits atomic.Int32
		req *http.Request
		body []byte
		readErr error
		served chan struct{} // one send per request, once it is answered
		}

		func receiveExport(t testing.T, reply func(w http.ResponseWriter)) exportReceiver {
		t.Helper()
		rcv := &exportReceiver{served: make(chan struct{}, 1)}
		rcv.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		rcv.hits.Add(1)
		rcv.req = r
		rcv.body, rcv.readErr = io.ReadAll(r.Body)
		reply(w)
		rcv.served <- struct{}{}
		}))
		t.Cleanup(rcv.srv.Close)
		return rcv
		}

		func noContent(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) }

		func tarEntries(t *testing.T, archive []byte) map[string]string {
		t.Helper()
		gz, err := gzip.NewReader(bytes.NewReader(archive))
		if err != nil {
		t.Fatalf("not gzip: %v", err)
		}
		out := map[string]string{}
		tr := tar.NewReader(gz)
		for {
		h, err := tr.Next()
		if err == io.EOF {
		return out
		}
		if err != nil {
		t.Fatalf("tar: %v", err)
		}
		b, _ := io.ReadAll(tr)
		out[h.Name] = string(b)
		}
		}

		func TestCmdExportWorld(t *testing.T) {
		root := t.TempDir()
		if err := os.MkdirAll(filepath.Join(root, "world", "region"), 0o755); err != nil {
		t.Fatal(err)
		}
		for name, body := range map[string]string{"server.properties": "motd=hi\n", "world/region/r.0.0.mca": "chunks"} {
		if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o600); err != nil {
		t.Fatal(err)
		}
		}
		rcv := receiveExport(t, noContent)
		t.Setenv(worldexport.TokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		code := cmdExport([]string{"--mode", "world", "--server", "survival", "--target-url", rcv.srv.URL + "/api/v1/internal/exports/ab",
		"--worlds-root", root}, &stdout, &stderr)
		if code != 0 {
		t.Fatalf("exit %d, stderr %q", code, stderr.String())
		}
		r := rcv.req
		if r.Method != http.MethodPut \|\| r.URL.Path != "/api/v1/internal/exports/ab" \|\| r.Header.Get("Authorization") != "Bearer tok" \|\|
		r.Header.Get("Content-Type") != "application/gzip" \|\| r.ContentLength != -1 \|\| strings.Join(r.TransferEncoding, ",") != "chunked" {
		t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
		}
		got := tarEntries(t, rcv.body)
		want := map[string]string{"server.properties": "motd=hi\n", "world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks"}
		if len(got) != len(want) {
		t.Fatalf("archive holds %v, want %v", got, want)
		}
		for name, body := range want {
		if b, ok := got[name]; !ok \|\| b != body {
		t.Errorf("%s = %q (present %v), want %q", name, b, ok, body)
		}
		}
		if !strings.Contains(stdout.String(), "server=survival mode=world downloaded") {
		t.Errorf("stdout = %q", stdout.String())
		}
		}

		// A world that cannot be read must never reach felis-api as a complete body:
		// the chunked upload is cut off, so felis-api aborts the browser's download.
		func TestCmdExportWorldReadErrorAbortsTheUpload(t *testing.T) {
		rcv := receiveExport(t, noContent)
		t.Setenv(worldexport.TokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
		if code != 1 {
		t.Fatalf("exit %d, want 1", code)
		}
		select {
		case <-rcv.served:
		if rcv.readErr == nil {
		t.Fatalf("felis-api read a complete %d-byte body from an unreadable world", len(rcv.body))
		}
		case <-time.After(2 * time.Second): // the request never reached the handler
		}
		}

		func TestCmdExportBackup(t *testing.T) {
		root := t.TempDir()
		archive := bytes.Repeat([]byte("felis"), 10_000)
		ref := filepath.Join(root, "survival-1.tar.gz")
		if err := os.WriteFile(ref, archive, 0o600); err != nil {
		t.Fatal(err)
		}
		args := func(url, ref string) []string {
		return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
		}

		t.Run("hands the archive over with its length", func(t *testing.T) {
		rcv := receiveExport(t, noContent)
		t.Setenv(worldexport.TokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 0 {
		t.Fatalf("exit %d, stderr %q", code, stderr.String())
		}
		if rcv.req.ContentLength != int64(len(archive)) \|\| !bytes.Equal(rcv.body, archive) \|\| rcv.req.Header.Get("Authorization") != "Bearer tok" {
		t.Fatalf("got %d bytes (length %d, auth %q), want the %d archive bytes",
		len(rcv.body), rcv.req.ContentLength, rcv.req.Header.Get("Authorization"), len(archive))
		}
		})

		t.Run("a ref outside the backup root is refused before any request", func(t *testing.T) {
		outside := filepath.Join(t.TempDir(), "secret.tar.gz")
		if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil {
		t.Fatal(err)
		}
		rcv := receiveExport(t, noContent)
		t.Setenv(worldexport.TokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		if code := cmdExport(args(rcv.srv.URL, outside), &stdout, &stderr); code != 1 {
		t.Fatalf("exit %d, want 1", code)
		}
		if n := rcv.hits.Load(); n != 0 {
		t.Fatalf("felis-api got %d requests", n)
		}
		})

		t.Run("a refusal exits 1 with felis-api's reason", func(t *testing.T) {
		rcv := receiveExport(t, func(w http.ResponseWriter) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusGone)
		io.WriteString(w, `{"error":{"code":"export_expired","message":"nobody opened the download"}}`)
		})
		t.Setenv(worldexport.TokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 1 {
		t.Fatalf("exit %d, want 1", code)
		}
		if got := stderr.String(); got != "felis export: felis-api answered 410 Gone: nobody opened the download\n" {
		t.Fatalf("stderr = %q", got)
		}
		})

		// The request carries the token and the internal face never redirects, so a
		// redirect is refused rather than followed with the token attached. A 302 or
		// 303 is the one net/http would follow on its own (as a GET, and to the same
		// host with the Authorization header still on it).
		for _, status := range []int{http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
		t.Run("a redirect is not followed: "+strconv.Itoa(status), func(t *testing.T) {
		elsewhere := receiveExport(t, noContent)
		redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.srv.URL, status))
		defer redirecting.Close()
		t.Setenv(worldexport.TokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		if code := cmdExport(args(redirecting.URL, ref), &stdout, &stderr); code != 1 {
		t.Fatalf("exit %d, want 1", code)
		}
		if n := elsewhere.hits.Load(); n != 0 {
		t.Fatalf("the redirect target got %d requests", n)
		}
		if got, want := stderr.String(), "felis export: felis-api answered "+strconv.Itoa(status)+" "+http.StatusText(status)+"\n"; got != want {
		t.Fatalf("stderr = %q, want %q", got, want)
		}
		})
		}
		}

		func TestCmdExportUsage(t *testing.T) {
		for name, tc := range map[string]struct {
		token string
		args []string
		}{
		"no token": {"", []string{"--mode", "world", "--target-url", "http://api/x"}},
		"no target": {"tok", []string{"--mode", "world"}},
		"unknown mode": {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
		"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
		} {
		t.Run(name, func(t *testing.T) {
		t.Setenv(worldexport.TokenEnv, tc.token)
		var stdout, stderr bytes.Buffer
		if code := cmdExport(tc.args, &stdout, &stderr); code != 2 {
		t.Fatalf("exit %d, want 2; stderr %q", code, stderr.String())
		}
		})
		}
		}

		// TestCmdExportWiring: the Job's `felis export` reaches cmdExport, and
		// felis-api's executor mounts the backup store at the path the archives were
		// written under, since a backup's ref is an absolute path there.
		func TestCmdExportWiring(t *testing.T) {
		t.Setenv(worldexport.TokenEnv, "")
		var stdout, stderr bytes.Buffer
		if code := run([]string{"export"}, &stdout, &stderr); code != 2 \|\|
		stderr.String() != "felis export: --target-url and "+worldexport.TokenEnv+" are required\n" {
		t.Fatalf("felis export = %d, stderr %q", code, stderr.String())
		}
		cfg := &config.Config{}
		cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups"
		want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"}
		if got := exportConfig(cfg, "felis:1", "felis-backups"); got != want {
		t.Fatalf("exportConfig = %+v, want %+v", got, want)
		}
		}

cmd/felis/run.go

+2 −0

Changes for cmd/felis/run.go: 2 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@ Commands:
		backup Archive a world into the backup store and record it (internal Job entrypoint)
		backup-now Archive every user server's world now, one at a time (or the named ones; -stop stops running ones first; prints the plan, -yes applies; requires root/sudo)
		files List, read, write, mkdir, delete, rename or upload one path in a stopped server's world (internal Job entrypoint)
		export Archive a stopped server's world, or read one of its backups, and hand it to felis-api for download (internal Job entrypoint)
		egress-gate Hold a build or game server pod until its egress NetworkPolicy is enforced (internal init container entrypoint)
		fetch-context Fetch and extract a submission's build context (internal Job entrypoint)
		scan-gate Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
		@@ -66,6 +67,7 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
		"backup": cmdBackup,
		"backup-now": cmdBackupNow,
		"files": cmdFiles,
		"export": cmdExport,
		"egress-gate": cmdEgressGate,
		"fetch-context": cmdFetchContext,
		"scan-gate": cmdScanGate,

docs/openapi.yaml

+261 −5

File changed.

Preview size limit exceeded, changes collapsed.