feat(backups): 主人和管理员可下载单个备份、导出停服世界,字节经一次性票据从导出 Job 流式转给浏览器,备份按 sha256 核对

This commit is contained in:
Lemon-miaow committed 2026-09-28 00:58:42 +08:00
1 parent 34b81ee8fe
commit 5dffadb40d
35 files changed
+4018 -70

No files matched your search

+24
View File
@@ -33,6 +33,7 @@ import (
"felis.lolicon.best/internal/restore"
"felis.lolicon.best/internal/retention"
"felis.lolicon.best/internal/submit"
"felis.lolicon.best/internal/worldexport"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
@@ -292,6 +293,17 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
fmt.Fprintln(stderr, "felis api: backup executor disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — backup endpoint returns 503")
}
// World export: a weak-SA Job mounts the world PVC, or the backup PVC, read-only
// and PUTs the archive to the internal face, which streams it on to the owner's
// browser (internal/worldexport). A backup export mounts the backup PVC, so it
// is wired under the restore gate; otherwise the export routes return 503.
var exporter api.Exporter
if felisImage != "" && backupPVC != "" {
exporter = worldexport.New(clientset, exportConfig(cfg, felisImage, backupPVC))
} else {
fmt.Fprintln(stderr, "felis api: world export disabled (needs FELIS_IMAGE and FELIS_BACKUP_PVC) — export endpoints return 503")
}
// Server file editor: a weak-SA Job mounts ONLY the target world PVC and runs
// `felis files`, printing its result for felis-api to read back through
// pods/log (see internal/fileedit). It needs FELIS_IMAGE but — unlike restore
@@ -368,6 +380,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
// namespace, where the internal face is reachable like it is for the login
// gate.
InternalBaseURL: internalAPIBaseURL(),
Exporter: exporter,
Submissions: submissions,
Mailer: mailer,
Schedules: repo,
@@ -687,6 +700,17 @@ func backupConfig(cfg *config.Config, image, backupPVC string) backupjob.Config
}
}
// exportConfig builds the world export executor's config. BackupRoot mirrors
// restoreConfig: the stored refs are absolute paths under [archive] local_path.
func exportConfig(cfg *config.Config, image, backupPVC string) worldexport.Config {
return worldexport.Config{
Namespace: cfg.K8s.Namespace,
Image: image,
BackupPVC: backupPVC,
BackupRoot: cfg.Archive.LocalPath,
}
}
// fileEditConfig builds the file editor's config from felis.toml plus the
// deployment-supplied image. It is the shortest of the three: the editor mounts
// only the world PVC, so it needs no archive coordinates at all, and everything
+153
View File
@@ -0,0 +1,153 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"felis.lolicon.best/internal/backup"
"felis.lolicon.best/internal/worldexport"
)
// cmdExport is the in-Pod entrypoint the export Job runs. internal/worldexport
// renders a Pod whose command is `/usr/local/bin/felis export`. It archives the
// mounted world (or opens one archive on the mounted backup store), PUTs the
// tar.gz to felis-api's internal face, and exits once felis-api says the
// owner's browser got all of it. It is NOT a user-facing command and is never
// invoked by hand.
//
// Like cmdRestore it holds no database credentials and never calls config.Load:
// felis-api made every decision (who may download what, that the server is
// stopped, which archive) before the Job existed. Its input is the flags below
// plus the one-time upload token in the environment, which opens this one
// export and nothing else.
//
// Exit status: 0 once felis-api answers 204 (the download completed), 1 when
// the archive could not be read or handed over, or felis-api refused it (the
// browser never came, left early, or the backup failed its digest check), 2 on
// bad flags. The last stderr line reaches the export's status and the jobs list.
func cmdExport(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("export", flag.ContinueOnError)
fs.SetOutput(stderr)
mode := fs.String("mode", "", "what to export: world or backup")
server := fs.String("server", "", "server name being exported (for logging)")
target := fs.String("target-url", "", "felis-api URL to PUT the archive to")
ref := fs.String("ref", "", "backup only: absolute path to the archive on the backup mount")
backupRoot := fs.String("backup-root", "/backups", "backup only: mount path of the backup PVC (the ref must resolve under it)")
worldsRoot := fs.String("worlds-root", "/world", "world only: mount path of the world PVC to archive")
if err := fs.Parse(args); err != nil {
return 2
}
token := os.Getenv(worldexport.TokenEnv)
if *target == "" || token == "" {
fmt.Fprintf(stderr, "felis export: --target-url and %s are required\n", worldexport.TokenEnv)
return 2
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
var err error
switch *mode {
case worldexport.ModeBackup:
if *ref == "" {
fmt.Fprintln(stderr, "felis export: --ref is required for a backup")
return 2
}
err = exportBackup(ctx, *target, token, *ref, *backupRoot)
case worldexport.ModeWorld:
err = exportWorld(ctx, *target, token, *worldsRoot, stdout)
default:
fmt.Fprintf(stderr, "felis export: --mode must be %s or %s\n", worldexport.ModeWorld, worldexport.ModeBackup)
return 2
}
if err != nil {
fmt.Fprintf(stderr, "felis export: %v\n", err)
return 1
}
fmt.Fprintf(stdout, "felis export: server=%s mode=%s downloaded\n", *server, *mode)
return 0
}
// exportBackup hands over one stored archive as it is, with its length, so the
// browser shows real progress and felis-api can check its recorded digest.
func exportBackup(ctx context.Context, target, token, ref, root string) error {
// Defense in depth, as in cmdRestore: the ref comes from felis-api, but this
// process opens it, so it confirms the ref stays on the backup mount.
if !refWithinRoot(ref, root) {
return fmt.Errorf("ref %q is not under backup root %q", ref, root)
}
f, err := os.Open(ref)
if err != nil {
return err
}
defer f.Close()
st, err := f.Stat()
if err != nil {
return err
}
return putExport(ctx, target, token, f, st.Size())
}
// exportWorld archives the world straight into the request body: nothing is
// staged, so a world bigger than the Pod's memory or any scratch disk exports
// the same. A read error mid-way aborts the chunked body, and felis-api cuts
// the browser's download off rather than end it.
func exportWorld(ctx context.Context, target, token, root string, stdout io.Writer) error {
pr, pw := io.Pipe()
skippedc := make(chan []string, 1)
go func() {
skipped, err := backup.WriteTarGz(ctx, pw, root)
pw.CloseWithError(err)
skippedc <- skipped
}()
err := putExport(ctx, target, token, pr, -1)
pr.CloseWithError(io.ErrClosedPipe) // stops the archiver if the PUT ended first
if skipped := <-skippedc; len(skipped) > 0 {
fmt.Fprintf(stdout, "felis export: left out %d entries a tar cannot hold (symbolic links, devices, sockets)\n", len(skipped))
}
return err
}
// putExport PUTs the archive to felis-api. There is no retry: the token opens
// the export once, so a second attempt could only be refused. Redirects are
// refused because the request carries the token and the internal face never
// redirects. felis-api answers only after the whole download, which the Job's
// activeDeadlineSeconds bounds, so the header timeout is a backstop for a
// wedged endpoint and not the real limit.
func putExport(ctx context.Context, target, token string, body io.Reader, size int64) error {
req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body)
if err != nil {
return err
}
req.ContentLength = size
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/gzip")
client := &http.Client{
Transport: &http.Transport{ResponseHeaderTimeout: 10 * time.Minute},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
return nil
}
var e struct {
Error struct {
Message string `json:"message"`
} `json:"error"`
}
if json.NewDecoder(io.LimitReader(resp.Body, 4<<10)).Decode(&e) == nil && e.Error.Message != "" {
return fmt.Errorf("felis-api answered %s: %s", resp.Status, e.Error.Message)
}
return fmt.Errorf("felis-api answered %s", resp.Status)
}
+243
View File
@@ -0,0 +1,243 @@
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/worldexport"
)
// exportReceiver stands in for felis-api's internal upload route: it records
// the PUT it gets (or the error reading it ended on) and answers with reply.
type exportReceiver struct {
srv *httptest.Server
hits atomic.Int32
req *http.Request
body []byte
readErr error
served chan struct{} // one send per request, once it is answered
}
func receiveExport(t *testing.T, reply func(w http.ResponseWriter)) *exportReceiver {
t.Helper()
rcv := &exportReceiver{served: make(chan struct{}, 1)}
rcv.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rcv.hits.Add(1)
rcv.req = r
rcv.body, rcv.readErr = io.ReadAll(r.Body)
reply(w)
rcv.served <- struct{}{}
}))
t.Cleanup(rcv.srv.Close)
return rcv
}
func noContent(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) }
func tarEntries(t *testing.T, archive []byte) map[string]string {
t.Helper()
gz, err := gzip.NewReader(bytes.NewReader(archive))
if err != nil {
t.Fatalf("not gzip: %v", err)
}
out := map[string]string{}
tr := tar.NewReader(gz)
for {
h, err := tr.Next()
if err == io.EOF {
return out
}
if err != nil {
t.Fatalf("tar: %v", err)
}
b, _ := io.ReadAll(tr)
out[h.Name] = string(b)
}
}
func TestCmdExportWorld(t *testing.T) {
root := t.TempDir()
if err := os.MkdirAll(filepath.Join(root, "world", "region"), 0o755); err != nil {
t.Fatal(err)
}
for name, body := range map[string]string{"server.properties": "motd=hi\n", "world/region/r.0.0.mca": "chunks"} {
if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
code := cmdExport([]string{"--mode", "world", "--server", "survival", "--target-url", rcv.srv.URL + "/api/v1/internal/exports/ab",
"--worlds-root", root}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
r := rcv.req
if r.Method != http.MethodPut || r.URL.Path != "/api/v1/internal/exports/ab" || r.Header.Get("Authorization") != "Bearer tok" ||
r.Header.Get("Content-Type") != "application/gzip" || r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
}
got := tarEntries(t, rcv.body)
want := map[string]string{"server.properties": "motd=hi\n", "world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks"}
if len(got) != len(want) {
t.Fatalf("archive holds %v, want %v", got, want)
}
for name, body := range want {
if b, ok := got[name]; !ok || b != body {
t.Errorf("%s = %q (present %v), want %q", name, b, ok, body)
}
}
if !strings.Contains(stdout.String(), "server=survival mode=world downloaded") {
t.Errorf("stdout = %q", stdout.String())
}
}
// A world that cannot be read must never reach felis-api as a complete body:
// the chunked upload is cut off, so felis-api aborts the browser's download.
func TestCmdExportWorldReadErrorAbortsTheUpload(t *testing.T) {
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
if code != 1 {
t.Fatalf("exit %d, want 1", code)
}
select {
case <-rcv.served:
if rcv.readErr == nil {
t.Fatalf("felis-api read a complete %d-byte body from an unreadable world", len(rcv.body))
}
case <-time.After(2 * time.Second): // the request never reached the handler
}
}
func TestCmdExportBackup(t *testing.T) {
root := t.TempDir()
archive := bytes.Repeat([]byte("felis"), 10_000)
ref := filepath.Join(root, "survival-1.tar.gz")
if err := os.WriteFile(ref, archive, 0o600); err != nil {
t.Fatal(err)
}
args := func(url, ref string) []string {
return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
}
t.Run("hands the archive over with its length", func(t *testing.T) {
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 0 {
t.Fatalf("exit %d, stderr %q", code, stderr.String())
}
if rcv.req.ContentLength != int64(len(archive)) || !bytes.Equal(rcv.body, archive) || rcv.req.Header.Get("Authorization") != "Bearer tok" {
t.Fatalf("got %d bytes (length %d, auth %q), want the %d archive bytes",
len(rcv.body), rcv.req.ContentLength, rcv.req.Header.Get("Authorization"), len(archive))
}
})
t.Run("a ref outside the backup root is refused before any request", func(t *testing.T) {
outside := filepath.Join(t.TempDir(), "secret.tar.gz")
if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil {
t.Fatal(err)
}
rcv := receiveExport(t, noContent)
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(rcv.srv.URL, outside), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if n := rcv.hits.Load(); n != 0 {
t.Fatalf("felis-api got %d requests", n)
}
})
t.Run("a refusal exits 1 with felis-api's reason", func(t *testing.T) {
rcv := receiveExport(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusGone)
io.WriteString(w, `{"error":{"code":"export_expired","message":"nobody opened the download"}}`)
})
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if got := stderr.String(); got != "felis export: felis-api answered 410 Gone: nobody opened the download\n" {
t.Fatalf("stderr = %q", got)
}
})
// The request carries the token and the internal face never redirects, so a
// redirect is refused rather than followed with the token attached. A 302 or
// 303 is the one net/http would follow on its own (as a GET, and to the same
// host with the Authorization header still on it).
for _, status := range []int{http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
t.Run("a redirect is not followed: "+strconv.Itoa(status), func(t *testing.T) {
elsewhere := receiveExport(t, noContent)
redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.srv.URL, status))
defer redirecting.Close()
t.Setenv(worldexport.TokenEnv, "tok")
var stdout, stderr bytes.Buffer
if code := cmdExport(args(redirecting.URL, ref), &stdout, &stderr); code != 1 {
t.Fatalf("exit %d, want 1", code)
}
if n := elsewhere.hits.Load(); n != 0 {
t.Fatalf("the redirect target got %d requests", n)
}
if got, want := stderr.String(), "felis export: felis-api answered "+strconv.Itoa(status)+" "+http.StatusText(status)+"\n"; got != want {
t.Fatalf("stderr = %q, want %q", got, want)
}
})
}
}
func TestCmdExportUsage(t *testing.T) {
for name, tc := range map[string]struct {
token string
args []string
}{
"no token": {"", []string{"--mode", "world", "--target-url", "http://api/x"}},
"no target": {"tok", []string{"--mode", "world"}},
"unknown mode": {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
} {
t.Run(name, func(t *testing.T) {
t.Setenv(worldexport.TokenEnv, tc.token)
var stdout, stderr bytes.Buffer
if code := cmdExport(tc.args, &stdout, &stderr); code != 2 {
t.Fatalf("exit %d, want 2; stderr %q", code, stderr.String())
}
})
}
}
// TestCmdExportWiring: the Job's `felis export` reaches cmdExport, and
// felis-api's executor mounts the backup store at the path the archives were
// written under, since a backup's ref is an absolute path there.
func TestCmdExportWiring(t *testing.T) {
t.Setenv(worldexport.TokenEnv, "")
var stdout, stderr bytes.Buffer
if code := run([]string{"export"}, &stdout, &stderr); code != 2 ||
stderr.String() != "felis export: --target-url and "+worldexport.TokenEnv+" are required\n" {
t.Fatalf("felis export = %d, stderr %q", code, stderr.String())
}
cfg := &config.Config{}
cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups"
want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"}
if got := exportConfig(cfg, "felis:1", "felis-backups"); got != want {
t.Fatalf("exportConfig = %+v, want %+v", got, want)
}
}
+2
View File
@@ -22,6 +22,7 @@ Commands:
backup Archive a world into the backup store and record it (internal Job entrypoint)
backup-now Archive every user server's world now, one at a time (or the named ones; -stop stops running ones first; prints the plan, -yes applies; requires root/sudo)
files List, read, write, mkdir, delete, rename or upload one path in a stopped server's world (internal Job entrypoint)
export Archive a stopped server's world, or read one of its backups, and hand it to felis-api for download (internal Job entrypoint)
egress-gate Hold a build or game server pod until its egress NetworkPolicy is enforced (internal init container entrypoint)
fetch-context Fetch and extract a submission's build context (internal Job entrypoint)
scan-gate Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
@@ -66,6 +67,7 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
"backup": cmdBackup,
"backup-now": cmdBackupNow,
"files": cmdFiles,
"export": cmdExport,
"egress-gate": cmdEgressGate,
"fetch-context": cmdFetchContext,
"scan-gate": cmdScanGate,
+261 -5
View File
@@ -122,7 +122,7 @@ tags:
- name: console
description: Read (SSE) and write (RCON) server console (external face, app tier).
- name: backups
description: World backup listing and restore (external face, app tier).
description: World backup listing, restore and export (external face, app tier).
- name: schedules
description: Scheduled tasks of your own servers (external face, app tier).
- name: account
@@ -731,6 +731,34 @@ components:
type: integer
description: World entries the archive could not hold (symbolic links, devices, sockets). Omitted when zero.
ExportTicket:
type: object
description: >-
An export just started (internal/api/exports.go exportTicketView). The
ticket opens GET /exports/{ticket} and its download for the user who
started it, and nobody else.
required: [ticket, state, filename]
properties:
ticket: { type: string, description: 64 hex characters. }
state: { type: string, const: pending }
filename: { type: string, description: What the download saves as. }
ExportStatus:
type: object
description: Where an export stands (internal/api/exports.go exportStatusView).
required: [state]
properties:
state:
type: string
enum: [pending, ready, failed]
description: >-
pending while its Job starts; ready once the archive waits for the
download, which must begin within 90 seconds; failed when the Job
died first.
message:
type: string
description: Why a failed export's Job died. Omitted otherwise.
Build:
type: object
description: One image build (internal/build Build).
@@ -1298,6 +1326,52 @@ paths:
'404':
$ref: '#/components/responses/NotFound'
/api/v1/internal/exports/{id}:
put:
tags: [backups]
operationId: internalExportUpload
summary: Hand one export's archive over for download (one-time bearer token).
description: >-
The export Job PUTs the tar.gz here, chunked for a world and with its
Content-Length for a backup. The Job holds no service token, so the route
is public on the internal face and the bearer token minted with the
export is the whole check; an unknown id, a wrong or missing token and a
token already used are all the same 404. The request then waits, body
unread, up to 90 seconds for the owner's browser to open the download,
and is read at the browser's pace: the 16 KiB/s minimum body rate does
not apply, and the body fails only after 2 minutes without a byte. It
answers once the download has ended.
x-felis-face: [internal]
x-felis-tier: public
security: []
parameters:
- { name: id, in: path, required: true, schema: { type: string } }
- name: Authorization
in: header
required: true
description: Bearer followed by the token minted with the export.
schema: { type: string }
requestBody:
required: true
content:
application/gzip:
schema: { type: string, format: binary }
responses:
'204':
$ref: '#/components/responses/NoContent'
'404':
$ref: '#/components/responses/NotFound'
'409':
description: The backup did not match the sha256 recorded when it was written, and the download was aborted (backup_corrupt).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'410':
description: Nobody opened the download within 90 seconds, or the browser left before the archive ended (export_expired).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
/api/v1/internal/servers/{name}/join-event:
post:
tags: [servers-internal]
@@ -4066,14 +4140,196 @@ paths:
'507':
$ref: '#/components/responses/InsufficientStorage'
# ------------------------------------------------------ world export (app) ---
/api/v1/servers/{name}/backups/{id}/export:
post:
tags: [backups]
operationId: exportBackup
summary: Start downloading one backup (owner-or-admin plus a former-owner match).
description: >-
Starts a Job that reads the archive from the backup store and hands it to
felis-api, which streams it to the browser (poll GET /exports/{ticket},
then open its download). The archive is checked against the sha256
recorded when it was written as it streams; a mismatch aborts the
download. A user gets 404 for a backup outside their scope, as their
list never shows it. One export per user at a time, 2 across the
install, 6 per user per hour.
x-felis-face: [external]
x-felis-tier: app
security: [{ sessionCookie: [] }]
parameters:
- { name: name, in: path, required: true, schema: { type: string } }
- { name: id, in: path, required: true, schema: { type: string } }
responses:
'202':
description: Export started.
content:
application/json:
schema: { $ref: '#/components/schemas/ExportTicket' }
'400':
description: Malformed server name (bad_name).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'401':
$ref: '#/components/responses/Unauthorized'
'403':
$ref: '#/components/responses/Forbidden'
'404':
description: Unknown server, or no present backup with this id in the caller's scope (no_backup).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'409':
description: The backup failed a read-back (backup_corrupt).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'429':
description: An export limit is reached (export_busy); Retry-After gives the seconds to wait.
headers:
Retry-After:
schema: { type: integer }
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'503':
$ref: '#/components/responses/ServiceUnavailable'
/api/v1/servers/{name}/world/export:
post:
tags: [backups]
operationId: exportWorld
summary: Start downloading a stopped server's world as it is now (owner-or-admin).
description: >-
Starts a Job that archives the server's data volume, read-only, and hands
it to felis-api, which streams it to the browser (poll GET
/exports/{ticket}, then open its download). The server must be fully
stopped, and it cannot start until the download has ended or the Job's
2 hour deadline passes. Same limits as a backup export.
x-felis-face: [external]
x-felis-tier: app
security: [{ sessionCookie: [] }]
parameters:
- { name: name, in: path, required: true, schema: { type: string } }
responses:
'202':
description: Export started.
content:
application/json:
schema: { $ref: '#/components/schemas/ExportTicket' }
'400':
description: Malformed server name (bad_name).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'401':
$ref: '#/components/responses/Unauthorized'
'403':
$ref: '#/components/responses/Forbidden'
'404':
description: Unknown server.
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'409':
description: Server is not stopped (not_stopped), has no world volume yet (no_world_volume), or a restore, backup, file write or export already holds its world volume (maintenance_in_progress).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'429':
description: An export limit is reached (export_busy); Retry-After gives the seconds to wait.
headers:
Retry-After:
schema: { type: integer }
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'503':
$ref: '#/components/responses/ServiceUnavailable'
/api/v1/exports/{ticket}:
get:
tags: [backups]
operationId: exportStatus
summary: Where an export stands (the user who started it only).
description: >-
The panel polls this until the state reads ready, then opens the
download. Another user's ticket is 404, as an unknown one is.
x-felis-face: [external]
x-felis-tier: app
security: [{ sessionCookie: [] }]
parameters:
- { name: ticket, in: path, required: true, schema: { type: string } }
responses:
'200':
description: The export's state.
content:
application/json:
schema: { $ref: '#/components/schemas/ExportStatus' }
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'410':
description: The export was downloaded, or expired before it was (export_expired).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
/api/v1/exports/{ticket}/download:
get:
tags: [backups]
operationId: exportDownload
summary: Download a ready export (once, by the user who started it).
description: >-
The first request spends the ticket, whatever becomes of it. The archive
streams as the Job sends it, with Content-Length when it is known; a
download that cannot finish (the Job died, or a backup did not match its
recorded sha256) is cut off, so the browser reports it failed. HEAD is
refused, since it would spend the ticket on no body.
x-felis-face: [external]
x-felis-tier: app
security: [{ sessionCookie: [] }]
parameters:
- { name: ticket, in: path, required: true, schema: { type: string } }
responses:
'200':
description: The tar.gz, as an attachment.
headers:
Content-Disposition:
schema: { type: string }
content:
application/gzip:
schema: { type: string, format: binary }
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'405':
description: A method other than GET (method_not_allowed).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'409':
description: The export is still pending (export_not_ready).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'410':
description: The export was downloaded, failed, or expired before it was (export_expired).
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
# -------------------------------------------------- async job status (app) ---
/api/v1/servers/{name}/jobs:
get:
tags: [backups]
operationId: listServerJobs
summary: Latest async world operations (backup/restore) for a server (owner-or-admin).
summary: Latest async world operations (backup, restore, export) for a server (owner-or-admin).
description: >-
Backup and restore run as cluster Jobs, so a 202 that later failed left
Backup, restore and export run as cluster Jobs, so a 202 that later failed left
its only trace in the Job object. This route projects the newest such
Jobs, newest first, so failures are observable without kubectl. State is
"running" | "succeeded" | "failed".
@@ -4084,7 +4340,7 @@ paths:
- { name: name, in: path, required: true, schema: { type: string } }
responses:
'200':
description: The server's newest backup/restore jobs.
description: The server's newest backup, restore and export jobs.
content:
application/json:
schema:
@@ -4099,7 +4355,7 @@ paths:
required: [name, kind, state]
properties:
name: { type: string }
kind: { type: string, enum: [backup, restore] }
kind: { type: string, enum: [backup, restore, export_world, export_backup] }
state: { type: string, enum: [running, succeeded, failed] }
message: { type: string }
started_at: { type: string, format: date-time }
+70 -5
View File
@@ -346,9 +346,9 @@ grade the two halves separately.
A server's world volume is ReadWriteOnce, and on a single node RWO lets a game
pod and a restore Job mount it side by side. So felis-api serialises them per
server: a restore, a backup, or a file change takes the world, and until its Job
finishes every wake (panel or join) and every other world operation on that
server gets `409 maintenance_in_progress`. File reads and listings never hold
server: a restore, a backup, a world download or a file change takes the world,
and until its Job finishes every wake (panel or join) and every other world
operation on that server gets `409 maintenance_in_progress`. File reads and listings never hold
it. The operator applies the same rule when `desiredState` is flipped to
`Running` by anything other than felis-api: the StatefulSet is not scaled up,
and the `Ready` condition reads `MaintenanceInProgress` until the Job ends.
@@ -356,9 +356,11 @@ and the `Ready` condition reads `MaintenanceInProgress` until the Job ends.
What holds the world, in order:
1. An unfinished Job labelled `felis.lolicon.best/server=<name>` with
`app.kubernetes.io/managed-by` `felis-restore`, `felis-backup`, or
`app.kubernetes.io/managed-by` `felis-restore`, `felis-backup`,
`felis-files` with any `felis.lolicon.best/files-mode` but `list` or `read`
(a save, new file, new folder, rename, delete or upload; §18):
(a save, new file, new folder, rename, delete or upload; §18), or
`felis-export` with any `felis.lolicon.best/export-mode` but `backup` (a
world download, held until the download ends; §10):
```sh
kubectl -n minecraft get jobs -l felis.lolicon.best/server=<name>
@@ -1213,6 +1215,68 @@ sudo k3s kubectl -n felis exec deploy/felis-postgres -c postgres -- psql -U post
[GO-TESTED: `TestDeleteBackup`, `TestExpiredBackupsDeleted`,
`TestSyncExpiresOnlyPastRetention`; PG-TESTED: `TestOwnerDeletedBackup`]
### Downloading a backup or the world (export)
The download button on a backup row (`POST /api/v1/servers/{name}/backups/{id}/export`)
and "Export world" in the page header (`POST /api/v1/servers/{name}/world/export`)
hand the owner a `.tar.gz`. Who may download a backup follows delete and restore:
an admin any, a user only one of a world they owned (`404 no_backup` otherwise), a
backup of another server is `403`, and a corrupt one `409 backup_corrupt`. The
world export needs the server stopped (`409 not_stopped`) and a world volume
(`409 no_world_volume`), and it takes the world like a backup does (§3b), so
nothing starts the server until the download ends. The audit actions are
`backup.export` and `world.export`.
Neither archive is staged anywhere. felis-api starts a `felis-export` Job in the
server namespace (managed-by `felis-export`, label `felis.lolicon.best/export-mode`
`world` or `backup`), which mounts the world or the backup volume read-only and
PUTs the archive to felis-api's internal face with a one-time token; felis-api
streams it straight to the browser. The panel polls `GET /api/v1/exports/{ticket}`
until it reads `ready`, then opens `GET /api/v1/exports/{ticket}/download` as a
plain link. The ticket belongs to the user who asked, opens the download once,
and has these clocks:
| Limit | Value | When it runs out |
|---|---|---|
| Job reaching felis-api | 10 min | the ticket answers `410 export_expired` |
| Browser opening the download once `ready` | 90 s | the Job's upload gets `410`, the ticket is spent |
| Either end sending nothing mid-transfer | 2 min | the download is cut off |
| The Job as a whole | 2 h (`activeDeadlineSeconds`) | Kubernetes ends it |
At most one export per user runs at a time, two across the platform, and six per
user per hour; past any of them the start is `429 export_busy` with `Retry-After`.
A deployment without `FELIS_IMAGE` and `FELIS_BACKUP_PVC` answers
`503 export_unavailable` (felis-api logs `world export disabled` at start).
A backup download carries the archive's length and is checked against the sha256
recorded when it was written. The last chunk is held back until the digest
matches, so a mismatch cuts the download off short, the browser shows it failed,
and the Job exits with `409 backup_corrupt`. A digest mismatch does not mark the
backup corrupt; the reaper's next read-back does. A world download has no length
and no digest: a read error in the Job cuts it off the same way.
Why an export failed shows under Recent operations as the Job's last line. A Job
that fails before it reaches felis-api (an archive missing from the backup
volume, say) also ends the panel's wait with that line. The rest happen after
the panel has handed the download to the browser:
- `felis-api answered 410 Gone: this export has expired …`: nobody opened the
download within 90 s (the tab was closed, or the browser blocked the download).
- `felis-api answered 410 Gone: the download ended before the archive did …`:
the browser cancelled or lost the connection mid-way.
- `felis-api answered 404 Not Found: no such export`: felis-api restarted since
the export began, or the ticket had already run out. Tickets live only in
felis-api's memory, so start the export again.
A world export whose Job never reaches felis-api keeps the world until the Job's
deadline; `kubectl -n minecraft delete job -l app.kubernetes.io/managed-by=felis-export,felis.lolicon.best/server=<name>`
releases it at once. [GO-TESTED: `TestExportBackupGate`, `TestExportWorldGate`,
`TestExportLimits`, `TestExportRendezvous`, `TestExportExpiry`,
`TestExportStatusReportsFailedJob`, `TestExportRegistryRaces`,
`TestExportBackupDigest`, `TestExportUploadPace`, `TestK8sExportJobs`,
`TestExportJobIsolation`, `TestExportJobDefaults`, `TestCmdExportWorld`,
`TestCmdExportBackup`, `TestCmdExportWiring`]
### Every world at once: `felis backup-now`
A world lives only in its volume, and the off-site copy holds only its archives.
@@ -3243,6 +3307,7 @@ PG-TESTED: `TestScheduleStoreRunCAS`, `TestDueSchedules`, `TestSchedulesFollowTh
| Registry push/pull unreachable | §9 |
| World deleted unexpectedly / backup skipped | §10 |
| Reaper `awaiting_stop` stays above 0; `corrupt=` / backup shown as damaged; `orphan_archives` | §10 |
| Backup or world download refused or cut off (`export_busy`, `export_expired`, `export_unavailable`) | §10 |
| Idle auto-stop not firing; player count 0; `PlayersCounted=False` | §11 |
| A config field seems ignored | §12 |
| PVC left behind after delete | §13 |
+24
View File
@@ -98,6 +98,12 @@ type API struct {
FileStage *fileedit.Stage
InternalBaseURL string
// Exporter starts the Job behind a world or backup download (exports.go),
// which PUTs the archive to InternalBaseURL. Optional like Restorer: the
// export routes report 503 unless both are set, after the owner-or-admin,
// backup and stopped gates.
Exporter Exporter
// Schedules stores the servers' scheduled tasks (schedules.go), which
// RunSchedules fires. Optional: when nil the schedule routes report 503 and
// RunSchedules does nothing.
@@ -222,6 +228,9 @@ type API struct {
streamCapOnce sync.Once
streamCap *streamLimiter
exportsOnce sync.Once
exports *exportRegistry
authDoorOnce sync.Once
authDoorBuckets *bucketSet
mailOnce sync.Once
@@ -459,6 +468,13 @@ func (a *API) internalAPIRoutes() []apiRoute {
// because that Job holds no service token; the one-time bearer token minted
// with the upload is the check (handlers_files.go).
{Method: "GET", Pattern: "/api/v1/internal/file-uploads/{id}", Public: true, h: a.handleInternalFileUpload},
// An export Job's archive, held open until the owner's browser downloads
// it. Public for the same reason as file uploads: the Job holds no service
// token, and the one-time bearer token minted with the export is the check
// (exports.go). The body is read at the browser's pace, so the handler
// lifts the minimum-rate body deadline and applies its own stall bound.
{Method: "PUT", Pattern: "/api/v1/internal/exports/{id}", Public: true, h: a.handleInternalExportUpload},
}
}
@@ -558,6 +574,14 @@ func (a *API) externalAPIRoutes() []apiRoute {
{Method: "GET", Pattern: "/api/v1/servers/{name}/jobs", h: a.handleServerJobs},
{Method: "POST", Pattern: "/api/v1/servers/{name}/restore-backup", h: a.handleRestoreBackup},
{Method: "POST", Pattern: "/api/v1/servers/{name}/backup", h: a.handleBackupNow},
// World export (exports.go): download a backup, or a stopped server's world
// as it is now, straight to the browser. App-tier with the restore gate
// inside each start route; the ticket routes answer only the user who
// started the export.
{Method: "POST", Pattern: "/api/v1/servers/{name}/backups/{id}/export", h: a.handleExportBackup},
{Method: "POST", Pattern: "/api/v1/servers/{name}/world/export", h: a.handleExportWorld},
{Method: "GET", Pattern: "/api/v1/exports/{ticket}", h: a.handleExportStatus},
{Method: "GET", Pattern: "/api/v1/exports/{ticket}/download", h: a.handleExportDownload},
// Server file manager: list, read, write, make a folder, delete, rename and
// upload in a STOPPED server's world volume (handlers_files.go). App-tier,
// exactly like the backup pair above and for the same reason — every route
+4 -3
View File
@@ -255,8 +255,9 @@ type fakeSession struct {
// fakeBackup mirrors a world_backups row: the client-facing view plus the
// server-side backup_ref the list queries never expose.
type fakeBackup struct {
view BackupView
ref string
view BackupView
ref string
sha256 string
}
// fakeLinkCode mirrors an account_link_codes row.
@@ -1051,7 +1052,7 @@ func (f *fakeRepo) BackupByID(_ context.Context, id string) (*BackupRecord, erro
return &BackupRecord{
ID: b.view.ID, ServerName: b.view.ServerName,
FormerOwner: b.view.FormerOwner, BackupRef: b.ref,
SizeBytes: b.view.SizeBytes, Corrupt: b.view.Corrupt,
SizeBytes: b.view.SizeBytes, Corrupt: b.view.Corrupt, SHA256: b.sha256,
}, nil
}
}
+677
View File
@@ -0,0 +1,677 @@
package api
import (
"context"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"log"
"mime"
"net/http"
"strconv"
"strings"
"sync"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/worldexport"
)
// World export. The owner downloads a tar.gz of their world, either as it is
// now (the server stopped) or as one of its backups, straight into the browser:
//
// 1. POST /servers/{name}/world/export or /servers/{name}/backups/{id}/export
// checks the caller and the server, admits the export against the limits
// below and starts a one-shot felis-export Job (internal/worldexport) that
// reads the world or the archive read-only. It answers 202 with a ticket:
// 256 random bits, good for the caller who started it and nobody else.
// 2. The Job PUTs the archive to the internal face (PUT
// /api/v1/internal/exports/{id} with the one-time token it was started
// with), and that request waits, body unread, for the browser.
// 3. The panel polls GET /exports/{ticket} until it reads ready, then points a
// hidden <a download> at GET /exports/{ticket}/download, which claims the
// waiting PUT and copies its body into the response 32 KiB at a time. The
// archive passes through felis-api's memory once and never touches a disk
// it owns, and the Job moves at the browser's pace.
//
// A backup is checked against the sha256 recorded when it was written as it
// streams, and the last read is held back until the digest is known: a mismatch
// aborts the response, so the browser reports a failed download and never keeps
// a complete-looking corrupt file, and the Job is told backup_corrupt.
//
// Tickets live in felis-api's memory. A restart forgets them, and a Job that
// then PUTs finds nothing and fails, which the jobs list shows.
// Exporter starts the Job that archives a world or a backup and hands it to the
// internal upload route (internal/worldexport). Optional: when nil the export
// routes answer 503.
type Exporter interface {
Start(ctx context.Context, r worldexport.Request) (job string, err error)
}
// Limits on exports. Each one keeps a Job, a connection and a 64 KiB copy
// buffer alive for as long as a download takes, and a world export also keeps
// its server from starting.
const (
exportMaxActive = 2 // admitted and not yet over, install-wide
exportMaxPerUser = 1
exportPerHour = 6 // started by one user in any hour
exportCopyBuffer = 32 << 10
)
// Timings. Vars only so a test can shrink them.
var (
// exportClaimTTL is how long the Job's upload waits for the browser.
exportClaimTTL = 90 * time.Second
// exportPendingTTL is how long an export may take to reach ready: the Job
// is scheduled, pulls its image and connects well inside it.
exportPendingTTL = 10 * time.Minute
// exportStall is the longest either end may go without moving a byte.
exportStall = 2 * time.Minute
// exportKeepSpent is how long a finished ticket still answers 410
// export_expired before it reads as unknown.
exportKeepSpent = 10 * time.Minute
)
// States of an export.
const (
exportPending = "pending" // the Job has not connected yet
exportReady = "ready" // its upload is waiting for the browser
exportStreaming = "streaming" // the browser is downloading
exportFailed = "failed" // the Job died before it connected
exportSpent = "spent" // downloaded, or given up
)
// exportTicketView answers both export routes.
type exportTicketView struct {
Ticket string `json:"ticket"`
State string `json:"state"`
Filename string `json:"filename"`
}
// exportStatusView answers GET /exports/{ticket}.
type exportStatusView struct {
State string `json:"state"`
Message string `json:"message,omitempty"`
}
var errExportDigest = errors.New("the archive does not match the sha256 recorded when it was written")
func errExportExpired() error {
return newError(http.StatusGone, "export_expired", "this export has expired or was already downloaded; start a new one")
}
func errNoExport() error {
return newError(http.StatusNotFound, "not_found", "no such export")
}
type exportEntry struct {
ticket string
id string
tokenHash [32]byte
userID string
server string
mode string
sha256 string // what a backup must hash to; empty for a world
filename string
job string
state string
message string
at time.Time // when it entered its state
upload *exportUpload
}
// exportUpload is the Job's PUT, parked until a browser claims it.
type exportUpload struct {
body io.Reader
size int64 // -1 when the Job streams it chunked
claimed chan struct{}
done chan error // how the download ended; buffered
}
func (e *exportEntry) active() bool {
return e.state == exportPending || e.state == exportReady || e.state == exportStreaming
}
type exportRegistry struct {
mu sync.Mutex
byTicket map[string]*exportEntry
byID map[string]*exportEntry
starts map[string][]time.Time // per user, oldest first
}
func (a *API) exportTickets() *exportRegistry {
a.exportsOnce.Do(func() {
a.exports = &exportRegistry{
byTicket: map[string]*exportEntry{},
byID: map[string]*exportEntry{},
starts: map[string][]time.Time{},
}
})
return a.exports
}
// sweepLocked expires what has waited too long and forgets what ended long ago.
func (g *exportRegistry) sweepLocked(now time.Time) {
for t, e := range g.byTicket {
switch {
case e.state == exportPending && now.Sub(e.at) >= exportPendingTTL:
e.state, e.at = exportSpent, now
case !e.active() && now.Sub(e.at) >= exportKeepSpent:
delete(g.byTicket, t)
delete(g.byID, e.id)
}
}
for u, ts := range g.starts {
for len(ts) > 0 && now.Sub(ts[0]) >= time.Hour {
ts = ts[1:]
}
if len(ts) == 0 {
delete(g.starts, u)
} else {
g.starts[u] = ts
}
}
}
func randomHex(n int) string {
b := make([]byte, n)
_, _ = rand.Read(b)
return hex.EncodeToString(b)
}
// admit reserves the export e describes (its user, server, mode, filename and
// digest) and returns its upload token, or refuses it with export_busy.
func (g *exportRegistry) admit(e *exportEntry, now time.Time) (string, error) {
g.mu.Lock()
defer g.mu.Unlock()
g.sweepLocked(now)
active, mine := 0, 0
for _, o := range g.byTicket {
if o.active() {
active++
if o.userID == e.userID {
mine++
}
}
}
switch starts := g.starts[e.userID]; {
case mine >= exportMaxPerUser:
return "", newError(http.StatusTooManyRequests, "export_busy",
"you already have an export in progress; download it or let it expire first").retryAfter(exportClaimTTL)
case active >= exportMaxActive:
return "", newError(http.StatusTooManyRequests, "export_busy",
"%d exports are already in progress; retry in a few minutes", exportMaxActive).retryAfter(time.Minute)
case len(starts) >= exportPerHour:
return "", newError(http.StatusTooManyRequests, "export_busy",
"you have started %d exports in the last hour; retry later", exportPerHour).retryAfter(starts[0].Add(time.Hour).Sub(now))
}
token := randomHex(32)
e.ticket, e.id, e.tokenHash = randomHex(32), randomHex(8), sha256.Sum256([]byte(token))
e.state, e.at = exportPending, now
g.byTicket[e.ticket] = e
g.byID[e.id] = e
g.starts[e.userID] = append(g.starts[e.userID], now)
return token, nil
}
// drop forgets an export that never got its Job, and gives its start back.
func (g *exportRegistry) drop(e *exportEntry) {
g.mu.Lock()
defer g.mu.Unlock()
delete(g.byTicket, e.ticket)
delete(g.byID, e.id)
ts := g.starts[e.userID]
for i := len(ts) - 1; i >= 0; i-- {
if ts[i].Equal(e.at) {
g.starts[e.userID] = append(ts[:i:i], ts[i+1:]...)
break
}
}
}
func (g *exportRegistry) started(e *exportEntry, job string) {
g.mu.Lock()
defer g.mu.Unlock()
e.job = job
}
// lookup returns a copy of userID's export by its ticket. Another user's reads
// as unknown, and a spent one as export_expired.
func (g *exportRegistry) lookup(ticket, userID string, now time.Time) (exportEntry, error) {
g.mu.Lock()
defer g.mu.Unlock()
g.sweepLocked(now)
e, ok := g.byTicket[ticket]
switch {
case !ok || e.userID != userID:
return exportEntry{}, errNoExport()
case e.state == exportSpent || e.state == exportStreaming:
return exportEntry{}, errExportExpired()
}
return *e, nil
}
// fail records that the export's Job died before it connected, and reports
// false when the export has moved on since.
func (g *exportRegistry) fail(ticket, message string, now time.Time) bool {
g.mu.Lock()
defer g.mu.Unlock()
e, ok := g.byTicket[ticket]
if !ok || e.state != exportPending {
return false
}
e.state, e.message, e.at = exportFailed, message, now
return true
}
// arrive parks the Job's upload on the export the id and token open. Unknown
// ids, wrong tokens and exports past pending all read the same.
func (g *exportRegistry) arrive(id, token string, up *exportUpload, now time.Time) bool {
g.mu.Lock()
defer g.mu.Unlock()
g.sweepLocked(now)
e, ok := g.byID[id]
if !ok {
return false
}
sum := sha256.Sum256([]byte(token))
if subtle.ConstantTimeCompare(sum[:], e.tokenHash[:]) != 1 || e.state != exportPending {
return false
}
e.state, e.at, e.upload = exportReady, now, up
return true
}
// giveUp spends an export whose upload nobody claimed, and reports false when a
// claim got there first.
func (g *exportRegistry) giveUp(up *exportUpload, now time.Time) bool {
g.mu.Lock()
defer g.mu.Unlock()
for _, e := range g.byID {
if e.upload == up && e.state == exportReady {
e.state, e.at = exportSpent, now
return true
}
}
return false
}
// claim hands userID's ready export to its download: from here on the ticket is
// spent whatever happens to the download.
func (g *exportRegistry) claim(ticket, userID string, now time.Time) (exportEntry, error) {
g.mu.Lock()
defer g.mu.Unlock()
g.sweepLocked(now)
e, ok := g.byTicket[ticket]
switch {
case !ok || e.userID != userID:
return exportEntry{}, errNoExport()
case e.state == exportPending:
return exportEntry{}, newError(http.StatusConflict, "export_not_ready",
"the export is still being prepared; wait until its status reads ready")
case e.state != exportReady:
return exportEntry{}, errExportExpired()
}
e.state, e.at = exportStreaming, now
close(e.upload.claimed)
return *e, nil
}
func (g *exportRegistry) finish(ticket string, now time.Time) {
g.mu.Lock()
defer g.mu.Unlock()
if e, ok := g.byTicket[ticket]; ok {
e.state, e.at = exportSpent, now
}
}
// handleExportBackup starts the download of one backup (POST
// /api/v1/servers/{name}/backups/{id}/export). The gate is restore's and
// delete's together:
//
// ① name validation; ② an unknown server is 404; ③ owner-or-admin, else 403
// ④ the backup must exist, and a non-admin must be its former owner: any
// other id reads as unknown (404 no_backup), as it does in their list
// ⑤ cross-server guard: the backup must be this server's (403)
// ⑥ an archive that failed its read-back is 409 backup_corrupt
//
// It needs no stopped server and takes no world lock: the Job reads the backup
// store only.
func (a *API) handleExportBackup(w http.ResponseWriter, r *http.Request) {
p := principalFromContext(r.Context())
name, rec, ok := a.exportGate(w, r)
if !ok {
return
}
backup, err := a.Repo.BackupByID(r.Context(), r.PathValue("id"))
if err == nil && !p.IsAdmin() && backup.FormerOwner != p.UserID {
err = ErrNotFound
}
if errors.Is(err, ErrNotFound) {
writeError(w, r, newError(http.StatusNotFound, "no_backup", "no matching backup exists"))
return
}
if err != nil {
writeError(w, r, err)
return
}
if backup.ServerName != name {
writeError(w, r, errForbidden)
return
}
if backup.Corrupt {
writeError(w, r, newError(http.StatusConflict, "backup_corrupt",
"this backup did not read back intact and cannot be exported; pick another"))
return
}
if a.Exporter == nil || a.InternalBaseURL == "" {
writeError(w, r, errExportUnavailable())
return
}
e := &exportEntry{userID: p.UserID, server: name, mode: worldexport.ModeBackup,
filename: fmt.Sprintf("%s-backup-%s.tar.gz", name, backup.ID), sha256: backup.SHA256}
token, err := a.exportTickets().admit(e, a.now())
if err != nil {
writeError(w, r, err)
return
}
if !a.startExport(w, r, e, token, backup.BackupRef) {
return
}
a.auditEntry(r, AuditEntry{Actor: auditActor(p), ActorUserID: p.UserID, Action: "backup.export", ServerName: rec.Name,
Payload: auditPayload(map[string]any{"backup_id": backup.ID, "size_bytes": backup.SizeBytes})})
writeJSON(w, http.StatusAccepted, exportTicketView{Ticket: e.ticket, State: exportPending, Filename: e.filename})
}
// handleExportWorld starts the download of a stopped server's world as it is
// now (POST /api/v1/servers/{name}/world/export). The gate is a backup's: owner
// or admin, the server fully stopped, a world volume to read, and the
// world-volume lock, which the export Job then holds (as KindExport) until the
// download ends, so the server cannot start under it and tear the archive.
func (a *API) handleExportWorld(w http.ResponseWriter, r *http.Request) {
p := principalFromContext(r.Context())
name, rec, ok := a.exportGate(w, r)
if !ok {
return
}
const notStopped = "stop the server before exporting its world"
info, err := a.Cluster.GetServer(r.Context(), name)
if err != nil {
a.writeLookupError(w, r, err)
return
}
if info.Ready || info.DesiredState != string(v1alpha1.DesiredStopped) {
writeError(w, r, newError(http.StatusConflict, "not_stopped", notStopped))
return
}
if exists, err := a.Cluster.WorldVolumeExists(r.Context(), name); err != nil {
writeError(w, r, err)
return
} else if !exists {
writeError(w, r, errNoWorldVolume())
return
}
if a.Exporter == nil || a.InternalBaseURL == "" {
writeError(w, r, errExportUnavailable())
return
}
reg := a.exportTickets()
e := &exportEntry{userID: p.UserID, server: name, mode: worldexport.ModeWorld,
filename: fmt.Sprintf("%s-world-%s.tar.gz", name, a.now().UTC().Format("20060102-150405"))}
token, err := reg.admit(e, a.now())
if err != nil {
writeError(w, r, err)
return
}
release, ok := a.acquireWorld(w, r, name, maintenance.KindExport, notStopped)
if !ok {
reg.drop(e)
return
}
defer release()
if !a.startExport(w, r, e, token, "") {
return
}
a.auditEntry(r, AuditEntry{Actor: auditActor(p), ActorUserID: p.UserID, Action: "world.export", ServerName: rec.Name})
writeJSON(w, http.StatusAccepted, exportTicketView{Ticket: e.ticket, State: exportPending, Filename: e.filename})
}
func errExportUnavailable() error {
return newError(http.StatusServiceUnavailable, "export_unavailable", "world export is not configured")
}
// exportGate is the front half both export routes share: a valid name, a known
// server, and a caller who owns it or is an admin. The ticket is bound to the
// caller's account, so a principal without one cannot start an export.
func (a *API) exportGate(w http.ResponseWriter, r *http.Request) (string, *ServerRecord, bool) {
p := principalFromContext(r.Context())
name := r.PathValue("name")
if err := naming.ValidateServerName(name); err != nil {
writeError(w, r, newError(http.StatusBadRequest, "bad_name", "invalid server name: %v", err))
return "", nil, false
}
rec, err := a.Repo.ServerByName(r.Context(), name)
if err != nil {
a.writeLookupError(w, r, err)
return "", nil, false
}
if p.UserID == "" || !a.isOwnerOrAdmin(p, rec) {
writeError(w, r, errForbidden)
return "", nil, false
}
return name, rec, true
}
// startExport creates the admitted export's Job, or forgets the export and
// writes the error.
func (a *API) startExport(w http.ResponseWriter, r *http.Request, e *exportEntry, token, backupRef string) bool {
job, err := a.Exporter.Start(r.Context(), worldexport.Request{
Server: e.server, Mode: e.mode, BackupRef: backupRef, ID: e.id, Token: token,
TargetURL: a.InternalBaseURL + "/api/v1/internal/exports/" + e.id,
})
if err != nil {
a.exportTickets().drop(e)
writeError(w, r, err)
return false
}
a.exportTickets().started(e, job)
return true
}
// handleExportStatus answers GET /api/v1/exports/{ticket}: pending while the
// Job gets going, ready once its upload waits for the browser, failed (with the
// Job's error) when it died first. A downloaded or expired ticket is 410
// export_expired; an unknown one, or another user's, is 404.
func (a *API) handleExportStatus(w http.ResponseWriter, r *http.Request) {
p := principalFromContext(r.Context())
reg := a.exportTickets()
e, err := reg.lookup(r.PathValue("ticket"), p.UserID, a.now())
if err != nil {
writeError(w, r, err)
return
}
// A Job list that fails leaves the export pending: the next poll asks again,
// and exportPendingTTL ends it either way.
if e.state == exportPending && e.job != "" && a.JobStatus != nil {
jobs, _ := a.JobStatus.LatestJobs(r.Context(), e.server)
for _, j := range jobs {
if j.Name == e.job && j.State == "failed" && reg.fail(e.ticket, j.Message, a.now()) {
e.state, e.message = exportFailed, j.Message
}
}
}
writeJSON(w, http.StatusOK, exportStatusView{State: e.state, Message: e.message})
}
// handleExportDownload streams a ready export to its browser (GET
// /api/v1/exports/{ticket}/download). The claim spends the ticket before the
// first byte moves, so a second tab, a retry or a HEAD probe never gets a
// second copy; a HEAD is refused outright, since claiming on it would spend the
// ticket on a response with no body.
func (a *API) handleExportDownload(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.Header().Set("Allow", http.MethodGet)
writeError(w, r, newError(http.StatusMethodNotAllowed, "method_not_allowed",
"%s is not allowed here; use GET", r.Method))
return
}
p := principalFromContext(r.Context())
reg := a.exportTickets()
e, err := reg.claim(r.PathValue("ticket"), p.UserID, a.now())
if err != nil {
writeError(w, r, err)
return
}
defer reg.finish(e.ticket, a.now())
h := w.Header()
h.Set("Content-Type", "application/gzip")
if cd := mime.FormatMediaType("attachment", map[string]string{"filename": e.filename}); cd != "" {
h.Set("Content-Disposition", cd)
} else {
h.Set("Content-Disposition", "attachment")
}
h.Set("X-Content-Type-Options", "nosniff")
h.Set("Cache-Control", "no-store")
if e.upload.size >= 0 {
h.Set("Content-Length", strconv.FormatInt(e.upload.size, 10))
}
w.WriteHeader(http.StatusOK)
err = copyExport(w, e.upload.body, e.sha256)
e.upload.done <- err
if err != nil {
log.Printf("api: export %s of %s ended early: %v", e.id, e.server, err)
// Abort the response rather than end it: a download cut short must
// read as failed in the browser, never as a complete file.
panic(http.ErrAbortHandler)
}
}
// copyExport copies body into w through a fixed 32 KiB buffer, restarting w's
// write deadline on every write. With want set, the last read is held back
// until the whole body hashes to it.
func copyExport(w http.ResponseWriter, body io.Reader, want string) error {
out := &heldWriter{w: w, rc: http.NewResponseController(w)}
var sum hash.Hash
if want != "" {
sum = sha256.New()
body = io.TeeReader(body, sum)
}
if _, err := io.CopyBuffer(out, body, make([]byte, exportCopyBuffer)); err != nil {
return err
}
if sum != nil && !strings.EqualFold(hex.EncodeToString(sum.Sum(nil)), want) {
return errExportDigest
}
if err := out.flush(); err != nil {
return err
}
_ = out.rc.SetWriteDeadline(time.Time{}) // the connection may serve another request
return nil
}
// heldWriter passes each write on one behind, keeping the latest back until
// flush, so the end of an archive reaches the browser only once it is checked.
// It has no ReadFrom, so io.CopyBuffer uses the buffer it is given.
type heldWriter struct {
w io.Writer
rc *http.ResponseController
held []byte
}
func (h *heldWriter) Write(p []byte) (int, error) {
if err := h.flush(); err != nil {
return 0, err
}
h.held = append(h.held[:0], p...)
return len(p), nil
}
func (h *heldWriter) flush() error {
if len(h.held) == 0 {
return nil
}
_ = h.rc.SetWriteDeadline(time.Now().Add(exportStall))
_, err := h.w.Write(h.held)
h.held = h.held[:0]
return err
}
// handleInternalExportUpload takes an export Job's archive (PUT
// /api/v1/internal/exports/{id}). The one-time bearer token minted with the
// export is the check, as for file uploads: an unknown id, a wrong token and a
// token already used all read the same 404. The request then waits for the
// browser, up to exportClaimTTL, and answers once the download has ended: 204
// when it got the whole archive, 409 backup_corrupt when the archive did not
// match its recorded digest, 410 export_expired when nobody came for it or the
// browser left early.
func (a *API) handleInternalExportUpload(w http.ResponseWriter, r *http.Request) {
token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
if !ok {
writeError(w, r, errNoExport())
return
}
rc := takeBodyDeadline(w, r)
up := &exportUpload{
body: &stallBody{r: r.Body, rc: rc}, size: r.ContentLength,
claimed: make(chan struct{}), done: make(chan error, 1),
}
reg := a.exportTickets()
if !reg.arrive(r.PathValue("id"), token, up, a.now()) {
writeError(w, r, errNoExport())
return
}
timer := time.NewTimer(exportClaimTTL)
defer timer.Stop()
select {
case <-up.claimed:
case <-timer.C:
if reg.giveUp(up, a.now()) {
writeError(w, r, errExportExpired())
return
}
case <-r.Context().Done():
if reg.giveUp(up, a.now()) {
return
}
}
switch err := <-up.done; {
case err == nil:
w.WriteHeader(http.StatusNoContent)
case errors.Is(err, errExportDigest):
writeError(w, r, newError(http.StatusConflict, "backup_corrupt", "%v", err))
default:
writeError(w, r, newError(http.StatusGone, "export_expired", "the download ended before the archive did: %v", err))
}
}
// stallBody restarts the connection's read deadline on every read, so reading
// fails only once no byte has come for exportStall, and clears it when the body
// ends, as deadlineBody does.
type stallBody struct {
r io.Reader
rc *http.ResponseController
done bool
}
func (b *stallBody) Read(p []byte) (int, error) {
if b.done {
return b.r.Read(p)
}
_ = b.rc.SetReadDeadline(time.Now().Add(exportStall))
n, err := b.r.Read(p)
if err != nil {
b.done = true
_ = b.rc.SetReadDeadline(time.Time{})
}
return n, err
}
+921
View File
@@ -0,0 +1,921 @@
package api
import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"regexp"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
"felis.lolicon.best/internal/worldexport"
batchv1 "k8s.io/api/batch/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
type fakeExporter struct {
reqs []worldexport.Request
err error
}
func (f *fakeExporter) Start(_ context.Context, r worldexport.Request) (string, error) {
f.reqs = append(f.reqs, r)
if f.err != nil {
return "", f.err
}
return worldexport.JobName(r.Server, r.ID), nil
}
// asUser authenticates each request as the principal its X-Test-User header
// names, so one handler serves several users.
type asUser map[string]*Principal
func (u asUser) Authenticate(r *http.Request) (*Principal, error) {
if p, ok := u[r.Header.Get("X-Test-User")]; ok {
return p, nil
}
return nil, errUnauthorized
}
var exportUsers = asUser{
"owner1": {UserID: "owner1", Email: "[email protected]", Role: "user"},
"owner3": {UserID: "owner3", Email: "[email protected]", Role: "user"},
"newowner": {UserID: "newowner", Email: "[email protected]", Role: "user"},
"stranger": {UserID: "stranger", Email: "[email protected]", Role: "user"},
"admin1": {UserID: "admin1", Email: "[email protected]", Role: "admin", ViaAdminAccess: true},
"nouser": {Email: "[email protected]", Role: "admin", ViaAdminAccess: true},
}
func as(user string) map[string]string { return map[string]string{"X-Test-User": user} }
const (
exportBase = "http://felis-api-internal.felis.svc:8081"
worldPath = "/api/v1/servers/survival/world/export"
backupPath = "/api/v1/servers/survival/backups/bk1/export"
exportStamp = "20231114-221320" // newTestAPI's clock, UTC
)
var (
hex64 = regexp.MustCompile(`^[0-9a-f]{64}$`)
hex16 = regexp.MustCompile(`^[0-9a-f]{16}$`)
)
// exportFixture: owner1 owns survival (stopped) and creative, owner3 owns gamma
// (stopped); bk1 is survival's backup, bk2 creative's, both formerly owner1's.
func exportFixture() (*API, *fakeRepo, *fakeCluster, *fakeExporter) {
repo := newFakeRepo()
for name, owner := range map[string]string{"survival": "owner1", "creative": "owner1", "gamma": "owner3"} {
repo.byName[name] = &ServerRecord{Name: name, OwnerID: owner}
}
repo.backups = []fakeBackup{
{view: BackupView{ID: "bk1", ServerName: "survival", FormerOwner: "owner1", Status: "present", SizeBytes: 1024},
ref: "/backups/survival-bk1.tar.gz"},
{view: BackupView{ID: "bk2", ServerName: "creative", FormerOwner: "owner1", Status: "present", SizeBytes: 2048},
ref: "/backups/creative-bk2.tar.gz"},
}
cl := newFakeCluster()
for _, name := range []string{"survival", "gamma"} {
cl.byName[name] = &ServerInfo{Name: name, Phase: "Stopped", DesiredState: string(v1alpha1.DesiredStopped)}
}
ex := &fakeExporter{}
a := newTestAPI(repo, cl)
a.External = exportUsers
a.Exporter, a.InternalBaseURL = ex, exportBase
return a, repo, cl, ex
}
func beginExport(t *testing.T, h http.Handler, path, user string) exportTicketView {
t.Helper()
w := do(h, "POST", path, "", as(user))
if w.Code != http.StatusAccepted {
t.Fatalf("POST %s as %s = %d, want 202 (%s)", path, user, w.Code, w.Body.String())
}
var v exportTicketView
if err := json.Unmarshal(w.Body.Bytes(), &v); err != nil {
t.Fatalf("ticket not JSON: %v (%s)", err, w.Body.String())
}
return v
}
func exportState(t *testing.T, h http.Handler, ticket, user string) (int, exportStatusView) {
t.Helper()
w := do(h, "GET", "/api/v1/exports/"+ticket, "", as(user))
var v exportStatusView
if w.Code == http.StatusOK {
if err := json.Unmarshal(w.Body.Bytes(), &v); err != nil {
t.Fatalf("status not JSON: %v (%s)", err, w.Body.String())
}
} else {
v.State = decodeErr(t, w)
}
return w.Code, v
}
func waitExportReady(t *testing.T, h http.Handler, ticket, user string) {
t.Helper()
for deadline := time.Now().Add(5 * time.Second); ; {
if code, v := exportState(t, h, ticket, user); code == http.StatusOK && v.State == exportReady {
return
}
if time.Now().After(deadline) {
t.Fatal("the export never read ready")
}
time.Sleep(2 * time.Millisecond)
}
}
// uploadExport serves the Job's PUT on the internal face in the background.
func uploadExport(h http.Handler, id, token string, body io.Reader) <-chan *httptest.ResponseRecorder {
out := make(chan *httptest.ResponseRecorder, 1)
go func() {
r := httptest.NewRequest("PUT", "/api/v1/internal/exports/"+id, body)
r.Header.Set("Authorization", "Bearer "+token)
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
out <- w
}()
return out
}
func awaitUpload(t *testing.T, up <-chan *httptest.ResponseRecorder) *httptest.ResponseRecorder {
t.Helper()
select {
case w := <-up:
return w
case <-time.After(5 * time.Second):
t.Fatal("the upload never answered")
return nil
}
}
// doSoon is do for a request that must be refused outright. A download that
// claimed the export by mistake would block on the parked body, and an upload
// let in by mistake would wait for a browser, so either fails the test after a
// bound instead of hanging it.
func doSoon(t *testing.T, h http.Handler, method, path, body string, headers map[string]string) *httptest.ResponseRecorder {
t.Helper()
out := make(chan *httptest.ResponseRecorder, 1)
go func() { out <- do(h, method, path, body, headers) }()
select {
case w := <-out:
return w
case <-time.After(2 * time.Second):
t.Fatalf("%s %s never answered: it claimed the export", method, path)
return nil
}
}
func randomBytes(n int) []byte {
b := make([]byte, n)
_, _ = rand.Read(b)
return b
}
func TestExportBackupGate(t *testing.T) {
t.Run("former owner starts a backup export", func(t *testing.T) {
a, repo, cl, ex := exportFixture()
v := beginExport(t, a.ExternalHandler(), backupPath, "owner1")
if !hex64.MatchString(v.Ticket) || v.State != "pending" || v.Filename != "survival-backup-bk1.tar.gz" {
t.Fatalf("ticket = %+v", v)
}
if len(ex.reqs) != 1 {
t.Fatalf("exporter started %d Jobs, want 1", len(ex.reqs))
}
r := ex.reqs[0]
if r.Server != "survival" || r.Mode != worldexport.ModeBackup || r.BackupRef != "/backups/survival-bk1.tar.gz" ||
!hex16.MatchString(r.ID) || !hex64.MatchString(r.Token) || r.Token == v.Ticket ||
r.TargetURL != exportBase+"/api/v1/internal/exports/"+r.ID {
t.Fatalf("export request = %+v", r)
}
if len(cl.acquired) != 0 {
t.Fatalf("a backup export took the world lock: %v", cl.acquired)
}
if len(repo.audits) != 1 || repo.audits[0].Action != "backup.export" || repo.audits[0].ServerName != "survival" ||
repo.audits[0].ActorUserID != "owner1" || string(repo.audits[0].Payload) != `{"backup_id":"bk1","size_bytes":1024}` {
t.Fatalf("audit = %+v", repo.audits)
}
})
for _, tc := range []struct {
name, user, path string
edit func(*API, *fakeRepo)
code int
errCode string
}{
{name: "admin exports another's world", user: "admin1", path: backupPath,
edit: func(_ *API, r *fakeRepo) { r.backups[0].view.FormerOwner = "someone" }, code: http.StatusAccepted},
{name: "stranger", user: "stranger", path: backupPath, code: http.StatusForbidden, errCode: "forbidden"},
{name: "principal without an account", user: "nouser", path: backupPath, code: http.StatusForbidden, errCode: "forbidden"},
{name: "current owner who is not the former owner", user: "newowner", path: backupPath,
edit: func(_ *API, r *fakeRepo) { r.byName["survival"].OwnerID = "newowner" },
code: http.StatusNotFound, errCode: "no_backup"},
{name: "unknown backup", user: "owner1", path: "/api/v1/servers/survival/backups/nope/export",
code: http.StatusNotFound, errCode: "no_backup"},
{name: "another server's backup", user: "owner1", path: "/api/v1/servers/survival/backups/bk2/export",
code: http.StatusForbidden, errCode: "forbidden"},
{name: "corrupt backup", user: "owner1", path: backupPath,
edit: func(_ *API, r *fakeRepo) { r.backups[0].view.Corrupt = true }, code: http.StatusConflict, errCode: "backup_corrupt"},
{name: "bad name", user: "owner1", path: "/api/v1/servers/Bad_Name/backups/bk1/export",
code: http.StatusBadRequest, errCode: "bad_name"},
{name: "unknown server", user: "owner1", path: "/api/v1/servers/nosuch/backups/bk1/export",
code: http.StatusNotFound, errCode: "not_found"},
{name: "no exporter", user: "owner1", path: backupPath,
edit: func(a *API, _ *fakeRepo) { a.Exporter = nil }, code: http.StatusServiceUnavailable, errCode: "export_unavailable"},
{name: "no internal URL", user: "owner1", path: backupPath,
edit: func(a *API, _ *fakeRepo) { a.InternalBaseURL = "" }, code: http.StatusServiceUnavailable, errCode: "export_unavailable"},
} {
t.Run(tc.name, func(t *testing.T) {
a, repo, _, ex := exportFixture()
if tc.edit != nil {
tc.edit(a, repo)
}
w := do(a.ExternalHandler(), "POST", tc.path, "", as(tc.user))
if w.Code != tc.code {
t.Fatalf("code = %d, want %d (%s)", w.Code, tc.code, w.Body.String())
}
if tc.code == http.StatusAccepted {
if len(ex.reqs) != 1 {
t.Fatalf("exporter started %d Jobs, want 1", len(ex.reqs))
}
return
}
if got := decodeErr(t, w); got != tc.errCode {
t.Errorf("error code = %q, want %q", got, tc.errCode)
}
if len(ex.reqs) != 0 || len(repo.audits) != 0 {
t.Errorf("a refused export started %d Jobs and wrote %d audits", len(ex.reqs), len(repo.audits))
}
})
}
}
func TestExportWorldGate(t *testing.T) {
t.Run("owner exports a stopped world under the world lock", func(t *testing.T) {
a, repo, cl, ex := exportFixture()
v := beginExport(t, a.ExternalHandler(), worldPath, "owner1")
if v.Filename != "survival-world-"+exportStamp+".tar.gz" || v.State != "pending" {
t.Fatalf("ticket = %+v", v)
}
if len(ex.reqs) != 1 || ex.reqs[0].Mode != worldexport.ModeWorld || ex.reqs[0].BackupRef != "" || ex.reqs[0].Server != "survival" {
t.Fatalf("export requests = %+v", ex.reqs)
}
if strings.Join(cl.acquired, ",") != "survival:"+maintenance.KindExport || strings.Join(cl.released, ",") != "survival" {
t.Fatalf("lock acquired %v, released %v", cl.acquired, cl.released)
}
if len(repo.audits) != 1 || repo.audits[0].Action != "world.export" || repo.audits[0].ServerName != "survival" {
t.Fatalf("audit = %+v", repo.audits)
}
})
for _, tc := range []struct {
name, user string
edit func(*API, *fakeCluster)
code int
errCode string
}{
{name: "admin", user: "admin1", code: http.StatusAccepted},
{name: "stranger", user: "stranger", code: http.StatusForbidden, errCode: "forbidden"},
{name: "running", user: "owner1", edit: func(_ *API, c *fakeCluster) { c.byName["survival"].Ready = true },
code: http.StatusConflict, errCode: "not_stopped"},
{name: "coming up", user: "owner1",
edit: func(_ *API, c *fakeCluster) { c.byName["survival"].DesiredState = string(v1alpha1.DesiredRunning) },
code: http.StatusConflict, errCode: "not_stopped"},
{name: "no world volume", user: "owner1", edit: func(_ *API, c *fakeCluster) { c.noWorld["survival"] = true },
code: http.StatusConflict, errCode: "no_world_volume"},
{name: "world busy", user: "owner1",
edit: func(_ *API, c *fakeCluster) {
c.maintErr["survival"] = &MaintenanceBusyError{Kind: maintenance.KindRestore}
},
code: http.StatusConflict, errCode: "maintenance_in_progress"},
{name: "no exporter", user: "owner1", edit: func(a *API, _ *fakeCluster) { a.Exporter = nil },
code: http.StatusServiceUnavailable, errCode: "export_unavailable"},
{name: "no internal URL", user: "owner1", edit: func(a *API, _ *fakeCluster) { a.InternalBaseURL = "" },
code: http.StatusServiceUnavailable, errCode: "export_unavailable"},
} {
t.Run(tc.name, func(t *testing.T) {
a, repo, cl, ex := exportFixture()
if tc.edit != nil {
tc.edit(a, cl)
}
w := do(a.ExternalHandler(), "POST", worldPath, "", as(tc.user))
if w.Code != tc.code {
t.Fatalf("code = %d, want %d (%s)", w.Code, tc.code, w.Body.String())
}
if tc.code == http.StatusAccepted {
if len(ex.reqs) != 1 {
t.Fatalf("exporter started %d Jobs, want 1", len(ex.reqs))
}
return
}
if got := decodeErr(t, w); got != tc.errCode {
t.Errorf("error code = %q, want %q", got, tc.errCode)
}
if len(ex.reqs) != 0 || len(repo.audits) != 0 || len(cl.released) != 0 {
t.Errorf("a refused export started %d Jobs, wrote %d audits, released %v", len(ex.reqs), len(repo.audits), cl.released)
}
})
}
// What anyone refused by a running world export hears.
t.Run("a world export holding the lock is named", func(t *testing.T) {
a, _, cl, _ := exportFixture()
cl.maintErr["survival"] = &MaintenanceBusyError{Kind: maintenance.KindExport}
w := do(a.ExternalHandler(), "POST", worldPath, "", as("admin1"))
var raw map[string]map[string]string
_ = json.Unmarshal(w.Body.Bytes(), &raw)
if want := "a world export is running on this server's world; retry once it finishes"; w.Code != http.StatusConflict ||
raw["error"]["code"] != "maintenance_in_progress" || raw["error"]["message"] != want {
t.Fatalf("busy = %d %s, want 409 %q", w.Code, w.Body.String(), want)
}
})
// A refusal after admission gives the export back: the owner is not left
// blocked by an export that never got a Job.
t.Run("lock conflict and a failed Job are refunded", func(t *testing.T) {
a, _, cl, ex := exportFixture()
h := a.ExternalHandler()
cl.maintErr["survival"] = &MaintenanceBusyError{Kind: maintenance.KindBackup}
if w := do(h, "POST", worldPath, "", as("owner1")); w.Code != http.StatusConflict {
t.Fatalf("busy world: code = %d, want 409", w.Code)
}
delete(cl.maintErr, "survival")
ex.err = errors.New("jobs is forbidden")
if w := do(h, "POST", worldPath, "", as("owner1")); w.Code != http.StatusInternalServerError {
t.Fatalf("failed Job: code = %d, want 500 (%s)", w.Code, w.Body.String())
}
if strings.Join(cl.released, ",") != "survival" {
t.Fatalf("a failed Job left the lock held: released %v", cl.released)
}
ex.err = nil
beginExport(t, h, worldPath, "owner1")
if n := len(a.exportTickets().starts["owner1"]); n != 1 {
t.Fatalf("hourly starts = %d, want only the export that got a Job", n)
}
})
}
func TestExportLimits(t *testing.T) {
t.Run("one per user", func(t *testing.T) {
a, _, _, ex := exportFixture()
h := a.ExternalHandler()
beginExport(t, h, worldPath, "owner1")
w := do(h, "POST", backupPath, "", as("owner1"))
if w.Code != http.StatusTooManyRequests || decodeErr(t, w) != "export_busy" || w.Header().Get("Retry-After") != "90" {
t.Fatalf("second export: %d %s Retry-After %q", w.Code, w.Body.String(), w.Header().Get("Retry-After"))
}
if len(ex.reqs) != 1 {
t.Fatalf("exporter started %d Jobs, want 1", len(ex.reqs))
}
})
t.Run("two across the install", func(t *testing.T) {
a, _, cl, ex := exportFixture()
h := a.ExternalHandler()
beginExport(t, h, worldPath, "owner1")
beginExport(t, h, "/api/v1/servers/creative/backups/bk2/export", "admin1")
w := do(h, "POST", "/api/v1/servers/gamma/world/export", "", as("owner3"))
if w.Code != http.StatusTooManyRequests || decodeErr(t, w) != "export_busy" || w.Header().Get("Retry-After") != "60" {
t.Fatalf("third export: %d %s Retry-After %q", w.Code, w.Body.String(), w.Header().Get("Retry-After"))
}
if len(ex.reqs) != 2 || strings.Join(cl.acquired, ",") != "survival:export" {
t.Fatalf("a refused export started a Job or took gamma's lock: %d Jobs, acquired %v", len(ex.reqs), cl.acquired)
}
})
t.Run("six per user per hour", func(t *testing.T) {
defer func(old time.Duration) { exportPendingTTL = old }(exportPendingTTL)
exportPendingTTL = time.Minute
a, _, _, _ := exportFixture()
var clock atomic.Int64
clock.Store(1_700_000_000)
a.Now = func() time.Time { return time.Unix(clock.Load(), 0) }
h := a.ExternalHandler()
for i := range exportPerHour {
clock.Store(1_700_000_000 + int64(i)*120) // each start outlives the last one's pending TTL
beginExport(t, h, worldPath, "owner1")
}
clock.Store(1_700_000_000 + 12*60)
w := do(h, "POST", worldPath, "", as("owner1"))
if w.Code != http.StatusTooManyRequests || decodeErr(t, w) != "export_busy" || w.Header().Get("Retry-After") != "2880" {
t.Fatalf("seventh export: %d %s Retry-After %q", w.Code, w.Body.String(), w.Header().Get("Retry-After"))
}
clock.Store(1_700_000_000 + 3600)
beginExport(t, h, worldPath, "owner1")
})
}
// TestExportRendezvous walks one world export end to end through the handlers:
// the Job's PUT parks until the owner's download claims it, the archive streams
// through untouched, the ticket works once and for its user only.
func TestExportRendezvous(t *testing.T) {
a, repo, _, ex := exportFixture()
var clock atomic.Int64
clock.Store(1_700_000_000)
a.Now = func() time.Time { return time.Unix(clock.Load(), 0) }
ext, in := a.ExternalHandler(), a.InternalHandler()
v := beginExport(t, ext, worldPath, "owner1")
job := ex.reqs[0]
download := "/api/v1/exports/" + v.Ticket + "/download"
if code, s := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusOK || s != (exportStatusView{State: "pending"}) {
t.Fatalf("fresh status = %d %+v", code, s)
}
for _, user := range []string{"stranger", "admin1"} {
if code, s := exportState(t, ext, v.Ticket, user); code != http.StatusNotFound || s.State != "not_found" {
t.Fatalf("%s reads the owner's ticket: %d %+v", user, code, s)
}
}
if w := do(ext, "GET", download, "", as("owner1")); w.Code != http.StatusConflict || decodeErr(t, w) != "export_not_ready" {
t.Fatalf("download while pending = %d %s", w.Code, w.Body.String())
}
for name, hdr := range map[string]map[string]string{
"no token": nil,
"wrong token": {"Authorization": "Bearer " + strings.Repeat("0", 64)},
"the ticket": {"Authorization": "Bearer " + v.Ticket},
} {
if w := doSoon(t, in, "PUT", "/api/v1/internal/exports/"+job.ID, "x", hdr); w.Code != http.StatusNotFound || decodeErr(t, w) != "not_found" {
t.Fatalf("PUT with %s = %d %s", name, w.Code, w.Body.String())
}
}
if w := doSoon(t, in, "PUT", "/api/v1/internal/exports/ffffffffffffffff", "x", map[string]string{"Authorization": "Bearer " + job.Token}); w.Code != http.StatusNotFound {
t.Fatalf("PUT to another id = %d", w.Code)
}
archive := randomBytes(3*exportCopyBuffer + 4321)
pr, pw := io.Pipe()
up := uploadExport(in, job.ID, job.Token, pr)
waitExportReady(t, ext, v.Ticket, "owner1")
if w := doSoon(t, in, "PUT", "/api/v1/internal/exports/"+job.ID, "x", map[string]string{"Authorization": "Bearer " + job.Token}); w.Code != http.StatusNotFound {
t.Fatalf("a second PUT with the spent token = %d, want 404", w.Code)
}
// Neither a HEAD nor another user's GET claims it.
if w := doSoon(t, ext, "HEAD", download, "", as("owner1")); w.Code != http.StatusMethodNotAllowed || w.Header().Get("Allow") != "GET" {
t.Fatalf("HEAD = %d, Allow %q", w.Code, w.Header().Get("Allow"))
}
if w := doSoon(t, ext, "GET", download, "", as("stranger")); w.Code != http.StatusNotFound {
t.Fatalf("stranger's download = %d", w.Code)
}
if code, s := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusOK || s.State != exportReady {
t.Fatalf("after HEAD and a stranger: %d %+v, want still ready", code, s)
}
got := make(chan *httptest.ResponseRecorder, 1)
go func() { got <- do(ext, "GET", download, "", as("owner1")) }()
_, _ = pw.Write(archive[:10_000]) // returns once the download has read it
if code, s := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusGone || s.State != "export_expired" {
t.Fatalf("status mid-download = %d %+v, want 410 export_expired", code, s)
}
if w := doSoon(t, ext, "GET", download, "", as("owner1")); w.Code != http.StatusGone || decodeErr(t, w) != "export_expired" {
t.Fatalf("a second download mid-stream = %d %s", w.Code, w.Body.String())
}
go func() {
for rest := archive[10_000:]; len(rest) > 0; {
n := min(len(rest), 10_000)
_, _ = pw.Write(rest[:n])
rest = rest[n:]
}
pw.Close()
}()
w := <-got
if w.Code != http.StatusOK || !bytes.Equal(w.Body.Bytes(), archive) {
t.Fatalf("download = %d, %d bytes; want 200 and the %d archive bytes", w.Code, w.Body.Len(), len(archive))
}
h := w.Header()
if h.Get("Content-Type") != "application/gzip" || h.Get("X-Content-Type-Options") != "nosniff" || h.Get("Cache-Control") != "no-store" ||
h.Get("Content-Disposition") != "attachment; filename=survival-world-"+exportStamp+".tar.gz" || h.Get("Content-Length") != "" {
t.Fatalf("download headers = %v", h)
}
if w := awaitUpload(t, up); w.Code != http.StatusNoContent {
t.Fatalf("upload answered %d %s, want 204", w.Code, w.Body.String())
}
if w := do(ext, "GET", download, "", as("owner1")); w.Code != http.StatusGone || decodeErr(t, w) != "export_expired" {
t.Fatalf("second download = %d %s", w.Code, w.Body.String())
}
if code, s := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusGone || s.State != "export_expired" {
t.Fatalf("status after download = %d %+v", code, s)
}
clock.Add(int64(exportKeepSpent / time.Second))
if code, _ := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusNotFound {
t.Fatalf("status long after = %d, want 404", code)
}
if len(repo.audits) != 1 {
t.Fatalf("audits = %+v, want the start only", repo.audits)
}
beginExport(t, ext, worldPath, "owner1") // the finished export no longer counts
}
func TestExportExpiry(t *testing.T) {
t.Run("a Job that never connects", func(t *testing.T) {
a, _, _, ex := exportFixture()
var clock atomic.Int64
clock.Store(1_700_000_000)
a.Now = func() time.Time { return time.Unix(clock.Load(), 0) }
ext := a.ExternalHandler()
v := beginExport(t, ext, worldPath, "owner1")
clock.Add(int64(exportPendingTTL/time.Second) - 1)
if code, s := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusOK || s.State != "pending" {
t.Fatalf("just inside the pending TTL: %d %+v", code, s)
}
clock.Add(1)
if code, s := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusGone || s.State != "export_expired" {
t.Fatalf("past the pending TTL: %d %+v", code, s)
}
job := ex.reqs[0]
if w := doSoon(t, a.InternalHandler(), "PUT", "/api/v1/internal/exports/"+job.ID, "x", map[string]string{"Authorization": "Bearer " + job.Token}); w.Code != http.StatusNotFound {
t.Fatalf("a late Job's PUT = %d, want 404", w.Code)
}
})
t.Run("a browser that never comes", func(t *testing.T) {
defer func(old time.Duration) { exportClaimTTL = old }(exportClaimTTL)
exportClaimTTL = 30 * time.Millisecond
a, _, _, ex := exportFixture()
ext := a.ExternalHandler()
v := beginExport(t, ext, backupPath, "owner1")
job := ex.reqs[0]
w := awaitUpload(t, uploadExport(a.InternalHandler(), job.ID, job.Token, strings.NewReader("archive")))
if w.Code != http.StatusGone || decodeErr(t, w) != "export_expired" {
t.Fatalf("unclaimed upload = %d %s, want 410 export_expired", w.Code, w.Body.String())
}
if w := do(ext, "GET", "/api/v1/exports/"+v.Ticket+"/download", "", as("owner1")); w.Code != http.StatusGone {
t.Fatalf("download after the claim TTL = %d, want 410", w.Code)
}
})
}
// TestExportStatusReportsFailedJob: a Job that dies before it connects turns
// the export failed with the Job's own error, and frees the owner's slot.
func TestExportStatusReportsFailedJob(t *testing.T) {
a, _, _, ex := exportFixture()
js := &fakeJobStatus{}
a.JobStatus = js
ext := a.ExternalHandler()
v := beginExport(t, ext, worldPath, "owner1")
job := ex.reqs[0]
name := worldexport.JobName("survival", job.ID)
js.jobs = []AsyncJob{
{Name: "export-survival-0000000000000000", Kind: "export_world", State: "failed", Message: "someone else's"},
{Name: name, Kind: "export_world", State: "running"},
}
if code, s := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusOK || s != (exportStatusView{State: "pending"}) {
t.Fatalf("running Job: %d %+v", code, s)
}
js.jobs[1] = AsyncJob{Name: name, Kind: "export_world", State: "failed", Message: "felis export: open /world: permission denied"}
want := exportStatusView{State: "failed", Message: "felis export: open /world: permission denied"}
if code, s := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusOK || s != want || js.got != "survival" {
t.Fatalf("failed Job: %d %+v (asked about %q)", code, s, js.got)
}
js.jobs = nil
if code, s := exportState(t, ext, v.Ticket, "owner1"); code != http.StatusOK || s != want {
t.Fatalf("failed export once the Job is gone: %d %+v", code, s)
}
if w := doSoon(t, a.InternalHandler(), "PUT", "/api/v1/internal/exports/"+job.ID, "x", map[string]string{"Authorization": "Bearer " + job.Token}); w.Code != http.StatusNotFound {
t.Fatalf("PUT on a failed export = %d, want 404", w.Code)
}
beginExport(t, ext, backupPath, "owner1")
}
// TestExportRegistryRaces: two transitions the handlers reach only in a race.
// A Job reported failed just as its upload arrived must not fail the ready
// export, and the claim timer firing as the browser claims must not spend the
// export under its download.
func TestExportRegistryRaces(t *testing.T) {
now := time.Unix(1_700_000_000, 0)
reg := (&API{}).exportTickets()
e := &exportEntry{userID: "owner1", server: "survival", mode: worldexport.ModeWorld, filename: "survival.tar.gz"}
token, err := reg.admit(e, now)
if err != nil {
t.Fatal(err)
}
up := &exportUpload{body: strings.NewReader("x"), size: 1, claimed: make(chan struct{}), done: make(chan error, 1)}
if !reg.arrive(e.id, token, up, now) {
t.Fatal("the upload did not arrive")
}
if reg.fail(e.ticket, "felis export: killed", now) {
t.Fatal("a late Job failure failed a ready export")
}
if got, err := reg.lookup(e.ticket, "owner1", now); err != nil || got.state != exportReady || got.message != "" {
t.Fatalf("after a late Job failure: state %q, message %q, err %v; want ready", got.state, got.message, err)
}
if _, err := reg.claim(e.ticket, "owner1", now); err != nil {
t.Fatalf("claim: %v", err)
}
if reg.giveUp(up, now) {
t.Fatal("the claim timer spent an export being downloaded")
}
if e.state != exportStreaming {
t.Fatalf("state after a late giveUp = %q, want streaming", e.state)
}
}
// exportServers runs both faces for real, so deadlines apply and an aborted
// download reaches the client as an error.
func exportServers(t *testing.T, a *API) (ext, in *httptest.Server) {
ext, in = httptest.NewServer(a.ExternalHandler()), httptest.NewServer(a.InternalHandler())
t.Cleanup(func() { ext.Close(); in.Close() })
return ext, in
}
func realUpload(t *testing.T, in *httptest.Server, job worldexport.Request, body io.Reader, size int64) <-chan *http.Response {
req, err := http.NewRequest("PUT", in.URL+"/api/v1/internal/exports/"+job.ID, body)
if err != nil {
t.Fatal(err)
}
req.ContentLength = size
req.Header.Set("Authorization", "Bearer "+job.Token)
out := make(chan *http.Response, 1)
go func() {
resp, err := http.DefaultClient.Do(req)
if err != nil {
out <- &http.Response{StatusCode: -1, Status: err.Error(), Body: io.NopCloser(strings.NewReader(""))}
return
}
out <- resp
}()
return out
}
// realDownload fetches the owner's download: the response (nil when the request
// failed outright), the bytes read and the error that ended the read.
func realDownload(ext *httptest.Server, ticket string) (*http.Response, []byte, error) {
req, _ := http.NewRequest("GET", ext.URL+"/api/v1/exports/"+ticket+"/download", nil)
req.Header.Set("X-Test-User", "owner1")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
got, err := io.ReadAll(resp.Body)
return resp, got, err
}
func awaitResponse(t *testing.T, ch <-chan *http.Response) *http.Response {
t.Helper()
select {
case resp := <-ch:
t.Cleanup(func() { resp.Body.Close() })
return resp
case <-time.After(5 * time.Second):
t.Fatal("the upload never answered")
return nil
}
}
// TestExportBackupDigest: a backup streams with its length and is checked
// against the sha256 recorded when it was written. A match downloads whole; a
// mismatch withholds the tail and aborts, so the browser never holds a
// complete-looking corrupt file, and the Job hears backup_corrupt.
func TestExportBackupDigest(t *testing.T) {
archive := randomBytes(3*exportCopyBuffer + 4321)
sum := sha256.Sum256(archive)
for _, tc := range []struct {
name string
digest string
}{
{"recorded digest matches", hex.EncodeToString(sum[:])},
{"no digest recorded", ""},
{"digest mismatch", strings.Repeat("ab", 32)},
} {
t.Run(tc.name, func(t *testing.T) {
a, repo, _, ex := exportFixture()
repo.backups[0].sha256 = tc.digest
ext, in := exportServers(t, a)
v := beginExport(t, a.ExternalHandler(), backupPath, "owner1")
up := realUpload(t, in, ex.reqs[0], bytes.NewReader(archive), int64(len(archive)))
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
resp, got, err := realDownload(ext, v.Ticket)
if resp == nil {
t.Fatalf("download: %v", err)
}
if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Length") != strconv.Itoa(len(archive)) {
t.Fatalf("download = %d, Content-Length %q", resp.StatusCode, resp.Header.Get("Content-Length"))
}
upResp := awaitResponse(t, up)
if tc.digest == strings.Repeat("ab", 32) {
if err == nil || len(got) >= len(archive) || !bytes.Equal(got, archive[:len(got)]) {
t.Fatalf("mismatch: read %d of %d bytes, err %v; want an error short of the end", len(got), len(archive), err)
}
if upResp.StatusCode != http.StatusConflict || errCode(mustRead(t, upResp.Body)) != "backup_corrupt" {
t.Fatalf("upload answered %d, want 409 backup_corrupt", upResp.StatusCode)
}
return
}
if err != nil || !bytes.Equal(got, archive) {
t.Fatalf("read %d of %d bytes, err %v", len(got), len(archive), err)
}
if upResp.StatusCode != http.StatusNoContent {
t.Fatalf("upload answered %d, want 204", upResp.StatusCode)
}
})
}
// The tail is withheld from the response writer itself, not only from
// whatever the connection had yet to send: with every write captured, a
// mismatch still ends short of the archive.
t.Run("the tail waits for the digest", func(t *testing.T) {
a, repo, _, ex := exportFixture()
repo.backups[0].sha256 = strings.Repeat("ab", 32)
ext := a.ExternalHandler()
v := beginExport(t, ext, backupPath, "owner1")
up := uploadExport(a.InternalHandler(), ex.reqs[0].ID, ex.reqs[0].Token, bytes.NewReader(archive))
waitExportReady(t, ext, v.Ticket, "owner1")
w := httptest.NewRecorder()
func() {
defer func() {
if p := recover(); p != http.ErrAbortHandler {
t.Errorf("the download ended with %v, want the abort", p)
}
}()
r := httptest.NewRequest("GET", "/api/v1/exports/"+v.Ticket+"/download", nil)
r.Header.Set("X-Test-User", "owner1")
ext.ServeHTTP(w, r)
}()
if got := w.Body.Bytes(); len(got) != 3*exportCopyBuffer || !bytes.Equal(got, archive[:len(got)]) {
t.Fatalf("wrote %d of %d bytes, want all but the last read (%d)", len(got), len(archive), 3*exportCopyBuffer)
}
if w := awaitUpload(t, up); w.Code != http.StatusConflict || decodeErr(t, w) != "backup_corrupt" {
t.Fatalf("upload answered %d %s, want 409 backup_corrupt", w.Code, w.Body.String())
}
})
}
// zeros reads as an endless run of zero bytes.
type zeros struct{}
func (zeros) Read(p []byte) (int, error) {
clear(p)
return len(p), nil
}
func mustRead(t *testing.T, r io.Reader) []byte {
t.Helper()
b, err := io.ReadAll(r)
if err != nil {
t.Fatal(err)
}
return b
}
// TestExportUploadPace: the Job's upload waits for the browser longer than the
// body-deadline grace, and then moves at the browser's pace, without being cut
// off; a body that stops moving for exportStall is.
func TestExportUploadPace(t *testing.T) {
defer func(g time.Duration, r float64, s time.Duration) { bodyGrace, bodyMinRate, exportStall = g, r, s }(bodyGrace, bodyMinRate, exportStall)
bodyGrace, bodyMinRate, exportStall = 100*time.Millisecond, 1<<30, 300*time.Millisecond
t.Run("a slow claim is not a slow body", func(t *testing.T) {
a, _, _, ex := exportFixture()
ext, in := exportServers(t, a)
v := beginExport(t, a.ExternalHandler(), worldPath, "owner1")
archive := randomBytes(64 << 10)
pr, pw := io.Pipe()
go func() {
_, _ = pw.Write(archive[:1000])
time.Sleep(3 * bodyGrace)
_, _ = pw.Write(archive[1000:])
pw.Close()
}()
up := realUpload(t, in, ex.reqs[0], pr, -1)
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
time.Sleep(3 * bodyGrace)
_, got, err := realDownload(ext, v.Ticket)
if err != nil || !bytes.Equal(got, archive) {
t.Fatalf("read %d of %d bytes, err %v", len(got), len(archive), err)
}
if resp := awaitResponse(t, up); resp.StatusCode != http.StatusNoContent {
t.Fatalf("upload answered %d, want 204", resp.StatusCode)
}
})
t.Run("a stalled body is cut off", func(t *testing.T) {
a, _, _, ex := exportFixture()
ext, in := exportServers(t, a)
v := beginExport(t, a.ExternalHandler(), worldPath, "owner1")
pr, pw := io.Pipe()
defer pw.Close()
go func() { _, _ = pw.Write(make([]byte, 1000)) }() // then nothing, ever
up := realUpload(t, in, ex.reqs[0], pr, -1)
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
done := make(chan error, 1)
go func() { _, _, err := realDownload(ext, v.Ticket); done <- err }()
select {
case err := <-done:
if err == nil {
t.Fatal("a stalled archive downloaded as complete")
}
case <-time.After(5 * time.Second):
t.Fatal("a stalled upload was never cut off")
}
pw.CloseWithError(errors.New("test over"))
awaitResponse(t, up)
})
t.Run("a browser that stops reading is cut off", func(t *testing.T) {
a, _, _, ex := exportFixture()
ext, in := exportServers(t, a)
v := beginExport(t, a.ExternalHandler(), worldPath, "owner1")
realUpload(t, in, ex.reqs[0], io.LimitReader(zeros{}, 1<<30), -1)
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
req, _ := http.NewRequest("GET", ext.URL+"/api/v1/exports/"+v.Ticket+"/download", nil)
req.Header.Set("X-Test-User", "owner1")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { resp.Body.Close() }) // before the servers close, so a held handler ends
// Read nothing: the socket buffers fill and the next write waits.
reg := a.exportTickets()
for deadline := time.Now().Add(5 * time.Second); ; {
reg.mu.Lock()
state := reg.byTicket[v.Ticket].state
reg.mu.Unlock()
if state == exportSpent {
break
}
if time.Now().After(deadline) {
t.Fatalf("a browser that stopped reading held the download open: state %q", state)
}
time.Sleep(10 * time.Millisecond)
}
if _, err := io.ReadAll(resp.Body); err == nil {
t.Fatal("the cut-off download read as complete")
}
})
}
// TestK8sExportJobs: export Jobs show in the jobs list by what they archive,
// and never count as world work the backup scheduler waits for.
func TestK8sExportJobs(t *testing.T) {
scheme := runtime.NewScheme()
if err := clientgoscheme.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
job := func(id, mode string) *batchv1.Job {
j, err := worldexport.ExportJob(worldexport.JobParams{
Server: "survival", ID: id, Mode: mode,
WorldPVC: "world-survival-0", BackupPVC: "felis-backups", BackupRef: "/backups/a.tar.gz",
TargetURL: exportBase + "/x", Token: "t", Namespace: "minecraft", Image: "felis:1",
})
if err != nil {
t.Fatal(err)
}
return j
}
world, backupJob := job("1111111111111111", worldexport.ModeWorld), job("2222222222222222", worldexport.ModeBackup)
restoring := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "restore-survival-cc",
Labels: map[string]string{jobServerLabel: "survival", jobManagedByLabel: jobManagedByRestore}}}
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(world, backupJob, restoring).
WithStatusSubresource(&batchv1.Job{}).Build()
k := NewK8sJobStatus(c, "minecraft")
ctx := context.Background()
if n, err := k.RunningWorldJobs(ctx); err != nil || n != 1 {
t.Fatalf("RunningWorldJobs = %d, %v; want the restore only", n, err)
}
jobs, err := k.LatestJobs(ctx, "survival")
if err != nil {
t.Fatal(err)
}
kinds := map[string]string{}
for _, j := range jobs {
kinds[j.Name] = j.Kind + "/" + j.State
}
want := map[string]string{world.Name: "export_world/running", backupJob.Name: "export_backup/running", restoring.Name: "restore/running"}
if len(kinds) != len(want) {
t.Fatalf("jobs = %v, want %v", kinds, want)
}
for name, k := range want {
if kinds[name] != k {
t.Errorf("%s = %q, want %q", name, kinds[name], k)
}
}
// The kinds agree with maintenance.JobKind: a world export holds the world,
// a backup export does not.
if kind, holds := maintenance.JobKind(world); kind != maintenance.KindExport || !holds {
t.Errorf("JobKind(world export) = %q, %v", kind, holds)
}
if _, holds := maintenance.JobKind(backupJob); holds {
t.Error("a backup export holds the world")
}
}
+1 -1
View File
@@ -14,7 +14,7 @@ import (
// object an operator with kubectl could read. The route is the API-side outlet.
type AsyncJob struct {
Name string `json:"name"`
Kind string `json:"kind"` // "backup" | "restore"
Kind string `json:"kind"` // "backup" | "restore" | "export_world" | "export_backup"
State string `json:"state"` // "running" | "succeeded" | "failed"
Message string `json:"message,omitempty"`
StartedAt time.Time `json:"started_at,omitzero"`
+16 -5
View File
@@ -25,6 +25,7 @@ const (
jobManagedByBackup = "felis-backup"
jobManagedByRestore = "felis-restore"
jobManagedByExport = "felis-export"
// jobBackupReasonLabel marks the backup Job of a scheduled restore point
// (backupjob.LabelReason).
@@ -45,9 +46,9 @@ func NewK8sJobStatus(c client.Client, namespace string) *K8sJobStatus {
return &K8sJobStatus{c: c, namespace: namespace}
}
// LatestJobs lists this server's backup/restore Jobs newest-first, capped so a
// long history cannot balloon the response. Jobs the label selector catches but
// another component created (unknown managed-by) are dropped.
// LatestJobs lists this server's backup, restore and export Jobs newest-first,
// capped so a long history cannot balloon the response. Jobs the label selector
// catches but another component created (unknown managed-by) are dropped.
func (k *K8sJobStatus) LatestJobs(ctx context.Context, serverName string) ([]AsyncJob, error) {
var list batchv1.JobList
if err := k.c.List(ctx, &list, client.InNamespace(k.namespace),
@@ -147,7 +148,10 @@ func jobToAsyncJob(j *batchv1.Job) (AsyncJob, bool) {
}
// RunningWorldJobs counts the backup and restore Jobs of every server that
// have yet to finish (BackupScheduler waits for them).
// have yet to finish (BackupScheduler waits for them). Export Jobs are left
// out: one moves at its browser's pace, a download left running for an hour
// must not hold every world's restore point back, and the export limits
// (exportMaxActive) already bound the load they add.
func (k *K8sJobStatus) RunningWorldJobs(ctx context.Context) (int, error) {
var list batchv1.JobList
if err := k.c.List(ctx, &list, client.InNamespace(k.namespace), client.HasLabels{jobManagedByLabel}); err != nil {
@@ -156,7 +160,7 @@ func (k *K8sJobStatus) RunningWorldJobs(ctx context.Context) (int, error) {
n := 0
for i := range list.Items {
j := &list.Items[i]
if _, ok := jobOutcome(j); ok && !maintenance.JobFinished(j) {
if _, ok := jobOutcome(j); ok && j.Labels[jobManagedByLabel] != jobManagedByExport && !maintenance.JobFinished(j) {
n++
}
}
@@ -225,6 +229,13 @@ func jobOutcome(j *batchv1.Job) (AsyncJob, bool) {
kind = "backup"
case jobManagedByRestore:
kind = "restore"
case jobManagedByExport:
// As maintenance.JobKind reads it: only a Job that says it reads a
// backup is not a world export.
kind = "export_world"
if j.Labels[maintenance.LabelExportMode] == maintenance.ExportModeBackup {
kind = "export_backup"
}
default:
return AsyncJob{}, false
}
+5 -3
View File
@@ -11,9 +11,9 @@ import (
)
// maintenanceError maps the world-volume lock's refusals onto their 409s:
// maintenance_in_progress while a restore, backup or file write holds the volume,
// not_stopped (with the caller's wording) while the server is not fully down.
// Anything else passes through unchanged.
// maintenance_in_progress while a restore, backup, file write or world export
// holds the volume, not_stopped (with the caller's wording) while the server is
// not fully down. Anything else passes through unchanged.
func maintenanceError(err error, notStopped string) error {
var busy *MaintenanceBusyError
switch {
@@ -37,6 +37,8 @@ func maintenanceLabel(kind string) string {
return "a backup"
case maintenance.KindFileWrite:
return "a file write"
case maintenance.KindExport:
return "a world export"
case maintenance.KindReap:
return "the idle-world reaper"
}
+12
View File
@@ -210,6 +210,18 @@ func (b *deadlineBody) Read(p []byte) (int, error) {
return n, err
}
// takeBodyDeadline lifts withBodyDeadline from a request whose body is read at
// someone else's pace (an export's upload waits for the browser, then moves at
// its speed), and returns the controller the handler bounds it with instead.
func takeBodyDeadline(w http.ResponseWriter, r *http.Request) *http.ResponseController {
if b, ok := r.Body.(*deadlineBody); ok {
r.Body = b.ReadCloser
_ = b.rc.SetReadDeadline(time.Time{})
return b.rc
}
return http.NewResponseController(w)
}
// requireInternal enforces service-token auth for the internal face and stashes
// the caller the token belongs to, which callersOnly checks against the route.
// It never applies Zero Trust (spec §14 red line).
+2
View File
@@ -40,6 +40,8 @@ func TestOpenAPISchemasMatchWireStructs(t *testing.T) {
"MyServerView": MyServerView{},
"AllowlistEntry": AllowlistEntry{},
"BackupView": BackupView{},
"ExportTicket": exportTicketView{},
"ExportStatus": exportStatusView{},
"Schedule": Schedule{},
"Build": build.Build{},
"Image": build.Image{},
+2 -2
View File
@@ -958,11 +958,11 @@ func (p *PGRepo) LatestBackup(ctx context.Context, serverName string) (*BackupRe
// BackupByID returns a single present backup by its id, or ErrNotFound.
func (p *PGRepo) BackupByID(ctx context.Context, id string) (*BackupRecord, error) {
const q = `SELECT id, server_name, COALESCE(former_owner, ''), backup_ref, COALESCE(size_bytes, 0),
corrupt_at IS NOT NULL
corrupt_at IS NOT NULL, COALESCE(sha256, '')
FROM world_backups WHERE id = $1 AND status = 'present'`
var b BackupRecord
switch err := p.db.QueryRowContext(ctx, q, id).Scan(
&b.ID, &b.ServerName, &b.FormerOwner, &b.BackupRef, &b.SizeBytes, &b.Corrupt); {
&b.ID, &b.ServerName, &b.FormerOwner, &b.BackupRef, &b.SizeBytes, &b.Corrupt, &b.SHA256); {
case errors.Is(err, sql.ErrNoRows):
return nil, ErrNotFound
case err != nil:
+4
View File
@@ -143,6 +143,10 @@ type BackupRecord struct {
SizeBytes int64
// Corrupt reports that the archive failed a read-back (see BackupView).
Corrupt bool
// SHA256 is the digest recorded when the archive was written, which an
// export checks the bytes against as they stream. Empty for an archive from
// before digests were kept.
SHA256 string
}
// StaffUser is the login-side projection of a users row (spec §B passwordless
+8
View File
@@ -426,6 +426,14 @@ func writeTarGz(ctx context.Context, w io.Writer, srcDir string) (tarStats, erro
return st, nil
}
// WriteTarGz archives srcDir into w laid out exactly as Archive lays out a
// backup, so an exported world restores like any other archive, and returns the
// entries it left out. The world export Job streams it straight into its upload.
func WriteTarGz(ctx context.Context, w io.Writer, srcDir string) ([]string, error) {
st, err := writeTarGz(ctx, w, srcDir)
return st.skipped, err
}
// dirMeta is a directory's recorded permission bits and modification time,
// applied once nothing more is written into it.
type dirMeta struct {
+27 -8
View File
@@ -1,6 +1,6 @@
// Package maintenance is the per-server mutual exclusion between a game server
// and the Jobs that write or snapshot its world volume (restore, backup, file
// write). The world PVC is ReadWriteOnce, and RWO is exclusive per NODE: on a
// write, world export). The world PVC is ReadWriteOnce, and RWO is exclusive per NODE: on a
// single-node cluster the game pod and a restore pod mount it side by side, so
// the access mode alone guards nothing. A server woken mid-restore boots on a
// half-extracted world and the restore then prunes what it wrote; a server woken
@@ -29,7 +29,10 @@
//
// File reads and listings are not holders. They mount the volume read-only for a
// second or two, and a server starting beside one cannot hurt either side, so
// nobody waits for them.
// nobody waits for them. A world export mounts it read-only too, but it holds:
// it archives the world for as long as the download takes, and a server started
// beside it would hand the owner a torn archive. Exporting a backup reads only
// the backup store and holds nothing.
package maintenance
import (
@@ -50,14 +53,18 @@ const (
Grace = 2 * time.Minute
// LabelServer / LabelManagedBy are the labels every maintenance executor puts
// on its Job (internal/restore, internal/backupjob, internal/fileedit keep
// their own copies; maintenance_test pins them against these).
// on its Job (internal/restore, internal/backupjob, internal/fileedit and
// internal/worldexport keep their own copies; maintenance_test pins them
// against these).
LabelServer = "felis.lolicon.best/server"
LabelManagedBy = "app.kubernetes.io/managed-by"
// LabelFilesMode is the file operation (list, read, write, mkdir, delete,
// rename, upload) a files Job performs. Every one but list and read holds the
// volume.
LabelFilesMode = "felis.lolicon.best/files-mode"
// LabelExportMode is what an export Job archives: ExportModeWorld (the live
// world, which holds the volume) or ExportModeBackup (a stored archive).
LabelExportMode = "felis.lolicon.best/export-mode"
// LabelThenRestore marks a backup Job that is the safety snapshot in front of
// a restore. Its value is the chain's state: ThenRestorePending until felis-api
@@ -85,6 +92,7 @@ const (
KindRestore = "restore"
KindBackup = "backup"
KindFileWrite = "file-write"
KindExport = "export"
// KindReap is the reaper archiving an idle world and reclaiming its volume.
// It runs no Job: the reaper holds the Annotation itself and rewrites it
// well inside Grace for as long as it works on the world.
@@ -98,11 +106,18 @@ const (
FilesModeRead = "read"
)
// The LabelExportMode values.
const (
ExportModeWorld = "world"
ExportModeBackup = "backup"
)
// JobKind names the holder a Job represents, or reports false for a Job that
// holds nothing (a file read, a build, anything else in the namespace). A files
// Job holds unless it is a list or a read, so an operation this build does not
// know — and a Job without LabelFilesMode, which can only be an old one still
// inside its TTL — counts as a change: over-counting is the safe side.
// holds nothing (a file read, a backup export, a build, anything else in the
// namespace). A files Job holds unless it is a list or a read, so an operation
// this build does not know — and a Job without LabelFilesMode, which can only be
// an old one still inside its TTL — counts as a change: over-counting is the
// safe side. An export Job holds unless it names a backup, for the same reason.
func JobKind(j *batchv1.Job) (string, bool) {
switch j.Labels[LabelManagedBy] {
case "felis-restore":
@@ -113,6 +128,10 @@ func JobKind(j *batchv1.Job) (string, bool) {
if mode := j.Labels[LabelFilesMode]; mode != FilesModeList && mode != FilesModeRead {
return KindFileWrite, true
}
case "felis-export":
if j.Labels[LabelExportMode] != ExportModeBackup {
return KindExport, true
}
}
return "", false
}
+31
View File
@@ -7,6 +7,7 @@ import (
"felis.lolicon.best/internal/backupjob"
"felis.lolicon.best/internal/fileedit"
"felis.lolicon.best/internal/restore"
"felis.lolicon.best/internal/worldexport"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
)
@@ -66,6 +67,20 @@ func filesJob(t *testing.T, server, op string) batchv1.Job {
return *j
}
func exportJob(t *testing.T, server, mode string) batchv1.Job {
t.Helper()
j, err := worldexport.ExportJob(worldexport.JobParams{
Server: server, ID: "0011223344556677", Mode: mode, WorldPVC: "world-" + server + "-0",
BackupPVC: "felis-backups", BackupRef: "/backups/a.tar.gz", TargetURL: "http://api/x", Token: "t",
Namespace: "minecraft", ServiceAccount: "felis-restore", Image: "felis:1",
BackupRoot: "/backups", WorldsRoot: "/world",
})
if err != nil {
t.Fatalf("ExportJob: %v", err)
}
return *j
}
func finished(j batchv1.Job, cond batchv1.JobConditionType) batchv1.Job {
j.Status.Conditions = append(j.Status.Conditions, batchv1.JobCondition{Type: cond, Status: corev1.ConditionTrue})
return j
@@ -89,6 +104,8 @@ func TestJobKindMatchesTheExecutors(t *testing.T) {
{"file upload", filesJob(t, "survival", fileedit.OpUpload), KindFileWrite, true},
{"file read", filesJob(t, "survival", fileedit.OpRead), "", false},
{"file list", filesJob(t, "survival", fileedit.OpList), "", false},
{"world export", exportJob(t, "survival", worldexport.ModeWorld), KindExport, true},
{"backup export", exportJob(t, "survival", worldexport.ModeBackup), "", false},
} {
kind, ok := JobKind(&tc.job)
if kind != tc.kind || ok != tc.ok {
@@ -208,3 +225,17 @@ func TestHolderFromLock(t *testing.T) {
}
}
}
// An export Job that does not say it reads a backup holds, like a files Job
// with an unknown mode: a world export a newer build labels differently must
// not let the server start under it.
func TestExportJobWithoutModeHolds(t *testing.T) {
j := exportJob(t, "survival", worldexport.ModeBackup)
if j.Labels[LabelExportMode] != ExportModeBackup {
t.Fatalf("export mode label = %q", j.Labels[LabelExportMode])
}
delete(j.Labels, LabelExportMode)
if kind, ok := JobKind(&j); !ok || kind != KindExport {
t.Fatalf("an export Job without a mode = %q, %v; want export", kind, ok)
}
}
+2 -2
View File
@@ -344,8 +344,8 @@ func TestBackupReadBack(t *testing.T) {
if b, err := repo.BackupByID(ctx, newer); err != nil || !b.Corrupt {
t.Fatalf("BackupByID(corrupt) = (%+v, %v); want Corrupt", b, err)
}
if b, err := repo.BackupByID(ctx, older); err != nil || b.Corrupt {
t.Fatalf("BackupByID(intact) = (%+v, %v); want not Corrupt", b, err)
if b, err := repo.BackupByID(ctx, older); err != nil || b.Corrupt || b.SHA256 != "aa" {
t.Fatalf("BackupByID(intact) = (%+v, %v); want not Corrupt, sha256 aa", b, err)
}
views, _, err := repo.AllBackups(ctx, api.BackupListOpts{Server: name, Limit: api.MaxBackupListLimit})
+237
View File
@@ -0,0 +1,237 @@
package worldexport
import (
"fmt"
"time"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// Label keys applied to export objects, the same ones the other executors use
// (internal/maintenance and internal/api keep their own copies;
// maintenance_test pins them against these).
const (
LabelManagedBy = "app.kubernetes.io/managed-by"
LabelComponent = "app.kubernetes.io/component"
LabelServer = "felis.lolicon.best/server"
// LabelMode is what the Job archives (ModeWorld or ModeBackup). A world
// export holds the world volume; a backup export does not.
LabelMode = "felis.lolicon.best/export-mode"
managedByValue = "felis-export"
componentValue = "world-export"
worldVolume = "world"
backupVolume = "backup"
containerName = "export"
felisBinaryPath = "/usr/local/bin/felis"
)
// The two things an export can archive.
const (
ModeWorld = "world"
ModeBackup = "backup"
)
// TokenEnv carries the one-time upload token into the Pod. It is the only
// secret the Pod holds, so it rides the environment and not argv, where a
// process listing on the node would show it.
const TokenEnv = "FELIS_EXPORT_TOKEN"
// JobParams are the rendered inputs to an export Job. ExportJob is a pure
// function of them, so the Job shape is unit-tested without a cluster.
type JobParams struct {
Server string
// ID names this export: it is the tail of the Job name and of the internal
// upload path, so two exports of one server never collide.
ID string
Mode string
// WorldPVC is mounted for ModeWorld, BackupPVC and BackupRef for ModeBackup.
WorldPVC string
BackupPVC string
BackupRef string
// TargetURL is where the Pod PUTs the archive (felis-api's internal face),
// and Token the one-time bearer token that opens it.
TargetURL string
Token string
Namespace string
ServiceAccount string
Image string
BackupRoot string
WorldsRoot string
Deadline time.Duration
CPULimit string
MemLimit string
RunAsUser int64
RunAsGroup int64
FSGroup int64
TTLAfterFinished time.Duration
}
// JobName is the Job an export runs as.
func JobName(server, id string) string { return "export-" + server + "-" + id }
func exportLabels(p JobParams) map[string]string {
return map[string]string{
LabelManagedBy: managedByValue,
LabelComponent: componentValue,
LabelServer: p.Server,
LabelMode: p.Mode,
}
}
// ExportJob renders the export Job. Every isolation guarantee lives here and is
// asserted by jobspec_test.go:
//
// - the weak felis-restore SA with its token auto-mount disabled, so the Pod
// cannot reach the K8s API;
// - EXACTLY one volume, read-only in the claim and in the mount: the world PVC
// for a world export, the backup PVC for a backup export. No Secret or
// ConfigMap: the one credential in the Pod is the upload token, which opens
// this one export and nothing else;
// - root with DAC_OVERRIDE and nothing more: the world is the game uid's
// mode-0600 files, and Pod Security baseline, which the minecraft namespace
// enforces, admits DAC_OVERRIDE but not the narrower DAC_READ_SEARCH. No
// privilege or escalation, a read-only root filesystem;
// - backoffLimit 0 (the token is spent by the first attempt, a retry could
// only fail) and activeDeadlineSeconds, so a download left hanging cannot
// hold the world forever; ttlSecondsAfterFinished GCs the finished Job.
//
// The container runs `/usr/local/bin/felis export` (cmd/felis). The backup ref is
// an absolute path, so the backup PVC is mounted at BackupRoot, the path the
// archives were written under, as the restore Job mounts it.
func ExportJob(p JobParams) (*batchv1.Job, error) {
if p.Image == "" {
return nil, fmt.Errorf("worldexport: image is empty")
}
if p.ID == "" || p.TargetURL == "" || p.Token == "" {
return nil, fmt.Errorf("worldexport: an export needs an id, a target URL and a token")
}
var (
args = []string{"--mode", p.Mode, "--server", p.Server, "--target-url", p.TargetURL}
volume corev1.Volume
mount corev1.VolumeMount
)
switch p.Mode {
case ModeWorld:
if p.WorldPVC == "" {
return nil, fmt.Errorf("worldexport: world PVC name is required")
}
args = append(args, "--worlds-root", p.WorldsRoot)
volume = readOnlyClaim(worldVolume, p.WorldPVC)
mount = corev1.VolumeMount{Name: worldVolume, MountPath: p.WorldsRoot, ReadOnly: true}
case ModeBackup:
if p.BackupPVC == "" || p.BackupRef == "" {
return nil, fmt.Errorf("worldexport: backup PVC name and archive ref are required")
}
args = append(args, "--ref", p.BackupRef, "--backup-root", p.BackupRoot)
volume = readOnlyClaim(backupVolume, p.BackupPVC)
mount = corev1.VolumeMount{Name: backupVolume, MountPath: p.BackupRoot, ReadOnly: true}
default:
return nil, fmt.Errorf("worldexport: unknown mode %q", p.Mode)
}
limits, err := resourceLimits(p.CPULimit, p.MemLimit)
if err != nil {
return nil, err
}
deadline := int64(p.Deadline / time.Second)
if deadline <= 0 {
deadline = int64(defaultDeadline / time.Second)
}
ttl := int32(p.TTLAfterFinished / time.Second)
if ttl <= 0 {
ttl = int32(defaultTTL / time.Second)
}
container := corev1.Container{
Name: containerName,
Image: p.Image,
Command: []string{felisBinaryPath, "export"},
Args: args,
Env: []corev1.EnvVar{{Name: TokenEnv, Value: p.Token}},
VolumeMounts: []corev1.VolumeMount{mount},
Resources: corev1.ResourceRequirements{Limits: limits, Requests: limits},
SecurityContext: &corev1.SecurityContext{
Privileged: boolPtr(false),
AllowPrivilegeEscalation: boolPtr(false),
ReadOnlyRootFilesystem: boolPtr(true),
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"ALL"},
Add: []corev1.Capability{"DAC_OVERRIDE"},
},
},
// The exit error reaches the export's status and GET
// /servers/{name}/jobs through the terminated state.
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
}
sc := &corev1.PodSecurityContext{
RunAsNonRoot: boolPtr(false),
RunAsUser: int64Ptr(p.RunAsUser),
RunAsGroup: int64Ptr(p.RunAsGroup),
}
if p.FSGroup > 0 {
sc.FSGroup = int64Ptr(p.FSGroup)
}
return &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: JobName(p.Server, p.ID),
Namespace: p.Namespace,
Labels: exportLabels(p),
},
Spec: batchv1.JobSpec{
BackoffLimit: int32Ptr(0),
ActiveDeadlineSeconds: int64Ptr(deadline),
TTLSecondsAfterFinished: int32Ptr(ttl),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: exportLabels(p)},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyNever,
ServiceAccountName: p.ServiceAccount,
AutomountServiceAccountToken: boolPtr(false),
SecurityContext: sc,
Containers: []corev1.Container{container},
Volumes: []corev1.Volume{volume},
},
},
},
}, nil
}
func readOnlyClaim(name, claim string) corev1.Volume {
return corev1.Volume{
Name: name,
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: claim, ReadOnly: true},
},
}
}
// resourceLimits parses the CPU/memory limits into a ResourceList.
func resourceLimits(cpu, mem string) (corev1.ResourceList, error) {
if cpu == "" {
cpu = defaultCPULimit
}
if mem == "" {
mem = defaultMemLimit
}
cpuQty, err := resource.ParseQuantity(cpu)
if err != nil {
return nil, fmt.Errorf("worldexport: invalid cpu limit %q: %w", cpu, err)
}
memQty, err := resource.ParseQuantity(mem)
if err != nil {
return nil, fmt.Errorf("worldexport: invalid memory limit %q: %w", mem, err)
}
return corev1.ResourceList{corev1.ResourceCPU: cpuQty, corev1.ResourceMemory: memQty}, nil
}
func boolPtr(b bool) *bool { return &b }
func int32Ptr(i int32) *int32 { return &i }
func int64Ptr(i int64) *int64 { return &i }
+178
View File
@@ -0,0 +1,178 @@
package worldexport
import (
"context"
"slices"
"strings"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
const secretToken = "5ecret5ecret5ecret5ecret5ecret5ecret5ecret5ecret5ecret5ecret5ecr"
func params(mode string) JobParams {
return JobParams{
Server: "survival", ID: "0011223344556677", Mode: mode,
WorldPVC: "world-survival-0", BackupPVC: "felis-backups",
BackupRef: "/backups/survival-1.tar.gz",
TargetURL: "http://felis-api-internal.felis.svc.cluster.local:8081/api/v1/internal/exports/0011223344556677",
Token: secretToken,
Namespace: "minecraft", ServiceAccount: "felis-restore", Image: "felis:1",
BackupRoot: "/backups", WorldsRoot: "/world", Deadline: time.Hour,
CPULimit: "1", MemLimit: "256Mi", TTLAfterFinished: 5 * time.Minute,
}
}
// The export Pod reads a world or an archive and hands it to felis-api. It gets
// exactly the one volume it reads, read-only at both ends, no Secret, no API
// token, and no capability beyond DAC_OVERRIDE: a compromised export can read
// that one volume and talk to that one upload URL, nothing more.
func TestExportJobIsolation(t *testing.T) {
for _, tc := range []struct {
mode, volume, claim, mountPath string
args []string
}{
{ModeWorld, worldVolume, "world-survival-0", "/world", []string{"--worlds-root", "/world"}},
{ModeBackup, backupVolume, "felis-backups", "/backups", []string{"--ref", "/backups/survival-1.tar.gz", "--backup-root", "/backups"}},
} {
job, err := ExportJob(params(tc.mode))
if err != nil {
t.Fatalf("%s: ExportJob: %v", tc.mode, err)
}
if job.Name != "export-survival-0011223344556677" || job.Namespace != "minecraft" {
t.Errorf("%s: job = %s/%s", tc.mode, job.Namespace, job.Name)
}
for _, labels := range []map[string]string{job.Labels, job.Spec.Template.Labels} {
if labels[LabelManagedBy] != "felis-export" || labels[LabelServer] != "survival" || labels[LabelMode] != tc.mode {
t.Errorf("%s: labels = %v", tc.mode, labels)
}
}
spec := job.Spec.Template.Spec
if spec.ServiceAccountName != "felis-restore" {
t.Errorf("%s: service account = %q", tc.mode, spec.ServiceAccountName)
}
if spec.AutomountServiceAccountToken == nil || *spec.AutomountServiceAccountToken {
t.Errorf("%s: the SA token is mounted", tc.mode)
}
if spec.RestartPolicy != corev1.RestartPolicyNever {
t.Errorf("%s: restart policy = %q", tc.mode, spec.RestartPolicy)
}
if len(spec.Volumes) != 1 {
t.Fatalf("%s: volumes = %+v, want exactly one", tc.mode, spec.Volumes)
}
v := spec.Volumes[0]
if v.Name != tc.volume || v.PersistentVolumeClaim == nil || v.PersistentVolumeClaim.ClaimName != tc.claim || !v.PersistentVolumeClaim.ReadOnly {
t.Errorf("%s: volume = %+v, want %s read-only", tc.mode, v, tc.claim)
}
if len(spec.Containers) != 1 || len(spec.InitContainers) != 0 {
t.Fatalf("%s: containers = %d, init = %d", tc.mode, len(spec.Containers), len(spec.InitContainers))
}
c := spec.Containers[0]
if len(c.VolumeMounts) != 1 || c.VolumeMounts[0] != (corev1.VolumeMount{Name: tc.volume, MountPath: tc.mountPath, ReadOnly: true}) {
t.Errorf("%s: mounts = %+v", tc.mode, c.VolumeMounts)
}
if len(c.EnvFrom) != 0 || len(c.Env) != 1 || c.Env[0] != (corev1.EnvVar{Name: TokenEnv, Value: secretToken}) {
t.Errorf("%s: env = %+v, envFrom = %+v; want only the token", tc.mode, c.Env, c.EnvFrom)
}
if strings.Contains(strings.Join(c.Args, " "), secretToken) {
t.Errorf("%s: the token rides argv: %v", tc.mode, c.Args)
}
wantArgs := append([]string{"--mode", tc.mode, "--server", "survival", "--target-url", params(tc.mode).TargetURL}, tc.args...)
if !slices.Equal(c.Command, []string{felisBinaryPath, "export"}) || !slices.Equal(c.Args, wantArgs) {
t.Errorf("%s: command = %v %v", tc.mode, c.Command, c.Args)
}
sc := c.SecurityContext
if sc == nil || sc.Privileged == nil || *sc.Privileged || sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation ||
sc.ReadOnlyRootFilesystem == nil || !*sc.ReadOnlyRootFilesystem {
t.Errorf("%s: container security context = %+v", tc.mode, sc)
}
if sc.Capabilities == nil || !slices.Equal(sc.Capabilities.Drop, []corev1.Capability{"ALL"}) ||
!slices.Equal(sc.Capabilities.Add, []corev1.Capability{"DAC_OVERRIDE"}) {
t.Errorf("%s: capabilities = %+v", tc.mode, sc.Capabilities)
}
if c.TerminationMessagePolicy != corev1.TerminationMessageFallbackToLogsOnError {
t.Errorf("%s: termination message policy = %q", tc.mode, c.TerminationMessagePolicy)
}
if psc := spec.SecurityContext; psc == nil || psc.RunAsUser == nil || *psc.RunAsUser != 0 || psc.FSGroup != nil {
t.Errorf("%s: pod security context = %+v", tc.mode, psc)
}
if b := job.Spec.BackoffLimit; b == nil || *b != 0 {
t.Errorf("%s: backoffLimit = %v, want 0", tc.mode, b)
}
if d := job.Spec.ActiveDeadlineSeconds; d == nil || *d != 3600 {
t.Errorf("%s: activeDeadlineSeconds = %v, want 3600", tc.mode, d)
}
if ttl := job.Spec.TTLSecondsAfterFinished; ttl == nil || *ttl != 300 {
t.Errorf("%s: ttlSecondsAfterFinished = %v, want 300", tc.mode, ttl)
}
}
}
func TestExportJobRefusesIncompleteParams(t *testing.T) {
for name, edit := range map[string]func(*JobParams){
"no image": func(p *JobParams) { p.Image = "" },
"no token": func(p *JobParams) { p.Token = "" },
"no target": func(p *JobParams) { p.TargetURL = "" },
"no id": func(p *JobParams) { p.ID = "" },
"unknown mode": func(p *JobParams) { p.Mode = "both" },
"world without claim": func(p *JobParams) { p.WorldPVC = "" },
} {
p := params(ModeWorld)
edit(&p)
if _, err := ExportJob(p); err == nil {
t.Errorf("%s: ExportJob accepted %+v", name, p)
}
}
p := params(ModeBackup)
p.BackupRef = ""
if _, err := ExportJob(p); err == nil {
t.Error("a backup export without a ref was accepted")
}
}
// TestExportJobDefaults: a caller that leaves the deadline and the TTL unset
// still gets a Job that ends and is collected.
func TestExportJobDefaults(t *testing.T) {
p := params(ModeWorld)
p.Deadline, p.TTLAfterFinished = 0, 0
job, err := ExportJob(p)
if err != nil {
t.Fatal(err)
}
if d := job.Spec.ActiveDeadlineSeconds; d == nil || *d != 7200 {
t.Errorf("activeDeadlineSeconds = %v, want 7200", d)
}
if ttl := job.Spec.TTLSecondsAfterFinished; ttl == nil || *ttl != 600 {
t.Errorf("ttlSecondsAfterFinished = %v, want 600", ttl)
}
}
func TestStartCreatesTheJob(t *testing.T) {
cs := fake.NewSimpleClientset()
e := New(cs, Config{Image: "felis:1", BackupPVC: "felis-backups"})
name, err := e.Start(context.Background(), Request{
Server: "survival", Mode: ModeWorld, ID: "0011223344556677",
TargetURL: "http://api:8081/api/v1/internal/exports/0011223344556677", Token: secretToken,
})
if err != nil || name != "export-survival-0011223344556677" {
t.Fatalf("Start = %q, %v", name, err)
}
job, err := cs.BatchV1().Jobs("minecraft").Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
t.Fatalf("the Job is not in the minecraft namespace: %v", err)
}
if claim := job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName; claim != "world-survival-0" {
t.Errorf("world claim = %q, want world-survival-0", claim)
}
if d := *job.Spec.ActiveDeadlineSeconds; d != int64(defaultDeadline/time.Second) {
t.Errorf("deadline = %d, want the %s default", d, defaultDeadline)
}
if _, err := e.Start(context.Background(), Request{Server: "survival", Mode: ModeWorld, ID: "0011223344556677",
TargetURL: "http://api:8081/x", Token: secretToken}); err == nil {
t.Error("a second Job of the same name was reported as created")
}
}
+125
View File
@@ -0,0 +1,125 @@
// Package worldexport is the executor behind the world export routes (POST
// /servers/{name}/world/export and POST /servers/{name}/backups/{id}/export): a
// one-shot Job in the minecraft namespace that reads a stopped server's world,
// or one of its stored archives, and PUTs the tar.gz to felis-api's internal
// face, which streams it on to the owner's browser as it arrives
// (internal/api/exports.go). Nothing is staged on the way: felis-api never
// mounts a world or the backup store, and the archive never lands on a disk it
// owns.
//
// Trust model as in internal/restore: the Pod runs under the weak felis-restore
// SA with no API token, mounts one volume read-only, and holds no database URL
// or Secret. The only credential it gets is the one-time token of this one
// upload. felis-api makes every authorization decision before the Job exists.
package worldexport
import (
"context"
"fmt"
"time"
"felis.lolicon.best/internal/naming"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
// Request is one export as felis-api admitted it.
type Request struct {
Server string
Mode string // ModeWorld or ModeBackup
// BackupRef is the archive a ModeBackup export reads.
BackupRef string
// ID names the export (JobName) and TargetURL/Token are where and how the
// Pod hands the archive over.
ID string
TargetURL string
Token string
}
// Config parameterises the executor. Image and BackupPVC have no safe default:
// cmd/felis leaves the API's Exporter nil when either is missing, and the
// routes answer 503.
type Config struct {
Namespace string
ServiceAccount string
Image string
BackupPVC string
// BackupRoot MUST be the path the archives were written under
// (cfg.Archive.LocalPath): the stored refs are absolute paths.
BackupRoot string
WorldsRoot string
// Deadline caps the Pod's wall-clock. The archive moves at the browser's
// pace, so it is longer than a backup's, and it is also the longest a world
// export can keep the server from starting.
Deadline time.Duration
CPULimit string
MemLimit string
RunAsUser int64
RunAsGroup int64
FSGroup int64
TTLAfterFinished time.Duration
}
const (
defaultNamespace = "minecraft"
defaultServiceAccount = "felis-restore"
defaultBackupRoot = "/backups"
defaultWorldsRoot = "/world"
defaultDeadline = 2 * time.Hour
defaultCPULimit = "1"
defaultMemLimit = "256Mi"
defaultTTL = 10 * time.Minute
)
func (c Config) withDefaults() Config {
if c.Namespace == "" {
c.Namespace = defaultNamespace
}
if c.ServiceAccount == "" {
c.ServiceAccount = defaultServiceAccount
}
if c.BackupRoot == "" {
c.BackupRoot = defaultBackupRoot
}
if c.WorldsRoot == "" {
c.WorldsRoot = defaultWorldsRoot
}
if c.Deadline <= 0 {
c.Deadline = defaultDeadline
}
return c
}
// Exporter is the production internal/api.Exporter. It creates the Job with
// jobs:create, which felis-api already holds in the minecraft namespace.
type Exporter struct {
cs kubernetes.Interface
cfg Config
}
// New builds an Exporter over the typed clientset.
func New(cs kubernetes.Interface, cfg Config) *Exporter {
return &Exporter{cs: cs, cfg: cfg.withDefaults()}
}
// Start creates the export Job for r and returns its name.
func (e *Exporter) Start(ctx context.Context, r Request) (string, error) {
c := e.cfg
job, err := ExportJob(JobParams{
Server: r.Server, ID: r.ID, Mode: r.Mode,
WorldPVC: naming.WorldPVCName(r.Server), BackupPVC: c.BackupPVC, BackupRef: r.BackupRef,
TargetURL: r.TargetURL, Token: r.Token,
Namespace: c.Namespace, ServiceAccount: c.ServiceAccount, Image: c.Image,
BackupRoot: c.BackupRoot, WorldsRoot: c.WorldsRoot, Deadline: c.Deadline,
CPULimit: c.CPULimit, MemLimit: c.MemLimit,
RunAsUser: c.RunAsUser, RunAsGroup: c.RunAsGroup, FSGroup: c.FSGroup,
TTLAfterFinished: c.TTLAfterFinished,
})
if err != nil {
return "", err
}
if _, err := e.cs.BatchV1().Jobs(c.Namespace).Create(ctx, job, metav1.CreateOptions{}); err != nil {
return "", fmt.Errorf("worldexport: create export job: %w", err)
}
return job.Name, nil
}
+17 -2
View File
@@ -15,19 +15,34 @@
"backup_now": "Back up now",
"backup_in_progress": "Starting backup…",
"backup_started": "Backup started — its progress shows under recent operations below.",
"backup_world_busy": "A backup or restore is already running; back up once it finishes.",
"backup_world_busy": "A backup, restore or world export is already running; back up once it finishes.",
"backup_requires_stopped": "Stop the server before backing it up.",
"export_world": "Export world",
"export_world_hint": "Download the server's current world as a .tar.gz archive.",
"export_preparing": "Preparing…",
"export_requires_stopped": "Stop the server before exporting its world.",
"export_world_busy": "A backup, restore or export is running on this world; export once it finishes.",
"export_another": "Another download is being prepared; wait for it to start.",
"export_progress_world": "Archiving the world — the download starts by itself once it is ready. Keep this page open; the server can't start until the download ends.",
"export_progress_backup": "Preparing the backup — the download starts by itself in a moment. Keep this page open.",
"export_started_world": "Download started: your browser is saving {{filename}}. The server can start again once it finishes.",
"export_started_backup": "Download started: your browser is saving {{filename}}.",
"export_failed": "The export failed and nothing was downloaded.",
"export_failed_because": "The export failed: {{reason}}",
"download_btn": "Download backup",
"expires_in": "Expires {{when}}",
"expired": "Expired",
"former_owner": "Former owner: {{owner}}",
"empty_title": "No backups for this server",
"empty_hint": "A scheduled backup is saved once the server has run and stopped — or create one now with “Back up now”.",
"jobs_title": "Recent operations",
"jobs_empty": "No recent backup or restore operations.",
"jobs_empty": "No recent backup, restore or export operations.",
"job_backup": "Backup",
"job_restore": "Restore",
"job_pre_restore": "Pre-restore snapshot",
"job_scheduled": "Scheduled backup",
"job_export_world": "World export",
"job_export_backup": "Backup download",
"job_running": "Running",
"job_succeeded": "Succeeded",
"job_failed": "Failed",
+5 -1
View File
@@ -25,13 +25,17 @@
"no_backup": "There's no restorable backup for this server yet.",
"backup_corrupt": "This backup failed its read-back check and can't be restored intact — pick another backup.",
"not_stopped": "Stop the server completely first — this changes its world volume, which the running server holds.",
"maintenance_in_progress": "This server's world is busy with a restore, backup, file write or idle reclaim — try again once it finishes. A restore, backup or file write usually takes a minute or two; an idle reclaim can take longer on a large world.",
"maintenance_in_progress": "This server's world is busy with a restore, backup, world export, file write or idle reclaim — try again once it finishes. A restore, backup or file write usually takes a minute or two; a world export lasts until its download ends, and an idle reclaim can take longer on a large world.",
"no_world_volume": "This server has no world volume yet — start it once so it is created, then retry.",
"file_changed": "This file changed after you opened it (another manager saved it, or the server rewrote it on its last run), so your save was not written, to keep that change.",
"volume_full": "The server's volume is full, so the change was not written and the files there are unchanged. Delete files it no longer needs, or ask an admin to grow its volume.",
"backup_cooldown": "This server was backed up moments ago, and manual backups have a cooldown — try again in a few minutes.",
"backup_store_full": "The backup store is full, so manual backups are paused — ask an administrator to free space.",
"restore_unavailable": "Restore isn't available right now — try again later.",
"export_busy": "Too many downloads are being prepared right now (one at a time per person, six an hour) — try again in a few minutes.",
"export_expired": "This download has expired or was already used — start the export again.",
"export_not_ready": "The download isn't ready yet — wait a moment and try again.",
"export_unavailable": "Downloading worlds and backups isn't set up on this deployment — ask an administrator.",
"session_expired": "Your session expired — please sign in again.",
"forbidden": "You are not allowed to do that.",
"self_protected": "You can't do that to the account you're signed in with.",
+17 -2
View File
@@ -15,19 +15,34 @@
"backup_now": "立即备份",
"backup_in_progress": "正在启动备份……",
"backup_started": "已开始备份——进度可在下方“最近操作”中查看。",
"backup_world_busy": "已有备份或回档在进行,等它结束再备份。",
"backup_world_busy": "已有备份、恢复或世界导出在进行,等它结束再备份。",
"backup_requires_stopped": "备份前需要先停止服务器。",
"export_world": "导出世界",
"export_world_hint": "把服务器当前的世界打包成 .tar.gz 下载。",
"export_preparing": "准备中……",
"export_requires_stopped": "导出世界前需要先停止服务器。",
"export_world_busy": "这个世界正在备份、恢复或导出,等它结束再导出。",
"export_another": "另一个下载正在准备,等它开始后再试。",
"export_progress_world": "正在打包世界——准备好后会自动开始下载。请保持此页面打开;下载结束前服务器无法启动。",
"export_progress_backup": "正在准备备份——稍后会自动开始下载。请保持此页面打开。",
"export_started_world": "已开始下载:浏览器正在保存 {{filename}}。下载结束后服务器即可再次启动。",
"export_started_backup": "已开始下载:浏览器正在保存 {{filename}}。",
"export_failed": "导出失败,没有下载任何内容。",
"export_failed_because": "导出失败:{{reason}}",
"download_btn": "下载备份",
"expires_in": "{{when}}过期",
"expired": "已过期",
"former_owner": "原世界所有者:{{owner}}",
"empty_title": "暂无备份",
"empty_hint": "服务器运行过并停止后会自动生成定时备份,也可以点“立即备份”马上创建。",
"jobs_title": "最近操作",
"jobs_empty": "暂无备份或恢复操作记录。",
"jobs_empty": "暂无备份、恢复或导出操作记录。",
"job_backup": "备份",
"job_restore": "恢复",
"job_pre_restore": "恢复前快照",
"job_scheduled": "定时备份",
"job_export_world": "世界导出",
"job_export_backup": "备份下载",
"job_running": "进行中",
"job_succeeded": "成功",
"job_failed": "失败",
+5 -1
View File
@@ -25,13 +25,17 @@
"no_backup": "这台服务器暂时没有可回档的备份。",
"backup_corrupt": "这份备份回读校验未通过,已无法完整恢复——请选择另一份备份。",
"not_stopped": "请先把服务器完全停止——这项操作要改动世界存储卷,运行中的服务器独占着它。",
"maintenance_in_progress": "这台服务器的世界正在回档、备份、写入文件或闲置回收——等它完成后再试。回档、备份和写文件通常一两分钟,闲置回收视世界大小可能更久。",
"maintenance_in_progress": "这台服务器的世界正在回档、备份、导出、写入文件或闲置回收——等它完成后再试。回档、备份和写文件通常一两分钟,导出要等下载结束,闲置回收视世界大小可能更久。",
"no_world_volume": "这台服务器还没有世界卷——先启动一次让它创建,然后再试。",
"file_changed": "这个文件在你打开之后被改过了(另一位管理者保存过,或者服务器上次运行时改写了它)。为了不覆盖那次修改,这次保存没有写入。",
"volume_full": "服务器的存储卷已满,这次改动没有写入,原有文件保持不变。删掉用不着的文件,或者请管理员给它扩容。",
"backup_cooldown": "这台服务器刚备份过,手动备份之间有冷却时间——请过几分钟再试。",
"backup_store_full": "备份存储已满,暂时无法手动备份——请联系管理员清理空间。",
"restore_unavailable": "回档功能当前不可用,请稍后再试。",
"export_busy": "现在正在准备的下载太多了(每人同时一个、每小时最多六个)——请过几分钟再试。",
"export_expired": "这个下载已过期或已经用过了——请重新导出。",
"export_not_ready": "下载还没准备好——请稍等片刻再试。",
"export_unavailable": "当前部署没有开通世界和备份下载——请联系管理员。",
"session_expired": "会话已过期——请重新登录。",
"forbidden": "你无权执行此操作。",
"self_protected": "不能对当前登录的账号执行此操作。",
+70 -1
View File
@@ -7,8 +7,10 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
// falsy, and EVERY admin silently renders as a non-admin while typecheck/build stay
// green. This fixture is the canonical mirror of handlers_user.go handleMe.
// apiBase stays "" (paths read bare) unless a test that builds a whole URL sets it.
const cfg = vi.hoisted(() => ({ apiBase: "" }));
vi.mock("./config", () => ({
loadConfig: async () => ({ apiBase: "", rootDomain: "example.test" }),
loadConfig: async () => ({ apiBase: cfg.apiBase, rootDomain: "example.test" }),
}));
// Imported after the mock so api.ts picks up the mocked loadConfig.
@@ -1600,3 +1602,70 @@ describe("copy for the generic server codes", () => {
);
});
});
describe("world export wire shapes", () => {
beforeEach(() => vi.restoreAllMocks());
afterEach(() => {
vi.unstubAllGlobals();
cfg.apiBase = "";
});
function sent(fetchSpy: typeof fetch): [string, RequestInit] {
const [url, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
return [String(url), opts as RequestInit];
}
const ticket = { ticket: "ab".repeat(32), state: "pending", filename: "survival-backup-bk1.tar.gz" };
it("startBackupExport POSTs to /servers/{name}/backups/{id}/export with no body", async () => {
const fetchSpy = fakeFetch(ticket, { status: 202 });
vi.stubGlobal("fetch", fetchSpy);
expect(await api.startBackupExport("survival", "bk/1")).toEqual(ticket);
const [url, opts] = sent(fetchSpy);
expect(url).toBe("/servers/survival/backups/bk%2F1/export");
expect(opts.method).toBe("POST");
expect(opts.body).toBeUndefined();
});
it("startWorldExport POSTs to /servers/{name}/world/export", async () => {
const fetchSpy = fakeFetch(ticket, { status: 202 });
vi.stubGlobal("fetch", fetchSpy);
await api.startWorldExport("survival");
const [url, opts] = sent(fetchSpy);
expect(url).toBe("/servers/survival/world/export");
expect(opts.method).toBe("POST");
});
it("exportStatus GETs /exports/{ticket}", async () => {
const fetchSpy = fakeFetch({ state: "failed", message: "felis export: no such file" });
vi.stubGlobal("fetch", fetchSpy);
expect(await api.exportStatus("t/1")).toEqual({ state: "failed", message: "felis export: no such file" });
const [url, opts] = sent(fetchSpy);
expect(url).toBe("/exports/t%2F1");
expect(opts.method).toBe("GET");
});
it("exportDownloadURL is the full download address, and sends nothing", async () => {
cfg.apiBase = "https://api.example.test/api/v1";
const fetchSpy = fakeFetch({});
vi.stubGlobal("fetch", fetchSpy);
expect(await api.exportDownloadURL("t/1")).toBe("https://api.example.test/api/v1/exports/t%2F1/download");
await expect(api.exportDownloadURL("..")).rejects.toMatchObject({ code: "bad_path_param" });
expect(fetchSpy).not.toHaveBeenCalled();
});
it("words the export refusals itself", () => {
expect(humanizeError({ status: 429, code: "export_busy", message: "raw" })).toBe(
"Too many downloads are being prepared right now (one at a time per person, six an hour) — try again in a few minutes.",
);
expect(humanizeError({ status: 410, code: "export_expired", message: "raw" })).toBe(
"This download has expired or was already used — start the export again.",
);
expect(humanizeError({ status: 409, code: "export_not_ready", message: "raw" })).toBe(
"The download isn't ready yet — wait a moment and try again.",
);
expect(humanizeError({ status: 503, code: "export_unavailable", message: "raw" })).toBe(
"Downloading worlds and backups isn't set up on this deployment — ask an administrator.",
);
});
});
+36
View File
@@ -10,6 +10,8 @@ import type {
ContextUploadProgress,
CreateServerRequest,
CreateUserRequest,
ExportStatus,
ExportTicket,
FleetServer,
Identity,
KickResult,
@@ -673,6 +675,29 @@ export const api = rejectingSync({
backupNow: (name: string) =>
request<{ name: string; status: string }>("POST", urlPath`/servers/${name}/backup`),
// World export: a copy of one backup (startBackupExport, the same scope as
// delete and restore) or of a stopped server's current world (startWorldExport,
// 409 not_stopped otherwise; the world is held until the download ends). The
// reply is a one-time ticket bound to the caller. An export Job then hands the
// archive to felis-api: exportStatus reads "pending" until it has, and once
// "ready" the browser must open exportDownloadURL within 90 s. The download
// streams through felis-api, so it is opened as a link (an <a download>) and
// never read into memory. 429 export_busy past the limits (one per person at a
// time, two across the platform, six an hour); 410 export_expired once the
// ticket was used or its time ran out.
startBackupExport: (name: string, id: string) =>
request<ExportTicket>("POST", urlPath`/servers/${name}/backups/${id}/export`),
startWorldExport: (name: string) =>
request<ExportTicket>("POST", urlPath`/servers/${name}/world/export`),
exportStatus: (ticket: string) => request<ExportStatus>("GET", urlPath`/exports/${ticket}`),
exportDownloadURL: async (ticket: string) => {
const { apiBase } = await loadConfig();
return `${apiBase}${urlPath`/exports/${ticket}/download`}`;
},
// serverJobs lists the newest backup/restore Jobs of one server, newest first
// (GET /servers/{name}/jobs). Owner-or-admin gated server-side; a Job's
// failure text rides `message`. The backend answers 503 until the job-status
@@ -1223,6 +1248,17 @@ export function humanizeError(e: unknown): string {
return t("backup_store_full");
case "restore_unavailable":
return t("restore_unavailable");
// World export: over the per-person, platform or hourly limit (429), a
// ticket already used or past its time (410), a download asked for before
// the archive arrived (409), or export not wired on this deployment (503).
case "export_busy":
return t("export_busy");
case "export_expired":
return t("export_expired");
case "export_not_ready":
return t("export_not_ready");
case "export_unavailable":
return t("export_unavailable");
// Server create/edit (spec §22): the portability regex + reservation list are
// enforced server-side, and the create form's own checks are weaker, so these
// refusals reach the dialog as-is.
+373 -4
View File
@@ -159,6 +159,26 @@ export interface paths {
patch?: never;
trace?: never;
};
"/api/v1/internal/exports/{id}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
/**
* Hand one export's archive over for download (one-time bearer token).
* @description The export Job PUTs the tar.gz here, chunked for a world and with its Content-Length for a backup. The Job holds no service token, so the route is public on the internal face and the bearer token minted with the export is the whole check; an unknown id, a wrong or missing token and a token already used are all the same 404. The request then waits, body unread, up to 90 seconds for the owner's browser to open the download, and is read at the browser's pace: the 16 KiB/s minimum body rate does not apply, and the body fails only after 2 minutes without a byte. It answers once the download has ended.
*/
put: operations["internalExportUpload"];
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/internal/servers/{name}/join-event": {
parameters: {
query?: never;
@@ -1254,6 +1274,86 @@ export interface paths {
patch?: never;
trace?: never;
};
"/api/v1/servers/{name}/backups/{id}/export": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Start downloading one backup (owner-or-admin plus a former-owner match).
* @description Starts a Job that reads the archive from the backup store and hands it to felis-api, which streams it to the browser (poll GET /exports/{ticket}, then open its download). The archive is checked against the sha256 recorded when it was written as it streams; a mismatch aborts the download. A user gets 404 for a backup outside their scope, as their list never shows it. One export per user at a time, 2 across the install, 6 per user per hour.
*/
post: operations["exportBackup"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/servers/{name}/world/export": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Start downloading a stopped server's world as it is now (owner-or-admin).
* @description Starts a Job that archives the server's data volume, read-only, and hands it to felis-api, which streams it to the browser (poll GET /exports/{ticket}, then open its download). The server must be fully stopped, and it cannot start until the download has ended or the Job's 2 hour deadline passes. Same limits as a backup export.
*/
post: operations["exportWorld"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/exports/{ticket}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Where an export stands (the user who started it only).
* @description The panel polls this until the state reads ready, then opens the download. Another user's ticket is 404, as an unknown one is.
*/
get: operations["exportStatus"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/exports/{ticket}/download": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Download a ready export (once, by the user who started it).
* @description The first request spends the ticket, whatever becomes of it. The archive streams as the Job sends it, with Content-Length when it is known; a download that cannot finish (the Job died, or a backup did not match its recorded sha256) is cut off, so the browser reports it failed. HEAD is refused, since it would spend the ticket on no body.
*/
get: operations["exportDownload"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/servers/{name}/jobs": {
parameters: {
query?: never;
@@ -1262,8 +1362,8 @@ export interface paths {
cookie?: never;
};
/**
* Latest async world operations (backup/restore) for a server (owner-or-admin).
* @description Backup and restore run as cluster Jobs, so a 202 that later failed left its only trace in the Job object. This route projects the newest such Jobs, newest first, so failures are observable without kubectl. State is "running" | "succeeded" | "failed".
* Latest async world operations (backup, restore, export) for a server (owner-or-admin).
* @description Backup, restore and export run as cluster Jobs, so a 202 that later failed left its only trace in the Job object. This route projects the newest such Jobs, newest first, so failures are observable without kubectl. State is "running" | "succeeded" | "failed".
*/
get: operations["listServerJobs"];
put?: never;
@@ -2799,6 +2899,25 @@ export interface components {
/** @description World entries the archive could not hold (symbolic links, devices, sockets). Omitted when zero. */
skipped_entries?: number;
};
/** @description An export just started (internal/api/exports.go exportTicketView). The ticket opens GET /exports/{ticket} and its download for the user who started it, and nobody else. */
ExportTicket: {
/** @description 64 hex characters. */
ticket: string;
/** @constant */
state: "pending";
/** @description What the download saves as. */
filename: string;
};
/** @description Where an export stands (internal/api/exports.go exportStatusView). */
ExportStatus: {
/**
* @description pending while its Job starts; ready once the archive waits for the download, which must begin within 90 seconds; failed when the Job died first.
* @enum {string}
*/
state: "pending" | "ready" | "failed";
/** @description Why a failed export's Job died. Omitted otherwise. */
message?: string;
};
/** @description One image build (internal/build Build). */
Build: {
id: string;
@@ -3491,6 +3610,46 @@ export interface operations {
404: components["responses"]["NotFound"];
};
};
internalExportUpload: {
parameters: {
query?: never;
header: {
/** @description Bearer followed by the token minted with the export. */
Authorization: string;
};
path: {
id: string;
};
cookie?: never;
};
requestBody: {
content: {
"application/gzip": string;
};
};
responses: {
204: components["responses"]["NoContent"];
404: components["responses"]["NotFound"];
/** @description The backup did not match the sha256 recorded when it was written, and the download was aborted (backup_corrupt). */
409: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
/** @description Nobody opened the download within 90 seconds, or the browser left before the archive ended (export_expired). */
410: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
};
};
joinEvent: {
parameters: {
query?: never;
@@ -6132,6 +6291,216 @@ export interface operations {
507: components["responses"]["InsufficientStorage"];
};
};
exportBackup: {
parameters: {
query?: never;
header?: never;
path: {
name: string;
id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Export started. */
202: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ExportTicket"];
};
};
/** @description Malformed server name (bad_name). */
400: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
401: components["responses"]["Unauthorized"];
403: components["responses"]["Forbidden"];
/** @description Unknown server, or no present backup with this id in the caller's scope (no_backup). */
404: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
/** @description The backup failed a read-back (backup_corrupt). */
409: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
/** @description An export limit is reached (export_busy); Retry-After gives the seconds to wait. */
429: {
headers: {
"Retry-After"?: number;
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
503: components["responses"]["ServiceUnavailable"];
};
};
exportWorld: {
parameters: {
query?: never;
header?: never;
path: {
name: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Export started. */
202: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ExportTicket"];
};
};
/** @description Malformed server name (bad_name). */
400: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
401: components["responses"]["Unauthorized"];
403: components["responses"]["Forbidden"];
/** @description Unknown server. */
404: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
/** @description Server is not stopped (not_stopped), has no world volume yet (no_world_volume), or a restore, backup, file write or export already holds its world volume (maintenance_in_progress). */
409: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
/** @description An export limit is reached (export_busy); Retry-After gives the seconds to wait. */
429: {
headers: {
"Retry-After"?: number;
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
503: components["responses"]["ServiceUnavailable"];
};
};
exportStatus: {
parameters: {
query?: never;
header?: never;
path: {
ticket: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description The export's state. */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ExportStatus"];
};
};
401: components["responses"]["Unauthorized"];
404: components["responses"]["NotFound"];
/** @description The export was downloaded, or expired before it was (export_expired). */
410: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
};
};
exportDownload: {
parameters: {
query?: never;
header?: never;
path: {
ticket: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description The tar.gz, as an attachment. */
200: {
headers: {
"Content-Disposition"?: string;
[name: string]: unknown;
};
content: {
"application/gzip": string;
};
};
401: components["responses"]["Unauthorized"];
404: components["responses"]["NotFound"];
/** @description A method other than GET (method_not_allowed). */
405: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
/** @description The export is still pending (export_not_ready). */
409: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
/** @description The export was downloaded, failed, or expired before it was (export_expired). */
410: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["Error"];
};
};
};
};
listServerJobs: {
parameters: {
query?: never;
@@ -6143,7 +6512,7 @@ export interface operations {
};
requestBody?: never;
responses: {
/** @description The server's newest backup/restore jobs. */
/** @description The server's newest backup, restore and export jobs. */
200: {
headers: {
[name: string]: unknown;
@@ -6154,7 +6523,7 @@ export interface operations {
jobs: {
name: string;
/** @enum {string} */
kind: "backup" | "restore";
kind: "backup" | "restore" | "export_world" | "export_backup";
/** @enum {string} */
state: "running" | "succeeded" | "failed";
message?: string;
+2
View File
@@ -39,6 +39,8 @@ export type WireParity = [
Holds<Parity<T.Schedule, S["Schedule"]>>,
Holds<Parity<T.ScheduleInput, S["ScheduleInput"]>>,
Holds<Parity<T.BackupView, S["BackupView"]>>,
Holds<Parity<T.ExportTicket, S["ExportTicket"]>>,
Holds<Parity<T.ExportStatus, S["ExportStatus"]>>,
Holds<Parity<T.Build, S["Build"]>>,
Holds<Parity<T.BuildScan, S["BuildScan"]>>,
Holds<Parity<T.ScanSummary, S["ScanSummary"]>>,
+27 -3
View File
@@ -227,11 +227,13 @@ export interface BackupView {
}
/** ServerJob is one row of GET /api/v1/servers/{name}/jobs — the observable
* outcome of an async backup/restore Job. The API only enqueues Jobs, so this
* projection is how a 202 that later failed becomes visible in the panel. */
* outcome of an async backup/restore/export Job. The API only enqueues Jobs, so
* this projection is how a 202 that later failed becomes visible in the panel.
* An export_world Job holds the world until its download ends; an
* export_backup Job only reads the backup store. */
export interface ServerJob {
name: string;
kind: string; // "backup" | "restore"
kind: string; // "backup" | "restore" | "export_world" | "export_backup"
state: string; // "running" | "succeeded" | "failed"
message?: string;
started_at?: string;
@@ -247,6 +249,28 @@ export interface ServerJob {
scheduled?: boolean;
}
// ---- World export (internal/api/exports.go exportTicketView, exportStatusView) ----
/** ExportTicket is the 202 of POST /servers/{name}/backups/{id}/export and
* /servers/{name}/world/export: the one-time handle of a download being
* prepared. Only the viewer who asked can read or use it, and it opens the
* download once. `filename` is what the browser saves. */
export interface ExportTicket {
ticket: string;
state: "pending";
filename: string;
}
/** ExportStatus is GET /exports/{ticket}: "pending" while the export Job is on
* its way, "ready" once felis-api holds the archive and waits (90 s) for the
* browser to fetch it, "failed" when the Job gave up, with its reason in
* message. A ticket already downloaded or past its time answers 410
* export_expired instead. */
export interface ExportStatus {
state: "pending" | "ready" | "failed";
message?: string;
}
// ---- Scheduled tasks (internal/api/schedules.go Schedule, scheduleInput) ----
export type ScheduleAction = "command" | "restart" | "stop" | "start" | "backup";
+243 -1
View File
@@ -16,6 +16,10 @@ const calls = vi.hoisted(() => ({
stop: vi.fn(),
restoreBackup: vi.fn(),
deleteBackup: vi.fn(),
startBackupExport: vi.fn(),
startWorldExport: vi.fn(),
exportStatus: vi.fn(),
exportDownloadURL: vi.fn(),
}));
vi.mock("@/lib/tier", () => ({
useTier: () => ({
@@ -203,7 +207,7 @@ describe("ServerBackups back up now", () => {
renderPage();
await screen.findByText("Page 1 of 3");
await vi.waitFor(() => expect(backUp().disabled).toBe(true));
expect(backUp().title).toBe("A backup or restore is already running; back up once it finishes.");
expect(backUp().title).toBe("A backup, restore or world export is already running; back up once it finishes.");
});
it("keeps the page when a reread fails, and says the status may be stale", async () => {
@@ -369,3 +373,241 @@ describe("ServerBackups delete", () => {
expect(screen.queryByText(/^Page \d+ of/)).toBeNull();
});
});
describe("ServerBackups export", () => {
const row = (when: string) => screen.getByText(when).closest("tr") as HTMLElement;
const downloadIn = (when: string) =>
within(row(when)).getByRole("button", { name: "Download backup" }) as HTMLButtonElement;
const exportWorld = () => screen.getByRole("button", { name: /^(Export world|Preparing…)$/ }) as HTMLButtonElement;
const backUp = () => screen.getByRole("button", { name: "Back up now" }) as HTMLButtonElement;
const text = (key: string, opts?: Record<string, string>) => i18next.t(key, opts);
// Every hidden link the page clicks, as it was at the click.
function watchLinks() {
const clicked: { href: string | null; download: string; hidden: boolean; connected: boolean }[] = [];
vi.spyOn(HTMLAnchorElement.prototype, "click").mockImplementation(function (this: HTMLAnchorElement) {
clicked.push({
href: this.getAttribute("href"),
download: this.download,
hidden: this.hidden,
connected: this.isConnected,
});
});
return clicked;
}
// readyAfter answers "pending" for the first n reads of the ticket, then "ready".
function readyAfter(n: number) {
let reads = 0;
calls.exportStatus.mockImplementation(async () => ({ state: reads++ < n ? "pending" : "ready" }));
}
beforeEach(() => {
calls.listBackups.mockResolvedValue({ backups: [backup("bk-1", 3), backup("bk-2", 4)], total: 2 });
calls.exportDownloadURL.mockImplementation(async (ticket: string) => `/api/v1/exports/${ticket}/download`);
});
it("downloads a backup through a hidden link once felis-api holds it, never before", async () => {
const clicked = watchLinks();
calls.startBackupExport.mockResolvedValue({ ticket: "t1", state: "pending", filename: "survival-backup-bk-1.tar.gz" });
readyAfter(2);
renderPage();
await screen.findByText("3 hours ago");
vi.useFakeTimers();
await act(async () => {
fireEvent.click(downloadIn("3 hours ago"));
});
expect(calls.startBackupExport).toHaveBeenCalledExactlyOnceWith("survival", "bk-1");
expect(screen.getByRole("status").textContent).toBe(text("backups:export_progress_backup"));
// One download at a time: every other export waits, and says why.
expect(downloadIn("3 hours ago").disabled).toBe(true);
expect(downloadIn("4 hours ago").disabled).toBe(true);
expect(downloadIn("4 hours ago").parentElement?.title).toBe(text("backups:export_another"));
expect(exportWorld().disabled).toBe(true);
await act(() => vi.advanceTimersByTimeAsync(4_000));
expect(calls.exportStatus).toHaveBeenCalledTimes(2);
expect(calls.exportStatus).toHaveBeenCalledWith("t1");
expect(clicked).toEqual([]);
await act(() => vi.advanceTimersByTimeAsync(2_000));
expect(calls.exportDownloadURL).toHaveBeenCalledExactlyOnceWith("t1");
expect(clicked).toEqual([
{ href: "/api/v1/exports/t1/download", download: "survival-backup-bk-1.tar.gz", hidden: true, connected: true },
]);
expect(document.querySelectorAll("a[download]")).toHaveLength(0);
expect(screen.getByRole("status").textContent).toBe(
text("backups:export_started_backup", { filename: "survival-backup-bk-1.tar.gz" }),
);
expect(downloadIn("4 hours ago").disabled).toBe(false);
expect(exportWorld().disabled).toBe(false);
await act(() => vi.advanceTimersByTimeAsync(10_000));
expect(calls.exportStatus).toHaveBeenCalledTimes(3);
// The next download can start once this one has.
await act(async () => {
fireEvent.click(downloadIn("4 hours ago"));
});
expect(calls.startBackupExport).toHaveBeenLastCalledWith("survival", "bk-2");
expect(calls.startBackupExport).toHaveBeenCalledTimes(2);
});
it("starts one export for a double click", async () => {
watchLinks();
calls.startBackupExport.mockResolvedValue({ ticket: "t1", state: "pending", filename: "a.tar.gz" });
readyAfter(0);
renderPage();
await screen.findByText("3 hours ago");
await act(async () => {
fireEvent.click(downloadIn("3 hours ago"));
fireEvent.click(downloadIn("3 hours ago"));
});
expect(calls.startBackupExport).toHaveBeenCalledTimes(1);
});
it("exports the stopped world and holds the world buttons meanwhile", async () => {
const clicked = watchLinks();
calls.startWorldExport.mockResolvedValue({ ticket: "t2", state: "pending", filename: "survival-world-20231114-221320.tar.gz" });
readyAfter(1);
renderPage();
await screen.findByText("3 hours ago");
expect(exportWorld().disabled).toBe(false);
expect(exportWorld().title).toBe(text("backups:export_world_hint"));
// The export Job holds the world as soon as it exists.
calls.serverJobs.mockResolvedValue([{ name: "export-survival-aa", kind: "export_world", state: "running" }]);
vi.useFakeTimers();
await act(async () => {
fireEvent.click(exportWorld());
});
expect(calls.startWorldExport).toHaveBeenCalledExactlyOnceWith("survival");
expect(calls.startBackupExport).not.toHaveBeenCalled();
expect(exportWorld().textContent).toBe("Preparing…");
expect(screen.getByRole("status").textContent).toBe(text("backups:export_progress_world"));
expect(backUp().disabled).toBe(true);
await act(() => vi.advanceTimersByTimeAsync(4_000));
expect(clicked).toEqual([
{ href: "/api/v1/exports/t2/download", download: "survival-world-20231114-221320.tar.gz", hidden: true, connected: true },
]);
expect(screen.getByRole("status").textContent).toBe(
text("backups:export_started_world", { filename: "survival-world-20231114-221320.tar.gz" }),
);
// Until the download ends the Job still holds the world.
expect(exportWorld().textContent).toBe("Export world");
expect(exportWorld().disabled).toBe(true);
expect(exportWorld().title).toBe(text("backups:export_world_busy"));
expect(backUp().title).toBe(text("backups:backup_world_busy"));
expect(screen.getByText("World export", { selector: "li span" })).toBeTruthy();
});
it("offers the world export only while the server is stopped", async () => {
calls.status.mockResolvedValue({ name: "survival", displayName: "Survival", phase: "Running", desiredState: "Running" });
renderPage();
await screen.findByText("3 hours ago");
expect(exportWorld().disabled).toBe(true);
expect(exportWorld().title).toBe(text("backups:export_requires_stopped"));
// A backup is still downloadable while the server runs.
expect(downloadIn("3 hours ago").disabled).toBe(false);
});
it("waits for a backup or restore that holds the world, but not for a backup download", async () => {
calls.serverJobs.mockResolvedValue([{ name: "backup-survival-aa", kind: "backup", state: "running" }]);
const { unmount } = renderPage();
await screen.findByText("3 hours ago");
await vi.waitFor(() => expect(exportWorld().disabled).toBe(true));
expect(exportWorld().title).toBe(text("backups:export_world_busy"));
unmount();
calls.serverJobs.mockResolvedValue([{ name: "export-survival-bb", kind: "export_backup", state: "running" }]);
renderPage();
expect(await screen.findByText("Backup download", { selector: "li span" })).toBeTruthy();
expect(exportWorld().disabled).toBe(false);
expect(backUp().disabled).toBe(false);
});
it("offers a download on exactly the rows a restore is offered on", async () => {
calls.listBackups.mockResolvedValue({
backups: [
backup("bk-ok", 3),
backup("bk-bad", 4, true),
{ ...backup("bk-old", 5), expires_at: new Date(Date.now() - 3600_000).toISOString() },
],
total: 3,
});
renderPage();
await screen.findByText("3 hours ago");
for (const [when, offered] of [
["3 hours ago", true],
["4 hours ago", false],
["5 hours ago", false],
] as const) {
expect(within(row(when)).queryByRole("button", { name: "Restore" }) !== null).toBe(offered);
expect(within(row(when)).queryByRole("button", { name: "Download backup" }) !== null).toBe(offered);
}
});
it("says why a failed export produced nothing", async () => {
const clicked = watchLinks();
calls.startBackupExport.mockResolvedValue({ ticket: "t3", state: "pending", filename: "a.tar.gz" });
calls.exportStatus
.mockResolvedValueOnce({ state: "pending" })
.mockResolvedValue({ state: "failed", message: "felis export: ref is not under backup root" });
renderPage();
await screen.findByText("3 hours ago");
vi.useFakeTimers();
await act(async () => {
fireEvent.click(downloadIn("3 hours ago"));
});
await act(() => vi.advanceTimersByTimeAsync(4_000));
expect(screen.getByRole("alert").textContent).toBe(
text("backups:export_failed_because", { reason: "felis export: ref is not under backup root" }),
);
expect(clicked).toEqual([]);
expect(calls.exportDownloadURL).not.toHaveBeenCalled();
expect(downloadIn("3 hours ago").disabled).toBe(false);
await act(() => vi.advanceTimersByTimeAsync(10_000));
expect(calls.exportStatus).toHaveBeenCalledTimes(2);
});
it.each([
["the limits", "backup", { status: 429, code: "export_busy", message: "one export at a time per user" }, "errors:export_busy"],
["a stale ticket", "status", { status: 410, code: "export_expired", message: "this export has expired" }, "errors:export_expired"],
["an unwired deployment", "backup", { status: 503, code: "export_unavailable", message: "export is not configured" }, "errors:export_unavailable"],
["a server that is not stopped", "world", { status: 409, code: "not_stopped", message: "stop the server" }, "backups:export_requires_stopped"],
] as const)("words a refusal for %s", async (_, where, err, key) => {
const clicked = watchLinks();
calls.startBackupExport.mockResolvedValue({ ticket: "t4", state: "pending", filename: "a.tar.gz" });
calls.startWorldExport.mockRejectedValue(err);
if (where === "backup") calls.startBackupExport.mockRejectedValue(err);
calls.exportStatus.mockRejectedValue(err);
renderPage();
await screen.findByText("3 hours ago");
vi.useFakeTimers();
await act(async () => {
fireEvent.click(where === "world" ? exportWorld() : downloadIn("3 hours ago"));
});
await act(() => vi.advanceTimersByTimeAsync(2_000));
expect(screen.getByRole("alert").textContent).toBe(text(key));
expect(clicked).toEqual([]);
expect(exportWorld().disabled).toBe(false);
});
it("stops reading the ticket once the page is left", async () => {
const clicked = watchLinks();
calls.startBackupExport.mockResolvedValue({ ticket: "t5", state: "pending", filename: "a.tar.gz" });
readyAfter(1);
const { unmount } = renderPage();
await screen.findByText("3 hours ago");
vi.useFakeTimers();
await act(async () => {
fireEvent.click(downloadIn("3 hours ago"));
});
await act(() => vi.advanceTimersByTimeAsync(2_000));
expect(calls.exportStatus).toHaveBeenCalledTimes(1);
unmount();
await act(() => vi.advanceTimersByTimeAsync(10_000));
expect(calls.exportStatus).toHaveBeenCalledTimes(1);
expect(clicked).toEqual([]);
});
});
+194 -21
View File
@@ -5,6 +5,7 @@ import {
Archive,
CheckCircle2,
Clock,
Download,
HardDrive,
Loader2,
RotateCcw,
@@ -45,13 +46,15 @@ const BACKUP_PAGE_SIZE = 20;
const CELL = "whitespace-nowrap md:px-4 md:py-3.5";
/** BackupRow is one backup in the table, with its own restore and delete actions. `isLatest`
* marks the row a restore with no pick recovers: the newest one that is not
* corrupt, which is what the backend's LatestBackup selects. Under the reason it
* shows what the reaper's read-back found — corrupt (restore refused), verified,
* or entries the archive could not hold. `showOwner` surfaces the former owner
* (admins list every world's backups; a user only ever sees their own). `only`
* says this is the one backup the server has, which the delete confirm names. */
/** BackupRow is one backup in the table, with its own restore, download and delete
* actions. `isLatest` marks the row a restore with no pick recovers: the newest
* one that is not corrupt, which is what the backend's LatestBackup selects.
* Under the reason it shows what the reaper's read-back found — corrupt (restore
* refused), verified, or entries the archive could not hold. `showOwner`
* surfaces the former owner (admins list every world's backups; a user only
* ever sees their own). `only` says this is the one backup the server has,
* which the delete confirm names. `exporting` is the download the page is
* preparing, if any. */
function BackupRow({
b,
isLatest,
@@ -61,8 +64,10 @@ function BackupRow({
serverName,
jobs,
only,
exporting,
onReloadStatus,
onDeleted,
onExport,
}: {
b: BackupView;
isLatest: boolean;
@@ -72,8 +77,10 @@ function BackupRow({
serverName: string;
jobs: ServerJob[];
only: boolean;
exporting: ExportTarget | null;
onReloadStatus: () => void;
onDeleted: (gone: boolean) => void;
onExport: (target: ExportTarget) => void;
}) {
const { t } = useTranslation("backups");
const expired = isExpired(b.expires_at, now);
@@ -147,14 +154,17 @@ function BackupRow({
{t("corrupt_short")}
</span>
) : !expired ? (
<RestoreControls
serverName={serverName}
backup={b}
now={now}
locale={locale}
onReloadStatus={onReloadStatus}
buttonVariant={isLatest ? "destructive" : "outline"}
/>
<>
<RestoreControls
serverName={serverName}
backup={b}
now={now}
locale={locale}
onReloadStatus={onReloadStatus}
buttonVariant={isLatest ? "destructive" : "outline"}
/>
<DownloadBackupButton id={b.id} exporting={exporting} onExport={onExport} />
</>
) : (
<span className="text-xs text-muted-foreground/40 font-medium px-3 py-1.5">
{t("expired")}
@@ -185,6 +195,57 @@ function restoreMayRead(jobs: ServerJob[], id: string): boolean {
);
}
/** ExportTarget is what a download copies: the server's current world, or one of
* its backups. */
type ExportTarget = { kind: "world" } | { kind: "backup"; id: string };
/** DownloadBackupButton asks for a copy of one backup. The page prepares the
* download (one at a time, as felis-api allows one per person), so the button
* spins while it is this backup's and waits while it is another's. */
function DownloadBackupButton({
id,
exporting,
onExport,
}: {
id: string;
exporting: ExportTarget | null;
onExport: (target: ExportTarget) => void;
}) {
const { t } = useTranslation("backups");
const mine = exporting?.kind === "backup" && exporting.id === id;
// A disabled button takes no pointer events, so the reason sits on a wrapper.
return (
<span title={exporting && !mine ? t("export_another") : undefined}>
<Button
variant="ghost"
size="icon"
className="h-8 w-8 text-muted-foreground hover:bg-primary/10 hover:text-primary"
onClick={() => onExport({ kind: "backup", id })}
disabled={exporting !== null}
aria-label={t("download_btn")}
title={t("download_btn")}
>
{mine ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Download className="h-3.5 w-3.5" />}
</Button>
</span>
);
}
/** saveDownload hands a ready export to the browser the way a link would: a
* hidden <a download> it clicks once. felis-api answers with an attachment, so
* the browser's own download manager streams the archive to disk; reading it
* into a Blob first would hold a whole world in the tab's memory. */
function saveDownload(url: string, filename: string) {
const a = document.createElement("a");
a.href = url;
a.download = filename;
a.rel = "noopener";
a.hidden = true;
document.body.appendChild(a);
a.click();
a.remove();
}
/** DeleteBackupButton deletes one backup behind a confirm that says what goes: the
* row leaves the list at once and can no longer be restored, and the archive and
* its off-site copy follow at the next cleanup and sync. A reclaimed or released
@@ -409,6 +470,9 @@ const POLL_MS = 2500;
// them in game and holds the stop 30 s, then the pre-stop save and the pod's own
// shutdown save follow.
const MAX_POLLS = 48;
// How often an export being prepared rereads its ticket: the Job needs a few
// seconds to start, and a ready ticket waits 90 s for the browser.
const EXPORT_POLL_MS = 2000;
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
function RestoreControls({
@@ -566,7 +630,8 @@ function RestoreControls({
/** ServerBackups is the per-server backup surface (/servers/:name/backups): view the
* world archives kept for this server and (B2) roll the world back to any of them
* that has not expired. It owns its own gating — ownership from /me/servers, since GET status
* that has not expired, or download one, or the stopped world itself, as a
* .tar.gz. It owns its own gating — ownership from /me/servers, since GET status
* never carries `owned` — but deliberately does NOT gate on readiness the way
* ServerPlayers does: backups are read from Postgres, not RCON, and a restore in
* fact requires the server to be STOPPED, so this page must work while it is asleep. */
@@ -667,6 +732,68 @@ export function ServerBackups() {
backupsQ.reload();
}
// The download being prepared. The ref guards a second click landing before
// the render that disables the buttons; alive stops the wait once the page
// is left (the export is then abandoned: nobody fetches it).
const [exporting, setExporting] = useState<ExportTarget | null>(null);
const exportRun = useRef(false);
const alive = useRef(true);
useEffect(() => {
alive.current = true;
return () => {
alive.current = false;
};
}, []);
// A download is prepared before it starts: the POST answers a one-time ticket,
// an export Job hands the archive to felis-api, and this reads the ticket
// until felis-api holds it. Only then does the browser fetch it, as a link, so
// any size of world streams straight to disk. A ready ticket waits 90 s for
// that fetch, so it follows at once. A pending ticket ends on its own (410
// export_expired after 10 min), which bounds the wait.
async function runExport(target: ExportTarget) {
if (exportRun.current) return;
exportRun.current = true;
setExporting(target);
setBackupMsg(null);
try {
const tk =
target.kind === "world" ? await api.startWorldExport(name) : await api.startBackupExport(name, target.id);
jobsQ.reload(); // a world export now holds the world
for (;;) {
await sleep(EXPORT_POLL_MS);
if (!alive.current) return;
const s = await api.exportStatus(tk.ticket);
if (s.state === "ready") break;
if (s.state === "failed") {
setBackupMsg({
kind: "error",
text: s.message ? t("export_failed_because", { reason: s.message }) : t("export_failed"),
});
return;
}
}
saveDownload(await api.exportDownloadURL(tk.ticket), tk.filename);
setBackupMsg({
kind: "success",
text: t(target.kind === "world" ? "export_started_world" : "export_started_backup", { filename: tk.filename }),
});
} catch (e: any) {
if (!alive.current) return;
setBackupMsg({
kind: "error",
text:
target.kind === "world" && e && e.code === "not_stopped" ? t("export_requires_stopped") : humanizeError(e),
});
} finally {
exportRun.current = false;
if (alive.current) {
setExporting(null);
jobsQ.reload();
}
}
}
const back = (
<BackLink to={`/servers/${name}`} label={t("back_to_console")} />
);
@@ -699,10 +826,13 @@ export function ServerBackups() {
const total = backupsQ.data?.total ?? 0;
const latestID = page === 1 ? all.find((b) => !b.corrupt)?.id : undefined;
// A server just woken still reads Stopped until its pod starts, so the gate is
// where it is heading. A backup or restore already under way holds the world
// (the API would answer 409), so the button waits for it too.
// where it is heading. A backup, restore or world export already under way
// holds the world (the API would answer 409), so the buttons wait for it too;
// a backup download only reads the backup store and holds nothing.
const phase = shownPhase(statusQ.data);
const worldBusy = (jobsQ.data ?? []).some((j) => j.state === "running" || j.then_restore === "pending");
const worldBusy = (jobsQ.data ?? []).some(
(j) => (j.state === "running" && j.kind !== "export_backup") || j.then_restore === "pending",
);
const header = (
<PageHeader
@@ -711,6 +841,32 @@ export function ServerBackups() {
subtitle={t("title")}
actions={
<div className="flex items-center gap-2">
{owned && (
<Button
size="sm"
variant="outline"
onClick={() => runExport({ kind: "world" })}
disabled={exporting !== null || phase !== "Stopped" || worldBusy}
title={
exporting?.kind === "backup"
? t("export_another")
: exporting
? undefined
: phase !== "Stopped"
? t("export_requires_stopped")
: worldBusy
? t("export_world_busy")
: t("export_world_hint")
}
>
{exporting?.kind === "world" ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
<Download className="h-4 w-4" />
)}
{exporting?.kind === "world" ? t("export_preparing") : t("export_world")}
</Button>
)}
{owned && (
<Button
size="sm"
@@ -753,6 +909,15 @@ export function ServerBackups() {
<NotYours title={t("not_yours_title")} body={t("not_yours_body")} />
) : (
<div className="space-y-4">
{exporting && (
<div
role="status"
className="flex items-center gap-2 rounded-md border border-sky-500/20 bg-sky-500/10 p-3 text-sm text-sky-600 dark:text-sky-400"
>
<Loader2 className="h-4 w-4 shrink-0 animate-spin" />
<p>{t(exporting.kind === "world" ? "export_progress_world" : "export_progress_backup")}</p>
</div>
)}
{backupMsg && <MessageLine kind={backupMsg.kind} message={backupMsg.text} />}
<p className="text-sm text-muted-foreground">{t("subtitle")}</p>
{backupsQ.loading && !backupsQ.data ? (
@@ -787,11 +952,13 @@ export function ServerBackups() {
serverName={name}
jobs={jobsQ.data ?? []}
only={total === 1}
exporting={exporting}
onReloadStatus={() => {
statusQ.reload();
jobsQ.reload();
}}
onDeleted={handleDeleted}
onExport={runExport}
/>
))}
</tbody>
@@ -811,8 +978,8 @@ export function ServerBackups() {
</>
)}
{/* Async world-operation history: every backup/restore 202 lands here, so a
Job that later failed stays visible (with its message) without kubectl. */}
{/* Async world-operation history: every backup/restore/export 202 lands here,
so a Job that later failed stays visible (with its message) without kubectl. */}
{jobsQ.error ? (
<ErrorState error={jobsQ.error} onRetry={jobsQ.reload} />
) : (
@@ -833,6 +1000,8 @@ export function ServerBackups() {
<div className="flex min-w-0 items-center gap-2">
{j.kind === "restore" ? (
<RotateCcw className="h-3.5 w-3.5 shrink-0 text-muted-foreground/60" />
) : j.kind === "export_world" || j.kind === "export_backup" ? (
<Download className="h-3.5 w-3.5 shrink-0 text-muted-foreground/60" />
) : (
<Archive className="h-3.5 w-3.5 shrink-0 text-muted-foreground/60" />
)}
@@ -845,6 +1014,10 @@ export function ServerBackups() {
: j.scheduled
? t("job_scheduled")
: t("job_backup")
: j.kind === "export_world"
? t("job_export_world")
: j.kind === "export_backup"
? t("job_export_backup")
: j.kind}
</span>
{at && (